/* This function may work on any distribution of the bipartite graph */
int
Zoltan_ZG_Register(ZZ* zz, ZG* graph, int* properties)
{
static char *yo = "Zoltan_ZG_Register";
int ierr = ZOLTAN_OK;
int *props;
struct Zoltan_DD_Struct *dd;
int size;
ZOLTAN_ID_PTR GID;
ZOLTAN_TRACE_ENTER(zz, yo);
size = graph->mtx.mtx.nY;
dd = graph->mtx.mtx.ddY;
if (graph->bipartite) { /* Need to construct another properties array with only the fixed elements ! */
int vertlno;
if (graph->fixObj) {
dd = graph->mtx.mtx.ddX;
}
props = (int*)ZOLTAN_MALLOC(sizeof(int)*size);
if (graph->mtx.mtx.nY && props == NULL) MEMORY_ERROR;
GID = ZOLTAN_MALLOC_GID_ARRAY(zz, size);
if (size && GID == NULL) MEMORY_ERROR;
for (size = 0, vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++vertlno) {
if (graph->fixed_vertices[vertlno]) {
props[size] = properties[vertlno];
ZOLTAN_SET_GID(zz, GID+ size*zz->Num_GID,
graph->mtx.mtx.yGID+vertlno*zz->Num_GID);
size ++;
}
}
}
else {
props = properties;
GID = graph->mtx.mtx.yGID;
if (graph->mtx.mtx.ddY == NULL) {
ierr = Zoltan_DD_Create (&graph->mtx.mtx.ddY, zz->Communicator, 1, zz->Num_GID,
1, graph->mtx.mtx.globalY/zz->Num_Proc, 0);
CHECK_IERR;
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(graph->mtx.mtx.ddY, graph->mtx.mtx.globalX/zz->Num_Proc);
}
dd = graph->mtx.mtx.ddY;
}
/* Make our new numbering public */
ierr = Zoltan_DD_Update (dd, GID, NULL, NULL, props, size);
CHECK_IERR;
End:
if (graph->bipartite) {
ZOLTAN_FREE(&props);
ZOLTAN_FREE(&GID);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int
Zoltan_Matrix_Vertex_Info(ZZ* zz, const Zoltan_matrix * const m,
ZOLTAN_ID_PTR lid,
float *wwgt, int *input_part)
{
static char *yo = "Zoltan_Matrix_Vertex_Info";
int ierr = ZOLTAN_OK;
int nX;
ZOLTAN_ID_PTR l_gid = NULL;
ZOLTAN_ID_PTR l_lid = NULL;
float * l_xwgt = NULL;
int *l_input_part = NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (m->completed == 0) {
ierr = ZOLTAN_FATAL;
goto End;
}
ierr = Zoltan_Get_Obj_List(zz, &nX, &l_gid, &l_lid,
zz->Obj_Weight_Dim, &l_xwgt,
&l_input_part);
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, zz->Num_LID,
zz->Obj_Weight_Dim*sizeof(float)/sizeof(int),
nX, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (dd, l_gid, l_lid, (ZOLTAN_ID_PTR) l_xwgt,l_input_part, nX);
ZOLTAN_FREE(&l_gid);
ZOLTAN_FREE(&l_lid);
ZOLTAN_FREE(&l_xwgt);
ZOLTAN_FREE(&l_input_part);
ierr = Zoltan_DD_Find (dd, m->yGID, lid, (ZOLTAN_ID_PTR)wwgt, input_part,
m->nY, NULL);
End:
if (dd != NULL)
Zoltan_DD_Destroy(&dd);
ZOLTAN_FREE(&l_gid);
ZOLTAN_FREE(&l_lid);
ZOLTAN_FREE(&l_xwgt);
ZOLTAN_FREE(&l_input_part);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
size_t findUniqueGidsCommon(
size_t num_keys,
int num_gid,
ZOLTAN_ID_PTR ddkeys,
char *ddnewgids,
MPI_Comm mpicomm
)
{
int num_lid = 0; // Local IDs not needed
int debug_level = 0;
int num_user = sizeof(gno_t);
Zoltan_DD_Struct *dd = NULL;
Zoltan_DD_Create(&dd, mpicomm, num_gid, num_lid, num_user, num_keys,
debug_level);
ZOLTAN_ID_PTR ddnotneeded = NULL; // Local IDs not needed
Zoltan_DD_Update(dd, ddkeys, ddnotneeded, ddnewgids, NULL, int(num_keys));
//////////
// Insert unique GIDs for DD entries in User data here.
// Get value of first gid on this rank
ssize_t nDDEntries = (ssize_t)(dd->nodecnt);
ssize_t firstIdx;
MPI_Scan(&nDDEntries, &firstIdx, 1, MPI_LONG_LONG, MPI_SUM, mpicomm);
firstIdx -= nDDEntries; // do not include this rank's entries in prefix sum
// Loop over all directory entries, updating their userdata with updated gid
DD_NodeIdx cnt = 0;
for (DD_NodeIdx i = 0; i < dd->nodelistlen; i++) {
DD_Node *ptr = &(dd->nodelist[i]);
if (!(ptr->free)) {
char *userchar = (char*)(ptr->gid + (dd->gid_length + dd->lid_length));
gno_t *newgid = (gno_t*) userchar;
*newgid = gno_t(firstIdx + cnt);
cnt++;
}
}
///////////
// Retrieve the global numbers and put in the result gids vector
Zoltan_DD_Find(dd, ddkeys, ddnotneeded, ddnewgids, NULL, int(num_keys), NULL);
Zoltan_DD_Destroy(&dd);
ssize_t nUnique = 0;
MPI_Allreduce(&nDDEntries, &nUnique, 1, MPI_LONG_LONG, MPI_SUM, mpicomm);
return size_t(nUnique);
}
int build_elem_dd(MESH_INFO_PTR mesh)
{
/* Create a distributed directory of the elements so we can track their
* processor assignment after migrations.
*/
int maxelems;
MPI_Allreduce(&(mesh->num_elems), &maxelems, 1, MPI_INT, MPI_MAX,
MPI_COMM_WORLD);
if (Zoltan_DD_Create(&(mesh->dd), MPI_COMM_WORLD, 1, 0, 0, maxelems, 0) != 0){
Gen_Error(0, "fatal: NULL returned from Zoltan_DD_Create()\n");
return 0;
}
return update_elem_dd(mesh);
}
int
Zoltan_PHG_2ways_hyperedge_partition (
ZZ *zz, /* Input : Zoltan data structure */
HGraph *hg,
Partition parts,
Zoltan_PHG_Tree *tree,
struct Zoltan_DD_Struct * gnoToGID,
struct Zoltan_DD_Struct **dd,
int *numParts,
int **sizeParts
)
{
int ierr = ZOLTAN_OK;
char *yo = "Zoltan_PHG_2ways_hyperedge_partition";
int nEdge, hEdge;
int *interval;
int *partnumber = NULL;
int tree_size;
int *rowpart =NULL;
int *rowGNO = NULL;
ZOLTAN_ID_PTR rowGID=NULL;
int index;
int offset;
ZOLTAN_TRACE_ENTER(zz, yo);
nEdge = hg->nEdge;
fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y, hg->nVtx, nEdge);
interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int));
if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR;
tree_size = get_tree_size(tree) + 1;
for (index = 0 ; index < nEdge ; ++index){
SET_MIN_NODE(interval, index, tree_size);
SET_MAX_NODE(interval, index, -1);
}
/* Update interval with the local knowledge */
/* XXX: I loop on the hyperedges, as I think it's more cache friendly
* and it allows me to discoupled the computation if a non blocking MPI_Reduce is
* available
*/
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){
int part;
int max = -1; /* Trick : we use the initialized values */
int min = tree_size + 1;
for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) {
part = parts[hg->hvertex[index]];
max = MAX(max, part);
min = MIN(min, part);
}
SET_MIN_NODE(interval, hEdge, min);
SET_MAX_NODE(interval, hEdge, max);
}
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm);
/* Now I have to compute the partition of hyperedges according to the "interval"
* and the tree */
/* First, compute the partition number corresponding to the nodes in the tree */
partnumber = compute_part_number(tree);
if (partnumber == NULL) {
ierr = ZOLTAN_FATAL;
goto End;
}
(*numParts) = get_tree_size(tree);
rowpart = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int));
if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR;
rowGNO = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int));
if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR;
(*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int));
if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR;
offset = hg->dist_y[hg->comm->myProc_y];
/* Then we search we is the hyperedge in the tree */
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
int node;
node = find_interval_in_tree(tree, interval+2*hEdge);
rowpart[hEdge] = partnumber[node];
(*sizeParts)[rowpart[hEdge]] ++;
rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge);
#if 0
fprintf (stderr, "%d : %d (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]);
#endif
}
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
/* Compute number of elements per parts */
/* TODO: support processor which are not part of the distribution */
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm);
/* Export results to data directory */
/* First, get the GIDs of our edges */
rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge);
if (nEdge && rowGID == NULL) MEMORY_ERROR;
ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL,
nEdge, NULL);
ZOLTAN_FREE(&rowGNO);
ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, (ZOLTAN_ID_PTR) rowpart, NULL, NULL, nEdge);
#ifdef CEDRIC_PRINT
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]);
}
#endif
End:
ZOLTAN_FREE(&rowGID);
ZOLTAN_FREE(&rowGNO);
ZOLTAN_FREE(&rowpart);
if (partnumber != NULL)
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/* Performs a permutation of the matrix, perm_y A perm_y^t.
* TODO: at this time we only do symmetric permutations (don't know xGNO !).
*/
int
Zoltan_Matrix_Permute(ZZ* zz, Zoltan_matrix *m, int* perm_y)
{
static char *yo = "Zoltan_Matrix_Permute";
int ierr = ZOLTAN_OK;
int *pinGNO = NULL;
ZOLTAN_ID_PTR yGID=NULL;
float *ywgt=NULL;
struct Zoltan_DD_Struct *dd;
ZOLTAN_TRACE_ENTER(zz, yo);
/* First apply y permutation */
if (m->completed) { /* We directly know the good arrays */
yGID = m->yGID;
ywgt = m->ywgt;
if (m->ddY == NULL || m->ddY != m->ddX) { /* We have to create again the DD */
/* We have to define ddY : yGNO, yGID, ywgt */
ierr = Zoltan_DD_Create (&m->ddY, zz->Communicator, 1, zz->Num_GID,
m->ywgtdim*sizeof(float)/sizeof(int), m->globalY/zz->Num_Proc, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(m->ddY, m->globalY/zz->Num_Proc);
}
}
else { /* We have to get these fields */
/* Update data directories */
yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY);
ywgt = (float*) ZOLTAN_MALLOC(m->nY * sizeof(float) * m->ywgtdim);
if (m->nY && (yGID == NULL || (m->ywgtdim && ywgt == NULL)))
MEMORY_ERROR;
/* Get Informations about Y */
Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL,
m->nY, NULL);
}
memcpy (m->yGNO, perm_y, m->nY*sizeof(int));
/* Get Informations about Y */
Zoltan_DD_Update (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL,
m->nY);
ZOLTAN_FREE (&yGID);
ZOLTAN_FREE (&ywgt);
/* We have to define dd : old_yGNO, new_yGNO */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, 1, 1, 0, m->globalY/zz->Num_Proc, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, m->globalY/zz->Num_Proc);
Zoltan_DD_Update (dd, (ZOLTAN_ID_PTR)m->yGNO, (ZOLTAN_ID_PTR)perm_y, NULL, NULL, m->nY);
pinGNO = (int*)ZOLTAN_MALLOC(m->nPins*sizeof(int));
if (m->nPins && pinGNO == NULL)
MEMORY_ERROR;
Zoltan_DD_Find (dd, (ZOLTAN_ID_PTR)m->pinGNO, (ZOLTAN_ID_PTR)pinGNO, NULL, NULL,
m->nY, NULL);
ZOLTAN_FREE(&m->pinGNO);
m->pinGNO = pinGNO;
pinGNO = NULL;
End:
ZOLTAN_FREE (&pinGNO);
ZOLTAN_FREE (&yGID);
ZOLTAN_FREE (&ywgt);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int
Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix)
{
static char *yo = "Zoltan_Matrix_Build";
int ierr = ZOLTAN_OK;
int nX;
int *xGNO = NULL;
ZOLTAN_ID_PTR xLID=NULL;
ZOLTAN_ID_PTR xGID=NULL;
ZOLTAN_ID_PTR yGID=NULL;
ZOLTAN_ID_PTR pinID=NULL;
float *xwgt = NULL;
int * Input_Parts=NULL;
struct Zoltan_DD_Struct *dd = NULL;
int *proclist = NULL;
int *xpid = NULL;
int i;
ZOLTAN_TRACE_ENTER(zz, yo);
memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */
memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options));
/**************************************************/
/* Obtain vertex information from the application */
/**************************************************/
ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID,
zz->Obj_Weight_Dim, &xwgt,
&Input_Parts);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data");
goto End;
}
ZOLTAN_FREE(&Input_Parts);
ZOLTAN_FREE(&xwgt);
/*******************************************************************/
/* Assign vertex consecutive numbers (gnos) */
/*******************************************************************/
if (matrix->opts.speed == MATRIX_FULL_DD) { /* Zoltan computes a translation */
if (nX) {
xGNO = (int*) ZOLTAN_MALLOC(nX*sizeof(int));
if (xGNO == NULL)
MEMORY_ERROR;
}
ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices");
goto End;
}
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, 1,
0, nX, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL, NULL, nX);
}
else { /* We don't want to use the DD */
xGNO = (int *) xGID;
MPI_Allreduce(&nX, &matrix->globalX, 1, MPI_INT, MPI_SUM, zz->Communicator);
}
/* I store : xGNO, xGID, xpid, */
ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID,
1, matrix->globalX/zz->Num_Proc, 0);
CHECK_IERR;
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
/* Associate all the data with our xGNO */
xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int));
if (nX >0 && xpid == NULL) MEMORY_ERROR;
for (i = 0 ; i < nX ; ++i)
xpid[i] = zz->Proc;
Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (ZOLTAN_ID_PTR) xpid, NULL, nX);
ZOLTAN_FREE(&xpid);
if (matrix->opts.pinwgt)
matrix->pinwgtdim = zz->Edge_Weight_Dim;
else
matrix->pinwgtdim = 0;
ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt);
CHECK_IERR;
matrix->nY_ori = matrix->nY;
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){
goto End;
}
if (matrix->opts.enforceSquare && matrix->redist) {
/* Convert yGID to yGNO using the same translation as x */
/* Needed for graph : rowID = colID */
/* y and x may have different distributions */
matrix->yGNO = (int*)ZOLTAN_MALLOC(matrix->nY * sizeof(int));
if (matrix->nY && matrix->yGNO == NULL)
MEMORY_ERROR;
ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL,
matrix->nY, NULL);
if (ierr != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n");
ierr = ZOLTAN_FATAL;
goto End;
}
}
if (matrix->opts.local) { /* keep only local edges */
proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int));
if (matrix->nPins && proclist == NULL) MEMORY_ERROR;
}
else
proclist = NULL;
/* Convert pinID to pinGNO using the same translation as x */
if (matrix->opts.speed == MATRIX_FULL_DD) {
matrix->pinGNO = (int*)ZOLTAN_MALLOC(matrix->nPins* sizeof(int));
if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) MEMORY_ERROR;
ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL,
matrix->nPins, proclist);
if (ierr != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n");
ierr = ZOLTAN_FATAL;
goto End;
}
ZOLTAN_FREE(&pinID);
Zoltan_DD_Destroy(&dd);
dd = NULL;
}
else {
matrix->pinGNO = (int *) pinID;
pinID = NULL;
}
/* if (matrix->opts.local) { /\* keep only local edges *\/ */
/* int *nnz_list; /\* nnz offset to delete *\/ */
/* int nnz; /\* number of nnz to delete *\/ */
/* int i; */
/* nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */
/* if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */
/* for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */
/* if (proclist[i] == zz->Proc) continue; */
/* nnz_list[nnz++] = i; */
/* } */
/* ZOLTAN_FREE(&proclist); */
/* Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */
/* } */
if (!matrix->opts.enforceSquare) {
/* Hyperedges name translation is different from the one of vertices */
matrix->yGNO = (int*)ZOLTAN_CALLOC(matrix->nY, sizeof(int));
if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR;
/* int nGlobalEdges = 0; */
ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY,
matrix->opts.randomize, &matrix->globalY);
CHECK_IERR;
/* /\**************************************************************************************** */
/* * If it is desired to remove dense edges, divide the list of edges into */
/* * two lists. The ZHG structure will contain the removed edges (if final_output is true), */
/* * and the kept edges will be returned. */
/* ****************************************************************************************\/ */
/* totalNumEdges = zhg->globalHedges; */
/* ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */
/* &nLocalEdges, &nGlobalEdges, &nPins, */
/* &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */
/* if (nGlobalEdges < totalNumEdges){ */
/* /\* re-assign edge global numbers if any edges were removed *\/ */
/* ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */
/* randomizeInitDist, &totalNumEdges); */
/* if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */
/* goto End; */
/* } */
/* } */
/* We have to define ddY : yGNO, yGID, ywgt */
ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, 1, zz->Num_GID,
0, matrix->globalY/zz->Num_Proc, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc);
/* Associate all the data with our yGNO */
Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO,
yGID, NULL, NULL, matrix->nY);
}
End:
ZOLTAN_FREE(&xpid);
ZOLTAN_FREE(&xLID);
ZOLTAN_FREE(&xGNO);
ZOLTAN_FREE(&xGID);
ZOLTAN_FREE(&xwgt);
ZOLTAN_FREE(&Input_Parts);
ZOLTAN_FREE(&proclist);
if (dd != NULL)
Zoltan_DD_Destroy(&dd);
/* Already stored in the DD */
ZOLTAN_FREE(&yGID);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int main (int argc, char *argv[])
{
Zoltan_DD_Directory *dd;
ZTIMER *zt;
int myproc; /* MPI rank, my processor's MPI ID */
int nproc; /* MPI size, number of processors */
ZOLTAN_ID_PTR glist = NULL; /* list of GIDs */
ZOLTAN_ID_PTR llist = NULL; /* list of LIDs */
ZOLTAN_ID_PTR ulist = NULL; /* list of user data of type
ZOLTAN_ID_TYPE */
int *plist = NULL; /* list of partitions */
int *olist = NULL; /* list of owners */
Param param ; /* program's adjustable parameters */
char *store = NULL; /* non directory storage of test data */
Data *data = NULL; /* pointer into store */
/* these are all temporary variables */
int new_owner;
int count;
int i;
char *p;
int err;
int errcount;
int loop;
char str[100]; /* for building output messages */
static int timer[7] = {-1,-1,-1,-1,-1,-1,-1};
char *yo = "DD_Main";
/* initialize MPI communications, ignore errors */
MPI_Init (&argc, &argv);
MPI_Comm_rank (MPI_COMM_WORLD, &myproc);
MPI_Comm_size (MPI_COMM_WORLD, &nproc);
get_params (¶m); /* read input parameters */
zt = Zoltan_Timer_Create(1);
MACRO_TIMER_START(5, "Total time", 0);
MACRO_TIMER_START(0, "DD_Create time", 0);
err = Zoltan_DD_Create (&dd, MPI_COMM_WORLD, param.glen, param.llen,
param.ulen, param.tlen, param.debug_level);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Create");
MACRO_TIMER_STOP(0);
param.slen = sizeof (Data) + sizeof(ZOLTAN_ID_TYPE)
* (param.glen + param.llen + param.ulen);
param.slen = Zoltan_Align(param.slen);
store = (char*) ZOLTAN_MALLOC (param.count * param.slen);
/* allocate storage for various lists */
glist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC(sizeof(ZOLTAN_ID_TYPE) * param.count
* param.glen);
plist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count);
olist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count);
if (param.llen)
llist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE)
* param.count * param.llen);
if (param.ulen)
ulist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE)
* param.count * param.ulen) ;
if (store == NULL || glist == NULL || (param.llen != 0 && llist == NULL)
|| (param.ulen != 0 && ulist == NULL) || plist == NULL || olist == NULL) {
ZOLTAN_PRINT_ERROR (myproc, yo, "Unable to malloc storage lists");
return ZOLTAN_MEMERR;
}
initialize_data (¶m, store, nproc);
/* create & update directory with myproc's initial simulated GIDs */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data *)p)->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data*)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data *)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen,
((Data *)p)->id + (param.glen + param.llen));
plist [count] = ((Data *)p)->partition;
++count;
}
MACRO_TIMER_START (1, "DD_Update timer", 0);
err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
i = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen) {
ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id);
++i;
}
MACRO_TIMER_START(2, "DD_Find timer", 0);
err = Zoltan_DD_Find (dd, glist, llist, ulist, plist, param.count, olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
errcount = 0;
for (i = 0; i < param.count; i++)
if (olist[i] != ((Data *) (store + i * param.slen))->new_owner)
++errcount;
if (errcount)
sprintf (str, "FIRST TEST FAILED, errcount is %d", errcount);
else
sprintf (str, "FIRST TEST SUCCESSFUL");
ZOLTAN_PRINT_INFO (myproc, yo, str);
/* repeatedly simulate moving "dots" around the system */
for (loop = 0; loop < param.nloops; loop++)
{
for (p = store; p < store + param.count * param.slen; p += param.slen)
((Data*) p)->old_owner = ((Data*) p)->new_owner;
/* randomly exchange some dots and randomly reassign others */
for (i = 0; i < (param.pmove * param.count)/100; i++) {
Data *d1, *d2;
d1 = (Data*) (store + param.slen * (rand() % param.count));
d2 = (Data*) (store + param.slen * (rand() % param.count));
new_owner = d1->new_owner;
d1->new_owner = d2->new_owner;
d2->new_owner = new_owner;
}
for (i = 0; i < (param.count * param.pscatter)/100; i++)
((Data*) (store + param.slen *(rand() % param.count)))->new_owner
= rand() % nproc;
/* determine which new GIDs myproc gained, and update directory */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == myproc) {
((Data*)p)->id[param.glen] = count; /* set LID */
ZOLTAN_SET_ID (param.glen, glist + count * param.glen,
((Data *)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data*)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen,
((Data*)p)->id + (param.glen + param.llen));
plist [count] = ((Data *)p)->partition;
++count;
}
MACRO_TIMER_START(1, "DD_Update", 0);
err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
/* use directory to "find" GIDs */
i = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen) {
ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id);
++i;
}
MACRO_TIMER_START(2, "DD_Find timer", 0);
if (loop % 5 == 0)
err = Zoltan_DD_Find(dd,glist,NULL,ulist,plist,param.count,olist);
else if (loop % 7 == 0)
err = Zoltan_DD_Find(dd,glist,llist,NULL,plist,param.count,olist);
else if (loop % 9 == 0)
err = Zoltan_DD_Find(dd,glist,llist,ulist,NULL,param.count,olist);
else if (loop % 2 == 0)
err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,NULL);
else
err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
if (loop % 2) {
errcount = 0 ;
for (i = 0; i < param.count; i++)
if (olist[i] != ((Data *)(store + i * param.slen))->new_owner)
++errcount;
if (errcount)
sprintf (str,"LOOP %d TEST FAILED, errcount is %d",loop,errcount);
else
sprintf (str, "LOOP %d TEST SUCCESSFUL", loop);
ZOLTAN_PRINT_INFO (myproc, yo, str) ;
}
else {
sprintf (str, "LOOP %d Completed", loop);
ZOLTAN_PRINT_INFO (myproc, yo, str);
}
/* randomly remove a percentage of GIDs from the directory */
count = 0;
for (i = 0; i < (param.count * param.pdelete)/100; i++) {
data = (Data*) (store + param.slen * (rand() % param.count));
if (data->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id);
++count;
}
data->new_owner = NO_PROC;
}
MACRO_TIMER_START(3, "DD_Remove timer", 0);
err = Zoltan_DD_Remove (dd, glist, count);
MACRO_TIMER_STOP(3);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Remove");
/* update directory (put directory entries back in) */
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == NO_PROC)
((Data*)p)->new_owner = loop % nproc; /* place in new location */
count = 0;
for (p = store; p < store + param.count * param.slen; p += param.slen)
if (((Data*)p)->new_owner == myproc) {
ZOLTAN_SET_ID(param.glen,glist+count*param.glen,((Data*)p)->id);
if (param.llen)
ZOLTAN_SET_ID (param.llen, llist + count * param.llen,
((Data*)p)->id + param.glen);
if (param.ulen)
ZOLTAN_SET_ID(param.ulen, ulist + count * param.ulen,
((Data *)p)->id + (param.glen + param.llen));
plist [count] = ((Data*)p)->partition;
++count;
}
MACRO_TIMER_START(1, "DD_Update timer", 0);
err = Zoltan_DD_Update (dd, glist, NULL, NULL, NULL, count);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update");
MACRO_TIMER_STOP(1);
}
/* now Find directory info for GIDs myproc now owns and validate */
count = 0;
for (i = 0; i < param.count; i++) {
data = (Data *) (store + i * param.slen);
if (data->new_owner == myproc) {
ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id);
++count;
}
}
MACRO_TIMER_START(2, "DD_Find", 0);
err = Zoltan_DD_Find (dd, glist, NULL, NULL, NULL, count, olist);
if (err != ZOLTAN_OK)
ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find");
MACRO_TIMER_STOP(2);
errcount = 0;
for (i = 0; i < count; i++)
if (olist[i] != myproc)
++errcount;
if (errcount) {
sprintf (str, "errcount is %d", errcount);
ZOLTAN_PRINT_ERROR (myproc, yo, str);
}
else
ZOLTAN_PRINT_INFO (myproc, yo, "TEST SUCCESSFUL");
Zoltan_DD_Stats (dd);
/* done, now free memory, stop MPI & directory, return */
ZOLTAN_FREE (&store);
ZOLTAN_FREE (&glist);
ZOLTAN_FREE (&plist);
ZOLTAN_FREE (&olist);
if (param.llen) ZOLTAN_FREE (&llist);
if (param.ulen) ZOLTAN_FREE (&ulist);
ZOLTAN_PRINT_INFO (myproc, yo, "Completing program");
MACRO_TIMER_START(4, "DD_Destroy", 0);
Zoltan_DD_Destroy (&dd);
MACRO_TIMER_STOP(4);
MACRO_TIMER_STOP(5);
Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stderr);
MPI_Finalize ();
return ZOLTAN_OK;
}
int Zoltan_Color_Test(
ZZ *zz, /* Zoltan structure */
int *num_gid_entries, /* # of entries for a global id */
int *num_lid_entries, /* # of entries for a local id */
int num_obj, /* Input: number of objects */
ZOLTAN_ID_PTR global_ids, /* Input: global ids of the vertices */
/* The application must allocate enough space */
ZOLTAN_ID_PTR local_ids, /* Input: local ids of the vertices */
/* The application must allocate enough space */
int *color_exp /* Input: Colors assigned to local vertices */
)
{
static char *yo = "color_test_fn";
int nvtx = num_obj; /* number of vertices */
int i, j;
int ierr = ZOLTAN_OK;
int ferr = ZOLTAN_OK; /* final error signal */
char coloring_problem; /* Input: which coloring to perform;
currently only supports D1, D2 coloring and variants */
char coloring_problemStr[MAX_PARAM_STRING_LEN]; /* string version coloring problem name */
int ss=100;
char comm_pattern='S', coloring_order='I', coloring_method='F';
int comm[2],gcomm[2];
int *color=NULL;
int *adjproc=NULL, *xadj=NULL;
ZOLTAN_GNO_TYPE gvtx; /* number of global vertices */
ZOLTAN_GNO_TYPE *vtxdist=NULL, *adjncy=NULL;
ZG graph;
ZOLTAN_GNO_TYPE *requested_GNOs = NULL; /* Return GNOs of
the requested GIDs. */
int *loc_partialD2 = NULL; /* local binary array showing which vertices to be colored */
int *partialD2 = NULL; /* global binary array showing which vertices to be colored */
struct Zoltan_DD_Struct *dd_color; /* DDirectory for colors */
ZOLTAN_GNO_TYPE *local_GNOs = NULL;
ZOLTAN_GNO_TYPE *global_GNOs = NULL;
memset (&graph, 0, sizeof(ZG));
/* PARAMETER SETTINGS */
Zoltan_Bind_Param(Color_params, "COLORING_PROBLEM", (void *) &coloring_problemStr);
Zoltan_Bind_Param(Color_params, "SUPERSTEP_SIZE", (void *) &ss);
Zoltan_Bind_Param(Color_params, "COMM_PATTERN", (void *) &comm_pattern);
Zoltan_Bind_Param(Color_params, "VERTEX_VISIT_ORDER", (void *) &coloring_order);
Zoltan_Bind_Param(Color_params, "COLORING_METHOD", (void *) &coloring_method);
strncpy(coloring_problemStr, "distance-1", MAX_PARAM_STRING_LEN);
Zoltan_Assign_Param_Vals(zz->Params, Color_params, zz->Debug_Level, zz->Proc,
zz->Debug_Proc);
/* Check validity of parameters - they should be consistent with Zoltan_Color */
if (!strcasecmp(coloring_problemStr, "distance-1"))
coloring_problem = '1';
else if (!strcasecmp(coloring_problemStr, "distance-2"))
coloring_problem = '2';
else if (!strcasecmp(coloring_problemStr, "partial-distance-2")
|| !strcasecmp(coloring_problemStr, "bipartite"))
coloring_problem = 'P';
else {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Unknown coloring requested. Using Distance-1 coloring.");
coloring_problem = '1';
}
if (ss == 0) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid superstep size. Using default value 100.");
ss = 100;
}
if (comm_pattern != 'S' && comm_pattern != 'A') {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid communication pattern. Using synchronous communication (S).");
comm_pattern = 'S';
}
if (comm_pattern == 'A' && (coloring_problem == '2' || coloring_problem == 'P')) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Asynchronous communication pattern is not implemented for distance-2 coloring and its variants. Using synchronous communication (S).");
comm_pattern = 'S';
}
if (coloring_order != 'I' && coloring_order != 'B' && coloring_order != 'U' && coloring_order != 'L' && coloring_order != 'N' && coloring_order != 'S') {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring order. Using internal first coloring order (I).");
coloring_order = 'I';
}
if (coloring_order == 'U' && (coloring_problem == '2' || coloring_problem == 'P')) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Interleaved coloring order is not implemented for distance-2 coloring and its variants. Using internal first coloring order (I).");
coloring_order = 'I';
}
if (coloring_method !='F') {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring method. Using first fit method (F).");
coloring_method = 'F';
}
/* Compute Max number of array entries per ID over all processors.
This is a sanity-maintaining step; we don't want different
processors to have different values for these numbers. */
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
/* Return if this processor is not in the Zoltan structure's
communicator. */
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz))
return ZOLTAN_OK;
/* BUILD THE GRAPH */
/* Check that the user has allocated space for the return args. */
if (!color_exp)
ZOLTAN_COLOR_ERROR(ZOLTAN_FATAL, "Color argument is NULL. Please give colors of local vertices.");
requested_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(num_obj *
sizeof(ZOLTAN_GNO_TYPE));
Zoltan_ZG_Build (zz, &graph, 0, 1, num_obj,
global_ids, requested_GNOs);
Zoltan_ZG_Export (zz, &graph,
&gvtx, &nvtx, NULL, NULL,
&vtxdist, &xadj, &adjncy, &adjproc,
NULL, NULL);
if (gvtx > (ZOLTAN_GNO_TYPE)INT_MAX){
if (zz->Proc == 0){
fprintf(stderr,
"Zoltan_Color_Test assumes number of vertices (%ld) is less than INT_MAX\n",gvtx);
}
ierr = ZOLTAN_FATAL;
goto End;
}
KDDKDDKDD(zz->Proc, "COLORTEST DD");
/* Exchange global color information */
if (vtxdist[zz->Num_Proc] && !(color = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int))))
MEMORY_ERROR;
if (nvtx && !(local_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(nvtx * sizeof(ZOLTAN_GNO_TYPE))))
MEMORY_ERROR;
for (i=0; i<nvtx; ++i)
local_GNOs[i] = i+vtxdist[zz->Proc];
if (vtxdist[zz->Num_Proc] && !(global_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(vtxdist[zz->Num_Proc] * sizeof(ZOLTAN_GNO_TYPE))))
MEMORY_ERROR;
for (i=0; i<vtxdist[zz->Num_Proc]; ++i)
global_GNOs[i] = i;
ierr = Zoltan_DD_Create (&dd_color, zz->Communicator,
sizeof(ZOLTAN_GNO_TYPE)/sizeof(ZOLTAN_ID_TYPE), 0, 0, num_obj, 0);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot construct DDirectory.");
/* Put req obs with 1 but first inialize the rest with 0 */
ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)local_GNOs, NULL,
NULL, color, nvtx);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory.");
ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)requested_GNOs, NULL,
NULL, color_exp, num_obj);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory.");
/* Get requested colors from the DDirectory. */
ierr = Zoltan_DD_Find (dd_color, (ZOLTAN_ID_PTR)global_GNOs, NULL, NULL,
color, vtxdist[zz->Num_Proc], NULL);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot find object in DDirectory.");
/* Free DDirectory */
Zoltan_DD_Destroy(&dd_color);
ZOLTAN_FREE(&local_GNOs);
ZOLTAN_FREE(&global_GNOs);
KDDKDDKDD(zz->Proc, "COLORTEST CHECK");
if (coloring_problem == 'P' || coloring_problem == '2') {
if (vtxdist[zz->Num_Proc] && !(partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int))))
MEMORY_ERROR;
if (vtxdist[zz->Num_Proc] && !(loc_partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int))))
MEMORY_ERROR;
if (coloring_problem == 'P') {
for (i=0; i<num_obj; ++i) {
int gno=requested_GNOs[i];
loc_partialD2[gno] = 1;
}
MPI_Allreduce(loc_partialD2, partialD2, vtxdist[zz->Num_Proc], MPI_INT, MPI_LOR, zz->Communicator);
} else {
for (i=0; i<vtxdist[zz->Num_Proc]; ++i)
partialD2[i] = 1;
}
}
/* Check if there is an error in coloring */
if (coloring_problem == '1') {
for (i=0; i<nvtx; i++) {
int gno = i + (int)vtxdist[zz->Proc];
if (color[gno] <= 0) { /* object i is not colored properly */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]);
break;
}
for (j = xadj[i]; j < xadj[i+1]; ++j) {
int v = (int)adjncy[j];
if (color[gno] == color[v]) { /* neighbors have the same color */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! u:%d, v:%d, cu:%d, cv:%d\n", gno, v, color[gno], color[v]);
break;
}
}
if (ierr == ZOLTAN_FATAL)
break;
}
} else if (coloring_problem == '2' || coloring_problem == 'P') {
for (i=0; i<nvtx; i++) {
int gno = i + (int)vtxdist[zz->Proc];
if (partialD2[gno] && color[gno] <= 0) { /* object i is not colored properly */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]);
break;
}
for (j = xadj[i]; j < xadj[i+1]; ++j) {
int v = (int)adjncy[j], k;
if (partialD2[v] && color[v] <= 0) {
ierr = ZOLTAN_FATAL;
printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]);
break;
}
if (partialD2[gno] && partialD2[v] && color[gno] == color[v]) { /* d-1 neighbors have the same color */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]);
break;
}
for (k = j+1; k < xadj[i+1]; ++k) {
int w = (int)adjncy[k];
if (partialD2[v] && partialD2[w] && color[v] == color[w]) { /* d-2 neighbors have the same color */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! d2-neigh: v:%d, w:%d, cv:%d, cw:%d pv:%d pw:%d\n", v, w, color[v], color[w], partialD2[v], partialD2[w]);
break;
}
}
}
if (ierr == ZOLTAN_FATAL)
break;
}
} else
ZOLTAN_COLOR_ERROR(ZOLTAN_WARN, "Zoltan_Color_Test is only implemented for distance-1 and distance-2 coloring. Unknown coloring, skipping verification.");
End:
KDDKDDKDD(zz->Proc, "COLORTEST DONE");
if (ierr==ZOLTAN_FATAL)
ierr = 2;
MPI_Allreduce(&ierr, &ferr, 1, MPI_INT, MPI_MAX, zz->Communicator);
if (ferr == 2)
ierr = ZOLTAN_FATAL;
else
ierr = ZOLTAN_OK;
Zoltan_ZG_Free (&graph);
ZOLTAN_FREE(&adjproc);
ZOLTAN_FREE(&color);
ZOLTAN_FREE(&requested_GNOs);
ZOLTAN_FREE(&partialD2);
ZOLTAN_FREE(&loc_partialD2);
return ierr;
}
int
Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix,
int request_GNOs, /* Input: Flag indicating calling code
needs translation of extra GIDs
to GNOs; partial 2D coloring
needs this feature. */
int num_requested, /* Input: Local # of GIDs needing
translation to GNOs. */
ZOLTAN_ID_PTR requested_GIDs, /* Input: Calling code requests the
GNOs for these GIDs */
ZOLTAN_GNO_TYPE *requested_GNOs /* Output: Return GNOs of
the requested GIDs. */
)
{
static char *yo = "Zoltan_Matrix_Build";
int ierr = ZOLTAN_OK;
int nX;
ZOLTAN_GNO_TYPE tmp;
ZOLTAN_GNO_TYPE *xGNO = NULL;
ZOLTAN_ID_PTR xLID=NULL;
ZOLTAN_ID_PTR xGID=NULL;
ZOLTAN_ID_PTR yGID=NULL;
ZOLTAN_ID_PTR pinID=NULL;
float *xwgt = NULL;
int * Input_Parts=NULL;
struct Zoltan_DD_Struct *dd = NULL;
int *proclist = NULL;
int *xpid = NULL;
int i;
int gno_size_for_dd;
MPI_Datatype zoltan_gno_mpi_type;
int use_full_dd = (opt->speed == MATRIX_FULL_DD);
int fast_build_base = opt->fast_build_base;
matrix->opts.speed = opt->speed;
matrix->opts.fast_build_base = opt->fast_build_base;
ZOLTAN_TRACE_ENTER(zz, yo);
if (num_requested && (!requested_GIDs || !requested_GNOs)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error in requested input; needed arrays are NULL.\n");
}
/* ZOLTAN_GNO_TYPE is >= ZOLTAN_ID_TYPE */
gno_size_for_dd = sizeof(ZOLTAN_GNO_TYPE) / sizeof(ZOLTAN_ID_TYPE);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */
memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options));
/**************************************************/
/* Obtain vertex information from the application */
/**************************************************/
ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID,
zz->Obj_Weight_Dim, &xwgt,
&Input_Parts);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data");
goto End;
}
ZOLTAN_FREE(&Input_Parts);
ZOLTAN_FREE(&xwgt);
/*******************************************************************/
/* Assign vertex consecutive numbers (gnos) */
/*******************************************************************/
if (use_full_dd) {
/* Zoltan computes a translation */
/* Have to use Data Directory if request_GNOs is true. */
if (nX) {
xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE));
if (xGNO == NULL)
MEMORY_ERROR;
}
ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices");
goto End;
}
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, gno_size_for_dd, 0, nX, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL, NULL, nX);
if (request_GNOs) {
Zoltan_DD_Find(dd, requested_GIDs, (ZOLTAN_ID_PTR) requested_GNOs,
NULL, NULL, num_requested, NULL);
}
}
else { /* We don't want to use the DD */
/*
* KDDKDD 2/10/11 This code cannot work when NUM_GID_ENTRIES>1.
* KDDKDD 2/10/11 The assumption is that, if a user sets the
* KDDKDD 2/10/11 appropriate parameter to enable this code, the user
* KDDKDD 2/10/11 knows that his GIDs are compatible with integers.
*/
if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){
xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE));
if (nX && xGNO == NULL)
MEMORY_ERROR;
for (i=0; i < nX; i++)
xGNO[i] = (ZOLTAN_GNO_TYPE)xGID[i] - fast_build_base;
}
else {
xGNO = (ZOLTAN_GNO_TYPE *)xGID;
if (fast_build_base)
for (i = 0; i < nX; i++)
xGNO[i] -= fast_build_base;
}
for (i = 0; i < num_requested; i++)
requested_GNOs[i] = (ZOLTAN_GNO_TYPE)requested_GIDs[i]
- fast_build_base;
tmp = (ZOLTAN_GNO_TYPE)nX;
MPI_Allreduce(&tmp, &matrix->globalX, 1, zoltan_gno_mpi_type, MPI_SUM, zz->Communicator);
}
/* I store : xGNO, xGID, xpid, */
ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, gno_size_for_dd, zz->Num_GID,
sizeof(int), matrix->globalX/zz->Num_Proc, 0);
CHECK_IERR;
/* Hope a linear assignment will help a little */
if (matrix->globalX/zz->Num_Proc)
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
/* Associate all the data with our xGNO */
xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int));
if (nX >0 && xpid == NULL) MEMORY_ERROR;
for (i = 0 ; i < nX ; ++i)
xpid[i] = zz->Proc;
Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (char *)xpid, NULL, nX);
ZOLTAN_FREE(&xpid);
if (matrix->opts.pinwgt)
matrix->pinwgtdim = zz->Edge_Weight_Dim;
else
matrix->pinwgtdim = 0;
ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt, use_full_dd);
CHECK_IERR;
matrix->nY_ori = matrix->nY;
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){
goto End;
}
if (matrix->opts.enforceSquare && matrix->redist) {
/* Convert yGID to yGNO using the same translation as x */
/* Needed for graph : rowID = colID */
/* y and x may have different distributions */
matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nY * sizeof(ZOLTAN_GNO_TYPE));
if (matrix->nY && matrix->yGNO == NULL) {
ZOLTAN_FREE(&pinID);
MEMORY_ERROR;
}
ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL,
matrix->nY, NULL);
if (ierr != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n");
ierr = ZOLTAN_FATAL;
ZOLTAN_FREE(&pinID);
goto End;
}
}
if (matrix->opts.local) { /* keep only local edges */
proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int));
if (matrix->nPins && proclist == NULL) {
ZOLTAN_FREE(&pinID);
MEMORY_ERROR;
}
}
else
proclist = NULL;
/* Convert pinID to pinGNO using the same translation as x */
if (use_full_dd) {
matrix->pinGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nPins* sizeof(ZOLTAN_GNO_TYPE));
if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) {
ZOLTAN_FREE(&pinID);
MEMORY_ERROR;
}
ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL,
matrix->nPins, proclist);
if (ierr != ZOLTAN_OK) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n");
ierr = ZOLTAN_FATAL;
goto End;
}
ZOLTAN_FREE(&pinID);
Zoltan_DD_Destroy(&dd);
dd = NULL;
}
else {
if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){
matrix->pinGNO = (ZOLTAN_GNO_TYPE *)ZOLTAN_MALLOC(matrix->nPins * sizeof(ZOLTAN_GNO_TYPE));
if (matrix->nPins && !matrix->pinGNO){
ZOLTAN_FREE(&pinID);
MEMORY_ERROR;
}
for (i=0; i < matrix->nPins; i++)
matrix->pinGNO[i] = (ZOLTAN_GNO_TYPE)pinID[i] - fast_build_base;
ZOLTAN_FREE(&pinID);
}
else{
matrix->pinGNO = (ZOLTAN_GNO_TYPE *) pinID;
if (fast_build_base)
for (i=0; i < matrix->nPins; i++)
matrix->pinGNO[i] -= fast_build_base;
pinID = NULL;
}
}
/* if (matrix->opts.local) { /\* keep only local edges *\/ */
/* int *nnz_list; /\* nnz offset to delete *\/ */
/* int nnz; /\* number of nnz to delete *\/ */
/* int i; */
/* nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */
/* if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */
/* for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */
/* if (proclist[i] == zz->Proc) continue; */
/* nnz_list[nnz++] = i; */
/* } */
/* ZOLTAN_FREE(&proclist); */
/* Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */
/* } */
if (!matrix->opts.enforceSquare) {
/* Hyperedges name translation is different from the one of vertices */
matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_CALLOC(matrix->nY, sizeof(ZOLTAN_GNO_TYPE));
if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR;
/* int nGlobalEdges = 0; */
ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY,
matrix->opts.randomize, &matrix->globalY);
CHECK_IERR;
/* /\**************************************************************************************** */
/* * If it is desired to remove dense edges, divide the list of edges into */
/* * two lists. The ZHG structure will contain the removed edges (if final_output is true), */
/* * and the kept edges will be returned. */
/* ****************************************************************************************\/ */
/* totalNumEdges = zhg->globalHedges; */
/* ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */
/* &nLocalEdges, &nGlobalEdges, &nPins, */
/* &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */
/* if (nGlobalEdges < totalNumEdges){ */
/* /\* re-assign edge global numbers if any edges were removed *\/ */
/* ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */
/* randomizeInitDist, &totalNumEdges); */
/* if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */
/* goto End; */
/* } */
/* } */
/* We have to define ddY : yGNO, yGID, ywgt */
ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, gno_size_for_dd, zz->Num_GID,
0, matrix->globalY/zz->Num_Proc, 0);
/* Hope a linear assignment will help a little */
if (matrix->globalY/zz->Num_Proc)
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc);
/* Associate all the data with our yGNO */
Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, NULL, NULL, matrix->nY);
}
End:
ZOLTAN_FREE(&xpid);
ZOLTAN_FREE(&xLID);
ZOLTAN_FREE(&xGNO);
ZOLTAN_FREE(&xGID);
ZOLTAN_FREE(&xwgt);
ZOLTAN_FREE(&Input_Parts);
ZOLTAN_FREE(&proclist);
if (dd != NULL)
Zoltan_DD_Destroy(&dd);
/* Already stored in the DD */
ZOLTAN_FREE(&yGID);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int main(int argc, char *argv[])
{
int i, rc;
int myRank, numProcs;
float ver;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport, start_gid, num_nbors;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int *parts=NULL;
ZOLTAN_ID_PTR lids=NULL;
FILE *fp;
struct Zoltan_DD_Struct *dd;
GRAPH_DATA myGraph;
int gid_length = 1; /* our global IDs consist of 1 integer */
int lid_length = 1; /* our local IDs consist of 1 integer */
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
exit(0);
}
/******************************************************************
** Read graph from input file and distribute it
******************************************************************/
fp = fopen(fname, "r");
if (!fp){
if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
MPI_Finalize();
exit(1);
}
fclose(fp);
read_input_file(myRank, numProcs, fname, &myGraph);
/*fprintf(stderr,"%d have %d objects\n",myRank,myGraph.numMyVertices);*/
/******************************************************************
** Create a distributed data directory which maps vertex
** global IDs to their current partition number. We'll use this
** after migrating vertices, to update the partition in which
** our vertices neighbors are.
**
** Our local IDs (array "lids") are of type ZOLTAN_ID_TYPE because
** we are using Zoltan's distributed data directory. It assumes
** that a global ID is a sequence of "gid_length" ZOLTAN_ID_TYPEs.
** It assumes that a local ID is a sequence of "lid_length"
** ZOLTAN_ID_TYPEs.
******************************************************************/
rc = Zoltan_DD_Create(&dd, MPI_COMM_WORLD,
gid_length, /* length of a global ID */
lid_length, /* length of a local ID */
0, /* length of user data */
myGraph.numMyVertices, /* hash table size */
0); /* debug level */
parts = malloc(myGraph.numMyVertices * sizeof(int));
lids = malloc(myGraph.numMyVertices * sizeof(ZOLTAN_ID_TYPE));
for (i=0; i < myGraph.numMyVertices; i++){
parts[i] = myRank; /* part number of this vertex */
lids[i] = (ZOLTAN_ID_TYPE)i; /* local ID on my process for this vertex */
}
rc = Zoltan_DD_Update(dd,
myGraph.vertexGID,
lids,
NULL,
parts,
myGraph.numMyVertices);
myGraph.dd = dd;
/******************************************************************
** Create a Zoltan library structure for this instance of load
** balancing. Set the parameters and query functions that will
** govern the library's calculation. See the Zoltan User's
** Guide for the definition of these and many other parameters.
******************************************************************/
zz = Zoltan_Create(MPI_COMM_WORLD);
/* General parameters */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
/* Graph parameters */
Zoltan_Set_Param(zz, "CHECK_GRAPH", "2");
Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */
/* Query functions, defined in this source file */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph);
Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph);
Zoltan_Set_Obj_Size_Multi_Fn(zz, get_message_sizes,&myGraph);
Zoltan_Set_Pack_Obj_Multi_Fn(zz, pack_object_messages,&myGraph);
Zoltan_Set_Unpack_Obj_Multi_Fn(zz, unpack_object_messages,&myGraph);
Zoltan_Set_Mid_Migrate_PP_Fn(zz, mid_migrate,&myGraph);
/******************************************************************
** Visualize the graph partitioning before calling Zoltan.
******************************************************************/
if (myRank== 0){
printf("\nGraph partition before calling Zoltan\n");
}
showGraphPartitions(myRank, myGraph.dd);
/******************************************************************
** Zoltan can now partition the simple graph.
** In this simple example, we assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/*fprintf(stderr,"%d export %d import %d\n",myRank,numExport,numImport);*/
/******************************************************************
** Update the data directory with the new partition numbers
******************************************************************/
for (i=0; i < numExport; i++){
parts[exportLocalGids[i]] = exportToPart[i];
}
rc = Zoltan_DD_Update(dd,
myGraph.vertexGID,
lids,
NULL,
parts,
myGraph.numMyVertices);
/******************************************************************
** Migrate vertices to new partitions
******************************************************************/
rc = Zoltan_Migrate(zz,
numImport, importGlobalGids, importLocalGids,
importProcs, importToPart,
numExport, exportGlobalGids, exportLocalGids,
exportProcs, exportToPart);
/******************************************************************
** Use the data dictionary to find neighbors' partitions
******************************************************************/
start_gid = myGraph.numMyVertices - numImport;
num_nbors = myGraph.nborIndex[myGraph.numMyVertices] - myGraph.nborIndex[start_gid];
rc = Zoltan_DD_Find(dd,
(ZOLTAN_ID_PTR)(myGraph.nborGID + start_gid), NULL, NULL,
myGraph.nborPart + start_gid, num_nbors, NULL);
/******************************************************************
** Visualize the graph partitioning after calling Zoltan.
******************************************************************/
if (myRank == 0){
printf("Graph partition after calling Zoltan\n");
}
showGraphPartitions(myRank, myGraph.dd);
/******************************************************************
** Free the arrays allocated by Zoltan_LB_Partition, and free
** the storage allocated for the Zoltan structure.
******************************************************************/
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
Zoltan_Destroy(&zz);
/**********************
** all done ***********
**********************/
MPI_Finalize();
if (myGraph.vertex_capacity > 0){
free(myGraph.vertexGID);
free(myGraph.nborIndex);
if (myGraph.nbor_capacity > 0){
free(myGraph.nborGID);
free(myGraph.nborPart);
}
}
if (parts) free(parts);
if (lids) free(lids);
return 0;
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int num_gid_entries, /* # of entries for a global id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm /* iperm[rank[i]]=i, only for sequential ordering */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
int * vtxdist = NULL;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL, *local_iperm=NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value");
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.rank = rank;
zz->Order.iperm = iperm;
zz->Order.gids = gids;
zz->Order.lids = lids;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
#ifdef HAVE_MPI
strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN);
#else
strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN);
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
zz->Order.start_index = opt.start_index;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && rank)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
/* strcpy(opt.order_type, "GLOBAL"); */
}
#endif /* ZOLTAN_SCOTCH */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
if (!strcmp(opt.order_type, "GLOBAL"))
strcpy (opt.order_type, "DIST");
if (!strcmp(opt.order_type, "LOCAL"))
strcpy (opt.order_type, "SERIAL");
strcpy(zz->Order.order_type, opt.order_type);
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 3,
&local_gids, &local_rank, &local_iperm);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
if ((!(opt.return_args & RETURN_RANK) && (rank != NULL))
|| (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) {
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr) {
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) {
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) {
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* I store : GNO, rank, iperm */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
ZOLTAN_FREE(&local_iperm);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
int Zoltan_Order (
struct Zoltan_Struct *zz,
int num_gid_entries,
int num_obj,
ZOLTAN_ID_PTR gids,
ZOLTAN_ID_PTR permuted_global_ids
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids.
* num_gid_entries
* Output:
* permuted_global_ids
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
char msg[253];
sprintf(msg, "num_gid_entries=%d is not equal to parameter setting "
"NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
#ifdef HAVE_MPI
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "GLOBAL");
#else
strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "LOCAL");
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
/*
* Check that the user has allocated space for the return args.
*/
if (num_obj && !(gids && permuted_global_ids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
else if (!strcmp(opt.method, "LOCAL_HSFC"))
{
Order_fn = Zoltan_LocalHSFC_Order;
strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PTSCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_SCOTCH */
#ifdef ZOLTAN_HUND
else if (!strcasecmp(opt.method, "HUND")) {
ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL);
goto End;
}
#endif /* ZOLTAN_HUND */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/* TODO : Ask why useful ! */
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/************************************
* Check for required query function
************************************/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
/* TODO allocate all this stuff with the graph */
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 2,
&local_gids, &local_rank);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* TODO: At this time, I consider rank == permuted_global_ids */
/* I store : GNO, rank, permuted GID */
/* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0);
/* Hope a linear assignment will help a little */
if (local_num_obj)
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", permuted_global_ids[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
#ifdef ZOLTAN_HUND
End:
#endif /*ZOLTAN_HUND*/
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
static int create_a_graph()
{
int rc, i, sum, n, j;
float wgts[4];
long gid, nbors[4], count[2];
ldiv_t result;
float c;
c = sqrt((float)numGlobalVertices);
c = (c < 3.0) ? 3.0 : c;
cylCount = cylSize = (long)c;
numGlobalVertices = cylCount * cylSize;
result = ldiv(numGlobalVertices, (long)numProcs);
numMyVertices = (int)result.quot + (myRank < result.rem ? 1 : 0);
count[0] = (long)numMyVertices;
MPI_Scan(count+0, count+1, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD);
myFirstGID = count[1] - count[0];
numMyPins = 0;
for (i=0, gid = myFirstGID; i < numMyVertices; i++, gid++){
numMyPins += num_neighbors(gid);
}
vtxGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * numMyVertices);
nborIndex = (int *)malloc(sizeof(int) * (numMyVertices + 1));
nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * numMyPins);
nborProc = (int *)malloc(sizeof(int) * numMyPins);
edgeWgt = (float *)malloc(sizeof(float) * numMyPins);
if (numMyPins && !(vtxGID || nborIndex || nborGID || nborProc || edgeWgt)){
fprintf(stderr,"%d out of memory\n",myRank);
return 1;
}
nborIndex[0] = 0;
for (i=0, gid=myFirstGID, n=0; i < numMyVertices; i++, gid++){
vtxGID[i] = (ZOLTAN_ID_TYPE)gid;
sum = get_nbor_info(gid, nbors, wgts);
for (j=0; j < sum; j++, n++){
nborGID[n] = (ZOLTAN_ID_TYPE)nbors[j];
edgeWgt[n] = wgts[j];
if (nborGID[n] < myFirstGID)
nborProc[n] = myRank - 1;
else if (nborGID[n] >= myFirstGID + numMyVertices)
nborProc[n] = myRank + 1;
else
nborProc[n] = myRank;
}
nborIndex[i+1] = nborIndex[i] + sum;
}
rc = Zoltan_DD_Create(&dd, MPI_COMM_WORLD, 1, 0, 0, (int)(numGlobalVertices / numProcs), 0);
if ((rc != ZOLTAN_OK) && (rc != ZOLTAN_WARN)){
fprintf(stderr,"%d DD Create failure\n",myRank);
MPI_Abort(MPI_COMM_WORLD, 1);
}
rc = Zoltan_DD_Update(dd, vtxGID, NULL, NULL, NULL, numMyVertices);
if ((rc != ZOLTAN_OK) && (rc != ZOLTAN_WARN)){
fprintf(stderr,"%d DD Update failure in create\n",myRank);
return 1;
}
return 0;
}
int
Zoltan_Matrix_Sym(ZZ* zz, Zoltan_matrix *matrix, int bipartite)
{
static char *yo = "Zoltan_Matrix_Sym";
int ierr = ZOLTAN_OK;
Zoltan_Arc *tr_tab = NULL;
int i, j, cnt;
ZOLTAN_ID_PTR yGID = NULL;
int *ypid=NULL;
float *pinwgt=NULL;
int * ybipart = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (bipartite || !matrix->opts.enforceSquare) {
matrix->bipartite = 1;
matrix->redist = 1;
}
if (matrix->ywgtdim != zz->Obj_Weight_Dim)
FATAL_ERROR("Cannot form bipartite graph: vertex and edge weights are not consistant");
matrix->opts.symmetrize = 1;
/* Update the data directories */
tr_tab = (Zoltan_Arc*) ZOLTAN_MALLOC(sizeof(Zoltan_Arc)*(matrix->nPins*2+matrix->nY));
if (matrix->nPins && tr_tab == NULL) MEMORY_ERROR;
pinwgt = (float*)ZOLTAN_MALLOC((matrix->nPins*2+matrix->nY)*matrix->pinwgtdim*sizeof(float));
for (i = 0 ; i < 2 ; ++i) /* Copy pin weights */
memcpy(pinwgt + i*matrix->nPins*matrix->pinwgtdim*sizeof(float),
matrix->pinwgt, matrix->nPins*matrix->pinwgtdim*sizeof(float));
for (i=0, cnt = 0 ; i < matrix->nY ; ++i) {
for (j = matrix->ystart[i] ; j < matrix->yend[i] ; ++j) {
tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */
tr_tab[cnt].GNO[1] = matrix->pinGNO[j];
memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim,
matrix->pinwgtdim*sizeof(float));
cnt ++;
tr_tab[cnt].GNO[0] = matrix->pinGNO[j]; /* Symmetric arc */
tr_tab[cnt].GNO[1] = matrix->yGNO[i] + bipartite*matrix->globalX; /* new ordering */
memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim,
matrix->pinwgtdim*sizeof(float));
cnt ++;
}
if (matrix->ystart[i] == matrix->yend[i]) { /* Singleton */
tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */
tr_tab[cnt].GNO[1] = -1;
cnt ++;
}
}
ZOLTAN_FREE(&matrix->pinwgt);
Zoltan_Matrix_Remove_DupArcs(zz, cnt, tr_tab, pinwgt, matrix);
ZOLTAN_FREE(&tr_tab);
ZOLTAN_FREE(&pinwgt);
if (bipartite) {
int endX;
int * yGNO = NULL;
/* Update data directories */
yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nY);
ypid = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int));
ybipart = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int));
for (endX = 0 ; endX < matrix->nY ; ++endX) {
if (matrix->yGNO[endX] >= matrix->globalX)
break;
ybipart[endX] = 0;
}
/* Get Informations about X */
Zoltan_DD_Find (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, NULL,
endX, NULL);
yGNO = (int*)ZOLTAN_MALLOC(endX*sizeof(int));
for (i = endX ; i < matrix->nY ; ++i) {
yGNO[i-endX] = matrix->yGNO[i] - matrix->globalX;
/* TODO: add a something to have the correct ypid */
ybipart[endX] = 1;
}
/* Get Informations about Y */
Zoltan_DD_Find (matrix->ddY, (ZOLTAN_ID_PTR)yGNO,
yGID + endX*zz->Num_GID,
NULL, NULL,
matrix->nY-endX, NULL);
if (matrix->ddY != matrix->ddX)
Zoltan_DD_Destroy (&matrix->ddY);
Zoltan_DD_Destroy (&matrix->ddX);
matrix->globalX += matrix->globalY;
matrix->globalY = matrix->globalX;
/* I store : xGNO, xGID, xpid, bipart */
ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID,
1, matrix->globalX/zz->Num_Proc, 0);
matrix->ddY = matrix->ddX;
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
/* Associate all the data with our xyGNO */
Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, ybipart,
matrix->nY);
}
End:
ZOLTAN_FREE(&ybipart);
ZOLTAN_FREE(&ypid);
ZOLTAN_FREE(&pinwgt);
ZOLTAN_FREE(&yGID);
ZOLTAN_FREE(&tr_tab);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}