void Zoltan_Matrix_Free(Zoltan_matrix *m, int delete_flag) { if (m->yend != m->ystart + 1) ZOLTAN_FREE(&m->yend); if (FIELD_QUERY_DO_FREE(delete_flag, FIELD_YSTART)) ZOLTAN_FREE(&m->ystart); ZOLTAN_FREE(&m->yGNO); if (FIELD_QUERY_DO_FREE(delete_flag, FIELD_PINGNO)) ZOLTAN_FREE(&m->pinGNO); if (FIELD_QUERY_DO_FREE(delete_flag, FIELD_PINWGT)) ZOLTAN_FREE(&m->pinwgt); if (FIELD_QUERY_DO_FREE(delete_flag, FIELD_YGID)) ZOLTAN_FREE(&m->yGID); ZOLTAN_FREE(&m->ybipart); ZOLTAN_FREE(&m->ypid); if (m->ddY != m->ddX && m->ddY != NULL) Zoltan_DD_Destroy(&m->ddY); if (m->ddX != NULL) Zoltan_DD_Destroy(&m->ddX); memset (m, 0, sizeof(Zoltan_matrix)); }
int Zoltan_Matrix_Complete(ZZ* zz,Zoltan_matrix* m) { static char *yo = "Zoltan_Matrix_Complete"; int ierr = ZOLTAN_OK; if(m->completed) return (ZOLTAN_OK); ZOLTAN_TRACE_ENTER(zz, yo); if (m->yend != m->ystart +1 ) {/* Not in compact mode yet */ int y; /* I have to also rewrites all the pinarrays */ for (y = 1 ; y <= m->nY ; ++y) { int length; if (m->ystart[y] == m->yend[y-1]) /* No hole */ continue; length = m->yend[y]-m->ystart[y]; memcpy(m->pinGNO+m->yend[y-1], m->pinGNO+m->ystart[y], length*sizeof(ZOLTAN_GNO_TYPE)); memcpy(m->pinwgt+m->yend[y-1]*m->pinwgtdim, m->pinGNO+m->ystart[y]*m->pinwgtdim, length*sizeof(float)*m->pinwgtdim); m->ystart[y] = m->yend[y-1]; m->yend[y] = m->ystart[y] + length; } ZOLTAN_FREE(&m->yend); m->yend = m->ystart + 1; } /* Update data directories */ m->yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY); m->ypid = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int)); if (m->bipartite) m->ybipart = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int)); if (m->nY && ((m->yGID == NULL) || (m->ypid == NULL) || (m->bipartite && m->ybipart == NULL))) MEMORY_ERROR; /* Get Informations about Y */ Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, m->yGID, (char *)m->ypid, m->ybipart, m->nY, NULL); if (m->ddY != m->ddX) { Zoltan_DD_Destroy(&m->ddY); m->ddY = NULL; } m->completed = 1; End: ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Matrix_Vertex_Info(ZZ* zz, const Zoltan_matrix * const m, ZOLTAN_ID_PTR lid, float *wwgt, int *input_part) { static char *yo = "Zoltan_Matrix_Vertex_Info"; int ierr = ZOLTAN_OK; int nX; ZOLTAN_ID_PTR l_gid = NULL; ZOLTAN_ID_PTR l_lid = NULL; float * l_xwgt = NULL; int *l_input_part = NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (m->completed == 0) { ierr = ZOLTAN_FATAL; goto End; } ierr = Zoltan_Get_Obj_List(zz, &nX, &l_gid, &l_lid, zz->Obj_Weight_Dim, &l_xwgt, &l_input_part); ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, zz->Num_LID, zz->Obj_Weight_Dim*sizeof(float)/sizeof(int), nX, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (dd, l_gid, l_lid, (ZOLTAN_ID_PTR) l_xwgt,l_input_part, nX); ZOLTAN_FREE(&l_gid); ZOLTAN_FREE(&l_lid); ZOLTAN_FREE(&l_xwgt); ZOLTAN_FREE(&l_input_part); ierr = Zoltan_DD_Find (dd, m->yGID, lid, (ZOLTAN_ID_PTR)wwgt, input_part, m->nY, NULL); End: if (dd != NULL) Zoltan_DD_Destroy(&dd); ZOLTAN_FREE(&l_gid); ZOLTAN_FREE(&l_lid); ZOLTAN_FREE(&l_xwgt); ZOLTAN_FREE(&l_input_part); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
size_t findUniqueGidsCommon( size_t num_keys, int num_gid, ZOLTAN_ID_PTR ddkeys, char *ddnewgids, MPI_Comm mpicomm ) { int num_lid = 0; // Local IDs not needed int debug_level = 0; int num_user = sizeof(gno_t); Zoltan_DD_Struct *dd = NULL; Zoltan_DD_Create(&dd, mpicomm, num_gid, num_lid, num_user, num_keys, debug_level); ZOLTAN_ID_PTR ddnotneeded = NULL; // Local IDs not needed Zoltan_DD_Update(dd, ddkeys, ddnotneeded, ddnewgids, NULL, int(num_keys)); ////////// // Insert unique GIDs for DD entries in User data here. // Get value of first gid on this rank ssize_t nDDEntries = (ssize_t)(dd->nodecnt); ssize_t firstIdx; MPI_Scan(&nDDEntries, &firstIdx, 1, MPI_LONG_LONG, MPI_SUM, mpicomm); firstIdx -= nDDEntries; // do not include this rank's entries in prefix sum // Loop over all directory entries, updating their userdata with updated gid DD_NodeIdx cnt = 0; for (DD_NodeIdx i = 0; i < dd->nodelistlen; i++) { DD_Node *ptr = &(dd->nodelist[i]); if (!(ptr->free)) { char *userchar = (char*)(ptr->gid + (dd->gid_length + dd->lid_length)); gno_t *newgid = (gno_t*) userchar; *newgid = gno_t(firstIdx + cnt); cnt++; } } /////////// // Retrieve the global numbers and put in the result gids vector Zoltan_DD_Find(dd, ddkeys, ddnotneeded, ddnewgids, NULL, int(num_keys), NULL); Zoltan_DD_Destroy(&dd); ssize_t nUnique = 0; MPI_Allreduce(&nDDEntries, &nUnique, 1, MPI_LONG_LONG, MPI_SUM, mpicomm); return size_t(nUnique); }
static void free_graph() { if (dd) Zoltan_DD_Destroy(&dd); if (vtxGID) free(vtxGID); if (nborIndex) free(nborIndex); if (nborGID) free(nborGID); if (nborProc) free(nborProc); if (edgeWgt) free(edgeWgt); vtxGID = nborGID = NULL; nborIndex = nborProc = NULL; edgeWgt = NULL; dd=NULL; }
int Zoltan_DD_Copy_To(Zoltan_DD_Directory **toptr, Zoltan_DD_Directory *from) { static char *yo = "Zoltan_DD_Copy_To"; int i, proc = 0; Zoltan_DD_Directory *to= NULL; if (!toptr){ return ZOLTAN_FATAL; } if (*toptr){ Zoltan_DD_Destroy(toptr); } if (from){ proc = from->my_proc; to = *toptr = (Zoltan_DD_Directory *)ZOLTAN_MALLOC( sizeof (Zoltan_DD_Directory) + (from->table_length * sizeof(DD_Node*))); if (!to){ ZOLTAN_PRINT_ERROR(proc, yo, "Insufficient memory."); return ZOLTAN_MEMERR; } *to = *from; MPI_Comm_dup(from->comm, &(to->comm)); for (i=0; i<to->table_length; i++){ allocate_copy_list(&(to->table[i]), from->table[i], to->node_size); } } return ZOLTAN_OK; }
int Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix) { static char *yo = "Zoltan_Matrix_Build"; int ierr = ZOLTAN_OK; int nX; int *xGNO = NULL; ZOLTAN_ID_PTR xLID=NULL; ZOLTAN_ID_PTR xGID=NULL; ZOLTAN_ID_PTR yGID=NULL; ZOLTAN_ID_PTR pinID=NULL; float *xwgt = NULL; int * Input_Parts=NULL; struct Zoltan_DD_Struct *dd = NULL; int *proclist = NULL; int *xpid = NULL; int i; ZOLTAN_TRACE_ENTER(zz, yo); memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */ memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options)); /**************************************************/ /* Obtain vertex information from the application */ /**************************************************/ ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID, zz->Obj_Weight_Dim, &xwgt, &Input_Parts); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data"); goto End; } ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&xwgt); /*******************************************************************/ /* Assign vertex consecutive numbers (gnos) */ /*******************************************************************/ if (matrix->opts.speed == MATRIX_FULL_DD) { /* Zoltan computes a translation */ if (nX) { xGNO = (int*) ZOLTAN_MALLOC(nX*sizeof(int)); if (xGNO == NULL) MEMORY_ERROR; } ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices"); goto End; } ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, 1, 0, nX, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL, NULL, nX); } else { /* We don't want to use the DD */ xGNO = (int *) xGID; MPI_Allreduce(&nX, &matrix->globalX, 1, MPI_INT, MPI_SUM, zz->Communicator); } /* I store : xGNO, xGID, xpid, */ ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID, 1, matrix->globalX/zz->Num_Proc, 0); CHECK_IERR; /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc); /* Associate all the data with our xGNO */ xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int)); if (nX >0 && xpid == NULL) MEMORY_ERROR; for (i = 0 ; i < nX ; ++i) xpid[i] = zz->Proc; Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (ZOLTAN_ID_PTR) xpid, NULL, nX); ZOLTAN_FREE(&xpid); if (matrix->opts.pinwgt) matrix->pinwgtdim = zz->Edge_Weight_Dim; else matrix->pinwgtdim = 0; ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt); CHECK_IERR; matrix->nY_ori = matrix->nY; if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){ goto End; } if (matrix->opts.enforceSquare && matrix->redist) { /* Convert yGID to yGNO using the same translation as x */ /* Needed for graph : rowID = colID */ /* y and x may have different distributions */ matrix->yGNO = (int*)ZOLTAN_MALLOC(matrix->nY * sizeof(int)); if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR; ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL, matrix->nY, NULL); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n"); ierr = ZOLTAN_FATAL; goto End; } } if (matrix->opts.local) { /* keep only local edges */ proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); if (matrix->nPins && proclist == NULL) MEMORY_ERROR; } else proclist = NULL; /* Convert pinID to pinGNO using the same translation as x */ if (matrix->opts.speed == MATRIX_FULL_DD) { matrix->pinGNO = (int*)ZOLTAN_MALLOC(matrix->nPins* sizeof(int)); if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) MEMORY_ERROR; ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL, matrix->nPins, proclist); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n"); ierr = ZOLTAN_FATAL; goto End; } ZOLTAN_FREE(&pinID); Zoltan_DD_Destroy(&dd); dd = NULL; } else { matrix->pinGNO = (int *) pinID; pinID = NULL; } /* if (matrix->opts.local) { /\* keep only local edges *\/ */ /* int *nnz_list; /\* nnz offset to delete *\/ */ /* int nnz; /\* number of nnz to delete *\/ */ /* int i; */ /* nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */ /* if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */ /* for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */ /* if (proclist[i] == zz->Proc) continue; */ /* nnz_list[nnz++] = i; */ /* } */ /* ZOLTAN_FREE(&proclist); */ /* Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */ /* } */ if (!matrix->opts.enforceSquare) { /* Hyperedges name translation is different from the one of vertices */ matrix->yGNO = (int*)ZOLTAN_CALLOC(matrix->nY, sizeof(int)); if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR; /* int nGlobalEdges = 0; */ ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY, matrix->opts.randomize, &matrix->globalY); CHECK_IERR; /* /\**************************************************************************************** */ /* * If it is desired to remove dense edges, divide the list of edges into */ /* * two lists. The ZHG structure will contain the removed edges (if final_output is true), */ /* * and the kept edges will be returned. */ /* ****************************************************************************************\/ */ /* totalNumEdges = zhg->globalHedges; */ /* ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */ /* &nLocalEdges, &nGlobalEdges, &nPins, */ /* &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */ /* if (nGlobalEdges < totalNumEdges){ */ /* /\* re-assign edge global numbers if any edges were removed *\/ */ /* ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */ /* randomizeInitDist, &totalNumEdges); */ /* if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */ /* goto End; */ /* } */ /* } */ /* We have to define ddY : yGNO, yGID, ywgt */ ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, 1, zz->Num_GID, 0, matrix->globalY/zz->Num_Proc, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc); /* Associate all the data with our yGNO */ Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, NULL, NULL, matrix->nY); } End: ZOLTAN_FREE(&xpid); ZOLTAN_FREE(&xLID); ZOLTAN_FREE(&xGNO); ZOLTAN_FREE(&xGID); ZOLTAN_FREE(&xwgt); ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&proclist); if (dd != NULL) Zoltan_DD_Destroy(&dd); /* Already stored in the DD */ ZOLTAN_FREE(&yGID); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int main (int argc, char *argv[]) { Zoltan_DD_Directory *dd; ZTIMER *zt; int myproc; /* MPI rank, my processor's MPI ID */ int nproc; /* MPI size, number of processors */ ZOLTAN_ID_PTR glist = NULL; /* list of GIDs */ ZOLTAN_ID_PTR llist = NULL; /* list of LIDs */ ZOLTAN_ID_PTR ulist = NULL; /* list of user data of type ZOLTAN_ID_TYPE */ int *plist = NULL; /* list of partitions */ int *olist = NULL; /* list of owners */ Param param ; /* program's adjustable parameters */ char *store = NULL; /* non directory storage of test data */ Data *data = NULL; /* pointer into store */ /* these are all temporary variables */ int new_owner; int count; int i; char *p; int err; int errcount; int loop; char str[100]; /* for building output messages */ static int timer[7] = {-1,-1,-1,-1,-1,-1,-1}; char *yo = "DD_Main"; /* initialize MPI communications, ignore errors */ MPI_Init (&argc, &argv); MPI_Comm_rank (MPI_COMM_WORLD, &myproc); MPI_Comm_size (MPI_COMM_WORLD, &nproc); get_params (¶m); /* read input parameters */ zt = Zoltan_Timer_Create(1); MACRO_TIMER_START(5, "Total time", 0); MACRO_TIMER_START(0, "DD_Create time", 0); err = Zoltan_DD_Create (&dd, MPI_COMM_WORLD, param.glen, param.llen, param.ulen, param.tlen, param.debug_level); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Create"); MACRO_TIMER_STOP(0); param.slen = sizeof (Data) + sizeof(ZOLTAN_ID_TYPE) * (param.glen + param.llen + param.ulen); param.slen = Zoltan_Align(param.slen); store = (char*) ZOLTAN_MALLOC (param.count * param.slen); /* allocate storage for various lists */ glist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC(sizeof(ZOLTAN_ID_TYPE) * param.count * param.glen); plist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count); olist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count); if (param.llen) llist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE) * param.count * param.llen); if (param.ulen) ulist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE) * param.count * param.ulen) ; if (store == NULL || glist == NULL || (param.llen != 0 && llist == NULL) || (param.ulen != 0 && ulist == NULL) || plist == NULL || olist == NULL) { ZOLTAN_PRINT_ERROR (myproc, yo, "Unable to malloc storage lists"); return ZOLTAN_MEMERR; } initialize_data (¶m, store, nproc); /* create & update directory with myproc's initial simulated GIDs */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data *)p)->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data*)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data *)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen, ((Data *)p)->id + (param.glen + param.llen)); plist [count] = ((Data *)p)->partition; ++count; } MACRO_TIMER_START (1, "DD_Update timer", 0); err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); i = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) { ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id); ++i; } MACRO_TIMER_START(2, "DD_Find timer", 0); err = Zoltan_DD_Find (dd, glist, llist, ulist, plist, param.count, olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); errcount = 0; for (i = 0; i < param.count; i++) if (olist[i] != ((Data *) (store + i * param.slen))->new_owner) ++errcount; if (errcount) sprintf (str, "FIRST TEST FAILED, errcount is %d", errcount); else sprintf (str, "FIRST TEST SUCCESSFUL"); ZOLTAN_PRINT_INFO (myproc, yo, str); /* repeatedly simulate moving "dots" around the system */ for (loop = 0; loop < param.nloops; loop++) { for (p = store; p < store + param.count * param.slen; p += param.slen) ((Data*) p)->old_owner = ((Data*) p)->new_owner; /* randomly exchange some dots and randomly reassign others */ for (i = 0; i < (param.pmove * param.count)/100; i++) { Data *d1, *d2; d1 = (Data*) (store + param.slen * (rand() % param.count)); d2 = (Data*) (store + param.slen * (rand() % param.count)); new_owner = d1->new_owner; d1->new_owner = d2->new_owner; d2->new_owner = new_owner; } for (i = 0; i < (param.count * param.pscatter)/100; i++) ((Data*) (store + param.slen *(rand() % param.count)))->new_owner = rand() % nproc; /* determine which new GIDs myproc gained, and update directory */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == myproc) { ((Data*)p)->id[param.glen] = count; /* set LID */ ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data *)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data*)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen, ((Data*)p)->id + (param.glen + param.llen)); plist [count] = ((Data *)p)->partition; ++count; } MACRO_TIMER_START(1, "DD_Update", 0); err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); /* use directory to "find" GIDs */ i = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) { ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id); ++i; } MACRO_TIMER_START(2, "DD_Find timer", 0); if (loop % 5 == 0) err = Zoltan_DD_Find(dd,glist,NULL,ulist,plist,param.count,olist); else if (loop % 7 == 0) err = Zoltan_DD_Find(dd,glist,llist,NULL,plist,param.count,olist); else if (loop % 9 == 0) err = Zoltan_DD_Find(dd,glist,llist,ulist,NULL,param.count,olist); else if (loop % 2 == 0) err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,NULL); else err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); if (loop % 2) { errcount = 0 ; for (i = 0; i < param.count; i++) if (olist[i] != ((Data *)(store + i * param.slen))->new_owner) ++errcount; if (errcount) sprintf (str,"LOOP %d TEST FAILED, errcount is %d",loop,errcount); else sprintf (str, "LOOP %d TEST SUCCESSFUL", loop); ZOLTAN_PRINT_INFO (myproc, yo, str) ; } else { sprintf (str, "LOOP %d Completed", loop); ZOLTAN_PRINT_INFO (myproc, yo, str); } /* randomly remove a percentage of GIDs from the directory */ count = 0; for (i = 0; i < (param.count * param.pdelete)/100; i++) { data = (Data*) (store + param.slen * (rand() % param.count)); if (data->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id); ++count; } data->new_owner = NO_PROC; } MACRO_TIMER_START(3, "DD_Remove timer", 0); err = Zoltan_DD_Remove (dd, glist, count); MACRO_TIMER_STOP(3); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Remove"); /* update directory (put directory entries back in) */ for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == NO_PROC) ((Data*)p)->new_owner = loop % nproc; /* place in new location */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == myproc) { ZOLTAN_SET_ID(param.glen,glist+count*param.glen,((Data*)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data*)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID(param.ulen, ulist + count * param.ulen, ((Data *)p)->id + (param.glen + param.llen)); plist [count] = ((Data*)p)->partition; ++count; } MACRO_TIMER_START(1, "DD_Update timer", 0); err = Zoltan_DD_Update (dd, glist, NULL, NULL, NULL, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); } /* now Find directory info for GIDs myproc now owns and validate */ count = 0; for (i = 0; i < param.count; i++) { data = (Data *) (store + i * param.slen); if (data->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id); ++count; } } MACRO_TIMER_START(2, "DD_Find", 0); err = Zoltan_DD_Find (dd, glist, NULL, NULL, NULL, count, olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); errcount = 0; for (i = 0; i < count; i++) if (olist[i] != myproc) ++errcount; if (errcount) { sprintf (str, "errcount is %d", errcount); ZOLTAN_PRINT_ERROR (myproc, yo, str); } else ZOLTAN_PRINT_INFO (myproc, yo, "TEST SUCCESSFUL"); Zoltan_DD_Stats (dd); /* done, now free memory, stop MPI & directory, return */ ZOLTAN_FREE (&store); ZOLTAN_FREE (&glist); ZOLTAN_FREE (&plist); ZOLTAN_FREE (&olist); if (param.llen) ZOLTAN_FREE (&llist); if (param.ulen) ZOLTAN_FREE (&ulist); ZOLTAN_PRINT_INFO (myproc, yo, "Completing program"); MACRO_TIMER_START(4, "DD_Destroy", 0); Zoltan_DD_Destroy (&dd); MACRO_TIMER_STOP(4); MACRO_TIMER_STOP(5); Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stderr); MPI_Finalize (); return ZOLTAN_OK; }
int Zoltan_Color_Test( ZZ *zz, /* Zoltan structure */ int *num_gid_entries, /* # of entries for a global id */ int *num_lid_entries, /* # of entries for a local id */ int num_obj, /* Input: number of objects */ ZOLTAN_ID_PTR global_ids, /* Input: global ids of the vertices */ /* The application must allocate enough space */ ZOLTAN_ID_PTR local_ids, /* Input: local ids of the vertices */ /* The application must allocate enough space */ int *color_exp /* Input: Colors assigned to local vertices */ ) { static char *yo = "color_test_fn"; int nvtx = num_obj; /* number of vertices */ int i, j; int ierr = ZOLTAN_OK; int ferr = ZOLTAN_OK; /* final error signal */ char coloring_problem; /* Input: which coloring to perform; currently only supports D1, D2 coloring and variants */ char coloring_problemStr[MAX_PARAM_STRING_LEN]; /* string version coloring problem name */ int ss=100; char comm_pattern='S', coloring_order='I', coloring_method='F'; int comm[2],gcomm[2]; int *color=NULL; int *adjproc=NULL, *xadj=NULL; ZOLTAN_GNO_TYPE gvtx; /* number of global vertices */ ZOLTAN_GNO_TYPE *vtxdist=NULL, *adjncy=NULL; ZG graph; ZOLTAN_GNO_TYPE *requested_GNOs = NULL; /* Return GNOs of the requested GIDs. */ int *loc_partialD2 = NULL; /* local binary array showing which vertices to be colored */ int *partialD2 = NULL; /* global binary array showing which vertices to be colored */ struct Zoltan_DD_Struct *dd_color; /* DDirectory for colors */ ZOLTAN_GNO_TYPE *local_GNOs = NULL; ZOLTAN_GNO_TYPE *global_GNOs = NULL; memset (&graph, 0, sizeof(ZG)); /* PARAMETER SETTINGS */ Zoltan_Bind_Param(Color_params, "COLORING_PROBLEM", (void *) &coloring_problemStr); Zoltan_Bind_Param(Color_params, "SUPERSTEP_SIZE", (void *) &ss); Zoltan_Bind_Param(Color_params, "COMM_PATTERN", (void *) &comm_pattern); Zoltan_Bind_Param(Color_params, "VERTEX_VISIT_ORDER", (void *) &coloring_order); Zoltan_Bind_Param(Color_params, "COLORING_METHOD", (void *) &coloring_method); strncpy(coloring_problemStr, "distance-1", MAX_PARAM_STRING_LEN); Zoltan_Assign_Param_Vals(zz->Params, Color_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* Check validity of parameters - they should be consistent with Zoltan_Color */ if (!strcasecmp(coloring_problemStr, "distance-1")) coloring_problem = '1'; else if (!strcasecmp(coloring_problemStr, "distance-2")) coloring_problem = '2'; else if (!strcasecmp(coloring_problemStr, "partial-distance-2") || !strcasecmp(coloring_problemStr, "bipartite")) coloring_problem = 'P'; else { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Unknown coloring requested. Using Distance-1 coloring."); coloring_problem = '1'; } if (ss == 0) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid superstep size. Using default value 100."); ss = 100; } if (comm_pattern != 'S' && comm_pattern != 'A') { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid communication pattern. Using synchronous communication (S)."); comm_pattern = 'S'; } if (comm_pattern == 'A' && (coloring_problem == '2' || coloring_problem == 'P')) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Asynchronous communication pattern is not implemented for distance-2 coloring and its variants. Using synchronous communication (S)."); comm_pattern = 'S'; } if (coloring_order != 'I' && coloring_order != 'B' && coloring_order != 'U' && coloring_order != 'L' && coloring_order != 'N' && coloring_order != 'S') { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring order. Using internal first coloring order (I)."); coloring_order = 'I'; } if (coloring_order == 'U' && (coloring_problem == '2' || coloring_problem == 'P')) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Interleaved coloring order is not implemented for distance-2 coloring and its variants. Using internal first coloring order (I)."); coloring_order = 'I'; } if (coloring_method !='F') { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring method. Using first fit method (F)."); coloring_method = 'F'; } /* Compute Max number of array entries per ID over all processors. This is a sanity-maintaining step; we don't want different processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; /* Return if this processor is not in the Zoltan structure's communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) return ZOLTAN_OK; /* BUILD THE GRAPH */ /* Check that the user has allocated space for the return args. */ if (!color_exp) ZOLTAN_COLOR_ERROR(ZOLTAN_FATAL, "Color argument is NULL. Please give colors of local vertices."); requested_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(num_obj * sizeof(ZOLTAN_GNO_TYPE)); Zoltan_ZG_Build (zz, &graph, 0, 1, num_obj, global_ids, requested_GNOs); Zoltan_ZG_Export (zz, &graph, &gvtx, &nvtx, NULL, NULL, &vtxdist, &xadj, &adjncy, &adjproc, NULL, NULL); if (gvtx > (ZOLTAN_GNO_TYPE)INT_MAX){ if (zz->Proc == 0){ fprintf(stderr, "Zoltan_Color_Test assumes number of vertices (%ld) is less than INT_MAX\n",gvtx); } ierr = ZOLTAN_FATAL; goto End; } KDDKDDKDD(zz->Proc, "COLORTEST DD"); /* Exchange global color information */ if (vtxdist[zz->Num_Proc] && !(color = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int)))) MEMORY_ERROR; if (nvtx && !(local_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(nvtx * sizeof(ZOLTAN_GNO_TYPE)))) MEMORY_ERROR; for (i=0; i<nvtx; ++i) local_GNOs[i] = i+vtxdist[zz->Proc]; if (vtxdist[zz->Num_Proc] && !(global_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(vtxdist[zz->Num_Proc] * sizeof(ZOLTAN_GNO_TYPE)))) MEMORY_ERROR; for (i=0; i<vtxdist[zz->Num_Proc]; ++i) global_GNOs[i] = i; ierr = Zoltan_DD_Create (&dd_color, zz->Communicator, sizeof(ZOLTAN_GNO_TYPE)/sizeof(ZOLTAN_ID_TYPE), 0, 0, num_obj, 0); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot construct DDirectory."); /* Put req obs with 1 but first inialize the rest with 0 */ ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)local_GNOs, NULL, NULL, color, nvtx); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory."); ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)requested_GNOs, NULL, NULL, color_exp, num_obj); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory."); /* Get requested colors from the DDirectory. */ ierr = Zoltan_DD_Find (dd_color, (ZOLTAN_ID_PTR)global_GNOs, NULL, NULL, color, vtxdist[zz->Num_Proc], NULL); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot find object in DDirectory."); /* Free DDirectory */ Zoltan_DD_Destroy(&dd_color); ZOLTAN_FREE(&local_GNOs); ZOLTAN_FREE(&global_GNOs); KDDKDDKDD(zz->Proc, "COLORTEST CHECK"); if (coloring_problem == 'P' || coloring_problem == '2') { if (vtxdist[zz->Num_Proc] && !(partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int)))) MEMORY_ERROR; if (vtxdist[zz->Num_Proc] && !(loc_partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int)))) MEMORY_ERROR; if (coloring_problem == 'P') { for (i=0; i<num_obj; ++i) { int gno=requested_GNOs[i]; loc_partialD2[gno] = 1; } MPI_Allreduce(loc_partialD2, partialD2, vtxdist[zz->Num_Proc], MPI_INT, MPI_LOR, zz->Communicator); } else { for (i=0; i<vtxdist[zz->Num_Proc]; ++i) partialD2[i] = 1; } } /* Check if there is an error in coloring */ if (coloring_problem == '1') { for (i=0; i<nvtx; i++) { int gno = i + (int)vtxdist[zz->Proc]; if (color[gno] <= 0) { /* object i is not colored properly */ ierr = ZOLTAN_FATAL; printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]); break; } for (j = xadj[i]; j < xadj[i+1]; ++j) { int v = (int)adjncy[j]; if (color[gno] == color[v]) { /* neighbors have the same color */ ierr = ZOLTAN_FATAL; printf("Error in coloring! u:%d, v:%d, cu:%d, cv:%d\n", gno, v, color[gno], color[v]); break; } } if (ierr == ZOLTAN_FATAL) break; } } else if (coloring_problem == '2' || coloring_problem == 'P') { for (i=0; i<nvtx; i++) { int gno = i + (int)vtxdist[zz->Proc]; if (partialD2[gno] && color[gno] <= 0) { /* object i is not colored properly */ ierr = ZOLTAN_FATAL; printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]); break; } for (j = xadj[i]; j < xadj[i+1]; ++j) { int v = (int)adjncy[j], k; if (partialD2[v] && color[v] <= 0) { ierr = ZOLTAN_FATAL; printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]); break; } if (partialD2[gno] && partialD2[v] && color[gno] == color[v]) { /* d-1 neighbors have the same color */ ierr = ZOLTAN_FATAL; printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]); break; } for (k = j+1; k < xadj[i+1]; ++k) { int w = (int)adjncy[k]; if (partialD2[v] && partialD2[w] && color[v] == color[w]) { /* d-2 neighbors have the same color */ ierr = ZOLTAN_FATAL; printf("Error in coloring! d2-neigh: v:%d, w:%d, cv:%d, cw:%d pv:%d pw:%d\n", v, w, color[v], color[w], partialD2[v], partialD2[w]); break; } } } if (ierr == ZOLTAN_FATAL) break; } } else ZOLTAN_COLOR_ERROR(ZOLTAN_WARN, "Zoltan_Color_Test is only implemented for distance-1 and distance-2 coloring. Unknown coloring, skipping verification."); End: KDDKDDKDD(zz->Proc, "COLORTEST DONE"); if (ierr==ZOLTAN_FATAL) ierr = 2; MPI_Allreduce(&ierr, &ferr, 1, MPI_INT, MPI_MAX, zz->Communicator); if (ferr == 2) ierr = ZOLTAN_FATAL; else ierr = ZOLTAN_OK; Zoltan_ZG_Free (&graph); ZOLTAN_FREE(&adjproc); ZOLTAN_FREE(&color); ZOLTAN_FREE(&requested_GNOs); ZOLTAN_FREE(&partialD2); ZOLTAN_FREE(&loc_partialD2); return ierr; }
int Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix, int request_GNOs, /* Input: Flag indicating calling code needs translation of extra GIDs to GNOs; partial 2D coloring needs this feature. */ int num_requested, /* Input: Local # of GIDs needing translation to GNOs. */ ZOLTAN_ID_PTR requested_GIDs, /* Input: Calling code requests the GNOs for these GIDs */ ZOLTAN_GNO_TYPE *requested_GNOs /* Output: Return GNOs of the requested GIDs. */ ) { static char *yo = "Zoltan_Matrix_Build"; int ierr = ZOLTAN_OK; int nX; ZOLTAN_GNO_TYPE tmp; ZOLTAN_GNO_TYPE *xGNO = NULL; ZOLTAN_ID_PTR xLID=NULL; ZOLTAN_ID_PTR xGID=NULL; ZOLTAN_ID_PTR yGID=NULL; ZOLTAN_ID_PTR pinID=NULL; float *xwgt = NULL; int * Input_Parts=NULL; struct Zoltan_DD_Struct *dd = NULL; int *proclist = NULL; int *xpid = NULL; int i; int gno_size_for_dd; MPI_Datatype zoltan_gno_mpi_type; int use_full_dd = (opt->speed == MATRIX_FULL_DD); int fast_build_base = opt->fast_build_base; matrix->opts.speed = opt->speed; matrix->opts.fast_build_base = opt->fast_build_base; ZOLTAN_TRACE_ENTER(zz, yo); if (num_requested && (!requested_GIDs || !requested_GNOs)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error in requested input; needed arrays are NULL.\n"); } /* ZOLTAN_GNO_TYPE is >= ZOLTAN_ID_TYPE */ gno_size_for_dd = sizeof(ZOLTAN_GNO_TYPE) / sizeof(ZOLTAN_ID_TYPE); zoltan_gno_mpi_type = Zoltan_mpi_gno_type(); memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */ memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options)); /**************************************************/ /* Obtain vertex information from the application */ /**************************************************/ ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID, zz->Obj_Weight_Dim, &xwgt, &Input_Parts); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data"); goto End; } ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&xwgt); /*******************************************************************/ /* Assign vertex consecutive numbers (gnos) */ /*******************************************************************/ if (use_full_dd) { /* Zoltan computes a translation */ /* Have to use Data Directory if request_GNOs is true. */ if (nX) { xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE)); if (xGNO == NULL) MEMORY_ERROR; } ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices"); goto End; } ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, gno_size_for_dd, 0, nX, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL, NULL, nX); if (request_GNOs) { Zoltan_DD_Find(dd, requested_GIDs, (ZOLTAN_ID_PTR) requested_GNOs, NULL, NULL, num_requested, NULL); } } else { /* We don't want to use the DD */ /* * KDDKDD 2/10/11 This code cannot work when NUM_GID_ENTRIES>1. * KDDKDD 2/10/11 The assumption is that, if a user sets the * KDDKDD 2/10/11 appropriate parameter to enable this code, the user * KDDKDD 2/10/11 knows that his GIDs are compatible with integers. */ if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){ xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE)); if (nX && xGNO == NULL) MEMORY_ERROR; for (i=0; i < nX; i++) xGNO[i] = (ZOLTAN_GNO_TYPE)xGID[i] - fast_build_base; } else { xGNO = (ZOLTAN_GNO_TYPE *)xGID; if (fast_build_base) for (i = 0; i < nX; i++) xGNO[i] -= fast_build_base; } for (i = 0; i < num_requested; i++) requested_GNOs[i] = (ZOLTAN_GNO_TYPE)requested_GIDs[i] - fast_build_base; tmp = (ZOLTAN_GNO_TYPE)nX; MPI_Allreduce(&tmp, &matrix->globalX, 1, zoltan_gno_mpi_type, MPI_SUM, zz->Communicator); } /* I store : xGNO, xGID, xpid, */ ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, gno_size_for_dd, zz->Num_GID, sizeof(int), matrix->globalX/zz->Num_Proc, 0); CHECK_IERR; /* Hope a linear assignment will help a little */ if (matrix->globalX/zz->Num_Proc) Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc); /* Associate all the data with our xGNO */ xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int)); if (nX >0 && xpid == NULL) MEMORY_ERROR; for (i = 0 ; i < nX ; ++i) xpid[i] = zz->Proc; Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (char *)xpid, NULL, nX); ZOLTAN_FREE(&xpid); if (matrix->opts.pinwgt) matrix->pinwgtdim = zz->Edge_Weight_Dim; else matrix->pinwgtdim = 0; ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt, use_full_dd); CHECK_IERR; matrix->nY_ori = matrix->nY; if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){ goto End; } if (matrix->opts.enforceSquare && matrix->redist) { /* Convert yGID to yGNO using the same translation as x */ /* Needed for graph : rowID = colID */ /* y and x may have different distributions */ matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nY * sizeof(ZOLTAN_GNO_TYPE)); if (matrix->nY && matrix->yGNO == NULL) { ZOLTAN_FREE(&pinID); MEMORY_ERROR; } ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL, matrix->nY, NULL); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n"); ierr = ZOLTAN_FATAL; ZOLTAN_FREE(&pinID); goto End; } } if (matrix->opts.local) { /* keep only local edges */ proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); if (matrix->nPins && proclist == NULL) { ZOLTAN_FREE(&pinID); MEMORY_ERROR; } } else proclist = NULL; /* Convert pinID to pinGNO using the same translation as x */ if (use_full_dd) { matrix->pinGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nPins* sizeof(ZOLTAN_GNO_TYPE)); if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) { ZOLTAN_FREE(&pinID); MEMORY_ERROR; } ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL, matrix->nPins, proclist); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n"); ierr = ZOLTAN_FATAL; goto End; } ZOLTAN_FREE(&pinID); Zoltan_DD_Destroy(&dd); dd = NULL; } else { if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){ matrix->pinGNO = (ZOLTAN_GNO_TYPE *)ZOLTAN_MALLOC(matrix->nPins * sizeof(ZOLTAN_GNO_TYPE)); if (matrix->nPins && !matrix->pinGNO){ ZOLTAN_FREE(&pinID); MEMORY_ERROR; } for (i=0; i < matrix->nPins; i++) matrix->pinGNO[i] = (ZOLTAN_GNO_TYPE)pinID[i] - fast_build_base; ZOLTAN_FREE(&pinID); } else{ matrix->pinGNO = (ZOLTAN_GNO_TYPE *) pinID; if (fast_build_base) for (i=0; i < matrix->nPins; i++) matrix->pinGNO[i] -= fast_build_base; pinID = NULL; } } /* if (matrix->opts.local) { /\* keep only local edges *\/ */ /* int *nnz_list; /\* nnz offset to delete *\/ */ /* int nnz; /\* number of nnz to delete *\/ */ /* int i; */ /* nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */ /* if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */ /* for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */ /* if (proclist[i] == zz->Proc) continue; */ /* nnz_list[nnz++] = i; */ /* } */ /* ZOLTAN_FREE(&proclist); */ /* Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */ /* } */ if (!matrix->opts.enforceSquare) { /* Hyperedges name translation is different from the one of vertices */ matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_CALLOC(matrix->nY, sizeof(ZOLTAN_GNO_TYPE)); if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR; /* int nGlobalEdges = 0; */ ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY, matrix->opts.randomize, &matrix->globalY); CHECK_IERR; /* /\**************************************************************************************** */ /* * If it is desired to remove dense edges, divide the list of edges into */ /* * two lists. The ZHG structure will contain the removed edges (if final_output is true), */ /* * and the kept edges will be returned. */ /* ****************************************************************************************\/ */ /* totalNumEdges = zhg->globalHedges; */ /* ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */ /* &nLocalEdges, &nGlobalEdges, &nPins, */ /* &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */ /* if (nGlobalEdges < totalNumEdges){ */ /* /\* re-assign edge global numbers if any edges were removed *\/ */ /* ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */ /* randomizeInitDist, &totalNumEdges); */ /* if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */ /* goto End; */ /* } */ /* } */ /* We have to define ddY : yGNO, yGID, ywgt */ ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, gno_size_for_dd, zz->Num_GID, 0, matrix->globalY/zz->Num_Proc, 0); /* Hope a linear assignment will help a little */ if (matrix->globalY/zz->Num_Proc) Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc); /* Associate all the data with our yGNO */ Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, NULL, NULL, matrix->nY); } End: ZOLTAN_FREE(&xpid); ZOLTAN_FREE(&xLID); ZOLTAN_FREE(&xGNO); ZOLTAN_FREE(&xGID); ZOLTAN_FREE(&xwgt); ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&proclist); if (dd != NULL) Zoltan_DD_Destroy(&dd); /* Already stored in the DD */ ZOLTAN_FREE(&yGID); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int num_gid_entries, /* # of entries for a global id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm /* iperm[rank[i]]=i, only for sequential ordering */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; int * vtxdist = NULL; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL, *local_iperm=NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value"); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.rank = rank; zz->Order.iperm = iperm; zz->Order.gids = gids; zz->Order.lids = lids; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); #ifdef HAVE_MPI strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN); #else strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); zz->Order.start_index = opt.start_index; /* * Check that the user has allocated space for the return args. */ if (!(gids && rank)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); /* strcpy(opt.order_type, "GLOBAL"); */ } #endif /* ZOLTAN_SCOTCH */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } if (!strcmp(opt.order_type, "GLOBAL")) strcpy (opt.order_type, "DIST"); if (!strcmp(opt.order_type, "LOCAL")) strcpy (opt.order_type, "SERIAL"); strcpy(zz->Order.order_type, opt.order_type); /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. * Compute gid according to the local graph. */ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 3, &local_gids, &local_rank, &local_iperm); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ if ((!(opt.return_args & RETURN_RANK) && (rank != NULL)) || (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) { ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr) { /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) { /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) { /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* I store : GNO, rank, iperm */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); ZOLTAN_FREE(&local_iperm); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
int main(int argc, char *argv[]) { /* Local declarations. */ struct Zoltan_Struct *zz = NULL; char *cmd_file; char cmesg[256]; /* for error messages */ float version; int Proc, Num_Proc; int iteration; int error, gerror; int print_output = 1; MESH_INFO mesh; /* mesh information struct */ PARIO_INFO pio_info; PROB_INFO prob; /***************************** BEGIN EXECUTION ******************************/ /* initialize MPI */ MPI_Init(&argc, &argv); #ifdef VAMPIR VT_initialize(&argc, &argv); #endif /* get some machine information */ MPI_Comm_rank(MPI_COMM_WORLD, &Proc); MPI_Comm_size(MPI_COMM_WORLD, &Num_Proc); my_rank = Proc; #ifdef HOST_LINUX signal(SIGSEGV, meminfo_signal_handler); signal(SIGINT, meminfo_signal_handler); signal(SIGTERM, meminfo_signal_handler); signal(SIGABRT, meminfo_signal_handler); signal(SIGFPE, meminfo_signal_handler); #endif #ifdef ZOLTAN_PURIFY printf("%d of %d ZDRIVE LAUNCH pid = %d file = %s\n", Proc, Num_Proc, getpid(), argv[1]); #endif /* Initialize flags */ Test.DDirectory = 0; Test.Local_Parts = 0; Test.Fixed_Objects = 0; Test.Drops = 0; Test.RCB_Box = 0; Test.Multi_Callbacks = 0; Test.Graph_Callbacks = 1; Test.Hypergraph_Callbacks = 1; Test.Gen_Files = 0; Test.Null_Lists = NO_NULL_LISTS; Test.Dynamic_Weights = .0; Test.Dynamic_Graph = .0; Test.Vtx_Inc = 0; Output.Text = 1; Output.Gnuplot = 0; Output.Nemesis = 0; Output.Plot_Partition = 0; Output.Mesh_Info_File = 0; /* Interpret the command line */ switch(argc) { case 1: cmd_file = "zdrive.inp"; break; case 2: cmd_file = argv[1]; break; default: fprintf(stderr, "MAIN: ERROR in command line,"); if(Proc == 0) { fprintf(stderr, " usage:\n"); fprintf(stderr, "\t%s [command file]", DRIVER_NAME); } exit(1); break; } /* initialize Zoltan */ if ((error = Zoltan_Initialize(argc, argv, &version)) != ZOLTAN_OK) { sprintf(cmesg, "fatal: Zoltan_Initialize returned error code, %d", error); Gen_Error(0, cmesg); error_report(Proc); print_output = 0; goto End; } /* initialize some variables */ initialize_mesh(&mesh, Proc); pio_info.dsk_list_cnt = -1; pio_info.file_comp = STANDARD; pio_info.num_dsk_ctrlrs = -1; pio_info.pdsk_add_fact = -1; pio_info.zeros = -1; pio_info.file_type = -1; pio_info.chunk_reader = 0; pio_info.init_dist_type = -1; pio_info.init_size = ZOLTAN_ID_INVALID; pio_info.init_dim = -1; pio_info.init_vwgt_dim = -1; pio_info.init_dist_pins = -1; pio_info.pdsk_root[0] = '\0'; pio_info.pdsk_subdir[0] = '\0'; pio_info.pexo_fname[0] = '\0'; prob.method[0] = '\0'; prob.num_params = 0; prob.params = NULL; /* Read in the ascii input file */ error = gerror = 0; if (Proc == 0) { printf("\n\nReading the command file, %s\n", cmd_file); if (!read_cmd_file(cmd_file, &prob, &pio_info, NULL)) { sprintf(cmesg,"fatal: Could not read in the command file" " \"%s\"!\n", cmd_file); Gen_Error(0, cmesg); error_report(Proc); print_output = 0; error = 1; } if (!check_inp(&prob, &pio_info)) { Gen_Error(0, "fatal: Error in user specified parameters.\n"); error_report(Proc); print_output = 0; error = 1; } print_input_info(stdout, Num_Proc, &prob, &pio_info, version); } MPI_Allreduce(&error, &gerror, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); if (gerror) goto End; /* broadcast the command info to all of the processor */ brdcst_cmd_info(Proc, &prob, &pio_info, &mesh); Zoltan_Set_Param(NULL, "DEBUG_MEMORY", "1"); print_output = Output.Text; /* * Create a Zoltan structure. */ if ((zz = Zoltan_Create(MPI_COMM_WORLD)) == NULL) { Gen_Error(0, "fatal: NULL returned from Zoltan_Create()\n"); return 0; } if (!setup_zoltan(zz, Proc, &prob, &mesh, &pio_info)) { Gen_Error(0, "fatal: Error returned from setup_zoltan\n"); error_report(Proc); print_output = 0; goto End; } /* srand(Proc); Different seeds on different procs. */ srand(1); /* Same seed everywhere. */ if (Test.Dynamic_Weights){ /* Set obj weight dim to 1; can be overridden by user parameter */ Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "1"); } /* Loop over read and balance for a number of iterations */ /* (Useful for testing REUSE parameters in Zoltan.) */ for (iteration = 1; iteration <= Number_Iterations; iteration++) { if (Proc == 0) { printf("Starting iteration %d\n", iteration); fflush(stdout); } /* * now read in the mesh and element information. * This is the only function call to do this. Upon return, * the mesh struct and the elements array should be filled. */ if (iteration == 1) { if (!read_mesh(Proc, Num_Proc, &prob, &pio_info, &mesh)) { Gen_Error(0, "fatal: Error returned from read_mesh\n"); error_report(Proc); print_output = 0; goto End; } /* * Create a Zoltan DD for tracking elements during repartitioning. */ if (mesh.data_type == ZOLTAN_HYPERGRAPH && !build_elem_dd(&mesh)) { Gen_Error(0, "fatal: Error returned from build_elem_dd\n"); error_report(Proc); print_output = 0; goto End; } } #ifdef KDDKDD_COOL_TEST /* KDD Cool test of changing number of partitions */ sprintf(cmesg, "%d", Num_Proc * iteration); Zoltan_Set_Param(zz, "NUM_GLOBAL_PARTS", cmesg); #endif /* * Produce files to verify input. */ if (iteration == 1) { if (Debug_Driver > 2) { if (!output_results(cmd_file,"in",Proc,Num_Proc,&prob,&pio_info,&mesh)){ Gen_Error(0, "fatal: Error returned from output_results\n"); error_report(Proc); } if (Output.Gnuplot) if (!output_gnu(cmd_file,"in",Proc,Num_Proc,&prob,&pio_info,&mesh)) { Gen_Error(0, "warning: Error returned from output_gnu\n"); error_report(Proc); } } if (Test.Vtx_Inc<0){ /* Read Citeseer data from file */ FILE *fp; int i=0; if (Proc==0){ fp = fopen("months.txt", "r"); if (!fp) printf("ERROR: Couldn't open file months.txt\n"); while (fscanf(fp, "%d", &CITESEER[i])==1){ ++i; } fclose(fp); } MPI_Bcast (CITESEER, 200, MPI_INT, 0, MPI_COMM_WORLD); } } if (Test.Dynamic_Graph > 0.0){ if (mesh.data_type == ZOLTAN_GRAPH) { remove_random_vertices(&mesh, iteration, Test.Dynamic_Graph); } else{ Gen_Error(0, "fatal: \"test dynamic graph\" only works on graphs, not hypergraphs\n"); error_report(Proc); print_output = 0; goto End; } } if (Test.Vtx_Inc){ if (mesh.data_type == ZOLTAN_HYPERGRAPH ) { if (Test.Vtx_Inc>0) mesh.visible_nvtx += Test.Vtx_Inc; /* Increment uniformly */ else mesh.visible_nvtx = CITESEER[iteration-1]; /* Citeseer document matrix. */ } else{ Gen_Error(0, "fatal: \"vertex increment\" only works on hypergraphs\n"); error_report(Proc); print_output = 0; goto End; } } /* * now run Zoltan to get a new load balance and perform * the migration */ #ifdef IGNORE_FIRST_ITERATION_STATS if (iteration == 1) { /* Exercise partitioner once on Tbird because first run is slow. */ /* Lee Ann suspects Tbird is loading shared libraries. */ struct Zoltan_Struct *zzcopy; zzcopy = Zoltan_Copy(zz); /* Don't do any migration or accumulate any stats. */ if (Proc == 0) printf("%d KDDKDD IGNORING FIRST ITERATION STATS\n", Proc); Zoltan_Set_Param(zzcopy, "RETURN_LISTS", "NONE"); Zoltan_Set_Param(zzcopy, "FINAL_OUTPUT", "0"); Zoltan_Set_Param(zzcopy, "USE_TIMERS", "0"); if (!run_zoltan(zzcopy, Proc, &prob, &mesh, &pio_info)) { Gen_Error(0, "fatal: Error returned from run_zoltan\n"); error_report(Proc); print_output = 0; goto End; } Zoltan_Destroy(&zzcopy); } #endif /* IGNORE_FIRST_ITERATION_STATS */ #ifdef RANDOM_DIST if (iteration % 2 == 0) { char LB_METHOD[1024]; if (Proc == 0) printf("%d CCCC Randomizing the input\n", Proc); strcpy(LB_METHOD, prob.method); strcpy(prob.method, "RANDOM"); Zoltan_Set_Param(zz, "LB_METHOD", "RANDOM"); Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); if (!run_zoltan(zz, Proc, &prob, &mesh, &pio_info)) { Gen_Error(0, "fatal: Error returned from run_zoltan\n"); error_report(Proc); print_output = 0; goto End; } Zoltan_Set_Param(zz, "RETURN_LISTS", "NONE"); Zoltan_Set_Param(zz, "LB_METHOD", LB_METHOD); strcpy(prob.method, LB_METHOD); if (Proc == 0) printf("%d CCCC Randomizing the input -- END\n", Proc); } #endif /* RANDOM_DIST */ if (!run_zoltan(zz, Proc, &prob, &mesh, &pio_info)) { Gen_Error(0, "fatal: Error returned from run_zoltan\n"); error_report(Proc); print_output = 0; goto End; } /* Reset the mesh data structure for next iteration. */ if (iteration < Number_Iterations) { int i, j; float tmp; float twiddle = 0.01; char str[4]; /* Perturb coordinates of mesh */ if (mesh.data_type == ZOLTAN_GRAPH){ for (i = 0; i < mesh.num_elems; i++) { for (j = 0; j < mesh.num_dims; j++) { /* tmp = ((float) rand())/RAND_MAX; *//* Equiv. to sjplimp's test */ tmp = (float) (i % 10) / 10.; mesh.elements[i].coord[0][j] += twiddle * (2.0*tmp-1.0); mesh.elements[i].avg_coord[j] = mesh.elements[i].coord[0][j]; } } /* Increase weights in some parts */ if (Test.Dynamic_Weights){ /* Randomly pick 10% of parts to "refine" */ /* Note: Assumes at least 10 parts! */ /* Increase vertex weight, and also edge weights? TODO */ j = (int) ((10.0*rand())/RAND_MAX + .5); for (i = 0; i < mesh.num_elems; i++) { if ((mesh.elements[i].my_part%10) == j){ mesh.elements[i].cpu_wgt[0] = Test.Dynamic_Weights*(1+rand()%5); } } } } /* change the ParMETIS Seed */ sprintf(str, "%d", iteration); #ifdef ZOLTAN_PARMETIS Zoltan_Set_Param(zz, "PARMETIS_SEED", str); #endif } } /* End of loop over read and balance */ if (Proc == 0) { printf("FILE %s: Total: %e seconds in Partitioning\n", cmd_file, Total_Partition_Time); printf("FILE %s: Average: %e seconds per Iteration\n", cmd_file, Total_Partition_Time/Number_Iterations); } End: Zoltan_Destroy(&zz); if (mesh.dd) Zoltan_DD_Destroy(&(mesh.dd)); Zoltan_Memory_Stats(); /* * output the results */ if (print_output) { if (!output_results(cmd_file,"out",Proc,Num_Proc,&prob,&pio_info,&mesh)) { Gen_Error(0, "fatal: Error returned from output_results\n"); error_report(Proc); } if (Output.Gnuplot) { if (!output_gnu(cmd_file,"out",Proc,Num_Proc,&prob,&pio_info,&mesh)) { Gen_Error(0, "warning: Error returned from output_gnu\n"); error_report(Proc); } } } free_mesh_arrays(&mesh); if (prob.params != NULL) free(prob.params); MPI_Finalize(); #ifdef VAMPIR VT_finalize(); #endif return 0; }
int Zoltan_Order ( struct Zoltan_Struct *zz, int num_gid_entries, int num_obj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR permuted_global_ids ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids. * num_gid_entries * Output: * permuted_global_ids * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { char msg[253]; sprintf(msg, "num_gid_entries=%d is not equal to parameter setting " "NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ #ifdef HAVE_MPI strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "GLOBAL"); #else strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "LOCAL"); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* * Check that the user has allocated space for the return args. */ if (num_obj && !(gids && permuted_global_ids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "LOCAL_HSFC")) { Order_fn = Zoltan_LocalHSFC_Order; strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/ } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PTSCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_SCOTCH */ #ifdef ZOLTAN_HUND else if (!strcasecmp(opt.method, "HUND")) { ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL); goto End; } #endif /* ZOLTAN_HUND */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* TODO : Ask why useful ! */ /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /************************************ * Check for required query function ************************************/ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } /* TODO allocate all this stuff with the graph */ local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); /* * Call the actual ordering function. * Compute gid according to the local graph. */ ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 2, &local_gids, &local_rank); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* TODO: At this time, I consider rank == permuted_global_ids */ /* I store : GNO, rank, permuted GID */ /* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0); /* Hope a linear assignment will help a little */ if (local_num_obj) Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", permuted_global_ids[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } #ifdef ZOLTAN_HUND End: #endif /*ZOLTAN_HUND*/ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int main(int argc, char *argv[]) { int i, rc; int myRank, numProcs; float ver; struct Zoltan_Struct *zz; /* mfh 18 Apr 2014: Commented out unused variables */ /* FIXME (mfh 18 Apr 2014) Hey, is start_gid a GID? Shouldn't it not be an int then? */ int changes, numGidEntries, numLidEntries, numImport, numExport; /* , start_gid, num_nbors; */ ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids; int *importProcs, *importToPart, *exportProcs, *exportToPart; int *parts=NULL; ZOLTAN_ID_PTR lids=NULL; FILE *fp; struct Zoltan_DD_Struct *dd; GRAPH_DATA myGraph; int gid_length = 1; /* our global IDs consist of 1 integer */ int lid_length = 1; /* our local IDs consist of 1 integer */ /****************************************************************** ** Initialize MPI and Zoltan ******************************************************************/ MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &myRank); MPI_Comm_size(MPI_COMM_WORLD, &numProcs); rc = Zoltan_Initialize(argc, argv, &ver); if (rc != ZOLTAN_OK){ printf("Failed to initialize Zoltan -- sorry...\n"); MPI_Finalize(); exit(0); } /****************************************************************** ** Read graph from input file and distribute it ******************************************************************/ fp = fopen(fname, "r"); if (!fp){ if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname); MPI_Finalize(); exit(1); } fclose(fp); read_input_file(myRank, numProcs, fname, &myGraph); /*fprintf(stderr,"%d have %d objects\n",myRank,myGraph.numMyVertices);*/ /****************************************************************** ** Create a distributed data directory which maps vertex ** global IDs to their current part number. We'll use this ** after migrating vertices, to update the parts in which ** our vertices neighbors are. ** ** Our local IDs (array "lids") are of type ZOLTAN_ID_TYPE because ** we are using Zoltan's distributed data directory. It assumes ** that a global ID is a sequence of "gid_length" ZOLTAN_ID_TYPEs. ** It assumes that a local ID is a sequence of "lid_length" ** ZOLTAN_ID_TYPEs. ******************************************************************/ rc = Zoltan_DD_Create(&dd, MPI_COMM_WORLD, gid_length, /* length of a global ID */ lid_length, /* length of a local ID */ 0, /* length of user data */ myGraph.numMyVertices, /* hash table size */ 0); /* debug level */ parts = (int *) malloc(myGraph.numMyVertices * sizeof(int)); lids = (ZOLTAN_ID_TYPE *) malloc(myGraph.numMyVertices * sizeof(ZOLTAN_ID_TYPE)); for (i=0; i < myGraph.numMyVertices; i++){ parts[i] = myRank; /* part number of this vertex */ lids[i] = (ZOLTAN_ID_TYPE)i; /* local ID on my process for this vertex */ } rc = Zoltan_DD_Update(dd, myGraph.vertexGID, lids, NULL, parts, myGraph.numMyVertices); myGraph.dd = dd; /****************************************************************** ** Create a Zoltan library structure for this instance of load ** balancing. Set the parameters and query functions that will ** govern the library's calculation. See the Zoltan User's ** Guide for the definition of these and many other parameters. ******************************************************************/ zz = Zoltan_Create(MPI_COMM_WORLD); /* General parameters */ Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"); Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION"); Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); /* Graph parameters */ Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */ /* Query functions, defined in this source file */ Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph); Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph); Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph); Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph); Zoltan_Set_Obj_Size_Multi_Fn(zz, get_message_sizes,&myGraph); Zoltan_Set_Pack_Obj_Multi_Fn(zz, pack_object_messages,&myGraph); Zoltan_Set_Unpack_Obj_Multi_Fn(zz, unpack_object_messages,&myGraph); Zoltan_Set_Mid_Migrate_PP_Fn(zz, mid_migrate,&myGraph); /****************************************************************** ** Visualize the graph partition before calling Zoltan. ******************************************************************/ if (myRank== 0){ printf("\nGraph partition before calling Zoltan\n"); } printGraph(myRank, &myGraph); showGraphParts(myRank, myGraph.dd); /****************************************************************** ** Zoltan can now partition the simple graph. ** In this simple example, we assume the number of parts is ** equal to the number of processes. Process rank 0 will own ** part 0, process rank 1 will own part 1, and so on. ******************************************************************/ rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */ &changes, /* 1 if partition was changed, 0 otherwise */ &numGidEntries, /* Number of integers used for a global ID */ &numLidEntries, /* Number of integers used for a local ID */ &numImport, /* Number of vertices to be sent to me */ &importGlobalGids, /* Global IDs of vertices to be sent to me */ &importLocalGids, /* Local IDs of vertices to be sent to me */ &importProcs, /* Process rank for source of each incoming vertex */ &importToPart, /* New part for each incoming vertex */ &numExport, /* Number of vertices I must send to other processes*/ &exportGlobalGids, /* Global IDs of the vertices I must send */ &exportLocalGids, /* Local IDs of the vertices I must send */ &exportProcs, /* Process to which I send each of the vertices */ &exportToPart); /* Partition to which each vertex will belong */ if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); Zoltan_Destroy(&zz); exit(0); } /*fprintf(stderr,"%d export %d import %d\n",myRank,numExport,numImport);*/ /****************************************************************** ** Update the data directory with the new part numbers ******************************************************************/ for (i=0; i < numExport; i++){ parts[exportLocalGids[i]] = exportToPart[i]; } rc = Zoltan_DD_Update(dd, myGraph.vertexGID, lids, NULL, parts, myGraph.numMyVertices); /****************************************************************** ** Migrate vertices to new parts ******************************************************************/ rc = Zoltan_Migrate(zz, numImport, importGlobalGids, importLocalGids, importProcs, importToPart, numExport, exportGlobalGids, exportLocalGids, exportProcs, exportToPart); /****************************************************************** ** Use the data dictionary to find neighbors' parts ******************************************************************/ rc = Zoltan_DD_Find(dd, (ZOLTAN_ID_PTR)(myGraph.nborGID), NULL, NULL, myGraph.nborPart, myGraph.nborIndex[myGraph.numMyVertices], NULL); /****************************************************************** ** Visualize the graph partition after calling Zoltan. ******************************************************************/ if (myRank == 0){ printf("Graph partition after calling Zoltan\n"); } printGraph(myRank, &myGraph); showGraphParts(myRank, myGraph.dd); /****************************************************************** ** Free the arrays allocated by Zoltan_LB_Partition, and free ** the storage allocated for the Zoltan structure. ******************************************************************/ Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart); Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart); Zoltan_Destroy(&zz); /********************** ** all done *********** **********************/ if (myGraph.vertex_capacity > 0){ free(myGraph.vertexGID); free(myGraph.nborIndex); if (myGraph.nbor_capacity > 0){ free(myGraph.nborGID); free(myGraph.nborPart); } } Zoltan_DD_Destroy(&dd); if (parts) free(parts); if (lids) free(lids); MPI_Finalize(); return 0; }
int Zoltan_Matrix_Sym(ZZ* zz, Zoltan_matrix *matrix, int bipartite) { static char *yo = "Zoltan_Matrix_Sym"; int ierr = ZOLTAN_OK; Zoltan_Arc *tr_tab = NULL; int i, j, cnt; ZOLTAN_ID_PTR yGID = NULL; int *ypid=NULL; float *pinwgt=NULL; int * ybipart = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (bipartite || !matrix->opts.enforceSquare) { matrix->bipartite = 1; matrix->redist = 1; } if (matrix->ywgtdim != zz->Obj_Weight_Dim) FATAL_ERROR("Cannot form bipartite graph: vertex and edge weights are not consistant"); matrix->opts.symmetrize = 1; /* Update the data directories */ tr_tab = (Zoltan_Arc*) ZOLTAN_MALLOC(sizeof(Zoltan_Arc)*(matrix->nPins*2+matrix->nY)); if (matrix->nPins && tr_tab == NULL) MEMORY_ERROR; pinwgt = (float*)ZOLTAN_MALLOC((matrix->nPins*2+matrix->nY)*matrix->pinwgtdim*sizeof(float)); for (i = 0 ; i < 2 ; ++i) /* Copy pin weights */ memcpy(pinwgt + i*matrix->nPins*matrix->pinwgtdim*sizeof(float), matrix->pinwgt, matrix->nPins*matrix->pinwgtdim*sizeof(float)); for (i=0, cnt = 0 ; i < matrix->nY ; ++i) { for (j = matrix->ystart[i] ; j < matrix->yend[i] ; ++j) { tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */ tr_tab[cnt].GNO[1] = matrix->pinGNO[j]; memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim, matrix->pinwgtdim*sizeof(float)); cnt ++; tr_tab[cnt].GNO[0] = matrix->pinGNO[j]; /* Symmetric arc */ tr_tab[cnt].GNO[1] = matrix->yGNO[i] + bipartite*matrix->globalX; /* new ordering */ memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim, matrix->pinwgtdim*sizeof(float)); cnt ++; } if (matrix->ystart[i] == matrix->yend[i]) { /* Singleton */ tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */ tr_tab[cnt].GNO[1] = -1; cnt ++; } } ZOLTAN_FREE(&matrix->pinwgt); Zoltan_Matrix_Remove_DupArcs(zz, cnt, tr_tab, pinwgt, matrix); ZOLTAN_FREE(&tr_tab); ZOLTAN_FREE(&pinwgt); if (bipartite) { int endX; int * yGNO = NULL; /* Update data directories */ yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nY); ypid = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int)); ybipart = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int)); for (endX = 0 ; endX < matrix->nY ; ++endX) { if (matrix->yGNO[endX] >= matrix->globalX) break; ybipart[endX] = 0; } /* Get Informations about X */ Zoltan_DD_Find (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, NULL, endX, NULL); yGNO = (int*)ZOLTAN_MALLOC(endX*sizeof(int)); for (i = endX ; i < matrix->nY ; ++i) { yGNO[i-endX] = matrix->yGNO[i] - matrix->globalX; /* TODO: add a something to have the correct ypid */ ybipart[endX] = 1; } /* Get Informations about Y */ Zoltan_DD_Find (matrix->ddY, (ZOLTAN_ID_PTR)yGNO, yGID + endX*zz->Num_GID, NULL, NULL, matrix->nY-endX, NULL); if (matrix->ddY != matrix->ddX) Zoltan_DD_Destroy (&matrix->ddY); Zoltan_DD_Destroy (&matrix->ddX); matrix->globalX += matrix->globalY; matrix->globalY = matrix->globalX; /* I store : xGNO, xGID, xpid, bipart */ ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID, 1, matrix->globalX/zz->Num_Proc, 0); matrix->ddY = matrix->ddX; /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc); /* Associate all the data with our xyGNO */ Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, ybipart, matrix->nY); } End: ZOLTAN_FREE(&ybipart); ZOLTAN_FREE(&ypid); ZOLTAN_FREE(&pinwgt); ZOLTAN_FREE(&yGID); ZOLTAN_FREE(&tr_tab); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }