int update_hvertex_proc(MESH_INFO_PTR mesh) { if (Zoltan_DD_Find(mesh->dd, (ZOLTAN_ID_PTR)mesh->hvertex, NULL, NULL, NULL, mesh->hindex[mesh->nhedges], mesh->hvertex_proc) != 0) { Gen_Error(0, "fatal: NULL returned from Zoltan_DD_Find()\n"); return 0; } return 1; }
int Zoltan_Matrix_Complete(ZZ* zz,Zoltan_matrix* m) { static char *yo = "Zoltan_Matrix_Complete"; int ierr = ZOLTAN_OK; if(m->completed) return (ZOLTAN_OK); ZOLTAN_TRACE_ENTER(zz, yo); if (m->yend != m->ystart +1 ) {/* Not in compact mode yet */ int y; /* I have to also rewrites all the pinarrays */ for (y = 1 ; y <= m->nY ; ++y) { int length; if (m->ystart[y] == m->yend[y-1]) /* No hole */ continue; length = m->yend[y]-m->ystart[y]; memcpy(m->pinGNO+m->yend[y-1], m->pinGNO+m->ystart[y], length*sizeof(ZOLTAN_GNO_TYPE)); memcpy(m->pinwgt+m->yend[y-1]*m->pinwgtdim, m->pinGNO+m->ystart[y]*m->pinwgtdim, length*sizeof(float)*m->pinwgtdim); m->ystart[y] = m->yend[y-1]; m->yend[y] = m->ystart[y] + length; } ZOLTAN_FREE(&m->yend); m->yend = m->ystart + 1; } /* Update data directories */ m->yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY); m->ypid = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int)); if (m->bipartite) m->ybipart = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int)); if (m->nY && ((m->yGID == NULL) || (m->ypid == NULL) || (m->bipartite && m->ybipart == NULL))) MEMORY_ERROR; /* Get Informations about Y */ Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, m->yGID, (char *)m->ypid, m->ybipart, m->nY, NULL); if (m->ddY != m->ddX) { Zoltan_DD_Destroy(&m->ddY); m->ddY = NULL; } m->completed = 1; End: ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
/* This function may work with any distribution of the bipartite graph */ int Zoltan_ZG_Query (ZZ* zz, const ZG* const graph, ZOLTAN_ID_PTR GID, int GID_length, int* properties) { struct Zoltan_DD_Struct *dd; dd = graph->mtx.mtx.ddY; /* if (graph->bipartite && graph->fixObj) */ /* dd = graph->mtx.mtx.ddX; */ return Zoltan_DD_Find(dd, GID, NULL, NULL, properties, GID_length, NULL); }
int Zoltan_Matrix_Vertex_Info(ZZ* zz, const Zoltan_matrix * const m, ZOLTAN_ID_PTR lid, float *wwgt, int *input_part) { static char *yo = "Zoltan_Matrix_Vertex_Info"; int ierr = ZOLTAN_OK; int nX; ZOLTAN_ID_PTR l_gid = NULL; ZOLTAN_ID_PTR l_lid = NULL; float * l_xwgt = NULL; int *l_input_part = NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (m->completed == 0) { ierr = ZOLTAN_FATAL; goto End; } ierr = Zoltan_Get_Obj_List(zz, &nX, &l_gid, &l_lid, zz->Obj_Weight_Dim, &l_xwgt, &l_input_part); ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, zz->Num_LID, zz->Obj_Weight_Dim*sizeof(float)/sizeof(int), nX, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (dd, l_gid, l_lid, (ZOLTAN_ID_PTR) l_xwgt,l_input_part, nX); ZOLTAN_FREE(&l_gid); ZOLTAN_FREE(&l_lid); ZOLTAN_FREE(&l_xwgt); ZOLTAN_FREE(&l_input_part); ierr = Zoltan_DD_Find (dd, m->yGID, lid, (ZOLTAN_ID_PTR)wwgt, input_part, m->nY, NULL); End: if (dd != NULL) Zoltan_DD_Destroy(&dd); ZOLTAN_FREE(&l_gid); ZOLTAN_FREE(&l_lid); ZOLTAN_FREE(&l_xwgt); ZOLTAN_FREE(&l_input_part); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
size_t findUniqueGidsCommon( size_t num_keys, int num_gid, ZOLTAN_ID_PTR ddkeys, char *ddnewgids, MPI_Comm mpicomm ) { int num_lid = 0; // Local IDs not needed int debug_level = 0; int num_user = sizeof(gno_t); Zoltan_DD_Struct *dd = NULL; Zoltan_DD_Create(&dd, mpicomm, num_gid, num_lid, num_user, num_keys, debug_level); ZOLTAN_ID_PTR ddnotneeded = NULL; // Local IDs not needed Zoltan_DD_Update(dd, ddkeys, ddnotneeded, ddnewgids, NULL, int(num_keys)); ////////// // Insert unique GIDs for DD entries in User data here. // Get value of first gid on this rank ssize_t nDDEntries = (ssize_t)(dd->nodecnt); ssize_t firstIdx; MPI_Scan(&nDDEntries, &firstIdx, 1, MPI_LONG_LONG, MPI_SUM, mpicomm); firstIdx -= nDDEntries; // do not include this rank's entries in prefix sum // Loop over all directory entries, updating their userdata with updated gid DD_NodeIdx cnt = 0; for (DD_NodeIdx i = 0; i < dd->nodelistlen; i++) { DD_Node *ptr = &(dd->nodelist[i]); if (!(ptr->free)) { char *userchar = (char*)(ptr->gid + (dd->gid_length + dd->lid_length)); gno_t *newgid = (gno_t*) userchar; *newgid = gno_t(firstIdx + cnt); cnt++; } } /////////// // Retrieve the global numbers and put in the result gids vector Zoltan_DD_Find(dd, ddkeys, ddnotneeded, ddnewgids, NULL, int(num_keys), NULL); Zoltan_DD_Destroy(&dd); ssize_t nUnique = 0; MPI_Allreduce(&nDDEntries, &nUnique, 1, MPI_LONG_LONG, MPI_SUM, mpicomm); return size_t(nUnique); }
/* Return an array of locally owned GID */ ZOLTAN_ID_PTR Zoltan_Matrix_Get_GID(ZZ* zz, Zoltan_matrix* m) { ZOLTAN_ID_PTR yGID; yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY); if (m->nY && yGID == NULL) return (NULL); /* Get Informations about Y */ Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, NULL, NULL, m->nY, NULL); return (yGID); }
int main (int argc, char *argv[]) { Zoltan_DD_Directory *dd; ZTIMER *zt; int myproc; /* MPI rank, my processor's MPI ID */ int nproc; /* MPI size, number of processors */ ZOLTAN_ID_PTR glist = NULL; /* list of GIDs */ ZOLTAN_ID_PTR llist = NULL; /* list of LIDs */ ZOLTAN_ID_PTR ulist = NULL; /* list of user data of type ZOLTAN_ID_TYPE */ int *plist = NULL; /* list of partitions */ int *olist = NULL; /* list of owners */ Param param ; /* program's adjustable parameters */ char *store = NULL; /* non directory storage of test data */ Data *data = NULL; /* pointer into store */ /* these are all temporary variables */ int new_owner; int count; int i; char *p; int err; int errcount; int loop; char str[100]; /* for building output messages */ static int timer[7] = {-1,-1,-1,-1,-1,-1,-1}; char *yo = "DD_Main"; /* initialize MPI communications, ignore errors */ MPI_Init (&argc, &argv); MPI_Comm_rank (MPI_COMM_WORLD, &myproc); MPI_Comm_size (MPI_COMM_WORLD, &nproc); get_params (¶m); /* read input parameters */ zt = Zoltan_Timer_Create(1); MACRO_TIMER_START(5, "Total time", 0); MACRO_TIMER_START(0, "DD_Create time", 0); err = Zoltan_DD_Create (&dd, MPI_COMM_WORLD, param.glen, param.llen, param.ulen, param.tlen, param.debug_level); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Create"); MACRO_TIMER_STOP(0); param.slen = sizeof (Data) + sizeof(ZOLTAN_ID_TYPE) * (param.glen + param.llen + param.ulen); param.slen = Zoltan_Align(param.slen); store = (char*) ZOLTAN_MALLOC (param.count * param.slen); /* allocate storage for various lists */ glist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC(sizeof(ZOLTAN_ID_TYPE) * param.count * param.glen); plist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count); olist = (int*) ZOLTAN_MALLOC (sizeof(int) * param.count); if (param.llen) llist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE) * param.count * param.llen); if (param.ulen) ulist = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC (sizeof (ZOLTAN_ID_TYPE) * param.count * param.ulen) ; if (store == NULL || glist == NULL || (param.llen != 0 && llist == NULL) || (param.ulen != 0 && ulist == NULL) || plist == NULL || olist == NULL) { ZOLTAN_PRINT_ERROR (myproc, yo, "Unable to malloc storage lists"); return ZOLTAN_MEMERR; } initialize_data (¶m, store, nproc); /* create & update directory with myproc's initial simulated GIDs */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data *)p)->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data*)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data *)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen, ((Data *)p)->id + (param.glen + param.llen)); plist [count] = ((Data *)p)->partition; ++count; } MACRO_TIMER_START (1, "DD_Update timer", 0); err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); i = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) { ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id); ++i; } MACRO_TIMER_START(2, "DD_Find timer", 0); err = Zoltan_DD_Find (dd, glist, llist, ulist, plist, param.count, olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); errcount = 0; for (i = 0; i < param.count; i++) if (olist[i] != ((Data *) (store + i * param.slen))->new_owner) ++errcount; if (errcount) sprintf (str, "FIRST TEST FAILED, errcount is %d", errcount); else sprintf (str, "FIRST TEST SUCCESSFUL"); ZOLTAN_PRINT_INFO (myproc, yo, str); /* repeatedly simulate moving "dots" around the system */ for (loop = 0; loop < param.nloops; loop++) { for (p = store; p < store + param.count * param.slen; p += param.slen) ((Data*) p)->old_owner = ((Data*) p)->new_owner; /* randomly exchange some dots and randomly reassign others */ for (i = 0; i < (param.pmove * param.count)/100; i++) { Data *d1, *d2; d1 = (Data*) (store + param.slen * (rand() % param.count)); d2 = (Data*) (store + param.slen * (rand() % param.count)); new_owner = d1->new_owner; d1->new_owner = d2->new_owner; d2->new_owner = new_owner; } for (i = 0; i < (param.count * param.pscatter)/100; i++) ((Data*) (store + param.slen *(rand() % param.count)))->new_owner = rand() % nproc; /* determine which new GIDs myproc gained, and update directory */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == myproc) { ((Data*)p)->id[param.glen] = count; /* set LID */ ZOLTAN_SET_ID (param.glen, glist + count * param.glen, ((Data *)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data*)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID (param.ulen, ulist + count * param.ulen, ((Data*)p)->id + (param.glen + param.llen)); plist [count] = ((Data *)p)->partition; ++count; } MACRO_TIMER_START(1, "DD_Update", 0); err = Zoltan_DD_Update (dd, glist, llist, ulist, plist, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); /* use directory to "find" GIDs */ i = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) { ZOLTAN_SET_ID (param.glen, glist + i * param.glen, ((Data *)p)->id); ++i; } MACRO_TIMER_START(2, "DD_Find timer", 0); if (loop % 5 == 0) err = Zoltan_DD_Find(dd,glist,NULL,ulist,plist,param.count,olist); else if (loop % 7 == 0) err = Zoltan_DD_Find(dd,glist,llist,NULL,plist,param.count,olist); else if (loop % 9 == 0) err = Zoltan_DD_Find(dd,glist,llist,ulist,NULL,param.count,olist); else if (loop % 2 == 0) err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,NULL); else err = Zoltan_DD_Find(dd,glist,llist,ulist,plist,param.count,olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); if (loop % 2) { errcount = 0 ; for (i = 0; i < param.count; i++) if (olist[i] != ((Data *)(store + i * param.slen))->new_owner) ++errcount; if (errcount) sprintf (str,"LOOP %d TEST FAILED, errcount is %d",loop,errcount); else sprintf (str, "LOOP %d TEST SUCCESSFUL", loop); ZOLTAN_PRINT_INFO (myproc, yo, str) ; } else { sprintf (str, "LOOP %d Completed", loop); ZOLTAN_PRINT_INFO (myproc, yo, str); } /* randomly remove a percentage of GIDs from the directory */ count = 0; for (i = 0; i < (param.count * param.pdelete)/100; i++) { data = (Data*) (store + param.slen * (rand() % param.count)); if (data->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id); ++count; } data->new_owner = NO_PROC; } MACRO_TIMER_START(3, "DD_Remove timer", 0); err = Zoltan_DD_Remove (dd, glist, count); MACRO_TIMER_STOP(3); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Remove"); /* update directory (put directory entries back in) */ for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == NO_PROC) ((Data*)p)->new_owner = loop % nproc; /* place in new location */ count = 0; for (p = store; p < store + param.count * param.slen; p += param.slen) if (((Data*)p)->new_owner == myproc) { ZOLTAN_SET_ID(param.glen,glist+count*param.glen,((Data*)p)->id); if (param.llen) ZOLTAN_SET_ID (param.llen, llist + count * param.llen, ((Data*)p)->id + param.glen); if (param.ulen) ZOLTAN_SET_ID(param.ulen, ulist + count * param.ulen, ((Data *)p)->id + (param.glen + param.llen)); plist [count] = ((Data*)p)->partition; ++count; } MACRO_TIMER_START(1, "DD_Update timer", 0); err = Zoltan_DD_Update (dd, glist, NULL, NULL, NULL, count); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Update"); MACRO_TIMER_STOP(1); } /* now Find directory info for GIDs myproc now owns and validate */ count = 0; for (i = 0; i < param.count; i++) { data = (Data *) (store + i * param.slen); if (data->new_owner == myproc) { ZOLTAN_SET_ID (param.glen, glist + count * param.glen, data->id); ++count; } } MACRO_TIMER_START(2, "DD_Find", 0); err = Zoltan_DD_Find (dd, glist, NULL, NULL, NULL, count, olist); if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (myproc, yo, "Failed return from DD Find"); MACRO_TIMER_STOP(2); errcount = 0; for (i = 0; i < count; i++) if (olist[i] != myproc) ++errcount; if (errcount) { sprintf (str, "errcount is %d", errcount); ZOLTAN_PRINT_ERROR (myproc, yo, str); } else ZOLTAN_PRINT_INFO (myproc, yo, "TEST SUCCESSFUL"); Zoltan_DD_Stats (dd); /* done, now free memory, stop MPI & directory, return */ ZOLTAN_FREE (&store); ZOLTAN_FREE (&glist); ZOLTAN_FREE (&plist); ZOLTAN_FREE (&olist); if (param.llen) ZOLTAN_FREE (&llist); if (param.ulen) ZOLTAN_FREE (&ulist); ZOLTAN_PRINT_INFO (myproc, yo, "Completing program"); MACRO_TIMER_START(4, "DD_Destroy", 0); Zoltan_DD_Destroy (&dd); MACRO_TIMER_STOP(4); MACRO_TIMER_STOP(5); Zoltan_Timer_PrintAll(zt, 0, MPI_COMM_WORLD, stderr); MPI_Finalize (); return ZOLTAN_OK; }
int Zoltan_Color_Test( ZZ *zz, /* Zoltan structure */ int *num_gid_entries, /* # of entries for a global id */ int *num_lid_entries, /* # of entries for a local id */ int num_obj, /* Input: number of objects */ ZOLTAN_ID_PTR global_ids, /* Input: global ids of the vertices */ /* The application must allocate enough space */ ZOLTAN_ID_PTR local_ids, /* Input: local ids of the vertices */ /* The application must allocate enough space */ int *color_exp /* Input: Colors assigned to local vertices */ ) { static char *yo = "color_test_fn"; int nvtx = num_obj; /* number of vertices */ int i, j; int ierr = ZOLTAN_OK; int ferr = ZOLTAN_OK; /* final error signal */ char coloring_problem; /* Input: which coloring to perform; currently only supports D1, D2 coloring and variants */ char coloring_problemStr[MAX_PARAM_STRING_LEN]; /* string version coloring problem name */ int ss=100; char comm_pattern='S', coloring_order='I', coloring_method='F'; int comm[2],gcomm[2]; int *color=NULL; int *adjproc=NULL, *xadj=NULL; ZOLTAN_GNO_TYPE gvtx; /* number of global vertices */ ZOLTAN_GNO_TYPE *vtxdist=NULL, *adjncy=NULL; ZG graph; ZOLTAN_GNO_TYPE *requested_GNOs = NULL; /* Return GNOs of the requested GIDs. */ int *loc_partialD2 = NULL; /* local binary array showing which vertices to be colored */ int *partialD2 = NULL; /* global binary array showing which vertices to be colored */ struct Zoltan_DD_Struct *dd_color; /* DDirectory for colors */ ZOLTAN_GNO_TYPE *local_GNOs = NULL; ZOLTAN_GNO_TYPE *global_GNOs = NULL; memset (&graph, 0, sizeof(ZG)); /* PARAMETER SETTINGS */ Zoltan_Bind_Param(Color_params, "COLORING_PROBLEM", (void *) &coloring_problemStr); Zoltan_Bind_Param(Color_params, "SUPERSTEP_SIZE", (void *) &ss); Zoltan_Bind_Param(Color_params, "COMM_PATTERN", (void *) &comm_pattern); Zoltan_Bind_Param(Color_params, "VERTEX_VISIT_ORDER", (void *) &coloring_order); Zoltan_Bind_Param(Color_params, "COLORING_METHOD", (void *) &coloring_method); strncpy(coloring_problemStr, "distance-1", MAX_PARAM_STRING_LEN); Zoltan_Assign_Param_Vals(zz->Params, Color_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* Check validity of parameters - they should be consistent with Zoltan_Color */ if (!strcasecmp(coloring_problemStr, "distance-1")) coloring_problem = '1'; else if (!strcasecmp(coloring_problemStr, "distance-2")) coloring_problem = '2'; else if (!strcasecmp(coloring_problemStr, "partial-distance-2") || !strcasecmp(coloring_problemStr, "bipartite")) coloring_problem = 'P'; else { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Unknown coloring requested. Using Distance-1 coloring."); coloring_problem = '1'; } if (ss == 0) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid superstep size. Using default value 100."); ss = 100; } if (comm_pattern != 'S' && comm_pattern != 'A') { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid communication pattern. Using synchronous communication (S)."); comm_pattern = 'S'; } if (comm_pattern == 'A' && (coloring_problem == '2' || coloring_problem == 'P')) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Asynchronous communication pattern is not implemented for distance-2 coloring and its variants. Using synchronous communication (S)."); comm_pattern = 'S'; } if (coloring_order != 'I' && coloring_order != 'B' && coloring_order != 'U' && coloring_order != 'L' && coloring_order != 'N' && coloring_order != 'S') { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring order. Using internal first coloring order (I)."); coloring_order = 'I'; } if (coloring_order == 'U' && (coloring_problem == '2' || coloring_problem == 'P')) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Interleaved coloring order is not implemented for distance-2 coloring and its variants. Using internal first coloring order (I)."); coloring_order = 'I'; } if (coloring_method !='F') { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring method. Using first fit method (F)."); coloring_method = 'F'; } /* Compute Max number of array entries per ID over all processors. This is a sanity-maintaining step; we don't want different processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; /* Return if this processor is not in the Zoltan structure's communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) return ZOLTAN_OK; /* BUILD THE GRAPH */ /* Check that the user has allocated space for the return args. */ if (!color_exp) ZOLTAN_COLOR_ERROR(ZOLTAN_FATAL, "Color argument is NULL. Please give colors of local vertices."); requested_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(num_obj * sizeof(ZOLTAN_GNO_TYPE)); Zoltan_ZG_Build (zz, &graph, 0, 1, num_obj, global_ids, requested_GNOs); Zoltan_ZG_Export (zz, &graph, &gvtx, &nvtx, NULL, NULL, &vtxdist, &xadj, &adjncy, &adjproc, NULL, NULL); if (gvtx > (ZOLTAN_GNO_TYPE)INT_MAX){ if (zz->Proc == 0){ fprintf(stderr, "Zoltan_Color_Test assumes number of vertices (%ld) is less than INT_MAX\n",gvtx); } ierr = ZOLTAN_FATAL; goto End; } KDDKDDKDD(zz->Proc, "COLORTEST DD"); /* Exchange global color information */ if (vtxdist[zz->Num_Proc] && !(color = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int)))) MEMORY_ERROR; if (nvtx && !(local_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(nvtx * sizeof(ZOLTAN_GNO_TYPE)))) MEMORY_ERROR; for (i=0; i<nvtx; ++i) local_GNOs[i] = i+vtxdist[zz->Proc]; if (vtxdist[zz->Num_Proc] && !(global_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(vtxdist[zz->Num_Proc] * sizeof(ZOLTAN_GNO_TYPE)))) MEMORY_ERROR; for (i=0; i<vtxdist[zz->Num_Proc]; ++i) global_GNOs[i] = i; ierr = Zoltan_DD_Create (&dd_color, zz->Communicator, sizeof(ZOLTAN_GNO_TYPE)/sizeof(ZOLTAN_ID_TYPE), 0, 0, num_obj, 0); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot construct DDirectory."); /* Put req obs with 1 but first inialize the rest with 0 */ ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)local_GNOs, NULL, NULL, color, nvtx); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory."); ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)requested_GNOs, NULL, NULL, color_exp, num_obj); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory."); /* Get requested colors from the DDirectory. */ ierr = Zoltan_DD_Find (dd_color, (ZOLTAN_ID_PTR)global_GNOs, NULL, NULL, color, vtxdist[zz->Num_Proc], NULL); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot find object in DDirectory."); /* Free DDirectory */ Zoltan_DD_Destroy(&dd_color); ZOLTAN_FREE(&local_GNOs); ZOLTAN_FREE(&global_GNOs); KDDKDDKDD(zz->Proc, "COLORTEST CHECK"); if (coloring_problem == 'P' || coloring_problem == '2') { if (vtxdist[zz->Num_Proc] && !(partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int)))) MEMORY_ERROR; if (vtxdist[zz->Num_Proc] && !(loc_partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int)))) MEMORY_ERROR; if (coloring_problem == 'P') { for (i=0; i<num_obj; ++i) { int gno=requested_GNOs[i]; loc_partialD2[gno] = 1; } MPI_Allreduce(loc_partialD2, partialD2, vtxdist[zz->Num_Proc], MPI_INT, MPI_LOR, zz->Communicator); } else { for (i=0; i<vtxdist[zz->Num_Proc]; ++i) partialD2[i] = 1; } } /* Check if there is an error in coloring */ if (coloring_problem == '1') { for (i=0; i<nvtx; i++) { int gno = i + (int)vtxdist[zz->Proc]; if (color[gno] <= 0) { /* object i is not colored properly */ ierr = ZOLTAN_FATAL; printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]); break; } for (j = xadj[i]; j < xadj[i+1]; ++j) { int v = (int)adjncy[j]; if (color[gno] == color[v]) { /* neighbors have the same color */ ierr = ZOLTAN_FATAL; printf("Error in coloring! u:%d, v:%d, cu:%d, cv:%d\n", gno, v, color[gno], color[v]); break; } } if (ierr == ZOLTAN_FATAL) break; } } else if (coloring_problem == '2' || coloring_problem == 'P') { for (i=0; i<nvtx; i++) { int gno = i + (int)vtxdist[zz->Proc]; if (partialD2[gno] && color[gno] <= 0) { /* object i is not colored properly */ ierr = ZOLTAN_FATAL; printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]); break; } for (j = xadj[i]; j < xadj[i+1]; ++j) { int v = (int)adjncy[j], k; if (partialD2[v] && color[v] <= 0) { ierr = ZOLTAN_FATAL; printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]); break; } if (partialD2[gno] && partialD2[v] && color[gno] == color[v]) { /* d-1 neighbors have the same color */ ierr = ZOLTAN_FATAL; printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]); break; } for (k = j+1; k < xadj[i+1]; ++k) { int w = (int)adjncy[k]; if (partialD2[v] && partialD2[w] && color[v] == color[w]) { /* d-2 neighbors have the same color */ ierr = ZOLTAN_FATAL; printf("Error in coloring! d2-neigh: v:%d, w:%d, cv:%d, cw:%d pv:%d pw:%d\n", v, w, color[v], color[w], partialD2[v], partialD2[w]); break; } } } if (ierr == ZOLTAN_FATAL) break; } } else ZOLTAN_COLOR_ERROR(ZOLTAN_WARN, "Zoltan_Color_Test is only implemented for distance-1 and distance-2 coloring. Unknown coloring, skipping verification."); End: KDDKDDKDD(zz->Proc, "COLORTEST DONE"); if (ierr==ZOLTAN_FATAL) ierr = 2; MPI_Allreduce(&ierr, &ferr, 1, MPI_INT, MPI_MAX, zz->Communicator); if (ferr == 2) ierr = ZOLTAN_FATAL; else ierr = ZOLTAN_OK; Zoltan_ZG_Free (&graph); ZOLTAN_FREE(&adjproc); ZOLTAN_FREE(&color); ZOLTAN_FREE(&requested_GNOs); ZOLTAN_FREE(&partialD2); ZOLTAN_FREE(&loc_partialD2); return ierr; }
int Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix, int request_GNOs, /* Input: Flag indicating calling code needs translation of extra GIDs to GNOs; partial 2D coloring needs this feature. */ int num_requested, /* Input: Local # of GIDs needing translation to GNOs. */ ZOLTAN_ID_PTR requested_GIDs, /* Input: Calling code requests the GNOs for these GIDs */ ZOLTAN_GNO_TYPE *requested_GNOs /* Output: Return GNOs of the requested GIDs. */ ) { static char *yo = "Zoltan_Matrix_Build"; int ierr = ZOLTAN_OK; int nX; ZOLTAN_GNO_TYPE tmp; ZOLTAN_GNO_TYPE *xGNO = NULL; ZOLTAN_ID_PTR xLID=NULL; ZOLTAN_ID_PTR xGID=NULL; ZOLTAN_ID_PTR yGID=NULL; ZOLTAN_ID_PTR pinID=NULL; float *xwgt = NULL; int * Input_Parts=NULL; struct Zoltan_DD_Struct *dd = NULL; int *proclist = NULL; int *xpid = NULL; int i; int gno_size_for_dd; MPI_Datatype zoltan_gno_mpi_type; int use_full_dd = (opt->speed == MATRIX_FULL_DD); int fast_build_base = opt->fast_build_base; matrix->opts.speed = opt->speed; matrix->opts.fast_build_base = opt->fast_build_base; ZOLTAN_TRACE_ENTER(zz, yo); if (num_requested && (!requested_GIDs || !requested_GNOs)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error in requested input; needed arrays are NULL.\n"); } /* ZOLTAN_GNO_TYPE is >= ZOLTAN_ID_TYPE */ gno_size_for_dd = sizeof(ZOLTAN_GNO_TYPE) / sizeof(ZOLTAN_ID_TYPE); zoltan_gno_mpi_type = Zoltan_mpi_gno_type(); memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */ memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options)); /**************************************************/ /* Obtain vertex information from the application */ /**************************************************/ ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID, zz->Obj_Weight_Dim, &xwgt, &Input_Parts); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data"); goto End; } ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&xwgt); /*******************************************************************/ /* Assign vertex consecutive numbers (gnos) */ /*******************************************************************/ if (use_full_dd) { /* Zoltan computes a translation */ /* Have to use Data Directory if request_GNOs is true. */ if (nX) { xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE)); if (xGNO == NULL) MEMORY_ERROR; } ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices"); goto End; } ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, gno_size_for_dd, 0, nX, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL, NULL, nX); if (request_GNOs) { Zoltan_DD_Find(dd, requested_GIDs, (ZOLTAN_ID_PTR) requested_GNOs, NULL, NULL, num_requested, NULL); } } else { /* We don't want to use the DD */ /* * KDDKDD 2/10/11 This code cannot work when NUM_GID_ENTRIES>1. * KDDKDD 2/10/11 The assumption is that, if a user sets the * KDDKDD 2/10/11 appropriate parameter to enable this code, the user * KDDKDD 2/10/11 knows that his GIDs are compatible with integers. */ if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){ xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE)); if (nX && xGNO == NULL) MEMORY_ERROR; for (i=0; i < nX; i++) xGNO[i] = (ZOLTAN_GNO_TYPE)xGID[i] - fast_build_base; } else { xGNO = (ZOLTAN_GNO_TYPE *)xGID; if (fast_build_base) for (i = 0; i < nX; i++) xGNO[i] -= fast_build_base; } for (i = 0; i < num_requested; i++) requested_GNOs[i] = (ZOLTAN_GNO_TYPE)requested_GIDs[i] - fast_build_base; tmp = (ZOLTAN_GNO_TYPE)nX; MPI_Allreduce(&tmp, &matrix->globalX, 1, zoltan_gno_mpi_type, MPI_SUM, zz->Communicator); } /* I store : xGNO, xGID, xpid, */ ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, gno_size_for_dd, zz->Num_GID, sizeof(int), matrix->globalX/zz->Num_Proc, 0); CHECK_IERR; /* Hope a linear assignment will help a little */ if (matrix->globalX/zz->Num_Proc) Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc); /* Associate all the data with our xGNO */ xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int)); if (nX >0 && xpid == NULL) MEMORY_ERROR; for (i = 0 ; i < nX ; ++i) xpid[i] = zz->Proc; Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (char *)xpid, NULL, nX); ZOLTAN_FREE(&xpid); if (matrix->opts.pinwgt) matrix->pinwgtdim = zz->Edge_Weight_Dim; else matrix->pinwgtdim = 0; ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt, use_full_dd); CHECK_IERR; matrix->nY_ori = matrix->nY; if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){ goto End; } if (matrix->opts.enforceSquare && matrix->redist) { /* Convert yGID to yGNO using the same translation as x */ /* Needed for graph : rowID = colID */ /* y and x may have different distributions */ matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nY * sizeof(ZOLTAN_GNO_TYPE)); if (matrix->nY && matrix->yGNO == NULL) { ZOLTAN_FREE(&pinID); MEMORY_ERROR; } ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL, matrix->nY, NULL); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n"); ierr = ZOLTAN_FATAL; ZOLTAN_FREE(&pinID); goto End; } } if (matrix->opts.local) { /* keep only local edges */ proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); if (matrix->nPins && proclist == NULL) { ZOLTAN_FREE(&pinID); MEMORY_ERROR; } } else proclist = NULL; /* Convert pinID to pinGNO using the same translation as x */ if (use_full_dd) { matrix->pinGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nPins* sizeof(ZOLTAN_GNO_TYPE)); if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) { ZOLTAN_FREE(&pinID); MEMORY_ERROR; } ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL, matrix->nPins, proclist); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n"); ierr = ZOLTAN_FATAL; goto End; } ZOLTAN_FREE(&pinID); Zoltan_DD_Destroy(&dd); dd = NULL; } else { if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){ matrix->pinGNO = (ZOLTAN_GNO_TYPE *)ZOLTAN_MALLOC(matrix->nPins * sizeof(ZOLTAN_GNO_TYPE)); if (matrix->nPins && !matrix->pinGNO){ ZOLTAN_FREE(&pinID); MEMORY_ERROR; } for (i=0; i < matrix->nPins; i++) matrix->pinGNO[i] = (ZOLTAN_GNO_TYPE)pinID[i] - fast_build_base; ZOLTAN_FREE(&pinID); } else{ matrix->pinGNO = (ZOLTAN_GNO_TYPE *) pinID; if (fast_build_base) for (i=0; i < matrix->nPins; i++) matrix->pinGNO[i] -= fast_build_base; pinID = NULL; } } /* if (matrix->opts.local) { /\* keep only local edges *\/ */ /* int *nnz_list; /\* nnz offset to delete *\/ */ /* int nnz; /\* number of nnz to delete *\/ */ /* int i; */ /* nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */ /* if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */ /* for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */ /* if (proclist[i] == zz->Proc) continue; */ /* nnz_list[nnz++] = i; */ /* } */ /* ZOLTAN_FREE(&proclist); */ /* Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */ /* } */ if (!matrix->opts.enforceSquare) { /* Hyperedges name translation is different from the one of vertices */ matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_CALLOC(matrix->nY, sizeof(ZOLTAN_GNO_TYPE)); if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR; /* int nGlobalEdges = 0; */ ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY, matrix->opts.randomize, &matrix->globalY); CHECK_IERR; /* /\**************************************************************************************** */ /* * If it is desired to remove dense edges, divide the list of edges into */ /* * two lists. The ZHG structure will contain the removed edges (if final_output is true), */ /* * and the kept edges will be returned. */ /* ****************************************************************************************\/ */ /* totalNumEdges = zhg->globalHedges; */ /* ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */ /* &nLocalEdges, &nGlobalEdges, &nPins, */ /* &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */ /* if (nGlobalEdges < totalNumEdges){ */ /* /\* re-assign edge global numbers if any edges were removed *\/ */ /* ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */ /* randomizeInitDist, &totalNumEdges); */ /* if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */ /* goto End; */ /* } */ /* } */ /* We have to define ddY : yGNO, yGID, ywgt */ ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, gno_size_for_dd, zz->Num_GID, 0, matrix->globalY/zz->Num_Proc, 0); /* Hope a linear assignment will help a little */ if (matrix->globalY/zz->Num_Proc) Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc); /* Associate all the data with our yGNO */ Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, NULL, NULL, matrix->nY); } End: ZOLTAN_FREE(&xpid); ZOLTAN_FREE(&xLID); ZOLTAN_FREE(&xGNO); ZOLTAN_FREE(&xGID); ZOLTAN_FREE(&xwgt); ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&proclist); if (dd != NULL) Zoltan_DD_Destroy(&dd); /* Already stored in the DD */ ZOLTAN_FREE(&yGID); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
static void showGraphPartitions(int me, struct Zoltan_DD_Struct *dd) { int i, j, rc, part, cuts, prevPart, nparts=0; int *partCount; float imbal, localImbal, sum; ZOLTAN_ID_TYPE gid[25]; int parts[25]; for (i=0; i < 25; i++){ gid[i] = i+1; } rc = Zoltan_DD_Find(dd, gid, NULL, NULL, parts, 25, NULL); if (me > 0){ return; } for (i=0; i < 25; i++){ if (parts[i] > nparts) nparts = parts[i]; } nparts++; partCount = (int *)calloc(sizeof(int), nparts); cuts = 0; for (i=20; i >= 0; i-=5){ for (j=0; j < 5; j++){ part = parts[i + j]; partCount[part]++; if (j > 0){ if (part == prevPart){ printf("-----%d",part); } else{ printf("--x--%d",part); cuts++; prevPart = part; } } else{ printf("%d",part); prevPart = part; } } printf("\n"); if (i > 0){ for (j=0; j < 5; j++){ if (parts[i+j] != parts[i+j-5]){ printf("x "); cuts++; } else{ printf("| "); } } printf("\n"); } } printf("\n"); for (sum=0, i=0; i < nparts; i++){ sum += partCount[i]; } imbal = 0; for (i=0; i < nparts; i++){ /* An imbalance measure. 1.0 is perfect balance, larger is worse */ localImbal = (nparts * partCount[i]) / sum; if (localImbal > imbal) imbal = localImbal; } printf("Object imbalance (1.0 perfect, larger numbers are worse): %f\n",imbal); printf("Total number of edge cuts: %d\n\n",cuts); if (nparts) free(partCount); }
/* if !copy, inmat is not usable after this call */ int Zoltan_Matrix2d_Distribute (ZZ* zz, Zoltan_matrix inmat, /* Cannot be const as we can share it inside outmat */ Zoltan_matrix_2d *outmat, int copy) { static char *yo = "Zoltan_Matrix2d_Distribute"; int ierr = ZOLTAN_OK; int nProc_x, nProc_y; int myProc_x, myProc_y; int i, j, cnt; int *proclist = NULL; Zoltan_Arc *nonzeros= NULL, *sendbuf= NULL; ZOLTAN_GNO_TYPE *perm_y = NULL; float *wgtarray = NULL; float *tmpwgtarray = NULL; int msg_tag = 1021982; ZOLTAN_COMM_OBJ *plan; MPI_Comm communicator = MPI_COMM_NULL; int nProc; ZOLTAN_GNO_TYPE *yGNO = NULL; ZOLTAN_GNO_TYPE *pinGNO = NULL; ZOLTAN_GNO_TYPE tmp_gno; void *partdata = NULL; MPI_Datatype zoltan_gno_mpi_type; ZOLTAN_TRACE_ENTER(zz, yo); zoltan_gno_mpi_type = Zoltan_mpi_gno_type(); memcpy(&outmat->mtx, &inmat, sizeof(Zoltan_matrix)); if(copy) { /* TODO: We need to copy the arrays also */ Zoltan_Matrix_Reset (&outmat->mtx); /* Copy also directories */ outmat->mtx.ddX = Zoltan_DD_Copy (inmat.ddX); if (inmat.ddY == inmat.ddX) outmat->mtx.ddY = outmat->mtx.ddX; else outmat->mtx.ddY = Zoltan_DD_Copy (inmat.ddY); } communicator = outmat->comm->Communicator; nProc = outmat->comm->nProc; nProc_x = outmat->comm->nProc_x; nProc_y = outmat->comm->nProc_y; myProc_x = outmat->comm->myProc_x; myProc_y = outmat->comm->myProc_y; KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_Duplicates"); ierr = Zoltan_Matrix_Remove_Duplicates(zz, outmat->mtx, &outmat->mtx); /* KDDKDDKDD FIX INDENTATION OF THIS BLOCK */ if (inmat.opts.speed != MATRIX_NO_REDIST) { if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) { /* I need to know the original distribution */ if (outmat->mtx.ddX != outmat->mtx.ddY) { /* No initial distribution */ outmat->hashDistFct = (distFnct *)&Zoltan_Distribute_Linear; } else { int *cmember = NULL; cmember = (int*)ZOLTAN_MALLOC(outmat->mtx.nY*sizeof(int)); if (outmat->mtx.nY > 0 && cmember == NULL) MEMORY_ERROR; Zoltan_DD_Find (outmat->mtx.ddY, (ZOLTAN_ID_PTR)outmat->mtx.yGNO, NULL, (char *)cmember, NULL, outmat->mtx.nY, NULL); KDDKDDKDD(zz->Proc, " Zoltan_Distribute_Partition_Register"); partdata = Zoltan_Distribute_Partition_Register(zz, outmat->mtx.nY, outmat->mtx.yGNO, cmember, zz->Num_Proc, zz->Num_Proc); ZOLTAN_FREE(&cmember); Zoltan_Distribute_Set(outmat, (distFnct *)&Zoltan_Distribute_Origin, partdata); } } /* * Build comm plan for sending non-zeros to their target processors in * 2D data distribution. */ /* TRICK: create fake arc (edgeno, -1) for empty Y. Upper bound for size might be nPins + nY */ proclist = (int *)ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY) *sizeof(int)); sendbuf = (Zoltan_Arc*) ZOLTAN_MALLOC((outmat->mtx.nPins +outmat->mtx.nY)* sizeof(Zoltan_Arc)); if ((outmat->mtx.nPins + outmat->mtx.nY >0) && (proclist == NULL || sendbuf == NULL)) MEMORY_ERROR; wgtarray = (float*) ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY)*outmat->mtx.pinwgtdim*sizeof(float)); if (outmat->mtx.nPins*outmat->mtx.pinwgtdim && !wgtarray) MEMORY_ERROR; yGNO = outmat->mtx.yGNO; pinGNO = outmat->mtx.pinGNO; KDDKDDKDD(zz->Proc, " CommPlan Hash"); cnt = 0; for (i = 0; i < outmat->mtx.nY; i++) { ZOLTAN_GNO_TYPE edge_gno=-1; /* processor row for the edge */ edge_gno = yGNO[i]; for (j = outmat->mtx.ystart[i]; j < outmat->mtx.yend[i]; j++) { ZOLTAN_GNO_TYPE vtx_gno=-1; /* processor column for the vertex */ vtx_gno = pinGNO[j]; proclist[cnt] = (*outmat->hashDistFct)(edge_gno, vtx_gno, outmat->hashDistData, &sendbuf[cnt].part_y); if (proclist[cnt] < 0) /* Discard this nnz */ continue; sendbuf[cnt].GNO[0] = edge_gno; sendbuf[cnt].GNO[1] = vtx_gno; memcpy(wgtarray+cnt*outmat->mtx.pinwgtdim, outmat->mtx.pinwgt+j*outmat->mtx.pinwgtdim, outmat->mtx.pinwgtdim*sizeof(float)); cnt++; } if(outmat->mtx.ystart[i] == outmat->mtx.yend[i]) { proclist[cnt] = (*outmat->hashDistFct)(edge_gno, -1, outmat->hashDistData, &sendbuf[cnt].part_y); if (proclist[cnt] < 0) /* Discard this nnz */ continue; sendbuf[cnt].GNO[0] = edge_gno; sendbuf[cnt].GNO[1] = -1; memset(wgtarray+cnt*outmat->mtx.pinwgtdim, 0,outmat->mtx.pinwgtdim*sizeof(float)); cnt++; } } if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) Zoltan_Distribute_Partition_Free(&outmat->hashDistData); if (outmat->mtx.yend != outmat->mtx.ystart + 1) ZOLTAN_FREE(&outmat->mtx.yend); outmat->mtx.yend = NULL; ZOLTAN_FREE(&outmat->mtx.ystart); ZOLTAN_FREE(&outmat->mtx.yGNO); ZOLTAN_FREE(&outmat->mtx.pinGNO); ZOLTAN_FREE(&outmat->mtx.pinwgt); ZOLTAN_FREE(&outmat->mtx.yGID); /* * Send pins to their target processors. * They become non-zeros in the 2D data distribution. */ KDDKDDKDD(zz->Proc, " CommPlan Create"); msg_tag--; ierr = Zoltan_Comm_Create(&plan, cnt, proclist, communicator, msg_tag, &outmat->mtx.nPins); ZOLTAN_FREE(&proclist); nonzeros = (Zoltan_Arc *) ZOLTAN_MALLOC((outmat->mtx.nPins) * sizeof(Zoltan_Arc)); if (outmat->mtx.nPins && nonzeros == NULL) MEMORY_ERROR; msg_tag--; Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(Zoltan_Arc), (char *) nonzeros); ZOLTAN_FREE(&sendbuf); if (outmat->mtx.pinwgtdim) { /* We have to take care about weights */ tmpwgtarray = (float*) ZOLTAN_MALLOC(outmat->mtx.nPins*outmat->mtx.pinwgtdim*sizeof(float)); if (outmat->mtx.nPins && tmpwgtarray == NULL) MEMORY_ERROR; msg_tag--; Zoltan_Comm_Do(plan, msg_tag, (char *) wgtarray, outmat->mtx.pinwgtdim*sizeof(float), (char *) tmpwgtarray); ZOLTAN_FREE(&wgtarray); } Zoltan_Comm_Destroy(&plan); /* Unpack the non-zeros received. */ KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_DupArcs"); /* TODO: do take care about singletons */ Zoltan_Matrix_Remove_DupArcs(zz, outmat->mtx.nPins, (Zoltan_Arc*)nonzeros, tmpwgtarray, &outmat->mtx); } /* Now we just have to change numbering */ outmat->dist_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_y+1), sizeof(ZOLTAN_GNO_TYPE)); outmat->dist_x = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_x+1), sizeof(ZOLTAN_GNO_TYPE)); if (outmat->dist_y == NULL || outmat->dist_x == NULL) MEMORY_ERROR; /* FIXME: Work only in 1D */ tmp_gno = (ZOLTAN_GNO_TYPE)outmat->mtx.nY; MPI_Allgather(&tmp_gno, 1, zoltan_gno_mpi_type, outmat->dist_y+1, 1, zoltan_gno_mpi_type, communicator); for (i = 1 ; i <= nProc_y ; i ++) { outmat->dist_y[i] += outmat->dist_y[i-1]; } outmat->dist_x[1] = outmat->mtx.globalX; perm_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(outmat->mtx.nY * sizeof(ZOLTAN_GNO_TYPE)); if (outmat->mtx.nY > 0 && perm_y == NULL) MEMORY_ERROR; for (i = 0 ; i < outmat->mtx.nY ; ++i){ perm_y[i] = i + outmat->dist_y[myProc_y]; } KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute"); Zoltan_Matrix_Permute(zz, &outmat->mtx, perm_y); KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute done"); End: ZOLTAN_FREE(&perm_y); ZOLTAN_FREE(&proclist); ZOLTAN_FREE(&sendbuf); ZOLTAN_FREE(&nonzeros); ZOLTAN_FREE(&tmpwgtarray); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Order ( struct Zoltan_Struct *zz, int num_gid_entries, int num_obj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR permuted_global_ids ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids. * num_gid_entries * Output: * permuted_global_ids * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { char msg[253]; sprintf(msg, "num_gid_entries=%d is not equal to parameter setting " "NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ #ifdef HAVE_MPI strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "GLOBAL"); #else strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "LOCAL"); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* * Check that the user has allocated space for the return args. */ if (num_obj && !(gids && permuted_global_ids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "LOCAL_HSFC")) { Order_fn = Zoltan_LocalHSFC_Order; strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/ } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PTSCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_SCOTCH */ #ifdef ZOLTAN_HUND else if (!strcasecmp(opt.method, "HUND")) { ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL); goto End; } #endif /* ZOLTAN_HUND */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* TODO : Ask why useful ! */ /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /************************************ * Check for required query function ************************************/ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } /* TODO allocate all this stuff with the graph */ local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); /* * Call the actual ordering function. * Compute gid according to the local graph. */ ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 2, &local_gids, &local_rank); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* TODO: At this time, I consider rank == permuted_global_ids */ /* I store : GNO, rank, permuted GID */ /* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0); /* Hope a linear assignment will help a little */ if (local_num_obj) Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", permuted_global_ids[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } #ifdef ZOLTAN_HUND End: #endif /*ZOLTAN_HUND*/ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
static int migrate_graph(int num_exports, int num_imports, ZOLTAN_ID_TYPE *export_lids, ZOLTAN_ID_TYPE *import_gids) { int i, j, k, nextv, nextp, npins, numNewVertices, numNewPins, sum; long nbors[4]; float wgts[4]; int rc, startlocv, startloce; int status = 0; numNewVertices = numMyVertices - num_exports + num_imports; for (i=0; i < num_exports; i++){ vtxGID[export_lids[i]] = ZOLTAN_ID_INVALID; } for (i=0, nextv=0, nextp=0; i < numMyVertices; i++){ npins = nborIndex[i+1] - nborIndex[i]; if (vtxGID[i] != ZOLTAN_ID_INVALID){ if (i > nextv){ vtxGID[nextv] = vtxGID[i]; for (j=nborIndex[i], k=0; j < nborIndex[i+1]; j++, k++){ nborGID[nextp+k] = nborGID[j]; edgeWgt[nextp+k] = edgeWgt[j]; /* skip nborProc because we don't know what it is yet */ } nborIndex[nextv+1] = nborIndex[nextv] + npins; } nextv++; nextp += npins; } } numNewPins = nextp; startlocv = nextv; startloce = nextp; for (i=0; i < num_imports; i++){ numNewPins += num_neighbors(import_gids[i]); } status = reallocate_buffers(numNewVertices, numNewPins); if (status == 0){ for (i=0; i < num_imports; i++, nextv++){ vtxGID[nextv] = import_gids[i]; sum = get_nbor_info(import_gids[i], nbors, wgts); for (j=0; j < sum; j++, nextp++){ nborGID[nextp] = (ZOLTAN_ID_TYPE)nbors[j]; edgeWgt[nextp] = wgts[j]; } nborIndex[nextv+1] = nborIndex[nextv] + sum; } } else{ fprintf(stderr,"memory allocation failure in reallocate buffers\n"); return 1; } numMyVertices = numNewVertices; numMyPins = numNewPins; rc = Zoltan_DD_Update(dd, vtxGID+startlocv, NULL, NULL, NULL, num_imports); if ((rc != ZOLTAN_OK) && (rc != ZOLTAN_WARN)){ status = 1; } rc = Zoltan_DD_Find(dd, nborGID+startloce, NULL, NULL, NULL, numNewPins - startloce, nborProc+startloce); if ((rc != ZOLTAN_OK) && (rc != ZOLTAN_WARN)){ status = 1; } return status; }
int Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix) { static char *yo = "Zoltan_Matrix_Build"; int ierr = ZOLTAN_OK; int nX; int *xGNO = NULL; ZOLTAN_ID_PTR xLID=NULL; ZOLTAN_ID_PTR xGID=NULL; ZOLTAN_ID_PTR yGID=NULL; ZOLTAN_ID_PTR pinID=NULL; float *xwgt = NULL; int * Input_Parts=NULL; struct Zoltan_DD_Struct *dd = NULL; int *proclist = NULL; int *xpid = NULL; int i; ZOLTAN_TRACE_ENTER(zz, yo); memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */ memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options)); /**************************************************/ /* Obtain vertex information from the application */ /**************************************************/ ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID, zz->Obj_Weight_Dim, &xwgt, &Input_Parts); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data"); goto End; } ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&xwgt); /*******************************************************************/ /* Assign vertex consecutive numbers (gnos) */ /*******************************************************************/ if (matrix->opts.speed == MATRIX_FULL_DD) { /* Zoltan computes a translation */ if (nX) { xGNO = (int*) ZOLTAN_MALLOC(nX*sizeof(int)); if (xGNO == NULL) MEMORY_ERROR; } ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices"); goto End; } ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, 1, 0, nX, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL, NULL, nX); } else { /* We don't want to use the DD */ xGNO = (int *) xGID; MPI_Allreduce(&nX, &matrix->globalX, 1, MPI_INT, MPI_SUM, zz->Communicator); } /* I store : xGNO, xGID, xpid, */ ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID, 1, matrix->globalX/zz->Num_Proc, 0); CHECK_IERR; /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc); /* Associate all the data with our xGNO */ xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int)); if (nX >0 && xpid == NULL) MEMORY_ERROR; for (i = 0 ; i < nX ; ++i) xpid[i] = zz->Proc; Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (ZOLTAN_ID_PTR) xpid, NULL, nX); ZOLTAN_FREE(&xpid); if (matrix->opts.pinwgt) matrix->pinwgtdim = zz->Edge_Weight_Dim; else matrix->pinwgtdim = 0; ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt); CHECK_IERR; matrix->nY_ori = matrix->nY; if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){ goto End; } if (matrix->opts.enforceSquare && matrix->redist) { /* Convert yGID to yGNO using the same translation as x */ /* Needed for graph : rowID = colID */ /* y and x may have different distributions */ matrix->yGNO = (int*)ZOLTAN_MALLOC(matrix->nY * sizeof(int)); if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR; ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL, matrix->nY, NULL); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n"); ierr = ZOLTAN_FATAL; goto End; } } if (matrix->opts.local) { /* keep only local edges */ proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); if (matrix->nPins && proclist == NULL) MEMORY_ERROR; } else proclist = NULL; /* Convert pinID to pinGNO using the same translation as x */ if (matrix->opts.speed == MATRIX_FULL_DD) { matrix->pinGNO = (int*)ZOLTAN_MALLOC(matrix->nPins* sizeof(int)); if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) MEMORY_ERROR; ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL, matrix->nPins, proclist); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n"); ierr = ZOLTAN_FATAL; goto End; } ZOLTAN_FREE(&pinID); Zoltan_DD_Destroy(&dd); dd = NULL; } else { matrix->pinGNO = (int *) pinID; pinID = NULL; } /* if (matrix->opts.local) { /\* keep only local edges *\/ */ /* int *nnz_list; /\* nnz offset to delete *\/ */ /* int nnz; /\* number of nnz to delete *\/ */ /* int i; */ /* nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */ /* if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */ /* for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */ /* if (proclist[i] == zz->Proc) continue; */ /* nnz_list[nnz++] = i; */ /* } */ /* ZOLTAN_FREE(&proclist); */ /* Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */ /* } */ if (!matrix->opts.enforceSquare) { /* Hyperedges name translation is different from the one of vertices */ matrix->yGNO = (int*)ZOLTAN_CALLOC(matrix->nY, sizeof(int)); if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR; /* int nGlobalEdges = 0; */ ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY, matrix->opts.randomize, &matrix->globalY); CHECK_IERR; /* /\**************************************************************************************** */ /* * If it is desired to remove dense edges, divide the list of edges into */ /* * two lists. The ZHG structure will contain the removed edges (if final_output is true), */ /* * and the kept edges will be returned. */ /* ****************************************************************************************\/ */ /* totalNumEdges = zhg->globalHedges; */ /* ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */ /* &nLocalEdges, &nGlobalEdges, &nPins, */ /* &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */ /* if (nGlobalEdges < totalNumEdges){ */ /* /\* re-assign edge global numbers if any edges were removed *\/ */ /* ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */ /* randomizeInitDist, &totalNumEdges); */ /* if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */ /* goto End; */ /* } */ /* } */ /* We have to define ddY : yGNO, yGID, ywgt */ ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, 1, zz->Num_GID, 0, matrix->globalY/zz->Num_Proc, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc); /* Associate all the data with our yGNO */ Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, NULL, NULL, matrix->nY); } End: ZOLTAN_FREE(&xpid); ZOLTAN_FREE(&xLID); ZOLTAN_FREE(&xGNO); ZOLTAN_FREE(&xGID); ZOLTAN_FREE(&xwgt); ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&proclist); if (dd != NULL) Zoltan_DD_Destroy(&dd); /* Already stored in the DD */ ZOLTAN_FREE(&yGID); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int num_gid_entries, /* # of entries for a global id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm /* iperm[rank[i]]=i, only for sequential ordering */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; int * vtxdist = NULL; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL, *local_iperm=NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value"); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.rank = rank; zz->Order.iperm = iperm; zz->Order.gids = gids; zz->Order.lids = lids; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); #ifdef HAVE_MPI strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN); #else strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); zz->Order.start_index = opt.start_index; /* * Check that the user has allocated space for the return args. */ if (!(gids && rank)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); /* strcpy(opt.order_type, "GLOBAL"); */ } #endif /* ZOLTAN_SCOTCH */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } if (!strcmp(opt.order_type, "GLOBAL")) strcpy (opt.order_type, "DIST"); if (!strcmp(opt.order_type, "LOCAL")) strcpy (opt.order_type, "SERIAL"); strcpy(zz->Order.order_type, opt.order_type); /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. * Compute gid according to the local graph. */ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 3, &local_gids, &local_rank, &local_iperm); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ if ((!(opt.return_args & RETURN_RANK) && (rank != NULL)) || (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) { ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr) { /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) { /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) { /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* I store : GNO, rank, iperm */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); ZOLTAN_FREE(&local_iperm); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
/* Performs a permutation of the matrix, perm_y A perm_y^t. * TODO: at this time we only do symmetric permutations (don't know xGNO !). */ int Zoltan_Matrix_Permute(ZZ* zz, Zoltan_matrix *m, int* perm_y) { static char *yo = "Zoltan_Matrix_Permute"; int ierr = ZOLTAN_OK; int *pinGNO = NULL; ZOLTAN_ID_PTR yGID=NULL; float *ywgt=NULL; struct Zoltan_DD_Struct *dd; ZOLTAN_TRACE_ENTER(zz, yo); /* First apply y permutation */ if (m->completed) { /* We directly know the good arrays */ yGID = m->yGID; ywgt = m->ywgt; if (m->ddY == NULL || m->ddY != m->ddX) { /* We have to create again the DD */ /* We have to define ddY : yGNO, yGID, ywgt */ ierr = Zoltan_DD_Create (&m->ddY, zz->Communicator, 1, zz->Num_GID, m->ywgtdim*sizeof(float)/sizeof(int), m->globalY/zz->Num_Proc, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(m->ddY, m->globalY/zz->Num_Proc); } } else { /* We have to get these fields */ /* Update data directories */ yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY); ywgt = (float*) ZOLTAN_MALLOC(m->nY * sizeof(float) * m->ywgtdim); if (m->nY && (yGID == NULL || (m->ywgtdim && ywgt == NULL))) MEMORY_ERROR; /* Get Informations about Y */ Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL, m->nY, NULL); } memcpy (m->yGNO, perm_y, m->nY*sizeof(int)); /* Get Informations about Y */ Zoltan_DD_Update (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL, m->nY); ZOLTAN_FREE (&yGID); ZOLTAN_FREE (&ywgt); /* We have to define dd : old_yGNO, new_yGNO */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, 1, 1, 0, m->globalY/zz->Num_Proc, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, m->globalY/zz->Num_Proc); Zoltan_DD_Update (dd, (ZOLTAN_ID_PTR)m->yGNO, (ZOLTAN_ID_PTR)perm_y, NULL, NULL, m->nY); pinGNO = (int*)ZOLTAN_MALLOC(m->nPins*sizeof(int)); if (m->nPins && pinGNO == NULL) MEMORY_ERROR; Zoltan_DD_Find (dd, (ZOLTAN_ID_PTR)m->pinGNO, (ZOLTAN_ID_PTR)pinGNO, NULL, NULL, m->nY, NULL); ZOLTAN_FREE(&m->pinGNO); m->pinGNO = pinGNO; pinGNO = NULL; End: ZOLTAN_FREE (&pinGNO); ZOLTAN_FREE (&yGID); ZOLTAN_FREE (&ywgt); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int main(int argc, char *argv[]) { int i, rc; int myRank, numProcs; float ver; struct Zoltan_Struct *zz; int changes, numGidEntries, numLidEntries, numImport, numExport, start_gid, num_nbors; ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids; int *importProcs, *importToPart, *exportProcs, *exportToPart; int *parts=NULL; ZOLTAN_ID_PTR lids=NULL; FILE *fp; struct Zoltan_DD_Struct *dd; GRAPH_DATA myGraph; int gid_length = 1; /* our global IDs consist of 1 integer */ int lid_length = 1; /* our local IDs consist of 1 integer */ /****************************************************************** ** Initialize MPI and Zoltan ******************************************************************/ MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &myRank); MPI_Comm_size(MPI_COMM_WORLD, &numProcs); rc = Zoltan_Initialize(argc, argv, &ver); if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); exit(0); } /****************************************************************** ** Read graph from input file and distribute it ******************************************************************/ fp = fopen(fname, "r"); if (!fp){ if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname); MPI_Finalize(); exit(1); } fclose(fp); read_input_file(myRank, numProcs, fname, &myGraph); /*fprintf(stderr,"%d have %d objects\n",myRank,myGraph.numMyVertices);*/ /****************************************************************** ** Create a distributed data directory which maps vertex ** global IDs to their current partition number. We'll use this ** after migrating vertices, to update the partition in which ** our vertices neighbors are. ** ** Our local IDs (array "lids") are of type ZOLTAN_ID_TYPE because ** we are using Zoltan's distributed data directory. It assumes ** that a global ID is a sequence of "gid_length" ZOLTAN_ID_TYPEs. ** It assumes that a local ID is a sequence of "lid_length" ** ZOLTAN_ID_TYPEs. ******************************************************************/ rc = Zoltan_DD_Create(&dd, MPI_COMM_WORLD, gid_length, /* length of a global ID */ lid_length, /* length of a local ID */ 0, /* length of user data */ myGraph.numMyVertices, /* hash table size */ 0); /* debug level */ parts = malloc(myGraph.numMyVertices * sizeof(int)); lids = malloc(myGraph.numMyVertices * sizeof(ZOLTAN_ID_TYPE)); for (i=0; i < myGraph.numMyVertices; i++){ parts[i] = myRank; /* part number of this vertex */ lids[i] = (ZOLTAN_ID_TYPE)i; /* local ID on my process for this vertex */ } rc = Zoltan_DD_Update(dd, myGraph.vertexGID, lids, NULL, parts, myGraph.numMyVertices); myGraph.dd = dd; /****************************************************************** ** Create a Zoltan library structure for this instance of load ** balancing. Set the parameters and query functions that will ** govern the library's calculation. See the Zoltan User's ** Guide for the definition of these and many other parameters. ******************************************************************/ zz = Zoltan_Create(MPI_COMM_WORLD); /* General parameters */ Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"); Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION"); Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); /* Graph parameters */ Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */ /* Query functions, defined in this source file */ Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph); Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph); Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph); Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph); Zoltan_Set_Obj_Size_Multi_Fn(zz, get_message_sizes,&myGraph); Zoltan_Set_Pack_Obj_Multi_Fn(zz, pack_object_messages,&myGraph); Zoltan_Set_Unpack_Obj_Multi_Fn(zz, unpack_object_messages,&myGraph); Zoltan_Set_Mid_Migrate_PP_Fn(zz, mid_migrate,&myGraph); /****************************************************************** ** Visualize the graph partitioning before calling Zoltan. ******************************************************************/ if (myRank== 0){ printf("\nGraph partition before calling Zoltan\n"); } showGraphPartitions(myRank, myGraph.dd); /****************************************************************** ** Zoltan can now partition the simple graph. ** In this simple example, we assume the number of partitions is ** equal to the number of processes. Process rank 0 will own ** partition 0, process rank 1 will own partition 1, and so on. ******************************************************************/ rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */ &changes, /* 1 if partitioning was changed, 0 otherwise */ &numGidEntries, /* Number of integers used for a global ID */ &numLidEntries, /* Number of integers used for a local ID */ &numImport, /* Number of vertices to be sent to me */ &importGlobalGids, /* Global IDs of vertices to be sent to me */ &importLocalGids, /* Local IDs of vertices to be sent to me */ &importProcs, /* Process rank for source of each incoming vertex */ &importToPart, /* New partition for each incoming vertex */ &numExport, /* Number of vertices I must send to other processes*/ &exportGlobalGids, /* Global IDs of the vertices I must send */ &exportLocalGids, /* Local IDs of the vertices I must send */ &exportProcs, /* Process to which I send each of the vertices */ &exportToPart); /* Partition to which each vertex will belong */ if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); Zoltan_Destroy(&zz); exit(0); } /*fprintf(stderr,"%d export %d import %d\n",myRank,numExport,numImport);*/ /****************************************************************** ** Update the data directory with the new partition numbers ******************************************************************/ for (i=0; i < numExport; i++){ parts[exportLocalGids[i]] = exportToPart[i]; } rc = Zoltan_DD_Update(dd, myGraph.vertexGID, lids, NULL, parts, myGraph.numMyVertices); /****************************************************************** ** Migrate vertices to new partitions ******************************************************************/ rc = Zoltan_Migrate(zz, numImport, importGlobalGids, importLocalGids, importProcs, importToPart, numExport, exportGlobalGids, exportLocalGids, exportProcs, exportToPart); /****************************************************************** ** Use the data dictionary to find neighbors' partitions ******************************************************************/ start_gid = myGraph.numMyVertices - numImport; num_nbors = myGraph.nborIndex[myGraph.numMyVertices] - myGraph.nborIndex[start_gid]; rc = Zoltan_DD_Find(dd, (ZOLTAN_ID_PTR)(myGraph.nborGID + start_gid), NULL, NULL, myGraph.nborPart + start_gid, num_nbors, NULL); /****************************************************************** ** Visualize the graph partitioning after calling Zoltan. ******************************************************************/ if (myRank == 0){ printf("Graph partition after calling Zoltan\n"); } showGraphPartitions(myRank, myGraph.dd); /****************************************************************** ** Free the arrays allocated by Zoltan_LB_Partition, and free ** the storage allocated for the Zoltan structure. ******************************************************************/ Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart); Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart); Zoltan_Destroy(&zz); /********************** ** all done *********** **********************/ MPI_Finalize(); if (myGraph.vertex_capacity > 0){ free(myGraph.vertexGID); free(myGraph.nborIndex); if (myGraph.nbor_capacity > 0){ free(myGraph.nborGID); free(myGraph.nborPart); } } if (parts) free(parts); if (lids) free(lids); return 0; }
int Zoltan_PHG_2ways_hyperedge_partition ( ZZ *zz, /* Input : Zoltan data structure */ HGraph *hg, Partition parts, Zoltan_PHG_Tree *tree, struct Zoltan_DD_Struct * gnoToGID, struct Zoltan_DD_Struct **dd, int *numParts, int **sizeParts ) { int ierr = ZOLTAN_OK; char *yo = "Zoltan_PHG_2ways_hyperedge_partition"; int nEdge, hEdge; int *interval; int *partnumber = NULL; int tree_size; int *rowpart =NULL; int *rowGNO = NULL; ZOLTAN_ID_PTR rowGID=NULL; int index; int offset; ZOLTAN_TRACE_ENTER(zz, yo); nEdge = hg->nEdge; fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y, hg->nVtx, nEdge); interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int)); if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR; tree_size = get_tree_size(tree) + 1; for (index = 0 ; index < nEdge ; ++index){ SET_MIN_NODE(interval, index, tree_size); SET_MAX_NODE(interval, index, -1); } /* Update interval with the local knowledge */ /* XXX: I loop on the hyperedges, as I think it's more cache friendly * and it allows me to discoupled the computation if a non blocking MPI_Reduce is * available */ for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){ int part; int max = -1; /* Trick : we use the initialized values */ int min = tree_size + 1; for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) { part = parts[hg->hvertex[index]]; max = MAX(max, part); min = MIN(min, part); } SET_MIN_NODE(interval, hEdge, min); SET_MAX_NODE(interval, hEdge, max); } /* Update results to view the complete hyperedges */ Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm); /* Now I have to compute the partition of hyperedges according to the "interval" * and the tree */ /* First, compute the partition number corresponding to the nodes in the tree */ partnumber = compute_part_number(tree); if (partnumber == NULL) { ierr = ZOLTAN_FATAL; goto End; } (*numParts) = get_tree_size(tree); rowpart = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int)); if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR; rowGNO = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int)); if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR; (*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int)); if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR; offset = hg->dist_y[hg->comm->myProc_y]; /* Then we search we is the hyperedge in the tree */ for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) { int node; node = find_interval_in_tree(tree, interval+2*hEdge); rowpart[hEdge] = partnumber[node]; (*sizeParts)[rowpart[hEdge]] ++; rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge); #if 0 fprintf (stderr, "%d : %d (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]); #endif } partnumber += 1; ZOLTAN_FREE(&partnumber); ZOLTAN_FREE(&interval); /* Compute number of elements per parts */ /* TODO: support processor which are not part of the distribution */ /* Update results to view the complete hyperedges */ Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm); /* Export results to data directory */ /* First, get the GIDs of our edges */ rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge); if (nEdge && rowGID == NULL) MEMORY_ERROR; ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL, nEdge, NULL); ZOLTAN_FREE(&rowGNO); ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, (ZOLTAN_ID_PTR) rowpart, NULL, NULL, nEdge); #ifdef CEDRIC_PRINT for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) { fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]); } #endif End: ZOLTAN_FREE(&rowGID); ZOLTAN_FREE(&rowGNO); ZOLTAN_FREE(&rowpart); if (partnumber != NULL) partnumber += 1; ZOLTAN_FREE(&partnumber); ZOLTAN_FREE(&interval); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Matrix_Sym(ZZ* zz, Zoltan_matrix *matrix, int bipartite) { static char *yo = "Zoltan_Matrix_Sym"; int ierr = ZOLTAN_OK; Zoltan_Arc *tr_tab = NULL; int i, j, cnt; ZOLTAN_ID_PTR yGID = NULL; int *ypid=NULL; float *pinwgt=NULL; int * ybipart = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (bipartite || !matrix->opts.enforceSquare) { matrix->bipartite = 1; matrix->redist = 1; } if (matrix->ywgtdim != zz->Obj_Weight_Dim) FATAL_ERROR("Cannot form bipartite graph: vertex and edge weights are not consistant"); matrix->opts.symmetrize = 1; /* Update the data directories */ tr_tab = (Zoltan_Arc*) ZOLTAN_MALLOC(sizeof(Zoltan_Arc)*(matrix->nPins*2+matrix->nY)); if (matrix->nPins && tr_tab == NULL) MEMORY_ERROR; pinwgt = (float*)ZOLTAN_MALLOC((matrix->nPins*2+matrix->nY)*matrix->pinwgtdim*sizeof(float)); for (i = 0 ; i < 2 ; ++i) /* Copy pin weights */ memcpy(pinwgt + i*matrix->nPins*matrix->pinwgtdim*sizeof(float), matrix->pinwgt, matrix->nPins*matrix->pinwgtdim*sizeof(float)); for (i=0, cnt = 0 ; i < matrix->nY ; ++i) { for (j = matrix->ystart[i] ; j < matrix->yend[i] ; ++j) { tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */ tr_tab[cnt].GNO[1] = matrix->pinGNO[j]; memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim, matrix->pinwgtdim*sizeof(float)); cnt ++; tr_tab[cnt].GNO[0] = matrix->pinGNO[j]; /* Symmetric arc */ tr_tab[cnt].GNO[1] = matrix->yGNO[i] + bipartite*matrix->globalX; /* new ordering */ memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim, matrix->pinwgtdim*sizeof(float)); cnt ++; } if (matrix->ystart[i] == matrix->yend[i]) { /* Singleton */ tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */ tr_tab[cnt].GNO[1] = -1; cnt ++; } } ZOLTAN_FREE(&matrix->pinwgt); Zoltan_Matrix_Remove_DupArcs(zz, cnt, tr_tab, pinwgt, matrix); ZOLTAN_FREE(&tr_tab); ZOLTAN_FREE(&pinwgt); if (bipartite) { int endX; int * yGNO = NULL; /* Update data directories */ yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nY); ypid = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int)); ybipart = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int)); for (endX = 0 ; endX < matrix->nY ; ++endX) { if (matrix->yGNO[endX] >= matrix->globalX) break; ybipart[endX] = 0; } /* Get Informations about X */ Zoltan_DD_Find (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, NULL, endX, NULL); yGNO = (int*)ZOLTAN_MALLOC(endX*sizeof(int)); for (i = endX ; i < matrix->nY ; ++i) { yGNO[i-endX] = matrix->yGNO[i] - matrix->globalX; /* TODO: add a something to have the correct ypid */ ybipart[endX] = 1; } /* Get Informations about Y */ Zoltan_DD_Find (matrix->ddY, (ZOLTAN_ID_PTR)yGNO, yGID + endX*zz->Num_GID, NULL, NULL, matrix->nY-endX, NULL); if (matrix->ddY != matrix->ddX) Zoltan_DD_Destroy (&matrix->ddY); Zoltan_DD_Destroy (&matrix->ddX); matrix->globalX += matrix->globalY; matrix->globalY = matrix->globalX; /* I store : xGNO, xGID, xpid, bipart */ ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID, 1, matrix->globalX/zz->Num_Proc, 0); matrix->ddY = matrix->ddX; /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc); /* Associate all the data with our xyGNO */ Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, ybipart, matrix->nY); } End: ZOLTAN_FREE(&ybipart); ZOLTAN_FREE(&ypid); ZOLTAN_FREE(&pinwgt); ZOLTAN_FREE(&yGID); ZOLTAN_FREE(&tr_tab); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }