int Zoltan_Distribute_layout (ZZ *zz, const PHGComm * const inlayout, int loRank, int hiRank, int reqx, int reqy, PHGComm *outlayout) { MPI_Group allgrp, newgrp; int *ranks; MPI_Comm nmpicomm; MPI_Comm ompicomm; int myProc; int i; int nProc; ompicomm = (inlayout != NULL)?inlayout->Communicator:zz->Communicator; myProc = (inlayout != NULL)?inlayout->myProc:zz->Proc; nProc= (inlayout != NULL)?inlayout->nProc:zz->Num_Proc; if (((reqx != 1) && (reqy != 1) && (nProc > 3)) && Zoltan_PHG_isPrime(nProc)) nProc--; Zoltan_PHGComm_Init(outlayout); /* create a new communicator for procs[lo..hi] */ MPI_Comm_group(ompicomm, &allgrp); ranks = (int *) ZOLTAN_MALLOC(nProc * sizeof(int)); if (!ranks) return ZOLTAN_MEMERR; for (i=loRank; i<=hiRank; ++i) ranks[i-loRank] = i; MPI_Group_incl(allgrp, nProc, ranks, &newgrp); MPI_Comm_create(ompicomm, newgrp, &nmpicomm); MPI_Group_free(&newgrp); MPI_Group_free(&allgrp); ZOLTAN_FREE(&ranks); return (Zoltan_PHG_Set_2D_Proc_Distrib(zz, nmpicomm, myProc-loRank, nProc, reqx, reqy, outlayout)); }
/* This function needs a distribution : rows then cols to work properly */ int Zoltan_ZG_Build (ZZ* zz, ZG* graph, int local) { static char *yo = "Zoltan_ZG_Build"; int ierr = ZOLTAN_OK; int diag; int *diagarray=NULL; Zoltan_matrix_options opt; char symmetrization[MAX_PARAM_STRING_LEN+1]; char bipartite_type[MAX_PARAM_STRING_LEN+1]; char weigth_type[MAX_PARAM_STRING_LEN+1]; char matrix_build_type[MAX_PARAM_STRING_LEN+1]; int bipartite = 0; #ifdef CC_TIMERS double times[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */ double gtimes[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */ char *timenames[9]= {"", "setup", "matrix build", "diag", "symmetrize", "dist lin", "2D dist", "complete", "clean up"}; MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); #endif /* CC_TIMERS */ ZOLTAN_TRACE_ENTER(zz, yo); memset (graph, 0, sizeof(ZG)); /* Read graph build parameters */ Zoltan_Bind_Param(ZG_params, "GRAPH_SYMMETRIZE", (void *) &symmetrization); Zoltan_Bind_Param(ZG_params, "GRAPH_SYM_WEIGHT", (void *) &weigth_type); Zoltan_Bind_Param(ZG_params, "GRAPH_BIPARTITE_TYPE", (void *) &bipartite_type); Zoltan_Bind_Param(ZG_params, "GRAPH_BUILD_TYPE", (void*) &matrix_build_type); /* Set default values */ strncpy(symmetrization, "NONE", MAX_PARAM_STRING_LEN); strncpy(bipartite_type, "OBJ", MAX_PARAM_STRING_LEN); strncpy(weigth_type, "ADD", MAX_PARAM_STRING_LEN); strncpy(matrix_build_type, "NORMAL", MAX_PARAM_STRING_LEN); Zoltan_Assign_Param_Vals(zz->Params, ZG_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); Zoltan_Matrix2d_Init(&graph->mtx); graph->mtx.comm = (PHGComm*)ZOLTAN_MALLOC (sizeof(PHGComm)); if (graph->mtx.comm == NULL) MEMORY_ERROR; Zoltan_PHGComm_Init (graph->mtx.comm); memset(&opt, 0, sizeof(Zoltan_matrix_options)); opt.enforceSquare = 1; /* We want a graph: square matrix */ if (!strcasecmp(weigth_type, "ADD")) opt.pinwgtop = ADD_WEIGHT; else if (!strcasecmp(weigth_type, "MAX")) opt.pinwgtop = MAX_WEIGHT; else if (!strcasecmp(weigth_type, "CMP")) opt.pinwgtop = MAX_WEIGHT; opt.pinwgt = 1; opt.randomize = 0; opt.local = local; opt.keep_distribution = 1; if (strcasecmp(symmetrization, "NONE")) { opt.symmetrize = 1; } if (!strcasecmp(matrix_build_type, "FAST")) opt.speed = MATRIX_FAST; else if (!strcasecmp(matrix_build_type, "FAST_NO_DUP")) opt.speed = MATRIX_NO_REDIST; else opt.speed = MATRIX_FULL_DD; #ifdef CC_TIMERS times[1] = Zoltan_Time(zz->Timer); #endif ierr = Zoltan_Matrix_Build(zz, &opt, &graph->mtx.mtx); CHECK_IERR; #ifdef CC_TIMERS times[2] = Zoltan_Time(zz->Timer); #endif ierr = Zoltan_Matrix_Mark_Diag (zz, &graph->mtx.mtx, &diag, &diagarray); CHECK_IERR; if (diag) { /* Some Diagonal Terms have to be removed */ ierr = Zoltan_Matrix_Delete_nnz(zz, &graph->mtx.mtx, diag, diagarray); ZOLTAN_FREE(&diagarray); CHECK_IERR; } #ifdef CC_TIMERS times[3] = Zoltan_Time(zz->Timer); #endif if (opt.symmetrize) { if (!strcasecmp(symmetrization, "BIPARTITE")) bipartite = 1; ierr = Zoltan_Matrix_Sym(zz, &graph->mtx.mtx, bipartite); CHECK_IERR; } #ifdef CC_TIMERS times[4] = Zoltan_Time(zz->Timer); #endif ierr = Zoltan_Distribute_LinearY(zz, graph->mtx.comm); CHECK_IERR; #ifdef CC_TIMERS times[5] = Zoltan_Time(zz->Timer); MPI_Barrier(zz->Communicator); #endif ierr = Zoltan_Matrix2d_Distribute (zz, graph->mtx.mtx, &graph->mtx, 0); CHECK_IERR; #ifdef CC_TIMERS times[6] = Zoltan_Time(zz->Timer); #endif ierr = Zoltan_Matrix_Complete(zz, &graph->mtx.mtx); #ifdef CC_TIMERS times[7] = Zoltan_Time(zz->Timer); #endif if (bipartite) { int vertlno; int limit; int offset; graph->bipartite = 1; graph->fixed_vertices = graph->mtx.mtx.ybipart; /* graph->fixed_vertices = (int*) ZOLTAN_MALLOC(graph->mtx.mtx.nY*sizeof(int)); */ /* if (graph->mtx.mtx.nY && graph->fixed_vertices == NULL) MEMORY_ERROR; */ /* limit = graph->mtx.mtx.offsetY; */ /* /\* What kind of vertices do we want to keep ? *\/ */ /* graph->fixObj = !strcasecmp(bipartite_type, "OBJ"); /\* Non-zero value means "objects" *\/ */ /* offset = graph->mtx.mtx.offsetY - graph->mtx.dist_y[graph->mtx.comm->myProc_y]; */ /* if (graph->fixObj) /\* What kind of vertices do we want to keep ? *\/ */ /* for (vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++ vertlno) */ /* graph->fixed_vertices[vertlno] = (vertlno < offset); */ /* else */ /* for (vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++ vertlno) */ /* graph->fixed_vertices[vertlno] = (vertlno >= offset); */ } #ifdef CC_TIMERS MPI_Barrier(zz->Communicator); times[8] = Zoltan_Time(zz->Timer); MPI_Reduce(times, gtimes, 9, MPI_DOUBLE, MPI_MAX, 0, zz->Communicator); if (!zz->Proc) { int i; printf("Total Build Time in Proc-0: %.2lf Max: %.2lf\n", times[8]-times[0], gtimes[8]-times[0]); for (i=1; i<9; ++i) printf("%-13s in Proc-0: %8.2lf Max: %8.2lf\n", timenames[i], times[i]-times[i-1], gtimes[i]-gtimes[i-1]); } #endif End: ZOLTAN_FREE(&diagarray); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }