int Zoltan_Compute_Destinations(
ZZ *zz,
int num_in,
ZOLTAN_ID_PTR in_global_ids,
ZOLTAN_ID_PTR in_local_ids,
int *in_procs,
int *num_out,
ZOLTAN_ID_PTR *out_global_ids,
ZOLTAN_ID_PTR *out_local_ids,
int **out_procs
)
{
/*
* Wrapper around Zoltan_Invert_Lists, with NULL for excluded partition arrays.
* Maintained for backward compatibility.
* Arguments are analogous to Zoltan_Invert_Lists.
*/
char *yo = "Zoltan_Compute_Destinations";
int ierr;
ZOLTAN_TRACE_ENTER(zz, yo);
ierr = Zoltan_Invert_Lists(zz, num_in,
in_global_ids, in_local_ids, in_procs, NULL,
num_out,
out_global_ids, out_local_ids, out_procs, NULL);
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
/* This function compute the indices of the diagonal terms.
This function needs that diagonal terms are declared at most
1 time locally.
*/
int
Zoltan_Matrix_Mark_Diag(ZZ* zz, const Zoltan_matrix* const m,
int *n_nnz, int **nnz)
{
static char *yo = "Zoltan_Matrix_Mark_Diag";
int ierr = ZOLTAN_OK;
int y;
ZOLTAN_TRACE_ENTER(zz, yo);
(*nnz) = (int*)ZOLTAN_MALLOC(m->nY*sizeof(int));
if (m->nY && (*nnz) == NULL)
MEMORY_ERROR;
(*n_nnz) = 0;
for (y = 0 ; y < m->nY ; ++y) {
int pin;
for (pin = m->ystart[y] ; pin < m->yend[y] ; ++pin) {
if (m->pinGNO[pin] == m->yGNO[y]) {
(*nnz)[(*n_nnz)] = pin;
(*n_nnz)++;
}
}
}
if (*n_nnz == 0) ZOLTAN_FREE(nnz); /* Avoid memory leaks */
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int
Zoltan_ZG_Vertex_Info(ZZ* zz, const ZG *const graph,
ZOLTAN_ID_PTR *pgid, ZOLTAN_ID_PTR *plid, float **pwwgt, int **pinput_part) {
static char *yo = "Zoltan_ZG_Vertex_Info";
int ierr = ZOLTAN_OK;
float *wgt = NULL;
int *input_part = NULL;
ZOLTAN_ID_PTR lid = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
AFFECT_NOT_NULL(pgid, graph->mtx.mtx.yGID);
if (pwwgt != NULL) {
wgt = *pwwgt = (float*) ZOLTAN_MALLOC(graph->mtx.mtx.nY*zz->Obj_Weight_Dim*sizeof(float));
if (graph->mtx.mtx.nY >0 && zz->Obj_Weight_Dim > 0 && *pwwgt == NULL) MEMORY_ERROR;
}
if (pinput_part != NULL) {
input_part = *pinput_part = (int*) ZOLTAN_MALLOC(graph->mtx.mtx.nY*sizeof(int));
if (graph->mtx.mtx.nY > 0 && *pinput_part == NULL) MEMORY_ERROR;
}
if (plid != NULL) {
lid = *plid = ZOLTAN_MALLOC_LID_ARRAY(zz, graph->mtx.mtx.nY);
if (graph->mtx.mtx.nY >0 && zz->Num_LID >0 && *plid == NULL)
MEMORY_ERROR;
}
ierr = Zoltan_Matrix_Vertex_Info(zz, &graph->mtx.mtx, lid,
wgt, input_part);
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/* This function may work on any distribution of the bipartite graph */
int
Zoltan_ZG_Register(ZZ* zz, ZG* graph, int* properties)
{
static char *yo = "Zoltan_ZG_Register";
int ierr = ZOLTAN_OK;
int *props;
struct Zoltan_DD_Struct *dd;
int size;
ZOLTAN_ID_PTR GID;
ZOLTAN_TRACE_ENTER(zz, yo);
size = graph->mtx.mtx.nY;
dd = graph->mtx.mtx.ddY;
if (graph->bipartite) { /* Need to construct another properties array with only the fixed elements ! */
int vertlno;
if (graph->fixObj) {
dd = graph->mtx.mtx.ddX;
}
props = (int*)ZOLTAN_MALLOC(sizeof(int)*size);
if (graph->mtx.mtx.nY && props == NULL) MEMORY_ERROR;
GID = ZOLTAN_MALLOC_GID_ARRAY(zz, size);
if (size && GID == NULL) MEMORY_ERROR;
for (size = 0, vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++vertlno) {
if (graph->fixed_vertices[vertlno]) {
props[size] = properties[vertlno];
ZOLTAN_SET_GID(zz, GID+ size*zz->Num_GID,
graph->mtx.mtx.yGID+vertlno*zz->Num_GID);
size ++;
}
}
}
else {
props = properties;
GID = graph->mtx.mtx.yGID;
if (graph->mtx.mtx.ddY == NULL) {
ierr = Zoltan_DD_Create (&graph->mtx.mtx.ddY, zz->Communicator, 1, zz->Num_GID,
1, graph->mtx.mtx.globalY/zz->Num_Proc, 0);
CHECK_IERR;
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(graph->mtx.mtx.ddY, graph->mtx.mtx.globalX/zz->Num_Proc);
}
dd = graph->mtx.mtx.ddY;
}
/* Make our new numbering public */
ierr = Zoltan_DD_Update (dd, GID, NULL, NULL, props, size);
CHECK_IERR;
End:
if (graph->bipartite) {
ZOLTAN_FREE(&props);
ZOLTAN_FREE(&GID);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/* This function removes nnz which are listed as arguments (list of indexes in
pin* arrays.
nnz array have to be sorted.
*/
int
Zoltan_Matrix_Delete_nnz(ZZ* zz, Zoltan_matrix* m,
const int n_nnz, const int* const nnz)
{
static char *yo = "Zoltan_Matrix_Delete_nnz";
int ierr = ZOLTAN_OK;
int i;
int y;
if (n_nnz == 0)
return (ZOLTAN_OK);
ZOLTAN_TRACE_ENTER(zz, yo);
if (m->yend == m->ystart + 1) { /* Cannot do this "efficiently" in compact mode */
m->yend = (int*)ZOLTAN_MALLOC(m->nY*sizeof(int));
if (m->nY && m->yend == NULL)
MEMORY_ERROR;
memcpy(m->yend, m->ystart+1, m->nY*sizeof(int));
}
/* Loop over elements we have to remove */
for (i = 0, y=0; i < n_nnz ; ) {
int lenght=0;
int n_removed = 0;
while (y < m->nY && !(m->ystart[y] <= nnz[i] && m->yend[y] > nnz[i] )) {
y++;
}
if (y >= m->nY){
ierr = ZOLTAN_WARN;
break;
}
while (i<n_nnz && nnz[i] < m->yend[y]) {
if (i+1 < n_nnz) lenght = MIN(nnz[i+1], m->yend[y]);
else lenght = m->yend[y];
lenght -= nnz[i]+1; /* We remove at least nnz[i] */
memmove(m->pinGNO+nnz[i], m->pinGNO+nnz[i]+1, lenght*sizeof(int));
memmove(m->pinwgt+nnz[i]*m->pinwgtdim, m->pinwgt+(nnz[i]+1)*m->pinwgtdim,
lenght*sizeof(float)*m->pinwgtdim);
n_removed ++;
i++;
}
m->yend[y] -= n_removed;
}
m->nPins -= n_nnz;
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int
Zoltan_Matrix_Complete(ZZ* zz,Zoltan_matrix* m)
{
static char *yo = "Zoltan_Matrix_Complete";
int ierr = ZOLTAN_OK;
if(m->completed)
return (ZOLTAN_OK);
ZOLTAN_TRACE_ENTER(zz, yo);
if (m->yend != m->ystart +1 ) {/* Not in compact mode yet */
int y;
/* I have to also rewrites all the pinarrays */
for (y = 1 ; y <= m->nY ; ++y) {
int length;
if (m->ystart[y] == m->yend[y-1]) /* No hole */
continue;
length = m->yend[y]-m->ystart[y];
memcpy(m->pinGNO+m->yend[y-1], m->pinGNO+m->ystart[y], length*sizeof(ZOLTAN_GNO_TYPE));
memcpy(m->pinwgt+m->yend[y-1]*m->pinwgtdim,
m->pinGNO+m->ystart[y]*m->pinwgtdim, length*sizeof(float)*m->pinwgtdim);
m->ystart[y] = m->yend[y-1];
m->yend[y] = m->ystart[y] + length;
}
ZOLTAN_FREE(&m->yend);
m->yend = m->ystart + 1;
}
/* Update data directories */
m->yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY);
m->ypid = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int));
if (m->bipartite)
m->ybipart = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int));
if (m->nY && ((m->yGID == NULL) || (m->ypid == NULL) || (m->bipartite && m->ybipart == NULL)))
MEMORY_ERROR;
/* Get Informations about Y */
Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, m->yGID, (char *)m->ypid, m->ybipart,
m->nY, NULL);
if (m->ddY != m->ddX) {
Zoltan_DD_Destroy(&m->ddY);
m->ddY = NULL;
}
m->completed = 1;
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Help_Migrate(
ZZ *zz,
int num_import,
ZOLTAN_ID_PTR import_global_ids,
ZOLTAN_ID_PTR import_local_ids,
int *import_procs,
int num_export,
ZOLTAN_ID_PTR export_global_ids,
ZOLTAN_ID_PTR export_local_ids,
int *export_procs
)
{
/*
* Wrapper around Zoltan_Migrate with NULL pointers for partition arrays.
* Maintained for backward compatibility.
* Arguments are same as for Zoltan_Migrate.
*/
char *yo = "Zoltan_Help_Migrate";
int ierr;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->LB.PartDist != NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Non-uniform distribution of partitions over processors is specified; "
"use Zoltan_Migrate\n");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Migrate.Pre_Migrate_PP || zz->Migrate.Mid_Migrate_PP ||
zz->Migrate.Post_Migrate_PP) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Partition information not available in Zoltan_Help_Migrate for "
"ZOLTAN_*_MIGRATE_PP_FNs; use ZOLTAN_*_MIGRATE_FNs instead.");
ierr = ZOLTAN_FATAL;
goto End;
}
/*
* Wrapper (for backward compatilibity) around Zoltan_Migrate.
* Passes NULL for partition assignment arrays.
*/
ierr = Zoltan_Migrate(zz, num_import, import_global_ids, import_local_ids,
import_procs, NULL,
num_export, export_global_ids, export_local_ids,
export_procs, NULL);
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
int
Zoltan_Matrix_Construct_CSR(ZZ *zz, int size, Zoltan_Arc *arcs, float* pinwgt,
Zoltan_matrix *outmat, int offset)
{
static char *yo = "Zoltan_Matrix_Remove_DupArcs";
int *tmparray=NULL;
int ierr = ZOLTAN_OK;
int nY, nPin;
int i;
ZOLTAN_TRACE_ENTER(zz, yo);
tmparray = (int*)ZOLTAN_CALLOC(outmat->nY, sizeof(int));
/* Count degree for each vertex */
for (i = 0 ; i < size ; i++) {
int lno = arcs[i].yGNO - offset;
if (arcs[i].pinGNO != -1)
tmparray[lno] ++;
}
outmat->ystart[0] = 0;
outmat->yend = outmat->ystart + 1;
for (i = 0 ; i < outmat->nY ; i++) { /* Assume compact mode */
outmat->yend[i] = outmat->ystart[i] + tmparray[i] ;
}
memset(tmparray, 0, sizeof(int)*outmat->nY);
outmat->nPins = 0;
for(i = 0 ; i <size; i++) {
int lno = arcs[i].yGNO - offset;
if (arcs[i].pinGNO == -1)
continue;
outmat->pinGNO[outmat->ystart[lno] + tmparray[lno]] = arcs[i].pinGNO;
tmparray[lno]++;
outmat->nPins ++;
}
outmat->pinGNO = (int *) ZOLTAN_REALLOC(outmat->pinGNO, outmat->nPins * sizeof(int));
outmat->pinwgt = (float *) ZOLTAN_REALLOC(outmat->pinwgt,
outmat->nPins*outmat->pinwgtdim*sizeof(float));
End:
ZOLTAN_FREE(&tmparray);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int
Zoltan_Matrix_Vertex_Info(ZZ* zz, const Zoltan_matrix * const m,
ZOLTAN_ID_PTR lid,
float *wwgt, int *input_part)
{
static char *yo = "Zoltan_Matrix_Vertex_Info";
int ierr = ZOLTAN_OK;
int nX;
ZOLTAN_ID_PTR l_gid = NULL;
ZOLTAN_ID_PTR l_lid = NULL;
float * l_xwgt = NULL;
int *l_input_part = NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (m->completed == 0) {
ierr = ZOLTAN_FATAL;
goto End;
}
ierr = Zoltan_Get_Obj_List(zz, &nX, &l_gid, &l_lid,
zz->Obj_Weight_Dim, &l_xwgt,
&l_input_part);
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, zz->Num_LID,
zz->Obj_Weight_Dim*sizeof(float)/sizeof(int),
nX, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (dd, l_gid, l_lid, (ZOLTAN_ID_PTR) l_xwgt,l_input_part, nX);
ZOLTAN_FREE(&l_gid);
ZOLTAN_FREE(&l_lid);
ZOLTAN_FREE(&l_xwgt);
ZOLTAN_FREE(&l_input_part);
ierr = Zoltan_DD_Find (dd, m->yGID, lid, (ZOLTAN_ID_PTR)wwgt, input_part,
m->nY, NULL);
End:
if (dd != NULL)
Zoltan_DD_Destroy(&dd);
ZOLTAN_FREE(&l_gid);
ZOLTAN_FREE(&l_lid);
ZOLTAN_FREE(&l_xwgt);
ZOLTAN_FREE(&l_input_part);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/* Function that removes locale duplicated nnz */
int
Zoltan_Matrix_Remove_Duplicates(ZZ *zz, Zoltan_matrix inmat, Zoltan_matrix *outmat)
{
static char *yo = "Zoltan_Matrix_Remove_Duplicates";
int ierr = ZOLTAN_OK;
Zoltan_Arc *arcs = NULL;
float *pinwgt = NULL;
int freeflag = 0;
int size;
int i, j, cnt;
ZOLTAN_TRACE_ENTER(zz, yo);
if (inmat.opts.symmetrize == 0) /* No symmetrization, we hope no duplicates ...*/
goto End;
size = inmat.nPins + inmat.nY; /* We add fake arcs for non connected vertices */
arcs = (Zoltan_Arc*) ZOLTAN_MALLOC(size*sizeof(Zoltan_Arc));
if (inmat.nPins && arcs == NULL) MEMORY_ERROR;
for (i = 0, cnt=0 ; i < inmat.nY; i++) {
/* Fake arc in order to be sure to keep this vertex */
arcs[cnt].yGNO = inmat.yGNO[i];
arcs[cnt].pinGNO = -1;
arcs[cnt].offset = -1;
cnt++;
for (j = inmat.ystart[i]; j < inmat.yend[i]; j++) {
arcs[cnt].yGNO = inmat.yGNO[i];
arcs[cnt].pinGNO = inmat.pinGNO[j];
arcs[cnt].offset = j;
cnt ++;
}
}
pinwgt = inmat.pinwgt;
if (pinwgt == outmat->pinwgt) {
freeflag = 1;
outmat->pinwgt = (float*) ZOLTAN_MALLOC(inmat.pinwgtdim*inmat.nPins*sizeof(float));
if (inmat.pinwgtdim && inmat.nPins && outmat->pinwgt == NULL) MEMORY_ERROR;
}
ierr = Zoltan_Matrix_Remove_DupArcs(zz, size, arcs, pinwgt,outmat);
if (freeflag)
ZOLTAN_FREE(&pinwgt);
End:
ZOLTAN_FREE(&arcs);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Get_Obj_List_Special_Malloc(
ZZ *zz,
int *num_obj,
ZOLTAN_ID_PTR *global_ids,
ZOLTAN_ID_PTR *local_ids,
int wdim,
float **objwgts,
int **parts
)
{
char *yo = "Zoltan_Get_Obj_List_Special_Malloc";
int rc=ZOLTAN_OK;
ZOLTAN_TRACE_ENTER(zz, yo);
rc = _Zoltan_Get_Obj_List(zz, num_obj, global_ids, local_ids,
wdim, objwgts, parts, 1);
ZOLTAN_TRACE_EXIT(zz, yo);
return rc;
}
int
Zoltan_Matrix2d_adjproc (ZZ* zz, const Zoltan_matrix_2d * const mat, int **adjproc)
{
static char *yo = "Zoltan_Matrix2d_adjproc";
int ierr = ZOLTAN_OK;
int i;
ZOLTAN_TRACE_ENTER(zz, yo);
*adjproc = (int*) ZOLTAN_MALLOC(mat->mtx.nPins*sizeof(int));
if (mat->mtx.nPins && (*adjproc == NULL))
MEMORY_ERROR;
for (i = 0 ; i < mat->mtx.nPins ; ++i ) {
(*adjproc)[i] = EDGE_TO_PROC_Y(mat, mat->mtx.pinGNO[i]);
}
End:
ZOLTAN_TRACE_EXIT(zz, yo);
KDDKDDKDD(zz->Proc, "Zoltan_Matrix2d_adjproc done");
return (ierr);
}
int Zoltan_LB_Partition(
ZZ *zz,
int *changes,
int *num_gid_entries,
int *num_lid_entries,
int *num_import_objs,
ZOLTAN_ID_PTR *import_global_ids,
ZOLTAN_ID_PTR *import_local_ids,
int **import_procs,
int **import_to_part,
int *num_export_objs,
ZOLTAN_ID_PTR *export_global_ids,
ZOLTAN_ID_PTR *export_local_ids,
int **export_procs,
int **export_to_part
)
{
/*
* Wrapper around Zoltan_LB to generate partition information.
* Arguments correspond directly with arguments of Zoltan_LB.
*/
char *yo = "Zoltan_LB_Partition";
int ierr = ZOLTAN_OK; /* Error code */
ZOLTAN_TRACE_ENTER(zz, yo);
ierr = Zoltan_LB(zz, 1, changes, num_gid_entries, num_lid_entries,
num_import_objs, import_global_ids, import_local_ids,
import_procs, import_to_part,
num_export_objs, export_global_ids,
export_local_ids, export_procs, export_to_part);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
/* Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
ZZ *zz, /* Zoltan data structure */
HGraph *hg, /* Input hypergraph to be partitioned */
int p, /* Input: number partitions to be generated */
float *part_sizes, /* Input: array of length p containing percentages
of work to be assigned to each partition */
Partition parts, /* Input: initial partition #s; aligned with vtx
arrays.
Output: computed partition #s */
PHGPartParams *hgp, /* Input: parameters for hgraph partitioning. */
int level)
{
PHGComm *hgc = hg->comm;
VCycle *vcycle=NULL, *del=NULL;
int i, err = ZOLTAN_OK;
int prevVcnt = 2*hg->dist_x[hgc->nProc_x];
int prevVedgecnt = 2*hg->dist_y[hgc->nProc_y];
char *yo = "Zoltan_PHG_Partition";
static int timer_match = -1, /* Timers for various stages */
timer_coarse = -1, /* Declared static so we can accumulate */
timer_refine = -1, /* times over calls to Zoltan_PHG_Partition */
timer_coarsepart = -1,
timer_project = -1,
timer_vcycle = -1; /* times everything in Vcycle not included
in above timers */
int do_timing = (hgp->use_timers > 1);
int vcycle_timing = (hgp->use_timers > 4);
ZOLTAN_TRACE_ENTER(zz, yo);
if (do_timing) {
if (timer_vcycle < 0)
timer_vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
if (timer_match < 0)
timer_match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
if (timer_coarse < 0)
timer_coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
if (timer_coarsepart < 0)
timer_coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
"Coarse_Partition");
if (timer_refine < 0)
timer_refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
if (timer_project < 0)
timer_project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
return ZOLTAN_MEMERR;
}
/****** Coarsening ******/
#define COARSEN_FRACTION_LIMIT 0.9 /* Stop if we don't make much progress */
while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > hg->redl)
&& ((hg->dist_x[hgc->nProc_x] < (int) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5))
|| (hg->dist_y[hgc->nProc_y] < (int) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5)))
&& hg->dist_y[hgc->nProc_y] && hgp->matching) {
int *match = NULL;
VCycle *coarser=NULL;
prevVcnt = hg->dist_x[hgc->nProc_x];
prevVedgecnt = hg->dist_y[hgc->nProc_y];
#ifdef _DEBUG
/* UVC: load balance stats */
Zoltan_PHG_LoadBalStat(zz, hg);
#endif
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc,
"START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_match, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_match < 0) {
char str[80];
sprintf(str, "VC Matching %d", hg->info);
vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
}
/* Allocate and initialize Matching Array */
if (hg->nVtx && !(match = (int*) ZOLTAN_MALLOC (hg->nVtx*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
return ZOLTAN_MEMERR;
}
for (i = 0; i < hg->nVtx; i++)
match[i] = i;
/* Calculate matching (packing or grouping) */
err = Zoltan_PHG_Matching (zz, hg, match, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
ZOLTAN_FREE ((void**) &match);
goto End;
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_match, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_coarse, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_coarse < 0) {
char str[80];
sprintf(str, "VC Coarsening %d", hg->info);
vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
}
if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
ZOLTAN_FREE ((void**) &match);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
goto End;
}
/* Construct coarse hypergraph and LevelMap */
err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
&vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData,
&vcycle->comm_plan, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
hgc->Communicator);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_coarse, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
ZOLTAN_FREE ((void**) &match);
if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
goto End;
vcycle = coarser;
hg = vcycle->hg;
}
if (hgp->output_level >= PHG_DEBUG_LIST) {
uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
if (hgp->output_level > PHG_DEBUG_LIST) {
err = Zoltan_HG_Info(zz, hg);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
}
}
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
"coarsening plot");
/* free array that may have been allocated in matching */
if (hgp->vtx_scal) ZOLTAN_FREE(&(hgp->vtx_scal));
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_coarsepart, hgc->Communicator);
}
/****** Coarse Partitioning ******/
err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
goto End;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_coarsepart, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
del = vcycle;
/****** Uncoarsening/Refinement ******/
while (vcycle) {
VCycle *finer = vcycle->finer;
hg = vcycle->hg;
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_refine, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_refine < 0) {
char str[80];
sprintf(str, "VC Refinement %d", hg->info);
vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
}
err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_refine, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
hgc->Communicator);
if (hgp->output_level >= PHG_DEBUG_LIST)
uprintf(hgc,
"FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl,
hgp->redm_str,
hgp->coarsepartition_str, hgp->refinement_str, p,
Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, p, vcycle->Part),
Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));
if (hgp->output_level >= PHG_DEBUG_PLOT)
Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
"partitioned plot");
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_project, hgc->Communicator);
}
if (vcycle_timing) {
if (vcycle->timer_project < 0) {
char str[80];
sprintf(str, "VC Project Up %d", hg->info);
vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
}
ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
}
/* Project coarse partition to fine partition */
if (finer) {
int *rbuffer;
/* easy to undo internal matches */
for (i = 0; i < finer->hg->nVtx; i++)
if (finer->LevelMap[i] >= 0)
finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
/* fill sendbuffer with part data for external matches I owned */
for (i = 0; i < finer->LevelCnt; i++) {
++i; /* skip return lno */
finer->LevelData[i] = finer->Part[finer->LevelData[i]];
}
/* allocate rec buffer */
rbuffer = NULL;
if (finer->LevelSndCnt > 0) {
rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
if (!rbuffer) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
return ZOLTAN_MEMERR;
}
}
/* get partition assignments from owners of externally matchted vtxs */
Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1,
(char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);
/* process data to undo external matches */
for (i = 0; i < 2 * finer->LevelSndCnt;) {
int lno, partition;
lno = rbuffer[i++];
partition = rbuffer[i++];
finer->Part[lno] = partition;
}
ZOLTAN_FREE (&rbuffer);
Zoltan_Comm_Destroy (&finer->comm_plan);
}
if (do_timing) {
ZOLTAN_TIMER_STOP(zz->ZTime, timer_project, hgc->Communicator);
ZOLTAN_TIMER_START(zz->ZTime, timer_vcycle, hgc->Communicator);
}
if (vcycle_timing)
ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
hgc->Communicator);
vcycle = finer;
} /* while (vcycle) */
End:
vcycle = del;
while (vcycle) {
if (vcycle_timing) {
Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
Zoltan_Timer_Destroy(&vcycle->timer);
}
if (vcycle->finer) { /* cleanup by level */
Zoltan_HG_HGraph_Free (vcycle->hg);
Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part, &vcycle->LevelMap,
&vcycle->LevelData, &vcycle->hg);
}
else /* cleanup top level */
Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
&vcycle->LevelData);
del = vcycle;
vcycle = vcycle->finer;
ZOLTAN_FREE(&del);
}
if (do_timing)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_vcycle, hgc->Communicator);
ZOLTAN_TRACE_EXIT(zz, yo) ;
return err;
}
int Zoltan_Migrate(
ZZ *zz, /* Zoltan structure. */
int num_import, /* Number of non-local objects assigned to the
processor in the new decomposition. */
ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects
assigned to this processor in the new
decomposition; this field can be NULL if
the application doesn't provide import IDs.*/
ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects
assigned to the processor in the new
decomposition; this field can be NULL if the
application does not provide import IDs. */
int *import_procs, /* Array of processor IDs of processors owning
the non-local objects that are assigned to
this processor in the new decomposition; this
field can be NULL if the application does
not provide import IDs. */
int *import_to_part, /* Array of partition numbers to which imported
objects should be assigned. */
int num_export, /* Number of objs to be exported
to other processors to establish the new
decomposition. */
ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of
objects to be exported to other processors
to establish the new decomposition. */
ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of
objects to be exported to other processors
to establish the new decomposition. */
int *export_procs, /* Array of processor IDs
to which objects will be exported
to establish the new decomposition. */
int *export_to_part /* Array of partition numbers to which exported
objects should be assigned. */
)
{
/*
* Routine to help perform migration. If migration pre-processing routine
* (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn.
* It then calls a function to obtain the size of the migrating objects
* (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified
* object packing routine (ZOLTAN_PACK_OBJ_FN) for each object
* to be exported. It develops the needed communication map to move the
* objects to other processors. It performs the communication according
* to the map, and then calls an application-specified object unpacking
* routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported.
*/
char *yo = "Zoltan_Migrate";
int num_gid_entries, num_lid_entries; /* lengths of global & local ids */
int *sizes = NULL; /* sizes (in bytes) of the object data for export. */
int id_size; /* size (in bytes) of ZOLTAN_GID + padding for
alignment */
int tag_size; /* size (in bytes) of ZOLTAN_GID + one int
(for message size) */
char *export_buf = NULL; /* buffer for packing export data. */
char *import_buf = NULL; /* buffer for receiving imported data. */
char *tmp; /* temporary pointer into buffers. */
int i; /* loop counter. */
int tmp_size; /* size of a single object's data. */
int *idx = NULL; /* index used for multi-fn packs and unpacks. */
int idx_cnt = 0; /* index counter for idx array. */
ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm
buf */
ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass
NULL to query functions when NUM_LID_ENTRIES=0. */
ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */
ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */
int msgtag, msgtag2; /* Tags for communication routines */
int total_send_size; /* Total size of outcoming message (in #items) */
int total_recv_size; /* Total size of incoming message (in #items) */
int aligned_int; /* size of an int padded for alignment */
int dest; /* temporary destination partition. */
int include_parts = 0; /* flag indicating whether partition info is
provided */
int ierr = ZOLTAN_OK;
int actual_num_exp = 0;
int actual_exp_allocated = 0;
ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */
int *actual_exp_procs = NULL; /* are changing partition but not */
int *actual_exp_to_part = NULL; /* processor may not be included. */
int actual_num_imp = 0;
int actual_imp_allocated = 0;
ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */
int *actual_imp_procs = NULL; /* are changing partition but not */
int *actual_imp_to_part = NULL; /* processor may not be included. */
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
goto End;
}
/*
* Check that all procs use the same id types.
*/
ierr = check_input(zz,
((num_export >= 0 && export_to_part) ||
(num_import >= 0 && import_to_part)),
&include_parts);
if (ierr != ZOLTAN_OK)
goto End;
num_gid_entries = zz->Num_GID;
num_lid_entries = zz->Num_LID;
/*
* Check that all necessary query functions are available.
*/
if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (num_export >= 0) {
/* Build the actual export arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_export, export_global_ids, export_local_ids,
export_procs, export_to_part,
&actual_num_exp, &actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part,
&actual_exp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on actual exports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs,
zz->Communicator, msgtag, &actual_num_imp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
}
else if (num_import >= 0) {
/* Build the actual import arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_import, import_global_ids, import_local_ids,
import_procs, import_to_part,
&actual_num_imp, &actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part,
&actual_imp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on imports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs,
zz->Communicator, msgtag, &actual_num_exp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
/* Compute actual export lists for packing objects */
if (actual_num_exp > 0) {
actual_exp_allocated = 1;
actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp);
actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp);
actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp);
if (include_parts)
actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp);
if (actual_exp_gids == NULL ||
(num_lid_entries && actual_exp_lids == NULL) ||
actual_exp_procs == NULL ||
(import_to_part != NULL && actual_exp_to_part == NULL)) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
ierr = ZOLTAN_MEMERR;
goto End;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) actual_exp_gids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) actual_exp_lids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part,
(int) sizeof(int), (char *) actual_exp_to_part);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
/* Create inverse plan (i.e., plan based on exports) so can set
* variable sizes.
* (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable
* but does not tell how large to allocate receive buffer.
*/
ierr = Zoltan_Comm_Invert_Plan(&imp_plan);
exp_plan = imp_plan;
imp_plan = NULL;
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Migrate.Pre_Migrate_PP != NULL) {
zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Pre_Migrate != NULL) {
zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_FN function.");
goto End;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing");
id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE));
/* Note that alignment is not strictly necessary
when ZOLTAN_ID_TYPE is int or unsigned int. */
aligned_int = Zoltan_Align(sizeof(int));
tag_size = id_size + aligned_int;
/*
* For each object, allow space for its global ID and its data plus
* one int (for the object data size).
* Zoltan will pack the global IDs; the application must pack the data
* through the pack routine. Zoltan needs the global IDs for unpacking,
* as the order of the data received during communication is not
* necessarily the same order as import_global_ids[].
* Zoltan also needs to communicate the sizes of the objects because
* only the sender knows the size of each object.
*/
if (actual_num_exp > 0) {
sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!sizes) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Get_Obj_Size_Multi != NULL) {
zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids, sizes, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE_MULTI function.");
goto End;
}
}
else {
for (i = 0; i < actual_num_exp; i++) {
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]),
lid, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE function.");
goto End;
}
}
}
total_send_size = 0;
for (i = 0; i < actual_num_exp; i++) {
sizes[i] = Zoltan_Align(sizes[i]);
total_send_size += sizes[i] + tag_size;
}
export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char));
if (!export_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Pack_Obj_Multi != NULL) {
/* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */
idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Pack the objects for export.
*/
idx_cnt = 0;
tmp = export_buf;
for (i = 0; i < actual_num_exp; i++) {
/* Pack the object's global ID */
tmp_id = (ZOLTAN_ID_PTR) tmp;
ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries]));
tmp += id_size;
/* Pack the object's size */
*((int *)tmp) = sizes[i];
tmp += aligned_int;
/* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Pack_Obj_Multi != NULL) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n",
zz->Proc, yo);
}
zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids,
(actual_exp_to_part!=NULL ? actual_exp_to_part
: actual_exp_procs),
sizes, idx, export_buf, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ_MULTI function.");
goto End;
}
}
else {
tmp = export_buf + tag_size;
for (i = 0; i < actual_num_exp; i++) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries]));
printf("size = %d bytes\n", sizes[i]);
}
/* Pack the object's data */
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i]
: actual_exp_procs[i]);
zz->Pack_Obj(zz->Pack_Obj_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]), lid, dest,
sizes[i], tmp, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ function.");
goto End;
}
tmp += sizes[i] + tag_size;
}
}
ZOLTAN_FREE(&idx);
tmp_id = NULL;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects");
/* Modify sizes[] to contain message sizes, not object sizes */
for (i=0; i<actual_num_exp; i++) {
sizes[i] += tag_size;
}
msgtag--;
ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize.");
goto End;
}
if (actual_num_imp > 0) {
import_buf = (char *) ZOLTAN_MALLOC(total_recv_size);
if (!import_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Send the export data using the communication plan.
*/
msgtag2 = 32765;
ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
/*
* Free whatever memory we can.
*/
Zoltan_Comm_Destroy(&exp_plan);
ZOLTAN_FREE(&export_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication");
/*
* Perform application-specified processing before unpacking the data.
*/
if (zz->Migrate.Mid_Migrate_PP != NULL) {
zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Mid_Migrate != NULL) {
zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_FN function.");
goto End;
}
}
/*
* Unpack the object data.
*/
if (actual_num_imp > 0) {
if (zz->Unpack_Obj_Multi != NULL) {
/* Allocate and fill input arrays for Unpack_Obj_Multi. */
sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp);
idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
if (!sizes || !tmp_id || !idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
tmp = import_buf;
idx_cnt = 0;
for (i = 0; i < actual_num_imp; i++) {
/* Unpack the object's global ID */
ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp);
tmp += id_size;
/* Unpack the object's size */
sizes[i] = *((int *)tmp);
tmp += aligned_int;
/* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n",
zz->Proc,yo);
}
zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries,
actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr);
ZOLTAN_FREE(&import_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_FREE(&tmp_id);
ZOLTAN_FREE(&idx);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_MULTI_FN.");
goto End;
}
}
else {
tmp = import_buf;
for (i = 0; i < actual_num_imp; i++) {
tmp_size = *((int *)(tmp + id_size));
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp);
printf("size = %d bytes\n", tmp_size);
}
/* Unpack the object's data */
zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries,
(ZOLTAN_ID_PTR) tmp, tmp_size,
tmp + tag_size, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_FN.");
goto End;
}
tmp += (tmp_size + tag_size);
}
ZOLTAN_FREE(&import_buf);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects");
if (zz->Migrate.Post_Migrate_PP != NULL) {
zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Post_Migrate != NULL) {
zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_FN function.");
goto End;
}
}
End:
if (actual_exp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
}
if (actual_imp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part);
}
if (ierr < 0) {
if (exp_plan) Zoltan_Comm_Destroy(&exp_plan);
Zoltan_Multifree(__FILE__, __LINE__, 5,
&import_buf, &tmp_id, &sizes, &idx, &export_buf);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Random(
ZZ *zz, /* The Zoltan structure. */
float *part_sizes, /* Input: Array of size zz->LB.Num_Global_Parts
* zz->Obj_Weight_Dim
containing the percentage of work to be
assigned to each partition. */
int *num_import, /* Return -1. Random uses only export lists. */
ZOLTAN_ID_PTR *import_global_ids, /* Not used. */
ZOLTAN_ID_PTR *import_local_ids, /* Not used. */
int **import_procs, /* Not used. */
int **import_to_part, /* Not used. */
int *num_export, /* Output: Number of objects to export. */
ZOLTAN_ID_PTR *export_global_ids, /* Output: GIDs to export. */
ZOLTAN_ID_PTR *export_local_ids, /* Output: LIDs to export. */
int **export_procs, /* Output: Processsors to export to. */
int **export_to_part /* Output: Partitions to export to. */
)
{
int ierr = ZOLTAN_OK;
int i, count, num_obj;
int max_export;
double rand_frac = 1.0; /* Default is to move all objects. */
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int *parts = NULL;
float *dummy = NULL;
static char *yo = "Zoltan_Random";
static int first_time = 1;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Synchronize the random number generator.
* This synchronization is needed only for sanity in our nightly testing.
* If some other operation (eg., Zoltan_LB_Eval) changes the status of
* the random number generator, the answers here will change. They won't
* be wrong, but they will be different from our accepted answers.
*/
if (first_time) {
Zoltan_Srand(zz->Seed, NULL);
Zoltan_Rand(NULL);
first_time=0;
}
/* No import lists computed. */
*num_import = -1;
/* Get parameter values. */
Zoltan_Bind_Param(Random_params, "RANDOM_MOVE_FRACTION", (void *) &rand_frac);
Zoltan_Assign_Param_Vals(zz->Params, Random_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
/* Get list of local objects. */
ierr = Zoltan_Get_Obj_List(zz, &num_obj, &global_ids, &local_ids, 0,
&dummy, &parts);
/* Bound number of objects to export. */
max_export = 1.5*rand_frac*num_obj;
/* Allocate export lists. */
*export_global_ids = *export_local_ids = NULL;
*export_procs = *export_to_part = NULL;
if (max_export > 0) {
if (!Zoltan_Special_Malloc(zz, (void **)export_global_ids, max_export,
ZOLTAN_SPECIAL_MALLOC_GID)
|| !Zoltan_Special_Malloc(zz, (void **)export_local_ids, max_export,
ZOLTAN_SPECIAL_MALLOC_LID)
|| !Zoltan_Special_Malloc(zz, (void **)export_procs, max_export,
ZOLTAN_SPECIAL_MALLOC_INT)
|| !Zoltan_Special_Malloc(zz, (void **)export_to_part, max_export,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/* Randomly assign ids to procs. */
count=0;
for (i=0; i<num_obj; i++){
/* Randomly select some objects to move (export) */
if ((count<max_export) && (Zoltan_Rand(NULL)<rand_frac*ZOLTAN_RAND_MAX)){
/* export_global_ids[count] = global_ids[i]; */
ZOLTAN_SET_GID(zz, &((*export_global_ids)[count*zz->Num_GID]),
&global_ids[i*zz->Num_GID]);
if (local_ids)
/* export_local_ids[count] = local_ids[i]; */
ZOLTAN_SET_LID(zz, &((*export_local_ids)[count*zz->Num_LID]),
&local_ids[i*zz->Num_LID]);
/* Randomly pick new partition number. */
(*export_to_part)[count] = Zoltan_Rand_InRange(NULL, zz->LB.Num_Global_Parts);
/* Processor number is derived from partition number. */
(*export_procs)[count] = Zoltan_LB_Part_To_Proc(zz,
(*export_to_part)[count], &global_ids[i*zz->Num_GID]);
/* printf("Debug: Export gid %u to part %d and proc %d.\n", (*export_global_ids)[count], (*export_to_part)[count], (*export_procs)[count]); */
++count;
}
}
(*num_export) = count;
End:
/* Free local memory, but not export lists. */
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_FREE(&parts);
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
static int Zoltan_LB(
ZZ *zz,
int include_parts, /* Flag indicating whether to generate
part informtion;
0 if called by Zoltan_LB_Balance,
1 if called by Zoltan_LB_Partition. */
int *changes, /* Set to zero or one depending on if
Zoltan determines a new
decomposition or not:
zero - No changes to the decomposition
were made by the load-balancing
algorithm; migration is not needed.
one - A new decomposition is suggested
by the load-balancer; migration is
needed to establish the new
decomposition. */
int *num_gid_entries, /* The number of array entries in a global ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Global_ID_Entries. */
int *num_lid_entries, /* The number of array entries in a local ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Local_ID_Entries. */
int *num_import_objs, /* The number of non-local objects in the
processor's new decomposition. */
ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
int **import_procs, /* Array of processor IDs for processors
currently owning non-local objects (i.e.,
objs to be imported) in this processor's
new decomposition. */
int **import_to_part, /* Partition to which the objects should be
imported. */
int *num_export_objs, /* The number of local objects that need to
be exported from the processor to establish
the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
int **export_procs, /* Array of destination processor IDs for
objects that need to be exported
to establish the new decomposition. */
int **export_to_part /* Partition to which objects should be
exported. */
)
{
/*
* Main load-balancing routine.
* Input: a Zoltan structure with appropriate function pointers set.
* Output:
* changes
* num_import_objs
* import_global_ids
* import_local_ids
* import_procs
* import_to_part
* num_export_objs
* export_global_ids
* export_local_ids
* export_procs
* export_to_part
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_LB";
int gmax; /* Maximum number of imported/exported objects
over all processors. */
int error = ZOLTAN_OK; /* Error code */
double start_time, end_time;
double lb_time[2] = {0.0,0.0};
char msg[256];
int comm[3],gcomm[3];
float *part_sizes = NULL, *fdummy = NULL;
int wgt_dim, part_dim;
int all_num_obj, i, ts, idIdx;
struct Hash_Node **ht;
int *export_all_procs, *export_all_to_part, *parts=NULL;
ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL;
ZOLTAN_ID_PTR gid;
#ifdef ZOLTAN_OVIS
struct OVIS_parameters ovisParameters;
#endif
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS){
printf("Build configuration:\n");
Zoltan_Print_Configuration(" ");
printf("\n");
Zoltan_Print_Key_Params(zz);
}
start_time = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_OVIS
Zoltan_OVIS_Setup(zz, &ovisParameters);
if (zz->Proc == 0)
printf("OVIS PARAMETERS %s %s %d %f\n",
ovisParameters.hello,
ovisParameters.dll,
ovisParameters.outputLevel,
ovisParameters.minVersion);
ovis_enabled(zz->Proc, ovisParameters.dll);
#endif
/*
* Compute Max number of array entries per ID over all processors.
* Compute Max number of return arguments for Zoltan_LB_Balance.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
comm[2] = zz->LB.Return_Lists;
MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
zz->LB.Return_Lists = gcomm[2];
/* assume no changes */
*changes = 0;
*num_import_objs = *num_export_objs = 0;
*import_global_ids = NULL;
*import_local_ids = NULL;
*import_procs = NULL;
*import_to_part = NULL;
*export_global_ids = NULL;
*export_local_ids = NULL;
*export_procs = NULL;
*export_to_part = NULL;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz))
goto End;
if (zz->LB.Method == NONE) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s Balancing method selected == NONE; no balancing performed\n",
yo);
error = ZOLTAN_WARN;
goto End;
}
/*
* Sync the random number generator across processors.
*/
Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator);
/* Since generating a new partition, need to free old mapping vector */
zz->LB.OldRemap = zz->LB.Remap;
zz->LB.Remap = NULL;
error = Zoltan_LB_Build_PartDist(zz);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN)
goto End;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
int i, np, fp;
for (i = 0; i < zz->Num_Proc; i++) {
Zoltan_LB_Proc_To_Part(zz, i, &np, &fp);
printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n",
zz->Proc, i, np, fp);
}
}
/*
* Generate parts sizes.
*/
#ifdef ZOLTAN_OVIS
/* set part sizes computed by OVIS, if requested. Processes set only their own value */
{
float part_sizes[1];
int part_ids[1], wgt_idx[1];
wgt_idx[0] = 0;
part_ids[0] = 0;
ovis_getPartsize(&(part_sizes[0]));
printf("Rank %d ps %f\n",zz->Proc, part_sizes[0]);
/* clear out old part size info first */
Zoltan_LB_Set_Part_Sizes(zz, 0, -1, NULL, NULL, NULL);
Zoltan_LB_Set_Part_Sizes(zz, 0, 1, part_ids, wgt_idx, part_sizes);
}
#endif
wgt_dim = zz->Obj_Weight_Dim;
part_dim = ((wgt_dim > 0) ? wgt_dim : 1);
part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim
* zz->LB.Num_Global_Parts);
if (part_sizes == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
error = ZOLTAN_MEMERR;
goto End;
}
/* Get part sizes. */
Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim,
part_sizes);
#ifdef ZOLTAN_OVIS
/* if (ovisParameters.outputlevel > 3) */
{
int myRank = zz->Proc;
if (myRank == 0){
int i, j;
for (i = 0; i < zz->LB.Num_Global_Parts; i++){
for (j = 0; j < part_dim; j++){
printf("Rank %d AG: part_sizes[%d] = %f (Num_Global_Parts = %d, part_dim = %d)\n",zz->Proc,
(i*part_dim+j), part_sizes[i*part_dim+j],zz->LB.Num_Global_Parts, part_dim);
}
}
}
}
#endif
/*
* Call the actual load-balancing function.
*/
error = zz->LB.LB_Fn(zz, part_sizes,
num_import_objs, import_global_ids, import_local_ids,
import_procs, import_to_part,
num_export_objs, export_global_ids, export_local_ids,
export_procs, export_to_part);
ZOLTAN_FREE(&part_sizes);
if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){
sprintf(msg, "Partitioning routine returned code %d.", error);
#ifdef HOST_LINUX
if ((error == ZOLTAN_MEMERR) && (Zoltan_Memory_Get_Debug() > 0)){
Zoltan_write_linux_meminfo(0, "State of /proc/meminfo after malloc failure\n", 0);
}
#endif
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
else if (error){
if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) {
sprintf(msg, "Partitioning routine returned code %d.", error);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning");
if (*num_import_objs >= 0)
MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
else /* use export data */
MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
if (gmax == 0) {
/*
* Decomposition was not changed by the load balancing; no migration
* is needed.
*/
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s No changes to the decomposition due to partitioning; "
"no migration is needed.\n", yo);
/*
* Reset num_import_objs and num_export_objs; don't want to return
* -1 for the arrays that weren't returned by ZOLTAN_LB_FN.
*/
*num_import_objs = *num_export_objs = 0;
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){
/*
* This parameter setting requires that all local objects
* and their assignments appear in the export list.
*/
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
}
goto End;
}
/*
* Check whether we know the import data, export data, or both.
*
* If we were given the import data,
* we know what the new decomposition should look like on the
* processor, but we don't know which of our local objects we have
* to export to other processors to establish the new decomposition.
* Reverse the argument if we were given the export data.
*
* Unless we were given both maps, compute the inverse map.
*/
if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) {
if (*num_import_objs >= 0)
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
if (*num_export_objs >= 0)
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
*num_import_objs = *num_export_objs = -1;
}
if (*num_import_objs >= 0){
if (*num_export_objs >= 0) {
/* Both maps already available; nothing to do. */;
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Export lists are requested; compute export map */
error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids,
*import_local_ids, *import_procs,
*import_to_part,
num_export_objs, export_global_ids,
export_local_ids, export_procs,
export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Method returned import lists, but only export lists were desired. */
/* Import lists not needed; free them. */
*num_import_objs = -1;
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
}
}
}
else { /* (*num_import_objs < 0) */
if (*num_export_objs >= 0) {
/* Only export lists have been returned. */
if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Compute import map */
error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids,
*export_local_ids, *export_procs,
*export_to_part,
num_import_objs, import_global_ids,
import_local_ids, import_procs,
import_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Method returned export lists, but only import lists are desired. */
/* Export lists not needed; free them. */
*num_export_objs = -1;
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
}
}
}
else { /* *num_export_objs < 0 && *num_import_objs < 0) */
if (zz->LB.Return_Lists) {
/* No map at all available */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned "
"neither import nor export data.");
error = ZOLTAN_WARN;
goto End;
}
}
}
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/*
* Normally, Zoltan_LB returns in the export lists all local
* objects that are moving off processor, or that are assigned
* to a part on the local processor that is not the
* default part. This setting of Return_Lists requests
* that all local objects be included in the export list.
*/
if (*num_export_objs == 0){
/* all local objects are remaining on processor */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (*num_export_objs) {
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End;
}
else{
all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error);
if (*num_export_objs < all_num_obj){
/* Create a lookup table for exported IDs */
ts = Zoltan_Recommended_Hash_Size(*num_export_objs);
ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts);
/* Create a list of all gids, lids and parts */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj,
&all_global_ids, &all_local_ids,
wgt_dim, &fdummy, &parts);
if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){
ZOLTAN_FREE(&fdummy);
if ((Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_procs,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) ||
(Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_to_part,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){
error = ZOLTAN_MEMERR;
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){
sprintf(msg, "Error building complete export list; "
"%d returned by Zoltan_Get_Obj_List\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
gid = all_global_ids;
for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){
idIdx = search_hash_table(zz, gid, ht, ts);
if (idIdx >= 0){
export_all_procs[i] = (*export_procs)[idIdx];
export_all_to_part[i] = (*export_to_part)[idIdx];
}
else{
export_all_procs[i] = zz->Proc;
export_all_to_part[i] = parts[i];
}
}
free_hash_table(ht, ts);
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
Zoltan_Special_Free(zz, (void **)(void*)&parts,
ZOLTAN_SPECIAL_MALLOC_INT);
*export_global_ids = all_global_ids;
*export_local_ids = all_local_ids;
*export_procs = export_all_procs;
*export_to_part = export_all_to_part;
*num_export_objs = all_num_obj;
}
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments");
end_time = Zoltan_Time(zz->Timer);
lb_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc);
for (i = 0; i < *num_import_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID]));
printf(" To part: %4d",
(*import_to_part != NULL ? (*import_to_part)[i]
: zz->Proc));
printf(" From processor: %4d\n", (*import_procs)[i]);
}
printf("\n");
printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc);
for (i = 0; i < *num_export_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID]));
printf(" To part: %4d",
(*export_to_part != NULL ? (*export_to_part)[i]
: (*export_procs)[i]));
printf(" To processor: %4d\n", (*export_procs)[i]);
}
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/*
* If the Help_Migrate flag is set, perform migration for the application.
*/
if (zz->Migrate.Auto_Migrate) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration");
start_time = Zoltan_Time(zz->Timer);
error = Zoltan_Migrate(zz,
*num_import_objs, *import_global_ids,
*import_local_ids, *import_procs, *import_to_part,
*num_export_objs, *export_global_ids,
*export_local_ids, *export_procs, *export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error in auto-migration; %d returned from "
"Zoltan_Help_Migrate\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
end_time = Zoltan_Time(zz->Timer);
lb_time[1] = end_time - start_time;
ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration");
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0],
"ZOLTAN Partition: ");
if (zz->Migrate.Auto_Migrate)
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1],
"ZOLTAN Migrate: ");
}
*changes = 1;
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (error);
}
int Zoltan_LB_Balance(
ZZ *zz,
int *changes,
int *num_gid_entries,
int *num_lid_entries,
int *num_import_objs,
ZOLTAN_ID_PTR *import_global_ids,
ZOLTAN_ID_PTR *import_local_ids,
int **import_procs,
int *num_export_objs,
ZOLTAN_ID_PTR *export_global_ids,
ZOLTAN_ID_PTR *export_local_ids,
int **export_procs
)
{
/*
* Wrapper around Zoltan_LB for backward compatibility with
* previous Zoltan versions.
* Appropriate only when (# requested parts == # processors), uniformly
* distributed.
* Arguments correspond directly with arguments of Zoltan_LB.
*/
char *yo = "Zoltan_LB_Balance";
int ierr = ZOLTAN_OK; /* Error code */
int *import_to_part = NULL; /* Array used as dummy arg in partitioning. */
int *export_to_part = NULL; /* Array used as dummy arg in partitioning. */
#ifdef ZOLTAN_OVIS
struct OVIS_parameters ovisParameters;
#endif
ZOLTAN_TRACE_ENTER(zz, yo);
/* Determine whether part parameters were set. Report error if
* values are unreasonable. */
if ((zz->LB.Num_Global_Parts_Param != -1 &&
zz->LB.Num_Global_Parts_Param != zz->Num_Proc) ||
(zz->LB.Num_Local_Parts_Param != -1 &&
zz->LB.Num_Local_Parts_Param != 1)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Non-uniform distribution of parts over processors is specified; "
"use Zoltan_LB_Partition.");
ierr = ZOLTAN_FATAL;
goto End;
}
#ifdef ZOLTAN_OVIS
Zoltan_OVIS_Setup(zz, &ovisParameters);
if (ovisParameters.outputLevel > 5){
int error = ZOLTAN_OK; /* Error code */
int acg_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error);
printf ("Entering Proc %d num_objs %d\n", zz->Proc, acg_num_obj);
}
#endif
ierr = Zoltan_LB(zz, 0, changes, num_gid_entries, num_lid_entries,
num_import_objs, import_global_ids, import_local_ids,
import_procs, &import_to_part,
num_export_objs, export_global_ids,
export_local_ids, export_procs, &export_to_part);
End:
#ifdef ZOLTAN_OVIS
if (ovisParameters.outputLevel > 5){
printf ("Exiting Proc %d num_import_objs %d num_export_objs %d\n", zz->Proc, *num_import_objs, *num_export_objs);
}
#endif
/* Not returning import/export part information; free it if allocated. */
if (import_to_part != NULL)
Zoltan_Special_Free(zz, (void **)(void*) &import_to_part,
ZOLTAN_SPECIAL_MALLOC_INT);
if (export_to_part != NULL)
Zoltan_Special_Free(zz, (void **)(void*) &export_to_part,
ZOLTAN_SPECIAL_MALLOC_INT);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
int Zoltan_Get_Coordinates(
ZZ *zz,
int num_obj, /* Input: number of objects */
ZOLTAN_ID_PTR global_ids, /* Input: global IDs of objects */
ZOLTAN_ID_PTR local_ids, /* Input: local IDs of objects; may be NULL. */
int *num_dim, /* Output: dimension of coordinates */
double **coords /* Output: array of coordinates; malloc'ed by
fn if NULL upon input. */
)
{
char *yo = "Zoltan_Get_Coordinates";
int i,j,rc;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
int alloced_coords = 0;
ZOLTAN_ID_PTR lid; /* Temporary pointers to local IDs; used to pass
NULL to query functions when NUM_LID_ENTRIES == 0. */
double dist[3];
double im[3][3];
double deg_ratio;
double x;
int order[3];
int reduce_dimensions, d, axis_aligned;
int target_dim;
int ierr = ZOLTAN_OK;
char msg[256];
ZZ_Transform *tran;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Error check -- make sure needed callback functions are registered. */
if (zz->Get_Num_Geom == NULL ||
(zz->Get_Geom == NULL && zz->Get_Geom_Multi == NULL)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_GEOM_FN and "
"either ZOLTAN_GEOM_MULTI_FN or ZOLTAN_GEOM_FN");
goto End;
}
/* Get problem dimension. */
*num_dim = zz->Get_Num_Geom(zz->Get_Num_Geom_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_GET_NUM_GEOM_FN");
goto End;
}
if (*num_dim < 0 || *num_dim > 3) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Invalid dimension returned from ZOLTAN_NUM_GEOM_FN");
goto End;
}
/* Get coordinates for object; allocate memory if not already provided. */
if (*num_dim > 0 && num_obj > 0) {
if (*coords == NULL) {
alloced_coords = 1;
*coords = (double *) ZOLTAN_MALLOC(num_obj * (*num_dim) * sizeof(double));
if (*coords == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error");
goto End;
}
}
if (zz->Get_Geom_Multi != NULL) {
zz->Get_Geom_Multi(zz->Get_Geom_Multi_Data, zz->Num_GID, zz->Num_LID,
num_obj, global_ids, local_ids, *num_dim, *coords,
&ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_GET_GEOM_MULTI_FN");
goto End;
}
}
else {
for (i = 0; i < num_obj; i++) {
lid = (num_lid_entries ? &(local_ids[i*num_lid_entries]) : NULL);
zz->Get_Geom(zz->Get_Geom_Data, num_gid_entries, num_lid_entries,
global_ids + i*num_gid_entries, lid,
(*coords) + i*(*num_dim), &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_GET_GEOM_FN");
goto End;
}
}
}
}
/*
* For RCB, RIB, and HSFC: if REDUCE_DIMENSIONS was selected, compute the
* center of mass and inertial matrix of the coordinates.
*
* For 3D problems: If the geometry is "flat", transform the points so the
* two primary directions lie along the X and Y coordinate axes and project
* to the Z=0 plane. If in addition the geometry is "skinny", project to
* the X axis. (This creates a 2D or 1D problem respectively.)
*
* For 2D problems: If the geometry is essentially a line, transform it's
* primary direction to the X axis and project to the X axis, yielding a
* 1D problem.
*
* Return these points to the partitioning algorithm, in effect partitioning
* in only the 2 (or 1) significant dimensions.
*/
if (((*num_dim == 3) || (*num_dim == 2)) &&
((zz->LB.Method==RCB) || (zz->LB.Method==RIB) || (zz->LB.Method==HSFC))){
Zoltan_Bind_Param(Reduce_Dim_Params, "KEEP_CUTS", (void *)&i);
Zoltan_Bind_Param(Reduce_Dim_Params, "REDUCE_DIMENSIONS",
(void *)&reduce_dimensions);
Zoltan_Bind_Param(Reduce_Dim_Params, "DEGENERATE_RATIO", (void *)°_ratio);
i = 0;
reduce_dimensions = 0;
deg_ratio = 10.0;
Zoltan_Assign_Param_Vals(zz->Params, Reduce_Dim_Params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
if (reduce_dimensions == 0){
goto End;
}
if (deg_ratio <= 1){
if (zz->Proc == 0){
ZOLTAN_PRINT_WARN(0, yo, "DEGENERATE_RATIO <= 1, setting it to 10.0");
}
deg_ratio = 10.0;
}
if (zz->LB.Method == RCB){
tran = &(((RCB_STRUCT *)(zz->LB.Data_Structure))->Tran);
}
else if (zz->LB.Method == RIB){
tran = &(((RIB_STRUCT *)(zz->LB.Data_Structure))->Tran);
}
else{
tran = &(((HSFC_Data*)(zz->LB.Data_Structure))->tran);
}
d = *num_dim;
if (tran->Target_Dim >= 0){
/*
* On a previous load balancing call, we determined whether
* or not the geometry was degenerate. If the geometry was
* determined to be not degenerate, then we assume it is still
* not degenerate, and we skip the degeneracy calculation.
*/
if (tran->Target_Dim > 0){
/*
* The geometry *was* degenerate. We test the extent
* of the geometry along the principal directions determined
* last time to determine if it is still degenerate with that
* orientation. If so, we transform the coordinates using the
* same transformation we used last time. If not, we do the
* entire degeneracy calculation again.
*/
if ((tran->Axis_Order[0] >= 0) &&
(tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){
axis_aligned = 1;
}
else{
axis_aligned = 0;
}
projected_distances(zz, *coords, num_obj, tran->CM,
tran->Evecs, dist, d, axis_aligned, tran->Axis_Order);
target_dim = get_target_dimension(dist, order, deg_ratio, d);
if (target_dim > 0){
transform_coordinates(*coords, num_obj, d, tran);
}
else{
/* Set's Target_Dim to -1, flag to recompute degeneracy */
Zoltan_Initialize_Transformation(tran);
}
}
}
if (tran->Target_Dim < 0){
tran->Target_Dim = 0;
/*
* Get the center of mass and inertial matrix of coordinates. Ignore
* vertex weights, we are only interested in geometry. Global operation.
*/
if (d == 2){
inertial_matrix2D(zz, *coords, num_obj, tran->CM, im);
}
else{
inertial_matrix3D(zz, *coords, num_obj, tran->CM, im);
}
/*
* The inertial matrix is a 3x3 or 2x2 real symmetric matrix. Get its
* three or two orthonormal eigenvectors. These usually indicate the
* orientation of the geometry.
*/
rc = eigenvectors(im, tran->Evecs, d);
if (rc){
if (zz->Proc == 0){
ZOLTAN_PRINT_WARN(0, yo, "REDUCE_DIMENSIONS calculation failed");
}
goto End;
}
/*
* Here we check to see if the eigenvectors are very close
* to the coordinate axes. If so, we can more quickly
* determine whether the geometry is degenerate, and also more
* quickly transform the geometry to the lower dimensional
* space.
*/
axis_aligned = 0;
for (i=0; i<d; i++){
tran->Axis_Order[i] = -1;
}
for (j=0; j<d; j++){
for (i=0; i<d; i++){
x = fabs(tran->Evecs[i][j]);
if (NEAR_ONE(x)){
tran->Axis_Order[j] = i; /* e'vector j is very close to i axis */
break;
}
}
if (tran->Axis_Order[j] < 0){
break;
}
}
if ((tran->Axis_Order[0] >= 0) &&
(tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){
axis_aligned = 1;
}
/*
* Calculate the extent of the geometry along the three lines defined
* by the direction of the eigenvectors through the center of mass.
*/
projected_distances(zz, *coords, num_obj, tran->CM, tran->Evecs, dist,
d, axis_aligned, tran->Axis_Order);
/*
* Decide whether these distances indicate the geometry is
* very flat in one or two directions.
*/
target_dim = get_target_dimension(dist, order, deg_ratio, d);
if (target_dim > 0){
/*
* Yes, geometry is degenerate
*/
if ((zz->Debug_Level > 0) && (zz->Proc == 0)){
if (d == 2){
sprintf(msg,
"Geometry (~%lf x %lf), exceeds %lf to 1.0 ratio",
dist[order[0]], dist[order[1]], deg_ratio);
}
else{
sprintf(msg,
"Geometry (~%lf x %lf x %lf), exceeds %lf to 1.0 ratio",
dist[order[0]], dist[order[1]], dist[order[2]], deg_ratio);
}
ZOLTAN_PRINT_INFO(zz->Proc, yo, msg);
sprintf(msg, "We'll treat it as %d dimensional",target_dim);
ZOLTAN_PRINT_INFO(zz->Proc, yo, msg);
}
if (axis_aligned){
/*
** Create new geometry, transforming the primary direction
** to the X-axis, and the secondary to the Y-axis.
*/
tran->Permutation[0] = tran->Axis_Order[order[0]];
if (target_dim == 2){
tran->Permutation[1] = tran->Axis_Order[order[1]];
}
}
else{
/*
* Reorder the eigenvectors (they're the columns of evecs) from
* longest projected distance to shorted projected distance. Compute
* the transpose (the inverse) of the matrix. This will transform
* the geometry to align along the X-Y plane, or along the X axis.
*/
for (i=0; i< target_dim; i++){
tran->Transformation[i][2] = 0.0;
for (j=0; j<d; j++){
tran->Transformation[i][j] = tran->Evecs[j][order[i]];
}
}
for (i=target_dim; i< 3; i++){
for (j=0; j<3; j++){
tran->Transformation[i][j] = 0.0;
}
}
}
tran->Target_Dim = target_dim;
transform_coordinates(*coords, num_obj, d, tran);
} /* If geometry is very flat */
} /* If REDUCE_DIMENSIONS is true */
} /* If 2-D or 3-D rcb, rib or hsfc */
End:
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no coordinates returned.");
if (alloced_coords) ZOLTAN_FREE(coords);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
int Zoltan_LB_Set_Part_Sizes(ZZ *zz, int global_num,
int len, int *part_ids, int *wgt_idx, float *part_sizes)
{
/*
* Function to set the desired partition sizes. This function
* only sets values locally. Later, Zoltan_LB_Get_Part_Sizes
* collects all the information across processors.
*
* Input:
* zz -- The Zoltan structure to which this method
* applies.
* global_num -- Global partition numbers? (0 for local numbers)
* len -- Length of arrays wgt_idx, part_idx, part_sizes
* part_ids -- Array of partition ids (local or global)
* wgt_idx -- Array of indices between 0 and Obj_Wgt_Dim-1
* part_sizes -- Array of floats that gives the desired partition
* size for each weight and each partition, i.e.,
* part_sizes[i] corresponds to wgt_idx[i] and part_id[i]
*
* Output:
* zz->LB.* -- Appropriate fields set to designated values.
* Return value -- Error code.
*/
char *yo = "Zoltan_LB_Set_Part_Sizes";
int i, j, maxlen=0;
int error = ZOLTAN_OK;
const int INIT_NUM_PART = 64; /* Initial allocation for Part_Info array. */
ZOLTAN_TRACE_ENTER(zz, yo);
/* len = -1 will nullify all partition sizes set on this proc */
if (len == -1){
zz->LB.Part_Info_Len = 0;
goto End;
}
/* Verify input. */
if ((part_ids==NULL) || (part_sizes==NULL)){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Invalid input argument NULL.");
error = ZOLTAN_FATAL;
goto End;
}
/* Do we need more space? */
if ((!zz->LB.Part_Info) || (zz->LB.Part_Info_Max_Len==0)){
maxlen = INIT_NUM_PART; /* Start with space for INIT_NUM_PART */
zz->LB.Part_Info = (struct Zoltan_part_info *) ZOLTAN_MALLOC(maxlen *
sizeof(struct Zoltan_part_info));
}
if (zz->LB.Part_Info_Len + len > zz->LB.Part_Info_Max_Len){
maxlen = 2*(zz->LB.Part_Info_Len + len); /* Double the length */
zz->LB.Part_Info = (struct Zoltan_part_info *) ZOLTAN_REALLOC(
zz->LB.Part_Info, maxlen * sizeof(struct Zoltan_part_info));
}
if (zz->LB.Part_Info == NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
error = ZOLTAN_MEMERR;
goto End;
}
/* Add new data to partition info array. */
for (i=0,j=zz->LB.Part_Info_Len; i<len; i++,j++){
zz->LB.Part_Info[j].Size = part_sizes[i];
zz->LB.Part_Info[j].Part_id = part_ids[i];
zz->LB.Part_Info[j].Idx = (wgt_idx ? wgt_idx[i] : 0);
zz->LB.Part_Info[j].Global_num = global_num;
}
/* Update values in LB. */
zz->LB.Part_Info_Len += len;
if (maxlen > zz->LB.Part_Info_Max_Len)
zz->LB.Part_Info_Max_Len = maxlen;
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return error;
}
int Zoltan_LB_Get_Part_Sizes(ZZ *zz,
int num_global_parts, int part_dim, float *part_sizes)
{
/*
* Function to get the scaled partition sizes.
*
* Input:
* zz -- The Zoltan structure to which this method
* applies.
* num_global_parts -- Number of global partitions.
* (This usually equals lb->Num_Global_Parts)
* part_dim -- The number of object weights per partition.
* (This usually equals lb->Obj_Wgt_Dim.)
*
* Output:
* part_sizes -- Array of floats that gives the set partition
* sizes, scaled such that they sum to one.
*/
int i, j, nparts, fpart;
float *temp_part_sizes=NULL, *sum=NULL;
int error = ZOLTAN_OK;
char msg[128];
static char *yo = "Zoltan_LB_Get_Part_Sizes";
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: num_global_parts = %d\n", zz->Proc, num_global_parts);
/* Barrier to make sure all procs have finished Zoltan_LB_Set_Part_Sizes */
MPI_Barrier(zz->Communicator);
/* For convenience, if no weights are used, set part_dim to 1 */
if (part_dim==0) part_dim = 1;
if (part_sizes == NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument part_sizes is NULL.");
error = ZOLTAN_FATAL;
goto End;
}
/* Find max Part_Info_Len over all procs to see if they are all zero. */
MPI_Allreduce((void*) &(zz->LB.Part_Info_Len), (void*) &j,
1, MPI_INT, MPI_MAX, zz->Communicator);
if (j == 0){
/* Uniform partition sizes. */
zz->LB.Uniform_Parts = 1;
for (i = 0; i < num_global_parts*part_dim; i++)
part_sizes[i] = 1.0 / (float)num_global_parts;
}
else {
/* Get the partition sizes set by the user (application).
* Each processor puts its data in a part_dim * num_global_parts
* array. Then we gather all the data across processors.
* Out-of-range partition size data is ignored.
*/
zz->LB.Uniform_Parts = 0;
/* Pack LB.Part_Info into temp array */
temp_part_sizes = (float *)ZOLTAN_MALLOC(num_global_parts*part_dim
*sizeof(float));
sum = (float *)ZOLTAN_MALLOC(part_dim*sizeof(float));
if ((!temp_part_sizes) || (!sum)){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
error = ZOLTAN_MEMERR;
goto End;
}
for (i = 0; i < num_global_parts*part_dim; i++){
temp_part_sizes[i] = -1.0;
}
for (i = 0; i < zz->LB.Part_Info_Len; i++){
/* Only assemble partition sizes for partitions and weights
in the requested range. */
if (zz->LB.Part_Info[i].Idx < part_dim){
j = zz->LB.Part_Info[i].Part_id;
if (zz->LB.Part_Info[i].Global_num == 0) {
Zoltan_LB_Proc_To_Part(zz, zz->Proc, &nparts, &fpart);
j += fpart;
}
if (j >= num_global_parts){
sprintf(msg, "Partition number %d is >= num_global_parts %d.",
j, num_global_parts);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
error = ZOLTAN_WARN;
}
else
temp_part_sizes[j*part_dim + zz->LB.Part_Info[i].Idx]
= zz->LB.Part_Info[i].Size;
}
}
/* Reduce over all procs */
MPI_Allreduce((void*) temp_part_sizes, (void*) part_sizes,
num_global_parts*part_dim, MPI_FLOAT, MPI_MAX, zz->Communicator);
/* Check for errors. Scale the sizes so they sum to one for each weight. */
for (j = 0; j < part_dim; j++)
sum[j] = 0.0;
for (i = 0; i < num_global_parts; i++){
for (j = 0; j < part_dim; j++){
if (part_sizes[i*part_dim+j]<0)
part_sizes[i*part_dim+j] = 1.0; /* default value if not set */
sum[j] += part_sizes[i*part_dim+j];
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
printf("[%1d] In %s: Partition size %1d (before scaling) = ",
zz->Proc, yo, i);
for (j = 0; j < part_dim; j++)
printf("%f, ", part_sizes[i*part_dim+j]);
printf("\n");
}
}
/* Check for sum[j] == 0 (error). */
for (j = 0; j < part_dim; j++) {
if (sum[j] == 0.0) {
sprintf(msg, "Sum of weights (component %1d) is zero.", j);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
error = ZOLTAN_FATAL;
goto End;
}
}
/* Normalize partition sizes */
for (i = 0; i < num_global_parts; i++)
for (j = 0; j < part_dim; j++)
part_sizes[i*part_dim+j] /= sum[j];
}
End:
if (temp_part_sizes) ZOLTAN_FREE(&temp_part_sizes);
if (sum) ZOLTAN_FREE(&sum);
ZOLTAN_TRACE_EXIT(zz, yo);
return error;
}
/* TODO: Add an option to deal with disconnected vertices */
int
Zoltan_Matrix_Remove_DupArcs(ZZ *zz, int size, Zoltan_Arc *arcs, float* pinwgt,
Zoltan_matrix *outmat)
{
static char *yo = "Zoltan_Matrix_Remove_DupArcs";
int ierr = ZOLTAN_OK;
WgtFctPtr wgtfct;
int nY, nPin;
int i;
int prev_pinGNO;
#ifdef CC_TIMERS
double time;
#endif
ZOLTAN_TRACE_ENTER(zz, yo);
#ifdef CC_TIMERS
time = MPI_Wtime();
#endif
switch (outmat->opts.pinwgtop) {
case MAX_WEIGHT:
wgtfct = &wgtFctMax;
break;
case CMP_WEIGHT:
wgtfct = &wgtFctCmp;
break;
case ADD_WEIGHT:
default:
wgtfct = &wgtFctAdd;
}
qsort ((void*)arcs, size, sizeof(Zoltan_Arc),
(int (*)(const void*,const void*))compar_arcs);
#ifdef CC_TIMERS
fprintf(stderr, "(%d) remove arcs (qsort): %g\n", zz->Proc, MPI_Wtime()-time);
#endif
prev_pinGNO = -2;
for (i = 0, nY=-1, nPin=-1; i < size ; ++i) {
int nnew = 0;
int yGNO = arcs[i].yGNO;
int pinGNO = arcs[i].pinGNO;
nnew = ((nY < 0) || (outmat->yGNO[nY] != yGNO));
if (nnew) {
nY++;
outmat->ystart[nY] = nPin + 1;
outmat->yGNO[nY] = yGNO;
prev_pinGNO = -1;
if (pinGNO < 0)
continue;
}
nnew = nnew ||(pinGNO != prev_pinGNO);
if (nnew) { /* New edge */
nPin ++;
outmat->pinGNO[nPin] = pinGNO;
memcpy(outmat->pinwgt + nPin*outmat->pinwgtdim, pinwgt + arcs[i].offset*outmat->pinwgtdim,
outmat->pinwgtdim*sizeof(float));
}
else { /* Duplicate */
wgtfct(outmat->pinwgt + nPin* outmat->pinwgtdim,
pinwgt + arcs[i].offset*outmat->pinwgtdim, outmat->pinwgtdim);
}
prev_pinGNO = outmat->pinGNO[nPin];
}
nY ++;
outmat->ystart[nY] = nPin+1; /* compact mode */
outmat->nPins = nPin+1;
outmat->nY = nY;
/* Try to minimize memory */
/* We reduce memory, thus I don't think these realloc can fail */
if (outmat->yend != outmat->ystart + 1)
ZOLTAN_FREE(&outmat->yend);
outmat->pinGNO = (int *) ZOLTAN_REALLOC(outmat->pinGNO, outmat->nPins * sizeof(int));
outmat->pinwgt = (float *) ZOLTAN_REALLOC(outmat->pinwgt,
outmat->nPins*outmat->pinwgtdim*sizeof(float));
outmat->yend = outmat->ystart + 1;
#ifdef CC_TIMERS
fprintf(stderr, "(%d) remove arcs: %g\n", zz->Proc, MPI_Wtime()-time);
#endif
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
static int Zoltan_Reftree_Partition(ZZ *zz, float *part_sizes, int *num_export,
ZOLTAN_ID_PTR *export_global_ids, ZOLTAN_ID_PTR *export_local_ids,
int **export_to_partition, int **export_procs)
{
/*
* Function to partition the grid and fill the export arrays.
*/
char *yo = "Zoltan_Reftree_Partition";
char msg[256];
int num_exp; /* count the number of export objects */
ZOLTAN_REFTREE *root; /* root of the tree */
float *cutoff; /* the relative sizes of the partitions */
int part; /* partition under construction */
float current_size; /* amount of weight consumed so far */
int num_part; /* number of partitions */
int ierr; /* error flag */
int wdim; /* Max(zz->Obj_Weight_Dim, 1) */
ZOLTAN_TRACE_ENTER(zz, yo);
root = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->reftree_root;
/*
* determine the size of the partitions and tolerance interval
*/
num_part = zz->LB.Num_Global_Parts;
/* Set the cutoff points of summed weights for the end of each partition */
/* TEMP HA
Determine a partition of unity for the sizes of the partitions
relative to the processing power of the processors, to support
heterogeneous architectures.
cutoff(0) = is the percent of work that should be assigned to
partition 0
cutoff(i) - cutoff(i-1) is the percent of work that should be
assigned to partition i.
cutoff(num_part-1) = 1.0
When support for heterogeneous architectures is added to Zoltan,
there should be a "vector of partition sizes" passed into this
routine, which would be used to define cutoff. For now, it is
just set to be equal sizes.
*/
cutoff = (float *)ZOLTAN_MALLOC((num_part)*sizeof(float));
if (cutoff == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
cutoff[0] = part_sizes[0]; /* TEMP SINGLE WEIGHT */
wdim = ((zz->Obj_Weight_Dim == 0) ? 1 : zz->Obj_Weight_Dim);
for (part = 1; part < num_part; part++)
cutoff[part] = cutoff[part-1] + part_sizes[part*wdim]; /* TEMP SINGLE WEIGHT */
cutoff[num_part-1] = 1.0; /* just to make sure roundoff doesn't bite us */
/* multiply cutoff by the total weight so that cutoff gives the desired
actual weight of each partition instead of relative sizes */
/* TEMP HA
It is quite possible that once we know what is passed into this
routine to determine relative partition sizes, then this can be combined
with the above partition of unity to simplify the code. I just did it
this way to make it clear what is happening.
*/
for (part=0; part<num_part; part++) {
cutoff[part] = cutoff[part]*root->summed_weight[0]; /* TEMP SINGLE WEIGHT */
}
/*
* traverse the tree to define the partition and count the number of exports
*/
num_exp = 0;
part = 0;
current_size = 0.0;
ierr = Zoltan_Reftree_Part_Recursive(zz,root,&part,¤t_size,&num_exp,
cutoff,num_part);
ZOLTAN_FREE(&cutoff);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
/*
* if no exports, we're done
*/
if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) {
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Candidate Lists not supported in REFTREE;"
"change RETURN_LISTS parameter.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_FATAL);
}
else if (num_exp == 0) {
*num_export = 0;
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
/*
* allocate space for the export lists
*/
if (!Zoltan_Special_Malloc(zz,(void **)export_global_ids,num_exp,
ZOLTAN_SPECIAL_MALLOC_GID)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!Zoltan_Special_Malloc(zz,(void **)export_local_ids,num_exp,
ZOLTAN_SPECIAL_MALLOC_LID)) {
Zoltan_Special_Free(zz,(void **)export_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!Zoltan_Special_Malloc(zz,(void **)export_to_partition,num_exp,
ZOLTAN_SPECIAL_MALLOC_INT)) {
Zoltan_Special_Free(zz,(void **)export_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void **)export_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!Zoltan_Special_Malloc(zz,(void **)export_procs,num_exp,
ZOLTAN_SPECIAL_MALLOC_INT)) {
Zoltan_Special_Free(zz,(void **)export_to_partition,ZOLTAN_SPECIAL_MALLOC_INT);
Zoltan_Special_Free(zz,(void **)export_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void **)export_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
/*
* traverse the tree to make the export lists
*/
*num_export = 0;
ierr = Zoltan_Reftree_Export_Lists(zz,root,num_export,export_global_ids,
export_local_ids,export_to_partition,export_procs);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (num_exp != *num_export) {
sprintf(msg, "num_exp = %d not equal to num_export = %d.",
num_exp,*num_export);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_WARN);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
int
Zoltan_PHG_2ways_hyperedge_partition (
ZZ *zz, /* Input : Zoltan data structure */
HGraph *hg,
Partition parts,
Zoltan_PHG_Tree *tree,
struct Zoltan_DD_Struct * gnoToGID,
struct Zoltan_DD_Struct **dd,
int *numParts,
int **sizeParts
)
{
int ierr = ZOLTAN_OK;
char *yo = "Zoltan_PHG_2ways_hyperedge_partition";
int nEdge, hEdge;
int *interval;
int *partnumber = NULL;
int tree_size;
int *rowpart =NULL;
int *rowGNO = NULL;
ZOLTAN_ID_PTR rowGID=NULL;
int index;
int offset;
ZOLTAN_TRACE_ENTER(zz, yo);
nEdge = hg->nEdge;
fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y, hg->nVtx, nEdge);
interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int));
if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR;
tree_size = get_tree_size(tree) + 1;
for (index = 0 ; index < nEdge ; ++index){
SET_MIN_NODE(interval, index, tree_size);
SET_MAX_NODE(interval, index, -1);
}
/* Update interval with the local knowledge */
/* XXX: I loop on the hyperedges, as I think it's more cache friendly
* and it allows me to discoupled the computation if a non blocking MPI_Reduce is
* available
*/
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){
int part;
int max = -1; /* Trick : we use the initialized values */
int min = tree_size + 1;
for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) {
part = parts[hg->hvertex[index]];
max = MAX(max, part);
min = MIN(min, part);
}
SET_MIN_NODE(interval, hEdge, min);
SET_MAX_NODE(interval, hEdge, max);
}
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm);
/* Now I have to compute the partition of hyperedges according to the "interval"
* and the tree */
/* First, compute the partition number corresponding to the nodes in the tree */
partnumber = compute_part_number(tree);
if (partnumber == NULL) {
ierr = ZOLTAN_FATAL;
goto End;
}
(*numParts) = get_tree_size(tree);
rowpart = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int));
if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR;
rowGNO = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int));
if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR;
(*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int));
if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR;
offset = hg->dist_y[hg->comm->myProc_y];
/* Then we search we is the hyperedge in the tree */
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
int node;
node = find_interval_in_tree(tree, interval+2*hEdge);
rowpart[hEdge] = partnumber[node];
(*sizeParts)[rowpart[hEdge]] ++;
rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge);
#if 0
fprintf (stderr, "%d : %d (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]);
#endif
}
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
/* Compute number of elements per parts */
/* TODO: support processor which are not part of the distribution */
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm);
/* Export results to data directory */
/* First, get the GIDs of our edges */
rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge);
if (nEdge && rowGID == NULL) MEMORY_ERROR;
ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL,
nEdge, NULL);
ZOLTAN_FREE(&rowGNO);
ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, (ZOLTAN_ID_PTR) rowpart, NULL, NULL, nEdge);
#ifdef CEDRIC_PRINT
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]);
}
#endif
End:
ZOLTAN_FREE(&rowGID);
ZOLTAN_FREE(&rowGNO);
ZOLTAN_FREE(&rowpart);
if (partnumber != NULL)
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
static int rib_fn(
ZZ *zz, /* The Zoltan structure with info for
the RIB balancer. */
int *num_import, /* Number of non-local objects assigned to
this processor in the new decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
num_import returns the number of input
objects as given by ZOLTAN_NUM_OBJ_FN. */
ZOLTAN_ID_PTR *import_global_ids, /* Returned value: array of global IDs for
non-local objects in this processor's new
decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
this array contains GIDs for all input
objs as given by ZOLTAN_OBJ_LIST_FN.*/
ZOLTAN_ID_PTR *import_local_ids, /* Returned value: array of local IDs for
non-local objects in this processor's new
decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
this array contains LIDs for all input
objs as given by ZOLTAN_OBJ_LIST_FN.*/
int **import_procs, /* Returned value: array of processor IDs for
processors owning the non-local objects in
this processor's new decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
the returned array is NULL. */
int **import_to_part, /* Returned value: array of parts to
which objects are imported.
When LB.Return_Lists==CANDIDATE_LISTS,
the returned array is NULL. */
int *num_export, /* Returned value only when
LB.Return_Lists==CANDIDATE_LISTS; number of
input objs as given by ZOLTAN_NUM_OBJ_FN */
ZOLTAN_ID_PTR *export_global_ids, /* Returned value only when
LB.Return_Lists==CANDIDATE_LISTS; for each
input obj (from ZOLTAN_OBJ_LIST_FN),
return a candidate obj from the part to which
the obj is assigned; used in PHG matching */
double overalloc, /* amount to overallocate by when realloc
of dot array must be done.
1.0 = no extra; 1.5 = 50% extra; etc. */
int wgtflag, /* No. of weights per dot supplied by user. */
int check_geom, /* Check input & output for consistency? */
int stats, /* Print timing & count summary? */
int gen_tree, /* (0) do not (1) do generate full treept */
int average_cuts, /* (0) don't (1) compute the cut to be the
average of the closest dots. */
float *part_sizes /* Input: Array of size
zz->Num_Global_Parts * max(zz->Obj_Weight_Dim, 1)
containing the percentage of work to be
assigned to each part. */
)
{
char yo[] = "rib_fn";
int proc,nprocs; /* my proc id, total # of procs */
struct Dot_Struct *dotpt; /* temporary pointer to local dot arrays */
int pdotnum; /* # of dots - decomposition changes it */
int *dotmark = NULL; /* which side of median for each dot */
int dotnum; /* number of dots */
int dotmax = 0; /* max # of dots arrays can hold */
int dottop; /* dots >= this index are new */
int proclower; /* 1st proc in lower set */
int procmid; /* 1st proc in upper set */
int partlower; /* 1st part in lower set */
int partmid; /* 1st part in upper set */
int set; /* which set processor is in = 0/1 */
int old_set; /* set processor was in last cut = 0/1 */
int root; /* part that stores last cut */
int num_procs; /* number of procs in current set */
int num_parts; /* number of parts in current set */
int ierr = ZOLTAN_OK; /* error flag. */
double *value = NULL; /* temp array for median_find */
double *wgts = NULL; /* temp array for serial_rib */
double valuehalf; /* median cut position */
double cm[3]; /* Center of mass of objects */
double evec[3]; /* Eigenvector defining direction */
int first_guess = 0; /* flag if first guess for median search */
int allocflag; /* have to re-allocate space */
double time1=0,time2=0; /* timers */
double time3=0,time4=0; /* timers */
double timestart=0,timestop=0; /* timers */
double timers[4]={0.,0.,0.,0.};
/* diagnostic timers
0 = start-up time before recursion
1 = time before median iterations
2 = time in median iterations
3 = communication time */
ZOLTAN_GNO_TYPE counters[7]; /* diagnostic counts
0 = unused
1 = # of dots sent
2 = # of dots received
3 = most dots this proc ever owns
4 = most dot memory this proc ever allocs
5 = # of times a previous cut is re-used
6 = # of reallocs of dot array */
int i, j; /* local variables */
int use_ids; /* When true, global and local IDs will be
stored along with dots in the RCB_STRUCT.
When false, storage, manipulation, and
communication of IDs is avoided.
Set by call to Zoltan_RB_Use_IDs(). */
RIB_STRUCT *rib = NULL; /* Pointer to data structures for RIB */
struct rib_tree *treept = NULL; /* tree of cuts - single cut on exit*/
double start_time, end_time;
double lb_time[2]={0,0};
int tfs[2], tmp_tfs[2]; /* added for Tflops_Special; max number
of procs and parts over all processors
in each iteration (while loop) of
parallel partitioning. */
int old_nprocs; /* added for Tflops_Special */
int old_nparts; /* added for Tflops_Special */
double valuelo; /* smallest value of value[i] */
double valuehi; /* largest value of value[i] */
double weight[RB_MAX_WGTS]; /* weight for current set */
double weightlo[RB_MAX_WGTS]; /* weight of lower side of cut */
double weighthi[RB_MAX_WGTS]; /* weight of upper side of cut */
double fractionlo[RB_MAX_WGTS]; /* desired wt in lower half */
int *dotlist = NULL; /* list of dots for find_median.
allocated above find_median for
better efficiency (don't necessarily
have to realloc for each find_median).*/
int rectilinear_blocks = 0; /* parameter for find_median (not used by rib) */
int fp=0; /* first part assigned to this proc. */
int np=0; /* number of parts assigned to this proc. */
int wgtdim; /* max(wgtflag,1) */
int *dindx = NULL, *tmpdindx = NULL;
/* MPI data types and user functions */
MPI_Comm local_comm, tmp_comm;
int free_comm = FALSE; /* Flag indicating whether MPI_Comm_free
should be called on local_comm at end. */
ZOLTAN_TRACE_ENTER(zz, yo);
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
MPI_Barrier(zz->Communicator);
timestart = time1 = Zoltan_Time(zz->Timer);
}
/* setup for parallel */
proc = zz->Proc;
nprocs = zz->Num_Proc;
num_parts = zz->LB.Num_Global_Parts;
/*
* Determine whether to store, manipulate, and communicate global and
* local IDs.
*/
use_ids = Zoltan_RB_Use_IDs(zz);
/*
* Build the RIB Data structure and
* set pointers to information in it.
*/
start_time = Zoltan_Time(zz->Timer);
ierr = Zoltan_RIB_Build_Structure(zz, &pdotnum, &dotmax, wgtflag, overalloc,
use_ids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RIB_Build_Structure.");
goto End;
}
rib = (RIB_STRUCT *) (zz->LB.Data_Structure);
treept = rib->Tree_Ptr;
end_time = Zoltan_Time(zz->Timer);
lb_time[0] = end_time - start_time;
start_time = end_time;
/* local copies of calling parameters */
dottop = dotnum = pdotnum;
/* initialize timers and counters */
counters[0] = 0;
counters[1] = 0;
counters[2] = 0;
counters[3] = dotnum;
counters[4] = dotmax;
counters[5] = 0;
counters[6] = 0;
/* Ensure there are dots */
MPI_Allreduce(&dotnum, &i, 1, MPI_INT, MPI_MAX, zz->Communicator);
if (i == 0){
if (proc == 0){
ZOLTAN_PRINT_WARN(proc, yo, "RIB partitioning called with no objects");
}
timestart = timestop = 0;
goto EndReporting;
}
/* If using RIB for matching, need to generate candidate lists.
* Candidate lists include input GIDs, LIDs as provided by the application.
* We need to capture that input here before we move any dots!
* We return it in the import lists.
* Candidates will be computed after partitioning and returned in the
* export lists.
*/
if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) {
ierr = Zoltan_RB_Candidates_Copy_Input(zz, dotnum,
rib->Global_IDs, rib->Local_IDs,
&rib->Dots,
num_import,
import_global_ids, import_local_ids,
import_procs, import_to_part);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc,yo,
"Error returned from Zoltan_RB_Return_Arguments.");
goto End;
}
}
/* create mark and list arrays for dots */
allocflag = 0;
if (dotmax > 0) {
if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))
|| !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double)))
|| !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) {
ierr = ZOLTAN_MEMERR;
goto End;
}
}
else {
dotmark = NULL;
value = NULL;
dotlist = NULL;
}
/* set dot weights = 1.0 if user didn't and determine total weight */
dotpt = &rib->Dots;
if (dotpt->nWeights == 0) {
weightlo[0] = (double) dotnum;
dotpt->uniformWeight = 1.0;
wgtdim = 1;
}
else {
double *wgt;
for (j=0; j<dotpt->nWeights; j++){
weightlo[j] = 0.0;
wgt = dotpt->Weight + j;
for (i=0; i < dotnum; i++){
weightlo[j] += *wgt;
wgt += dotpt->nWeights;
}
}
wgtdim = dotpt->nWeights;
}
MPI_Allreduce(weightlo, weight, wgtdim, MPI_DOUBLE, MPI_SUM, zz->Communicator);
if (check_geom) {
ierr = Zoltan_RB_check_geom_input(zz, dotpt, dotnum);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_check_geom_input");
goto End;
}
}
/* create local communicator for use in recursion */
if (zz->Tflops_Special)
local_comm = zz->Communicator;
else {
MPI_Comm_dup(zz->Communicator,&local_comm);
free_comm = TRUE;
}
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
time2 = Zoltan_Time(zz->Timer);
timers[0] = time2 - time1;
}
/* recursively halve until just one part or proc in set */
old_nprocs = num_procs = nprocs;
old_nparts = num_parts;
partlower = 0;
root = 0;
old_set = 1;
ierr = Zoltan_LB_Proc_To_Part(zz, proc, &np, &fp);
for (i = fp; i < (fp + np); i++) {
treept[i].parent = 0;
treept[i].left_leaf = 0;
}
if (zz->Tflops_Special) {
proclower = 0;
tfs[0] = nprocs;
tfs[1] = num_parts;
}
while ((num_parts > 1 && num_procs > 1) ||
(zz->Tflops_Special && tfs[0] > 1 && tfs[1] > 1)) {
ierr = Zoltan_Divide_Machine(zz, zz->Obj_Weight_Dim, part_sizes,
proc, local_comm, &set,
&proclower, &procmid, &num_procs,
&partlower, &partmid, &num_parts,
fractionlo);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo, "Error in Zoltan_Divide_Machine.");
goto End;
}
/* tfs[0] is max number of processors in all sets over all processors -
* tfs[1] is max number of parts in all sets over all processors -
* force all processors to go through all levels of parallel rib */
if (zz->Tflops_Special) {
tmp_tfs[0] = num_procs;
tmp_tfs[1] = num_parts;
MPI_Allreduce(tmp_tfs, tfs, 2, MPI_INT, MPI_MAX, local_comm);
}
/* create mark array and active list for dots */
if (allocflag) {
allocflag = 0;
ZOLTAN_FREE(&dotmark);
ZOLTAN_FREE(&value);
ZOLTAN_FREE(&dotlist);
if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))
|| !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double)))
|| !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) {
ierr = ZOLTAN_MEMERR;
goto End;
}
}
dotpt = &rib->Dots;
if (old_nparts > 1 && old_nprocs > 1) { /* test added for Tflops_Special;
compute values only if looping
to decompose, not if looping to
keep Tflops_Special happy. */
ierr = compute_rib_direction(zz, zz->Tflops_Special, rib->Num_Geom,
&valuelo, &valuehi, dotpt, NULL, dotnum,
wgtflag, cm, evec, value,
local_comm, proc, old_nprocs, proclower);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from compute_rib_direction");
goto End;
}
}
else { /* For Tflops_Special: initialize value when looping only
for Tflops_Special */
for (i = 0; i < dotmax; i++)
value[i] = 0.0;
valuelo = valuehi = 0.0;
}
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME))
time2 = Zoltan_Time(zz->Timer);
if (!Zoltan_RB_find_median(
zz->Tflops_Special, value, dotpt->Weight, dotpt->uniformWeight, dotmark, dotnum, proc,
fractionlo, local_comm, &valuehalf, first_guess,
nprocs, old_nprocs, proclower, old_nparts,
wgtflag, valuelo, valuehi, weight[0], weightlo,
weighthi, dotlist, rectilinear_blocks, average_cuts)) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_find_median.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (set) /* set weight for current part */
for (j=0; j<wgtdim; j++) weight[j] = weighthi[j];
else
for (j=0; j<wgtdim; j++) weight[j] = weightlo[j];
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME))
time3 = Zoltan_Time(zz->Timer);
/* store cut info in tree only if proc "owns" partmid */
/* test of partmid > 0 prevents treept[0] being set when this cut is
only removing low-numbered processors (proclower to procmid-1) that
have no parts in them from the processors remaining to
be partitioned. */
if (partmid > 0 && partmid == fp) {
treept[partmid].cm[0] = cm[0];
treept[partmid].cm[1] = cm[1];
treept[partmid].cm[2] = cm[2];
treept[partmid].ev[0] = evec[0];
treept[partmid].ev[1] = evec[1];
treept[partmid].ev[2] = evec[2];
treept[partmid].cut = valuehalf;
treept[partmid].parent = old_set ? -(root+1) : root+1;
/* The following two will get overwritten when the information
is assembled if this is not a terminal cut */
treept[partmid].left_leaf = -partlower;
treept[partmid].right_leaf = -partmid;
}
if (old_nprocs > 1 && partmid > 0 && partmid != partlower + old_nparts) {
/* old_nprocs > 1 test: Don't reset these values if proc is in loop only
* because of other procs for Tflops_Special.
* partmid > 0 test: Don't reset these values if low-numbered processors
* (proclower to procmid-1) have zero parts and this cut is removing
* them from the processors remaining to be partitioned.
* partmid != partlower + old_nparts test: Don't reset these values if
* cut is removing high-numbered processors with zero parts from
* the processors remaining to be partitioned.
*/
old_set = set;
root = partmid;
}
ierr = Zoltan_RB_Send_Outgoing(zz, &(rib->Global_IDs), &(rib->Local_IDs),
&(rib->Dots), &dotmark,
&dottop, &dotnum, &dotmax,
set, &allocflag, overalloc,
stats, counters, use_ids,
local_comm, proclower,
old_nprocs, partlower, partmid);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_Send_Outgoing.");
goto End;
}
/* create new communicators */
if (zz->Tflops_Special) {
if (set) {
proclower = procmid;
partlower = partmid;
}
old_nprocs = num_procs;
old_nparts = num_parts;
}
else {
if (set) partlower = partmid;
MPI_Comm_split(local_comm,set,proc,&tmp_comm);
MPI_Comm_free(&local_comm);
local_comm = tmp_comm;
old_nprocs = num_procs;
old_nparts = num_parts;
}
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
time4 = Zoltan_Time(zz->Timer);
timers[1] += time2 - time1;
timers[2] += time3 - time2;
timers[3] += time4 - time3;
}
}
/* have recursed all the way to a single processor sub-domain */
/* Send dots to correct processors for their parts. This is needed
most notably when a processor has zero parts on it, but still has
some dots after the parallel partitioning. */
ierr = Zoltan_RB_Send_To_Part(zz, &(rib->Global_IDs), &(rib->Local_IDs),
&(rib->Dots), &dotmark, &dottop,
&dotnum, &dotmax, &allocflag, overalloc,
stats, counters, use_ids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_RB_Send_To_Part");
goto End;
}
/* All dots are now on the processors they will end up on; now generate
* more parts if needed. */
if (num_parts > 1) {
if (dotpt->nWeights)
wgts = (double *) ZOLTAN_MALLOC(dotpt->nWeights * dotnum * sizeof(double));
dindx = (int *) ZOLTAN_MALLOC(dotnum * 2 * sizeof(int));
tmpdindx = dindx + dotnum;
if (allocflag) {
ZOLTAN_FREE(&dotmark);
ZOLTAN_FREE(&value);
ZOLTAN_FREE(&dotlist);
if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))
|| !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double)))
|| !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
for (i = 0; i < dotnum; i++)
dindx[i] = i;
ierr = serial_rib(zz, &rib->Dots, dotmark, dotlist, old_set, root,
rib->Num_Geom, weight[0], dotnum, num_parts,
&(dindx[0]), &(tmpdindx[0]), partlower,
proc, wgtflag, stats, gen_tree,
rectilinear_blocks, average_cuts,
treept, value, wgts, part_sizes);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from serial_rib");
goto End;
}
ZOLTAN_FREE(&wgts);
}
end_time = Zoltan_Time(zz->Timer);
lb_time[1] = end_time - start_time;
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
MPI_Barrier(zz->Communicator);
timestop = time1 = Zoltan_Time(zz->Timer);
}
/* error checking and statistics */
if (check_geom) {
ierr = Zoltan_RB_check_geom_output(zz, &rib->Dots, part_sizes, np, fp,
dotnum, pdotnum, NULL);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_check_geom_output");
goto End;
}
}
EndReporting:
/* update calling routine parameters */
start_time = Zoltan_Time(zz->Timer);
pdotnum = dotnum;
/* Perform remapping (if requested) */
if (zz->LB.Remap_Flag) {
ierr = Zoltan_RB_Remap(zz, &(rib->Global_IDs), &(rib->Local_IDs),
&(rib->Dots), &dotnum, &dotmax,
&allocflag, overalloc, stats, counters, use_ids);
/* Note: dottop is no longer valid after remapping. Remapping might
destroy the nice local-followed-by-non-local ordering of the
dots array. Do not use dottop after remapping. */
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_RB_Remap.");
goto End;
}
}
/* build return arguments */
if (zz->LB.Return_Lists != ZOLTAN_LB_NO_LISTS &&
zz->LB.Return_Lists != ZOLTAN_LB_CANDIDATE_LISTS) {
/* zz->LB.Return_Lists is true ==> use_ids is true */
ierr = Zoltan_RB_Return_Arguments(zz, rib->Global_IDs, rib->Local_IDs,
&rib->Dots, num_import,
import_global_ids, import_local_ids,
import_procs, import_to_part,
dotnum);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_Return_Arguments.");
goto End;
}
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) {
/* Select a candidate for each part and return it in the export_GIDs. */
ierr = Zoltan_RB_Candidates_Output(zz, dotnum, dindx,
rib->Global_IDs, rib->Local_IDs,
&rib->Dots,
*num_import, *import_global_ids,
num_export, export_global_ids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc,yo,
"Error returned from Zoltan_RB_Return_Candidates.");
goto End;
}
}
ZOLTAN_FREE(&dindx);
if (gen_tree) {
int *displ, *recvcount;
int sendcount;
struct rib_tree *treetmp = NULL; /* temporary tree of cuts; used to keep
valgrind from reporting overlapped
memory in MPI_Allgatherv */
treetmp = (struct rib_tree *)
ZOLTAN_MALLOC(zz->LB.Num_Global_Parts* sizeof(struct rib_tree));
displ = (int *) ZOLTAN_MALLOC(2 * zz->Num_Proc * sizeof(int));
if (!displ || !treetmp) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
recvcount = displ + zz->Num_Proc;
ierr = Zoltan_RB_Tree_Gatherv(zz, sizeof(struct rib_tree), &sendcount,
recvcount, displ);
/*
* Create copy of treept so that MPI_Allgatherv doesn't use same
* memory for sending and receiving; removes valgrind warning.
*/
for (i = 0; i < zz->LB.Num_Global_Parts; i++)
treetmp[i] = treept[i];
MPI_Allgatherv(&treetmp[fp], sendcount, MPI_BYTE, treept, recvcount, displ,
MPI_BYTE, zz->Communicator);
for (i = 1; i < zz->LB.Num_Global_Parts; i++){
if (treept[i].parent > 0)
treept[treept[i].parent - 1].left_leaf = i;
else if (treept[i].parent < 0)
treept[-treept[i].parent - 1].right_leaf = i;
}
ZOLTAN_FREE(&displ);
ZOLTAN_FREE(&treetmp);
}
else {
treept[0].right_leaf = -1;
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
print_rib_tree(zz, np, fp, &(treept[fp]));
end_time = Zoltan_Time(zz->Timer);
lb_time[0] += (end_time - start_time);
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME))
Zoltan_RB_stats(zz, timestop-timestart, &rib->Dots, dotnum,
part_sizes, timers, counters, stats, NULL, NULL, FALSE);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) {
if (zz->Proc == zz->Debug_Proc)
printf("ZOLTAN RIB Times: \n");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, lb_time[0],
"ZOLTAN Build: ");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, lb_time[1],
"ZOLTAN RIB: ");
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
/* zz->Debug_Level >= ZOLTAN_DEBUG_ALL ==> use_ids is true */
Zoltan_RB_Print_All(zz, rib->Global_IDs, &rib->Dots,
dotnum, *num_import,
*import_global_ids, *import_procs);
}
End:
/* Free memory allocated by the algorithm. */
if (free_comm) MPI_Comm_free(&local_comm);
ZOLTAN_FREE(&dotmark);
ZOLTAN_FREE(&value);
ZOLTAN_FREE(&dotlist);
if (!gen_tree && /* don't need parts */
rib && (rib->Tran.Target_Dim < 0)) { /* don't need transformation */
/* Free all memory used. */
Zoltan_RIB_Free_Structure(zz);
}
else if (rib != NULL) {
/* Free only Dots and IDs; keep other structures. */
ZOLTAN_FREE(&(rib->Global_IDs));
ZOLTAN_FREE(&(rib->Local_IDs));
Zoltan_Free_And_Reset_Dot_Structure(&(rib->Dots));
}
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
/* Performs a permutation of the matrix, perm_y A perm_y^t.
* TODO: at this time we only do symmetric permutations (don't know xGNO !).
*/
int
Zoltan_Matrix_Permute(ZZ* zz, Zoltan_matrix *m, int* perm_y)
{
static char *yo = "Zoltan_Matrix_Permute";
int ierr = ZOLTAN_OK;
int *pinGNO = NULL;
ZOLTAN_ID_PTR yGID=NULL;
float *ywgt=NULL;
struct Zoltan_DD_Struct *dd;
ZOLTAN_TRACE_ENTER(zz, yo);
/* First apply y permutation */
if (m->completed) { /* We directly know the good arrays */
yGID = m->yGID;
ywgt = m->ywgt;
if (m->ddY == NULL || m->ddY != m->ddX) { /* We have to create again the DD */
/* We have to define ddY : yGNO, yGID, ywgt */
ierr = Zoltan_DD_Create (&m->ddY, zz->Communicator, 1, zz->Num_GID,
m->ywgtdim*sizeof(float)/sizeof(int), m->globalY/zz->Num_Proc, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(m->ddY, m->globalY/zz->Num_Proc);
}
}
else { /* We have to get these fields */
/* Update data directories */
yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY);
ywgt = (float*) ZOLTAN_MALLOC(m->nY * sizeof(float) * m->ywgtdim);
if (m->nY && (yGID == NULL || (m->ywgtdim && ywgt == NULL)))
MEMORY_ERROR;
/* Get Informations about Y */
Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL,
m->nY, NULL);
}
memcpy (m->yGNO, perm_y, m->nY*sizeof(int));
/* Get Informations about Y */
Zoltan_DD_Update (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL,
m->nY);
ZOLTAN_FREE (&yGID);
ZOLTAN_FREE (&ywgt);
/* We have to define dd : old_yGNO, new_yGNO */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, 1, 1, 0, m->globalY/zz->Num_Proc, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, m->globalY/zz->Num_Proc);
Zoltan_DD_Update (dd, (ZOLTAN_ID_PTR)m->yGNO, (ZOLTAN_ID_PTR)perm_y, NULL, NULL, m->nY);
pinGNO = (int*)ZOLTAN_MALLOC(m->nPins*sizeof(int));
if (m->nPins && pinGNO == NULL)
MEMORY_ERROR;
Zoltan_DD_Find (dd, (ZOLTAN_ID_PTR)m->pinGNO, (ZOLTAN_ID_PTR)pinGNO, NULL, NULL,
m->nY, NULL);
ZOLTAN_FREE(&m->pinGNO);
m->pinGNO = pinGNO;
pinGNO = NULL;
End:
ZOLTAN_FREE (&pinGNO);
ZOLTAN_FREE (&yGID);
ZOLTAN_FREE (&ywgt);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
static int Zoltan_Reftree_Sum_Weights(ZZ *zz)
{
/*
* Function to sum the weights in the refinement tree. On input the
* refinement tree should be valid and have weight set. On output the
* values in summed_weight at each node is the sum of the weights in the
* subtree with that node as root.
* This function also sets assigned_to_me for interior nodes to be
* 1 if the entire subtree is assigned to this processor
* 0 if none of the subtree is assigned to this processor
* -1 if some of the subtree is assigned to this processor
*/
char *yo = "Zoltan_Reftree_Sum_Weights";
ZOLTAN_REFTREE *root; /* Root of the refinement tree */
int wdim; /* Dimension of the weight array */
int i,j; /* loop counters */
int count; /* counter */
ZOLTAN_ID_PTR leaf_list = NULL;
/* leaves for which some proc requests weight */
ZOLTAN_ID_PTR all_leaflist = NULL;
/* leaf_list from all processors */
int reqsize; /* length of leaf_list */
int *reqsize_all; /* reqsize from all processors */
int sum_reqsize; /* sum of all reqsize */
int *displs; /* running sum of all reqsize */
int my_start; /* position in leaf_list of this proc's list */
int nproc; /* number of processors */
ZOLTAN_REFTREE *node; /* a node in the refinement tree */
struct Zoltan_Reftree_hash_node **hashtab; /* hash table */
int hashsize; /* dimension of hash table */
float *send_float; /* sending message of floats */
float *req_weights; /* the requested weights */
int num_gid_entries = zz->Num_GID; /* Number of array entries in a global ID */
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* set the root and hash table
*/
root = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->reftree_root;
if (root == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Refinement tree not defined.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_FATAL);
}
hashtab = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table;
hashsize = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table_size;
/*
* Determine the dimension of the weight array
*/
if (zz->Obj_Weight_Dim == 0) {
wdim = 1;
} else {
wdim = zz->Obj_Weight_Dim;
}
/*
* In the first pass, sum the weights of the nodes that are assigned to
* this processor, and count the leaves that are not.
*/
count = 0;
for (i=0; i<root->num_child; i++) {
Zoltan_Reftree_Sum_My_Weights(zz,&(root->children[i]),&count,wdim);
}
root->assigned_to_me = -1;
/*
* Make a list of the leaves that are not assigned to this processor
*/
if (count == 0)
leaf_list = ZOLTAN_MALLOC_GID(zz);
else
leaf_list = ZOLTAN_MALLOC_GID_ARRAY(zz, count);
if (leaf_list == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
count = 0;
Zoltan_Reftree_List_Other_Leaves(zz, root,leaf_list,&count);
/*
* Get the unknown leaf weights from other processors.
*/
nproc = zz->Num_Proc;
reqsize = count;
/*
* Build a list of all processor's request list by concatinating them in
* the order of the processor ranks
*/
/*
* Determine the request size of all processors
*/
reqsize_all = (int *)ZOLTAN_MALLOC(nproc*sizeof(int));
displs = (int *)ZOLTAN_MALLOC(nproc*sizeof(int));
if (reqsize_all == NULL || displs == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 3, &displs,
&reqsize_all,
&leaf_list);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
MPI_Allgather((void *)&reqsize,1,MPI_INT,(void *)reqsize_all,1,MPI_INT,
zz->Communicator);
displs[0] = 0;
for (i=1; i<nproc; i++) displs[i] = displs[i-1]+reqsize_all[i-1];
sum_reqsize = displs[nproc-1] + reqsize_all[nproc-1];
my_start = displs[zz->Proc];
/*
* If sum_reqsize is 0, nothing needs to be communciated
*/
if (sum_reqsize == 0) {
Zoltan_Multifree(__FILE__, __LINE__, 3, &displs,
&reqsize_all,
&leaf_list);
}
else {
/*
* Gather the request list from all processors
*/
all_leaflist = ZOLTAN_MALLOC_GID_ARRAY(zz, sum_reqsize);
if (all_leaflist == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 4, &all_leaflist,
&displs,
&reqsize_all,
&leaf_list);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
/* KDDKDD Changed MPI_BYTE to ZOLTAN_ID_MPI_TYPE */
/* Account for number of array entries in an ID. */
for (i=0; i<nproc; i++) {
reqsize_all[i] = reqsize_all[i]*num_gid_entries;
displs[i] = displs[i]*num_gid_entries;
}
MPI_Allgatherv((void *)leaf_list,reqsize*num_gid_entries,ZOLTAN_ID_MPI_TYPE,
(void *)all_leaflist,reqsize_all,displs,ZOLTAN_ID_MPI_TYPE,
zz->Communicator);
ZOLTAN_FREE(&displs);
ZOLTAN_FREE(&leaf_list);
for (i=0; i<nproc; i++) reqsize_all[i] = reqsize_all[i]/num_gid_entries;
/*
* Create a list with the partial sums this processor has
*/
send_float = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*sum_reqsize);
if (send_float == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 3, &send_float,
&all_leaflist,
&reqsize_all);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
for (i=0; i<sum_reqsize; i++) {
node = Zoltan_Reftree_hash_lookup(zz, hashtab,
&(all_leaflist[i*num_gid_entries]),
hashsize);
if (node == NULL)
for (j=0; j<wdim; j++) send_float[i*wdim+j] = 0.0;
else
for (j=0; j<wdim; j++) send_float[i*wdim+j] = node->my_sum_weight[j];
}
/*
* Sum the weights over all the processors
*/
if (reqsize == 0)
req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim);
else
req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*reqsize);
if (req_weights == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 4, &req_weights,
&send_float,
&all_leaflist,
&reqsize_all);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
MPI_Reduce_scatter((void *)send_float, (void *)req_weights, reqsize_all,
MPI_FLOAT, MPI_SUM, zz->Communicator);
ZOLTAN_FREE(&send_float);
ZOLTAN_FREE(&reqsize_all);
/*
* Set the weights this processor requested
*/
for (i=0; i<count; i++) {
node = Zoltan_Reftree_hash_lookup(zz, hashtab,
&(all_leaflist[(i+my_start)*num_gid_entries]),
hashsize);
for (j=0; j<wdim; j++) node->summed_weight[j] = req_weights[i*wdim+j];
}
ZOLTAN_FREE(&req_weights);
ZOLTAN_FREE(&all_leaflist);
}
/*
* All the leaves now have summed_weight set.
* Sum the weights throughout the tree.
*/
Zoltan_Reftree_Sum_All_Weights(zz,root,wdim);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
static int _Zoltan_Get_Obj_List(
ZZ *zz,
int *num_obj,
ZOLTAN_ID_PTR *global_ids,
ZOLTAN_ID_PTR *local_ids,
int wdim,
float **objwgts,
int **parts,
int special_malloc
)
{
char *yo = "_Zoltan_Get_Obj_List";
int i, n;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
int alloced_gids = 0, alloced_lids = 0;
int gid_off, lid_off;
ZOLTAN_ID_PTR lid, next_lid; /* Temporary pointers to local IDs; used to pass
NULL to query functions when
NUM_LID_ENTRIES == 0. */
float *next_objwgt; /* Temporarry pointer to an object weight; used
to pass NULL to query functions when wdim=0 */
int ierr = ZOLTAN_OK;
ZOLTAN_TRACE_ENTER(zz, yo);
*num_obj = 0;
*objwgts = NULL;
if (zz->Get_Num_Obj != NULL) {
*num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
goto End;
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (*num_obj > 0) {
/*
* Test global_ids and local_ids for NULL.
* Should be NULL for doing partitioning.
* Should not be NULL if doing ordering.
*/
if (special_malloc){
if (*global_ids == NULL) {
Zoltan_Special_Malloc(zz, (void **)global_ids,
*num_obj, ZOLTAN_SPECIAL_MALLOC_GID);
alloced_gids = 1;
}
if (*local_ids == NULL) {
Zoltan_Special_Malloc(zz, (void **)local_ids,
*num_obj, ZOLTAN_SPECIAL_MALLOC_LID);
alloced_lids = 1;
}
Zoltan_Special_Malloc(zz, (void **)parts,
*num_obj, ZOLTAN_SPECIAL_MALLOC_INT);
}
else{
if (*global_ids == NULL) {
*global_ids = ZOLTAN_MALLOC_GID_ARRAY(zz, *num_obj);
alloced_gids = 1;
}
if (*local_ids == NULL) {
*local_ids = ZOLTAN_MALLOC_LID_ARRAY(zz, *num_obj);
alloced_lids = 1;
}
*parts = (int *) ZOLTAN_MALLOC(*num_obj * sizeof(int));
}
if (wdim > 0)
*objwgts = (float*) ZOLTAN_MALLOC (sizeof(float) * *num_obj * wdim);
if ((*global_ids == NULL) || (num_lid_entries > 0 && *local_ids == NULL) ||
(*parts == NULL) ||
(wdim > 0 && *objwgts == NULL)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory Error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Get_Obj_List != NULL){
/* Get object list directly */
zz->Get_Obj_List(zz->Get_Obj_List_Data,
num_gid_entries, num_lid_entries,
*global_ids, *local_ids,
wdim, *objwgts, &ierr);
}
else if ((zz->Get_First_Obj != NULL) && (zz->Get_Next_Obj != NULL)){
/* Use iterator functions to loop through object list */
if (zz->Get_First_Obj(zz->Get_First_Obj_Data,
num_gid_entries, num_lid_entries,
*global_ids, *local_ids,
wdim, *objwgts, &ierr)){
n = *num_obj;
i = 0;
while (!ierr && (i<n-1)){
gid_off = i * num_gid_entries;
lid_off = i * num_lid_entries;
lid = (num_lid_entries ? &((*local_ids)[lid_off]) : NULL);
next_lid = (num_lid_entries ? &((*local_ids)[lid_off+num_lid_entries])
: NULL);
next_objwgt = (wdim ? (*objwgts) + (i+1)*wdim : NULL);
zz->Get_Next_Obj(zz->Get_Next_Obj_Data,
num_gid_entries, num_lid_entries,
&((*global_ids)[gid_off]), lid,
&((*global_ids)[gid_off+num_gid_entries]),
next_lid,
wdim, next_objwgt, &ierr);
i++;
}
}
}
else { /* No way to get objects */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_OBJ_LIST_FN or "
"ZOLTAN_FIRST_OBJ_FN/ZOLTAN_NEXT_OBJ_FN.");
ierr = ZOLTAN_FATAL;
goto End;
}
/* Get partition information for objects. */
/* Call user-callback if provided; otherwise, all parts == zz->Proc */
if (zz->Get_Part == NULL && zz->Get_Part_Multi == NULL) {
for (i = 0; i < *num_obj; i++)
(*parts)[i] = zz->Proc;
}
else if (zz->Get_Part_Multi) {
zz->Get_Part_Multi(zz->Get_Part_Multi_Data,
num_gid_entries, num_lid_entries, *num_obj,
*global_ids, *local_ids, *parts, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_PART_MULTI_FN");
goto End;
}
}
else {
for (i = 0; i < *num_obj; i++) {
lid = (num_lid_entries ? &((*local_ids)[i*num_lid_entries]) : NULL);
(*parts)[i] = zz->Get_Part(zz->Get_Part_Data,
num_gid_entries, num_lid_entries,
&((*global_ids)[i*num_gid_entries]), lid, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_PART_FN");
goto End;
}
}
}
}
End:
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no lists returned.");
if (special_malloc){
if (alloced_gids)
Zoltan_Special_Free(zz, (void **)global_ids, ZOLTAN_SPECIAL_MALLOC_GID);
if (alloced_lids)
Zoltan_Special_Free(zz, (void **)local_ids, ZOLTAN_SPECIAL_MALLOC_LID);
Zoltan_Special_Free(zz, (void **)parts, ZOLTAN_SPECIAL_MALLOC_INT);
}
else{
if (alloced_gids) ZOLTAN_FREE(global_ids);
if (alloced_lids) ZOLTAN_FREE(local_ids);
ZOLTAN_FREE(parts);
}
ZOLTAN_FREE(objwgts);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
int Zoltan_Reftree_Part(
ZZ *zz, /* The Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_import, /* Not computed, set to -1 */
ZOLTAN_ID_PTR *import_global_ids, /* Not computed */
ZOLTAN_ID_PTR *import_local_ids, /* Not computed */
int **import_procs, /* Not computed */
int **import_to_part, /* Not computed */
int *num_export, /* Number of objects to be exported */
ZOLTAN_ID_PTR *export_global_ids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *export_local_ids, /* local ids of objects to be exported */
int **export_procs, /* list of processors to export to */
int **export_to_partition /* list of partitions to export to */
)
{
char *yo = "Zoltan_Reftree_Part";
int ierr; /* error code returned by called routines */
int final_ierr; /* error code returned by this routine */
double time0 = 0, time1= 0, time2 = 0, time3 =0, time4 =0;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Initializations in case of early exit. */
*num_export = -1;
*num_import = -1;
final_ierr = ZOLTAN_OK;
/*
* initialize the tree (first call only)
*/
if (zz->LB.Data_Structure == NULL) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time0 = Zoltan_Time(zz->Timer);
ierr = Zoltan_Reftree_Init(zz);
if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned by Zoltan_Reftree_Init.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time1 = Zoltan_Time(zz->Timer);
} else {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) {
time1 = Zoltan_Time(zz->Timer);
time0 = time1 + 1.0;
}
}
/*
* build the refinement tree
*/
ierr = Zoltan_Reftree_Build(zz);
if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned by Zoltan_Reftree_Build.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time2 = Zoltan_Time(zz->Timer);
/*
* sum the weights in the tree
*/
ierr = Zoltan_Reftree_Sum_Weights(zz);
if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned by Zoltan_Reftree_Sum_Weights.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time3 = Zoltan_Time(zz->Timer);
/*
* determine the new partition
*/
ierr = Zoltan_Reftree_Partition(zz, part_sizes, num_export, export_global_ids,
export_local_ids, export_to_partition, export_procs);
if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned by Zoltan_Reftree_Partition.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time4 = Zoltan_Time(zz->Timer);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) {
if (time0 <= time1) {
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time1-time0,
"REFTREE Time to initialize :");
}
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time2-time1,
"REFTREE Time to build tree :");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time3-time2,
"REFTREE Time to sum weights:");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time4-time3,
"REFTREE Time to partition :");
}
ZOLTAN_TRACE_EXIT(zz, yo);
return(final_ierr);
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int *num_gid_entries, /* # of entries for a global id */
int *num_lid_entries, /* # of entries for a local id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm, /* inverse permutation of rank */
ZOS *order_info /* Method-specific ordering info. Currently not used. */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* iperm, inverse permutation of rank
* order_info, a Zoltan Ordering Struct with additional info.
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
int *vtxdist;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
strncpy(opt.order_type, "GLOBAL", MAX_PARAM_STRING_LEN);
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
if (opt.use_order_info == 0) order_info = NULL;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && lids && rank && iperm)){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL){
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
*/
ierr = (*Order_fn)(zz, num_obj, gids, lids, rank, iperm, &opt, order_info);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN){
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr){
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK)){
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, iperm, rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM)){
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, rank, iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i, nobjs;
nobjs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &i);
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < nobjs; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(*num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
printf("ZOLTAN: inverse permutation on Proc %d\n", zz->Proc);
for (i = 0; i < nobjs; i++) {
printf("iperm[%3d] = %3d\n", i, iperm[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
