/* Function that removes locale duplicated nnz */
int
Zoltan_Matrix_Remove_Duplicates(ZZ *zz, Zoltan_matrix inmat, Zoltan_matrix *outmat)
{
static char *yo = "Zoltan_Matrix_Remove_Duplicates";
int ierr = ZOLTAN_OK;
Zoltan_Arc *arcs = NULL;
float *pinwgt = NULL;
int freeflag = 0;
int size;
int i, j, cnt;
ZOLTAN_TRACE_ENTER(zz, yo);
if (inmat.opts.symmetrize == 0) /* No symmetrization, we hope no duplicates ...*/
goto End;
size = inmat.nPins + inmat.nY; /* We add fake arcs for non connected vertices */
arcs = (Zoltan_Arc*) ZOLTAN_MALLOC(size*sizeof(Zoltan_Arc));
if (inmat.nPins && arcs == NULL) MEMORY_ERROR;
for (i = 0, cnt=0 ; i < inmat.nY; i++) {
/* Fake arc in order to be sure to keep this vertex */
arcs[cnt].yGNO = inmat.yGNO[i];
arcs[cnt].pinGNO = -1;
arcs[cnt].offset = -1;
cnt++;
for (j = inmat.ystart[i]; j < inmat.yend[i]; j++) {
arcs[cnt].yGNO = inmat.yGNO[i];
arcs[cnt].pinGNO = inmat.pinGNO[j];
arcs[cnt].offset = j;
cnt ++;
}
}
pinwgt = inmat.pinwgt;
if (pinwgt == outmat->pinwgt) {
freeflag = 1;
outmat->pinwgt = (float*) ZOLTAN_MALLOC(inmat.pinwgtdim*inmat.nPins*sizeof(float));
if (inmat.pinwgtdim && inmat.nPins && outmat->pinwgt == NULL) MEMORY_ERROR;
}
ierr = Zoltan_Matrix_Remove_DupArcs(zz, size, arcs, pinwgt,outmat);
if (freeflag)
ZOLTAN_FREE(&pinwgt);
End:
ZOLTAN_FREE(&arcs);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/* if !copy, inmat is not usable after this call */
int
Zoltan_Matrix2d_Distribute (ZZ* zz, Zoltan_matrix inmat, /* Cannot be const as we can share it inside outmat */
Zoltan_matrix_2d *outmat, int copy)
{
static char *yo = "Zoltan_Matrix2d_Distribute";
int ierr = ZOLTAN_OK;
int nProc_x, nProc_y;
int myProc_x, myProc_y;
int i, j, cnt;
int *proclist = NULL;
Zoltan_Arc *nonzeros= NULL, *sendbuf= NULL;
ZOLTAN_GNO_TYPE *perm_y = NULL;
float *wgtarray = NULL;
float *tmpwgtarray = NULL;
int msg_tag = 1021982;
ZOLTAN_COMM_OBJ *plan;
MPI_Comm communicator = MPI_COMM_NULL;
int nProc;
ZOLTAN_GNO_TYPE *yGNO = NULL;
ZOLTAN_GNO_TYPE *pinGNO = NULL;
ZOLTAN_GNO_TYPE tmp_gno;
void *partdata = NULL;
MPI_Datatype zoltan_gno_mpi_type;
ZOLTAN_TRACE_ENTER(zz, yo);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
memcpy(&outmat->mtx, &inmat, sizeof(Zoltan_matrix));
if(copy) {
/* TODO: We need to copy the arrays also */
Zoltan_Matrix_Reset (&outmat->mtx);
/* Copy also directories */
outmat->mtx.ddX = Zoltan_DD_Copy (inmat.ddX);
if (inmat.ddY == inmat.ddX)
outmat->mtx.ddY = outmat->mtx.ddX;
else
outmat->mtx.ddY = Zoltan_DD_Copy (inmat.ddY);
}
communicator = outmat->comm->Communicator;
nProc = outmat->comm->nProc;
nProc_x = outmat->comm->nProc_x;
nProc_y = outmat->comm->nProc_y;
myProc_x = outmat->comm->myProc_x;
myProc_y = outmat->comm->myProc_y;
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_Duplicates");
ierr = Zoltan_Matrix_Remove_Duplicates(zz, outmat->mtx, &outmat->mtx);
/* KDDKDDKDD FIX INDENTATION OF THIS BLOCK */
if (inmat.opts.speed != MATRIX_NO_REDIST) {
if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) {
/* I need to know the original distribution */
if (outmat->mtx.ddX != outmat->mtx.ddY) { /* No initial distribution */
outmat->hashDistFct = (distFnct *)&Zoltan_Distribute_Linear;
}
else {
int *cmember = NULL;
cmember = (int*)ZOLTAN_MALLOC(outmat->mtx.nY*sizeof(int));
if (outmat->mtx.nY > 0 && cmember == NULL) MEMORY_ERROR;
Zoltan_DD_Find (outmat->mtx.ddY, (ZOLTAN_ID_PTR)outmat->mtx.yGNO, NULL, (char *)cmember, NULL,
outmat->mtx.nY, NULL);
KDDKDDKDD(zz->Proc, " Zoltan_Distribute_Partition_Register");
partdata = Zoltan_Distribute_Partition_Register(zz, outmat->mtx.nY, outmat->mtx.yGNO,
cmember, zz->Num_Proc, zz->Num_Proc);
ZOLTAN_FREE(&cmember);
Zoltan_Distribute_Set(outmat, (distFnct *)&Zoltan_Distribute_Origin, partdata);
}
}
/*
* Build comm plan for sending non-zeros to their target processors in
* 2D data distribution.
*/
/* TRICK: create fake arc (edgeno, -1) for empty Y. Upper bound for size might be nPins + nY */
proclist = (int *)ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY) *sizeof(int));
sendbuf = (Zoltan_Arc*) ZOLTAN_MALLOC((outmat->mtx.nPins +outmat->mtx.nY)* sizeof(Zoltan_Arc));
if ((outmat->mtx.nPins + outmat->mtx.nY >0) && (proclist == NULL || sendbuf == NULL)) MEMORY_ERROR;
wgtarray = (float*) ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY)*outmat->mtx.pinwgtdim*sizeof(float));
if (outmat->mtx.nPins*outmat->mtx.pinwgtdim && !wgtarray) MEMORY_ERROR;
yGNO = outmat->mtx.yGNO;
pinGNO = outmat->mtx.pinGNO;
KDDKDDKDD(zz->Proc, " CommPlan Hash");
cnt = 0;
for (i = 0; i < outmat->mtx.nY; i++) {
ZOLTAN_GNO_TYPE edge_gno=-1;
/* processor row for the edge */
edge_gno = yGNO[i];
for (j = outmat->mtx.ystart[i]; j < outmat->mtx.yend[i]; j++) {
ZOLTAN_GNO_TYPE vtx_gno=-1;
/* processor column for the vertex */
vtx_gno = pinGNO[j];
proclist[cnt] = (*outmat->hashDistFct)(edge_gno, vtx_gno, outmat->hashDistData,
&sendbuf[cnt].part_y);
if (proclist[cnt] < 0) /* Discard this nnz */
continue;
sendbuf[cnt].GNO[0] = edge_gno;
sendbuf[cnt].GNO[1] = vtx_gno;
memcpy(wgtarray+cnt*outmat->mtx.pinwgtdim, outmat->mtx.pinwgt+j*outmat->mtx.pinwgtdim,
outmat->mtx.pinwgtdim*sizeof(float));
cnt++;
}
if(outmat->mtx.ystart[i] == outmat->mtx.yend[i]) {
proclist[cnt] = (*outmat->hashDistFct)(edge_gno, -1, outmat->hashDistData,
&sendbuf[cnt].part_y);
if (proclist[cnt] < 0) /* Discard this nnz */
continue;
sendbuf[cnt].GNO[0] = edge_gno;
sendbuf[cnt].GNO[1] = -1;
memset(wgtarray+cnt*outmat->mtx.pinwgtdim, 0,outmat->mtx.pinwgtdim*sizeof(float));
cnt++;
}
}
if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin)
Zoltan_Distribute_Partition_Free(&outmat->hashDistData);
if (outmat->mtx.yend != outmat->mtx.ystart + 1)
ZOLTAN_FREE(&outmat->mtx.yend);
outmat->mtx.yend = NULL;
ZOLTAN_FREE(&outmat->mtx.ystart);
ZOLTAN_FREE(&outmat->mtx.yGNO);
ZOLTAN_FREE(&outmat->mtx.pinGNO);
ZOLTAN_FREE(&outmat->mtx.pinwgt);
ZOLTAN_FREE(&outmat->mtx.yGID);
/*
* Send pins to their target processors.
* They become non-zeros in the 2D data distribution.
*/
KDDKDDKDD(zz->Proc, " CommPlan Create");
msg_tag--;
ierr = Zoltan_Comm_Create(&plan, cnt, proclist, communicator, msg_tag, &outmat->mtx.nPins);
ZOLTAN_FREE(&proclist);
nonzeros = (Zoltan_Arc *) ZOLTAN_MALLOC((outmat->mtx.nPins) * sizeof(Zoltan_Arc));
if (outmat->mtx.nPins && nonzeros == NULL) MEMORY_ERROR;
msg_tag--;
Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(Zoltan_Arc), (char *) nonzeros);
ZOLTAN_FREE(&sendbuf);
if (outmat->mtx.pinwgtdim) { /* We have to take care about weights */
tmpwgtarray = (float*) ZOLTAN_MALLOC(outmat->mtx.nPins*outmat->mtx.pinwgtdim*sizeof(float));
if (outmat->mtx.nPins && tmpwgtarray == NULL) MEMORY_ERROR;
msg_tag--;
Zoltan_Comm_Do(plan, msg_tag, (char *) wgtarray, outmat->mtx.pinwgtdim*sizeof(float),
(char *) tmpwgtarray);
ZOLTAN_FREE(&wgtarray);
}
Zoltan_Comm_Destroy(&plan);
/* Unpack the non-zeros received. */
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_DupArcs");
/* TODO: do take care about singletons */
Zoltan_Matrix_Remove_DupArcs(zz, outmat->mtx.nPins, (Zoltan_Arc*)nonzeros, tmpwgtarray,
&outmat->mtx);
}
/* Now we just have to change numbering */
outmat->dist_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_y+1), sizeof(ZOLTAN_GNO_TYPE));
outmat->dist_x = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_x+1), sizeof(ZOLTAN_GNO_TYPE));
if (outmat->dist_y == NULL || outmat->dist_x == NULL) MEMORY_ERROR;
/* FIXME: Work only in 1D */
tmp_gno = (ZOLTAN_GNO_TYPE)outmat->mtx.nY;
MPI_Allgather(&tmp_gno, 1, zoltan_gno_mpi_type, outmat->dist_y+1, 1, zoltan_gno_mpi_type, communicator);
for (i = 1 ; i <= nProc_y ; i ++) {
outmat->dist_y[i] += outmat->dist_y[i-1];
}
outmat->dist_x[1] = outmat->mtx.globalX;
perm_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(outmat->mtx.nY * sizeof(ZOLTAN_GNO_TYPE));
if (outmat->mtx.nY > 0 && perm_y == NULL) MEMORY_ERROR;
for (i = 0 ; i < outmat->mtx.nY ; ++i){
perm_y[i] = i + outmat->dist_y[myProc_y];
}
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute");
Zoltan_Matrix_Permute(zz, &outmat->mtx, perm_y);
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute done");
End:
ZOLTAN_FREE(&perm_y);
ZOLTAN_FREE(&proclist);
ZOLTAN_FREE(&sendbuf);
ZOLTAN_FREE(&nonzeros);
ZOLTAN_FREE(&tmpwgtarray);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int
Zoltan_Matrix_Sym(ZZ* zz, Zoltan_matrix *matrix, int bipartite)
{
static char *yo = "Zoltan_Matrix_Sym";
int ierr = ZOLTAN_OK;
Zoltan_Arc *tr_tab = NULL;
int i, j, cnt;
ZOLTAN_ID_PTR yGID = NULL;
int *ypid=NULL;
float *pinwgt=NULL;
int * ybipart = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (bipartite || !matrix->opts.enforceSquare) {
matrix->bipartite = 1;
matrix->redist = 1;
}
if (matrix->ywgtdim != zz->Obj_Weight_Dim)
FATAL_ERROR("Cannot form bipartite graph: vertex and edge weights are not consistant");
matrix->opts.symmetrize = 1;
/* Update the data directories */
tr_tab = (Zoltan_Arc*) ZOLTAN_MALLOC(sizeof(Zoltan_Arc)*(matrix->nPins*2+matrix->nY));
if (matrix->nPins && tr_tab == NULL) MEMORY_ERROR;
pinwgt = (float*)ZOLTAN_MALLOC((matrix->nPins*2+matrix->nY)*matrix->pinwgtdim*sizeof(float));
for (i = 0 ; i < 2 ; ++i) /* Copy pin weights */
memcpy(pinwgt + i*matrix->nPins*matrix->pinwgtdim*sizeof(float),
matrix->pinwgt, matrix->nPins*matrix->pinwgtdim*sizeof(float));
for (i=0, cnt = 0 ; i < matrix->nY ; ++i) {
for (j = matrix->ystart[i] ; j < matrix->yend[i] ; ++j) {
tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */
tr_tab[cnt].GNO[1] = matrix->pinGNO[j];
memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim,
matrix->pinwgtdim*sizeof(float));
cnt ++;
tr_tab[cnt].GNO[0] = matrix->pinGNO[j]; /* Symmetric arc */
tr_tab[cnt].GNO[1] = matrix->yGNO[i] + bipartite*matrix->globalX; /* new ordering */
memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim,
matrix->pinwgtdim*sizeof(float));
cnt ++;
}
if (matrix->ystart[i] == matrix->yend[i]) { /* Singleton */
tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */
tr_tab[cnt].GNO[1] = -1;
cnt ++;
}
}
ZOLTAN_FREE(&matrix->pinwgt);
Zoltan_Matrix_Remove_DupArcs(zz, cnt, tr_tab, pinwgt, matrix);
ZOLTAN_FREE(&tr_tab);
ZOLTAN_FREE(&pinwgt);
if (bipartite) {
int endX;
int * yGNO = NULL;
/* Update data directories */
yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nY);
ypid = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int));
ybipart = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int));
for (endX = 0 ; endX < matrix->nY ; ++endX) {
if (matrix->yGNO[endX] >= matrix->globalX)
break;
ybipart[endX] = 0;
}
/* Get Informations about X */
Zoltan_DD_Find (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, NULL,
endX, NULL);
yGNO = (int*)ZOLTAN_MALLOC(endX*sizeof(int));
for (i = endX ; i < matrix->nY ; ++i) {
yGNO[i-endX] = matrix->yGNO[i] - matrix->globalX;
/* TODO: add a something to have the correct ypid */
ybipart[endX] = 1;
}
/* Get Informations about Y */
Zoltan_DD_Find (matrix->ddY, (ZOLTAN_ID_PTR)yGNO,
yGID + endX*zz->Num_GID,
NULL, NULL,
matrix->nY-endX, NULL);
if (matrix->ddY != matrix->ddX)
Zoltan_DD_Destroy (&matrix->ddY);
Zoltan_DD_Destroy (&matrix->ddX);
matrix->globalX += matrix->globalY;
matrix->globalY = matrix->globalX;
/* I store : xGNO, xGID, xpid, bipart */
ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID,
1, matrix->globalX/zz->Num_Proc, 0);
matrix->ddY = matrix->ddX;
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
/* Associate all the data with our xyGNO */
Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, ybipart,
matrix->nY);
}
End:
ZOLTAN_FREE(&ybipart);
ZOLTAN_FREE(&ypid);
ZOLTAN_FREE(&pinwgt);
ZOLTAN_FREE(&yGID);
ZOLTAN_FREE(&tr_tab);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
