/*--------------------------------------------------------------------*/
int
main ( int argc, char *argv[] )
/*
---------------------------------------------------
read in a DSTree object, read in a Graph file,
read in a DV cutoffs file, get the stages IV object
based on domain weight and write it to a file.
created -- 97jun12, cca
---------------------------------------------------
*/
{
char *inCutoffDVfileName, *inDSTreeFileName,
*inGraphFileName, *outIVfileName ;
double t1, t2 ;
DV *cutoffDV ;
Graph *graph ;
int msglvl, rc ;
IV *stagesIV ;
DSTree *dstree ;
FILE *msgFile ;
if ( argc != 7 ) {
fprintf(stdout,
"\n\n usage : %s msglvl msgFile inDSTreeFile inGraphFile "
"\n inCutoffDVfile outFile"
"\n msglvl -- message level"
"\n msgFile -- message file"
"\n inDSTreeFile -- input file, must be *.dstreef or *.dstreeb"
"\n inGraphFile -- input file, must be *.graphf or *.graphb"
"\n inCutoffDVfile -- input file, must be *.dvf or *.dvb"
"\n outFile -- output file, must be *.ivf or *.ivb"
"\n", argv[0]) ;
return(0) ;
}
msglvl = atoi(argv[1]) ;
if ( strcmp(argv[2], "stdout") == 0 ) {
msgFile = stdout ;
} else if ( (msgFile = fopen(argv[2], "a")) == NULL ) {
fprintf(stderr, "\n fatal error in %s"
"\n unable to open file %s\n",
argv[0], argv[2]) ;
return(-1) ;
}
inDSTreeFileName = argv[3] ;
inGraphFileName = argv[4] ;
inCutoffDVfileName = argv[5] ;
outIVfileName = argv[6] ;
fprintf(msgFile,
"\n %s "
"\n msglvl -- %d"
"\n msgFile -- %s"
"\n inDSTreeFileName -- %s"
"\n inGraphFileName -- %s"
"\n inCutoffDVfileName -- %s"
"\n outFile -- %s"
"\n",
argv[0], msglvl, argv[2], inDSTreeFileName,
inGraphFileName, inCutoffDVfileName, outIVfileName) ;
fflush(msgFile) ;
/*
-------------------------
read in the DSTree object
-------------------------
*/
if ( strcmp(inDSTreeFileName, "none") == 0 ) {
fprintf(msgFile, "\n no file to read from") ;
spoolesFatal();
}
dstree = DSTree_new() ;
MARKTIME(t1) ;
rc = DSTree_readFromFile(dstree, inDSTreeFileName) ;
MARKTIME(t2) ;
fprintf(msgFile, "\n CPU %9.5f : read in dstree from file %s",
t2 - t1, inDSTreeFileName) ;
if ( rc != 1 ) {
fprintf(msgFile, "\n return value %d from DSTree_readFromFile(%p,%s)",
rc, dstree, inDSTreeFileName) ;
spoolesFatal();
}
fprintf(msgFile, "\n\n after reading DSTree object from file %s",
inDSTreeFileName) ;
if ( msglvl > 2 ) {
DSTree_writeForHumanEye(dstree, msgFile) ;
} else {
DSTree_writeStats(dstree, msgFile) ;
}
fflush(msgFile) ;
/*
-------------------------
read in the Graph object
-------------------------
*/
if ( strcmp(inGraphFileName, "none") == 0 ) {
fprintf(msgFile, "\n no file to read from") ;
spoolesFatal();
}
graph = Graph_new() ;
MARKTIME(t1) ;
rc = Graph_readFromFile(graph, inGraphFileName) ;
MARKTIME(t2) ;
fprintf(msgFile, "\n CPU %9.5f : read in graph from file %s",
t2 - t1, inGraphFileName) ;
if ( rc != 1 ) {
fprintf(msgFile, "\n return value %d from Graph_readFromFile(%p,%s)",
rc, graph, inGraphFileName) ;
spoolesFatal();
}
fprintf(msgFile, "\n\n after reading Graph object from file %s",
inGraphFileName) ;
if ( msglvl > 2 ) {
Graph_writeForHumanEye(graph, msgFile) ;
} else {
Graph_writeStats(graph, msgFile) ;
}
fflush(msgFile) ;
/*
-----------------------------
read in the cutoffs DV object
-----------------------------
*/
if ( strcmp(inCutoffDVfileName, "none") == 0 ) {
fprintf(msgFile, "\n no file to read from") ;
spoolesFatal();
}
cutoffDV = DV_new() ;
MARKTIME(t1) ;
rc = DV_readFromFile(cutoffDV, inCutoffDVfileName) ;
MARKTIME(t2) ;
fprintf(msgFile, "\n CPU %9.5f : read in graph from file %s",
t2 - t1, inCutoffDVfileName) ;
if ( rc != 1 ) {
fprintf(msgFile, "\n return value %d from DV_readFromFile(%p,%s)",
rc, cutoffDV, inCutoffDVfileName) ;
spoolesFatal();
}
fprintf(msgFile, "\n\n after reading DV object from file %s",
inCutoffDVfileName) ;
if ( msglvl > 0 ) {
DV_writeForHumanEye(cutoffDV, msgFile) ;
} else {
DV_writeStats(cutoffDV, msgFile) ;
}
fflush(msgFile) ;
/*
---------------------
get the stages vector
---------------------
*/
stagesIV = DSTree_stagesViaDomainWeight(dstree,
graph->vwghts, cutoffDV) ;
if ( msglvl > 2 ) {
IV_writeForHumanEye(stagesIV, msgFile) ;
} else {
IV_writeStats(stagesIV, msgFile) ;
}
fflush(msgFile) ;
/*
---------------------------
write out the DSTree object
---------------------------
*/
if ( stagesIV != NULL && strcmp(outIVfileName, "none") != 0 ) {
MARKTIME(t1) ;
rc = IV_writeToFile(stagesIV, outIVfileName) ;
MARKTIME(t2) ;
fprintf(msgFile, "\n CPU %9.5f : write dstree to file %s",
t2 - t1, outIVfileName) ;
if ( rc != 1 ) {
fprintf(msgFile,
"\n return value %d from IV_writeToFile(%p,%s)",
rc, stagesIV, outIVfileName) ;
}
}
/*
----------------------
free the DSTree object
----------------------
*/
DSTree_free(dstree) ;
if ( stagesIV != NULL ) {
IV_free(stagesIV) ;
}
fprintf(msgFile, "\n") ;
fclose(msgFile) ;
return(1) ; }
/*
----------------------------------------------------------
purpose -- to construct the map from fronts to processors,
and compute operations for each processor.
maptype -- type of map for parallel factorization
maptype = 1 --> wrap map
maptype = 2 --> balanced map
maptype = 3 --> subtree-subset map
maptype = 4 --> domain decomposition map
cutoff -- used when maptype = 4 as upper bound on
relative domain size
return value --
1 -- success
-1 -- bridge is NULL
-2 -- front tree is NULL
created -- 98sep25, cca
----------------------------------------------------------
*/
int
BridgeMPI_factorSetup (
BridgeMPI *bridge,
int maptype,
double cutoff
) {
double t1, t2 ;
DV *cumopsDV ;
ETree *frontETree ;
FILE *msgFile ;
int msglvl, nproc ;
/*
---------------
check the input
---------------
*/
MARKTIME(t1) ;
if ( bridge == NULL ) {
fprintf(stderr, "\n error in BridgeMPI_factorSetup()"
"\n bridge is NULL") ;
return(-1) ;
}
if ( (frontETree = bridge->frontETree) == NULL ) {
fprintf(stderr, "\n error in BridgeMPI_factorSetup()"
"\n frontETree is NULL") ;
return(-2) ;
}
nproc = bridge->nproc ;
msglvl = bridge->msglvl ;
msgFile = bridge->msgFile ;
/*
-------------------------------------------
allocate and initialize the cumopsDV object
-------------------------------------------
*/
if ( (cumopsDV = bridge->cumopsDV) == NULL ) {
cumopsDV = bridge->cumopsDV = DV_new() ;
}
DV_setSize(cumopsDV, nproc) ;
DV_zero(cumopsDV) ;
/*
----------------------------
create the owners map object
----------------------------
*/
switch ( maptype ) {
case 1 :
bridge->ownersIV = ETree_wrapMap(frontETree, bridge->type,
bridge->symmetryflag, cumopsDV) ;
break ;
case 2 :
bridge->ownersIV = ETree_balancedMap(frontETree, bridge->type,
bridge->symmetryflag, cumopsDV) ;
break ;
case 3 :
bridge->ownersIV = ETree_subtreeSubsetMap(frontETree, bridge->type,
bridge->symmetryflag, cumopsDV) ;
break ;
case 4 :
bridge->ownersIV = ETree_ddMap(frontETree, bridge->type,
bridge->symmetryflag, cumopsDV, cutoff) ;
break ;
default :
bridge->ownersIV = ETree_ddMap(frontETree, bridge->type,
bridge->symmetryflag, cumopsDV, 1./(2*nproc)) ;
break ;
}
MARKTIME(t2) ;
bridge->cpus[7] = t2 - t1 ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n parallel factor setup") ;
fprintf(msgFile, "\n type = %d, symmetryflag = %d",
bridge->type, bridge->symmetryflag) ;
fprintf(msgFile, "\n total factor operations = %.0f",
DV_sum(cumopsDV)) ;
fprintf(msgFile,
"\n upper bound on speedup due to load balance = %.2f",
DV_max(cumopsDV)/DV_sum(cumopsDV)) ;
fprintf(msgFile, "\n operations distributions over threads") ;
DV_writeForHumanEye(cumopsDV, msgFile) ;
fflush(msgFile) ;
}
if ( msglvl > 2 ) {
fprintf(msgFile, "\n\n owners map IV object") ;
IV_writeForHumanEye(bridge->ownersIV, msgFile) ;
fflush(msgFile) ;
}
/*
----------------------------
create the vertex map object
----------------------------
*/
bridge->vtxmapIV = IV_new() ;
IV_init(bridge->vtxmapIV, bridge->neqns, NULL) ;
IVgather(bridge->neqns, IV_entries(bridge->vtxmapIV),
IV_entries(bridge->ownersIV),
ETree_vtxToFront(bridge->frontETree)) ;
if ( msglvl > 2 ) {
fprintf(msgFile, "\n\n vertex map IV object") ;
IV_writeForHumanEye(bridge->vtxmapIV, msgFile) ;
fflush(msgFile) ;
}
return(1) ;
}
PetscErrorCode MatFactorNumeric_SeqSpooles(Mat F,Mat A,const MatFactorInfo *info)
{
Mat_Spooles *lu = (Mat_Spooles*)(F)->spptr;
ChvManager *chvmanager ;
Chv *rootchv ;
IVL *adjIVL;
PetscErrorCode ierr;
PetscInt nz,nrow=A->rmap->n,irow,nedges,neqns=A->cmap->n,*ai,*aj,i,*diag=0,fierr;
PetscScalar *av;
double cputotal,facops;
#if defined(PETSC_USE_COMPLEX)
PetscInt nz_row,*aj_tmp;
PetscScalar *av_tmp;
#else
PetscInt *ivec1,*ivec2,j;
double *dvec;
#endif
PetscBool isSeqAIJ,isMPIAIJ;
PetscFunctionBegin;
if (lu->flg == DIFFERENT_NONZERO_PATTERN) { /* first numeric factorization */
(F)->ops->solve = MatSolve_SeqSpooles;
(F)->assembled = PETSC_TRUE;
/* set Spooles options */
ierr = SetSpoolesOptions(A, &lu->options);CHKERRQ(ierr);
lu->mtxA = InpMtx_new();
}
/* copy A to Spooles' InpMtx object */
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&isSeqAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&isMPIAIJ);CHKERRQ(ierr);
if (isSeqAIJ){
Mat_SeqAIJ *mat = (Mat_SeqAIJ*)A->data;
ai=mat->i; aj=mat->j; av=mat->a;
if (lu->options.symflag == SPOOLES_NONSYMMETRIC) {
nz=mat->nz;
} else { /* SPOOLES_SYMMETRIC || SPOOLES_HERMITIAN */
nz=(mat->nz + A->rmap->n)/2;
diag=mat->diag;
}
} else { /* A is SBAIJ */
Mat_SeqSBAIJ *mat = (Mat_SeqSBAIJ*)A->data;
ai=mat->i; aj=mat->j; av=mat->a;
nz=mat->nz;
}
InpMtx_init(lu->mtxA, INPMTX_BY_ROWS, lu->options.typeflag, nz, 0);
#if defined(PETSC_USE_COMPLEX)
for (irow=0; irow<nrow; irow++) {
if ( lu->options.symflag == SPOOLES_NONSYMMETRIC || !(isSeqAIJ || isMPIAIJ)){
nz_row = ai[irow+1] - ai[irow];
aj_tmp = aj + ai[irow];
av_tmp = av + ai[irow];
} else {
nz_row = ai[irow+1] - diag[irow];
aj_tmp = aj + diag[irow];
av_tmp = av + diag[irow];
}
for (i=0; i<nz_row; i++){
InpMtx_inputComplexEntry(lu->mtxA, irow, *aj_tmp++,PetscRealPart(*av_tmp),PetscImaginaryPart(*av_tmp));
av_tmp++;
}
}
#else
ivec1 = InpMtx_ivec1(lu->mtxA);
ivec2 = InpMtx_ivec2(lu->mtxA);
dvec = InpMtx_dvec(lu->mtxA);
if ( lu->options.symflag == SPOOLES_NONSYMMETRIC || !isSeqAIJ){
for (irow = 0; irow < nrow; irow++){
for (i = ai[irow]; i<ai[irow+1]; i++) ivec1[i] = irow;
}
IVcopy(nz, ivec2, aj);
DVcopy(nz, dvec, av);
} else {
nz = 0;
for (irow = 0; irow < nrow; irow++){
for (j = diag[irow]; j<ai[irow+1]; j++) {
ivec1[nz] = irow;
ivec2[nz] = aj[j];
dvec[nz] = av[j];
nz++;
}
}
}
InpMtx_inputRealTriples(lu->mtxA, nz, ivec1, ivec2, dvec);
#endif
InpMtx_changeStorageMode(lu->mtxA, INPMTX_BY_VECTORS);
if ( lu->options.msglvl > 0 ) {
int err;
printf("\n\n input matrix");
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n input matrix");CHKERRQ(ierr);
InpMtx_writeForHumanEye(lu->mtxA, lu->options.msgFile);
err = fflush(lu->options.msgFile);
if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
}
if ( lu->flg == DIFFERENT_NONZERO_PATTERN){ /* first numeric factorization */
/*---------------------------------------------------
find a low-fill ordering
(1) create the Graph object
(2) order the graph
-------------------------------------------------------*/
if (lu->options.useQR){
adjIVL = InpMtx_adjForATA(lu->mtxA);
} else {
adjIVL = InpMtx_fullAdjacency(lu->mtxA);
}
nedges = IVL_tsize(adjIVL);
lu->graph = Graph_new();
Graph_init2(lu->graph, 0, neqns, 0, nedges, neqns, nedges, adjIVL, NULL, NULL);
if ( lu->options.msglvl > 2 ) {
int err;
if (lu->options.useQR){
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n graph of A^T A");CHKERRQ(ierr);
} else {
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n graph of the input matrix");CHKERRQ(ierr);
}
Graph_writeForHumanEye(lu->graph, lu->options.msgFile);
err = fflush(lu->options.msgFile);
if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
}
switch (lu->options.ordering) {
case 0:
lu->frontETree = orderViaBestOfNDandMS(lu->graph,
lu->options.maxdomainsize, lu->options.maxzeros, lu->options.maxsize,
lu->options.seed, lu->options.msglvl, lu->options.msgFile); break;
case 1:
lu->frontETree = orderViaMMD(lu->graph,lu->options.seed,lu->options.msglvl,lu->options.msgFile); break;
case 2:
lu->frontETree = orderViaMS(lu->graph, lu->options.maxdomainsize,
lu->options.seed,lu->options.msglvl,lu->options.msgFile); break;
case 3:
lu->frontETree = orderViaND(lu->graph, lu->options.maxdomainsize,
lu->options.seed,lu->options.msglvl,lu->options.msgFile); break;
default:
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Unknown Spooles's ordering");
}
if ( lu->options.msglvl > 0 ) {
int err;
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n front tree from ordering");CHKERRQ(ierr);
ETree_writeForHumanEye(lu->frontETree, lu->options.msgFile);
err = fflush(lu->options.msgFile);
if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
}
/* get the permutation, permute the front tree */
lu->oldToNewIV = ETree_oldToNewVtxPerm(lu->frontETree);
lu->oldToNew = IV_entries(lu->oldToNewIV);
lu->newToOldIV = ETree_newToOldVtxPerm(lu->frontETree);
if (!lu->options.useQR) ETree_permuteVertices(lu->frontETree, lu->oldToNewIV);
/* permute the matrix */
if (lu->options.useQR){
InpMtx_permute(lu->mtxA, NULL, lu->oldToNew);
} else {
InpMtx_permute(lu->mtxA, lu->oldToNew, lu->oldToNew);
if ( lu->options.symflag == SPOOLES_SYMMETRIC) {
InpMtx_mapToUpperTriangle(lu->mtxA);
}
#if defined(PETSC_USE_COMPLEX)
if ( lu->options.symflag == SPOOLES_HERMITIAN ) {
InpMtx_mapToUpperTriangleH(lu->mtxA);
}
#endif
InpMtx_changeCoordType(lu->mtxA, INPMTX_BY_CHEVRONS);
}
InpMtx_changeStorageMode(lu->mtxA, INPMTX_BY_VECTORS);
/* get symbolic factorization */
if (lu->options.useQR){
lu->symbfacIVL = SymbFac_initFromGraph(lu->frontETree, lu->graph);
IVL_overwrite(lu->symbfacIVL, lu->oldToNewIV);
IVL_sortUp(lu->symbfacIVL);
ETree_permuteVertices(lu->frontETree, lu->oldToNewIV);
} else {
lu->symbfacIVL = SymbFac_initFromInpMtx(lu->frontETree, lu->mtxA);
}
if ( lu->options.msglvl > 2 ) {
int err;
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n old-to-new permutation vector");CHKERRQ(ierr);
IV_writeForHumanEye(lu->oldToNewIV, lu->options.msgFile);
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n new-to-old permutation vector");CHKERRQ(ierr);
IV_writeForHumanEye(lu->newToOldIV, lu->options.msgFile);
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n front tree after permutation");CHKERRQ(ierr);
ETree_writeForHumanEye(lu->frontETree, lu->options.msgFile);
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n input matrix after permutation");CHKERRQ(ierr);
InpMtx_writeForHumanEye(lu->mtxA, lu->options.msgFile);
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n symbolic factorization");CHKERRQ(ierr);
IVL_writeForHumanEye(lu->symbfacIVL, lu->options.msgFile);
err = fflush(lu->options.msgFile);
if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
}
lu->frontmtx = FrontMtx_new();
lu->mtxmanager = SubMtxManager_new();
SubMtxManager_init(lu->mtxmanager, NO_LOCK, 0);
} else { /* new num factorization using previously computed symbolic factor */
if (lu->options.pivotingflag) { /* different FrontMtx is required */
FrontMtx_free(lu->frontmtx);
lu->frontmtx = FrontMtx_new();
} else {
FrontMtx_clearData (lu->frontmtx);
}
SubMtxManager_free(lu->mtxmanager);
lu->mtxmanager = SubMtxManager_new();
SubMtxManager_init(lu->mtxmanager, NO_LOCK, 0);
/* permute mtxA */
if (lu->options.useQR){
InpMtx_permute(lu->mtxA, NULL, lu->oldToNew);
} else {
InpMtx_permute(lu->mtxA, lu->oldToNew, lu->oldToNew);
if ( lu->options.symflag == SPOOLES_SYMMETRIC ) {
InpMtx_mapToUpperTriangle(lu->mtxA);
}
InpMtx_changeCoordType(lu->mtxA, INPMTX_BY_CHEVRONS);
}
InpMtx_changeStorageMode(lu->mtxA, INPMTX_BY_VECTORS);
if ( lu->options.msglvl > 2 ) {
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n input matrix after permutation");CHKERRQ(ierr);
InpMtx_writeForHumanEye(lu->mtxA, lu->options.msgFile);
}
} /* end of if( lu->flg == DIFFERENT_NONZERO_PATTERN) */
if (lu->options.useQR){
FrontMtx_init(lu->frontmtx, lu->frontETree, lu->symbfacIVL, lu->options.typeflag,
SPOOLES_SYMMETRIC, FRONTMTX_DENSE_FRONTS,
SPOOLES_NO_PIVOTING, NO_LOCK, 0, NULL,
lu->mtxmanager, lu->options.msglvl, lu->options.msgFile);
} else {
FrontMtx_init(lu->frontmtx, lu->frontETree, lu->symbfacIVL, lu->options.typeflag, lu->options.symflag,
FRONTMTX_DENSE_FRONTS, lu->options.pivotingflag, NO_LOCK, 0, NULL,
lu->mtxmanager, lu->options.msglvl, lu->options.msgFile);
}
if ( lu->options.symflag == SPOOLES_SYMMETRIC ) { /* || SPOOLES_HERMITIAN ? */
if ( lu->options.patchAndGoFlag == 1 ) {
lu->frontmtx->patchinfo = PatchAndGoInfo_new();
PatchAndGoInfo_init(lu->frontmtx->patchinfo, 1, lu->options.toosmall, lu->options.fudge,
lu->options.storeids, lu->options.storevalues);
} else if ( lu->options.patchAndGoFlag == 2 ) {
lu->frontmtx->patchinfo = PatchAndGoInfo_new();
PatchAndGoInfo_init(lu->frontmtx->patchinfo, 2, lu->options.toosmall, lu->options.fudge,
lu->options.storeids, lu->options.storevalues);
}
}
/* numerical factorization */
chvmanager = ChvManager_new();
ChvManager_init(chvmanager, NO_LOCK, 1);
DVfill(10, lu->cpus, 0.0);
if (lu->options.useQR){
facops = 0.0 ;
FrontMtx_QR_factor(lu->frontmtx, lu->mtxA, chvmanager,
lu->cpus, &facops, lu->options.msglvl, lu->options.msgFile);
if ( lu->options.msglvl > 1 ) {
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n factor matrix");CHKERRQ(ierr);
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n facops = %9.2f", facops);CHKERRQ(ierr);
}
} else {
IVfill(20, lu->stats, 0);
rootchv = FrontMtx_factorInpMtx(lu->frontmtx, lu->mtxA, lu->options.tau, 0.0,
chvmanager, &fierr, lu->cpus,lu->stats,lu->options.msglvl,lu->options.msgFile);
if (rootchv) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MAT_LU_ZRPVT,"\n matrix found to be singular");
if (fierr >= 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"\n error encountered at front %D", fierr);
if(lu->options.FrontMtxInfo){
ierr = PetscPrintf(PETSC_COMM_SELF,"\n %8d pivots, %8d pivot tests, %8d delayed rows and columns\n",lu->stats[0], lu->stats[1], lu->stats[2]);CHKERRQ(ierr);
cputotal = lu->cpus[8] ;
if ( cputotal > 0.0 ) {
ierr = PetscPrintf(PETSC_COMM_SELF,
"\n cpus cpus/totaltime"
"\n initialize fronts %8.3f %6.2f"
"\n load original entries %8.3f %6.2f"
"\n update fronts %8.3f %6.2f"
"\n assemble postponed data %8.3f %6.2f"
"\n factor fronts %8.3f %6.2f"
"\n extract postponed data %8.3f %6.2f"
"\n store factor entries %8.3f %6.2f"
"\n miscellaneous %8.3f %6.2f"
"\n total time %8.3f \n",
lu->cpus[0], 100.*lu->cpus[0]/cputotal,
lu->cpus[1], 100.*lu->cpus[1]/cputotal,
lu->cpus[2], 100.*lu->cpus[2]/cputotal,
lu->cpus[3], 100.*lu->cpus[3]/cputotal,
lu->cpus[4], 100.*lu->cpus[4]/cputotal,
lu->cpus[5], 100.*lu->cpus[5]/cputotal,
lu->cpus[6], 100.*lu->cpus[6]/cputotal,
lu->cpus[7], 100.*lu->cpus[7]/cputotal, cputotal);CHKERRQ(ierr);
}
}
}
ChvManager_free(chvmanager);
if ( lu->options.msglvl > 0 ) {
int err;
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n factor matrix");CHKERRQ(ierr);
FrontMtx_writeForHumanEye(lu->frontmtx, lu->options.msgFile);
err = fflush(lu->options.msgFile);
if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
}
if ( lu->options.symflag == SPOOLES_SYMMETRIC ) { /* || SPOOLES_HERMITIAN ? */
if ( lu->options.patchAndGoFlag == 1 ) {
if ( lu->frontmtx->patchinfo->fudgeIV != NULL ) {
if (lu->options.msglvl > 0 ){
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n small pivots found at these locations");CHKERRQ(ierr);
IV_writeForHumanEye(lu->frontmtx->patchinfo->fudgeIV, lu->options.msgFile);
}
}
PatchAndGoInfo_free(lu->frontmtx->patchinfo);
} else if ( lu->options.patchAndGoFlag == 2 ) {
if (lu->options.msglvl > 0 ){
if ( lu->frontmtx->patchinfo->fudgeIV != NULL ) {
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n small pivots found at these locations");CHKERRQ(ierr);
IV_writeForHumanEye(lu->frontmtx->patchinfo->fudgeIV, lu->options.msgFile);
}
if ( lu->frontmtx->patchinfo->fudgeDV != NULL ) {
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n perturbations");CHKERRQ(ierr);
DV_writeForHumanEye(lu->frontmtx->patchinfo->fudgeDV, lu->options.msgFile);
}
}
PatchAndGoInfo_free(lu->frontmtx->patchinfo);
}
}
/* post-process the factorization */
FrontMtx_postProcess(lu->frontmtx, lu->options.msglvl, lu->options.msgFile);
if ( lu->options.msglvl > 2 ) {
int err;
ierr = PetscFPrintf(PETSC_COMM_SELF,lu->options.msgFile, "\n\n factor matrix after post-processing");CHKERRQ(ierr);
FrontMtx_writeForHumanEye(lu->frontmtx, lu->options.msgFile);
err = fflush(lu->options.msgFile);
if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
}
lu->flg = SAME_NONZERO_PATTERN;
lu->CleanUpSpooles = PETSC_TRUE;
PetscFunctionReturn(0);
}
SPOOLESSolverMT::SPOOLESSolverMT(const SparseMatrix * A, int numThreads, int verbose)
{
n = A->Getn();
this->verbose = verbose;
msgFile = fopen("SPOOLES.message","w");
// prepare SPOOLES input matrix
if (verbose >= 1)
printf("Converting matrix to SPOOLES format...\n");
InpMtx * mtxA = InpMtx_new();
InpMtx_init(mtxA, INPMTX_BY_ROWS, SPOOLES_REAL, A->GetNumEntries(), n);
for(int row=0; row<n; row++)
{
int rowLength = A->GetRowLength(row);
for(int j=0; j< rowLength; j++)
{
if (A->GetColumnIndex(row,j) >= row)
InpMtx_inputRealEntry(mtxA, row, A->GetColumnIndex(row, j), A->GetEntry(row, j) );
}
}
InpMtx_changeStorageMode(mtxA, INPMTX_BY_VECTORS);
//InpMtx_writeForHumanEye(mtxA, msgFile);
// compute the factorization
if (verbose >= 1)
printf("Factoring the %d x %d matrix...\n",n,n);
BridgeMT * bridgeMT = BridgeMT_new();
BridgeMT_setMatrixParams(bridgeMT, n, SPOOLES_REAL, SPOOLES_SYMMETRIC);
BridgeMT_setMessageInfo(bridgeMT, 1, msgFile);
int rc = BridgeMT_setup(bridgeMT, mtxA);
if (rc != 1)
{
printf("Error: BridgeMT setup returned exit code %d.\n", rc);
throw 1;
}
int type = 1; // real entries
int nfront, nfind, nfent;
double nfactorops;
rc = BridgeMT_factorStats(bridgeMT, type, SPOOLES_SYMMETRIC, &nfront,
&nfind, &nfent, &nsolveops, &nfactorops);
if ( rc != 1 )
{
printf("Error: BridgeMT_factorStats returned exit code %d.\n", rc);
throw 1;
}
fprintf(msgFile,
"\n\n factor matrix statistics"
"\n %d fronts, %d indices, %d entries"
"\n %d solve operations, %12.4e factor operations",
nfront, nfind, nfent, nsolveops, nfactorops) ;
fflush(msgFile) ;
// setup the parallel factorization
rc = BridgeMT_factorSetup(bridgeMT, numThreads, 0, 0.0) ;
fprintf(msgFile, "\n\n ----- PARALLEL FACTOR SETUP -----\n") ;
fprintf(msgFile,
"\n CPU %8.3f : time to setup parallel factorization",
bridgeMT->cpus[5]) ;
fprintf(msgFile, "\n total factor operations = %.0f",
DV_sum(bridgeMT->cumopsDV)) ;
fprintf(msgFile,
"\n upper bound on speedup due to load balance = %.2f",
DV_sum(bridgeMT->cumopsDV)/DV_max(bridgeMT->cumopsDV)) ;
fprintf(msgFile, "\n operations distributions over threads") ;
DV_writeForHumanEye(bridgeMT->cumopsDV, msgFile) ;
fflush(msgFile) ;
// factor the matrix
int permuteflag = 1 ;
int error;
rc = BridgeMT_factor(bridgeMT, mtxA, permuteflag, &error);
if ( rc == 1 )
{
fprintf(msgFile, "\n\n factorization completed successfully\n") ;
}
else
{
printf("Error: factorization returned exit code %d (error %d).\n", rc, error);
throw 1;
}
fprintf(msgFile, "\n\n ----- FACTORIZATION -----\n") ;
fprintf(msgFile,
"\n CPU %8.3f : time to permute original matrix"
"\n CPU %8.3f : time to initialize factor matrix"
"\n CPU %8.3f : time to compute factorization"
"\n CPU %8.3f : time to post-process factorization"
"\n CPU %8.3f : total factorization time\n",
bridgeMT->cpus[6],
bridgeMT->cpus[7],
bridgeMT->cpus[8],
bridgeMT->cpus[9],
bridgeMT->cpus[10]) ;
fprintf(msgFile, "\n\n factorization statistics"
"\n %d pivots, %d pivot tests, %d delayed vertices"
"\n %d entries in D, %d entries in L, %d entries in U",
bridgeMT->stats[0], bridgeMT->stats[1], bridgeMT->stats[2],
bridgeMT->stats[3], bridgeMT->stats[4], bridgeMT->stats[5]) ;
fprintf(msgFile,
"\n\n factorization: raw mflops %8.3f, overall mflops %8.3f",
1.e-6*nfactorops/bridgeMT->cpus[8],
1.e-6*nfactorops/bridgeMT->cpus[10]) ;
fflush(msgFile) ;
// construct dense SPOOLES matrix for rhs and x
DenseMtx *mtx_rhs = DenseMtx_new();
DenseMtx_init(mtx_rhs, SPOOLES_REAL, 0, 0, n, 1, 1, n);
mtx_rhsPointer = (void*) mtx_rhs;
DenseMtx *mtx_x = DenseMtx_new();
DenseMtx_init(mtx_x, SPOOLES_REAL, 0, 0, n, 1, 1, n);
mtx_xPointer = (void*) mtx_x;
bridgeMTPointer = (void*) bridgeMT;
APointer = (void*) mtxA;
if (verbose >= 1)
printf("Factorization completed.\n");
}
/*--------------------------------------------------------------------*/
int
main ( int argc, char *argv[] ) {
/*
--------------------------------------------------
all-in-one program to solve A X = B
using a multithreaded factorization and solve
We use a patch-and-go strategy
for the factorization without pivoting
(1) read in matrix entries and form DInpMtx object
(2) form Graph object
(3) order matrix and form front tree
(4) get the permutation, permute the matrix and
front tree and get the symbolic factorization
(5) compute the numeric factorization
(6) read in right hand side entries
(7) compute the solution
created -- 98jun04, cca
--------------------------------------------------
*/
/*--------------------------------------------------------------------*/
char *matrixFileName, *rhsFileName ;
DenseMtx *mtxB, *mtxX ;
Chv *rootchv ;
ChvManager *chvmanager ;
double fudge, imag, real, tau = 100., toosmall, value ;
double cpus[10] ;
DV *cumopsDV ;
ETree *frontETree ;
FrontMtx *frontmtx ;
FILE *inputFile, *msgFile ;
Graph *graph ;
InpMtx *mtxA ;
int error, ient, irow, jcol, jrhs, jrow, lookahead, msglvl,
ncol, nedges, nent, neqns, nfront, nrhs, nrow, nthread,
patchAndGoFlag, seed,
storeids, storevalues, symmetryflag, type ;
int *newToOld, *oldToNew ;
int stats[20] ;
IV *newToOldIV, *oldToNewIV, *ownersIV ;
IVL *adjIVL, *symbfacIVL ;
SolveMap *solvemap ;
SubMtxManager *mtxmanager ;
/*--------------------------------------------------------------------*/
/*
--------------------
get input parameters
--------------------
*/
if ( argc != 14 ) {
fprintf(stdout, "\n"
"\n usage: %s msglvl msgFile type symmetryflag patchAndGoFlag"
"\n fudge toosmall storeids storevalues"
"\n matrixFileName rhsFileName seed"
"\n msglvl -- message level"
"\n msgFile -- message file"
"\n type -- type of entries"
"\n 1 (SPOOLES_REAL) -- real entries"
"\n 2 (SPOOLES_COMPLEX) -- complex entries"
"\n symmetryflag -- type of matrix"
"\n 0 (SPOOLES_SYMMETRIC) -- symmetric entries"
"\n 1 (SPOOLES_HERMITIAN) -- Hermitian entries"
"\n 2 (SPOOLES_NONSYMMETRIC) -- nonsymmetric entries"
"\n patchAndGoFlag -- flag for the patch-and-go strategy"
"\n 0 -- none, stop factorization"
"\n 1 -- optimization strategy"
"\n 2 -- structural analysis strategy"
"\n fudge -- perturbation parameter"
"\n toosmall -- upper bound on a small pivot"
"\n storeids -- flag to store ids of small pivots"
"\n storevalues -- flag to store perturbations"
"\n matrixFileName -- matrix file name, format"
"\n nrow ncol nent"
"\n irow jcol entry"
"\n ..."
"\n note: indices are zero based"
"\n rhsFileName -- right hand side file name, format"
"\n nrow nrhs "
"\n ..."
"\n jrow entry(jrow,0) ... entry(jrow,nrhs-1)"
"\n ..."
"\n seed -- random number seed, used for ordering"
"\n nthread -- number of threads"
"\n", argv[0]) ;
return(0) ;
}
msglvl = atoi(argv[1]) ;
if ( strcmp(argv[2], "stdout") == 0 ) {
msgFile = stdout ;
} else if ( (msgFile = fopen(argv[2], "a")) == NULL ) {
fprintf(stderr, "\n fatal error in %s"
"\n unable to open file %s\n",
argv[0], argv[2]) ;
return(-1) ;
}
type = atoi(argv[3]) ;
symmetryflag = atoi(argv[4]) ;
patchAndGoFlag = atoi(argv[5]) ;
fudge = atof(argv[6]) ;
toosmall = atof(argv[7]) ;
storeids = atoi(argv[8]) ;
storevalues = atoi(argv[9]) ;
matrixFileName = argv[10] ;
rhsFileName = argv[11] ;
seed = atoi(argv[12]) ;
nthread = atoi(argv[13]) ;
/*--------------------------------------------------------------------*/
/*
--------------------------------------------
STEP 1: read the entries from the input file
and create the InpMtx object
--------------------------------------------
*/
if ( (inputFile = fopen(matrixFileName, "r")) == NULL ) {
fprintf(stderr, "\n unable to open file %s", matrixFileName) ;
spoolesFatal();
}
fscanf(inputFile, "%d %d %d", &nrow, &ncol, &nent) ;
neqns = nrow ;
mtxA = InpMtx_new() ;
InpMtx_init(mtxA, INPMTX_BY_ROWS, type, nent, 0) ;
if ( type == SPOOLES_REAL ) {
for ( ient = 0 ; ient < nent ; ient++ ) {
fscanf(inputFile, "%d %d %le", &irow, &jcol, &value) ;
InpMtx_inputRealEntry(mtxA, irow, jcol, value) ;
}
} else {
for ( ient = 0 ; ient < nent ; ient++ ) {
fscanf(inputFile, "%d %d %le %le", &irow, &jcol, &real, &imag) ;
InpMtx_inputComplexEntry(mtxA, irow, jcol, real, imag) ;
}
}
fclose(inputFile) ;
InpMtx_changeStorageMode(mtxA, INPMTX_BY_VECTORS) ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n input matrix") ;
InpMtx_writeForHumanEye(mtxA, msgFile) ;
fflush(msgFile) ;
}
/*--------------------------------------------------------------------*/
/*
-------------------------------------------------
STEP 2 : find a low-fill ordering
(1) create the Graph object
(2) order the graph using multiple minimum degree
-------------------------------------------------
*/
graph = Graph_new() ;
adjIVL = InpMtx_fullAdjacency(mtxA) ;
nedges = IVL_tsize(adjIVL) ;
Graph_init2(graph, 0, neqns, 0, nedges, neqns, nedges, adjIVL,
NULL, NULL) ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n graph of the input matrix") ;
Graph_writeForHumanEye(graph, msgFile) ;
fflush(msgFile) ;
}
frontETree = orderViaMMD(graph, seed, msglvl, msgFile) ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n front tree from ordering") ;
ETree_writeForHumanEye(frontETree, msgFile) ;
fflush(msgFile) ;
}
/*--------------------------------------------------------------------*/
/*
-----------------------------------------------------
STEP 3: get the permutation, permute the matrix and
front tree and get the symbolic factorization
-----------------------------------------------------
*/
oldToNewIV = ETree_oldToNewVtxPerm(frontETree) ;
oldToNew = IV_entries(oldToNewIV) ;
newToOldIV = ETree_newToOldVtxPerm(frontETree) ;
newToOld = IV_entries(newToOldIV) ;
ETree_permuteVertices(frontETree, oldToNewIV) ;
InpMtx_permute(mtxA, oldToNew, oldToNew) ;
InpMtx_mapToUpperTriangle(mtxA) ;
InpMtx_changeCoordType(mtxA, INPMTX_BY_CHEVRONS) ;
InpMtx_changeStorageMode(mtxA, INPMTX_BY_VECTORS) ;
symbfacIVL = SymbFac_initFromInpMtx(frontETree, mtxA) ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n old-to-new permutation vector") ;
IV_writeForHumanEye(oldToNewIV, msgFile) ;
fprintf(msgFile, "\n\n new-to-old permutation vector") ;
IV_writeForHumanEye(newToOldIV, msgFile) ;
fprintf(msgFile, "\n\n front tree after permutation") ;
ETree_writeForHumanEye(frontETree, msgFile) ;
fprintf(msgFile, "\n\n input matrix after permutation") ;
InpMtx_writeForHumanEye(mtxA, msgFile) ;
fprintf(msgFile, "\n\n symbolic factorization") ;
IVL_writeForHumanEye(symbfacIVL, msgFile) ;
fflush(msgFile) ;
}
/*--------------------------------------------------------------------*/
/*
------------------------------------------
STEP 4: initialize the front matrix object
and the PatchAndGoInfo object to handle
small pivots
------------------------------------------
*/
frontmtx = FrontMtx_new() ;
mtxmanager = SubMtxManager_new() ;
SubMtxManager_init(mtxmanager, LOCK_IN_PROCESS, 0) ;
FrontMtx_init(frontmtx, frontETree, symbfacIVL, type, symmetryflag,
FRONTMTX_DENSE_FRONTS, SPOOLES_NO_PIVOTING, LOCK_IN_PROCESS,
0, NULL, mtxmanager, msglvl, msgFile) ;
if ( patchAndGoFlag == 1 ) {
frontmtx->patchinfo = PatchAndGoInfo_new() ;
PatchAndGoInfo_init(frontmtx->patchinfo, 1, toosmall, fudge,
storeids, storevalues) ;
} else if ( patchAndGoFlag == 2 ) {
frontmtx->patchinfo = PatchAndGoInfo_new() ;
PatchAndGoInfo_init(frontmtx->patchinfo, 2, toosmall, fudge,
storeids, storevalues) ;
}
/*--------------------------------------------------------------------*/
/*
------------------------------------------
STEP 5: setup the domain decomposition map
------------------------------------------
*/
if ( nthread > (nfront = FrontMtx_nfront(frontmtx)) ) {
nthread = nfront ;
}
cumopsDV = DV_new() ;
DV_init(cumopsDV, nthread, NULL) ;
ownersIV = ETree_ddMap(frontETree, type, symmetryflag,
cumopsDV, 1./(2.*nthread)) ;
DV_free(cumopsDV) ;
/*--------------------------------------------------------------------*/
/*
-----------------------------------------------------
STEP 6: compute the numeric factorization in parallel
-----------------------------------------------------
*/
chvmanager = ChvManager_new() ;
ChvManager_init(chvmanager, LOCK_IN_PROCESS, 1) ;
DVfill(10, cpus, 0.0) ;
IVfill(20, stats, 0) ;
lookahead = 0 ;
rootchv = FrontMtx_MT_factorInpMtx(frontmtx, mtxA, tau, 0.0,
chvmanager, ownersIV, lookahead,
&error, cpus, stats, msglvl, msgFile) ;
if ( patchAndGoFlag == 1 ) {
if ( frontmtx->patchinfo->fudgeIV != NULL ) {
fprintf(msgFile, "\n small pivots found at these locations") ;
IV_writeForHumanEye(frontmtx->patchinfo->fudgeIV, msgFile) ;
}
PatchAndGoInfo_free(frontmtx->patchinfo) ;
} else if ( patchAndGoFlag == 2 ) {
if ( frontmtx->patchinfo->fudgeIV != NULL ) {
fprintf(msgFile, "\n small pivots found at these locations") ;
IV_writeForHumanEye(frontmtx->patchinfo->fudgeIV, msgFile) ;
}
if ( frontmtx->patchinfo->fudgeDV != NULL ) {
fprintf(msgFile, "\n perturbations") ;
DV_writeForHumanEye(frontmtx->patchinfo->fudgeDV, msgFile) ;
}
PatchAndGoInfo_free(frontmtx->patchinfo) ;
}
ChvManager_free(chvmanager) ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n factor matrix") ;
FrontMtx_writeForHumanEye(frontmtx, msgFile) ;
fflush(msgFile) ;
}
if ( rootchv != NULL ) {
fprintf(msgFile, "\n\n matrix found to be singular\n") ;
spoolesFatal();
}
if ( error >= 0 ) {
fprintf(msgFile, "\n\n fatal error at front %d\n", error) ;
spoolesFatal();
}
/*
--------------------------------------
STEP 7: post-process the factorization
--------------------------------------
*/
FrontMtx_postProcess(frontmtx, msglvl, msgFile) ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n factor matrix after post-processing") ;
FrontMtx_writeForHumanEye(frontmtx, msgFile) ;
fflush(msgFile) ;
}
/*--------------------------------------------------------------------*/
/*
-----------------------------------------
STEP 8: read the right hand side matrix B
-----------------------------------------
*/
if ( (inputFile = fopen(rhsFileName, "r")) == NULL ) {
fprintf(stderr, "\n unable to open file %s", rhsFileName) ;
spoolesFatal();
}
fscanf(inputFile, "%d %d", &nrow, &nrhs) ;
mtxB = DenseMtx_new() ;
DenseMtx_init(mtxB, type, 0, 0, neqns, nrhs, 1, neqns) ;
DenseMtx_zero(mtxB) ;
if ( type == SPOOLES_REAL ) {
for ( irow = 0 ; irow < nrow ; irow++ ) {
fscanf(inputFile, "%d", &jrow) ;
for ( jrhs = 0 ; jrhs < nrhs ; jrhs++ ) {
fscanf(inputFile, "%le", &value) ;
DenseMtx_setRealEntry(mtxB, jrow, jrhs, value) ;
}
}
} else {
for ( irow = 0 ; irow < nrow ; irow++ ) {
fscanf(inputFile, "%d", &jrow) ;
for ( jrhs = 0 ; jrhs < nrhs ; jrhs++ ) {
fscanf(inputFile, "%le %le", &real, &imag) ;
DenseMtx_setComplexEntry(mtxB, jrow, jrhs, real, imag) ;
}
}
}
fclose(inputFile) ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n rhs matrix in original ordering") ;
DenseMtx_writeForHumanEye(mtxB, msgFile) ;
fflush(msgFile) ;
}
/*--------------------------------------------------------------------*/
/*
--------------------------------------------------------------
STEP 9: permute the right hand side into the original ordering
--------------------------------------------------------------
*/
DenseMtx_permuteRows(mtxB, oldToNewIV) ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n right hand side matrix in new ordering") ;
DenseMtx_writeForHumanEye(mtxB, msgFile) ;
fflush(msgFile) ;
}
/*--------------------------------------------------------------------*/
/*
--------------------------------------------------------
STEP 10: get the solve map object for the parallel solve
--------------------------------------------------------
*/
solvemap = SolveMap_new() ;
SolveMap_ddMap(solvemap, type, FrontMtx_upperBlockIVL(frontmtx),
FrontMtx_lowerBlockIVL(frontmtx), nthread, ownersIV,
FrontMtx_frontTree(frontmtx), seed, msglvl, msgFile) ;
/*--------------------------------------------------------------------*/
/*
--------------------------------------------
STEP 11: solve the linear system in parallel
--------------------------------------------
*/
mtxX = DenseMtx_new() ;
DenseMtx_init(mtxX, type, 0, 0, neqns, nrhs, 1, neqns) ;
DenseMtx_zero(mtxX) ;
FrontMtx_MT_solve(frontmtx, mtxX, mtxB, mtxmanager, solvemap,
cpus, msglvl, msgFile) ;
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n solution matrix in new ordering") ;
DenseMtx_writeForHumanEye(mtxX, msgFile) ;
fflush(msgFile) ;
}
/*--------------------------------------------------------------------*/
/*
--------------------------------------------------------
STEP 12: permute the solution into the original ordering
--------------------------------------------------------
*/
DenseMtx_permuteRows(mtxX, newToOldIV) ;
if ( msglvl > 0 ) {
fprintf(msgFile, "\n\n solution matrix in original ordering") ;
DenseMtx_writeForHumanEye(mtxX, msgFile) ;
fflush(msgFile) ;
}
/*--------------------------------------------------------------------*/
/*
-----------
free memory
-----------
*/
FrontMtx_free(frontmtx) ;
DenseMtx_free(mtxX) ;
DenseMtx_free(mtxB) ;
IV_free(newToOldIV) ;
IV_free(oldToNewIV) ;
InpMtx_free(mtxA) ;
ETree_free(frontETree) ;
IVL_free(symbfacIVL) ;
SubMtxManager_free(mtxmanager) ;
Graph_free(graph) ;
SolveMap_free(solvemap) ;
IV_free(ownersIV) ;
/*--------------------------------------------------------------------*/
return(1) ; }
/*
------------------------------------------------
purpose -- to get simple x[] and y[] coordinates
for the tree vertices
return values --
1 -- normal return
-1 -- tree is NULL
-2 -- heightflag is invalid
-3 -- coordflag is invalid
-4 -- xDV is NULL
-5 -- yDV is NULL
created -- 99jan07, cca
------------------------------------------------
*/
int
Tree_getSimpleCoords (
Tree *tree,
char heightflag,
char coordflag,
DV *xDV,
DV *yDV
) {
double *x, *y ;
int count, I, J, n, nleaves ;
int *fch, *par, *sib ;
/*
---------------
check the input
---------------
*/
if ( tree == NULL ) {
fprintf(stderr, "\n error in Tree_getSimpleCoords()"
"\n tree is NULL\n") ;
return(-1) ;
}
if ( heightflag != 'D' && heightflag != 'H' ) {
fprintf(stderr, "\n error in Tree_getSimpleCoords()"
"\n invalid heightflag = %c\n", heightflag) ;
return(-2) ;
}
if ( coordflag != 'C' && coordflag != 'P' ) {
fprintf(stderr, "\n error in Tree_getSimpleCoords()"
"\n invalid coordflag = %c\n", coordflag) ;
return(-3) ;
}
if ( xDV == NULL ) {
fprintf(stderr, "\n error in Tree_getSimpleCoords()"
"\n xDV is NULL\n") ;
return(-4) ;
}
if ( yDV == NULL ) {
fprintf(stderr, "\n error in Tree_getSimpleCoords()"
"\n yDV is NULL\n") ;
return(-5) ;
}
n = tree->n ;
par = tree->par ;
fch = tree->fch ;
sib = tree->sib ;
DV_setSize(xDV, n) ;
DV_setSize(yDV, n) ;
x = DV_entries(xDV) ;
y = DV_entries(yDV) ;
switch ( heightflag ) {
case 'D' : {
int J, K, maxdepth ;
for ( J = Tree_preOTfirst(tree), maxdepth = 0 ;
J != -1 ;
J = Tree_preOTnext(tree, J) ) {
if ( (K = par[J]) == -1 ) {
y[J] = 0.0 ;
} else {
y[J] = y[K] + 1.0 ;
}
if ( maxdepth < y[J] ) {
maxdepth = y[J] ;
}
}
if ( coordflag == 'C' ) {
for ( J = 0 ; J < n ; J++ ) {
y[J] = maxdepth - y[J] ;
}
}
} break ;
case 'H' : {
int height, I, J, maxheight ;
for ( J = Tree_postOTfirst(tree), maxheight = 0 ;
J != -1 ;
J = Tree_postOTnext(tree, J) ) {
if ( (I = fch[J]) == -1 ) {
y[J] = 0.0 ;
} else {
height = y[I] ;
for ( I = sib[I] ; I != -1 ; I = sib[I] ) {
if ( height < y[I] ) {
height = y[I] ;
}
}
y[J] = height + 1.0 ;
}
if ( maxheight < y[J] ) {
maxheight = y[J] ;
}
}
if ( coordflag == 'P' ) {
for ( J = 0 ; J < n ; J++ ) {
y[J] = maxheight - y[J] ;
}
}
} break ;
default :
break ;
}
#if MYDEBUG > 0
fprintf(stdout, "\n\n y") ;
DV_writeForHumanEye(yDV, stdout) ;
#endif
DV_zero(xDV) ;
nleaves = 0 ;
for ( J = Tree_postOTfirst(tree) ;
J != -1 ;
J = Tree_postOTnext(tree, J) ) {
if ( fch[J] == -1 ) {
x[J] = nleaves++ ;
} else {
for ( I = fch[J], count = 0 ; I != -1 ; I = sib[I] ) {
x[J] += x[I] ;
count++ ;
}
x[J] /= count ;
}
}
if ( coordflag == 'C' ) {
for ( J = 0 ; J < n ; J++ ) {
x[J] = x[J] / nleaves ;
}
} else {
double r, theta ;
for ( J = 0 ; J < n ; J++ ) {
theta = 6.283185 * x[J] / nleaves ;
r = y[J] ;
x[J] = r * cos(theta) ;
y[J] = r * sin(theta) ;
}
}
#if MYDEBUG > 0
fprintf(stdout, "\n\n x") ;
DV_writeForHumanEye(xDV, stdout) ;
#endif
return(1) ; }
/*--------------------------------------------------------------------*/
int
main ( int argc, char *argv[] )
/*
----------------------------------------
draw the tree
created -- 99jan23, cca
----------------------------------------
*/
{
char coordflag, heightflag ;
char *inTagsFileName, *inTreeFileName, *outEPSfileName ;
double fontsize, radius, t1, t2 ;
double bbox[4], frame[4] ;
DV *xDV, *yDV ;
int ierr, msglvl, rc, tagsflag ;
IV *tagsIV ;
Tree *tree ;
FILE *msgFile ;
if ( argc != 19 ) {
fprintf(stdout,
"\n\n usage : %s msglvl msgFile inTreeFile inTagsFile outEPSfile "
"\n heightflag coordflag radius bbox[4] frame[4] tagflag fontsize"
"\n msglvl -- message level"
"\n msgFile -- message file"
"\n inTreeFile -- input file, must be *.treef or *.treeb"
"\n inTagsFile -- input file, must be *.ivf or *.ivb or none"
"\n outEPSfile -- output file"
"\n heightflag -- height flag"
"\n 'D' -- use depth metric"
"\n 'H' -- use height metric"
"\n coordflag -- coordinate flag"
"\n 'C' -- use (x,y) Cartesian coordinates"
"\n 'P' -- use (r,theta) polar coordinates"
"\n radius -- radius of node"
"\n bbox[4] -- bounding box"
"\n frame[4] -- frame for plot"
"\n fontsize -- size of fonts (in points)"
"\n tagflag -- if 1, draw labels, otherwise, do not"
"\n", argv[0]) ;
return(0) ;
}
msglvl = atoi(argv[1]) ;
if ( strcmp(argv[2], "stdout") == 0 ) {
msgFile = stdout ;
} else if ( (msgFile = fopen(argv[2], "a")) == NULL ) {
fprintf(stderr, "\n fatal error in %s"
"\n unable to open file %s\n",
argv[0], argv[2]) ;
return(-1) ;
}
inTreeFileName = argv[3] ;
inTagsFileName = argv[4] ;
outEPSfileName = argv[5] ;
heightflag = argv[6][0] ;
coordflag = argv[7][0] ;
radius = atof(argv[8]) ;
bbox[0] = atof(argv[9]) ;
bbox[1] = atof(argv[10]) ;
bbox[2] = atof(argv[11]) ;
bbox[3] = atof(argv[12]) ;
frame[0] = atof(argv[13]) ;
frame[1] = atof(argv[14]) ;
frame[2] = atof(argv[15]) ;
frame[3] = atof(argv[16]) ;
fontsize = atof(argv[17]) ;
tagsflag = atoi(argv[18]) ;
fprintf(msgFile,
"\n %s "
"\n msglvl -- %d"
"\n msgFile -- %s"
"\n inTreeFile -- %s"
"\n inTagsFile -- %s"
"\n outEPSfile -- %s"
"\n heightflag -- %c"
"\n coordflag -- %d"
"\n radius -- %.3g"
"\n bbox -- %.3g %.3g %.3g %.3g"
"\n frame -- %.3g %.3g %.3g %.3g"
"\n fontsize -- %.3g"
"\n",
argv[0], msglvl, argv[2], inTreeFileName, inTagsFileName,
outEPSfileName, heightflag, coordflag, radius,
bbox[0], bbox[1], bbox[2], bbox[3],
frame[0], frame[1], frame[2], frame[3], fontsize, tagsflag) ;
fflush(msgFile) ;
/*
------------------------
read in the Tree object
------------------------
*/
if ( strcmp(inTreeFileName, "none") == 0 ) {
fprintf(msgFile, "\n no file to read from") ;
exit(0) ;
}
tree = Tree_new() ;
MARKTIME(t1) ;
rc = Tree_readFromFile(tree, inTreeFileName) ;
/*
Tree_setFchSibRoot(tree) ;
*/
Tree_leftJustify(tree) ;
MARKTIME(t2) ;
fprintf(msgFile, "\n CPU %9.5f : read in tree from file %s",
t2 - t1, inTreeFileName) ;
if ( rc != 1 ) {
fprintf(msgFile, "\n return value %d from Tree_readFromFile(%p,%s)",
rc, tree, inTreeFileName) ;
exit(-1) ;
}
fprintf(msgFile, "\n\n after reading Tree object from file %s",
inTreeFileName) ;
if ( msglvl > 2 ) {
Tree_writeForHumanEye(tree, msgFile) ;
} else {
Tree_writeStats(tree, msgFile) ;
}
fflush(msgFile) ;
if ( Tree_maxNchild(tree) > 2 ) {
fprintf(msgFile, "\n\n maximum number of children = %d",
Tree_maxNchild(tree)) ;
}
if ( strcmp(inTagsFileName, "none") != 0 ) {
/*
--------------------------
read in the tags IV object
--------------------------
*/
tagsIV = IV_new() ;
MARKTIME(t1) ;
rc = IV_readFromFile(tagsIV, inTagsFileName) ;
MARKTIME(t2) ;
fprintf(msgFile, "\n CPU %9.5f : read in tagsIV from file %s",
t2 - t1, inTagsFileName) ;
if ( rc != 1 ) {
fprintf(msgFile, "\n return value %d from IV_readFromFile(%p,%s)",
rc, tagsIV, inTagsFileName) ;
exit(-1) ;
}
fprintf(msgFile, "\n\n after reading IV object from file %s",
inTagsFileName) ;
if ( msglvl > 2 ) {
IV_writeForHumanEye(tagsIV, msgFile) ;
} else {
IV_writeStats(tagsIV, msgFile) ;
}
fflush(msgFile) ;
if ( IV_size(tagsIV) != tree->n ) {
fprintf(stderr,
"\n fatal error, IV_size(tagsIV) = %d, tree->n = %d",
IV_size(tagsIV), tree->n) ;
exit(-1) ;
}
} else {
tagsIV = NULL ;
}
/*
-------------------------------
get the coordinates of the tree
-------------------------------
*/
xDV = DV_new() ;
yDV = DV_new() ;
rc = Tree_getSimpleCoords(tree, heightflag, coordflag, xDV, yDV) ;
if ( rc != 1 ) {
fprintf(stderr, "\n error return %d from Tree_getSimpleCoords()",rc);
exit(-1) ;
}
if ( msglvl > 1 ) {
fprintf(msgFile, "\n\n x-coordinates") ;
DV_writeForHumanEye(xDV, msgFile) ;
fprintf(msgFile, "\n\n y-coordinates") ;
DV_writeForHumanEye(yDV, msgFile) ;
fflush(msgFile) ;
}
/*
-------------
draw the Tree
-------------
*/
rc = Tree_drawToEPS(tree, outEPSfileName, xDV, yDV, radius, NULL,
tagsflag, fontsize, tagsIV, bbox, frame, NULL) ;
if ( rc != 1 ) {
fprintf(stderr, "\n error return %d from Tree_drawToEPSfile()", rc) ;
exit(-1) ;
}
/*
---------------------
free the Tree object
---------------------
*/
Tree_free(tree) ;
if ( tagsIV != NULL ) {
IV_free(tagsIV) ;
}
fprintf(msgFile, "\n") ;
fclose(msgFile) ;
return(1) ; }
//static void factor_MT(struct factorinfo *pfi, InpMtx *mtxA, int size, FILE *msgFile, int symmetryflag)
void factor_MT(struct factorinfo *pfi, InpMtx *mtxA, int size, FILE *msgFile, int symmetryflag)
{
Graph *graph;
IV *ownersIV;
IVL *symbfacIVL;
Chv *rootchv;
/* Initialize pfi: */
pfi->size = size;
pfi->msgFile = msgFile;
DVfill(10, pfi->cpus, 0.0);
/*
* STEP 1 : find a low-fill ordering
* (1) create the Graph object
*/
ssolve_creategraph(&graph, &pfi->frontETree, mtxA, size, msgFile);
/*
* STEP 2: get the permutation, permute the matrix and
* front tree and get the symbolic factorization
*/
ssolve_permuteA(&pfi->oldToNewIV, &pfi->newToOldIV, &symbfacIVL, pfi->frontETree,
mtxA, msgFile, symmetryflag);
/*
* STEP 3: Prepare distribution to multiple threads/cpus
*/
{
DV *cumopsDV;
int nfront;
nfront = ETree_nfront(pfi->frontETree);
pfi->nthread = num_cpus;
if (pfi->nthread > nfront)
pfi->nthread = nfront;
cumopsDV = DV_new();
DV_init(cumopsDV, pfi->nthread, NULL);
ownersIV = ETree_ddMap(pfi->frontETree, SPOOLES_REAL, symmetryflag,
cumopsDV, 1. / (2. * pfi->nthread));
if (DEBUG_LVL > 1) {
fprintf(msgFile,
"\n\n map from fronts to threads");
IV_writeForHumanEye(ownersIV, msgFile);
fprintf(msgFile,
"\n\n factor operations for each front");
DV_writeForHumanEye(cumopsDV, msgFile);
fflush(msgFile);
} else {
fprintf(msgFile, "\n\n Using %d threads\n",
pfi->nthread);
}
DV_free(cumopsDV);
}
/*
* STEP 4: initialize the front matrix object
*/
{
pfi->frontmtx = FrontMtx_new();
pfi->mtxmanager = SubMtxManager_new();
SubMtxManager_init(pfi->mtxmanager, LOCK_IN_PROCESS, 0);
FrontMtx_init(pfi->frontmtx, pfi->frontETree, symbfacIVL, SPOOLES_REAL,
symmetryflag, FRONTMTX_DENSE_FRONTS,
SPOOLES_PIVOTING, LOCK_IN_PROCESS, 0, NULL,
pfi->mtxmanager, DEBUG_LVL, pfi->msgFile);
}
/*
* STEP 5: compute the numeric factorization in parallel
*/
{
ChvManager *chvmanager;
int stats[20];
int error;
chvmanager = ChvManager_new();
ChvManager_init(chvmanager, LOCK_IN_PROCESS, 1);
IVfill(20, stats, 0);
rootchv = FrontMtx_MT_factorInpMtx(pfi->frontmtx, mtxA, MAGIC_TAU, MAGIC_DTOL,
chvmanager, ownersIV, 0,
&error, pfi->cpus, stats, DEBUG_LVL,
pfi->msgFile);
ChvManager_free(chvmanager);
if (DEBUG_LVL > 1) {
fprintf(msgFile, "\n\n factor matrix");
FrontMtx_writeForHumanEye(pfi->frontmtx, pfi->msgFile);
fflush(pfi->msgFile);
}
if (rootchv != NULL) {
fprintf(pfi->msgFile, "\n\n matrix found to be singular\n");
exit(-1);
}
if (error >= 0) {
fprintf(pfi->msgFile, "\n\n fatal error at front %d", error);
exit(-1);
}
}
/*
* STEP 6: post-process the factorization
*/
ssolve_postfactor(pfi->frontmtx, pfi->msgFile);
/*
* STEP 7: get the solve map object for the parallel solve
*/
{
pfi->solvemap = SolveMap_new();
SolveMap_ddMap(pfi->solvemap, symmetryflag,
FrontMtx_upperBlockIVL(pfi->frontmtx),
FrontMtx_lowerBlockIVL(pfi->frontmtx), pfi->nthread, ownersIV,
FrontMtx_frontTree(pfi->frontmtx), RNDSEED, DEBUG_LVL,
pfi->msgFile);
}
/* cleanup: */
InpMtx_free(mtxA);
IVL_free(symbfacIVL);
Graph_free(graph);
IV_free(ownersIV);
}
