static PetscErrorCode PCBDDCScalingSetUp_Deluxe_Private(PC pc)
{
PC_BDDC *pcbddc=(PC_BDDC*)pc->data;
PCBDDCDeluxeScaling deluxe_ctx=pcbddc->deluxe_ctx;
PCBDDCSubSchurs sub_schurs = pcbddc->sub_schurs;
PetscScalar *matdata,*matdata2;
PetscInt i,max_subset_size,cum,cum2;
const PetscInt *idxs;
PetscBool newsetup = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!sub_schurs) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_PLIB,"Missing PCBDDCSubSchurs");
if (!sub_schurs->n_subs) PetscFunctionReturn(0);
/* Allocate arrays for subproblems */
if (!deluxe_ctx->seq_n) {
deluxe_ctx->seq_n = sub_schurs->n_subs;
ierr = PetscCalloc5(deluxe_ctx->seq_n,&deluxe_ctx->seq_scctx,deluxe_ctx->seq_n,&deluxe_ctx->seq_work1,deluxe_ctx->seq_n,&deluxe_ctx->seq_work2,deluxe_ctx->seq_n,&deluxe_ctx->seq_mat,deluxe_ctx->seq_n,&deluxe_ctx->seq_mat_inv_sum);CHKERRQ(ierr);
newsetup = PETSC_TRUE;
} else if (deluxe_ctx->seq_n != sub_schurs->n_subs) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Number of deluxe subproblems %D is different from the sub_schurs %D",deluxe_ctx->seq_n,sub_schurs->n_subs);
/* the change of basis is just a reference to sub_schurs->change (if any) */
deluxe_ctx->change = sub_schurs->change;
deluxe_ctx->change_with_qr = sub_schurs->change_with_qr;
/* Create objects for deluxe */
max_subset_size = 0;
for (i=0;i<sub_schurs->n_subs;i++) {
PetscInt subset_size;
ierr = ISGetLocalSize(sub_schurs->is_subs[i],&subset_size);CHKERRQ(ierr);
max_subset_size = PetscMax(subset_size,max_subset_size);
}
if (newsetup) {
ierr = PetscMalloc1(2*max_subset_size,&deluxe_ctx->workspace);CHKERRQ(ierr);
}
cum = cum2 = 0;
ierr = ISGetIndices(sub_schurs->is_Ej_all,&idxs);CHKERRQ(ierr);
ierr = MatSeqAIJGetArray(sub_schurs->S_Ej_all,&matdata);CHKERRQ(ierr);
ierr = MatSeqAIJGetArray(sub_schurs->sum_S_Ej_all,&matdata2);CHKERRQ(ierr);
for (i=0;i<deluxe_ctx->seq_n;i++) {
PetscInt subset_size;
ierr = ISGetLocalSize(sub_schurs->is_subs[i],&subset_size);CHKERRQ(ierr);
if (newsetup) {
IS sub;
/* work vectors */
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,subset_size,deluxe_ctx->workspace,&deluxe_ctx->seq_work1[i]);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,subset_size,deluxe_ctx->workspace+subset_size,&deluxe_ctx->seq_work2[i]);CHKERRQ(ierr);
/* scatters */
ierr = ISCreateGeneral(PETSC_COMM_SELF,subset_size,idxs+cum,PETSC_COPY_VALUES,&sub);CHKERRQ(ierr);
ierr = VecScatterCreate(pcbddc->work_scaling,sub,deluxe_ctx->seq_work1[i],NULL,&deluxe_ctx->seq_scctx[i]);CHKERRQ(ierr);
ierr = ISDestroy(&sub);CHKERRQ(ierr);
}
/* S_E_j */
ierr = MatDestroy(&deluxe_ctx->seq_mat[i]);CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF,subset_size,subset_size,matdata+cum2,&deluxe_ctx->seq_mat[i]);CHKERRQ(ierr);
/* \sum_k S^k_E_j */
ierr = MatDestroy(&deluxe_ctx->seq_mat_inv_sum[i]);CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF,subset_size,subset_size,matdata2+cum2,&deluxe_ctx->seq_mat_inv_sum[i]);CHKERRQ(ierr);
ierr = MatSetOption(deluxe_ctx->seq_mat_inv_sum[i],MAT_SPD,sub_schurs->is_posdef);CHKERRQ(ierr);
ierr = MatSetOption(deluxe_ctx->seq_mat_inv_sum[i],MAT_HERMITIAN,sub_schurs->is_hermitian);CHKERRQ(ierr);
if (sub_schurs->is_hermitian) {
ierr = MatCholeskyFactor(deluxe_ctx->seq_mat_inv_sum[i],NULL,NULL);CHKERRQ(ierr);
} else {
ierr = MatLUFactor(deluxe_ctx->seq_mat_inv_sum[i],NULL,NULL,NULL);CHKERRQ(ierr);
}
if (pcbddc->deluxe_singlemat) {
Mat X,Y;
if (!sub_schurs->is_hermitian) {
ierr = MatTranspose(deluxe_ctx->seq_mat[i],MAT_INITIAL_MATRIX,&X);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)deluxe_ctx->seq_mat[i]);CHKERRQ(ierr);
X = deluxe_ctx->seq_mat[i];
}
ierr = MatDuplicate(X,MAT_DO_NOT_COPY_VALUES,&Y);CHKERRQ(ierr);
if (!sub_schurs->is_hermitian) {
ierr = PCBDDCMatTransposeMatSolve_SeqDense(deluxe_ctx->seq_mat_inv_sum[i],X,Y);CHKERRQ(ierr);
} else {
ierr = MatMatSolve(deluxe_ctx->seq_mat_inv_sum[i],X,Y);CHKERRQ(ierr);
}
ierr = MatDestroy(&deluxe_ctx->seq_mat_inv_sum[i]);CHKERRQ(ierr);
ierr = MatDestroy(&deluxe_ctx->seq_mat[i]);CHKERRQ(ierr);
ierr = MatDestroy(&X);CHKERRQ(ierr);
if (deluxe_ctx->change) {
Mat C,CY;
if (!deluxe_ctx->change_with_qr) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Only QR based change of basis");
ierr = KSPGetOperators(deluxe_ctx->change[i],&C,NULL);CHKERRQ(ierr);
ierr = MatMatMult(C,Y,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&CY);CHKERRQ(ierr);
ierr = MatMatTransposeMult(CY,C,MAT_REUSE_MATRIX,PETSC_DEFAULT,&Y);CHKERRQ(ierr);
ierr = MatDestroy(&CY);CHKERRQ(ierr);
}
ierr = MatTranspose(Y,MAT_INPLACE_MATRIX,&Y);CHKERRQ(ierr);
deluxe_ctx->seq_mat[i] = Y;
}
cum += subset_size;
cum2 += subset_size*subset_size;
}
ierr = ISRestoreIndices(sub_schurs->is_Ej_all,&idxs);CHKERRQ(ierr);
ierr = MatSeqAIJRestoreArray(sub_schurs->S_Ej_all,&matdata);CHKERRQ(ierr);
ierr = MatSeqAIJRestoreArray(sub_schurs->sum_S_Ej_all,&matdata2);CHKERRQ(ierr);
if (pcbddc->deluxe_singlemat) {
deluxe_ctx->change = NULL;
deluxe_ctx->change_with_qr = PETSC_FALSE;
}
if (deluxe_ctx->change && !deluxe_ctx->change_with_qr) {
for (i=0;i<deluxe_ctx->seq_n;i++) {
if (newsetup) {
PC pc;
ierr = KSPGetPC(deluxe_ctx->change[i],&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCLU);CHKERRQ(ierr);
ierr = KSPSetFromOptions(deluxe_ctx->change[i]);CHKERRQ(ierr);
}
ierr = KSPSetUp(deluxe_ctx->change[i]);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
static PetscErrorCode PCSetUp_Cholesky(PC pc)
{
PetscErrorCode ierr;
PetscBool flg;
PC_Cholesky *dir = (PC_Cholesky*)pc->data;
const MatSolverPackage stype;
MatFactorError err;
PetscFunctionBegin;
pc->failedreason = PC_NOERROR;
if (dir->hdr.reusefill && pc->setupcalled) ((PC_Factor*)dir)->info.fill = dir->hdr.actualfill;
ierr = MatSetErrorIfFailure(pc->pmat,pc->erroriffailure);
CHKERRQ(ierr);
if (dir->hdr.inplace) {
if (dir->row && dir->col && (dir->row != dir->col)) {
ierr = ISDestroy(&dir->row);
CHKERRQ(ierr);
}
ierr = ISDestroy(&dir->col);
CHKERRQ(ierr);
ierr = MatGetOrdering(pc->pmat,((PC_Factor*)dir)->ordering,&dir->row,&dir->col);
CHKERRQ(ierr);
if (dir->col && (dir->row != dir->col)) { /* only use row ordering for SBAIJ */
ierr = ISDestroy(&dir->col);
CHKERRQ(ierr);
}
if (dir->row) {
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)dir->row);
CHKERRQ(ierr);
}
ierr = MatCholeskyFactor(pc->pmat,dir->row,&((PC_Factor*)dir)->info);
CHKERRQ(ierr);
ierr = MatFactorGetError(pc->pmat,&err);
CHKERRQ(ierr);
if (err) { /* Factor() fails */
pc->failedreason = (PCFailedReason)err;
PetscFunctionReturn(0);
}
((PC_Factor*)dir)->fact = pc->pmat;
} else {
MatInfo info;
if (!pc->setupcalled) {
ierr = MatGetOrdering(pc->pmat,((PC_Factor*)dir)->ordering,&dir->row,&dir->col);
CHKERRQ(ierr);
/* check if dir->row == dir->col */
ierr = ISEqual(dir->row,dir->col,&flg);
CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"row and column permutations must equal");
ierr = ISDestroy(&dir->col);
CHKERRQ(ierr); /* only pass one ordering into CholeskyFactor */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_factor_nonzeros_along_diagonal",&flg,NULL);
CHKERRQ(ierr);
if (flg) {
PetscReal tol = 1.e-10;
ierr = PetscOptionsGetReal(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_factor_nonzeros_along_diagonal",&tol,NULL);
CHKERRQ(ierr);
ierr = MatReorderForNonzeroDiagonal(pc->pmat,tol,dir->row,dir->row);
CHKERRQ(ierr);
}
if (dir->row) {
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)dir->row);
CHKERRQ(ierr);
}
if (!((PC_Factor*)dir)->fact) {
ierr = MatGetFactor(pc->pmat,((PC_Factor*)dir)->solvertype,MAT_FACTOR_CHOLESKY,&((PC_Factor*)dir)->fact);
CHKERRQ(ierr);
}
ierr = MatCholeskyFactorSymbolic(((PC_Factor*)dir)->fact,pc->pmat,dir->row,&((PC_Factor*)dir)->info);
CHKERRQ(ierr);
ierr = MatGetInfo(((PC_Factor*)dir)->fact,MAT_LOCAL,&info);
CHKERRQ(ierr);
dir->hdr.actualfill = info.fill_ratio_needed;
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)((PC_Factor*)dir)->fact);
CHKERRQ(ierr);
} else if (pc->flag != SAME_NONZERO_PATTERN) {
if (!dir->hdr.reuseordering) {
ierr = ISDestroy(&dir->row);
CHKERRQ(ierr);
ierr = MatGetOrdering(pc->pmat,((PC_Factor*)dir)->ordering,&dir->row,&dir->col);
CHKERRQ(ierr);
ierr = ISDestroy(&dir->col);
CHKERRQ(ierr); /* only use dir->row ordering in CholeskyFactor */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_factor_nonzeros_along_diagonal",&flg,NULL);
CHKERRQ(ierr);
if (flg) {
PetscReal tol = 1.e-10;
ierr = PetscOptionsGetReal(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_factor_nonzeros_along_diagonal",&tol,NULL);
CHKERRQ(ierr);
ierr = MatReorderForNonzeroDiagonal(pc->pmat,tol,dir->row,dir->row);
CHKERRQ(ierr);
}
if (dir->row) {
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)dir->row);
CHKERRQ(ierr);
}
}
ierr = MatDestroy(&((PC_Factor*)dir)->fact);
CHKERRQ(ierr);
ierr = MatGetFactor(pc->pmat,((PC_Factor*)dir)->solvertype,MAT_FACTOR_CHOLESKY,&((PC_Factor*)dir)->fact);
CHKERRQ(ierr);
ierr = MatCholeskyFactorSymbolic(((PC_Factor*)dir)->fact,pc->pmat,dir->row,&((PC_Factor*)dir)->info);
CHKERRQ(ierr);
ierr = MatGetInfo(((PC_Factor*)dir)->fact,MAT_LOCAL,&info);
CHKERRQ(ierr);
dir->hdr.actualfill = info.fill_ratio_needed;
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)((PC_Factor*)dir)->fact);
CHKERRQ(ierr);
} else {
ierr = MatFactorGetError(((PC_Factor*)dir)->fact,&err);
CHKERRQ(ierr);
if (err == MAT_FACTOR_NUMERIC_ZEROPIVOT) {
ierr = MatFactorClearError(((PC_Factor*)dir)->fact);
CHKERRQ(ierr);
pc->failedreason = PC_NOERROR;
}
}
ierr = MatFactorGetError(((PC_Factor*)dir)->fact,&err);
CHKERRQ(ierr);
if (err) { /* FactorSymbolic() fails */
pc->failedreason = (PCFailedReason)err;
PetscFunctionReturn(0);
}
ierr = MatCholeskyFactorNumeric(((PC_Factor*)dir)->fact,pc->pmat,&((PC_Factor*)dir)->info);
CHKERRQ(ierr);
ierr = MatFactorGetError(((PC_Factor*)dir)->fact,&err);
CHKERRQ(ierr);
if (err) { /* FactorNumeric() fails */
pc->failedreason = (PCFailedReason)err;
}
}
ierr = PCFactorGetMatSolverPackage(pc,&stype);
CHKERRQ(ierr);
if (!stype) {
const MatSolverPackage solverpackage;
ierr = MatFactorGetSolverPackage(((PC_Factor*)dir)->fact,&solverpackage);
CHKERRQ(ierr);
ierr = PCFactorSetMatSolverPackage(pc,solverpackage);
CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode PCSetUp_Cholesky(PC pc)
{
PetscErrorCode ierr;
PetscBool flg;
PC_Cholesky *dir = (PC_Cholesky*)pc->data;
PetscFunctionBegin;
if (dir->reusefill && pc->setupcalled) ((PC_Factor*)dir)->info.fill = dir->actualfill;
if (dir->inplace) {
if (dir->row && dir->col && (dir->row != dir->col)) {
ierr = ISDestroy(&dir->row);CHKERRQ(ierr);
}
ierr = ISDestroy(&dir->col);CHKERRQ(ierr);
ierr = MatGetOrdering(pc->pmat,((PC_Factor*)dir)->ordering,&dir->row,&dir->col);CHKERRQ(ierr);
if (dir->col && (dir->row != dir->col)) { /* only use row ordering for SBAIJ */
ierr = ISDestroy(&dir->col);CHKERRQ(ierr);
}
if (dir->row) {ierr = PetscLogObjectParent(pc,dir->row);CHKERRQ(ierr);}
ierr = MatCholeskyFactor(pc->pmat,dir->row,&((PC_Factor*)dir)->info);CHKERRQ(ierr);
((PC_Factor*)dir)->fact = pc->pmat;
} else {
MatInfo info;
if (!pc->setupcalled) {
ierr = MatGetOrdering(pc->pmat,((PC_Factor*)dir)->ordering,&dir->row,&dir->col);CHKERRQ(ierr);
/* check if dir->row == dir->col */
ierr = ISEqual(dir->row,dir->col,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"row and column permutations must equal");
ierr = ISDestroy(&dir->col);CHKERRQ(ierr); /* only pass one ordering into CholeskyFactor */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_factor_nonzeros_along_diagonal",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscReal tol = 1.e-10;
ierr = PetscOptionsGetReal(((PetscObject)pc)->prefix,"-pc_factor_nonzeros_along_diagonal",&tol,PETSC_NULL);CHKERRQ(ierr);
ierr = MatReorderForNonzeroDiagonal(pc->pmat,tol,dir->row,dir->row);CHKERRQ(ierr);
}
if (dir->row) {ierr = PetscLogObjectParent(pc,dir->row);CHKERRQ(ierr);}
if (!((PC_Factor*)dir)->fact){
ierr = MatGetFactor(pc->pmat,((PC_Factor*)dir)->solvertype,MAT_FACTOR_CHOLESKY,&((PC_Factor*)dir)->fact);CHKERRQ(ierr);
}
ierr = MatCholeskyFactorSymbolic(((PC_Factor*)dir)->fact,pc->pmat,dir->row,&((PC_Factor*)dir)->info);CHKERRQ(ierr);
ierr = MatGetInfo(((PC_Factor*)dir)->fact,MAT_LOCAL,&info);CHKERRQ(ierr);
dir->actualfill = info.fill_ratio_needed;
ierr = PetscLogObjectParent(pc,((PC_Factor*)dir)->fact);CHKERRQ(ierr);
} else if (pc->flag != SAME_NONZERO_PATTERN) {
if (!dir->reuseordering) {
ierr = ISDestroy(&dir->row);CHKERRQ(ierr);
ierr = MatGetOrdering(pc->pmat,((PC_Factor*)dir)->ordering,&dir->row,&dir->col);CHKERRQ(ierr);
ierr = ISDestroy(&dir->col);CHKERRQ(ierr); /* only use dir->row ordering in CholeskyFactor */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_factor_nonzeros_along_diagonal",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscReal tol = 1.e-10;
ierr = PetscOptionsGetReal(((PetscObject)pc)->prefix,"-pc_factor_nonzeros_along_diagonal",&tol,PETSC_NULL);CHKERRQ(ierr);
ierr = MatReorderForNonzeroDiagonal(pc->pmat,tol,dir->row,dir->row);CHKERRQ(ierr);
}
if (dir->row) {ierr = PetscLogObjectParent(pc,dir->row);CHKERRQ(ierr);}
}
ierr = MatDestroy(&((PC_Factor*)dir)->fact);CHKERRQ(ierr);
ierr = MatGetFactor(pc->pmat,((PC_Factor*)dir)->solvertype,MAT_FACTOR_CHOLESKY,&((PC_Factor*)dir)->fact);CHKERRQ(ierr);
ierr = MatCholeskyFactorSymbolic(((PC_Factor*)dir)->fact,pc->pmat,dir->row,&((PC_Factor*)dir)->info);CHKERRQ(ierr);
ierr = MatGetInfo(((PC_Factor*)dir)->fact,MAT_LOCAL,&info);CHKERRQ(ierr);
dir->actualfill = info.fill_ratio_needed;
ierr = PetscLogObjectParent(pc,((PC_Factor*)dir)->fact);CHKERRQ(ierr);
}
ierr = MatCholeskyFactorNumeric(((PC_Factor*)dir)->fact,pc->pmat,&((PC_Factor*)dir)->info);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
Mat A,F,B,X,C,Aher,G;
Vec b,x,c,d,e;
PetscErrorCode ierr;
PetscInt m = 5,n,p,i,j,nrows,ncols;
PetscScalar *v,*barray,rval;
PetscReal norm,tol=1.e-12;
PetscMPIInt size,rank;
PetscRandom rand;
const PetscInt *rows,*cols;
IS isrows,iscols;
PetscBool mats_view=PETSC_FALSE;
MatFactorInfo finfo;
PetscInitialize(&argc,&argv,(char*) 0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
/* Get local dimensions of matrices */
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
n = m;
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
p = m/2;
ierr = PetscOptionsGetInt(NULL,"-p",&p,NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,"-mats_view",&mats_view);CHKERRQ(ierr);
/* Create matrix A */
ierr = PetscPrintf(PETSC_COMM_WORLD," Create Elemental matrix A\n");CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
/* Set local matrix entries */
ierr = MatGetOwnershipIS(A,&isrows,&iscols);CHKERRQ(ierr);
ierr = ISGetLocalSize(isrows,&nrows);CHKERRQ(ierr);
ierr = ISGetIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscols,&ncols);CHKERRQ(ierr);
ierr = ISGetIndices(iscols,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(nrows*ncols,&v);CHKERRQ(ierr);
for (i=0; i<nrows; i++) {
for (j=0; j<ncols; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
v[i*ncols+j] = rval;
}
}
ierr = MatSetValues(A,nrows,rows,ncols,cols,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = ISRestoreIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISRestoreIndices(iscols,&cols);CHKERRQ(ierr);
ierr = ISDestroy(&isrows);CHKERRQ(ierr);
ierr = ISDestroy(&iscols);CHKERRQ(ierr);
ierr = PetscFree(v);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "A: nrows %d, m %d; ncols %d, n %d\n",nrows,m,ncols,n);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create rhs matrix B */
ierr = PetscPrintf(PETSC_COMM_WORLD," Create rhs matrix B\n");CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,m,p,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(B,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(B);CHKERRQ(ierr);
ierr = MatSetUp(B);CHKERRQ(ierr);
ierr = MatGetOwnershipIS(B,&isrows,&iscols);CHKERRQ(ierr);
ierr = ISGetLocalSize(isrows,&nrows);CHKERRQ(ierr);
ierr = ISGetIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscols,&ncols);CHKERRQ(ierr);
ierr = ISGetIndices(iscols,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(nrows*ncols,&v);CHKERRQ(ierr);
for (i=0; i<nrows; i++) {
for (j=0; j<ncols; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
v[i*ncols+j] = rval;
}
}
ierr = MatSetValues(B,nrows,rows,ncols,cols,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = ISRestoreIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISRestoreIndices(iscols,&cols);CHKERRQ(ierr);
ierr = ISDestroy(&isrows);CHKERRQ(ierr);
ierr = ISDestroy(&iscols);CHKERRQ(ierr);
ierr = PetscFree(v);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "B: nrows %d, m %d; ncols %d, p %d\n",nrows,m,ncols,p);CHKERRQ(ierr);
ierr = MatView(B,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create rhs vector b and solution x (same size as b) */
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecSetSizes(b,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(b);CHKERRQ(ierr);
ierr = VecGetArray(b,&barray);CHKERRQ(ierr);
for (j=0; j<m; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
barray[j] = rval;
}
ierr = VecRestoreArray(b,&barray);CHKERRQ(ierr);
ierr = VecAssemblyBegin(b);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d] b: m %d\n",rank,m);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
ierr = VecView(b,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = VecDuplicate(b,&x);CHKERRQ(ierr);
/* Create matrix X - same size as B */
ierr = PetscPrintf(PETSC_COMM_WORLD," Create solution matrix X\n");CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&X);CHKERRQ(ierr);
ierr = MatSetSizes(X,m,p,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(X,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(X);CHKERRQ(ierr);
ierr = MatSetUp(X);CHKERRQ(ierr);
ierr = MatAssemblyBegin(X,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(X,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Cholesky factorization */
/*------------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD," Create Elemental matrix Aher\n");CHKERRQ(ierr);
ierr = MatHermitianTranspose(A,MAT_INITIAL_MATRIX,&Aher);CHKERRQ(ierr);
ierr = MatAXPY(Aher,1.0,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr); /* Aher = A + A^T */
if (!rank) { /* add 100.0 to diagonals of Aher to make it spd */
PetscInt M,N;
ierr = MatGetSize(Aher,&M,&N);CHKERRQ(ierr);
for (i=0; i<M; i++) {
rval = 100.0;
ierr = MatSetValues(Aher,1,&i,1,&i,&rval,ADD_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(Aher,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Aher,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Aher:\n");CHKERRQ(ierr);
ierr = MatView(Aher,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Cholesky factorization */
/*------------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD," Test Cholesky Solver \n");CHKERRQ(ierr);
/* In-place Cholesky */
/* Create matrix factor G, then copy Aher to G */
ierr = MatCreate(PETSC_COMM_WORLD,&G);CHKERRQ(ierr);
ierr = MatSetSizes(G,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(G,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(G);CHKERRQ(ierr);
ierr = MatSetUp(G);CHKERRQ(ierr);
ierr = MatAssemblyBegin(G,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(G,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatCopy(Aher,G,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
/* Only G = U^T * U is implemented for now */
ierr = MatCholeskyFactor(G,0,0);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Cholesky Factor G:\n");CHKERRQ(ierr);
ierr = MatView(G,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Solve U^T * U x = b and U^T * U X = B */
ierr = MatSolve(G,b,x);CHKERRQ(ierr);
ierr = MatMatSolve(G,B,X);CHKERRQ(ierr);
ierr = MatDestroy(&G);CHKERRQ(ierr);
/* Out-place Cholesky */
ierr = MatGetFactor(Aher,MATSOLVERELEMENTAL,MAT_FACTOR_CHOLESKY,&G);CHKERRQ(ierr);
ierr = MatCholeskyFactorSymbolic(G,Aher,0,&finfo);CHKERRQ(ierr);
ierr = MatCholeskyFactorNumeric(G,Aher,&finfo);CHKERRQ(ierr);
if (mats_view) {
ierr = MatView(G,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = MatSolve(G,b,x);CHKERRQ(ierr);
ierr = MatMatSolve(G,B,X);CHKERRQ(ierr);
ierr = MatDestroy(&G);CHKERRQ(ierr);
/* Check norm(Aher*x - b) */
ierr = VecCreate(PETSC_COMM_WORLD,&c);CHKERRQ(ierr);
ierr = VecSetSizes(c,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(c);CHKERRQ(ierr);
ierr = MatMult(Aher,x,c);CHKERRQ(ierr);
ierr = VecAXPY(c,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(c,NORM_1,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |Aher*x - b| for Cholesky %g\n",(double)norm);CHKERRQ(ierr);
}
/* Check norm(Aher*X - B) */
ierr = MatMatMult(Aher,X,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&C);CHKERRQ(ierr);
ierr = MatAXPY(C,-1.0,B,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatNorm(C,NORM_1,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |Aher*X - B| for Cholesky %g\n",(double)norm);CHKERRQ(ierr);
}
/* LU factorization */
/*------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD," Test LU Solver \n");CHKERRQ(ierr);
/* In-place LU */
/* Create matrix factor F, then copy A to F */
ierr = MatCreate(PETSC_COMM_WORLD,&F);CHKERRQ(ierr);
ierr = MatSetSizes(F,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(F,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(F);CHKERRQ(ierr);
ierr = MatSetUp(F);CHKERRQ(ierr);
ierr = MatAssemblyBegin(F,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(F,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatCopy(A,F,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
/* Create vector d to test MatSolveAdd() */
ierr = VecDuplicate(x,&d);CHKERRQ(ierr);
ierr = VecCopy(x,d);CHKERRQ(ierr);
/* PF=LU or F=LU factorization - perms is ignored by Elemental;
set finfo.dtcol !0 or 0 to enable/disable partial pivoting */
finfo.dtcol = 0.1;
ierr = MatLUFactor(F,0,0,&finfo);CHKERRQ(ierr);
/* Solve LUX = PB or LUX = B */
ierr = MatSolveAdd(F,b,d,x);CHKERRQ(ierr);
ierr = MatMatSolve(F,B,X);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
/* Check norm(A*X - B) */
ierr = VecCreate(PETSC_COMM_WORLD,&e);CHKERRQ(ierr);
ierr = VecSetSizes(e,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(e);CHKERRQ(ierr);
ierr = MatMult(A,x,c);CHKERRQ(ierr);
ierr = MatMult(A,d,e);CHKERRQ(ierr);
ierr = VecAXPY(c,-1.0,e);CHKERRQ(ierr);
ierr = VecAXPY(c,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(c,NORM_1,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |A*x - b| for LU %g\n",(double)norm);CHKERRQ(ierr);
}
ierr = MatMatMult(A,X,MAT_REUSE_MATRIX,PETSC_DEFAULT,&C);CHKERRQ(ierr);
ierr = MatAXPY(C,-1.0,B,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatNorm(C,NORM_1,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |A*X - B| for LU %g\n",(double)norm);CHKERRQ(ierr);
}
/* Out-place LU */
ierr = MatGetFactor(A,MATSOLVERELEMENTAL,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(F,A,0,0,&finfo);CHKERRQ(ierr);
ierr = MatLUFactorNumeric(F,A,&finfo);CHKERRQ(ierr);
if (mats_view) {
ierr = MatView(F,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = MatSolve(F,b,x);CHKERRQ(ierr);
ierr = MatMatSolve(F,B,X);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
/* Free space */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&Aher);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&c);CHKERRQ(ierr);
ierr = VecDestroy(&d);CHKERRQ(ierr);
ierr = VecDestroy(&e);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
For MATSEQDENSE matrix, the factorization is just a thin wrapper to LAPACK \n\n";
#include <petscmat.h>
#undef __FUNCT__
#define __FUNCT__ "main"
int main(int argc,char **argv)
{
Mat mat,F,RHS,SOLU;
MatInfo info;
PetscErrorCode ierr;
PetscInt m = 10,n = 10,i,j,rstart,rend,nrhs=2;
PetscScalar value = 1.0;
Vec x,y,b,ytmp;
PetscReal norm,tol=1.e-15;
PetscMPIInt size;
PetscScalar *rhs_array,*solu_array;
PetscRandom rand;
PetscScalar *array,rval;
PetscInitialize(&argc,&argv,(char*) 0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"This is a uniprocessor example only!");
/* create single vectors */
ierr = VecCreate(PETSC_COMM_WORLD,&y);CHKERRQ(ierr);
ierr = VecSetSizes(y,PETSC_DECIDE,m);CHKERRQ(ierr);
ierr = VecSetFromOptions(y);CHKERRQ(ierr);
ierr = VecDuplicate(y,&x);CHKERRQ(ierr);
ierr = VecDuplicate(y,&ytmp);CHKERRQ(ierr);
ierr = VecSet(x,value);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecSetSizes(b,PETSC_DECIDE,n);CHKERRQ(ierr);
ierr = VecSetFromOptions(b);CHKERRQ(ierr);
/* create multiple vectors RHS and SOLU */
ierr = MatCreate(PETSC_COMM_WORLD,&RHS);CHKERRQ(ierr);
ierr = MatSetSizes(RHS,PETSC_DECIDE,PETSC_DECIDE,n,nrhs);CHKERRQ(ierr);
ierr = MatSetType(RHS,MATDENSE);CHKERRQ(ierr);
ierr = MatSetFromOptions(RHS);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(RHS,NULL);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatDenseGetArray(RHS,&array);CHKERRQ(ierr);
for (j=0; j<nrhs; j++) {
for (i=0; i<n; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
array[n*j+i] = rval;
}
}
ierr = MatDenseRestoreArray(RHS,&array);CHKERRQ(ierr);
ierr = MatDuplicate(RHS,MAT_DO_NOT_COPY_VALUES,&SOLU);CHKERRQ(ierr);
/* create matrix */
ierr = MatCreateSeqDense(PETSC_COMM_WORLD,m,n,NULL,&mat);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(mat,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
value = (PetscReal)i+1;
ierr = MatSetValues(mat,1,&i,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatGetInfo(mat,MAT_LOCAL,&info);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"matrix nonzeros = %D, allocated nonzeros = %D\n",
(PetscInt)info.nz_used,(PetscInt)info.nz_allocated);CHKERRQ(ierr);
/* Cholesky factorization - perm and factinfo are ignored by LAPACK */
/* in-place Cholesky */
ierr = MatMult(mat,x,b);CHKERRQ(ierr);
ierr = MatDuplicate(mat,MAT_COPY_VALUES,&F);CHKERRQ(ierr);
ierr = MatCholeskyFactor(F,0,0);CHKERRQ(ierr);
ierr = MatSolve(F,b,y);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for Cholesky %G\n",norm);CHKERRQ(ierr);
}
/* out-place Cholesky */
ierr = MatGetFactor(mat,MATSOLVERPETSC,MAT_FACTOR_CHOLESKY,&F);CHKERRQ(ierr);
ierr = MatCholeskyFactorSymbolic(F,mat,0,0);CHKERRQ(ierr);
ierr = MatCholeskyFactorNumeric(F,mat,0);CHKERRQ(ierr);
ierr = MatSolve(F,b,y);CHKERRQ(ierr);
value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for Cholesky %G\n",norm);CHKERRQ(ierr);
}
ierr = MatDestroy(&F);CHKERRQ(ierr);
/* LU factorization - perms and factinfo are ignored by LAPACK */
i = m-1; value = 1.0;
ierr = MatSetValues(mat,1,&i,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatMult(mat,x,b);CHKERRQ(ierr);
ierr = MatDuplicate(mat,MAT_COPY_VALUES,&F);CHKERRQ(ierr);
/* in-place LU */
ierr = MatLUFactor(F,0,0,0);CHKERRQ(ierr);
ierr = MatSolve(F,b,y);CHKERRQ(ierr);
value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for LU %G\n",norm);CHKERRQ(ierr);
}
ierr = MatMatSolve(F,RHS,SOLU);CHKERRQ(ierr);
ierr = MatDenseGetArray(SOLU,&solu_array);CHKERRQ(ierr);
ierr = MatDenseGetArray(RHS,&rhs_array);CHKERRQ(ierr);
for (j=0; j<nrhs; j++) {
ierr = VecPlaceArray(y,solu_array+j*m);CHKERRQ(ierr);
ierr = VecPlaceArray(b,rhs_array+j*m);CHKERRQ(ierr);
ierr = MatMult(mat,y,ytmp);CHKERRQ(ierr);
ierr = VecAXPY(ytmp,-1.0,b);CHKERRQ(ierr); /* ytmp = mat*SOLU[:,j] - RHS[:,j] */
ierr = VecNorm(ytmp,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: Norm of residual for LU %G\n",norm);CHKERRQ(ierr);
}
ierr = VecResetArray(b);CHKERRQ(ierr);
ierr = VecResetArray(y);CHKERRQ(ierr);
}
ierr = MatDenseRestoreArray(RHS,&rhs_array);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(SOLU,&solu_array);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
/* out-place LU */
ierr = MatGetFactor(mat,MATSOLVERPETSC,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(F,mat,0,0,0);CHKERRQ(ierr);
ierr = MatLUFactorNumeric(F,mat,0);CHKERRQ(ierr);
ierr = MatSolve(F,b,y);CHKERRQ(ierr);
value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for LU %G\n",norm);CHKERRQ(ierr);
}
/* free space */
ierr = MatDestroy(&F);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = MatDestroy(&RHS);CHKERRQ(ierr);
ierr = MatDestroy(&SOLU);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&ytmp);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}