//=======================================================
int EpetraExt_HypreIJMatrix::RightScale(const Epetra_Vector& X) {
// First we need to import off-processor values of the vector
Epetra_Import Importer(RowMatrixColMap(), RowMatrixRowMap());
Epetra_Vector Import_Vector(RowMatrixColMap(), true);
EPETRA_CHK_ERR(Import_Vector.Import(X, Importer, Insert, 0));
for(int i = 0; i < NumMyRows_; i++){
//Vector-scalar mult on ith column
int num_entries;
double *values;
int *indices;
// Get values and indices of ith row of matrix
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixGetRow(ParMatrix_,i+MyRowStart_, &num_entries, &indices, &values));
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixRestoreRow(ParMatrix_,i+MyRowStart_,&num_entries,&indices,&values));
Teuchos::Array<int> new_indices; new_indices.resize(num_entries);
Teuchos::Array<double> new_values; new_values.resize(num_entries);
for(int j = 0; j < num_entries; j++){
// Multiply column j with jth element
int index = RowMatrixColMap().LID(indices[j]);
TEUCHOS_TEST_FOR_EXCEPTION(index < 0, std::logic_error, "Index is negtive.");
new_values[j] = values[j] * Import_Vector[index];
new_indices[j] = indices[j];
}
// Finally set values of the Matrix for this row
int rows[1];
rows[0] = i+MyRowStart_;
EPETRA_CHK_ERR(HYPRE_IJMatrixSetValues(Matrix_, 1, &num_entries, rows, &new_indices[0], &new_values[0]));
}
HaveNumericConstants_ = false;
UpdateFlops(NumGlobalNonzeros());
return 0;
} //RightScale()
void EuclidRestoreRow(void *A, HYPRE_Int row, HYPRE_Int *len, HYPRE_Int **ind, double **val)
{
START_FUNC_DH
HYPRE_Int ierr;
HYPRE_ParCSRMatrix mat = (HYPRE_ParCSRMatrix) A;
ierr = HYPRE_ParCSRMatrixRestoreRow(mat, row, len, ind, val);
if (ierr) {
hypre_sprintf(msgBuf_dh, "HYPRE_ParCSRMatrixRestoreRow(row= %i) returned %i", row+1, ierr);
SET_V_ERROR(msgBuf_dh);
}
END_FUNC_DH
}
int32_t
impl_bHYPRE_IJParCSRMatrix_GetRow(
/* in */ bHYPRE_IJParCSRMatrix self,
/* in */ int32_t row,
/* out */ int32_t* size,
/* out array<int,column-major> */ struct sidl_int__array** col_ind,
/* out array<double,column-major> */ struct sidl_double__array** values,
/* out */ sidl_BaseInterface *_ex)
{
*_ex = 0;
{
/* DO-NOT-DELETE splicer.begin(bHYPRE.IJParCSRMatrix.GetRow) */
/* Insert the implementation of the GetRow method here... */
int ierr=0;
void * object;
struct bHYPRE_IJParCSRMatrix__data * data;
HYPRE_IJMatrix ij_A;
HYPRE_ParCSRMatrix bHYPREP_A;
int * iindices[1];
double * dvalues[1];
data = bHYPRE_IJParCSRMatrix__get_data( self );
ij_A = data -> ij_A;
ierr += HYPRE_IJMatrixGetObject( ij_A, &object );
bHYPREP_A = (HYPRE_ParCSRMatrix) object;
*col_ind = sidl_int__array_create1d( size[0] );
*values = sidl_double__array_create1d( size[0] );
*iindices = sidlArrayAddr1( *col_ind, 0 );
*dvalues = sidlArrayAddr1( *values, 0 );
/* RestoreRow doesn't do anything but reset a parameter. Its
* function is to make sure the user who calls GetRow is aware that
* the data in the output arrays will be changed. */
HYPRE_ParCSRMatrixRestoreRow( bHYPREP_A, row, size, iindices, dvalues );
ierr += HYPRE_ParCSRMatrixGetRow( bHYPREP_A, row, size, iindices, dvalues );
return( ierr );
/* DO-NOT-DELETE splicer.end(bHYPRE.IJParCSRMatrix.GetRow) */
}
}
//=======================================================
int EpetraExt_HypreIJMatrix::LeftScale(const Epetra_Vector& X) {
for(int i = 0; i < NumMyRows_; i++){
//Vector-scalar mult on ith row
int num_entries;
int *indices;
double *values;
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixGetRow(ParMatrix_,i+MyRowStart_, &num_entries, &indices, &values));
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixRestoreRow(ParMatrix_,i+MyRowStart_, &num_entries, &indices, &values));
Teuchos::Array<double> new_values; new_values.resize(num_entries);
Teuchos::Array<int> new_indices; new_indices.resize(num_entries);
for(int j = 0; j < num_entries; j++){
// Scale this row with the appropriate values from the vector
new_values[j] = X[i]*values[j];
new_indices[j] = indices[j];
}
int rows[1];
rows[0] = i+MyRowStart_;
EPETRA_CHK_ERR(HYPRE_IJMatrixSetValues(Matrix_, 1, &num_entries, rows, &new_indices[0], &new_values[0]));
// Finally set values of the Matrix for this row
}
HaveNumericConstants_ = false;
UpdateFlops(NumGlobalNonzeros());
return 0;
} //LeftScale()
//=======================================================
int EpetraExt_HypreIJMatrix::ExtractMyRowCopy(int Row, int Length, int & NumEntries,
double * Values, int * Indices) const
{
// Get values and indices of ith row of matrix
int *indices;
double *values;
int num_entries;
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixGetRow(ParMatrix_, Row+RowMatrixRowMap().MinMyGID(), &num_entries, &indices, &values));
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixRestoreRow(ParMatrix_, Row+RowMatrixRowMap().MinMyGID(), &num_entries, &indices, &values));
NumEntries = num_entries;
if(Length < NumEntries){
printf("The arrays passed in are not large enough. Allocate more space.\n");
return -2;
}
for(int i = 0; i < NumEntries; i++){
Values[i] = values[i];
Indices[i] = RowMatrixColMap().LID(indices[i]);
}
return 0;
} //ExtractMyRowCopy()
HYPRE_Int main( HYPRE_Int argc, char *argv[] )
{
HYPRE_Int arg_index;
HYPRE_Int print_usage;
HYPRE_Int build_matrix_arg_index;
HYPRE_Int solver_id;
HYPRE_Int ierr,i,j;
HYPRE_Int num_iterations;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_Int num_procs, myid;
HYPRE_Int local_row;
HYPRE_Int time_index;
MPI_Comm comm;
HYPRE_Int M, N;
HYPRE_Int first_local_row, last_local_row;
HYPRE_Int first_local_col, last_local_col;
HYPRE_Int size, *col_ind;
HYPRE_Real *values;
/* parameters for BoomerAMG */
HYPRE_Real strong_threshold;
HYPRE_Int num_grid_sweeps;
HYPRE_Real relax_weight;
/* parameters for GMRES */
HYPRE_Int k_dim;
char *paramString = new char[100];
/*-----------------------------------------------------------
* Initialize some stuff
*-----------------------------------------------------------*/
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
/*-----------------------------------------------------------
* Set defaults
*-----------------------------------------------------------*/
build_matrix_arg_index = argc;
solver_id = 0;
strong_threshold = 0.25;
num_grid_sweeps = 2;
relax_weight = 0.5;
k_dim = 20;
/*-----------------------------------------------------------
* Parse command line
*-----------------------------------------------------------*/
print_usage = 0;
arg_index = 1;
while ( (arg_index < argc) && (!print_usage) )
{
if ( strcmp(argv[arg_index], "-solver") == 0 )
{
arg_index++;
solver_id = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-dbg") == 0 )
{
arg_index++;
atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
}
else
{
arg_index++;
}
}
/*-----------------------------------------------------------
* Print usage info
*-----------------------------------------------------------*/
if ( (print_usage) && (myid == 0) )
{
hypre_printf("\n");
hypre_printf("Usage: %s [<options>]\n", argv[0]);
hypre_printf("\n");
hypre_printf(" -solver <ID> : solver ID\n");
hypre_printf(" 0=DS-PCG 1=ParaSails-PCG \n");
hypre_printf(" 2=AMG-PCG 3=DS-GMRES \n");
hypre_printf(" 4=PILUT-GMRES 5=AMG-GMRES \n");
hypre_printf("\n");
hypre_printf(" -rlx <val> : relaxation type\n");
hypre_printf(" 0=Weighted Jacobi \n");
hypre_printf(" 1=Gauss-Seidel (very slow!) \n");
hypre_printf(" 3=Hybrid Jacobi/Gauss-Seidel \n");
hypre_printf("\n");
exit(1);
}
/*-----------------------------------------------------------
* Print driver parameters
*-----------------------------------------------------------*/
if (myid == 0)
{
hypre_printf("Running with these driver parameters:\n");
hypre_printf(" solver ID = %d\n", solver_id);
}
/*-----------------------------------------------------------
* Set up matrix
*-----------------------------------------------------------*/
strcpy(paramString, "LS Interface");
time_index = hypre_InitializeTiming(paramString);
hypre_BeginTiming(time_index);
BuildParLaplacian27pt(argc, argv, build_matrix_arg_index, &parcsr_A);
/*-----------------------------------------------------------
* Copy the parcsr matrix into the LSI through interface calls
*-----------------------------------------------------------*/
ierr = HYPRE_ParCSRMatrixGetComm( parcsr_A, &comm );
ierr += HYPRE_ParCSRMatrixGetDims( parcsr_A, &M, &N );
ierr = HYPRE_ParCSRMatrixGetLocalRange( parcsr_A,
&first_local_row, &last_local_row ,
&first_local_col, &last_local_col );
HYPRE_LinSysCore H(hypre_MPI_COMM_WORLD);
HYPRE_Int numLocalEqns = last_local_row - first_local_row + 1;
H.createMatricesAndVectors(M,first_local_row+1,numLocalEqns);
HYPRE_Int index;
HYPRE_Int *rowLengths = new HYPRE_Int[numLocalEqns];
HYPRE_Int **colIndices = new HYPRE_Int*[numLocalEqns];
local_row = 0;
for (i=first_local_row; i<= last_local_row; i++)
{
ierr += HYPRE_ParCSRMatrixGetRow(parcsr_A,i,&size,&col_ind,&values );
rowLengths[local_row] = size;
colIndices[local_row] = new HYPRE_Int[size];
for (j=0; j<size; j++) colIndices[local_row][j] = col_ind[j] + 1;
local_row++;
HYPRE_ParCSRMatrixRestoreRow(parcsr_A,i,&size,&col_ind,&values);
}
H.allocateMatrix(colIndices, rowLengths);
delete [] rowLengths;
for (i=0; i< numLocalEqns; i++) delete [] colIndices[i];
delete [] colIndices;
HYPRE_Int *newColInd;
for (i=first_local_row; i<= last_local_row; i++)
{
ierr += HYPRE_ParCSRMatrixGetRow(parcsr_A,i,&size,&col_ind,&values );
newColInd = new HYPRE_Int[size];
for (j=0; j<size; j++) newColInd[j] = col_ind[j] + 1;
H.sumIntoSystemMatrix(i+1,size,(const HYPRE_Real*)values,
(const HYPRE_Int*)newColInd);
delete [] newColInd;
ierr += HYPRE_ParCSRMatrixRestoreRow(parcsr_A,i,&size,&col_ind,&values);
}
H.matrixLoadComplete();
HYPRE_ParCSRMatrixDestroy(parcsr_A);
/*-----------------------------------------------------------
* Set up the RHS and initial guess
*-----------------------------------------------------------*/
HYPRE_Real ddata=1.0;
HYPRE_Int status;
for (i=first_local_row; i<= last_local_row; i++)
{
index = i + 1;
H.sumIntoRHSVector(1,(const HYPRE_Real*) &ddata, (const HYPRE_Int*) &index);
}
hypre_EndTiming(time_index);
strcpy(paramString, "LS Interface");
hypre_PrintTiming(paramString, hypre_MPI_COMM_WORLD);
hypre_FinalizeTiming(time_index);
hypre_ClearTiming();
/*-----------------------------------------------------------
* Solve the system using PCG
*-----------------------------------------------------------*/
if ( solver_id == 0 )
{
strcpy(paramString, "solver cg");
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: DS-PCG\n");
strcpy(paramString, "preconditioner diagonal");
H.parameters(1, ¶mString);
}
else if ( solver_id == 1 )
{
strcpy(paramString, "solver cg");
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: ParaSails-PCG\n");
strcpy(paramString, "preconditioner parasails");
H.parameters(1, ¶mString);
strcpy(paramString, "parasailsNlevels 1");
H.parameters(1, ¶mString);
strcpy(paramString, "parasailsThreshold 0.1");
H.parameters(1, ¶mString);
}
else if ( solver_id == 2 )
{
strcpy(paramString, "solver cg");
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: AMG-PCG\n");
strcpy(paramString, "preconditioner boomeramg");
H.parameters(1, ¶mString);
strcpy(paramString, "amgCoarsenType falgout");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgStrongThreshold %e", strong_threshold);
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgNumSweeps %d", num_grid_sweeps);
H.parameters(1, ¶mString);
strcpy(paramString, "amgRelaxType jacobi");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgRelaxWeight %e", relax_weight);
H.parameters(1, ¶mString);
}
else if ( solver_id == 3 )
{
strcpy(paramString, "solver cg");
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: Poly-PCG\n");
strcpy(paramString, "preconditioner poly");
H.parameters(1, ¶mString);
strcpy(paramString, "polyOrder 9");
H.parameters(1, ¶mString);
}
else if ( solver_id == 4 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: DS-GMRES\n");
strcpy(paramString, "preconditioner diagonal");
H.parameters(1, ¶mString);
}
else if ( solver_id == 5 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: PILUT-GMRES\n");
strcpy(paramString, "preconditioner pilut");
H.parameters(1, ¶mString);
strcpy(paramString, "pilutRowSize 0");
H.parameters(1, ¶mString);
strcpy(paramString, "pilutDropTol 0.0");
H.parameters(1, ¶mString);
}
else if ( solver_id == 6 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: AMG-GMRES\n");
strcpy(paramString, "preconditioner boomeramg");
H.parameters(1, ¶mString);
strcpy(paramString, "amgCoarsenType falgout");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgStrongThreshold %e", strong_threshold);
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgNumSweeps %d", num_grid_sweeps);
H.parameters(1, ¶mString);
strcpy(paramString, "amgRelaxType jacobi");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "amgRelaxWeight %e", relax_weight);
H.parameters(1, ¶mString);
}
else if ( solver_id == 7 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: DDILUT-GMRES\n");
strcpy(paramString, "preconditioner ddilut");
H.parameters(1, ¶mString);
strcpy(paramString, "ddilutFillin 5.0");
H.parameters(1, ¶mString);
strcpy(paramString, "ddilutDropTol 0.0");
H.parameters(1, ¶mString);
}
else if ( solver_id == 8 )
{
strcpy(paramString, "solver gmres");
H.parameters(1, ¶mString);
hypre_sprintf(paramString, "gmresDim %d", k_dim);
H.parameters(1, ¶mString);
if (myid == 0) hypre_printf("Solver: POLY-GMRES\n");
strcpy(paramString, "preconditioner poly");
H.parameters(1, ¶mString);
strcpy(paramString, "polyOrder 5");
H.parameters(1, ¶mString);
}
strcpy(paramString, "Krylov Solve");
time_index = hypre_InitializeTiming(paramString);
hypre_BeginTiming(time_index);
H.launchSolver(status, num_iterations);
hypre_EndTiming(time_index);
strcpy(paramString, "Solve phase times");
hypre_PrintTiming(paramString, hypre_MPI_COMM_WORLD);
hypre_FinalizeTiming(time_index);
hypre_ClearTiming();
if (myid == 0)
{
hypre_printf("\n Iterations = %d\n", num_iterations);
hypre_printf("\n");
}
/*-----------------------------------------------------------
* Finalize things
*-----------------------------------------------------------*/
delete [] paramString;
hypre_MPI_Finalize();
return (0);
}
//=======================================================
EpetraExt_HypreIJMatrix::EpetraExt_HypreIJMatrix(HYPRE_IJMatrix matrix)
: Epetra_BasicRowMatrix(Epetra_MpiComm(hypre_IJMatrixComm(matrix))),
Matrix_(matrix),
ParMatrix_(0),
NumMyRows_(-1),
NumGlobalRows_(-1),
NumGlobalCols_(-1),
MyRowStart_(-1),
MyRowEnd_(-1),
MatType_(-1),
TransposeSolve_(false),
SolveOrPrec_(Solver)
{
IsSolverSetup_ = new bool[1];
IsPrecondSetup_ = new bool[1];
IsSolverSetup_[0] = false;
IsPrecondSetup_[0] = false;
// Initialize default values for global variables
int ierr = 0;
ierr += InitializeDefaults();
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't initialize default values.");
// Create array of global row ids
Teuchos::Array<int> GlobalRowIDs; GlobalRowIDs.resize(NumMyRows_);
for (int i = MyRowStart_; i <= MyRowEnd_; i++) {
GlobalRowIDs[i-MyRowStart_] = i;
}
// Create array of global column ids
int new_value = 0; int entries = 0;
std::set<int> Columns;
int num_entries;
double *values;
int *indices;
for(int i = 0; i < NumMyRows_; i++){
ierr += HYPRE_ParCSRMatrixGetRow(ParMatrix_, i+MyRowStart_, &num_entries, &indices, &values);
ierr += HYPRE_ParCSRMatrixRestoreRow(ParMatrix_, i+MyRowStart_,&num_entries,&indices,&values);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get row of matrix.");
entries = num_entries;
for(int j = 0; j < num_entries; j++){
// Insert column ids from this row into set
new_value = indices[j];
Columns.insert(new_value);
}
}
int NumMyCols = Columns.size();
Teuchos::Array<int> GlobalColIDs; GlobalColIDs.resize(NumMyCols);
std::set<int>::iterator it;
int counter = 0;
for (it = Columns.begin(); it != Columns.end(); it++) {
// Get column ids in order
GlobalColIDs[counter] = *it;
counter = counter + 1;
}
//printf("Proc[%d] Rows from %d to %d, num = %d\n", Comm().MyPID(), MyRowStart_,MyRowEnd_, NumMyRows_);
Epetra_Map RowMap(-1, NumMyRows_, &GlobalRowIDs[0], 0, Comm());
Epetra_Map ColMap(-1, NumMyCols, &GlobalColIDs[0], 0, Comm());
//Need to call SetMaps()
SetMaps(RowMap, ColMap);
// Get an MPI_Comm to create vectors.
// The vectors will be reused in Multiply(), so that they aren't recreated every time.
MPI_Comm comm;
ierr += HYPRE_ParCSRMatrixGetComm(ParMatrix_, &comm);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't get communicator from Hypre Matrix.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &X_hypre);
ierr += HYPRE_IJVectorSetObjectType(X_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(X_hypre);
ierr += HYPRE_IJVectorAssemble(X_hypre);
ierr += HYPRE_IJVectorGetObject(X_hypre, (void**) &par_x);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre X vector.");
ierr += HYPRE_IJVectorCreate(comm, MyRowStart_, MyRowEnd_, &Y_hypre);
ierr += HYPRE_IJVectorSetObjectType(Y_hypre, HYPRE_PARCSR);
ierr += HYPRE_IJVectorInitialize(Y_hypre);
ierr += HYPRE_IJVectorAssemble(Y_hypre);
ierr += HYPRE_IJVectorGetObject(Y_hypre, (void**) &par_y);
TEUCHOS_TEST_FOR_EXCEPTION(ierr != 0, std::logic_error, "Couldn't create Hypre Y vector.");
x_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) X_hypre));
x_local = hypre_ParVectorLocalVector(x_vec);
y_vec = (hypre_ParVector *) hypre_IJVectorObject(((hypre_IJVector *) Y_hypre));
y_local = hypre_ParVectorLocalVector(y_vec);
SolverCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_ParCSRPCGCreate;
SolverDestroyPtr_ = &HYPRE_ParCSRPCGDestroy;
SolverSetupPtr_ = &HYPRE_ParCSRPCGSetup;
SolverSolvePtr_ = &HYPRE_ParCSRPCGSolve;
SolverPrecondPtr_ = &HYPRE_ParCSRPCGSetPrecond;
CreateSolver();
PrecondCreatePtr_ = &EpetraExt_HypreIJMatrix::Hypre_EuclidCreate;
PrecondDestroyPtr_ = &HYPRE_EuclidDestroy;
PrecondSetupPtr_ = &HYPRE_EuclidSetup;
PrecondSolvePtr_ = &HYPRE_EuclidSolve;
CreatePrecond();
ComputeNumericConstants();
ComputeStructureConstants();
} //EpetraExt_HYPREIJMatrix(Hypre_IJMatrix) Constructor
HYPRE_Int HYPRE_ParCSRMLConstructMHMatrix(HYPRE_ParCSRMatrix A, MH_Matrix *mh_mat,
MPI_Comm comm, HYPRE_Int *partition,MH_Context *obj)
{
HYPRE_Int i, j, index, my_id, nprocs, msgid, *tempCnt;
HYPRE_Int sendProcCnt, *sendLeng, *sendProc, **sendList;
HYPRE_Int recvProcCnt, *recvLeng, *recvProc;
HYPRE_Int rowLeng, *colInd, startRow, endRow, localEqns;
HYPRE_Int *diagSize, *offdiagSize, externLeng, *externList, ncnt, nnz;
HYPRE_Int *rowptr, *columns, num_bdry;
double *colVal, *values;
hypre_MPI_Request *Request;
hypre_MPI_Status status;
/* -------------------------------------------------------- */
/* get machine information and local matrix information */
/* -------------------------------------------------------- */
hypre_MPI_Comm_rank(comm, &my_id);
hypre_MPI_Comm_size(comm, &nprocs);
startRow = partition[my_id];
endRow = partition[my_id+1] - 1;
localEqns = endRow - startRow + 1;
/* -------------------------------------------------------- */
/* probe A to find out about diagonal and off-diagonal */
/* block information */
/* -------------------------------------------------------- */
diagSize = (HYPRE_Int*) malloc( sizeof(HYPRE_Int) * localEqns );
offdiagSize = (HYPRE_Int*) malloc( sizeof(HYPRE_Int) * localEqns );
num_bdry = 0;
for ( i = startRow; i <= endRow; i++ )
{
diagSize[i-startRow] = offdiagSize[i-startRow] = 0;
HYPRE_ParCSRMatrixGetRow(A, i, &rowLeng, &colInd, &colVal);
for (j = 0; j < rowLeng; j++)
if ( colInd[j] < startRow || colInd[j] > endRow )
{
//if ( colVal[j] != 0.0 ) offdiagSize[i-startRow]++;
offdiagSize[i-startRow]++;
}
else
{
//if ( colVal[j] != 0.0 ) diagSize[i-startRow]++;
diagSize[i-startRow]++;
}
HYPRE_ParCSRMatrixRestoreRow(A, i, &rowLeng, &colInd, &colVal);
if ( diagSize[i-startRow] + offdiagSize[i-startRow] == 1 ) num_bdry++;
}
/* -------------------------------------------------------- */
/* construct external node list in global eqn numbers */
/* -------------------------------------------------------- */
externLeng = 0;
for ( i = 0; i < localEqns; i++ ) externLeng += offdiagSize[i];
if ( externLeng > 0 )
externList = (HYPRE_Int *) malloc( sizeof(HYPRE_Int) * externLeng);
else externList = NULL;
externLeng = 0;
for ( i = startRow; i <= endRow; i++ )
{
HYPRE_ParCSRMatrixGetRow(A, i, &rowLeng, &colInd, &colVal);
for (j = 0; j < rowLeng; j++)
{
if ( colInd[j] < startRow || colInd[j] > endRow )
//if ( colVal[j] != 0.0 ) externList[externLeng++] = colInd[j];
externList[externLeng++] = colInd[j];
}
HYPRE_ParCSRMatrixRestoreRow(A, i, &rowLeng, &colInd, &colVal);
}
qsort0( externList, 0, externLeng-1 );
ncnt = 0;
for ( i = 1; i < externLeng; i++ )
{
if ( externList[i] != externList[ncnt] )
externList[++ncnt] = externList[i];
}
externLeng = ncnt + 1;
/* -------------------------------------------------------- */
/* allocate the CSR matrix */
/* -------------------------------------------------------- */
nnz = 0;
for ( i = 0; i < localEqns; i++ ) nnz += diagSize[i] + offdiagSize[i];
rowptr = (HYPRE_Int *) malloc( (localEqns + 1) * sizeof(HYPRE_Int) );
columns = (HYPRE_Int *) malloc( nnz * sizeof(HYPRE_Int) );
values = (double *) malloc( nnz * sizeof(double) );
rowptr[0] = 0;
for ( i = 1; i <= localEqns; i++ )
rowptr[i] = rowptr[i-1] + diagSize[i-1] + offdiagSize[i-1];
free( diagSize );
free( offdiagSize );
/* -------------------------------------------------------- */
/* put the matrix data in the CSR matrix */
/* -------------------------------------------------------- */
rowptr[0] = 0;
ncnt = 0;
for ( i = startRow; i <= endRow; i++ )
{
HYPRE_ParCSRMatrixGetRow(A, i, &rowLeng, &colInd, &colVal);
for (j = 0; j < rowLeng; j++)
{
index = colInd[j];
//if ( colVal[j] != 0.0 )
{
if ( index < startRow || index > endRow )
{
columns[ncnt] = hypre_BinarySearch(externList,index,
externLeng );
columns[ncnt] += localEqns;
values [ncnt++] = colVal[j];
}
else
{
columns[ncnt] = index - startRow;
values[ncnt++] = colVal[j];
}
}
}
rowptr[i-startRow+1] = ncnt;
HYPRE_ParCSRMatrixRestoreRow(A, i, &rowLeng, &colInd, &colVal);
}
assert( ncnt == nnz );
/* -------------------------------------------------------- */
/* initialize the MH_Matrix data structure */
/* -------------------------------------------------------- */
mh_mat->Nrows = localEqns;
mh_mat->rowptr = rowptr;
mh_mat->colnum = columns;
mh_mat->values = values;
mh_mat->sendProcCnt = 0;
mh_mat->recvProcCnt = 0;
mh_mat->sendLeng = NULL;
mh_mat->recvLeng = NULL;
mh_mat->sendProc = NULL;
mh_mat->recvProc = NULL;
mh_mat->sendList = NULL;
mh_mat->map = externList;
/* -------------------------------------------------------- */
/* form the remote portion of the matrix */
/* -------------------------------------------------------- */
if ( nprocs > 1 )
{
/* ----------------------------------------------------- */
/* count number of elements to be received from each */
/* remote processor (assume sequential mapping) */
/* ----------------------------------------------------- */
tempCnt = (HYPRE_Int *) malloc( sizeof(HYPRE_Int) * nprocs );
for ( i = 0; i < nprocs; i++ ) tempCnt[i] = 0;
for ( i = 0; i < externLeng; i++ )
{
for ( j = 0; j < nprocs; j++ )
{
if ( externList[i] >= partition[j] &&
externList[i] < partition[j+1] )
{
tempCnt[j]++;
break;
}
}
}
/* ----------------------------------------------------- */
/* compile a list processors data is to be received from */
/* ----------------------------------------------------- */
recvProcCnt = 0;
for ( i = 0; i < nprocs; i++ )
if ( tempCnt[i] > 0 ) recvProcCnt++;
recvLeng = (HYPRE_Int*) malloc( sizeof(HYPRE_Int) * recvProcCnt );
recvProc = (HYPRE_Int*) malloc( sizeof(HYPRE_Int) * recvProcCnt );
recvProcCnt = 0;
for ( i = 0; i < nprocs; i++ )
{
if ( tempCnt[i] > 0 )
{
recvProc[recvProcCnt] = i;
recvLeng[recvProcCnt++] = tempCnt[i];
}
}
/* ----------------------------------------------------- */
/* each processor has to find out how many processors it */
/* has to send data to */
/* ----------------------------------------------------- */
sendLeng = (HYPRE_Int *) malloc( nprocs * sizeof(HYPRE_Int) );
for ( i = 0; i < nprocs; i++ ) tempCnt[i] = 0;
for ( i = 0; i < recvProcCnt; i++ ) tempCnt[recvProc[i]] = 1;
hypre_MPI_Allreduce(tempCnt, sendLeng, nprocs, HYPRE_MPI_INT, hypre_MPI_SUM, comm );
sendProcCnt = sendLeng[my_id];
free( sendLeng );
if ( sendProcCnt > 0 )
{
sendLeng = (HYPRE_Int *) malloc( sendProcCnt * sizeof(HYPRE_Int) );
sendProc = (HYPRE_Int *) malloc( sendProcCnt * sizeof(HYPRE_Int) );
sendList = (HYPRE_Int **) malloc( sendProcCnt * sizeof(HYPRE_Int*) );
}
else
{
sendLeng = sendProc = NULL;
sendList = NULL;
}
/* ----------------------------------------------------- */
/* each processor sends to all processors it expects to */
/* receive data about the lengths of data expected */
/* ----------------------------------------------------- */
msgid = 539;
for ( i = 0; i < recvProcCnt; i++ )
{
hypre_MPI_Send((void*) &recvLeng[i],1,HYPRE_MPI_INT,recvProc[i],msgid,comm);
}
for ( i = 0; i < sendProcCnt; i++ )
{
hypre_MPI_Recv((void*) &sendLeng[i],1,HYPRE_MPI_INT,hypre_MPI_ANY_SOURCE,msgid,
comm,&status);
sendProc[i] = status.hypre_MPI_SOURCE;
sendList[i] = (HYPRE_Int *) malloc( sendLeng[i] * sizeof(HYPRE_Int) );
if ( sendList[i] == NULL )
hypre_printf("allocate problem %d \n", sendLeng[i]);
}
/* ----------------------------------------------------- */
/* each processor sends to all processors it expects to */
/* receive data about the equation numbers */
/* ----------------------------------------------------- */
for ( i = 0; i < nprocs; i++ ) tempCnt[i] = 0;
ncnt = 1;
for ( i = 0; i < externLeng; i++ )
{
if ( externList[i] >= partition[ncnt] )
{
tempCnt[ncnt-1] = i;
i--;
ncnt++;
}
}
for ( i = ncnt-1; i < nprocs; i++ ) tempCnt[i] = externLeng;
/* ----------------------------------------------------- */
/* send the global equation numbers */
/* ----------------------------------------------------- */
msgid = 540;
for ( i = 0; i < recvProcCnt; i++ )
{
if ( recvProc[i] == 0 ) j = 0;
else j = tempCnt[recvProc[i]-1];
rowLeng = recvLeng[i];
hypre_MPI_Send((void*) &externList[j],rowLeng,HYPRE_MPI_INT,recvProc[i],
msgid,comm);
}
for ( i = 0; i < sendProcCnt; i++ )
{
rowLeng = sendLeng[i];
hypre_MPI_Recv((void*)sendList[i],rowLeng,HYPRE_MPI_INT,sendProc[i],
msgid,comm,&status);
}
/* ----------------------------------------------------- */
/* convert the send list from global to local numbers */
/* ----------------------------------------------------- */
for ( i = 0; i < sendProcCnt; i++ )
{
for ( j = 0; j < sendLeng[i]; j++ )
{
index = sendList[i][j] - startRow;
if ( index < 0 || index >= localEqns )
{
hypre_printf("%d : Construct MH matrix Error - index out ");
hypre_printf("of range%d\n", my_id, index);
}
sendList[i][j] = index;
}
}
/* ----------------------------------------------------- */
/* convert the send list from global to local numbers */
/* ----------------------------------------------------- */
mh_mat->sendProcCnt = sendProcCnt;
mh_mat->recvProcCnt = recvProcCnt;
mh_mat->sendLeng = sendLeng;
mh_mat->recvLeng = recvLeng;
mh_mat->sendProc = sendProc;
mh_mat->recvProc = recvProc;
mh_mat->sendList = sendList;
/* ----------------------------------------------------- */
/* clean up */
/* ----------------------------------------------------- */
free( tempCnt );
}
return 0;
}
int HYPRE_LSI_GetParCSRMatrix(HYPRE_IJMatrix Amat, int nrows, int nnz,
int *ia_ptr, int *ja_ptr, double *a_ptr)
{
int nz, i, j, ierr, rowSize, *colInd, nz_ptr, *colInd2;
int firstNnz;
double *colVal, *colVal2;
HYPRE_ParCSRMatrix A_csr;
nz = 0;
nz_ptr = 0;
ia_ptr[0] = nz_ptr;
/* ---old_IJ----------------------------------------------------------- */
/*A_csr = (HYPRE_ParCSRMatrix) HYPRE_IJMatrixGetLocalStorage(Amat);*/
/* ---new_IJ----------------------------------------------------------- */
HYPRE_IJMatrixGetObject(Amat, (void**) &A_csr);
/* -------------------------------------------------------------------- */
for ( i = 0; i < nrows; i++ )
{
ierr = HYPRE_ParCSRMatrixGetRow(A_csr,i,&rowSize,&colInd,&colVal);
assert(!ierr);
colInd2 = (int *) malloc(rowSize * sizeof(int));
colVal2 = (double *) malloc(rowSize * sizeof(double));
for ( j = 0; j < rowSize; j++ )
{
colInd2[j] = colInd[j];
colVal2[j] = colVal[j];
}
qsort1(colInd2, colVal2, 0, rowSize-1);
for ( j = 0; j < rowSize-1; j++ )
if ( colInd2[j] == colInd2[j+1] )
printf("HYPRE_LSI_GetParCSRMatrix-duplicate colind at row %d \n",i);
firstNnz = 0;
for ( j = 0; j < rowSize; j++ )
{
if ( colVal2[j] != 0.0 )
{
if (nz_ptr > 0 && firstNnz > 0 && colInd2[j] == ja_ptr[nz_ptr-1])
{
a_ptr[nz_ptr-1] += colVal2[j];
printf("HYPRE_LSI_GetParCSRMatrix:: repeated col in row %d\n",i);
}
else
{
ja_ptr[nz_ptr] = colInd2[j];
a_ptr[nz_ptr++] = colVal2[j];
if ( nz_ptr > nnz )
{
printf("HYPRE_LSI_GetParCSRMatrix Error (1) - %d %d.\n",i,
nrows);
exit(1);
}
firstNnz++;
}
} else nz++;
}
free( colInd2 );
free( colVal2 );
ia_ptr[i+1] = nz_ptr;
ierr = HYPRE_ParCSRMatrixRestoreRow(A_csr,i,&rowSize,&colInd,&colVal);
assert(!ierr);
}
/*
if ( nnz != nz_ptr )
{
printf("HYPRE_LSI_GetParCSRMatrix note : matrix sparsity has been \n");
printf(" changed since matConfigure - %d > %d ?\n", nnz, nz_ptr);
printf(" number of zeros = %d \n", nz );
}
*/
return nz_ptr;
}
bHYPRE_IJParCSRMatrix
impl_bHYPRE_IJParCSRMatrix_GenerateLaplacian(
/* in */ bHYPRE_MPICommunicator mpi_comm,
/* in */ int32_t nx,
/* in */ int32_t ny,
/* in */ int32_t nz,
/* in */ int32_t Px,
/* in */ int32_t Py,
/* in */ int32_t Pz,
/* in */ int32_t p,
/* in */ int32_t q,
/* in */ int32_t r,
/* in rarray[nvalues] */ double* values,
/* in */ int32_t nvalues,
/* in */ int32_t discretization,
/* out */ sidl_BaseInterface *_ex)
{
*_ex = 0;
{
/* DO-NOT-DELETE splicer.begin(bHYPRE.IJParCSRMatrix.GenerateLaplacian) */
/* Insert-Code-Here {bHYPRE.IJParCSRMatrix.GenerateLaplacian} (GenerateLaplacian method) */
/* The returned matrix represents a Laplacian with 7,9,or 27 point discretization
as specified. Initialize but not Assemble is called before returning. */
int ierr = 0;
bHYPRE_IJParCSRMatrix bHA;
HYPRE_ParCSRMatrix HA;
int first_local_row, last_local_row, first_local_col, last_local_col;
int local_num_rows, size, i;
int * row_sizes;
int * col_inds;
double * row_values;
int stride[1];
MPI_Comm comm = bHYPRE_MPICommunicator__get_data(mpi_comm)->mpi_comm;
;
hypre_assert( nvalues == 4 );
hypre_assert( discretization==7 || discretization==9 || discretization==27 );
hypre_assert( discretization==7 ); /* only 7-point 3D example implemented */
HA = (HYPRE_ParCSRMatrix) GenerateLaplacian(
comm, nx, ny, nz, Px, Py, Pz, p, q, r, values );
/* We need to return a bHYPRE_IJParCSRMatrix. Make a one and copy HA to it... */
ierr += HYPRE_ParCSRMatrixGetLocalRange(
HA, &first_local_row, &last_local_row , &first_local_col, &last_local_col );
local_num_rows = last_local_row - first_local_row + 1;
bHA = bHYPRE_IJParCSRMatrix_Create(
mpi_comm, first_local_row, last_local_row, first_local_col, last_local_col, _ex ); SIDL_CHECK(*_ex);
row_sizes = hypre_CTAlloc( int, local_num_rows );
size = discretization;
for (i=0; i < local_num_rows; i++)
{
row_sizes[i] = size;
}
ierr = bHYPRE_IJParCSRMatrix_SetRowSizes( bHA, row_sizes, local_num_rows, _ex ); SIDL_CHECK(*_ex);
hypre_TFree( row_sizes );
ierr = bHYPRE_IJParCSRMatrix_Initialize( bHA, _ex ); SIDL_CHECK(*_ex);
row_sizes = hypre_CTAlloc( int, 1 );
stride[0] = 1;
/* Copy row data to the new matrix... */
for (i=first_local_row; i<= last_local_row; i++)
{
ierr += HYPRE_ParCSRMatrixGetRow( HA, i, &size, &col_inds, &row_values );
ierr += bHYPRE_IJParCSRMatrix_SetValues(
bHA, 1, &size, &i, col_inds, row_values, size, _ex ); SIDL_CHECK(*_ex);
ierr += HYPRE_ParCSRMatrixRestoreRow( HA, i, &size, &col_inds, &row_values );
}
hypre_assert( ierr == 0 );
return bHA;
hypre_babel_exception_no_return(_ex);
/* DO-NOT-DELETE splicer.end(bHYPRE.IJParCSRMatrix.GenerateLaplacian) */
}
}
int HYPRE_LSI_PolySetup(HYPRE_Solver solver, HYPRE_ParCSRMatrix A_csr,
HYPRE_ParVector b, HYPRE_ParVector x )
{
int i, j, my_id, startRow, endRow, order;
int pos_diag, neg_diag;
int rowLeng, *colInd, *row_partition;
double *coefs=NULL, rowsum, max_norm, *colVal;
HYPRE_LSI_Poly *poly_ptr = (HYPRE_LSI_Poly *) solver;
#ifndef HYPRE_SEQUENTIAL
double dtemp;
#endif
/* ---------------------------------------------------------------- */
/* initialize structure */
/* ---------------------------------------------------------------- */
order = poly_ptr->order;
coefs = (double *) malloc((order+1) * sizeof(double));
poly_ptr->coefficients = coefs;
/* ---------------------------------------------------------------- */
/* compute matrix norm */
/* ---------------------------------------------------------------- */
HYPRE_ParCSRMatrixGetRowPartitioning( A_csr, &row_partition );
#ifdef HYPRE_SEQUENTIAL
my_id = 0;
#else
MPI_Comm_rank(poly_ptr->comm, &my_id);
#endif
startRow = row_partition[my_id];
endRow = row_partition[my_id+1] - 1;
hypre_TFree( row_partition );
poly_ptr->Nrows = endRow - startRow + 1;
max_norm = 0.0;
pos_diag = neg_diag = 0;
for ( i = startRow; i <= endRow; i++ )
{
HYPRE_ParCSRMatrixGetRow(A_csr, i, &rowLeng, &colInd, &colVal);
rowsum = 0.0;
for (j = 0; j < rowLeng; j++)
{
rowsum += habs(colVal[j]);
if ( colInd[j] == i && colVal[j] > 0.0 ) pos_diag++;
if ( colInd[j] == i && colVal[j] < 0.0 ) neg_diag++;
}
if ( rowsum > max_norm ) max_norm = rowsum;
HYPRE_ParCSRMatrixRestoreRow(A_csr, i, &rowLeng, &colInd, &colVal);
}
#ifndef HYPRE_SEQUENTIAL
MPI_Allreduce(&max_norm, &dtemp, 1, MPI_INT, MPI_MAX, poly_ptr->comm);
#endif
if ( pos_diag == 0 && neg_diag > 0 ) max_norm = - max_norm;
/* ---------------------------------------------------------------- */
/* fill in the coefficient table */
/* ---------------------------------------------------------------- */
switch ( order )
{
case 0: coefs[0] = 1.0; break;
case 1: coefs[0] = 5.0; coefs[1] = -1.0; break;
case 2: coefs[0] = 14.0; coefs[1] = -7.0; coefs[2] = 1.0;
break;
case 3: coefs[0] = 30.0; coefs[1] = -27.0; coefs[2] = 9.0;
coefs[3] = -1.0; break;
case 4: coefs[0] = 55.0; coefs[1] = -77.0; coefs[2] = 44.0;
coefs[3] = -11.0; coefs[4] = 1.0; break;
case 5: coefs[0] = 91.0; coefs[1] = -182.0; coefs[2] = 156.0;
coefs[3] = -65.0; coefs[4] = 13.0; coefs[5] = -1.0;
break;
case 6: coefs[0] = 140.0; coefs[1] = -378.0; coefs[2] = 450.0;
coefs[3] = -275.0; coefs[4] = 90.0; coefs[5] = -15.0;
coefs[6] = 1.0; break;
case 7: coefs[0] = 204.0; coefs[1] = -714.0; coefs[2] = 1122.0;
coefs[3] = -935.0; coefs[4] = 442.0; coefs[5] = -119.0;
coefs[6] = 17.0; coefs[7] = -1.0; break;
case 8: coefs[0] = 285.0; coefs[1] = -1254.0; coefs[2] = 2508.0;
coefs[3] = -2717.0; coefs[4] = 1729.0; coefs[5] = -665.0;
coefs[6] = 152.0; coefs[7] = -19.0; coefs[8] = 1.0;
break;
}
for( i = 0; i <= order; i++ )
coefs[i] *= pow( 4.0 / max_norm, (double) i);
return 0;
}
HYPRE_Int
HYPRE_IJMatrixPrint( HYPRE_IJMatrix matrix,
const char *filename )
{
MPI_Comm comm;
HYPRE_Int *row_partitioning;
HYPRE_Int *col_partitioning;
HYPRE_Int ilower, iupper, jlower, jupper;
HYPRE_Int i, j, ii;
HYPRE_Int ncols, *cols;
HYPRE_Complex *values;
HYPRE_Int myid;
char new_filename[255];
FILE *file;
void *object;
if (!matrix)
{
hypre_error_in_arg(1);
return hypre_error_flag;
}
if ( (hypre_IJMatrixObjectType(matrix) != HYPRE_PARCSR) )
{
hypre_error_in_arg(1);
return hypre_error_flag;
}
comm = hypre_IJMatrixComm(matrix);
hypre_MPI_Comm_rank(comm, &myid);
hypre_sprintf(new_filename,"%s.%05d", filename, myid);
if ((file = fopen(new_filename, "w")) == NULL)
{
hypre_error_in_arg(2);
return hypre_error_flag;
}
row_partitioning = hypre_IJMatrixRowPartitioning(matrix);
col_partitioning = hypre_IJMatrixColPartitioning(matrix);
#ifdef HYPRE_NO_GLOBAL_PARTITION
ilower = row_partitioning[0];
iupper = row_partitioning[1] - 1;
jlower = col_partitioning[0];
jupper = col_partitioning[1] - 1;
#else
ilower = row_partitioning[myid];
iupper = row_partitioning[myid+1] - 1;
jlower = col_partitioning[myid];
jupper = col_partitioning[myid+1] - 1;
#endif
hypre_fprintf(file, "%d %d %d %d\n", ilower, iupper, jlower, jupper);
HYPRE_IJMatrixGetObject(matrix, &object);
for (i = ilower; i <= iupper; i++)
{
if ( hypre_IJMatrixObjectType(matrix) == HYPRE_PARCSR )
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
ii = i - hypre_IJMatrixGlobalFirstRow(matrix);
#else
ii = i - row_partitioning[0];
#endif
HYPRE_ParCSRMatrixGetRow((HYPRE_ParCSRMatrix) object,
ii, &ncols, &cols, &values);
for (j = 0; j < ncols; j++)
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
cols[j] += hypre_IJMatrixGlobalFirstCol(matrix);
#else
cols[j] += col_partitioning[0];
#endif
}
}
for (j = 0; j < ncols; j++)
{
hypre_fprintf(file, "%d %d %.14e\n", i, cols[j], values[j]);
}
if ( hypre_IJMatrixObjectType(matrix) == HYPRE_PARCSR )
{
for (j = 0; j < ncols; j++)
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
cols[j] -= hypre_IJMatrixGlobalFirstCol(matrix);
#else
cols[j] -= col_partitioning[0];
#endif
}
HYPRE_ParCSRMatrixRestoreRow((HYPRE_ParCSRMatrix) object,
ii, &ncols, &cols, &values);
}
}
fclose(file);
return hypre_error_flag;
}
