HYPRE_Int
main( HYPRE_Int argc,
char *argv[] )
{
hypre_ParVector *vector1;
hypre_ParVector *vector2;
hypre_ParVector *tmp_vector;
HYPRE_Int num_procs, my_id;
HYPRE_Int global_size = 20;
HYPRE_Int local_size;
HYPRE_Int first_index;
HYPRE_Int num_vectors, vecstride, idxstride;
HYPRE_Int i, j;
HYPRE_Int *partitioning;
double prod;
double *data, *data2;
hypre_Vector *vector;
hypre_Vector *local_vector;
hypre_Vector *local_vector2;
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id );
hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs);
partitioning = NULL;
num_vectors = 3;
vector1 = hypre_ParMultiVectorCreate
( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors );
partitioning = hypre_ParVectorPartitioning(vector1);
hypre_ParVectorInitialize(vector1);
local_vector = hypre_ParVectorLocalVector(vector1);
data = hypre_VectorData(local_vector);
local_size = hypre_VectorSize(local_vector);
vecstride = hypre_VectorVectorStride(local_vector);
idxstride = hypre_VectorIndexStride(local_vector);
first_index = partitioning[my_id];
hypre_printf("vecstride=%i idxstride=%i local_size=%i num_vectors=%i",
vecstride, idxstride, local_size, num_vectors );
for (j=0; j<num_vectors; ++j )
for (i=0; i < local_size; i++)
data[ j*vecstride + i*idxstride ] = first_index+i + 100*j;
hypre_ParVectorPrint(vector1, "Vector");
local_vector2 = hypre_SeqMultiVectorCreate( global_size, num_vectors );
hypre_SeqVectorInitialize(local_vector2);
data2 = hypre_VectorData(local_vector2);
vecstride = hypre_VectorVectorStride(local_vector2);
idxstride = hypre_VectorIndexStride(local_vector2);
for (j=0; j<num_vectors; ++j )
for (i=0; i < global_size; i++)
data2[ j*vecstride + i*idxstride ] = i + 100*j;
/* partitioning = hypre_CTAlloc(HYPRE_Int,4);
partitioning[0] = 0;
partitioning[1] = 10;
partitioning[2] = 10;
partitioning[3] = 20;
*/
partitioning = hypre_CTAlloc(HYPRE_Int,1+num_procs);
hypre_GeneratePartitioning( global_size, num_procs, &partitioning );
vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning);
hypre_ParVectorSetPartitioningOwner(vector2,0);
hypre_ParVectorPrint(vector2, "Convert");
vector = hypre_ParVectorToVectorAll(vector2);
/*-----------------------------------------------------------
* Copy the vector into tmp_vector
*-----------------------------------------------------------*/
/* Read doesn't work for multivectors yet...
tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert");*/
tmp_vector = hypre_ParMultiVectorCreate
( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors );
hypre_ParVectorInitialize( tmp_vector );
hypre_ParVectorCopy( vector2, tmp_vector );
/*
tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning);
hypre_ParVectorSetPartitioningOwner(tmp_vector,0);
hypre_ParVectorInitialize(tmp_vector);
hypre_ParVectorCopy(vector1, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Copy");
*/
/*-----------------------------------------------------------
* Scale tmp_vector
*-----------------------------------------------------------*/
hypre_ParVectorScale(2.0, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Scale");
/*-----------------------------------------------------------
* Do an Axpy (2*vector - vector) = vector
*-----------------------------------------------------------*/
hypre_ParVectorAxpy(-1.0, vector1, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Axpy");
/*-----------------------------------------------------------
* Do an inner product vector* tmp_vector
*-----------------------------------------------------------*/
prod = hypre_ParVectorInnerProd(vector1, tmp_vector);
hypre_printf (" prod: %8.2f \n", prod);
/*-----------------------------------------------------------
* Finalize things
*-----------------------------------------------------------*/
hypre_ParVectorDestroy(vector1);
hypre_ParVectorDestroy(vector2);
hypre_ParVectorDestroy(tmp_vector);
hypre_SeqVectorDestroy(local_vector2);
if (vector) hypre_SeqVectorDestroy(vector);
/* Finalize MPI */
hypre_MPI_Finalize();
return 0;
}
HYPRE_Int hypre_AMESetup(void *esolver)
{
HYPRE_Int ne, *edge_bc;
hypre_AMEData *ame_data = esolver;
hypre_AMSData *ams_data = ame_data -> precond;
if (ams_data -> beta_is_zero)
{
ame_data -> t1 = hypre_ParVectorInDomainOf(ams_data -> G);
ame_data -> t2 = hypre_ParVectorInDomainOf(ams_data -> G);
}
else
{
ame_data -> t1 = ams_data -> r1;
ame_data -> t2 = ams_data -> g1;
}
ame_data -> t3 = ams_data -> r0;
/* Eliminate boundary conditions in G = [Gii, Gib; 0, Gbb], i.e.,
compute [Gii, 0; 0, 0] */
{
HYPRE_Int i, j, k, nv;
HYPRE_Int *offd_edge_bc;
hypre_ParCSRMatrix *Gt;
nv = hypre_ParCSRMatrixNumCols(ams_data -> G);
ne = hypre_ParCSRMatrixNumRows(ams_data -> G);
edge_bc = hypre_TAlloc(HYPRE_Int, ne);
for (i = 0; i < ne; i++)
edge_bc[i] = 0;
/* Find boundary (eliminated) edges */
{
hypre_CSRMatrix *Ad = hypre_ParCSRMatrixDiag(ams_data -> A);
HYPRE_Int *AdI = hypre_CSRMatrixI(Ad);
HYPRE_Int *AdJ = hypre_CSRMatrixJ(Ad);
HYPRE_Real *AdA = hypre_CSRMatrixData(Ad);
hypre_CSRMatrix *Ao = hypre_ParCSRMatrixOffd(ams_data -> A);
HYPRE_Int *AoI = hypre_CSRMatrixI(Ao);
HYPRE_Real *AoA = hypre_CSRMatrixData(Ao);
HYPRE_Real l1_norm;
/* A row (edge) is boundary if its off-diag l1 norm is less than eps */
HYPRE_Real eps = DBL_EPSILON * 1e+4;
for (i = 0; i < ne; i++)
{
l1_norm = 0.0;
for (j = AdI[i]; j < AdI[i+1]; j++)
if (AdJ[j] != i)
l1_norm += fabs(AdA[j]);
if (AoI)
for (j = AoI[i]; j < AoI[i+1]; j++)
l1_norm += fabs(AoA[j]);
if (l1_norm < eps)
edge_bc[i] = 1;
}
}
hypre_ParCSRMatrixTranspose(ams_data -> G, &Gt, 1);
/* Use a Matvec communication to find which of the edges
connected to local vertices are on the boundary */
{
hypre_ParCSRCommHandle *comm_handle;
hypre_ParCSRCommPkg *comm_pkg;
HYPRE_Int num_sends, *int_buf_data;
HYPRE_Int index, start;
offd_edge_bc = hypre_CTAlloc(HYPRE_Int, hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(Gt)));
hypre_MatvecCommPkgCreate(Gt);
comm_pkg = hypre_ParCSRMatrixCommPkg(Gt);
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
int_buf_data = hypre_CTAlloc(HYPRE_Int,
hypre_ParCSRCommPkgSendMapStart(comm_pkg,
num_sends));
index = 0;
for (i = 0; i < num_sends; i++)
{
start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
{
k = hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j);
int_buf_data[index++] = edge_bc[k];
}
}
comm_handle = hypre_ParCSRCommHandleCreate(11, comm_pkg,
int_buf_data, offd_edge_bc);
hypre_ParCSRCommHandleDestroy(comm_handle);
hypre_TFree(int_buf_data);
}
/* Eliminate boundary vertex entries in G^t */
{
hypre_CSRMatrix *Gtd = hypre_ParCSRMatrixDiag(Gt);
HYPRE_Int *GtdI = hypre_CSRMatrixI(Gtd);
HYPRE_Int *GtdJ = hypre_CSRMatrixJ(Gtd);
HYPRE_Real *GtdA = hypre_CSRMatrixData(Gtd);
hypre_CSRMatrix *Gto = hypre_ParCSRMatrixOffd(Gt);
HYPRE_Int *GtoI = hypre_CSRMatrixI(Gto);
HYPRE_Int *GtoJ = hypre_CSRMatrixJ(Gto);
HYPRE_Real *GtoA = hypre_CSRMatrixData(Gto);
HYPRE_Int bdr;
for (i = 0; i < nv; i++)
{
bdr = 0;
/* A vertex is boundary if it belongs to a boundary edge */
for (j = GtdI[i]; j < GtdI[i+1]; j++)
if (edge_bc[GtdJ[j]]) { bdr = 1; break; }
if (!bdr && GtoI)
for (j = GtoI[i]; j < GtoI[i+1]; j++)
if (offd_edge_bc[GtoJ[j]]) { bdr = 1; break; }
if (bdr)
{
for (j = GtdI[i]; j < GtdI[i+1]; j++)
/* if (!edge_bc[GtdJ[j]]) */
GtdA[j] = 0.0;
if (GtoI)
for (j = GtoI[i]; j < GtoI[i+1]; j++)
/* if (!offd_edge_bc[GtoJ[j]]) */
GtoA[j] = 0.0;
}
}
}
hypre_ParCSRMatrixTranspose(Gt, &ame_data -> G, 1);
hypre_ParCSRMatrixDestroy(Gt);
hypre_TFree(offd_edge_bc);
}
/* Compute G^t M G */
{
if (!hypre_ParCSRMatrixCommPkg(ame_data -> G))
hypre_MatvecCommPkgCreate(ame_data -> G);
if (!hypre_ParCSRMatrixCommPkg(ame_data -> M))
hypre_MatvecCommPkgCreate(ame_data -> M);
hypre_BoomerAMGBuildCoarseOperator(ame_data -> G,
ame_data -> M,
ame_data -> G,
&ame_data -> A_G);
hypre_ParCSRMatrixFixZeroRows(ame_data -> A_G);
}
/* Create AMG preconditioner and PCG-AMG solver for G^tMG */
{
HYPRE_BoomerAMGCreate(&ame_data -> B1_G);
HYPRE_BoomerAMGSetCoarsenType(ame_data -> B1_G, ams_data -> B_G_coarsen_type);
HYPRE_BoomerAMGSetAggNumLevels(ame_data -> B1_G, ams_data -> B_G_agg_levels);
HYPRE_BoomerAMGSetRelaxType(ame_data -> B1_G, ams_data -> B_G_relax_type);
HYPRE_BoomerAMGSetNumSweeps(ame_data -> B1_G, 1);
HYPRE_BoomerAMGSetMaxLevels(ame_data -> B1_G, 25);
HYPRE_BoomerAMGSetTol(ame_data -> B1_G, 0.0);
HYPRE_BoomerAMGSetMaxIter(ame_data -> B1_G, 1);
HYPRE_BoomerAMGSetStrongThreshold(ame_data -> B1_G, ams_data -> B_G_theta);
/* don't use exact solve on the coarsest level (matrix may be singular) */
HYPRE_BoomerAMGSetCycleRelaxType(ame_data -> B1_G,
ams_data -> B_G_relax_type,
3);
HYPRE_ParCSRPCGCreate(hypre_ParCSRMatrixComm(ame_data->A_G),
&ame_data -> B2_G);
HYPRE_PCGSetPrintLevel(ame_data -> B2_G, 0);
HYPRE_PCGSetTol(ame_data -> B2_G, 1e-12);
HYPRE_PCGSetMaxIter(ame_data -> B2_G, 20);
HYPRE_PCGSetPrecond(ame_data -> B2_G,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
ame_data -> B1_G);
HYPRE_ParCSRPCGSetup(ame_data -> B2_G,
(HYPRE_ParCSRMatrix)ame_data->A_G,
(HYPRE_ParVector)ame_data->t1,
(HYPRE_ParVector)ame_data->t2);
}
/* Setup LOBPCG */
{
HYPRE_Int seed = 75;
mv_InterfaceInterpreter* interpreter;
mv_MultiVectorPtr eigenvectors;
ame_data -> interpreter = hypre_CTAlloc(mv_InterfaceInterpreter,1);
interpreter = (mv_InterfaceInterpreter*) ame_data -> interpreter;
HYPRE_ParCSRSetupInterpreter(interpreter);
ame_data -> eigenvalues = hypre_CTAlloc(HYPRE_Real, ame_data -> block_size);
ame_data -> eigenvectors =
mv_MultiVectorCreateFromSampleVector(interpreter,
ame_data -> block_size,
ame_data -> t3);
eigenvectors = (mv_MultiVectorPtr) ame_data -> eigenvectors;
mv_MultiVectorSetRandom (eigenvectors, seed);
/* Make the initial vectors discretely divergence free */
{
HYPRE_Int i, j;
HYPRE_Real *data;
mv_TempMultiVector* tmp = mv_MultiVectorGetData(eigenvectors);
HYPRE_ParVector *v = (HYPRE_ParVector*)(tmp -> vector);
hypre_ParVector *vi;
for (i = 0; i < ame_data -> block_size; i++)
{
vi = (hypre_ParVector*) v[i];
data = hypre_VectorData(hypre_ParVectorLocalVector(vi));
for (j = 0; j < ne; j++)
if (edge_bc[j])
data[j] = 0.0;
hypre_AMEDiscrDivFreeComponent(esolver, vi);
}
}
}
hypre_TFree(edge_bc);
return hypre_error_flag;
}
/* ----------------------------------------------------------------------
* set_element
*
* Sets single element in hypre vector by accessing its raw block.
* Probably not the most efficient way to set the entire vector.
* --------------------------------------------------------------------*/
void set_element(N_Vector X, long int i, realtype val)
{
hypre_ParVector *Xvec = N_VGetVector_ParHyp(X);
realtype *Xdata = hypre_VectorData(hypre_ParVectorLocalVector(Xvec));
Xdata[i] = val;
}
HYPRE_Int
hypre_ParCSRMatrixMatvecT( double alpha,
hypre_ParCSRMatrix *A,
hypre_ParVector *x,
double beta,
hypre_ParVector *y )
{
hypre_ParCSRCommHandle **comm_handle;
hypre_ParCSRCommPkg *comm_pkg = hypre_ParCSRMatrixCommPkg(A);
hypre_CSRMatrix *diag = hypre_ParCSRMatrixDiag(A);
hypre_CSRMatrix *offd = hypre_ParCSRMatrixOffd(A);
hypre_Vector *x_local = hypre_ParVectorLocalVector(x);
hypre_Vector *y_local = hypre_ParVectorLocalVector(y);
hypre_Vector *y_tmp;
HYPRE_Int vecstride = hypre_VectorVectorStride( y_local );
HYPRE_Int idxstride = hypre_VectorIndexStride( y_local );
double *y_tmp_data, **y_buf_data;
double *y_local_data = hypre_VectorData(y_local);
HYPRE_Int num_rows = hypre_ParCSRMatrixGlobalNumRows(A);
HYPRE_Int num_cols = hypre_ParCSRMatrixGlobalNumCols(A);
HYPRE_Int num_cols_offd = hypre_CSRMatrixNumCols(offd);
HYPRE_Int x_size = hypre_ParVectorGlobalSize(x);
HYPRE_Int y_size = hypre_ParVectorGlobalSize(y);
HYPRE_Int num_vectors = hypre_VectorNumVectors(y_local);
HYPRE_Int i, j, jv, index, start, num_sends;
HYPRE_Int ierr = 0;
/*---------------------------------------------------------------------
* Check for size compatibility. MatvecT returns ierr = 1 if
* length of X doesn't equal the number of rows of A,
* ierr = 2 if the length of Y doesn't equal the number of
* columns of A, and ierr = 3 if both are true.
*
* Because temporary vectors are often used in MatvecT, none of
* these conditions terminates processing, and the ierr flag
* is informational only.
*--------------------------------------------------------------------*/
if (num_rows != x_size)
ierr = 1;
if (num_cols != y_size)
ierr = 2;
if (num_rows != x_size && num_cols != y_size)
ierr = 3;
/*-----------------------------------------------------------------------
*-----------------------------------------------------------------------*/
comm_handle = hypre_CTAlloc(hypre_ParCSRCommHandle*,num_vectors);
if ( num_vectors==1 )
{
y_tmp = hypre_SeqVectorCreate(num_cols_offd);
}
else
{
y_tmp = hypre_SeqMultiVectorCreate(num_cols_offd,num_vectors);
}
hypre_SeqVectorInitialize(y_tmp);
/*---------------------------------------------------------------------
* If there exists no CommPkg for A, a CommPkg is generated using
* equally load balanced partitionings
*--------------------------------------------------------------------*/
if (!comm_pkg)
{
hypre_MatvecCommPkgCreate(A);
comm_pkg = hypre_ParCSRMatrixCommPkg(A);
}
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
y_buf_data = hypre_CTAlloc( double*, num_vectors );
for ( jv=0; jv<num_vectors; ++jv )
y_buf_data[jv] = hypre_CTAlloc(double, hypre_ParCSRCommPkgSendMapStart
(comm_pkg, num_sends));
y_tmp_data = hypre_VectorData(y_tmp);
y_local_data = hypre_VectorData(y_local);
hypre_assert( idxstride==1 ); /* >>> only 'column' storage of multivectors implemented so far */
if (num_cols_offd) hypre_CSRMatrixMatvecT(alpha, offd, x_local, 0.0, y_tmp);
for ( jv=0; jv<num_vectors; ++jv )
{
/* >>> this is where we assume multivectors are 'column' storage */
comm_handle[jv] = hypre_ParCSRCommHandleCreate
( 2, comm_pkg, &(y_tmp_data[jv*num_cols_offd]), y_buf_data[jv] );
}
hypre_CSRMatrixMatvecT(alpha, diag, x_local, beta, y_local);
for ( jv=0; jv<num_vectors; ++jv )
{
hypre_ParCSRCommHandleDestroy(comm_handle[jv]);
comm_handle[jv] = NULL;
}
hypre_TFree(comm_handle);
if ( num_vectors==1 )
{
index = 0;
for (i = 0; i < num_sends; i++)
{
start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
y_local_data[hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j)]
+= y_buf_data[0][index++];
}
}
else
for ( jv=0; jv<num_vectors; ++jv )
{
index = 0;
for (i = 0; i < num_sends; i++)
{
start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
y_local_data[ jv*vecstride +
idxstride*hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j) ]
+= y_buf_data[jv][index++];
}
}
hypre_SeqVectorDestroy(y_tmp);
y_tmp = NULL;
for ( jv=0; jv<num_vectors; ++jv ) hypre_TFree(y_buf_data[jv]);
hypre_TFree(y_buf_data);
return ierr;
}
HYPRE_Int
hypre_ParVectorPrintIJ( hypre_ParVector *vector,
HYPRE_Int base_j,
const char *filename )
{
MPI_Comm comm;
HYPRE_Int global_size;
HYPRE_Int *partitioning;
double *local_data;
HYPRE_Int myid, num_procs, i, j, part0;
char new_filename[255];
FILE *file;
if (!vector)
{
hypre_error_in_arg(1);
return hypre_error_flag;
}
comm = hypre_ParVectorComm(vector);
global_size = hypre_ParVectorGlobalSize(vector);
partitioning = hypre_ParVectorPartitioning(vector);
/* multivector code not written yet >>> */
hypre_assert( hypre_ParVectorNumVectors(vector) == 1 );
if ( hypre_ParVectorNumVectors(vector) != 1 ) hypre_error_in_arg(1);
hypre_MPI_Comm_rank(comm, &myid);
hypre_MPI_Comm_size(comm, &num_procs);
hypre_sprintf(new_filename,"%s.%05d", filename, myid);
if ((file = fopen(new_filename, "w")) == NULL)
{
hypre_printf("Error: can't open output file %s\n", new_filename);
hypre_error_in_arg(3);
return hypre_error_flag;
}
local_data = hypre_VectorData(hypre_ParVectorLocalVector(vector));
hypre_fprintf(file, "%d \n", global_size);
#ifdef HYPRE_NO_GLOBAL_PARTITION
for (i=0; i <= 2; i++)
#else
for (i=0; i <= num_procs; i++)
#endif
{
hypre_fprintf(file, "%d \n", partitioning[i] + base_j);
}
#ifdef HYPRE_NO_GLOBAL_PARTITION
part0 = partitioning[0];
for (j = part0; j < partitioning[1]; j++)
#else
part0 = partitioning[myid];
for (j = part0; j < partitioning[myid+1]; j++)
#endif
{
hypre_fprintf(file, "%d %.14e\n", j + base_j, local_data[j-part0]);
}
fclose(file);
return hypre_error_flag;
}
hypre_ParVector *
hypre_ParVectorCreateFromBlock( MPI_Comm comm,
HYPRE_Int p_global_size,
HYPRE_Int *p_partitioning, HYPRE_Int block_size)
{
hypre_ParVector *vector;
HYPRE_Int num_procs, my_id, i;
HYPRE_Int global_size;
HYPRE_Int *new_partitioning; /* need to create a new partitioning - son't want to write over
what is passed in */
global_size = p_global_size*block_size;
vector = hypre_CTAlloc(hypre_ParVector, 1);
hypre_MPI_Comm_rank(comm,&my_id);
hypre_MPI_Comm_size(comm,&num_procs);
if (!p_partitioning)
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
hypre_GenerateLocalPartitioning(global_size, num_procs, my_id, &new_partitioning);
#else
hypre_GeneratePartitioning(global_size, num_procs, &new_partitioning);
#endif
}
else /* adjust for block_size */
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
new_partitioning = hypre_CTAlloc(HYPRE_Int, 2);
for(i = 0; i < 2; i++)
{
new_partitioning[i] = p_partitioning[i]*block_size;
}
#else
new_partitioning = hypre_CTAlloc(HYPRE_Int, num_procs + 1);
for(i = 0; i < num_procs + 1; i++)
{
new_partitioning[i] = p_partitioning[i]*block_size;
}
#endif
}
hypre_ParVectorComm(vector) = comm;
hypre_ParVectorGlobalSize(vector) = global_size;
#ifdef HYPRE_NO_GLOBAL_PARTITION
hypre_ParVectorFirstIndex(vector) = new_partitioning[0];
hypre_ParVectorLastIndex(vector) = new_partitioning[1]-1;
hypre_ParVectorPartitioning(vector) = new_partitioning;
hypre_ParVectorLocalVector(vector) =
hypre_SeqVectorCreate(new_partitioning[1]-new_partitioning[0]);
#else
hypre_ParVectorFirstIndex(vector) = new_partitioning[my_id];
hypre_ParVectorLastIndex(vector) = new_partitioning[my_id+1] -1;
hypre_ParVectorPartitioning(vector) = new_partitioning;
hypre_ParVectorLocalVector(vector) =
hypre_SeqVectorCreate(new_partitioning[my_id+1]-new_partitioning[my_id]);
#endif
/* set defaults */
hypre_ParVectorOwnsData(vector) = 1;
hypre_ParVectorOwnsPartitioning(vector) = 1;
return vector;
}
HYPRE_Int
main( HYPRE_Int argc,
char *argv[] )
{
hypre_ParVector *vector1;
hypre_ParVector *vector2;
hypre_ParVector *tmp_vector;
HYPRE_Int num_procs, my_id;
HYPRE_Int global_size = 20;
HYPRE_Int local_size;
HYPRE_Int first_index;
HYPRE_Int i;
HYPRE_Int *partitioning;
HYPRE_Complex prod;
HYPRE_Complex *data, *data2;
hypre_Vector *vector;
hypre_Vector *local_vector;
hypre_Vector *local_vector2;
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id );
hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs);
partitioning = NULL;
vector1 = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning);
partitioning = hypre_ParVectorPartitioning(vector1);
hypre_ParVectorInitialize(vector1);
local_vector = hypre_ParVectorLocalVector(vector1);
data = hypre_VectorData(local_vector);
local_size = hypre_VectorSize(local_vector);
first_index = partitioning[my_id];
for (i=0; i < local_size; i++)
data[i] = first_index+i;
/*
hypre_ParVectorPrint(vector1, "Vector");
*/
local_vector2 = hypre_SeqVectorCreate(global_size);
hypre_SeqVectorInitialize(local_vector2);
data2 = hypre_VectorData(local_vector2);
for (i=0; i < global_size; i++)
data2[i] = i+1;
/* partitioning = hypre_CTAlloc(HYPRE_Int,4);
partitioning[0] = 0;
partitioning[1] = 10;
partitioning[2] = 10;
partitioning[3] = 20;
*/
vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning);
hypre_ParVectorSetPartitioningOwner(vector2,0);
hypre_ParVectorPrint(vector2, "Convert");
vector = hypre_ParVectorToVectorAll(vector2);
/*-----------------------------------------------------------
* Copy the vector into tmp_vector
*-----------------------------------------------------------*/
tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert");
/*
tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning);
hypre_ParVectorSetPartitioningOwner(tmp_vector,0);
hypre_ParVectorInitialize(tmp_vector);
hypre_ParVectorCopy(vector1, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Copy");
*/
/*-----------------------------------------------------------
* Scale tmp_vector
*-----------------------------------------------------------*/
hypre_ParVectorScale(2.0, tmp_vector);
/*
hypre_ParVectorPrint(tmp_vector,"Scale");
*/
/*-----------------------------------------------------------
* Do an Axpy (2*vector - vector) = vector
*-----------------------------------------------------------*/
hypre_ParVectorAxpy(-1.0, vector1, tmp_vector);
/*
hypre_ParVectorPrint(tmp_vector,"Axpy");
*/
/*-----------------------------------------------------------
* Do an inner product vector* tmp_vector
*-----------------------------------------------------------*/
prod = hypre_ParVectorInnerProd(vector1, tmp_vector);
hypre_printf (" prod: %8.2f \n", prod);
/*-----------------------------------------------------------
* Finalize things
*-----------------------------------------------------------*/
hypre_ParVectorDestroy(vector1);
hypre_ParVectorDestroy(vector2);
hypre_ParVectorDestroy(tmp_vector);
hypre_SeqVectorDestroy(local_vector2);
if (vector) hypre_SeqVectorDestroy(vector);
/* Finalize MPI */
hypre_MPI_Finalize();
return 0;
}
hypre_ParVector *
hypre_VectorToParVector (MPI_Comm comm, hypre_Vector *v, HYPRE_Int *vec_starts)
{
HYPRE_Int global_size;
HYPRE_Int local_size;
HYPRE_Int num_vectors;
HYPRE_Int num_procs, my_id;
HYPRE_Int global_vecstride, vecstride, idxstride;
hypre_ParVector *par_vector;
hypre_Vector *local_vector;
double *v_data;
double *local_data;
hypre_MPI_Request *requests;
hypre_MPI_Status *status, status0;
HYPRE_Int i, j, k, p;
hypre_MPI_Comm_size(comm,&num_procs);
hypre_MPI_Comm_rank(comm,&my_id);
if (my_id == 0)
{
global_size = hypre_VectorSize(v);
v_data = hypre_VectorData(v);
num_vectors = hypre_VectorNumVectors(v); /* for multivectors */
global_vecstride = hypre_VectorVectorStride(v);
}
hypre_MPI_Bcast(&global_size,1,HYPRE_MPI_INT,0,comm);
hypre_MPI_Bcast(&num_vectors,1,HYPRE_MPI_INT,0,comm);
hypre_MPI_Bcast(&global_vecstride,1,HYPRE_MPI_INT,0,comm);
if ( num_vectors==1 )
par_vector = hypre_ParVectorCreate(comm, global_size, vec_starts);
else
par_vector = hypre_ParMultiVectorCreate(comm, global_size, vec_starts, num_vectors);
vec_starts = hypre_ParVectorPartitioning(par_vector);
local_size = vec_starts[my_id+1] - vec_starts[my_id];
hypre_ParVectorInitialize(par_vector);
local_vector = hypre_ParVectorLocalVector(par_vector);
local_data = hypre_VectorData(local_vector);
vecstride = hypre_VectorVectorStride(local_vector);
idxstride = hypre_VectorIndexStride(local_vector);
hypre_assert( idxstride==1 ); /* <<< so far only the only implemented multivector StorageMethod is 0 <<< */
if (my_id == 0)
{
requests = hypre_CTAlloc(hypre_MPI_Request,num_vectors*(num_procs-1));
status = hypre_CTAlloc(hypre_MPI_Status,num_vectors*(num_procs-1));
k = 0;
for ( p=1; p<num_procs; p++)
for ( j=0; j<num_vectors; ++j )
{
hypre_MPI_Isend( &v_data[vec_starts[p]]+j*global_vecstride,
(vec_starts[p+1]-vec_starts[p]),
hypre_MPI_DOUBLE, p, 0, comm, &requests[k++] );
}
if ( num_vectors==1 )
{
for (i=0; i < local_size; i++)
local_data[i] = v_data[i];
}
else
for ( j=0; j<num_vectors; ++j )
{
for (i=0; i < local_size; i++)
local_data[i+j*vecstride] = v_data[i+j*global_vecstride];
}
hypre_MPI_Waitall(num_procs-1,requests, status);
hypre_TFree(requests);
hypre_TFree(status);
}
else
{
for ( j=0; j<num_vectors; ++j )
hypre_MPI_Recv( local_data+j*vecstride, local_size, hypre_MPI_DOUBLE, 0, 0, comm,&status0 );
}
return par_vector;
}
HYPRE_Int
main( HYPRE_Int argc,
char *argv[] )
{
hypre_CSRMatrix *matrix;
hypre_CSRMatrix *matrix1;
hypre_ParCSRMatrix *par_matrix;
hypre_Vector *x_local;
hypre_Vector *y_local;
hypre_Vector *y2_local;
hypre_ParVector *x;
hypre_ParVector *x2;
hypre_ParVector *y;
hypre_ParVector *y2;
HYPRE_Int vecstride_x, idxstride_x, vecstride_y, idxstride_y;
HYPRE_Int num_procs, my_id;
HYPRE_Int local_size;
HYPRE_Int num_vectors;
HYPRE_Int global_num_rows, global_num_cols;
HYPRE_Int first_index;
HYPRE_Int i, j, ierr=0;
double *data, *data2;
HYPRE_Int *row_starts, *col_starts;
char file_name[80];
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs);
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id);
hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs);
if (my_id == 0)
{
matrix = hypre_CSRMatrixRead("input");
hypre_printf(" read input\n");
}
row_starts = NULL;
col_starts = NULL;
par_matrix = hypre_CSRMatrixToParCSRMatrix(hypre_MPI_COMM_WORLD, matrix,
row_starts, col_starts);
hypre_printf(" converted\n");
matrix1 = hypre_ParCSRMatrixToCSRMatrixAll(par_matrix);
hypre_sprintf(file_name,"matrix1.%d",my_id);
if (matrix1) hypre_CSRMatrixPrint(matrix1, file_name);
hypre_ParCSRMatrixPrint(par_matrix,"matrix");
hypre_ParCSRMatrixPrintIJ(par_matrix,0,0,"matrixIJ");
par_matrix = hypre_ParCSRMatrixRead(hypre_MPI_COMM_WORLD,"matrix");
global_num_cols = hypre_ParCSRMatrixGlobalNumCols(par_matrix);
hypre_printf(" global_num_cols %d\n", global_num_cols);
global_num_rows = hypre_ParCSRMatrixGlobalNumRows(par_matrix);
col_starts = hypre_ParCSRMatrixColStarts(par_matrix);
first_index = col_starts[my_id];
local_size = col_starts[my_id+1] - first_index;
num_vectors = 3;
x = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_cols,
col_starts, num_vectors );
hypre_ParVectorSetPartitioningOwner(x,0);
hypre_ParVectorInitialize(x);
x_local = hypre_ParVectorLocalVector(x);
data = hypre_VectorData(x_local);
vecstride_x = hypre_VectorVectorStride(x_local);
idxstride_x = hypre_VectorIndexStride(x_local);
for ( j=0; j<num_vectors; ++j )
for (i=0; i < local_size; i++)
data[i*idxstride_x + j*vecstride_x] = first_index+i+1 + 100*j;
x2 = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_cols,
col_starts, num_vectors );
hypre_ParVectorSetPartitioningOwner(x2,0);
hypre_ParVectorInitialize(x2);
hypre_ParVectorSetConstantValues(x2,2.0);
row_starts = hypre_ParCSRMatrixRowStarts(par_matrix);
first_index = row_starts[my_id];
local_size = row_starts[my_id+1] - first_index;
y = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_rows,
row_starts, num_vectors );
hypre_ParVectorSetPartitioningOwner(y,0);
hypre_ParVectorInitialize(y);
y_local = hypre_ParVectorLocalVector(y);
y2 = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_rows,
row_starts, num_vectors );
hypre_ParVectorSetPartitioningOwner(y2,0);
hypre_ParVectorInitialize(y2);
y2_local = hypre_ParVectorLocalVector(y2);
data2 = hypre_VectorData(y2_local);
vecstride_y = hypre_VectorVectorStride(y2_local);
idxstride_y = hypre_VectorIndexStride(y2_local);
for ( j=0; j<num_vectors; ++j )
for (i=0; i < local_size; i++)
data2[i*idxstride_y+j*vecstride_y] = first_index+i+1 + 100*j;
hypre_ParVectorSetConstantValues(y,1.0);
hypre_printf(" initialized vectors, first_index=%i\n", first_index);
hypre_ParVectorPrint(x, "vectorx");
hypre_ParVectorPrint(y, "vectory");
hypre_MatvecCommPkgCreate(par_matrix);
hypre_ParCSRMatrixMatvec ( 1.0, par_matrix, x, 1.0, y);
hypre_printf(" did matvec\n");
hypre_ParVectorPrint(y, "result");
ierr = hypre_ParCSRMatrixMatvecT ( 1.0, par_matrix, y2, 1.0, x2);
hypre_printf(" did matvecT %d\n", ierr);
hypre_ParVectorPrint(x2, "transp");
hypre_ParCSRMatrixDestroy(par_matrix);
hypre_ParVectorDestroy(x);
hypre_ParVectorDestroy(x2);
hypre_ParVectorDestroy(y);
hypre_ParVectorDestroy(y2);
if (my_id == 0) hypre_CSRMatrixDestroy(matrix);
if (matrix1) hypre_CSRMatrixDestroy(matrix1);
/* Finalize MPI */
hypre_MPI_Finalize();
return 0;
}
HYPRE_Int
hypre_seqAMGCycle( hypre_ParAMGData *amg_data,
HYPRE_Int p_level,
hypre_ParVector **Par_F_array,
hypre_ParVector **Par_U_array )
{
hypre_ParVector *Aux_U;
hypre_ParVector *Aux_F;
/* Local variables */
HYPRE_Int Solve_err_flag = 0;
HYPRE_Int n;
HYPRE_Int i;
hypre_Vector *u_local;
HYPRE_Real *u_data;
HYPRE_Int first_index;
/* Acquire seq data */
MPI_Comm new_comm = hypre_ParAMGDataNewComm(amg_data);
HYPRE_Solver coarse_solver = hypre_ParAMGDataCoarseSolver(amg_data);
hypre_ParCSRMatrix *A_coarse = hypre_ParAMGDataACoarse(amg_data);
hypre_ParVector *F_coarse = hypre_ParAMGDataFCoarse(amg_data);
hypre_ParVector *U_coarse = hypre_ParAMGDataUCoarse(amg_data);
HYPRE_Int redundant = hypre_ParAMGDataRedundant(amg_data);
Aux_U = Par_U_array[p_level];
Aux_F = Par_F_array[p_level];
first_index = hypre_ParVectorFirstIndex(Aux_U);
u_local = hypre_ParVectorLocalVector(Aux_U);
u_data = hypre_VectorData(u_local);
n = hypre_VectorSize(u_local);
/*if (A_coarse)*/
if (hypre_ParAMGDataParticipate(amg_data))
{
HYPRE_Real *f_data;
hypre_Vector *f_local;
hypre_Vector *tmp_vec;
HYPRE_Int nf;
HYPRE_Int local_info;
HYPRE_Real *recv_buf = NULL;
HYPRE_Int *displs = NULL;
HYPRE_Int *info = NULL;
HYPRE_Int size;
HYPRE_Int new_num_procs, my_id;
hypre_MPI_Comm_size(new_comm, &new_num_procs);
hypre_MPI_Comm_rank(new_comm, &my_id);
f_local = hypre_ParVectorLocalVector(Aux_F);
f_data = hypre_VectorData(f_local);
nf = hypre_VectorSize(f_local);
/* first f */
info = hypre_CTAlloc(HYPRE_Int, new_num_procs);
local_info = nf;
if (redundant)
hypre_MPI_Allgather(&local_info, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, new_comm);
else
hypre_MPI_Gather(&local_info, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, 0, new_comm);
if (redundant || my_id ==0)
{
displs = hypre_CTAlloc(HYPRE_Int, new_num_procs+1);
displs[0] = 0;
for (i=1; i < new_num_procs+1; i++)
displs[i] = displs[i-1]+info[i-1];
size = displs[new_num_procs];
if (F_coarse)
{
tmp_vec = hypre_ParVectorLocalVector(F_coarse);
recv_buf = hypre_VectorData(tmp_vec);
}
}
if (redundant)
hypre_MPI_Allgatherv ( f_data, nf, HYPRE_MPI_REAL,
recv_buf, info, displs,
HYPRE_MPI_REAL, new_comm );
else
hypre_MPI_Gatherv ( f_data, nf, HYPRE_MPI_REAL,
recv_buf, info, displs,
HYPRE_MPI_REAL, 0, new_comm );
if (redundant || my_id ==0)
{
tmp_vec = hypre_ParVectorLocalVector(U_coarse);
recv_buf = hypre_VectorData(tmp_vec);
}
/*then u */
if (redundant)
{
hypre_MPI_Allgatherv ( u_data, n, HYPRE_MPI_REAL,
recv_buf, info, displs,
HYPRE_MPI_REAL, new_comm );
hypre_TFree(displs);
hypre_TFree(info);
}
else
hypre_MPI_Gatherv ( u_data, n, HYPRE_MPI_REAL,
recv_buf, info, displs,
HYPRE_MPI_REAL, 0, new_comm );
/* clean up */
if (redundant || my_id ==0)
{
hypre_BoomerAMGSolve(coarse_solver, A_coarse, F_coarse, U_coarse);
}
/*copy my part of U to parallel vector */
if (redundant)
{
HYPRE_Real *local_data;
local_data = hypre_VectorData(hypre_ParVectorLocalVector(U_coarse));
for (i = 0; i < n; i++)
{
u_data[i] = local_data[first_index+i];
}
}
else
{
HYPRE_Real *local_data=NULL;
if (my_id == 0)
local_data = hypre_VectorData(hypre_ParVectorLocalVector(U_coarse));
hypre_MPI_Scatterv ( local_data, info, displs, HYPRE_MPI_REAL,
u_data, n, HYPRE_MPI_REAL, 0, new_comm );
/*if (my_id == 0)
local_data = hypre_VectorData(hypre_ParVectorLocalVector(F_coarse));
hypre_MPI_Scatterv ( local_data, info, displs, HYPRE_MPI_REAL,
f_data, n, HYPRE_MPI_REAL, 0, new_comm );*/
if (my_id == 0) hypre_TFree(displs);
hypre_TFree(info);
}
}
return(Solve_err_flag);
}
/******************************************************************************
*
* hypre_IJVectorAddToValuesPar
*
* adds to a potentially noncontiguous set of IJVectorPar components
*
*****************************************************************************/
HYPRE_Int
hypre_IJVectorAddToValuesPar(hypre_IJVector *vector,
HYPRE_Int num_values,
const HYPRE_Int *indices,
const double *values )
{
HYPRE_Int my_id;
HYPRE_Int i, j, vec_start, vec_stop;
double *data;
HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector);
HYPRE_Int *IJpartitioning = hypre_IJVectorPartitioning(vector);
hypre_ParVector *par_vector = hypre_IJVectorObject(vector);
hypre_AuxParVector *aux_vector = hypre_IJVectorTranslator(vector);
MPI_Comm comm = hypre_IJVectorComm(vector);
hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector);
/* If no components are to be retrieved, perform no checking and return */
if (num_values < 1) return 0;
hypre_MPI_Comm_rank(comm, &my_id);
/* If par_vector == NULL or partitioning == NULL or local_vector == NULL
let user know of catastrophe and exit */
if (!par_vector)
{
if (print_level)
{
hypre_printf("par_vector == NULL -- ");
hypre_printf("hypre_IJVectorAddToValuesPar\n");
hypre_printf("**** Vector storage is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
if (!IJpartitioning)
{
if (print_level)
{
hypre_printf("IJpartitioning == NULL -- ");
hypre_printf("hypre_IJVectorAddToValuesPar\n");
hypre_printf("**** IJVector partitioning is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
if (!local_vector)
{
if (print_level)
{
hypre_printf("local_vector == NULL -- ");
hypre_printf("hypre_IJVectorAddToValuesPar\n");
hypre_printf("**** Vector local data is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
#ifdef HYPRE_NO_GLOBAL_PARTITION
vec_start = IJpartitioning[0];
vec_stop = IJpartitioning[1]-1;
#else
vec_start = IJpartitioning[my_id];
vec_stop = IJpartitioning[my_id+1]-1;
#endif
if (vec_start > vec_stop)
{
if (print_level)
{
hypre_printf("vec_start > vec_stop -- ");
hypre_printf("hypre_IJVectorAddToValuesPar\n");
hypre_printf("**** This vector partitioning should not occur ****\n");
}
hypre_error_in_arg(1);
}
/* Determine whether indices points to local indices only,
and if not, store indices and values into auxiliary vector structure
If indices == NULL, assume that num_values components are to be
set in a block starting at vec_start.
NOTE: If indices == NULL off processor values are ignored!!! */
/* if (indices)
{
for (i = 0; i < num_values; i++)
{
ierr += (indices[i] < vec_start);
ierr += (indices[i] >= vec_stop);
}
}
if (ierr)
{
hypre_printf("indices beyond local range -- ");
hypre_printf("hypre_IJVectorAddToValuesPar\n");
hypre_printf("**** Indices specified are unusable ****\n");
exit(1);
} */
data = hypre_VectorData(local_vector);
if (indices)
{
HYPRE_Int current_num_elmts
= hypre_AuxParVectorCurrentNumElmts(aux_vector);
HYPRE_Int max_off_proc_elmts
= hypre_AuxParVectorMaxOffProcElmts(aux_vector);
HYPRE_Int *off_proc_i = hypre_AuxParVectorOffProcI(aux_vector);
double *off_proc_data = hypre_AuxParVectorOffProcData(aux_vector);
for (j = 0; j < num_values; j++)
{
i = indices[j];
if (i < vec_start || i > vec_stop)
{
/* if elements outside processor boundaries, store in off processor
stash */
if (!max_off_proc_elmts)
{
max_off_proc_elmts = 100;
hypre_AuxParVectorMaxOffProcElmts(aux_vector) =
max_off_proc_elmts;
hypre_AuxParVectorOffProcI(aux_vector)
= hypre_CTAlloc(HYPRE_Int,max_off_proc_elmts);
hypre_AuxParVectorOffProcData(aux_vector)
= hypre_CTAlloc(double,max_off_proc_elmts);
off_proc_i = hypre_AuxParVectorOffProcI(aux_vector);
off_proc_data = hypre_AuxParVectorOffProcData(aux_vector);
}
else if (current_num_elmts + 1 > max_off_proc_elmts)
{
max_off_proc_elmts += 10;
off_proc_i = hypre_TReAlloc(off_proc_i,HYPRE_Int,max_off_proc_elmts);
off_proc_data = hypre_TReAlloc(off_proc_data,double,
max_off_proc_elmts);
hypre_AuxParVectorMaxOffProcElmts(aux_vector)
= max_off_proc_elmts;
hypre_AuxParVectorOffProcI(aux_vector) = off_proc_i;
hypre_AuxParVectorOffProcData(aux_vector) = off_proc_data;
}
off_proc_i[current_num_elmts] = i;
off_proc_data[current_num_elmts++] = values[j];
hypre_AuxParVectorCurrentNumElmts(aux_vector)=current_num_elmts;
}
/******************************************************************************
*
* hypre_IJVectorSetValuesPar
*
* sets a potentially noncontiguous set of components of an IJVectorPar
*
*****************************************************************************/
HYPRE_Int
hypre_IJVectorSetValuesPar(hypre_IJVector *vector,
HYPRE_Int num_values,
const HYPRE_Int *indices,
const double *values )
{
HYPRE_Int my_id;
HYPRE_Int i, j, vec_start, vec_stop;
double *data;
HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector);
HYPRE_Int *IJpartitioning = hypre_IJVectorPartitioning(vector);
hypre_ParVector *par_vector = hypre_IJVectorObject(vector);
hypre_AuxParVector *aux_vector = hypre_IJVectorTranslator(vector);
MPI_Comm comm = hypre_IJVectorComm(vector);
hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector);
/* If no components are to be set, perform no checking and return */
if (num_values < 1) return 0;
hypre_MPI_Comm_rank(comm, &my_id);
/* If par_vector == NULL or partitioning == NULL or local_vector == NULL
let user know of catastrophe and exit */
if (!par_vector)
{
if (print_level)
{
hypre_printf("par_vector == NULL -- ");
hypre_printf("hypre_IJVectorSetValuesPar\n");
hypre_printf("**** Vector storage is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
if (!IJpartitioning)
{
if (print_level)
{
hypre_printf("IJpartitioning == NULL -- ");
hypre_printf("hypre_IJVectorSetValuesPar\n");
hypre_printf("**** IJVector partitioning is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
if (!local_vector)
{
if (print_level)
{
hypre_printf("local_vector == NULL -- ");
hypre_printf("hypre_IJVectorSetValuesPar\n");
hypre_printf("**** Vector local data is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
#ifdef HYPRE_NO_GLOBAL_PARTITION
vec_start = IJpartitioning[0];
vec_stop = IJpartitioning[1]-1;
#else
vec_start = IJpartitioning[my_id];
vec_stop = IJpartitioning[my_id+1]-1;
#endif
if (vec_start > vec_stop)
{
if (print_level)
{
hypre_printf("vec_start > vec_stop -- ");
hypre_printf("hypre_IJVectorSetValuesPar\n");
hypre_printf("**** This vector partitioning should not occur ****\n");
}
hypre_error_in_arg(1);
}
/* Determine whether indices points to local indices only,
and if not, store indices and values into auxiliary vector structure
If indices == NULL, assume that num_values components are to be
set in a block starting at vec_start.
NOTE: If indices == NULL off processor values are ignored!!! */
data = hypre_VectorData(local_vector);
if (indices)
{
HYPRE_Int current_num_elmts
= hypre_AuxParVectorCurrentNumElmts(aux_vector);
/*HYPRE_Int max_off_proc_elmts
= hypre_AuxParVectorMaxOffProcElmts(aux_vector);*/
HYPRE_Int *off_proc_i = hypre_AuxParVectorOffProcI(aux_vector);
/*double *off_proc_data = hypre_AuxParVectorOffProcData(aux_vector);*/
HYPRE_Int cancel_indx = hypre_AuxParVectorCancelIndx(aux_vector);
HYPRE_Int ii;
for (j = 0; j < num_values; j++)
{
i = indices[j];
if (i < vec_start || i > vec_stop)
{
for (ii = 0; ii < current_num_elmts; ii++)
{
if (i == off_proc_i[ii])
{
off_proc_i[ii] = -1;
cancel_indx++;
}
}
hypre_AuxParVectorCancelIndx(aux_vector) = cancel_indx;
}
/* if elements outside processor boundaries, search for previous
occurrences and cancel them */
/* if elements outside processor boundaries, store in off processor
stash */
/*if (!max_off_proc_elmts)
{
max_off_proc_elmts = 100;
hypre_AuxParVectorMaxOffProcElmts(aux_vector) =
max_off_proc_elmts;
hypre_AuxParVectorOffProcI(aux_vector)
= hypre_CTAlloc(HYPRE_Int,max_off_proc_elmts);
hypre_AuxParVectorOffProcData(aux_vector)
= hypre_CTAlloc(double,max_off_proc_elmts);
off_proc_i = hypre_AuxParVectorOffProcI(aux_vector);
off_proc_data = hypre_AuxParVectorOffProcData(aux_vector);
}
else if (current_num_elmts + 1 > max_off_proc_elmts)
{
max_off_proc_elmts += 10;
off_proc_i = hypre_TReAlloc(off_proc_i,HYPRE_Int,max_off_proc_elmts);
off_proc_data = hypre_TReAlloc(off_proc_data,double,
max_off_proc_elmts);
hypre_AuxParVectorMaxOffProcElmts(aux_vector)
= max_off_proc_elmts;
hypre_AuxParVectorOffProcI(aux_vector) = off_proc_i;
hypre_AuxParVectorOffProcData(aux_vector) = off_proc_data;
}
off_proc_i[current_num_elmts] = i;
off_proc_data[current_num_elmts++] = values[j];
hypre_AuxParVectorCurrentNumElmts(aux_vector)=current_num_elmts;
}*/
else /* local values are inserted into the vector */
{
i -= vec_start;
data[i] = values[j];
}
}
}
else
{
if (num_values > vec_stop - vec_start + 1)
{
if (print_level)
{
hypre_printf("Warning! Indices beyond local range not identified!\n ");
hypre_printf("Off processor values have been ignored!\n");
}
num_values = vec_stop - vec_start +1;
}
for (j = 0; j < num_values; j++)
data[j] = values[j];
}
return hypre_error_flag;
}
/******************************************************************************
*
* hypre_IJVectorZeroValuesPar
*
* zeroes all local components of an IJVectorPar
*
*****************************************************************************/
HYPRE_Int
hypre_IJVectorZeroValuesPar(hypre_IJVector *vector)
{
HYPRE_Int my_id;
HYPRE_Int i, vec_start, vec_stop;
double *data;
hypre_ParVector *par_vector = hypre_IJVectorObject(vector);
MPI_Comm comm = hypre_IJVectorComm(vector);
HYPRE_Int *partitioning = hypre_ParVectorPartitioning(par_vector);
hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector);
HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector);
hypre_MPI_Comm_rank(comm, &my_id);
/* If par_vector == NULL or partitioning == NULL or local_vector == NULL
let user know of catastrophe and exit */
if (!par_vector)
{
if (print_level)
{
hypre_printf("par_vector == NULL -- ");
hypre_printf("hypre_IJVectorZeroValuesPar\n");
hypre_printf("**** Vector storage is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
if (!partitioning)
{
if (print_level)
{
hypre_printf("partitioning == NULL -- ");
hypre_printf("hypre_IJVectorZeroValuesPar\n");
hypre_printf("**** Vector partitioning is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
if (!local_vector)
{
if (print_level)
{
hypre_printf("local_vector == NULL -- ");
hypre_printf("hypre_IJVectorZeroValuesPar\n");
hypre_printf("**** Vector local data is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
#ifdef HYPRE_NO_GLOBAL_PARTITION
vec_start = partitioning[0];
vec_stop = partitioning[1];
#else
vec_start = partitioning[my_id];
vec_stop = partitioning[my_id+1];
#endif
if (vec_start > vec_stop)
{
if (print_level)
{
hypre_printf("vec_start > vec_stop -- ");
hypre_printf("hypre_IJVectorZeroValuesPar\n");
hypre_printf("**** This vector partitioning should not occur ****\n");
}
hypre_error_in_arg(1);
}
data = hypre_VectorData( local_vector );
for (i = 0; i < vec_stop - vec_start; i++)
data[i] = 0.;
return hypre_error_flag;
}
HYPRE_Complex hypre_ParVectorLocalSumElts( hypre_ParVector * vector )
{
return hypre_VectorSumElts( hypre_ParVectorLocalVector(vector) );
}
HYPRE_Int hypre_CreateLambda(void *amg_vdata)
{
hypre_ParAMGData *amg_data = amg_vdata;
/* Data Structure variables */
MPI_Comm comm;
hypre_ParCSRMatrix **A_array;
hypre_ParVector **F_array;
hypre_ParVector **U_array;
hypre_ParCSRMatrix *A_tmp;
hypre_ParCSRMatrix *Lambda;
hypre_CSRMatrix *L_diag;
hypre_CSRMatrix *L_offd;
hypre_CSRMatrix *A_tmp_diag;
hypre_CSRMatrix *A_tmp_offd;
hypre_ParVector *Xtilde;
hypre_ParVector *Rtilde;
hypre_Vector *Xtilde_local;
hypre_Vector *Rtilde_local;
hypre_ParCSRCommPkg *comm_pkg;
hypre_ParCSRCommPkg *L_comm_pkg = NULL;
hypre_ParCSRCommHandle *comm_handle;
HYPRE_Real *L_diag_data;
HYPRE_Real *L_offd_data;
HYPRE_Real *buf_data = NULL;
HYPRE_Real *tmp_data;
HYPRE_Real *x_data;
HYPRE_Real *r_data;
HYPRE_Real *l1_norms;
HYPRE_Real *A_tmp_diag_data;
HYPRE_Real *A_tmp_offd_data;
HYPRE_Real *D_data = NULL;
HYPRE_Real *D_data_offd = NULL;
HYPRE_Int *L_diag_i;
HYPRE_Int *L_diag_j;
HYPRE_Int *L_offd_i;
HYPRE_Int *L_offd_j;
HYPRE_Int *A_tmp_diag_i;
HYPRE_Int *A_tmp_offd_i;
HYPRE_Int *A_tmp_diag_j;
HYPRE_Int *A_tmp_offd_j;
HYPRE_Int *L_recv_ptr = NULL;
HYPRE_Int *L_send_ptr = NULL;
HYPRE_Int *L_recv_procs = NULL;
HYPRE_Int *L_send_procs = NULL;
HYPRE_Int *L_send_map_elmts = NULL;
HYPRE_Int *recv_procs;
HYPRE_Int *send_procs;
HYPRE_Int *send_map_elmts;
HYPRE_Int *send_map_starts;
HYPRE_Int *recv_vec_starts;
HYPRE_Int *all_send_procs = NULL;
HYPRE_Int *all_recv_procs = NULL;
HYPRE_Int *remap = NULL;
HYPRE_Int *level_start;
HYPRE_Int addlvl;
HYPRE_Int additive;
HYPRE_Int mult_additive;
HYPRE_Int num_levels;
HYPRE_Int num_add_lvls;
HYPRE_Int num_procs;
HYPRE_Int num_sends, num_recvs;
HYPRE_Int num_sends_L = 0;
HYPRE_Int num_recvs_L = 0;
HYPRE_Int send_data_L = 0;
HYPRE_Int num_rows_L = 0;
HYPRE_Int num_rows_tmp = 0;
HYPRE_Int num_cols_offd_L = 0;
HYPRE_Int num_cols_offd = 0;
HYPRE_Int level, i, j, k;
HYPRE_Int this_proc, cnt, cnt_diag, cnt_offd;
HYPRE_Int cnt_recv, cnt_send, cnt_row, row_start;
HYPRE_Int start_diag, start_offd, indx, cnt_map;
HYPRE_Int start, j_indx, index, cnt_level;
HYPRE_Int max_sends, max_recvs;
/* Local variables */
HYPRE_Int Solve_err_flag = 0;
HYPRE_Int num_threads;
HYPRE_Int num_nonzeros_diag;
HYPRE_Int num_nonzeros_offd;
HYPRE_Real **l1_norms_ptr = NULL;
HYPRE_Real *relax_weight = NULL;
HYPRE_Real relax_type;
/* Acquire data and allocate storage */
num_threads = hypre_NumThreads();
A_array = hypre_ParAMGDataAArray(amg_data);
F_array = hypre_ParAMGDataFArray(amg_data);
U_array = hypre_ParAMGDataUArray(amg_data);
additive = hypre_ParAMGDataAdditive(amg_data);
mult_additive = hypre_ParAMGDataMultAdditive(amg_data);
num_levels = hypre_ParAMGDataNumLevels(amg_data);
relax_weight = hypre_ParAMGDataRelaxWeight(amg_data);
relax_type = hypre_ParAMGDataGridRelaxType(amg_data)[1];
comm = hypre_ParCSRMatrixComm(A_array[0]);
hypre_MPI_Comm_size(comm,&num_procs);
l1_norms_ptr = hypre_ParAMGDataL1Norms(amg_data);
addlvl = hypre_max(additive, mult_additive);
num_add_lvls = num_levels+1-addlvl;
level_start = hypre_CTAlloc(HYPRE_Int, num_add_lvls+1);
send_data_L = 0;
num_rows_L = 0;
num_cols_offd_L = 0;
num_nonzeros_diag = 0;
num_nonzeros_offd = 0;
level_start[0] = 0;
cnt = 1;
max_sends = 0;
max_recvs = 0;
for (i=addlvl; i < num_levels; i++)
{
A_tmp = A_array[i];
A_tmp_diag = hypre_ParCSRMatrixDiag(A_tmp);
A_tmp_offd = hypre_ParCSRMatrixOffd(A_tmp);
A_tmp_diag_i = hypre_CSRMatrixI(A_tmp_diag);
A_tmp_offd_i = hypre_CSRMatrixI(A_tmp_offd);
num_rows_tmp = hypre_CSRMatrixNumRows(A_tmp_diag);
num_cols_offd = hypre_CSRMatrixNumCols(A_tmp_offd);
num_rows_L += num_rows_tmp;
level_start[cnt] = level_start[cnt-1] + num_rows_tmp;
cnt++;
num_cols_offd_L += num_cols_offd;
num_nonzeros_diag += A_tmp_diag_i[num_rows_tmp];
num_nonzeros_offd += A_tmp_offd_i[num_rows_tmp];
comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp);
if (comm_pkg)
{
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
max_sends += num_sends;
if (num_sends)
send_data_L += hypre_ParCSRCommPkgSendMapStart(comm_pkg,num_sends);
max_recvs += hypre_ParCSRCommPkgNumRecvs(comm_pkg);
}
}
if (max_sends >= num_procs ||max_recvs >= num_procs)
{
max_sends = num_procs;
max_recvs = num_procs;
}
if (max_sends) all_send_procs = hypre_CTAlloc(HYPRE_Int, max_sends);
if (max_recvs) all_recv_procs = hypre_CTAlloc(HYPRE_Int, max_recvs);
cnt_send = 0;
cnt_recv = 0;
if (max_sends || max_recvs)
{
if (max_sends < num_procs && max_recvs < num_procs)
{
for (i=addlvl; i < num_levels; i++)
{
A_tmp = A_array[i];
comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp);
if (comm_pkg)
{
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg);
send_procs = hypre_ParCSRCommPkgSendProcs(comm_pkg);
recv_procs = hypre_ParCSRCommPkgRecvProcs(comm_pkg);
for (j = 0; j < num_sends; j++)
all_send_procs[cnt_send++] = send_procs[j];
for (j = 0; j < num_recvs; j++)
all_recv_procs[cnt_recv++] = recv_procs[j];
}
}
if (max_sends)
{
qsort0(all_send_procs, 0, max_sends-1);
num_sends_L = 1;
this_proc = all_send_procs[0];
for (i=1; i < max_sends; i++)
{
if (all_send_procs[i] > this_proc)
{
this_proc = all_send_procs[i];
all_send_procs[num_sends_L++] = this_proc;
}
}
L_send_procs = hypre_CTAlloc(HYPRE_Int, num_sends_L);
for (j=0; j < num_sends_L; j++)
L_send_procs[j] = all_send_procs[j];
hypre_TFree(all_send_procs);
}
if (max_recvs)
{
qsort0(all_recv_procs, 0, max_recvs-1);
num_recvs_L = 1;
this_proc = all_recv_procs[0];
for (i=1; i < max_recvs; i++)
{
if (all_recv_procs[i] > this_proc)
{
this_proc = all_recv_procs[i];
all_recv_procs[num_recvs_L++] = this_proc;
}
}
L_recv_procs = hypre_CTAlloc(HYPRE_Int, num_recvs_L);
for (j=0; j < num_recvs_L; j++)
L_recv_procs[j] = all_recv_procs[j];
hypre_TFree(all_recv_procs);
}
L_recv_ptr = hypre_CTAlloc(HYPRE_Int, num_recvs_L+1);
L_send_ptr = hypre_CTAlloc(HYPRE_Int, num_sends_L+1);
for (i=addlvl; i < num_levels; i++)
{
A_tmp = A_array[i];
comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp);
if (comm_pkg)
{
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg);
send_procs = hypre_ParCSRCommPkgSendProcs(comm_pkg);
recv_procs = hypre_ParCSRCommPkgRecvProcs(comm_pkg);
send_map_starts = hypre_ParCSRCommPkgSendMapStarts(comm_pkg);
recv_vec_starts = hypre_ParCSRCommPkgRecvVecStarts(comm_pkg);
}
else
{
num_sends = 0;
num_recvs = 0;
}
for (k = 0; k < num_sends; k++)
{
this_proc = hypre_BinarySearch(L_send_procs,send_procs[k],num_sends_L);
L_send_ptr[this_proc+1] += send_map_starts[k+1]-send_map_starts[k];
}
for (k = 0; k < num_recvs; k++)
{
this_proc = hypre_BinarySearch(L_recv_procs,recv_procs[k],num_recvs_L);
L_recv_ptr[this_proc+1] += recv_vec_starts[k+1]-recv_vec_starts[k];
}
}
L_recv_ptr[0] = 0;
for (i=1; i < num_recvs_L; i++)
L_recv_ptr[i+1] += L_recv_ptr[i];
L_send_ptr[0] = 0;
for (i=1; i < num_sends_L; i++)
L_send_ptr[i+1] += L_send_ptr[i];
}
else
{
num_recvs_L = 0;
num_sends_L = 0;
for (i=addlvl; i < num_levels; i++)
{
A_tmp = A_array[i];
comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp);
if (comm_pkg)
{
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg);
send_procs = hypre_ParCSRCommPkgSendProcs(comm_pkg);
recv_procs = hypre_ParCSRCommPkgRecvProcs(comm_pkg);
send_map_starts = hypre_ParCSRCommPkgSendMapStarts(comm_pkg);
recv_vec_starts = hypre_ParCSRCommPkgRecvVecStarts(comm_pkg);
for (j = 0; j < num_sends; j++)
{
this_proc = send_procs[j];
if (all_send_procs[this_proc] == 0)
num_sends_L++;
all_send_procs[this_proc] += send_map_starts[j+1]-send_map_starts[j];
}
for (j = 0; j < num_recvs; j++)
{
this_proc = recv_procs[j];
if (all_recv_procs[this_proc] == 0)
num_recvs_L++;
all_recv_procs[this_proc] += recv_vec_starts[j+1]-recv_vec_starts[j];
}
}
}
if (max_sends)
{
L_send_procs = hypre_CTAlloc(HYPRE_Int, num_sends_L);
L_send_ptr = hypre_CTAlloc(HYPRE_Int, num_sends_L+1);
num_sends_L = 0;
for (j=0; j < num_procs; j++)
{
this_proc = all_send_procs[j];
if (this_proc)
{
L_send_procs[num_sends_L++] = j;
L_send_ptr[num_sends_L] = this_proc + L_send_ptr[num_sends_L-1];
}
}
}
if (max_recvs)
{
L_recv_procs = hypre_CTAlloc(HYPRE_Int, num_recvs_L);
L_recv_ptr = hypre_CTAlloc(HYPRE_Int, num_recvs_L+1);
num_recvs_L = 0;
for (j=0; j < num_procs; j++)
{
this_proc = all_recv_procs[j];
if (this_proc)
{
L_recv_procs[num_recvs_L++] = j;
L_recv_ptr[num_recvs_L] = this_proc + L_recv_ptr[num_recvs_L-1];
}
}
}
}
}
if (max_sends) hypre_TFree(all_send_procs);
if (max_recvs) hypre_TFree(all_recv_procs);
L_diag = hypre_CSRMatrixCreate(num_rows_L, num_rows_L, num_nonzeros_diag);
L_offd = hypre_CSRMatrixCreate(num_rows_L, num_cols_offd_L, num_nonzeros_offd);
hypre_CSRMatrixInitialize(L_diag);
hypre_CSRMatrixInitialize(L_offd);
if (num_nonzeros_diag)
{
L_diag_data = hypre_CSRMatrixData(L_diag);
L_diag_j = hypre_CSRMatrixJ(L_diag);
}
L_diag_i = hypre_CSRMatrixI(L_diag);
if (num_nonzeros_offd)
{
L_offd_data = hypre_CSRMatrixData(L_offd);
L_offd_j = hypre_CSRMatrixJ(L_offd);
}
L_offd_i = hypre_CSRMatrixI(L_offd);
if (num_rows_L) D_data = hypre_CTAlloc(HYPRE_Real,num_rows_L);
if (send_data_L)
{
L_send_map_elmts = hypre_CTAlloc(HYPRE_Int, send_data_L);
buf_data = hypre_CTAlloc(HYPRE_Real,send_data_L);
}
if (num_cols_offd_L)
{
D_data_offd = hypre_CTAlloc(HYPRE_Real,num_cols_offd_L);
/*L_col_map_offd = hypre_CTAlloc(HYPRE_Int, num_cols_offd_L);*/
remap = hypre_CTAlloc(HYPRE_Int, num_cols_offd_L);
}
Rtilde = hypre_CTAlloc(hypre_ParVector, 1);
Rtilde_local = hypre_SeqVectorCreate(num_rows_L);
hypre_SeqVectorInitialize(Rtilde_local);
hypre_ParVectorLocalVector(Rtilde) = Rtilde_local;
hypre_ParVectorOwnsData(Rtilde) = 1;
Xtilde = hypre_CTAlloc(hypre_ParVector, 1);
Xtilde_local = hypre_SeqVectorCreate(num_rows_L);
hypre_SeqVectorInitialize(Xtilde_local);
hypre_ParVectorLocalVector(Xtilde) = Xtilde_local;
hypre_ParVectorOwnsData(Xtilde) = 1;
x_data = hypre_VectorData(hypre_ParVectorLocalVector(Xtilde));
r_data = hypre_VectorData(hypre_ParVectorLocalVector(Rtilde));
cnt = 0;
cnt_level = 0;
cnt_diag = 0;
cnt_offd = 0;
cnt_row = 1;
L_diag_i[0] = 0;
L_offd_i[0] = 0;
for (level=addlvl; level < num_levels; level++)
{
row_start = level_start[cnt_level];
if (level != 0)
{
tmp_data = hypre_VectorData(hypre_ParVectorLocalVector(F_array[level]));
if (tmp_data) hypre_TFree(tmp_data);
hypre_VectorData(hypre_ParVectorLocalVector(F_array[level])) = &r_data[row_start];
hypre_VectorOwnsData(hypre_ParVectorLocalVector(F_array[level])) = 0;
tmp_data = hypre_VectorData(hypre_ParVectorLocalVector(U_array[level]));
if (tmp_data) hypre_TFree(tmp_data);
hypre_VectorData(hypre_ParVectorLocalVector(U_array[level])) = &x_data[row_start];
hypre_VectorOwnsData(hypre_ParVectorLocalVector(U_array[level])) = 0;
}
cnt_level++;
start_diag = L_diag_i[cnt_row-1];
start_offd = L_offd_i[cnt_row-1];
A_tmp = A_array[level];
A_tmp_diag = hypre_ParCSRMatrixDiag(A_tmp);
A_tmp_offd = hypre_ParCSRMatrixOffd(A_tmp);
comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp);
A_tmp_diag_i = hypre_CSRMatrixI(A_tmp_diag);
A_tmp_offd_i = hypre_CSRMatrixI(A_tmp_offd);
A_tmp_diag_j = hypre_CSRMatrixJ(A_tmp_diag);
A_tmp_offd_j = hypre_CSRMatrixJ(A_tmp_offd);
A_tmp_diag_data = hypre_CSRMatrixData(A_tmp_diag);
A_tmp_offd_data = hypre_CSRMatrixData(A_tmp_offd);
num_rows_tmp = hypre_CSRMatrixNumRows(A_tmp_diag);
if (comm_pkg)
{
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg);
send_procs = hypre_ParCSRCommPkgSendProcs(comm_pkg);
recv_procs = hypre_ParCSRCommPkgRecvProcs(comm_pkg);
send_map_starts = hypre_ParCSRCommPkgSendMapStarts(comm_pkg);
send_map_elmts = hypre_ParCSRCommPkgSendMapElmts(comm_pkg);
recv_vec_starts = hypre_ParCSRCommPkgRecvVecStarts(comm_pkg);
}
else
{
num_sends = 0;
num_recvs = 0;
}
/* Compute new combined communication package */
for (i=0; i < num_sends; i++)
{
this_proc = hypre_BinarySearch(L_send_procs,send_procs[i],num_sends_L);
indx = L_send_ptr[this_proc];
for (j=send_map_starts[i]; j < send_map_starts[i+1]; j++)
{
L_send_map_elmts[indx++] = row_start + send_map_elmts[j];
}
L_send_ptr[this_proc] = indx;
}
cnt_map = 0;
for (i = 0; i < num_recvs; i++)
{
this_proc = hypre_BinarySearch(L_recv_procs,recv_procs[i],num_recvs_L);
indx = L_recv_ptr[this_proc];
for (j=recv_vec_starts[i]; j < recv_vec_starts[i+1]; j++)
{
remap[cnt_map++] = indx++;
}
L_recv_ptr[this_proc] = indx;
}
/* Compute Lambda */
if (relax_type == 0)
{
HYPRE_Real rlx_wt = relax_weight[level];
#ifdef HYPRE_USING_OPENMP
#pragma omp for private(i) HYPRE_SMP_SCHEDULE
#endif
for (i=0; i < num_rows_tmp; i++)
{
D_data[i] = rlx_wt/A_tmp_diag_data[A_tmp_diag_i[i]];
L_diag_i[cnt_row+i] = start_diag + A_tmp_diag_i[i+1];
L_offd_i[cnt_row+i] = start_offd + A_tmp_offd_i[i+1];
}
}
else
{
l1_norms = l1_norms_ptr[level];
#ifdef HYPRE_USING_OPENMP
#pragma omp for private(i) HYPRE_SMP_SCHEDULE
#endif
for (i=0; i < num_rows_tmp; i++)
{
D_data[i] = 1.0/l1_norms[i];
L_diag_i[cnt_row+i] = start_diag + A_tmp_diag_i[i+1];
L_offd_i[cnt_row+i] = start_offd + A_tmp_offd_i[i+1];
}
}
if (num_procs > 1)
{
index = 0;
for (i=0; i < num_sends; i++)
{
start = send_map_starts[i];
for (j=start; j < send_map_starts[i+1]; j++)
buf_data[index++] = D_data[send_map_elmts[j]];
}
comm_handle = hypre_ParCSRCommHandleCreate(1, comm_pkg,
buf_data, D_data_offd);
hypre_ParCSRCommHandleDestroy(comm_handle);
}
for (i = 0; i < num_rows_tmp; i++)
{
j_indx = A_tmp_diag_i[i];
L_diag_data[cnt_diag] = (2.0 - A_tmp_diag_data[j_indx]*D_data[i])*D_data[i];
L_diag_j[cnt_diag++] = i+row_start;
for (j=A_tmp_diag_i[i]+1; j < A_tmp_diag_i[i+1]; j++)
{
j_indx = A_tmp_diag_j[j];
L_diag_data[cnt_diag] = (- A_tmp_diag_data[j]*D_data[j_indx])*D_data[i];
L_diag_j[cnt_diag++] = j_indx+row_start;
}
for (j=A_tmp_offd_i[i]; j < A_tmp_offd_i[i+1]; j++)
{
j_indx = A_tmp_offd_j[j];
L_offd_data[cnt_offd] = (- A_tmp_offd_data[j]*D_data_offd[j_indx])*D_data[i];
L_offd_j[cnt_offd++] = remap[j_indx];
}
}
cnt_row += num_rows_tmp;
}
if (L_send_ptr)
{
for (i=num_sends_L-1; i > 0; i--)
L_send_ptr[i] = L_send_ptr[i-1];
L_send_ptr[0] = 0;
}
else
L_send_ptr = hypre_CTAlloc(HYPRE_Int,1);
if (L_recv_ptr)
{
for (i=num_recvs_L-1; i > 0; i--)
L_recv_ptr[i] = L_recv_ptr[i-1];
L_recv_ptr[0] = 0;
}
else
L_recv_ptr = hypre_CTAlloc(HYPRE_Int,1);
L_comm_pkg = hypre_CTAlloc(hypre_ParCSRCommPkg,1);
hypre_ParCSRCommPkgNumRecvs(L_comm_pkg) = num_recvs_L;
hypre_ParCSRCommPkgNumSends(L_comm_pkg) = num_sends_L;
hypre_ParCSRCommPkgRecvProcs(L_comm_pkg) = L_recv_procs;
hypre_ParCSRCommPkgSendProcs(L_comm_pkg) = L_send_procs;
hypre_ParCSRCommPkgRecvVecStarts(L_comm_pkg) = L_recv_ptr;
hypre_ParCSRCommPkgSendMapStarts(L_comm_pkg) = L_send_ptr;
hypre_ParCSRCommPkgSendMapElmts(L_comm_pkg) = L_send_map_elmts;
hypre_ParCSRCommPkgComm(L_comm_pkg) = comm;
Lambda = hypre_CTAlloc(hypre_ParCSRMatrix, 1);
hypre_ParCSRMatrixDiag(Lambda) = L_diag;
hypre_ParCSRMatrixOffd(Lambda) = L_offd;
hypre_ParCSRMatrixCommPkg(Lambda) = L_comm_pkg;
hypre_ParCSRMatrixComm(Lambda) = comm;
hypre_ParCSRMatrixOwnsData(Lambda) = 1;
hypre_ParAMGDataLambda(amg_data) = Lambda;
hypre_ParAMGDataRtilde(amg_data) = Rtilde;
hypre_ParAMGDataXtilde(amg_data) = Xtilde;
hypre_TFree(D_data_offd);
hypre_TFree(D_data);
if (num_procs > 1) hypre_TFree(buf_data);
hypre_TFree(remap);
hypre_TFree(buf_data);
hypre_TFree(level_start);
return Solve_err_flag;
}
double hypre_ParVectorLocalSumElts( hypre_ParVector * vector )
{
return hypre_VectorSumElts( hypre_ParVectorLocalVector(vector) );
}
HYPRE_Int
hypre_BoomerAMGAdditiveCycle( void *amg_vdata)
{
hypre_ParAMGData *amg_data = amg_vdata;
/* Data Structure variables */
hypre_ParCSRMatrix **A_array;
hypre_ParCSRMatrix **P_array;
hypre_ParCSRMatrix **R_array;
hypre_ParCSRMatrix *Lambda;
hypre_ParVector **F_array;
hypre_ParVector **U_array;
hypre_ParVector *Vtemp;
hypre_ParVector *Ztemp;
hypre_ParVector *Xtilde, *Rtilde;
HYPRE_Int **CF_marker_array;
HYPRE_Int num_levels;
HYPRE_Int addlvl;
HYPRE_Int additive;
HYPRE_Int mult_additive;
HYPRE_Int simple;
HYPRE_Int i, num_rows;
HYPRE_Int n_global;
HYPRE_Int rlx_order;
/* Local variables */
HYPRE_Int Solve_err_flag = 0;
HYPRE_Int level;
HYPRE_Int coarse_grid;
HYPRE_Int fine_grid;
HYPRE_Int relax_type;
HYPRE_Int rlx_down;
HYPRE_Int rlx_up;
HYPRE_Int *grid_relax_type;
HYPRE_Real **l1_norms;
HYPRE_Real alpha, beta;
HYPRE_Int num_threads;
HYPRE_Real *u_data;
HYPRE_Real *f_data;
HYPRE_Real *v_data;
HYPRE_Real *l1_norms_lvl;
HYPRE_Real *D_inv;
HYPRE_Real *x_global;
HYPRE_Real *r_global;
HYPRE_Real *relax_weight;
HYPRE_Real *omega;
#if 0
HYPRE_Real *D_mat;
HYPRE_Real *S_vec;
#endif
/* Acquire data and allocate storage */
num_threads = hypre_NumThreads();
A_array = hypre_ParAMGDataAArray(amg_data);
F_array = hypre_ParAMGDataFArray(amg_data);
U_array = hypre_ParAMGDataUArray(amg_data);
P_array = hypre_ParAMGDataPArray(amg_data);
R_array = hypre_ParAMGDataRArray(amg_data);
CF_marker_array = hypre_ParAMGDataCFMarkerArray(amg_data);
Vtemp = hypre_ParAMGDataVtemp(amg_data);
Ztemp = hypre_ParAMGDataZtemp(amg_data);
num_levels = hypre_ParAMGDataNumLevels(amg_data);
additive = hypre_ParAMGDataAdditive(amg_data);
mult_additive = hypre_ParAMGDataMultAdditive(amg_data);
simple = hypre_ParAMGDataSimple(amg_data);
grid_relax_type = hypre_ParAMGDataGridRelaxType(amg_data);
Lambda = hypre_ParAMGDataLambda(amg_data);
Xtilde = hypre_ParAMGDataXtilde(amg_data);
Rtilde = hypre_ParAMGDataRtilde(amg_data);
l1_norms = hypre_ParAMGDataL1Norms(amg_data);
D_inv = hypre_ParAMGDataDinv(amg_data);
grid_relax_type = hypre_ParAMGDataGridRelaxType(amg_data);
relax_weight = hypre_ParAMGDataRelaxWeight(amg_data);
omega = hypre_ParAMGDataOmega(amg_data);
rlx_order = hypre_ParAMGDataRelaxOrder(amg_data);
/* Initialize */
addlvl = hypre_max(additive, mult_additive);
addlvl = hypre_max(addlvl, simple);
Solve_err_flag = 0;
/*---------------------------------------------------------------------
* Main loop of cycling --- multiplicative version --- V-cycle
*--------------------------------------------------------------------*/
/* down cycle */
relax_type = grid_relax_type[1];
rlx_down = grid_relax_type[1];
rlx_up = grid_relax_type[2];
for (level = 0; level < num_levels-1; level++)
{
fine_grid = level;
coarse_grid = level + 1;
u_data = hypre_VectorData(hypre_ParVectorLocalVector(U_array[fine_grid]));
f_data = hypre_VectorData(hypre_ParVectorLocalVector(F_array[fine_grid]));
v_data = hypre_VectorData(hypre_ParVectorLocalVector(Vtemp));
l1_norms_lvl = l1_norms[level];
hypre_ParVectorSetConstantValues(U_array[coarse_grid], 0.0);
if (level < addlvl) /* multiplicative version */
{
/* smoothing step */
if (rlx_down == 0)
{
HYPRE_Real *A_data = hypre_CSRMatrixData(hypre_ParCSRMatrixDiag(A_array[fine_grid]));
HYPRE_Int *A_i = hypre_CSRMatrixI(hypre_ParCSRMatrixDiag(A_array[fine_grid]));
hypre_ParVectorCopy(F_array[fine_grid],Vtemp);
num_rows = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A_array[fine_grid]));
#ifdef HYPRE_USING_OPENMP
#pragma omp parallel for private(i) HYPRE_SMP_SCHEDULE
#endif
for (i = 0; i < num_rows; i++)
u_data[i] = relax_weight[level]*v_data[i] / A_data[A_i[i]];
}
else if (rlx_down != 18)
{
/*hypre_BoomerAMGRelax(A_array[fine_grid],F_array[fine_grid],NULL,rlx_down,0,*/
hypre_BoomerAMGRelaxIF(A_array[fine_grid],F_array[fine_grid],
CF_marker_array[fine_grid], rlx_down,rlx_order,1,
relax_weight[fine_grid], omega[fine_grid],
l1_norms_lvl, U_array[fine_grid], Vtemp, Ztemp);
hypre_ParVectorCopy(F_array[fine_grid],Vtemp);
}
else
{
hypre_ParVectorCopy(F_array[fine_grid],Vtemp);
num_rows = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A_array[fine_grid]));
#ifdef HYPRE_USING_OPENMP
#pragma omp parallel for private(i) HYPRE_SMP_SCHEDULE
#endif
for (i = 0; i < num_rows; i++)
u_data[i] += v_data[i] / l1_norms_lvl[i];
}
alpha = -1.0;
beta = 1.0;
hypre_ParCSRMatrixMatvec(alpha, A_array[fine_grid], U_array[fine_grid],
beta, Vtemp);
alpha = 1.0;
beta = 0.0;
hypre_ParCSRMatrixMatvecT(alpha,R_array[fine_grid],Vtemp,
beta,F_array[coarse_grid]);
}
else /* additive version */
{
hypre_ParVectorCopy(F_array[fine_grid],Vtemp);
if (level == 0) /* compute residual */
{
hypre_ParVectorCopy(Vtemp, Rtilde);
hypre_ParVectorCopy(U_array[fine_grid],Xtilde);
}
alpha = 1.0;
beta = 0.0;
hypre_ParCSRMatrixMatvecT(alpha,R_array[fine_grid],Vtemp,
beta,F_array[coarse_grid]);
}
}
/* solve coarse grid */
if (addlvl < num_levels)
{
if (simple > -1)
{
x_global = hypre_VectorData(hypre_ParVectorLocalVector(Xtilde));
r_global = hypre_VectorData(hypre_ParVectorLocalVector(Rtilde));
n_global = hypre_VectorSize(hypre_ParVectorLocalVector(Xtilde));
#ifdef HYPRE_USING_OPENMP
#pragma omp parallel for private(i) HYPRE_SMP_SCHEDULE
#endif
for (i=0; i < n_global; i++)
x_global[i] += D_inv[i]*r_global[i];
}
else
hypre_ParCSRMatrixMatvec(1.0, Lambda, Rtilde, 1.0, Xtilde);
if (addlvl == 0) hypre_ParVectorCopy(Xtilde, U_array[0]);
}
else
{
fine_grid = num_levels -1;
hypre_ParCSRRelax(A_array[fine_grid], F_array[fine_grid],
1, 1, l1_norms[fine_grid],
1.0, 1.0 ,0,0,0,0,
U_array[fine_grid], Vtemp, Ztemp);
}
/* up cycle */
relax_type = grid_relax_type[2];
for (level = num_levels-1; level > 0; level--)
{
fine_grid = level - 1;
coarse_grid = level;
if (level <= addlvl) /* multiplicative version */
{
alpha = 1.0;
beta = 1.0;
hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid],
U_array[coarse_grid],
beta, U_array[fine_grid]);
if (rlx_up != 18)
/*hypre_BoomerAMGRelax(A_array[fine_grid],F_array[fine_grid],NULL,rlx_up,0,*/
hypre_BoomerAMGRelaxIF(A_array[fine_grid],F_array[fine_grid],
CF_marker_array[fine_grid],
rlx_up,rlx_order,2,
relax_weight[fine_grid], omega[fine_grid],
l1_norms[fine_grid], U_array[fine_grid], Vtemp, Ztemp);
else if (rlx_order)
{
HYPRE_Int loc_relax_points[2];
loc_relax_points[0] = -1;
loc_relax_points[1] = 1;
for (i=0; i < 2; i++)
hypre_ParCSRRelax_L1_Jacobi(A_array[fine_grid],F_array[fine_grid],
CF_marker_array[fine_grid],
loc_relax_points[i],
1.0, l1_norms[fine_grid],
U_array[fine_grid], Vtemp);
}
else
hypre_ParCSRRelax(A_array[fine_grid], F_array[fine_grid],
1, 1, l1_norms[fine_grid],
1.0, 1.0 ,0,0,0,0,
U_array[fine_grid], Vtemp, Ztemp);
}
else /* additive version */
{
alpha = 1.0;
beta = 1.0;
hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid],
U_array[coarse_grid],
beta, U_array[fine_grid]);
}
}
return(Solve_err_flag);
}
hypre_Vector *
hypre_ParVectorToVectorAll (hypre_ParVector *par_v)
{
MPI_Comm comm = hypre_ParVectorComm(par_v);
HYPRE_Int global_size = hypre_ParVectorGlobalSize(par_v);
#ifndef HYPRE_NO_GLOBAL_PARTITION
HYPRE_Int *vec_starts = hypre_ParVectorPartitioning(par_v);
#endif
hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_v);
HYPRE_Int num_procs, my_id;
HYPRE_Int num_vectors = hypre_ParVectorNumVectors(par_v);
hypre_Vector *vector;
double *vector_data;
double *local_data;
HYPRE_Int local_size;
hypre_MPI_Request *requests;
hypre_MPI_Status *status;
HYPRE_Int i, j;
HYPRE_Int *used_procs;
HYPRE_Int num_types, num_requests;
HYPRE_Int vec_len, proc_id;
#ifdef HYPRE_NO_GLOBAL_PARTITION
HYPRE_Int *new_vec_starts;
HYPRE_Int num_contacts;
HYPRE_Int contact_proc_list[1];
HYPRE_Int contact_send_buf[1];
HYPRE_Int contact_send_buf_starts[2];
HYPRE_Int max_response_size;
HYPRE_Int *response_recv_buf=NULL;
HYPRE_Int *response_recv_buf_starts = NULL;
hypre_DataExchangeResponse response_obj;
hypre_ProcListElements send_proc_obj;
HYPRE_Int *send_info = NULL;
hypre_MPI_Status status1;
HYPRE_Int count, tag1 = 112, tag2 = 223;
HYPRE_Int start;
#endif
hypre_MPI_Comm_size(comm, &num_procs);
hypre_MPI_Comm_rank(comm, &my_id);
#ifdef HYPRE_NO_GLOBAL_PARTITION
local_size = hypre_ParVectorLastIndex(par_v) -
hypre_ParVectorFirstIndex(par_v) + 1;
/* determine procs which hold data of par_v and store ids in used_procs */
/* we need to do an exchange data for this. If I own row then I will contact
processor 0 with the endpoint of my local range */
if (local_size > 0)
{
num_contacts = 1;
contact_proc_list[0] = 0;
contact_send_buf[0] = hypre_ParVectorLastIndex(par_v);
contact_send_buf_starts[0] = 0;
contact_send_buf_starts[1] = 1;
}
else
{
num_contacts = 0;
contact_send_buf_starts[0] = 0;
contact_send_buf_starts[1] = 0;
}
/*build the response object*/
/*send_proc_obj will be for saving info from contacts */
send_proc_obj.length = 0;
send_proc_obj.storage_length = 10;
send_proc_obj.id = hypre_CTAlloc(HYPRE_Int, send_proc_obj.storage_length);
send_proc_obj.vec_starts = hypre_CTAlloc(HYPRE_Int, send_proc_obj.storage_length + 1);
send_proc_obj.vec_starts[0] = 0;
send_proc_obj.element_storage_length = 10;
send_proc_obj.elements = hypre_CTAlloc(HYPRE_Int, send_proc_obj.element_storage_length);
max_response_size = 0; /* each response is null */
response_obj.fill_response = hypre_FillResponseParToVectorAll;
response_obj.data1 = NULL;
response_obj.data2 = &send_proc_obj; /*this is where we keep info from contacts*/
hypre_DataExchangeList(num_contacts,
contact_proc_list, contact_send_buf,
contact_send_buf_starts, sizeof(HYPRE_Int),
sizeof(HYPRE_Int), &response_obj,
max_response_size, 1,
comm, (void**) &response_recv_buf,
&response_recv_buf_starts);
/* now processor 0 should have a list of ranges for processors that have rows -
these are in send_proc_obj - it needs to create the new list of processors
and also an array of vec starts - and send to those who own row*/
if (my_id)
{
if (local_size)
{
/* look for a message from processor 0 */
hypre_MPI_Probe(0, tag1, comm, &status1);
hypre_MPI_Get_count(&status1, HYPRE_MPI_INT, &count);
send_info = hypre_CTAlloc(HYPRE_Int, count);
hypre_MPI_Recv(send_info, count, HYPRE_MPI_INT, 0, tag1, comm, &status1);
/* now unpack */
num_types = send_info[0];
used_procs = hypre_CTAlloc(HYPRE_Int, num_types);
new_vec_starts = hypre_CTAlloc(HYPRE_Int, num_types+1);
for (i=1; i<= num_types; i++)
{
used_procs[i-1] = send_info[i];
}
for (i=num_types+1; i< count; i++)
{
new_vec_starts[i-num_types-1] = send_info[i] ;
}
}
else /* clean up and exit */
{
hypre_TFree(send_proc_obj.vec_starts);
hypre_TFree(send_proc_obj.id);
hypre_TFree(send_proc_obj.elements);
if(response_recv_buf) hypre_TFree(response_recv_buf);
if(response_recv_buf_starts) hypre_TFree(response_recv_buf_starts);
return NULL;
}
}
else /* my_id ==0 */
{
num_types = send_proc_obj.length;
used_procs = hypre_CTAlloc(HYPRE_Int, num_types);
new_vec_starts = hypre_CTAlloc(HYPRE_Int, num_types+1);
new_vec_starts[0] = 0;
for (i=0; i< num_types; i++)
{
used_procs[i] = send_proc_obj.id[i];
new_vec_starts[i+1] = send_proc_obj.elements[i]+1;
}
qsort0(used_procs, 0, num_types-1);
qsort0(new_vec_starts, 0, num_types);
/*now we need to put into an array to send */
count = 2*num_types+2;
send_info = hypre_CTAlloc(HYPRE_Int, count);
send_info[0] = num_types;
for (i=1; i<= num_types; i++)
{
send_info[i] = used_procs[i-1];
}
for (i=num_types+1; i< count; i++)
{
send_info[i] = new_vec_starts[i-num_types-1];
}
requests = hypre_CTAlloc(hypre_MPI_Request, num_types);
status = hypre_CTAlloc(hypre_MPI_Status, num_types);
/* don't send to myself - these are sorted so my id would be first*/
start = 0;
if (used_procs[0] == 0)
{
start = 1;
}
for (i=start; i < num_types; i++)
{
hypre_MPI_Isend(send_info, count, HYPRE_MPI_INT, used_procs[i], tag1, comm, &requests[i-start]);
}
hypre_MPI_Waitall(num_types-start, requests, status);
hypre_TFree(status);
hypre_TFree(requests);
}
/* clean up */
hypre_TFree(send_proc_obj.vec_starts);
hypre_TFree(send_proc_obj.id);
hypre_TFree(send_proc_obj.elements);
hypre_TFree(send_info);
if(response_recv_buf) hypre_TFree(response_recv_buf);
if(response_recv_buf_starts) hypre_TFree(response_recv_buf_starts);
/* now proc 0 can exit if it has no rows */
if (!local_size) {
hypre_TFree(used_procs);
hypre_TFree(new_vec_starts);
return NULL;
}
/* everyone left has rows and knows: new_vec_starts, num_types, and used_procs */
/* this vector should be rather small */
local_data = hypre_VectorData(local_vector);
vector = hypre_SeqVectorCreate(global_size);
hypre_VectorNumVectors(vector) = num_vectors;
hypre_SeqVectorInitialize(vector);
vector_data = hypre_VectorData(vector);
num_requests = 2*num_types;
requests = hypre_CTAlloc(hypre_MPI_Request, num_requests);
status = hypre_CTAlloc(hypre_MPI_Status, num_requests);
/* initialize data exchange among used_procs and generate vector - here we
send to ourself also*/
j = 0;
for (i = 0; i < num_types; i++)
{
proc_id = used_procs[i];
vec_len = new_vec_starts[i+1] - new_vec_starts[i];
hypre_MPI_Irecv(&vector_data[new_vec_starts[i]], num_vectors*vec_len, hypre_MPI_DOUBLE,
proc_id, tag2, comm, &requests[j++]);
}
for (i = 0; i < num_types; i++)
{
hypre_MPI_Isend(local_data, num_vectors*local_size, hypre_MPI_DOUBLE, used_procs[i],
tag2, comm, &requests[j++]);
}
hypre_MPI_Waitall(num_requests, requests, status);
if (num_requests)
{
hypre_TFree(requests);
hypre_TFree(status);
hypre_TFree(used_procs);
}
hypre_TFree(new_vec_starts);
#else
local_size = vec_starts[my_id+1] - vec_starts[my_id];
/* if my_id contains no data, return NULL */
if (!local_size)
return NULL;
local_data = hypre_VectorData(local_vector);
vector = hypre_SeqVectorCreate(global_size);
hypre_VectorNumVectors(vector) = num_vectors;
hypre_SeqVectorInitialize(vector);
vector_data = hypre_VectorData(vector);
/* determine procs which hold data of par_v and store ids in used_procs */
num_types = -1;
for (i=0; i < num_procs; i++)
if (vec_starts[i+1]-vec_starts[i])
num_types++;
num_requests = 2*num_types;
used_procs = hypre_CTAlloc(HYPRE_Int, num_types);
j = 0;
for (i=0; i < num_procs; i++)
if (vec_starts[i+1]-vec_starts[i] && i-my_id)
used_procs[j++] = i;
requests = hypre_CTAlloc(hypre_MPI_Request, num_requests);
status = hypre_CTAlloc(hypre_MPI_Status, num_requests);
/* initialize data exchange among used_procs and generate vector */
j = 0;
for (i = 0; i < num_types; i++)
{
proc_id = used_procs[i];
vec_len = vec_starts[proc_id+1] - vec_starts[proc_id];
hypre_MPI_Irecv(&vector_data[vec_starts[proc_id]], num_vectors*vec_len, hypre_MPI_DOUBLE,
proc_id, 0, comm, &requests[j++]);
}
for (i = 0; i < num_types; i++)
{
hypre_MPI_Isend(local_data, num_vectors*local_size, hypre_MPI_DOUBLE, used_procs[i],
0, comm, &requests[j++]);
}
for (i=0; i < num_vectors*local_size; i++)
vector_data[vec_starts[my_id]+i] = local_data[i];
hypre_MPI_Waitall(num_requests, requests, status);
if (num_requests)
{
hypre_TFree(used_procs);
hypre_TFree(requests);
hypre_TFree(status);
}
#endif
return vector;
}
HYPRE_Int hypre_CreateDinv(void *amg_vdata)
{
hypre_ParAMGData *amg_data = amg_vdata;
/* Data Structure variables */
hypre_ParCSRMatrix **A_array;
hypre_ParVector **F_array;
hypre_ParVector **U_array;
hypre_ParCSRMatrix *A_tmp;
hypre_CSRMatrix *A_tmp_diag;
hypre_ParVector *Xtilde;
hypre_ParVector *Rtilde;
hypre_Vector *Xtilde_local;
hypre_Vector *Rtilde_local;
HYPRE_Real *x_data;
HYPRE_Real *r_data;
HYPRE_Real *tmp_data;
HYPRE_Real *D_inv = NULL;
HYPRE_Real *relax_weight = NULL;
HYPRE_Real relax_type;
HYPRE_Int addlvl;
HYPRE_Int num_levels;
HYPRE_Int num_add_lvls;
HYPRE_Int num_rows_L;
HYPRE_Int num_rows_A;
HYPRE_Int num_rows_tmp;
HYPRE_Int level, i;
/* Local variables */
HYPRE_Int Solve_err_flag = 0;
HYPRE_Int num_threads;
HYPRE_Real **l1_norms_ptr = NULL;
HYPRE_Real *l1_norms;
HYPRE_Int l1_start;
/* Acquire data and allocate storage */
num_threads = hypre_NumThreads();
A_array = hypre_ParAMGDataAArray(amg_data);
F_array = hypre_ParAMGDataFArray(amg_data);
U_array = hypre_ParAMGDataUArray(amg_data);
addlvl = hypre_ParAMGDataSimple(amg_data);
num_levels = hypre_ParAMGDataNumLevels(amg_data);
relax_weight = hypre_ParAMGDataRelaxWeight(amg_data);
relax_type = hypre_ParAMGDataGridRelaxType(amg_data)[1];
num_rows_A = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A_array[0]));
l1_norms_ptr = hypre_ParAMGDataL1Norms(amg_data);
/* smooth_option = hypre_ParAMGDataSmoothOption(amg_data); */
num_add_lvls = num_levels+1-addlvl;
num_rows_L = 0;
for (i=addlvl; i < num_levels; i++)
{
A_tmp = A_array[i];
A_tmp_diag = hypre_ParCSRMatrixDiag(A_tmp);
num_rows_tmp = hypre_CSRMatrixNumRows(A_tmp_diag);
num_rows_L += num_rows_tmp;
}
Rtilde = hypre_CTAlloc(hypre_ParVector, 1);
Rtilde_local = hypre_SeqVectorCreate(num_rows_L);
hypre_SeqVectorInitialize(Rtilde_local);
hypre_ParVectorLocalVector(Rtilde) = Rtilde_local;
hypre_ParVectorOwnsData(Rtilde) = 1;
Xtilde = hypre_CTAlloc(hypre_ParVector, 1);
Xtilde_local = hypre_SeqVectorCreate(num_rows_L);
hypre_SeqVectorInitialize(Xtilde_local);
hypre_ParVectorLocalVector(Xtilde) = Xtilde_local;
hypre_ParVectorOwnsData(Xtilde) = 1;
x_data = hypre_VectorData(hypre_ParVectorLocalVector(Xtilde));
r_data = hypre_VectorData(hypre_ParVectorLocalVector(Rtilde));
D_inv = hypre_CTAlloc(HYPRE_Real, num_rows_L);
l1_start = 0;
for (level=addlvl; level < num_levels; level++)
{
if (level != 0)
{
tmp_data = hypre_VectorData(hypre_ParVectorLocalVector(F_array[level]));
if (tmp_data) hypre_TFree(tmp_data);
hypre_VectorData(hypre_ParVectorLocalVector(F_array[level])) = &r_data[l1_start];
hypre_VectorOwnsData(hypre_ParVectorLocalVector(F_array[level])) = 0;
tmp_data = hypre_VectorData(hypre_ParVectorLocalVector(U_array[level]));
if (tmp_data) hypre_TFree(tmp_data);
hypre_VectorData(hypre_ParVectorLocalVector(U_array[level])) = &x_data[l1_start];
hypre_VectorOwnsData(hypre_ParVectorLocalVector(U_array[level])) = 0;
}
A_tmp = A_array[level];
A_tmp_diag = hypre_ParCSRMatrixDiag(A_tmp);
num_rows_tmp = hypre_CSRMatrixNumRows(A_tmp_diag);
if (relax_type == 0)
{
HYPRE_Real rlx_wt = relax_weight[level];
HYPRE_Int *A_tmp_diag_i = hypre_CSRMatrixI(A_tmp_diag);
HYPRE_Real *A_tmp_diag_data = hypre_CSRMatrixData(A_tmp_diag);
#ifdef HYPRE_USING_OPENMP
#pragma omp for private(i) HYPRE_SMP_SCHEDULE
#endif
for (i=0; i < num_rows_tmp; i++)
D_inv[l1_start+i] = rlx_wt/A_tmp_diag_data[A_tmp_diag_i[i]];
}
else
{
l1_norms = l1_norms_ptr[level];
#ifdef HYPRE_USING_OPENMP
#pragma omp for private(i) HYPRE_SMP_SCHEDULE
#endif
for (i=0; i < num_rows_tmp; i++)
D_inv[l1_start+i] = 1.0/l1_norms[i];
}
l1_start += num_rows_tmp;
}
hypre_ParAMGDataDinv(amg_data) = D_inv;
hypre_ParAMGDataRtilde(amg_data) = Rtilde;
hypre_ParAMGDataXtilde(amg_data) = Xtilde;
return Solve_err_flag;
}
HYPRE_Int
hypre_ParVectorReadIJ( MPI_Comm comm,
const char *filename,
HYPRE_Int *base_j_ptr,
hypre_ParVector **vector_ptr)
{
HYPRE_Int global_size;
hypre_ParVector *vector;
hypre_Vector *local_vector;
double *local_data;
HYPRE_Int *partitioning;
HYPRE_Int base_j;
HYPRE_Int myid, num_procs, i, j, J;
char new_filename[255];
FILE *file;
hypre_MPI_Comm_size(comm, &num_procs);
hypre_MPI_Comm_rank(comm, &myid);
hypre_sprintf(new_filename,"%s.%05d", filename, myid);
if ((file = fopen(new_filename, "r")) == NULL)
{
hypre_printf("Error: can't open output file %s\n", new_filename);
hypre_error(HYPRE_ERROR_GENERIC);
return hypre_error_flag;
}
hypre_fscanf(file, "%d", &global_size);
#ifdef HYPRE_NO_GLOBAL_PARTITION
/* this may need to be changed so that the base is available in the file! */
partitioning = hypre_CTAlloc(HYPRE_Int,2);
hypre_fscanf(file, "%d", partitioning);
for (i = 0; i < 2; i++)
{
hypre_fscanf(file, "%d", partitioning+i);
}
/* This is not yet implemented correctly! */
base_j = 0;
#else
partitioning = hypre_CTAlloc(HYPRE_Int,num_procs+1);
hypre_fscanf(file, "%d", partitioning);
for (i = 1; i <= num_procs; i++)
{
hypre_fscanf(file, "%d", partitioning+i);
partitioning[i] -= partitioning[0];
}
base_j = partitioning[0];
partitioning[0] = 0;
#endif
vector = hypre_ParVectorCreate(comm, global_size,
partitioning);
hypre_ParVectorInitialize(vector);
local_vector = hypre_ParVectorLocalVector(vector);
local_data = hypre_VectorData(local_vector);
#ifdef HYPRE_NO_GLOBAL_PARTITION
for (j = 0; j < partitioning[1] - partitioning[0]; j++)
#else
for (j = 0; j < partitioning[myid+1] - partitioning[myid]; j++)
#endif
{
hypre_fscanf(file, "%d %le", &J, local_data + j);
}
fclose(file);
*base_j_ptr = base_j;
*vector_ptr = vector;
/* multivector code not written yet >>> */
hypre_assert( hypre_ParVectorNumVectors(vector) == 1 );
if ( hypre_ParVectorNumVectors(vector) != 1 ) hypre_error(HYPRE_ERROR_GENERIC);
return hypre_error_flag;
}
HYPRE_Int
hypre_BoomerAMGCycle( void *amg_vdata,
hypre_ParVector **F_array,
hypre_ParVector **U_array )
{
hypre_ParAMGData *amg_data = amg_vdata;
HYPRE_Solver *smoother;
/* Data Structure variables */
hypre_ParCSRMatrix **A_array;
hypre_ParCSRMatrix **P_array;
hypre_ParCSRMatrix **R_array;
hypre_ParVector *Utemp;
hypre_ParVector *Vtemp;
hypre_ParVector *Rtemp;
hypre_ParVector *Ptemp;
hypre_ParVector *Ztemp;
hypre_ParVector *Aux_U;
hypre_ParVector *Aux_F;
hypre_ParCSRBlockMatrix **A_block_array;
hypre_ParCSRBlockMatrix **P_block_array;
hypre_ParCSRBlockMatrix **R_block_array;
HYPRE_Real *Ztemp_data;
HYPRE_Real *Ptemp_data;
HYPRE_Int **CF_marker_array;
/* HYPRE_Int **unknown_map_array;
HYPRE_Int **point_map_array;
HYPRE_Int **v_at_point_array; */
HYPRE_Real cycle_op_count;
HYPRE_Int cycle_type;
HYPRE_Int num_levels;
HYPRE_Int max_levels;
HYPRE_Real *num_coeffs;
HYPRE_Int *num_grid_sweeps;
HYPRE_Int *grid_relax_type;
HYPRE_Int **grid_relax_points;
HYPRE_Int block_mode;
HYPRE_Real *max_eig_est;
HYPRE_Real *min_eig_est;
HYPRE_Int cheby_order;
HYPRE_Real cheby_fraction;
/* Local variables */
HYPRE_Int *lev_counter;
HYPRE_Int Solve_err_flag;
HYPRE_Int k;
HYPRE_Int i, j, jj;
HYPRE_Int level;
HYPRE_Int cycle_param;
HYPRE_Int coarse_grid;
HYPRE_Int fine_grid;
HYPRE_Int Not_Finished;
HYPRE_Int num_sweep;
HYPRE_Int cg_num_sweep = 1;
HYPRE_Int relax_type;
HYPRE_Int relax_points;
HYPRE_Int relax_order;
HYPRE_Int relax_local;
HYPRE_Int old_version = 0;
HYPRE_Real *relax_weight;
HYPRE_Real *omega;
HYPRE_Real alfa, beta, gammaold;
HYPRE_Real gamma = 1.0;
HYPRE_Int local_size;
/* HYPRE_Int *smooth_option; */
HYPRE_Int smooth_type;
HYPRE_Int smooth_num_levels;
HYPRE_Int num_threads, my_id;
HYPRE_Real alpha;
HYPRE_Real **l1_norms = NULL;
HYPRE_Real *l1_norms_level;
HYPRE_Int seq_cg = 0;
MPI_Comm comm;
#if 0
HYPRE_Real *D_mat;
HYPRE_Real *S_vec;
#endif
/* Acquire data and allocate storage */
num_threads = hypre_NumThreads();
A_array = hypre_ParAMGDataAArray(amg_data);
P_array = hypre_ParAMGDataPArray(amg_data);
R_array = hypre_ParAMGDataRArray(amg_data);
CF_marker_array = hypre_ParAMGDataCFMarkerArray(amg_data);
Vtemp = hypre_ParAMGDataVtemp(amg_data);
Rtemp = hypre_ParAMGDataRtemp(amg_data);
Ptemp = hypre_ParAMGDataPtemp(amg_data);
Ztemp = hypre_ParAMGDataZtemp(amg_data);
num_levels = hypre_ParAMGDataNumLevels(amg_data);
max_levels = hypre_ParAMGDataMaxLevels(amg_data);
cycle_type = hypre_ParAMGDataCycleType(amg_data);
A_block_array = hypre_ParAMGDataABlockArray(amg_data);
P_block_array = hypre_ParAMGDataPBlockArray(amg_data);
R_block_array = hypre_ParAMGDataRBlockArray(amg_data);
block_mode = hypre_ParAMGDataBlockMode(amg_data);
num_grid_sweeps = hypre_ParAMGDataNumGridSweeps(amg_data);
grid_relax_type = hypre_ParAMGDataGridRelaxType(amg_data);
grid_relax_points = hypre_ParAMGDataGridRelaxPoints(amg_data);
relax_order = hypre_ParAMGDataRelaxOrder(amg_data);
relax_weight = hypre_ParAMGDataRelaxWeight(amg_data);
omega = hypre_ParAMGDataOmega(amg_data);
smooth_type = hypre_ParAMGDataSmoothType(amg_data);
smooth_num_levels = hypre_ParAMGDataSmoothNumLevels(amg_data);
l1_norms = hypre_ParAMGDataL1Norms(amg_data);
/* smooth_option = hypre_ParAMGDataSmoothOption(amg_data); */
max_eig_est = hypre_ParAMGDataMaxEigEst(amg_data);
min_eig_est = hypre_ParAMGDataMinEigEst(amg_data);
cheby_order = hypre_ParAMGDataChebyOrder(amg_data);
cheby_fraction = hypre_ParAMGDataChebyFraction(amg_data);
cycle_op_count = hypre_ParAMGDataCycleOpCount(amg_data);
lev_counter = hypre_CTAlloc(HYPRE_Int, num_levels);
if (hypre_ParAMGDataParticipate(amg_data)) seq_cg = 1;
/* Initialize */
Solve_err_flag = 0;
if (grid_relax_points) old_version = 1;
num_coeffs = hypre_CTAlloc(HYPRE_Real, num_levels);
num_coeffs[0] = hypre_ParCSRMatrixDNumNonzeros(A_array[0]);
comm = hypre_ParCSRMatrixComm(A_array[0]);
hypre_MPI_Comm_rank(comm,&my_id);
if (block_mode)
{
for (j = 1; j < num_levels; j++)
num_coeffs[j] = hypre_ParCSRBlockMatrixNumNonzeros(A_block_array[j]);
}
else
{
for (j = 1; j < num_levels; j++)
num_coeffs[j] = hypre_ParCSRMatrixDNumNonzeros(A_array[j]);
}
/*---------------------------------------------------------------------
* Initialize cycling control counter
*
* Cycling is controlled using a level counter: lev_counter[k]
*
* Each time relaxation is performed on level k, the
* counter is decremented by 1. If the counter is then
* negative, we go to the next finer level. If non-
* negative, we go to the next coarser level. The
* following actions control cycling:
*
* a. lev_counter[0] is initialized to 1.
* b. lev_counter[k] is initialized to cycle_type for k>0.
*
* c. During cycling, when going down to level k, lev_counter[k]
* is set to the max of (lev_counter[k],cycle_type)
*---------------------------------------------------------------------*/
Not_Finished = 1;
lev_counter[0] = 1;
for (k = 1; k < num_levels; ++k)
{
lev_counter[k] = cycle_type;
}
level = 0;
cycle_param = 1;
smoother = hypre_ParAMGDataSmoother(amg_data);
if (smooth_num_levels > 0)
{
if (smooth_type == 7 || smooth_type == 8
|| smooth_type == 17 || smooth_type == 18
|| smooth_type == 9 || smooth_type == 19)
{
HYPRE_Int actual_local_size = hypre_ParVectorActualLocalSize(Vtemp);
Utemp = hypre_ParVectorCreate(comm,hypre_ParVectorGlobalSize(Vtemp),
hypre_ParVectorPartitioning(Vtemp));
hypre_ParVectorOwnsPartitioning(Utemp) = 0;
local_size
= hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp));
if (local_size < actual_local_size)
{
hypre_VectorData(hypre_ParVectorLocalVector(Utemp)) =
hypre_CTAlloc(HYPRE_Complex, actual_local_size);
hypre_ParVectorActualLocalSize(Utemp) = actual_local_size;
}
else
hypre_ParVectorInitialize(Utemp);
}
}
/*---------------------------------------------------------------------
* Main loop of cycling
*--------------------------------------------------------------------*/
while (Not_Finished)
{
if (num_levels > 1)
{
local_size
= hypre_VectorSize(hypre_ParVectorLocalVector(F_array[level]));
hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp)) = local_size;
if (smooth_num_levels <= level)
{
cg_num_sweep = 1;
num_sweep = num_grid_sweeps[cycle_param];
Aux_U = U_array[level];
Aux_F = F_array[level];
}
else if (smooth_type > 9)
{
hypre_VectorSize(hypre_ParVectorLocalVector(Ztemp)) = local_size;
hypre_VectorSize(hypre_ParVectorLocalVector(Rtemp)) = local_size;
hypre_VectorSize(hypre_ParVectorLocalVector(Ptemp)) = local_size;
Ztemp_data = hypre_VectorData(hypre_ParVectorLocalVector(Ztemp));
Ptemp_data = hypre_VectorData(hypre_ParVectorLocalVector(Ptemp));
hypre_ParVectorSetConstantValues(Ztemp,0);
alpha = -1.0;
beta = 1.0;
hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level],
U_array[level], beta, F_array[level], Rtemp);
cg_num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data);
num_sweep = num_grid_sweeps[cycle_param];
Aux_U = Ztemp;
Aux_F = Rtemp;
}
else
{
cg_num_sweep = 1;
num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data);
Aux_U = U_array[level];
Aux_F = F_array[level];
}
relax_type = grid_relax_type[cycle_param];
}
else /* AB: 4/08: removed the max_levels > 1 check - should do this when max-levels = 1 also */
{
/* If no coarsening occurred, apply a simple smoother once */
Aux_U = U_array[level];
Aux_F = F_array[level];
num_sweep = 1;
/* TK: Use the user relax type (instead of 0) to allow for setting a
convergent smoother (e.g. in the solution of singular problems). */
relax_type = hypre_ParAMGDataUserRelaxType(amg_data);
}
if (l1_norms != NULL)
l1_norms_level = l1_norms[level];
else
l1_norms_level = NULL;
if (cycle_param == 3 && seq_cg)
{
hypre_seqAMGCycle(amg_data, level, F_array, U_array);
}
else
{
/*------------------------------------------------------------------
* Do the relaxation num_sweep times
*-----------------------------------------------------------------*/
for (jj = 0; jj < cg_num_sweep; jj++)
{
if (smooth_num_levels > level && smooth_type > 9)
hypre_ParVectorSetConstantValues(Aux_U,0);
for (j = 0; j < num_sweep; j++)
{
if (num_levels == 1 && max_levels > 1)
{
relax_points = 0;
relax_local = 0;
}
else
{
if (old_version)
relax_points = grid_relax_points[cycle_param][j];
relax_local = relax_order;
}
/*-----------------------------------------------
* VERY sloppy approximation to cycle complexity
*-----------------------------------------------*/
if (old_version && level < num_levels -1)
{
switch (relax_points)
{
case 1:
cycle_op_count += num_coeffs[level+1];
break;
case -1:
cycle_op_count += (num_coeffs[level]-num_coeffs[level+1]);
break;
}
}
else
{
cycle_op_count += num_coeffs[level];
}
/*-----------------------------------------------
Choose Smoother
-----------------------------------------------*/
if (smooth_num_levels > level &&
(smooth_type == 7 || smooth_type == 8 ||
smooth_type == 9 || smooth_type == 19 ||
smooth_type == 17 || smooth_type == 18))
{
hypre_VectorSize(hypre_ParVectorLocalVector(Utemp)) = local_size;
alpha = -1.0;
beta = 1.0;
hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level],
U_array[level], beta, Aux_F, Vtemp);
if (smooth_type == 8 || smooth_type == 18)
HYPRE_ParCSRParaSailsSolve(smoother[level],
(HYPRE_ParCSRMatrix) A_array[level],
(HYPRE_ParVector) Vtemp,
(HYPRE_ParVector) Utemp);
else if (smooth_type == 7 || smooth_type == 17)
HYPRE_ParCSRPilutSolve(smoother[level],
(HYPRE_ParCSRMatrix) A_array[level],
(HYPRE_ParVector) Vtemp,
(HYPRE_ParVector) Utemp);
else if (smooth_type == 9 || smooth_type == 19)
HYPRE_EuclidSolve(smoother[level],
(HYPRE_ParCSRMatrix) A_array[level],
(HYPRE_ParVector) Vtemp,
(HYPRE_ParVector) Utemp);
hypre_ParVectorAxpy(relax_weight[level],Utemp,Aux_U);
}
else if (smooth_num_levels > level &&
(smooth_type == 6 || smooth_type == 16))
{
HYPRE_SchwarzSolve(smoother[level],
(HYPRE_ParCSRMatrix) A_array[level],
(HYPRE_ParVector) Aux_F,
(HYPRE_ParVector) Aux_U);
}
/*else if (relax_type == 99)*/
else if (relax_type == 9 || relax_type == 99)
{ /* Gaussian elimination */
hypre_GaussElimSolve(amg_data, level, relax_type);
}
else if (relax_type == 18)
{ /* L1 - Jacobi*/
if (relax_order == 1 && cycle_param < 3)
{
/* need to do CF - so can't use the AMS one */
HYPRE_Int i;
HYPRE_Int loc_relax_points[2];
if (cycle_type < 2)
{
loc_relax_points[0] = 1;
loc_relax_points[1] = -1;
}
else
{
loc_relax_points[0] = -1;
loc_relax_points[1] = 1;
}
for (i=0; i < 2; i++)
hypre_ParCSRRelax_L1_Jacobi(A_array[level],
Aux_F,
CF_marker_array[level],
loc_relax_points[i],
relax_weight[level],
l1_norms[level],
Aux_U,
Vtemp);
}
else /* not CF - so use through AMS */
{
if (num_threads == 1)
hypre_ParCSRRelax(A_array[level],
Aux_F,
1,
1,
l1_norms_level,
relax_weight[level],
omega[level],0,0,0,0,
Aux_U,
Vtemp,
Ztemp);
else
hypre_ParCSRRelaxThreads(A_array[level],
Aux_F,
1,
1,
l1_norms_level,
relax_weight[level],
omega[level],
Aux_U,
Vtemp,
Ztemp);
}
}
else if (relax_type == 15)
{ /* CG */
if (j ==0) /* do num sweep iterations of CG */
hypre_ParCSRRelax_CG( smoother[level],
A_array[level],
Aux_F,
Aux_U,
num_sweep);
}
else if (relax_type == 16)
{ /* scaled Chebyshev */
HYPRE_Int scale = 1;
HYPRE_Int variant = 0;
hypre_ParCSRRelax_Cheby(A_array[level],
Aux_F,
max_eig_est[level],
min_eig_est[level],
cheby_fraction, cheby_order, scale,
variant, Aux_U, Vtemp, Ztemp );
}
else if (relax_type ==17)
{
hypre_BoomerAMGRelax_FCFJacobi(A_array[level],
Aux_F,
CF_marker_array[level],
relax_weight[level],
Aux_U,
Vtemp);
}
else if (old_version)
{
Solve_err_flag = hypre_BoomerAMGRelax(A_array[level],
Aux_F,
CF_marker_array[level],
relax_type, relax_points,
relax_weight[level],
omega[level],
l1_norms_level,
Aux_U,
Vtemp,
Ztemp);
}
else
{
/* smoother than can have CF ordering */
if (block_mode)
{
Solve_err_flag = hypre_BoomerAMGBlockRelaxIF(A_block_array[level],
Aux_F,
CF_marker_array[level],
relax_type,
relax_local,
cycle_param,
relax_weight[level],
omega[level],
Aux_U,
Vtemp);
}
else
{
Solve_err_flag = hypre_BoomerAMGRelaxIF(A_array[level],
Aux_F,
CF_marker_array[level],
relax_type,
relax_local,
cycle_param,
relax_weight[level],
omega[level],
l1_norms_level,
Aux_U,
Vtemp,
Ztemp);
}
}
if (Solve_err_flag != 0)
return(Solve_err_flag);
}
if (smooth_num_levels > level && smooth_type > 9)
{
gammaold = gamma;
gamma = hypre_ParVectorInnerProd(Rtemp,Ztemp);
if (jj == 0)
hypre_ParVectorCopy(Ztemp,Ptemp);
else
{
beta = gamma/gammaold;
for (i=0; i < local_size; i++)
Ptemp_data[i] = Ztemp_data[i] + beta*Ptemp_data[i];
}
hypre_ParCSRMatrixMatvec(1.0,A_array[level],Ptemp,0.0,Vtemp);
alfa = gamma /hypre_ParVectorInnerProd(Ptemp,Vtemp);
hypre_ParVectorAxpy(alfa,Ptemp,U_array[level]);
hypre_ParVectorAxpy(-alfa,Vtemp,Rtemp);
}
}
}
/*------------------------------------------------------------------
* Decrement the control counter and determine which grid to visit next
*-----------------------------------------------------------------*/
--lev_counter[level];
if (lev_counter[level] >= 0 && level != num_levels-1)
{
/*---------------------------------------------------------------
* Visit coarser level next.
* Compute residual using hypre_ParCSRMatrixMatvec.
* Perform restriction using hypre_ParCSRMatrixMatvecT.
* Reset counters and cycling parameters for coarse level
*--------------------------------------------------------------*/
fine_grid = level;
coarse_grid = level + 1;
hypre_ParVectorSetConstantValues(U_array[coarse_grid], 0.0);
alpha = -1.0;
beta = 1.0;
if (block_mode)
{
hypre_ParVectorCopy(F_array[fine_grid],Vtemp);
hypre_ParCSRBlockMatrixMatvec(alpha, A_block_array[fine_grid], U_array[fine_grid],
beta, Vtemp);
}
else
{
// JSP: avoid unnecessary copy using out-of-place version of SpMV
hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[fine_grid], U_array[fine_grid],
beta, F_array[fine_grid], Vtemp);
}
alpha = 1.0;
beta = 0.0;
if (block_mode)
{
hypre_ParCSRBlockMatrixMatvecT(alpha,R_block_array[fine_grid],Vtemp,
beta,F_array[coarse_grid]);
}
else
{
hypre_ParCSRMatrixMatvecT(alpha,R_array[fine_grid],Vtemp,
beta,F_array[coarse_grid]);
}
++level;
lev_counter[level] = hypre_max(lev_counter[level],cycle_type);
cycle_param = 1;
if (level == num_levels-1) cycle_param = 3;
}
else if (level != 0)
{
/*---------------------------------------------------------------
* Visit finer level next.
* Interpolate and add correction using hypre_ParCSRMatrixMatvec.
* Reset counters and cycling parameters for finer level.
*--------------------------------------------------------------*/
fine_grid = level - 1;
coarse_grid = level;
alpha = 1.0;
beta = 1.0;
if (block_mode)
{
hypre_ParCSRBlockMatrixMatvec(alpha, P_block_array[fine_grid],
U_array[coarse_grid],
beta, U_array[fine_grid]);
}
else
{
hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid],
U_array[coarse_grid],
beta, U_array[fine_grid]);
}
--level;
cycle_param = 2;
}
else
{
Not_Finished = 0;
}
}
hypre_ParAMGDataCycleOpCount(amg_data) = cycle_op_count;
hypre_TFree(lev_counter);
hypre_TFree(num_coeffs);
if (smooth_num_levels > 0)
{
if (smooth_type == 7 || smooth_type == 8 || smooth_type == 9 ||
smooth_type == 17 || smooth_type == 18 || smooth_type == 19 )
hypre_ParVectorDestroy(Utemp);
}
return(Solve_err_flag);
}
static int f(realtype t, N_Vector u, N_Vector udot, void *user_data)
{
realtype ui, ult, urt, hordc, horac, hdiff, hadv;
realtype *udata, *udotdata, *z;
int i;
int npes, my_pe, my_length, my_pe_m1, my_pe_p1, last_pe;
UserData data;
MPI_Status status;
MPI_Comm comm;
HYPRE_ParVector uhyp;
HYPRE_ParVector udothyp;
/* Extract hypre vectors */
uhyp = N_VGetVector_ParHyp(u);
udothyp = N_VGetVector_ParHyp(udot);
/* Access hypre vectors local data */
udata = hypre_VectorData(hypre_ParVectorLocalVector(uhyp));
udotdata = hypre_VectorData(hypre_ParVectorLocalVector(udothyp));
/* Extract needed problem constants from data */
data = (UserData) user_data;
hordc = data->hdcoef;
horac = data->hacoef;
/* Extract parameters for parhyp computation. */
comm = data->comm;
npes = data->npes; /* Number of processes */
my_pe = data->my_pe; /* Current process number */
my_length = hypre_ParVectorLastIndex(uhyp) /* Local length of uhyp */
- hypre_ParVectorFirstIndex(uhyp) + 1;
z = data->z;
/* Compute related parameters. */
my_pe_m1 = my_pe - 1;
my_pe_p1 = my_pe + 1;
last_pe = npes - 1;
/* Store local segment of u in the working array z. */
for (i = 1; i <= my_length; i++)
z[i] = udata[i - 1];
/* Pass needed data to processes before and after current process. */
if (my_pe != 0)
MPI_Send(&z[1], 1, PVEC_REAL_MPI_TYPE, my_pe_m1, 0, comm);
if (my_pe != last_pe)
MPI_Send(&z[my_length], 1, PVEC_REAL_MPI_TYPE, my_pe_p1, 0, comm);
/* Receive needed data from processes before and after current process. */
if (my_pe != 0)
MPI_Recv(&z[0], 1, PVEC_REAL_MPI_TYPE, my_pe_m1, 0, comm, &status);
else
z[0] = ZERO;
if (my_pe != last_pe)
MPI_Recv(&z[my_length+1], 1, PVEC_REAL_MPI_TYPE, my_pe_p1, 0, comm,
&status);
else
z[my_length + 1] = ZERO;
/* Loop over all grid points in current process. */
for (i=1; i<=my_length; i++) {
/* Extract u at x_i and two neighboring points */
ui = z[i];
ult = z[i-1];
urt = z[i+1];
/* Set diffusion and advection terms and load into udot */
hdiff = hordc*(ult - RCONST(2.0)*ui + urt);
hadv = horac*(urt - ult);
udotdata[i-1] = hdiff + hadv;
}
return(0);
}
static int Precond(realtype tn, N_Vector u, N_Vector fu,
booleantype jok, booleantype *jcurPtr,
realtype gamma, void *user_data,
N_Vector vtemp1, N_Vector vtemp2, N_Vector vtemp3)
{
realtype c1, c2, cydn, cyup, diag, ydn, yup, q4coef, dely, verdco, hordco;
realtype **(*P)[MYSUB], **(*Jbd)[MYSUB];
int nvmxsub, ier, offset;
long int *(*pivot)[MYSUB];
int lx, ly, jy, isuby;
realtype *udata, **a, **j;
HYPRE_ParVector uhyp;
UserData data;
/* Make local copies of pointers in user_data, pointer to u's data,
and PE index pair */
data = (UserData) user_data;
P = data->P;
Jbd = data->Jbd;
pivot = data->pivot;
isuby = data->isuby;
nvmxsub = data->nvmxsub;
uhyp = N_VGetVector_ParHyp(u);
udata = hypre_VectorData(hypre_ParVectorLocalVector(uhyp));
if (jok) {
/* jok = TRUE: Copy Jbd to P */
for (ly = 0; ly < MYSUB; ly++)
for (lx = 0; lx < MXSUB; lx++)
denseCopy(Jbd[lx][ly], P[lx][ly], NVARS, NVARS);
*jcurPtr = FALSE;
}
else {
/* jok = FALSE: Generate Jbd from scratch and copy to P */
/* Make local copies of problem variables, for efficiency */
q4coef = data->q4;
dely = data->dy;
verdco = data->vdco;
hordco = data->hdco;
/* Compute 2x2 diagonal Jacobian blocks (using q4 values
c*omputed on the last f call). Load into P. */
for (ly = 0; ly < MYSUB; ly++) {
jy = ly + isuby*MYSUB;
ydn = YMIN + (jy - RCONST(0.5))*dely;
yup = ydn + dely;
cydn = verdco*SUNRexp(RCONST(0.2)*ydn);
cyup = verdco*SUNRexp(RCONST(0.2)*yup);
diag = -(cydn + cyup + RCONST(2.0)*hordco);
for (lx = 0; lx < MXSUB; lx++) {
offset = lx*NVARS + ly*nvmxsub;
c1 = udata[offset];
c2 = udata[offset+1];
j = Jbd[lx][ly];
a = P[lx][ly];
IJth(j,1,1) = (-Q1*C3 - Q2*c2) + diag;
IJth(j,1,2) = -Q2*c1 + q4coef;
IJth(j,2,1) = Q1*C3 - Q2*c2;
IJth(j,2,2) = (-Q2*c1 - q4coef) + diag;
denseCopy(j, a, NVARS, NVARS);
}
}
*jcurPtr = TRUE;
}
/* Scale by -gamma */
for (ly = 0; ly < MYSUB; ly++)
for (lx = 0; lx < MXSUB; lx++)
denseScale(-gamma, P[lx][ly], NVARS, NVARS);
/* Add identity matrix and do LU decompositions on blocks in place */
for (lx = 0; lx < MXSUB; lx++) {
for (ly = 0; ly < MYSUB; ly++) {
denseAddIdentity(P[lx][ly], NVARS);
ier = denseGETRF(P[lx][ly], NVARS, NVARS, pivot[lx][ly]);
if (ier != 0) return(1);
}
}
return(0);
}
/* ----------------------------------------------------------------------
* get_element
*
* Reads single element from hypre vector by accessing its raw block.
* Probably not the most efficient way to get the vector values.
* --------------------------------------------------------------------*/
realtype get_element(N_Vector X, long int i)
{
hypre_ParVector *Xvec = N_VGetVector_ParHyp(X);
const realtype *Xdata = hypre_VectorData(hypre_ParVectorLocalVector(Xvec));
return Xdata[i];
}
/*
Function: hypre_ParCSRMatrixEliminateAXB
This function eliminates the global rows and columns of a matrix
A corresponding to given lists of sorted (!) local row numbers,
so that the solution to the system A*X = B is X_b for the given rows.
The elimination is done as follows:
(input) (output)
/ A_ii | A_ib \ / A_ii | 0 \
A = | -----+----- | ---> | -----+----- |
\ A_bi | A_bb / \ 0 | I /
/ X_i \ / X_i \
X = | --- | ---> | --- | (no change)
\ X_b / \ X_b /
/ B_i \ / B_i - A_ib * X_b \
B = | --- | ---> | ---------------- |
\ B_b / \ X_b /
*/
void hypre_ParCSRMatrixEliminateAXB(hypre_ParCSRMatrix *A,
HYPRE_Int num_rowscols_to_elim,
HYPRE_Int *rowscols_to_elim,
hypre_ParVector *X,
hypre_ParVector *B)
{
hypre_CSRMatrix *diag = hypre_ParCSRMatrixDiag(A);
hypre_CSRMatrix *offd = hypre_ParCSRMatrixOffd(A);
HYPRE_Int diag_nrows = hypre_CSRMatrixNumRows(diag);
HYPRE_Int offd_ncols = hypre_CSRMatrixNumCols(offd);
hypre_Vector *Xlocal = hypre_ParVectorLocalVector(X);
hypre_Vector *Blocal = hypre_ParVectorLocalVector(B);
HYPRE_Real *Bdata = hypre_VectorData(Blocal);
HYPRE_Real *Xdata = hypre_VectorData(Xlocal);
HYPRE_Int num_offd_cols_to_elim;
HYPRE_Int *offd_cols_to_elim;
HYPRE_Real *eliminate_coefs;
/* figure out which offd cols should be eliminated and with what coef */
hypre_ParCSRCommHandle *comm_handle;
hypre_ParCSRCommPkg *comm_pkg;
HYPRE_Int num_sends;
HYPRE_Int index, start;
HYPRE_Int i, j, k, irow;
HYPRE_Real *eliminate_row = hypre_CTAlloc(HYPRE_Real, diag_nrows);
HYPRE_Real *eliminate_col = hypre_CTAlloc(HYPRE_Real, offd_ncols);
HYPRE_Real *buf_data, coef;
/* make sure A has a communication package */
comm_pkg = hypre_ParCSRMatrixCommPkg(A);
if (!comm_pkg)
{
hypre_MatvecCommPkgCreate(A);
comm_pkg = hypre_ParCSRMatrixCommPkg(A);
}
/* HACK: rows that shouldn't be eliminated are marked with quiet NaN;
those that should are set to the boundary value from X; this is to
avoid sending complex type (int+double) or communicating twice. */
for (i = 0; i < diag_nrows; i++)
{
eliminate_row[i] = std::numeric_limits<HYPRE_Real>::quiet_NaN();
}
for (i = 0; i < num_rowscols_to_elim; i++)
{
irow = rowscols_to_elim[i];
eliminate_row[irow] = Xdata[irow];
}
/* use a Matvec communication pattern to find (in eliminate_col)
which of the local offd columns are to be eliminated */
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
buf_data = hypre_CTAlloc(HYPRE_Real,
hypre_ParCSRCommPkgSendMapStart(comm_pkg,
num_sends));
index = 0;
for (i = 0; i < num_sends; i++)
{
start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
{
k = hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j);
buf_data[index++] = eliminate_row[k];
}
}
comm_handle = hypre_ParCSRCommHandleCreate(1, comm_pkg,
buf_data, eliminate_col);
/* do sequential part of the elimination while stuff is getting sent */
hypre_CSRMatrixEliminateAXB(diag, num_rowscols_to_elim, rowscols_to_elim,
Xlocal, Blocal);
/* finish the communication */
hypre_ParCSRCommHandleDestroy(comm_handle);
/* received eliminate_col[], count offd columns to eliminate */
num_offd_cols_to_elim = 0;
for (i = 0; i < offd_ncols; i++)
{
coef = eliminate_col[i];
if (coef == coef) // test for NaN
{
num_offd_cols_to_elim++;
}
}
offd_cols_to_elim = hypre_CTAlloc(HYPRE_Int, num_offd_cols_to_elim);
eliminate_coefs = hypre_CTAlloc(HYPRE_Real, num_offd_cols_to_elim);
/* get a list of offd column indices and coefs */
num_offd_cols_to_elim = 0;
for (i = 0; i < offd_ncols; i++)
{
coef = eliminate_col[i];
if (coef == coef) // test for NaN
{
offd_cols_to_elim[num_offd_cols_to_elim] = i;
eliminate_coefs[num_offd_cols_to_elim] = coef;
num_offd_cols_to_elim++;
}
}
hypre_TFree(buf_data);
hypre_TFree(eliminate_row);
hypre_TFree(eliminate_col);
/* eliminate the off-diagonal part */
hypre_CSRMatrixEliminateOffdColsAXB(offd, num_offd_cols_to_elim,
offd_cols_to_elim,
eliminate_coefs, Blocal);
hypre_CSRMatrixEliminateOffdRowsAXB(offd, num_rowscols_to_elim,
rowscols_to_elim);
/* set boundary values in the rhs */
for (int i = 0; i < num_rowscols_to_elim; i++)
{
irow = rowscols_to_elim[i];
Bdata[irow] = Xdata[irow];
}
hypre_TFree(offd_cols_to_elim);
hypre_TFree(eliminate_coefs);
}
HYPRE_Int
hypre_ParCSRMatrixMatvec( double alpha,
hypre_ParCSRMatrix *A,
hypre_ParVector *x,
double beta,
hypre_ParVector *y )
{
hypre_ParCSRCommHandle **comm_handle;
hypre_ParCSRCommPkg *comm_pkg = hypre_ParCSRMatrixCommPkg(A);
hypre_CSRMatrix *diag = hypre_ParCSRMatrixDiag(A);
hypre_CSRMatrix *offd = hypre_ParCSRMatrixOffd(A);
hypre_Vector *x_local = hypre_ParVectorLocalVector(x);
hypre_Vector *y_local = hypre_ParVectorLocalVector(y);
HYPRE_Int num_rows = hypre_ParCSRMatrixGlobalNumRows(A);
HYPRE_Int num_cols = hypre_ParCSRMatrixGlobalNumCols(A);
hypre_Vector *x_tmp;
HYPRE_Int x_size = hypre_ParVectorGlobalSize(x);
HYPRE_Int y_size = hypre_ParVectorGlobalSize(y);
HYPRE_Int num_vectors = hypre_VectorNumVectors(x_local);
HYPRE_Int num_cols_offd = hypre_CSRMatrixNumCols(offd);
HYPRE_Int ierr = 0;
HYPRE_Int num_sends, i, j, jv, index, start;
HYPRE_Int vecstride = hypre_VectorVectorStride( x_local );
HYPRE_Int idxstride = hypre_VectorIndexStride( x_local );
double *x_tmp_data, **x_buf_data;
double *x_local_data = hypre_VectorData(x_local);
/*---------------------------------------------------------------------
* Check for size compatibility. ParMatvec returns ierr = 11 if
* length of X doesn't equal the number of columns of A,
* ierr = 12 if the length of Y doesn't equal the number of rows
* of A, and ierr = 13 if both are true.
*
* Because temporary vectors are often used in ParMatvec, none of
* these conditions terminates processing, and the ierr flag
* is informational only.
*--------------------------------------------------------------------*/
hypre_assert( idxstride>0 );
if (num_cols != x_size)
ierr = 11;
if (num_rows != y_size)
ierr = 12;
if (num_cols != x_size && num_rows != y_size)
ierr = 13;
hypre_assert( hypre_VectorNumVectors(y_local)==num_vectors );
if ( num_vectors==1 )
x_tmp = hypre_SeqVectorCreate( num_cols_offd );
else
{
hypre_assert( num_vectors>1 );
x_tmp = hypre_SeqMultiVectorCreate( num_cols_offd, num_vectors );
}
hypre_SeqVectorInitialize(x_tmp);
x_tmp_data = hypre_VectorData(x_tmp);
comm_handle = hypre_CTAlloc(hypre_ParCSRCommHandle*,num_vectors);
/*---------------------------------------------------------------------
* If there exists no CommPkg for A, a CommPkg is generated using
* equally load balanced partitionings
*--------------------------------------------------------------------*/
if (!comm_pkg)
{
hypre_MatvecCommPkgCreate(A);
comm_pkg = hypre_ParCSRMatrixCommPkg(A);
}
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
x_buf_data = hypre_CTAlloc( double*, num_vectors );
for ( jv=0; jv<num_vectors; ++jv )
x_buf_data[jv] = hypre_CTAlloc(double, hypre_ParCSRCommPkgSendMapStart
(comm_pkg, num_sends));
if ( num_vectors==1 )
{
index = 0;
for (i = 0; i < num_sends; i++)
{
start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
x_buf_data[0][index++]
= x_local_data[hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j)];
}
}
else
for ( jv=0; jv<num_vectors; ++jv )
{
index = 0;
for (i = 0; i < num_sends; i++)
{
start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
x_buf_data[jv][index++]
= x_local_data[
jv*vecstride +
idxstride*hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j) ];
}
}
hypre_assert( idxstride==1 );
/* >>> ... The assert is because the following loop only works for 'column' storage of a multivector <<<
>>> This needs to be fixed to work more generally, at least for 'row' storage. <<<
>>> This in turn, means either change CommPkg so num_sends is no.zones*no.vectors (not no.zones)
>>> or, less dangerously, put a stride in the logic of CommHandleCreate (stride either from a
>>> new arg or a new variable inside CommPkg). Or put the num_vector iteration inside
>>> CommHandleCreate (perhaps a new multivector variant of it).
*/
for ( jv=0; jv<num_vectors; ++jv )
{
comm_handle[jv] = hypre_ParCSRCommHandleCreate
( 1, comm_pkg, x_buf_data[jv], &(x_tmp_data[jv*num_cols_offd]) );
}
hypre_CSRMatrixMatvec( alpha, diag, x_local, beta, y_local);
for ( jv=0; jv<num_vectors; ++jv )
{
hypre_ParCSRCommHandleDestroy(comm_handle[jv]);
comm_handle[jv] = NULL;
}
hypre_TFree(comm_handle);
if (num_cols_offd) hypre_CSRMatrixMatvec( alpha, offd, x_tmp, 1.0, y_local);
hypre_SeqVectorDestroy(x_tmp);
x_tmp = NULL;
for ( jv=0; jv<num_vectors; ++jv ) hypre_TFree(x_buf_data[jv]);
hypre_TFree(x_buf_data);
return ierr;
}
HYPRE_Int hypre_BoomerAMGRelaxT( hypre_ParCSRMatrix *A,
hypre_ParVector *f,
HYPRE_Int *cf_marker,
HYPRE_Int relax_type,
HYPRE_Int relax_points,
double relax_weight,
hypre_ParVector *u,
hypre_ParVector *Vtemp )
{
hypre_CSRMatrix *A_diag = hypre_ParCSRMatrixDiag(A);
double *A_diag_data = hypre_CSRMatrixData(A_diag);
HYPRE_Int *A_diag_i = hypre_CSRMatrixI(A_diag);
HYPRE_Int n_global= hypre_ParCSRMatrixGlobalNumRows(A);
HYPRE_Int n = hypre_CSRMatrixNumRows(A_diag);
HYPRE_Int first_index = hypre_ParVectorFirstIndex(u);
hypre_Vector *u_local = hypre_ParVectorLocalVector(u);
double *u_data = hypre_VectorData(u_local);
hypre_Vector *Vtemp_local = hypre_ParVectorLocalVector(Vtemp);
double *Vtemp_data = hypre_VectorData(Vtemp_local);
hypre_CSRMatrix *A_CSR;
HYPRE_Int *A_CSR_i;
HYPRE_Int *A_CSR_j;
double *A_CSR_data;
hypre_Vector *f_vector;
double *f_vector_data;
HYPRE_Int i;
HYPRE_Int jj;
HYPRE_Int column;
HYPRE_Int relax_error = 0;
double *A_mat;
double *b_vec;
double zero = 0.0;
/*-----------------------------------------------------------------------
* Switch statement to direct control based on relax_type:
* relax_type = 7 -> Jacobi (uses ParMatvec)
* relax_type = 9 -> Direct Solve
*-----------------------------------------------------------------------*/
switch (relax_type)
{
case 7: /* Jacobi (uses ParMatvec) */
{
/*-----------------------------------------------------------------
* Copy f into temporary vector.
*-----------------------------------------------------------------*/
hypre_ParVectorCopy(f,Vtemp);
/*-----------------------------------------------------------------
* Perform MatvecT Vtemp=f-A^Tu
*-----------------------------------------------------------------*/
hypre_ParCSRMatrixMatvecT(-1.0,A, u, 1.0, Vtemp);
for (i = 0; i < n; i++)
{
/*-----------------------------------------------------------
* If diagonal is nonzero, relax point i; otherwise, skip it.
*-----------------------------------------------------------*/
if (A_diag_data[A_diag_i[i]] != zero)
{
u_data[i] += relax_weight * Vtemp_data[i]
/ A_diag_data[A_diag_i[i]];
}
}
}
break;
case 9: /* Direct solve: use gaussian elimination */
{
/*-----------------------------------------------------------------
* Generate CSR matrix from ParCSRMatrix A
*-----------------------------------------------------------------*/
if (n)
{
A_CSR = hypre_ParCSRMatrixToCSRMatrixAll(A);
f_vector = hypre_ParVectorToVectorAll(f);
A_CSR_i = hypre_CSRMatrixI(A_CSR);
A_CSR_j = hypre_CSRMatrixJ(A_CSR);
A_CSR_data = hypre_CSRMatrixData(A_CSR);
f_vector_data = hypre_VectorData(f_vector);
A_mat = hypre_CTAlloc(double, n_global*n_global);
b_vec = hypre_CTAlloc(double, n_global);
/*---------------------------------------------------------------
* Load transpose of CSR matrix into A_mat.
*---------------------------------------------------------------*/
for (i = 0; i < n_global; i++)
{
for (jj = A_CSR_i[i]; jj < A_CSR_i[i+1]; jj++)
{
column = A_CSR_j[jj];
A_mat[column*n_global+i] = A_CSR_data[jj];
}
b_vec[i] = f_vector_data[i];
}
relax_error = gselim(A_mat,b_vec,n_global);
for (i = 0; i < n; i++)
{
u_data[i] = b_vec[first_index+i];
}
hypre_TFree(A_mat);
hypre_TFree(b_vec);
hypre_CSRMatrixDestroy(A_CSR);
A_CSR = NULL;
hypre_SeqVectorDestroy(f_vector);
f_vector = NULL;
}
}
break;
}
return(relax_error);
}
