hypre_ParVector *
hypre_ParVectorCreate( MPI_Comm comm,
HYPRE_Int global_size,
HYPRE_Int *partitioning)
{
hypre_ParVector *vector;
HYPRE_Int num_procs, my_id;
if (global_size < 0)
{
hypre_error_in_arg(2);
return NULL;
}
vector = hypre_CTAlloc(hypre_ParVector, 1);
hypre_MPI_Comm_rank(comm,&my_id);
if (!partitioning)
{
hypre_MPI_Comm_size(comm,&num_procs);
#ifdef HYPRE_NO_GLOBAL_PARTITION
hypre_GenerateLocalPartitioning(global_size, num_procs, my_id, &partitioning);
#else
hypre_GeneratePartitioning(global_size, num_procs, &partitioning);
#endif
}
hypre_ParVectorAssumedPartition(vector) = NULL;
hypre_ParVectorComm(vector) = comm;
hypre_ParVectorGlobalSize(vector) = global_size;
#ifdef HYPRE_NO_GLOBAL_PARTITION
hypre_ParVectorFirstIndex(vector) = partitioning[0];
hypre_ParVectorLastIndex(vector) = partitioning[1]-1;
hypre_ParVectorPartitioning(vector) = partitioning;
hypre_ParVectorLocalVector(vector) =
hypre_SeqVectorCreate(partitioning[1]-partitioning[0]);
#else
hypre_ParVectorFirstIndex(vector) = partitioning[my_id];
hypre_ParVectorLastIndex(vector) = partitioning[my_id+1] -1;
hypre_ParVectorPartitioning(vector) = partitioning;
hypre_ParVectorLocalVector(vector) =
hypre_SeqVectorCreate(partitioning[my_id+1]-partitioning[my_id]);
#endif
/* set defaults */
hypre_ParVectorOwnsData(vector) = 1;
hypre_ParVectorOwnsPartitioning(vector) = 1;
return vector;
}
hypre_ParVector
*hypre_ParVectorRead( MPI_Comm comm,
const char *file_name )
{
char new_file_name[80];
hypre_ParVector *par_vector;
HYPRE_Int my_id, num_procs;
HYPRE_Int *partitioning;
HYPRE_Int global_size, i;
FILE *fp;
hypre_MPI_Comm_rank(comm,&my_id);
hypre_MPI_Comm_size(comm,&num_procs);
partitioning = hypre_CTAlloc(HYPRE_Int,num_procs+1);
hypre_sprintf(new_file_name,"%s.INFO.%d",file_name,my_id);
fp = fopen(new_file_name, "r");
hypre_fscanf(fp, "%d\n", &global_size);
#ifdef HYPRE_NO_GLOBAL_PARTITION
for (i=0; i < 2; i++)
hypre_fscanf(fp, "%d\n", &partitioning[i]);
fclose (fp);
#else
for (i=0; i < num_procs; i++)
hypre_fscanf(fp, "%d\n", &partitioning[i]);
fclose (fp);
partitioning[num_procs] = global_size;
#endif
par_vector = hypre_CTAlloc(hypre_ParVector, 1);
hypre_ParVectorComm(par_vector) = comm;
hypre_ParVectorGlobalSize(par_vector) = global_size;
#ifdef HYPRE_NO_GLOBAL_PARTITION
hypre_ParVectorFirstIndex(par_vector) = partitioning[0];
hypre_ParVectorLastIndex(par_vector) = partitioning[1]-1;
#else
hypre_ParVectorFirstIndex(par_vector) = partitioning[my_id];
hypre_ParVectorLastIndex(par_vector) = partitioning[my_id+1]-1;
#endif
hypre_ParVectorPartitioning(par_vector) = partitioning;
hypre_ParVectorOwnsData(par_vector) = 1;
hypre_ParVectorOwnsPartitioning(par_vector) = 1;
hypre_sprintf(new_file_name,"%s.%d",file_name,my_id);
hypre_ParVectorLocalVector(par_vector) = hypre_SeqVectorRead(new_file_name);
/* multivector code not written yet >>> */
hypre_assert( hypre_ParVectorNumVectors(par_vector) == 1 );
return par_vector;
}
void *
hypre_ParKrylovCreateVector( void *vvector )
{
hypre_ParVector *vector = vvector;
hypre_ParVector *new_vector;
new_vector = hypre_ParVectorCreate( hypre_ParVectorComm(vector),
hypre_ParVectorGlobalSize(vector),
hypre_ParVectorPartitioning(vector) );
hypre_ParVectorSetPartitioningOwner(new_vector,0);
hypre_ParVectorInitialize(new_vector);
return ( (void *) new_vector );
}
hypre_ParVector *
hypre_ParVectorCloneShallow( hypre_ParVector *x )
{
hypre_ParVector * y = hypre_ParVectorCreate(
hypre_ParVectorComm(x), hypre_ParVectorGlobalSize(x), hypre_ParVectorPartitioning(x) );
hypre_ParVectorOwnsData(y) = 1;
/* ...This vector owns its local vector, although the local vector doesn't own _its_ data */
hypre_ParVectorOwnsPartitioning(y) = 0;
hypre_SeqVectorDestroy( hypre_ParVectorLocalVector(y) );
hypre_ParVectorLocalVector(y) = hypre_SeqVectorCloneShallow(
hypre_ParVectorLocalVector(x) );
hypre_ParVectorFirstIndex(y) = hypre_ParVectorFirstIndex(x);
return y;
}
int HYPRE_ParCSRCotreeSetup(HYPRE_Solver solver, HYPRE_ParCSRMatrix A,
HYPRE_ParVector b, HYPRE_ParVector x)
{
int *partition, *new_partition, nprocs, *tindices, ii;
void *vsolver = (void *) solver;
/*
void *vA = (void *) A;
void *vb = (void *) b;
void *vx = (void *) x;
*/
hypre_CotreeData *cotree_data = (hypre_CotreeData *) vsolver;
hypre_ParCSRMatrix **submatrices;
hypre_ParVector *new_vector;
MPI_Comm comm;
cotree_data->Aee = (hypre_ParCSRMatrix *) A;
hypre_ParCSRMatrixGenSpanningTree(cotree_data->Gen, &tindices, 1);
submatrices = (hypre_ParCSRMatrix **) malloc(sizeof(hypre_ParCSRMatrix *));
hypre_ParCSRMatrixExtractSubmatrices(cotree_data->Aee, tindices,
&submatrices);
cotree_data->Att = submatrices[0];
cotree_data->Atc = submatrices[1];
cotree_data->Act = submatrices[2];
cotree_data->Acc = submatrices[3];
hypre_ParCSRMatrixExtractRowSubmatrices(cotree_data->Gen, tindices,
&submatrices);
cotree_data->Gt = submatrices[0];
cotree_data->Gc = submatrices[1];
free(submatrices);
comm = hypre_ParCSRMatrixComm((hypre_ParCSRMatrix *) A);
MPI_Comm_size(comm, &nprocs);
partition = hypre_ParVectorPartitioning((hypre_ParVector *) b);
new_partition = (int *) malloc((nprocs+1) * sizeof(int));
for (ii = 0; ii <= nprocs; ii++) new_partition[ii] = partition[ii];
/* partition = hypre_ParVectorPartitioning((hypre_ParVector *) b); */
new_vector = hypre_ParVectorCreate(hypre_ParVectorComm((hypre_ParVector *)b),
(int) hypre_ParVectorGlobalSize((hypre_ParVector *) b),
new_partition);
hypre_ParVectorInitialize(new_vector);
cotree_data->w = new_vector;
return 0;
}
/******************************************************************************
*
* hypre_IJVectorInitializePar
*
* initializes ParVector of IJVectorPar
*
*****************************************************************************/
HYPRE_Int
hypre_IJVectorInitializePar(hypre_IJVector *vector)
{
hypre_ParVector *par_vector = hypre_IJVectorObject(vector);
hypre_AuxParVector *aux_vector = hypre_IJVectorTranslator(vector);
HYPRE_Int *partitioning = hypre_ParVectorPartitioning(par_vector);
hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector);
HYPRE_Int my_id;
HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector);
MPI_Comm comm = hypre_IJVectorComm(vector);
hypre_MPI_Comm_rank(comm,&my_id);
if (!partitioning)
{
if (print_level)
{
hypre_printf("No ParVector partitioning for initialization -- ");
hypre_printf("hypre_IJVectorInitializePar\n");
}
hypre_error_in_arg(1);
}
#ifdef HYPRE_NO_GLOBAL_PARTITION
hypre_VectorSize(local_vector) = partitioning[1] -
partitioning[0];
#else
hypre_VectorSize(local_vector) = partitioning[my_id+1] -
partitioning[my_id];
#endif
hypre_ParVectorInitialize(par_vector);
if (!aux_vector)
{
hypre_AuxParVectorCreate(&aux_vector);
hypre_IJVectorTranslator(vector) = aux_vector;
}
hypre_AuxParVectorInitialize(aux_vector);
return hypre_error_flag;
}
void *
hypre_ParKrylovCreateVectorArray(HYPRE_Int n, void *vvector )
{
hypre_ParVector *vector = vvector;
hypre_ParVector **new_vector;
HYPRE_Int i;
new_vector = hypre_CTAlloc(hypre_ParVector*,n);
for (i=0; i < n; i++)
{
new_vector[i] = hypre_ParVectorCreate( hypre_ParVectorComm(vector),
hypre_ParVectorGlobalSize(vector),
hypre_ParVectorPartitioning(vector) );
hypre_ParVectorSetPartitioningOwner(new_vector[i],0);
hypre_ParVectorInitialize(new_vector[i]);
}
return ( (void *) new_vector );
}
HYPRE_Int
hypre_ParVectorPrint( hypre_ParVector *vector,
const char *file_name )
{
char new_file_name[80];
hypre_Vector *local_vector;
MPI_Comm comm;
HYPRE_Int my_id, num_procs, i;
HYPRE_Int *partitioning;
HYPRE_Int global_size;
FILE *fp;
if (!vector)
{
hypre_error_in_arg(1);
return hypre_error_flag;
}
local_vector = hypre_ParVectorLocalVector(vector);
comm = hypre_ParVectorComm(vector);
partitioning = hypre_ParVectorPartitioning(vector);
global_size = hypre_ParVectorGlobalSize(vector);
hypre_MPI_Comm_rank(comm,&my_id);
hypre_MPI_Comm_size(comm,&num_procs);
hypre_sprintf(new_file_name,"%s.%d",file_name,my_id);
hypre_SeqVectorPrint(local_vector,new_file_name);
hypre_sprintf(new_file_name,"%s.INFO.%d",file_name,my_id);
fp = fopen(new_file_name, "w");
hypre_fprintf(fp, "%d\n", global_size);
#ifdef HYPRE_NO_GLOBAL_PARTITION
for (i=0; i < 2; i++)
hypre_fprintf(fp, "%d\n", partitioning[i]);
#else
for (i=0; i < num_procs; i++)
hypre_fprintf(fp, "%d\n", partitioning[i]);
#endif
fclose (fp);
return hypre_error_flag;
}
HYPRE_Int
hypre_ParVectorDestroy( hypre_ParVector *vector )
{
if (vector)
{
if ( hypre_ParVectorOwnsData(vector) )
{
hypre_SeqVectorDestroy(hypre_ParVectorLocalVector(vector));
}
if ( hypre_ParVectorOwnsPartitioning(vector) )
{
hypre_TFree(hypre_ParVectorPartitioning(vector));
}
if (hypre_ParVectorAssumedPartition(vector))
hypre_ParVectorDestroyAssumedPartition(vector);
hypre_TFree(vector);
}
return hypre_error_flag;
}
HYPRE_Int
main( HYPRE_Int argc,
char *argv[] )
{
hypre_ParVector *vector1;
hypre_ParVector *vector2;
hypre_ParVector *tmp_vector;
HYPRE_Int num_procs, my_id;
HYPRE_Int global_size = 20;
HYPRE_Int local_size;
HYPRE_Int first_index;
HYPRE_Int num_vectors, vecstride, idxstride;
HYPRE_Int i, j;
HYPRE_Int *partitioning;
double prod;
double *data, *data2;
hypre_Vector *vector;
hypre_Vector *local_vector;
hypre_Vector *local_vector2;
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id );
hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs);
partitioning = NULL;
num_vectors = 3;
vector1 = hypre_ParMultiVectorCreate
( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors );
partitioning = hypre_ParVectorPartitioning(vector1);
hypre_ParVectorInitialize(vector1);
local_vector = hypre_ParVectorLocalVector(vector1);
data = hypre_VectorData(local_vector);
local_size = hypre_VectorSize(local_vector);
vecstride = hypre_VectorVectorStride(local_vector);
idxstride = hypre_VectorIndexStride(local_vector);
first_index = partitioning[my_id];
hypre_printf("vecstride=%i idxstride=%i local_size=%i num_vectors=%i",
vecstride, idxstride, local_size, num_vectors );
for (j=0; j<num_vectors; ++j )
for (i=0; i < local_size; i++)
data[ j*vecstride + i*idxstride ] = first_index+i + 100*j;
hypre_ParVectorPrint(vector1, "Vector");
local_vector2 = hypre_SeqMultiVectorCreate( global_size, num_vectors );
hypre_SeqVectorInitialize(local_vector2);
data2 = hypre_VectorData(local_vector2);
vecstride = hypre_VectorVectorStride(local_vector2);
idxstride = hypre_VectorIndexStride(local_vector2);
for (j=0; j<num_vectors; ++j )
for (i=0; i < global_size; i++)
data2[ j*vecstride + i*idxstride ] = i + 100*j;
/* partitioning = hypre_CTAlloc(HYPRE_Int,4);
partitioning[0] = 0;
partitioning[1] = 10;
partitioning[2] = 10;
partitioning[3] = 20;
*/
partitioning = hypre_CTAlloc(HYPRE_Int,1+num_procs);
hypre_GeneratePartitioning( global_size, num_procs, &partitioning );
vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning);
hypre_ParVectorSetPartitioningOwner(vector2,0);
hypre_ParVectorPrint(vector2, "Convert");
vector = hypre_ParVectorToVectorAll(vector2);
/*-----------------------------------------------------------
* Copy the vector into tmp_vector
*-----------------------------------------------------------*/
/* Read doesn't work for multivectors yet...
tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert");*/
tmp_vector = hypre_ParMultiVectorCreate
( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors );
hypre_ParVectorInitialize( tmp_vector );
hypre_ParVectorCopy( vector2, tmp_vector );
/*
tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning);
hypre_ParVectorSetPartitioningOwner(tmp_vector,0);
hypre_ParVectorInitialize(tmp_vector);
hypre_ParVectorCopy(vector1, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Copy");
*/
/*-----------------------------------------------------------
* Scale tmp_vector
*-----------------------------------------------------------*/
hypre_ParVectorScale(2.0, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Scale");
/*-----------------------------------------------------------
* Do an Axpy (2*vector - vector) = vector
*-----------------------------------------------------------*/
hypre_ParVectorAxpy(-1.0, vector1, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Axpy");
/*-----------------------------------------------------------
* Do an inner product vector* tmp_vector
*-----------------------------------------------------------*/
prod = hypre_ParVectorInnerProd(vector1, tmp_vector);
hypre_printf (" prod: %8.2f \n", prod);
/*-----------------------------------------------------------
* Finalize things
*-----------------------------------------------------------*/
hypre_ParVectorDestroy(vector1);
hypre_ParVectorDestroy(vector2);
hypre_ParVectorDestroy(tmp_vector);
hypre_SeqVectorDestroy(local_vector2);
if (vector) hypre_SeqVectorDestroy(vector);
/* Finalize MPI */
hypre_MPI_Finalize();
return 0;
}
HYPRE_Int
hypre_BoomerAMGCycle( void *amg_vdata,
hypre_ParVector **F_array,
hypre_ParVector **U_array )
{
hypre_ParAMGData *amg_data = amg_vdata;
HYPRE_Solver *smoother;
/* Data Structure variables */
hypre_ParCSRMatrix **A_array;
hypre_ParCSRMatrix **P_array;
hypre_ParCSRMatrix **R_array;
hypre_ParVector *Utemp;
hypre_ParVector *Vtemp;
hypre_ParVector *Rtemp;
hypre_ParVector *Ptemp;
hypre_ParVector *Ztemp;
hypre_ParVector *Aux_U;
hypre_ParVector *Aux_F;
hypre_ParCSRBlockMatrix **A_block_array;
hypre_ParCSRBlockMatrix **P_block_array;
hypre_ParCSRBlockMatrix **R_block_array;
HYPRE_Real *Ztemp_data;
HYPRE_Real *Ptemp_data;
HYPRE_Int **CF_marker_array;
/* HYPRE_Int **unknown_map_array;
HYPRE_Int **point_map_array;
HYPRE_Int **v_at_point_array; */
HYPRE_Real cycle_op_count;
HYPRE_Int cycle_type;
HYPRE_Int num_levels;
HYPRE_Int max_levels;
HYPRE_Real *num_coeffs;
HYPRE_Int *num_grid_sweeps;
HYPRE_Int *grid_relax_type;
HYPRE_Int **grid_relax_points;
HYPRE_Int block_mode;
HYPRE_Real *max_eig_est;
HYPRE_Real *min_eig_est;
HYPRE_Int cheby_order;
HYPRE_Real cheby_fraction;
/* Local variables */
HYPRE_Int *lev_counter;
HYPRE_Int Solve_err_flag;
HYPRE_Int k;
HYPRE_Int i, j, jj;
HYPRE_Int level;
HYPRE_Int cycle_param;
HYPRE_Int coarse_grid;
HYPRE_Int fine_grid;
HYPRE_Int Not_Finished;
HYPRE_Int num_sweep;
HYPRE_Int cg_num_sweep = 1;
HYPRE_Int relax_type;
HYPRE_Int relax_points;
HYPRE_Int relax_order;
HYPRE_Int relax_local;
HYPRE_Int old_version = 0;
HYPRE_Real *relax_weight;
HYPRE_Real *omega;
HYPRE_Real alfa, beta, gammaold;
HYPRE_Real gamma = 1.0;
HYPRE_Int local_size;
/* HYPRE_Int *smooth_option; */
HYPRE_Int smooth_type;
HYPRE_Int smooth_num_levels;
HYPRE_Int num_threads, my_id;
HYPRE_Real alpha;
HYPRE_Real **l1_norms = NULL;
HYPRE_Real *l1_norms_level;
HYPRE_Int seq_cg = 0;
MPI_Comm comm;
#if 0
HYPRE_Real *D_mat;
HYPRE_Real *S_vec;
#endif
/* Acquire data and allocate storage */
num_threads = hypre_NumThreads();
A_array = hypre_ParAMGDataAArray(amg_data);
P_array = hypre_ParAMGDataPArray(amg_data);
R_array = hypre_ParAMGDataRArray(amg_data);
CF_marker_array = hypre_ParAMGDataCFMarkerArray(amg_data);
Vtemp = hypre_ParAMGDataVtemp(amg_data);
Rtemp = hypre_ParAMGDataRtemp(amg_data);
Ptemp = hypre_ParAMGDataPtemp(amg_data);
Ztemp = hypre_ParAMGDataZtemp(amg_data);
num_levels = hypre_ParAMGDataNumLevels(amg_data);
max_levels = hypre_ParAMGDataMaxLevels(amg_data);
cycle_type = hypre_ParAMGDataCycleType(amg_data);
A_block_array = hypre_ParAMGDataABlockArray(amg_data);
P_block_array = hypre_ParAMGDataPBlockArray(amg_data);
R_block_array = hypre_ParAMGDataRBlockArray(amg_data);
block_mode = hypre_ParAMGDataBlockMode(amg_data);
num_grid_sweeps = hypre_ParAMGDataNumGridSweeps(amg_data);
grid_relax_type = hypre_ParAMGDataGridRelaxType(amg_data);
grid_relax_points = hypre_ParAMGDataGridRelaxPoints(amg_data);
relax_order = hypre_ParAMGDataRelaxOrder(amg_data);
relax_weight = hypre_ParAMGDataRelaxWeight(amg_data);
omega = hypre_ParAMGDataOmega(amg_data);
smooth_type = hypre_ParAMGDataSmoothType(amg_data);
smooth_num_levels = hypre_ParAMGDataSmoothNumLevels(amg_data);
l1_norms = hypre_ParAMGDataL1Norms(amg_data);
/* smooth_option = hypre_ParAMGDataSmoothOption(amg_data); */
max_eig_est = hypre_ParAMGDataMaxEigEst(amg_data);
min_eig_est = hypre_ParAMGDataMinEigEst(amg_data);
cheby_order = hypre_ParAMGDataChebyOrder(amg_data);
cheby_fraction = hypre_ParAMGDataChebyFraction(amg_data);
cycle_op_count = hypre_ParAMGDataCycleOpCount(amg_data);
lev_counter = hypre_CTAlloc(HYPRE_Int, num_levels);
if (hypre_ParAMGDataParticipate(amg_data)) seq_cg = 1;
/* Initialize */
Solve_err_flag = 0;
if (grid_relax_points) old_version = 1;
num_coeffs = hypre_CTAlloc(HYPRE_Real, num_levels);
num_coeffs[0] = hypre_ParCSRMatrixDNumNonzeros(A_array[0]);
comm = hypre_ParCSRMatrixComm(A_array[0]);
hypre_MPI_Comm_rank(comm,&my_id);
if (block_mode)
{
for (j = 1; j < num_levels; j++)
num_coeffs[j] = hypre_ParCSRBlockMatrixNumNonzeros(A_block_array[j]);
}
else
{
for (j = 1; j < num_levels; j++)
num_coeffs[j] = hypre_ParCSRMatrixDNumNonzeros(A_array[j]);
}
/*---------------------------------------------------------------------
* Initialize cycling control counter
*
* Cycling is controlled using a level counter: lev_counter[k]
*
* Each time relaxation is performed on level k, the
* counter is decremented by 1. If the counter is then
* negative, we go to the next finer level. If non-
* negative, we go to the next coarser level. The
* following actions control cycling:
*
* a. lev_counter[0] is initialized to 1.
* b. lev_counter[k] is initialized to cycle_type for k>0.
*
* c. During cycling, when going down to level k, lev_counter[k]
* is set to the max of (lev_counter[k],cycle_type)
*---------------------------------------------------------------------*/
Not_Finished = 1;
lev_counter[0] = 1;
for (k = 1; k < num_levels; ++k)
{
lev_counter[k] = cycle_type;
}
level = 0;
cycle_param = 1;
smoother = hypre_ParAMGDataSmoother(amg_data);
if (smooth_num_levels > 0)
{
if (smooth_type == 7 || smooth_type == 8
|| smooth_type == 17 || smooth_type == 18
|| smooth_type == 9 || smooth_type == 19)
{
HYPRE_Int actual_local_size = hypre_ParVectorActualLocalSize(Vtemp);
Utemp = hypre_ParVectorCreate(comm,hypre_ParVectorGlobalSize(Vtemp),
hypre_ParVectorPartitioning(Vtemp));
hypre_ParVectorOwnsPartitioning(Utemp) = 0;
local_size
= hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp));
if (local_size < actual_local_size)
{
hypre_VectorData(hypre_ParVectorLocalVector(Utemp)) =
hypre_CTAlloc(HYPRE_Complex, actual_local_size);
hypre_ParVectorActualLocalSize(Utemp) = actual_local_size;
}
else
hypre_ParVectorInitialize(Utemp);
}
}
/*---------------------------------------------------------------------
* Main loop of cycling
*--------------------------------------------------------------------*/
while (Not_Finished)
{
if (num_levels > 1)
{
local_size
= hypre_VectorSize(hypre_ParVectorLocalVector(F_array[level]));
hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp)) = local_size;
if (smooth_num_levels <= level)
{
cg_num_sweep = 1;
num_sweep = num_grid_sweeps[cycle_param];
Aux_U = U_array[level];
Aux_F = F_array[level];
}
else if (smooth_type > 9)
{
hypre_VectorSize(hypre_ParVectorLocalVector(Ztemp)) = local_size;
hypre_VectorSize(hypre_ParVectorLocalVector(Rtemp)) = local_size;
hypre_VectorSize(hypre_ParVectorLocalVector(Ptemp)) = local_size;
Ztemp_data = hypre_VectorData(hypre_ParVectorLocalVector(Ztemp));
Ptemp_data = hypre_VectorData(hypre_ParVectorLocalVector(Ptemp));
hypre_ParVectorSetConstantValues(Ztemp,0);
alpha = -1.0;
beta = 1.0;
hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level],
U_array[level], beta, F_array[level], Rtemp);
cg_num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data);
num_sweep = num_grid_sweeps[cycle_param];
Aux_U = Ztemp;
Aux_F = Rtemp;
}
else
{
cg_num_sweep = 1;
num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data);
Aux_U = U_array[level];
Aux_F = F_array[level];
}
relax_type = grid_relax_type[cycle_param];
}
else /* AB: 4/08: removed the max_levels > 1 check - should do this when max-levels = 1 also */
{
/* If no coarsening occurred, apply a simple smoother once */
Aux_U = U_array[level];
Aux_F = F_array[level];
num_sweep = 1;
/* TK: Use the user relax type (instead of 0) to allow for setting a
convergent smoother (e.g. in the solution of singular problems). */
relax_type = hypre_ParAMGDataUserRelaxType(amg_data);
}
if (l1_norms != NULL)
l1_norms_level = l1_norms[level];
else
l1_norms_level = NULL;
if (cycle_param == 3 && seq_cg)
{
hypre_seqAMGCycle(amg_data, level, F_array, U_array);
}
else
{
/*------------------------------------------------------------------
* Do the relaxation num_sweep times
*-----------------------------------------------------------------*/
for (jj = 0; jj < cg_num_sweep; jj++)
{
if (smooth_num_levels > level && smooth_type > 9)
hypre_ParVectorSetConstantValues(Aux_U,0);
for (j = 0; j < num_sweep; j++)
{
if (num_levels == 1 && max_levels > 1)
{
relax_points = 0;
relax_local = 0;
}
else
{
if (old_version)
relax_points = grid_relax_points[cycle_param][j];
relax_local = relax_order;
}
/*-----------------------------------------------
* VERY sloppy approximation to cycle complexity
*-----------------------------------------------*/
if (old_version && level < num_levels -1)
{
switch (relax_points)
{
case 1:
cycle_op_count += num_coeffs[level+1];
break;
case -1:
cycle_op_count += (num_coeffs[level]-num_coeffs[level+1]);
break;
}
}
else
{
cycle_op_count += num_coeffs[level];
}
/*-----------------------------------------------
Choose Smoother
-----------------------------------------------*/
if (smooth_num_levels > level &&
(smooth_type == 7 || smooth_type == 8 ||
smooth_type == 9 || smooth_type == 19 ||
smooth_type == 17 || smooth_type == 18))
{
hypre_VectorSize(hypre_ParVectorLocalVector(Utemp)) = local_size;
alpha = -1.0;
beta = 1.0;
hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level],
U_array[level], beta, Aux_F, Vtemp);
if (smooth_type == 8 || smooth_type == 18)
HYPRE_ParCSRParaSailsSolve(smoother[level],
(HYPRE_ParCSRMatrix) A_array[level],
(HYPRE_ParVector) Vtemp,
(HYPRE_ParVector) Utemp);
else if (smooth_type == 7 || smooth_type == 17)
HYPRE_ParCSRPilutSolve(smoother[level],
(HYPRE_ParCSRMatrix) A_array[level],
(HYPRE_ParVector) Vtemp,
(HYPRE_ParVector) Utemp);
else if (smooth_type == 9 || smooth_type == 19)
HYPRE_EuclidSolve(smoother[level],
(HYPRE_ParCSRMatrix) A_array[level],
(HYPRE_ParVector) Vtemp,
(HYPRE_ParVector) Utemp);
hypre_ParVectorAxpy(relax_weight[level],Utemp,Aux_U);
}
else if (smooth_num_levels > level &&
(smooth_type == 6 || smooth_type == 16))
{
HYPRE_SchwarzSolve(smoother[level],
(HYPRE_ParCSRMatrix) A_array[level],
(HYPRE_ParVector) Aux_F,
(HYPRE_ParVector) Aux_U);
}
/*else if (relax_type == 99)*/
else if (relax_type == 9 || relax_type == 99)
{ /* Gaussian elimination */
hypre_GaussElimSolve(amg_data, level, relax_type);
}
else if (relax_type == 18)
{ /* L1 - Jacobi*/
if (relax_order == 1 && cycle_param < 3)
{
/* need to do CF - so can't use the AMS one */
HYPRE_Int i;
HYPRE_Int loc_relax_points[2];
if (cycle_type < 2)
{
loc_relax_points[0] = 1;
loc_relax_points[1] = -1;
}
else
{
loc_relax_points[0] = -1;
loc_relax_points[1] = 1;
}
for (i=0; i < 2; i++)
hypre_ParCSRRelax_L1_Jacobi(A_array[level],
Aux_F,
CF_marker_array[level],
loc_relax_points[i],
relax_weight[level],
l1_norms[level],
Aux_U,
Vtemp);
}
else /* not CF - so use through AMS */
{
if (num_threads == 1)
hypre_ParCSRRelax(A_array[level],
Aux_F,
1,
1,
l1_norms_level,
relax_weight[level],
omega[level],0,0,0,0,
Aux_U,
Vtemp,
Ztemp);
else
hypre_ParCSRRelaxThreads(A_array[level],
Aux_F,
1,
1,
l1_norms_level,
relax_weight[level],
omega[level],
Aux_U,
Vtemp,
Ztemp);
}
}
else if (relax_type == 15)
{ /* CG */
if (j ==0) /* do num sweep iterations of CG */
hypre_ParCSRRelax_CG( smoother[level],
A_array[level],
Aux_F,
Aux_U,
num_sweep);
}
else if (relax_type == 16)
{ /* scaled Chebyshev */
HYPRE_Int scale = 1;
HYPRE_Int variant = 0;
hypre_ParCSRRelax_Cheby(A_array[level],
Aux_F,
max_eig_est[level],
min_eig_est[level],
cheby_fraction, cheby_order, scale,
variant, Aux_U, Vtemp, Ztemp );
}
else if (relax_type ==17)
{
hypre_BoomerAMGRelax_FCFJacobi(A_array[level],
Aux_F,
CF_marker_array[level],
relax_weight[level],
Aux_U,
Vtemp);
}
else if (old_version)
{
Solve_err_flag = hypre_BoomerAMGRelax(A_array[level],
Aux_F,
CF_marker_array[level],
relax_type, relax_points,
relax_weight[level],
omega[level],
l1_norms_level,
Aux_U,
Vtemp,
Ztemp);
}
else
{
/* smoother than can have CF ordering */
if (block_mode)
{
Solve_err_flag = hypre_BoomerAMGBlockRelaxIF(A_block_array[level],
Aux_F,
CF_marker_array[level],
relax_type,
relax_local,
cycle_param,
relax_weight[level],
omega[level],
Aux_U,
Vtemp);
}
else
{
Solve_err_flag = hypre_BoomerAMGRelaxIF(A_array[level],
Aux_F,
CF_marker_array[level],
relax_type,
relax_local,
cycle_param,
relax_weight[level],
omega[level],
l1_norms_level,
Aux_U,
Vtemp,
Ztemp);
}
}
if (Solve_err_flag != 0)
return(Solve_err_flag);
}
if (smooth_num_levels > level && smooth_type > 9)
{
gammaold = gamma;
gamma = hypre_ParVectorInnerProd(Rtemp,Ztemp);
if (jj == 0)
hypre_ParVectorCopy(Ztemp,Ptemp);
else
{
beta = gamma/gammaold;
for (i=0; i < local_size; i++)
Ptemp_data[i] = Ztemp_data[i] + beta*Ptemp_data[i];
}
hypre_ParCSRMatrixMatvec(1.0,A_array[level],Ptemp,0.0,Vtemp);
alfa = gamma /hypre_ParVectorInnerProd(Ptemp,Vtemp);
hypre_ParVectorAxpy(alfa,Ptemp,U_array[level]);
hypre_ParVectorAxpy(-alfa,Vtemp,Rtemp);
}
}
}
/*------------------------------------------------------------------
* Decrement the control counter and determine which grid to visit next
*-----------------------------------------------------------------*/
--lev_counter[level];
if (lev_counter[level] >= 0 && level != num_levels-1)
{
/*---------------------------------------------------------------
* Visit coarser level next.
* Compute residual using hypre_ParCSRMatrixMatvec.
* Perform restriction using hypre_ParCSRMatrixMatvecT.
* Reset counters and cycling parameters for coarse level
*--------------------------------------------------------------*/
fine_grid = level;
coarse_grid = level + 1;
hypre_ParVectorSetConstantValues(U_array[coarse_grid], 0.0);
alpha = -1.0;
beta = 1.0;
if (block_mode)
{
hypre_ParVectorCopy(F_array[fine_grid],Vtemp);
hypre_ParCSRBlockMatrixMatvec(alpha, A_block_array[fine_grid], U_array[fine_grid],
beta, Vtemp);
}
else
{
// JSP: avoid unnecessary copy using out-of-place version of SpMV
hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[fine_grid], U_array[fine_grid],
beta, F_array[fine_grid], Vtemp);
}
alpha = 1.0;
beta = 0.0;
if (block_mode)
{
hypre_ParCSRBlockMatrixMatvecT(alpha,R_block_array[fine_grid],Vtemp,
beta,F_array[coarse_grid]);
}
else
{
hypre_ParCSRMatrixMatvecT(alpha,R_array[fine_grid],Vtemp,
beta,F_array[coarse_grid]);
}
++level;
lev_counter[level] = hypre_max(lev_counter[level],cycle_type);
cycle_param = 1;
if (level == num_levels-1) cycle_param = 3;
}
else if (level != 0)
{
/*---------------------------------------------------------------
* Visit finer level next.
* Interpolate and add correction using hypre_ParCSRMatrixMatvec.
* Reset counters and cycling parameters for finer level.
*--------------------------------------------------------------*/
fine_grid = level - 1;
coarse_grid = level;
alpha = 1.0;
beta = 1.0;
if (block_mode)
{
hypre_ParCSRBlockMatrixMatvec(alpha, P_block_array[fine_grid],
U_array[coarse_grid],
beta, U_array[fine_grid]);
}
else
{
hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid],
U_array[coarse_grid],
beta, U_array[fine_grid]);
}
--level;
cycle_param = 2;
}
else
{
Not_Finished = 0;
}
}
hypre_ParAMGDataCycleOpCount(amg_data) = cycle_op_count;
hypre_TFree(lev_counter);
hypre_TFree(num_coeffs);
if (smooth_num_levels > 0)
{
if (smooth_type == 7 || smooth_type == 8 || smooth_type == 9 ||
smooth_type == 17 || smooth_type == 18 || smooth_type == 19 )
hypre_ParVectorDestroy(Utemp);
}
return(Solve_err_flag);
}
/******************************************************************************
*
* hypre_IJVectorZeroValuesPar
*
* zeroes all local components of an IJVectorPar
*
*****************************************************************************/
HYPRE_Int
hypre_IJVectorZeroValuesPar(hypre_IJVector *vector)
{
HYPRE_Int my_id;
HYPRE_Int i, vec_start, vec_stop;
double *data;
hypre_ParVector *par_vector = hypre_IJVectorObject(vector);
MPI_Comm comm = hypre_IJVectorComm(vector);
HYPRE_Int *partitioning = hypre_ParVectorPartitioning(par_vector);
hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector);
HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector);
hypre_MPI_Comm_rank(comm, &my_id);
/* If par_vector == NULL or partitioning == NULL or local_vector == NULL
let user know of catastrophe and exit */
if (!par_vector)
{
if (print_level)
{
hypre_printf("par_vector == NULL -- ");
hypre_printf("hypre_IJVectorZeroValuesPar\n");
hypre_printf("**** Vector storage is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
if (!partitioning)
{
if (print_level)
{
hypre_printf("partitioning == NULL -- ");
hypre_printf("hypre_IJVectorZeroValuesPar\n");
hypre_printf("**** Vector partitioning is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
if (!local_vector)
{
if (print_level)
{
hypre_printf("local_vector == NULL -- ");
hypre_printf("hypre_IJVectorZeroValuesPar\n");
hypre_printf("**** Vector local data is either unallocated or orphaned ****\n");
}
hypre_error_in_arg(1);
}
#ifdef HYPRE_NO_GLOBAL_PARTITION
vec_start = partitioning[0];
vec_stop = partitioning[1];
#else
vec_start = partitioning[my_id];
vec_stop = partitioning[my_id+1];
#endif
if (vec_start > vec_stop)
{
if (print_level)
{
hypre_printf("vec_start > vec_stop -- ");
hypre_printf("hypre_IJVectorZeroValuesPar\n");
hypre_printf("**** This vector partitioning should not occur ****\n");
}
hypre_error_in_arg(1);
}
data = hypre_VectorData( local_vector );
for (i = 0; i < vec_stop - vec_start; i++)
data[i] = 0.;
return hypre_error_flag;
}
HYPRE_Int
hypre_ParVectorPrintIJ( hypre_ParVector *vector,
HYPRE_Int base_j,
const char *filename )
{
MPI_Comm comm;
HYPRE_Int global_size;
HYPRE_Int *partitioning;
double *local_data;
HYPRE_Int myid, num_procs, i, j, part0;
char new_filename[255];
FILE *file;
if (!vector)
{
hypre_error_in_arg(1);
return hypre_error_flag;
}
comm = hypre_ParVectorComm(vector);
global_size = hypre_ParVectorGlobalSize(vector);
partitioning = hypre_ParVectorPartitioning(vector);
/* multivector code not written yet >>> */
hypre_assert( hypre_ParVectorNumVectors(vector) == 1 );
if ( hypre_ParVectorNumVectors(vector) != 1 ) hypre_error_in_arg(1);
hypre_MPI_Comm_rank(comm, &myid);
hypre_MPI_Comm_size(comm, &num_procs);
hypre_sprintf(new_filename,"%s.%05d", filename, myid);
if ((file = fopen(new_filename, "w")) == NULL)
{
hypre_printf("Error: can't open output file %s\n", new_filename);
hypre_error_in_arg(3);
return hypre_error_flag;
}
local_data = hypre_VectorData(hypre_ParVectorLocalVector(vector));
hypre_fprintf(file, "%d \n", global_size);
#ifdef HYPRE_NO_GLOBAL_PARTITION
for (i=0; i <= 2; i++)
#else
for (i=0; i <= num_procs; i++)
#endif
{
hypre_fprintf(file, "%d \n", partitioning[i] + base_j);
}
#ifdef HYPRE_NO_GLOBAL_PARTITION
part0 = partitioning[0];
for (j = part0; j < partitioning[1]; j++)
#else
part0 = partitioning[myid];
for (j = part0; j < partitioning[myid+1]; j++)
#endif
{
hypre_fprintf(file, "%d %.14e\n", j + base_j, local_data[j-part0]);
}
fclose(file);
return hypre_error_flag;
}
hypre_Vector *
hypre_ParVectorToVectorAll (hypre_ParVector *par_v)
{
MPI_Comm comm = hypre_ParVectorComm(par_v);
HYPRE_Int global_size = hypre_ParVectorGlobalSize(par_v);
#ifndef HYPRE_NO_GLOBAL_PARTITION
HYPRE_Int *vec_starts = hypre_ParVectorPartitioning(par_v);
#endif
hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_v);
HYPRE_Int num_procs, my_id;
HYPRE_Int num_vectors = hypre_ParVectorNumVectors(par_v);
hypre_Vector *vector;
double *vector_data;
double *local_data;
HYPRE_Int local_size;
hypre_MPI_Request *requests;
hypre_MPI_Status *status;
HYPRE_Int i, j;
HYPRE_Int *used_procs;
HYPRE_Int num_types, num_requests;
HYPRE_Int vec_len, proc_id;
#ifdef HYPRE_NO_GLOBAL_PARTITION
HYPRE_Int *new_vec_starts;
HYPRE_Int num_contacts;
HYPRE_Int contact_proc_list[1];
HYPRE_Int contact_send_buf[1];
HYPRE_Int contact_send_buf_starts[2];
HYPRE_Int max_response_size;
HYPRE_Int *response_recv_buf=NULL;
HYPRE_Int *response_recv_buf_starts = NULL;
hypre_DataExchangeResponse response_obj;
hypre_ProcListElements send_proc_obj;
HYPRE_Int *send_info = NULL;
hypre_MPI_Status status1;
HYPRE_Int count, tag1 = 112, tag2 = 223;
HYPRE_Int start;
#endif
hypre_MPI_Comm_size(comm, &num_procs);
hypre_MPI_Comm_rank(comm, &my_id);
#ifdef HYPRE_NO_GLOBAL_PARTITION
local_size = hypre_ParVectorLastIndex(par_v) -
hypre_ParVectorFirstIndex(par_v) + 1;
/* determine procs which hold data of par_v and store ids in used_procs */
/* we need to do an exchange data for this. If I own row then I will contact
processor 0 with the endpoint of my local range */
if (local_size > 0)
{
num_contacts = 1;
contact_proc_list[0] = 0;
contact_send_buf[0] = hypre_ParVectorLastIndex(par_v);
contact_send_buf_starts[0] = 0;
contact_send_buf_starts[1] = 1;
}
else
{
num_contacts = 0;
contact_send_buf_starts[0] = 0;
contact_send_buf_starts[1] = 0;
}
/*build the response object*/
/*send_proc_obj will be for saving info from contacts */
send_proc_obj.length = 0;
send_proc_obj.storage_length = 10;
send_proc_obj.id = hypre_CTAlloc(HYPRE_Int, send_proc_obj.storage_length);
send_proc_obj.vec_starts = hypre_CTAlloc(HYPRE_Int, send_proc_obj.storage_length + 1);
send_proc_obj.vec_starts[0] = 0;
send_proc_obj.element_storage_length = 10;
send_proc_obj.elements = hypre_CTAlloc(HYPRE_Int, send_proc_obj.element_storage_length);
max_response_size = 0; /* each response is null */
response_obj.fill_response = hypre_FillResponseParToVectorAll;
response_obj.data1 = NULL;
response_obj.data2 = &send_proc_obj; /*this is where we keep info from contacts*/
hypre_DataExchangeList(num_contacts,
contact_proc_list, contact_send_buf,
contact_send_buf_starts, sizeof(HYPRE_Int),
sizeof(HYPRE_Int), &response_obj,
max_response_size, 1,
comm, (void**) &response_recv_buf,
&response_recv_buf_starts);
/* now processor 0 should have a list of ranges for processors that have rows -
these are in send_proc_obj - it needs to create the new list of processors
and also an array of vec starts - and send to those who own row*/
if (my_id)
{
if (local_size)
{
/* look for a message from processor 0 */
hypre_MPI_Probe(0, tag1, comm, &status1);
hypre_MPI_Get_count(&status1, HYPRE_MPI_INT, &count);
send_info = hypre_CTAlloc(HYPRE_Int, count);
hypre_MPI_Recv(send_info, count, HYPRE_MPI_INT, 0, tag1, comm, &status1);
/* now unpack */
num_types = send_info[0];
used_procs = hypre_CTAlloc(HYPRE_Int, num_types);
new_vec_starts = hypre_CTAlloc(HYPRE_Int, num_types+1);
for (i=1; i<= num_types; i++)
{
used_procs[i-1] = send_info[i];
}
for (i=num_types+1; i< count; i++)
{
new_vec_starts[i-num_types-1] = send_info[i] ;
}
}
else /* clean up and exit */
{
hypre_TFree(send_proc_obj.vec_starts);
hypre_TFree(send_proc_obj.id);
hypre_TFree(send_proc_obj.elements);
if(response_recv_buf) hypre_TFree(response_recv_buf);
if(response_recv_buf_starts) hypre_TFree(response_recv_buf_starts);
return NULL;
}
}
else /* my_id ==0 */
{
num_types = send_proc_obj.length;
used_procs = hypre_CTAlloc(HYPRE_Int, num_types);
new_vec_starts = hypre_CTAlloc(HYPRE_Int, num_types+1);
new_vec_starts[0] = 0;
for (i=0; i< num_types; i++)
{
used_procs[i] = send_proc_obj.id[i];
new_vec_starts[i+1] = send_proc_obj.elements[i]+1;
}
qsort0(used_procs, 0, num_types-1);
qsort0(new_vec_starts, 0, num_types);
/*now we need to put into an array to send */
count = 2*num_types+2;
send_info = hypre_CTAlloc(HYPRE_Int, count);
send_info[0] = num_types;
for (i=1; i<= num_types; i++)
{
send_info[i] = used_procs[i-1];
}
for (i=num_types+1; i< count; i++)
{
send_info[i] = new_vec_starts[i-num_types-1];
}
requests = hypre_CTAlloc(hypre_MPI_Request, num_types);
status = hypre_CTAlloc(hypre_MPI_Status, num_types);
/* don't send to myself - these are sorted so my id would be first*/
start = 0;
if (used_procs[0] == 0)
{
start = 1;
}
for (i=start; i < num_types; i++)
{
hypre_MPI_Isend(send_info, count, HYPRE_MPI_INT, used_procs[i], tag1, comm, &requests[i-start]);
}
hypre_MPI_Waitall(num_types-start, requests, status);
hypre_TFree(status);
hypre_TFree(requests);
}
/* clean up */
hypre_TFree(send_proc_obj.vec_starts);
hypre_TFree(send_proc_obj.id);
hypre_TFree(send_proc_obj.elements);
hypre_TFree(send_info);
if(response_recv_buf) hypre_TFree(response_recv_buf);
if(response_recv_buf_starts) hypre_TFree(response_recv_buf_starts);
/* now proc 0 can exit if it has no rows */
if (!local_size) {
hypre_TFree(used_procs);
hypre_TFree(new_vec_starts);
return NULL;
}
/* everyone left has rows and knows: new_vec_starts, num_types, and used_procs */
/* this vector should be rather small */
local_data = hypre_VectorData(local_vector);
vector = hypre_SeqVectorCreate(global_size);
hypre_VectorNumVectors(vector) = num_vectors;
hypre_SeqVectorInitialize(vector);
vector_data = hypre_VectorData(vector);
num_requests = 2*num_types;
requests = hypre_CTAlloc(hypre_MPI_Request, num_requests);
status = hypre_CTAlloc(hypre_MPI_Status, num_requests);
/* initialize data exchange among used_procs and generate vector - here we
send to ourself also*/
j = 0;
for (i = 0; i < num_types; i++)
{
proc_id = used_procs[i];
vec_len = new_vec_starts[i+1] - new_vec_starts[i];
hypre_MPI_Irecv(&vector_data[new_vec_starts[i]], num_vectors*vec_len, hypre_MPI_DOUBLE,
proc_id, tag2, comm, &requests[j++]);
}
for (i = 0; i < num_types; i++)
{
hypre_MPI_Isend(local_data, num_vectors*local_size, hypre_MPI_DOUBLE, used_procs[i],
tag2, comm, &requests[j++]);
}
hypre_MPI_Waitall(num_requests, requests, status);
if (num_requests)
{
hypre_TFree(requests);
hypre_TFree(status);
hypre_TFree(used_procs);
}
hypre_TFree(new_vec_starts);
#else
local_size = vec_starts[my_id+1] - vec_starts[my_id];
/* if my_id contains no data, return NULL */
if (!local_size)
return NULL;
local_data = hypre_VectorData(local_vector);
vector = hypre_SeqVectorCreate(global_size);
hypre_VectorNumVectors(vector) = num_vectors;
hypre_SeqVectorInitialize(vector);
vector_data = hypre_VectorData(vector);
/* determine procs which hold data of par_v and store ids in used_procs */
num_types = -1;
for (i=0; i < num_procs; i++)
if (vec_starts[i+1]-vec_starts[i])
num_types++;
num_requests = 2*num_types;
used_procs = hypre_CTAlloc(HYPRE_Int, num_types);
j = 0;
for (i=0; i < num_procs; i++)
if (vec_starts[i+1]-vec_starts[i] && i-my_id)
used_procs[j++] = i;
requests = hypre_CTAlloc(hypre_MPI_Request, num_requests);
status = hypre_CTAlloc(hypre_MPI_Status, num_requests);
/* initialize data exchange among used_procs and generate vector */
j = 0;
for (i = 0; i < num_types; i++)
{
proc_id = used_procs[i];
vec_len = vec_starts[proc_id+1] - vec_starts[proc_id];
hypre_MPI_Irecv(&vector_data[vec_starts[proc_id]], num_vectors*vec_len, hypre_MPI_DOUBLE,
proc_id, 0, comm, &requests[j++]);
}
for (i = 0; i < num_types; i++)
{
hypre_MPI_Isend(local_data, num_vectors*local_size, hypre_MPI_DOUBLE, used_procs[i],
0, comm, &requests[j++]);
}
for (i=0; i < num_vectors*local_size; i++)
vector_data[vec_starts[my_id]+i] = local_data[i];
hypre_MPI_Waitall(num_requests, requests, status);
if (num_requests)
{
hypre_TFree(used_procs);
hypre_TFree(requests);
hypre_TFree(status);
}
#endif
return vector;
}
hypre_ParVector *
hypre_VectorToParVector (MPI_Comm comm, hypre_Vector *v, HYPRE_Int *vec_starts)
{
HYPRE_Int global_size;
HYPRE_Int local_size;
HYPRE_Int num_vectors;
HYPRE_Int num_procs, my_id;
HYPRE_Int global_vecstride, vecstride, idxstride;
hypre_ParVector *par_vector;
hypre_Vector *local_vector;
double *v_data;
double *local_data;
hypre_MPI_Request *requests;
hypre_MPI_Status *status, status0;
HYPRE_Int i, j, k, p;
hypre_MPI_Comm_size(comm,&num_procs);
hypre_MPI_Comm_rank(comm,&my_id);
if (my_id == 0)
{
global_size = hypre_VectorSize(v);
v_data = hypre_VectorData(v);
num_vectors = hypre_VectorNumVectors(v); /* for multivectors */
global_vecstride = hypre_VectorVectorStride(v);
}
hypre_MPI_Bcast(&global_size,1,HYPRE_MPI_INT,0,comm);
hypre_MPI_Bcast(&num_vectors,1,HYPRE_MPI_INT,0,comm);
hypre_MPI_Bcast(&global_vecstride,1,HYPRE_MPI_INT,0,comm);
if ( num_vectors==1 )
par_vector = hypre_ParVectorCreate(comm, global_size, vec_starts);
else
par_vector = hypre_ParMultiVectorCreate(comm, global_size, vec_starts, num_vectors);
vec_starts = hypre_ParVectorPartitioning(par_vector);
local_size = vec_starts[my_id+1] - vec_starts[my_id];
hypre_ParVectorInitialize(par_vector);
local_vector = hypre_ParVectorLocalVector(par_vector);
local_data = hypre_VectorData(local_vector);
vecstride = hypre_VectorVectorStride(local_vector);
idxstride = hypre_VectorIndexStride(local_vector);
hypre_assert( idxstride==1 ); /* <<< so far only the only implemented multivector StorageMethod is 0 <<< */
if (my_id == 0)
{
requests = hypre_CTAlloc(hypre_MPI_Request,num_vectors*(num_procs-1));
status = hypre_CTAlloc(hypre_MPI_Status,num_vectors*(num_procs-1));
k = 0;
for ( p=1; p<num_procs; p++)
for ( j=0; j<num_vectors; ++j )
{
hypre_MPI_Isend( &v_data[vec_starts[p]]+j*global_vecstride,
(vec_starts[p+1]-vec_starts[p]),
hypre_MPI_DOUBLE, p, 0, comm, &requests[k++] );
}
if ( num_vectors==1 )
{
for (i=0; i < local_size; i++)
local_data[i] = v_data[i];
}
else
for ( j=0; j<num_vectors; ++j )
{
for (i=0; i < local_size; i++)
local_data[i+j*vecstride] = v_data[i+j*global_vecstride];
}
hypre_MPI_Waitall(num_procs-1,requests, status);
hypre_TFree(requests);
hypre_TFree(status);
}
else
{
for ( j=0; j<num_vectors; ++j )
hypre_MPI_Recv( local_data+j*vecstride, local_size, hypre_MPI_DOUBLE, 0, 0, comm,&status0 );
}
return par_vector;
}
HYPRE_Int
main( HYPRE_Int argc,
char *argv[] )
{
hypre_ParVector *vector1;
hypre_ParVector *vector2;
hypre_ParVector *tmp_vector;
HYPRE_Int num_procs, my_id;
HYPRE_Int global_size = 20;
HYPRE_Int local_size;
HYPRE_Int first_index;
HYPRE_Int i;
HYPRE_Int *partitioning;
HYPRE_Complex prod;
HYPRE_Complex *data, *data2;
hypre_Vector *vector;
hypre_Vector *local_vector;
hypre_Vector *local_vector2;
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id );
hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs);
partitioning = NULL;
vector1 = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning);
partitioning = hypre_ParVectorPartitioning(vector1);
hypre_ParVectorInitialize(vector1);
local_vector = hypre_ParVectorLocalVector(vector1);
data = hypre_VectorData(local_vector);
local_size = hypre_VectorSize(local_vector);
first_index = partitioning[my_id];
for (i=0; i < local_size; i++)
data[i] = first_index+i;
/*
hypre_ParVectorPrint(vector1, "Vector");
*/
local_vector2 = hypre_SeqVectorCreate(global_size);
hypre_SeqVectorInitialize(local_vector2);
data2 = hypre_VectorData(local_vector2);
for (i=0; i < global_size; i++)
data2[i] = i+1;
/* partitioning = hypre_CTAlloc(HYPRE_Int,4);
partitioning[0] = 0;
partitioning[1] = 10;
partitioning[2] = 10;
partitioning[3] = 20;
*/
vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning);
hypre_ParVectorSetPartitioningOwner(vector2,0);
hypre_ParVectorPrint(vector2, "Convert");
vector = hypre_ParVectorToVectorAll(vector2);
/*-----------------------------------------------------------
* Copy the vector into tmp_vector
*-----------------------------------------------------------*/
tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert");
/*
tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning);
hypre_ParVectorSetPartitioningOwner(tmp_vector,0);
hypre_ParVectorInitialize(tmp_vector);
hypre_ParVectorCopy(vector1, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Copy");
*/
/*-----------------------------------------------------------
* Scale tmp_vector
*-----------------------------------------------------------*/
hypre_ParVectorScale(2.0, tmp_vector);
/*
hypre_ParVectorPrint(tmp_vector,"Scale");
*/
/*-----------------------------------------------------------
* Do an Axpy (2*vector - vector) = vector
*-----------------------------------------------------------*/
hypre_ParVectorAxpy(-1.0, vector1, tmp_vector);
/*
hypre_ParVectorPrint(tmp_vector,"Axpy");
*/
/*-----------------------------------------------------------
* Do an inner product vector* tmp_vector
*-----------------------------------------------------------*/
prod = hypre_ParVectorInnerProd(vector1, tmp_vector);
hypre_printf (" prod: %8.2f \n", prod);
/*-----------------------------------------------------------
* Finalize things
*-----------------------------------------------------------*/
hypre_ParVectorDestroy(vector1);
hypre_ParVectorDestroy(vector2);
hypre_ParVectorDestroy(tmp_vector);
hypre_SeqVectorDestroy(local_vector2);
if (vector) hypre_SeqVectorDestroy(vector);
/* Finalize MPI */
hypre_MPI_Finalize();
return 0;
}
hypre_ParMultiVector *
hypre_ParMultiVectorTempRead(MPI_Comm comm, const char *fileName)
/* ***** temporary implementation ****** */
{
HYPRE_Int i, n, id;
double * dest;
double * src;
HYPRE_Int count;
HYPRE_Int retcode;
char temp_string[128];
hypre_ParMultiVector * x;
hypre_ParVector * temp_vec;
/* calculate the number of files */
hypre_MPI_Comm_rank( comm, &id );
n = 0;
do {
hypre_sprintf( temp_string, "test -f %s.%d.%d", fileName, n, id );
if (!(retcode=system(temp_string))) /* zero retcode mean file exists */
n++;
} while (!retcode);
if ( n == 0 ) return NULL;
/* now read the first vector using hypre_ParVectorRead into temp_vec */
hypre_sprintf(temp_string,"%s.%d",fileName,0);
temp_vec = hypre_ParVectorRead(comm, temp_string);
/* this vector WON'T own partitioning */
hypre_ParVectorSetPartitioningOwner(temp_vec,0);
/* now create multivector using temp_vec as a sample */
x = hypre_ParMultiVectorCreate(hypre_ParVectorComm(temp_vec),
hypre_ParVectorGlobalSize(temp_vec),hypre_ParVectorPartitioning(temp_vec),n);
/* this vector WILL own the partitioning */
hypre_ParMultiVectorSetPartitioningOwner(x,1);
hypre_ParMultiVectorInitialize(x);
/* read data from first and all other vectors into "x" */
i = 0;
do {
/* copy data from current vector */
dest = x->local_vector->data + i*(x->local_vector->size);
src = temp_vec->local_vector->data;
count = temp_vec->local_vector->size;
memcpy(dest,src, count*sizeof(double));
/* destroy current vector */
hypre_ParVectorDestroy(temp_vec);
/* read the data to new current vector, if there are more vectors to read */
if (i<n-1)
{
hypre_sprintf(temp_string,"%s.%d",fileName,i+1);
temp_vec = hypre_ParVectorRead(comm, temp_string);
}
} while (++i<n);
return x;
}
hypre_ParVector *
hypre_ParVectorCreateFromBlock( MPI_Comm comm,
HYPRE_Int p_global_size,
HYPRE_Int *p_partitioning, HYPRE_Int block_size)
{
hypre_ParVector *vector;
HYPRE_Int num_procs, my_id, i;
HYPRE_Int global_size;
HYPRE_Int *new_partitioning; /* need to create a new partitioning - son't want to write over
what is passed in */
global_size = p_global_size*block_size;
vector = hypre_CTAlloc(hypre_ParVector, 1);
hypre_MPI_Comm_rank(comm,&my_id);
hypre_MPI_Comm_size(comm,&num_procs);
if (!p_partitioning)
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
hypre_GenerateLocalPartitioning(global_size, num_procs, my_id, &new_partitioning);
#else
hypre_GeneratePartitioning(global_size, num_procs, &new_partitioning);
#endif
}
else /* adjust for block_size */
{
#ifdef HYPRE_NO_GLOBAL_PARTITION
new_partitioning = hypre_CTAlloc(HYPRE_Int, 2);
for(i = 0; i < 2; i++)
{
new_partitioning[i] = p_partitioning[i]*block_size;
}
#else
new_partitioning = hypre_CTAlloc(HYPRE_Int, num_procs + 1);
for(i = 0; i < num_procs + 1; i++)
{
new_partitioning[i] = p_partitioning[i]*block_size;
}
#endif
}
hypre_ParVectorComm(vector) = comm;
hypre_ParVectorGlobalSize(vector) = global_size;
#ifdef HYPRE_NO_GLOBAL_PARTITION
hypre_ParVectorFirstIndex(vector) = new_partitioning[0];
hypre_ParVectorLastIndex(vector) = new_partitioning[1]-1;
hypre_ParVectorPartitioning(vector) = new_partitioning;
hypre_ParVectorLocalVector(vector) =
hypre_SeqVectorCreate(new_partitioning[1]-new_partitioning[0]);
#else
hypre_ParVectorFirstIndex(vector) = new_partitioning[my_id];
hypre_ParVectorLastIndex(vector) = new_partitioning[my_id+1] -1;
hypre_ParVectorPartitioning(vector) = new_partitioning;
hypre_ParVectorLocalVector(vector) =
hypre_SeqVectorCreate(new_partitioning[my_id+1]-new_partitioning[my_id]);
#endif
/* set defaults */
hypre_ParVectorOwnsData(vector) = 1;
hypre_ParVectorOwnsPartitioning(vector) = 1;
return vector;
}
