HYPRE_Int
HYPRE_ParMultiVectorCreate( MPI_Comm comm,
HYPRE_Int global_size,
HYPRE_Int *partitioning,
HYPRE_Int number_vectors,
HYPRE_ParVector *vector )
{
if (!vector)
{
hypre_error_in_arg(5);
return hypre_error_flag;
}
*vector = (HYPRE_ParVector)
hypre_ParMultiVectorCreate( comm, global_size, partitioning, number_vectors );
return hypre_error_flag;
}
HYPRE_Int
main( HYPRE_Int argc,
char *argv[] )
{
hypre_ParVector *vector1;
hypre_ParVector *vector2;
hypre_ParVector *tmp_vector;
HYPRE_Int num_procs, my_id;
HYPRE_Int global_size = 20;
HYPRE_Int local_size;
HYPRE_Int first_index;
HYPRE_Int num_vectors, vecstride, idxstride;
HYPRE_Int i, j;
HYPRE_Int *partitioning;
double prod;
double *data, *data2;
hypre_Vector *vector;
hypre_Vector *local_vector;
hypre_Vector *local_vector2;
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id );
hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs);
partitioning = NULL;
num_vectors = 3;
vector1 = hypre_ParMultiVectorCreate
( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors );
partitioning = hypre_ParVectorPartitioning(vector1);
hypre_ParVectorInitialize(vector1);
local_vector = hypre_ParVectorLocalVector(vector1);
data = hypre_VectorData(local_vector);
local_size = hypre_VectorSize(local_vector);
vecstride = hypre_VectorVectorStride(local_vector);
idxstride = hypre_VectorIndexStride(local_vector);
first_index = partitioning[my_id];
hypre_printf("vecstride=%i idxstride=%i local_size=%i num_vectors=%i",
vecstride, idxstride, local_size, num_vectors );
for (j=0; j<num_vectors; ++j )
for (i=0; i < local_size; i++)
data[ j*vecstride + i*idxstride ] = first_index+i + 100*j;
hypre_ParVectorPrint(vector1, "Vector");
local_vector2 = hypre_SeqMultiVectorCreate( global_size, num_vectors );
hypre_SeqVectorInitialize(local_vector2);
data2 = hypre_VectorData(local_vector2);
vecstride = hypre_VectorVectorStride(local_vector2);
idxstride = hypre_VectorIndexStride(local_vector2);
for (j=0; j<num_vectors; ++j )
for (i=0; i < global_size; i++)
data2[ j*vecstride + i*idxstride ] = i + 100*j;
/* partitioning = hypre_CTAlloc(HYPRE_Int,4);
partitioning[0] = 0;
partitioning[1] = 10;
partitioning[2] = 10;
partitioning[3] = 20;
*/
partitioning = hypre_CTAlloc(HYPRE_Int,1+num_procs);
hypre_GeneratePartitioning( global_size, num_procs, &partitioning );
vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning);
hypre_ParVectorSetPartitioningOwner(vector2,0);
hypre_ParVectorPrint(vector2, "Convert");
vector = hypre_ParVectorToVectorAll(vector2);
/*-----------------------------------------------------------
* Copy the vector into tmp_vector
*-----------------------------------------------------------*/
/* Read doesn't work for multivectors yet...
tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert");*/
tmp_vector = hypre_ParMultiVectorCreate
( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors );
hypre_ParVectorInitialize( tmp_vector );
hypre_ParVectorCopy( vector2, tmp_vector );
/*
tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning);
hypre_ParVectorSetPartitioningOwner(tmp_vector,0);
hypre_ParVectorInitialize(tmp_vector);
hypre_ParVectorCopy(vector1, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Copy");
*/
/*-----------------------------------------------------------
* Scale tmp_vector
*-----------------------------------------------------------*/
hypre_ParVectorScale(2.0, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Scale");
/*-----------------------------------------------------------
* Do an Axpy (2*vector - vector) = vector
*-----------------------------------------------------------*/
hypre_ParVectorAxpy(-1.0, vector1, tmp_vector);
hypre_ParVectorPrint(tmp_vector,"Axpy");
/*-----------------------------------------------------------
* Do an inner product vector* tmp_vector
*-----------------------------------------------------------*/
prod = hypre_ParVectorInnerProd(vector1, tmp_vector);
hypre_printf (" prod: %8.2f \n", prod);
/*-----------------------------------------------------------
* Finalize things
*-----------------------------------------------------------*/
hypre_ParVectorDestroy(vector1);
hypre_ParVectorDestroy(vector2);
hypre_ParVectorDestroy(tmp_vector);
hypre_SeqVectorDestroy(local_vector2);
if (vector) hypre_SeqVectorDestroy(vector);
/* Finalize MPI */
hypre_MPI_Finalize();
return 0;
}
hypre_ParMultiVector *
hypre_ParMultiVectorTempRead(MPI_Comm comm, const char *fileName)
/* ***** temporary implementation ****** */
{
HYPRE_Int i, n, id;
double * dest;
double * src;
HYPRE_Int count;
HYPRE_Int retcode;
char temp_string[128];
hypre_ParMultiVector * x;
hypre_ParVector * temp_vec;
/* calculate the number of files */
hypre_MPI_Comm_rank( comm, &id );
n = 0;
do {
hypre_sprintf( temp_string, "test -f %s.%d.%d", fileName, n, id );
if (!(retcode=system(temp_string))) /* zero retcode mean file exists */
n++;
} while (!retcode);
if ( n == 0 ) return NULL;
/* now read the first vector using hypre_ParVectorRead into temp_vec */
hypre_sprintf(temp_string,"%s.%d",fileName,0);
temp_vec = hypre_ParVectorRead(comm, temp_string);
/* this vector WON'T own partitioning */
hypre_ParVectorSetPartitioningOwner(temp_vec,0);
/* now create multivector using temp_vec as a sample */
x = hypre_ParMultiVectorCreate(hypre_ParVectorComm(temp_vec),
hypre_ParVectorGlobalSize(temp_vec),hypre_ParVectorPartitioning(temp_vec),n);
/* this vector WILL own the partitioning */
hypre_ParMultiVectorSetPartitioningOwner(x,1);
hypre_ParMultiVectorInitialize(x);
/* read data from first and all other vectors into "x" */
i = 0;
do {
/* copy data from current vector */
dest = x->local_vector->data + i*(x->local_vector->size);
src = temp_vec->local_vector->data;
count = temp_vec->local_vector->size;
memcpy(dest,src, count*sizeof(double));
/* destroy current vector */
hypre_ParVectorDestroy(temp_vec);
/* read the data to new current vector, if there are more vectors to read */
if (i<n-1)
{
hypre_sprintf(temp_string,"%s.%d",fileName,i+1);
temp_vec = hypre_ParVectorRead(comm, temp_string);
}
} while (++i<n);
return x;
}
hypre_ParVector *
hypre_VectorToParVector (MPI_Comm comm, hypre_Vector *v, HYPRE_Int *vec_starts)
{
HYPRE_Int global_size;
HYPRE_Int local_size;
HYPRE_Int num_vectors;
HYPRE_Int num_procs, my_id;
HYPRE_Int global_vecstride, vecstride, idxstride;
hypre_ParVector *par_vector;
hypre_Vector *local_vector;
double *v_data;
double *local_data;
hypre_MPI_Request *requests;
hypre_MPI_Status *status, status0;
HYPRE_Int i, j, k, p;
hypre_MPI_Comm_size(comm,&num_procs);
hypre_MPI_Comm_rank(comm,&my_id);
if (my_id == 0)
{
global_size = hypre_VectorSize(v);
v_data = hypre_VectorData(v);
num_vectors = hypre_VectorNumVectors(v); /* for multivectors */
global_vecstride = hypre_VectorVectorStride(v);
}
hypre_MPI_Bcast(&global_size,1,HYPRE_MPI_INT,0,comm);
hypre_MPI_Bcast(&num_vectors,1,HYPRE_MPI_INT,0,comm);
hypre_MPI_Bcast(&global_vecstride,1,HYPRE_MPI_INT,0,comm);
if ( num_vectors==1 )
par_vector = hypre_ParVectorCreate(comm, global_size, vec_starts);
else
par_vector = hypre_ParMultiVectorCreate(comm, global_size, vec_starts, num_vectors);
vec_starts = hypre_ParVectorPartitioning(par_vector);
local_size = vec_starts[my_id+1] - vec_starts[my_id];
hypre_ParVectorInitialize(par_vector);
local_vector = hypre_ParVectorLocalVector(par_vector);
local_data = hypre_VectorData(local_vector);
vecstride = hypre_VectorVectorStride(local_vector);
idxstride = hypre_VectorIndexStride(local_vector);
hypre_assert( idxstride==1 ); /* <<< so far only the only implemented multivector StorageMethod is 0 <<< */
if (my_id == 0)
{
requests = hypre_CTAlloc(hypre_MPI_Request,num_vectors*(num_procs-1));
status = hypre_CTAlloc(hypre_MPI_Status,num_vectors*(num_procs-1));
k = 0;
for ( p=1; p<num_procs; p++)
for ( j=0; j<num_vectors; ++j )
{
hypre_MPI_Isend( &v_data[vec_starts[p]]+j*global_vecstride,
(vec_starts[p+1]-vec_starts[p]),
hypre_MPI_DOUBLE, p, 0, comm, &requests[k++] );
}
if ( num_vectors==1 )
{
for (i=0; i < local_size; i++)
local_data[i] = v_data[i];
}
else
for ( j=0; j<num_vectors; ++j )
{
for (i=0; i < local_size; i++)
local_data[i+j*vecstride] = v_data[i+j*global_vecstride];
}
hypre_MPI_Waitall(num_procs-1,requests, status);
hypre_TFree(requests);
hypre_TFree(status);
}
else
{
for ( j=0; j<num_vectors; ++j )
hypre_MPI_Recv( local_data+j*vecstride, local_size, hypre_MPI_DOUBLE, 0, 0, comm,&status0 );
}
return par_vector;
}
HYPRE_Int
main( HYPRE_Int argc,
char *argv[] )
{
hypre_CSRMatrix *matrix;
hypre_CSRMatrix *matrix1;
hypre_ParCSRMatrix *par_matrix;
hypre_Vector *x_local;
hypre_Vector *y_local;
hypre_Vector *y2_local;
hypre_ParVector *x;
hypre_ParVector *x2;
hypre_ParVector *y;
hypre_ParVector *y2;
HYPRE_Int vecstride_x, idxstride_x, vecstride_y, idxstride_y;
HYPRE_Int num_procs, my_id;
HYPRE_Int local_size;
HYPRE_Int num_vectors;
HYPRE_Int global_num_rows, global_num_cols;
HYPRE_Int first_index;
HYPRE_Int i, j, ierr=0;
double *data, *data2;
HYPRE_Int *row_starts, *col_starts;
char file_name[80];
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs);
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id);
hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs);
if (my_id == 0)
{
matrix = hypre_CSRMatrixRead("input");
hypre_printf(" read input\n");
}
row_starts = NULL;
col_starts = NULL;
par_matrix = hypre_CSRMatrixToParCSRMatrix(hypre_MPI_COMM_WORLD, matrix,
row_starts, col_starts);
hypre_printf(" converted\n");
matrix1 = hypre_ParCSRMatrixToCSRMatrixAll(par_matrix);
hypre_sprintf(file_name,"matrix1.%d",my_id);
if (matrix1) hypre_CSRMatrixPrint(matrix1, file_name);
hypre_ParCSRMatrixPrint(par_matrix,"matrix");
hypre_ParCSRMatrixPrintIJ(par_matrix,0,0,"matrixIJ");
par_matrix = hypre_ParCSRMatrixRead(hypre_MPI_COMM_WORLD,"matrix");
global_num_cols = hypre_ParCSRMatrixGlobalNumCols(par_matrix);
hypre_printf(" global_num_cols %d\n", global_num_cols);
global_num_rows = hypre_ParCSRMatrixGlobalNumRows(par_matrix);
col_starts = hypre_ParCSRMatrixColStarts(par_matrix);
first_index = col_starts[my_id];
local_size = col_starts[my_id+1] - first_index;
num_vectors = 3;
x = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_cols,
col_starts, num_vectors );
hypre_ParVectorSetPartitioningOwner(x,0);
hypre_ParVectorInitialize(x);
x_local = hypre_ParVectorLocalVector(x);
data = hypre_VectorData(x_local);
vecstride_x = hypre_VectorVectorStride(x_local);
idxstride_x = hypre_VectorIndexStride(x_local);
for ( j=0; j<num_vectors; ++j )
for (i=0; i < local_size; i++)
data[i*idxstride_x + j*vecstride_x] = first_index+i+1 + 100*j;
x2 = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_cols,
col_starts, num_vectors );
hypre_ParVectorSetPartitioningOwner(x2,0);
hypre_ParVectorInitialize(x2);
hypre_ParVectorSetConstantValues(x2,2.0);
row_starts = hypre_ParCSRMatrixRowStarts(par_matrix);
first_index = row_starts[my_id];
local_size = row_starts[my_id+1] - first_index;
y = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_rows,
row_starts, num_vectors );
hypre_ParVectorSetPartitioningOwner(y,0);
hypre_ParVectorInitialize(y);
y_local = hypre_ParVectorLocalVector(y);
y2 = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_rows,
row_starts, num_vectors );
hypre_ParVectorSetPartitioningOwner(y2,0);
hypre_ParVectorInitialize(y2);
y2_local = hypre_ParVectorLocalVector(y2);
data2 = hypre_VectorData(y2_local);
vecstride_y = hypre_VectorVectorStride(y2_local);
idxstride_y = hypre_VectorIndexStride(y2_local);
for ( j=0; j<num_vectors; ++j )
for (i=0; i < local_size; i++)
data2[i*idxstride_y+j*vecstride_y] = first_index+i+1 + 100*j;
hypre_ParVectorSetConstantValues(y,1.0);
hypre_printf(" initialized vectors, first_index=%i\n", first_index);
hypre_ParVectorPrint(x, "vectorx");
hypre_ParVectorPrint(y, "vectory");
hypre_MatvecCommPkgCreate(par_matrix);
hypre_ParCSRMatrixMatvec ( 1.0, par_matrix, x, 1.0, y);
hypre_printf(" did matvec\n");
hypre_ParVectorPrint(y, "result");
ierr = hypre_ParCSRMatrixMatvecT ( 1.0, par_matrix, y2, 1.0, x2);
hypre_printf(" did matvecT %d\n", ierr);
hypre_ParVectorPrint(x2, "transp");
hypre_ParCSRMatrixDestroy(par_matrix);
hypre_ParVectorDestroy(x);
hypre_ParVectorDestroy(x2);
hypre_ParVectorDestroy(y);
hypre_ParVectorDestroy(y2);
if (my_id == 0) hypre_CSRMatrixDestroy(matrix);
if (matrix1) hypre_CSRMatrixDestroy(matrix1);
/* Finalize MPI */
hypre_MPI_Finalize();
return 0;
}