HYPRE_Int HYPRE_ParMultiVectorCreate( MPI_Comm comm, HYPRE_Int global_size, HYPRE_Int *partitioning, HYPRE_Int number_vectors, HYPRE_ParVector *vector ) { if (!vector) { hypre_error_in_arg(5); return hypre_error_flag; } *vector = (HYPRE_ParVector) hypre_ParMultiVectorCreate( comm, global_size, partitioning, number_vectors ); return hypre_error_flag; }
HYPRE_Int main( HYPRE_Int argc, char *argv[] ) { hypre_ParVector *vector1; hypre_ParVector *vector2; hypre_ParVector *tmp_vector; HYPRE_Int num_procs, my_id; HYPRE_Int global_size = 20; HYPRE_Int local_size; HYPRE_Int first_index; HYPRE_Int num_vectors, vecstride, idxstride; HYPRE_Int i, j; HYPRE_Int *partitioning; double prod; double *data, *data2; hypre_Vector *vector; hypre_Vector *local_vector; hypre_Vector *local_vector2; /* Initialize MPI */ hypre_MPI_Init(&argc, &argv); hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs ); hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id ); hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs); partitioning = NULL; num_vectors = 3; vector1 = hypre_ParMultiVectorCreate ( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors ); partitioning = hypre_ParVectorPartitioning(vector1); hypre_ParVectorInitialize(vector1); local_vector = hypre_ParVectorLocalVector(vector1); data = hypre_VectorData(local_vector); local_size = hypre_VectorSize(local_vector); vecstride = hypre_VectorVectorStride(local_vector); idxstride = hypre_VectorIndexStride(local_vector); first_index = partitioning[my_id]; hypre_printf("vecstride=%i idxstride=%i local_size=%i num_vectors=%i", vecstride, idxstride, local_size, num_vectors ); for (j=0; j<num_vectors; ++j ) for (i=0; i < local_size; i++) data[ j*vecstride + i*idxstride ] = first_index+i + 100*j; hypre_ParVectorPrint(vector1, "Vector"); local_vector2 = hypre_SeqMultiVectorCreate( global_size, num_vectors ); hypre_SeqVectorInitialize(local_vector2); data2 = hypre_VectorData(local_vector2); vecstride = hypre_VectorVectorStride(local_vector2); idxstride = hypre_VectorIndexStride(local_vector2); for (j=0; j<num_vectors; ++j ) for (i=0; i < global_size; i++) data2[ j*vecstride + i*idxstride ] = i + 100*j; /* partitioning = hypre_CTAlloc(HYPRE_Int,4); partitioning[0] = 0; partitioning[1] = 10; partitioning[2] = 10; partitioning[3] = 20; */ partitioning = hypre_CTAlloc(HYPRE_Int,1+num_procs); hypre_GeneratePartitioning( global_size, num_procs, &partitioning ); vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning); hypre_ParVectorSetPartitioningOwner(vector2,0); hypre_ParVectorPrint(vector2, "Convert"); vector = hypre_ParVectorToVectorAll(vector2); /*----------------------------------------------------------- * Copy the vector into tmp_vector *-----------------------------------------------------------*/ /* Read doesn't work for multivectors yet... tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert");*/ tmp_vector = hypre_ParMultiVectorCreate ( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors ); hypre_ParVectorInitialize( tmp_vector ); hypre_ParVectorCopy( vector2, tmp_vector ); /* tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning); hypre_ParVectorSetPartitioningOwner(tmp_vector,0); hypre_ParVectorInitialize(tmp_vector); hypre_ParVectorCopy(vector1, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Copy"); */ /*----------------------------------------------------------- * Scale tmp_vector *-----------------------------------------------------------*/ hypre_ParVectorScale(2.0, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Scale"); /*----------------------------------------------------------- * Do an Axpy (2*vector - vector) = vector *-----------------------------------------------------------*/ hypre_ParVectorAxpy(-1.0, vector1, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Axpy"); /*----------------------------------------------------------- * Do an inner product vector* tmp_vector *-----------------------------------------------------------*/ prod = hypre_ParVectorInnerProd(vector1, tmp_vector); hypre_printf (" prod: %8.2f \n", prod); /*----------------------------------------------------------- * Finalize things *-----------------------------------------------------------*/ hypre_ParVectorDestroy(vector1); hypre_ParVectorDestroy(vector2); hypre_ParVectorDestroy(tmp_vector); hypre_SeqVectorDestroy(local_vector2); if (vector) hypre_SeqVectorDestroy(vector); /* Finalize MPI */ hypre_MPI_Finalize(); return 0; }
hypre_ParMultiVector * hypre_ParMultiVectorTempRead(MPI_Comm comm, const char *fileName) /* ***** temporary implementation ****** */ { HYPRE_Int i, n, id; double * dest; double * src; HYPRE_Int count; HYPRE_Int retcode; char temp_string[128]; hypre_ParMultiVector * x; hypre_ParVector * temp_vec; /* calculate the number of files */ hypre_MPI_Comm_rank( comm, &id ); n = 0; do { hypre_sprintf( temp_string, "test -f %s.%d.%d", fileName, n, id ); if (!(retcode=system(temp_string))) /* zero retcode mean file exists */ n++; } while (!retcode); if ( n == 0 ) return NULL; /* now read the first vector using hypre_ParVectorRead into temp_vec */ hypre_sprintf(temp_string,"%s.%d",fileName,0); temp_vec = hypre_ParVectorRead(comm, temp_string); /* this vector WON'T own partitioning */ hypre_ParVectorSetPartitioningOwner(temp_vec,0); /* now create multivector using temp_vec as a sample */ x = hypre_ParMultiVectorCreate(hypre_ParVectorComm(temp_vec), hypre_ParVectorGlobalSize(temp_vec),hypre_ParVectorPartitioning(temp_vec),n); /* this vector WILL own the partitioning */ hypre_ParMultiVectorSetPartitioningOwner(x,1); hypre_ParMultiVectorInitialize(x); /* read data from first and all other vectors into "x" */ i = 0; do { /* copy data from current vector */ dest = x->local_vector->data + i*(x->local_vector->size); src = temp_vec->local_vector->data; count = temp_vec->local_vector->size; memcpy(dest,src, count*sizeof(double)); /* destroy current vector */ hypre_ParVectorDestroy(temp_vec); /* read the data to new current vector, if there are more vectors to read */ if (i<n-1) { hypre_sprintf(temp_string,"%s.%d",fileName,i+1); temp_vec = hypre_ParVectorRead(comm, temp_string); } } while (++i<n); return x; }
hypre_ParVector * hypre_VectorToParVector (MPI_Comm comm, hypre_Vector *v, HYPRE_Int *vec_starts) { HYPRE_Int global_size; HYPRE_Int local_size; HYPRE_Int num_vectors; HYPRE_Int num_procs, my_id; HYPRE_Int global_vecstride, vecstride, idxstride; hypre_ParVector *par_vector; hypre_Vector *local_vector; double *v_data; double *local_data; hypre_MPI_Request *requests; hypre_MPI_Status *status, status0; HYPRE_Int i, j, k, p; hypre_MPI_Comm_size(comm,&num_procs); hypre_MPI_Comm_rank(comm,&my_id); if (my_id == 0) { global_size = hypre_VectorSize(v); v_data = hypre_VectorData(v); num_vectors = hypre_VectorNumVectors(v); /* for multivectors */ global_vecstride = hypre_VectorVectorStride(v); } hypre_MPI_Bcast(&global_size,1,HYPRE_MPI_INT,0,comm); hypre_MPI_Bcast(&num_vectors,1,HYPRE_MPI_INT,0,comm); hypre_MPI_Bcast(&global_vecstride,1,HYPRE_MPI_INT,0,comm); if ( num_vectors==1 ) par_vector = hypre_ParVectorCreate(comm, global_size, vec_starts); else par_vector = hypre_ParMultiVectorCreate(comm, global_size, vec_starts, num_vectors); vec_starts = hypre_ParVectorPartitioning(par_vector); local_size = vec_starts[my_id+1] - vec_starts[my_id]; hypre_ParVectorInitialize(par_vector); local_vector = hypre_ParVectorLocalVector(par_vector); local_data = hypre_VectorData(local_vector); vecstride = hypre_VectorVectorStride(local_vector); idxstride = hypre_VectorIndexStride(local_vector); hypre_assert( idxstride==1 ); /* <<< so far only the only implemented multivector StorageMethod is 0 <<< */ if (my_id == 0) { requests = hypre_CTAlloc(hypre_MPI_Request,num_vectors*(num_procs-1)); status = hypre_CTAlloc(hypre_MPI_Status,num_vectors*(num_procs-1)); k = 0; for ( p=1; p<num_procs; p++) for ( j=0; j<num_vectors; ++j ) { hypre_MPI_Isend( &v_data[vec_starts[p]]+j*global_vecstride, (vec_starts[p+1]-vec_starts[p]), hypre_MPI_DOUBLE, p, 0, comm, &requests[k++] ); } if ( num_vectors==1 ) { for (i=0; i < local_size; i++) local_data[i] = v_data[i]; } else for ( j=0; j<num_vectors; ++j ) { for (i=0; i < local_size; i++) local_data[i+j*vecstride] = v_data[i+j*global_vecstride]; } hypre_MPI_Waitall(num_procs-1,requests, status); hypre_TFree(requests); hypre_TFree(status); } else { for ( j=0; j<num_vectors; ++j ) hypre_MPI_Recv( local_data+j*vecstride, local_size, hypre_MPI_DOUBLE, 0, 0, comm,&status0 ); } return par_vector; }
HYPRE_Int main( HYPRE_Int argc, char *argv[] ) { hypre_CSRMatrix *matrix; hypre_CSRMatrix *matrix1; hypre_ParCSRMatrix *par_matrix; hypre_Vector *x_local; hypre_Vector *y_local; hypre_Vector *y2_local; hypre_ParVector *x; hypre_ParVector *x2; hypre_ParVector *y; hypre_ParVector *y2; HYPRE_Int vecstride_x, idxstride_x, vecstride_y, idxstride_y; HYPRE_Int num_procs, my_id; HYPRE_Int local_size; HYPRE_Int num_vectors; HYPRE_Int global_num_rows, global_num_cols; HYPRE_Int first_index; HYPRE_Int i, j, ierr=0; double *data, *data2; HYPRE_Int *row_starts, *col_starts; char file_name[80]; /* Initialize MPI */ hypre_MPI_Init(&argc, &argv); hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs); hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id); hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs); if (my_id == 0) { matrix = hypre_CSRMatrixRead("input"); hypre_printf(" read input\n"); } row_starts = NULL; col_starts = NULL; par_matrix = hypre_CSRMatrixToParCSRMatrix(hypre_MPI_COMM_WORLD, matrix, row_starts, col_starts); hypre_printf(" converted\n"); matrix1 = hypre_ParCSRMatrixToCSRMatrixAll(par_matrix); hypre_sprintf(file_name,"matrix1.%d",my_id); if (matrix1) hypre_CSRMatrixPrint(matrix1, file_name); hypre_ParCSRMatrixPrint(par_matrix,"matrix"); hypre_ParCSRMatrixPrintIJ(par_matrix,0,0,"matrixIJ"); par_matrix = hypre_ParCSRMatrixRead(hypre_MPI_COMM_WORLD,"matrix"); global_num_cols = hypre_ParCSRMatrixGlobalNumCols(par_matrix); hypre_printf(" global_num_cols %d\n", global_num_cols); global_num_rows = hypre_ParCSRMatrixGlobalNumRows(par_matrix); col_starts = hypre_ParCSRMatrixColStarts(par_matrix); first_index = col_starts[my_id]; local_size = col_starts[my_id+1] - first_index; num_vectors = 3; x = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_cols, col_starts, num_vectors ); hypre_ParVectorSetPartitioningOwner(x,0); hypre_ParVectorInitialize(x); x_local = hypre_ParVectorLocalVector(x); data = hypre_VectorData(x_local); vecstride_x = hypre_VectorVectorStride(x_local); idxstride_x = hypre_VectorIndexStride(x_local); for ( j=0; j<num_vectors; ++j ) for (i=0; i < local_size; i++) data[i*idxstride_x + j*vecstride_x] = first_index+i+1 + 100*j; x2 = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_cols, col_starts, num_vectors ); hypre_ParVectorSetPartitioningOwner(x2,0); hypre_ParVectorInitialize(x2); hypre_ParVectorSetConstantValues(x2,2.0); row_starts = hypre_ParCSRMatrixRowStarts(par_matrix); first_index = row_starts[my_id]; local_size = row_starts[my_id+1] - first_index; y = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_rows, row_starts, num_vectors ); hypre_ParVectorSetPartitioningOwner(y,0); hypre_ParVectorInitialize(y); y_local = hypre_ParVectorLocalVector(y); y2 = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_rows, row_starts, num_vectors ); hypre_ParVectorSetPartitioningOwner(y2,0); hypre_ParVectorInitialize(y2); y2_local = hypre_ParVectorLocalVector(y2); data2 = hypre_VectorData(y2_local); vecstride_y = hypre_VectorVectorStride(y2_local); idxstride_y = hypre_VectorIndexStride(y2_local); for ( j=0; j<num_vectors; ++j ) for (i=0; i < local_size; i++) data2[i*idxstride_y+j*vecstride_y] = first_index+i+1 + 100*j; hypre_ParVectorSetConstantValues(y,1.0); hypre_printf(" initialized vectors, first_index=%i\n", first_index); hypre_ParVectorPrint(x, "vectorx"); hypre_ParVectorPrint(y, "vectory"); hypre_MatvecCommPkgCreate(par_matrix); hypre_ParCSRMatrixMatvec ( 1.0, par_matrix, x, 1.0, y); hypre_printf(" did matvec\n"); hypre_ParVectorPrint(y, "result"); ierr = hypre_ParCSRMatrixMatvecT ( 1.0, par_matrix, y2, 1.0, x2); hypre_printf(" did matvecT %d\n", ierr); hypre_ParVectorPrint(x2, "transp"); hypre_ParCSRMatrixDestroy(par_matrix); hypre_ParVectorDestroy(x); hypre_ParVectorDestroy(x2); hypre_ParVectorDestroy(y); hypre_ParVectorDestroy(y2); if (my_id == 0) hypre_CSRMatrixDestroy(matrix); if (matrix1) hypre_CSRMatrixDestroy(matrix1); /* Finalize MPI */ hypre_MPI_Finalize(); return 0; }