hypre_ParVector * hypre_ParVectorCreate( MPI_Comm comm, HYPRE_Int global_size, HYPRE_Int *partitioning) { hypre_ParVector *vector; HYPRE_Int num_procs, my_id; if (global_size < 0) { hypre_error_in_arg(2); return NULL; } vector = hypre_CTAlloc(hypre_ParVector, 1); hypre_MPI_Comm_rank(comm,&my_id); if (!partitioning) { hypre_MPI_Comm_size(comm,&num_procs); #ifdef HYPRE_NO_GLOBAL_PARTITION hypre_GenerateLocalPartitioning(global_size, num_procs, my_id, &partitioning); #else hypre_GeneratePartitioning(global_size, num_procs, &partitioning); #endif } hypre_ParVectorAssumedPartition(vector) = NULL; hypre_ParVectorComm(vector) = comm; hypre_ParVectorGlobalSize(vector) = global_size; #ifdef HYPRE_NO_GLOBAL_PARTITION hypre_ParVectorFirstIndex(vector) = partitioning[0]; hypre_ParVectorLastIndex(vector) = partitioning[1]-1; hypre_ParVectorPartitioning(vector) = partitioning; hypre_ParVectorLocalVector(vector) = hypre_SeqVectorCreate(partitioning[1]-partitioning[0]); #else hypre_ParVectorFirstIndex(vector) = partitioning[my_id]; hypre_ParVectorLastIndex(vector) = partitioning[my_id+1] -1; hypre_ParVectorPartitioning(vector) = partitioning; hypre_ParVectorLocalVector(vector) = hypre_SeqVectorCreate(partitioning[my_id+1]-partitioning[my_id]); #endif /* set defaults */ hypre_ParVectorOwnsData(vector) = 1; hypre_ParVectorOwnsPartitioning(vector) = 1; return vector; }
hypre_ParVector *hypre_ParVectorRead( MPI_Comm comm, const char *file_name ) { char new_file_name[80]; hypre_ParVector *par_vector; HYPRE_Int my_id, num_procs; HYPRE_Int *partitioning; HYPRE_Int global_size, i; FILE *fp; hypre_MPI_Comm_rank(comm,&my_id); hypre_MPI_Comm_size(comm,&num_procs); partitioning = hypre_CTAlloc(HYPRE_Int,num_procs+1); hypre_sprintf(new_file_name,"%s.INFO.%d",file_name,my_id); fp = fopen(new_file_name, "r"); hypre_fscanf(fp, "%d\n", &global_size); #ifdef HYPRE_NO_GLOBAL_PARTITION for (i=0; i < 2; i++) hypre_fscanf(fp, "%d\n", &partitioning[i]); fclose (fp); #else for (i=0; i < num_procs; i++) hypre_fscanf(fp, "%d\n", &partitioning[i]); fclose (fp); partitioning[num_procs] = global_size; #endif par_vector = hypre_CTAlloc(hypre_ParVector, 1); hypre_ParVectorComm(par_vector) = comm; hypre_ParVectorGlobalSize(par_vector) = global_size; #ifdef HYPRE_NO_GLOBAL_PARTITION hypre_ParVectorFirstIndex(par_vector) = partitioning[0]; hypre_ParVectorLastIndex(par_vector) = partitioning[1]-1; #else hypre_ParVectorFirstIndex(par_vector) = partitioning[my_id]; hypre_ParVectorLastIndex(par_vector) = partitioning[my_id+1]-1; #endif hypre_ParVectorPartitioning(par_vector) = partitioning; hypre_ParVectorOwnsData(par_vector) = 1; hypre_ParVectorOwnsPartitioning(par_vector) = 1; hypre_sprintf(new_file_name,"%s.%d",file_name,my_id); hypre_ParVectorLocalVector(par_vector) = hypre_SeqVectorRead(new_file_name); /* multivector code not written yet >>> */ hypre_assert( hypre_ParVectorNumVectors(par_vector) == 1 ); return par_vector; }
void * hypre_ParKrylovCreateVector( void *vvector ) { hypre_ParVector *vector = vvector; hypre_ParVector *new_vector; new_vector = hypre_ParVectorCreate( hypre_ParVectorComm(vector), hypre_ParVectorGlobalSize(vector), hypre_ParVectorPartitioning(vector) ); hypre_ParVectorSetPartitioningOwner(new_vector,0); hypre_ParVectorInitialize(new_vector); return ( (void *) new_vector ); }
hypre_ParVector * hypre_ParVectorCloneShallow( hypre_ParVector *x ) { hypre_ParVector * y = hypre_ParVectorCreate( hypre_ParVectorComm(x), hypre_ParVectorGlobalSize(x), hypre_ParVectorPartitioning(x) ); hypre_ParVectorOwnsData(y) = 1; /* ...This vector owns its local vector, although the local vector doesn't own _its_ data */ hypre_ParVectorOwnsPartitioning(y) = 0; hypre_SeqVectorDestroy( hypre_ParVectorLocalVector(y) ); hypre_ParVectorLocalVector(y) = hypre_SeqVectorCloneShallow( hypre_ParVectorLocalVector(x) ); hypre_ParVectorFirstIndex(y) = hypre_ParVectorFirstIndex(x); return y; }
int HYPRE_ParCSRCotreeSetup(HYPRE_Solver solver, HYPRE_ParCSRMatrix A, HYPRE_ParVector b, HYPRE_ParVector x) { int *partition, *new_partition, nprocs, *tindices, ii; void *vsolver = (void *) solver; /* void *vA = (void *) A; void *vb = (void *) b; void *vx = (void *) x; */ hypre_CotreeData *cotree_data = (hypre_CotreeData *) vsolver; hypre_ParCSRMatrix **submatrices; hypre_ParVector *new_vector; MPI_Comm comm; cotree_data->Aee = (hypre_ParCSRMatrix *) A; hypre_ParCSRMatrixGenSpanningTree(cotree_data->Gen, &tindices, 1); submatrices = (hypre_ParCSRMatrix **) malloc(sizeof(hypre_ParCSRMatrix *)); hypre_ParCSRMatrixExtractSubmatrices(cotree_data->Aee, tindices, &submatrices); cotree_data->Att = submatrices[0]; cotree_data->Atc = submatrices[1]; cotree_data->Act = submatrices[2]; cotree_data->Acc = submatrices[3]; hypre_ParCSRMatrixExtractRowSubmatrices(cotree_data->Gen, tindices, &submatrices); cotree_data->Gt = submatrices[0]; cotree_data->Gc = submatrices[1]; free(submatrices); comm = hypre_ParCSRMatrixComm((hypre_ParCSRMatrix *) A); MPI_Comm_size(comm, &nprocs); partition = hypre_ParVectorPartitioning((hypre_ParVector *) b); new_partition = (int *) malloc((nprocs+1) * sizeof(int)); for (ii = 0; ii <= nprocs; ii++) new_partition[ii] = partition[ii]; /* partition = hypre_ParVectorPartitioning((hypre_ParVector *) b); */ new_vector = hypre_ParVectorCreate(hypre_ParVectorComm((hypre_ParVector *)b), (int) hypre_ParVectorGlobalSize((hypre_ParVector *) b), new_partition); hypre_ParVectorInitialize(new_vector); cotree_data->w = new_vector; return 0; }
/****************************************************************************** * * hypre_IJVectorInitializePar * * initializes ParVector of IJVectorPar * *****************************************************************************/ HYPRE_Int hypre_IJVectorInitializePar(hypre_IJVector *vector) { hypre_ParVector *par_vector = hypre_IJVectorObject(vector); hypre_AuxParVector *aux_vector = hypre_IJVectorTranslator(vector); HYPRE_Int *partitioning = hypre_ParVectorPartitioning(par_vector); hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector); HYPRE_Int my_id; HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector); MPI_Comm comm = hypre_IJVectorComm(vector); hypre_MPI_Comm_rank(comm,&my_id); if (!partitioning) { if (print_level) { hypre_printf("No ParVector partitioning for initialization -- "); hypre_printf("hypre_IJVectorInitializePar\n"); } hypre_error_in_arg(1); } #ifdef HYPRE_NO_GLOBAL_PARTITION hypre_VectorSize(local_vector) = partitioning[1] - partitioning[0]; #else hypre_VectorSize(local_vector) = partitioning[my_id+1] - partitioning[my_id]; #endif hypre_ParVectorInitialize(par_vector); if (!aux_vector) { hypre_AuxParVectorCreate(&aux_vector); hypre_IJVectorTranslator(vector) = aux_vector; } hypre_AuxParVectorInitialize(aux_vector); return hypre_error_flag; }
void * hypre_ParKrylovCreateVectorArray(HYPRE_Int n, void *vvector ) { hypre_ParVector *vector = vvector; hypre_ParVector **new_vector; HYPRE_Int i; new_vector = hypre_CTAlloc(hypre_ParVector*,n); for (i=0; i < n; i++) { new_vector[i] = hypre_ParVectorCreate( hypre_ParVectorComm(vector), hypre_ParVectorGlobalSize(vector), hypre_ParVectorPartitioning(vector) ); hypre_ParVectorSetPartitioningOwner(new_vector[i],0); hypre_ParVectorInitialize(new_vector[i]); } return ( (void *) new_vector ); }
HYPRE_Int hypre_ParVectorPrint( hypre_ParVector *vector, const char *file_name ) { char new_file_name[80]; hypre_Vector *local_vector; MPI_Comm comm; HYPRE_Int my_id, num_procs, i; HYPRE_Int *partitioning; HYPRE_Int global_size; FILE *fp; if (!vector) { hypre_error_in_arg(1); return hypre_error_flag; } local_vector = hypre_ParVectorLocalVector(vector); comm = hypre_ParVectorComm(vector); partitioning = hypre_ParVectorPartitioning(vector); global_size = hypre_ParVectorGlobalSize(vector); hypre_MPI_Comm_rank(comm,&my_id); hypre_MPI_Comm_size(comm,&num_procs); hypre_sprintf(new_file_name,"%s.%d",file_name,my_id); hypre_SeqVectorPrint(local_vector,new_file_name); hypre_sprintf(new_file_name,"%s.INFO.%d",file_name,my_id); fp = fopen(new_file_name, "w"); hypre_fprintf(fp, "%d\n", global_size); #ifdef HYPRE_NO_GLOBAL_PARTITION for (i=0; i < 2; i++) hypre_fprintf(fp, "%d\n", partitioning[i]); #else for (i=0; i < num_procs; i++) hypre_fprintf(fp, "%d\n", partitioning[i]); #endif fclose (fp); return hypre_error_flag; }
HYPRE_Int hypre_ParVectorDestroy( hypre_ParVector *vector ) { if (vector) { if ( hypre_ParVectorOwnsData(vector) ) { hypre_SeqVectorDestroy(hypre_ParVectorLocalVector(vector)); } if ( hypre_ParVectorOwnsPartitioning(vector) ) { hypre_TFree(hypre_ParVectorPartitioning(vector)); } if (hypre_ParVectorAssumedPartition(vector)) hypre_ParVectorDestroyAssumedPartition(vector); hypre_TFree(vector); } return hypre_error_flag; }
HYPRE_Int main( HYPRE_Int argc, char *argv[] ) { hypre_ParVector *vector1; hypre_ParVector *vector2; hypre_ParVector *tmp_vector; HYPRE_Int num_procs, my_id; HYPRE_Int global_size = 20; HYPRE_Int local_size; HYPRE_Int first_index; HYPRE_Int num_vectors, vecstride, idxstride; HYPRE_Int i, j; HYPRE_Int *partitioning; double prod; double *data, *data2; hypre_Vector *vector; hypre_Vector *local_vector; hypre_Vector *local_vector2; /* Initialize MPI */ hypre_MPI_Init(&argc, &argv); hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs ); hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id ); hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs); partitioning = NULL; num_vectors = 3; vector1 = hypre_ParMultiVectorCreate ( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors ); partitioning = hypre_ParVectorPartitioning(vector1); hypre_ParVectorInitialize(vector1); local_vector = hypre_ParVectorLocalVector(vector1); data = hypre_VectorData(local_vector); local_size = hypre_VectorSize(local_vector); vecstride = hypre_VectorVectorStride(local_vector); idxstride = hypre_VectorIndexStride(local_vector); first_index = partitioning[my_id]; hypre_printf("vecstride=%i idxstride=%i local_size=%i num_vectors=%i", vecstride, idxstride, local_size, num_vectors ); for (j=0; j<num_vectors; ++j ) for (i=0; i < local_size; i++) data[ j*vecstride + i*idxstride ] = first_index+i + 100*j; hypre_ParVectorPrint(vector1, "Vector"); local_vector2 = hypre_SeqMultiVectorCreate( global_size, num_vectors ); hypre_SeqVectorInitialize(local_vector2); data2 = hypre_VectorData(local_vector2); vecstride = hypre_VectorVectorStride(local_vector2); idxstride = hypre_VectorIndexStride(local_vector2); for (j=0; j<num_vectors; ++j ) for (i=0; i < global_size; i++) data2[ j*vecstride + i*idxstride ] = i + 100*j; /* partitioning = hypre_CTAlloc(HYPRE_Int,4); partitioning[0] = 0; partitioning[1] = 10; partitioning[2] = 10; partitioning[3] = 20; */ partitioning = hypre_CTAlloc(HYPRE_Int,1+num_procs); hypre_GeneratePartitioning( global_size, num_procs, &partitioning ); vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning); hypre_ParVectorSetPartitioningOwner(vector2,0); hypre_ParVectorPrint(vector2, "Convert"); vector = hypre_ParVectorToVectorAll(vector2); /*----------------------------------------------------------- * Copy the vector into tmp_vector *-----------------------------------------------------------*/ /* Read doesn't work for multivectors yet... tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert");*/ tmp_vector = hypre_ParMultiVectorCreate ( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors ); hypre_ParVectorInitialize( tmp_vector ); hypre_ParVectorCopy( vector2, tmp_vector ); /* tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning); hypre_ParVectorSetPartitioningOwner(tmp_vector,0); hypre_ParVectorInitialize(tmp_vector); hypre_ParVectorCopy(vector1, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Copy"); */ /*----------------------------------------------------------- * Scale tmp_vector *-----------------------------------------------------------*/ hypre_ParVectorScale(2.0, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Scale"); /*----------------------------------------------------------- * Do an Axpy (2*vector - vector) = vector *-----------------------------------------------------------*/ hypre_ParVectorAxpy(-1.0, vector1, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Axpy"); /*----------------------------------------------------------- * Do an inner product vector* tmp_vector *-----------------------------------------------------------*/ prod = hypre_ParVectorInnerProd(vector1, tmp_vector); hypre_printf (" prod: %8.2f \n", prod); /*----------------------------------------------------------- * Finalize things *-----------------------------------------------------------*/ hypre_ParVectorDestroy(vector1); hypre_ParVectorDestroy(vector2); hypre_ParVectorDestroy(tmp_vector); hypre_SeqVectorDestroy(local_vector2); if (vector) hypre_SeqVectorDestroy(vector); /* Finalize MPI */ hypre_MPI_Finalize(); return 0; }
HYPRE_Int hypre_BoomerAMGCycle( void *amg_vdata, hypre_ParVector **F_array, hypre_ParVector **U_array ) { hypre_ParAMGData *amg_data = amg_vdata; HYPRE_Solver *smoother; /* Data Structure variables */ hypre_ParCSRMatrix **A_array; hypre_ParCSRMatrix **P_array; hypre_ParCSRMatrix **R_array; hypre_ParVector *Utemp; hypre_ParVector *Vtemp; hypre_ParVector *Rtemp; hypre_ParVector *Ptemp; hypre_ParVector *Ztemp; hypre_ParVector *Aux_U; hypre_ParVector *Aux_F; hypre_ParCSRBlockMatrix **A_block_array; hypre_ParCSRBlockMatrix **P_block_array; hypre_ParCSRBlockMatrix **R_block_array; HYPRE_Real *Ztemp_data; HYPRE_Real *Ptemp_data; HYPRE_Int **CF_marker_array; /* HYPRE_Int **unknown_map_array; HYPRE_Int **point_map_array; HYPRE_Int **v_at_point_array; */ HYPRE_Real cycle_op_count; HYPRE_Int cycle_type; HYPRE_Int num_levels; HYPRE_Int max_levels; HYPRE_Real *num_coeffs; HYPRE_Int *num_grid_sweeps; HYPRE_Int *grid_relax_type; HYPRE_Int **grid_relax_points; HYPRE_Int block_mode; HYPRE_Real *max_eig_est; HYPRE_Real *min_eig_est; HYPRE_Int cheby_order; HYPRE_Real cheby_fraction; /* Local variables */ HYPRE_Int *lev_counter; HYPRE_Int Solve_err_flag; HYPRE_Int k; HYPRE_Int i, j, jj; HYPRE_Int level; HYPRE_Int cycle_param; HYPRE_Int coarse_grid; HYPRE_Int fine_grid; HYPRE_Int Not_Finished; HYPRE_Int num_sweep; HYPRE_Int cg_num_sweep = 1; HYPRE_Int relax_type; HYPRE_Int relax_points; HYPRE_Int relax_order; HYPRE_Int relax_local; HYPRE_Int old_version = 0; HYPRE_Real *relax_weight; HYPRE_Real *omega; HYPRE_Real alfa, beta, gammaold; HYPRE_Real gamma = 1.0; HYPRE_Int local_size; /* HYPRE_Int *smooth_option; */ HYPRE_Int smooth_type; HYPRE_Int smooth_num_levels; HYPRE_Int num_threads, my_id; HYPRE_Real alpha; HYPRE_Real **l1_norms = NULL; HYPRE_Real *l1_norms_level; HYPRE_Int seq_cg = 0; MPI_Comm comm; #if 0 HYPRE_Real *D_mat; HYPRE_Real *S_vec; #endif /* Acquire data and allocate storage */ num_threads = hypre_NumThreads(); A_array = hypre_ParAMGDataAArray(amg_data); P_array = hypre_ParAMGDataPArray(amg_data); R_array = hypre_ParAMGDataRArray(amg_data); CF_marker_array = hypre_ParAMGDataCFMarkerArray(amg_data); Vtemp = hypre_ParAMGDataVtemp(amg_data); Rtemp = hypre_ParAMGDataRtemp(amg_data); Ptemp = hypre_ParAMGDataPtemp(amg_data); Ztemp = hypre_ParAMGDataZtemp(amg_data); num_levels = hypre_ParAMGDataNumLevels(amg_data); max_levels = hypre_ParAMGDataMaxLevels(amg_data); cycle_type = hypre_ParAMGDataCycleType(amg_data); A_block_array = hypre_ParAMGDataABlockArray(amg_data); P_block_array = hypre_ParAMGDataPBlockArray(amg_data); R_block_array = hypre_ParAMGDataRBlockArray(amg_data); block_mode = hypre_ParAMGDataBlockMode(amg_data); num_grid_sweeps = hypre_ParAMGDataNumGridSweeps(amg_data); grid_relax_type = hypre_ParAMGDataGridRelaxType(amg_data); grid_relax_points = hypre_ParAMGDataGridRelaxPoints(amg_data); relax_order = hypre_ParAMGDataRelaxOrder(amg_data); relax_weight = hypre_ParAMGDataRelaxWeight(amg_data); omega = hypre_ParAMGDataOmega(amg_data); smooth_type = hypre_ParAMGDataSmoothType(amg_data); smooth_num_levels = hypre_ParAMGDataSmoothNumLevels(amg_data); l1_norms = hypre_ParAMGDataL1Norms(amg_data); /* smooth_option = hypre_ParAMGDataSmoothOption(amg_data); */ max_eig_est = hypre_ParAMGDataMaxEigEst(amg_data); min_eig_est = hypre_ParAMGDataMinEigEst(amg_data); cheby_order = hypre_ParAMGDataChebyOrder(amg_data); cheby_fraction = hypre_ParAMGDataChebyFraction(amg_data); cycle_op_count = hypre_ParAMGDataCycleOpCount(amg_data); lev_counter = hypre_CTAlloc(HYPRE_Int, num_levels); if (hypre_ParAMGDataParticipate(amg_data)) seq_cg = 1; /* Initialize */ Solve_err_flag = 0; if (grid_relax_points) old_version = 1; num_coeffs = hypre_CTAlloc(HYPRE_Real, num_levels); num_coeffs[0] = hypre_ParCSRMatrixDNumNonzeros(A_array[0]); comm = hypre_ParCSRMatrixComm(A_array[0]); hypre_MPI_Comm_rank(comm,&my_id); if (block_mode) { for (j = 1; j < num_levels; j++) num_coeffs[j] = hypre_ParCSRBlockMatrixNumNonzeros(A_block_array[j]); } else { for (j = 1; j < num_levels; j++) num_coeffs[j] = hypre_ParCSRMatrixDNumNonzeros(A_array[j]); } /*--------------------------------------------------------------------- * Initialize cycling control counter * * Cycling is controlled using a level counter: lev_counter[k] * * Each time relaxation is performed on level k, the * counter is decremented by 1. If the counter is then * negative, we go to the next finer level. If non- * negative, we go to the next coarser level. The * following actions control cycling: * * a. lev_counter[0] is initialized to 1. * b. lev_counter[k] is initialized to cycle_type for k>0. * * c. During cycling, when going down to level k, lev_counter[k] * is set to the max of (lev_counter[k],cycle_type) *---------------------------------------------------------------------*/ Not_Finished = 1; lev_counter[0] = 1; for (k = 1; k < num_levels; ++k) { lev_counter[k] = cycle_type; } level = 0; cycle_param = 1; smoother = hypre_ParAMGDataSmoother(amg_data); if (smooth_num_levels > 0) { if (smooth_type == 7 || smooth_type == 8 || smooth_type == 17 || smooth_type == 18 || smooth_type == 9 || smooth_type == 19) { HYPRE_Int actual_local_size = hypre_ParVectorActualLocalSize(Vtemp); Utemp = hypre_ParVectorCreate(comm,hypre_ParVectorGlobalSize(Vtemp), hypre_ParVectorPartitioning(Vtemp)); hypre_ParVectorOwnsPartitioning(Utemp) = 0; local_size = hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp)); if (local_size < actual_local_size) { hypre_VectorData(hypre_ParVectorLocalVector(Utemp)) = hypre_CTAlloc(HYPRE_Complex, actual_local_size); hypre_ParVectorActualLocalSize(Utemp) = actual_local_size; } else hypre_ParVectorInitialize(Utemp); } } /*--------------------------------------------------------------------- * Main loop of cycling *--------------------------------------------------------------------*/ while (Not_Finished) { if (num_levels > 1) { local_size = hypre_VectorSize(hypre_ParVectorLocalVector(F_array[level])); hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp)) = local_size; if (smooth_num_levels <= level) { cg_num_sweep = 1; num_sweep = num_grid_sweeps[cycle_param]; Aux_U = U_array[level]; Aux_F = F_array[level]; } else if (smooth_type > 9) { hypre_VectorSize(hypre_ParVectorLocalVector(Ztemp)) = local_size; hypre_VectorSize(hypre_ParVectorLocalVector(Rtemp)) = local_size; hypre_VectorSize(hypre_ParVectorLocalVector(Ptemp)) = local_size; Ztemp_data = hypre_VectorData(hypre_ParVectorLocalVector(Ztemp)); Ptemp_data = hypre_VectorData(hypre_ParVectorLocalVector(Ptemp)); hypre_ParVectorSetConstantValues(Ztemp,0); alpha = -1.0; beta = 1.0; hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level], U_array[level], beta, F_array[level], Rtemp); cg_num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data); num_sweep = num_grid_sweeps[cycle_param]; Aux_U = Ztemp; Aux_F = Rtemp; } else { cg_num_sweep = 1; num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data); Aux_U = U_array[level]; Aux_F = F_array[level]; } relax_type = grid_relax_type[cycle_param]; } else /* AB: 4/08: removed the max_levels > 1 check - should do this when max-levels = 1 also */ { /* If no coarsening occurred, apply a simple smoother once */ Aux_U = U_array[level]; Aux_F = F_array[level]; num_sweep = 1; /* TK: Use the user relax type (instead of 0) to allow for setting a convergent smoother (e.g. in the solution of singular problems). */ relax_type = hypre_ParAMGDataUserRelaxType(amg_data); } if (l1_norms != NULL) l1_norms_level = l1_norms[level]; else l1_norms_level = NULL; if (cycle_param == 3 && seq_cg) { hypre_seqAMGCycle(amg_data, level, F_array, U_array); } else { /*------------------------------------------------------------------ * Do the relaxation num_sweep times *-----------------------------------------------------------------*/ for (jj = 0; jj < cg_num_sweep; jj++) { if (smooth_num_levels > level && smooth_type > 9) hypre_ParVectorSetConstantValues(Aux_U,0); for (j = 0; j < num_sweep; j++) { if (num_levels == 1 && max_levels > 1) { relax_points = 0; relax_local = 0; } else { if (old_version) relax_points = grid_relax_points[cycle_param][j]; relax_local = relax_order; } /*----------------------------------------------- * VERY sloppy approximation to cycle complexity *-----------------------------------------------*/ if (old_version && level < num_levels -1) { switch (relax_points) { case 1: cycle_op_count += num_coeffs[level+1]; break; case -1: cycle_op_count += (num_coeffs[level]-num_coeffs[level+1]); break; } } else { cycle_op_count += num_coeffs[level]; } /*----------------------------------------------- Choose Smoother -----------------------------------------------*/ if (smooth_num_levels > level && (smooth_type == 7 || smooth_type == 8 || smooth_type == 9 || smooth_type == 19 || smooth_type == 17 || smooth_type == 18)) { hypre_VectorSize(hypre_ParVectorLocalVector(Utemp)) = local_size; alpha = -1.0; beta = 1.0; hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level], U_array[level], beta, Aux_F, Vtemp); if (smooth_type == 8 || smooth_type == 18) HYPRE_ParCSRParaSailsSolve(smoother[level], (HYPRE_ParCSRMatrix) A_array[level], (HYPRE_ParVector) Vtemp, (HYPRE_ParVector) Utemp); else if (smooth_type == 7 || smooth_type == 17) HYPRE_ParCSRPilutSolve(smoother[level], (HYPRE_ParCSRMatrix) A_array[level], (HYPRE_ParVector) Vtemp, (HYPRE_ParVector) Utemp); else if (smooth_type == 9 || smooth_type == 19) HYPRE_EuclidSolve(smoother[level], (HYPRE_ParCSRMatrix) A_array[level], (HYPRE_ParVector) Vtemp, (HYPRE_ParVector) Utemp); hypre_ParVectorAxpy(relax_weight[level],Utemp,Aux_U); } else if (smooth_num_levels > level && (smooth_type == 6 || smooth_type == 16)) { HYPRE_SchwarzSolve(smoother[level], (HYPRE_ParCSRMatrix) A_array[level], (HYPRE_ParVector) Aux_F, (HYPRE_ParVector) Aux_U); } /*else if (relax_type == 99)*/ else if (relax_type == 9 || relax_type == 99) { /* Gaussian elimination */ hypre_GaussElimSolve(amg_data, level, relax_type); } else if (relax_type == 18) { /* L1 - Jacobi*/ if (relax_order == 1 && cycle_param < 3) { /* need to do CF - so can't use the AMS one */ HYPRE_Int i; HYPRE_Int loc_relax_points[2]; if (cycle_type < 2) { loc_relax_points[0] = 1; loc_relax_points[1] = -1; } else { loc_relax_points[0] = -1; loc_relax_points[1] = 1; } for (i=0; i < 2; i++) hypre_ParCSRRelax_L1_Jacobi(A_array[level], Aux_F, CF_marker_array[level], loc_relax_points[i], relax_weight[level], l1_norms[level], Aux_U, Vtemp); } else /* not CF - so use through AMS */ { if (num_threads == 1) hypre_ParCSRRelax(A_array[level], Aux_F, 1, 1, l1_norms_level, relax_weight[level], omega[level],0,0,0,0, Aux_U, Vtemp, Ztemp); else hypre_ParCSRRelaxThreads(A_array[level], Aux_F, 1, 1, l1_norms_level, relax_weight[level], omega[level], Aux_U, Vtemp, Ztemp); } } else if (relax_type == 15) { /* CG */ if (j ==0) /* do num sweep iterations of CG */ hypre_ParCSRRelax_CG( smoother[level], A_array[level], Aux_F, Aux_U, num_sweep); } else if (relax_type == 16) { /* scaled Chebyshev */ HYPRE_Int scale = 1; HYPRE_Int variant = 0; hypre_ParCSRRelax_Cheby(A_array[level], Aux_F, max_eig_est[level], min_eig_est[level], cheby_fraction, cheby_order, scale, variant, Aux_U, Vtemp, Ztemp ); } else if (relax_type ==17) { hypre_BoomerAMGRelax_FCFJacobi(A_array[level], Aux_F, CF_marker_array[level], relax_weight[level], Aux_U, Vtemp); } else if (old_version) { Solve_err_flag = hypre_BoomerAMGRelax(A_array[level], Aux_F, CF_marker_array[level], relax_type, relax_points, relax_weight[level], omega[level], l1_norms_level, Aux_U, Vtemp, Ztemp); } else { /* smoother than can have CF ordering */ if (block_mode) { Solve_err_flag = hypre_BoomerAMGBlockRelaxIF(A_block_array[level], Aux_F, CF_marker_array[level], relax_type, relax_local, cycle_param, relax_weight[level], omega[level], Aux_U, Vtemp); } else { Solve_err_flag = hypre_BoomerAMGRelaxIF(A_array[level], Aux_F, CF_marker_array[level], relax_type, relax_local, cycle_param, relax_weight[level], omega[level], l1_norms_level, Aux_U, Vtemp, Ztemp); } } if (Solve_err_flag != 0) return(Solve_err_flag); } if (smooth_num_levels > level && smooth_type > 9) { gammaold = gamma; gamma = hypre_ParVectorInnerProd(Rtemp,Ztemp); if (jj == 0) hypre_ParVectorCopy(Ztemp,Ptemp); else { beta = gamma/gammaold; for (i=0; i < local_size; i++) Ptemp_data[i] = Ztemp_data[i] + beta*Ptemp_data[i]; } hypre_ParCSRMatrixMatvec(1.0,A_array[level],Ptemp,0.0,Vtemp); alfa = gamma /hypre_ParVectorInnerProd(Ptemp,Vtemp); hypre_ParVectorAxpy(alfa,Ptemp,U_array[level]); hypre_ParVectorAxpy(-alfa,Vtemp,Rtemp); } } } /*------------------------------------------------------------------ * Decrement the control counter and determine which grid to visit next *-----------------------------------------------------------------*/ --lev_counter[level]; if (lev_counter[level] >= 0 && level != num_levels-1) { /*--------------------------------------------------------------- * Visit coarser level next. * Compute residual using hypre_ParCSRMatrixMatvec. * Perform restriction using hypre_ParCSRMatrixMatvecT. * Reset counters and cycling parameters for coarse level *--------------------------------------------------------------*/ fine_grid = level; coarse_grid = level + 1; hypre_ParVectorSetConstantValues(U_array[coarse_grid], 0.0); alpha = -1.0; beta = 1.0; if (block_mode) { hypre_ParVectorCopy(F_array[fine_grid],Vtemp); hypre_ParCSRBlockMatrixMatvec(alpha, A_block_array[fine_grid], U_array[fine_grid], beta, Vtemp); } else { // JSP: avoid unnecessary copy using out-of-place version of SpMV hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[fine_grid], U_array[fine_grid], beta, F_array[fine_grid], Vtemp); } alpha = 1.0; beta = 0.0; if (block_mode) { hypre_ParCSRBlockMatrixMatvecT(alpha,R_block_array[fine_grid],Vtemp, beta,F_array[coarse_grid]); } else { hypre_ParCSRMatrixMatvecT(alpha,R_array[fine_grid],Vtemp, beta,F_array[coarse_grid]); } ++level; lev_counter[level] = hypre_max(lev_counter[level],cycle_type); cycle_param = 1; if (level == num_levels-1) cycle_param = 3; } else if (level != 0) { /*--------------------------------------------------------------- * Visit finer level next. * Interpolate and add correction using hypre_ParCSRMatrixMatvec. * Reset counters and cycling parameters for finer level. *--------------------------------------------------------------*/ fine_grid = level - 1; coarse_grid = level; alpha = 1.0; beta = 1.0; if (block_mode) { hypre_ParCSRBlockMatrixMatvec(alpha, P_block_array[fine_grid], U_array[coarse_grid], beta, U_array[fine_grid]); } else { hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid], U_array[coarse_grid], beta, U_array[fine_grid]); } --level; cycle_param = 2; } else { Not_Finished = 0; } } hypre_ParAMGDataCycleOpCount(amg_data) = cycle_op_count; hypre_TFree(lev_counter); hypre_TFree(num_coeffs); if (smooth_num_levels > 0) { if (smooth_type == 7 || smooth_type == 8 || smooth_type == 9 || smooth_type == 17 || smooth_type == 18 || smooth_type == 19 ) hypre_ParVectorDestroy(Utemp); } return(Solve_err_flag); }
/****************************************************************************** * * hypre_IJVectorZeroValuesPar * * zeroes all local components of an IJVectorPar * *****************************************************************************/ HYPRE_Int hypre_IJVectorZeroValuesPar(hypre_IJVector *vector) { HYPRE_Int my_id; HYPRE_Int i, vec_start, vec_stop; double *data; hypre_ParVector *par_vector = hypre_IJVectorObject(vector); MPI_Comm comm = hypre_IJVectorComm(vector); HYPRE_Int *partitioning = hypre_ParVectorPartitioning(par_vector); hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector); HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector); hypre_MPI_Comm_rank(comm, &my_id); /* If par_vector == NULL or partitioning == NULL or local_vector == NULL let user know of catastrophe and exit */ if (!par_vector) { if (print_level) { hypre_printf("par_vector == NULL -- "); hypre_printf("hypre_IJVectorZeroValuesPar\n"); hypre_printf("**** Vector storage is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } if (!partitioning) { if (print_level) { hypre_printf("partitioning == NULL -- "); hypre_printf("hypre_IJVectorZeroValuesPar\n"); hypre_printf("**** Vector partitioning is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } if (!local_vector) { if (print_level) { hypre_printf("local_vector == NULL -- "); hypre_printf("hypre_IJVectorZeroValuesPar\n"); hypre_printf("**** Vector local data is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } #ifdef HYPRE_NO_GLOBAL_PARTITION vec_start = partitioning[0]; vec_stop = partitioning[1]; #else vec_start = partitioning[my_id]; vec_stop = partitioning[my_id+1]; #endif if (vec_start > vec_stop) { if (print_level) { hypre_printf("vec_start > vec_stop -- "); hypre_printf("hypre_IJVectorZeroValuesPar\n"); hypre_printf("**** This vector partitioning should not occur ****\n"); } hypre_error_in_arg(1); } data = hypre_VectorData( local_vector ); for (i = 0; i < vec_stop - vec_start; i++) data[i] = 0.; return hypre_error_flag; }
HYPRE_Int hypre_ParVectorPrintIJ( hypre_ParVector *vector, HYPRE_Int base_j, const char *filename ) { MPI_Comm comm; HYPRE_Int global_size; HYPRE_Int *partitioning; double *local_data; HYPRE_Int myid, num_procs, i, j, part0; char new_filename[255]; FILE *file; if (!vector) { hypre_error_in_arg(1); return hypre_error_flag; } comm = hypre_ParVectorComm(vector); global_size = hypre_ParVectorGlobalSize(vector); partitioning = hypre_ParVectorPartitioning(vector); /* multivector code not written yet >>> */ hypre_assert( hypre_ParVectorNumVectors(vector) == 1 ); if ( hypre_ParVectorNumVectors(vector) != 1 ) hypre_error_in_arg(1); hypre_MPI_Comm_rank(comm, &myid); hypre_MPI_Comm_size(comm, &num_procs); hypre_sprintf(new_filename,"%s.%05d", filename, myid); if ((file = fopen(new_filename, "w")) == NULL) { hypre_printf("Error: can't open output file %s\n", new_filename); hypre_error_in_arg(3); return hypre_error_flag; } local_data = hypre_VectorData(hypre_ParVectorLocalVector(vector)); hypre_fprintf(file, "%d \n", global_size); #ifdef HYPRE_NO_GLOBAL_PARTITION for (i=0; i <= 2; i++) #else for (i=0; i <= num_procs; i++) #endif { hypre_fprintf(file, "%d \n", partitioning[i] + base_j); } #ifdef HYPRE_NO_GLOBAL_PARTITION part0 = partitioning[0]; for (j = part0; j < partitioning[1]; j++) #else part0 = partitioning[myid]; for (j = part0; j < partitioning[myid+1]; j++) #endif { hypre_fprintf(file, "%d %.14e\n", j + base_j, local_data[j-part0]); } fclose(file); return hypre_error_flag; }
hypre_Vector * hypre_ParVectorToVectorAll (hypre_ParVector *par_v) { MPI_Comm comm = hypre_ParVectorComm(par_v); HYPRE_Int global_size = hypre_ParVectorGlobalSize(par_v); #ifndef HYPRE_NO_GLOBAL_PARTITION HYPRE_Int *vec_starts = hypre_ParVectorPartitioning(par_v); #endif hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_v); HYPRE_Int num_procs, my_id; HYPRE_Int num_vectors = hypre_ParVectorNumVectors(par_v); hypre_Vector *vector; double *vector_data; double *local_data; HYPRE_Int local_size; hypre_MPI_Request *requests; hypre_MPI_Status *status; HYPRE_Int i, j; HYPRE_Int *used_procs; HYPRE_Int num_types, num_requests; HYPRE_Int vec_len, proc_id; #ifdef HYPRE_NO_GLOBAL_PARTITION HYPRE_Int *new_vec_starts; HYPRE_Int num_contacts; HYPRE_Int contact_proc_list[1]; HYPRE_Int contact_send_buf[1]; HYPRE_Int contact_send_buf_starts[2]; HYPRE_Int max_response_size; HYPRE_Int *response_recv_buf=NULL; HYPRE_Int *response_recv_buf_starts = NULL; hypre_DataExchangeResponse response_obj; hypre_ProcListElements send_proc_obj; HYPRE_Int *send_info = NULL; hypre_MPI_Status status1; HYPRE_Int count, tag1 = 112, tag2 = 223; HYPRE_Int start; #endif hypre_MPI_Comm_size(comm, &num_procs); hypre_MPI_Comm_rank(comm, &my_id); #ifdef HYPRE_NO_GLOBAL_PARTITION local_size = hypre_ParVectorLastIndex(par_v) - hypre_ParVectorFirstIndex(par_v) + 1; /* determine procs which hold data of par_v and store ids in used_procs */ /* we need to do an exchange data for this. If I own row then I will contact processor 0 with the endpoint of my local range */ if (local_size > 0) { num_contacts = 1; contact_proc_list[0] = 0; contact_send_buf[0] = hypre_ParVectorLastIndex(par_v); contact_send_buf_starts[0] = 0; contact_send_buf_starts[1] = 1; } else { num_contacts = 0; contact_send_buf_starts[0] = 0; contact_send_buf_starts[1] = 0; } /*build the response object*/ /*send_proc_obj will be for saving info from contacts */ send_proc_obj.length = 0; send_proc_obj.storage_length = 10; send_proc_obj.id = hypre_CTAlloc(HYPRE_Int, send_proc_obj.storage_length); send_proc_obj.vec_starts = hypre_CTAlloc(HYPRE_Int, send_proc_obj.storage_length + 1); send_proc_obj.vec_starts[0] = 0; send_proc_obj.element_storage_length = 10; send_proc_obj.elements = hypre_CTAlloc(HYPRE_Int, send_proc_obj.element_storage_length); max_response_size = 0; /* each response is null */ response_obj.fill_response = hypre_FillResponseParToVectorAll; response_obj.data1 = NULL; response_obj.data2 = &send_proc_obj; /*this is where we keep info from contacts*/ hypre_DataExchangeList(num_contacts, contact_proc_list, contact_send_buf, contact_send_buf_starts, sizeof(HYPRE_Int), sizeof(HYPRE_Int), &response_obj, max_response_size, 1, comm, (void**) &response_recv_buf, &response_recv_buf_starts); /* now processor 0 should have a list of ranges for processors that have rows - these are in send_proc_obj - it needs to create the new list of processors and also an array of vec starts - and send to those who own row*/ if (my_id) { if (local_size) { /* look for a message from processor 0 */ hypre_MPI_Probe(0, tag1, comm, &status1); hypre_MPI_Get_count(&status1, HYPRE_MPI_INT, &count); send_info = hypre_CTAlloc(HYPRE_Int, count); hypre_MPI_Recv(send_info, count, HYPRE_MPI_INT, 0, tag1, comm, &status1); /* now unpack */ num_types = send_info[0]; used_procs = hypre_CTAlloc(HYPRE_Int, num_types); new_vec_starts = hypre_CTAlloc(HYPRE_Int, num_types+1); for (i=1; i<= num_types; i++) { used_procs[i-1] = send_info[i]; } for (i=num_types+1; i< count; i++) { new_vec_starts[i-num_types-1] = send_info[i] ; } } else /* clean up and exit */ { hypre_TFree(send_proc_obj.vec_starts); hypre_TFree(send_proc_obj.id); hypre_TFree(send_proc_obj.elements); if(response_recv_buf) hypre_TFree(response_recv_buf); if(response_recv_buf_starts) hypre_TFree(response_recv_buf_starts); return NULL; } } else /* my_id ==0 */ { num_types = send_proc_obj.length; used_procs = hypre_CTAlloc(HYPRE_Int, num_types); new_vec_starts = hypre_CTAlloc(HYPRE_Int, num_types+1); new_vec_starts[0] = 0; for (i=0; i< num_types; i++) { used_procs[i] = send_proc_obj.id[i]; new_vec_starts[i+1] = send_proc_obj.elements[i]+1; } qsort0(used_procs, 0, num_types-1); qsort0(new_vec_starts, 0, num_types); /*now we need to put into an array to send */ count = 2*num_types+2; send_info = hypre_CTAlloc(HYPRE_Int, count); send_info[0] = num_types; for (i=1; i<= num_types; i++) { send_info[i] = used_procs[i-1]; } for (i=num_types+1; i< count; i++) { send_info[i] = new_vec_starts[i-num_types-1]; } requests = hypre_CTAlloc(hypre_MPI_Request, num_types); status = hypre_CTAlloc(hypre_MPI_Status, num_types); /* don't send to myself - these are sorted so my id would be first*/ start = 0; if (used_procs[0] == 0) { start = 1; } for (i=start; i < num_types; i++) { hypre_MPI_Isend(send_info, count, HYPRE_MPI_INT, used_procs[i], tag1, comm, &requests[i-start]); } hypre_MPI_Waitall(num_types-start, requests, status); hypre_TFree(status); hypre_TFree(requests); } /* clean up */ hypre_TFree(send_proc_obj.vec_starts); hypre_TFree(send_proc_obj.id); hypre_TFree(send_proc_obj.elements); hypre_TFree(send_info); if(response_recv_buf) hypre_TFree(response_recv_buf); if(response_recv_buf_starts) hypre_TFree(response_recv_buf_starts); /* now proc 0 can exit if it has no rows */ if (!local_size) { hypre_TFree(used_procs); hypre_TFree(new_vec_starts); return NULL; } /* everyone left has rows and knows: new_vec_starts, num_types, and used_procs */ /* this vector should be rather small */ local_data = hypre_VectorData(local_vector); vector = hypre_SeqVectorCreate(global_size); hypre_VectorNumVectors(vector) = num_vectors; hypre_SeqVectorInitialize(vector); vector_data = hypre_VectorData(vector); num_requests = 2*num_types; requests = hypre_CTAlloc(hypre_MPI_Request, num_requests); status = hypre_CTAlloc(hypre_MPI_Status, num_requests); /* initialize data exchange among used_procs and generate vector - here we send to ourself also*/ j = 0; for (i = 0; i < num_types; i++) { proc_id = used_procs[i]; vec_len = new_vec_starts[i+1] - new_vec_starts[i]; hypre_MPI_Irecv(&vector_data[new_vec_starts[i]], num_vectors*vec_len, hypre_MPI_DOUBLE, proc_id, tag2, comm, &requests[j++]); } for (i = 0; i < num_types; i++) { hypre_MPI_Isend(local_data, num_vectors*local_size, hypre_MPI_DOUBLE, used_procs[i], tag2, comm, &requests[j++]); } hypre_MPI_Waitall(num_requests, requests, status); if (num_requests) { hypre_TFree(requests); hypre_TFree(status); hypre_TFree(used_procs); } hypre_TFree(new_vec_starts); #else local_size = vec_starts[my_id+1] - vec_starts[my_id]; /* if my_id contains no data, return NULL */ if (!local_size) return NULL; local_data = hypre_VectorData(local_vector); vector = hypre_SeqVectorCreate(global_size); hypre_VectorNumVectors(vector) = num_vectors; hypre_SeqVectorInitialize(vector); vector_data = hypre_VectorData(vector); /* determine procs which hold data of par_v and store ids in used_procs */ num_types = -1; for (i=0; i < num_procs; i++) if (vec_starts[i+1]-vec_starts[i]) num_types++; num_requests = 2*num_types; used_procs = hypre_CTAlloc(HYPRE_Int, num_types); j = 0; for (i=0; i < num_procs; i++) if (vec_starts[i+1]-vec_starts[i] && i-my_id) used_procs[j++] = i; requests = hypre_CTAlloc(hypre_MPI_Request, num_requests); status = hypre_CTAlloc(hypre_MPI_Status, num_requests); /* initialize data exchange among used_procs and generate vector */ j = 0; for (i = 0; i < num_types; i++) { proc_id = used_procs[i]; vec_len = vec_starts[proc_id+1] - vec_starts[proc_id]; hypre_MPI_Irecv(&vector_data[vec_starts[proc_id]], num_vectors*vec_len, hypre_MPI_DOUBLE, proc_id, 0, comm, &requests[j++]); } for (i = 0; i < num_types; i++) { hypre_MPI_Isend(local_data, num_vectors*local_size, hypre_MPI_DOUBLE, used_procs[i], 0, comm, &requests[j++]); } for (i=0; i < num_vectors*local_size; i++) vector_data[vec_starts[my_id]+i] = local_data[i]; hypre_MPI_Waitall(num_requests, requests, status); if (num_requests) { hypre_TFree(used_procs); hypre_TFree(requests); hypre_TFree(status); } #endif return vector; }
hypre_ParVector * hypre_VectorToParVector (MPI_Comm comm, hypre_Vector *v, HYPRE_Int *vec_starts) { HYPRE_Int global_size; HYPRE_Int local_size; HYPRE_Int num_vectors; HYPRE_Int num_procs, my_id; HYPRE_Int global_vecstride, vecstride, idxstride; hypre_ParVector *par_vector; hypre_Vector *local_vector; double *v_data; double *local_data; hypre_MPI_Request *requests; hypre_MPI_Status *status, status0; HYPRE_Int i, j, k, p; hypre_MPI_Comm_size(comm,&num_procs); hypre_MPI_Comm_rank(comm,&my_id); if (my_id == 0) { global_size = hypre_VectorSize(v); v_data = hypre_VectorData(v); num_vectors = hypre_VectorNumVectors(v); /* for multivectors */ global_vecstride = hypre_VectorVectorStride(v); } hypre_MPI_Bcast(&global_size,1,HYPRE_MPI_INT,0,comm); hypre_MPI_Bcast(&num_vectors,1,HYPRE_MPI_INT,0,comm); hypre_MPI_Bcast(&global_vecstride,1,HYPRE_MPI_INT,0,comm); if ( num_vectors==1 ) par_vector = hypre_ParVectorCreate(comm, global_size, vec_starts); else par_vector = hypre_ParMultiVectorCreate(comm, global_size, vec_starts, num_vectors); vec_starts = hypre_ParVectorPartitioning(par_vector); local_size = vec_starts[my_id+1] - vec_starts[my_id]; hypre_ParVectorInitialize(par_vector); local_vector = hypre_ParVectorLocalVector(par_vector); local_data = hypre_VectorData(local_vector); vecstride = hypre_VectorVectorStride(local_vector); idxstride = hypre_VectorIndexStride(local_vector); hypre_assert( idxstride==1 ); /* <<< so far only the only implemented multivector StorageMethod is 0 <<< */ if (my_id == 0) { requests = hypre_CTAlloc(hypre_MPI_Request,num_vectors*(num_procs-1)); status = hypre_CTAlloc(hypre_MPI_Status,num_vectors*(num_procs-1)); k = 0; for ( p=1; p<num_procs; p++) for ( j=0; j<num_vectors; ++j ) { hypre_MPI_Isend( &v_data[vec_starts[p]]+j*global_vecstride, (vec_starts[p+1]-vec_starts[p]), hypre_MPI_DOUBLE, p, 0, comm, &requests[k++] ); } if ( num_vectors==1 ) { for (i=0; i < local_size; i++) local_data[i] = v_data[i]; } else for ( j=0; j<num_vectors; ++j ) { for (i=0; i < local_size; i++) local_data[i+j*vecstride] = v_data[i+j*global_vecstride]; } hypre_MPI_Waitall(num_procs-1,requests, status); hypre_TFree(requests); hypre_TFree(status); } else { for ( j=0; j<num_vectors; ++j ) hypre_MPI_Recv( local_data+j*vecstride, local_size, hypre_MPI_DOUBLE, 0, 0, comm,&status0 ); } return par_vector; }
HYPRE_Int main( HYPRE_Int argc, char *argv[] ) { hypre_ParVector *vector1; hypre_ParVector *vector2; hypre_ParVector *tmp_vector; HYPRE_Int num_procs, my_id; HYPRE_Int global_size = 20; HYPRE_Int local_size; HYPRE_Int first_index; HYPRE_Int i; HYPRE_Int *partitioning; HYPRE_Complex prod; HYPRE_Complex *data, *data2; hypre_Vector *vector; hypre_Vector *local_vector; hypre_Vector *local_vector2; /* Initialize MPI */ hypre_MPI_Init(&argc, &argv); hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs ); hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id ); hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs); partitioning = NULL; vector1 = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning); partitioning = hypre_ParVectorPartitioning(vector1); hypre_ParVectorInitialize(vector1); local_vector = hypre_ParVectorLocalVector(vector1); data = hypre_VectorData(local_vector); local_size = hypre_VectorSize(local_vector); first_index = partitioning[my_id]; for (i=0; i < local_size; i++) data[i] = first_index+i; /* hypre_ParVectorPrint(vector1, "Vector"); */ local_vector2 = hypre_SeqVectorCreate(global_size); hypre_SeqVectorInitialize(local_vector2); data2 = hypre_VectorData(local_vector2); for (i=0; i < global_size; i++) data2[i] = i+1; /* partitioning = hypre_CTAlloc(HYPRE_Int,4); partitioning[0] = 0; partitioning[1] = 10; partitioning[2] = 10; partitioning[3] = 20; */ vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning); hypre_ParVectorSetPartitioningOwner(vector2,0); hypre_ParVectorPrint(vector2, "Convert"); vector = hypre_ParVectorToVectorAll(vector2); /*----------------------------------------------------------- * Copy the vector into tmp_vector *-----------------------------------------------------------*/ tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert"); /* tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning); hypre_ParVectorSetPartitioningOwner(tmp_vector,0); hypre_ParVectorInitialize(tmp_vector); hypre_ParVectorCopy(vector1, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Copy"); */ /*----------------------------------------------------------- * Scale tmp_vector *-----------------------------------------------------------*/ hypre_ParVectorScale(2.0, tmp_vector); /* hypre_ParVectorPrint(tmp_vector,"Scale"); */ /*----------------------------------------------------------- * Do an Axpy (2*vector - vector) = vector *-----------------------------------------------------------*/ hypre_ParVectorAxpy(-1.0, vector1, tmp_vector); /* hypre_ParVectorPrint(tmp_vector,"Axpy"); */ /*----------------------------------------------------------- * Do an inner product vector* tmp_vector *-----------------------------------------------------------*/ prod = hypre_ParVectorInnerProd(vector1, tmp_vector); hypre_printf (" prod: %8.2f \n", prod); /*----------------------------------------------------------- * Finalize things *-----------------------------------------------------------*/ hypre_ParVectorDestroy(vector1); hypre_ParVectorDestroy(vector2); hypre_ParVectorDestroy(tmp_vector); hypre_SeqVectorDestroy(local_vector2); if (vector) hypre_SeqVectorDestroy(vector); /* Finalize MPI */ hypre_MPI_Finalize(); return 0; }
hypre_ParMultiVector * hypre_ParMultiVectorTempRead(MPI_Comm comm, const char *fileName) /* ***** temporary implementation ****** */ { HYPRE_Int i, n, id; double * dest; double * src; HYPRE_Int count; HYPRE_Int retcode; char temp_string[128]; hypre_ParMultiVector * x; hypre_ParVector * temp_vec; /* calculate the number of files */ hypre_MPI_Comm_rank( comm, &id ); n = 0; do { hypre_sprintf( temp_string, "test -f %s.%d.%d", fileName, n, id ); if (!(retcode=system(temp_string))) /* zero retcode mean file exists */ n++; } while (!retcode); if ( n == 0 ) return NULL; /* now read the first vector using hypre_ParVectorRead into temp_vec */ hypre_sprintf(temp_string,"%s.%d",fileName,0); temp_vec = hypre_ParVectorRead(comm, temp_string); /* this vector WON'T own partitioning */ hypre_ParVectorSetPartitioningOwner(temp_vec,0); /* now create multivector using temp_vec as a sample */ x = hypre_ParMultiVectorCreate(hypre_ParVectorComm(temp_vec), hypre_ParVectorGlobalSize(temp_vec),hypre_ParVectorPartitioning(temp_vec),n); /* this vector WILL own the partitioning */ hypre_ParMultiVectorSetPartitioningOwner(x,1); hypre_ParMultiVectorInitialize(x); /* read data from first and all other vectors into "x" */ i = 0; do { /* copy data from current vector */ dest = x->local_vector->data + i*(x->local_vector->size); src = temp_vec->local_vector->data; count = temp_vec->local_vector->size; memcpy(dest,src, count*sizeof(double)); /* destroy current vector */ hypre_ParVectorDestroy(temp_vec); /* read the data to new current vector, if there are more vectors to read */ if (i<n-1) { hypre_sprintf(temp_string,"%s.%d",fileName,i+1); temp_vec = hypre_ParVectorRead(comm, temp_string); } } while (++i<n); return x; }
hypre_ParVector * hypre_ParVectorCreateFromBlock( MPI_Comm comm, HYPRE_Int p_global_size, HYPRE_Int *p_partitioning, HYPRE_Int block_size) { hypre_ParVector *vector; HYPRE_Int num_procs, my_id, i; HYPRE_Int global_size; HYPRE_Int *new_partitioning; /* need to create a new partitioning - son't want to write over what is passed in */ global_size = p_global_size*block_size; vector = hypre_CTAlloc(hypre_ParVector, 1); hypre_MPI_Comm_rank(comm,&my_id); hypre_MPI_Comm_size(comm,&num_procs); if (!p_partitioning) { #ifdef HYPRE_NO_GLOBAL_PARTITION hypre_GenerateLocalPartitioning(global_size, num_procs, my_id, &new_partitioning); #else hypre_GeneratePartitioning(global_size, num_procs, &new_partitioning); #endif } else /* adjust for block_size */ { #ifdef HYPRE_NO_GLOBAL_PARTITION new_partitioning = hypre_CTAlloc(HYPRE_Int, 2); for(i = 0; i < 2; i++) { new_partitioning[i] = p_partitioning[i]*block_size; } #else new_partitioning = hypre_CTAlloc(HYPRE_Int, num_procs + 1); for(i = 0; i < num_procs + 1; i++) { new_partitioning[i] = p_partitioning[i]*block_size; } #endif } hypre_ParVectorComm(vector) = comm; hypre_ParVectorGlobalSize(vector) = global_size; #ifdef HYPRE_NO_GLOBAL_PARTITION hypre_ParVectorFirstIndex(vector) = new_partitioning[0]; hypre_ParVectorLastIndex(vector) = new_partitioning[1]-1; hypre_ParVectorPartitioning(vector) = new_partitioning; hypre_ParVectorLocalVector(vector) = hypre_SeqVectorCreate(new_partitioning[1]-new_partitioning[0]); #else hypre_ParVectorFirstIndex(vector) = new_partitioning[my_id]; hypre_ParVectorLastIndex(vector) = new_partitioning[my_id+1] -1; hypre_ParVectorPartitioning(vector) = new_partitioning; hypre_ParVectorLocalVector(vector) = hypre_SeqVectorCreate(new_partitioning[my_id+1]-new_partitioning[my_id]); #endif /* set defaults */ hypre_ParVectorOwnsData(vector) = 1; hypre_ParVectorOwnsPartitioning(vector) = 1; return vector; }