HYPRE_Int main( HYPRE_Int argc, char *argv[] ) { hypre_ParVector *vector1; hypre_ParVector *vector2; hypre_ParVector *tmp_vector; HYPRE_Int num_procs, my_id; HYPRE_Int global_size = 20; HYPRE_Int local_size; HYPRE_Int first_index; HYPRE_Int num_vectors, vecstride, idxstride; HYPRE_Int i, j; HYPRE_Int *partitioning; double prod; double *data, *data2; hypre_Vector *vector; hypre_Vector *local_vector; hypre_Vector *local_vector2; /* Initialize MPI */ hypre_MPI_Init(&argc, &argv); hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs ); hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id ); hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs); partitioning = NULL; num_vectors = 3; vector1 = hypre_ParMultiVectorCreate ( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors ); partitioning = hypre_ParVectorPartitioning(vector1); hypre_ParVectorInitialize(vector1); local_vector = hypre_ParVectorLocalVector(vector1); data = hypre_VectorData(local_vector); local_size = hypre_VectorSize(local_vector); vecstride = hypre_VectorVectorStride(local_vector); idxstride = hypre_VectorIndexStride(local_vector); first_index = partitioning[my_id]; hypre_printf("vecstride=%i idxstride=%i local_size=%i num_vectors=%i", vecstride, idxstride, local_size, num_vectors ); for (j=0; j<num_vectors; ++j ) for (i=0; i < local_size; i++) data[ j*vecstride + i*idxstride ] = first_index+i + 100*j; hypre_ParVectorPrint(vector1, "Vector"); local_vector2 = hypre_SeqMultiVectorCreate( global_size, num_vectors ); hypre_SeqVectorInitialize(local_vector2); data2 = hypre_VectorData(local_vector2); vecstride = hypre_VectorVectorStride(local_vector2); idxstride = hypre_VectorIndexStride(local_vector2); for (j=0; j<num_vectors; ++j ) for (i=0; i < global_size; i++) data2[ j*vecstride + i*idxstride ] = i + 100*j; /* partitioning = hypre_CTAlloc(HYPRE_Int,4); partitioning[0] = 0; partitioning[1] = 10; partitioning[2] = 10; partitioning[3] = 20; */ partitioning = hypre_CTAlloc(HYPRE_Int,1+num_procs); hypre_GeneratePartitioning( global_size, num_procs, &partitioning ); vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning); hypre_ParVectorSetPartitioningOwner(vector2,0); hypre_ParVectorPrint(vector2, "Convert"); vector = hypre_ParVectorToVectorAll(vector2); /*----------------------------------------------------------- * Copy the vector into tmp_vector *-----------------------------------------------------------*/ /* Read doesn't work for multivectors yet... tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert");*/ tmp_vector = hypre_ParMultiVectorCreate ( hypre_MPI_COMM_WORLD, global_size, partitioning, num_vectors ); hypre_ParVectorInitialize( tmp_vector ); hypre_ParVectorCopy( vector2, tmp_vector ); /* tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning); hypre_ParVectorSetPartitioningOwner(tmp_vector,0); hypre_ParVectorInitialize(tmp_vector); hypre_ParVectorCopy(vector1, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Copy"); */ /*----------------------------------------------------------- * Scale tmp_vector *-----------------------------------------------------------*/ hypre_ParVectorScale(2.0, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Scale"); /*----------------------------------------------------------- * Do an Axpy (2*vector - vector) = vector *-----------------------------------------------------------*/ hypre_ParVectorAxpy(-1.0, vector1, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Axpy"); /*----------------------------------------------------------- * Do an inner product vector* tmp_vector *-----------------------------------------------------------*/ prod = hypre_ParVectorInnerProd(vector1, tmp_vector); hypre_printf (" prod: %8.2f \n", prod); /*----------------------------------------------------------- * Finalize things *-----------------------------------------------------------*/ hypre_ParVectorDestroy(vector1); hypre_ParVectorDestroy(vector2); hypre_ParVectorDestroy(tmp_vector); hypre_SeqVectorDestroy(local_vector2); if (vector) hypre_SeqVectorDestroy(vector); /* Finalize MPI */ hypre_MPI_Finalize(); return 0; }
HYPRE_Int hypre_AMESetup(void *esolver) { HYPRE_Int ne, *edge_bc; hypre_AMEData *ame_data = esolver; hypre_AMSData *ams_data = ame_data -> precond; if (ams_data -> beta_is_zero) { ame_data -> t1 = hypre_ParVectorInDomainOf(ams_data -> G); ame_data -> t2 = hypre_ParVectorInDomainOf(ams_data -> G); } else { ame_data -> t1 = ams_data -> r1; ame_data -> t2 = ams_data -> g1; } ame_data -> t3 = ams_data -> r0; /* Eliminate boundary conditions in G = [Gii, Gib; 0, Gbb], i.e., compute [Gii, 0; 0, 0] */ { HYPRE_Int i, j, k, nv; HYPRE_Int *offd_edge_bc; hypre_ParCSRMatrix *Gt; nv = hypre_ParCSRMatrixNumCols(ams_data -> G); ne = hypre_ParCSRMatrixNumRows(ams_data -> G); edge_bc = hypre_TAlloc(HYPRE_Int, ne); for (i = 0; i < ne; i++) edge_bc[i] = 0; /* Find boundary (eliminated) edges */ { hypre_CSRMatrix *Ad = hypre_ParCSRMatrixDiag(ams_data -> A); HYPRE_Int *AdI = hypre_CSRMatrixI(Ad); HYPRE_Int *AdJ = hypre_CSRMatrixJ(Ad); HYPRE_Real *AdA = hypre_CSRMatrixData(Ad); hypre_CSRMatrix *Ao = hypre_ParCSRMatrixOffd(ams_data -> A); HYPRE_Int *AoI = hypre_CSRMatrixI(Ao); HYPRE_Real *AoA = hypre_CSRMatrixData(Ao); HYPRE_Real l1_norm; /* A row (edge) is boundary if its off-diag l1 norm is less than eps */ HYPRE_Real eps = DBL_EPSILON * 1e+4; for (i = 0; i < ne; i++) { l1_norm = 0.0; for (j = AdI[i]; j < AdI[i+1]; j++) if (AdJ[j] != i) l1_norm += fabs(AdA[j]); if (AoI) for (j = AoI[i]; j < AoI[i+1]; j++) l1_norm += fabs(AoA[j]); if (l1_norm < eps) edge_bc[i] = 1; } } hypre_ParCSRMatrixTranspose(ams_data -> G, &Gt, 1); /* Use a Matvec communication to find which of the edges connected to local vertices are on the boundary */ { hypre_ParCSRCommHandle *comm_handle; hypre_ParCSRCommPkg *comm_pkg; HYPRE_Int num_sends, *int_buf_data; HYPRE_Int index, start; offd_edge_bc = hypre_CTAlloc(HYPRE_Int, hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(Gt))); hypre_MatvecCommPkgCreate(Gt); comm_pkg = hypre_ParCSRMatrixCommPkg(Gt); num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg); int_buf_data = hypre_CTAlloc(HYPRE_Int, hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends)); index = 0; for (i = 0; i < num_sends; i++) { start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i); for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++) { k = hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j); int_buf_data[index++] = edge_bc[k]; } } comm_handle = hypre_ParCSRCommHandleCreate(11, comm_pkg, int_buf_data, offd_edge_bc); hypre_ParCSRCommHandleDestroy(comm_handle); hypre_TFree(int_buf_data); } /* Eliminate boundary vertex entries in G^t */ { hypre_CSRMatrix *Gtd = hypre_ParCSRMatrixDiag(Gt); HYPRE_Int *GtdI = hypre_CSRMatrixI(Gtd); HYPRE_Int *GtdJ = hypre_CSRMatrixJ(Gtd); HYPRE_Real *GtdA = hypre_CSRMatrixData(Gtd); hypre_CSRMatrix *Gto = hypre_ParCSRMatrixOffd(Gt); HYPRE_Int *GtoI = hypre_CSRMatrixI(Gto); HYPRE_Int *GtoJ = hypre_CSRMatrixJ(Gto); HYPRE_Real *GtoA = hypre_CSRMatrixData(Gto); HYPRE_Int bdr; for (i = 0; i < nv; i++) { bdr = 0; /* A vertex is boundary if it belongs to a boundary edge */ for (j = GtdI[i]; j < GtdI[i+1]; j++) if (edge_bc[GtdJ[j]]) { bdr = 1; break; } if (!bdr && GtoI) for (j = GtoI[i]; j < GtoI[i+1]; j++) if (offd_edge_bc[GtoJ[j]]) { bdr = 1; break; } if (bdr) { for (j = GtdI[i]; j < GtdI[i+1]; j++) /* if (!edge_bc[GtdJ[j]]) */ GtdA[j] = 0.0; if (GtoI) for (j = GtoI[i]; j < GtoI[i+1]; j++) /* if (!offd_edge_bc[GtoJ[j]]) */ GtoA[j] = 0.0; } } } hypre_ParCSRMatrixTranspose(Gt, &ame_data -> G, 1); hypre_ParCSRMatrixDestroy(Gt); hypre_TFree(offd_edge_bc); } /* Compute G^t M G */ { if (!hypre_ParCSRMatrixCommPkg(ame_data -> G)) hypre_MatvecCommPkgCreate(ame_data -> G); if (!hypre_ParCSRMatrixCommPkg(ame_data -> M)) hypre_MatvecCommPkgCreate(ame_data -> M); hypre_BoomerAMGBuildCoarseOperator(ame_data -> G, ame_data -> M, ame_data -> G, &ame_data -> A_G); hypre_ParCSRMatrixFixZeroRows(ame_data -> A_G); } /* Create AMG preconditioner and PCG-AMG solver for G^tMG */ { HYPRE_BoomerAMGCreate(&ame_data -> B1_G); HYPRE_BoomerAMGSetCoarsenType(ame_data -> B1_G, ams_data -> B_G_coarsen_type); HYPRE_BoomerAMGSetAggNumLevels(ame_data -> B1_G, ams_data -> B_G_agg_levels); HYPRE_BoomerAMGSetRelaxType(ame_data -> B1_G, ams_data -> B_G_relax_type); HYPRE_BoomerAMGSetNumSweeps(ame_data -> B1_G, 1); HYPRE_BoomerAMGSetMaxLevels(ame_data -> B1_G, 25); HYPRE_BoomerAMGSetTol(ame_data -> B1_G, 0.0); HYPRE_BoomerAMGSetMaxIter(ame_data -> B1_G, 1); HYPRE_BoomerAMGSetStrongThreshold(ame_data -> B1_G, ams_data -> B_G_theta); /* don't use exact solve on the coarsest level (matrix may be singular) */ HYPRE_BoomerAMGSetCycleRelaxType(ame_data -> B1_G, ams_data -> B_G_relax_type, 3); HYPRE_ParCSRPCGCreate(hypre_ParCSRMatrixComm(ame_data->A_G), &ame_data -> B2_G); HYPRE_PCGSetPrintLevel(ame_data -> B2_G, 0); HYPRE_PCGSetTol(ame_data -> B2_G, 1e-12); HYPRE_PCGSetMaxIter(ame_data -> B2_G, 20); HYPRE_PCGSetPrecond(ame_data -> B2_G, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, ame_data -> B1_G); HYPRE_ParCSRPCGSetup(ame_data -> B2_G, (HYPRE_ParCSRMatrix)ame_data->A_G, (HYPRE_ParVector)ame_data->t1, (HYPRE_ParVector)ame_data->t2); } /* Setup LOBPCG */ { HYPRE_Int seed = 75; mv_InterfaceInterpreter* interpreter; mv_MultiVectorPtr eigenvectors; ame_data -> interpreter = hypre_CTAlloc(mv_InterfaceInterpreter,1); interpreter = (mv_InterfaceInterpreter*) ame_data -> interpreter; HYPRE_ParCSRSetupInterpreter(interpreter); ame_data -> eigenvalues = hypre_CTAlloc(HYPRE_Real, ame_data -> block_size); ame_data -> eigenvectors = mv_MultiVectorCreateFromSampleVector(interpreter, ame_data -> block_size, ame_data -> t3); eigenvectors = (mv_MultiVectorPtr) ame_data -> eigenvectors; mv_MultiVectorSetRandom (eigenvectors, seed); /* Make the initial vectors discretely divergence free */ { HYPRE_Int i, j; HYPRE_Real *data; mv_TempMultiVector* tmp = mv_MultiVectorGetData(eigenvectors); HYPRE_ParVector *v = (HYPRE_ParVector*)(tmp -> vector); hypre_ParVector *vi; for (i = 0; i < ame_data -> block_size; i++) { vi = (hypre_ParVector*) v[i]; data = hypre_VectorData(hypre_ParVectorLocalVector(vi)); for (j = 0; j < ne; j++) if (edge_bc[j]) data[j] = 0.0; hypre_AMEDiscrDivFreeComponent(esolver, vi); } } } hypre_TFree(edge_bc); return hypre_error_flag; }
/* ---------------------------------------------------------------------- * set_element * * Sets single element in hypre vector by accessing its raw block. * Probably not the most efficient way to set the entire vector. * --------------------------------------------------------------------*/ void set_element(N_Vector X, long int i, realtype val) { hypre_ParVector *Xvec = N_VGetVector_ParHyp(X); realtype *Xdata = hypre_VectorData(hypre_ParVectorLocalVector(Xvec)); Xdata[i] = val; }
HYPRE_Int hypre_ParCSRMatrixMatvecT( double alpha, hypre_ParCSRMatrix *A, hypre_ParVector *x, double beta, hypre_ParVector *y ) { hypre_ParCSRCommHandle **comm_handle; hypre_ParCSRCommPkg *comm_pkg = hypre_ParCSRMatrixCommPkg(A); hypre_CSRMatrix *diag = hypre_ParCSRMatrixDiag(A); hypre_CSRMatrix *offd = hypre_ParCSRMatrixOffd(A); hypre_Vector *x_local = hypre_ParVectorLocalVector(x); hypre_Vector *y_local = hypre_ParVectorLocalVector(y); hypre_Vector *y_tmp; HYPRE_Int vecstride = hypre_VectorVectorStride( y_local ); HYPRE_Int idxstride = hypre_VectorIndexStride( y_local ); double *y_tmp_data, **y_buf_data; double *y_local_data = hypre_VectorData(y_local); HYPRE_Int num_rows = hypre_ParCSRMatrixGlobalNumRows(A); HYPRE_Int num_cols = hypre_ParCSRMatrixGlobalNumCols(A); HYPRE_Int num_cols_offd = hypre_CSRMatrixNumCols(offd); HYPRE_Int x_size = hypre_ParVectorGlobalSize(x); HYPRE_Int y_size = hypre_ParVectorGlobalSize(y); HYPRE_Int num_vectors = hypre_VectorNumVectors(y_local); HYPRE_Int i, j, jv, index, start, num_sends; HYPRE_Int ierr = 0; /*--------------------------------------------------------------------- * Check for size compatibility. MatvecT returns ierr = 1 if * length of X doesn't equal the number of rows of A, * ierr = 2 if the length of Y doesn't equal the number of * columns of A, and ierr = 3 if both are true. * * Because temporary vectors are often used in MatvecT, none of * these conditions terminates processing, and the ierr flag * is informational only. *--------------------------------------------------------------------*/ if (num_rows != x_size) ierr = 1; if (num_cols != y_size) ierr = 2; if (num_rows != x_size && num_cols != y_size) ierr = 3; /*----------------------------------------------------------------------- *-----------------------------------------------------------------------*/ comm_handle = hypre_CTAlloc(hypre_ParCSRCommHandle*,num_vectors); if ( num_vectors==1 ) { y_tmp = hypre_SeqVectorCreate(num_cols_offd); } else { y_tmp = hypre_SeqMultiVectorCreate(num_cols_offd,num_vectors); } hypre_SeqVectorInitialize(y_tmp); /*--------------------------------------------------------------------- * If there exists no CommPkg for A, a CommPkg is generated using * equally load balanced partitionings *--------------------------------------------------------------------*/ if (!comm_pkg) { hypre_MatvecCommPkgCreate(A); comm_pkg = hypre_ParCSRMatrixCommPkg(A); } num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg); y_buf_data = hypre_CTAlloc( double*, num_vectors ); for ( jv=0; jv<num_vectors; ++jv ) y_buf_data[jv] = hypre_CTAlloc(double, hypre_ParCSRCommPkgSendMapStart (comm_pkg, num_sends)); y_tmp_data = hypre_VectorData(y_tmp); y_local_data = hypre_VectorData(y_local); hypre_assert( idxstride==1 ); /* >>> only 'column' storage of multivectors implemented so far */ if (num_cols_offd) hypre_CSRMatrixMatvecT(alpha, offd, x_local, 0.0, y_tmp); for ( jv=0; jv<num_vectors; ++jv ) { /* >>> this is where we assume multivectors are 'column' storage */ comm_handle[jv] = hypre_ParCSRCommHandleCreate ( 2, comm_pkg, &(y_tmp_data[jv*num_cols_offd]), y_buf_data[jv] ); } hypre_CSRMatrixMatvecT(alpha, diag, x_local, beta, y_local); for ( jv=0; jv<num_vectors; ++jv ) { hypre_ParCSRCommHandleDestroy(comm_handle[jv]); comm_handle[jv] = NULL; } hypre_TFree(comm_handle); if ( num_vectors==1 ) { index = 0; for (i = 0; i < num_sends; i++) { start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i); for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++) y_local_data[hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j)] += y_buf_data[0][index++]; } } else for ( jv=0; jv<num_vectors; ++jv ) { index = 0; for (i = 0; i < num_sends; i++) { start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i); for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++) y_local_data[ jv*vecstride + idxstride*hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j) ] += y_buf_data[jv][index++]; } } hypre_SeqVectorDestroy(y_tmp); y_tmp = NULL; for ( jv=0; jv<num_vectors; ++jv ) hypre_TFree(y_buf_data[jv]); hypre_TFree(y_buf_data); return ierr; }
HYPRE_Int hypre_ParVectorPrintIJ( hypre_ParVector *vector, HYPRE_Int base_j, const char *filename ) { MPI_Comm comm; HYPRE_Int global_size; HYPRE_Int *partitioning; double *local_data; HYPRE_Int myid, num_procs, i, j, part0; char new_filename[255]; FILE *file; if (!vector) { hypre_error_in_arg(1); return hypre_error_flag; } comm = hypre_ParVectorComm(vector); global_size = hypre_ParVectorGlobalSize(vector); partitioning = hypre_ParVectorPartitioning(vector); /* multivector code not written yet >>> */ hypre_assert( hypre_ParVectorNumVectors(vector) == 1 ); if ( hypre_ParVectorNumVectors(vector) != 1 ) hypre_error_in_arg(1); hypre_MPI_Comm_rank(comm, &myid); hypre_MPI_Comm_size(comm, &num_procs); hypre_sprintf(new_filename,"%s.%05d", filename, myid); if ((file = fopen(new_filename, "w")) == NULL) { hypre_printf("Error: can't open output file %s\n", new_filename); hypre_error_in_arg(3); return hypre_error_flag; } local_data = hypre_VectorData(hypre_ParVectorLocalVector(vector)); hypre_fprintf(file, "%d \n", global_size); #ifdef HYPRE_NO_GLOBAL_PARTITION for (i=0; i <= 2; i++) #else for (i=0; i <= num_procs; i++) #endif { hypre_fprintf(file, "%d \n", partitioning[i] + base_j); } #ifdef HYPRE_NO_GLOBAL_PARTITION part0 = partitioning[0]; for (j = part0; j < partitioning[1]; j++) #else part0 = partitioning[myid]; for (j = part0; j < partitioning[myid+1]; j++) #endif { hypre_fprintf(file, "%d %.14e\n", j + base_j, local_data[j-part0]); } fclose(file); return hypre_error_flag; }
hypre_ParVector * hypre_ParVectorCreateFromBlock( MPI_Comm comm, HYPRE_Int p_global_size, HYPRE_Int *p_partitioning, HYPRE_Int block_size) { hypre_ParVector *vector; HYPRE_Int num_procs, my_id, i; HYPRE_Int global_size; HYPRE_Int *new_partitioning; /* need to create a new partitioning - son't want to write over what is passed in */ global_size = p_global_size*block_size; vector = hypre_CTAlloc(hypre_ParVector, 1); hypre_MPI_Comm_rank(comm,&my_id); hypre_MPI_Comm_size(comm,&num_procs); if (!p_partitioning) { #ifdef HYPRE_NO_GLOBAL_PARTITION hypre_GenerateLocalPartitioning(global_size, num_procs, my_id, &new_partitioning); #else hypre_GeneratePartitioning(global_size, num_procs, &new_partitioning); #endif } else /* adjust for block_size */ { #ifdef HYPRE_NO_GLOBAL_PARTITION new_partitioning = hypre_CTAlloc(HYPRE_Int, 2); for(i = 0; i < 2; i++) { new_partitioning[i] = p_partitioning[i]*block_size; } #else new_partitioning = hypre_CTAlloc(HYPRE_Int, num_procs + 1); for(i = 0; i < num_procs + 1; i++) { new_partitioning[i] = p_partitioning[i]*block_size; } #endif } hypre_ParVectorComm(vector) = comm; hypre_ParVectorGlobalSize(vector) = global_size; #ifdef HYPRE_NO_GLOBAL_PARTITION hypre_ParVectorFirstIndex(vector) = new_partitioning[0]; hypre_ParVectorLastIndex(vector) = new_partitioning[1]-1; hypre_ParVectorPartitioning(vector) = new_partitioning; hypre_ParVectorLocalVector(vector) = hypre_SeqVectorCreate(new_partitioning[1]-new_partitioning[0]); #else hypre_ParVectorFirstIndex(vector) = new_partitioning[my_id]; hypre_ParVectorLastIndex(vector) = new_partitioning[my_id+1] -1; hypre_ParVectorPartitioning(vector) = new_partitioning; hypre_ParVectorLocalVector(vector) = hypre_SeqVectorCreate(new_partitioning[my_id+1]-new_partitioning[my_id]); #endif /* set defaults */ hypre_ParVectorOwnsData(vector) = 1; hypre_ParVectorOwnsPartitioning(vector) = 1; return vector; }
HYPRE_Int main( HYPRE_Int argc, char *argv[] ) { hypre_ParVector *vector1; hypre_ParVector *vector2; hypre_ParVector *tmp_vector; HYPRE_Int num_procs, my_id; HYPRE_Int global_size = 20; HYPRE_Int local_size; HYPRE_Int first_index; HYPRE_Int i; HYPRE_Int *partitioning; HYPRE_Complex prod; HYPRE_Complex *data, *data2; hypre_Vector *vector; hypre_Vector *local_vector; hypre_Vector *local_vector2; /* Initialize MPI */ hypre_MPI_Init(&argc, &argv); hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs ); hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id ); hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs); partitioning = NULL; vector1 = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning); partitioning = hypre_ParVectorPartitioning(vector1); hypre_ParVectorInitialize(vector1); local_vector = hypre_ParVectorLocalVector(vector1); data = hypre_VectorData(local_vector); local_size = hypre_VectorSize(local_vector); first_index = partitioning[my_id]; for (i=0; i < local_size; i++) data[i] = first_index+i; /* hypre_ParVectorPrint(vector1, "Vector"); */ local_vector2 = hypre_SeqVectorCreate(global_size); hypre_SeqVectorInitialize(local_vector2); data2 = hypre_VectorData(local_vector2); for (i=0; i < global_size; i++) data2[i] = i+1; /* partitioning = hypre_CTAlloc(HYPRE_Int,4); partitioning[0] = 0; partitioning[1] = 10; partitioning[2] = 10; partitioning[3] = 20; */ vector2 = hypre_VectorToParVector(hypre_MPI_COMM_WORLD,local_vector2,partitioning); hypre_ParVectorSetPartitioningOwner(vector2,0); hypre_ParVectorPrint(vector2, "Convert"); vector = hypre_ParVectorToVectorAll(vector2); /*----------------------------------------------------------- * Copy the vector into tmp_vector *-----------------------------------------------------------*/ tmp_vector = hypre_ParVectorRead(hypre_MPI_COMM_WORLD, "Convert"); /* tmp_vector = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_size,partitioning); hypre_ParVectorSetPartitioningOwner(tmp_vector,0); hypre_ParVectorInitialize(tmp_vector); hypre_ParVectorCopy(vector1, tmp_vector); hypre_ParVectorPrint(tmp_vector,"Copy"); */ /*----------------------------------------------------------- * Scale tmp_vector *-----------------------------------------------------------*/ hypre_ParVectorScale(2.0, tmp_vector); /* hypre_ParVectorPrint(tmp_vector,"Scale"); */ /*----------------------------------------------------------- * Do an Axpy (2*vector - vector) = vector *-----------------------------------------------------------*/ hypre_ParVectorAxpy(-1.0, vector1, tmp_vector); /* hypre_ParVectorPrint(tmp_vector,"Axpy"); */ /*----------------------------------------------------------- * Do an inner product vector* tmp_vector *-----------------------------------------------------------*/ prod = hypre_ParVectorInnerProd(vector1, tmp_vector); hypre_printf (" prod: %8.2f \n", prod); /*----------------------------------------------------------- * Finalize things *-----------------------------------------------------------*/ hypre_ParVectorDestroy(vector1); hypre_ParVectorDestroy(vector2); hypre_ParVectorDestroy(tmp_vector); hypre_SeqVectorDestroy(local_vector2); if (vector) hypre_SeqVectorDestroy(vector); /* Finalize MPI */ hypre_MPI_Finalize(); return 0; }
hypre_ParVector * hypre_VectorToParVector (MPI_Comm comm, hypre_Vector *v, HYPRE_Int *vec_starts) { HYPRE_Int global_size; HYPRE_Int local_size; HYPRE_Int num_vectors; HYPRE_Int num_procs, my_id; HYPRE_Int global_vecstride, vecstride, idxstride; hypre_ParVector *par_vector; hypre_Vector *local_vector; double *v_data; double *local_data; hypre_MPI_Request *requests; hypre_MPI_Status *status, status0; HYPRE_Int i, j, k, p; hypre_MPI_Comm_size(comm,&num_procs); hypre_MPI_Comm_rank(comm,&my_id); if (my_id == 0) { global_size = hypre_VectorSize(v); v_data = hypre_VectorData(v); num_vectors = hypre_VectorNumVectors(v); /* for multivectors */ global_vecstride = hypre_VectorVectorStride(v); } hypre_MPI_Bcast(&global_size,1,HYPRE_MPI_INT,0,comm); hypre_MPI_Bcast(&num_vectors,1,HYPRE_MPI_INT,0,comm); hypre_MPI_Bcast(&global_vecstride,1,HYPRE_MPI_INT,0,comm); if ( num_vectors==1 ) par_vector = hypre_ParVectorCreate(comm, global_size, vec_starts); else par_vector = hypre_ParMultiVectorCreate(comm, global_size, vec_starts, num_vectors); vec_starts = hypre_ParVectorPartitioning(par_vector); local_size = vec_starts[my_id+1] - vec_starts[my_id]; hypre_ParVectorInitialize(par_vector); local_vector = hypre_ParVectorLocalVector(par_vector); local_data = hypre_VectorData(local_vector); vecstride = hypre_VectorVectorStride(local_vector); idxstride = hypre_VectorIndexStride(local_vector); hypre_assert( idxstride==1 ); /* <<< so far only the only implemented multivector StorageMethod is 0 <<< */ if (my_id == 0) { requests = hypre_CTAlloc(hypre_MPI_Request,num_vectors*(num_procs-1)); status = hypre_CTAlloc(hypre_MPI_Status,num_vectors*(num_procs-1)); k = 0; for ( p=1; p<num_procs; p++) for ( j=0; j<num_vectors; ++j ) { hypre_MPI_Isend( &v_data[vec_starts[p]]+j*global_vecstride, (vec_starts[p+1]-vec_starts[p]), hypre_MPI_DOUBLE, p, 0, comm, &requests[k++] ); } if ( num_vectors==1 ) { for (i=0; i < local_size; i++) local_data[i] = v_data[i]; } else for ( j=0; j<num_vectors; ++j ) { for (i=0; i < local_size; i++) local_data[i+j*vecstride] = v_data[i+j*global_vecstride]; } hypre_MPI_Waitall(num_procs-1,requests, status); hypre_TFree(requests); hypre_TFree(status); } else { for ( j=0; j<num_vectors; ++j ) hypre_MPI_Recv( local_data+j*vecstride, local_size, hypre_MPI_DOUBLE, 0, 0, comm,&status0 ); } return par_vector; }
HYPRE_Int main( HYPRE_Int argc, char *argv[] ) { hypre_CSRMatrix *matrix; hypre_CSRMatrix *matrix1; hypre_ParCSRMatrix *par_matrix; hypre_Vector *x_local; hypre_Vector *y_local; hypre_Vector *y2_local; hypre_ParVector *x; hypre_ParVector *x2; hypre_ParVector *y; hypre_ParVector *y2; HYPRE_Int vecstride_x, idxstride_x, vecstride_y, idxstride_y; HYPRE_Int num_procs, my_id; HYPRE_Int local_size; HYPRE_Int num_vectors; HYPRE_Int global_num_rows, global_num_cols; HYPRE_Int first_index; HYPRE_Int i, j, ierr=0; double *data, *data2; HYPRE_Int *row_starts, *col_starts; char file_name[80]; /* Initialize MPI */ hypre_MPI_Init(&argc, &argv); hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs); hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &my_id); hypre_printf(" my_id: %d num_procs: %d\n", my_id, num_procs); if (my_id == 0) { matrix = hypre_CSRMatrixRead("input"); hypre_printf(" read input\n"); } row_starts = NULL; col_starts = NULL; par_matrix = hypre_CSRMatrixToParCSRMatrix(hypre_MPI_COMM_WORLD, matrix, row_starts, col_starts); hypre_printf(" converted\n"); matrix1 = hypre_ParCSRMatrixToCSRMatrixAll(par_matrix); hypre_sprintf(file_name,"matrix1.%d",my_id); if (matrix1) hypre_CSRMatrixPrint(matrix1, file_name); hypre_ParCSRMatrixPrint(par_matrix,"matrix"); hypre_ParCSRMatrixPrintIJ(par_matrix,0,0,"matrixIJ"); par_matrix = hypre_ParCSRMatrixRead(hypre_MPI_COMM_WORLD,"matrix"); global_num_cols = hypre_ParCSRMatrixGlobalNumCols(par_matrix); hypre_printf(" global_num_cols %d\n", global_num_cols); global_num_rows = hypre_ParCSRMatrixGlobalNumRows(par_matrix); col_starts = hypre_ParCSRMatrixColStarts(par_matrix); first_index = col_starts[my_id]; local_size = col_starts[my_id+1] - first_index; num_vectors = 3; x = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_cols, col_starts, num_vectors ); hypre_ParVectorSetPartitioningOwner(x,0); hypre_ParVectorInitialize(x); x_local = hypre_ParVectorLocalVector(x); data = hypre_VectorData(x_local); vecstride_x = hypre_VectorVectorStride(x_local); idxstride_x = hypre_VectorIndexStride(x_local); for ( j=0; j<num_vectors; ++j ) for (i=0; i < local_size; i++) data[i*idxstride_x + j*vecstride_x] = first_index+i+1 + 100*j; x2 = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_cols, col_starts, num_vectors ); hypre_ParVectorSetPartitioningOwner(x2,0); hypre_ParVectorInitialize(x2); hypre_ParVectorSetConstantValues(x2,2.0); row_starts = hypre_ParCSRMatrixRowStarts(par_matrix); first_index = row_starts[my_id]; local_size = row_starts[my_id+1] - first_index; y = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_rows, row_starts, num_vectors ); hypre_ParVectorSetPartitioningOwner(y,0); hypre_ParVectorInitialize(y); y_local = hypre_ParVectorLocalVector(y); y2 = hypre_ParMultiVectorCreate( hypre_MPI_COMM_WORLD, global_num_rows, row_starts, num_vectors ); hypre_ParVectorSetPartitioningOwner(y2,0); hypre_ParVectorInitialize(y2); y2_local = hypre_ParVectorLocalVector(y2); data2 = hypre_VectorData(y2_local); vecstride_y = hypre_VectorVectorStride(y2_local); idxstride_y = hypre_VectorIndexStride(y2_local); for ( j=0; j<num_vectors; ++j ) for (i=0; i < local_size; i++) data2[i*idxstride_y+j*vecstride_y] = first_index+i+1 + 100*j; hypre_ParVectorSetConstantValues(y,1.0); hypre_printf(" initialized vectors, first_index=%i\n", first_index); hypre_ParVectorPrint(x, "vectorx"); hypre_ParVectorPrint(y, "vectory"); hypre_MatvecCommPkgCreate(par_matrix); hypre_ParCSRMatrixMatvec ( 1.0, par_matrix, x, 1.0, y); hypre_printf(" did matvec\n"); hypre_ParVectorPrint(y, "result"); ierr = hypre_ParCSRMatrixMatvecT ( 1.0, par_matrix, y2, 1.0, x2); hypre_printf(" did matvecT %d\n", ierr); hypre_ParVectorPrint(x2, "transp"); hypre_ParCSRMatrixDestroy(par_matrix); hypre_ParVectorDestroy(x); hypre_ParVectorDestroy(x2); hypre_ParVectorDestroy(y); hypre_ParVectorDestroy(y2); if (my_id == 0) hypre_CSRMatrixDestroy(matrix); if (matrix1) hypre_CSRMatrixDestroy(matrix1); /* Finalize MPI */ hypre_MPI_Finalize(); return 0; }
HYPRE_Int hypre_seqAMGCycle( hypre_ParAMGData *amg_data, HYPRE_Int p_level, hypre_ParVector **Par_F_array, hypre_ParVector **Par_U_array ) { hypre_ParVector *Aux_U; hypre_ParVector *Aux_F; /* Local variables */ HYPRE_Int Solve_err_flag = 0; HYPRE_Int n; HYPRE_Int i; hypre_Vector *u_local; HYPRE_Real *u_data; HYPRE_Int first_index; /* Acquire seq data */ MPI_Comm new_comm = hypre_ParAMGDataNewComm(amg_data); HYPRE_Solver coarse_solver = hypre_ParAMGDataCoarseSolver(amg_data); hypre_ParCSRMatrix *A_coarse = hypre_ParAMGDataACoarse(amg_data); hypre_ParVector *F_coarse = hypre_ParAMGDataFCoarse(amg_data); hypre_ParVector *U_coarse = hypre_ParAMGDataUCoarse(amg_data); HYPRE_Int redundant = hypre_ParAMGDataRedundant(amg_data); Aux_U = Par_U_array[p_level]; Aux_F = Par_F_array[p_level]; first_index = hypre_ParVectorFirstIndex(Aux_U); u_local = hypre_ParVectorLocalVector(Aux_U); u_data = hypre_VectorData(u_local); n = hypre_VectorSize(u_local); /*if (A_coarse)*/ if (hypre_ParAMGDataParticipate(amg_data)) { HYPRE_Real *f_data; hypre_Vector *f_local; hypre_Vector *tmp_vec; HYPRE_Int nf; HYPRE_Int local_info; HYPRE_Real *recv_buf = NULL; HYPRE_Int *displs = NULL; HYPRE_Int *info = NULL; HYPRE_Int size; HYPRE_Int new_num_procs, my_id; hypre_MPI_Comm_size(new_comm, &new_num_procs); hypre_MPI_Comm_rank(new_comm, &my_id); f_local = hypre_ParVectorLocalVector(Aux_F); f_data = hypre_VectorData(f_local); nf = hypre_VectorSize(f_local); /* first f */ info = hypre_CTAlloc(HYPRE_Int, new_num_procs); local_info = nf; if (redundant) hypre_MPI_Allgather(&local_info, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, new_comm); else hypre_MPI_Gather(&local_info, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, 0, new_comm); if (redundant || my_id ==0) { displs = hypre_CTAlloc(HYPRE_Int, new_num_procs+1); displs[0] = 0; for (i=1; i < new_num_procs+1; i++) displs[i] = displs[i-1]+info[i-1]; size = displs[new_num_procs]; if (F_coarse) { tmp_vec = hypre_ParVectorLocalVector(F_coarse); recv_buf = hypre_VectorData(tmp_vec); } } if (redundant) hypre_MPI_Allgatherv ( f_data, nf, HYPRE_MPI_REAL, recv_buf, info, displs, HYPRE_MPI_REAL, new_comm ); else hypre_MPI_Gatherv ( f_data, nf, HYPRE_MPI_REAL, recv_buf, info, displs, HYPRE_MPI_REAL, 0, new_comm ); if (redundant || my_id ==0) { tmp_vec = hypre_ParVectorLocalVector(U_coarse); recv_buf = hypre_VectorData(tmp_vec); } /*then u */ if (redundant) { hypre_MPI_Allgatherv ( u_data, n, HYPRE_MPI_REAL, recv_buf, info, displs, HYPRE_MPI_REAL, new_comm ); hypre_TFree(displs); hypre_TFree(info); } else hypre_MPI_Gatherv ( u_data, n, HYPRE_MPI_REAL, recv_buf, info, displs, HYPRE_MPI_REAL, 0, new_comm ); /* clean up */ if (redundant || my_id ==0) { hypre_BoomerAMGSolve(coarse_solver, A_coarse, F_coarse, U_coarse); } /*copy my part of U to parallel vector */ if (redundant) { HYPRE_Real *local_data; local_data = hypre_VectorData(hypre_ParVectorLocalVector(U_coarse)); for (i = 0; i < n; i++) { u_data[i] = local_data[first_index+i]; } } else { HYPRE_Real *local_data=NULL; if (my_id == 0) local_data = hypre_VectorData(hypre_ParVectorLocalVector(U_coarse)); hypre_MPI_Scatterv ( local_data, info, displs, HYPRE_MPI_REAL, u_data, n, HYPRE_MPI_REAL, 0, new_comm ); /*if (my_id == 0) local_data = hypre_VectorData(hypre_ParVectorLocalVector(F_coarse)); hypre_MPI_Scatterv ( local_data, info, displs, HYPRE_MPI_REAL, f_data, n, HYPRE_MPI_REAL, 0, new_comm );*/ if (my_id == 0) hypre_TFree(displs); hypre_TFree(info); } } return(Solve_err_flag); }
/****************************************************************************** * * hypre_IJVectorAddToValuesPar * * adds to a potentially noncontiguous set of IJVectorPar components * *****************************************************************************/ HYPRE_Int hypre_IJVectorAddToValuesPar(hypre_IJVector *vector, HYPRE_Int num_values, const HYPRE_Int *indices, const double *values ) { HYPRE_Int my_id; HYPRE_Int i, j, vec_start, vec_stop; double *data; HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector); HYPRE_Int *IJpartitioning = hypre_IJVectorPartitioning(vector); hypre_ParVector *par_vector = hypre_IJVectorObject(vector); hypre_AuxParVector *aux_vector = hypre_IJVectorTranslator(vector); MPI_Comm comm = hypre_IJVectorComm(vector); hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector); /* If no components are to be retrieved, perform no checking and return */ if (num_values < 1) return 0; hypre_MPI_Comm_rank(comm, &my_id); /* If par_vector == NULL or partitioning == NULL or local_vector == NULL let user know of catastrophe and exit */ if (!par_vector) { if (print_level) { hypre_printf("par_vector == NULL -- "); hypre_printf("hypre_IJVectorAddToValuesPar\n"); hypre_printf("**** Vector storage is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } if (!IJpartitioning) { if (print_level) { hypre_printf("IJpartitioning == NULL -- "); hypre_printf("hypre_IJVectorAddToValuesPar\n"); hypre_printf("**** IJVector partitioning is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } if (!local_vector) { if (print_level) { hypre_printf("local_vector == NULL -- "); hypre_printf("hypre_IJVectorAddToValuesPar\n"); hypre_printf("**** Vector local data is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } #ifdef HYPRE_NO_GLOBAL_PARTITION vec_start = IJpartitioning[0]; vec_stop = IJpartitioning[1]-1; #else vec_start = IJpartitioning[my_id]; vec_stop = IJpartitioning[my_id+1]-1; #endif if (vec_start > vec_stop) { if (print_level) { hypre_printf("vec_start > vec_stop -- "); hypre_printf("hypre_IJVectorAddToValuesPar\n"); hypre_printf("**** This vector partitioning should not occur ****\n"); } hypre_error_in_arg(1); } /* Determine whether indices points to local indices only, and if not, store indices and values into auxiliary vector structure If indices == NULL, assume that num_values components are to be set in a block starting at vec_start. NOTE: If indices == NULL off processor values are ignored!!! */ /* if (indices) { for (i = 0; i < num_values; i++) { ierr += (indices[i] < vec_start); ierr += (indices[i] >= vec_stop); } } if (ierr) { hypre_printf("indices beyond local range -- "); hypre_printf("hypre_IJVectorAddToValuesPar\n"); hypre_printf("**** Indices specified are unusable ****\n"); exit(1); } */ data = hypre_VectorData(local_vector); if (indices) { HYPRE_Int current_num_elmts = hypre_AuxParVectorCurrentNumElmts(aux_vector); HYPRE_Int max_off_proc_elmts = hypre_AuxParVectorMaxOffProcElmts(aux_vector); HYPRE_Int *off_proc_i = hypre_AuxParVectorOffProcI(aux_vector); double *off_proc_data = hypre_AuxParVectorOffProcData(aux_vector); for (j = 0; j < num_values; j++) { i = indices[j]; if (i < vec_start || i > vec_stop) { /* if elements outside processor boundaries, store in off processor stash */ if (!max_off_proc_elmts) { max_off_proc_elmts = 100; hypre_AuxParVectorMaxOffProcElmts(aux_vector) = max_off_proc_elmts; hypre_AuxParVectorOffProcI(aux_vector) = hypre_CTAlloc(HYPRE_Int,max_off_proc_elmts); hypre_AuxParVectorOffProcData(aux_vector) = hypre_CTAlloc(double,max_off_proc_elmts); off_proc_i = hypre_AuxParVectorOffProcI(aux_vector); off_proc_data = hypre_AuxParVectorOffProcData(aux_vector); } else if (current_num_elmts + 1 > max_off_proc_elmts) { max_off_proc_elmts += 10; off_proc_i = hypre_TReAlloc(off_proc_i,HYPRE_Int,max_off_proc_elmts); off_proc_data = hypre_TReAlloc(off_proc_data,double, max_off_proc_elmts); hypre_AuxParVectorMaxOffProcElmts(aux_vector) = max_off_proc_elmts; hypre_AuxParVectorOffProcI(aux_vector) = off_proc_i; hypre_AuxParVectorOffProcData(aux_vector) = off_proc_data; } off_proc_i[current_num_elmts] = i; off_proc_data[current_num_elmts++] = values[j]; hypre_AuxParVectorCurrentNumElmts(aux_vector)=current_num_elmts; }
/****************************************************************************** * * hypre_IJVectorSetValuesPar * * sets a potentially noncontiguous set of components of an IJVectorPar * *****************************************************************************/ HYPRE_Int hypre_IJVectorSetValuesPar(hypre_IJVector *vector, HYPRE_Int num_values, const HYPRE_Int *indices, const double *values ) { HYPRE_Int my_id; HYPRE_Int i, j, vec_start, vec_stop; double *data; HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector); HYPRE_Int *IJpartitioning = hypre_IJVectorPartitioning(vector); hypre_ParVector *par_vector = hypre_IJVectorObject(vector); hypre_AuxParVector *aux_vector = hypre_IJVectorTranslator(vector); MPI_Comm comm = hypre_IJVectorComm(vector); hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector); /* If no components are to be set, perform no checking and return */ if (num_values < 1) return 0; hypre_MPI_Comm_rank(comm, &my_id); /* If par_vector == NULL or partitioning == NULL or local_vector == NULL let user know of catastrophe and exit */ if (!par_vector) { if (print_level) { hypre_printf("par_vector == NULL -- "); hypre_printf("hypre_IJVectorSetValuesPar\n"); hypre_printf("**** Vector storage is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } if (!IJpartitioning) { if (print_level) { hypre_printf("IJpartitioning == NULL -- "); hypre_printf("hypre_IJVectorSetValuesPar\n"); hypre_printf("**** IJVector partitioning is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } if (!local_vector) { if (print_level) { hypre_printf("local_vector == NULL -- "); hypre_printf("hypre_IJVectorSetValuesPar\n"); hypre_printf("**** Vector local data is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } #ifdef HYPRE_NO_GLOBAL_PARTITION vec_start = IJpartitioning[0]; vec_stop = IJpartitioning[1]-1; #else vec_start = IJpartitioning[my_id]; vec_stop = IJpartitioning[my_id+1]-1; #endif if (vec_start > vec_stop) { if (print_level) { hypre_printf("vec_start > vec_stop -- "); hypre_printf("hypre_IJVectorSetValuesPar\n"); hypre_printf("**** This vector partitioning should not occur ****\n"); } hypre_error_in_arg(1); } /* Determine whether indices points to local indices only, and if not, store indices and values into auxiliary vector structure If indices == NULL, assume that num_values components are to be set in a block starting at vec_start. NOTE: If indices == NULL off processor values are ignored!!! */ data = hypre_VectorData(local_vector); if (indices) { HYPRE_Int current_num_elmts = hypre_AuxParVectorCurrentNumElmts(aux_vector); /*HYPRE_Int max_off_proc_elmts = hypre_AuxParVectorMaxOffProcElmts(aux_vector);*/ HYPRE_Int *off_proc_i = hypre_AuxParVectorOffProcI(aux_vector); /*double *off_proc_data = hypre_AuxParVectorOffProcData(aux_vector);*/ HYPRE_Int cancel_indx = hypre_AuxParVectorCancelIndx(aux_vector); HYPRE_Int ii; for (j = 0; j < num_values; j++) { i = indices[j]; if (i < vec_start || i > vec_stop) { for (ii = 0; ii < current_num_elmts; ii++) { if (i == off_proc_i[ii]) { off_proc_i[ii] = -1; cancel_indx++; } } hypre_AuxParVectorCancelIndx(aux_vector) = cancel_indx; } /* if elements outside processor boundaries, search for previous occurrences and cancel them */ /* if elements outside processor boundaries, store in off processor stash */ /*if (!max_off_proc_elmts) { max_off_proc_elmts = 100; hypre_AuxParVectorMaxOffProcElmts(aux_vector) = max_off_proc_elmts; hypre_AuxParVectorOffProcI(aux_vector) = hypre_CTAlloc(HYPRE_Int,max_off_proc_elmts); hypre_AuxParVectorOffProcData(aux_vector) = hypre_CTAlloc(double,max_off_proc_elmts); off_proc_i = hypre_AuxParVectorOffProcI(aux_vector); off_proc_data = hypre_AuxParVectorOffProcData(aux_vector); } else if (current_num_elmts + 1 > max_off_proc_elmts) { max_off_proc_elmts += 10; off_proc_i = hypre_TReAlloc(off_proc_i,HYPRE_Int,max_off_proc_elmts); off_proc_data = hypre_TReAlloc(off_proc_data,double, max_off_proc_elmts); hypre_AuxParVectorMaxOffProcElmts(aux_vector) = max_off_proc_elmts; hypre_AuxParVectorOffProcI(aux_vector) = off_proc_i; hypre_AuxParVectorOffProcData(aux_vector) = off_proc_data; } off_proc_i[current_num_elmts] = i; off_proc_data[current_num_elmts++] = values[j]; hypre_AuxParVectorCurrentNumElmts(aux_vector)=current_num_elmts; }*/ else /* local values are inserted into the vector */ { i -= vec_start; data[i] = values[j]; } } } else { if (num_values > vec_stop - vec_start + 1) { if (print_level) { hypre_printf("Warning! Indices beyond local range not identified!\n "); hypre_printf("Off processor values have been ignored!\n"); } num_values = vec_stop - vec_start +1; } for (j = 0; j < num_values; j++) data[j] = values[j]; } return hypre_error_flag; }
/****************************************************************************** * * hypre_IJVectorZeroValuesPar * * zeroes all local components of an IJVectorPar * *****************************************************************************/ HYPRE_Int hypre_IJVectorZeroValuesPar(hypre_IJVector *vector) { HYPRE_Int my_id; HYPRE_Int i, vec_start, vec_stop; double *data; hypre_ParVector *par_vector = hypre_IJVectorObject(vector); MPI_Comm comm = hypre_IJVectorComm(vector); HYPRE_Int *partitioning = hypre_ParVectorPartitioning(par_vector); hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector); HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector); hypre_MPI_Comm_rank(comm, &my_id); /* If par_vector == NULL or partitioning == NULL or local_vector == NULL let user know of catastrophe and exit */ if (!par_vector) { if (print_level) { hypre_printf("par_vector == NULL -- "); hypre_printf("hypre_IJVectorZeroValuesPar\n"); hypre_printf("**** Vector storage is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } if (!partitioning) { if (print_level) { hypre_printf("partitioning == NULL -- "); hypre_printf("hypre_IJVectorZeroValuesPar\n"); hypre_printf("**** Vector partitioning is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } if (!local_vector) { if (print_level) { hypre_printf("local_vector == NULL -- "); hypre_printf("hypre_IJVectorZeroValuesPar\n"); hypre_printf("**** Vector local data is either unallocated or orphaned ****\n"); } hypre_error_in_arg(1); } #ifdef HYPRE_NO_GLOBAL_PARTITION vec_start = partitioning[0]; vec_stop = partitioning[1]; #else vec_start = partitioning[my_id]; vec_stop = partitioning[my_id+1]; #endif if (vec_start > vec_stop) { if (print_level) { hypre_printf("vec_start > vec_stop -- "); hypre_printf("hypre_IJVectorZeroValuesPar\n"); hypre_printf("**** This vector partitioning should not occur ****\n"); } hypre_error_in_arg(1); } data = hypre_VectorData( local_vector ); for (i = 0; i < vec_stop - vec_start; i++) data[i] = 0.; return hypre_error_flag; }
HYPRE_Complex hypre_ParVectorLocalSumElts( hypre_ParVector * vector ) { return hypre_VectorSumElts( hypre_ParVectorLocalVector(vector) ); }
HYPRE_Int hypre_CreateLambda(void *amg_vdata) { hypre_ParAMGData *amg_data = amg_vdata; /* Data Structure variables */ MPI_Comm comm; hypre_ParCSRMatrix **A_array; hypre_ParVector **F_array; hypre_ParVector **U_array; hypre_ParCSRMatrix *A_tmp; hypre_ParCSRMatrix *Lambda; hypre_CSRMatrix *L_diag; hypre_CSRMatrix *L_offd; hypre_CSRMatrix *A_tmp_diag; hypre_CSRMatrix *A_tmp_offd; hypre_ParVector *Xtilde; hypre_ParVector *Rtilde; hypre_Vector *Xtilde_local; hypre_Vector *Rtilde_local; hypre_ParCSRCommPkg *comm_pkg; hypre_ParCSRCommPkg *L_comm_pkg = NULL; hypre_ParCSRCommHandle *comm_handle; HYPRE_Real *L_diag_data; HYPRE_Real *L_offd_data; HYPRE_Real *buf_data = NULL; HYPRE_Real *tmp_data; HYPRE_Real *x_data; HYPRE_Real *r_data; HYPRE_Real *l1_norms; HYPRE_Real *A_tmp_diag_data; HYPRE_Real *A_tmp_offd_data; HYPRE_Real *D_data = NULL; HYPRE_Real *D_data_offd = NULL; HYPRE_Int *L_diag_i; HYPRE_Int *L_diag_j; HYPRE_Int *L_offd_i; HYPRE_Int *L_offd_j; HYPRE_Int *A_tmp_diag_i; HYPRE_Int *A_tmp_offd_i; HYPRE_Int *A_tmp_diag_j; HYPRE_Int *A_tmp_offd_j; HYPRE_Int *L_recv_ptr = NULL; HYPRE_Int *L_send_ptr = NULL; HYPRE_Int *L_recv_procs = NULL; HYPRE_Int *L_send_procs = NULL; HYPRE_Int *L_send_map_elmts = NULL; HYPRE_Int *recv_procs; HYPRE_Int *send_procs; HYPRE_Int *send_map_elmts; HYPRE_Int *send_map_starts; HYPRE_Int *recv_vec_starts; HYPRE_Int *all_send_procs = NULL; HYPRE_Int *all_recv_procs = NULL; HYPRE_Int *remap = NULL; HYPRE_Int *level_start; HYPRE_Int addlvl; HYPRE_Int additive; HYPRE_Int mult_additive; HYPRE_Int num_levels; HYPRE_Int num_add_lvls; HYPRE_Int num_procs; HYPRE_Int num_sends, num_recvs; HYPRE_Int num_sends_L = 0; HYPRE_Int num_recvs_L = 0; HYPRE_Int send_data_L = 0; HYPRE_Int num_rows_L = 0; HYPRE_Int num_rows_tmp = 0; HYPRE_Int num_cols_offd_L = 0; HYPRE_Int num_cols_offd = 0; HYPRE_Int level, i, j, k; HYPRE_Int this_proc, cnt, cnt_diag, cnt_offd; HYPRE_Int cnt_recv, cnt_send, cnt_row, row_start; HYPRE_Int start_diag, start_offd, indx, cnt_map; HYPRE_Int start, j_indx, index, cnt_level; HYPRE_Int max_sends, max_recvs; /* Local variables */ HYPRE_Int Solve_err_flag = 0; HYPRE_Int num_threads; HYPRE_Int num_nonzeros_diag; HYPRE_Int num_nonzeros_offd; HYPRE_Real **l1_norms_ptr = NULL; HYPRE_Real *relax_weight = NULL; HYPRE_Real relax_type; /* Acquire data and allocate storage */ num_threads = hypre_NumThreads(); A_array = hypre_ParAMGDataAArray(amg_data); F_array = hypre_ParAMGDataFArray(amg_data); U_array = hypre_ParAMGDataUArray(amg_data); additive = hypre_ParAMGDataAdditive(amg_data); mult_additive = hypre_ParAMGDataMultAdditive(amg_data); num_levels = hypre_ParAMGDataNumLevels(amg_data); relax_weight = hypre_ParAMGDataRelaxWeight(amg_data); relax_type = hypre_ParAMGDataGridRelaxType(amg_data)[1]; comm = hypre_ParCSRMatrixComm(A_array[0]); hypre_MPI_Comm_size(comm,&num_procs); l1_norms_ptr = hypre_ParAMGDataL1Norms(amg_data); addlvl = hypre_max(additive, mult_additive); num_add_lvls = num_levels+1-addlvl; level_start = hypre_CTAlloc(HYPRE_Int, num_add_lvls+1); send_data_L = 0; num_rows_L = 0; num_cols_offd_L = 0; num_nonzeros_diag = 0; num_nonzeros_offd = 0; level_start[0] = 0; cnt = 1; max_sends = 0; max_recvs = 0; for (i=addlvl; i < num_levels; i++) { A_tmp = A_array[i]; A_tmp_diag = hypre_ParCSRMatrixDiag(A_tmp); A_tmp_offd = hypre_ParCSRMatrixOffd(A_tmp); A_tmp_diag_i = hypre_CSRMatrixI(A_tmp_diag); A_tmp_offd_i = hypre_CSRMatrixI(A_tmp_offd); num_rows_tmp = hypre_CSRMatrixNumRows(A_tmp_diag); num_cols_offd = hypre_CSRMatrixNumCols(A_tmp_offd); num_rows_L += num_rows_tmp; level_start[cnt] = level_start[cnt-1] + num_rows_tmp; cnt++; num_cols_offd_L += num_cols_offd; num_nonzeros_diag += A_tmp_diag_i[num_rows_tmp]; num_nonzeros_offd += A_tmp_offd_i[num_rows_tmp]; comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp); if (comm_pkg) { num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg); max_sends += num_sends; if (num_sends) send_data_L += hypre_ParCSRCommPkgSendMapStart(comm_pkg,num_sends); max_recvs += hypre_ParCSRCommPkgNumRecvs(comm_pkg); } } if (max_sends >= num_procs ||max_recvs >= num_procs) { max_sends = num_procs; max_recvs = num_procs; } if (max_sends) all_send_procs = hypre_CTAlloc(HYPRE_Int, max_sends); if (max_recvs) all_recv_procs = hypre_CTAlloc(HYPRE_Int, max_recvs); cnt_send = 0; cnt_recv = 0; if (max_sends || max_recvs) { if (max_sends < num_procs && max_recvs < num_procs) { for (i=addlvl; i < num_levels; i++) { A_tmp = A_array[i]; comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp); if (comm_pkg) { num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg); num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg); send_procs = hypre_ParCSRCommPkgSendProcs(comm_pkg); recv_procs = hypre_ParCSRCommPkgRecvProcs(comm_pkg); for (j = 0; j < num_sends; j++) all_send_procs[cnt_send++] = send_procs[j]; for (j = 0; j < num_recvs; j++) all_recv_procs[cnt_recv++] = recv_procs[j]; } } if (max_sends) { qsort0(all_send_procs, 0, max_sends-1); num_sends_L = 1; this_proc = all_send_procs[0]; for (i=1; i < max_sends; i++) { if (all_send_procs[i] > this_proc) { this_proc = all_send_procs[i]; all_send_procs[num_sends_L++] = this_proc; } } L_send_procs = hypre_CTAlloc(HYPRE_Int, num_sends_L); for (j=0; j < num_sends_L; j++) L_send_procs[j] = all_send_procs[j]; hypre_TFree(all_send_procs); } if (max_recvs) { qsort0(all_recv_procs, 0, max_recvs-1); num_recvs_L = 1; this_proc = all_recv_procs[0]; for (i=1; i < max_recvs; i++) { if (all_recv_procs[i] > this_proc) { this_proc = all_recv_procs[i]; all_recv_procs[num_recvs_L++] = this_proc; } } L_recv_procs = hypre_CTAlloc(HYPRE_Int, num_recvs_L); for (j=0; j < num_recvs_L; j++) L_recv_procs[j] = all_recv_procs[j]; hypre_TFree(all_recv_procs); } L_recv_ptr = hypre_CTAlloc(HYPRE_Int, num_recvs_L+1); L_send_ptr = hypre_CTAlloc(HYPRE_Int, num_sends_L+1); for (i=addlvl; i < num_levels; i++) { A_tmp = A_array[i]; comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp); if (comm_pkg) { num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg); num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg); send_procs = hypre_ParCSRCommPkgSendProcs(comm_pkg); recv_procs = hypre_ParCSRCommPkgRecvProcs(comm_pkg); send_map_starts = hypre_ParCSRCommPkgSendMapStarts(comm_pkg); recv_vec_starts = hypre_ParCSRCommPkgRecvVecStarts(comm_pkg); } else { num_sends = 0; num_recvs = 0; } for (k = 0; k < num_sends; k++) { this_proc = hypre_BinarySearch(L_send_procs,send_procs[k],num_sends_L); L_send_ptr[this_proc+1] += send_map_starts[k+1]-send_map_starts[k]; } for (k = 0; k < num_recvs; k++) { this_proc = hypre_BinarySearch(L_recv_procs,recv_procs[k],num_recvs_L); L_recv_ptr[this_proc+1] += recv_vec_starts[k+1]-recv_vec_starts[k]; } } L_recv_ptr[0] = 0; for (i=1; i < num_recvs_L; i++) L_recv_ptr[i+1] += L_recv_ptr[i]; L_send_ptr[0] = 0; for (i=1; i < num_sends_L; i++) L_send_ptr[i+1] += L_send_ptr[i]; } else { num_recvs_L = 0; num_sends_L = 0; for (i=addlvl; i < num_levels; i++) { A_tmp = A_array[i]; comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp); if (comm_pkg) { num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg); num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg); send_procs = hypre_ParCSRCommPkgSendProcs(comm_pkg); recv_procs = hypre_ParCSRCommPkgRecvProcs(comm_pkg); send_map_starts = hypre_ParCSRCommPkgSendMapStarts(comm_pkg); recv_vec_starts = hypre_ParCSRCommPkgRecvVecStarts(comm_pkg); for (j = 0; j < num_sends; j++) { this_proc = send_procs[j]; if (all_send_procs[this_proc] == 0) num_sends_L++; all_send_procs[this_proc] += send_map_starts[j+1]-send_map_starts[j]; } for (j = 0; j < num_recvs; j++) { this_proc = recv_procs[j]; if (all_recv_procs[this_proc] == 0) num_recvs_L++; all_recv_procs[this_proc] += recv_vec_starts[j+1]-recv_vec_starts[j]; } } } if (max_sends) { L_send_procs = hypre_CTAlloc(HYPRE_Int, num_sends_L); L_send_ptr = hypre_CTAlloc(HYPRE_Int, num_sends_L+1); num_sends_L = 0; for (j=0; j < num_procs; j++) { this_proc = all_send_procs[j]; if (this_proc) { L_send_procs[num_sends_L++] = j; L_send_ptr[num_sends_L] = this_proc + L_send_ptr[num_sends_L-1]; } } } if (max_recvs) { L_recv_procs = hypre_CTAlloc(HYPRE_Int, num_recvs_L); L_recv_ptr = hypre_CTAlloc(HYPRE_Int, num_recvs_L+1); num_recvs_L = 0; for (j=0; j < num_procs; j++) { this_proc = all_recv_procs[j]; if (this_proc) { L_recv_procs[num_recvs_L++] = j; L_recv_ptr[num_recvs_L] = this_proc + L_recv_ptr[num_recvs_L-1]; } } } } } if (max_sends) hypre_TFree(all_send_procs); if (max_recvs) hypre_TFree(all_recv_procs); L_diag = hypre_CSRMatrixCreate(num_rows_L, num_rows_L, num_nonzeros_diag); L_offd = hypre_CSRMatrixCreate(num_rows_L, num_cols_offd_L, num_nonzeros_offd); hypre_CSRMatrixInitialize(L_diag); hypre_CSRMatrixInitialize(L_offd); if (num_nonzeros_diag) { L_diag_data = hypre_CSRMatrixData(L_diag); L_diag_j = hypre_CSRMatrixJ(L_diag); } L_diag_i = hypre_CSRMatrixI(L_diag); if (num_nonzeros_offd) { L_offd_data = hypre_CSRMatrixData(L_offd); L_offd_j = hypre_CSRMatrixJ(L_offd); } L_offd_i = hypre_CSRMatrixI(L_offd); if (num_rows_L) D_data = hypre_CTAlloc(HYPRE_Real,num_rows_L); if (send_data_L) { L_send_map_elmts = hypre_CTAlloc(HYPRE_Int, send_data_L); buf_data = hypre_CTAlloc(HYPRE_Real,send_data_L); } if (num_cols_offd_L) { D_data_offd = hypre_CTAlloc(HYPRE_Real,num_cols_offd_L); /*L_col_map_offd = hypre_CTAlloc(HYPRE_Int, num_cols_offd_L);*/ remap = hypre_CTAlloc(HYPRE_Int, num_cols_offd_L); } Rtilde = hypre_CTAlloc(hypre_ParVector, 1); Rtilde_local = hypre_SeqVectorCreate(num_rows_L); hypre_SeqVectorInitialize(Rtilde_local); hypre_ParVectorLocalVector(Rtilde) = Rtilde_local; hypre_ParVectorOwnsData(Rtilde) = 1; Xtilde = hypre_CTAlloc(hypre_ParVector, 1); Xtilde_local = hypre_SeqVectorCreate(num_rows_L); hypre_SeqVectorInitialize(Xtilde_local); hypre_ParVectorLocalVector(Xtilde) = Xtilde_local; hypre_ParVectorOwnsData(Xtilde) = 1; x_data = hypre_VectorData(hypre_ParVectorLocalVector(Xtilde)); r_data = hypre_VectorData(hypre_ParVectorLocalVector(Rtilde)); cnt = 0; cnt_level = 0; cnt_diag = 0; cnt_offd = 0; cnt_row = 1; L_diag_i[0] = 0; L_offd_i[0] = 0; for (level=addlvl; level < num_levels; level++) { row_start = level_start[cnt_level]; if (level != 0) { tmp_data = hypre_VectorData(hypre_ParVectorLocalVector(F_array[level])); if (tmp_data) hypre_TFree(tmp_data); hypre_VectorData(hypre_ParVectorLocalVector(F_array[level])) = &r_data[row_start]; hypre_VectorOwnsData(hypre_ParVectorLocalVector(F_array[level])) = 0; tmp_data = hypre_VectorData(hypre_ParVectorLocalVector(U_array[level])); if (tmp_data) hypre_TFree(tmp_data); hypre_VectorData(hypre_ParVectorLocalVector(U_array[level])) = &x_data[row_start]; hypre_VectorOwnsData(hypre_ParVectorLocalVector(U_array[level])) = 0; } cnt_level++; start_diag = L_diag_i[cnt_row-1]; start_offd = L_offd_i[cnt_row-1]; A_tmp = A_array[level]; A_tmp_diag = hypre_ParCSRMatrixDiag(A_tmp); A_tmp_offd = hypre_ParCSRMatrixOffd(A_tmp); comm_pkg = hypre_ParCSRMatrixCommPkg(A_tmp); A_tmp_diag_i = hypre_CSRMatrixI(A_tmp_diag); A_tmp_offd_i = hypre_CSRMatrixI(A_tmp_offd); A_tmp_diag_j = hypre_CSRMatrixJ(A_tmp_diag); A_tmp_offd_j = hypre_CSRMatrixJ(A_tmp_offd); A_tmp_diag_data = hypre_CSRMatrixData(A_tmp_diag); A_tmp_offd_data = hypre_CSRMatrixData(A_tmp_offd); num_rows_tmp = hypre_CSRMatrixNumRows(A_tmp_diag); if (comm_pkg) { num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg); num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg); send_procs = hypre_ParCSRCommPkgSendProcs(comm_pkg); recv_procs = hypre_ParCSRCommPkgRecvProcs(comm_pkg); send_map_starts = hypre_ParCSRCommPkgSendMapStarts(comm_pkg); send_map_elmts = hypre_ParCSRCommPkgSendMapElmts(comm_pkg); recv_vec_starts = hypre_ParCSRCommPkgRecvVecStarts(comm_pkg); } else { num_sends = 0; num_recvs = 0; } /* Compute new combined communication package */ for (i=0; i < num_sends; i++) { this_proc = hypre_BinarySearch(L_send_procs,send_procs[i],num_sends_L); indx = L_send_ptr[this_proc]; for (j=send_map_starts[i]; j < send_map_starts[i+1]; j++) { L_send_map_elmts[indx++] = row_start + send_map_elmts[j]; } L_send_ptr[this_proc] = indx; } cnt_map = 0; for (i = 0; i < num_recvs; i++) { this_proc = hypre_BinarySearch(L_recv_procs,recv_procs[i],num_recvs_L); indx = L_recv_ptr[this_proc]; for (j=recv_vec_starts[i]; j < recv_vec_starts[i+1]; j++) { remap[cnt_map++] = indx++; } L_recv_ptr[this_proc] = indx; } /* Compute Lambda */ if (relax_type == 0) { HYPRE_Real rlx_wt = relax_weight[level]; #ifdef HYPRE_USING_OPENMP #pragma omp for private(i) HYPRE_SMP_SCHEDULE #endif for (i=0; i < num_rows_tmp; i++) { D_data[i] = rlx_wt/A_tmp_diag_data[A_tmp_diag_i[i]]; L_diag_i[cnt_row+i] = start_diag + A_tmp_diag_i[i+1]; L_offd_i[cnt_row+i] = start_offd + A_tmp_offd_i[i+1]; } } else { l1_norms = l1_norms_ptr[level]; #ifdef HYPRE_USING_OPENMP #pragma omp for private(i) HYPRE_SMP_SCHEDULE #endif for (i=0; i < num_rows_tmp; i++) { D_data[i] = 1.0/l1_norms[i]; L_diag_i[cnt_row+i] = start_diag + A_tmp_diag_i[i+1]; L_offd_i[cnt_row+i] = start_offd + A_tmp_offd_i[i+1]; } } if (num_procs > 1) { index = 0; for (i=0; i < num_sends; i++) { start = send_map_starts[i]; for (j=start; j < send_map_starts[i+1]; j++) buf_data[index++] = D_data[send_map_elmts[j]]; } comm_handle = hypre_ParCSRCommHandleCreate(1, comm_pkg, buf_data, D_data_offd); hypre_ParCSRCommHandleDestroy(comm_handle); } for (i = 0; i < num_rows_tmp; i++) { j_indx = A_tmp_diag_i[i]; L_diag_data[cnt_diag] = (2.0 - A_tmp_diag_data[j_indx]*D_data[i])*D_data[i]; L_diag_j[cnt_diag++] = i+row_start; for (j=A_tmp_diag_i[i]+1; j < A_tmp_diag_i[i+1]; j++) { j_indx = A_tmp_diag_j[j]; L_diag_data[cnt_diag] = (- A_tmp_diag_data[j]*D_data[j_indx])*D_data[i]; L_diag_j[cnt_diag++] = j_indx+row_start; } for (j=A_tmp_offd_i[i]; j < A_tmp_offd_i[i+1]; j++) { j_indx = A_tmp_offd_j[j]; L_offd_data[cnt_offd] = (- A_tmp_offd_data[j]*D_data_offd[j_indx])*D_data[i]; L_offd_j[cnt_offd++] = remap[j_indx]; } } cnt_row += num_rows_tmp; } if (L_send_ptr) { for (i=num_sends_L-1; i > 0; i--) L_send_ptr[i] = L_send_ptr[i-1]; L_send_ptr[0] = 0; } else L_send_ptr = hypre_CTAlloc(HYPRE_Int,1); if (L_recv_ptr) { for (i=num_recvs_L-1; i > 0; i--) L_recv_ptr[i] = L_recv_ptr[i-1]; L_recv_ptr[0] = 0; } else L_recv_ptr = hypre_CTAlloc(HYPRE_Int,1); L_comm_pkg = hypre_CTAlloc(hypre_ParCSRCommPkg,1); hypre_ParCSRCommPkgNumRecvs(L_comm_pkg) = num_recvs_L; hypre_ParCSRCommPkgNumSends(L_comm_pkg) = num_sends_L; hypre_ParCSRCommPkgRecvProcs(L_comm_pkg) = L_recv_procs; hypre_ParCSRCommPkgSendProcs(L_comm_pkg) = L_send_procs; hypre_ParCSRCommPkgRecvVecStarts(L_comm_pkg) = L_recv_ptr; hypre_ParCSRCommPkgSendMapStarts(L_comm_pkg) = L_send_ptr; hypre_ParCSRCommPkgSendMapElmts(L_comm_pkg) = L_send_map_elmts; hypre_ParCSRCommPkgComm(L_comm_pkg) = comm; Lambda = hypre_CTAlloc(hypre_ParCSRMatrix, 1); hypre_ParCSRMatrixDiag(Lambda) = L_diag; hypre_ParCSRMatrixOffd(Lambda) = L_offd; hypre_ParCSRMatrixCommPkg(Lambda) = L_comm_pkg; hypre_ParCSRMatrixComm(Lambda) = comm; hypre_ParCSRMatrixOwnsData(Lambda) = 1; hypre_ParAMGDataLambda(amg_data) = Lambda; hypre_ParAMGDataRtilde(amg_data) = Rtilde; hypre_ParAMGDataXtilde(amg_data) = Xtilde; hypre_TFree(D_data_offd); hypre_TFree(D_data); if (num_procs > 1) hypre_TFree(buf_data); hypre_TFree(remap); hypre_TFree(buf_data); hypre_TFree(level_start); return Solve_err_flag; }
double hypre_ParVectorLocalSumElts( hypre_ParVector * vector ) { return hypre_VectorSumElts( hypre_ParVectorLocalVector(vector) ); }
HYPRE_Int hypre_BoomerAMGAdditiveCycle( void *amg_vdata) { hypre_ParAMGData *amg_data = amg_vdata; /* Data Structure variables */ hypre_ParCSRMatrix **A_array; hypre_ParCSRMatrix **P_array; hypre_ParCSRMatrix **R_array; hypre_ParCSRMatrix *Lambda; hypre_ParVector **F_array; hypre_ParVector **U_array; hypre_ParVector *Vtemp; hypre_ParVector *Ztemp; hypre_ParVector *Xtilde, *Rtilde; HYPRE_Int **CF_marker_array; HYPRE_Int num_levels; HYPRE_Int addlvl; HYPRE_Int additive; HYPRE_Int mult_additive; HYPRE_Int simple; HYPRE_Int i, num_rows; HYPRE_Int n_global; HYPRE_Int rlx_order; /* Local variables */ HYPRE_Int Solve_err_flag = 0; HYPRE_Int level; HYPRE_Int coarse_grid; HYPRE_Int fine_grid; HYPRE_Int relax_type; HYPRE_Int rlx_down; HYPRE_Int rlx_up; HYPRE_Int *grid_relax_type; HYPRE_Real **l1_norms; HYPRE_Real alpha, beta; HYPRE_Int num_threads; HYPRE_Real *u_data; HYPRE_Real *f_data; HYPRE_Real *v_data; HYPRE_Real *l1_norms_lvl; HYPRE_Real *D_inv; HYPRE_Real *x_global; HYPRE_Real *r_global; HYPRE_Real *relax_weight; HYPRE_Real *omega; #if 0 HYPRE_Real *D_mat; HYPRE_Real *S_vec; #endif /* Acquire data and allocate storage */ num_threads = hypre_NumThreads(); A_array = hypre_ParAMGDataAArray(amg_data); F_array = hypre_ParAMGDataFArray(amg_data); U_array = hypre_ParAMGDataUArray(amg_data); P_array = hypre_ParAMGDataPArray(amg_data); R_array = hypre_ParAMGDataRArray(amg_data); CF_marker_array = hypre_ParAMGDataCFMarkerArray(amg_data); Vtemp = hypre_ParAMGDataVtemp(amg_data); Ztemp = hypre_ParAMGDataZtemp(amg_data); num_levels = hypre_ParAMGDataNumLevels(amg_data); additive = hypre_ParAMGDataAdditive(amg_data); mult_additive = hypre_ParAMGDataMultAdditive(amg_data); simple = hypre_ParAMGDataSimple(amg_data); grid_relax_type = hypre_ParAMGDataGridRelaxType(amg_data); Lambda = hypre_ParAMGDataLambda(amg_data); Xtilde = hypre_ParAMGDataXtilde(amg_data); Rtilde = hypre_ParAMGDataRtilde(amg_data); l1_norms = hypre_ParAMGDataL1Norms(amg_data); D_inv = hypre_ParAMGDataDinv(amg_data); grid_relax_type = hypre_ParAMGDataGridRelaxType(amg_data); relax_weight = hypre_ParAMGDataRelaxWeight(amg_data); omega = hypre_ParAMGDataOmega(amg_data); rlx_order = hypre_ParAMGDataRelaxOrder(amg_data); /* Initialize */ addlvl = hypre_max(additive, mult_additive); addlvl = hypre_max(addlvl, simple); Solve_err_flag = 0; /*--------------------------------------------------------------------- * Main loop of cycling --- multiplicative version --- V-cycle *--------------------------------------------------------------------*/ /* down cycle */ relax_type = grid_relax_type[1]; rlx_down = grid_relax_type[1]; rlx_up = grid_relax_type[2]; for (level = 0; level < num_levels-1; level++) { fine_grid = level; coarse_grid = level + 1; u_data = hypre_VectorData(hypre_ParVectorLocalVector(U_array[fine_grid])); f_data = hypre_VectorData(hypre_ParVectorLocalVector(F_array[fine_grid])); v_data = hypre_VectorData(hypre_ParVectorLocalVector(Vtemp)); l1_norms_lvl = l1_norms[level]; hypre_ParVectorSetConstantValues(U_array[coarse_grid], 0.0); if (level < addlvl) /* multiplicative version */ { /* smoothing step */ if (rlx_down == 0) { HYPRE_Real *A_data = hypre_CSRMatrixData(hypre_ParCSRMatrixDiag(A_array[fine_grid])); HYPRE_Int *A_i = hypre_CSRMatrixI(hypre_ParCSRMatrixDiag(A_array[fine_grid])); hypre_ParVectorCopy(F_array[fine_grid],Vtemp); num_rows = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A_array[fine_grid])); #ifdef HYPRE_USING_OPENMP #pragma omp parallel for private(i) HYPRE_SMP_SCHEDULE #endif for (i = 0; i < num_rows; i++) u_data[i] = relax_weight[level]*v_data[i] / A_data[A_i[i]]; } else if (rlx_down != 18) { /*hypre_BoomerAMGRelax(A_array[fine_grid],F_array[fine_grid],NULL,rlx_down,0,*/ hypre_BoomerAMGRelaxIF(A_array[fine_grid],F_array[fine_grid], CF_marker_array[fine_grid], rlx_down,rlx_order,1, relax_weight[fine_grid], omega[fine_grid], l1_norms_lvl, U_array[fine_grid], Vtemp, Ztemp); hypre_ParVectorCopy(F_array[fine_grid],Vtemp); } else { hypre_ParVectorCopy(F_array[fine_grid],Vtemp); num_rows = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A_array[fine_grid])); #ifdef HYPRE_USING_OPENMP #pragma omp parallel for private(i) HYPRE_SMP_SCHEDULE #endif for (i = 0; i < num_rows; i++) u_data[i] += v_data[i] / l1_norms_lvl[i]; } alpha = -1.0; beta = 1.0; hypre_ParCSRMatrixMatvec(alpha, A_array[fine_grid], U_array[fine_grid], beta, Vtemp); alpha = 1.0; beta = 0.0; hypre_ParCSRMatrixMatvecT(alpha,R_array[fine_grid],Vtemp, beta,F_array[coarse_grid]); } else /* additive version */ { hypre_ParVectorCopy(F_array[fine_grid],Vtemp); if (level == 0) /* compute residual */ { hypre_ParVectorCopy(Vtemp, Rtilde); hypre_ParVectorCopy(U_array[fine_grid],Xtilde); } alpha = 1.0; beta = 0.0; hypre_ParCSRMatrixMatvecT(alpha,R_array[fine_grid],Vtemp, beta,F_array[coarse_grid]); } } /* solve coarse grid */ if (addlvl < num_levels) { if (simple > -1) { x_global = hypre_VectorData(hypre_ParVectorLocalVector(Xtilde)); r_global = hypre_VectorData(hypre_ParVectorLocalVector(Rtilde)); n_global = hypre_VectorSize(hypre_ParVectorLocalVector(Xtilde)); #ifdef HYPRE_USING_OPENMP #pragma omp parallel for private(i) HYPRE_SMP_SCHEDULE #endif for (i=0; i < n_global; i++) x_global[i] += D_inv[i]*r_global[i]; } else hypre_ParCSRMatrixMatvec(1.0, Lambda, Rtilde, 1.0, Xtilde); if (addlvl == 0) hypre_ParVectorCopy(Xtilde, U_array[0]); } else { fine_grid = num_levels -1; hypre_ParCSRRelax(A_array[fine_grid], F_array[fine_grid], 1, 1, l1_norms[fine_grid], 1.0, 1.0 ,0,0,0,0, U_array[fine_grid], Vtemp, Ztemp); } /* up cycle */ relax_type = grid_relax_type[2]; for (level = num_levels-1; level > 0; level--) { fine_grid = level - 1; coarse_grid = level; if (level <= addlvl) /* multiplicative version */ { alpha = 1.0; beta = 1.0; hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid], U_array[coarse_grid], beta, U_array[fine_grid]); if (rlx_up != 18) /*hypre_BoomerAMGRelax(A_array[fine_grid],F_array[fine_grid],NULL,rlx_up,0,*/ hypre_BoomerAMGRelaxIF(A_array[fine_grid],F_array[fine_grid], CF_marker_array[fine_grid], rlx_up,rlx_order,2, relax_weight[fine_grid], omega[fine_grid], l1_norms[fine_grid], U_array[fine_grid], Vtemp, Ztemp); else if (rlx_order) { HYPRE_Int loc_relax_points[2]; loc_relax_points[0] = -1; loc_relax_points[1] = 1; for (i=0; i < 2; i++) hypre_ParCSRRelax_L1_Jacobi(A_array[fine_grid],F_array[fine_grid], CF_marker_array[fine_grid], loc_relax_points[i], 1.0, l1_norms[fine_grid], U_array[fine_grid], Vtemp); } else hypre_ParCSRRelax(A_array[fine_grid], F_array[fine_grid], 1, 1, l1_norms[fine_grid], 1.0, 1.0 ,0,0,0,0, U_array[fine_grid], Vtemp, Ztemp); } else /* additive version */ { alpha = 1.0; beta = 1.0; hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid], U_array[coarse_grid], beta, U_array[fine_grid]); } } return(Solve_err_flag); }
hypre_Vector * hypre_ParVectorToVectorAll (hypre_ParVector *par_v) { MPI_Comm comm = hypre_ParVectorComm(par_v); HYPRE_Int global_size = hypre_ParVectorGlobalSize(par_v); #ifndef HYPRE_NO_GLOBAL_PARTITION HYPRE_Int *vec_starts = hypre_ParVectorPartitioning(par_v); #endif hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_v); HYPRE_Int num_procs, my_id; HYPRE_Int num_vectors = hypre_ParVectorNumVectors(par_v); hypre_Vector *vector; double *vector_data; double *local_data; HYPRE_Int local_size; hypre_MPI_Request *requests; hypre_MPI_Status *status; HYPRE_Int i, j; HYPRE_Int *used_procs; HYPRE_Int num_types, num_requests; HYPRE_Int vec_len, proc_id; #ifdef HYPRE_NO_GLOBAL_PARTITION HYPRE_Int *new_vec_starts; HYPRE_Int num_contacts; HYPRE_Int contact_proc_list[1]; HYPRE_Int contact_send_buf[1]; HYPRE_Int contact_send_buf_starts[2]; HYPRE_Int max_response_size; HYPRE_Int *response_recv_buf=NULL; HYPRE_Int *response_recv_buf_starts = NULL; hypre_DataExchangeResponse response_obj; hypre_ProcListElements send_proc_obj; HYPRE_Int *send_info = NULL; hypre_MPI_Status status1; HYPRE_Int count, tag1 = 112, tag2 = 223; HYPRE_Int start; #endif hypre_MPI_Comm_size(comm, &num_procs); hypre_MPI_Comm_rank(comm, &my_id); #ifdef HYPRE_NO_GLOBAL_PARTITION local_size = hypre_ParVectorLastIndex(par_v) - hypre_ParVectorFirstIndex(par_v) + 1; /* determine procs which hold data of par_v and store ids in used_procs */ /* we need to do an exchange data for this. If I own row then I will contact processor 0 with the endpoint of my local range */ if (local_size > 0) { num_contacts = 1; contact_proc_list[0] = 0; contact_send_buf[0] = hypre_ParVectorLastIndex(par_v); contact_send_buf_starts[0] = 0; contact_send_buf_starts[1] = 1; } else { num_contacts = 0; contact_send_buf_starts[0] = 0; contact_send_buf_starts[1] = 0; } /*build the response object*/ /*send_proc_obj will be for saving info from contacts */ send_proc_obj.length = 0; send_proc_obj.storage_length = 10; send_proc_obj.id = hypre_CTAlloc(HYPRE_Int, send_proc_obj.storage_length); send_proc_obj.vec_starts = hypre_CTAlloc(HYPRE_Int, send_proc_obj.storage_length + 1); send_proc_obj.vec_starts[0] = 0; send_proc_obj.element_storage_length = 10; send_proc_obj.elements = hypre_CTAlloc(HYPRE_Int, send_proc_obj.element_storage_length); max_response_size = 0; /* each response is null */ response_obj.fill_response = hypre_FillResponseParToVectorAll; response_obj.data1 = NULL; response_obj.data2 = &send_proc_obj; /*this is where we keep info from contacts*/ hypre_DataExchangeList(num_contacts, contact_proc_list, contact_send_buf, contact_send_buf_starts, sizeof(HYPRE_Int), sizeof(HYPRE_Int), &response_obj, max_response_size, 1, comm, (void**) &response_recv_buf, &response_recv_buf_starts); /* now processor 0 should have a list of ranges for processors that have rows - these are in send_proc_obj - it needs to create the new list of processors and also an array of vec starts - and send to those who own row*/ if (my_id) { if (local_size) { /* look for a message from processor 0 */ hypre_MPI_Probe(0, tag1, comm, &status1); hypre_MPI_Get_count(&status1, HYPRE_MPI_INT, &count); send_info = hypre_CTAlloc(HYPRE_Int, count); hypre_MPI_Recv(send_info, count, HYPRE_MPI_INT, 0, tag1, comm, &status1); /* now unpack */ num_types = send_info[0]; used_procs = hypre_CTAlloc(HYPRE_Int, num_types); new_vec_starts = hypre_CTAlloc(HYPRE_Int, num_types+1); for (i=1; i<= num_types; i++) { used_procs[i-1] = send_info[i]; } for (i=num_types+1; i< count; i++) { new_vec_starts[i-num_types-1] = send_info[i] ; } } else /* clean up and exit */ { hypre_TFree(send_proc_obj.vec_starts); hypre_TFree(send_proc_obj.id); hypre_TFree(send_proc_obj.elements); if(response_recv_buf) hypre_TFree(response_recv_buf); if(response_recv_buf_starts) hypre_TFree(response_recv_buf_starts); return NULL; } } else /* my_id ==0 */ { num_types = send_proc_obj.length; used_procs = hypre_CTAlloc(HYPRE_Int, num_types); new_vec_starts = hypre_CTAlloc(HYPRE_Int, num_types+1); new_vec_starts[0] = 0; for (i=0; i< num_types; i++) { used_procs[i] = send_proc_obj.id[i]; new_vec_starts[i+1] = send_proc_obj.elements[i]+1; } qsort0(used_procs, 0, num_types-1); qsort0(new_vec_starts, 0, num_types); /*now we need to put into an array to send */ count = 2*num_types+2; send_info = hypre_CTAlloc(HYPRE_Int, count); send_info[0] = num_types; for (i=1; i<= num_types; i++) { send_info[i] = used_procs[i-1]; } for (i=num_types+1; i< count; i++) { send_info[i] = new_vec_starts[i-num_types-1]; } requests = hypre_CTAlloc(hypre_MPI_Request, num_types); status = hypre_CTAlloc(hypre_MPI_Status, num_types); /* don't send to myself - these are sorted so my id would be first*/ start = 0; if (used_procs[0] == 0) { start = 1; } for (i=start; i < num_types; i++) { hypre_MPI_Isend(send_info, count, HYPRE_MPI_INT, used_procs[i], tag1, comm, &requests[i-start]); } hypre_MPI_Waitall(num_types-start, requests, status); hypre_TFree(status); hypre_TFree(requests); } /* clean up */ hypre_TFree(send_proc_obj.vec_starts); hypre_TFree(send_proc_obj.id); hypre_TFree(send_proc_obj.elements); hypre_TFree(send_info); if(response_recv_buf) hypre_TFree(response_recv_buf); if(response_recv_buf_starts) hypre_TFree(response_recv_buf_starts); /* now proc 0 can exit if it has no rows */ if (!local_size) { hypre_TFree(used_procs); hypre_TFree(new_vec_starts); return NULL; } /* everyone left has rows and knows: new_vec_starts, num_types, and used_procs */ /* this vector should be rather small */ local_data = hypre_VectorData(local_vector); vector = hypre_SeqVectorCreate(global_size); hypre_VectorNumVectors(vector) = num_vectors; hypre_SeqVectorInitialize(vector); vector_data = hypre_VectorData(vector); num_requests = 2*num_types; requests = hypre_CTAlloc(hypre_MPI_Request, num_requests); status = hypre_CTAlloc(hypre_MPI_Status, num_requests); /* initialize data exchange among used_procs and generate vector - here we send to ourself also*/ j = 0; for (i = 0; i < num_types; i++) { proc_id = used_procs[i]; vec_len = new_vec_starts[i+1] - new_vec_starts[i]; hypre_MPI_Irecv(&vector_data[new_vec_starts[i]], num_vectors*vec_len, hypre_MPI_DOUBLE, proc_id, tag2, comm, &requests[j++]); } for (i = 0; i < num_types; i++) { hypre_MPI_Isend(local_data, num_vectors*local_size, hypre_MPI_DOUBLE, used_procs[i], tag2, comm, &requests[j++]); } hypre_MPI_Waitall(num_requests, requests, status); if (num_requests) { hypre_TFree(requests); hypre_TFree(status); hypre_TFree(used_procs); } hypre_TFree(new_vec_starts); #else local_size = vec_starts[my_id+1] - vec_starts[my_id]; /* if my_id contains no data, return NULL */ if (!local_size) return NULL; local_data = hypre_VectorData(local_vector); vector = hypre_SeqVectorCreate(global_size); hypre_VectorNumVectors(vector) = num_vectors; hypre_SeqVectorInitialize(vector); vector_data = hypre_VectorData(vector); /* determine procs which hold data of par_v and store ids in used_procs */ num_types = -1; for (i=0; i < num_procs; i++) if (vec_starts[i+1]-vec_starts[i]) num_types++; num_requests = 2*num_types; used_procs = hypre_CTAlloc(HYPRE_Int, num_types); j = 0; for (i=0; i < num_procs; i++) if (vec_starts[i+1]-vec_starts[i] && i-my_id) used_procs[j++] = i; requests = hypre_CTAlloc(hypre_MPI_Request, num_requests); status = hypre_CTAlloc(hypre_MPI_Status, num_requests); /* initialize data exchange among used_procs and generate vector */ j = 0; for (i = 0; i < num_types; i++) { proc_id = used_procs[i]; vec_len = vec_starts[proc_id+1] - vec_starts[proc_id]; hypre_MPI_Irecv(&vector_data[vec_starts[proc_id]], num_vectors*vec_len, hypre_MPI_DOUBLE, proc_id, 0, comm, &requests[j++]); } for (i = 0; i < num_types; i++) { hypre_MPI_Isend(local_data, num_vectors*local_size, hypre_MPI_DOUBLE, used_procs[i], 0, comm, &requests[j++]); } for (i=0; i < num_vectors*local_size; i++) vector_data[vec_starts[my_id]+i] = local_data[i]; hypre_MPI_Waitall(num_requests, requests, status); if (num_requests) { hypre_TFree(used_procs); hypre_TFree(requests); hypre_TFree(status); } #endif return vector; }
HYPRE_Int hypre_CreateDinv(void *amg_vdata) { hypre_ParAMGData *amg_data = amg_vdata; /* Data Structure variables */ hypre_ParCSRMatrix **A_array; hypre_ParVector **F_array; hypre_ParVector **U_array; hypre_ParCSRMatrix *A_tmp; hypre_CSRMatrix *A_tmp_diag; hypre_ParVector *Xtilde; hypre_ParVector *Rtilde; hypre_Vector *Xtilde_local; hypre_Vector *Rtilde_local; HYPRE_Real *x_data; HYPRE_Real *r_data; HYPRE_Real *tmp_data; HYPRE_Real *D_inv = NULL; HYPRE_Real *relax_weight = NULL; HYPRE_Real relax_type; HYPRE_Int addlvl; HYPRE_Int num_levels; HYPRE_Int num_add_lvls; HYPRE_Int num_rows_L; HYPRE_Int num_rows_A; HYPRE_Int num_rows_tmp; HYPRE_Int level, i; /* Local variables */ HYPRE_Int Solve_err_flag = 0; HYPRE_Int num_threads; HYPRE_Real **l1_norms_ptr = NULL; HYPRE_Real *l1_norms; HYPRE_Int l1_start; /* Acquire data and allocate storage */ num_threads = hypre_NumThreads(); A_array = hypre_ParAMGDataAArray(amg_data); F_array = hypre_ParAMGDataFArray(amg_data); U_array = hypre_ParAMGDataUArray(amg_data); addlvl = hypre_ParAMGDataSimple(amg_data); num_levels = hypre_ParAMGDataNumLevels(amg_data); relax_weight = hypre_ParAMGDataRelaxWeight(amg_data); relax_type = hypre_ParAMGDataGridRelaxType(amg_data)[1]; num_rows_A = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A_array[0])); l1_norms_ptr = hypre_ParAMGDataL1Norms(amg_data); /* smooth_option = hypre_ParAMGDataSmoothOption(amg_data); */ num_add_lvls = num_levels+1-addlvl; num_rows_L = 0; for (i=addlvl; i < num_levels; i++) { A_tmp = A_array[i]; A_tmp_diag = hypre_ParCSRMatrixDiag(A_tmp); num_rows_tmp = hypre_CSRMatrixNumRows(A_tmp_diag); num_rows_L += num_rows_tmp; } Rtilde = hypre_CTAlloc(hypre_ParVector, 1); Rtilde_local = hypre_SeqVectorCreate(num_rows_L); hypre_SeqVectorInitialize(Rtilde_local); hypre_ParVectorLocalVector(Rtilde) = Rtilde_local; hypre_ParVectorOwnsData(Rtilde) = 1; Xtilde = hypre_CTAlloc(hypre_ParVector, 1); Xtilde_local = hypre_SeqVectorCreate(num_rows_L); hypre_SeqVectorInitialize(Xtilde_local); hypre_ParVectorLocalVector(Xtilde) = Xtilde_local; hypre_ParVectorOwnsData(Xtilde) = 1; x_data = hypre_VectorData(hypre_ParVectorLocalVector(Xtilde)); r_data = hypre_VectorData(hypre_ParVectorLocalVector(Rtilde)); D_inv = hypre_CTAlloc(HYPRE_Real, num_rows_L); l1_start = 0; for (level=addlvl; level < num_levels; level++) { if (level != 0) { tmp_data = hypre_VectorData(hypre_ParVectorLocalVector(F_array[level])); if (tmp_data) hypre_TFree(tmp_data); hypre_VectorData(hypre_ParVectorLocalVector(F_array[level])) = &r_data[l1_start]; hypre_VectorOwnsData(hypre_ParVectorLocalVector(F_array[level])) = 0; tmp_data = hypre_VectorData(hypre_ParVectorLocalVector(U_array[level])); if (tmp_data) hypre_TFree(tmp_data); hypre_VectorData(hypre_ParVectorLocalVector(U_array[level])) = &x_data[l1_start]; hypre_VectorOwnsData(hypre_ParVectorLocalVector(U_array[level])) = 0; } A_tmp = A_array[level]; A_tmp_diag = hypre_ParCSRMatrixDiag(A_tmp); num_rows_tmp = hypre_CSRMatrixNumRows(A_tmp_diag); if (relax_type == 0) { HYPRE_Real rlx_wt = relax_weight[level]; HYPRE_Int *A_tmp_diag_i = hypre_CSRMatrixI(A_tmp_diag); HYPRE_Real *A_tmp_diag_data = hypre_CSRMatrixData(A_tmp_diag); #ifdef HYPRE_USING_OPENMP #pragma omp for private(i) HYPRE_SMP_SCHEDULE #endif for (i=0; i < num_rows_tmp; i++) D_inv[l1_start+i] = rlx_wt/A_tmp_diag_data[A_tmp_diag_i[i]]; } else { l1_norms = l1_norms_ptr[level]; #ifdef HYPRE_USING_OPENMP #pragma omp for private(i) HYPRE_SMP_SCHEDULE #endif for (i=0; i < num_rows_tmp; i++) D_inv[l1_start+i] = 1.0/l1_norms[i]; } l1_start += num_rows_tmp; } hypre_ParAMGDataDinv(amg_data) = D_inv; hypre_ParAMGDataRtilde(amg_data) = Rtilde; hypre_ParAMGDataXtilde(amg_data) = Xtilde; return Solve_err_flag; }
HYPRE_Int hypre_ParVectorReadIJ( MPI_Comm comm, const char *filename, HYPRE_Int *base_j_ptr, hypre_ParVector **vector_ptr) { HYPRE_Int global_size; hypre_ParVector *vector; hypre_Vector *local_vector; double *local_data; HYPRE_Int *partitioning; HYPRE_Int base_j; HYPRE_Int myid, num_procs, i, j, J; char new_filename[255]; FILE *file; hypre_MPI_Comm_size(comm, &num_procs); hypre_MPI_Comm_rank(comm, &myid); hypre_sprintf(new_filename,"%s.%05d", filename, myid); if ((file = fopen(new_filename, "r")) == NULL) { hypre_printf("Error: can't open output file %s\n", new_filename); hypre_error(HYPRE_ERROR_GENERIC); return hypre_error_flag; } hypre_fscanf(file, "%d", &global_size); #ifdef HYPRE_NO_GLOBAL_PARTITION /* this may need to be changed so that the base is available in the file! */ partitioning = hypre_CTAlloc(HYPRE_Int,2); hypre_fscanf(file, "%d", partitioning); for (i = 0; i < 2; i++) { hypre_fscanf(file, "%d", partitioning+i); } /* This is not yet implemented correctly! */ base_j = 0; #else partitioning = hypre_CTAlloc(HYPRE_Int,num_procs+1); hypre_fscanf(file, "%d", partitioning); for (i = 1; i <= num_procs; i++) { hypre_fscanf(file, "%d", partitioning+i); partitioning[i] -= partitioning[0]; } base_j = partitioning[0]; partitioning[0] = 0; #endif vector = hypre_ParVectorCreate(comm, global_size, partitioning); hypre_ParVectorInitialize(vector); local_vector = hypre_ParVectorLocalVector(vector); local_data = hypre_VectorData(local_vector); #ifdef HYPRE_NO_GLOBAL_PARTITION for (j = 0; j < partitioning[1] - partitioning[0]; j++) #else for (j = 0; j < partitioning[myid+1] - partitioning[myid]; j++) #endif { hypre_fscanf(file, "%d %le", &J, local_data + j); } fclose(file); *base_j_ptr = base_j; *vector_ptr = vector; /* multivector code not written yet >>> */ hypre_assert( hypre_ParVectorNumVectors(vector) == 1 ); if ( hypre_ParVectorNumVectors(vector) != 1 ) hypre_error(HYPRE_ERROR_GENERIC); return hypre_error_flag; }
HYPRE_Int hypre_BoomerAMGCycle( void *amg_vdata, hypre_ParVector **F_array, hypre_ParVector **U_array ) { hypre_ParAMGData *amg_data = amg_vdata; HYPRE_Solver *smoother; /* Data Structure variables */ hypre_ParCSRMatrix **A_array; hypre_ParCSRMatrix **P_array; hypre_ParCSRMatrix **R_array; hypre_ParVector *Utemp; hypre_ParVector *Vtemp; hypre_ParVector *Rtemp; hypre_ParVector *Ptemp; hypre_ParVector *Ztemp; hypre_ParVector *Aux_U; hypre_ParVector *Aux_F; hypre_ParCSRBlockMatrix **A_block_array; hypre_ParCSRBlockMatrix **P_block_array; hypre_ParCSRBlockMatrix **R_block_array; HYPRE_Real *Ztemp_data; HYPRE_Real *Ptemp_data; HYPRE_Int **CF_marker_array; /* HYPRE_Int **unknown_map_array; HYPRE_Int **point_map_array; HYPRE_Int **v_at_point_array; */ HYPRE_Real cycle_op_count; HYPRE_Int cycle_type; HYPRE_Int num_levels; HYPRE_Int max_levels; HYPRE_Real *num_coeffs; HYPRE_Int *num_grid_sweeps; HYPRE_Int *grid_relax_type; HYPRE_Int **grid_relax_points; HYPRE_Int block_mode; HYPRE_Real *max_eig_est; HYPRE_Real *min_eig_est; HYPRE_Int cheby_order; HYPRE_Real cheby_fraction; /* Local variables */ HYPRE_Int *lev_counter; HYPRE_Int Solve_err_flag; HYPRE_Int k; HYPRE_Int i, j, jj; HYPRE_Int level; HYPRE_Int cycle_param; HYPRE_Int coarse_grid; HYPRE_Int fine_grid; HYPRE_Int Not_Finished; HYPRE_Int num_sweep; HYPRE_Int cg_num_sweep = 1; HYPRE_Int relax_type; HYPRE_Int relax_points; HYPRE_Int relax_order; HYPRE_Int relax_local; HYPRE_Int old_version = 0; HYPRE_Real *relax_weight; HYPRE_Real *omega; HYPRE_Real alfa, beta, gammaold; HYPRE_Real gamma = 1.0; HYPRE_Int local_size; /* HYPRE_Int *smooth_option; */ HYPRE_Int smooth_type; HYPRE_Int smooth_num_levels; HYPRE_Int num_threads, my_id; HYPRE_Real alpha; HYPRE_Real **l1_norms = NULL; HYPRE_Real *l1_norms_level; HYPRE_Int seq_cg = 0; MPI_Comm comm; #if 0 HYPRE_Real *D_mat; HYPRE_Real *S_vec; #endif /* Acquire data and allocate storage */ num_threads = hypre_NumThreads(); A_array = hypre_ParAMGDataAArray(amg_data); P_array = hypre_ParAMGDataPArray(amg_data); R_array = hypre_ParAMGDataRArray(amg_data); CF_marker_array = hypre_ParAMGDataCFMarkerArray(amg_data); Vtemp = hypre_ParAMGDataVtemp(amg_data); Rtemp = hypre_ParAMGDataRtemp(amg_data); Ptemp = hypre_ParAMGDataPtemp(amg_data); Ztemp = hypre_ParAMGDataZtemp(amg_data); num_levels = hypre_ParAMGDataNumLevels(amg_data); max_levels = hypre_ParAMGDataMaxLevels(amg_data); cycle_type = hypre_ParAMGDataCycleType(amg_data); A_block_array = hypre_ParAMGDataABlockArray(amg_data); P_block_array = hypre_ParAMGDataPBlockArray(amg_data); R_block_array = hypre_ParAMGDataRBlockArray(amg_data); block_mode = hypre_ParAMGDataBlockMode(amg_data); num_grid_sweeps = hypre_ParAMGDataNumGridSweeps(amg_data); grid_relax_type = hypre_ParAMGDataGridRelaxType(amg_data); grid_relax_points = hypre_ParAMGDataGridRelaxPoints(amg_data); relax_order = hypre_ParAMGDataRelaxOrder(amg_data); relax_weight = hypre_ParAMGDataRelaxWeight(amg_data); omega = hypre_ParAMGDataOmega(amg_data); smooth_type = hypre_ParAMGDataSmoothType(amg_data); smooth_num_levels = hypre_ParAMGDataSmoothNumLevels(amg_data); l1_norms = hypre_ParAMGDataL1Norms(amg_data); /* smooth_option = hypre_ParAMGDataSmoothOption(amg_data); */ max_eig_est = hypre_ParAMGDataMaxEigEst(amg_data); min_eig_est = hypre_ParAMGDataMinEigEst(amg_data); cheby_order = hypre_ParAMGDataChebyOrder(amg_data); cheby_fraction = hypre_ParAMGDataChebyFraction(amg_data); cycle_op_count = hypre_ParAMGDataCycleOpCount(amg_data); lev_counter = hypre_CTAlloc(HYPRE_Int, num_levels); if (hypre_ParAMGDataParticipate(amg_data)) seq_cg = 1; /* Initialize */ Solve_err_flag = 0; if (grid_relax_points) old_version = 1; num_coeffs = hypre_CTAlloc(HYPRE_Real, num_levels); num_coeffs[0] = hypre_ParCSRMatrixDNumNonzeros(A_array[0]); comm = hypre_ParCSRMatrixComm(A_array[0]); hypre_MPI_Comm_rank(comm,&my_id); if (block_mode) { for (j = 1; j < num_levels; j++) num_coeffs[j] = hypre_ParCSRBlockMatrixNumNonzeros(A_block_array[j]); } else { for (j = 1; j < num_levels; j++) num_coeffs[j] = hypre_ParCSRMatrixDNumNonzeros(A_array[j]); } /*--------------------------------------------------------------------- * Initialize cycling control counter * * Cycling is controlled using a level counter: lev_counter[k] * * Each time relaxation is performed on level k, the * counter is decremented by 1. If the counter is then * negative, we go to the next finer level. If non- * negative, we go to the next coarser level. The * following actions control cycling: * * a. lev_counter[0] is initialized to 1. * b. lev_counter[k] is initialized to cycle_type for k>0. * * c. During cycling, when going down to level k, lev_counter[k] * is set to the max of (lev_counter[k],cycle_type) *---------------------------------------------------------------------*/ Not_Finished = 1; lev_counter[0] = 1; for (k = 1; k < num_levels; ++k) { lev_counter[k] = cycle_type; } level = 0; cycle_param = 1; smoother = hypre_ParAMGDataSmoother(amg_data); if (smooth_num_levels > 0) { if (smooth_type == 7 || smooth_type == 8 || smooth_type == 17 || smooth_type == 18 || smooth_type == 9 || smooth_type == 19) { HYPRE_Int actual_local_size = hypre_ParVectorActualLocalSize(Vtemp); Utemp = hypre_ParVectorCreate(comm,hypre_ParVectorGlobalSize(Vtemp), hypre_ParVectorPartitioning(Vtemp)); hypre_ParVectorOwnsPartitioning(Utemp) = 0; local_size = hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp)); if (local_size < actual_local_size) { hypre_VectorData(hypre_ParVectorLocalVector(Utemp)) = hypre_CTAlloc(HYPRE_Complex, actual_local_size); hypre_ParVectorActualLocalSize(Utemp) = actual_local_size; } else hypre_ParVectorInitialize(Utemp); } } /*--------------------------------------------------------------------- * Main loop of cycling *--------------------------------------------------------------------*/ while (Not_Finished) { if (num_levels > 1) { local_size = hypre_VectorSize(hypre_ParVectorLocalVector(F_array[level])); hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp)) = local_size; if (smooth_num_levels <= level) { cg_num_sweep = 1; num_sweep = num_grid_sweeps[cycle_param]; Aux_U = U_array[level]; Aux_F = F_array[level]; } else if (smooth_type > 9) { hypre_VectorSize(hypre_ParVectorLocalVector(Ztemp)) = local_size; hypre_VectorSize(hypre_ParVectorLocalVector(Rtemp)) = local_size; hypre_VectorSize(hypre_ParVectorLocalVector(Ptemp)) = local_size; Ztemp_data = hypre_VectorData(hypre_ParVectorLocalVector(Ztemp)); Ptemp_data = hypre_VectorData(hypre_ParVectorLocalVector(Ptemp)); hypre_ParVectorSetConstantValues(Ztemp,0); alpha = -1.0; beta = 1.0; hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level], U_array[level], beta, F_array[level], Rtemp); cg_num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data); num_sweep = num_grid_sweeps[cycle_param]; Aux_U = Ztemp; Aux_F = Rtemp; } else { cg_num_sweep = 1; num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data); Aux_U = U_array[level]; Aux_F = F_array[level]; } relax_type = grid_relax_type[cycle_param]; } else /* AB: 4/08: removed the max_levels > 1 check - should do this when max-levels = 1 also */ { /* If no coarsening occurred, apply a simple smoother once */ Aux_U = U_array[level]; Aux_F = F_array[level]; num_sweep = 1; /* TK: Use the user relax type (instead of 0) to allow for setting a convergent smoother (e.g. in the solution of singular problems). */ relax_type = hypre_ParAMGDataUserRelaxType(amg_data); } if (l1_norms != NULL) l1_norms_level = l1_norms[level]; else l1_norms_level = NULL; if (cycle_param == 3 && seq_cg) { hypre_seqAMGCycle(amg_data, level, F_array, U_array); } else { /*------------------------------------------------------------------ * Do the relaxation num_sweep times *-----------------------------------------------------------------*/ for (jj = 0; jj < cg_num_sweep; jj++) { if (smooth_num_levels > level && smooth_type > 9) hypre_ParVectorSetConstantValues(Aux_U,0); for (j = 0; j < num_sweep; j++) { if (num_levels == 1 && max_levels > 1) { relax_points = 0; relax_local = 0; } else { if (old_version) relax_points = grid_relax_points[cycle_param][j]; relax_local = relax_order; } /*----------------------------------------------- * VERY sloppy approximation to cycle complexity *-----------------------------------------------*/ if (old_version && level < num_levels -1) { switch (relax_points) { case 1: cycle_op_count += num_coeffs[level+1]; break; case -1: cycle_op_count += (num_coeffs[level]-num_coeffs[level+1]); break; } } else { cycle_op_count += num_coeffs[level]; } /*----------------------------------------------- Choose Smoother -----------------------------------------------*/ if (smooth_num_levels > level && (smooth_type == 7 || smooth_type == 8 || smooth_type == 9 || smooth_type == 19 || smooth_type == 17 || smooth_type == 18)) { hypre_VectorSize(hypre_ParVectorLocalVector(Utemp)) = local_size; alpha = -1.0; beta = 1.0; hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level], U_array[level], beta, Aux_F, Vtemp); if (smooth_type == 8 || smooth_type == 18) HYPRE_ParCSRParaSailsSolve(smoother[level], (HYPRE_ParCSRMatrix) A_array[level], (HYPRE_ParVector) Vtemp, (HYPRE_ParVector) Utemp); else if (smooth_type == 7 || smooth_type == 17) HYPRE_ParCSRPilutSolve(smoother[level], (HYPRE_ParCSRMatrix) A_array[level], (HYPRE_ParVector) Vtemp, (HYPRE_ParVector) Utemp); else if (smooth_type == 9 || smooth_type == 19) HYPRE_EuclidSolve(smoother[level], (HYPRE_ParCSRMatrix) A_array[level], (HYPRE_ParVector) Vtemp, (HYPRE_ParVector) Utemp); hypre_ParVectorAxpy(relax_weight[level],Utemp,Aux_U); } else if (smooth_num_levels > level && (smooth_type == 6 || smooth_type == 16)) { HYPRE_SchwarzSolve(smoother[level], (HYPRE_ParCSRMatrix) A_array[level], (HYPRE_ParVector) Aux_F, (HYPRE_ParVector) Aux_U); } /*else if (relax_type == 99)*/ else if (relax_type == 9 || relax_type == 99) { /* Gaussian elimination */ hypre_GaussElimSolve(amg_data, level, relax_type); } else if (relax_type == 18) { /* L1 - Jacobi*/ if (relax_order == 1 && cycle_param < 3) { /* need to do CF - so can't use the AMS one */ HYPRE_Int i; HYPRE_Int loc_relax_points[2]; if (cycle_type < 2) { loc_relax_points[0] = 1; loc_relax_points[1] = -1; } else { loc_relax_points[0] = -1; loc_relax_points[1] = 1; } for (i=0; i < 2; i++) hypre_ParCSRRelax_L1_Jacobi(A_array[level], Aux_F, CF_marker_array[level], loc_relax_points[i], relax_weight[level], l1_norms[level], Aux_U, Vtemp); } else /* not CF - so use through AMS */ { if (num_threads == 1) hypre_ParCSRRelax(A_array[level], Aux_F, 1, 1, l1_norms_level, relax_weight[level], omega[level],0,0,0,0, Aux_U, Vtemp, Ztemp); else hypre_ParCSRRelaxThreads(A_array[level], Aux_F, 1, 1, l1_norms_level, relax_weight[level], omega[level], Aux_U, Vtemp, Ztemp); } } else if (relax_type == 15) { /* CG */ if (j ==0) /* do num sweep iterations of CG */ hypre_ParCSRRelax_CG( smoother[level], A_array[level], Aux_F, Aux_U, num_sweep); } else if (relax_type == 16) { /* scaled Chebyshev */ HYPRE_Int scale = 1; HYPRE_Int variant = 0; hypre_ParCSRRelax_Cheby(A_array[level], Aux_F, max_eig_est[level], min_eig_est[level], cheby_fraction, cheby_order, scale, variant, Aux_U, Vtemp, Ztemp ); } else if (relax_type ==17) { hypre_BoomerAMGRelax_FCFJacobi(A_array[level], Aux_F, CF_marker_array[level], relax_weight[level], Aux_U, Vtemp); } else if (old_version) { Solve_err_flag = hypre_BoomerAMGRelax(A_array[level], Aux_F, CF_marker_array[level], relax_type, relax_points, relax_weight[level], omega[level], l1_norms_level, Aux_U, Vtemp, Ztemp); } else { /* smoother than can have CF ordering */ if (block_mode) { Solve_err_flag = hypre_BoomerAMGBlockRelaxIF(A_block_array[level], Aux_F, CF_marker_array[level], relax_type, relax_local, cycle_param, relax_weight[level], omega[level], Aux_U, Vtemp); } else { Solve_err_flag = hypre_BoomerAMGRelaxIF(A_array[level], Aux_F, CF_marker_array[level], relax_type, relax_local, cycle_param, relax_weight[level], omega[level], l1_norms_level, Aux_U, Vtemp, Ztemp); } } if (Solve_err_flag != 0) return(Solve_err_flag); } if (smooth_num_levels > level && smooth_type > 9) { gammaold = gamma; gamma = hypre_ParVectorInnerProd(Rtemp,Ztemp); if (jj == 0) hypre_ParVectorCopy(Ztemp,Ptemp); else { beta = gamma/gammaold; for (i=0; i < local_size; i++) Ptemp_data[i] = Ztemp_data[i] + beta*Ptemp_data[i]; } hypre_ParCSRMatrixMatvec(1.0,A_array[level],Ptemp,0.0,Vtemp); alfa = gamma /hypre_ParVectorInnerProd(Ptemp,Vtemp); hypre_ParVectorAxpy(alfa,Ptemp,U_array[level]); hypre_ParVectorAxpy(-alfa,Vtemp,Rtemp); } } } /*------------------------------------------------------------------ * Decrement the control counter and determine which grid to visit next *-----------------------------------------------------------------*/ --lev_counter[level]; if (lev_counter[level] >= 0 && level != num_levels-1) { /*--------------------------------------------------------------- * Visit coarser level next. * Compute residual using hypre_ParCSRMatrixMatvec. * Perform restriction using hypre_ParCSRMatrixMatvecT. * Reset counters and cycling parameters for coarse level *--------------------------------------------------------------*/ fine_grid = level; coarse_grid = level + 1; hypre_ParVectorSetConstantValues(U_array[coarse_grid], 0.0); alpha = -1.0; beta = 1.0; if (block_mode) { hypre_ParVectorCopy(F_array[fine_grid],Vtemp); hypre_ParCSRBlockMatrixMatvec(alpha, A_block_array[fine_grid], U_array[fine_grid], beta, Vtemp); } else { // JSP: avoid unnecessary copy using out-of-place version of SpMV hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[fine_grid], U_array[fine_grid], beta, F_array[fine_grid], Vtemp); } alpha = 1.0; beta = 0.0; if (block_mode) { hypre_ParCSRBlockMatrixMatvecT(alpha,R_block_array[fine_grid],Vtemp, beta,F_array[coarse_grid]); } else { hypre_ParCSRMatrixMatvecT(alpha,R_array[fine_grid],Vtemp, beta,F_array[coarse_grid]); } ++level; lev_counter[level] = hypre_max(lev_counter[level],cycle_type); cycle_param = 1; if (level == num_levels-1) cycle_param = 3; } else if (level != 0) { /*--------------------------------------------------------------- * Visit finer level next. * Interpolate and add correction using hypre_ParCSRMatrixMatvec. * Reset counters and cycling parameters for finer level. *--------------------------------------------------------------*/ fine_grid = level - 1; coarse_grid = level; alpha = 1.0; beta = 1.0; if (block_mode) { hypre_ParCSRBlockMatrixMatvec(alpha, P_block_array[fine_grid], U_array[coarse_grid], beta, U_array[fine_grid]); } else { hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid], U_array[coarse_grid], beta, U_array[fine_grid]); } --level; cycle_param = 2; } else { Not_Finished = 0; } } hypre_ParAMGDataCycleOpCount(amg_data) = cycle_op_count; hypre_TFree(lev_counter); hypre_TFree(num_coeffs); if (smooth_num_levels > 0) { if (smooth_type == 7 || smooth_type == 8 || smooth_type == 9 || smooth_type == 17 || smooth_type == 18 || smooth_type == 19 ) hypre_ParVectorDestroy(Utemp); } return(Solve_err_flag); }
static int f(realtype t, N_Vector u, N_Vector udot, void *user_data) { realtype ui, ult, urt, hordc, horac, hdiff, hadv; realtype *udata, *udotdata, *z; int i; int npes, my_pe, my_length, my_pe_m1, my_pe_p1, last_pe; UserData data; MPI_Status status; MPI_Comm comm; HYPRE_ParVector uhyp; HYPRE_ParVector udothyp; /* Extract hypre vectors */ uhyp = N_VGetVector_ParHyp(u); udothyp = N_VGetVector_ParHyp(udot); /* Access hypre vectors local data */ udata = hypre_VectorData(hypre_ParVectorLocalVector(uhyp)); udotdata = hypre_VectorData(hypre_ParVectorLocalVector(udothyp)); /* Extract needed problem constants from data */ data = (UserData) user_data; hordc = data->hdcoef; horac = data->hacoef; /* Extract parameters for parhyp computation. */ comm = data->comm; npes = data->npes; /* Number of processes */ my_pe = data->my_pe; /* Current process number */ my_length = hypre_ParVectorLastIndex(uhyp) /* Local length of uhyp */ - hypre_ParVectorFirstIndex(uhyp) + 1; z = data->z; /* Compute related parameters. */ my_pe_m1 = my_pe - 1; my_pe_p1 = my_pe + 1; last_pe = npes - 1; /* Store local segment of u in the working array z. */ for (i = 1; i <= my_length; i++) z[i] = udata[i - 1]; /* Pass needed data to processes before and after current process. */ if (my_pe != 0) MPI_Send(&z[1], 1, PVEC_REAL_MPI_TYPE, my_pe_m1, 0, comm); if (my_pe != last_pe) MPI_Send(&z[my_length], 1, PVEC_REAL_MPI_TYPE, my_pe_p1, 0, comm); /* Receive needed data from processes before and after current process. */ if (my_pe != 0) MPI_Recv(&z[0], 1, PVEC_REAL_MPI_TYPE, my_pe_m1, 0, comm, &status); else z[0] = ZERO; if (my_pe != last_pe) MPI_Recv(&z[my_length+1], 1, PVEC_REAL_MPI_TYPE, my_pe_p1, 0, comm, &status); else z[my_length + 1] = ZERO; /* Loop over all grid points in current process. */ for (i=1; i<=my_length; i++) { /* Extract u at x_i and two neighboring points */ ui = z[i]; ult = z[i-1]; urt = z[i+1]; /* Set diffusion and advection terms and load into udot */ hdiff = hordc*(ult - RCONST(2.0)*ui + urt); hadv = horac*(urt - ult); udotdata[i-1] = hdiff + hadv; } return(0); }
static int Precond(realtype tn, N_Vector u, N_Vector fu, booleantype jok, booleantype *jcurPtr, realtype gamma, void *user_data, N_Vector vtemp1, N_Vector vtemp2, N_Vector vtemp3) { realtype c1, c2, cydn, cyup, diag, ydn, yup, q4coef, dely, verdco, hordco; realtype **(*P)[MYSUB], **(*Jbd)[MYSUB]; int nvmxsub, ier, offset; long int *(*pivot)[MYSUB]; int lx, ly, jy, isuby; realtype *udata, **a, **j; HYPRE_ParVector uhyp; UserData data; /* Make local copies of pointers in user_data, pointer to u's data, and PE index pair */ data = (UserData) user_data; P = data->P; Jbd = data->Jbd; pivot = data->pivot; isuby = data->isuby; nvmxsub = data->nvmxsub; uhyp = N_VGetVector_ParHyp(u); udata = hypre_VectorData(hypre_ParVectorLocalVector(uhyp)); if (jok) { /* jok = TRUE: Copy Jbd to P */ for (ly = 0; ly < MYSUB; ly++) for (lx = 0; lx < MXSUB; lx++) denseCopy(Jbd[lx][ly], P[lx][ly], NVARS, NVARS); *jcurPtr = FALSE; } else { /* jok = FALSE: Generate Jbd from scratch and copy to P */ /* Make local copies of problem variables, for efficiency */ q4coef = data->q4; dely = data->dy; verdco = data->vdco; hordco = data->hdco; /* Compute 2x2 diagonal Jacobian blocks (using q4 values c*omputed on the last f call). Load into P. */ for (ly = 0; ly < MYSUB; ly++) { jy = ly + isuby*MYSUB; ydn = YMIN + (jy - RCONST(0.5))*dely; yup = ydn + dely; cydn = verdco*SUNRexp(RCONST(0.2)*ydn); cyup = verdco*SUNRexp(RCONST(0.2)*yup); diag = -(cydn + cyup + RCONST(2.0)*hordco); for (lx = 0; lx < MXSUB; lx++) { offset = lx*NVARS + ly*nvmxsub; c1 = udata[offset]; c2 = udata[offset+1]; j = Jbd[lx][ly]; a = P[lx][ly]; IJth(j,1,1) = (-Q1*C3 - Q2*c2) + diag; IJth(j,1,2) = -Q2*c1 + q4coef; IJth(j,2,1) = Q1*C3 - Q2*c2; IJth(j,2,2) = (-Q2*c1 - q4coef) + diag; denseCopy(j, a, NVARS, NVARS); } } *jcurPtr = TRUE; } /* Scale by -gamma */ for (ly = 0; ly < MYSUB; ly++) for (lx = 0; lx < MXSUB; lx++) denseScale(-gamma, P[lx][ly], NVARS, NVARS); /* Add identity matrix and do LU decompositions on blocks in place */ for (lx = 0; lx < MXSUB; lx++) { for (ly = 0; ly < MYSUB; ly++) { denseAddIdentity(P[lx][ly], NVARS); ier = denseGETRF(P[lx][ly], NVARS, NVARS, pivot[lx][ly]); if (ier != 0) return(1); } } return(0); }
/* ---------------------------------------------------------------------- * get_element * * Reads single element from hypre vector by accessing its raw block. * Probably not the most efficient way to get the vector values. * --------------------------------------------------------------------*/ realtype get_element(N_Vector X, long int i) { hypre_ParVector *Xvec = N_VGetVector_ParHyp(X); const realtype *Xdata = hypre_VectorData(hypre_ParVectorLocalVector(Xvec)); return Xdata[i]; }
/* Function: hypre_ParCSRMatrixEliminateAXB This function eliminates the global rows and columns of a matrix A corresponding to given lists of sorted (!) local row numbers, so that the solution to the system A*X = B is X_b for the given rows. The elimination is done as follows: (input) (output) / A_ii | A_ib \ / A_ii | 0 \ A = | -----+----- | ---> | -----+----- | \ A_bi | A_bb / \ 0 | I / / X_i \ / X_i \ X = | --- | ---> | --- | (no change) \ X_b / \ X_b / / B_i \ / B_i - A_ib * X_b \ B = | --- | ---> | ---------------- | \ B_b / \ X_b / */ void hypre_ParCSRMatrixEliminateAXB(hypre_ParCSRMatrix *A, HYPRE_Int num_rowscols_to_elim, HYPRE_Int *rowscols_to_elim, hypre_ParVector *X, hypre_ParVector *B) { hypre_CSRMatrix *diag = hypre_ParCSRMatrixDiag(A); hypre_CSRMatrix *offd = hypre_ParCSRMatrixOffd(A); HYPRE_Int diag_nrows = hypre_CSRMatrixNumRows(diag); HYPRE_Int offd_ncols = hypre_CSRMatrixNumCols(offd); hypre_Vector *Xlocal = hypre_ParVectorLocalVector(X); hypre_Vector *Blocal = hypre_ParVectorLocalVector(B); HYPRE_Real *Bdata = hypre_VectorData(Blocal); HYPRE_Real *Xdata = hypre_VectorData(Xlocal); HYPRE_Int num_offd_cols_to_elim; HYPRE_Int *offd_cols_to_elim; HYPRE_Real *eliminate_coefs; /* figure out which offd cols should be eliminated and with what coef */ hypre_ParCSRCommHandle *comm_handle; hypre_ParCSRCommPkg *comm_pkg; HYPRE_Int num_sends; HYPRE_Int index, start; HYPRE_Int i, j, k, irow; HYPRE_Real *eliminate_row = hypre_CTAlloc(HYPRE_Real, diag_nrows); HYPRE_Real *eliminate_col = hypre_CTAlloc(HYPRE_Real, offd_ncols); HYPRE_Real *buf_data, coef; /* make sure A has a communication package */ comm_pkg = hypre_ParCSRMatrixCommPkg(A); if (!comm_pkg) { hypre_MatvecCommPkgCreate(A); comm_pkg = hypre_ParCSRMatrixCommPkg(A); } /* HACK: rows that shouldn't be eliminated are marked with quiet NaN; those that should are set to the boundary value from X; this is to avoid sending complex type (int+double) or communicating twice. */ for (i = 0; i < diag_nrows; i++) { eliminate_row[i] = std::numeric_limits<HYPRE_Real>::quiet_NaN(); } for (i = 0; i < num_rowscols_to_elim; i++) { irow = rowscols_to_elim[i]; eliminate_row[irow] = Xdata[irow]; } /* use a Matvec communication pattern to find (in eliminate_col) which of the local offd columns are to be eliminated */ num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg); buf_data = hypre_CTAlloc(HYPRE_Real, hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends)); index = 0; for (i = 0; i < num_sends; i++) { start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i); for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++) { k = hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j); buf_data[index++] = eliminate_row[k]; } } comm_handle = hypre_ParCSRCommHandleCreate(1, comm_pkg, buf_data, eliminate_col); /* do sequential part of the elimination while stuff is getting sent */ hypre_CSRMatrixEliminateAXB(diag, num_rowscols_to_elim, rowscols_to_elim, Xlocal, Blocal); /* finish the communication */ hypre_ParCSRCommHandleDestroy(comm_handle); /* received eliminate_col[], count offd columns to eliminate */ num_offd_cols_to_elim = 0; for (i = 0; i < offd_ncols; i++) { coef = eliminate_col[i]; if (coef == coef) // test for NaN { num_offd_cols_to_elim++; } } offd_cols_to_elim = hypre_CTAlloc(HYPRE_Int, num_offd_cols_to_elim); eliminate_coefs = hypre_CTAlloc(HYPRE_Real, num_offd_cols_to_elim); /* get a list of offd column indices and coefs */ num_offd_cols_to_elim = 0; for (i = 0; i < offd_ncols; i++) { coef = eliminate_col[i]; if (coef == coef) // test for NaN { offd_cols_to_elim[num_offd_cols_to_elim] = i; eliminate_coefs[num_offd_cols_to_elim] = coef; num_offd_cols_to_elim++; } } hypre_TFree(buf_data); hypre_TFree(eliminate_row); hypre_TFree(eliminate_col); /* eliminate the off-diagonal part */ hypre_CSRMatrixEliminateOffdColsAXB(offd, num_offd_cols_to_elim, offd_cols_to_elim, eliminate_coefs, Blocal); hypre_CSRMatrixEliminateOffdRowsAXB(offd, num_rowscols_to_elim, rowscols_to_elim); /* set boundary values in the rhs */ for (int i = 0; i < num_rowscols_to_elim; i++) { irow = rowscols_to_elim[i]; Bdata[irow] = Xdata[irow]; } hypre_TFree(offd_cols_to_elim); hypre_TFree(eliminate_coefs); }
HYPRE_Int hypre_ParCSRMatrixMatvec( double alpha, hypre_ParCSRMatrix *A, hypre_ParVector *x, double beta, hypre_ParVector *y ) { hypre_ParCSRCommHandle **comm_handle; hypre_ParCSRCommPkg *comm_pkg = hypre_ParCSRMatrixCommPkg(A); hypre_CSRMatrix *diag = hypre_ParCSRMatrixDiag(A); hypre_CSRMatrix *offd = hypre_ParCSRMatrixOffd(A); hypre_Vector *x_local = hypre_ParVectorLocalVector(x); hypre_Vector *y_local = hypre_ParVectorLocalVector(y); HYPRE_Int num_rows = hypre_ParCSRMatrixGlobalNumRows(A); HYPRE_Int num_cols = hypre_ParCSRMatrixGlobalNumCols(A); hypre_Vector *x_tmp; HYPRE_Int x_size = hypre_ParVectorGlobalSize(x); HYPRE_Int y_size = hypre_ParVectorGlobalSize(y); HYPRE_Int num_vectors = hypre_VectorNumVectors(x_local); HYPRE_Int num_cols_offd = hypre_CSRMatrixNumCols(offd); HYPRE_Int ierr = 0; HYPRE_Int num_sends, i, j, jv, index, start; HYPRE_Int vecstride = hypre_VectorVectorStride( x_local ); HYPRE_Int idxstride = hypre_VectorIndexStride( x_local ); double *x_tmp_data, **x_buf_data; double *x_local_data = hypre_VectorData(x_local); /*--------------------------------------------------------------------- * Check for size compatibility. ParMatvec returns ierr = 11 if * length of X doesn't equal the number of columns of A, * ierr = 12 if the length of Y doesn't equal the number of rows * of A, and ierr = 13 if both are true. * * Because temporary vectors are often used in ParMatvec, none of * these conditions terminates processing, and the ierr flag * is informational only. *--------------------------------------------------------------------*/ hypre_assert( idxstride>0 ); if (num_cols != x_size) ierr = 11; if (num_rows != y_size) ierr = 12; if (num_cols != x_size && num_rows != y_size) ierr = 13; hypre_assert( hypre_VectorNumVectors(y_local)==num_vectors ); if ( num_vectors==1 ) x_tmp = hypre_SeqVectorCreate( num_cols_offd ); else { hypre_assert( num_vectors>1 ); x_tmp = hypre_SeqMultiVectorCreate( num_cols_offd, num_vectors ); } hypre_SeqVectorInitialize(x_tmp); x_tmp_data = hypre_VectorData(x_tmp); comm_handle = hypre_CTAlloc(hypre_ParCSRCommHandle*,num_vectors); /*--------------------------------------------------------------------- * If there exists no CommPkg for A, a CommPkg is generated using * equally load balanced partitionings *--------------------------------------------------------------------*/ if (!comm_pkg) { hypre_MatvecCommPkgCreate(A); comm_pkg = hypre_ParCSRMatrixCommPkg(A); } num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg); x_buf_data = hypre_CTAlloc( double*, num_vectors ); for ( jv=0; jv<num_vectors; ++jv ) x_buf_data[jv] = hypre_CTAlloc(double, hypre_ParCSRCommPkgSendMapStart (comm_pkg, num_sends)); if ( num_vectors==1 ) { index = 0; for (i = 0; i < num_sends; i++) { start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i); for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++) x_buf_data[0][index++] = x_local_data[hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j)]; } } else for ( jv=0; jv<num_vectors; ++jv ) { index = 0; for (i = 0; i < num_sends; i++) { start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i); for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++) x_buf_data[jv][index++] = x_local_data[ jv*vecstride + idxstride*hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j) ]; } } hypre_assert( idxstride==1 ); /* >>> ... The assert is because the following loop only works for 'column' storage of a multivector <<< >>> This needs to be fixed to work more generally, at least for 'row' storage. <<< >>> This in turn, means either change CommPkg so num_sends is no.zones*no.vectors (not no.zones) >>> or, less dangerously, put a stride in the logic of CommHandleCreate (stride either from a >>> new arg or a new variable inside CommPkg). Or put the num_vector iteration inside >>> CommHandleCreate (perhaps a new multivector variant of it). */ for ( jv=0; jv<num_vectors; ++jv ) { comm_handle[jv] = hypre_ParCSRCommHandleCreate ( 1, comm_pkg, x_buf_data[jv], &(x_tmp_data[jv*num_cols_offd]) ); } hypre_CSRMatrixMatvec( alpha, diag, x_local, beta, y_local); for ( jv=0; jv<num_vectors; ++jv ) { hypre_ParCSRCommHandleDestroy(comm_handle[jv]); comm_handle[jv] = NULL; } hypre_TFree(comm_handle); if (num_cols_offd) hypre_CSRMatrixMatvec( alpha, offd, x_tmp, 1.0, y_local); hypre_SeqVectorDestroy(x_tmp); x_tmp = NULL; for ( jv=0; jv<num_vectors; ++jv ) hypre_TFree(x_buf_data[jv]); hypre_TFree(x_buf_data); return ierr; }
HYPRE_Int hypre_BoomerAMGRelaxT( hypre_ParCSRMatrix *A, hypre_ParVector *f, HYPRE_Int *cf_marker, HYPRE_Int relax_type, HYPRE_Int relax_points, double relax_weight, hypre_ParVector *u, hypre_ParVector *Vtemp ) { hypre_CSRMatrix *A_diag = hypre_ParCSRMatrixDiag(A); double *A_diag_data = hypre_CSRMatrixData(A_diag); HYPRE_Int *A_diag_i = hypre_CSRMatrixI(A_diag); HYPRE_Int n_global= hypre_ParCSRMatrixGlobalNumRows(A); HYPRE_Int n = hypre_CSRMatrixNumRows(A_diag); HYPRE_Int first_index = hypre_ParVectorFirstIndex(u); hypre_Vector *u_local = hypre_ParVectorLocalVector(u); double *u_data = hypre_VectorData(u_local); hypre_Vector *Vtemp_local = hypre_ParVectorLocalVector(Vtemp); double *Vtemp_data = hypre_VectorData(Vtemp_local); hypre_CSRMatrix *A_CSR; HYPRE_Int *A_CSR_i; HYPRE_Int *A_CSR_j; double *A_CSR_data; hypre_Vector *f_vector; double *f_vector_data; HYPRE_Int i; HYPRE_Int jj; HYPRE_Int column; HYPRE_Int relax_error = 0; double *A_mat; double *b_vec; double zero = 0.0; /*----------------------------------------------------------------------- * Switch statement to direct control based on relax_type: * relax_type = 7 -> Jacobi (uses ParMatvec) * relax_type = 9 -> Direct Solve *-----------------------------------------------------------------------*/ switch (relax_type) { case 7: /* Jacobi (uses ParMatvec) */ { /*----------------------------------------------------------------- * Copy f into temporary vector. *-----------------------------------------------------------------*/ hypre_ParVectorCopy(f,Vtemp); /*----------------------------------------------------------------- * Perform MatvecT Vtemp=f-A^Tu *-----------------------------------------------------------------*/ hypre_ParCSRMatrixMatvecT(-1.0,A, u, 1.0, Vtemp); for (i = 0; i < n; i++) { /*----------------------------------------------------------- * If diagonal is nonzero, relax point i; otherwise, skip it. *-----------------------------------------------------------*/ if (A_diag_data[A_diag_i[i]] != zero) { u_data[i] += relax_weight * Vtemp_data[i] / A_diag_data[A_diag_i[i]]; } } } break; case 9: /* Direct solve: use gaussian elimination */ { /*----------------------------------------------------------------- * Generate CSR matrix from ParCSRMatrix A *-----------------------------------------------------------------*/ if (n) { A_CSR = hypre_ParCSRMatrixToCSRMatrixAll(A); f_vector = hypre_ParVectorToVectorAll(f); A_CSR_i = hypre_CSRMatrixI(A_CSR); A_CSR_j = hypre_CSRMatrixJ(A_CSR); A_CSR_data = hypre_CSRMatrixData(A_CSR); f_vector_data = hypre_VectorData(f_vector); A_mat = hypre_CTAlloc(double, n_global*n_global); b_vec = hypre_CTAlloc(double, n_global); /*--------------------------------------------------------------- * Load transpose of CSR matrix into A_mat. *---------------------------------------------------------------*/ for (i = 0; i < n_global; i++) { for (jj = A_CSR_i[i]; jj < A_CSR_i[i+1]; jj++) { column = A_CSR_j[jj]; A_mat[column*n_global+i] = A_CSR_data[jj]; } b_vec[i] = f_vector_data[i]; } relax_error = gselim(A_mat,b_vec,n_global); for (i = 0; i < n; i++) { u_data[i] = b_vec[first_index+i]; } hypre_TFree(A_mat); hypre_TFree(b_vec); hypre_CSRMatrixDestroy(A_CSR); A_CSR = NULL; hypre_SeqVectorDestroy(f_vector); f_vector = NULL; } } break; } return(relax_error); }