HYPRE_Int main( HYPRE_Int argc, char *argv[] ) { hypre_ParCSRMatrix *par_matrix, *g_matrix, **submatrices; hypre_CSRMatrix *A_diag, *A_offd; hypre_CSRBlockMatrix *diag; hypre_CSRBlockMatrix *offd; hypre_ParCSRBlockMatrix *par_blk_matrix, *par_blk_matrixT, *rap_matrix; hypre_Vector *x_local; hypre_Vector *y_local; hypre_ParVector *x; hypre_ParVector *y; HYPRE_Solver gmres_solver, precon; HYPRE_Int *diag_i, *diag_j, *offd_i, *offd_j; HYPRE_Int *diag_i2, *diag_j2, *offd_i2, *offd_j2; double *diag_d, *diag_d2, *offd_d, *offd_d2; HYPRE_Int mypid, local_size, nprocs; HYPRE_Int global_num_rows, global_num_cols, num_cols_offd; HYPRE_Int num_nonzeros_diag, num_nonzeros_offd, *colMap; HYPRE_Int ii, jj, kk, row, col, nnz, *indices, *colMap2; double *data, ddata, *y_data; HYPRE_Int *row_starts, *col_starts, *rstarts, *cstarts; HYPRE_Int *row_starts2, *col_starts2; HYPRE_Int block_size=2, bnnz=4, *index_set; FILE *fp; /* --------------------------------------------- */ /* Initialize MPI */ /* --------------------------------------------- */ hypre_MPI_Init(&argc, &argv); hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &mypid); hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &nprocs); /* build and fetch matrix */ MyBuildParLaplacian9pt((HYPRE_ParCSRMatrix *) &par_matrix); global_num_rows = hypre_ParCSRMatrixGlobalNumRows(par_matrix); global_num_cols = hypre_ParCSRMatrixGlobalNumCols(par_matrix); row_starts = hypre_ParCSRMatrixRowStarts(par_matrix); col_starts = hypre_ParCSRMatrixColStarts(par_matrix); A_diag = hypre_ParCSRMatrixDiag(par_matrix); A_offd = hypre_ParCSRMatrixOffd(par_matrix); num_cols_offd = hypre_CSRMatrixNumCols(A_offd); num_nonzeros_diag = hypre_CSRMatrixNumNonzeros(A_diag); num_nonzeros_offd = hypre_CSRMatrixNumNonzeros(A_offd); /* --------------------------------------------- */ /* build vector and apply matvec */ /* --------------------------------------------- */ x = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_num_cols,col_starts); hypre_ParVectorSetPartitioningOwner(x,0); hypre_ParVectorInitialize(x); x_local = hypre_ParVectorLocalVector(x); data = hypre_VectorData(x_local); local_size = col_starts[mypid+1] - col_starts[mypid]; for (ii = 0; ii < local_size; ii++) data[ii] = 1.0; y = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD,global_num_rows,row_starts); hypre_ParVectorSetPartitioningOwner(y,0); hypre_ParVectorInitialize(y); hypre_ParCSRMatrixMatvec (1.0, par_matrix, x, 0.0, y); ddata = hypre_ParVectorInnerProd(y, y); if (mypid == 0) hypre_printf("y inner product = %e\n", ddata); hypre_ParVectorDestroy(x); hypre_ParVectorDestroy(y); /* --------------------------------------------- */ /* build block matrix */ /* --------------------------------------------- */ rstarts = hypre_CTAlloc(HYPRE_Int, nprocs+1); for (ii = 0; ii <= nprocs; ii++) rstarts[ii] = row_starts[ii]; cstarts = hypre_CTAlloc(HYPRE_Int, nprocs+1); for (ii = 0; ii <= nprocs; ii++) cstarts[ii] = col_starts[ii]; par_blk_matrix = hypre_ParCSRBlockMatrixCreate(hypre_MPI_COMM_WORLD,block_size, global_num_rows, global_num_cols, rstarts, cstarts, num_cols_offd, num_nonzeros_diag, num_nonzeros_offd); colMap = hypre_ParCSRMatrixColMapOffd(par_matrix); if (num_cols_offd > 0) colMap2 = hypre_CTAlloc(HYPRE_Int, num_cols_offd); else colMap2 = NULL; for (ii = 0; ii < num_cols_offd; ii++) colMap2[ii] = colMap[ii]; hypre_ParCSRBlockMatrixColMapOffd(par_blk_matrix) = colMap2; diag_i = hypre_CSRMatrixI(hypre_ParCSRMatrixDiag(par_matrix)); diag_j = hypre_CSRMatrixJ(hypre_ParCSRMatrixDiag(par_matrix)); diag_d = hypre_CSRMatrixData(hypre_ParCSRMatrixDiag(par_matrix)); diag = hypre_ParCSRBlockMatrixDiag(par_blk_matrix); diag_i2 = hypre_CTAlloc(HYPRE_Int, local_size+1); diag_j2 = hypre_CTAlloc(HYPRE_Int, num_nonzeros_diag); diag_d2 = hypre_CTAlloc(double, num_nonzeros_diag*bnnz); for (ii = 0; ii <= local_size; ii++) diag_i2[ii] = diag_i[ii]; for (ii = 0; ii < num_nonzeros_diag; ii++) diag_j2[ii] = diag_j[ii]; hypre_CSRBlockMatrixI(diag) = diag_i2; hypre_CSRBlockMatrixJ(diag) = diag_j2; for (ii = 0; ii < num_nonzeros_diag; ii++) { for (jj = 0; jj < block_size; jj++) for (kk = 0; kk < block_size; kk++) { if (jj <= kk) diag_d2[ii*bnnz+jj*block_size+kk] = diag_d[ii]; else diag_d2[ii*bnnz+jj*block_size+kk] = 0.0; } } hypre_CSRBlockMatrixData(diag) = diag_d2; offd_i = hypre_CSRMatrixI(hypre_ParCSRMatrixOffd(par_matrix)); offd_j = hypre_CSRMatrixJ(hypre_ParCSRMatrixOffd(par_matrix)); offd_d = hypre_CSRMatrixData(hypre_ParCSRMatrixOffd(par_matrix)); offd = hypre_ParCSRBlockMatrixOffd(par_blk_matrix); offd_i2 = hypre_CTAlloc(HYPRE_Int, local_size+1); for (ii = 0; ii <= local_size; ii++) offd_i2[ii] = offd_i[ii]; hypre_CSRBlockMatrixI(offd) = offd_i2; if (num_cols_offd) { offd_j2 = hypre_CTAlloc(HYPRE_Int, num_nonzeros_offd); for (ii = 0; ii < num_nonzeros_offd; ii++) offd_j2[ii] = offd_j[ii]; hypre_CSRBlockMatrixJ(offd) = offd_j2; offd_d2 = hypre_CTAlloc(double, num_nonzeros_offd*bnnz); for (ii = 0; ii < num_nonzeros_offd; ii++) { for (jj = 0; jj < block_size; jj++) for (kk = 0; kk < block_size; kk++) { if (jj <= kk) offd_d2[ii*bnnz+jj*block_size+kk] = offd_d[ii]; else offd_d2[ii*bnnz+jj*block_size+kk] = 0.0; } } hypre_CSRBlockMatrixData(offd) = offd_d2; } else {
hypre_ParCSRBlockMatrix * hypre_ParCSRBlockMatrixConvertFromParCSRMatrix(hypre_ParCSRMatrix *matrix, HYPRE_Int matrix_C_block_size ) { MPI_Comm comm = hypre_ParCSRMatrixComm(matrix); hypre_CSRMatrix *diag = hypre_ParCSRMatrixDiag(matrix); hypre_CSRMatrix *offd = hypre_ParCSRMatrixOffd(matrix); HYPRE_Int global_num_rows = hypre_ParCSRMatrixGlobalNumRows(matrix); HYPRE_Int global_num_cols = hypre_ParCSRMatrixGlobalNumCols(matrix); HYPRE_Int *row_starts = hypre_ParCSRMatrixRowStarts(matrix); HYPRE_Int *col_starts = hypre_ParCSRMatrixColStarts(matrix); HYPRE_Int num_cols_offd = hypre_CSRMatrixNumCols(offd); HYPRE_Int *col_map_offd = hypre_ParCSRBlockMatrixColMapOffd(matrix); HYPRE_Int *map_to_node=NULL, *counter=NULL, *col_in_j_map=NULL; HYPRE_Int *matrix_C_col_map_offd = NULL; HYPRE_Int matrix_C_num_cols_offd; HYPRE_Int matrix_C_num_nonzeros_offd; HYPRE_Int num_rows, num_nodes; HYPRE_Int *offd_i = hypre_CSRMatrixI(offd); HYPRE_Int *offd_j = hypre_CSRMatrixJ(offd); HYPRE_Complex * offd_data = hypre_CSRMatrixData(offd); hypre_ParCSRBlockMatrix *matrix_C; HYPRE_Int *matrix_C_row_starts; HYPRE_Int *matrix_C_col_starts; hypre_CSRBlockMatrix *matrix_C_diag; hypre_CSRBlockMatrix *matrix_C_offd; HYPRE_Int *matrix_C_offd_i=NULL, *matrix_C_offd_j = NULL; HYPRE_Complex *matrix_C_offd_data = NULL; HYPRE_Int num_procs, i, j, k, k_map, count, index, start_index, pos, row; hypre_MPI_Comm_size(comm,&num_procs); #ifdef HYPRE_NO_GLOBAL_PARTITION matrix_C_row_starts = hypre_CTAlloc(HYPRE_Int, 2); matrix_C_col_starts = hypre_CTAlloc(HYPRE_Int, 2); for(i = 0; i < 2; i++) { matrix_C_row_starts[i] = row_starts[i]/matrix_C_block_size; matrix_C_col_starts[i] = col_starts[i]/matrix_C_block_size; } #else matrix_C_row_starts = hypre_CTAlloc(HYPRE_Int, num_procs + 1); matrix_C_col_starts = hypre_CTAlloc(HYPRE_Int, num_procs + 1); for(i = 0; i < num_procs + 1; i++) { matrix_C_row_starts[i] = row_starts[i]/matrix_C_block_size; matrix_C_col_starts[i] = col_starts[i]/matrix_C_block_size; } #endif /************* create the diagonal part ************/ matrix_C_diag = hypre_CSRBlockMatrixConvertFromCSRMatrix(diag, matrix_C_block_size); /******* the offd part *******************/ /* can't use the same function for the offd part - because this isn't square and the offd j entries aren't global numbering (have to consider the offd map) - need to look at col_map_offd first */ /* figure out the new number of offd columns (num rows is same as diag) */ num_cols_offd = hypre_CSRMatrixNumCols(offd); num_rows = hypre_CSRMatrixNumRows(diag); num_nodes = num_rows/matrix_C_block_size; matrix_C_offd_i = hypre_CTAlloc(HYPRE_Int, num_nodes + 1); matrix_C_num_cols_offd = 0; matrix_C_offd_i[0] = 0; matrix_C_num_nonzeros_offd = 0; if (num_cols_offd) { map_to_node = hypre_CTAlloc(HYPRE_Int, num_cols_offd); matrix_C_num_cols_offd = 1; map_to_node[0] = col_map_offd[0]/matrix_C_block_size; for (i=1; i < num_cols_offd; i++) { map_to_node[i] = col_map_offd[i]/matrix_C_block_size; if (map_to_node[i] > map_to_node[i-1]) matrix_C_num_cols_offd++; } matrix_C_col_map_offd = hypre_CTAlloc(HYPRE_Int, matrix_C_num_cols_offd); col_in_j_map = hypre_CTAlloc(HYPRE_Int, num_cols_offd); matrix_C_col_map_offd[0] = map_to_node[0]; col_in_j_map[0] = 0; count = 1; j = 1; /* fill in the col_map_off_d - these are global numbers. Then we need to map these to j entries (these have local numbers) */ for (i=1; i < num_cols_offd; i++) { if (map_to_node[i] > map_to_node[i-1]) { matrix_C_col_map_offd[count++] = map_to_node[i]; } col_in_j_map[j++] = count - 1; } /* now figure the nonzeros */ matrix_C_num_nonzeros_offd = 0; counter = hypre_CTAlloc(HYPRE_Int, matrix_C_num_cols_offd); for (i=0; i < matrix_C_num_cols_offd; i++) counter[i] = -1; for (i=0; i < num_nodes; i++) /* for each block row */ { matrix_C_offd_i[i] = matrix_C_num_nonzeros_offd; for (j=0; j < matrix_C_block_size; j++) { row = i*matrix_C_block_size+j; for (k=offd_i[row]; k < offd_i[row+1]; k++) /* go through single row */ { k_map = col_in_j_map[offd_j[k]]; /*nodal col - see if this has been in this block row (i) already*/ if (counter[k_map] < i) /* not yet counted for this nodal row */ { counter[k_map] = i; matrix_C_num_nonzeros_offd++; } } } } /* fill in final i entry */ matrix_C_offd_i[num_nodes] = matrix_C_num_nonzeros_offd; } /* create offd matrix */ matrix_C_offd = hypre_CSRBlockMatrixCreate(matrix_C_block_size, num_nodes, matrix_C_num_cols_offd, matrix_C_num_nonzeros_offd); /* assign i */ hypre_CSRBlockMatrixI(matrix_C_offd) = matrix_C_offd_i; /* create (and allocate j and data) */ if (matrix_C_num_nonzeros_offd) { matrix_C_offd_j = hypre_CTAlloc(HYPRE_Int, matrix_C_num_nonzeros_offd); matrix_C_offd_data = hypre_CTAlloc(HYPRE_Complex, matrix_C_num_nonzeros_offd*matrix_C_block_size* matrix_C_block_size); hypre_CSRBlockMatrixJ(matrix_C_offd) = matrix_C_offd_j; hypre_CSRMatrixData(matrix_C_offd) = matrix_C_offd_data; for (i=0; i < matrix_C_num_cols_offd; i++) counter[i] = -1; index = 0; /*keep track of entry in matrix_C_offd_j*/ start_index = 0; for (i=0; i < num_nodes; i++) /* for each block row */ { for (j=0; j < matrix_C_block_size; j++) /* for each row in block */ { row = i*matrix_C_block_size+j; for (k=offd_i[row]; k < offd_i[row+1]; k++) /* go through single row's cols */ { k_map = col_in_j_map[offd_j[k]]; /*nodal col for off_d */ if (counter[k_map] < start_index) /* not yet counted for this nodal row */ { counter[k_map] = index; matrix_C_offd_j[index] = k_map; /*copy the data: which position (corresponds to j array) + which row + which col */ pos = (index * matrix_C_block_size * matrix_C_block_size) + (j * matrix_C_block_size) + col_map_offd[offd_j[k]]%matrix_C_block_size; matrix_C_offd_data[pos] = offd_data[k]; index ++; } else /* this col has already been listed for this row */ { /*copy the data: which position (corresponds to j array) + which row + which col */ pos = (counter[k_map]* matrix_C_block_size * matrix_C_block_size) + (j * matrix_C_block_size) + col_map_offd[offd_j[k]]%matrix_C_block_size; matrix_C_offd_data[pos] = offd_data[k]; } } } start_index = index; /* first index for current nodal row */ } } /* *********create the new matrix *************/ matrix_C = hypre_ParCSRBlockMatrixCreate(comm, matrix_C_block_size, global_num_rows/matrix_C_block_size, global_num_cols/matrix_C_block_size, matrix_C_row_starts, matrix_C_col_starts, matrix_C_num_cols_offd, hypre_CSRBlockMatrixNumNonzeros(matrix_C_diag), matrix_C_num_nonzeros_offd); /* use the diag and off diag matrices we have already created */ hypre_CSRBlockMatrixDestroy(hypre_ParCSRMatrixDiag(matrix_C)); hypre_ParCSRBlockMatrixDiag(matrix_C) = matrix_C_diag; hypre_CSRBlockMatrixDestroy(hypre_ParCSRMatrixOffd(matrix_C)); hypre_ParCSRBlockMatrixOffd(matrix_C) = matrix_C_offd; hypre_ParCSRMatrixColMapOffd(matrix_C) = matrix_C_col_map_offd; /* *********don't bother to copy the comm_pkg *************/ hypre_ParCSRBlockMatrixCommPkg(matrix_C) = NULL; /* CLEAN UP !!!! */ hypre_TFree(map_to_node); hypre_TFree(col_in_j_map); hypre_TFree(counter); return matrix_C; }