void AZ_factor_subdomain(struct context *context, int N, int N_nz,
int *nz_used)
{
/****************************************************************************
Given an overlapped subdomain matrix, factor it according to the
chosen algorithm and store the result back in subdomain. Additionally,
store the number of nonzeros used in the factorization in nz_used.
Notes:
1) Matrix comes in as an MSR matrix.
2) context contains several fields which need to be appropriately
set. These fields are specific to the individual solvers.
3) The factorization overwrites the matrix. However, different
solvers will store the factorization in different formats.
Author: Ray Tuminaro, SNL, 9222 (3/98)
Return code: void
============
Parameter list:
===============
context On input, context contains the matrix to be
factored in context.A_overlapped (MSR format),
On output, context contains the factored matrix
which is stored in a format specific to the solver and
any additional parameters required by the backsolver.
N On input, the size of the linear system to be solved.
N_nz On input, the number of nonzero values in the matrix
to be factored.
nz_used On output, the number of nonzero values in the matrix
representing the factorization.
*******************************************************************************/
#ifdef HAVE_AZLU
int ifail, N_nz_matrix, *rnr;
double *fake_rhs, *aflag;
#endif
int i, j, *bindx, *bpntr, *iw;
double *cr, *unorm, *a, *val;
int *ind, *jnz, *ja, ifill;
double dtemp = (context->aztec_choices->params)[AZ_omega];
int N_blk_rows, name = context->A_overlapped->data_org[AZ_name];
char str[80];
/* Begin Aztec 2.1 mheroux mod */
#ifdef IFPACK
void *precon, *bmat;
double rthresh, athresh;
int N_int_blk, N_bord_blk, graph_fill;
#endif
/* End Aztec 2.1 mheroux mod */
bindx = context->A_overlapped->bindx;
*nz_used = bindx[N];
switch(context->aztec_choices->options[AZ_subdomain_solve]) {
/* Begin Aztec 2.1 mheroux mod */
case AZ_bilu_ifp:
#ifdef IFPACK
if (N == 0) return;
bindx = context->A_overlapped->bindx;
val = context->A_overlapped->val;
/* for bilu(k) with k > 1 , figure out the new sparsity pattern */
AZ_sort_msr(bindx, val, N);
/* Let IFPACK handle fillin */
graph_fill = (context->aztec_choices->options)[AZ_graph_fill];
(context->aztec_choices->options)[AZ_graph_fill] = 0;
/* recover some space so that there will */
/* be enough room to convert back to vbr */
i = AZ_compress_msr(&(context->A_overlapped->bindx),
&(context->A_overlapped->val), context->N_nz_allocated,
*nz_used, name, context);
context->N_nz = *nz_used;
context->N_nz_allocated = *nz_used;
AZ_msr2vbr_mem_efficient(N, &(context->A_overlapped->bindx),
&(context->A_overlapped->val),
&(context->A_overlapped->cpntr),
&(context->A_overlapped->bpntr),
&(context->A_overlapped->indx), &N_blk_rows,
(context->A_overlapped->data_org)[AZ_name],
context->tag,i);
context->A_overlapped->matrix_type = AZ_VBR_MATRIX;
/*ifp_initialize();*/
/* Create IFPACK encapsulation of Amat */
context->A_overlapped->rpntr = context->A_overlapped->cpntr;
N_int_blk = context->A_overlapped->data_org[AZ_N_int_blk];
N_bord_blk = context->A_overlapped->data_org[AZ_N_bord_blk];
context->A_overlapped->data_org[AZ_N_int_blk] = N_blk_rows;
context->A_overlapped->data_org[AZ_N_bord_blk] = 0;
(context->aztec_choices->options)[AZ_graph_fill] = graph_fill;
az2ifp_blockmatrix(&bmat, context->A_overlapped);
context->A_overlapped->data_org[AZ_N_int_blk] = N_int_blk;
context->A_overlapped->data_org[AZ_N_bord_blk] = N_bord_blk;
rthresh = (context->aztec_choices->params)[AZ_rthresh];
athresh = (context->aztec_choices->params)[AZ_athresh];
ifill = (context->aztec_choices->options)[AZ_graph_fill];
ifp_preconditioner(&precon, bmat, IFP_BILUK, (double) ifill, 0.0,
IFP_SVD, rthresh, athresh);
if ((context->aztec_choices->options)[AZ_output]>0) {
ifp_biluk_stats(precon);
}
context->precon = precon;
break;
/* End Aztec 2.1 mheroux mod */
#else
AZ_perror("IFPACK not linked. Must compile with -DIFPACK");
#endif
case AZ_bilu:
if (N == 0) return;
bindx = context->A_overlapped->bindx;
val = context->A_overlapped->val;
/* for bilu(k) with k > 1 , figure out the new sparsity pattern */
AZ_sort_msr(bindx, val, N);
ifill = (context->aztec_choices->options)[AZ_graph_fill];
if (ifill > 0) {
*nz_used = AZ_fill_sparsity_pattern(context, ifill,
bindx, val, N);
}
/* recover some space so that there will */
/* be enough room to convert back to vbr */
i = AZ_compress_msr(&(context->A_overlapped->bindx),
&(context->A_overlapped->val), context->N_nz_allocated,
*nz_used, name, context);
context->N_nz = *nz_used;
context->N_nz_allocated = *nz_used;
AZ_msr2vbr_mem_efficient(N, &(context->A_overlapped->bindx),
&(context->A_overlapped->val),
&(context->A_overlapped->cpntr),
&(context->A_overlapped->bpntr),
&(context->A_overlapped->indx), &N_blk_rows,
(context->A_overlapped->data_org)[AZ_name],
context->tag,i);
context->A_overlapped->matrix_type = AZ_VBR_MATRIX;
bindx = context->A_overlapped->bindx;
bpntr = context->A_overlapped->bpntr;
val = context->A_overlapped->val;
sprintf(str,"ipvt %s",context->tag);
context->ipvt = (int *) AZ_manage_memory((N+1)*sizeof(int),
AZ_ALLOC, name, str, &i);
sprintf(str,"dblock %s",context->tag);
context->dblock= (int *) AZ_manage_memory((N_blk_rows+1)*
sizeof(int), AZ_ALLOC, name,
str, &i);
context->N_blk_rows = N_blk_rows;
/* set dblock to point to the diagonal block in each block row */
for (i = 0 ; i < N_blk_rows ; i++ ) {
for (j = bpntr[i] ; j < bpntr[i+1] ; j++ ) {
if (bindx[j] == i) context->dblock[i] = j;
}
}
AZ_fact_bilu(N_blk_rows, context->A_overlapped, context->dblock,
context->ipvt);
break;
case AZ_ilut:
cr = (double *) AZ_allocate((2*N+3+context->max_row)*sizeof(int)+
(2*N+2+context->max_row)*sizeof(double));
if (cr == NULL) AZ_perror("Out of space in ilut.\n");
unorm = &(cr[N+2]);
a = &(unorm[N]);
ind = (int *) &(a[context->max_row]);
jnz = &(ind[N+3]);
ja = &(jnz[N]);
sprintf(str,"iu %s",context->tag);
context->iu = (int *) AZ_manage_memory((N+1)*sizeof(int),
AZ_ALLOC, name, str, &i);
AZ_fact_ilut(&N, context->A_overlapped, a, ja,
(context->aztec_choices->params)[AZ_drop],
context->extra_fact_nz_per_row, N_nz - bindx[N],
context->iu,cr,unorm,ind, nz_used, jnz,
(context->aztec_choices->params)[AZ_rthresh],
(context->aztec_choices->params)[AZ_athresh]);
AZ_free(cr);
break;
case AZ_ilu:
dtemp = 0.0;
case AZ_rilu:
if (N == 0) return;
sprintf(str,"iu %s",context->tag);
bindx = context->A_overlapped->bindx;
val = context->A_overlapped->val;
/* for ilu(k) with k > 1 , figure out the new sparsity pattern */
AZ_sort_msr(bindx, val, N);
ifill = (context->aztec_choices->options)[AZ_graph_fill];
if (ifill > 0) {
*nz_used = AZ_fill_sparsity_pattern(context, ifill,
bindx, val, N);
}
context->iu= (int *) AZ_manage_memory((N+1)*sizeof(int),AZ_ALLOC,
name, str, &i);
iw = (int *) AZ_allocate((N+1)*sizeof(int));
if (iw == NULL) AZ_perror("Out of space in ilu.\n");
AZ_fact_rilu(N, nz_used, context->iu, iw, context->A_overlapped,
dtemp,
(context->aztec_choices->params)[AZ_rthresh],
(context->aztec_choices->params)[AZ_athresh]);
AZ_free(iw);
break;
case AZ_icc:
sprintf(str,"iu %s",context->tag);
bindx = context->A_overlapped->bindx;
val = context->A_overlapped->val;
/* for ilu(k) with k > 1 , figure out the new sparsity pattern */
AZ_sort_msr(bindx, val, N);
ifill = (context->aztec_choices->options)[AZ_graph_fill];
if (ifill > 0)
*nz_used = AZ_fill_sparsity_pattern(context, ifill,
bindx, val, N);
AZ_fact_chol(context->A_overlapped->bindx,
context->A_overlapped->val,N,
(context->aztec_choices->params)[AZ_rthresh],
(context->aztec_choices->params)[AZ_athresh]);
break;
case AZ_lu:
#ifdef HAVE_AZLU
if (N == 0) return;
aflag = (double *) AZ_allocate(8*sizeof(double));
rnr = (int *) AZ_allocate(N_nz*sizeof(int));
if (rnr == NULL) AZ_perror("Out of space in lu.\n");
sprintf(str,"iflag %s",context->tag);
context->iflag = (int *) AZ_manage_memory(10*sizeof(int), AZ_ALLOC,
name, str ,&i);
sprintf(str,"ha %s",context->tag);
context->ha = (int *) AZ_manage_memory(11*(N+1)*sizeof(int),
AZ_ALLOC, name, str, &i);
sprintf(str,"pivot %s",context->tag);
context->pivot = (double *) AZ_manage_memory((N+1)*sizeof(double),
AZ_ALLOC, name, str,&i);
aflag[0] = 16.0; aflag[2] = 1.0e8; aflag[3] = 1.0e-12;
aflag[1] = (context->aztec_choices->params)[AZ_drop];
/* set up flags for the sparse factorization solver */
context->iflag[0] = 1; context->iflag[1] = 2;
context->iflag[2] = 1; context->iflag[3] = 0;
context->iflag[4] = 2;
/* Note: if matrix is pos def, iflag[2] = 2 is cheaper */
N_nz_matrix = bindx[N] - 1;
AZ_msr2lu(N, context->A_overlapped, rnr);
/* Mark bindx so we can see what was not used later */
for (i = N_nz_matrix ; i < N_nz ; i++) bindx[i] = -7;
/* factor the matrix */
if (N == 1) {
context->A_overlapped->val[0]=1./context->A_overlapped->val[0];
}
else {
context->N_nz_factors = N_nz;
fake_rhs = (double *) AZ_allocate(N*sizeof(double));
if (fake_rhs == NULL) {
AZ_printf_out("Not enough memory inside subdomain_solve\n");
}
for (i = 0 ; i < N ; i++ ) fake_rhs[i] = 0.0;
AZ_fact_lu(fake_rhs, context->A_overlapped,aflag,
context->pivot, rnr, context->ha,
context->iflag, &N_nz_matrix,
&ifail, &(context->N_nz_factors),
&N, &N);
(context->iflag)[4] = 3;
AZ_free(fake_rhs);
/* find out what was not used by checking what was not touched */
*nz_used = N_nz;
for (i = N_nz_matrix; i < N_nz ; i++ ) {
if (bindx[i] != -7) *nz_used = i;
}
(*nz_used)++;
context->N_nz_factors = *nz_used;
}
AZ_free(rnr);
AZ_free(aflag);
#else
AZ_printf_err("AZ_lu unavailable: configure with --enable-aztecoo-azlu to make available\n");
exit(1);
#endif
break;
default:
if (context->aztec_choices->options[AZ_subdomain_solve]
>= AZ_SOLVER_PARAMS) {
AZ_printf_err("Unknown subdomain solver(%d)\n",
context->aztec_choices->options[AZ_subdomain_solve]);
exit(1);
}
}
}
void AZ_domain_decomp(double x[], AZ_MATRIX *Amat, int options[],
int proc_config[], double params[],
struct context *context)
/*******************************************************************************
Precondition 'x' using an overlapping domain decomposition method where a
solver specified by options[AZ_subdomain_solve] is used on the subdomains.
Note: if a factorization needs to be computed on the first iteration, this
will be done and stored for future iterations.
Author: Lydie Prevost, SNL, 9222
======= Revised by R. Tuminaro (4/97), SNL, 9222
Return code: void
============
Parameter list:
===============
N_unpadded: On input, number of rows in linear system (unpadded matrix)
that will be factored (after adding values for overlapping).
Nb_unpadded: On input, number of block rows in linear system (unpadded)
that will be factored (after adding values for overlapping).
N_nz_unpadded: On input, number of nonzeros in linear system (unpadded)
that will be factored (after adding values for overlapping).
x: On output, x[] is preconditioned by performing the subdomain
solve indicated by options[AZ_subdomain_solve].
val indx
bindx rpntr: On input, arrays containing matrix nonzeros (see manual).
cpntr bpntr
options: Determines specific solution method and other parameters. In
this routine, we are concerned with options[AZ_overlap]:
== AZ_none: nonoverlapping domain decomposition
== AZ_diag: use rows corresponding to external variables
but only keep the diagonal for these rows.
== k : Obtain rows that are a distance k away from
rows owned by this processor.
data_org: Contains information on matrix data distribution and
communication parameters (see manual).
*******************************************************************************/
{
int N_unpadded, Nb_unpadded, N_nz_unpadded;
double *x_pad = NULL, *x_reord = NULL, *ext_vals = NULL;
int N_nz, N_nz_padded, nz_used;
int mem_orig, mem_overlapped, mem_factor;
int name, i, bandwidth;
int *ordering = NULL;
double condest;
/*
double start_t;
*/
int estimated_requirements;
char str[80];
int *garbage;
int N;
int *padded_data_org = NULL, *bindx, *data_org;
double *val;
int *inv_ordering = NULL;
int *map = NULL;
AZ_MATRIX *A_overlapped = NULL;
struct aztec_choices aztec_choices;
/**************************** execution begins ******************************/
data_org = Amat->data_org;
bindx = Amat->bindx;
val = Amat->val;
N_unpadded = data_org[AZ_N_internal] + data_org[AZ_N_border];
Nb_unpadded = data_org[AZ_N_int_blk]+data_org[AZ_N_bord_blk];
if (data_org[AZ_matrix_type] == AZ_MSR_MATRIX)
N_nz_unpadded = bindx[N_unpadded];
else if (data_org[AZ_matrix_type] == AZ_VBR_MATRIX)
N_nz_unpadded = (Amat->indx)[(Amat->bpntr)[Nb_unpadded]];
else {
if (Amat->N_nz < 0)
AZ_matfree_Nnzs(Amat);
N_nz_unpadded = Amat->N_nz;
}
aztec_choices.options = options;
aztec_choices.params = params;
context->aztec_choices = &aztec_choices;
context->proc_config = proc_config;
name = data_org[AZ_name];
if ((options[AZ_pre_calc] >= AZ_reuse) && (context->Pmat_computed)) {
N = context->N;
N_nz = context->N_nz;
A_overlapped = context->A_overlapped;
A_overlapped->data_org = data_org;
A_overlapped->matvec = Amat->matvec;
}
else {
sprintf(str,"A_over %s",context->tag);
A_overlapped = (AZ_MATRIX *) AZ_manage_memory(sizeof(AZ_MATRIX),
AZ_ALLOC, name, str, &i);
AZ_matrix_init(A_overlapped, 0);
context->A_overlapped = A_overlapped;
A_overlapped->data_org = data_org;
A_overlapped->matvec = Amat->matvec;
A_overlapped->matrix_type = AZ_MSR_MATRIX;
AZ_init_subdomain_solver(context);
AZ_compute_matrix_size(Amat, options, N_nz_unpadded, N_unpadded,
&N_nz_padded, data_org[AZ_N_external],
&(context->max_row), &N, &N_nz, params[AZ_ilut_fill],
&(context->extra_fact_nz_per_row),
Nb_unpadded,&bandwidth);
estimated_requirements = N_nz;
if (N_nz*2 > N_nz) N_nz = N_nz*2; /* check for overflow */
/* Add extra memory to N_nz. */
/* This extra memory is used */
/* as temporary space during */
/* overlapping calculation */
/* Readjust N_nz by allocating auxilliary arrays and allocate */
/* the MSR matrix to check that there is enough space. */
/* block off some space for map and padded_data_org in dd_overlap */
garbage = (int *) AZ_allocate((AZ_send_list + 2*(N-N_unpadded) +10)*
sizeof(int));
AZ_hold_space(context, N);
if (N_nz*((int) sizeof(double)) < N_nz) N_nz=N_nz/2; /* check for overflow */
if (N_nz*((int) sizeof(double)) < N_nz) N_nz=N_nz/2; /* check for overflow */
if (N_nz*((int) sizeof(double)) < N_nz) N_nz=N_nz/2; /* check for overflow */
if (N_nz*((int) sizeof(double)) < N_nz) N_nz=N_nz/2; /* check for overflow */
if (N_nz*((int) sizeof(double)) < N_nz) N_nz=N_nz/2; /* check for overflow */
N_nz = AZ_adjust_N_nz_to_fit_memory(N_nz,
context->N_large_int_arrays,
context->N_large_dbl_arrays);
context->N_nz_factors = N_nz;
if (N_nz <= N_nz_unpadded) {
AZ_printf_out("Error: Not enough space for domain decomposition\n");
AZ_exit(1);
}
if (estimated_requirements > N_nz ) estimated_requirements = N_nz;
/* allocate matrix via AZ_manage_memory() */
sprintf(str,"bindx %s",context->tag);
A_overlapped->bindx =(int *) AZ_manage_memory(N_nz*sizeof(int),
AZ_ALLOC, name, str, &i);
sprintf(str,"val %s",context->tag);
A_overlapped->val =(double *) AZ_manage_memory(N_nz*sizeof(double),
AZ_ALLOC, name, str, &i);
context->N_nz_allocated = N_nz;
AZ_free_space_holder(context);
AZ_free(garbage);
/* convert to MSR if necessary */
if (data_org[AZ_matrix_type] == AZ_VBR_MATRIX)
AZ_vb2msr(Nb_unpadded,val,Amat->indx,bindx,Amat->rpntr,Amat->cpntr,
Amat->bpntr, A_overlapped->val, A_overlapped->bindx);
else if (data_org[AZ_matrix_type] == AZ_MSR_MATRIX) {
for (i = 0 ; i < N_nz_unpadded; i++ ) {
A_overlapped->bindx[i] = bindx[i];
A_overlapped->val[i] = val[i];
}
}
else AZ_matfree_2_msr(Amat,A_overlapped->val,A_overlapped->bindx,N_nz);
mem_orig = AZ_gsum_int(A_overlapped->bindx[N_unpadded],proc_config);
/*
start_t = AZ_second();
*/
AZ_pad_matrix(context, proc_config, N_unpadded, &N,
&(context->map), &(context->padded_data_org), &N_nz,
estimated_requirements);
/*
if (proc_config[AZ_node] == 0)
AZ_printf_out("matrix padding took %e seconds\n",AZ_second()-start_t);
*/
mem_overlapped = AZ_gsum_int(A_overlapped->bindx[N],proc_config);
if (options[AZ_reorder]) {
/*
start_t = AZ_second();
*/
AZ_find_MSR_ordering(A_overlapped->bindx,
&(context->ordering),N,
&(context->inv_ordering),name,context);
/*
if (proc_config[AZ_node] == 0)
AZ_printf_out("took %e seconds to find ordering\n", AZ_second()-start_t);
*/
/*
start_t = AZ_second();
*/
AZ_mat_reorder(N,A_overlapped->bindx,A_overlapped->val,
context->ordering, context->inv_ordering);
/*
if (proc_config[AZ_node] == 0)
AZ_printf_out("took %e seconds to reorder\n", AZ_second()-start_t);
*/
/* NOTE: ordering is freed inside AZ_mat_reorder */
#ifdef AZ_COL_REORDER
if (options[AZ_reorder]==2) {
AZ_mat_colperm(N,A_overlapped->bindx,A_overlapped->val,
&(context->ordering), name, context );
}
#endif
}
/* Do a factorization if needed. */
/*
start_t = AZ_second();
*/
AZ_factor_subdomain(context, N, N_nz, &nz_used);
if (options[AZ_output] > 0 && options[AZ_diagnostics]!=AZ_none) {
AZ_printf_out("\n*********************************************************************\n");
condest = AZ_condest(N, context);
AZ_printf_out("***** Condition number estimate for subdomain preconditioner on PE %d = %.4e\n",
proc_config[AZ_node], condest);
AZ_printf_out("*********************************************************************\n");
}
/*
start_t = AZ_second()-start_t;
max_time = AZ_gmax_double(start_t,proc_config);
min_time = AZ_gmin_double(start_t,proc_config);
if (proc_config[AZ_node] == 0)
AZ_printf_out("time for subdomain solvers ranges from %e to %e\n",
min_time,max_time);
*/
if ( A_overlapped->matrix_type == AZ_MSR_MATRIX)
AZ_compress_msr(&(A_overlapped->bindx), &(A_overlapped->val),
context->N_nz_allocated, nz_used, name, context);
context->N_nz = nz_used;
context->N = N;
context->N_nz_allocated = nz_used;
mem_factor = AZ_gsum_int(nz_used,proc_config);
if (proc_config[AZ_node] == 0)
AZ_print_header(options,mem_overlapped,mem_orig,mem_factor);
if (options[AZ_overlap] >= 1) {
sprintf(str,"x_pad %s",context->tag);
context->x_pad = (double *) AZ_manage_memory(N*sizeof(double),
AZ_ALLOC, name, str, &i);
sprintf(str,"ext_vals %s",context->tag);
context->ext_vals = (double *) AZ_manage_memory((N-N_unpadded)*
sizeof(double), AZ_ALLOC, name,
str, &i);
}
if (options[AZ_reorder]) {
sprintf(str,"x_reord %s",context->tag);
context->x_reord = (double *) AZ_manage_memory(N*sizeof(double),
AZ_ALLOC, name, str, &i);
}
}
/* Solve L u = x where the solution u overwrites x */
x_reord = context->x_reord;
inv_ordering = context->inv_ordering;
ordering = context->ordering;
x_pad = context->x_pad;
ext_vals = context->ext_vals;
padded_data_org = context->padded_data_org;
map = context->map;
if (x_pad == NULL) x_pad = x;
if (options[AZ_overlap] >= 1) {
for (i = 0 ; i < N_unpadded ; i++) x_pad[i] = x[i];
AZ_exchange_bdry(x_pad,padded_data_org, proc_config);
for (i = 0 ; i < N-N_unpadded ; i++ )
ext_vals[map[i]-N_unpadded] = x_pad[i+N_unpadded];
for (i = 0 ; i < N-N_unpadded ; i++ ) x_pad[i + N_unpadded] = ext_vals[i];
}
else if (options[AZ_overlap] == AZ_diag)
AZ_exchange_bdry(x_pad,data_org, proc_config);
if (x_reord == NULL) x_reord = x_pad;
if (options[AZ_reorder]) {
/* Apply row permutation to the right hand side */
/* ((P'A P)Pi') Pi P'x = P'rhs, b= P'rhs */
for (i = 0 ; i < N ; i++ ) x_reord[inv_ordering[i]] = x_pad[i];
}
AZ_solve_subdomain(x_reord,N, context);
#ifdef AZ_COL_REORDER
/* Apply column permutation to the solution */
if (options[AZ_reorder]==1){
/* ((P'A P) P'sol = P'rhs sol = P( P'sol) */
for (i = 0; i < N; i++) x_pad[i] = x_reord[inv_ordering[i]];
}
if (options[AZ_reorder]==2){
/*
* ((P'A P)Pi') Pi P'sol = P'rhs sol = P Pi'( Pi P'sol)
* Version 1:
* for (i = 0; i < N; i++) pi_sol[i] = x_reord[ordering[i]];
* for (j = 0; j < N; j++) x_pad[j] = pi_sol[inv_ordering[j]];
* Version 2:
*/
for (i = 0; i < N; i++) x_pad[i] = x_reord[ ordering[inv_ordering[i]] ];
}
#else
if (options[AZ_reorder])
for (i = 0; i < N; i++) x_pad[i] = x_reord[inv_ordering[i]];
#endif
AZ_combine_overlapped_values(options[AZ_type_overlap],padded_data_org,
options, x_pad, map,ext_vals,name,proc_config);
if (x_pad != x)
for (i = 0 ; i < N_unpadded ; i++ ) x[i] = x_pad[i];
} /* subdomain driver*/
