int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt nlocal,local[5],nneigh,*neigh,**ineigh,*numneigh;
ISLocalToGlobalMapping mapping;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 3) SETERRQ(PETSC_COMM_SELF,1,"Must run with three processors");
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (!rank) {
nlocal = 4; local[0] = 0; local[1] = 3; local[2] = 2; local[3] = 1;
} else if (rank == 1) {
nlocal = 4; local[0] = 3; local[1] = 5; local[2] = 4; local[3] = 2;
} else {
nlocal = 4; local[0] = 7; local[1] = 6; local[2] = 5; local[3] = 3;
}
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,nlocal,local,PETSC_COPY_VALUES,&mapping);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetInfo(mapping,&nneigh,&neigh,&numneigh,&ineigh);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreInfo(mapping,&nneigh,&neigh,&numneigh,&ineigh);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&mapping);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PETSC_EXTERN void PETSC_STDCALL islocaltoglobalmpnggetinfosize_(ISLocalToGlobalMapping *mapping,PetscInt *size,PetscInt *maxnumprocs,PetscErrorCode *ierr)
{
PetscInt i;
if (called) {*ierr = PETSC_ERR_ARG_WRONGSTATE; return;}
*ierr = ISLocalToGlobalMappingGetInfo(*mapping,size,&sprocs,&snumprocs,&sindices); if (*ierr) return;
*maxnumprocs = 0;
for (i=0; i<*size; i++) *maxnumprocs = PetscMax(*maxnumprocs,snumprocs[i]);
called = PETSC_TRUE;
}
PetscErrorCode PCISSetUp(PC pc)
{
PC_IS *pcis = (PC_IS*)(pc->data);
Mat_IS *matis;
PetscErrorCode ierr;
PetscBool flg,issbaij;
Vec counter;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)pc->pmat,MATIS,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONG,"Preconditioner type of Neumann Neumman requires matrix of type MATIS");
matis = (Mat_IS*)pc->pmat->data;
pcis->pure_neumann = matis->pure_neumann;
/* get info on mapping */
ierr = PetscObjectReference((PetscObject)matis->mapping);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&pcis->mapping);CHKERRQ(ierr);
pcis->mapping = matis->mapping;
ierr = ISLocalToGlobalMappingGetSize(pcis->mapping,&pcis->n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetInfo(pcis->mapping,&(pcis->n_neigh),&(pcis->neigh),&(pcis->n_shared),&(pcis->shared));CHKERRQ(ierr);
/* Creating local and global index sets for interior and inteface nodes. */
{
PetscInt n_I;
PetscInt *idx_I_local,*idx_B_local,*idx_I_global,*idx_B_global;
PetscInt *array;
PetscInt i,j;
/* Identifying interior and interface nodes, in local numbering */
ierr = PetscMalloc1(pcis->n,&array);CHKERRQ(ierr);
ierr = PetscMemzero(array,pcis->n*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0;i<pcis->n_neigh;i++)
for (j=0;j<pcis->n_shared[i];j++)
array[pcis->shared[i][j]] += 1;
ierr = PetscMalloc1(pcis->n,&idx_I_local);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n,&idx_B_local);CHKERRQ(ierr);
for (i=0, pcis->n_B=0, n_I=0; i<pcis->n; i++) {
if (!array[i]) {
idx_I_local[n_I] = i;
n_I++;
} else {
idx_B_local[pcis->n_B] = i;
pcis->n_B++;
}
}
/* Getting the global numbering */
idx_B_global = idx_I_local + n_I; /* Just avoiding allocating extra memory, since we have vacant space */
idx_I_global = idx_B_local + pcis->n_B;
ierr = ISLocalToGlobalMappingApply(pcis->mapping,pcis->n_B,idx_B_local,idx_B_global);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(pcis->mapping,n_I, idx_I_local,idx_I_global);CHKERRQ(ierr);
/* Creating the index sets. */
ierr = ISCreateGeneral(PETSC_COMM_SELF,pcis->n_B,idx_B_local,PETSC_COPY_VALUES, &pcis->is_B_local);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,pcis->n_B,idx_B_global,PETSC_COPY_VALUES,&pcis->is_B_global);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n_I,idx_I_local,PETSC_COPY_VALUES, &pcis->is_I_local);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n_I,idx_I_global,PETSC_COPY_VALUES,&pcis->is_I_global);CHKERRQ(ierr);
/* Freeing memory and restoring arrays */
ierr = PetscFree(idx_B_local);CHKERRQ(ierr);
ierr = PetscFree(idx_I_local);CHKERRQ(ierr);
ierr = PetscFree(array);CHKERRQ(ierr);
}
/*
Extracting the blocks A_II, A_BI, A_IB and A_BB from A. If the numbering
is such that interior nodes come first than the interface ones, we have
[ | ]
[ A_II | A_IB ]
A = [ | ]
[-----------+------]
[ A_BI | A_BB ]
*/
ierr = MatGetSubMatrix(matis->A,pcis->is_I_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_II);CHKERRQ(ierr);
ierr = MatGetSubMatrix(matis->A,pcis->is_B_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_BB);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr);
if (!issbaij) {
ierr = MatGetSubMatrix(matis->A,pcis->is_I_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_IB);CHKERRQ(ierr);
ierr = MatGetSubMatrix(matis->A,pcis->is_B_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_BI);CHKERRQ(ierr);
} else {
Mat newmat;
ierr = MatConvert(matis->A,MATSEQBAIJ,MAT_INITIAL_MATRIX,&newmat);CHKERRQ(ierr);
ierr = MatGetSubMatrix(newmat,pcis->is_I_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_IB);CHKERRQ(ierr);
ierr = MatGetSubMatrix(newmat,pcis->is_B_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_BI);CHKERRQ(ierr);
ierr = MatDestroy(&newmat);CHKERRQ(ierr);
}
/*
Creating work vectors and arrays
*/
ierr = VecDuplicate(matis->x,&pcis->vec1_N);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_N,&pcis->vec2_N);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pcis->n-pcis->n_B,&pcis->vec1_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec2_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec3_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec4_D);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pcis->n_B,&pcis->vec1_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_B,&pcis->vec2_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_B,&pcis->vec3_B);CHKERRQ(ierr);
ierr = MatCreateVecs(pc->pmat,&pcis->vec1_global,0);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n,&pcis->work_N);CHKERRQ(ierr);
/* Creating the scatter contexts */
ierr = VecScatterCreate(pcis->vec1_global,pcis->is_I_global,pcis->vec1_D,(IS)0,&pcis->global_to_D);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_N,pcis->is_B_local,pcis->vec1_B,(IS)0,&pcis->N_to_B);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_global,pcis->is_B_global,pcis->vec1_B,(IS)0,&pcis->global_to_B);CHKERRQ(ierr);
/* Creating scaling "matrix" D */
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_is_use_stiffness_scaling",&pcis->use_stiffness_scaling,NULL);CHKERRQ(ierr);
if (!pcis->D) {
ierr = VecDuplicate(pcis->vec1_B,&pcis->D);CHKERRQ(ierr);
if (!pcis->use_stiffness_scaling) {
ierr = VecSet(pcis->D,pcis->scaling_factor);CHKERRQ(ierr);
} else {
ierr = MatGetDiagonal(matis->A,pcis->vec1_N);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
}
ierr = VecCopy(pcis->D,pcis->vec1_B);CHKERRQ(ierr);
ierr = MatCreateVecs(pc->pmat,&counter,0);CHKERRQ(ierr); /* temporary auxiliar vector */
ierr = VecSet(counter,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,counter,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,counter,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,counter,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,counter,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPointwiseDivide(pcis->D,pcis->D,pcis->vec1_B);CHKERRQ(ierr);
ierr = VecDestroy(&counter);CHKERRQ(ierr);
/* See historical note 01, at the bottom of this file. */
/*
Creating the KSP contexts for the local Dirichlet and Neumann problems.
*/
if (pcis->computesolvers) {
PC pc_ctx;
/* Dirichlet */
ierr = KSPCreate(PETSC_COMM_SELF,&pcis->ksp_D);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcis->ksp_D,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(pcis->ksp_D,pcis->A_II,pcis->A_II);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcis->ksp_D,"is_localD_");CHKERRQ(ierr);
ierr = KSPGetPC(pcis->ksp_D,&pc_ctx);CHKERRQ(ierr);
ierr = PCSetType(pc_ctx,PCLU);CHKERRQ(ierr);
ierr = KSPSetType(pcis->ksp_D,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcis->ksp_D);CHKERRQ(ierr);
/* the vectors in the following line are dummy arguments, just telling the KSP the vector size. Values are not used */
ierr = KSPSetUp(pcis->ksp_D);CHKERRQ(ierr);
/* Neumann */
ierr = KSPCreate(PETSC_COMM_SELF,&pcis->ksp_N);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcis->ksp_N,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(pcis->ksp_N,matis->A,matis->A);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcis->ksp_N,"is_localN_");CHKERRQ(ierr);
ierr = KSPGetPC(pcis->ksp_N,&pc_ctx);CHKERRQ(ierr);
ierr = PCSetType(pc_ctx,PCLU);CHKERRQ(ierr);
ierr = KSPSetType(pcis->ksp_N,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcis->ksp_N);CHKERRQ(ierr);
{
PetscBool damp_fixed = PETSC_FALSE,
remove_nullspace_fixed = PETSC_FALSE,
set_damping_factor_floating = PETSC_FALSE,
not_damp_floating = PETSC_FALSE,
not_remove_nullspace_floating = PETSC_FALSE;
PetscReal fixed_factor,
floating_factor;
ierr = PetscOptionsGetReal(((PetscObject)pc_ctx)->prefix,"-pc_is_damp_fixed",&fixed_factor,&damp_fixed);CHKERRQ(ierr);
if (!damp_fixed) fixed_factor = 0.0;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_damp_fixed",&damp_fixed,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_remove_nullspace_fixed",&remove_nullspace_fixed,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(((PetscObject)pc_ctx)->prefix,"-pc_is_set_damping_factor_floating",
&floating_factor,&set_damping_factor_floating);CHKERRQ(ierr);
if (!set_damping_factor_floating) floating_factor = 0.0;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_set_damping_factor_floating",&set_damping_factor_floating,NULL);CHKERRQ(ierr);
if (!set_damping_factor_floating) floating_factor = 1.e-12;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_not_damp_floating",¬_damp_floating,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_not_remove_nullspace_floating",¬_remove_nullspace_floating,NULL);CHKERRQ(ierr);
if (pcis->pure_neumann) { /* floating subdomain */
if (!(not_damp_floating)) {
ierr = PCFactorSetShiftType(pc_ctx,MAT_SHIFT_NONZERO);CHKERRQ(ierr);
ierr = PCFactorSetShiftAmount(pc_ctx,floating_factor);CHKERRQ(ierr);
}
if (!(not_remove_nullspace_floating)) {
MatNullSpace nullsp;
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nullsp);CHKERRQ(ierr);
ierr = KSPSetNullSpace(pcis->ksp_N,nullsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullsp);CHKERRQ(ierr);
}
} else { /* fixed subdomain */
if (damp_fixed) {
ierr = PCFactorSetShiftType(pc_ctx,MAT_SHIFT_NONZERO);CHKERRQ(ierr);
ierr = PCFactorSetShiftAmount(pc_ctx,floating_factor);CHKERRQ(ierr);
}
if (remove_nullspace_fixed) {
MatNullSpace nullsp;
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nullsp);CHKERRQ(ierr);
ierr = KSPSetNullSpace(pcis->ksp_N,nullsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullsp);CHKERRQ(ierr);
}
}
}
/* the vectors in the following line are dummy arguments, just telling the KSP the vector size. Values are not used */
ierr = KSPSetUp(pcis->ksp_N);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCGraphSetUp(PCBDDCGraph graph, PetscInt custom_minimal_size, IS neumann_is, IS dirichlet_is, PetscInt n_ISForDofs, IS ISForDofs[], IS custom_primal_vertices)
{
IS subset,subset_n;
MPI_Comm comm;
const PetscInt *is_indices;
PetscInt n_neigh,*neigh,*n_shared,**shared,*queue_global;
PetscInt i,j,k,s,total_counts,nodes_touched,is_size;
PetscMPIInt commsize;
PetscBool same_set,mirrors_found;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidLogicalCollectiveInt(graph->l2gmap,custom_minimal_size,2);
if (neumann_is) {
PetscValidHeaderSpecific(neumann_is,IS_CLASSID,3);
PetscCheckSameComm(graph->l2gmap,1,neumann_is,3);
}
graph->has_dirichlet = PETSC_FALSE;
if (dirichlet_is) {
PetscValidHeaderSpecific(dirichlet_is,IS_CLASSID,4);
PetscCheckSameComm(graph->l2gmap,1,dirichlet_is,4);
graph->has_dirichlet = PETSC_TRUE;
}
PetscValidLogicalCollectiveInt(graph->l2gmap,n_ISForDofs,5);
for (i=0;i<n_ISForDofs;i++) {
PetscValidHeaderSpecific(ISForDofs[i],IS_CLASSID,6);
PetscCheckSameComm(graph->l2gmap,1,ISForDofs[i],6);
}
if (custom_primal_vertices) {
PetscValidHeaderSpecific(custom_primal_vertices,IS_CLASSID,6);
PetscCheckSameComm(graph->l2gmap,1,custom_primal_vertices,7);
}
ierr = PetscObjectGetComm((PetscObject)(graph->l2gmap),&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&commsize);CHKERRQ(ierr);
/* custom_minimal_size */
graph->custom_minimal_size = custom_minimal_size;
/* get info l2gmap and allocate work vectors */
ierr = ISLocalToGlobalMappingGetInfo(graph->l2gmap,&n_neigh,&neigh,&n_shared,&shared);CHKERRQ(ierr);
/* check if we have any local periodic nodes (periodic BCs) */
mirrors_found = PETSC_FALSE;
if (graph->nvtxs && n_neigh) {
for (i=0; i<n_shared[0]; i++) graph->count[shared[0][i]] += 1;
for (i=0; i<n_shared[0]; i++) {
if (graph->count[shared[0][i]] > 1) {
mirrors_found = PETSC_TRUE;
break;
}
}
}
/* compute local mirrors (if any) */
if (mirrors_found) {
IS to,from;
PetscInt *local_indices,*global_indices;
ierr = ISCreateStride(PETSC_COMM_SELF,graph->nvtxs,0,1,&to);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApplyIS(graph->l2gmap,to,&from);CHKERRQ(ierr);
/* get arrays of local and global indices */
ierr = PetscMalloc1(graph->nvtxs,&local_indices);CHKERRQ(ierr);
ierr = ISGetIndices(to,(const PetscInt**)&is_indices);CHKERRQ(ierr);
ierr = PetscMemcpy(local_indices,is_indices,graph->nvtxs*sizeof(PetscInt));CHKERRQ(ierr);
ierr = ISRestoreIndices(to,(const PetscInt**)&is_indices);CHKERRQ(ierr);
ierr = PetscMalloc1(graph->nvtxs,&global_indices);CHKERRQ(ierr);
ierr = ISGetIndices(from,(const PetscInt**)&is_indices);CHKERRQ(ierr);
ierr = PetscMemcpy(global_indices,is_indices,graph->nvtxs*sizeof(PetscInt));CHKERRQ(ierr);
ierr = ISRestoreIndices(from,(const PetscInt**)&is_indices);CHKERRQ(ierr);
/* allocate space for mirrors */
ierr = PetscMalloc2(graph->nvtxs,&graph->mirrors,graph->nvtxs,&graph->mirrors_set);CHKERRQ(ierr);
ierr = PetscMemzero(graph->mirrors,graph->nvtxs*sizeof(PetscInt));CHKERRQ(ierr);
graph->mirrors_set[0] = 0;
k=0;
for (i=0;i<n_shared[0];i++) {
j=shared[0][i];
if (graph->count[j] > 1) {
graph->mirrors[j]++;
k++;
}
}
/* allocate space for set of mirrors */
ierr = PetscMalloc1(k,&graph->mirrors_set[0]);CHKERRQ(ierr);
for (i=1;i<graph->nvtxs;i++)
graph->mirrors_set[i]=graph->mirrors_set[i-1]+graph->mirrors[i-1];
/* fill arrays */
ierr = PetscMemzero(graph->mirrors,graph->nvtxs*sizeof(PetscInt));CHKERRQ(ierr);
for (j=0;j<n_shared[0];j++) {
i=shared[0][j];
if (graph->count[i] > 1)
graph->mirrors_set[i][graph->mirrors[i]++]=global_indices[i];
}
ierr = PetscSortIntWithArray(graph->nvtxs,global_indices,local_indices);CHKERRQ(ierr);
for (i=0;i<graph->nvtxs;i++) {
if (graph->mirrors[i] > 0) {
ierr = PetscFindInt(graph->mirrors_set[i][0],graph->nvtxs,global_indices,&k);CHKERRQ(ierr);
j = global_indices[k];
while ( k > 0 && global_indices[k-1] == j) k--;
for (j=0;j<graph->mirrors[i];j++) {
graph->mirrors_set[i][j]=local_indices[k+j];
}
ierr = PetscSortInt(graph->mirrors[i],graph->mirrors_set[i]);CHKERRQ(ierr);
}
}
ierr = PetscFree(local_indices);CHKERRQ(ierr);
ierr = PetscFree(global_indices);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
}
ierr = PetscMemzero(graph->count,graph->nvtxs*sizeof(*graph->count));CHKERRQ(ierr);
/* Count total number of neigh per node */
k = 0;
for (i=1;i<n_neigh;i++) {
k += n_shared[i];
for (j=0;j<n_shared[i];j++) {
graph->count[shared[i][j]] += 1;
}
}
/* Allocate space for storing the set of neighbours for each node */
if (graph->nvtxs) {
ierr = PetscMalloc1(k,&graph->neighbours_set[0]);CHKERRQ(ierr);
}
for (i=1;i<graph->nvtxs;i++) { /* dont count myself */
graph->neighbours_set[i]=graph->neighbours_set[i-1]+graph->count[i-1];
}
/* Get information for sharing subdomains */
ierr = PetscMemzero(graph->count,graph->nvtxs*sizeof(*graph->count));CHKERRQ(ierr);
for (i=1;i<n_neigh;i++) { /* dont count myself */
s = n_shared[i];
for (j=0;j<s;j++) {
k = shared[i][j];
graph->neighbours_set[k][graph->count[k]] = neigh[i];
graph->count[k] += 1;
}
}
/* sort set of sharing subdomains */
for (i=0;i<graph->nvtxs;i++) {
ierr = PetscSortRemoveDupsInt(&graph->count[i],graph->neighbours_set[i]);CHKERRQ(ierr);
}
/* free memory allocated by ISLocalToGlobalMappingGetInfo */
ierr = ISLocalToGlobalMappingRestoreInfo(graph->l2gmap,&n_neigh,&neigh,&n_shared,&shared);CHKERRQ(ierr);
/*
Get info for dofs splitting
User can specify just a subset; an additional field is considered as a complementary field
*/
for (i=0;i<graph->nvtxs;i++) graph->which_dof[i] = n_ISForDofs; /* by default a dof belongs to the complement set */
for (i=0;i<n_ISForDofs;i++) {
ierr = ISGetLocalSize(ISForDofs[i],&is_size);CHKERRQ(ierr);
ierr = ISGetIndices(ISForDofs[i],(const PetscInt**)&is_indices);CHKERRQ(ierr);
for (j=0;j<is_size;j++) {
if (is_indices[j] > -1 && is_indices[j] < graph->nvtxs) { /* out of bounds indices (if any) are skipped */
graph->which_dof[is_indices[j]] = i;
}
}
ierr = ISRestoreIndices(ISForDofs[i],(const PetscInt**)&is_indices);CHKERRQ(ierr);
}
/* Take into account Neumann nodes */
if (neumann_is) {
ierr = ISGetLocalSize(neumann_is,&is_size);CHKERRQ(ierr);
ierr = ISGetIndices(neumann_is,(const PetscInt**)&is_indices);CHKERRQ(ierr);
for (i=0;i<is_size;i++) {
if (is_indices[i] > -1 && is_indices[i] < graph->nvtxs) { /* out of bounds indices (if any) are skipped */
graph->special_dof[is_indices[i]] = PCBDDCGRAPH_NEUMANN_MARK;
}
}
ierr = ISRestoreIndices(neumann_is,(const PetscInt**)&is_indices);CHKERRQ(ierr);
}
/* Take into account Dirichlet nodes (they overwrite any neumann boundary mark previously set) */
if (dirichlet_is) {
ierr = ISGetLocalSize(dirichlet_is,&is_size);CHKERRQ(ierr);
ierr = ISGetIndices(dirichlet_is,(const PetscInt**)&is_indices);CHKERRQ(ierr);
for (i=0;i<is_size;i++){
if (is_indices[i] > -1 && is_indices[i] < graph->nvtxs) { /* out of bounds indices (if any) are skipped */
if (commsize > graph->commsizelimit) { /* dirichlet nodes treated as internal */
ierr = PetscBTSet(graph->touched,is_indices[i]);CHKERRQ(ierr);
graph->subset[is_indices[i]] = 0;
}
graph->special_dof[is_indices[i]] = PCBDDCGRAPH_DIRICHLET_MARK;
}
}
ierr = ISRestoreIndices(dirichlet_is,(const PetscInt**)&is_indices);CHKERRQ(ierr);
}
/* mark local periodic nodes (if any) and adapt CSR graph (if any) */
if (graph->mirrors) {
for (i=0;i<graph->nvtxs;i++)
if (graph->mirrors[i])
graph->special_dof[i] = PCBDDCGRAPH_LOCAL_PERIODIC_MARK;
if (graph->xadj) {
PetscInt *new_xadj,*new_adjncy;
/* sort CSR graph */
for (i=0;i<graph->nvtxs;i++)
ierr = PetscSortInt(graph->xadj[i+1]-graph->xadj[i],&graph->adjncy[graph->xadj[i]]);CHKERRQ(ierr);
/* adapt local CSR graph in case of local periodicity */
k = 0;
for (i=0;i<graph->nvtxs;i++)
for (j=graph->xadj[i];j<graph->xadj[i+1];j++)
k += graph->mirrors[graph->adjncy[j]];
ierr = PetscMalloc1(graph->nvtxs+1,&new_xadj);CHKERRQ(ierr);
ierr = PetscMalloc1(k+graph->xadj[graph->nvtxs],&new_adjncy);CHKERRQ(ierr);
new_xadj[0] = 0;
for (i=0;i<graph->nvtxs;i++) {
k = graph->xadj[i+1]-graph->xadj[i];
ierr = PetscMemcpy(&new_adjncy[new_xadj[i]],&graph->adjncy[graph->xadj[i]],k*sizeof(PetscInt));CHKERRQ(ierr);
new_xadj[i+1] = new_xadj[i]+k;
for (j=graph->xadj[i];j<graph->xadj[i+1];j++) {
k = graph->mirrors[graph->adjncy[j]];
ierr = PetscMemcpy(&new_adjncy[new_xadj[i+1]],graph->mirrors_set[graph->adjncy[j]],k*sizeof(PetscInt));CHKERRQ(ierr);
new_xadj[i+1] += k;
}
k = new_xadj[i+1]-new_xadj[i];
ierr = PetscSortRemoveDupsInt(&k,&new_adjncy[new_xadj[i]]);CHKERRQ(ierr);
new_xadj[i+1] = new_xadj[i]+k;
}
/* set new CSR into graph */
ierr = PetscFree(graph->xadj);CHKERRQ(ierr);
ierr = PetscFree(graph->adjncy);CHKERRQ(ierr);
graph->xadj = new_xadj;
graph->adjncy = new_adjncy;
}
}
/* mark special nodes (if any) -> each will become a single node equivalence class */
if (custom_primal_vertices) {
ierr = ISGetLocalSize(custom_primal_vertices,&is_size);CHKERRQ(ierr);
ierr = ISGetIndices(custom_primal_vertices,(const PetscInt**)&is_indices);CHKERRQ(ierr);
for (i=0,j=0;i<is_size;i++){
if (is_indices[i] > -1 && is_indices[i] < graph->nvtxs && graph->special_dof[is_indices[i]] != PCBDDCGRAPH_DIRICHLET_MARK) { /* out of bounds indices (if any) are skipped */
graph->special_dof[is_indices[i]] = PCBDDCGRAPH_SPECIAL_MARK-j;
j++;
}
}
ierr = ISRestoreIndices(custom_primal_vertices,(const PetscInt**)&is_indices);CHKERRQ(ierr);
}
/* mark interior nodes (if commsize > graph->commsizelimit) as touched and belonging to partition number 0 */
if (commsize > graph->commsizelimit) {
for (i=0;i<graph->nvtxs;i++) {
if (!graph->count[i]) {
ierr = PetscBTSet(graph->touched,i);CHKERRQ(ierr);
graph->subset[i] = 0;
}
}
}
/* init graph structure and compute default subsets */
nodes_touched = 0;
for (i=0;i<graph->nvtxs;i++) {
if (PetscBTLookup(graph->touched,i)) {
nodes_touched++;
}
}
i = 0;
graph->ncc = 0;
total_counts = 0;
/* allocated space for queues */
if (commsize == graph->commsizelimit) {
ierr = PetscMalloc2(graph->nvtxs+1,&graph->cptr,graph->nvtxs,&graph->queue);CHKERRQ(ierr);
} else {
PetscInt nused = graph->nvtxs - nodes_touched;
ierr = PetscMalloc2(nused+1,&graph->cptr,nused,&graph->queue);CHKERRQ(ierr);
}
while (nodes_touched<graph->nvtxs) {
/* find first untouched node in local ordering */
while (PetscBTLookup(graph->touched,i)) i++;
ierr = PetscBTSet(graph->touched,i);CHKERRQ(ierr);
graph->subset[i] = graph->ncc+1;
graph->cptr[graph->ncc] = total_counts;
graph->queue[total_counts] = i;
total_counts++;
nodes_touched++;
/* now find all other nodes having the same set of sharing subdomains */
for (j=i+1;j<graph->nvtxs;j++) {
/* check for same number of sharing subdomains, dof number and same special mark */
if (!PetscBTLookup(graph->touched,j) && graph->count[i] == graph->count[j] && graph->which_dof[i] == graph->which_dof[j] && graph->special_dof[i] == graph->special_dof[j]) {
/* check for same set of sharing subdomains */
same_set = PETSC_TRUE;
for (k=0;k<graph->count[j];k++){
if (graph->neighbours_set[i][k] != graph->neighbours_set[j][k]) {
same_set = PETSC_FALSE;
}
}
/* I found a friend of mine */
if (same_set) {
ierr = PetscBTSet(graph->touched,j);CHKERRQ(ierr);
graph->subset[j] = graph->ncc+1;
nodes_touched++;
graph->queue[total_counts] = j;
total_counts++;
}
}
}
graph->ncc++;
}
/* set default number of subsets (at this point no info on csr and/or local_subs has been taken into account, so n_subsets = ncc */
graph->n_subsets = graph->ncc;
ierr = PetscMalloc1(graph->n_subsets,&graph->subset_ncc);CHKERRQ(ierr);
for (i=0;i<graph->n_subsets;i++) {
graph->subset_ncc[i] = 1;
}
/* final pointer */
graph->cptr[graph->ncc] = total_counts;
/* For consistency reasons (among neighbours), I need to sort (by global ordering) each connected component */
/* Get a reference node (min index in global ordering) for each subset for tagging messages */
ierr = PetscMalloc1(graph->ncc,&graph->subset_ref_node);CHKERRQ(ierr);
ierr = PetscMalloc1(graph->cptr[graph->ncc],&queue_global);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(graph->l2gmap,graph->cptr[graph->ncc],graph->queue,queue_global);CHKERRQ(ierr);
for (j=0;j<graph->ncc;j++) {
ierr = PetscSortIntWithArray(graph->cptr[j+1]-graph->cptr[j],&queue_global[graph->cptr[j]],&graph->queue[graph->cptr[j]]);CHKERRQ(ierr);
graph->subset_ref_node[j] = graph->queue[graph->cptr[j]];
}
ierr = PetscFree(queue_global);CHKERRQ(ierr);
graph->queue_sorted = PETSC_TRUE;
/* save information on subsets (needed when analyzing the connected components) */
if (graph->ncc) {
ierr = PetscMalloc2(graph->ncc,&graph->subset_size,graph->ncc,&graph->subset_idxs);CHKERRQ(ierr);
ierr = PetscMalloc1(graph->cptr[graph->ncc],&graph->subset_idxs[0]);CHKERRQ(ierr);
ierr = PetscMemzero(graph->subset_idxs[0],graph->cptr[graph->ncc]*sizeof(PetscInt));CHKERRQ(ierr);
for (j=1;j<graph->ncc;j++) {
graph->subset_size[j-1] = graph->cptr[j] - graph->cptr[j-1];
graph->subset_idxs[j] = graph->subset_idxs[j-1] + graph->subset_size[j-1];
}
graph->subset_size[graph->ncc-1] = graph->cptr[graph->ncc] - graph->cptr[graph->ncc-1];
ierr = PetscMemcpy(graph->subset_idxs[0],graph->queue,graph->cptr[graph->ncc]*sizeof(PetscInt));CHKERRQ(ierr);
}
/* renumber reference nodes */
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)(graph->l2gmap)),graph->ncc,graph->subset_ref_node,PETSC_COPY_VALUES,&subset_n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApplyIS(graph->l2gmap,subset_n,&subset);CHKERRQ(ierr);
ierr = ISDestroy(&subset_n);CHKERRQ(ierr);
ierr = ISRenumber(subset,NULL,NULL,&subset_n);CHKERRQ(ierr);
ierr = ISDestroy(&subset);CHKERRQ(ierr);
ierr = ISGetLocalSize(subset_n,&k);CHKERRQ(ierr);
if (k != graph->ncc) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Invalid size of new subset! %D != %D",k,graph->ncc);
ierr = ISGetIndices(subset_n,&is_indices);CHKERRQ(ierr);
ierr = PetscMemcpy(graph->subset_ref_node,is_indices,graph->ncc*sizeof(PetscInt));CHKERRQ(ierr);
ierr = ISRestoreIndices(subset_n,&is_indices);CHKERRQ(ierr);
ierr = ISDestroy(&subset_n);CHKERRQ(ierr);
/* free workspace */
graph->setupcalled = PETSC_TRUE;
PetscFunctionReturn(0);
}
