void
build_strength_matrix(Mat A, PetscReal theta, Mat* strength) {
//Variables for the new matrix structure
std::vector<PetscInt> rows, cols;
int cursor = 0;
PetscInt start;
PetscInt end;
MatGetOwnershipRange((A), &start, &end);
for (int row=start; row<end; row++) {
PetscInt ncols;
const PetscInt *col_indx;
const PetscScalar *col_value;
MatGetRow((A), row, &ncols, &col_indx, &col_value);
rows.push_back(cursor);
// First, find the threshhold for this row
PetscScalar strong_threshhold = -col_value[0];
for (int ii=0; ii<ncols; ii++) {
if (-col_value[ii] > strong_threshhold) {
strong_threshhold = -col_value[ii];
}
}
strong_threshhold *= theta;
//if the threshold is negative, assume that this row only has a diagonal entry and skip the row
if (strong_threshhold > 0) {
for (int ii=0; ii<ncols; ii++) {
if (-col_value[ii] >= strong_threshhold) {
cols.push_back(col_indx[ii]);
cursor++;
}
}
}
MatRestoreRow((A), row, &ncols, &col_indx, &col_value);
}
rows.push_back(cursor);
MatGetOwnershipRange((A), &start, &end);
std::vector<PetscScalar> data(cols.size());
//TODO: control for cases where cols and rows are split differently
//TODO: replace this PETSC_COMM_WORLD so the strength matrix is using the same communicator as the original matrix.
MatCreate(PETSC_COMM_WORLD, strength);
MatSetSizes(*strength, end-start, end-start, PETSC_DETERMINE, PETSC_DETERMINE);
MatSetType(*strength,MATMPIAIJ);
MatMPIAIJSetPreallocationCSR(*strength,&rows[0],&cols[0],&data[0]);
//TODO: the above code is a work around for a bug in the following function call:
//MatCreateMPIAIJWithArrays(PETSC_COMM_WORLD, end-start, end-start, PETSC_DETERMINE, PETSC_DETERMINE, &rows[0], &cols[0], &data[0], strength);
}
int main(int argc,char **args)
{
Mat A;
PetscInt *ia,*ja;
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 4) SETERRQ(PETSC_COMM_WORLD,1,"Must run with 4 processors");
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscMalloc1(5,&ia);CHKERRQ(ierr);
ierr = PetscMalloc1(16,&ja);CHKERRQ(ierr);
if (!rank) {
ja[0] = 1; ja[1] = 4; ja[2] = 0; ja[3] = 2; ja[4] = 5; ja[5] = 1; ja[6] = 3; ja[7] = 6;
ja[8] = 2; ja[9] = 7;
ia[0] = 0; ia[1] = 2; ia[2] = 5; ia[3] = 8; ia[4] = 10;
} else if (rank == 1) {
ja[0] = 0; ja[1] = 5; ja[2] = 8; ja[3] = 1; ja[4] = 4; ja[5] = 6; ja[6] = 9; ja[7] = 2;
ja[8] = 5; ja[9] = 7; ja[10] = 10; ja[11] = 3; ja[12] = 6; ja[13] = 11;
ia[0] = 0; ia[1] = 3; ia[2] = 7; ia[3] = 11; ia[4] = 14;
} else if (rank == 2) {
ja[0] = 4; ja[1] = 9; ja[2] = 12; ja[3] = 5; ja[4] = 8; ja[5] = 10; ja[6] = 13; ja[7] = 6;
ja[8] = 9; ja[9] = 11; ja[10] = 14; ja[11] = 7; ja[12] = 10; ja[13] = 15;
ia[0] = 0; ia[1] = 3; ia[2] = 7; ia[3] = 11; ia[4] = 14;
} else {
ja[0] = 8; ja[1] = 13; ja[2] = 9; ja[3] = 12; ja[4] = 14; ja[5] = 10; ja[6] = 13; ja[7] = 15;
ja[8] = 11; ja[9] = 14;
ia[0] = 0; ia[1] = 2; ia[2] = 5; ia[3] = 8; ia[4] = 10;
}
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,4,4,16,16);CHKERRQ(ierr);
ierr = MatSetType(A,MATMPIAIJ);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocationCSR(A,ia,ja,NULL);CHKERRQ(ierr);
ierr = PetscFree(ia);CHKERRQ(ierr);
ierr = PetscFree(ja);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
void setFieldSplitPC(Type* ptA, KSP ksp, KN<double>* const& fields, KN<String>* const& names, KN<Matrice_Creuse<PetscScalar>>* const& mT, KN<double>* const& pL) {
if(fields) {
PC pc;
KSPGetPC(ksp, &pc);
PCSetType(pc, PCFIELDSPLIT);
unsigned short* local = new unsigned short[fields->n + ptA->_last - ptA->_first]();
for(int i = 0; i < fields->n; ++i)
local[i] = std::round(fields->operator[](i));
unsigned short nb = fields->n > 0 ? *std::max_element(local, local + fields->n) : 0;
MPI_Allreduce(MPI_IN_PLACE, &nb, 1, MPI_UNSIGNED_SHORT, MPI_MAX, PETSC_COMM_WORLD);
local += fields->n;
if(fields->n)
HPDDM::Subdomain<PetscScalar>::template distributedVec<0>(ptA->_num, ptA->_first, ptA->_last, local - fields->n, local, fields->n);
unsigned long* counts = new unsigned long[nb]();
for(unsigned int i = 0; i < ptA->_last - ptA->_first; ++i)
if(local[i])
++counts[local[i] - 1];
PetscInt* idx = new PetscInt[*std::max_element(counts, counts + nb)];
for(unsigned short j = 0; j < nb; ++j) {
IS is;
unsigned short* pt = local;
for(unsigned int i = 0; i < counts[j]; ++pt) {
if(*pt == j + 1)
idx[i++] = ptA->_first + std::distance(local, pt);
}
ISCreateGeneral(PETSC_COMM_WORLD, counts[j], idx, PETSC_COPY_VALUES, &is);
PCFieldSplitSetIS(pc, names && j < names->size() ? (*(names->operator[](j))).c_str() : NULL, is);
ISDestroy(&is);
}
if(mT && mT->n > 0 && pL) {
int *is, *js;
PetscScalar *s;
unsigned int* re = new unsigned int[pL->n];
unsigned int nbSchur = 1;
for(int i = 0; i < pL->n; ++i)
re[i] = ((*pL)[i]) ? nbSchur++ : 0;
nbSchur--;
unsigned int* num;
unsigned int start, end, global;
ptA->_A->clearBuffer();
globalMapping(ptA->_A, num, start, end, global, re);
delete [] re;
re = new unsigned int[2 * nbSchur];
unsigned int* numSchur = re + nbSchur;
for(int i = 0, j = 0; i < pL->n; ++i) {
if((*pL)[i]) {
*numSchur++ = num[i];
re[static_cast<int>((*pL)[i]) - 1] = j++;
}
}
numSchur -= nbSchur;
delete [] num;
ptA->_S.resize(mT->n);
for(int k = 0; k < mT->n; ++k) {
MatriceMorse<PetscScalar>* mS = static_cast<MatriceMorse<PetscScalar>*>(&(*(mT->operator[](k)).A));
std::vector<std::vector<std::pair<int, PetscScalar>>> tmp(mS->n);
for(int i = 0; i < mS->n; ++i) {
unsigned int row = re[i];
tmp[row].reserve(mS->lg[i + 1] - mS->lg[i]);
for(int j = mS->lg[i]; j < mS->lg[i + 1]; ++j)
tmp[row].emplace_back(re[mS->cl[j]], mS->a[j]);
std::sort(tmp[row].begin(), tmp[row].end(), [](const std::pair<int, PetscScalar>& lhs, const std::pair<int, PetscScalar>& rhs) { return lhs.first < rhs.first; });
}
is = new int[mS->n + 1];
js = new int[mS->nbcoef];
s = new PetscScalar[mS->nbcoef];
is[0] = 0;
for(int i = 0; i < mS->n; ++i) {
for(int j = 0; j < tmp[i].size(); ++j) {
*js++ = tmp[i][j].first;
*s++ = tmp[i][j].second;
}
is[i + 1] = is[i] + tmp[i].size();
}
js -= mS->nbcoef;
s -= mS->nbcoef;
int* ia, *ja;
PetscScalar* c;
ia = ja = nullptr;
c = nullptr;
HPDDM::MatrixCSR<PetscScalar>* dN = new HPDDM::MatrixCSR<PetscScalar>(mS->n, mS->m, mS->nbcoef, s, is, js, mS->symetrique, true);
bool free = ptA->_A->HPDDM::template Subdomain<PetscScalar>::distributedCSR(numSchur, start, end, ia, ja, c, dN);
MatCreate(PETSC_COMM_WORLD, &ptA->_S[k]);
MatSetSizes(ptA->_S[k], end - start, end - start, global, global);
MatSetType(ptA->_S[k], MATMPIAIJ);
MatMPIAIJSetPreallocationCSR(ptA->_S[k], ia, ja, c);
MatSetOption(ptA->_S[k], MAT_NO_OFF_PROC_ENTRIES, PETSC_TRUE);
if(free) {
delete [] ia;
delete [] ja;
delete [] c;
}
delete dN;
}
delete [] re;
ptA->_A->setBuffer();
}
delete [] idx;
delete [] counts;
local -= fields->n;
delete [] local;
}
}
PetscErrorCode MatLoad_AIJ_HDF5(Mat mat, PetscViewer viewer)
{
PetscViewerFormat format;
const PetscInt *i_glob = NULL;
PetscInt *i = NULL;
const PetscInt *j = NULL;
const PetscScalar *a = NULL;
const char *a_name = NULL, *i_name = NULL, *j_name = NULL, *mat_name = NULL, *c_name = NULL;
PetscInt p, m, M, N;
PetscInt bs = mat->rmap->bs;
PetscBool flg;
IS is_i = NULL, is_j = NULL;
Vec vec_a = NULL;
PetscLayout jmap = NULL;
MPI_Comm comm;
PetscMPIInt rank, size;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
switch (format) {
case PETSC_VIEWER_HDF5_PETSC:
case PETSC_VIEWER_DEFAULT:
case PETSC_VIEWER_NATIVE:
case PETSC_VIEWER_HDF5_MAT:
break;
default:
SETERRQ1(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"PetscViewerFormat %s not supported for HDF5 input.",PetscViewerFormats[format]);
}
ierr = PetscObjectGetComm((PetscObject)mat,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject)mat,&mat_name);CHKERRQ(ierr);
ierr = PetscViewerHDF5GetAIJNames(viewer,&i_name,&j_name,&a_name,&c_name);CHKERRQ(ierr);
ierr = PetscOptionsBegin(comm,NULL,"Options for loading matrix from HDF5","Mat");CHKERRQ(ierr);
ierr = PetscOptionsInt("-matload_block_size","Set the blocksize used to store the matrix","MatLoad",bs,&bs,&flg);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
if (flg) {
ierr = MatSetBlockSize(mat, bs);CHKERRQ(ierr);
}
ierr = PetscViewerHDF5PushGroup(viewer,mat_name);CHKERRQ(ierr);
ierr = PetscViewerHDF5ReadAttribute(viewer,NULL,c_name,PETSC_INT,&N);CHKERRQ(ierr);
ierr = PetscViewerHDF5ReadSizes(viewer, i_name, NULL, &M);CHKERRQ(ierr);
--M; /* i has size M+1 as there is global number of nonzeros stored at the end */
if (format==PETSC_VIEWER_HDF5_MAT && !mat->symmetric) {
/* Swap row and columns layout for unallocated matrix. I want to avoid calling MatTranspose() just to transpose sparsity pattern and layout. */
if (!mat->preallocated) {
PetscLayout tmp;
tmp = mat->rmap; mat->rmap = mat->cmap; mat->cmap = tmp;
} else SETERRQ(comm,PETSC_ERR_SUP,"Not for preallocated matrix - we would need to transpose it here which we want to avoid");
}
/* If global sizes are set, check if they are consistent with that given in the file */
if (mat->rmap->N >= 0 && mat->rmap->N != M) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED,"Inconsistent # of rows: Matrix in file has (%D) and input matrix has (%D)",mat->rmap->N,M);
if (mat->cmap->N >= 0 && mat->cmap->N != N) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED,"Inconsistent # of cols: Matrix in file has (%D) and input matrix has (%D)",mat->cmap->N,N);
/* Determine ownership of all (block) rows and columns */
mat->rmap->N = M;
mat->cmap->N = N;
ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr);
m = mat->rmap->n;
/* Read array i (array of row indices) */
ierr = PetscMalloc1(m+1, &i);CHKERRQ(ierr); /* allocate i with one more position for local number of nonzeros on each rank */
i[0] = i[m] = 0; /* make the last entry always defined - the code block below overwrites it just on last rank */
if (rank == size-1) m++; /* in the loaded array i_glob, only the last rank has one more position with the global number of nonzeros */
M++;
ierr = ISCreate(comm,&is_i);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)is_i,i_name);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(is_i->map,m);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(is_i->map,M);CHKERRQ(ierr);
ierr = ISLoad(is_i,viewer);CHKERRQ(ierr);
ierr = ISGetIndices(is_i,&i_glob);CHKERRQ(ierr);
ierr = PetscMemcpy(i,i_glob,m*sizeof(PetscInt));CHKERRQ(ierr);
/* Reset m and M to the matrix sizes */
m = mat->rmap->n;
M--;
/* Create PetscLayout for j and a vectors; construct ranges first */
ierr = PetscLayoutCreate(comm,&jmap);CHKERRQ(ierr);
ierr = PetscCalloc1(size+1, &jmap->range);CHKERRQ(ierr);
ierr = MPI_Allgather(&i[0], 1, MPIU_INT, jmap->range, 1, MPIU_INT, comm);CHKERRQ(ierr);
/* Last rank has global number of nonzeros (= length of j and a arrays) in i[m] (last i entry) so broadcast it */
jmap->range[size] = i[m];
ierr = MPI_Bcast(&jmap->range[size], 1, MPIU_INT, size-1, comm);CHKERRQ(ierr);
for (p=size-1; p>0; p--) {
if (!jmap->range[p]) jmap->range[p] = jmap->range[p+1]; /* for ranks with 0 rows, take the value from the next processor */
}
i[m] = jmap->range[rank+1]; /* i[m] (last i entry) is equal to next rank's offset */
/* Deduce rstart, rend, n and N from the ranges */
ierr = PetscLayoutSetUp(jmap);CHKERRQ(ierr);
/* Convert global to local indexing of rows */
for (p=1; p<m+1; ++p) i[p] -= i[0];
i[0] = 0;
/* Read array j (array of column indices) */
ierr = ISCreate(comm,&is_j);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)is_j,j_name);CHKERRQ(ierr);
ierr = PetscLayoutDuplicate(jmap,&is_j->map);CHKERRQ(ierr);
ierr = ISLoad(is_j,viewer);CHKERRQ(ierr);
ierr = ISGetIndices(is_j,&j);CHKERRQ(ierr);
/* Read array a (array of values) */
ierr = VecCreate(comm,&vec_a);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)vec_a,a_name);CHKERRQ(ierr);
ierr = PetscLayoutDuplicate(jmap,&vec_a->map);CHKERRQ(ierr);
ierr = VecLoad(vec_a,viewer);CHKERRQ(ierr);
ierr = VecGetArrayRead(vec_a,&a);CHKERRQ(ierr);
/* populate matrix */
if (!((PetscObject)mat)->type_name) {
ierr = MatSetType(mat,MATAIJ);CHKERRQ(ierr);
}
ierr = MatSeqAIJSetPreallocationCSR(mat,i,j,a);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocationCSR(mat,i,j,a);CHKERRQ(ierr);
/*
ierr = MatSeqBAIJSetPreallocationCSR(mat,bs,i,j,a);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocationCSR(mat,bs,i,j,a);CHKERRQ(ierr);
*/
if (format==PETSC_VIEWER_HDF5_MAT && !mat->symmetric) {
/* Transpose the input matrix back */
ierr = MatTranspose(mat,MAT_INPLACE_MATRIX,&mat);CHKERRQ(ierr);
}
ierr = PetscViewerHDF5PopGroup(viewer);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&jmap);CHKERRQ(ierr);
ierr = PetscFree(i);CHKERRQ(ierr);
ierr = ISRestoreIndices(is_i,&i_glob);CHKERRQ(ierr);
ierr = ISRestoreIndices(is_j,&j);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(vec_a,&a);CHKERRQ(ierr);
ierr = ISDestroy(&is_i);CHKERRQ(ierr);
ierr = ISDestroy(&is_j);CHKERRQ(ierr);
ierr = VecDestroy(&vec_a);CHKERRQ(ierr);
PetscFunctionReturn(0);
}