shared_ptr<Epetra_CrsMatrix> sparseCholesky(const Epetra_CrsMatrix &mat) {
// Note: we assume the matrix mat is symmetric and positive-definite
size_t size = mat.NumGlobalCols();
if (mat.NumGlobalRows() != size)
throw std::invalid_argument("sparseCholesky(): matrix must be square");
int *rowOffsets = 0;
int *colIndices = 0;
double *values = 0;
mat.ExtractCrsDataPointers(rowOffsets, colIndices, values);
Epetra_SerialComm comm;
Epetra_LocalMap rowMap(static_cast<int>(size), 0 /* index_base */, comm);
Epetra_LocalMap columnMap(static_cast<int>(size), 0 /* index_base */, comm);
shared_ptr<Epetra_CrsMatrix> result = boost::make_shared<Epetra_CrsMatrix>(
Copy, rowMap, columnMap, mat.GlobalMaxNumEntries());
arma::Mat<double> localMat;
arma::Mat<double> localCholesky;
std::vector<bool> processed(size, false);
for (size_t r = 0; r < size; ++r) {
if (processed[r])
continue;
int localSize = rowOffsets[r + 1] - rowOffsets[r];
localMat.set_size(localSize, localSize);
localMat.fill(0.);
localCholesky.set_size(localSize, localSize);
for (int s = 0; s < localSize; ++s) {
int row = colIndices[rowOffsets[r] + s];
for (int c = 0; c < localSize; ++c) {
int col = colIndices[rowOffsets[row] + c];
if (col != colIndices[rowOffsets[r] + c])
throw std::invalid_argument("sparseCholesky(): matrix is not "
"block-diagonal");
localMat(s, c) = values[rowOffsets[row] + c];
}
}
assert(arma::norm(localMat - localMat.t(), "fro") <
1e-12 * arma::norm(localMat, "fro"));
localCholesky = arma::chol(localMat); // localCholesky: U
for (int s = 0; s < localSize; ++s) {
int row = colIndices[rowOffsets[r] + s];
processed[row] = true;
#ifndef NDEBUG
int errorCode =
#endif
result->InsertGlobalValues(row, s + 1 /* number of values */,
localCholesky.colptr(s),
colIndices + rowOffsets[r]);
assert(errorCode == 0);
}
}
result->FillComplete(columnMap, rowMap);
return result;
}
int TUnpackAndCombineIntoCrsArrays(const Epetra_CrsMatrix& SourceMatrix,
int NumSameIDs,
int NumRemoteIDs,
const int * RemoteLIDs,
int NumPermuteIDs,
const int *PermuteToLIDs,
const int *PermuteFromLIDs,
int LenImports,
char* Imports,
int TargetNumRows,
int TargetNumNonzeros,
int * CSR_rowptr,
int_type * CSR_colind,
double * CSR_vals,
const std::vector<int> &SourcePids,
std::vector<int> &TargetPids)
{
// What we really need to know is where in the CSR arrays each row should start (aka the rowptr).
// We do that by (a) having each row record it's size in the rowptr (b) doing a cumulative sum to get the rowptr values correct and
// (c) Having each row copied into the right colind / values locations.
// From Epetra_CrsMatrix UnpackAndCombine():
// Each segment of Exports will be filled by a packed row of information for each row as follows:
// 1st int: GRID of row where GRID is the global row ID for the source matrix
// next int: NumEntries, Number of indices in row.
// next NumEntries: The actual indices for the row.
int i,j,rv;
int N = TargetNumRows;
int mynnz = TargetNumNonzeros;
int MyPID = SourceMatrix.Comm().MyPID();
int SizeofIntType = sizeof(int_type);
// Zero the rowptr
for(i=0; i<N+1; i++) CSR_rowptr[i]=0;
// SameIDs: Always first, always in the same place
for(i=0; i<NumSameIDs; i++)
CSR_rowptr[i]=SourceMatrix.NumMyEntries(i);
// PermuteIDs: Still local, but reordered
for(i=0; i<NumPermuteIDs; i++)
CSR_rowptr[PermuteToLIDs[i]] = SourceMatrix.NumMyEntries(PermuteFromLIDs[i]);
// RemoteIDs: RemoteLIDs tells us the ID, we need to look up the length the hard way. See UnpackAndCombine for where this code came from
if(NumRemoteIDs > 0) {
double * dintptr = (double *) Imports;
int_type * intptr = (int_type *) dintptr;
int NumEntries = (int) intptr[1];
int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
for(i=0; i<NumRemoteIDs; i++) {
CSR_rowptr[RemoteLIDs[i]] += NumEntries;
if( i < (NumRemoteIDs-1) ) {
dintptr += IntSize + NumEntries;
intptr = (int_type *) dintptr;
NumEntries = (int) intptr[1];
IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
}
}
}
// If multiple procs contribute to a row;
std::vector<int> NewStartRow(N+1);
// Turn row length into a real CSR_rowptr
int last_len = CSR_rowptr[0];
CSR_rowptr[0] = 0;
for(i=1; i<N+1; i++){
int new_len = CSR_rowptr[i];
CSR_rowptr[i] = last_len + CSR_rowptr[i-1];
NewStartRow[i] = CSR_rowptr[i];
last_len = new_len;
}
// Preseed TargetPids with -1 for local
if(TargetPids.size()!=(size_t)mynnz) TargetPids.resize(mynnz);
TargetPids.assign(mynnz,-1);
// Grab pointers for SourceMatrix
int * Source_rowptr, * Source_colind;
double * Source_vals;
rv=SourceMatrix.ExtractCrsDataPointers(Source_rowptr,Source_colind,Source_vals);
if(rv) throw std::runtime_error("UnpackAndCombineIntoCrsArrays: failed in ExtractCrsDataPointers");
// SameIDs: Copy the data over
for(i=0; i<NumSameIDs; i++) {
int FromRow = Source_rowptr[i];
int ToRow = CSR_rowptr[i];
NewStartRow[i] += Source_rowptr[i+1]-Source_rowptr[i];
for(j=Source_rowptr[i]; j<Source_rowptr[i+1]; j++) {
CSR_vals[ToRow + j - FromRow] = Source_vals[j];
CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]);
TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1;
}
}
// PermuteIDs: Copy the data over
for(i=0; i<NumPermuteIDs; i++) {
int FromLID = PermuteFromLIDs[i];
int FromRow = Source_rowptr[FromLID];
int ToRow = CSR_rowptr[PermuteToLIDs[i]];
NewStartRow[PermuteToLIDs[i]] += Source_rowptr[FromLID+1]-Source_rowptr[FromLID];
for(j=Source_rowptr[FromLID]; j<Source_rowptr[FromLID+1]; j++) {
CSR_vals[ToRow + j - FromRow] = Source_vals[j];
CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]);
TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1;
}
}
// RemoteIDs: Loop structure following UnpackAndCombine
if(NumRemoteIDs > 0) {
double * dintptr = (double *) Imports;
int_type * intptr = (int_type *) dintptr;
int NumEntries = (int) intptr[1];
int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
double* valptr = dintptr + IntSize;
for (i=0; i<NumRemoteIDs; i++) {
int ToLID = RemoteLIDs[i];
int StartRow = NewStartRow[ToLID];
NewStartRow[ToLID]+=NumEntries;
double * values = valptr;
int_type * Indices = intptr + 2;
for(j=0; j<NumEntries; j++){
CSR_vals[StartRow + j] = values[j];
CSR_colind[StartRow + j] = Indices[2*j];
TargetPids[StartRow + j] = (Indices[2*j+1] != MyPID) ? Indices[2*j+1] : -1;
}
if( i < (NumRemoteIDs-1) ) {
dintptr += IntSize + NumEntries;
intptr = (int_type *) dintptr;
NumEntries = (int) intptr[1];
IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
valptr = dintptr + IntSize;
}
}
}
return 0;
}
