int TUnpackAndCombineIntoCrsArrays(const Epetra_CrsMatrix& SourceMatrix,
int NumSameIDs,
int NumRemoteIDs,
const int * RemoteLIDs,
int NumPermuteIDs,
const int *PermuteToLIDs,
const int *PermuteFromLIDs,
int LenImports,
char* Imports,
int TargetNumRows,
int TargetNumNonzeros,
int * CSR_rowptr,
int_type * CSR_colind,
double * CSR_vals,
const std::vector<int> &SourcePids,
std::vector<int> &TargetPids)
{
// What we really need to know is where in the CSR arrays each row should start (aka the rowptr).
// We do that by (a) having each row record it's size in the rowptr (b) doing a cumulative sum to get the rowptr values correct and
// (c) Having each row copied into the right colind / values locations.
// From Epetra_CrsMatrix UnpackAndCombine():
// Each segment of Exports will be filled by a packed row of information for each row as follows:
// 1st int: GRID of row where GRID is the global row ID for the source matrix
// next int: NumEntries, Number of indices in row.
// next NumEntries: The actual indices for the row.
int i,j,rv;
int N = TargetNumRows;
int mynnz = TargetNumNonzeros;
int MyPID = SourceMatrix.Comm().MyPID();
int SizeofIntType = sizeof(int_type);
// Zero the rowptr
for(i=0; i<N+1; i++) CSR_rowptr[i]=0;
// SameIDs: Always first, always in the same place
for(i=0; i<NumSameIDs; i++)
CSR_rowptr[i]=SourceMatrix.NumMyEntries(i);
// PermuteIDs: Still local, but reordered
for(i=0; i<NumPermuteIDs; i++)
CSR_rowptr[PermuteToLIDs[i]] = SourceMatrix.NumMyEntries(PermuteFromLIDs[i]);
// RemoteIDs: RemoteLIDs tells us the ID, we need to look up the length the hard way. See UnpackAndCombine for where this code came from
if(NumRemoteIDs > 0) {
double * dintptr = (double *) Imports;
int_type * intptr = (int_type *) dintptr;
int NumEntries = (int) intptr[1];
int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
for(i=0; i<NumRemoteIDs; i++) {
CSR_rowptr[RemoteLIDs[i]] += NumEntries;
if( i < (NumRemoteIDs-1) ) {
dintptr += IntSize + NumEntries;
intptr = (int_type *) dintptr;
NumEntries = (int) intptr[1];
IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
}
}
}
// If multiple procs contribute to a row;
std::vector<int> NewStartRow(N+1);
// Turn row length into a real CSR_rowptr
int last_len = CSR_rowptr[0];
CSR_rowptr[0] = 0;
for(i=1; i<N+1; i++){
int new_len = CSR_rowptr[i];
CSR_rowptr[i] = last_len + CSR_rowptr[i-1];
NewStartRow[i] = CSR_rowptr[i];
last_len = new_len;
}
// Preseed TargetPids with -1 for local
if(TargetPids.size()!=(size_t)mynnz) TargetPids.resize(mynnz);
TargetPids.assign(mynnz,-1);
// Grab pointers for SourceMatrix
int * Source_rowptr, * Source_colind;
double * Source_vals;
rv=SourceMatrix.ExtractCrsDataPointers(Source_rowptr,Source_colind,Source_vals);
if(rv) throw std::runtime_error("UnpackAndCombineIntoCrsArrays: failed in ExtractCrsDataPointers");
// SameIDs: Copy the data over
for(i=0; i<NumSameIDs; i++) {
int FromRow = Source_rowptr[i];
int ToRow = CSR_rowptr[i];
NewStartRow[i] += Source_rowptr[i+1]-Source_rowptr[i];
for(j=Source_rowptr[i]; j<Source_rowptr[i+1]; j++) {
CSR_vals[ToRow + j - FromRow] = Source_vals[j];
CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]);
TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1;
}
}
// PermuteIDs: Copy the data over
for(i=0; i<NumPermuteIDs; i++) {
int FromLID = PermuteFromLIDs[i];
int FromRow = Source_rowptr[FromLID];
int ToRow = CSR_rowptr[PermuteToLIDs[i]];
NewStartRow[PermuteToLIDs[i]] += Source_rowptr[FromLID+1]-Source_rowptr[FromLID];
for(j=Source_rowptr[FromLID]; j<Source_rowptr[FromLID+1]; j++) {
CSR_vals[ToRow + j - FromRow] = Source_vals[j];
CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]);
TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1;
}
}
// RemoteIDs: Loop structure following UnpackAndCombine
if(NumRemoteIDs > 0) {
double * dintptr = (double *) Imports;
int_type * intptr = (int_type *) dintptr;
int NumEntries = (int) intptr[1];
int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
double* valptr = dintptr + IntSize;
for (i=0; i<NumRemoteIDs; i++) {
int ToLID = RemoteLIDs[i];
int StartRow = NewStartRow[ToLID];
NewStartRow[ToLID]+=NumEntries;
double * values = valptr;
int_type * Indices = intptr + 2;
for(j=0; j<NumEntries; j++){
CSR_vals[StartRow + j] = values[j];
CSR_colind[StartRow + j] = Indices[2*j];
TargetPids[StartRow + j] = (Indices[2*j+1] != MyPID) ? Indices[2*j+1] : -1;
}
if( i < (NumRemoteIDs-1) ) {
dintptr += IntSize + NumEntries;
intptr = (int_type *) dintptr;
NumEntries = (int) intptr[1];
IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double));
valptr = dintptr + IntSize;
}
}
}
return 0;
}
