int TUnpackAndCombineIntoCrsArrays(const Epetra_CrsMatrix& SourceMatrix, int NumSameIDs, int NumRemoteIDs, const int * RemoteLIDs, int NumPermuteIDs, const int *PermuteToLIDs, const int *PermuteFromLIDs, int LenImports, char* Imports, int TargetNumRows, int TargetNumNonzeros, int * CSR_rowptr, int_type * CSR_colind, double * CSR_vals, const std::vector<int> &SourcePids, std::vector<int> &TargetPids) { // What we really need to know is where in the CSR arrays each row should start (aka the rowptr). // We do that by (a) having each row record it's size in the rowptr (b) doing a cumulative sum to get the rowptr values correct and // (c) Having each row copied into the right colind / values locations. // From Epetra_CrsMatrix UnpackAndCombine(): // Each segment of Exports will be filled by a packed row of information for each row as follows: // 1st int: GRID of row where GRID is the global row ID for the source matrix // next int: NumEntries, Number of indices in row. // next NumEntries: The actual indices for the row. int i,j,rv; int N = TargetNumRows; int mynnz = TargetNumNonzeros; int MyPID = SourceMatrix.Comm().MyPID(); int SizeofIntType = sizeof(int_type); // Zero the rowptr for(i=0; i<N+1; i++) CSR_rowptr[i]=0; // SameIDs: Always first, always in the same place for(i=0; i<NumSameIDs; i++) CSR_rowptr[i]=SourceMatrix.NumMyEntries(i); // PermuteIDs: Still local, but reordered for(i=0; i<NumPermuteIDs; i++) CSR_rowptr[PermuteToLIDs[i]] = SourceMatrix.NumMyEntries(PermuteFromLIDs[i]); // RemoteIDs: RemoteLIDs tells us the ID, we need to look up the length the hard way. See UnpackAndCombine for where this code came from if(NumRemoteIDs > 0) { double * dintptr = (double *) Imports; int_type * intptr = (int_type *) dintptr; int NumEntries = (int) intptr[1]; int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); for(i=0; i<NumRemoteIDs; i++) { CSR_rowptr[RemoteLIDs[i]] += NumEntries; if( i < (NumRemoteIDs-1) ) { dintptr += IntSize + NumEntries; intptr = (int_type *) dintptr; NumEntries = (int) intptr[1]; IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); } } } // If multiple procs contribute to a row; std::vector<int> NewStartRow(N+1); // Turn row length into a real CSR_rowptr int last_len = CSR_rowptr[0]; CSR_rowptr[0] = 0; for(i=1; i<N+1; i++){ int new_len = CSR_rowptr[i]; CSR_rowptr[i] = last_len + CSR_rowptr[i-1]; NewStartRow[i] = CSR_rowptr[i]; last_len = new_len; } // Preseed TargetPids with -1 for local if(TargetPids.size()!=(size_t)mynnz) TargetPids.resize(mynnz); TargetPids.assign(mynnz,-1); // Grab pointers for SourceMatrix int * Source_rowptr, * Source_colind; double * Source_vals; rv=SourceMatrix.ExtractCrsDataPointers(Source_rowptr,Source_colind,Source_vals); if(rv) throw std::runtime_error("UnpackAndCombineIntoCrsArrays: failed in ExtractCrsDataPointers"); // SameIDs: Copy the data over for(i=0; i<NumSameIDs; i++) { int FromRow = Source_rowptr[i]; int ToRow = CSR_rowptr[i]; NewStartRow[i] += Source_rowptr[i+1]-Source_rowptr[i]; for(j=Source_rowptr[i]; j<Source_rowptr[i+1]; j++) { CSR_vals[ToRow + j - FromRow] = Source_vals[j]; CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]); TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1; } } // PermuteIDs: Copy the data over for(i=0; i<NumPermuteIDs; i++) { int FromLID = PermuteFromLIDs[i]; int FromRow = Source_rowptr[FromLID]; int ToRow = CSR_rowptr[PermuteToLIDs[i]]; NewStartRow[PermuteToLIDs[i]] += Source_rowptr[FromLID+1]-Source_rowptr[FromLID]; for(j=Source_rowptr[FromLID]; j<Source_rowptr[FromLID+1]; j++) { CSR_vals[ToRow + j - FromRow] = Source_vals[j]; CSR_colind[ToRow + j - FromRow] = (int_type) SourceMatrix.GCID64(Source_colind[j]); TargetPids[ToRow + j - FromRow] = (SourcePids[Source_colind[j]] != MyPID) ? SourcePids[Source_colind[j]] : -1; } } // RemoteIDs: Loop structure following UnpackAndCombine if(NumRemoteIDs > 0) { double * dintptr = (double *) Imports; int_type * intptr = (int_type *) dintptr; int NumEntries = (int) intptr[1]; int IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); double* valptr = dintptr + IntSize; for (i=0; i<NumRemoteIDs; i++) { int ToLID = RemoteLIDs[i]; int StartRow = NewStartRow[ToLID]; NewStartRow[ToLID]+=NumEntries; double * values = valptr; int_type * Indices = intptr + 2; for(j=0; j<NumEntries; j++){ CSR_vals[StartRow + j] = values[j]; CSR_colind[StartRow + j] = Indices[2*j]; TargetPids[StartRow + j] = (Indices[2*j+1] != MyPID) ? Indices[2*j+1] : -1; } if( i < (NumRemoteIDs-1) ) { dintptr += IntSize + NumEntries; intptr = (int_type *) dintptr; NumEntries = (int) intptr[1]; IntSize = 1 + (((2*NumEntries+2)*SizeofIntType)/(int)sizeof(double)); valptr = dintptr + IntSize; } } } return 0; }