bool compare_matrices(const Epetra_CrsMatrix& A, const Epetra_CrsMatrix& B)
{
const Epetra_Map& Amap = A.RowMap();
const Epetra_Map& Bmap = B.RowMap();
if (!Amap.PointSameAs(Bmap)) {
return(false);
}
int numRows = Amap.NumMyElements();
int* rows = Amap.MyGlobalElements();
Epetra_Util util;
for(int i=0; i<numRows; ++i) {
int row = rows[i];
int rowLen = A.NumGlobalEntries(row);
if (rowLen != B.NumGlobalEntries(row)) {
return(false);
}
int* indices = new int[rowLen*2];
int* Bindices = indices+rowLen;
double* values = new double[rowLen*2];
double* Bvalues = values+rowLen;
A.ExtractGlobalRowCopy(row, rowLen, rowLen, values, indices);
B.ExtractGlobalRowCopy(row, rowLen, rowLen, Bvalues, Bindices);
util.Sort(true, rowLen, indices, 1, &values, 0, 0);
util.Sort(true, rowLen, Bindices, 1, &Bvalues, 0, 0);
bool same = true;
for(int j=0; j<rowLen; ++j) {
if (indices[j] != Bindices[j]) {
same = false;
break;
}
if (values[j] != Bvalues[j]) {
same = false;
break;
}
}
delete [] indices;
delete [] values;
if (!same) {
return(false);
}
}
return(true);
}
//=========================================================================
int Epetra_IntSerialDenseMatrix::Random() {
Epetra_Util util;
for(int j = 0; j < N_; j++) {
int* arrayPtr = A_ + (j * LDA_);
for(int i = 0; i < M_; i++) {
*arrayPtr++ = util.RandomInt();
}
}
return(0);
}
//==============================================================================
int LinearProblem_CrsSingletonFilter::CreatePostSolveArrays(const Epetra_IntVector & RowIDs,
const Epetra_MapColoring & RowMapColors,
const Epetra_IntVector & ColProfiles,
const Epetra_IntVector & NewColProfiles,
const Epetra_IntVector & ColHasRowWithSingleton) {
int j;
if (NumMyColSingletons_==0) return(0); // Nothing to do
Epetra_MapColoring & ColMapColors = *ColMapColors_;
int NumMyCols = FullMatrix()->NumMyCols();
// We will need these arrays for the post-solve phase
ColSingletonRowLIDs_ = new int[NumMyColSingletons_];
ColSingletonColLIDs_ = new int[NumMyColSingletons_];
ColSingletonPivotLIDs_ = new int[NumMyColSingletons_];
ColSingletonPivots_ = new double[NumMyColSingletons_];
// Register singleton columns (that were not already counted as singleton rows)
// Check to see if any columns disappeared because all associated rows were eliminated
int NumMyColSingletonstmp = 0;
for (j=0; j<NumMyCols; j++) {
int i = RowIDs[j];
if ( ColProfiles[j]==1 && RowMapColors[i]!=1 ) {
ColSingletonRowLIDs_[NumMyColSingletonstmp] = i;
ColSingletonColLIDs_[NumMyColSingletonstmp] = j;
NumMyColSingletonstmp++;
}
// Also check for columns that were eliminated implicitly by
// having all associated row eliminated
else if (NewColProfiles[j]==0 && ColHasRowWithSingleton[j]!=1 && RowMapColors[i]==0) {
ColMapColors[j] = 1;
}
}
assert(NumMyColSingletonstmp==NumMyColSingletons_); //Sanity check
Epetra_Util sorter;
sorter.Sort(true, NumMyColSingletons_, ColSingletonRowLIDs_, 0, 0, 1, &ColSingletonColLIDs_);
return(0);
}
//=========================================================================
int Epetra_MapColoring::GenerateLists() const {
int NumMyElements = Map().NumMyElements();
if (NumMyElements==0) return(0); // Nothing to do
if (ListsAreValid_) return(0); // Already been here
if (ListsAreGenerated_) DeleteLists(); // Delete any existing lists
// Scan the ElementColors to determine how many colors we have
NumColors_ = 1;
FirstColor_ = new ListItem(ElementColors_[0]); // Initialize First color in list
for (int i=1; i<NumMyElements; i++) if (!InItemList(ElementColors_[i])) NumColors_++;
// Create hash table that maps color IDs to the integers 0,...NumColors_
ColorIDs_ = new Epetra_HashTable<int>(NumColors_);
ListOfColors_ = new int[NumColors_];
ListItem * CurItem = FirstColor_;
{for (int i=0; i<NumColors_; i++) {
ColorIDs_->Add(CurItem->ItemValue, i); // Create hash table entry
ListOfColors_[i] = CurItem->ItemValue; // Put color value in a list of colors
CurItem = CurItem->NextItem;
}}
Epetra_Util util;
util.Sort(true, NumColors_, ListOfColors_, 0, 0, 0, 0, 0, 0); // Sort List of colors in ascending order
// Count the number of IDs of each color
ColorCount_ = new int[NumColors_];
{for (int i=0; i<NumColors_; i++) ColorCount_[i] = 0;}
{for (int i=0; i<NumMyElements; i++) ColorCount_[ColorIDs_->Get(ElementColors_[i])]++;}
// Finally build list of IDs grouped by color
ColorLists_ = new int *[NumColors_];
{for (int i=0; i<NumColors_; i++) ColorLists_[i] = new int[ColorCount_[i]];}
{for (int i=0; i<NumColors_; i++) ColorCount_[i] = 0;} // Reset so we can use for counting
{for (int i=0; i<NumMyElements; i++) {
int j = ColorIDs_->Get(ElementColors_[i]);
ColorLists_[j][ColorCount_[j]++] = i;
}}
ListsAreValid_ = true;
ListsAreGenerated_ = true;
return(0);
}
void Epetra_Export::Construct( const Epetra_BlockMap & sourceMap, const Epetra_BlockMap & targetMap)
{
int i;
// Build three ID lists:
// NumSameIDs - Number of IDs in TargetMap and SourceMap that are identical, up to the first
// nonidentical ID.
// NumPermuteIDs - Number of IDs in SourceMap that must be indirectly loaded but are on this processor.
// NumExportIDs - Number of IDs that are in SourceMap but not in TargetMap, and thus must be exported.
int NumSourceIDs = sourceMap.NumMyElements();
int NumTargetIDs = targetMap.NumMyElements();
int_type *TargetGIDs = 0;
if (NumTargetIDs>0) {
TargetGIDs = new int_type[NumTargetIDs];
targetMap.MyGlobalElements(TargetGIDs);
}
int_type * SourceGIDs = 0;
if (NumSourceIDs>0) {
SourceGIDs = new int_type[NumSourceIDs];
sourceMap.MyGlobalElements(SourceGIDs);
}
int MinIDs = EPETRA_MIN(NumSourceIDs, NumTargetIDs);
NumSameIDs_ = 0;
for (i=0; i< MinIDs; i++) if (TargetGIDs[i]==SourceGIDs[i]) NumSameIDs_++; else break;
// Find count of Source IDs that are truly remote and those that are local but permuted
NumPermuteIDs_ = 0;
NumExportIDs_ = 0;
for (i=NumSameIDs_; i< NumSourceIDs; i++)
if (targetMap.MyGID(SourceGIDs[i])) NumPermuteIDs_++; // Check if Source GID is a local Target GID
else NumExportIDs_++; // If not, then it is remote
// Define remote and permutation lists
int_type * ExportGIDs = 0;
if (NumExportIDs_>0) {
ExportLIDs_ = new int[NumExportIDs_];
ExportGIDs = new int_type[NumExportIDs_];
}
if (NumPermuteIDs_>0) {
PermuteToLIDs_ = new int[NumPermuteIDs_];
PermuteFromLIDs_ = new int[NumPermuteIDs_];
}
NumPermuteIDs_ = 0;
NumExportIDs_ = 0;
for (i=NumSameIDs_; i< NumSourceIDs; i++) {
if (targetMap.MyGID(SourceGIDs[i])) {
PermuteFromLIDs_[NumPermuteIDs_] = i;
PermuteToLIDs_[NumPermuteIDs_++] = targetMap.LID(SourceGIDs[i]);
}
else {
//NumSend_ +=sourceMap.ElementSize(i); // Count total number of entries to send
NumSend_ +=sourceMap.MaxElementSize(); // Count total number of entries to send (currently need max)
ExportGIDs[NumExportIDs_] = SourceGIDs[i];
ExportLIDs_[NumExportIDs_++] = i;
}
}
if ( NumExportIDs_>0 && !sourceMap.DistributedGlobal())
ReportError("Warning in Epetra_Export: Serial Export has remote IDs. (Exporting from Subset of Source Map)", 1);
// Test for distributed cases
int ierr = 0;
if (sourceMap.DistributedGlobal()) {
if (NumExportIDs_>0) ExportPIDs_ = new int[NumExportIDs_];
ierr = targetMap.RemoteIDList(NumExportIDs_, ExportGIDs, ExportPIDs_, 0); // Get remote PIDs
if( ierr ) throw ReportError("Error in Epetra_BlockMap::RemoteIDList", ierr);
//Get rid of IDs not in Target Map
if(NumExportIDs_>0) {
int cnt = 0;
for( i = 0; i < NumExportIDs_; ++i )
if( ExportPIDs_[i] == -1 ) ++cnt;
if( cnt ) {
int_type * NewExportGIDs = 0;
int * NewExportPIDs = 0;
int * NewExportLIDs = 0;
int cnt1 = NumExportIDs_-cnt;
if (cnt1) {
NewExportGIDs = new int_type[cnt1];
NewExportPIDs = new int[cnt1];
NewExportLIDs = new int[cnt1];
}
cnt = 0;
for( i = 0; i < NumExportIDs_; ++i )
if( ExportPIDs_[i] != -1 ) {
NewExportGIDs[cnt] = ExportGIDs[i];
NewExportPIDs[cnt] = ExportPIDs_[i];
NewExportLIDs[cnt] = ExportLIDs_[i];
++cnt;
}
assert(cnt==cnt1); // Sanity test
NumExportIDs_ = cnt;
delete [] ExportGIDs;
delete [] ExportPIDs_;
delete [] ExportLIDs_;
ExportGIDs = NewExportGIDs;
ExportPIDs_ = NewExportPIDs;
ExportLIDs_ = NewExportLIDs;
ReportError("Warning in Epetra_Export: Source IDs not found in Target Map (Do you want to export from subset of Source Map?)", 1 );
}
}
//Make sure Export IDs are ordered by processor
Epetra_Util util;
if(targetMap.GlobalIndicesLongLong()) {
// FIXME (mfh 11 Jul 2013) This breaks ANSI aliasing rules, if
// int_type != long long. On some compilers, it results in
// warnings such as this: "dereferencing type-punned pointer
// will break strict-aliasing rules".
util.Sort(true,NumExportIDs_,ExportPIDs_,0,0,1,&ExportLIDs_, 1, (long long **)&ExportGIDs);
}
else if(targetMap.GlobalIndicesInt()) {
int* ptrs[2] = {ExportLIDs_, (int*) ExportGIDs};
util.Sort(true,NumExportIDs_,ExportPIDs_,0,0, 2,&ptrs[0], 0, 0);
}
else {
throw ReportError("Epetra_Import::Epetra_Import: GlobalIndices Internal Error", -1);
}
Distor_ = sourceMap.Comm().CreateDistributor();
// Construct list of exports that calling processor needs to send as a result
// of everyone asking for what it needs to receive.
ierr = Distor_->CreateFromSends( NumExportIDs_, ExportPIDs_, true, NumRemoteIDs_);
if (ierr!=0) throw ReportError("Error in Epetra_Distributor.CreateFromSends()", ierr);
// Use comm plan with ExportGIDs to find out who is sending to us and
// get proper ordering of GIDs for remote entries
// (that we will convert to LIDs when done).
if (NumRemoteIDs_>0) RemoteLIDs_ = new int[NumRemoteIDs_]; // Allocate space for LIDs in target that are
// going to get something from off-processor.
char * cRemoteGIDs = 0; //Do will alloc memory for this object
int LenCRemoteGIDs = 0;
ierr = Distor_->Do(reinterpret_cast<char *> (ExportGIDs),
sizeof( int_type ),
LenCRemoteGIDs,
cRemoteGIDs);
if (ierr) throw ReportError("Error in Epetra_Distributor.Do()", ierr);
int_type * RemoteGIDs = reinterpret_cast<int_type*>(cRemoteGIDs);
// Remote IDs come in as GIDs, convert to LIDs
for (i=0; i< NumRemoteIDs_; i++) {
RemoteLIDs_[i] = targetMap.LID(RemoteGIDs[i]);
//NumRecv_ += targetMap.ElementSize(RemoteLIDs_[i]); // Count total number of entries to receive
NumRecv_ += targetMap.MaxElementSize(); // Count total number of entries to receive (currently need max)
}
if (LenCRemoteGIDs>0) delete [] cRemoteGIDs;
}
if (NumExportIDs_>0) delete [] ExportGIDs;
if (NumTargetIDs>0) delete [] TargetGIDs;
if (NumSourceIDs>0) delete [] SourceGIDs;
return;
}
//==============================================================================
// Epetra_Export constructor for a Epetra_BlockMap object
Epetra_Export::Epetra_Export( const Epetra_BlockMap & SourceMap, const Epetra_BlockMap & TargetMap)
: Epetra_Object("Epetra::Export"),
TargetMap_(TargetMap),
SourceMap_(SourceMap),
NumSameIDs_(0),
NumPermuteIDs_(0),
PermuteToLIDs_(0),
PermuteFromLIDs_(0),
NumRemoteIDs_(0),
RemoteLIDs_(0),
NumExportIDs_(0),
ExportLIDs_(0),
ExportPIDs_(0),
NumSend_(0),
NumRecv_(0),
Distor_(0)
{
int i;
// Build three ID lists:
// NumSameIDs - Number of IDs in TargetMap and SourceMap that are identical, up to the first
// nonidentical ID.
// NumPermuteIDs - Number of IDs in SourceMap that must be indirectly loaded but are on this processor.
// NumExportIDs - Number of IDs that are in SourceMap but not in TargetMap, and thus must be exported.
int NumSourceIDs = SourceMap.NumMyElements();
int NumTargetIDs = TargetMap.NumMyElements();
int *TargetGIDs = 0;
if (NumTargetIDs>0) {
TargetGIDs = new int[NumTargetIDs];
TargetMap.MyGlobalElements(TargetGIDs);
}
int * SourceGIDs = 0;
if (NumSourceIDs>0) {
SourceGIDs = new int[NumSourceIDs];
SourceMap.MyGlobalElements(SourceGIDs);
}
int MinIDs = EPETRA_MIN(NumSourceIDs, NumTargetIDs);
NumSameIDs_ = 0;
for (i=0; i< MinIDs; i++) if (TargetGIDs[i]==SourceGIDs[i]) NumSameIDs_++; else break;
// Find count of Source IDs that are truly remote and those that are local but permuted
NumPermuteIDs_ = 0;
NumExportIDs_ = 0;
for (i=NumSameIDs_; i< NumSourceIDs; i++)
if (TargetMap.MyGID(SourceGIDs[i])) NumPermuteIDs_++; // Check if Source GID is a local Target GID
else NumExportIDs_++; // If not, then it is remote
// Define remote and permutation lists
int * ExportGIDs = 0;
if (NumExportIDs_>0) {
ExportLIDs_ = new int[NumExportIDs_];
ExportGIDs = new int[NumExportIDs_];
}
if (NumPermuteIDs_>0) {
PermuteToLIDs_ = new int[NumPermuteIDs_];
PermuteFromLIDs_ = new int[NumPermuteIDs_];
}
NumPermuteIDs_ = 0;
NumExportIDs_ = 0;
for (i=NumSameIDs_; i< NumSourceIDs; i++) {
if (TargetMap.MyGID(SourceGIDs[i])) {
PermuteFromLIDs_[NumPermuteIDs_] = i;
PermuteToLIDs_[NumPermuteIDs_++] = TargetMap.LID(SourceGIDs[i]);
}
else {
//NumSend_ +=SourceMap.ElementSize(i); // Count total number of entries to send
NumSend_ +=SourceMap.MaxElementSize(); // Count total number of entries to send (currently need max)
ExportGIDs[NumExportIDs_] = SourceGIDs[i];
ExportLIDs_[NumExportIDs_++] = i;
}
}
if ( NumExportIDs_>0 && !SourceMap.DistributedGlobal())
ReportError("Warning in Epetra_Export: Serial Export has remote IDs. (Exporting from Subset of Source Map)", 1);
// Test for distributed cases
int ierr = 0;
if (SourceMap.DistributedGlobal()) {
if (NumExportIDs_>0) ExportPIDs_ = new int[NumExportIDs_];
ierr = TargetMap.RemoteIDList(NumExportIDs_, ExportGIDs, ExportPIDs_, 0); // Get remote PIDs
if( ierr ) throw ReportError("Error in Epetra_BlockMap::RemoteIDList", ierr);
//Get rid of IDs not in Target Map
if(NumExportIDs_>0) {
int cnt = 0;
for( i = 0; i < NumExportIDs_; ++i )
if( ExportPIDs_[i] == -1 ) ++cnt;
if( cnt ) {
int * NewExportGIDs = 0;
int * NewExportPIDs = 0;
int * NewExportLIDs = 0;
int cnt1 = NumExportIDs_-cnt;
if (cnt1) {
NewExportGIDs = new int[cnt1];
NewExportPIDs = new int[cnt1];
NewExportLIDs = new int[cnt1];
}
cnt = 0;
for( i = 0; i < NumExportIDs_; ++i )
if( ExportPIDs_[i] != -1 ) {
NewExportGIDs[cnt] = ExportGIDs[i];
NewExportPIDs[cnt] = ExportPIDs_[i];
NewExportLIDs[cnt] = ExportLIDs_[i];
++cnt;
}
assert(cnt==cnt1); // Sanity test
NumExportIDs_ = cnt;
delete [] ExportGIDs;
delete [] ExportPIDs_;
delete [] ExportLIDs_;
ExportGIDs = NewExportGIDs;
ExportPIDs_ = NewExportPIDs;
ExportLIDs_ = NewExportLIDs;
ReportError("Warning in Epetra_Export: Source IDs not found in Target Map (Do you want to export from subset of Source Map?)", 1 );
}
}
//Make sure Export IDs are ordered by processor
Epetra_Util util;
int * tmpPtr[2];
tmpPtr[0] = ExportLIDs_, tmpPtr[1] = ExportGIDs;
util.Sort(true,NumExportIDs_,ExportPIDs_,0,0,2,tmpPtr);
Distor_ = SourceMap.Comm().CreateDistributor();
// Construct list of exports that calling processor needs to send as a result
// of everyone asking for what it needs to receive.
ierr = Distor_->CreateFromSends( NumExportIDs_, ExportPIDs_, true, NumRemoteIDs_);
if (ierr!=0) throw ReportError("Error in Epetra_Distributor.CreateFromSends()", ierr);
// Use comm plan with ExportGIDs to find out who is sending to us and
// get proper ordering of GIDs for remote entries
// (that we will convert to LIDs when done).
if (NumRemoteIDs_>0) RemoteLIDs_ = new int[NumRemoteIDs_]; // Allocate space for LIDs in target that are
// going to get something from off-processor.
char * cRemoteGIDs = 0; //Do will alloc memory for this object
int LenCRemoteGIDs = 0;
ierr = Distor_->Do(reinterpret_cast<char *> (ExportGIDs),
sizeof( int ),
LenCRemoteGIDs,
cRemoteGIDs);
if (ierr) throw ReportError("Error in Epetra_Distributor.Do()", ierr);
int * RemoteGIDs = reinterpret_cast<int*>(cRemoteGIDs);
// Remote IDs come in as GIDs, convert to LIDs
for (i=0; i< NumRemoteIDs_; i++) {
RemoteLIDs_[i] = TargetMap.LID(RemoteGIDs[i]);
//NumRecv_ += TargetMap.ElementSize(RemoteLIDs_[i]); // Count total number of entries to receive
NumRecv_ += TargetMap.MaxElementSize(); // Count total number of entries to receive (currently need max)
}
if (NumExportIDs_>0) delete [] ExportGIDs;
if (LenCRemoteGIDs>0) delete [] cRemoteGIDs;
}
if (NumTargetIDs>0) delete [] TargetGIDs;
if (NumSourceIDs>0) delete [] SourceGIDs;
return;
}
void Epetra_Import::Construct( const Epetra_BlockMap & targetMap, const Epetra_BlockMap & sourceMap)
{
int i;
// Build three ID lists:
// NumSameIDs - Number of IDs in TargetMap and SourceMap that are identical, up to the first
// nonidentical ID.
// NumPermuteIDs - Number of IDs in SourceMap that must be indirectly loaded but are on this processor.
// NumRemoteIDs - Number of IDs that are in SourceMap but not in TargetMap, and thus must be imported.
int NumSourceIDs = sourceMap.NumMyElements();
int NumTargetIDs = targetMap.NumMyElements();
int_type *TargetGIDs = 0;
if (NumTargetIDs>0) {
TargetGIDs = new int_type[NumTargetIDs];
targetMap.MyGlobalElements(TargetGIDs);
}
int_type * SourceGIDs = 0;
if (NumSourceIDs>0) {
SourceGIDs = new int_type[NumSourceIDs];
sourceMap.MyGlobalElements(SourceGIDs);
}
int MinIDs = EPETRA_MIN(NumSourceIDs, NumTargetIDs);
NumSameIDs_ = 0;
for (i=0; i< MinIDs; i++) if (TargetGIDs[i]==SourceGIDs[i]) NumSameIDs_++; else break;
// Find count of Target IDs that are truly remote and those that are local but permuted
NumPermuteIDs_ = 0;
NumRemoteIDs_ = 0;
for (i=NumSameIDs_; i< NumTargetIDs; i++)
if (sourceMap.MyGID(TargetGIDs[i])) NumPermuteIDs_++; // Check if Target GID is a local Source GID
else NumRemoteIDs_++; // If not, then it is remote
// Define remote and permutation lists
int_type * RemoteGIDs=0;
RemoteLIDs_ = 0;
if (NumRemoteIDs_>0) {
RemoteLIDs_ = new int[NumRemoteIDs_];
RemoteGIDs = new int_type[NumRemoteIDs_];
}
if (NumPermuteIDs_>0) {
PermuteToLIDs_ = new int[NumPermuteIDs_];
PermuteFromLIDs_ = new int[NumPermuteIDs_];
}
NumPermuteIDs_ = 0;
NumRemoteIDs_ = 0;
for (i=NumSameIDs_; i< NumTargetIDs; i++) {
if (sourceMap.MyGID(TargetGIDs[i])) {
PermuteToLIDs_[NumPermuteIDs_] = i;
PermuteFromLIDs_[NumPermuteIDs_++] = sourceMap.LID(TargetGIDs[i]);
}
else {
//NumRecv_ +=TargetMap.ElementSize(i); // Count total number of entries to receive
NumRecv_ +=targetMap.MaxElementSize(); // Count total number of entries to receive (currently need max)
RemoteGIDs[NumRemoteIDs_] = TargetGIDs[i];
RemoteLIDs_[NumRemoteIDs_++] = i;
}
}
if( NumRemoteIDs_>0 && !sourceMap.DistributedGlobal() )
ReportError("Warning in Epetra_Import: Serial Import has remote IDs. (Importing to Subset of Target Map)", 1);
// Test for distributed cases
int * RemotePIDs = 0;
if (sourceMap.DistributedGlobal()) {
if (NumRemoteIDs_>0) RemotePIDs = new int[NumRemoteIDs_];
int ierr = sourceMap.RemoteIDList(NumRemoteIDs_, RemoteGIDs, RemotePIDs, 0); // Get remote PIDs
if (ierr) throw ReportError("Error in sourceMap.RemoteIDList call", ierr);
//Get rid of IDs that don't exist in SourceMap
if(NumRemoteIDs_>0) {
int cnt = 0;
for( i = 0; i < NumRemoteIDs_; ++i )
if( RemotePIDs[i] == -1 ) ++cnt;
if( cnt ) {
if( NumRemoteIDs_-cnt ) {
int_type * NewRemoteGIDs = new int_type[NumRemoteIDs_-cnt];
int * NewRemotePIDs = new int[NumRemoteIDs_-cnt];
int * NewRemoteLIDs = new int[NumRemoteIDs_-cnt];
cnt = 0;
for( i = 0; i < NumRemoteIDs_; ++i )
if( RemotePIDs[i] != -1 ) {
NewRemoteGIDs[cnt] = RemoteGIDs[i];
NewRemotePIDs[cnt] = RemotePIDs[i];
NewRemoteLIDs[cnt] = targetMap.LID(RemoteGIDs[i]);
++cnt;
}
NumRemoteIDs_ = cnt;
delete [] RemoteGIDs;
delete [] RemotePIDs;
delete [] RemoteLIDs_;
RemoteGIDs = NewRemoteGIDs;
RemotePIDs = NewRemotePIDs;
RemoteLIDs_ = NewRemoteLIDs;
ReportError("Warning in Epetra_Import: Target IDs not found in Source Map (Do you want to import to subset of Target Map?)", 1);
}
else { //valid RemoteIDs empty
NumRemoteIDs_ = 0;
delete [] RemoteGIDs;
RemoteGIDs = 0;
delete [] RemotePIDs;
RemotePIDs = 0;
}
}
}
//Sort Remote IDs by processor so DoReverses will work
Epetra_Util util;
if(targetMap.GlobalIndicesLongLong())
{
util.Sort(true,NumRemoteIDs_,RemotePIDs,0,0, 1,&RemoteLIDs_, 1,(long long**)&RemoteGIDs);
}
else if(targetMap.GlobalIndicesInt())
{
int* ptrs[2] = {RemoteLIDs_, (int*)RemoteGIDs};
util.Sort(true,NumRemoteIDs_,RemotePIDs,0,0,2,&ptrs[0], 0, 0);
}
else
{
throw ReportError("Epetra_Import::Epetra_Import: GlobalIndices Internal Error", -1);
}
Distor_ = sourceMap.Comm().CreateDistributor();
// Construct list of exports that calling processor needs to send as a result
// of everyone asking for what it needs to receive.
bool Deterministic = true;
int_type* tmp_ExportLIDs; //Export IDs come in as GIDs
ierr = Distor_->CreateFromRecvs( NumRemoteIDs_, RemoteGIDs, RemotePIDs,
Deterministic, NumExportIDs_, tmp_ExportLIDs, ExportPIDs_ );
if (ierr!=0) throw ReportError("Error in Epetra_Distributor.CreateFromRecvs()", ierr);
// Export IDs come in as GIDs, convert to LIDs
if(targetMap.GlobalIndicesLongLong())
{
ExportLIDs_ = new int[NumExportIDs_];
for (i=0; i< NumExportIDs_; i++) {
if (ExportPIDs_[i] < 0) throw ReportError("targetMap requested a GID that is not in the sourceMap.", -1);
ExportLIDs_[i] = sourceMap.LID(tmp_ExportLIDs[i]);
NumSend_ += sourceMap.MaxElementSize(); // Count total number of entries to send (currently need max)
}
delete[] tmp_ExportLIDs;
}
else if(targetMap.GlobalIndicesInt())
{
for (i=0; i< NumExportIDs_; i++) {
if (ExportPIDs_[i] < 0) throw ReportError("targetMap requested a GID that is not in the sourceMap.", -1);
tmp_ExportLIDs[i] = sourceMap.LID(tmp_ExportLIDs[i]);
NumSend_ += sourceMap.MaxElementSize(); // Count total number of entries to send (currently need max)
}
ExportLIDs_ = reinterpret_cast<int *>(tmp_ExportLIDs); // Can't reach here if tmp_ExportLIDs is long long.
}
else
{
throw ReportError("Epetra_Import::Epetra_Import: GlobalIndices Internal Error", -1);
}
}
if( NumRemoteIDs_>0 ) delete [] RemoteGIDs;
if( NumRemoteIDs_>0 ) delete [] RemotePIDs;
if (NumTargetIDs>0) delete [] TargetGIDs;
if (NumSourceIDs>0) delete [] SourceGIDs;
return;
}
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
int main(int argc, char *argv[])
{
using namespace std;
using namespace Teuchos;
using namespace PHX;
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
#endif
try {
RCP<Time> total_time = TimeMonitor::getNewTimer("Total Run Time");
TimeMonitor tm(*total_time);
bool print_debug_info = true;
cout << "\nStarting Epetra_VBR_Test Example!\n" << endl;
// *********************************************************
// * Build the Finite Element data structures
// *********************************************************
// Create the mesh, one strip of 2D elements.
const std::size_t num_local_cells = 5;
double domain_length = 1.0;
double dx = domain_length / static_cast<double>(num_local_cells);
std::vector<int> global_id(4);
global_id[0] = 0;
global_id[1] = 2;
global_id[2] = 3;
global_id[3] = 1;
std::vector<double> x_coords(4);
std::vector<double> y_coords(4);
std::vector<Element_Linear2D> cells;
for (std::size_t i = 0; i < num_local_cells; ++i) {
x_coords[0] = static_cast<double>(i) * dx;
x_coords[1] = x_coords[0] + dx;
x_coords[2] = x_coords[0] + dx;
x_coords[3] = static_cast<double>(i) * dx;
y_coords[0] = 0.0;
y_coords[1] = 0.0;
y_coords[2] = 1.0;
y_coords[3] = 1.0;
Element_Linear2D e(global_id, i, i, x_coords, y_coords);
cells.push_back(e);
// update global indices for next element
for (std::size_t i=0; i < global_id.size(); ++i)
global_id[i] += 2;
}
// Divide mesh into workset blocks
const std::size_t workset_size = 5;
std::vector<MyWorkset> worksets;
{
std::vector<Element_Linear2D>::iterator cell_it = cells.begin();
std::size_t count = 0;
MyWorkset w;
w.local_offset = cell_it->localElementIndex();
w.begin = cell_it;
for (; cell_it != cells.end(); ++cell_it) {
++count;
std::vector<Element_Linear2D>::iterator next = cell_it;
++next;
if ( count == workset_size || next == cells.end()) {
w.end = next;
w.num_cells = count;
worksets.push_back(w);
count = 0;
if (next != cells.end()) {
w.local_offset = next->localElementIndex();
w.begin = next;
}
}
}
}
if (print_debug_info) {
cout << "Printing Element Information" << endl;
for (std::size_t i = 0; i < worksets.size(); ++i) {
std::vector<Element_Linear2D>::iterator it = worksets[i].begin;
for (; it != worksets[i].end; ++it)
cout << *it << endl;
}
}
if (print_debug_info) {
for (std::size_t i = 0; i < worksets.size(); ++i) {
cout << "Printing Workset Information" << endl;
cout << "worksets[" << i << "]" << endl;
cout << " num_cells =" << worksets[i].num_cells << endl;
cout << " local_offset =" << worksets[i].local_offset << endl;
std::vector<Element_Linear2D>::iterator it = worksets[i].begin;
for (; it != worksets[i].end; ++it)
cout << " cell_local_index =" << it->localElementIndex() << endl;
}
cout << endl;
}
// *********************************************************
// * Build the Newton solver data structures
// *********************************************************
// Setup Nonlinear Problem (build Epetra_Vector and Epetra_CrsMatrix)
// Newton's method: J delta_x = -f
const std::size_t num_eq = 2;
const std::size_t num_nodes = 2 * (num_local_cells +1);
const std::size_t num_dof = num_nodes * num_eq;
RCP<Epetra_Vector> x;
RCP<Epetra_Vector> delta_x;
RCP<Epetra_Vector> f;
RCP<Epetra_VbrRowMatrix> Jac;
{
Epetra_SerialComm comm;
Epetra_BlockMap map(static_cast<int>(num_nodes), static_cast<int>(num_eq), 0, comm);
Epetra_DataAccess copy = ::Copy;
Epetra_CrsGraph graph(copy, map, 3);
std::vector<Element_Linear2D>::iterator e = cells.begin();
for (; e != cells.end(); ++e) {
for (int row = 0; row < e->numNodes(); ++row) {
for (int col = 0; col < e->numNodes(); ++col) {
int global_row = e->globalNodeId(row);
int global_col = e->globalNodeId(col);
graph.InsertGlobalIndices(global_row, 1, &global_col);
}
}
}
graph.FillComplete();
graph.Print(cout);
Epetra_SerialDenseMatrix block_matrix(2,2);
Epetra_SerialDenseMatrix diag_block_matrix(2,2);
RCP<Epetra_VbrMatrix> Jac_vbr = rcp(new Epetra_VbrMatrix(copy,graph));
Epetra_Util util;
e = cells.begin();
for (; e != cells.end(); ++e) {
for (int row = 0; row < e->numNodes(); ++row) {
int global_row = e->globalNodeId(row);
block_matrix(0,0) = util.RandomDouble();
block_matrix(0,1) = util.RandomDouble();
block_matrix(1,0) = util.RandomDouble();
block_matrix(1,1) = util.RandomDouble();
diag_block_matrix(0,0) = 100.0*util.RandomDouble();
diag_block_matrix(0,1) = util.RandomDouble();
diag_block_matrix(1,0) = util.RandomDouble();
diag_block_matrix(1,1) = 100.0*util.RandomDouble();
for (int col = 0; col < e->numNodes(); ++col) {
int global_col = e->globalNodeId(col);
Jac_vbr->BeginReplaceMyValues(global_row, 1, &global_col);
if (global_row==global_col)
Jac_vbr->SubmitBlockEntry(diag_block_matrix);
else
Jac_vbr->SubmitBlockEntry(block_matrix);
Jac_vbr->EndSubmitEntries();
}
}
}
Jac_vbr->FillComplete();
x = rcp(new Epetra_Vector(map));
delta_x = rcp(new Epetra_Vector(map));
f = rcp(new Epetra_Vector(map));
x->PutScalar(1.0);
Jac_vbr->Apply(*x,*f);
Jac =
rcpWithEmbeddedObjPostDestroy(new Epetra_VbrRowMatrix(Jac_vbr.get()),
Jac_vbr);
}
if (print_debug_info) {
x->Print(cout);
Jac->Print(cout);
f->Print(cout);
}
// *********************************************************
// * Build Preconditioner (Ifpack)
// *********************************************************
Ifpack Factory;
std::string PrecType = "ILU";
int OverlapLevel = 1;
RCP<Ifpack_Preconditioner> Prec =
Teuchos::rcp( Factory.Create(PrecType, &*Jac, OverlapLevel) );
ParameterList ifpackList;
ifpackList.set("fact: drop tolerance", 1e-9);
ifpackList.set("fact: level-of-fill", 1);
ifpackList.set("schwarz: combine mode", "Add");
IFPACK_CHK_ERR(Prec->SetParameters(ifpackList));
IFPACK_CHK_ERR(Prec->Initialize());
IFPACK_CHK_ERR(Prec->Compute());
IFPACK_CHK_ERR(Prec->Condest());
RCP<Belos::EpetraPrecOp> belosPrec =
rcp( new Belos::EpetraPrecOp( Prec ) );
// *********************************************************
// * Build linear solver (Belos)
// *********************************************************
// Linear solver parameters
typedef double ST;
typedef Teuchos::ScalarTraits<ST> SCT;
typedef SCT::magnitudeType MT;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Belos::MultiVecTraits<ST,MV> MVT;
typedef Belos::OperatorTraits<ST,MV,OP> OPT;
RCP<ParameterList> belosList = rcp(new ParameterList);
belosList->set<int>("Num Blocks", num_dof);
belosList->set<int>("Block Size", 1);
belosList->set<int>("Maximum Iterations", 400);
belosList->set<int>("Maximum Restarts", 0);
belosList->set<MT>( "Convergence Tolerance", 1.0e-4);
int verbosity = Belos::Errors + Belos::Warnings;
if (false) {
verbosity += Belos::TimingDetails + Belos::StatusTestDetails;
belosList->set<int>( "Output Frequency", -1);
}
if (print_debug_info) {
verbosity += Belos::Debug;
belosList->set<int>( "Output Frequency", -1);
}
belosList->set( "Verbosity", verbosity );
RCP<Epetra_MultiVector> F =
Teuchos::rcp_implicit_cast<Epetra_MultiVector>(f);
RCP<Epetra_MultiVector> DX =
Teuchos::rcp_implicit_cast<Epetra_MultiVector>(delta_x);
RCP<Belos::LinearProblem<double,MV,OP> > problem =
rcp(new Belos::LinearProblem<double,MV,OP>(Jac, DX, F) );
problem->setRightPrec( belosPrec );
RCP< Belos::SolverManager<double,MV,OP> > gmres_solver =
rcp( new Belos::BlockGmresSolMgr<double,MV,OP>(problem, belosList) );
// *********************************************************
// * Solve the system
// *********************************************************
RCP<Time> linear_solve_time =
TimeMonitor::getNewTimer("Linear Solve Time");
std::size_t num_gmres_iterations = 0;
{
TimeMonitor t(*linear_solve_time);
delta_x->PutScalar(0.0);
IFPACK_CHK_ERR(Prec->Compute());
problem->setProblem();
Belos::ReturnType ret = gmres_solver->solve();
int num_iters = gmres_solver->getNumIters();
num_gmres_iterations += num_iters;
if (print_debug_info)
std::cout << "Number of gmres iterations performed for this solve: "
<< num_iters << std::endl;
if (ret!=Belos::Converged) {
std::cout << std::endl << "WARNING: Belos did not converge!"
<< std::endl;
}
}
delta_x->Print(cout);
// *********************************************************************
// Finished all testing
// *********************************************************************
std::cout << "\nRun has completed successfully!\n" << std::endl;
// *********************************************************************
// *********************************************************************
}
catch (const std::exception& e) {
std::cout << "************************************************" << endl;
std::cout << "************************************************" << endl;
std::cout << "Exception Caught!" << endl;
std::cout << "Error message is below\n " << e.what() << endl;
std::cout << "************************************************" << endl;
}
catch (...) {
std::cout << "************************************************" << endl;
std::cout << "************************************************" << endl;
std::cout << "Unknown Exception Caught!" << endl;
std::cout << "************************************************" << endl;
}
TimeMonitor::summarize();
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return 0;
}
bool matrix_data::compare_local_data(const Epetra_CrsMatrix& A)
{
const Epetra_Map& map = A.RowMap();
int numMyRows = map.NumMyElements();
Epetra_Util util;
if(map.GlobalIndicesInt()) {
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
int* myRows_int = map.MyGlobalElements();
for(int i=0; i<numMyRows; ++i) {
int row = myRows_int[i];
int rowLen = A.NumGlobalEntries(row);
if (rowLen != rowlengths_[row]) {
return(false);
}
int* indices = new int[rowLen];
double* values = new double[rowLen];
A.ExtractGlobalRowCopy(row, rowLen, rowLen, values, indices);
util.Sort(true, rowLen, indices, 1, &values, 0, 0, 0, 0);
bool same = true;
int* this_indices = colindices_[row];
double* this_coefs = coefs_[row];
for(int j=0; j<rowLen; ++j) {
if (indices[j] != this_indices[j]) {
same = false; break;
}
if (values[j] != this_coefs[j]) {
same = false; break;
}
}
delete [] indices;
delete [] values;
if (!same) return(false);
}
#else
throw "matrix_data::compare_local_data: global index int but no API for it.";
#endif
}
else if(map.GlobalIndicesLongLong()) {
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
long long* myRows_LL = map.MyGlobalElements64();
for(int i=0; i<numMyRows; ++i) {
long long row = myRows_LL[i];
int rowLen = A.NumGlobalEntries(row);
if (rowLen != rowlengths_[row]) {
return(false);
}
long long* indices = new long long[rowLen];
double* values = new double[rowLen];
A.ExtractGlobalRowCopy(row, rowLen, rowLen, values, indices);
util.Sort(true, rowLen, indices, 1, &values, 0, 0, 0, 0);
bool same = true;
int* this_indices = colindices_[row];
double* this_coefs = coefs_[row];
for(int j=0; j<rowLen; ++j) {
if (indices[j] != this_indices[j]) {
same = false; break;
}
if (values[j] != this_coefs[j]) {
same = false; break;
}
}
delete [] indices;
delete [] values;
if (!same) return(false);
}
#else
throw "matrix_data::compare_local_data: global index long long but no API for it.";
#endif
}
else {
assert(false);
throw "matrix_data::compare_local_data: global index type of map unknown.";
}
return(true);
}
void Epetra_Import::Construct_Expert( const Epetra_BlockMap & targetMap, const Epetra_BlockMap & sourceMap, int NumRemotePIDs, const int * UserRemotePIDs,
const int & UserNumExportIDs, const int * UserExportLIDs, const int * UserExportPIDs)
{
int i,ierr;
// Build three ID lists:
// NumSameIDs - Number of IDs in TargetMap and SourceMap that are identical, up to the first
// nonidentical ID.
// NumPermuteIDs - Number of IDs in SourceMap that must be indirectly loaded but are on this processor.
// NumRemoteIDs - Number of IDs that are in SourceMap but not in TargetMap, and thus must be imported.
int NumSourceIDs = sourceMap.NumMyElements();
int NumTargetIDs = targetMap.NumMyElements();
int_type *TargetGIDs = 0;
if (NumTargetIDs>0) {
TargetGIDs = new int_type[NumTargetIDs];
targetMap.MyGlobalElements(TargetGIDs);
}
int_type * SourceGIDs = 0;
if (NumSourceIDs>0) {
SourceGIDs = new int_type[NumSourceIDs];
sourceMap.MyGlobalElements(SourceGIDs);
}
int MinIDs = EPETRA_MIN(NumSourceIDs, NumTargetIDs);
NumSameIDs_ = 0;
for (i=0; i< MinIDs; i++) if (TargetGIDs[i]==SourceGIDs[i]) NumSameIDs_++; else break;
// Find count of Target IDs that are truly remote and those that are local but permuted
NumPermuteIDs_ = 0;
NumRemoteIDs_ = 0;
for (i=NumSameIDs_; i< NumTargetIDs; i++)
if (sourceMap.MyGID(TargetGIDs[i])) NumPermuteIDs_++; // Check if Target GID is a local Source GID
else NumRemoteIDs_++; // If not, then it is remote
// Define remote and permutation lists
int_type * RemoteGIDs=0;
RemoteLIDs_ = 0;
if (NumRemoteIDs_>0) {
RemoteLIDs_ = new int[NumRemoteIDs_];
RemoteGIDs = new int_type[NumRemoteIDs_];
}
if (NumPermuteIDs_>0) {
PermuteToLIDs_ = new int[NumPermuteIDs_];
PermuteFromLIDs_ = new int[NumPermuteIDs_];
}
NumPermuteIDs_ = 0;
NumRemoteIDs_ = 0;
for (i=NumSameIDs_; i< NumTargetIDs; i++) {
if (sourceMap.MyGID(TargetGIDs[i])) {
PermuteToLIDs_[NumPermuteIDs_] = i;
PermuteFromLIDs_[NumPermuteIDs_++] = sourceMap.LID(TargetGIDs[i]);
}
else {
//NumRecv_ +=TargetMap.ElementSize(i); // Count total number of entries to receive
NumRecv_ +=targetMap.MaxElementSize(); // Count total number of entries to receive (currently need max)
RemoteGIDs[NumRemoteIDs_] = TargetGIDs[i];
RemoteLIDs_[NumRemoteIDs_++] = i;
}
}
if( NumRemoteIDs_>0 && !sourceMap.DistributedGlobal() )
ReportError("Warning in Epetra_Import: Serial Import has remote IDs. (Importing to Subset of Target Map)", 1);
// Test for distributed cases
int * RemotePIDs = 0;
if (sourceMap.DistributedGlobal()) {
if (NumRemoteIDs_>0) RemotePIDs = new int[NumRemoteIDs_];
#ifdef EPETRA_ENABLE_DEBUG
int myeq = (NumRemotePIDs==NumRemoteIDs_);
int globaleq=0;
sourceMap.Comm().MinAll(&myeq,&globaleq,1);
if(globaleq!=1) {
printf("[%d] UserRemotePIDs count wrong %d != %d\n",sourceMap.Comm().MyPID(),NumRemotePIDs,NumRemoteIDs_);
fflush(stdout);
sourceMap.Comm().Barrier();
sourceMap.Comm().Barrier();
sourceMap.Comm().Barrier();
throw ReportError("Epetra_Import: UserRemotePIDs count wrong");
}
#endif
if(NumRemotePIDs==NumRemoteIDs_){
// Since I need to sort these, I'll copy them
for(i=0; i<NumRemoteIDs_; i++) RemotePIDs[i] = UserRemotePIDs[i];
}
//Get rid of IDs that don't exist in SourceMap
if(NumRemoteIDs_>0) {
int cnt = 0;
for( i = 0; i < NumRemoteIDs_; ++i )
if( RemotePIDs[i] == -1 ) ++cnt;
if( cnt ) {
if( NumRemoteIDs_-cnt ) {
int_type * NewRemoteGIDs = new int_type[NumRemoteIDs_-cnt];
int * NewRemotePIDs = new int[NumRemoteIDs_-cnt];
int * NewRemoteLIDs = new int[NumRemoteIDs_-cnt];
cnt = 0;
for( i = 0; i < NumRemoteIDs_; ++i )
if( RemotePIDs[i] != -1 ) {
NewRemoteGIDs[cnt] = RemoteGIDs[i];
NewRemotePIDs[cnt] = RemotePIDs[i];
NewRemoteLIDs[cnt] = targetMap.LID(RemoteGIDs[i]);
++cnt;
}
NumRemoteIDs_ = cnt;
delete [] RemoteGIDs;
delete [] RemotePIDs;
delete [] RemoteLIDs_;
RemoteGIDs = NewRemoteGIDs;
RemotePIDs = NewRemotePIDs;
RemoteLIDs_ = NewRemoteLIDs;
ReportError("Warning in Epetra_Import: Target IDs not found in Source Map (Do you want to import to subset of Target Map?)", 1);
}
else { //valid RemoteIDs empty
NumRemoteIDs_ = 0;
delete [] RemoteGIDs;
RemoteGIDs = 0;
delete [] RemotePIDs;
RemotePIDs = 0;
}
}
}
//Sort Remote IDs by processor so DoReverses will work
Epetra_Util util;
if(targetMap.GlobalIndicesLongLong())
{
util.Sort(true,NumRemoteIDs_,RemotePIDs,0,0, 1,&RemoteLIDs_, 1,(long long**)&RemoteGIDs);
}
else if(targetMap.GlobalIndicesInt())
{
int* ptrs[2] = {RemoteLIDs_, (int*)RemoteGIDs};
util.Sort(true,NumRemoteIDs_,RemotePIDs,0,0,2,&ptrs[0], 0, 0);
}
else
{
throw ReportError("Epetra_Import::Epetra_Import: GlobalIndices Internal Error", -1);
}
// Build distributor & Export lists
Distor_ = sourceMap.Comm().CreateDistributor();
NumExportIDs_=UserNumExportIDs;
ExportLIDs_ = new int[NumExportIDs_];
ExportPIDs_ = new int[NumExportIDs_];
for(i=0; i<NumExportIDs_; i++) {
ExportPIDs_[i] = UserExportPIDs[i];
ExportLIDs_[i] = UserExportLIDs[i];
}
#ifdef HAVE_MPI
Epetra_MpiDistributor* MpiDistor = dynamic_cast< Epetra_MpiDistributor*>(Distor_);
bool Deterministic = true;
if(MpiDistor)
ierr=MpiDistor->CreateFromSendsAndRecvs(NumExportIDs_,ExportPIDs_,
NumRemoteIDs_, RemoteGIDs, RemotePIDs,Deterministic);
else ierr=-10;
#else
ierr=-20;
#endif
if (ierr!=0) throw ReportError("Error in Epetra_Distributor.CreateFromRecvs()", ierr);
}
if( NumRemoteIDs_>0 ) delete [] RemoteGIDs;
if( NumRemoteIDs_>0 ) delete [] RemotePIDs;
if (NumTargetIDs>0) delete [] TargetGIDs;
if (NumSourceIDs>0) delete [] SourceGIDs;
#ifdef EPETRA_ENABLE_DEBUG
// Sanity check to make sure we got the import right
Epetra_IntVector Source(sourceMap);
Epetra_IntVector Target(targetMap);
for(i=0; i<Source.MyLength(); i++)
Source[i] = (int) (Source.Map().GID(i) % INT_MAX);
Target.PutValue(-1);
Target.Import(Source,*this,Insert);
bool test_passed=true;
for(i=0; i<Target.MyLength(); i++){
if(Target[i] != Target.Map().GID(i) % INT_MAX) test_passed=false;
}
if(!test_passed) {
printf("[%d] PROCESSOR has a mismatch... prepearing to crash or hang!\n",sourceMap.Comm().MyPID());
fflush(stdout);
sourceMap.Comm().Barrier();
sourceMap.Comm().Barrier();
sourceMap.Comm().Barrier();
throw ReportError("Epetra_Import: ERROR. User provided IDs do not match what an import generates.");
}
#endif
return;
}
