bool Tpetra::Utils::parseRfmt(Teuchos::ArrayRCP<char> fmt, int &perline, int &width, int &prec, char &valformat) {
TEUCHOS_TEST_FOR_EXCEPT(fmt.size() != 0 && fmt[fmt.size()-1] != '\0');
std::transform(fmt.begin(), fmt.end(), fmt, static_cast < int(*)(int) > (std::toupper));
// find the first left paren '(' and the last right paren ')'
Teuchos::ArrayRCP<char>::iterator firstLeftParen = std::find( fmt.begin(), fmt.end(), '(');
Teuchos::ArrayRCP<char>::iterator lastRightParen = std::find(std::reverse_iterator<Teuchos::ArrayRCP<char>::iterator>(fmt.end()),
std::reverse_iterator<Teuchos::ArrayRCP<char>::iterator>(fmt.begin()),
')').base()-1;
// select the substring between the parens, including them
// if neither was found, set the string to empty
if (firstLeftParen == fmt.end() || lastRightParen == fmt.begin()) {
fmt.resize(0 + 1);
fmt[0] = '\0';
}
else {
fmt += (firstLeftParen - fmt.begin());
size_t newLen = lastRightParen - firstLeftParen + 1;
fmt.resize(newLen + 1);
fmt[newLen] = '\0';
}
if (std::find(fmt.begin(),fmt.end(),'P') != fmt.end()) {
// not supported
return true;
}
bool error = true;
if (std::sscanf(fmt.getRawPtr(),"(%d%c%d.%d)",&perline,&valformat,&width,&prec) == 4) {
if (valformat == 'E' || valformat == 'D' || valformat == 'F') {
error = false;
}
}
return error;
}
void dump(const MV& v, const std::string& name) {
std::cout << name << std::endl;
Teuchos::ArrayRCP<const Scalar> view = v.get1dView();
for (Teuchos::ArrayRCP<const Scalar>::iterator it = view.begin(); it != view.end(); ++it) {
std::cout << *it << std::endl;
}
std::cout << std::endl;
}
//-----------------------------------------------------------------------------
double TpetraVector::sum() const
{
dolfin_assert(!_x.is_null());
Teuchos::ArrayRCP<const double> arr = _x->getData(0);
const double _sum = std::accumulate(arr.begin(), arr.end(), 0.0);
return MPI::sum(mpi_comm(), _sum);
}
bool Tpetra::Utils::parseIfmt(Teuchos::ArrayRCP<char> fmt, int &perline, int &width) {
TEUCHOS_TEST_FOR_EXCEPT(fmt.size() != 0 && fmt[fmt.size()-1] != '\0');
// parses integers n and d out of (nId)
bool error = true;
std::transform(fmt.begin(), fmt.end(), fmt, static_cast < int(*)(int) > (std::toupper));
if (std::sscanf(fmt.getRawPtr(),"(%dI%d)",&perline,&width) == 2) {
error = false;
}
return error;
}
void Tpetra::Utils::readHBHeader(std::ifstream &fin, Teuchos::ArrayRCP<char> &Title, Teuchos::ArrayRCP<char> &Key, Teuchos::ArrayRCP<char> &Type,
int &Nrow, int &Ncol, int &Nnzero, int &Nrhs,
Teuchos::ArrayRCP<char> &Ptrfmt, Teuchos::ArrayRCP<char> &Indfmt, Teuchos::ArrayRCP<char> &Valfmt, Teuchos::ArrayRCP<char> &Rhsfmt,
int &Ptrcrd, int &Indcrd, int &Valcrd, int &Rhscrd, Teuchos::ArrayRCP<char> &Rhstype) {
int Totcrd, Neltvl, Nrhsix;
const int MAXLINE = 81;
char line[MAXLINE];
//
Title.resize(72 + 1); std::fill(Title.begin(), Title.end(), '\0');
Key.resize(8 + 1); std::fill(Key.begin(), Key.end(), '\0');
Type.resize(3 + 1); std::fill(Type.begin(), Type.end(), '\0');
Ptrfmt.resize(16 + 1); std::fill(Ptrfmt.begin(), Ptrfmt.end(), '\0');
Indfmt.resize(16 + 1); std::fill(Indfmt.begin(), Indfmt.end(), '\0');
Valfmt.resize(20 + 1); std::fill(Valfmt.begin(), Valfmt.end(), '\0');
Rhsfmt.resize(20 + 1); std::fill(Rhsfmt.begin(), Rhsfmt.end(), '\0');
//
const std::string errStr("Tpetra::Utils::readHBHeader(): Improperly formatted H/B file: ");
/* First line: (A72,A8) */
fin.getline(line,MAXLINE);
TEUCHOS_TEST_FOR_EXCEPTION( std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
(void)std::sscanf(line, "%72c%8[^\n]", Title.getRawPtr(), Key.getRawPtr());
/* Second line: (5I14) or (4I14) */
fin.getline(line,MAXLINE);
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
if ( std::sscanf(line,"%14d%14d%14d%14d%14d",&Totcrd,&Ptrcrd,&Indcrd,&Valcrd,&Rhscrd) != 5 ) {
Rhscrd = 0;
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%14d%14d%14d%14d",&Totcrd,&Ptrcrd,&Indcrd,&Valcrd) != 4, std::runtime_error, errStr << "error reading pointers (line 2)");
}
/* Third line: (A3, 11X, 4I14) */
fin.getline(line,MAXLINE);
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line, "%3c%14i%14i%14i%14i", Type.getRawPtr(),&Nrow,&Ncol,&Nnzero,&Neltvl) != 5 , std::runtime_error, errStr << "error reading matrix meta-data (line 3)");
std::transform(Type.begin(), Type.end(), Type.begin(), static_cast < int(*)(int) > (std::toupper));
/* Fourth line: */
fin.getline(line,MAXLINE);
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
if (Rhscrd != 0) {
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%16c%16c%20c%20c",Ptrfmt.getRawPtr(),Indfmt.getRawPtr(),Valfmt.getRawPtr(),Rhsfmt.getRawPtr()) != 4, std::runtime_error, errStr << "error reading formats (line 4)");
}
else {
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%16c%16c%20c",Ptrfmt.getRawPtr(),Indfmt.getRawPtr(),Valfmt.getRawPtr()) != 3, std::runtime_error, errStr << "error reading formats (line 4)");
}
/* (Optional) Fifth line: */
if (Rhscrd != 0 ) {
Rhstype.resize(3 + 1,'\0');
fin.getline(line,MAXLINE);
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%3c%14d%14d", Rhstype.getRawPtr(), &Nrhs, &Nrhsix) != 3, std::runtime_error, errStr << "error reading right-hand-side meta-data (line 5)");
}
}
void
Filtered_UniqueGlobalIndexer<LocalOrdinalT,GlobalOrdinalT>::
getOwnedAndGhostedNotFilteredIndicator(std::vector<int> & indicator) const
{
using Teuchos::RCP;
typedef GlobalOrdinalT GO;
typedef LocalOrdinalT LO;
typedef panzer::TpetraNodeType Node;
typedef Tpetra::Map<LO, GO, Node> Map;
typedef Tpetra::Vector<GO,LO,GO,Node> Vector;
typedef Tpetra::Import<LO,GO,Node> Import;
std::vector<GlobalOrdinalT> ownedIndices;
std::vector<GlobalOrdinalT> ghostedIndices;
// build owned and ghosted maps
getOwnedIndices(ownedIndices);
getOwnedAndGhostedIndices(ghostedIndices);
RCP<const Map> ownedMap
= Tpetra::createNonContigMap<LO,GO>(ownedIndices,getComm());
RCP<const Map> ghostedMap
= Tpetra::createNonContigMap<LO,GO>(ghostedIndices,getComm());
// allocate the owned vector, mark those GIDs as unfiltered
// (they are by definition)
Vector ownedActive(ownedMap);
ownedActive.putScalar(1);
// Initialize all indices to zero
Vector ghostedActive(ghostedMap);
ghostedActive.putScalar(0);
// do communication, marking unfiltered indices as 1 (filtered
// indices locally are marked as zero)
Import importer(ownedMap,ghostedMap);
ghostedActive.doImport(ownedActive,importer,Tpetra::INSERT);
Teuchos::ArrayRCP<const GO> data = ghostedActive.getData();
// copy communicated data (clear it out first)
indicator.clear();
indicator.insert(indicator.end(),data.begin(),data.end());
}
//---------------------------------------------------------------------------//
TEUCHOS_UNIT_TEST( FieldTools, tensor_field_test )
{
using namespace DataTransferKit;
// Setup random numbers for the test.
int num_rand = 4;
double rand_max = 10.0;
Teuchos::Array<double> random_numbers( num_rand );
std::srand( 1 );
for ( int i = 0; i < num_rand; ++i )
{
random_numbers[i] = rand_max * (double) std::rand() / RAND_MAX;
}
// Setup communication.
Teuchos::RCP< const Teuchos::Comm<int> > comm = getDefaultComm<int>();
int my_rank = comm->getRank();
int my_size = comm->getSize();
// Setup a field manager.
int field_dim = 9;
typedef FieldTraits<ArrayField> FT;
Teuchos::RCP<ArrayField> array_field =
Teuchos::rcp( new ArrayField( field_dim*(my_rank+1), field_dim ) );
FieldManager<ArrayField> field_manager( array_field, comm );
// Test the field tools.
typedef FieldTools<ArrayField> Tools;
// Dimension iterators.
FT::iterator dim_begin, dim_end;
for ( int d = 0; d < field_dim; ++d )
{
dim_begin = Tools::dimBegin( *field_manager.field(), d );
dim_end = Tools::dimEnd( *field_manager.field(), d );
TEST_ASSERT( std::distance( dim_begin, dim_end ) == my_rank+1 );
}
FT::const_iterator const_dim_begin, const_dim_end;
for ( int d = 0; d < field_dim; ++d )
{
const_dim_begin = Tools::dimBegin( *field_manager.field(), d );
const_dim_end = Tools::dimEnd( *field_manager.field(), d );
TEST_ASSERT( std::distance( const_dim_begin, const_dim_end )
== my_rank+1 );
}
// Length.
int global_size = 0;
for ( int i = 0; i < my_size; ++i )
{
global_size += field_dim*(i+1);
}
TEST_ASSERT(
Tools::globalSize( *field_manager.field(), field_manager.comm() )
== global_size );
// Scalar filling.
Tools::putScalar( *field_manager.field(), random_numbers[0] );
// Views.
double local_val;
Teuchos::ArrayRCP<const double> const_view =
Tools::view( *field_manager.field() );
Teuchos::ArrayRCP<const double>::const_iterator const_view_iterator;
for ( const_view_iterator = const_view.begin();
const_view_iterator != const_view.end();
++const_view_iterator )
{
local_val = random_numbers[0];
TEST_ASSERT( *const_view_iterator == local_val );
}
Teuchos::ArrayRCP<double> view =
Tools::nonConstView( *field_manager.field() );
Teuchos::ArrayRCP<double>::iterator view_iterator;
for ( view_iterator = view.begin();
view_iterator != view.end();
++view_iterator )
{
local_val = random_numbers[0];
TEST_ASSERT( *view_iterator == local_val );
}
Teuchos::ArrayRCP<double> copy =
Tools::copy( *field_manager.field() );
Teuchos::ArrayRCP<double>::const_iterator copy_iterator;
for ( copy_iterator = copy.begin();
copy_iterator != copy.end();
++copy_iterator )
{
local_val = random_numbers[0];
TEST_ASSERT( *copy_iterator == local_val );
}
// Multiple scalar filling.
Teuchos::Array<double> fillers( field_dim );
for ( int d = 0; d < field_dim; ++d )
{
fillers[d] = (d+1)*random_numbers[1];
}
Tools::putScalar( *field_manager.field(), fillers() );
// Dim iterators.
FT::const_iterator const_dim_iterator;
for ( int d = 0; d < field_dim; ++d )
{
for ( const_dim_iterator = Tools::dimBegin( *field_manager.field(), d );
const_dim_iterator != Tools::dimEnd( *field_manager.field(), d );
++const_dim_iterator )
{
local_val = (d+1) * random_numbers[1];
TEST_ASSERT( *const_dim_iterator == local_val );
}
}
FT::iterator dim_iterator;
for ( int d = 0; d < field_dim; ++d )
{
for ( dim_iterator = Tools::dimBegin( *field_manager.field(), d );
dim_iterator != Tools::dimEnd( *field_manager.field(), d );
++dim_iterator )
{
local_val = (d+1) * random_numbers[1];
TEST_ASSERT( *dim_iterator == local_val );
}
}
// Scaling.
Tools::scale( *field_manager.field(), random_numbers[2] );
for ( int d = 0; d < field_dim; ++d )
{
for ( dim_iterator = Tools::dimBegin( *field_manager.field(), d );
dim_iterator != Tools::dimEnd( *field_manager.field(), d );
++dim_iterator )
{
local_val = (d+1) * random_numbers[1] * random_numbers[2];
TEST_ASSERT( *dim_iterator == local_val );
}
}
// Multiple scaling.
Teuchos::Array<double> multipliers( field_dim );
for ( int d = 0; d < field_dim; ++d )
{
multipliers[d] = -(d+1) - random_numbers[3];
}
Tools::scale( *field_manager.field(), multipliers() );
for ( int d = 0; d < field_dim; ++d )
{
for ( dim_iterator = Tools::dimBegin( *field_manager.field(), d );
dim_iterator != Tools::dimEnd( *field_manager.field(), d );
++dim_iterator )
{
local_val = (d+1) * random_numbers[1]
* random_numbers[2] * (-(d+1) - random_numbers[3]);
TEST_ASSERT( *dim_iterator == local_val );
}
}
// Norms.
Teuchos::Array<double> inf_norms;
Tools::normInf( *field_manager.field(), field_manager.comm(), inf_norms );
for ( int d = 0; d < field_dim; ++d )
{
local_val = (d+1) * random_numbers[1] * random_numbers[2]
* (-(d+1) - random_numbers[3]);
TEST_ASSERT( softEquivalence( inf_norms[d], std::abs( local_val ) ) );
}
Teuchos::Array<double> one_norms;
Tools::norm1( *field_manager.field(), field_manager.comm(), one_norms );
for ( int d = 0; d < field_dim; ++d )
{
local_val = ( global_size / FT::dim( *field_manager.field() ) ) *
(d+1) * random_numbers[1]
* random_numbers[2] * (-(d+1) - random_numbers[3]);
TEST_ASSERT( softEquivalence( one_norms[d], std::abs( local_val ) ) );
}
Teuchos::Array<double> two_norms;
Tools::norm2( *field_manager.field(), field_manager.comm(), two_norms );
for ( int d = 0; d < field_dim; ++d )
{
local_val = ( global_size / FT::dim( *field_manager.field() ) ) *
std::pow(
std::abs( (d+1) * random_numbers[1] * random_numbers[2]
* (-(d+1) - random_numbers[3]) ), 2.0 );
local_val = std::pow( local_val, 1.0/2.0 );
TEST_ASSERT( softEquivalence( two_norms[d], local_val ) );
}
Teuchos::Array<double> q_norms;
int q = std::floor( (10 * std::rand()) / RAND_MAX ) + 1;
Tools::normQ( *field_manager.field(), field_manager.comm(), q, q_norms );
for ( int d = 0; d < field_dim; ++d )
{
local_val = ( global_size / FT::dim( *field_manager.field() ) ) *
std::pow(
std::abs( (d+1) * random_numbers[1] * random_numbers[2]
* (-(d+1) - random_numbers[3]) ), q );
local_val = std::pow( local_val, 1.0/q );
TEST_ASSERT( softEquivalence( q_norms[d], local_val ) );
}
// Average.
Teuchos::Array<double> averages;
Tools::average( *field_manager.field(), field_manager.comm(), averages );
for ( int d = 0; d < field_dim; ++d )
{
local_val = (d+1) * random_numbers[1] * random_numbers[2]
* (-(d+1) - random_numbers[3]);
TEST_ASSERT( softEquivalence( averages[d], local_val ) );
}
}
void Tpetra::Utils::readHBMatDouble(const std::string &filename, int &numRows, int &numCols, int &numNZ, std::string &type, Teuchos::ArrayRCP<int> &colPtrs, Teuchos::ArrayRCP<int> &rowInds, Teuchos::ArrayRCP<double> &vals) {
// NOTE: if complex, allocate enough space for 2*NNZ and interleave real and imaginary parts (real,imag)
// if pattern, don't touch parameter vals; do not allocate space space for values
try {
std::ifstream fin;
int ptrCrd, indCrd, valCrd;
Teuchos::ArrayRCP<char> Title, Key, Ptrfmt, Indfmt, Valfmt;
const int MAXSIZE = 81;
char lineBuf[MAXSIZE];
// nitty gritty
int ptrsPerLine, ptrWidth, indsPerLine, indWidth, valsPerLine, valWidth, valPrec;
char valFlag;
//
fin.open(filename.c_str(),std::ifstream::in);
{
// we don't care about RHS-related stuff, so declare those vars in an expiring scope
int Nrhs, rhsCrd;
Teuchos::ArrayRCP<char> Rhstype, Rhsfmt;
Teuchos::ArrayRCP<char> TypeArray;
Tpetra::Utils::readHBHeader(fin, Title, Key, TypeArray, numRows, numCols, numNZ, Nrhs,
Ptrfmt, Indfmt, Valfmt, Rhsfmt,
ptrCrd, indCrd, valCrd, rhsCrd, Rhstype);
if (TypeArray.size() > 0) {
type.resize(TypeArray.size()-1);
std::copy(TypeArray.begin(), TypeArray.end(), type.begin());
}
}
const std::string errStr("Tpetra::Utils::readHBHeader(): Improperly formatted H/B file.");
const bool readPatternOnly = (type[0] == 'P' || type[0] == 'p');
const bool readComplex = (type[0] == 'C' || type[0] == 'c');
/* Parse the array input formats from Line 3 of HB file */
TEUCHOS_TEST_FOR_EXCEPTION( Tpetra::Utils::parseIfmt(Ptrfmt,ptrsPerLine,ptrWidth) == true, std::runtime_error,
"Tpetra::Utils::readHBMatDouble(): error parsing. Invalid/unsupported file format.");
TEUCHOS_TEST_FOR_EXCEPTION( Tpetra::Utils::parseIfmt(Indfmt,indsPerLine,indWidth) == true, std::runtime_error,
"Tpetra::Utils::readHBMatDouble(): error parsing. Invalid/unsupported file format.");
if (readPatternOnly == false) {
TEUCHOS_TEST_FOR_EXCEPTION( Tpetra::Utils::parseRfmt(Valfmt,valsPerLine,valWidth,valPrec,valFlag) == true, std::runtime_error,
"Tpetra::Utils::readHBMatDouble(): error parsing. Invalid/unsupported file format.");
}
// special case this: the reason is that the number of colPtrs read is numCols+1, which is non-zero even if numCols == 0
// furthermore, if numCols == 0, there is nothing of interest to read
if (numCols == 0) return;
// allocate space for column pointers, row indices, and values
// if the file is empty, do not touch these ARCPs
colPtrs = Teuchos::arcp<int>(numCols+1);
if (numNZ > 0) {
rowInds = Teuchos::arcp<int>(numNZ);
if (readPatternOnly == false) {
if (readComplex) {
vals = Teuchos::arcp<double>(2*numNZ);
}
else {
vals = Teuchos::arcp<double>(numNZ);
}
}
}
/* Read column pointer array:
Specifically, read ptrCrd number of lines, and on each line, read ptrsPerLine number of integers, each of width ptrWidth
Store these in colPtrs */
{
int colPtrsRead = 0;
char NullSub = '\0';
for (int lno=0; lno < ptrCrd; ++lno) {
fin.getline(lineBuf, MAXSIZE);
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(lineBuf,"%*s") < 0, std::runtime_error, errStr);
char *linePtr = lineBuf;
for (int ptr=0; ptr < ptrsPerLine; ++ptr) {
if (colPtrsRead == numCols + 1) break;
int cptr;
// terminate the string at the end of the current ptr block, saving the character in that location
std::swap(NullSub,linePtr[ptrWidth]);
// read the ptr
std::sscanf(linePtr, "%d", &cptr);
// put the saved character back, and put the '\0' back into NullSub for use again
std::swap(NullSub,linePtr[ptrWidth]);
linePtr += ptrWidth;
colPtrs[colPtrsRead++] = cptr;
}
}
TEUCHOS_TEST_FOR_EXCEPT(colPtrsRead != numCols + 1);
}
/* Read row index array:
Specifically, read indCrd number of lines, and on each line, read indsPerLine number of integers, each of width indWidth
Store these in rowInds */
{
char NullSub = '\0';
int indicesRead = 0;
for (int lno=0; lno < indCrd; ++lno) {
fin.getline(lineBuf, MAXSIZE);
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(lineBuf,"%*s") < 0, std::runtime_error, errStr);
char *linePtr = lineBuf;
for (int indcntr=0; indcntr < indsPerLine; ++indcntr) {
if (indicesRead == numNZ) break;
int ind;
// terminate the string at the end of the current ind block, saving the character in that location
std::swap(NullSub,linePtr[indWidth]);
// read the ind
std::sscanf(linePtr, "%d", &ind);
// put the saved character back, and put the '\0' back into NullSub for use again
std::swap(NullSub,linePtr[indWidth]);
linePtr += indWidth;
rowInds[indicesRead++] = ind;
}
}
TEUCHOS_TEST_FOR_EXCEPT(indicesRead != numNZ);
}
/* Read array of values:
Specifically, read valCrd number of lines, and on each line, read valsPerLine number of real values, each of width/precision valWidth/valPrec
Store these in vals
If readComplex, then read twice as many non-zeros, and interleave real,imag into vals */
if (readPatternOnly == false) {
int totalNumVals;
if (readComplex) {
totalNumVals = 2*numNZ;
}
else {
totalNumVals = numNZ;
}
char NullSub = '\0';
int valsRead = 0;
for (int lno=0; lno < valCrd; ++lno) {
fin.getline(lineBuf, MAXSIZE);
TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(lineBuf,"%*s") < 0, std::runtime_error, errStr);
// if valFlag == 'D', then we need to convert [dD] in fp vals into [eE] that scanf can parse
if (valFlag == 'D') std::replace_if(lineBuf, lineBuf+MAXSIZE, std::bind2nd(std::equal_to<char>(), 'D'), 'E');
char *linePtr = lineBuf;
for (int valcntr=0; valcntr < valsPerLine; ++valcntr) {
if (valsRead == totalNumVals) break;
double val;
// terminate the string at the end of the current val block, saving the character in that location
std::swap(NullSub,linePtr[valWidth]);
// read the val
std::sscanf(linePtr, "%le", &val);
// put the saved character back, and put the '\0' back into NullSub for use again
std::swap(NullSub,linePtr[valWidth]);
linePtr += valWidth;
vals[valsRead++] = val;
}
}
TEUCHOS_TEST_FOR_EXCEPT(valsRead != totalNumVals);
}
fin.close();
}
catch (std::exception &e) {
TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
"Tpetra::Utils::readHBInfo() of filename \"" << filename << "\" caught exception: " << std::endl
<< e.what() << std::endl);
}
}
void
Tpetra::Utils::readHBMatrix(const std::string &filename,
const Teuchos::RCP<const Teuchos::Comm<int> > &comm,
const Teuchos::RCP<Node> &node,
Teuchos::RCP< Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> > &A,
Teuchos::RCP< const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > rowMap,
const Teuchos::RCP<ParameterList> ¶ms)
{
const int myRank = comm->getRank();
int numRows,numCols,numNZ;
Teuchos::ArrayRCP<Scalar> svals;
Teuchos::ArrayRCP<GlobalOrdinal> colinds;
Teuchos::ArrayRCP<int> rowptrs;
Teuchos::ArrayRCP<size_t> nnzPerRow;
int fail = 0;
if (myRank == 0) {
bool isSymmetric=false;
Teuchos::ArrayRCP<double> dvals;
Teuchos::ArrayRCP<int> colptrs, rowinds;
std::string type;
Tpetra::Utils::readHBMatDouble(filename,numRows,numCols,numNZ,type,colptrs,rowinds,dvals);
TEUCHOS_TEST_FOR_EXCEPT(type.size() != 3);
if (type[0] != 'R' && type[0] != 'r') {
// only real matrices right now
fail = 1;
}
if (fail == 0 && numNZ > 0) {
if (type[1] == 'S' || type[1] == 's') {
isSymmetric = true;
}
else {
isSymmetric = false;
}
}
if (fail == 0 && numNZ > 0) {
// find num non-zero per row
nnzPerRow = Teuchos::arcp<size_t>(numRows);
std::fill(nnzPerRow.begin(), nnzPerRow.end(), 0);
for (Teuchos::ArrayRCP<int>::const_iterator ri=rowinds.begin(); ri != rowinds.end(); ++ri) {
// count each row index towards its row
++nnzPerRow[*ri-1];
}
if (isSymmetric) {
// count each column toward the corresponding row as well
for (int c=0; c < numCols; ++c) {
// the diagonal was already counted; neglect it, if it exists
for (int i=colptrs[c]-1; i != colptrs[c+1]-1; ++i) {
if (rowinds[i] != c+1) {
++nnzPerRow[c];
++numNZ;
}
}
}
}
// allocate/set new matrix data
svals = Teuchos::arcp<Scalar>(numNZ);
colinds = Teuchos::arcp<GlobalOrdinal>(numNZ);
rowptrs = Teuchos::arcp<int>(numRows+1);
rowptrs[0] = 0;
#ifdef HAVE_TPETRA_DEBUG
Teuchos::ArrayRCP<size_t> nnzPerRow_debug(nnzPerRow.size());
std::copy(nnzPerRow.begin(), nnzPerRow.end(), nnzPerRow_debug.begin());
#endif
for (int j=1; j <= numRows; ++j) {
rowptrs[j] = rowptrs[j-1] + nnzPerRow[j-1];
nnzPerRow[j-1] = 0;
}
// translate from column-oriented to row-oriented
for (int col=0; col<numCols; ++col) {
for (int i=colptrs[col]-1; i != colptrs[col+1]-1; ++i) {
const int row = rowinds[i]-1;
// add entry to (row,col), with value dvals[i]
const size_t entry = rowptrs[row] + nnzPerRow[row];
svals[entry] = Teuchos::as<Scalar>(dvals[i]);
colinds[entry] = Teuchos::as<GlobalOrdinal>(col);
++nnzPerRow[row];
if (isSymmetric && row != col) {
// add entry to (col,row), with value dvals[i]
const size_t symentry = rowptrs[col] + nnzPerRow[col];
svals[symentry] = Teuchos::as<Scalar>(dvals[i]);
colinds[symentry] = Teuchos::as<GlobalOrdinal>(row);
++nnzPerRow[col];
}
}
}
#ifdef HAVE_TPETRA_DEBUG
{
bool isequal = true;
typename Teuchos::ArrayRCP<size_t>::const_iterator it1, it2;
for (it1 = nnzPerRow.begin(), it2 = nnzPerRow_debug.begin(); it1 != nnzPerRow.end(); ++it1, ++it2) {
if (*it1 != *it2) {
isequal = false;
break;
}
}
TEUCHOS_TEST_FOR_EXCEPTION(!isequal || nnzPerRow.size() != nnzPerRow_debug.size(), std::logic_error,
"Tpetra::Utils::readHBMatrix(): Logic error.");
}
#endif
}
// std::cout << "Matrix " << filename << " of type " << type << ": " << numRows << " by " << numCols << ", " << numNZ << " nonzeros" << std::endl;
}
// check for read errors
broadcast(*comm,0,&fail);
TEUCHOS_TEST_FOR_EXCEPTION(fail == 1, std::runtime_error, "Tpetra::Utils::readHBMatrix() can only read Real matrices.");
// distribute global matrix info
broadcast(*comm,0,&numRows);
broadcast(*comm,0,&numCols);
// create map with uniform partitioning
if (rowMap == Teuchos::null) {
rowMap = Teuchos::rcp(new Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node>((global_size_t)numRows,(GlobalOrdinal)0,comm,GloballyDistributed,node));
}
else {
TEUCHOS_TEST_FOR_EXCEPTION( rowMap->getGlobalNumElements() != (global_size_t)numRows, std::runtime_error,
"Tpetra::Utils::readHBMatrix(): specified map has incorrect number of elements.");
TEUCHOS_TEST_FOR_EXCEPTION( rowMap->isDistributed() == false && comm->getSize() > 1, std::runtime_error,
"Tpetra::Utils::readHBMatrix(): specified map is not distributed.");
}
Teuchos::ArrayRCP<size_t> myNNZ;
if (rowMap->getNodeNumElements()) {
myNNZ = Teuchos::arcp<size_t>(rowMap->getNodeNumElements());
}
if (myRank == 0) {
LocalOrdinal numRowsAlreadyDistributed = rowMap->getNodeNumElements();
std::copy(nnzPerRow.begin(), nnzPerRow.begin()+numRowsAlreadyDistributed,myNNZ);
for (int p=1; p < Teuchos::size(*comm); ++p) {
size_t numRowsForP;
Teuchos::receive(*comm,p,&numRowsForP);
if (numRowsForP) {
Teuchos::send<int,size_t>(*comm,numRowsForP,nnzPerRow(numRowsAlreadyDistributed,numRowsForP).getRawPtr(),p);
numRowsAlreadyDistributed += numRowsForP;
}
}
}
else {
const size_t numMyRows = rowMap->getNodeNumElements();
Teuchos::send(*comm,numMyRows,0);
if (numMyRows) {
Teuchos::receive<int,size_t>(*comm,0,numMyRows,myNNZ(0,numMyRows).getRawPtr());
}
}
nnzPerRow = Teuchos::null;
// create column map
Teuchos::RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > domMap;
if (numRows == numCols) {
domMap = rowMap;
}
else {
domMap = createUniformContigMapWithNode<LocalOrdinal,GlobalOrdinal,Node>(numCols,comm,node);
}
A = rcp(new Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node>(rowMap,myNNZ,Tpetra::StaticProfile));
// free this locally, A will keep it allocated as long as it is needed by A (up until allocation of nonzeros)
myNNZ = Teuchos::null;
if (myRank == 0 && numNZ > 0) {
for (int r=0; r < numRows; ++r) {
const LocalOrdinal nnz = rowptrs[r+1] - rowptrs[r];
if (nnz > 0) {
Teuchos::ArrayView<const GlobalOrdinal> inds = colinds(rowptrs[r],nnz);
Teuchos::ArrayView<const Scalar> vals = svals( rowptrs[r],nnz);
A->insertGlobalValues(r, inds, vals);
}
}
}
// don't need these anymore
colinds = Teuchos::null;
svals = Teuchos::null;
rowptrs = Teuchos::null;
A->fillComplete(domMap,rowMap,params);
}
TEUCHOS_UNIT_TEST(Zoltan, Build)
{
typedef Teuchos::ScalarTraits<Scalar> ST;
out << "version: " << MueLu::Version() << std::endl;
out << std::endl;
out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl;
out << "that has a random number of nonzeros per row. Good results have been precomputed" << std::endl;
out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl;
out << "once the Zoltan repartitioning has been applied." << std::endl;
out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl;
RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();
if (comm->getSize() > 5) {
out << std::endl;
out << "This test must be run on 1 to 5 processes." << std::endl;
TEST_EQUALITY(true, true);
return;
}
Level level;
RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
int nx=7;
int ny=nx;
GO numGlobalElements = nx*ny;
size_t maxEntriesPerRow=30;
// Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row.
RCP<const Map> map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm);
const size_t numMyElements = map->getNodeNumElements();
Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList();
RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more
// memory on the fly. While not super efficient, this
// ensures that no zeros are being stored. Thus, from
// Zoltan's perspective the matrix is imbalanced.
// Create a vector with random integer entries in [1,maxEntriesPerRow].
ST::seedrandom(666*comm->getRank());
RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false);
Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0);
for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) {
*i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1));
}
RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
fos->setOutputToRootOnly(-1);
Teuchos::Array<Scalar> vals(maxEntriesPerRow);
Teuchos::Array<GO> cols(maxEntriesPerRow);
for (size_t i = 0; i < numMyElements; ++i) {
Teuchos::ArrayView<SC> av(&vals[0],eprData[i]);
Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]);
//stick in ones for values
for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one();
//figure out valid column indices
GO start = std::max(myGlobalElements[i]-eprData[i]+1,0);
for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j;
A->insertGlobalValues(myGlobalElements[i], iv, av);
}
A->fillComplete();
level.Set("A",A);
//build coordinates
RCP<const Map> rowMap = A->getRowMap();
Teuchos::ParameterList list;
list.set("nx",nx);
list.set("ny",ny);
RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",rowMap,list);
level.Set("Coordinates",XYZ);
LO numPartitions = comm->getSize();
level.Set("number of partitions",numPartitions);
RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface());
//zoltan->SetNumberOfPartitions(numPartitions);
//zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints
level.Request("Partition",zoltan.get());
zoltan->Build(level);
RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get());
/* //TODO temporary code to have the trivial decomposition (no change)
ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
*i = comm->getRank();
decompEntries=Teuchos::null;
*/ //TODO end of temporary code
//Create vector whose local length is the global number of partitions.
//This vector will record the local number of nonzeros associated with each partition.
Teuchos::Array<GO> parts(numPartitions);
for (int i=0; i<numPartitions; ++i) parts[i] = i;
Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions);
RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView,
map->getIndexBase(),comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap);
//For the local rows in each partition, tally up the number of nonzeros. This is what
//Zoltan should be load-balancing.
Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0);
Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0);
for (size_t i=0; i<decomposition->getLocalLength();++i) {
Teuchos::ArrayView<const LO> c;
Teuchos::ArrayView<const SC> v;
A->getLocalRowView(i,c,v);
lpvData[decompData[i]] += v.size();
}
lpvData = Teuchos::null;
decompData = Teuchos::null;
//localPartsVec->describe(*fos,Teuchos::VERB_EXTREME);
//Send the local nnz tallies to pid 0, which can report the global sums.
size_t mysize=1;
if (comm->getRank() == 0) mysize = numPartitions;
RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(),
mysize,
map->getIndexBase(),
comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap);
RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap);
globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD);
ArrayRCP<GO> expectedResults(numPartitions);
switch (comm->getSize()) {
case 1:
expectedResults[0] = 807;
break;
case 2:
expectedResults[0] = 364;
expectedResults[1] = 363;
break;
case 3:
expectedResults[0] = 277;
expectedResults[1] = 261;
expectedResults[2] = 269;
break;
case 4:
expectedResults[0] = 195;
expectedResults[1] = 186;
expectedResults[2] = 177;
expectedResults[3] = 168;
break;
case 5:
expectedResults[0] = 161;
expectedResults[1] = 145;
expectedResults[2] = 148;
expectedResults[3] = 159;
expectedResults[4] = 157;
break;
default:
break;
};
//FIXME cool ... this next line causes a hang if locally the globalyTallyVec has no data.
//FIXME I get around this by making mysize (above) 1 instead of 0. Is this a bug or feature
//FIXME in getData?
ArrayRCP<const LO> gtvData = globalTallyVec->getData(0);
#ifdef __linux__
out << "Checking results..." << std::endl;
for (int i=0; i<numPartitions; ++i) {
if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]);
}
#endif
#ifdef MUELU_VISUALIZE_REPARTITIONING
//
//Now write everything to a comma-separate list that ParaView can grok
//
Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0);
Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1);
Teuchos::ArrayRCP<const GO> D = decomposition->getData(0);
RCP<std::ofstream> outFile;
std::string fileName = "zoltanResults.csv";
//write header information
if (comm->getRank() == 0) {
outFile = rcp(new std::ofstream(fileName.c_str()));
*outFile << "x coord, y coord, z coord, scalar" << std::endl;
}
comm->barrier();
//append coordinates
for (int j=0; j<comm->getSize(); ++j) {
int mypid = comm->getRank();
if (mypid == j) {
outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app));
for (int i=0; i < D.size(); ++i) {
*outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", " << D[i] << std::endl;
}
}
} //for (int i=0; i<comm->getSize(); ++i)
out << std::endl;
out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl;
out << " 1) Load the data file " << fileName << "." << std::endl;
out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl;
out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl;
out << " 4) Split screen horizontally (Icon, top right)." << std::endl;
out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl;
out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl;
out << std::endl;
out << " To display row weights next to each point:" << std::endl;
out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl;
out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl;
out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl;
#endif
} //Build
TEUCHOS_UNIT_TEST(Zoltan, Build3PDEs)
{
typedef Teuchos::ScalarTraits<Scalar> ST;
out << "version: " << MueLu::Version() << std::endl;
out << std::endl;
out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl;
out << "that has a random number of nonzeros per row and 3 DOFs per mesh point. Good results have been precomputed" << std::endl;
out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl;
out << "once the Zoltan repartitioning has been applied." << std::endl;
out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl;
RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();
if (comm->getSize() > 5) {
out << std::endl;
out << "This test must be run on 1 to 5 processes." << std::endl;
TEST_EQUALITY(true, true);
return;
}
Level level;
RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
int nx=9;
int ny=nx;
int dofsPerNode = 3;
GO numGlobalElements = nx*ny*dofsPerNode;
size_t maxEntriesPerRow=30;
RCP<const Map> map;
int numMyNodes = numGlobalElements / dofsPerNode;
if (comm->getSize() > 1) {
// In parallel, make sure that the dof's associated with a node all
// reside on the same processor.
int numNodes = numGlobalElements / dofsPerNode;
TEUCHOS_TEST_FOR_EXCEPTION( (numGlobalElements - numNodes * dofsPerNode) != 0, MueLu::Exceptions::RuntimeError,
"Number of matrix rows is not divisible by #dofs" );
int nproc = comm->getSize();
if (comm->getRank() < nproc-1) numMyNodes = numNodes / nproc;
else numMyNodes = numNodes - (numNodes/nproc) * (nproc-1);
map = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, numMyNodes*dofsPerNode, comm);
} else {
map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm);
}
const size_t numMyElements = map->getNodeNumElements();
Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList();
RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more
// memory on the fly. While not super efficient, this
// ensures that no zeros are being stored. Thus, from
// Zoltan's perspective the matrix is imbalanced.
// Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row.
// Create a vector with random integer entries in [1,maxEntriesPerRow].
ST::seedrandom(666*comm->getRank());
RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false);
Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0);
for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) {
*i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1));
}
RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
fos->setOutputToRootOnly(-1);
Teuchos::Array<Scalar> vals(maxEntriesPerRow);
Teuchos::Array<GO> cols(maxEntriesPerRow);
for (size_t i = 0; i < numMyElements; ++i) {
Teuchos::ArrayView<SC> av(&vals[0],eprData[i]);
Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]);
//stick in ones for values
for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one();
//figure out valid column indices
GO start = std::max(myGlobalElements[i]-eprData[i]+1,0);
for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j;
A->insertGlobalValues(myGlobalElements[i], iv, av);
}
A->fillComplete();
// Now treat the matrix as if it has 3 DOFs per node.
A->SetFixedBlockSize(dofsPerNode);
level.Set("A",A);
//build coordinates
Teuchos::ParameterList list;
list.set("nx",nx);
list.set("ny",ny);
RCP<const Map> coalescedMap = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements/dofsPerNode, numMyNodes, comm);
RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",coalescedMap,list);
// XYZ are the "coalesce" coordinates as it has been generated for 1 DOF/node and we are using them for 3 DOFS/node
// level.Set("Coordinates",XYZ); "Coordinates" == uncoalesce. "X,Y,ZCoordinates" == coalesce
{
RCP<MultiVector> coordinates = XYZ;
// making a copy because I don't want to keep 'open' the Xpetra_MultiVector
if (coordinates->getNumVectors() >= 1) {
Teuchos::ArrayRCP<const SC> coord = coordinates->getData(0);
Teuchos::ArrayRCP<SC> coordCpy(coord.size());
for(int i=0; i<coord.size(); i++) {
coordCpy[i] = coord[i];
}
level.Set("XCoordinates", coordCpy);
//std::cout << coordCpy << std::endl;
}
if (coordinates->getNumVectors() >= 2) {
Teuchos::ArrayRCP<const SC> coord = coordinates->getData(1);
Teuchos::ArrayRCP<SC> coordCpy(coord.size());
for(int i=0; i<coord.size(); i++) {
coordCpy[i] = coord[i];
}
level.Set("YCoordinates", coordCpy);
}
/*if (coordinates->getNumVectors() >= 3) {
Teuchos::ArrayRCP<const SC> coord = coordinates->getData(2);
Teuchos::ArrayRCP<SC> coordCpy(coord.size());
for(int i=0; i<coord.size(); i++) {
coordCpy[i] = coord[i];
}
level.Set("ZCoordinates", coordCpy);
}*/
}
//coalescedMap->describe(*fos,Teuchos::VERB_EXTREME);
//sleep(1); comm->barrier();
//XYZ->describe(*fos,Teuchos::VERB_EXTREME);
LO numPartitions = comm->getSize();
level.Set("number of partitions",numPartitions);
RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface());
//zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints
level.Request("Partition",zoltan.get());
zoltan->Build(level);
RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get());
/* //temporary code to have the trivial decomposition (no change)
ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
*i = comm->getRank();
decompEntries=Teuchos::null;
*/
//Create vector whose local length is the global number of partitions.
//This vector will record the local number of nonzeros associated with each partition.
Teuchos::Array<GO> parts(numPartitions);
for (int i=0; i<numPartitions; ++i) parts[i] = i;
Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions);
RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView,
map->getIndexBase(),comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap);
RCP<Xpetra::Vector<LO,LO,GO,NO> > nnzPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(A->getRowMap());
Teuchos::ArrayRCP<GO> nnzData = nnzPerRow->getDataNonConst(0);
//For the local rows in each partition, tally up the number of nonzeros. This is what
//Zoltan should be load-balancing.
Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0);
Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0);
for (size_t i=0; i<decomposition->getLocalLength();++i) {
Teuchos::ArrayView<const LO> c;
Teuchos::ArrayView<const SC> v;
A->getLocalRowView(i,c,v);
lpvData[decompData[i]] += v.size();
nnzData[i] = v.size();
}
lpvData = Teuchos::null;
decompData = Teuchos::null;
nnzData = Teuchos::null;
/*
if (comm->getRank() == 0)
std::cout << "nnz per row" << std::endl;
nnzPerRow->describe(*fos,Teuchos::VERB_EXTREME);
if (comm->getRank() == 0)
std::cout << "Row-to-partition assignment (from Zoltan)" << std::endl;
decomposition->describe(*fos,Teuchos::VERB_EXTREME);
if (comm->getRank() == 0)
std::cout << "#nonzeros per partition" << std::endl;
localPartsVec->describe(*fos,Teuchos::VERB_EXTREME);
*/
//Send the local nnz tallies to pid 0, which can report the global sums.
size_t mysize=1;
if (comm->getRank() == 0) mysize = numPartitions;
RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
Teuchos::OrdinalTraits<global_size_t>::invalid(),
mysize,
map->getIndexBase(),
comm);
RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap);
RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap);
globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD);
ArrayRCP<GO> expectedResults(numPartitions);
switch (comm->getSize()) {
case 1:
expectedResults[0] = 3951;
break;
case 2:
expectedResults[0] = 1955;
expectedResults[1] = 1910;
break;
case 3:
expectedResults[0] = 1326;
expectedResults[1] = 1340;
expectedResults[2] = 1321;
break;
case 4:
expectedResults[0] = 950;
expectedResults[1] = 922;
expectedResults[2] = 908;
expectedResults[3] = 936;
break;
case 5:
expectedResults[0] = 774;
expectedResults[1] = 735;
expectedResults[2] = 726;
expectedResults[3] = 771;
expectedResults[4] = 759;
break;
default:
break;
};
ArrayRCP<const LO> gtvData = globalTallyVec->getData(0);
#ifdef __linux__
out << "Checking results..." << std::endl;
for (int i=0; i<numPartitions; ++i) {
if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]);
}
#endif
#ifdef MUELU_VISUALIZE_REPARTITIONING
//
//Now write everything to a comma-separate list that ParaView can grok
//
Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0);
Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1);
Teuchos::ArrayRCP<const GO> D = decomposition->getData(0);
RCP<std::ofstream> outFile;
std::string fileName = "zoltanResults.csv";
//write header information
if (comm->getRank() == 0) {
outFile = rcp(new std::ofstream(fileName.c_str()));
*outFile << "x coord, y coord, z coord, partition, row weight" << std::endl;
}
comm->barrier();
//append coordinates
nnzData = nnzPerRow->getDataNonConst(0);
for (int j=0; j<comm->getSize(); ++j) {
int mypid = comm->getRank();
if (mypid == j) {
outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app));
int blockSize = A->GetFixedBlockSize();
//Coordinates are for coalesced system, D is for uncoalesced
for (int i=0; i < D.size()/blockSize; ++i) {
int nnz=0;
for (int k=0; k<blockSize; ++k) nnz += nnzData[i*blockSize+k];
*outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", "
<< D[i*blockSize] << ", " << nnz << std::endl;
}
}
} //for (int i=0; i<comm->getSize(); ++i)
out << std::endl;
out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl;
out << " 1) Load the data file " << fileName << "." << std::endl;
out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl;
out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl;
out << " 4) Split screen horizontally (Icon, top right)." << std::endl;
out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl;
out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl;
out << std::endl;
out << " To display row weights next to each point:" << std::endl;
out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl;
out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl;
out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl;
#endif
} //Build3PDEs