bool ML_Epetra::ValidateMLPParameters(const Teuchos::ParameterList &inList, int depth){
using Teuchos::ParameterList;
using Teuchos::Exceptions::InvalidParameterName;
using Teuchos::Exceptions::InvalidParameterType;
using Teuchos::Exceptions::InvalidParameterValue;
using std::cout;
using std::endl;
using std::string;
ParameterList List,*validList;
bool rv=true;
/* Build a copy of the list to be validated. */
for (ParameterList::ConstIterator param = inList.begin(); param != inList.end(); ++param) {
const std::string pname=inList.name(param);
if (pname.find("user-defined function",0) == std::string::npos) {
List.setEntry(pname,inList.entry(param));
}
}
List.setName(inList.name());
/* Get Defaults + Validate */
try {
validList = GetValidMLPParameters();
}
catch(...) {
std::cout << "Error in GetValidMLPParameters. Please report this bug to the ML "
"developers." << std::endl;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
exit(EXIT_FAILURE);
}
try {
List.validateParameters (*validList, depth, Teuchos::VALIDATE_USED_ENABLED,
Teuchos::VALIDATE_DEFAULTS_DISABLED);
}
#ifdef HAVE_IFPACK_DYNAMIC_FACTORY
catch(InvalidParameterName &excpt) {/*rv=false; std::cout<<excpt.what()<<std::endl;*/}
#else
catch(InvalidParameterName &excpt) {rv=false; std::cout<<excpt.what()<<std::endl;}
#endif
catch(InvalidParameterType &excpt) {rv=false; std::cout<<excpt.what()<<std::endl;}
catch(InvalidParameterValue &excpt) {rv=false; std::cout<<excpt.what()<<std::endl;}
catch(...) {rv=false;}
delete validList;
return rv;
}
bool
ComparisonHelper::metricComparisonTest(const RCP<const Comm<int> > &comm,
const Zoltan2::MetricValues<zscalar_t> & metric,
const Zoltan2::MetricValues<zscalar_t> & ref_metric,
const Teuchos::ParameterList & metricPlist,
ostringstream &msg)
{
// run a comparison of min and max agains a given metric
// return an error message on failure
bool pass = true;
string test_name = metricPlist.name() + " test";
double local_ref_value = ref_metric.getMaxImbalance()/ref_metric.getAvgImbalance();
double local_value = metric.getMaxImbalance()/metric.getAvgImbalance();
// reduce problem metric
double value;
Teuchos::Ptr<double> global(&value);
comm->barrier();
reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,local_value,global);
// reduce reference metric
double ref_value;
Teuchos::Ptr<double> globalRef(&ref_value);
comm->barrier();
reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,local_ref_value,globalRef);
// want to reduce value to max value for all procs
if (metricPlist.isParameter("lower"))
{
double min = metricPlist.get<double>("lower")*ref_value;
if(value < min)
{
msg << test_name << " FAILED: Minimum imbalance per part, "
<< value << ", less than specified allowable minimum, " << min << ".\n";
pass = false;
}else{
msg << test_name << " PASSED: Minimum imbalance per part, "
<< value << ", greater than specified allowable minimum, " << min << ".\n";
}
}
if(metricPlist.isParameter("upper" ) && pass != false) {
double max = metricPlist.get<double>("upper") * ref_value;
if (value > max)
{
msg << test_name << " FAILED: Maximum imbalance per part, "
<< value << ", greater than specified allowable maximum, " << max << ".\n";
pass = false;
}else{
msg << test_name << " PASSED: Maximum imbalance per part, "
<< value << ", less than specified allowable maximum, " << max << ".\n";
}
}
return pass;
}
// BDD, to do: print metrics even for pass
// reduce max metric to process 0
// print only on process 0 --- duh.
bool ComparisonHelper::timerComparisonTest(const RCP<const Comm<int> > &comm,
const double time,
const double ref_time,
const Teuchos::ParameterList & metricPlist,
ostringstream &msg)
{
// Reduce time from test
double global_time;
Teuchos::Ptr<double> global(&global_time);
comm->barrier();
reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,time,global);
// Reduce time from reference
double global_ref_time;
Teuchos::Ptr<double> globalRef(&global_ref_time);
comm->barrier();
reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,ref_time,globalRef);
// run a comparison of min and max agains a given metric
// return an error message on failure
bool pass = true;
string test_name = metricPlist.name() + " test";
if (metricPlist.isParameter("lower"))
{
double min = metricPlist.get<double>("lower")*global_ref_time;
if(global_time < min)
{
msg << test_name << " FAILED: Minimum time, "
<< time <<
"[s], less than specified allowable minimum time, " << min <<"[s]"<< ".\n";
pass = false;
}else{
msg << test_name << " PASSED: Minimum time, "
<< time <<
"[s], greater than specified allowable minimum time, " << min <<"[s]"<< ".\n";
}
}
if(metricPlist.isParameter("upper" ) && pass != false) {
double max = metricPlist.get<double>("upper") * global_ref_time;
if (global_time > max)
{
msg << test_name << " FAILED: Maximum time, "
<< global_time <<
"[s], greater than specified allowable maximum time, " << max <<"[s]"<< ".\n";
pass = false;
}else{
msg << test_name << " PASSED: Maximum time, "
<< global_time <<
"[s], less than specified allowable maximum time, " << max <<"[s]"<< ".\n";
}
}
return pass;
}
bool ML_Epetra::ValidateRefMaxwellParameters(const Teuchos::ParameterList &inList){
using Teuchos::ParameterList;
using Teuchos::Exceptions::InvalidParameterName;
using Teuchos::Exceptions::InvalidParameterType;
using Teuchos::Exceptions::InvalidParameterValue;
using std::cout;
using std::endl;
using std::string;
ParameterList List,*validList;
bool rv=true;
/* Build a list with level-specific stuff stripped */
//TODO this should be fixed
for (ParameterList::ConstIterator param = inList.begin(); param != inList.end(); ++param) {
const string pname=inList.name(param);
if (pname.find("(level",0) == string::npos) {
List.setEntry(pname,inList.entry(param));
}
}
List.setName(inList.name());
/* Get Defaults + Validate */
try{
validList = GetValidRefMaxwellParameters();
}
catch(...) {
cout << "Error in GetValidMLPParameters. Please report this bug to the ML "
"developers." << endl;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
exit(EXIT_FAILURE);
}
try {
List.validateParameters(*validList, 0, Teuchos::VALIDATE_USED_DISABLED,
Teuchos::VALIDATE_DEFAULTS_DISABLED);
}
catch(InvalidParameterName &excpt) {rv=false; cout<<excpt.what();}
catch(InvalidParameterType &excpt) {rv=false; cout<<excpt.what();}
catch(InvalidParameterValue &excpt) {rv=false; cout<<excpt.what();}
catch(...) {rv=false;}
delete validList;
return rv;
}
bool TouchContFileParameters( Teuchos::ParameterList & fileParams )
{
// Either int or double type
int dummyInt;
double dummyDouble;
// Looping the list
Teuchos::map<string, Teuchos::ParameterEntry>::const_iterator i;
for (i = fileParams.begin(); i !=fileParams.end(); ++i) {
if (fileParams.isType<int>(fileParams.name(i)))
dummyInt = fileParams.get<int>(fileParams.name(i));
if (fileParams.isType<double>(fileParams.name(i)))
dummyDouble = fileParams.get<double>(fileParams.name(i));
}
return true;
}
void buildResponseMap(const Teuchos::ParameterList & p,
std::map<std::string,std::pair<ResponseId,std::pair<std::list<std::string>,std::list<std::string> > > > & responses)
{
static Teuchos::RCP<const Teuchos::ParameterList> validList;
// build valid parameter list
if(validList==Teuchos::null) {
Teuchos::RCP<Teuchos::ParameterList> tmpList = Teuchos::rcp(new Teuchos::ParameterList);
tmpList->set<std::string>("Type","");
tmpList->set<std::string>("Field Name","");
tmpList->set<std::string>("Element Blocks","empty","Element blocks for this response",Teuchos::rcp(new CommaSeperatedEntryValidator));
tmpList->set<std::string>("Evaluation Types","empty","Evaluation types for this response",Teuchos::rcp(new CommaSeperatedEntryValidator));
validList = tmpList;
}
CommaSeperatedEntryValidator validator;
const std::string & sublistName = p.name();
std::vector<std::string> tokens;
responses.clear();
// loop over entries of parameter list, must satisfy response formatting
for(Teuchos::ParameterList::ConstIterator itr=p.begin(); itr!=p.end();++itr) {
const std::string & paramName = itr->first;
const Teuchos::ParameterEntry & pe = itr->second;
// make sure this is a parameter list
TEUCHOS_TEST_FOR_EXCEPTION(!pe.isList(),Teuchos::Exceptions::InvalidParameterValue,
"In list \""+sublistName+"\", the parameter \""+paramName+"\" is expected "
"to be a sublist. Response map cannot be built!");
// extract parameter list and validate
const Teuchos::ParameterList & respList = Teuchos::getValue<Teuchos::ParameterList>(pe);
respList.validateParameters(*validList);
const std::string & respLabel = paramName;
ResponseId & rid = responses[respLabel].first;
std::list<std::string> & eBlocks = responses[respLabel].second.first; // element blocks
std::list<std::string> & eTypes = responses[respLabel].second.second; // evaluation types
rid.type = respList.get<std::string>("Type");
rid.name = respList.get<std::string>("Field Name");
CommaSeperatedEntryValidator::split(respList.get<std::string>("Element Blocks"),",",tokens);
eBlocks.assign(tokens.begin(),tokens.end()); // this should automatically wipe out old values
CommaSeperatedEntryValidator::split(respList.get<std::string>("Evaluation Types"),",",tokens);
eTypes.assign(tokens.begin(),tokens.end()); // this should automatically wipe out old values
}
}
Operator<Node>::Operator(Teuchos::RCP<const ::Tpetra::CrsGraph<int,int,Node> > input_graph,
const Teuchos::ParameterList& paramlist, int base)
: input_graph_(input_graph),
operation_already_computed_(false),
base_(base)
{
input_map_ = input_graph->getRowMap(); //think this is safe
if(paramlist.name() != "EmptyParameterList")
{
setParameters(paramlist);
}
}
bool
ComparisonHelper::metricComparisonTest(const RCP<const Comm<int> > &comm,
const Zoltan2::MetricValues<zscalar_t> & metric,
const Zoltan2::MetricValues<zscalar_t> & ref_metric,
const Teuchos::ParameterList & metricPlist,
ostringstream &msg)
{
// run a comparison of min and max agains a given metric
// return an error message on failure
bool pass = true;
string test_name = metricPlist.name() + " test";
double ref_value = ref_metric.getMaxImbalance();
double value = metric.getMaxImbalance();
// want to reduce value to max value for all procs
if (metricPlist.isParameter("lower"))
{
double min = metricPlist.get<double>("lower")*ref_value;
if(value < min)
{
msg << test_name << " FAILED: imbalance per part, "
<< value << ", less than specified allowable minimum, " << min << ".\n";
pass = false;
}else{
msg << test_name << " PASSED: imbalance per part, "
<< value << ", greater than specified allowable minimum, " << min << ".\n";
}
}
if(metricPlist.isParameter("upper" ) && pass != false) {
double max = metricPlist.get<double>("upper") * ref_value;
if (value > max)
{
msg << test_name << " FAILED: imbalance per part, "
<< value << ", greater than specified allowable maximum, " << max << ".\n";
pass = false;
}else{
msg << test_name << " PASSED: imbalance per part, "
<< value << ", less than specified allowable maximum, " << max << ".\n";
}
}
return pass;
}
bool ComparisonHelper::metricComparisonTest(const RCP<const Comm<int> > &comm,
const MetricAnalyzerInfo & metric,
const MetricAnalyzerInfo & ref_metric,
const Teuchos::ParameterList & metricPlist,
ostringstream &msg)
{
// run a comparison of min and max against a given metric
// return an error message on failure
bool pass = true;
string test_name = metricPlist.name() + " test";
double ref_value = ref_metric.theValue;
double value = metric.theValue;
if (ref_value == 0) {
throw std::logic_error( "The parameter list had a 0 value for the reference value so a percentage cannot be calculated." );
}
double percentRatio = value / ref_value;
// want to reduce value to max value for all procs
if (ref_metric.bFoundLowerBound) {
double min = ref_metric.lowerValue;
if (percentRatio < min) {
msg << test_name << " FAILED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which less than specified allowable minimum percent, " << min << ".\n";
pass = false;
}
else {
msg << test_name << " PASSED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which is greater than specified allowable minimum percent, " << min << ".\n";
}
}
if (ref_metric.bFoundUpperBound) {
double max = ref_metric.upperValue;
if (percentRatio > max) {
msg << test_name << " FAILED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which is greater than specified allowable maximum percent, " << max << ".\n";
pass = false;
}
else {
msg << test_name << " PASSED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which is less than specified allowable maximum percent, " << max << ".\n";
}
}
return pass;
}
bool WriteHeaderToContFile( const string & fileName,
const Teuchos::ParameterList & fileParams )
{
// The file to open
ofstream oFile(fileName.c_str());
// Writing the header
oFile << "#" << setw(6) << "ID";
// Looping on the parameters
Teuchos::map<string, Teuchos::ParameterEntry>::const_iterator i;
for (i = fileParams.begin(); i !=fileParams.end(); ++i)
oFile << setw(15) << fileParams.name(i);
oFile << std::endl;
// Closing
oFile.close();
return true;
}
void Operator<Node>::paramsToUpper(Teuchos::ParameterList &plist, int &changed, bool rmUnderscore)
{
changed = 0;
// get a list of all parameter names in the list
std::vector<std::string> paramNames ;
Teuchos::ParameterList::ConstIterator pIter;
pIter = plist.begin();
while (1){
//////////////////////////////////////////////////////////////////////
// Compiler considered this while statement an error
// for ( pIter = plist.begin() ; pIter != plist.end() ; pIter++ ){
// }
//////////////////////////////////////////////////////////////////////
if (pIter == plist.end()) break;
const std::string & nm = plist.name(pIter);
paramNames.push_back(nm);
pIter++;
}
// Change parameter names and values to upper case
for (unsigned int i=0; i < paramNames.size(); i++){
std::string origName(paramNames[i]);
int paramNameChanged = 0;
stringToUpper(paramNames[i], paramNameChanged, rmUnderscore);
if (plist.isSublist(origName)){
Teuchos::ParameterList &sublist = plist.sublist(origName);
int sublistChanged=0;
paramsToUpper(sublist, sublistChanged, false);
if (paramNameChanged){
// this didn't work, so I need to remove the old sublist
// and create a new one
//
//sublist.setName(paramNames[i]);
Teuchos::ParameterList newlist(sublist);
plist.remove(origName);
plist.set(paramNames[i], newlist);
}
}
else if (plist.isParameter(origName)){
std::string paramVal(plist.get<std::string>(origName));
int paramValChanged=0;
stringToUpper(paramVal, paramValChanged);
if (paramNameChanged || paramValChanged){
if (paramNameChanged){
plist.remove(origName);
}
plist.set(paramNames[i], paramVal);
changed++;
}
}
} // next parameter or sublist
}
//! @name Constructor/Destructor
//@{
AMGXOperator(const Teuchos::RCP<Tpetra::CrsMatrix<SC,LO,GO,NO> > &inA, Teuchos::ParameterList ¶mListIn) {
RCP<const Teuchos::Comm<int> > comm = inA->getRowMap()->getComm();
int numProcs = comm->getSize();
int myRank = comm->getRank();
RCP<Teuchos::Time> amgxTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: initialize");
amgxTimer->start();
// Initialize
AMGX_SAFE_CALL(AMGX_initialize());
AMGX_SAFE_CALL(AMGX_initialize_plugins());
/*system*/
//AMGX_SAFE_CALL(AMGX_register_print_callback(&print_callback));
AMGX_SAFE_CALL(AMGX_install_signal_handler());
Teuchos::ParameterList configs = paramListIn.sublist("amgx:params", true);
if (configs.isParameter("json file")) {
AMGX_SAFE_CALL(AMGX_config_create_from_file(&Config_, (const char *) &configs.get<std::string>("json file")[0]));
} else {
std::ostringstream oss;
oss << "";
ParameterList::ConstIterator itr;
for (itr = configs.begin(); itr != configs.end(); ++itr) {
const std::string& name = configs.name(itr);
const ParameterEntry& entry = configs.entry(itr);
oss << name << "=" << filterValueToString(entry) << ", ";
}
oss << "\0";
std::string configString = oss.str();
if (configString == "") {
//print msg that using defaults
//GetOStream(Warnings0) << "Warning: No configuration parameters specified, using default AMGX configuration parameters. \n";
}
AMGX_SAFE_CALL(AMGX_config_create(&Config_, configString.c_str()));
}
// TODO: we probably need to add "exception_handling=1" to the parameter list
// to switch on internal error handling (with no need for AMGX_SAFE_CALL)
#define NEW_COMM
#ifdef NEW_COMM
// NOTE: MPI communicator used in AMGX_resources_create must exist in the scope of AMGX_matrix_comm_from_maps_one_ring
// FIXME: fix for serial comm
RCP<const Teuchos::MpiComm<int> > tmpic = Teuchos::rcp_dynamic_cast<const Teuchos::MpiComm<int> >(comm->duplicate());
TEUCHOS_TEST_FOR_EXCEPTION(tmpic.is_null(), Exceptions::RuntimeError, "Communicator is not MpiComm");
RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > rawMpiComm = tmpic->getRawMpiComm();
MPI_Comm mpiComm = *rawMpiComm;
#endif
// Construct AMGX resources
if (numProcs == 1) {
AMGX_resources_create_simple(&Resources_, Config_);
} else {
int numGPUDevices;
cudaGetDeviceCount(&numGPUDevices);
int device[] = {(comm->getRank() % numGPUDevices)};
AMGX_config_add_parameters(&Config_, "communicator=MPI");
#ifdef NEW_COMM
AMGX_resources_create(&Resources_, Config_, &mpiComm, 1/* number of GPU devices utilized by this rank */, device);
#else
AMGX_resources_create(&Resources_, Config_, MPI_COMM_WORLD, 1/* number of GPU devices utilized by this rank */, device);
#endif
}
AMGX_Mode mode = AMGX_mode_dDDI;
AMGX_solver_create(&Solver_, Resources_, mode, Config_);
AMGX_matrix_create(&A_, Resources_, mode);
AMGX_vector_create(&X_, Resources_, mode);
AMGX_vector_create(&Y_, Resources_, mode);
amgxTimer->stop();
amgxTimer->incrementNumCalls();
std::vector<int> amgx2muelu;
// Construct AMGX communication pattern
if (numProcs > 1) {
RCP<const Tpetra::Import<LO,GO> > importer = inA->getCrsGraph()->getImporter();
TEUCHOS_TEST_FOR_EXCEPTION(importer.is_null(), MueLu::Exceptions::RuntimeError, "The matrix A has no Import object.");
Tpetra::Distributor distributor = importer->getDistributor();
Array<int> sendRanks = distributor.getImagesTo();
Array<int> recvRanks = distributor.getImagesFrom();
std::sort(sendRanks.begin(), sendRanks.end());
std::sort(recvRanks.begin(), recvRanks.end());
bool match = true;
if (sendRanks.size() != recvRanks.size()) {
match = false;
} else {
for (int i = 0; i < sendRanks.size(); i++) {
if (recvRanks[i] != sendRanks[i])
match = false;
break;
}
}
TEUCHOS_TEST_FOR_EXCEPTION(!match, MueLu::Exceptions::RuntimeError, "AMGX requires that the processors that we send to and receive from are the same. "
"This is not the case: we send to {" << sendRanks << "} and receive from {" << recvRanks << "}");
int num_neighbors = sendRanks.size(); // does not include the calling process
const int* neighbors = &sendRanks[0];
// Later on, we'll have to organize the send and recv data by PIDs,
// i.e, a vector V of vectors, where V[i] is PID i's vector of data.
// Hence we need to be able to quickly look up an array index
// associated with each PID.
Tpetra::Details::HashTable<int,int> hashTable(3*num_neighbors);
for (int i = 0; i < num_neighbors; i++)
hashTable.add(neighbors[i], i);
// Get some information out
ArrayView<const int> exportLIDs = importer->getExportLIDs();
ArrayView<const int> exportPIDs = importer->getExportPIDs();
Array<int> importPIDs;
Tpetra::Import_Util::getPids(*importer, importPIDs, true/* make local -1 */);
// Construct the reordering for AMGX as in AMGX_matrix_upload_all documentation
RCP<const Map> rowMap = inA->getRowMap();
RCP<const Map> colMap = inA->getColMap();
int N = rowMap->getNodeNumElements(), Nc = colMap->getNodeNumElements();
muelu2amgx_.resize(Nc, -1);
int numUniqExports = 0;
for (int i = 0; i < exportLIDs.size(); i++)
if (muelu2amgx_[exportLIDs[i]] == -1) {
numUniqExports++;
muelu2amgx_[exportLIDs[i]] = -2;
}
int localOffset = 0, exportOffset = N - numUniqExports;
// Go through exported LIDs and put them at the end of LIDs
for (int i = 0; i < exportLIDs.size(); i++)
if (muelu2amgx_[exportLIDs[i]] < 0) // exportLIDs are not unique
muelu2amgx_[exportLIDs[i]] = exportOffset++;
// Go through all non-export LIDs, and put them at the beginning of LIDs
for (int i = 0; i < N; i++)
if (muelu2amgx_[i] == -1)
muelu2amgx_[i] = localOffset++;
// Go through the tail (imported LIDs), and order those by neighbors
int importOffset = N;
for (int k = 0; k < num_neighbors; k++)
for (int i = 0; i < importPIDs.size(); i++)
if (importPIDs[i] != -1 && hashTable.get(importPIDs[i]) == k)
muelu2amgx_[i] = importOffset++;
amgx2muelu.resize(muelu2amgx_.size());
for (int i = 0; i < muelu2amgx_.size(); i++)
amgx2muelu[muelu2amgx_[i]] = i;
// Construct send arrays
std::vector<std::vector<int> > sendDatas (num_neighbors);
std::vector<int> send_sizes(num_neighbors, 0);
for (int i = 0; i < exportPIDs.size(); i++) {
int index = hashTable.get(exportPIDs[i]);
sendDatas [index].push_back(muelu2amgx_[exportLIDs[i]]);
send_sizes[index]++;
}
// FIXME: sendDatas must be sorted (based on GIDs)
std::vector<const int*> send_maps(num_neighbors);
for (int i = 0; i < num_neighbors; i++)
send_maps[i] = &(sendDatas[i][0]);
// Debugging
printMaps(comm, sendDatas, amgx2muelu, neighbors, *importer->getTargetMap(), "send_map_vector");
// Construct recv arrays
std::vector<std::vector<int> > recvDatas (num_neighbors);
std::vector<int> recv_sizes(num_neighbors, 0);
for (int i = 0; i < importPIDs.size(); i++)
if (importPIDs[i] != -1) {
int index = hashTable.get(importPIDs[i]);
recvDatas [index].push_back(muelu2amgx_[i]);
recv_sizes[index]++;
}
// FIXME: recvDatas must be sorted (based on GIDs)
std::vector<const int*> recv_maps(num_neighbors);
for (int i = 0; i < num_neighbors; i++)
recv_maps[i] = &(recvDatas[i][0]);
// Debugging
printMaps(comm, recvDatas, amgx2muelu, neighbors, *importer->getTargetMap(), "recv_map_vector");
AMGX_SAFE_CALL(AMGX_matrix_comm_from_maps_one_ring(A_, 1, num_neighbors, neighbors, &send_sizes[0], &send_maps[0], &recv_sizes[0], &recv_maps[0]));
AMGX_vector_bind(X_, A_);
AMGX_vector_bind(Y_, A_);
}
RCP<Teuchos::Time> matrixTransformTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transform matrix");
matrixTransformTimer->start();
ArrayRCP<const size_t> ia_s;
ArrayRCP<const int> ja;
ArrayRCP<const double> a;
inA->getAllValues(ia_s, ja, a);
ArrayRCP<int> ia(ia_s.size());
for (int i = 0; i < ia.size(); i++)
ia[i] = Teuchos::as<int>(ia_s[i]);
N_ = inA->getNodeNumRows();
int nnz = inA->getNodeNumEntries();
matrixTransformTimer->stop();
matrixTransformTimer->incrementNumCalls();
// Upload matrix
// TODO Do we need to pin memory here through AMGX_pin_memory?
RCP<Teuchos::Time> matrixTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transfer matrix CPU->GPU");
matrixTimer->start();
if (numProcs == 1) {
AMGX_matrix_upload_all(A_, N_, nnz, 1, 1, &ia[0], &ja[0], &a[0], NULL);
} else {
// Transform the matrix
std::vector<int> ia_new(ia.size());
std::vector<int> ja_new(ja.size());
std::vector<double> a_new (a.size());
ia_new[0] = 0;
for (int i = 0; i < N_; i++) {
int oldRow = amgx2muelu[i];
ia_new[i+1] = ia_new[i] + (ia[oldRow+1] - ia[oldRow]);
for (int j = ia[oldRow]; j < ia[oldRow+1]; j++) {
int offset = j - ia[oldRow];
ja_new[ia_new[i] + offset] = muelu2amgx_[ja[j]];
a_new [ia_new[i] + offset] = a[j];
}
// Do bubble sort on two arrays
// NOTE: There are multiple possible optimizations here (even of bubble sort)
bool swapped;
do {
swapped = false;
for (int j = ia_new[i]; j < ia_new[i+1]-1; j++)
if (ja_new[j] > ja_new[j+1]) {
std::swap(ja_new[j], ja_new[j+1]);
std::swap(a_new [j], a_new [j+1]);
swapped = true;
}
} while (swapped == true);
}
AMGX_matrix_upload_all(A_, N_, nnz, 1, 1, &ia_new[0], &ja_new[0], &a_new[0], NULL);
}
matrixTimer->stop();
matrixTimer->incrementNumCalls();
domainMap_ = inA->getDomainMap();
rangeMap_ = inA->getRangeMap();
RCP<Teuchos::Time> realSetupTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: real setup");
realSetupTimer->start();
AMGX_solver_setup(Solver_, A_);
realSetupTimer->stop();
realSetupTimer->incrementNumCalls();
vectorTimer1_ = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transfer vectors CPU->GPU");
vectorTimer2_ = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transfer vector GPU->CPU");
}
int main(int argc, char *argv[]) {
using Teuchos::RCP; // reference count pointers
using Teuchos::rcp;
using Teuchos::TimeMonitor;
//
// MPI initialization using Teuchos
//
Teuchos::GlobalMPISession mpiSession(&argc, &argv, NULL);
RCP< const Teuchos::Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
//
// Parameters
//
Teuchos::CommandLineProcessor clp(false); // Note:
GO nx,ny,nz;
nx=500;
ny=500;
nz=100;
Galeri::Xpetra::Parameters<GO> matrixParameters(clp, nx, ny, nz, "Laplace2D"); // manage parameters of the test case
Xpetra::Parameters xpetraParameters(clp); // manage parameters of Xpetra
std::string xmlFileName = "scalingTest.xml"; clp.setOption("xml", &xmlFileName, "read parameters from a file. Otherwise, this example uses by default 'scalingTest.xml'");
int amgAsPrecond=1; clp.setOption("precond",&amgAsPrecond,"apply multigrid as preconditioner");
int amgAsSolver=0; clp.setOption("fixPoint",&amgAsSolver,"apply multigrid as solver");
bool printTimings=true; clp.setOption("timings","notimings",&printTimings,"print timings to screen");
switch (clp.parse(argc,argv)) {
case Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED: return EXIT_SUCCESS; break;
case Teuchos::CommandLineProcessor::PARSE_ERROR:
case Teuchos::CommandLineProcessor::PARSE_UNRECOGNIZED_OPTION: return EXIT_FAILURE; break;
case Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL: break;
}
if (comm->getRank() == 0) {
std::cout << "========================================================" << std::endl
<< xpetraParameters << matrixParameters;
}
RCP<TimeMonitor> globalTimeMonitor = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: S - Global Time")));
// read aggregation options from file
Teuchos::FileInputSource fileSrc(xmlFileName);
Teuchos::XMLObject fileXML = fileSrc.getObject();
Teuchos::XMLParameterListReader listReader;
Teuchos::ParameterList aggList = listReader.toParameterList(fileXML);
//std::cout << "===========aggList start===========" << std::endl;
//aggList.print(std::cout);
//std::cout << "===========aggList end===========" << std::endl;
// instantiate aggregate factory, set options from parameter list
RCP<MueLu::SingleLevelFactoryBase> aggFact;
if (aggList.name() == "UncoupledAggregationFactory") {
RCP<UncoupledAggregationFactory> ucFact = rcp( new UncoupledAggregationFactory() );
//ucFact->SetParameterList(aggList);
//FIXME hack until UCAgg uses PL interface
std::string ordering = aggList.get<std::string>("Ordering");
MueLu::AggOptions::Ordering eordering;
if (ordering=="Natural") eordering = MueLu::AggOptions::NATURAL;
if (ordering=="Graph") eordering = MueLu::AggOptions::GRAPH;
if (ordering=="Random") eordering = MueLu::AggOptions::RANDOM;
ucFact->SetOrdering(eordering);
ucFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected"));
ucFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate"));
aggFact = ucFact;
} else if (aggList.name() == "CoupledAggregationFactory") {
RCP<CoupledAggregationFactory> cFact = rcp( new CoupledAggregationFactory() );
//cFact->SetParameterList(aggList);
//FIXME hack until CoupledAgg uses PL interface
//cFact->SetOrdering(aggList.get<std::string>("Ordering"));
cFact->SetMaxNeighAlreadySelected(aggList.get<int>("MaxNeighAlreadySelected"));
cFact->SetMinNodesPerAggregate(aggList.get<int>("MinNodesPerAggregate"));
aggFact = cFact;
} else {
throw(MueLu::Exceptions::RuntimeError("List's name does not correspond to a known aggregation factory."));
}
//Teuchos::ParameterList tlist = aggFact->GetParameterList();
//std::cout << "===========verify List start===========" << std::endl;
//tlist.print(std::cout);
//std::cout << "===========verify List end===========" << std::endl;
// build matrix
RCP<TimeMonitor> tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("ScalingTest: 1 - Matrix Build")));
RCP<const Map> map;
RCP<MultiVector> coordinates;
// Retrieve matrix parameters (they may have been changed on the command line), and pass them to Galeri.
// Galeri will attempt to create a square-as-possible distribution of subdomains di, e.g.,
// d1 d2 d3
// d4 d5 d6
// d7 d8 d9
// d10 d11 d12
// A perfect distribution is only possible when the #processors is a perfect square.
// This *will* result in "strip" distribution if the #processors is a prime number or if the factors are very different in
// size. For example, np=14 will give a 7-by-2 distribution.
// If you don't want Galeri to do this, specify mx or my on the galeriList.
Teuchos::ParameterList pl = matrixParameters.GetParameterList();
Teuchos::ParameterList galeriList;
galeriList.set("nx", pl.get("nx",nx));
galeriList.set("ny", pl.get("ny",ny));
//galeriList.set("mx", comm->getSize());
//galeriList.set("my", 1);
if (matrixParameters.GetMatrixType() == "Laplace1D") {
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("1D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace2D" || matrixParameters.GetMatrixType() == "Star2D") {
map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian2D", comm, galeriList);
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",map,matrixParameters.GetParameterList());
}
else if (matrixParameters.GetMatrixType() == "Laplace3D") {
coordinates = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("3D",map,matrixParameters.GetParameterList());
//map = Galeri::Xpetra::CreateMap<LO, GO, Node>(xpetraParameters.GetLib(), "Cartesian3D", comm, galeriList); //TODO when available in Galeri
map = MapFactory::Build(xpetraParameters.GetLib(), matrixParameters.GetNumGlobalElements(), 0, comm);
}
if (comm->getRank() == 0) {
GO mx = galeriList.get("mx", -1);
GO my = galeriList.get("my", -1);
std::cout << "Processor subdomains in x direction: " << mx << std::endl
<< "Processor subdomains in y direction: " << my << std::endl
<< "========================================================" << std::endl;
}
RCP<Galeri::Xpetra::Problem<Map,CrsMatrixWrap,MultiVector> > Pr =
Galeri::Xpetra::BuildProblem<SC,LO,GO,Map,CrsMatrixWrap,MultiVector>(matrixParameters.GetMatrixType(), map, matrixParameters.GetParameterList());
RCP<Matrix> A = Pr->BuildMatrix();
tm = Teuchos::null;
Level level;
RCP<MueLu::FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
level.SetFactoryManager(factoryHandler);
level.SetLevelID(0);
level.Set("A", A);
level.Request("Aggregates", aggFact.get());
level.Request(*aggFact);
level.setVerbLevel(Teuchos::VERB_NONE);
aggFact->setVerbLevel(Teuchos::VERB_NONE);
tm = rcp (new TimeMonitor(*TimeMonitor::getNewTimer("aggregation time")));
aggFact->Build(level);
tm = Teuchos::null;
globalTimeMonitor = Teuchos::null;
if (printTimings)
TimeMonitor::summarize();
} //main