void* PosixDlopenLibLoader::call_dlopen(const URI& fpath)
{
void* hdl = nullptr;
hdl = dlopen (fpath.path().c_str(), RTLD_LAZY|RTLD_GLOBAL);
if( is_not_null(hdl) )
CFinfo << "dlopen() loaded library \'" << fpath.path() << "\'" << CFendl;
// library name
if ( is_null(hdl) && !fpath.is_absolute() )
{
// loop over the search paths and attempt to load the library
std::vector< URI >::const_iterator itr = m_search_paths.begin();
for ( ; itr != m_search_paths.end(); ++itr)
{
// CFinfo << "searching in [" << *itr << "]\n" << CFflush;
URI fullqname = *itr / fpath;
// CFinfo << "fullqname [" << fullqname.string() << "]\n" << CFflush;
hdl = dlopen (fullqname.path().c_str(), RTLD_LAZY|RTLD_GLOBAL);
if( hdl != nullptr )
{
CFinfo << "dlopen() loaded library \'" << fullqname.path() << "\'" << CFendl;
break;
}
}
}
return hdl; // will return nullptr if failed
}
Future<Nothing> HadoopFetcherPlugin::fetch(
const URI& uri,
const string& directory)
{
// TODO(jieyu): Validate the given URI.
if (!uri.has_path()) {
return Failure("URI path is not specified");
}
Try<Nothing> mkdir = os::mkdir(directory);
if (mkdir.isError()) {
return Failure(
"Failed to create directory '" +
directory + "': " + mkdir.error());
}
// NOTE: We ignore the scheme prefix if the host in URI is not
// specified. This is the case when the host is set using the hadoop
// configuration file.
//
// TODO(jieyu): Allow user to specify the name of the output file.
return hdfs.get()->copyToLocal(
(uri.has_host() ? stringify(uri) : uri.path()),
path::join(directory, Path(uri.path()).basename()));
}
bool operator == (const URI& lhs, const URI& rhs) noexcept {
return icase_equal(lhs.scheme(), rhs.scheme())
and (lhs.userinfo() == rhs.userinfo())
and icase_equal(lhs.host(), rhs.host())
and lhs.port() == rhs.port()
and lhs.path() == rhs.path()
and lhs.query() == rhs.query()
and lhs.fragment() == rhs.fragment();
}
void BinaryDataReader::trigger_file()
{
const URI file_uri = options().value<URI>("file");
if(!boost::filesystem::exists(file_uri.path()))
{
throw SetupError(FromHere(), "Input file " + file_uri.path() + " does not exist");
}
m_implementation.reset(new Implementation(file_uri, options().value<Uint>("rank")));
}
void Client::populate_from_url(Request& req, const URI& url)
{
// Set uri path (default "/")
req.set_uri((!url.path().empty()) ? URI{url.path()} : URI{"/"});
// Set Host: host(:port)
const auto port = url.port();
req.header().set_field(header::Host,
(port != 0xFFFF and port != 80) ?
url.host().to_string() + ":" + std::to_string(port)
: url.host().to_string()); // to_string madness
}
std::string build_filename(const URI& input, const Uint rank)
{
const URI my_dir = input.base_path();
const std::string basename = input.base_name();
const URI result(my_dir / (basename + "_P" + to_str(rank) + ".cfbin"));
return result.path();
}
Future<Nothing> CurlFetcherPlugin::fetch(
const URI& uri,
const string& directory)
{
// TODO(jieyu): Validate the given URI.
if (!uri.has_path()) {
return Failure("URI path is not specified");
}
if (!os::exists(directory)) {
Try<Nothing> mkdir = os::mkdir(directory);
if (mkdir.isError()) {
return Failure(
"Failed to create directory '" +
directory + "': " + mkdir.error());
}
}
// TODO(jieyu): Allow user to specify the name of the output file.
const string output = path::join(directory, Path(uri.path()).basename());
const vector<string> argv = {
"curl",
"-s", // Don’t show progress meter or error messages.
"-S", // Makes curl show an error message if it fails.
"-L", // Follow HTTP 3xx redirects.
"-w", "%{http_code}", // Display HTTP response code on stdout.
"-o", output, // Write output to the file.
strings::trim(stringify(uri))
};
Try<Subprocess> s = subprocess(
"curl",
argv,
Subprocess::PATH("/dev/null"),
Subprocess::PIPE(),
Subprocess::PIPE());
if (s.isError()) {
return Failure("Failed to exec the curl subprocess: " + s.error());
}
return await(
s.get().status(),
io::read(s.get().out().get()),
io::read(s.get().err().get()))
.then(_fetch);
}
// TODO(jieyu): Add a comment here.
static Future<int> download(
const URI& uri,
const string& directory,
const http::Headers& headers = http::Headers())
{
const string output = path::join(directory, Path(uri.path()).basename());
vector<string> argv = {
"curl",
"-s", // Don't show progress meter or error messages.
"-S", // Make curl show an error message if it fails.
"-L", // Follow HTTP 3xx redirects.
"-w", "%{http_code}", // Display HTTP response code on stdout.
"-o", output // Write output to the file.
};
// Add additional headers.
foreachpair (const string& key, const string& value, headers) {
argv.push_back("-H");
argv.push_back(key + ": " + value);
}
std::size_t URI::HashFn::operator()(const URI& uri) const
{
std::size_t seed = 0;
if (uri.scheme()) {
boost::hash_combine(seed, *uri.scheme());
}
if (uri.authority()) {
boost::hash_combine(seed, *uri.authority());
}
boost::hash_combine(seed, uri.path());
if (uri.query()) {
boost::hash_combine(seed, *uri.query());
}
if (uri.fragment()) {
boost::hash_combine(seed, *uri.fragment());
}
return seed;
}
Future<Nothing> CurlFetcherPlugin::fetch(
const URI& uri,
const string& directory)
{
// TODO(jieyu): Validate the given URI.
if (!uri.has_path()) {
return Failure("URI path is not specified");
}
Try<Nothing> mkdir = os::mkdir(directory);
if (mkdir.isError()) {
return Failure(
"Failed to create directory '" +
directory + "': " + mkdir.error());
}
// TODO(jieyu): Allow user to specify the name of the output file.
const string output = path::join(directory, Path(uri.path()).basename());
const vector<string> argv = {
"curl",
"-s", // Don’t show progress meter or error messages.
"-S", // Makes curl show an error message if it fails.
"-L", // Follow HTTP 3xx redirects.
"-w", "%{http_code}", // Display HTTP response code on stdout.
"-o", output, // Write output to the file.
strings::trim(stringify(uri))
};
Try<Subprocess> s = subprocess(
"curl",
argv,
Subprocess::PATH("/dev/null"),
Subprocess::PIPE(),
Subprocess::PIPE());
if (s.isError()) {
return Failure("Failed to exec the curl subprocess: " + s.error());
}
return await(
s.get().status(),
io::read(s.get().out().get()),
io::read(s.get().err().get()))
.then([](const tuple<
Future<Option<int>>,
Future<string>,
Future<string>>& t) -> Future<Nothing> {
Future<Option<int>> status = std::get<0>(t);
if (!status.isReady()) {
return Failure(
"Failed to get the exit status of the curl subprocess: " +
(status.isFailed() ? status.failure() : "discarded"));
}
if (status->isNone()) {
return Failure("Failed to reap the curl subprocess");
}
if (status->get() != 0) {
Future<string> error = std::get<2>(t);
if (!error.isReady()) {
return Failure(
"Failed to perform 'curl'. Reading stderr failed: " +
(error.isFailed() ? error.failure() : "discarded"));
}
return Failure("Failed to perform 'curl': " + error.get());
}
Future<string> output = std::get<1>(t);
if (!output.isReady()) {
return Failure(
"Failed to read stdout from 'curl': " +
(output.isFailed() ? output.failure() : "discarded"));
}
// Parse the output and get the HTTP response code.
Try<int> code = numify<int>(output.get());
if (code.isError()) {
return Failure("Unexpected output from 'curl': " + output.get());
}
if (code.get() != http::Status::OK) {
return Failure(
"Unexpected HTTP response code: " +
http::Status::string(code.get()));
}
return Nothing();
});
}
bool VFSFile::unlink( const URI &uri )
{
int32 err = 0;
return Sys::fal_unlink( uri.path(), err );
}
bool VFSFile::move( const URI &suri, const URI &duri )
{
int32 err = 0;
return Sys::fal_move( suri.path(), duri.path(), err );
}
bool VFSFile::rmdir( const URI &uri )
{
AutoCString filename( uri.path() );
return ::rmdir( filename.c_str() ) == 0;
}
bool VFSFile::mkdir( const URI &uri, uint32 mode )
{
AutoCString filename( uri.path() );
return ::mkdir( filename.c_str(), mode ) == 0;
}
bool VFSFile::move( const URI &suri, const URI &duri )
{
AutoCString filename( suri.path() );
AutoCString dest( duri.path() );
return ::rename( filename.c_str(), dest.c_str() ) == 0;
}
bool VFSFile::chmod( const URI &uri, int mode )
{
AutoCString filename( uri.path() );
return ::chmod( filename.c_str(), mode ) == 0;
}
bool VFSFile::chown( const URI &uri, int uid, int gid )
{
AutoCString filename( uri.path() );
return ::chown( filename.c_str(), uid, gid ) == 0;
}
bool VFSFile::mkdir( const URI &uri, uint32 mode )
{
int32 err = 0;
return Sys::fal_mkdir( uri.path(), err );
}
Future<Nothing> CopyFetcherPlugin::fetch(
const URI& uri,
const string& directory) const
{
// TODO(jojy): Validate the given URI.
if (!uri.has_path()) {
return Failure("URI path is not specified");
}
// TODO(jojy): Verify that the path is a file.
Try<Nothing> mkdir = os::mkdir(directory);
if (mkdir.isError()) {
return Failure(
"Failed to create directory '" +
directory + "': " + mkdir.error());
}
VLOG(1) << "Copying '" << uri.path() << "' to '" << directory << "'";
const vector<string> argv = {"cp", "-a", uri.path(), directory};
Try<Subprocess> s = subprocess(
"cp",
argv,
Subprocess::PATH(os::DEV_NULL),
Subprocess::PIPE(),
Subprocess::PIPE());
if (s.isError()) {
return Failure("Failed to exec the copy subprocess: " + s.error());
}
return await(
s.get().status(),
io::read(s.get().out().get()),
io::read(s.get().err().get()))
.then([](const tuple<
Future<Option<int>>,
Future<string>,
Future<string>>& t) -> Future<Nothing> {
Future<Option<int>> status = std::get<0>(t);
if (!status.isReady()) {
return Failure(
"Failed to get the exit status of the copy subprocess: " +
(status.isFailed() ? status.failure() : "discarded"));
}
if (status->isNone()) {
return Failure("Failed to reap the copy subprocess");
}
if (status->get() != 0) {
Future<string> error = std::get<2>(t);
if (!error.isReady()) {
return Failure(
"Failed to perform 'copy'. Reading stderr failed: " +
(error.isFailed() ? error.failure() : "discarded"));
}
return Failure("Failed to perform 'copy': " + error.get());
}
return Nothing();
});
}
void CEigenLSS::solve()
{
#ifdef CF_HAVE_TRILINOS
Timer timer;
const Uint nb_rows = size();
cf_assert(nb_rows == m_system_matrix.outerSize());
Epetra_SerialComm comm;
Epetra_Map map(nb_rows, 0, comm);
Epetra_Vector ep_rhs(View, map, m_rhs.data());
Epetra_Vector ep_sol(View, map, m_solution.data());
// Count non-zeros
std::vector<int> nnz(nb_rows, 0);
for(int row=0; row < nb_rows; ++row)
{
for(MatrixT::InnerIterator it(m_system_matrix, row); it; ++it)
{
++nnz[row];
}
cf_assert(nnz[row]);
}
Epetra_CrsMatrix ep_A(Copy, map, &nnz[0]);
time_matrix_construction = timer.elapsed(); timer.restart();
// Fill the matrix
for(int row=0; row < nb_rows; ++row)
{
std::vector<int> indices; indices.reserve(nnz[row]);
std::vector<Real> values; values.reserve(nnz[row]);
for(MatrixT::InnerIterator it(m_system_matrix, row); it; ++it)
{
indices.push_back(it.col());
values.push_back(it.value());
}
ep_A.InsertGlobalValues(row, nnz[row], &values[0], &indices[0]);
}
ep_A.FillComplete();
time_matrix_fill = timer.elapsed(); timer.restart();
///////////////////////////////////////////////////////////////////////////////////////////////
//BEGIN////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
Teuchos::RCP<Epetra_CrsMatrix> epetra_A=Teuchos::rcpFromRef(ep_A);
Teuchos::RCP<Epetra_Vector> epetra_x=Teuchos::rcpFromRef(ep_sol);
Teuchos::RCP<Epetra_Vector> epetra_b=Teuchos::rcpFromRef(ep_rhs);
const URI config_uri = option("config_file").value<URI>();
const std::string config_path = config_uri.path();
Stratimikos::DefaultLinearSolverBuilder linearSolverBuilder(config_path); // the most important in general setup
Teuchos::RCP<Teuchos::FancyOStream> out = Teuchos::VerboseObjectBase::getDefaultOStream(); // TODO: decouple from fancyostream to ostream or to C stdout when possible
typedef Teuchos::ParameterList::PrintOptions PLPrintOptions;
Teuchos::CommandLineProcessor clp(false); // false: don't throw exceptions
Teuchos::RCP<const Thyra::LinearOpBase<double> > A = Thyra::epetraLinearOp( epetra_A );
Teuchos::RCP<Thyra::VectorBase<double> > x = Thyra::create_Vector( epetra_x, A->domain() );
Teuchos::RCP<const Thyra::VectorBase<double> > b = Thyra::create_Vector( epetra_b, A->range() );
// r = b - A*x, initial L2 norm
double nrm_r=0.;
Real systemResidual=-1.;
{
Epetra_Vector epetra_r(*epetra_b);
Epetra_Vector epetra_A_x(epetra_A->OperatorRangeMap());
epetra_A->Apply(*epetra_x,epetra_A_x);
epetra_r.Update(-1.0,epetra_A_x,1.0);
epetra_r.Norm2(&nrm_r);
}
// Reading in the solver parameters from the parameters file and/or from
// the command line. This was setup by the command-line options
// set by the setupCLP(...) function above.
linearSolverBuilder.readParameters(0); // out.get() if want confirmation about the xml file within trilinos
Teuchos::RCP<Thyra::LinearOpWithSolveFactoryBase<double> > lowsFactory = linearSolverBuilder.createLinearSolveStrategy(""); // create linear solver strategy
lowsFactory->setVerbLevel(Teuchos::VERB_NONE); // set verbosity
// // print back default and current settings
// if (opts->trilinos.dumpDefault!=0) {
// fflush(stdout); cout << flush;
// _MMESSAGE_(0,1,"Dumping Trilinos/Stratimikos solver defaults to files: 'trilinos_default.txt' and 'trilinos_default.xml'...\n");
// fflush(stdout); cout << flush;
// std::ofstream ofs("./trilinos_default.txt");
// linearSolverBuilder.getValidParameters()->print(ofs,PLPrintOptions().indent(2).showTypes(true).showDoc(true)); // the last true-false is the deciding about whether printing documentation to option or not
// ofs.flush();ofs.close();
// ofs.open("./trilinos_default.xml");
// Teuchos::writeParameterListToXmlOStream(*linearSolverBuilder.getValidParameters(),ofs);
// ofs.flush();ofs.close();
// }
// if (opts->trilinos.dumpCurrXML!=0) {
// fflush(stdout); cout << flush;
// _MMESSAGE_(0,1,"Dumping Trilinos/Stratimikos current settings to file: 'trilinos_current.xml'...\n");
// fflush(stdout); cout << flush;
// linearSolverBuilder.writeParamsFile(*lowsFactory,"./trilinos_current.xml");
// }
time_solver_setup = timer.elapsed(); timer.restart();
// solve the matrix
Teuchos::RCP<Thyra::LinearOpWithSolveBase<double> > lows = Thyra::linearOpWithSolve(*lowsFactory, A); // create solver
Thyra::solve(*lows, Thyra::NOTRANS, *b, &*x); // solve
time_solve = timer.elapsed(); timer.restart();
// r = b - A*x, final L2 norm
{
Epetra_Vector epetra_r(*epetra_b);
Epetra_Vector epetra_A_x(epetra_A->OperatorRangeMap());
epetra_A->Apply(*epetra_x,epetra_A_x);
epetra_r.Update(-1.0,epetra_A_x,1.0);
systemResidual=1./nrm_r;
nrm_r=0.;
epetra_r.Norm2(&nrm_r);
systemResidual*=nrm_r;
}
time_residual = timer.elapsed();
///////////////////////////////////////////////////////////////////////////////////////////////
//END//////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////
#else // no trilinos
#ifdef CF_HAVE_SUPERLU
Eigen::SparseMatrix<Real> A(m_system_matrix);
Eigen::SparseLU<Eigen::SparseMatrix<Real>,Eigen::SuperLU> lu_of_A(A);
if(!lu_of_A.solve(rhs(), &m_solution))
throw Common::FailedToConverge(FromHere(), "Solution failed.");
#else // no trilinos and no superlu
RealMatrix A(m_system_matrix);
Eigen::FullPivLU<RealMatrix> lu_of_A(A);
m_solution = lu_of_A.solve(m_rhs);
#endif // end ifdef superlu
#endif // end ifdef trilinos
}
bool VFSFile::rmdir( const URI &uri )
{
int32 err = 0;
return Sys::fal_rmdir( uri.path(), err );
}
bool VFSFile::readStats( const URI& uri, FileStat &s )
{
return Sys::fal_stats( uri.path(), s );
}
void Component::complete_path ( URI& path ) const
{
using namespace boost::algorithm;
// CFinfo << "PATH [" << path.string() << "]\n" << CFflush;
cf_assert( path.scheme() == URI::Scheme::CPATH );
if(path.empty())
path = "./";
if ( is_null(m_raw_parent) )
throw InvalidURI(FromHere(), "Component \'" + name() + "\' has no parent");
if (m_root.expired())
throw InvalidURI(FromHere(), "Component \'" + name() + "\' has no root");
boost::shared_ptr<Component> parent = m_raw_parent->self();
boost::shared_ptr<Component> root = m_root.lock();
std::string sp = path.path();
if ( path.is_relative() ) // transform it to absolute
{
if ( starts_with(sp,"/") ) // remove leading "/" if any
boost::algorithm::replace_first(sp, "/", "" );
// substitute leading "../" for uri() of parent
if (starts_with(sp,".."))
{
std::string pfp = parent->uri().path();
boost::algorithm::replace_first(sp, "..", pfp);
}
// substitute leading "./" for uri() of this component
else if (starts_with(sp,"."))
{
boost::algorithm::replace_first(sp, ".", uri().path());
}
else
{
sp = uri().path()+"/"+sp;
}
}
cf_assert ( URI(sp).is_absolute() );
// break path in tokens and loop on them, while concatenaitng to a new path
boost::char_separator<char> sep("/");
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
tokenizer tok (sp,sep);
path = "/" ;
std::string last;
for(tokenizer::iterator el=tok.begin(); el!=tok.end(); ++el)
{
if ( equals (*el, ".") ) continue; // substitute any "/./" for nothing
if ( equals (*el, "..") ) // substitute any "../" for base path
path = path.base_path();
else
path /= *el;
}
// CFinfo << "FINAL PATH: [" << path.string() << "]\n" << CFflush;
cf_assert ( path.is_complete() );
}