TEST_F(MesosContainerizerProcessTest, MultipleURIs)
{
CommandInfo commandInfo;
CommandInfo::URI uri;
uri.set_value("hdfs:///uri1");
uri.set_executable(false);
commandInfo.add_uris()->MergeFrom(uri);
uri.set_value("hdfs:///uri2");
uri.set_executable(true);
commandInfo.add_uris()->MergeFrom(uri);
string directory = "/tmp/directory";
Option<string> user("user");
Flags flags;
flags.frameworks_home = "/tmp/frameworks";
flags.hadoop_home = "/tmp/hadoop";
map<string, string> environment =
fetcherEnvironment(commandInfo, directory, user, flags);
EXPECT_EQ(5u, environment.size());
EXPECT_EQ(
"hdfs:///uri1+0X hdfs:///uri2+1X", environment["MESOS_EXECUTOR_URIS"]);
EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
EXPECT_EQ(user.get(), environment["MESOS_USER"]);
EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
EXPECT_EQ(flags.hadoop_home, environment["HADOOP_HOME"]);
}
TEST_F(FetcherTest, FileLocalhostURI)
{
string fromDir = path::join(os::getcwd(), "from");
ASSERT_SOME(os::mkdir(fromDir));
string testFile = path::join(fromDir, "test");
EXPECT_FALSE(os::write(testFile, "data").isError());
string localFile = path::join(os::getcwd(), "test");
EXPECT_FALSE(os::exists(localFile));
slave::Flags flags;
flags.frameworks_home = "/tmp/frameworks";
CommandInfo commandInfo;
CommandInfo::URI* uri = commandInfo.add_uris();
uri->set_value(path::join("file://localhost", testFile));
map<string, string> env =
fetcher::environment(commandInfo, os::getcwd(), None(), flags);
Try<Subprocess> fetcherProcess =
process::subprocess(
path::join(mesos::internal::tests::flags.build_dir, "src/mesos-fetcher"),
env);
ASSERT_SOME(fetcherProcess);
Future<Option<int>> status = fetcherProcess.get().status();
AWAIT_READY(status);
ASSERT_SOME(status.get());
EXPECT_EQ(0, status.get().get());
EXPECT_TRUE(os::exists(localFile));
}
TEST_F(FetcherEnvironmentTest, MultipleURIs)
{
CommandInfo commandInfo;
CommandInfo::URI uri;
uri.set_value("hdfs:///uri1");
uri.set_executable(false);
commandInfo.add_uris()->MergeFrom(uri);
uri.set_value("hdfs:///uri2");
uri.set_executable(true);
commandInfo.add_uris()->MergeFrom(uri);
string directory = "/tmp/directory";
Option<string> user("user");
slave::Flags flags;
flags.frameworks_home = "/tmp/frameworks";
flags.hadoop_home = "/tmp/hadoop";
map<string, string> environment =
fetcher::environment(commandInfo, directory, user, flags);
EXPECT_EQ(5u, environment.size());
EXPECT_EQ(stringify(JSON::Protobuf(commandInfo)),
environment["MESOS_COMMAND_INFO"]);
EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
EXPECT_EQ(user.get(), environment["MESOS_USER"]);
EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
EXPECT_EQ(flags.hadoop_home, environment["HADOOP_HOME"]);
}
TEST_F(FetcherTest, ExtractNotExecutable)
{
// First construct a temporary file that can be fetched and archive
// with tar gzip.
Try<string> path = os::mktemp();
ASSERT_SOME(path);
ASSERT_SOME(os::write(path.get(), "hello world"));
// TODO(benh): Update os::tar so that we can capture or ignore
// stdout/stderr output.
ASSERT_SOME(os::tar(path.get(), path.get() + ".tar.gz"));
CommandInfo commandInfo;
CommandInfo::URI* uri = commandInfo.add_uris();
uri->set_value(path.get() + ".tar.gz");
uri->set_executable(false);
uri->set_extract(true);
Option<int> stdout = None();
Option<int> stderr = None();
// Redirect mesos-fetcher output if running the tests verbosely.
if (tests::flags.verbose) {
stdout = STDOUT_FILENO;
stderr = STDERR_FILENO;
}
slave::Flags flags;
flags.launcher_dir = path::join(tests::flags.build_dir, "src");
Future<Option<int>> run =
fetcher::run(commandInfo, os::getcwd(), None(), flags, stdout, stderr);
AWAIT_READY(run);
EXPECT_SOME_EQ(0, run.get());
ASSERT_TRUE(os::exists(path::join(".", path.get())));
ASSERT_SOME_EQ("hello world", os::read(path::join(".", path.get())));
Try<os::Permissions> permissions =
os::permissions(path::join(".", path.get()));
ASSERT_SOME(permissions);
EXPECT_FALSE(permissions.get().owner.x);
EXPECT_FALSE(permissions.get().group.x);
EXPECT_FALSE(permissions.get().others.x);
ASSERT_SOME(os::rm(path.get()));
}
inline std::size_t hash_value(const CommandInfo::URI& uri)
{
size_t seed = 0;
if (uri.extract()) {
seed += 11;
}
if (uri.executable()) {
seed += 2003;
}
boost::hash_combine(seed, uri.value());
return seed;
}
// Returns the resulting file or in case of extraction the destination
// directory (for logging).
static Try<string> fetchBypassingCache(
const CommandInfo::URI& uri,
const string& sandboxDirectory,
const Option<string>& frameworksHome)
{
LOG(INFO) << "Fetching directly into the sandbox directory";
// TODO(mrbrowning): Factor out duplicated processing of "output_file" field
// here and in fetchFromCache into a separate helper function.
if (uri.has_output_file()) {
string dirname = Path(uri.output_file()).dirname();
if (dirname != ".") {
Try<Nothing> result =
os::mkdir(path::join(sandboxDirectory, dirname), true);
if (result.isError()) {
return Error(
"Unable to create subdirectory " + dirname + " in sandbox");
}
}
}
Try<string> outputFile = uri.has_output_file()
? uri.output_file()
: Fetcher::basename(uri.value());
if (outputFile.isError()) {
return Error(outputFile.error());
}
string path = path::join(sandboxDirectory, outputFile.get());
Try<string> downloaded = download(uri.value(), path, frameworksHome);
if (downloaded.isError()) {
return Error(downloaded.error());
}
if (uri.executable()) {
return chmodExecutable(downloaded.get());
} else if (uri.extract()) {
Try<bool> extracted = extract(path, sandboxDirectory);
if (extracted.isError()) {
return Error(extracted.error());
} else if (!extracted.get()) {
LOG(WARNING) << "Copying instead of extracting resource from URI with "
<< "'extract' flag, because it does not seem to be an "
<< "archive: " << uri.value();
}
}
return downloaded;
}
TEST_F(FetcherTest, NoExtractExecutable)
{
// First construct a temporary file that can be fetched.
Try<string> path = os::mktemp();
ASSERT_SOME(path);
CommandInfo commandInfo;
CommandInfo::URI* uri = commandInfo.add_uris();
uri->set_value(path.get());
uri->set_executable(true);
uri->set_extract(false);
Option<int> stdout = None();
Option<int> stderr = None();
// Redirect mesos-fetcher output if running the tests verbosely.
if (tests::flags.verbose) {
stdout = STDOUT_FILENO;
stderr = STDERR_FILENO;
}
slave::Flags flags;
flags.launcher_dir = path::join(tests::flags.build_dir, "src");
Future<Option<int>> run =
fetcher::run(commandInfo, os::getcwd(), None(), flags, stdout, stderr);
AWAIT_READY(run);
EXPECT_SOME_EQ(0, run.get());
Try<string> basename = os::basename(path.get());
ASSERT_SOME(basename);
Try<os::Permissions> permissions = os::permissions(basename.get());
ASSERT_SOME(permissions);
EXPECT_TRUE(permissions.get().owner.x);
EXPECT_TRUE(permissions.get().group.x);
EXPECT_TRUE(permissions.get().others.x);
ASSERT_SOME(os::rm(path.get()));
}
inline bool operator == (
const CommandInfo::URI& left,
const CommandInfo::URI& right)
{
return left.has_executable() == right.has_executable() &&
(!left.has_executable() || (left.executable() == right.executable())) &&
left.value() == right.value();
}
// Returns the resulting file or in case of extraction the destination
// directory (for logging).
static Try<string> fetchBypassingCache(
const CommandInfo::URI& uri,
const string& sandboxDirectory,
const Option<string>& frameworksHome)
{
LOG(INFO) << "Fetching directly into the sandbox directory";
Try<string> basename = Fetcher::basename(uri.value());
if (basename.isError()) {
return Error("Failed to determine the basename of the URI '" +
uri.value() + "' with error: " + basename.error());
}
string path = path::join(sandboxDirectory, basename.get());
Try<string> downloaded = download(uri.value(), path, frameworksHome);
if (downloaded.isError()) {
return Error(downloaded.error());
}
if (uri.executable()) {
return chmodExecutable(downloaded.get());
} else if (uri.extract()) {
Try<bool> extracted = extract(path, sandboxDirectory);
if (extracted.isError()) {
return Error(extracted.error());
} else if (!extracted.get()) {
LOG(WARNING) << "Copying instead of extracting resource from URI with "
<< "'extract' flag, because it does not seem to be an "
<< "archive: " << uri.value();
}
}
return downloaded;
}
TEST_F(MesosContainerizerProcessTest, NoUser)
{
CommandInfo commandInfo;
CommandInfo::URI uri;
uri.set_value("hdfs:///uri");
uri.set_executable(false);
commandInfo.add_uris()->MergeFrom(uri);
string directory = "/tmp/directory";
Flags flags;
flags.frameworks_home = "/tmp/frameworks";
flags.hadoop_home = "/tmp/hadoop";
map<string, string> environment =
fetcherEnvironment(commandInfo, directory, None(), flags);
EXPECT_EQ(4u, environment.size());
EXPECT_EQ("hdfs:///uri+0X", environment["MESOS_EXECUTOR_URIS"]);
EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
EXPECT_EQ(flags.hadoop_home, environment["HADOOP_HOME"]);
}
TEST_F(FetcherEnvironmentTest, NoHadoop)
{
CommandInfo commandInfo;
CommandInfo::URI* uri = commandInfo.add_uris();
uri->set_value("hdfs:///uri");
uri->set_executable(false);
string directory = "/tmp/directory";
Option<string> user = "******";
slave::Flags flags;
flags.frameworks_home = "/tmp/frameworks";
map<string, string> environment =
fetcher::environment(commandInfo, directory, user, flags);
EXPECT_EQ(4u, environment.size());
EXPECT_EQ(stringify(JSON::Protobuf(commandInfo)),
environment["MESOS_COMMAND_INFO"]);
EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
EXPECT_EQ(user.get(), environment["MESOS_USER"]);
EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
}
TEST_F(FetcherTest, OSNetUriTest)
{
HttpProcess process;
spawn(process);
string url = "http://" + net::getHostname(process.self().node.ip).get() +
":" + stringify(process.self().node.port) + "/help";
string localFile = path::join(os::getcwd(), "help");
EXPECT_FALSE(os::exists(localFile));
slave::Flags flags;
flags.frameworks_home = "/tmp/frameworks";
CommandInfo commandInfo;
CommandInfo::URI* uri = commandInfo.add_uris();
uri->set_value(url);
map<string, string> env =
fetcher::environment(commandInfo, os::getcwd(), None(), flags);
Try<Subprocess> fetcherProcess =
process::subprocess(
path::join(mesos::internal::tests::flags.build_dir, "src/mesos-fetcher"),
env);
ASSERT_SOME(fetcherProcess);
Future<Option<int>> status = fetcherProcess.get().status();
AWAIT_READY(status);
ASSERT_SOME(status.get());
EXPECT_EQ(0, status.get().get());
EXPECT_TRUE(os::exists(localFile));
}
bool operator == (const CommandInfo::URI& left, const CommandInfo::URI& right)
{
return left.value() == right.value() &&
left.executable() == right.executable() &&
left.extract() == right.extract();
}
int main(int argc, char* argv[])
{
GOOGLE_PROTOBUF_VERIFY_VERSION;
CommandInfo commandInfo;
// Construct URIs from the encoded environment string.
const std::string& uris = os::getenv("MESOS_EXECUTOR_URIS");
foreach (const std::string& token, strings::tokenize(uris, " ")) {
// Delimiter between URI, execute permission and extract options
// Expected format: {URI}+[01][XN]
// {URI} - The actual URI for the asset to fetch
// [01] - 1 if the execute permission should be set else 0
// [XN] - X if we should extract the URI (if it's compressed) else N
size_t pos = token.rfind("+");
CHECK(pos != std::string::npos)
<< "Invalid executor uri token in env " << token;
CommandInfo::URI uri;
uri.set_value(token.substr(0, pos));
uri.set_executable(token.substr(pos + 1, 1) == "1");
uri.set_extract(token.substr(pos + 2, 1) == "X");
commandInfo.add_uris()->MergeFrom(uri);
}
CHECK(os::hasenv("MESOS_WORK_DIRECTORY"))
<< "Missing MESOS_WORK_DIRECTORY environment variable";
std::string directory = os::getenv("MESOS_WORK_DIRECTORY");
// We cannot use Some in the ternary expression because the compiler needs to
// be able to infer the type, thus the explicit Option<string>.
// TODO(idownes): Add an os::hasenv that returns an Option<string>.
Option<std::string> user = os::hasenv("MESOS_USER")
? Option<std::string>(os::getenv("MESOS_USER")) // Explicit so it compiles.
: None();
// Fetch each URI to a local file, chmod, then chown if a user is provided.
foreach (const CommandInfo::URI& uri, commandInfo.uris()) {
// Fetch the URI to a local file.
Try<string> fetched = fetch(uri.value(), directory);
if (fetched.isError()) {
EXIT(1) << "Failed to fetch: " << uri.value();
}
// Chmod the fetched URI if it's executable, else assume it's an archive
// that should be extracted.
if (uri.executable()) {
Try<Nothing> chmod = os::chmod(
fetched.get(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
if (chmod.isError()) {
EXIT(1) << "Failed to chmod " << fetched.get() << ": " << chmod.error();
}
} else if (uri.extract()) {
//TODO(idownes): Consider removing the archive once extracted.
// Try to extract the file if it's recognized as an archive.
Try<bool> extracted = extract(fetched.get(), directory);
if (extracted.isError()) {
EXIT(1) << "Failed to extract "
<< fetched.get() << ":" << extracted.error();
}
} else {
LOG(INFO) << "Skipped extracting path '" << fetched.get() << "'";
}
// Recursively chown the directory if a user is provided.
if (user.isSome()) {
Try<Nothing> chowned = os::chown(user.get(), directory);
if (chowned.isError()) {
EXIT(1) << "Failed to chown " << directory << ": " << chowned.error();
}
}
}
return 0;
}
