void GroupProcess::connected(bool reconnect)
{
if (!reconnect) {
// Authenticate if necessary (and we are connected for the first
// time, or after a session expiration).
if (auth.isSome()) {
LOG(INFO) << "Authenticating with ZooKeeper using " << auth.get().scheme;
int code = zk->authenticate(auth.get().scheme, auth.get().credentials);
if (code != ZOK) { // TODO(benh): Authentication retries?
Try<string> message = strings::format(
"Failed to authenticate with ZooKeeper: %s", zk->message(code));
error = message.isSome()
? message.get()
: "Failed to authenticate with ZooKeeper";
abort(); // Cancels everything pending.
return;
}
}
// Create directory path znodes as necessary.
CHECK(znode.size() == 0 || znode.at(znode.size() - 1) != '/');
size_t index = znode.find("/", 0);
while (index < string::npos) {
// Get out the prefix to create.
index = znode.find("/", index + 1);
const string& prefix = znode.substr(0, index);
LOG(INFO) << "Trying to create '" << prefix << "' in ZooKeeper";
// Create the node (even if it already exists).
int code = zk->create(prefix, "", acl, 0, NULL);
if (code == ZINVALIDSTATE || (code != ZOK && zk->retryable(code))) {
CHECK(zk->getState() != ZOO_AUTH_FAILED_STATE);
return; // Try again later.
} else if (code != ZOK && code != ZNODEEXISTS) {
Try<string> message = strings::format(
"Failed to create '%s' in ZooKeeper: %s",
prefix.c_str(), zk->message(code));
error = message.isSome()
? message.get()
: "Failed to create node in ZooKeeper";
abort(); // Cancels everything pending.
return;
}
}
}
state = CONNECTED;
sync(); // Handle pending (and cache memberships).
}
bool operator==(Try<T> lhs, Try<T> rhs)
{
if (lhs.isSome() != rhs.isSome()) {
return false;
}
if (lhs.isSome()) {
return lhs.get() == rhs.get();
}
return lhs.error() == rhs.error();
}
inline Result<Process> process(pid_t pid)
{
// Page size, used for memory accounting.
// NOTE: This is more portable than using getpagesize().
static const long pageSize = sysconf(_SC_PAGESIZE);
if (pageSize <= 0) {
return Error("Failed to get sysconf(_SC_PAGESIZE)");
}
// Number of clock ticks per second, used for cpu accounting.
static const long ticks = sysconf(_SC_CLK_TCK);
if (ticks <= 0) {
return Error("Failed to get sysconf(_SC_CLK_TCK)");
}
const Result<proc::ProcessStatus> status = proc::status(pid);
if (status.isError()) {
return Error(status.error());
}
if (status.isNone()) {
return None();
}
// There are known bugs with invalid utime / stime values coming
// from /proc/<pid>/stat on some Linux systems.
// See the following thread for details:
// http://mail-archives.apache.org/mod_mbox/incubator-mesos-dev/
// 201307.mbox/%3CCA+2n2er-Nemh0CsKLbHRkaHd=YCrNt17NLUPM2=TtEfsKOw4
// [email protected]%3E
// These are similar reports:
// http://lkml.indiana.edu/hypermail/linux/kernel/1207.1/01388.html
// https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1023214
Try<Duration> utime = Duration::create(status.get().utime / (double) ticks);
Try<Duration> stime = Duration::create(status.get().stime / (double) ticks);
// The command line from 'status.get().comm' is only "arg0" from
// "argv" (i.e., the canonical executable name). To get the entire
// command line we grab '/proc/[pid]/cmdline'.
Result<std::string> cmdline = proc::cmdline(pid);
return Process(status.get().pid,
status.get().ppid,
status.get().pgrp,
status.get().session,
Bytes(status.get().rss * pageSize),
utime.isSome() ? utime.get() : Option<Duration>::none(),
stime.isSome() ? stime.get() : Option<Duration>::none(),
cmdline.isSome() ? cmdline.get() : status.get().comm,
status.get().state == 'Z');
}
TEST(StringsTest, Format)
{
Try<std::string> result = strings::format("%s %s", "hello", "world");
ASSERT_TRUE(result.isSome());
EXPECT_EQ("hello world", result.get());
result = strings::format("hello %d", 42);
ASSERT_TRUE(result.isSome());
EXPECT_EQ("hello 42", result.get());
result = strings::format("hello %s", "fourty-two");
ASSERT_TRUE(result.isSome());
EXPECT_EQ("hello fourty-two", result.get());
}
Try<string> Environment::mkdtemp()
{
const ::testing::TestInfo* const testInfo =
::testing::UnitTest::GetInstance()->current_test_info();
if (testInfo == NULL) {
return Error("Failed to determine the current test information");
}
// We replace any slashes present in the test names (e.g. TYPED_TEST),
// to make sure the temporary directory resides under '/tmp/'.
const string& testCase =
strings::replace(testInfo->test_case_name(), "/", "_");
string testName = strings::replace(testInfo->name(), "/", "_");
// Adjust the test name to remove any 'DISABLED_' prefix (to make
// things easier to read). While this might seem alarming, if we are
// "running" a disabled test it must be the case that the test was
// explicitly enabled (e.g., via 'gtest_filter').
if (strings::startsWith(testName, "DISABLED_")) {
testName = strings::remove(testName, "DISABLED_", strings::PREFIX);
}
const string& path =
path::join("/tmp", strings::join("_", testCase, testName, "XXXXXX"));
Try<string> mkdtemp = os::mkdtemp(path);
if (mkdtemp.isSome()) {
directories.push_back(mkdtemp.get());
}
return mkdtemp;
}
/** * Creates a MasterInfo protobuf from the process's UPID. * * This is only used by the `StandaloneMasterDetector` (used in tests * and outside tests when ZK is not used). * * For example, when we start a slave with * `[email protected]:5050`, since the slave (and consequently * its detector) doesn't have enough information about `MasterInfo`, it * tries to construct it based on the only available information * (`UPID`). * * @param pid The process's assigned untyped PID. * @return A fully formed `MasterInfo` with the IP/hostname information * as derived from the `UPID`. */ MasterInfo createMasterInfo(const UPID& pid) { MasterInfo info; info.set_id(stringify(pid) + "-" + UUID::random().toString()); // NOTE: Currently, we store the ip in network order, which should // be fixed. See MESOS-1201 for more details. // TODO(marco): `ip` and `port` are deprecated in favor of `address`; // remove them both after the deprecation cycle. info.set_ip(pid.address.ip.in().get().s_addr); info.set_port(pid.address.port); info.mutable_address()->set_ip(stringify(pid.address.ip)); info.mutable_address()->set_port(pid.address.port); info.set_pid(pid); Try<string> hostname = net::getHostname(pid.address.ip); if (hostname.isSome()) { // Hostname is deprecated; but we need to update it // to maintain backward compatibility. // TODO(marco): Remove once we deprecate it. info.set_hostname(hostname.get()); info.mutable_address()->set_hostname(hostname.get()); } return info; }
Future<Response> FilesProcess::browse(const Request& request)
{
Option<string> path = request.url.query.get("path");
if (!path.isSome() || path.get().empty()) {
return BadRequest("Expecting 'path=value' in query.\n");
}
Result<string> resolvedPath = resolve(path.get());
if (resolvedPath.isError()) {
return InternalServerError(resolvedPath.error() + ".\n");
} else if (resolvedPath.isNone()) {
return NotFound();
}
// The result will be a sorted (on path) array of files and dirs:
// [{"name": "README", "path": "dir/README" "dir":False, "size":42}, ...]
map<string, JSON::Object> files;
Try<list<string> > entries = os::ls(resolvedPath.get());
if (entries.isSome()) {
foreach (const string& entry, entries.get()) {
struct stat s;
string fullPath = path::join(resolvedPath.get(), entry);
if (stat(fullPath.c_str(), &s) < 0) {
PLOG(WARNING) << "Found " << fullPath << " in ls but stat failed";
continue;
}
files[fullPath] = jsonFileInfo(path::join(path.get(), entry), s);
}
}
inline bool isdir(
const std::string& path,
const FollowSymlink follow = FollowSymlink::FOLLOW_SYMLINK)
{
Try<struct ::stat> s = internal::stat(path, follow);
return s.isSome() && S_ISDIR(s->st_mode);
}
inline bool islink(const std::string& path)
{
Try<internal::windows::SymbolicLink> symlink =
internal::windows::query_symbolic_link_data(path);
return symlink.isSome();
}
inline Try<long> mtime(const std::string& path)
{
Try<::internal::windows::SymbolicLink> symlink =
::internal::windows::query_symbolic_link_data(path);
if (symlink.isSome()) {
return Error(
"Requested mtime for '" + path +
"', but symbolic links don't have an mtime on Windows");
}
struct _stat s;
if (::_stat(path.c_str(), &s) < 0) {
return ErrnoError("Error invoking stat for '" + path + "'");
}
// To be safe, we assert that `st_mtime` is represented as `__int64`. To
// conform to the POSIX, we also cast `st_mtime` to `long`; we choose to make
// this conversion explicit because we expect the truncation to not cause
// information loss.
static_assert(
std::is_same<__int64, __time64_t>::value,
"Mesos assumes `__time64_t` is represented as `__int64`");
return static_cast<long>(s.st_mtime);
}
Future<double> _mem_free_bytes()
{
Try<os::Memory> memory = os::memory();
if (memory.isSome()) {
return memory.get().free.bytes();
}
return Failure("memory not available.");
}
Future<double> _cpus_total()
{
Try<long> cpus = os::cpus();
if (cpus.isSome()) {
return cpus.get();
}
return Failure("cpus not available.");
}
inline bool islink(const std::string& path)
{
// By definition, you don't followsymlinks when trying
// to find whether a path is a link. If you followed it,
// it wouldn't ever be a link.
Try<struct ::stat> s = internal::stat(path, DO_NOT_FOLLOW_SYMLINK);
return s.isSome() && S_ISLNK(s->st_mode);
}
Future<double> _load_15min()
{
Try<os::Load> load = os::loadavg();
if (load.isSome()) {
return load.get().fifteen;
}
return Failure("Failed to get loadavg: " + load.error());
}
// Gauge handlers.
Future<double> _load_1min()
{
Try<os::Load> load = os::loadavg();
if (load.isSome()) {
return load.get().one;
}
return Failure("loadavg not available.");
}
Future<double> _cpus_total()
{
Try<long> cpus = os::cpus();
if (cpus.isSome()) {
return cpus.get();
}
return Failure("Failed to get cpus: " + cpus.error());
}
Future<double> _mem_free_bytes()
{
Try<os::Memory> memory = os::memory();
if (memory.isSome()) {
return static_cast<double>(memory.get().free.bytes());
}
return Failure("Failed to get memory: " + memory.error());
}
static hashset<string> listfiles(const string& directory)
{
hashset<string> fileset;
Try<std::list<std::string> > entries = os::ls(directory);
if (entries.isSome()) {
foreach (const string& entry, entries.get()) {
fileset.insert(entry);
}
}
void FlagsBase::add(
T1* t1,
const Name& name,
const Option<Name>& alias,
const std::string& help,
const T2& t2,
F validate)
{
// Don't bother adding anything if the pointer is NULL.
if (t1 == NULL) {
return;
}
*t1 = t2; // Set the default.
Flag flag;
flag.name = name;
flag.alias = alias;
flag.help = help;
flag.boolean = typeid(T1) == typeid(bool);
// NOTE: We need to take FlagsBase* (or const FlagsBase&) as the
// first argument to match the function signature of the 'load',
// 'stringify', and 'validate' lambdas used in other overloads of
// FlagsBase::add. Since we don't need to use the pointer here we
// don't name it as a parameter.
flag.load = [t1](FlagsBase*, const std::string& value) -> Try<Nothing> {
// NOTE: 'fetch' "retrieves" the value if necessary and then
// invokes 'parse'. See 'fetch' for more details.
Try<T1> t = fetch<T1>(value);
if (t.isSome()) {
*t1 = t.get();
} else {
return Error("Failed to load value '" + value + "': " + t.error());
}
return Nothing();
};
flag.stringify = [t1](const FlagsBase&) -> Option<std::string> {
return stringify(*t1);
};
flag.validate = [t1, validate](const FlagsBase&) -> Option<Error> {
return validate(*t1);
};
// Update the help string to include the default value.
flag.help += help.size() > 0 && help.find_last_of("\n\r") != help.size() - 1
? " (default: " // On same line, add space.
: "(default: "; // On newline.
flag.help += stringify(t2);
flag.help += ")";
add(flag);
}
inline bool isJailed() {
int mib[4];
size_t len = 4;
::sysctlnametomib("security.jail.jailed", mib, &len);
Try<int> jailed = os::sysctl(mib[0], mib[1], mib[2]).integer();
if (jailed.isSome()) {
return jailed.get() == 1;
}
return false;
}
void usage(const char* argv0)
{
// Get a list of available commands.
const string& PATH = os::getenv("PATH");
list<string> commands;
foreach (const string& path, strings::split(PATH, ":")) {
Try<list<string> > matches = os::glob(path::join(path, "mesos-*"));
if (matches.isSome()) {
foreach (const string& match, matches.get()) {
Try<bool> access = os::access(match, X_OK);
if (access.isSome() && access.get()) {
Try<string> basename = os::basename(match);
if (basename.isSome()) {
commands.push_back(basename.get().substr(6));
}
}
}
}
}
Try<Nothing> ModuleManager::load(const Modules& modules)
{
Lock lock(&mutex);
initialize();
foreach (const Modules::Library& library, modules.libraries()) {
string libraryName;
if (library.has_file()) {
libraryName = library.file();
} else if (library.has_name()) {
libraryName = os::libraries::expandName(library.name());
} else {
return Error("Library name or path not provided");
}
if (!dynamicLibraries.contains(libraryName)) {
Owned<DynamicLibrary> dynamicLibrary(new DynamicLibrary());
Try<Nothing> result = dynamicLibrary->open(libraryName);
if (!result.isSome()) {
return Error("Error opening library: '" + libraryName + "'");
}
dynamicLibraries[libraryName] = dynamicLibrary;
}
// Load module manifests.
foreach (const string& moduleName, library.modules()) {
if (moduleName.empty()) {
return Error(
"Error: module name not provided with library '" +
library.file() + "'");
}
// Check for possible duplicate module names.
if (moduleBases.contains(moduleName)) {
return Error("Error loading duplicate module '" + moduleName + "'");
}
Try<void*> symbol = dynamicLibraries[libraryName]->loadSymbol(moduleName);
if (symbol.isError()) {
return Error(
"Error loading module '" + moduleName + "': " + symbol.error());
}
ModuleBase* moduleBase = (ModuleBase*) symbol.get();
Try<Nothing> result = verifyModule(moduleName, moduleBase);
if (result.isError()) {
return Error(
"Error verifying module '" + moduleName + "': " + result.error());
}
moduleBases[moduleName] = (ModuleBase*) symbol.get();
}
}
return Nothing();
}
// Returns a gzip decompressed version of the provided string.
inline Try<std::string> decompress(const std::string& compressed)
{
Decompressor decompressor;
Try<std::string> decompressed = decompressor.decompress(compressed);
// Ensure that the decompression stream does not expect more input.
if (decompressed.isSome() && !decompressor.finished()) {
return Error("More input is expected");
}
return decompressed;
}
void usage(const char* argv0)
{
// Get a list of available commands.
const Option<string> PATH = os::getenv("PATH");
list<string> commands;
if (PATH.isSome()) {
foreach (const string& path, strings::split(PATH.get(), ":")) {
Try<list<string> > matches = fs::list(path::join(path, "mesos-*"));
if (matches.isSome()) {
foreach (const string& match, matches.get()) {
Try<bool> access = os::access(match, X_OK);
if (access.isSome() && access.get()) {
string basename = Path(match).basename();
if (basename != "mesos-slave") {
commands.push_back(basename.substr(6));
}
}
}
}
}
Result<T> numify(const Option<std::string>& s)
{
if (s.isSome()) {
Try<T> t = numify<T>(s.get());
if (t.isSome()) {
return t.get();
} else if (t.isError()) {
return Error(t.error());
}
}
return None();
}
// TODO(jpeach): As we move namespace parameters from strings to CLONE
// constants, we should be able to eventually remove the internal uses
// of this function.
static set<string> namespaces()
{
set<string> result;
Try<std::list<string>> entries = os::ls("/proc/self/ns");
if (entries.isSome()) {
foreach (const string& entry, entries.get()) {
// Introduced in Linux 4.12, pid_for_children is a handle for the PID
// namespace of child processes created by the current process.
if (entry != "pid_for_children") {
result.insert(entry);
}
}
}
// HTTP endpoints.
Future<http::Response> stats(const http::Request& request)
{
JSON::Object object;
Try<os::Load> load = os::loadavg();
if (load.isSome()) {
object.values["avg_load_1min"] = load.get().one;
object.values["avg_load_5min"] = load.get().five;
object.values["avg_load_15min"] = load.get().fifteen;
}
Try<long> cpus = os::cpus();
if (cpus.isSome()) {
object.values["cpus_total"] = cpus.get();
}
Try<os::Memory> memory = os::memory();
if (memory.isSome()) {
object.values["mem_total_bytes"] = memory.get().total.bytes();
object.values["mem_free_bytes"] = memory.get().free.bytes();
}
return http::OK(object, request.query.get("jsonp"));
}
static Try<Nothing> load(
Option<T>* flag,
const std::tr1::function<Try<T>(const std::string&)>& parse,
const std::string& name,
const std::string& value)
{
Try<T> t = parse(value);
if (t.isSome()) {
*flag = Option<T>::some(t.get());
} else {
return Error("Failed to load value '" + value + "': " + t.error());
}
return Nothing();
}
MasterInfo createMasterInfo(const process::UPID& pid)
{
MasterInfo info;
info.set_id(stringify(pid) + "-" + UUID::random().toString());
info.set_ip(pid.address.ip);
info.set_port(pid.address.port);
info.set_pid(pid);
Try<string> hostname = net::getHostname(pid.address.ip);
if (hostname.isSome()) {
info.set_hostname(hostname.get());
}
return info;
}
void FlagsBase::add(
Option<T>* option,
const Name& name,
const Option<Name>& alias,
const std::string& help,
F validate)
{
// Don't bother adding anything if the pointer is NULL.
if (option == NULL) {
return;
}
Flag flag;
flag.name = name;
flag.alias = alias;
flag.help = help;
flag.boolean = typeid(T) == typeid(bool);
// NOTE: See comment above in T* overload of FlagsBase::add for why
// we need to take the FlagsBase* parameter.
flag.load = [option](FlagsBase*, const std::string& value) -> Try<Nothing> {
// NOTE: 'fetch' "retrieves" the value if necessary and then
// invokes 'parse'. See 'fetch' for more details.
Try<T> t = fetch<T>(value);
if (t.isSome()) {
*option = Some(t.get());
} else {
return Error("Failed to load value '" + value + "': " + t.error());
}
return Nothing();
};
flag.stringify = [option](const FlagsBase&) -> Option<std::string> {
if (option->isSome()) {
return stringify(option->get());
}
return None();
};
flag.validate = [option, validate](const FlagsBase&) -> Option<Error> {
return validate(*option);
};
add(flag);
}