Label createLabel(const string& key, const Option<string>& value) { Label label; label.set_key(key); if (value.isSome()) { label.set_value(value.get()); } return label; }
inline Try<Nothing> FlagsBase::load( const Option<std::string>& prefix, int argc, const char* const *argv, bool unknowns, bool duplicates) { std::map<std::string, Option<std::string>> values; // Grab the program name from argv[0]. programName_ = argc > 0 ? Path(argv[0]).basename() : ""; if (prefix.isSome()) { values = extract(prefix.get()); } // Read flags from the command line. for (int i = 1; i < argc; i++) { const std::string arg(strings::trim(argv[i])); // Stop parsing flags after '--' is encountered. if (arg == "--") { break; } // Skip anything that doesn't look like a flag. if (arg.find("--") != 0) { continue; } std::string name; Option<std::string> value = None(); size_t eq = arg.find_first_of("="); if (eq == std::string::npos && arg.find("--no-") == 0) { // --no-name name = arg.substr(2); } else if (eq == std::string::npos) { // --name name = arg.substr(2); } else { // --name=value name = arg.substr(2, eq - 2); value = arg.substr(eq + 1); } name = strings::lower(name); if (!duplicates) { if (values.count(name) > 0 || (name.find("no-") == 0 && values.count(name.substr(3)) > 0)) { return Error("Duplicate flag '" + name + "' on command line"); } } values[name] = value; } return load(values, unknowns); }
Future<http::Response> auth(const http::Request& request) { string encodedAuth = base64::encode("testuser:testpass"); Option<string> authHeader = request.headers.get("Authorization"); if (!authHeader.isSome() || (authHeader.get() != "Basic " + encodedAuth)) { return http::Unauthorized("testrealm"); } return http::OK(); }
Try<string> fetchWithLocalCopy( const string& uri, const string& directory, const Option<std::string>& frameworksHome) { string local = uri; bool fileUri = false; if (strings::startsWith(local, string(FILE_URI_LOCALHOST))) { local = local.substr(sizeof(FILE_URI_LOCALHOST) - 1); fileUri = true; } else if (strings::startsWith(local, string(FILE_URI_PREFIX))) { local = local.substr(sizeof(FILE_URI_PREFIX) - 1); fileUri = true; } if (fileUri && !strings::startsWith(local, "/")) { return Error("File URI only supports absolute paths"); } if (local.find_first_of("/") != 0) { // We got a non-Hadoop and non-absolute path. if (frameworksHome.isSome()) { local = path::join(frameworksHome.get(), local); LOG(INFO) << "Prepended the slave's frameworks_home flag value " << " to relative path, making it: '" << local << "'"; } else { LOG(ERROR) << "A relative path was passed for the resource but the " << "slave's frameworks_home flag is not set; " << "please either specify this slave configuration " << "option or avoid using a relative path"; return Error("Could not resolve relative URI"); } } Try<string> base = os::basename(local); if (base.isError()) { LOG(ERROR) << base.error(); return Error("Fetch of URI failed"); } // Copy the resource to the directory. string path = path::join(directory, base.get()); std::ostringstream command; command << "cp '" << local << "' '" << path << "'"; LOG(INFO) << "Copying resource from '" << local << "' to '" << directory << "'"; int status = os::system(command.str()); if (status != 0) { LOG(ERROR) << "Failed to copy '" << local << "' : Exit status " << status; return Error("Local copy failed"); } return path; }
Try<process::PID<slave::Slave> > MesosTest::StartSlave( slave::Containerizer* containerizer, MasterDetector* detector, const Option<slave::Flags>& flags) { return cluster.slaves.start( containerizer, detector, flags.isNone() ? CreateSlaveFlags() : flags.get()); }
Data(const std::string& _name, const Option<Duration>& window) : name(_name), lock(0), history(None()) { if (window.isSome()) { history = Owned<TimeSeries<double>>(new TimeSeries<double>(window.get())); } }
int main(int argc, char **argv) { GOOGLE_PROTOBUF_VERIFY_VERSION; // TODO(benh): Inherit from both slave::Flags and master::Flags in // order to pass those flags on to the master. Alternatively, add a // way to load flags and ignore unknowns in order to load // master::flags, then slave::Flags, then local::Flags. local::Flags flags; // The following flags are executable specific (e.g., since we only // have one instance of libprocess per execution, we only want to // advertise the port and ip option once, here). uint16_t port; flags.add(&port, "port", "Port to listen on", 5050); Option<string> ip; flags.add(&ip, "ip", "IP address to listen on"); bool help; flags.add(&help, "help", "Prints this help message", false); // Load flags from environment and command line but allow unknown // flags since we might have some master/slave flags as well. Try<Nothing> load = flags.load("MESOS_", argc, argv, true); if (load.isError()) { cerr << load.error() << endl; usage(argv[0], flags); exit(1); } if (help) { usage(argv[0], flags); exit(1); } // Initialize libprocess. os::setenv("LIBPROCESS_PORT", stringify(port)); if (ip.isSome()) { os::setenv("LIBPROCESS_IP", ip.get()); } process::initialize("master"); logging::initialize(argv[0], flags); process::wait(local::launch(flags)); return 0; }
inline Try<Warnings> FlagsBase::load( const Option<std::string>& prefix, int argc, const char* const *argv, bool unknowns, bool duplicates) { Multimap<std::string, Option<std::string>> values; // Grab the program name from argv[0]. programName_ = argc > 0 ? Path(argv[0]).basename() : ""; // Read flags from the command line. for (int i = 1; i < argc; i++) { const std::string arg(strings::trim(argv[i])); // Stop parsing flags after '--' is encountered. if (arg == "--") { break; } // Skip anything that doesn't look like a flag. if (arg.find("--") != 0) { continue; } std::string name; Option<std::string> value = None(); size_t eq = arg.find_first_of("="); if (eq == std::string::npos && arg.find("--no-") == 0) { // --no-name name = arg.substr(2); } else if (eq == std::string::npos) { // --name name = arg.substr(2); } else { // --name=value name = arg.substr(2, eq - 2); value = arg.substr(eq + 1); } name = strings::lower(name); values.put(name, value); } if (prefix.isSome()) { // Merge in flags from the environment. Command-line // flags take precedence over environment flags. foreachpair (const std::string& name, const Option<std::string>& value, extract(prefix.get())) { if (!values.contains(name)) { values.put(name, value); } } }
inline Try<pid_t> cloneChild( const string& path, vector<string> argv, const Setsid set_sid, const Option<map<string, string>>& environment, const Option<lambda::function< pid_t(const lambda::function<int()>&)>>& _clone, const vector<Subprocess::Hook>& parent_hooks, const Option<string>& working_directory, const Watchdog watchdog, const InputFileDescriptors stdinfds, const OutputFileDescriptors stdoutfds, const OutputFileDescriptors stderrfds) { // The real arguments that will be passed to 'os::execvpe'. We need // to construct them here before doing the clone as it might not be // async signal safe to perform the memory allocation. char** _argv = new char*[argv.size() + 1]; for (size_t i = 0; i < argv.size(); i++) { _argv[i] = (char*) argv[i].c_str(); } _argv[argv.size()] = nullptr; // Like above, we need to construct the environment that we'll pass // to 'os::execvpe' as it might not be async-safe to perform the // memory allocations. char** envp = os::raw::environment(); if (environment.isSome()) { // NOTE: We add 1 to the size for a `nullptr` terminator. envp = new char*[environment.get().size() + 1]; size_t index = 0; foreachpair (const string& key, const string& value, environment.get()) { string entry = key + "=" + value; envp[index] = new char[entry.size() + 1]; strncpy(envp[index], entry.c_str(), entry.size() + 1); ++index; } envp[index] = nullptr; }
Result<string> Fetcher::uriToLocalPath( const string& uri, const Option<string>& frameworksHome) { if (!strings::startsWith(uri, "file://") && strings::contains(uri, "://")) { return None(); } string path = uri; bool fileUri = false; if (strings::startsWith(path, FILE_URI_LOCALHOST)) { path = path.substr(FILE_URI_LOCALHOST.size()); fileUri = true; } else if (strings::startsWith(path, FILE_URI_PREFIX)) { path = path.substr(FILE_URI_PREFIX.size()); fileUri = true; } #ifndef __WINDOWS__ const bool isRelativePath = !strings::startsWith(path, "/"); if (isRelativePath) { if (fileUri) { return Error("File URI only supports absolute paths"); } if (frameworksHome.isNone() || frameworksHome.get().empty()) { return Error("A relative path was passed for the resource but the " "Mesos framework home was not specified. " "Please either provide this config option " "or avoid using a relative path"); } else { path = path::join(frameworksHome.get(), path); LOG(INFO) << "Prepended Mesos frameworks home to relative path, " << "making it: '" << path << "'"; } } #endif // __WINDOWS__ return path; }
Future<http::Response> Logging::toggle( const http::Request& request, const Option<http::authentication::Principal>&) { Option<std::string> level = request.url.query.get("level"); Option<std::string> duration = request.url.query.get("duration"); if (level.isNone() && duration.isNone()) { return http::OK(stringify(FLAGS_v) + "\n"); } if (level.isSome() && duration.isNone()) { return http::BadRequest("Expecting 'duration=value' in query.\n"); } else if (level.isNone() && duration.isSome()) { return http::BadRequest("Expecting 'level=value' in query.\n"); } Try<int> v = numify<int>(level.get()); if (v.isError()) { return http::BadRequest(v.error() + ".\n"); } if (v.get() < 0) { return http::BadRequest( "Invalid level '" + stringify(v.get()) + "'.\n"); } else if (v.get() < original) { return http::BadRequest( "'" + stringify(v.get()) + "' < original level.\n"); } Try<Duration> d = Duration::parse(duration.get()); if (d.isError()) { return http::BadRequest(d.error() + ".\n"); } return set_level(v.get(), d.get()) .then([]() -> http::Response { return http::OK(); }); }
TEST(ResourcesTest, SetSubtraction) { Resource disks1 = Resources::parse( "disks", "{sda1,sda2,sda3,sda4}", "*").get(); Resource disks2 = Resources::parse( "disks", "{sda2,sda3,sda4}", "*").get(); Resources r; r += disks1; r -= disks2; EXPECT_FALSE(r.empty()); Option<Value::Set> set = r.get<Value::Set>("disks"); ASSERT_SOME(set); EXPECT_EQ(1, set.get().item_size()); EXPECT_EQ("sda1", set.get().item(0)); }
inline Resource::ReservationInfo createReservationInfo( const Option<std::string>& principal = None()) { Resource::ReservationInfo info; if (principal.isSome()) { info.set_principal(principal.get()); } return info; }
inline Result<gid_t> getgid(const Option<std::string>& user = None()) { if (user.isNone()) { return ::getgid(); } struct passwd passwd; struct passwd* result = nullptr; int size = sysconf(_SC_GETPW_R_SIZE_MAX); if (size == -1) { // Initial value for buffer size. size = 1024; } while (true) { char* buffer = new char[size]; if (getpwnam_r(user.get().c_str(), &passwd, buffer, size, &result) == 0) { // The usual interpretation of POSIX is that getpwnam_r will // return 0 but set result == nullptr if the group is not found. if (result == nullptr) { delete[] buffer; return None(); } gid_t gid = passwd.pw_gid; delete[] buffer; return gid; } else { // RHEL7 (and possibly other systems) will return non-zero and // set one of the following errors for "The given name or uid // was not found." See 'man getpwnam_r'. We only check for the // errors explicitly listed, and do not consider the ellipsis. if (errno == ENOENT || errno == ESRCH || errno == EBADF || errno == EPERM) { delete[] buffer; return None(); } if (errno != ERANGE) { delete[] buffer; return ErrnoError("Failed to get username information"); } // getpwnam_r set ERANGE so try again with a larger buffer. size *= 2; delete[] buffer; } } UNREACHABLE(); }
virtual void resourceOffers(SchedulerDriver* driver, const vector<Offer>& offers) { foreach (const Offer& offer, offers) { cout << "Received offer " << offer.id() << " with " << offer.resources() << endl; static const Resources TASK_RESOURCES = Resources::parse( "cpus:" + stringify(CPUS_PER_TASK) + ";mem:" + stringify(MEM_PER_TASK)).get(); Resources remaining = offer.resources(); // Launch tasks. vector<TaskInfo> tasks; while (tasksLaunched < totalTasks && remaining.flatten().contains(TASK_RESOURCES)) { int taskId = tasksLaunched++; cout << "Launching task " << taskId << " using offer " << offer.id() << endl; TaskInfo task; task.set_name("Task " + lexical_cast<string>(taskId)); task.mutable_task_id()->set_value(lexical_cast<string>(taskId)); task.mutable_slave_id()->MergeFrom(offer.slave_id()); task.mutable_executor()->MergeFrom(executor); Try<Resources> flattened = TASK_RESOURCES.flatten(role); CHECK_SOME(flattened); Option<Resources> resources = remaining.find(flattened.get()); CHECK_SOME(resources); task.mutable_resources()->MergeFrom(resources.get()); remaining -= resources.get(); tasks.push_back(task); } driver->launchTasks(offer.id(), tasks); }
Try<PID<slave::Slave>> MesosTest::StartSlave( mesos::slave::ResourceEstimator* resourceEstimator, const Option<slave::Flags>& flags) { return cluster.slaves.start( flags.isNone() ? CreateSlaveFlags() : flags.get(), None(), None(), None(), None(), resourceEstimator); }
Resource::DiskInfo createDiskInfo( const Option<string>& persistenceID, const Option<string>& containerPath) { Resource::DiskInfo info; if (persistenceID.isSome()) { Resource::DiskInfo::Persistence persistence; persistence.set_id(persistenceID.get()); info.mutable_persistence()->CopyFrom(persistence); } if (containerPath.isSome()) { Volume volume; volume.set_container_path(containerPath.get()); volume.set_mode(Volume::RW); info.mutable_volume()->CopyFrom(volume); } return info; }
Try<Connection> MockCSIPlugin::startup(const Option<string>& address) { ServerBuilder builder; if (address.isSome()) { builder.AddListeningPort(address.get(), InsecureServerCredentials()); } builder.RegisterService(static_cast<Identity::Service*>(this)); builder.RegisterService(static_cast<Controller::Service*>(this)); builder.RegisterService(static_cast<Node::Service*>(this)); server = builder.BuildAndStart(); if (!server) { return Error("Unable to start a mock CSI plugin."); } return address.isSome() ? Connection(address.get()) : Connection(server->InProcessChannel(ChannelArguments())); }
Future<Nothing> tar( const Path& input, const Path& output, const Option<Path>& directory, const Option<Compression>& compression) { vector<string> argv = { "tar", "-c", // Create archive. "-f", // Output file. output }; // Add additional flags. if (directory.isSome()) { argv.emplace_back("-C"); argv.emplace_back(directory.get()); } if (compression.isSome()) { switch (compression.get()) { case Compression::GZIP: argv.emplace_back("-z"); break; case Compression::BZIP2: argv.emplace_back("-j"); break; case Compression::XZ: argv.emplace_back("-J"); break; default: UNREACHABLE(); } } argv.emplace_back(input); return launch("tar", argv) .then([] () {return Nothing();}); }
TEST(EnvTest, SimpleEnvTest) { const string key = "key"; const string value_1 = "value_1"; // Key currently does not exist in process environment. EXPECT_NONE(os::getenv(key)); // Set environment variable, check that it is there now. os::setenv(key, value_1, false); Option<string> result = os::getenv(key); ASSERT_SOME(result); EXPECT_EQ(value_1, result.get()); // Verify that if we set the environment variable without the `overwrite` // flag set, the value of that variable was not updated. const string value_2 = "value_2"; os::setenv(key, value_2, false); result = os::getenv(key); ASSERT_SOME(result); EXPECT_EQ(value_1, result.get()); // Now set environment variable, and set `overwrite` flag. os::setenv(key, value_2, true); result = os::getenv(key); ASSERT_SOME(result); EXPECT_EQ(value_2, result.get()); // Now verify that the default behavior sets the `overwrite` flag to true. const string env_value_3 = "even_newer_favorite_value"; os::setenv(key, env_value_3); result = os::getenv(key); ASSERT_SOME(result); EXPECT_EQ(env_value_3, result.get()); // Finally, unset the variable. os::unsetenv(key); result = os::getenv(key); EXPECT_NONE(result); }
// Returns the resulting file or in case of extraction the destination // directory (for logging). static Try<string> fetchThroughCache( const FetcherInfo::Item& item, const Option<string>& cacheDirectory, const string& sandboxDirectory, const Option<string>& frameworksHome) { if (cacheDirectory.isNone() || cacheDirectory.get().empty()) { return Error("Cache directory not specified"); } if (!item.has_cache_filename() || item.cache_filename().empty()) { // This should never happen if this program is used by the Mesos // slave and could then be a CHECK. But other uses are possible. return Error("No cache file name for: " + item.uri().value()); } CHECK_NE(FetcherInfo::Item::BYPASS_CACHE, item.action()) << "Unexpected fetcher action selector"; if (item.action() == FetcherInfo::Item::DOWNLOAD_AND_CACHE) { LOG(INFO) << "Downloading into cache"; Try<Nothing> mkdir = os::mkdir(cacheDirectory.get()); if (mkdir.isError()) { return Error("Failed to create fetcher cache directory '" + cacheDirectory.get() + "': " + mkdir.error()); } Try<string> downloaded = download( item.uri().value(), path::join(cacheDirectory.get(), item.cache_filename()), frameworksHome); if (downloaded.isError()) { return Error(downloaded.error()); } } return fetchFromCache(item, cacheDirectory.get(), sandboxDirectory); }
// For use with a MockScheduler, for example: // EXPECT_CALL(sched, resourceOffers(_, _)) // .WillOnce(LaunchTasks(TASKS, CPUS, MEM)); // Launches up to TASKS no-op tasks, if possible, // each with CPUS cpus and MEM memory. ACTION_P4(LaunchTasks, tasks, cpus, mem, role) { SchedulerDriver* driver = arg0; std::vector<Offer> offers = arg1; int numTasks = tasks; int launched = 0; for (size_t i = 0; i < offers.size(); i++) { const Offer& offer = offers[i]; const Resources TASK_RESOURCES = Resources::parse( "cpus:" + stringify(cpus) + ";mem:" + stringify(mem)).get(); int nextTaskId = 0; std::vector<TaskInfo> tasks; Resources remaining = offer.resources(); while (TASK_RESOURCES <= remaining.flatten() && launched < numTasks) { TaskInfo task; task.set_name("TestTask"); task.mutable_task_id()->set_value(stringify(nextTaskId++)); task.mutable_slave_id()->MergeFrom(offer.slave_id()); ExecutorInfo executor; executor.mutable_executor_id()->set_value("default"); executor.mutable_command()->set_value(":"); task.mutable_executor()->MergeFrom(executor); Option<Resources> resources = remaining.find(TASK_RESOURCES, role); CHECK_SOME(resources); task.mutable_resources()->MergeFrom(resources.get()); remaining -= resources.get(); tasks.push_back(task); launched++; } driver->launchTasks(offer.id(), tasks); } }
string createExecutorDirectory( const string& rootDir, const SlaveID& slaveId, const FrameworkID& frameworkId, const ExecutorID& executorId, const ContainerID& containerId, const Option<string>& user) { const string directory = getExecutorRunPath(rootDir, slaveId, frameworkId, executorId, containerId); Try<Nothing> mkdir = os::mkdir(directory); CHECK_SOME(mkdir) << "Failed to create executor directory '" << directory << "'"; // Remove the previous "latest" symlink. const string latest = getExecutorLatestRunPath(rootDir, slaveId, frameworkId, executorId); if (os::exists(latest)) { CHECK_SOME(os::rm(latest)) << "Failed to remove latest symlink '" << latest << "'"; } // Symlink the new executor directory to "latest". Try<Nothing> symlink = ::fs::symlink(directory, latest); CHECK_SOME(symlink) << "Failed to symlink directory '" << directory << "' to '" << latest << "'"; if (user.isSome()) { // Per MESOS-2592, we need to set the ownership of the executor // directory during its creation. We should not rely on subsequent // phases of the executor creation to ensure the ownership as // those may be conditional and in some cases leave the executor // directory owned by the slave user instead of the specified // framework or per-executor user. Try<Nothing> chown = os::chown(user.get(), directory); if (chown.isError()) { // TODO(nnielsen): We currently have tests which depend on using // user names which may not be available on the test machines. // Therefore, we cannot make the chown validation a hard // CHECK(). LOG(WARNING) << "Failed to chown executor directory '" << directory << "': " << chown.error(); } } return directory; }
void offers(const vector<Offer>& offers) { CHECK_EQ(SUBSCRIBED, state); static const Try<Resources> TASK_RESOURCES = Resources::parse(resources); if (TASK_RESOURCES.isError()) { EXIT(EXIT_FAILURE) << "Failed to parse resources '" << resources << "': " << TASK_RESOURCES.error(); } foreach (const Offer& offer, offers) { Resources offered = offer.resources(); if (!launched && offered.flatten().contains(TASK_RESOURCES.get())) { TaskInfo task; task.set_name(name); task.mutable_task_id()->set_value(name); task.mutable_agent_id()->MergeFrom(offer.agent_id()); // Takes resources first from the specified role, then from '*'. Option<Resources> resources = offered.find(TASK_RESOURCES.get().flatten(frameworkInfo.role())); CHECK_SOME(resources); task.mutable_resources()->CopyFrom(resources.get()); CommandInfo* commandInfo = task.mutable_command(); if (shell) { CHECK_SOME(command); commandInfo->set_shell(true); commandInfo->set_value(command.get()); } else { // TODO(gilbert): Treat 'command' as executable value and arguments. commandInfo->set_shell(false); } if (environment.isSome()) { Environment* environment_ = commandInfo->mutable_environment(); foreachpair ( const string& name, const string& value, environment.get()) { Environment::Variable* environmentVariable = environment_->add_variables(); environmentVariable->set_name(name); environmentVariable->set_value(value); } }
void executorFailed( const ExecutorID& executorId, const AgentID& _agentId, const Option<int>& status) { CHECK_EQ(SUBSCRIBED, state); LOG(INFO) << "Executor '" << executorId << "' lost on agent '" << _agentId << (status.isSome() ? "' with status: " + stringify(status.get()) : ""); agentId = None(); }
Future<string> SocketImpl::recv(const Option<ssize_t>& size) { // Default chunk size to attempt to receive when nothing is // specified represents roughly 16 pages. static const size_t DEFAULT_CHUNK = 16 * os::pagesize(); size_t chunk = (size.isNone() || size.get() < 0) ? DEFAULT_CHUNK : size.get(); Owned<string> buffer(new string()); boost::shared_array<char> data(new char[chunk]); return recv(data.get(), chunk) .then(lambda::bind(&_recv, shared_from_this(), size, buffer, chunk, data, lambda::_1)); }
Try<PID<slave::Slave>> MesosTest::StartSlave( mesos::slave::QoSController* qoSController, const Option<slave::Flags>& flags) { return cluster.slaves.start( flags.isNone() ? CreateSlaveFlags() : flags.get(), None(), None(), None(), None(), None(), qoSController); }
UpdateOperationStatusMessage createUpdateOperationStatusMessage( const id::UUID& operationUUID, const OperationStatus& status, const Option<OperationStatus>& latestStatus, const Option<FrameworkID>& frameworkId, const Option<SlaveID>& slaveId) { UpdateOperationStatusMessage update; if (frameworkId.isSome()) { update.mutable_framework_id()->CopyFrom(frameworkId.get()); } if (slaveId.isSome()) { update.mutable_slave_id()->CopyFrom(slaveId.get()); } update.mutable_status()->CopyFrom(status); if (latestStatus.isSome()) { update.mutable_latest_status()->CopyFrom(latestStatus.get()); } update.mutable_operation_uuid()->set_value(operationUUID.toBytes()); return update; }
Try<Nothing> mount(const Option<string>& source, const string& target, const Option<string>& type, unsigned long flags, const Option<string>& options) { return mount( source, target, type, flags, options.isSome() ? options.get().c_str() : nullptr); }
process::Future<std::vector<std::string>> unmocked_get(const Image& image) { if (!image.has_appc()) { return process::Failure("Expecting APPC image"); } Option<process::Shared<Rootfs>> rootfs = rootfses.at(image.appc().name()); if (rootfs.isSome()) { return std::vector<std::string>({rootfs.get()->root}); } return process::Failure("Cannot find image '" + image.appc().name()); }