TEST_F(FetcherEnvironmentTest, EmptyHadoop)
{
CommandInfo commandInfo;
CommandInfo::URI* uri = commandInfo.add_uris();
uri->set_value("hdfs:///uri");
uri->set_executable(false);
string directory = "/tmp/directory";
Option<string> user = "******";
slave::Flags flags;
flags.frameworks_home = "/tmp/frameworks";
flags.hadoop_home = "";
map<string, string> environment =
fetcher::environment(commandInfo, directory, user, flags);
EXPECT_EQ(4u, environment.size());
EXPECT_EQ(stringify(JSON::Protobuf(commandInfo)),
environment["MESOS_COMMAND_INFO"]);
EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
EXPECT_EQ(user.get(), environment["MESOS_USER"]);
EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
}
// For use with a MockScheduler, for example:
// EXPECT_CALL(sched, resourceOffers(_, _))
// .WillOnce(LaunchTasks(EXECUTOR, TASKS, CPUS, MEM, ROLE));
// Launches up to TASKS no-op tasks, if possible,
// each with CPUS cpus and MEM memory and EXECUTOR executor.
ACTION_P5(LaunchTasks, executor, tasks, cpus, mem, role)
{
SchedulerDriver* driver = arg0;
std::vector<Offer> offers = arg1;
int numTasks = tasks;
int launched = 0;
for (size_t i = 0; i < offers.size(); i++) {
const Offer& offer = offers[i];
const Resources TASK_RESOURCES = Resources::parse(
"cpus:" + stringify(cpus) + ";mem:" + stringify(mem)).get();
int nextTaskId = 0;
std::vector<TaskInfo> tasks;
Resources remaining = offer.resources();
while (TASK_RESOURCES <= remaining.flatten() && launched < numTasks) {
TaskInfo task;
task.set_name("TestTask");
task.mutable_task_id()->set_value(stringify(nextTaskId++));
task.mutable_slave_id()->MergeFrom(offer.slave_id());
task.mutable_executor()->MergeFrom(executor);
Option<Resources> resources = remaining.find(TASK_RESOURCES, role);
CHECK_SOME(resources);
task.mutable_resources()->MergeFrom(resources.get());
remaining -= resources.get();
tasks.push_back(task);
launched++;
}
driver->launchTasks(offer.id(), tasks);
}
}
// parser<bool> implementation
//
bool parser<bool>::parse(Option &O, StringRef ArgName,
StringRef Arg, bool &Value) {
if (Arg == "" || Arg == "true" || Arg == "TRUE" || Arg == "True" ||
Arg == "1") {
Value = true;
return false;
}
if (Arg == "false" || Arg == "FALSE" || Arg == "False" || Arg == "0") {
Value = false;
return false;
}
return O.error("'" + Arg +
"' is invalid value for boolean argument! Try 0 or 1");
}
Try<PID<slave::Slave>> MesosTest::StartSlave(
MockExecutor* executor,
mesos::slave::ResourceEstimator* resourceEstimator,
const Option<slave::Flags>& flags)
{
slave::Containerizer* containerizer = new TestContainerizer(executor);
Try<PID<slave::Slave>> pid = cluster.slaves.start(
flags.isNone() ? CreateSlaveFlags() : flags.get(),
containerizer,
None(),
None(),
None(),
resourceEstimator);
if (pid.isError()) {
delete containerizer;
return pid;
}
containerizers[pid.get()] = containerizer;
return pid;
}
void Options::init(const Json::Value &array)
{
for (Json::ValueIterator itr = array.begin(); itr != array.end(); itr++)
{
Json::Value value = (*itr);
Json::Value &name = value["name"];
Json::Value &type = value["type"];
string typeString = type.asString();
Option *option = createOption(typeString);
if (option != NULL)
{
option->init(value);
add(name.asString(), option);
}
else
{
cout << "Unknown data type: " << typeString << "\n";
cout.flush();
}
}
}
Try<Owned<HealthChecker>> HealthChecker::create(
const HealthCheck& check,
const string& launcherDir,
const UPID& executor,
const TaskID& taskID,
Option<pid_t> taskPid,
const vector<string>& namespaces)
{
// Validate the 'HealthCheck' protobuf.
Option<Error> error = validation::healthCheck(check);
if (error.isSome()) {
return error.get();
}
Owned<HealthCheckerProcess> process(new HealthCheckerProcess(
check,
launcherDir,
executor,
taskID,
taskPid,
namespaces));
return Owned<HealthChecker>(new HealthChecker(process));
}
// parser<boolOrDefault> implementation
//
bool parser<boolOrDefault>::parse(Option &O, const string& ArgName,
const string& Arg, boolOrDefault &Value) {
if (Arg == "" || Arg == "true" || Arg == "TRUE" || Arg == "True" ||
Arg == "1") {
Value = BOU_TRUE;
return false;
}
if (Arg == "false" || Arg == "FALSE" || Arg == "False" || Arg == "0") {
Value = BOU_FALSE;
return false;
}
return O.error("'" + Arg +
"' is invalid value for boolean argument! Try 0 or 1");
}
Future<bool> Master::QuotaHandler::authorizeSetQuota(
const Option<string>& principal,
const string& role) const
{
if (master->authorizer.isNone()) {
return true;
}
LOG(INFO) << "Authorizing principal '"
<< (principal.isSome() ? principal.get() : "ANY")
<< "' to request quota for role '" << role << "'";
mesos::ACL::SetQuota request;
if (principal.isSome()) {
request.mutable_principals()->add_values(principal.get());
} else {
request.mutable_principals()->set_type(mesos::ACL::Entity::ANY);
}
request.mutable_roles()->add_values(role);
return master->authorizer.get()->authorize(request);
}
TEST_F(MesosContainerizerProcessTest, Simple)
{
CommandInfo commandInfo;
CommandInfo::URI uri;
uri.set_value("hdfs:///uri");
uri.set_executable(false);
commandInfo.add_uris()->MergeFrom(uri);
string directory = "/tmp/directory";
Option<string> user = "******";
Flags flags;
flags.frameworks_home = "/tmp/frameworks";
flags.hadoop_home = "/tmp/hadoop";
map<string, string> environment =
fetcherEnvironment(commandInfo, directory, user, flags);
EXPECT_EQ(5u, environment.size());
EXPECT_EQ("hdfs:///uri+0X", environment["MESOS_EXECUTOR_URIS"]);
EXPECT_EQ(directory, environment["MESOS_WORK_DIRECTORY"]);
EXPECT_EQ(user.get(), environment["MESOS_USER"]);
EXPECT_EQ(flags.frameworks_home, environment["MESOS_FRAMEWORKS_HOME"]);
EXPECT_EQ(flags.hadoop_home, environment["HADOOP_HOME"]);
}
OK(const JSON::Value& value, const Option<std::string>& jsonp = None())
{
type = BODY;
status = "200 OK";
std::ostringstream out;
if (jsonp.isSome()) {
out << jsonp.get() << "(";
}
out << value;
if (jsonp.isSome()) {
out << ");";
headers["Content-Type"] = "text/javascript";
} else {
headers["Content-Type"] = "application/json";
}
headers["Content-Length"] = stringify(out.str().size());
body = out.str().data();
}
int ListAdjacent::ProcessOptions(int argc, char** argv) {
Option option;
int argc_old=argc;
while (*argv[0]=='-' && argc>1) {
option.reset();
switch (argv[0][1]) {
default:
EchoError(string("unknown option ")+argv[0]);
} // switch
if(!option.option.empty())
addToSwitches(option);
argc--;
argv++;
} // while
return argc_old-argc;
}
// TODO(bmahler): Refactor this to make the distinction between
// command tasks and executor tasks clearer.
inline TaskInfo createTask(
const SlaveID& slaveId,
const Resources& resources,
const std::string& command,
const Option<mesos::ExecutorID>& executorId = None(),
const std::string& name = "test-task",
const std::string& id = UUID::random().toString())
{
TaskInfo task;
task.set_name(name);
task.mutable_task_id()->set_value(id);
task.mutable_slave_id()->CopyFrom(slaveId);
task.mutable_resources()->CopyFrom(resources);
if (executorId.isSome()) {
ExecutorInfo executor;
executor.mutable_executor_id()->CopyFrom(executorId.get());
executor.mutable_command()->set_value(command);
task.mutable_executor()->CopyFrom(executor);
} else {
task.mutable_command()->set_value(command);
}
return task;
}
inline Result<std::string> cmdline(const Option<pid_t>& pid = None())
{
const std::string path = pid.isSome()
? "/proc/" + stringify(pid.get()) + "/cmdline"
: "/proc/cmdline";
std::ifstream file(path.c_str());
if (!file.is_open()) {
// Need to check if file exists AFTER we open it to guarantee
// process hasn't terminated (or if it has, we at least have a
// file which the kernel _should_ respect until a close).
if (!os::exists(path)) {
return None();
}
return Error("Failed to open '" + path + "'");
}
std::stringbuf buffer;
do {
// Read each argument in "argv", separated by null bytes.
file.get(buffer, '\0');
// Check for any read errors.
if (file.fail() && !file.eof()) {
file.close();
return Error("Failed to read '" + path + "'");
} else if (!file.eof()) {
file.get(); // Read the null byte.
buffer.sputc(' '); // Put a space between each command line argument.
}
} while (!file.eof());
return buffer.str();
}
void PawnShopChapter::Do(Character& character) const
{
//Must be a copy, as the copy can get cleared
std::vector<std::pair<Item,int>> items = this->GetItems();
while (1)
{
std::vector<Option> options;
options.push_back(CreateLeaveOption());
for (const auto& item: items)
{
if (!character.HasItem(item.first)) continue;
std::stringstream text;
text
<< "Sell "
<< ToPrettyStr(item.first) << " for "
<< item.second << " gold pieces"
;
Consequence consequence;
consequence.AddItemToRemove(item.first);
consequence.SetChangeGold(item.second);
Option option(text.str(),consequence);
options.push_back(option);
}
//Pawn shop
const Option chosen{m_chapter->RequestOption(options)};
if (chosen.GetConsequence().GetType() == ConsequenceType::leave) { break; }
assert(!chosen.GetConsequence().GetItemsToRemove().empty());
const Item item_sold{chosen.GetConsequence().GetItemsToRemove()[0]};
m_chapter->ShowText("You sold " + ToPrettyStr(item_sold) + "\n");
chosen.GetConsequence().Apply(character);
}
}
Expression toMicroSt::newDiscrete(Option<Expression> oe) const {
char buff[1024];
sprintf(buff,"d%d",disc_count++);
Name name(buff);
VarSymbolTable &syms=mmo_class.syms_ref();
if (oe) {
ClassModification cm;
cm.push_back(ElMod("start",ModEq(oe.get())));
ModClass mc(cm);
Modification m = mc;
syms.insert(name,VarInfo(TypePrefixes(1,discrete) , "Real", Option<Comment>(), m));
} else
syms.insert(name,VarInfo(TypePrefixes(1,discrete) , "Real"));
mmo_class.variables_ref().push_back(name);
return Reference(Ref(1,RefTuple(name,ExpList())));
}
TEST(OptionsTest, test_option_writing)
{
const Option option_i("my_int", (uint16_t)17);
EXPECT_TRUE(option_i.getName() == "my_int");
EXPECT_TRUE(option_i.getValue() == "17");
const Option option_s("my_string", "Yow.");
EXPECT_TRUE(option_s.getName() == "my_string");
EXPECT_TRUE(option_s.getValue() == "Yow.");
EXPECT_TRUE(option_s.getValue() == "Yow.");
}
void CLIOptions::printHelp(QTextStream& os) const
{
os << applicationName() << " [OPTIONS]" << endl << endl;
os << "OPTIONS:" << endl << endl;
QMapIterator<QString, Option> it(m_options);
while(it.hasNext()) {
it.next();
Option opt = it.value();
os << opt.names().join(", ");
if(opt.type() != QVariant::Bool) {
if(opt.type() == QVariant::Int || opt.type() == QVariant::Double) os << " " << "number";
else os << " " << "'string'";
}
os << ':';
QVariant def_val = opt.defaultValue();
if(def_val.isValid()) os << " [default(" << def_val.toString() << ")]";
if(opt.isMandatory()) os << " [MANDATORY]";
os << endl;
QString oc = opt.comment();
if(!oc.isEmpty()) os << "\t" << opt.comment() << endl;
}
}
int main(int argc, char** argv)
{
if (argc != 2) {
cerr << "Usage: " << argv[0] << " <master>" << endl;
return -1;
}
DockerNoExecutorScheduler scheduler;
FrameworkInfo framework;
framework.set_user(""); // Have Mesos fill in the current user.
framework.set_name("Docker No Executor Framework (C++)");
framework.set_checkpoint(true);
MesosSchedulerDriver* driver;
if (os::getenv("MESOS_AUTHENTICATE_FRAMEWORKS").isSome()) {
cout << "Enabling authentication for the framework" << endl;
Option<string> value = os::getenv("DEFAULT_PRINCIPAL");
if (value.isNone()) {
EXIT(EXIT_FAILURE)
<< "Expecting authentication principal in the environment";
}
Credential credential;
credential.set_principal(value.get());
framework.set_principal(value.get());
value = os::getenv("DEFAULT_SECRET");
if (value.isNone()) {
EXIT(EXIT_FAILURE)
<< "Expecting authentication secret in the environment";
}
credential.set_secret(value.get());
driver = new MesosSchedulerDriver(
&scheduler, framework, argv[1], credential);
} else {
framework.set_principal("no-executor-framework-cpp");
driver = new MesosSchedulerDriver(
&scheduler, framework, argv[1]);
}
int status = driver->run() == DRIVER_STOPPED ? 0 : 1;
// Ensure that the driver process terminates.
driver->stop();
delete driver;
return status;
}
// The return type matches the `clone` function parameter in
// `process::subprocess`.
static Option<lambda::function<pid_t(const lambda::function<int()>&)>>
getCustomCloneFunc(const Option<runtime::Plain>& plain)
{
if (plain.isNone() || plain->namespaces.empty()) {
return None();
}
#ifdef __linux__
return lambda::bind(
&cloneWithSetns,
lambda::_1,
plain->taskPid,
plain->namespaces);
#else
return None();
#endif // __linux__
}
GroupProcess::GroupProcess(
const string& _servers,
const Duration& _timeout,
const string& _znode,
const Option<Authentication>& _auth)
: servers(_servers),
timeout(_timeout),
znode(strings::remove(_znode, "/", strings::SUFFIX)),
auth(_auth),
acl(_auth.isSome()
? EVERYONE_READ_CREATOR_ALL
: ZOO_OPEN_ACL_UNSAFE),
watcher(NULL),
zk(NULL),
state(DISCONNECTED),
retrying(false)
{}
Future<Nothing> MesosContainerizerProcess::recover(
const Option<state::SlaveState>& state)
{
LOG(INFO) << "Recovering containerizer";
// Gather the executor run states that we will attempt to recover.
list<RunState> recoverable;
if (state.isSome()) {
foreachvalue (const FrameworkState& framework, state.get().frameworks) {
foreachvalue (const ExecutorState& executor, framework.executors) {
if (executor.info.isNone()) {
LOG(WARNING) << "Skipping recovery of executor '" << executor.id
<< "' of framework " << framework.id
<< " because its info could not be recovered";
continue;
}
if (executor.latest.isNone()) {
LOG(WARNING) << "Skipping recovery of executor '" << executor.id
<< "' of framework " << framework.id
<< " because its latest run could not be recovered";
continue;
}
// We are only interested in the latest run of the executor!
const ContainerID& containerId = executor.latest.get();
Option<RunState> run = executor.runs.get(containerId);
CHECK_SOME(run);
// We need the pid so the reaper can monitor the executor so skip this
// executor if it's not present. This is not an error because the slave
// will try to wait on the container which will return a failed
// Termination and everything will get cleaned up.
if (!run.get().forkedPid.isSome()) {
continue;
}
if (run.get().completed) {
VLOG(1) << "Skipping recovery of executor '" << executor.id
<< "' of framework " << framework.id
<< " because its latest run "
<< containerId << " is completed";
continue;
}
LOG(INFO) << "Recovering container '" << containerId
<< "' for executor '" << executor.id
<< "' of framework " << framework.id;
recoverable.push_back(run.get());
}
}
}
// Creates a null-terminated array of null-terminated strings that will be
// passed to `CreateProcess` as the `lpEnvironment` argument, as described by
// MSDN[1]. This array needs to be sorted in alphabetical order, but the `map`
// already takes care of that. Note that this function does not handle Unicode
// environments, so it should not be used in conjunction with the
// `CREATE_UNICODE_ENVIRONMENT` flag.
//
// [1] https://msdn.microsoft.com/en-us/library/windows/desktop/ms682425(v=vs.85).aspx
inline Option<string> createProcessEnvironment(
const Option<map<string, string>>& env)
{
if (env.isNone() || (env.isSome() && env.get().size() == 0)) {
return None();
}
Option<map<string, string>> cuEnv = getCUEnvironment();
if (cuEnv.isNone() || (cuEnv.isSome() && cuEnv.get().size() == 0)) {
return None();
}
map<string, string> combEnv = env.get();
foreachpair(const string& key, const string& value, cuEnv.get()) {
combEnv[key] = value;
}
string environmentString;
foreachpair (const string& key, const string& value, combEnv) {
environmentString += key + '=' + value + '\0';
}
GroupProcess::GroupProcess(
const string& _servers,
const Duration& _sessionTimeout,
const string& _znode,
const Option<Authentication>& _auth)
: ProcessBase(ID::generate("zookeeper-group")),
servers(_servers),
sessionTimeout(_sessionTimeout),
znode(strings::remove(_znode, "/", strings::SUFFIX)),
auth(_auth),
acl(_auth.isSome()
? EVERYONE_READ_CREATOR_ALL
: ZOO_OPEN_ACL_UNSAFE),
watcher(nullptr),
zk(nullptr),
state(DISCONNECTED),
retrying(false)
{}
/*!
* Creates and adds a button to the menu.
* \param x X coordinate
* \param y Y coordinate
* \param width Button width
* \param height Button height
* \param text Button label. If text starts with a '#' then
* text is a property in the current language file and it is
* replaced by its value.
* \param size Font size
* \param to Id of the next menu when button is clicked
* \param visible True if button is visible on screen
* \param centered True if text must centered regarding button width
* \param dark_widget Widget drawn in front of the button when it's not highlighted
* \param light_widget Widget drawn in front of the button when it's highlighted
*/
int Menu::addOption(int x, int y, int width, int height, const char *text, FontManager::EFontSize size,
int to, bool visible, bool centered, int dark_widget, int light_widget) {
Option *pOption = new Option(this, x, y, width, height, text, getMenuFont(size), to, visible, centered, dark_widget, light_widget);
actions_.push_back(pOption);
if (pOption->getHotKey().keyFunc != KFC_UNKNOWN || pOption->getHotKey().unicode != 0) {
// The option already has an acceleration key
registerHotKey(pOption->getHotKey().unicode, pOption->getId());
}
return pOption->getId();
}
Option<Error> validate(const RepeatedPtrField<Resource>& resources)
{
Option<Error> error = Resources::validate(resources);
if (error.isSome()) {
return Error("Invalid resources: " + error.get().message);
}
error = validateGpus(resources);
if (error.isSome()) {
return Error("Invalid 'gpus' resource: " + error.get().message);
}
error = validateDiskInfo(resources);
if (error.isSome()) {
return Error("Invalid DiskInfo: " + error.get().message);
}
error = validateDynamicReservationInfo(resources);
if (error.isSome()) {
return Error("Invalid ReservationInfo: " + error.get().message);
}
return None();
}
void InputDialog::initializeControl(Option const& opt, QDoubleSpinBox* dspin)
{
connectControl(opt, dspin);
dspin->setToolTip(prependRemName(opt.getName(), opt.getDescription()));
dspin->setRange(opt.doubleMin(), opt.doubleMax());
dspin->setSingleStep(opt.doubleStep());
Action* action = new Action(
boost::bind(&QDoubleSpinBox::setValue, dspin, opt.doubleDefault()) );
m_resetActions.push_back(action);
Update* update = new Update(
boost::bind(
static_cast<void(*)(QDoubleSpinBox*, QString const&)>(SetControl), dspin, _1));
QString name = opt.getName();
m_setUpdates[name] = update;
}
// Returns the time series within the (optional) time range.
std::vector<Value> get(
const Option<Time>& start = None(),
const Option<Time>& stop = None()) const
{
// Ignore invalid ranges.
if (start.isSome() && stop.isSome() && start.get() > stop.get()) {
return std::vector<Value>();
}
typename std::map<Time, T>::const_iterator lower = values.lower_bound(
start.isSome() ? start.get() : Time::EPOCH);
typename std::map<Time, T>::const_iterator upper = values.upper_bound(
stop.isSome() ? stop.get() : Time::MAX);
std::vector<Value> values;
while (lower != upper) {
values.push_back(Value(lower->first, lower->second));
++lower;
}
return values;
}
void Init(int64 wnd) {
hwnd = wnd;
int row = 0;
//SetData("-- Regular styles --");
Ready(false);
StaticText *st;
Add(0, 0, 0);
st = new StaticText;
*st= "-- click me --";
*st <<= THISBACK(Action);
GetList().SetCtrl(0, 0, st);
++row;
Add(0, 0, 0);
st = new StaticText;
*st = "-- Regular styles --";
GetList().SetCtrl(1, 0, st);
++row;
Option *po;
for(int i = 0; i < sizeof(styles)/sizeof(styles[0]); ++i, row++) {
po = new Option;
po->Set((stylesbits[i]&GetWindowLong((HWND)hwnd, GWL_STYLE) ? true : false));
po->SetLabel(styles[i]);
*po <<= THISBACK(Action);
Add(po->Get(), ::Format("0x%08x", (int64)stylesbits[i]), *(int64*)&stylesbits[i]);
GetList().SetCtrl(row, 0, po);
}
Add(0, 0);
st = new StaticText;
*st = "** Extendes styles **";
GetList().SetCtrl(row, 0, st);
++row;
for(int i = 0; i < sizeof(exstyles)/sizeof(exstyles[0]); row++, ++i){
po = new Option;
po->Set((stylesbits[i]&GetWindowLong((HWND)hwnd,GWL_EXSTYLE)) ? true : false);
po->SetLabel(exstyles[i]);
*po <<= THISBACK(Action);
Add(po->Get(), ::Format("0x%08x", (int64)exstylesbits[i]), *(int64*)&exstylesbits[i]);
GetList().SetCtrl(row, 0, po);
}
Ready(true);
/*GetList().SetCursor(0);
SetIndex(0);
SetData("-- click me --");*/
}
volume<short> make_mask_from_mesh(const volume<short> & image, const Mesh& m)
{
double xdim = (double) image.xdim();
double ydim = (double) image.ydim();
double zdim = (double) image.zdim();
volume<short> mask = image;
int xsize = mask.xsize();
int ysize = mask.ysize();
int zsize = mask.zsize();
vector<Pt> current;
current.clear();
Pt c(0., 0., 0.);
for (vector<Mpoint *>::const_iterator it=m._points.begin(); it!=m._points.end(); it++)
c+=(*it)->get_coord();
c*=(1./m._points.size());
c.X/=xdim; c.Y/=ydim; c.Z/=zdim;
current.push_back(c);
while (!current.empty())
{
Pt pc = current.back();
int x, y, z;
x=(int) pc.X; y=(int) pc.Y; z=(int) pc.Z;
current.pop_back();
mask.value(x, y, z) = 1;
if (0<=x-1 && mask.value(x-1, y, z)==0) current.push_back(Pt(x-1, y, z));
if (0<=y-1 && mask.value(x, y-1, z)==0) current.push_back(Pt(x, y-1, z));
if (0<=z-1 && mask.value(x, y, z-1)==0) current.push_back(Pt(x, y, z-1));
if (xsize>x+1 && mask.value(x+1, y, z)==0) current.push_back(Pt(x+1, y, z));
if (ysize>y+1 && mask.value(x, y+1, z)==0) current.push_back(Pt(x, y+1, z));
if (zsize>z+1 && mask.value(x, y, z+1)==0) current.push_back(Pt(x, y, z+1));
}
return mask;
}
bool Option::matches(OptSpecifier Opt) const {
// Aliases are never considered in matching, look through them.
const Option Alias = getAlias();
if (Alias.isValid())
return Alias.matches(Opt);
// Check exact match.
if (getID() == Opt.getID())
return true;
const Option Group = getGroup();
if (Group.isValid())
return Group.matches(Opt);
return false;
}
