TER
Taker::cross (Offer const& offer)
{
assert (!done ());
/* Before we call flow we must set the limit right; for buy semantics we
need to clamp the output. And we always want to clamp the input.
*/
Amounts limit (offer.amount());
if (! m_options.sell)
limit = offer.quality ().ceil_out (limit, m_remain.out);
limit = offer.quality().ceil_in (limit, m_remain.in);
assert (limit.in <= offer.amount().in);
assert (limit.out <= offer.amount().out);
assert (limit.in <= m_remain.in);
Amounts const amount (flow (limit, offer, account ()));
m_remain.out -= amount.out;
m_remain.in -= amount.in;
assert (m_remain.in >= zero);
return fill (offer, amount);
}
bool Offer::operator==(Offer& other)
{
if (this->destination == other.getDest() &&
this->type == other.getType() &&
this->price == other.getPrice())
return (true);
return (false);
}
inline TaskInfo createTask(
const Offer& offer,
const std::string& command,
const Option<mesos::ExecutorID>& executorId = None(),
const std::string& name = "test-task",
const std::string& id = UUID::random().toString())
{
return createTask(
offer.slave_id(), offer.resources(), command, executorId, name, id);
}
TER
Taker::cross (Offer const& offer)
{
// In direct crossings, at least one leg must not be XRP.
if (isXRP (offer.amount ().in) && isXRP (offer.amount ().out))
return tefINTERNAL;
auto const amount = do_cross (
offer.amount (), offer.quality (), offer.owner ());
return fill (amount, offer);
}
// Fill a direct offer.
// @param offer the offer we are going to use.
// @param amount the amount to flow through the offer.
// @returns: tesSUCCESS if successful, or an error code otherwise.
TER
Taker::fill (Offer const& offer, Amounts const& amount)
{
consume (offer, amount);
// Pay the taker, then the owner
TER result = view ().accountSend (offer.account(), account(), amount.out);
if (result == tesSUCCESS)
result = view ().accountSend (account(), offer.account(), amount.in);
return result;
}
// Performs funds transfers to fill the given offer and adjusts offer.
TER
Taker::fill (BasicTaker::Flow const& flow, Offer const& offer)
{
// adjust offer
consume_offer (offer, flow.order);
TER result = tesSUCCESS;
if (cross_type () != CrossType::XrpToIou)
{
assert (!isXRP (flow.order.in));
if(result == tesSUCCESS)
result = redeemIOU (account (), flow.issuers.in, flow.issuers.in.issue ());
if (result == tesSUCCESS)
result = issueIOU (offer.owner (), flow.order.in, flow.order.in.issue ());
}
else
{
assert (isXRP (flow.order.in));
if (result == tesSUCCESS)
result = transferXRP (account (), offer.owner (), flow.order.in);
}
// Now send funds from the account whose offer we're taking
if (cross_type () != CrossType::IouToXrp)
{
assert (!isXRP (flow.order.out));
if(result == tesSUCCESS)
result = redeemIOU (offer.owner (), flow.issuers.out, flow.issuers.out.issue ());
if (result == tesSUCCESS)
result = issueIOU (account (), flow.order.out, flow.order.out.issue ());
}
else
{
assert (isXRP (flow.order.out));
if (result == tesSUCCESS)
result = transferXRP (offer.owner (), account (), flow.order.out);
}
if (result == tesSUCCESS)
direct_crossings_++;
return result;
}
// TODO(benh): Move this into utils, make more generic, and use in
// other tests.
vector<TaskInfo> createTasks(const Offer& offer)
{
TaskInfo task;
task.set_name("test-task");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offer.slave_id());
task.mutable_resources()->MergeFrom(offer.resources());
task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
vector<TaskInfo> tasks;
tasks.push_back(task);
return tasks;
}
void TableListener::printOffer(IO2GOfferTableRow *offerRow, const char *sInstrument)
{
Offer *offer = mOffers->findOffer(offerRow->getOfferID());
if (offer)
{
if (offerRow->isTimeValid() && offerRow->isBidValid() && offerRow->isAskValid())
{
offer->setDate(offerRow->getTime());
offer->setBid(offerRow->getBid());
offer->setAsk(offerRow->getAsk());
}
}
else
{
offer = new Offer(offerRow->getOfferID(), offerRow->getInstrument(),
offerRow->getDigits(), offerRow->getPointSize(), offerRow->getTime(),
offerRow->getBid(), offerRow->getAsk());
mOffers->addOffer(offer);
}
if (!sInstrument || strlen(sInstrument) == 0 || strcmp(offerRow->getInstrument(), sInstrument) == 0)
{
std::cout << offer->getID() << ", " << offer->getInstrument() << ", "
<< "Bid=" << std::fixed << offer->getBid() << ", "
<< "Ask=" << std::fixed << offer->getAsk() << std::endl;
}
}
inline double getScalarResource(const Offer& offer, const std::string& name)
{
double value = 0.0;
for (int i = 0; i < offer.resources_size(); i++) {
const Resource& resource = offer.resources(i);
if (resource.name() == name &&
resource.type() == Value::SCALAR) {
value = resource.scalar().value();
}
}
return value;
}
inline TaskInfo createTask(
const Offer& offer,
const std::string& command,
const std::string& name = "test-task",
const std::string& id = UUID::random().toString())
{
TaskInfo task;
task.set_name(name);
task.mutable_task_id()->set_value(id);
task.mutable_slave_id()->MergeFrom(offer.slave_id());
task.mutable_resources()->MergeFrom(offer.resources());
task.mutable_command()->set_value(command);
return task;
}
// Adjust an offer to indicate that we are consuming some (or all) of it.
void
Taker::consume (Offer const& offer, Amounts const& consumed) const
{
Amounts const& remaining (offer.amount ());
assert (remaining.in > zero && remaining.out > zero);
assert (remaining.in >= consumed.in && remaining.out >= consumed.out);
offer.entry ()->setFieldAmount (sfTakerPays, remaining.in - consumed.in);
offer.entry ()->setFieldAmount (sfTakerGets, remaining.out - consumed.out);
view ().entryModify (offer.entry());
assert (offer.entry ()->getFieldAmount (sfTakerPays) >= zero);
assert (offer.entry ()->getFieldAmount (sfTakerGets) >= zero);
}
TER
Taker::cross (Offer const& offer)
{
assert (!done ());
Amounts limit (offer.amount());
if (m_options.sell)
limit = offer.quality().ceil_in (limit, m_remain.in);
else
limit = offer.quality ().ceil_out (limit, m_remain.out);
assert (limit.out <= offer.amount().out);
assert (limit.in <= offer.amount().in);
Amounts const amount (flow (limit, offer, account ()));
m_remain.out -= amount.out;
m_remain.in -= amount.in;
assert (m_remain.in >= zero);
return fill (offer, amount);
}
void
Taker::consume_offer (Offer const& offer, Amounts const& order)
{
if (order.in < zero)
Throw<std::logic_error> ("flow with negative input.");
if (order.out < zero)
Throw<std::logic_error> ("flow with negative output.");
if (journal_.debug) journal_.debug << "Consuming from offer " << offer;
if (journal_.trace)
{
auto const& available = offer.amount ();
journal_.trace << " in:" << format_amount (available.in);
journal_.trace << " out:" << format_amount(available.out);
}
offer.consume (view_, order);
}
TER
Taker::cross (Offer const& leg1, Offer const& leg2)
{
// In bridged crossings, XRP must can't be the input to the first leg
// or the output of the second leg.
if (isXRP (leg1.amount ().in) || isXRP (leg2.amount ().out))
return tefINTERNAL;
auto ret = do_cross (
leg1.amount (), leg1.quality (), leg1.owner (),
leg2.amount (), leg2.quality (), leg2.owner ());
return fill (ret.first, leg1, ret.second, leg2);
}
/** Calculate the amount particular user could get through an offer.
@param amount the maximum flow that is available to the taker.
@param offer the offer to flow through.
@param taker the person taking the offer.
@return the maximum amount that can flow through this offer.
*/
Amounts
Taker::flow (Amounts amount, Offer const& offer, Account const& taker)
{
// Limit taker's input by available funds less fees
Amount const taker_funds (view ().accountFunds (
taker, amount.in, fhZERO_IF_FROZEN));
// Get fee rate paid by taker
std::uint32_t const taker_charge_rate (rippleTransferRate (view (),
taker, offer.account (), amount.in.getIssuer()));
// Skip some math when there's no fee
if (taker_charge_rate == QUALITY_ONE)
{
amount = offer.quality ().ceil_in (amount, taker_funds);
}
else
{
Amount const taker_charge (amountFromRate (taker_charge_rate));
amount = offer.quality ().ceil_in (amount,
divide (taker_funds, taker_charge, taker_funds.issue ()));
}
// Best flow the owner can get.
// Start out assuming entire offer will flow.
Amounts owner_amount (amount);
// Limit owner's output by available funds less fees
Amount const owner_funds (view ().accountFunds (
offer.account (), owner_amount.out, fhZERO_IF_FROZEN));
// Get fee rate paid by owner
std::uint32_t const owner_charge_rate (rippleTransferRate (view (),
offer.account (), taker, amount.out.getIssuer()));
if (owner_charge_rate == QUALITY_ONE)
{
// Skip some math when there's no fee
owner_amount = offer.quality ().ceil_out (owner_amount, owner_funds);
}
else
{
Amount const owner_charge (amountFromRate (owner_charge_rate));
owner_amount = offer.quality ().ceil_out (owner_amount,
divide (owner_funds, owner_charge, owner_funds.issue ()));
}
// Calculate the amount that will flow through the offer
// This does not include the fees.
return (owner_amount.in < amount.in)
? owner_amount
: amount;
}
inline TaskInfo createTask(
const Offer& offer,
const std::string& command,
const Option<mesos::ExecutorID>& executorId = None(),
const std::string& name = "test-task",
const std::string& id = UUID::random().toString())
{
TaskInfo task;
task.set_name(name);
task.mutable_task_id()->set_value(id);
task.mutable_slave_id()->CopyFrom(offer.slave_id());
task.mutable_resources()->CopyFrom(offer.resources());
if (executorId.isSome()) {
ExecutorInfo executor;
executor.mutable_executor_id()->CopyFrom(executorId.get());
executor.mutable_command()->set_value(command);
task.mutable_executor()->CopyFrom(executor);
} else {
task.mutable_command()->set_value(command);
}
return task;
}
// Fill a bridged offer.
// @param leg1 the first leg we are going to use.
// @param amount1 the amount to flow through the first leg of the offer.
// @param leg2 the second leg we are going to use.
// @param amount2 the amount to flow through the second leg of the offer.
// @return tesSUCCESS if successful, or an error code otherwise.
TER
Taker::fill (
Offer const& leg1, Amounts const& amount1,
Offer const& leg2, Amounts const& amount2)
{
assert (amount1.out == amount2.in);
TER result (tesSUCCESS);
Amounts const remain1 (
leg1.entry ()->getFieldAmount (sfTakerPays) - amount1.in,
leg1.entry ()->getFieldAmount (sfTakerGets) - amount1.out);
leg1.entry ()->setFieldAmount (sfTakerPays, remain1.in);
leg1.entry ()->setFieldAmount (sfTakerGets, remain1.out);
view ().entryModify (leg1.entry());
Amounts const remain2 (
leg2.entry ()->getFieldAmount (sfTakerPays) - amount2.in,
leg2.entry ()->getFieldAmount (sfTakerGets) - amount2.out);
view ().entryModify (leg2.entry ());
// Execute the payments in order:
// Taker pays Leg1 (amount1.in - A currency)
// Leg1 pays Leg2 (amount1.out == amount2.in, XRP)
// Leg2 pays Taker (amount2.out - B currency)
result = view ().accountSend (m_account, leg1.account (), amount1.in);
if (result == tesSUCCESS)
result = view ().accountSend (leg1.account (), leg2.account (), amount1.out);
if (result == tesSUCCESS)
result = view ().accountSend (leg2.account (), m_account, amount2.out);
return result;
}
vector<TaskInfo> populateTasks(
const string& cmd,
CommandInfo healthCommand,
const Offer& offer,
int gracePeriodSeconds = 0,
const Option<int>& consecutiveFailures = None(),
const Option<map<string, string> >& env = None())
{
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->CopyFrom(offer.slave_id());
task.mutable_resources()->CopyFrom(offer.resources());
CommandInfo command;
command.set_value(cmd);
Environment::Variable* variable =
command.mutable_environment()->add_variables();
// We need to set the correct directory to launch health check process
// instead of the default for tests.
variable->set_name("MESOS_LAUNCHER_DIR");
variable->set_value(path::join(tests::flags.build_dir, "src"));
task.mutable_command()->CopyFrom(command);
HealthCheck healthCheck;
if (env.isSome()) {
foreachpair (const string& name, const string value, env.get()) {
Environment::Variable* variable =
healthCommand.mutable_environment()->mutable_variables()->Add();
variable->set_name(name);
variable->set_value(value);
}
}
// Fill a bridged offer.
// @param leg1 the first leg we are going to use.
// @param amount1 the amount to flow through the first leg of the offer.
// @param leg2 the second leg we are going to use.
// @param amount2 the amount to flow through the second leg of the offer.
// @return tesSUCCESS if successful, or an error code otherwise.
TER
Taker::fill (
Offer const& leg1, Amounts const& amount1,
Offer const& leg2, Amounts const& amount2)
{
assert (amount1.out == amount2.in);
consume (leg1, amount1);
consume (leg2, amount2);
/* It is possible that m_account is the same as leg1.account, leg2.account
* or both. This could happen when bridging over one's own offer. In that
* case, accountSend won't actually do a send, which is what we want.
*/
TER result = view ().accountSend (m_account, leg1.account (), amount1.in);
if (result == tesSUCCESS)
result = view ().accountSend (leg1.account (), leg2.account (), amount1.out);
if (result == tesSUCCESS)
result = view ().accountSend (leg2.account (), m_account, amount2.out);
return result;
}
TER
Taker::cross (Offer const& leg1, Offer const& leg2)
{
assert (!done ());
assert (leg1.amount ().out.isNative ());
assert (leg2.amount ().in.isNative ());
Amounts amount1 (leg1.amount());
Amounts amount2 (leg2.amount());
if (m_options.sell)
amount1 = leg1.quality().ceil_in (amount1, m_remain.in);
else
amount2 = leg2.quality().ceil_out (amount2, m_remain.out);
if (amount1.out <= amount2.in)
amount2 = leg2.quality().ceil_in (amount2, amount1.out);
else
amount1 = leg1.quality().ceil_out (amount1, amount2.in);
assert (amount1.out == amount2.in);
// As written, flow can't handle a 3-party transfer, but this works for
// us because the output of leg1 and the input leg2 are XRP.
Amounts flow1 (flow (amount1, leg1, m_account));
amount2 = leg2.quality().ceil_in (amount2, flow1.out);
Amounts flow2 (flow (amount2, leg2, m_account));
m_remain.out -= amount2.out;
m_remain.in -= amount1.in;
return fill (leg1, flow1, leg2, flow2);
}
jobject convert(JNIEnv* env, const Offer& offer)
{
string data;
offer.SerializeToString(&data);
// byte[] data = ..;
jbyteArray jdata = env->NewByteArray(data.size());
env->SetByteArrayRegion(jdata, 0, data.size(), (jbyte*) data.data());
// Offer offer = Offer.parseFrom(data);
jclass clazz = FindMesosClass(env, "org/apache/mesos/Protos$Offer");
jmethodID parseFrom =
env->GetStaticMethodID(clazz, "parseFrom",
"([B)Lorg/apache/mesos/Protos$Offer;");
jobject joffer = env->CallStaticObjectMethod(clazz, parseFrom, jdata);
return joffer;
}
/// todo: weigh result with a "distance" from the constraint boundary
float NumericalRelationship::utility(const Offer& offer) const
{
const Attribute* offerAttribute = offer.getAttribute(m_name);
//qDebug() << "Offer attribute of name " << m_name << offerAttribute;
if (!offerAttribute)
return 1.0; // we don't know -> no violation
const NumericalAttribute* numericalOfferAttribute = dynamic_cast<const NumericalAttribute*>(offerAttribute);
if (!numericalOfferAttribute)
return 0; // wrong type
float baseUtility = Relationship::utility(offer);
if (m_type & Relationship::SmallerThan &&
*numericalOfferAttribute > static_cast<const NumericalAttribute&>(m_attribute))
baseUtility = 0;
if (m_type & Relationship::LargerThan &&
*numericalOfferAttribute < static_cast<const NumericalAttribute&>(m_attribute))
baseUtility = 0;
//qDebug() << "Returning: " << baseUtility;
return baseUtility;
}
// Performs bridged funds transfers to fill the given offers and adjusts offers.
TER
Taker::fill (
BasicTaker::Flow const& flow1, Offer const& leg1,
BasicTaker::Flow const& flow2, Offer const& leg2)
{
// Adjust offers accordingly
consume_offer (leg1, flow1.order);
consume_offer (leg2, flow2.order);
TER result = tesSUCCESS;
// Taker to leg1: IOU
if (leg1.owner () != account ())
{
if (result == tesSUCCESS)
result = redeemIOU (account (), flow1.issuers.in, flow1.issuers.in.issue ());
if (result == tesSUCCESS)
result = issueIOU (leg1.owner (), flow1.order.in, flow1.order.in.issue ());
}
// leg1 to leg2: bridging over XRP
if (result == tesSUCCESS)
result = transferXRP (leg1.owner (), leg2.owner (), flow1.order.out);
// leg2 to Taker: IOU
if (leg2.owner () != account ())
{
if (result == tesSUCCESS)
result = redeemIOU (leg2.owner (), flow2.issuers.out, flow2.issuers.out.issue ());
if (result == tesSUCCESS)
result = issueIOU (account (), flow2.order.out, flow2.order.out.issue ());
}
if (result == tesSUCCESS)
{
bridge_crossings_++;
xrp_flow_ += flow1.order.out;
}
return result;
}
// Fill a direct offer.
// @param offer the offer we are going to use.
// @param amount the amount to flow through the offer.
// @returns: tesSUCCESS if successful, or an error code otherwise.
TER
Taker::fill (Offer const& offer, Amounts const& amount)
{
TER result (tesSUCCESS);
Amounts const remain (
offer.entry ()->getFieldAmount (sfTakerPays) - amount.in,
offer.entry ()->getFieldAmount (sfTakerGets) - amount.out);
offer.entry ()->setFieldAmount (sfTakerPays, remain.in);
offer.entry ()->setFieldAmount (sfTakerGets, remain.out);
view ().entryModify (offer.entry());
// Pay the taker, then the owner
result = view ().accountSend (offer.account(), account(), amount.out);
if (result == tesSUCCESS)
result = view ().accountSend (account(), offer.account(), amount.in);
return result;
}
// This test verifies the slave will destroy a container if, when
// receiving a terminal status task update, updating the container's
// resources fails.
TEST_F(SlaveTest, TerminalTaskContainerizerUpdateFails)
{
// Start a master.
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
EXPECT_CALL(exec, registered(_, _, _, _));
TestContainerizer containerizer(&exec);
// Start a slave.
Try<PID<Slave> > slave = StartSlave(&containerizer);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(_, _, _));
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
Offer offer = offers.get()[0];
// Start two tasks.
vector<TaskInfo> tasks;
tasks.push_back(createTask(
offer.slave_id(),
Resources::parse("cpus:0.1;mem:32").get(),
"sleep 1000",
exec.id));
tasks.push_back(createTask(
offer.slave_id(),
Resources::parse("cpus:0.1;mem:32").get(),
"sleep 1000",
exec.id));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
Future<TaskStatus> status1, status2, status3, status4;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status1))
.WillOnce(FutureArg<1>(&status2))
.WillOnce(FutureArg<1>(&status3))
.WillOnce(FutureArg<1>(&status4));
driver.launchTasks(offer.id(), tasks);
AWAIT_READY(status1);
EXPECT_EQ(TASK_RUNNING, status1.get().state());
AWAIT_READY(status2);
EXPECT_EQ(TASK_RUNNING, status2.get().state());
// Set up the containerizer so the next update() will fail.
EXPECT_CALL(containerizer, update(_, _))
.WillOnce(Return(process::Failure("update() failed")))
.WillRepeatedly(Return(Nothing()));
EXPECT_CALL(exec, killTask(_, _))
.WillOnce(SendStatusUpdateFromTaskID(TASK_KILLED));
// Kill one of the tasks. The failed update should result in the
// second task going lost when the container is destroyed.
driver.killTask(tasks[0].task_id());
AWAIT_READY(status3);
EXPECT_EQ(TASK_KILLED, status3.get().state());
AWAIT_READY(status4);
EXPECT_EQ(TASK_LOST, status4.get().state());
driver.stop();
driver.join();
Shutdown();
}
// In this test, the agent initially doesn't enable disk isolation
// but then restarts with XFS disk isolation enabled. We verify that
// the old container launched before the agent restart is
// successfully recovered.
TEST_F(ROOT_XFS_QuotaTest, RecoverOldContainers)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
Owned<MasterDetector> detector = master.get()->createDetector();
slave::Flags flags = CreateSlaveFlags();
// `CreateSlaveFlags()` enables `disk/xfs` so here we reset
// `isolation` to empty.
flags.isolation.clear();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
ASSERT_SOME(slave);
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_checkpoint(true);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(_, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(_, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
Offer offer = offers.get()[0];
TaskInfo task = createTask(
offer.slave_id(),
Resources::parse("cpus:1;mem:128;disk:1").get(),
"dd if=/dev/zero of=file bs=1024 count=1; sleep 1000");
Future<TaskStatus> startingStatus;
Future<TaskStatus> runningstatus;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&startingStatus))
.WillOnce(FutureArg<1>(&runningstatus));
driver.launchTasks(offer.id(), {task});
AWAIT_READY(startingStatus);
EXPECT_EQ(task.task_id(), startingStatus->task_id());
EXPECT_EQ(TASK_STARTING, startingStatus->state());
AWAIT_READY(runningstatus);
EXPECT_EQ(task.task_id(), runningstatus->task_id());
EXPECT_EQ(TASK_RUNNING, runningstatus->state());
{
Future<ResourceUsage> usage =
process::dispatch(slave.get()->pid, &Slave::usage);
AWAIT_READY(usage);
// We should have 1 executor using resources but it doesn't have
// disk limit enabled.
ASSERT_EQ(1, usage->executors().size());
const ResourceUsage_Executor& executor = usage->executors().Get(0);
ASSERT_TRUE(executor.has_statistics());
ASSERT_FALSE(executor.statistics().has_disk_limit_bytes());
}
// Restart the slave.
slave.get()->terminate();
Future<SlaveReregisteredMessage> slaveReregisteredMessage =
FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _);
// This time use the agent flags that include XFS disk isolation.
slave = StartSlave(detector.get(), CreateSlaveFlags());
ASSERT_SOME(slave);
// Wait for the slave to re-register.
AWAIT_READY(slaveReregisteredMessage);
{
Future<ResourceUsage> usage =
process::dispatch(slave.get()->pid, &Slave::usage);
AWAIT_READY(usage);
// We should still have 1 executor using resources but it doesn't
// have disk limit enabled.
ASSERT_EQ(1, usage->executors().size());
const ResourceUsage_Executor& executor = usage->executors().Get(0);
ASSERT_TRUE(executor.has_statistics());
ASSERT_FALSE(executor.statistics().has_disk_limit_bytes());
}
driver.stop();
driver.join();
}
TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_Statistics)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
slave::Flags flags = CreateSlaveFlags();
// We only care about memory cgroup for this test.
flags.isolation = "cgroups/mem";
flags.agent_subsystems = None();
Fetcher fetcher;
Try<MesosContainerizer*> _containerizer =
MesosContainerizer::create(flags, true, &fetcher);
ASSERT_SOME(_containerizer);
Owned<MesosContainerizer> containerizer(_containerizer.get());
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave =
StartSlave(detector.get(), containerizer.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(_, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(_, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
Offer offer = offers.get()[0];
// Run a task that triggers memory pressure event. We request 1G
// disk because we are going to write a 512 MB file repeatedly.
TaskInfo task = createTask(
offer.slave_id(),
Resources::parse("cpus:1;mem:256;disk:1024").get(),
"while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done");
Future<TaskStatus> running;
Future<TaskStatus> killed;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&running))
.WillOnce(FutureArg<1>(&killed))
.WillRepeatedly(Return()); // Ignore subsequent updates.
driver.launchTasks(offer.id(), {task});
AWAIT_READY(running);
EXPECT_EQ(task.task_id(), running.get().task_id());
EXPECT_EQ(TASK_RUNNING, running.get().state());
Future<hashset<ContainerID>> containers = containerizer->containers();
AWAIT_READY(containers);
ASSERT_EQ(1u, containers.get().size());
ContainerID containerId = *(containers.get().begin());
// Wait a while for some memory pressure events to occur.
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage = containerizer->usage(containerId);
AWAIT_READY(usage);
if (usage.get().mem_low_pressure_counter() > 0) {
// We will check the correctness of the memory pressure counters
// later, because the memory-hammering task is still active
// and potentially incrementing these counters.
break;
}
os::sleep(Milliseconds(100));
waited += Milliseconds(100);
} while (waited < Seconds(5));
EXPECT_LE(waited, Seconds(5));
// Pause the clock to ensure that the reaper doesn't reap the exited
// command executor and inform the containerizer/slave.
Clock::pause();
Clock::settle();
// Stop the memory-hammering task.
driver.killTask(task.task_id());
AWAIT_READY_FOR(killed, Seconds(120));
EXPECT_EQ(task.task_id(), killed->task_id());
EXPECT_EQ(TASK_KILLED, killed->state());
// Now check the correctness of the memory pressure counters.
Future<ResourceStatistics> usage = containerizer->usage(containerId);
AWAIT_READY(usage);
EXPECT_GE(usage.get().mem_low_pressure_counter(),
usage.get().mem_medium_pressure_counter());
EXPECT_GE(usage.get().mem_medium_pressure_counter(),
usage.get().mem_critical_pressure_counter());
Clock::resume();
driver.stop();
driver.join();
}
// Test that memory pressure listening is restarted after recovery.
TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_SlaveRecovery)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
slave::Flags flags = CreateSlaveFlags();
// We only care about memory cgroup for this test.
flags.isolation = "cgroups/mem";
Fetcher fetcher(flags);
Try<MesosContainerizer*> _containerizer =
MesosContainerizer::create(flags, true, &fetcher);
ASSERT_SOME(_containerizer);
Owned<MesosContainerizer> containerizer(_containerizer.get());
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave =
StartSlave(detector.get(), containerizer.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
// Enable checkpointing for the framework.
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_checkpoint(true);
MesosSchedulerDriver driver(
&sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(_, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(_, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
Offer offer = offers.get()[0];
// Run a task that triggers memory pressure event. We request 1G
// disk because we are going to write a 512 MB file repeatedly.
TaskInfo task = createTask(
offer.slave_id(),
Resources::parse("cpus:1;mem:256;disk:1024").get(),
"while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done");
Future<TaskStatus> starting;
Future<TaskStatus> running;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&starting))
.WillOnce(FutureArg<1>(&running))
.WillRepeatedly(Return()); // Ignore subsequent updates.
Future<Nothing> runningAck =
FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement);
Future<Nothing> startingAck =
FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement);
driver.launchTasks(offers.get()[0].id(), {task});
AWAIT_READY(starting);
EXPECT_EQ(task.task_id(), starting->task_id());
EXPECT_EQ(TASK_STARTING, starting->state());
AWAIT_READY(startingAck);
AWAIT_READY(running);
EXPECT_EQ(task.task_id(), running->task_id());
EXPECT_EQ(TASK_RUNNING, running->state());
// Wait for the ACK to be checkpointed.
AWAIT_READY_FOR(runningAck, Seconds(120));
// We restart the slave to let it recover.
slave.get()->terminate();
// Set up so we can wait until the new slave updates the container's
// resources (this occurs after the executor has re-registered).
Future<Nothing> update =
FUTURE_DISPATCH(_, &MesosContainerizerProcess::update);
// Use the same flags.
_containerizer = MesosContainerizer::create(flags, true, &fetcher);
ASSERT_SOME(_containerizer);
containerizer.reset(_containerizer.get());
Future<SlaveReregisteredMessage> reregistered =
FUTURE_PROTOBUF(SlaveReregisteredMessage(), master.get()->pid, _);
slave = StartSlave(detector.get(), containerizer.get(), flags);
ASSERT_SOME(slave);
AWAIT_READY(reregistered);
// Wait until the containerizer is updated.
AWAIT_READY(update);
Future<hashset<ContainerID>> containers = containerizer->containers();
AWAIT_READY(containers);
ASSERT_EQ(1u, containers->size());
ContainerID containerId = *(containers->begin());
// Wait a while for some memory pressure events to occur.
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage = containerizer->usage(containerId);
AWAIT_READY(usage);
if (usage->mem_low_pressure_counter() > 0) {
// We will check the correctness of the memory pressure counters
// later, because the memory-hammering task is still active
// and potentially incrementing these counters.
break;
}
os::sleep(Milliseconds(100));
waited += Milliseconds(100);
} while (waited < Seconds(5));
EXPECT_LE(waited, Seconds(5));
// Pause the clock to ensure that the reaper doesn't reap the exited
// command executor and inform the containerizer/slave.
Clock::pause();
Clock::settle();
Future<TaskStatus> killed;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&killed));
// Stop the memory-hammering task.
driver.killTask(task.task_id());
AWAIT_READY_FOR(killed, Seconds(120));
EXPECT_EQ(task.task_id(), killed->task_id());
EXPECT_EQ(TASK_KILLED, killed->state());
// Now check the correctness of the memory pressure counters.
Future<ResourceStatistics> usage = containerizer->usage(containerId);
AWAIT_READY(usage);
EXPECT_GE(usage->mem_low_pressure_counter(),
usage->mem_medium_pressure_counter());
EXPECT_GE(usage->mem_medium_pressure_counter(),
usage->mem_critical_pressure_counter());
Clock::resume();
driver.stop();
driver.join();
}
bool is_valid_offer(const Offer &offer)
{
return offer.valid();
}
// Test that memory pressure listening is restarted after recovery.
TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_SlaveRecovery)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
slave::Flags flags = CreateSlaveFlags();
// We only care about memory cgroup for this test.
flags.isolation = "cgroups/mem";
flags.slave_subsystems = None();
Fetcher fetcher;
Try<MesosContainerizer*> containerizer1 =
MesosContainerizer::create(flags, true, &fetcher);
ASSERT_SOME(containerizer1);
Try<PID<Slave>> slave = StartSlave(containerizer1.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
// Enable checkpointing for the framework.
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_checkpoint(true);
MesosSchedulerDriver driver(
&sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(_, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(_, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
Offer offer = offers.get()[0];
// Run a task that triggers memory pressure event. We request 1G
// disk because we are going to write a 512 MB file repeatedly.
TaskInfo task = createTask(
offer.slave_id(),
Resources::parse("cpus:1;mem:256;disk:1024").get(),
"while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done");
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status))
.WillRepeatedly(Return()); // Ignore subsequent updates.
driver.launchTasks(offers.get()[0].id(), {task});
AWAIT_READY(status);
EXPECT_EQ(task.task_id(), status.get().task_id());
EXPECT_EQ(TASK_RUNNING, status.get().state());
// We restart the slave to let it recover.
Stop(slave.get());
delete containerizer1.get();
// Set up so we can wait until the new slave updates the container's
// resources (this occurs after the executor has re-registered).
Future<Nothing> update =
FUTURE_DISPATCH(_, &MesosContainerizerProcess::update);
// Use the same flags.
Try<MesosContainerizer*> containerizer2 =
MesosContainerizer::create(flags, true, &fetcher);
ASSERT_SOME(containerizer2);
slave = StartSlave(containerizer2.get(), flags);
ASSERT_SOME(slave);
// Wait until the containerizer is updated.
AWAIT_READY(update);
Future<hashset<ContainerID>> containers = containerizer2.get()->containers();
AWAIT_READY(containers);
ASSERT_EQ(1u, containers.get().size());
ContainerID containerId = *(containers.get().begin());
// Wait a while for some memory pressure events to occur.
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage = containerizer2.get()->usage(containerId);
AWAIT_READY(usage);
if (usage.get().mem_low_pressure_counter() > 0) {
// We will check the correctness of the memory pressure counters
// later, because the memory-hammering task is still active
// and potentially incrementing these counters.
break;
}
os::sleep(Milliseconds(100));
waited += Milliseconds(100);
} while (waited < Seconds(5));
EXPECT_LE(waited, Seconds(5));
// Stop the memory-hammering task.
driver.killTask(task.task_id());
// Process any queued up events through before proceeding.
process::Clock::pause();
process::Clock::settle();
process::Clock::resume();
// Now check the correctness of the memory pressure counters.
Future<ResourceStatistics> usage = containerizer2.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_GE(usage.get().mem_low_pressure_counter(),
usage.get().mem_medium_pressure_counter());
EXPECT_GE(usage.get().mem_medium_pressure_counter(),
usage.get().mem_critical_pressure_counter());
driver.stop();
driver.join();
Shutdown();
delete containerizer2.get();
}
