// This test verifies that the client will receive a `BadRequest` response if it
// includes a stream ID header with a subscribe call.
TEST_P(SchedulerHttpApiTest, SubscribeWithStreamId)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
headers["Mesos-Stream-Id"] = UUID::random().toString();
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
}
TEST_P(SchedulerHttpApiTest, DefaultAccept)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = "*/*";
// Only subscribe needs to 'Accept' json or protobuf.
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response);
}
// This test sets an unsupported media type as Content-Type. This
// should result in a 415 (UnsupportedMediaType) response.
TEST_P(SchedulerHttpApiTest, UnsupportedContentMediaType)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
const string unknownMediaType = "application/unknown-media-type";
Future<Response> response = process::http::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
unknownMediaType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(UnsupportedMediaType().status, response);
}
void doReliableRegistration()
{
if (state == SUBSCRIBED || state == DISCONNECTED) {
return;
}
Call call;
call.set_type(Call::SUBSCRIBE);
call.mutable_framework_id()->CopyFrom(frameworkId);
call.mutable_executor_id()->CopyFrom(executorId);
Call::Subscribe* subscribe = call.mutable_subscribe();
// Send all unacknowledged updates.
foreach (const Call::Update& update, updates.values()) {
subscribe->add_unacknowledged_updates()->MergeFrom(update);
}
// Send the unacknowledged task.
if (task.isSome()) {
subscribe->add_unacknowledged_tasks()->MergeFrom(task.get());
}
mesos->send(call);
delay(Seconds(1), self(), &Self::doReliableRegistration);
}
// This test verifies if the role is invalid in scheduler's framework message,
// the event is error on the stream in response to a Subscribe call request.
TEST_P(SchedulerHttpApiTest, RejectFrameworkWithInvalidRole)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
v1::FrameworkInfo framework = v1::DEFAULT_FRAMEWORK_INFO;
// Set invalid role.
framework.set_role("/test/test1");
subscribe->mutable_framework_info()->CopyFrom(framework);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check event type is error.
ASSERT_EQ(Event::ERROR, event.get().get().type());
}
void doReliableRegistration()
{
if (state == SUBSCRIBED || state == DISCONNECTED) {
return;
}
Call call;
call.set_type(Call::SUBSCRIBE);
if (frameworkInfo.has_id()) {
call.mutable_framework_id()->CopyFrom(frameworkInfo.id());
}
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(frameworkInfo);
mesos->send(call);
process::delay(Seconds(1), self(), &Self::doReliableRegistration);
}
void ResourceProviderManagerProcess::subscribe(
const HttpConnection& http,
const Call::Subscribe& subscribe)
{
ResourceProviderInfo resourceProviderInfo =
subscribe.resource_provider_info();
resourceProviderInfo.mutable_id()->CopyFrom(newResourceProviderId());
ResourceProvider resourceProvider(resourceProviderInfo, http);
Event event;
event.set_type(Event::SUBSCRIBED);
event.mutable_subscribed()->mutable_provider_id()->CopyFrom(
resourceProvider.info.id());
if (!resourceProvider.http.send(event)) {
LOG(WARNING) << "Unable to send event to resource provider "
<< stringify(resourceProvider.info.id())
<< ": connection closed";
}
resourceProviders.put(resourceProviderInfo.id(), std::move(resourceProvider));
}
// This test verifies that we are able to downgrade from a HTTP based
// framework to PID.
TEST_P(SchedulerHttpApiTest, UpdateHttpToPidScheduler)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO;
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(frameworkInfo);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check event type is subscribed and the framework id is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
frameworkInfo.mutable_id()->
CopyFrom(event.get().get().subscribed().framework_id());
// Make sure it receives a heartbeat.
event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
ASSERT_EQ(Event::HEARTBEAT, event.get().get().type());
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, devolve(frameworkInfo), master.get()->pid, DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
driver.start();
AWAIT_READY(frameworkId);
ASSERT_EQ(evolve(frameworkId.get()), frameworkInfo.id());
driver.stop();
driver.join();
}
// This test checks that a scheduler exit shuts down the executor.
TEST_F(HttpFaultToleranceTest, SchedulerExit)
{
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false;
Try<Owned<cluster::Master>> master = StartMaster(flags);
ASSERT_SOME(master);
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
auto executor = std::make_shared<v1::MockHTTPExecutor>();
ExecutorID executorId = DEFAULT_EXECUTOR_ID;
TestContainerizer containerizer(executorId, executor);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected))
.WillRepeatedly(Return()); // Ignore future invocations.
ContentType contentType = ContentType::PROTOBUF;
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
Future<Event::Subscribed> subscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
Future<Event::Offers> offers;
EXPECT_CALL(*scheduler, offers(_, _))
.WillOnce(FutureArg<1>(&offers));
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
schedulerLibrary.send(call);
}
AWAIT_READY(subscribed);
v1::FrameworkID frameworkId(subscribed->framework_id());
AWAIT_READY(offers);
EXPECT_NE(0, offers->offers().size());
EXPECT_CALL(*executor, connected(_))
.WillOnce(v1::executor::SendSubscribe(frameworkId, evolve(executorId)));
EXPECT_CALL(*executor, subscribed(_, _));
Future<Nothing> launch;
EXPECT_CALL(*executor, launch(_, _))
.WillOnce(FutureSatisfy(&launch));
const v1::Offer& offer = offers->offers(0);
v1::TaskInfo taskInfo =
evolve(createTask(devolve(offer), "", DEFAULT_EXECUTOR_ID));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::ACCEPT);
Call::Accept* accept = call.mutable_accept();
accept->add_offer_ids()->CopyFrom(offer.id());
v1::Offer::Operation* operation = accept->add_operations();
operation->set_type(v1::Offer::Operation::LAUNCH);
operation->mutable_launch()->add_task_infos()->CopyFrom(taskInfo);
schedulerLibrary.send(call);
}
AWAIT_READY(launch);
EXPECT_CALL(*scheduler, disconnected(_))
.Times(AtMost(1));
Future<Nothing> shutdown;
EXPECT_CALL(*executor, shutdown(_))
.WillOnce(FutureSatisfy(&shutdown));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::TEARDOWN);
schedulerLibrary.send(call);
}
// Ensure that the executor receives a `Event::Shutdown` after the
// scheduler exit.
AWAIT_READY(shutdown);
}
// This test verifies that a framework attempting to subscribe
// after its failover timeout has elapsed is disallowed.
TEST_F(HttpFaultToleranceTest, SchedulerSubscribeAfterFailoverTimeout)
{
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false;
v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_failover_timeout(Weeks(2).secs());
Try<Owned<cluster::Master>> master = StartMaster(flags);
ASSERT_SOME(master);
Future<Nothing> deactivateFramework = FUTURE_DISPATCH(
_, &master::allocator::MesosAllocatorProcess::deactivateFramework);
v1::FrameworkID frameworkId;
ContentType contentType = ContentType::PROTOBUF;
// Launch the first (i.e., failing) scheduler and wait until it receives
// a `SUBSCRIBED` event to launch the second (i.e., failover) scheduler.
{
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected));
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
Future<Event::Subscribed> subscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(frameworkInfo);
schedulerLibrary.send(call);
}
AWAIT_READY(subscribed);
frameworkId = subscribed->framework_id();
}
// Wait until master schedules the framework for removal.
AWAIT_READY(deactivateFramework);
// Simulate framework failover timeout.
Clock::pause();
Clock::settle();
Try<Duration> failoverTimeout =
Duration::create(frameworkInfo.failover_timeout());
ASSERT_SOME(failoverTimeout);
Future<Nothing> frameworkFailoverTimeout =
FUTURE_DISPATCH(_, &Master::frameworkFailoverTimeout);
Clock::advance(failoverTimeout.get());
Clock::resume();
// Wait until master actually marks the framework as completed.
AWAIT_READY(frameworkFailoverTimeout);
// Now launch the second (i.e., failover) scheduler using the
// framework id recorded from the first scheduler.
{
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected))
.WillRepeatedly(Return()); // Ignore future invocations.
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
// Framework should get `Error` event because the framework with this id
// is marked as completed.
Future<Nothing> error;
EXPECT_CALL(*scheduler, error(_, _))
.WillOnce(FutureSatisfy(&error));
EXPECT_CALL(*scheduler, disconnected(_))
.Times(AtMost(1));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
schedulerLibrary.send(call);
}
AWAIT_READY(error);
}
}
// This test checks that a failed over scheduler gets the retried status update
// when the original instance dies without acknowledging the update.
TEST_F(HttpFaultToleranceTest, SchedulerFailoverStatusUpdate)
{
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false;
Try<Owned<cluster::Master>> master = StartMaster(flags);
ASSERT_SOME(master);
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
auto executor = std::make_shared<v1::MockHTTPExecutor>();
ExecutorID executorId = DEFAULT_EXECUTOR_ID;
TestContainerizer containerizer(executorId, executor);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected))
.WillRepeatedly(Return()); // Ignore future invocations.
ContentType contentType = ContentType::PROTOBUF;
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
Future<Event::Subscribed> subscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
Future<Event::Offers> offers;
EXPECT_CALL(*scheduler, offers(_, _))
.WillOnce(FutureArg<1>(&offers));
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
schedulerLibrary.send(call);
}
AWAIT_READY(subscribed);
v1::FrameworkID frameworkId(subscribed->framework_id());
AWAIT_READY(offers);
EXPECT_NE(0, offers->offers().size());
EXPECT_CALL(*executor, connected(_))
.WillOnce(v1::executor::SendSubscribe(frameworkId, evolve(executorId)));
EXPECT_CALL(*executor, subscribed(_, _));
EXPECT_CALL(*executor, launch(_, _))
.WillOnce(v1::executor::SendUpdateFromTask(
frameworkId, evolve(executorId), v1::TASK_RUNNING));
Future<Nothing> acknowledged;
EXPECT_CALL(*executor, acknowledged(_, _))
.WillOnce(FutureSatisfy(&acknowledged));
Future<Event::Update> update;
EXPECT_CALL(*scheduler, update(_, _))
.WillOnce(FutureArg<1>(&update));
const v1::Offer& offer = offers->offers(0);
v1::TaskInfo taskInfo =
evolve(createTask(devolve(offer), "", executorId));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::ACCEPT);
Call::Accept* accept = call.mutable_accept();
accept->add_offer_ids()->CopyFrom(offer.id());
v1::Offer::Operation* operation = accept->add_operations();
operation->set_type(v1::Offer::Operation::LAUNCH);
operation->mutable_launch()->add_task_infos()->CopyFrom(taskInfo);
schedulerLibrary.send(call);
}
AWAIT_READY(acknowledged);
AWAIT_READY(update);
EXPECT_EQ(v1::TASK_RUNNING, update->status().state());
EXPECT_EQ(executorId, devolve(update->status().executor_id()));
EXPECT_TRUE(update->status().has_executor_id());
EXPECT_TRUE(update->status().has_uuid());
// Failover the scheduler without acknowledging the status update.
auto scheduler2 = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected2;
EXPECT_CALL(*scheduler2, connected(_))
.WillOnce(FutureSatisfy(&connected2));
// Failover to another scheduler instance.
v1::scheduler::TestMesos schedulerLibrary2(
master.get()->pid,
contentType,
scheduler2);
AWAIT_READY(connected2);
// The previously connected scheduler instance should receive an
// error/disconnected event.
Future<Nothing> error;
EXPECT_CALL(*scheduler, error(_, _))
.WillOnce(FutureSatisfy(&error));
Future<Nothing> disconnected;
EXPECT_CALL(*scheduler, disconnected(_))
.WillOnce(FutureSatisfy(&disconnected));
EXPECT_CALL(*scheduler2, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler2, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
// Scheduler2 should receive the retried status update.
Future<Nothing> update2;
EXPECT_CALL(*scheduler2, update(_, _))
.WillOnce(FutureSatisfy(&update2))
.WillRepeatedly(Return()); // Ignore subsequent updates.
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
schedulerLibrary2.send(call);
}
AWAIT_READY(error);
AWAIT_READY(disconnected);
AWAIT_READY(subscribed);
EXPECT_EQ(frameworkId, subscribed->framework_id());
Clock::pause();
// Now advance time enough for the reliable timeout to kick in and
// another status update to be sent.
Clock::advance(slave::STATUS_UPDATE_RETRY_INTERVAL_MIN);
AWAIT_READY(update2);
EXPECT_CALL(*executor, shutdown(_))
.Times(AtMost(1));
EXPECT_CALL(*executor, disconnected(_))
.Times(AtMost(1));
}
// This test ensures that the failed over scheduler is able to send a message
// to the executor.
TEST_F(HttpFaultToleranceTest, SchedulerFailoverFrameworkToExecutorMessage)
{
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false;
Try<Owned<cluster::Master>> master = StartMaster(flags);
ASSERT_SOME(master);
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
auto executor = std::make_shared<v1::MockHTTPExecutor>();
ExecutorID executorId = DEFAULT_EXECUTOR_ID;
TestContainerizer containerizer(executorId, executor);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected))
.WillRepeatedly(Return()); // Ignore future invocations.
ContentType contentType = ContentType::PROTOBUF;
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
Future<Event::Subscribed> subscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
Future<Event::Offers> offers;
EXPECT_CALL(*scheduler, offers(_, _))
.WillOnce(FutureArg<1>(&offers));
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
schedulerLibrary.send(call);
}
AWAIT_READY(subscribed);
v1::FrameworkID frameworkId(subscribed->framework_id());
AWAIT_READY(offers);
EXPECT_NE(0, offers->offers().size());
EXPECT_CALL(*executor, connected(_))
.WillOnce(v1::executor::SendSubscribe(frameworkId, evolve(executorId)));
EXPECT_CALL(*executor, subscribed(_, _));
Future<Nothing> launch;
EXPECT_CALL(*executor, launch(_, _))
.WillOnce(FutureSatisfy(&launch));
const v1::Offer& offer = offers->offers(0);
v1::TaskInfo taskInfo =
evolve(createTask(devolve(offer), "", executorId));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::ACCEPT);
Call::Accept* accept = call.mutable_accept();
accept->add_offer_ids()->CopyFrom(offer.id());
v1::Offer::Operation* operation = accept->add_operations();
operation->set_type(v1::Offer::Operation::LAUNCH);
operation->mutable_launch()->add_task_infos()->CopyFrom(taskInfo);
schedulerLibrary.send(call);
}
AWAIT_READY(launch);
auto scheduler2 = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected2;
EXPECT_CALL(*scheduler2, connected(_))
.WillOnce(FutureSatisfy(&connected2));
// Failover to another scheduler instance.
v1::scheduler::TestMesos schedulerLibrary2(
master.get()->pid,
contentType,
scheduler2);
AWAIT_READY(connected2);
// The previously connected scheduler instance should receive an
// error/disconnected event.
Future<Nothing> error;
EXPECT_CALL(*scheduler, error(_, _))
.WillOnce(FutureSatisfy(&error));
Future<Nothing> disconnected;
EXPECT_CALL(*scheduler, disconnected(_))
.WillOnce(FutureSatisfy(&disconnected));
EXPECT_CALL(*scheduler2, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler2, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
schedulerLibrary2.send(call);
}
AWAIT_READY(error);
AWAIT_READY(disconnected);
AWAIT_READY(subscribed);
EXPECT_EQ(frameworkId, subscribed->framework_id());
Future<v1::executor::Event::Message> message;
EXPECT_CALL(*executor, message(_, _))
.WillOnce(FutureArg<1>(&message));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::MESSAGE);
Call::Message* message = call.mutable_message();
message->mutable_agent_id()->CopyFrom(offer.agent_id());
message->mutable_executor_id()->CopyFrom(v1::DEFAULT_EXECUTOR_ID);
message->set_data("hello world");
schedulerLibrary2.send(call);
}
AWAIT_READY(message);
ASSERT_EQ("hello world", message->data());
EXPECT_CALL(*executor, shutdown(_))
.Times(AtMost(1));
EXPECT_CALL(*executor, disconnected(_))
.Times(AtMost(1));
}
// This test verifies that the scheduler will receive a `BadRequest` response
// when it tries to acknowledge a status update with a malformed UUID.
TEST_P(SchedulerHttpApiTest, MalformedUUID)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
v1::FrameworkID frameworkId;
string streamId;
// Subscribe once to get a valid stream ID.
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
ASSERT_EQ(Response::PIPE, response.get().type);
ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
streamId = response.get().headers.at("Mesos-Stream-Id");
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check that the event type is subscribed and the framework ID is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
frameworkId = event.get().get().subscribed().framework_id();
EXPECT_NE("", frameworkId.value());
}
// Make an acknowledge call with a malformed UUID. This should result in a
// `BadResponse`.
{
headers["Mesos-Stream-Id"] = streamId;
Call call;
call.set_type(Call::ACKNOWLEDGE);
// Set the framework ID in the subscribe call.
call.mutable_framework_id()->CopyFrom(frameworkId);
Call::Acknowledge* acknowledge = call.mutable_acknowledge();
acknowledge->mutable_task_id()->set_value("task-id");
acknowledge->mutable_agent_id()->set_value("agent-id");
// Set a malformed uuid.
acknowledge->set_uuid("bad-uuid");
Future<Response> response = process::http::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
AWAIT_EXPECT_RESPONSE_BODY_EQ(
"Failed to validate scheduler::Call: Not a valid UUID", response);
}
}
// This test verifies that the scheduler will receive a `BadRequest` response
// when a teardown call is made with an incorrect stream ID header.
TEST_P(SchedulerHttpApiTest, TeardownWrongStreamId)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
v1::FrameworkID frameworkId;
string streamId;
// Subscribe once to get a valid stream ID.
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
streamId = response.get().headers.at("Mesos-Stream-Id");
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check that the event type is subscribed and the framework ID is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
EXPECT_NE("", event.get().get().subscribed().framework_id().value());
frameworkId = event.get().get().subscribed().framework_id();
}
// Subscribe again to invalidate the first stream ID and acquire another one.
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
// Set the framework ID in the subscribe call.
call.mutable_framework_id()->CopyFrom(frameworkId);
subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
// Make sure that the new stream ID is different.
ASSERT_NE(streamId, response.get().headers.at("Mesos-Stream-Id"));
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
EXPECT_NE("", event.get().get().subscribed().framework_id().value());
}
{
Call call;
call.set_type(Call::TEARDOWN);
call.mutable_framework_id()->CopyFrom(frameworkId);
// Send the first (now incorrect) stream ID with the teardown call.
headers["Mesos-Stream-Id"] = streamId;
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
}
}
// This test verifies if the scheduler is able to receive a Subscribed
// event and heartbeat events on the stream in response to a Subscribe
// call request.
TEST_P(SchedulerHttpApiTest, Subscribe)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
EXPECT_NE("", response.get().headers.at("Mesos-Stream-Id"));
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check event type is subscribed and the framework id is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
EXPECT_NE("", event.get().get().subscribed().framework_id().value());
// Make sure it receives a heartbeat.
event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
ASSERT_EQ(Event::HEARTBEAT, event.get().get().type());
// Advance the clock to receive another heartbeat.
Clock::pause();
Clock::advance(DEFAULT_HEARTBEAT_INTERVAL);
event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
ASSERT_EQ(Event::HEARTBEAT, event.get().get().type());
}
// This test verifies if the scheduler can subscribe on retrying,
// e.g. after a ZK blip.
TEST_P(SchedulerHttpApiTest, SubscribedOnRetry)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
v1::FrameworkID frameworkId;
{
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
ASSERT_EQ(Response::PIPE, response.get().type);
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
frameworkId = event.get().get().subscribed().framework_id();
// Check event type is subscribed and the framework id is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
EXPECT_NE("", event.get().get().subscribed().framework_id().value());
}
{
call.mutable_framework_id()->CopyFrom(frameworkId);
subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
// Check if we were successfully able to subscribe after the blip.
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check event type is subscribed and the same framework id is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
EXPECT_EQ(frameworkId, event.get().get().subscribed().framework_id());
// Make sure it receives a heartbeat.
event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
ASSERT_EQ(Event::HEARTBEAT, event.get().get().type());
}
}
// This test verifies if we are able to upgrade from a PID based
// scheduler to HTTP scheduler.
TEST_P(SchedulerHttpApiTest, UpdatePidToHttpScheduler)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_failover_timeout(Weeks(2).secs());
MockScheduler sched;
StandaloneMasterDetector detector(master.get()->pid);
TestingMesosSchedulerDriver driver(&sched, &detector, devolve(frameworkInfo));
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
// Check that driver is notified with an error when the http
// framework is connected.
Future<FrameworkErrorMessage> errorMessage =
FUTURE_PROTOBUF(FrameworkErrorMessage(), _, _);
EXPECT_CALL(sched, error(_, _));
driver.start();
AWAIT_READY(frameworkId);
EXPECT_NE("", frameworkId.get().value());
// Now try to subscribe as an HTTP framework.
Call call;
call.set_type(Call::SUBSCRIBE);
call.mutable_framework_id()->CopyFrom(evolve(frameworkId.get()));
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(frameworkInfo);
subscribe->mutable_framework_info()->mutable_id()->
CopyFrom(evolve(frameworkId.get()));
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check event type is subscribed and the framework id is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
EXPECT_EQ(evolve(frameworkId.get()),
event.get().get().subscribed().framework_id());
// Make sure it receives a heartbeat.
event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
ASSERT_EQ(Event::HEARTBEAT, event.get().get().type());
driver.stop();
driver.join();
}
