// Test that memory pressure listening is restarted after recovery. TEST_F(MemoryPressureMesosTest, ROOT_CGROUPS_SlaveRecovery) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); // We only care about memory cgroup for this test. flags.isolation = "cgroups/mem"; Fetcher fetcher(flags); Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), containerizer.get(), flags); ASSERT_SOME(slave); MockScheduler sched; // Enable checkpointing for the framework. FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); MesosSchedulerDriver driver( &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); Offer offer = offers.get()[0]; // Run a task that triggers memory pressure event. We request 1G // disk because we are going to write a 512 MB file repeatedly. TaskInfo task = createTask( offer.slave_id(), Resources::parse("cpus:1;mem:256;disk:1024").get(), "while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done"); Future<TaskStatus> starting; Future<TaskStatus> running; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&starting)) .WillOnce(FutureArg<1>(&running)) .WillRepeatedly(Return()); // Ignore subsequent updates. Future<Nothing> runningAck = FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement); Future<Nothing> startingAck = FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(starting); EXPECT_EQ(task.task_id(), starting->task_id()); EXPECT_EQ(TASK_STARTING, starting->state()); AWAIT_READY_FOR(startingAck, Seconds(120)); AWAIT_READY(running); EXPECT_EQ(task.task_id(), running->task_id()); EXPECT_EQ(TASK_RUNNING, running->state()); // Wait for the ACK to be checkpointed. AWAIT_READY_FOR(runningAck, Seconds(120)); // We restart the slave to let it recover. slave.get()->terminate(); // Set up so we can wait until the new slave updates the container's // resources (this occurs after the executor has reregistered). Future<Nothing> update = FUTURE_DISPATCH(_, &MesosContainerizerProcess::update); // Use the same flags. _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); containerizer.reset(_containerizer.get()); Future<SlaveReregisteredMessage> reregistered = FUTURE_PROTOBUF(SlaveReregisteredMessage(), master.get()->pid, _); slave = StartSlave(detector.get(), containerizer.get(), flags); ASSERT_SOME(slave); AWAIT_READY(reregistered); // Wait until the containerizer is updated. AWAIT_READY(update); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers->size()); ContainerID containerId = *(containers->begin()); // Wait a while for some memory pressure events to occur. Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); if (usage->mem_low_pressure_counter() > 0) { // We will check the correctness of the memory pressure counters // later, because the memory-hammering task is still active // and potentially incrementing these counters. break; } os::sleep(Milliseconds(100)); waited += Milliseconds(100); } while (waited < Seconds(5)); EXPECT_LE(waited, Seconds(5)); // Pause the clock to ensure that the reaper doesn't reap the exited // command executor and inform the containerizer/slave. Clock::pause(); Clock::settle(); Future<TaskStatus> killed; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&killed)); // Stop the memory-hammering task. driver.killTask(task.task_id()); AWAIT_READY_FOR(killed, Seconds(120)); EXPECT_EQ(task.task_id(), killed->task_id()); EXPECT_EQ(TASK_KILLED, killed->state()); // Now check the correctness of the memory pressure counters. Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); EXPECT_GE(usage->mem_low_pressure_counter(), usage->mem_medium_pressure_counter()); EXPECT_GE(usage->mem_medium_pressure_counter(), usage->mem_critical_pressure_counter()); Clock::resume(); driver.stop(); driver.join(); }
// This test verifies that the slave reports pending tasks when // re-registering, otherwise the master will report them as being // lost. TEST_F(MasterSlaveReconciliationTest, SlaveReregisterPendingTask) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); StandaloneMasterDetector detector(master.get()->pid); Try<Owned<cluster::Slave>> slave = StartSlave(&detector); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); // No TASK_LOST updates should occur! EXPECT_CALL(sched, statusUpdate(&driver, _)) .Times(0); // We drop the _runTask dispatch to ensure the task remains // pending in the slave. Future<Nothing> _runTask = DROP_DISPATCH(slave.get()->pid, &Slave::_runTask); TaskInfo task1; task1.set_name("test task"); task1.mutable_task_id()->set_value("1"); task1.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task1.mutable_resources()->MergeFrom(offers.get()[0].resources()); task1.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); driver.launchTasks(offers.get()[0].id(), {task1}); AWAIT_READY(_runTask); Future<SlaveReregisteredMessage> slaveReregisteredMessage = FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _); // Simulate a spurious master change event (e.g., due to ZooKeeper // expiration) at the slave to force re-registration. detector.appoint(master.get()->pid); AWAIT_READY(slaveReregisteredMessage); Clock::pause(); Clock::settle(); Clock::resume(); driver.stop(); driver.join(); }
// This test verifies that when the slave re-registers, we correctly // send the information about actively running frameworks. TEST_F(MasterSlaveReconciliationTest, SlaveReregisterFrameworks) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task; task.set_name("test task"); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); EXPECT_CALL(exec, registered(_, _, _, _)); // Send an update right away. EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<Nothing> _statusUpdate = FUTURE_DISPATCH(_, &Slave::_statusUpdate); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)) .WillRepeatedly(Return()); // Ignore retried update due to update framework. driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(_statusUpdate); Future<ReregisterSlaveMessage> reregisterSlave = FUTURE_PROTOBUF(ReregisterSlaveMessage(), _, _); // Simulate a spurious master change event (e.g., due to ZooKeeper // expiration) at the slave to force re-registration. detector.appoint(master.get()->pid); // Expect to receive the 'ReregisterSlaveMessage' containing the // active frameworks. AWAIT_READY(reregisterSlave); EXPECT_EQ(1u, reregisterSlave.get().frameworks().size()); Clock::pause(); Clock::settle(); AWAIT_READY(status); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
// The purpose of this test is to ensure that when slaves are removed // from the master, and then attempt to send exited executor messages, // we send a ShutdownMessage to the slave. Why? Because during a // network partition, the master will remove a partitioned slave, thus // sending its tasks to LOST. At this point, when the partition is // removed, the slave may attempt to send exited executor messages if // it was unaware that the master removed it. We've already // notified frameworks that the tasks under the executors were LOST, // so we have to have the slave shut down. TEST_F(PartitionTest, PartitionedSlaveExitedExecutor) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); // Allow the master to PING the slave, but drop all PONG messages // from the slave. Note that we don't match on the master / slave // PIDs because it's actually the SlaveObserver Process that sends // the pings. Future<Message> ping = FUTURE_MESSAGE(Eq("PING"), _, _); DROP_MESSAGES(Eq("PONG"), _, _); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Try<PID<Slave> > slave = StartSlave(&containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId));\ Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); driver.start(); AWAIT_READY(frameworkId); AWAIT_READY(offers); ASSERT_NE(0u, offers.get().size()); // Launch a task. This allows us to have the slave send an // ExitedExecutorMessage. TaskID taskId; taskId.set_value("1"); TaskInfo task; task.set_name(""); task.mutable_task_id()->MergeFrom(taskId); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); task.mutable_executor()->mutable_command()->set_value("sleep 60"); vector<TaskInfo> tasks; tasks.push_back(task); // Set up the expectations for launching the task. EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); // Drop all the status updates from the slave, so that we can // ensure the ExitedExecutorMessage is what triggers the slave // shutdown. DROP_PROTOBUFS(StatusUpdateMessage(), _, master.get()); driver.launchTasks(offers.get()[0].id(), tasks); // Drop the first shutdown message from the master (simulated // partition) and allow the second shutdown message to pass when // triggered by the ExitedExecutorMessage. Future<ShutdownMessage> shutdownMessage = DROP_PROTOBUF(ShutdownMessage(), _, slave.get()); Future<TaskStatus> lostStatus; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&lostStatus)); Future<Nothing> slaveLost; EXPECT_CALL(sched, slaveLost(&driver, _)) .WillOnce(FutureSatisfy(&slaveLost)); Clock::pause(); // Now, induce a partition of the slave by having the master // timeout the slave. uint32_t pings = 0; while (true) { AWAIT_READY(ping); pings++; if (pings == master::MAX_SLAVE_PING_TIMEOUTS) { break; } ping = FUTURE_MESSAGE(Eq("PING"), _, _); Clock::advance(master::SLAVE_PING_TIMEOUT); Clock::settle(); } Clock::advance(master::SLAVE_PING_TIMEOUT); Clock::settle(); // The master will have notified the framework of the lost task. AWAIT_READY(lostStatus); EXPECT_EQ(TASK_LOST, lostStatus.get().state()); // Wait for the master to attempt to shut down the slave. AWAIT_READY(shutdownMessage); // The master will notify the framework that the slave was lost. AWAIT_READY(slaveLost); shutdownMessage = FUTURE_PROTOBUF(ShutdownMessage(), _, slave.get()); // Induce an ExitedExecutorMessage from the slave. containerizer.destroy( frameworkId.get(), DEFAULT_EXECUTOR_INFO.executor_id()); // Upon receiving the message, the master will shutdown the slave. AWAIT_READY(shutdownMessage); Clock::resume(); driver.stop(); driver.join(); Shutdown(); }
// This test verifies that the master reconciles tasks that are // missing from a re-registering slave. In this case, we drop the // RunTaskMessage so the slave should send TASK_LOST. TEST_F(MasterSlaveReconciliationTest, ReconcileLostTask) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); StandaloneMasterDetector detector(master.get()->pid); Try<Owned<cluster::Slave>> slave = StartSlave(&detector); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task; task.set_name("test task"); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); // We now launch a task and drop the corresponding RunTaskMessage on // the slave, to ensure that only the master knows about this task. Future<RunTaskMessage> runTaskMessage = DROP_PROTOBUF(RunTaskMessage(), _, _); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(runTaskMessage); Future<SlaveReregisteredMessage> slaveReregisteredMessage = FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _); Future<StatusUpdateMessage> statusUpdateMessage = FUTURE_PROTOBUF(StatusUpdateMessage(), _, master.get()->pid); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); // Simulate a spurious master change event (e.g., due to ZooKeeper // expiration) at the slave to force re-registration. detector.appoint(master.get()->pid); AWAIT_READY(slaveReregisteredMessage); // Make sure the slave generated the TASK_LOST. AWAIT_READY(statusUpdateMessage); AWAIT_READY(status); ASSERT_EQ(task.task_id(), status.get().task_id()); ASSERT_EQ(TASK_LOST, status.get().state()); // Before we obtain the metrics, ensure that the master has finished // processing the status update so metrics have been updated. Clock::pause(); Clock::settle(); Clock::resume(); // Check metrics. JSON::Object stats = Metrics(); EXPECT_EQ(1u, stats.values.count("master/tasks_lost")); EXPECT_EQ(1u, stats.values["master/tasks_lost"]); EXPECT_EQ( 1u, stats.values.count( "master/task_lost/source_slave/reason_reconciliation")); EXPECT_EQ( 1u, stats.values["master/task_lost/source_slave/reason_reconciliation"]); driver.stop(); driver.join(); }
// This test ensures that when explicit acknowledgements are enabled, // acknowledgements for master-generated updates are dropped by the // driver. We test this by creating an invalid task that uses no // resources. TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgementsMasterGeneratedUpdate) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get()); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, false, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. // Ensure no status update acknowledgements are sent to the master. EXPECT_NO_FUTURE_CALLS( mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _ , master.get()->pid); driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers->size()); // Launch a task using no resources. TaskInfo task; task.set_name(""); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); vector<TaskInfo> tasks; tasks.push_back(task); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); driver.launchTasks(offers.get()[0].id(), tasks); AWAIT_READY(status); ASSERT_EQ(TASK_ERROR, status->state()); ASSERT_EQ(TaskStatus::SOURCE_MASTER, status->source()); ASSERT_EQ(TaskStatus::REASON_TASK_INVALID, status->reason()); // Now send the acknowledgement. driver.acknowledgeStatusUpdate(status.get()); // Settle the clock to ensure driver processes the acknowledgement, // which should get dropped due to having come from the master. Clock::pause(); Clock::settle(); driver.stop(); driver.join(); }
bool InteractiveMarker::processMessage( const visualization_msgs::InteractiveMarker& message ) { boost::recursive_mutex::scoped_lock lock(mutex_); // copy values name_ = message.name; description_ = message.description; if ( message.controls.size() == 0 ) { Q_EMIT statusUpdate( StatusProperty::Ok, name_, "Marker empty."); return false; } scale_ = message.scale; reference_frame_ = message.header.frame_id; reference_time_ = message.header.stamp; frame_locked_ = (message.header.stamp == ros::Time(0)); position_ = Ogre::Vector3( message.pose.position.x, message.pose.position.y, message.pose.position.z ); orientation_ = Ogre::Quaternion( message.pose.orientation.w, message.pose.orientation.x, message.pose.orientation.y, message.pose.orientation.z ); pose_changed_ =false; time_since_last_feedback_ = 0; // setup axes axes_->setPosition(position_); axes_->setOrientation(orientation_); axes_->set( scale_, scale_*0.05 ); has_menu_ = message.menu_entries.size() > 0; updateReferencePose(); // Instead of just erasing all the old controls and making new ones // here, we want to preserve as much as possible from the old ones, // so that we don't lose the drag action in progress if a control is // being dragged when this update comes in. // // Controls are stored in a map from control name to control // pointer, so we loop over the incoming control messages looking // for names we already know about. When we find them, we just call // the control's processMessage() function to update it. When we // don't find them, we create a new Control. We also keep track of // which control names we used to have but which are not present in // the incoming message, which we use to delete the unwanted // controls. // Make set of old-names called old-names-to-delete. std::set<std::string> old_names_to_delete; M_ControlPtr::const_iterator ci; for( ci = controls_.begin(); ci != controls_.end(); ci++ ) { old_names_to_delete.insert( (*ci).first ); } // Loop over new array: for ( unsigned i = 0; i < message.controls.size(); i++ ) { const visualization_msgs::InteractiveMarkerControl& control_message = message.controls[ i ]; M_ControlPtr::iterator search_iter = controls_.find( control_message.name ); InteractiveMarkerControlPtr control; // If message->name in map, if( search_iter != controls_.end() ) { // Use existing control control = (*search_iter).second; } else { // Else make new control control = boost::make_shared<InteractiveMarkerControl>( context_, reference_node_, this ); controls_[ control_message.name ] = control; } // Update the control with the message data control->processMessage( control_message ); control->setShowVisualAids( show_visual_aids_ ); // Remove message->name from old-names-to-delete old_names_to_delete.erase( control_message.name ); } // Loop over old-names-to-delete std::set<std::string>::iterator si; for( si = old_names_to_delete.begin(); si != old_names_to_delete.end(); si++ ) { // Remove Control object from map for name-to-delete controls_.erase( *si ); } description_control_ = boost::make_shared<InteractiveMarkerControl>( context_, reference_node_, this ); description_control_->processMessage( interactive_markers::makeTitle( message )); //create menu menu_entries_.clear(); menu_.reset(); if ( has_menu_ ) { menu_.reset( new QMenu() ); top_level_menu_ids_.clear(); // Put all menu entries into the menu_entries_ map and create the // tree of menu entry ids. for ( unsigned m=0; m < message.menu_entries.size(); m++ ) { const visualization_msgs::MenuEntry& entry = message.menu_entries[ m ]; MenuNode node; node.entry = entry; menu_entries_[ entry.id ] = node; if( entry.parent_id == 0 ) { top_level_menu_ids_.push_back( entry.id ); } else { // Find the parent node and add this entry to the parent's list of children. std::map< uint32_t, MenuNode >::iterator parent_it = menu_entries_.find( entry.parent_id ); if( parent_it == menu_entries_.end() ) { ROS_ERROR("interactive marker menu entry %u found before its parent id %u. Ignoring.", entry.id, entry.parent_id); } else { (*parent_it).second.child_ids.push_back( entry.id ); } } } populateMenu( menu_.get(), top_level_menu_ids_ ); } if ( frame_locked_ ) { std::ostringstream s; s << "Locked to frame " << reference_frame_; Q_EMIT statusUpdate( StatusProperty::Ok, name_, s.str() ); } else { Q_EMIT statusUpdate( StatusProperty::Ok, name_, "Position is fixed." ); } return true; }
TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_Statistics) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); // We only care about memory cgroup for this test. flags.isolation = "cgroups/mem"; flags.agent_subsystems = None(); Fetcher fetcher; Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), containerizer.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); Offer offer = offers.get()[0]; // Run a task that triggers memory pressure event. We request 1G // disk because we are going to write a 512 MB file repeatedly. TaskInfo task = createTask( offer.slave_id(), Resources::parse("cpus:1;mem:256;disk:1024").get(), "while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done"); Future<TaskStatus> running; Future<TaskStatus> killed; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&running)) .WillOnce(FutureArg<1>(&killed)) .WillRepeatedly(Return()); // Ignore subsequent updates. driver.launchTasks(offer.id(), {task}); AWAIT_READY(running); EXPECT_EQ(task.task_id(), running.get().task_id()); EXPECT_EQ(TASK_RUNNING, running.get().state()); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers.get().size()); ContainerID containerId = *(containers.get().begin()); // Wait a while for some memory pressure events to occur. Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); if (usage.get().mem_low_pressure_counter() > 0) { // We will check the correctness of the memory pressure counters // later, because the memory-hammering task is still active // and potentially incrementing these counters. break; } os::sleep(Milliseconds(100)); waited += Milliseconds(100); } while (waited < Seconds(5)); EXPECT_LE(waited, Seconds(5)); // Pause the clock to ensure that the reaper doesn't reap the exited // command executor and inform the containerizer/slave. Clock::pause(); Clock::settle(); // Stop the memory-hammering task. driver.killTask(task.task_id()); AWAIT_READY(killed); EXPECT_EQ(task.task_id(), killed->task_id()); EXPECT_EQ(TASK_KILLED, killed->state()); // Now check the correctness of the memory pressure counters. Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); EXPECT_GE(usage.get().mem_low_pressure_counter(), usage.get().mem_medium_pressure_counter()); EXPECT_GE(usage.get().mem_medium_pressure_counter(), usage.get().mem_critical_pressure_counter()); Clock::resume(); driver.stop(); driver.join(); }
TEST_F(StatusUpdateManagerTest, RetryStatusUpdate) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); slave::Flags flags = CreateSlaveFlags(); Try<PID<Slave> > slave = StartSlave(&exec, flags); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)) .Times(1); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); EXPECT_CALL(exec, registered(_, _, _, _)) .Times(1); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<StatusUpdateMessage> statusUpdateMessage = DROP_PROTOBUF(StatusUpdateMessage(), master.get(), _); Clock::pause(); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(statusUpdateMessage); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); Clock::advance(slave::STATUS_UPDATE_RETRY_INTERVAL_MIN); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); Clock::resume(); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); }
bool InitDriveThread::method_resizePartitions() { int newStartOfRescuePartition = getFileContents("/sys/class/block/mmcblk0p1/start").trimmed().toInt(); int newSizeOfRescuePartition = sizeofBootFilesInKB()*1.024/1000 + 100; if (!umountSystemPartition()) { emit error(tr("Error unmounting system partition.")); return false; } if (!QFile::exists("/dev/mmcblk0p1")) { // SD card does not have a MBR. // Warn user that their SD card does not have an MBR and ask // if they would like us to create one for them QMessageBox::StandardButton answer; emit query(tr("Would you like NOOBS to create one for you?\nWARNING: This will erase all data on your SD card"), tr("Error: No MBR present on SD Card"), &answer); if(answer == QMessageBox::Yes) { emit statusUpdate(tr("Zeroing partition table")); if (!zeroMbr()) { emit error(tr("Error zero'ing MBR/GPT. SD card may be broken or advertising wrong capacity.")); return false; } // Create MBR containing single FAT partition emit statusUpdate(tr("Writing new MBR")); QProcess proc; proc.setProcessChannelMode(proc.MergedChannels); proc.start("/usr/sbin/parted /dev/mmcblk0 --script -- mktable msdos mkpartfs primary fat32 8192s -1"); proc.waitForFinished(-1); if (proc.exitCode() != 0) { // Warn user if we failed to create an MBR on their card emit error(tr("Error creating MBR")+"\n"+proc.readAll()); return false; } qDebug() << "Created missing MBR on SD card. parted output:" << proc.readAll(); // Advise user that their SD card has now been formatted // suitably for installing NOOBS and that they will have to // re-copy the files before rebooting emit error(tr("SD card has now been formatted ready for NOOBS installation. Please re-copy the NOOBS files onto the card and reboot")); return false; } else { emit error(tr("SD card has not been formatted correctly. Please reformat using the SD Association Formatting Tool and try again.")); return false; } } emit statusUpdate(tr("Removing partitions 2,3,4")); QFile f("/dev/mmcblk0"); f.open(f.ReadWrite); // Seek to partition entry 2 f.seek(462); // Zero out partition 2,3,4 to prevent parted complaining about invalid constraints f.write(QByteArray(16*3, '\0')); f.flush(); // Tell Linux to re-read the partition table ioctl(f.handle(), BLKRRPART); f.close(); QThread::msleep(500); emit statusUpdate(tr("Resizing FAT partition")); /* Relocating the start of the FAT partition is a write intensive operation * only move it when it is not aligned on a MiB boundary already */ if (newStartOfRescuePartition < 2048 || newStartOfRescuePartition % 2048 != 0) { newStartOfRescuePartition = PARTITION_ALIGNMENT; /* 4 MiB */ } QString cmd = "/usr/sbin/parted --script /dev/mmcblk0 resize 1 "+QString::number(newStartOfRescuePartition)+"s "+QString::number(newSizeOfRescuePartition)+"M"; qDebug() << "Executing" << cmd; QProcess p; QProcessEnvironment env = QProcessEnvironment::systemEnvironment(); /* Suppress parted's big fat warning about its file system manipulation code not being robust. It distracts from any real error messages that may follow it. */ env.insert("PARTED_SUPPRESS_FILE_SYSTEM_MANIPULATION_WARNING", "1"); p.setProcessEnvironment(env); p.setProcessChannelMode(p.MergedChannels); p.start(cmd); p.closeWriteChannel(); p.waitForFinished(-1); if (p.exitCode() != 0) { emit error(tr("Error resizing existing FAT partition")+"\n"+p.readAll()); return false; } qDebug() << "parted done, output:" << p.readAll(); QThread::msleep(500); emit statusUpdate(tr("Creating extended partition")); QByteArray partitionTable; int startOfOurPartition = getFileContents("/sys/class/block/mmcblk0p1/start").trimmed().toInt(); int sizeOfOurPartition = getFileContents("/sys/class/block/mmcblk0p1/size").trimmed().toInt(); int startOfExtended = startOfOurPartition+sizeOfOurPartition; // Align start of settings partition on 4 MiB boundary int startOfSettings = startOfExtended + PARTITION_GAP; if (startOfSettings % PARTITION_ALIGNMENT != 0) startOfSettings += PARTITION_ALIGNMENT-(startOfSettings % PARTITION_ALIGNMENT); // Primary partitions partitionTable = QByteArray::number(startOfOurPartition)+","+QByteArray::number(sizeOfOurPartition)+",0E\n"; /* FAT partition */ partitionTable += QByteArray::number(startOfExtended)+",,X\n"; /* Extended partition with all remaining space */ partitionTable += "0,0\n"; partitionTable += "0,0\n"; // Logical partitions partitionTable += QByteArray::number(startOfSettings)+","+QByteArray::number(SETTINGS_PARTITION_SIZE)+",L\n"; /* Settings partition */ qDebug() << "Writing partition table" << partitionTable; /* Let sfdisk write a proper partition table */ cmd = QString("/sbin/sfdisk -uS /dev/mmcblk0"); QProcess proc; proc.setProcessChannelMode(proc.MergedChannels); proc.start(cmd); proc.write(partitionTable); proc.closeWriteChannel(); proc.waitForFinished(-1); if (proc.exitCode() != 0) { emit error(tr("Error creating extended partition")+"\n"+proc.readAll()); return false; } qDebug() << "sfdisk done, output:" << proc.readAll(); QThread::msleep(500); /* For reasons unknown Linux sometimes * only finds /dev/mmcblk0p2 and /dev/mmcblk0p1 goes missing */ if (!QFile::exists("/dev/mmcblk0p1")) { /* Probe again */ QProcess::execute("/usr/sbin/partprobe"); QThread::msleep(1500); } QProcess::execute("/sbin/mlabel p:RECOVERY"); return true; }
void InitDriveThread::run() { QDir dir; emit statusUpdate("Waiting for SD card to be ready"); while (!QFile::exists("/dev/mmcblk0")) { QThread::usleep(100); } emit statusUpdate(tr("Mounting FAT partition")); mountSystemPartition(); // Try to resize existing partitions if (!method_resizePartitions()) return; emit statusUpdate(tr("Formatting settings partition")); if (!formatSettingsPartition()) { emit error(tr("Error formatting settings partition")); return; } emit statusUpdate(tr("Mounting FAT partition")); if (!mountSystemPartition()) { emit error(tr("Error mounting system partition.")); return; } dir.mkdir("/mnt/os"); emit statusUpdate(tr("Editing cmdline.txt")); QString cmdlinefilename = "/mnt/recovery.cmdline"; if (!QFile::exists(cmdlinefilename)) cmdlinefilename = "/mnt/cmdline.txt"; /* Remove "runinstaller" from cmdline.txt */ QFile f(cmdlinefilename); if (!f.open(f.ReadOnly)) { emit error(tr("Error opening %1").arg(cmdlinefilename)); return; } QByteArray line = f.readAll().trimmed(); line = line.replace("runinstaller", "").trimmed(); f.close(); f.open(f.WriteOnly); f.write(line); f.close(); #ifdef RISCOS_BLOB_FILENAME if (QFile::exists(RISCOS_BLOB_FILENAME)) { emit statusUpdate(tr("Writing RiscOS blob")); if (!writeRiscOSblob()) { emit error(tr("Error writing RiscOS blob")); return; } } #endif /* Finish writing */ emit statusUpdate(tr("Unmounting boot partition")); umountSystemPartition(); emit statusUpdate(tr("Finish writing to disk (sync)")); sync(); /* Perform a quick test to verify our changes were written * Drop page cache to make sure we are reading from card, and not from cache */ QFile dc("/proc/sys/vm/drop_caches"); dc.open(f.WriteOnly); dc.write("3\n"); dc.close(); emit statusUpdate(tr("Mounting boot partition again")); mountSystemPartition(); /* Verify that cmdline.txt was written correctly */ f.open(f.ReadOnly); QByteArray cmdlineread = f.readAll(); f.close(); umountSystemPartition(); if (cmdlineread != line) { emit error(tr("SD card broken (writes do not persist)")); return; } emit completed(); }
// This test checks the behavior of passed invalid limits. TEST_F(PosixRLimitsIsolatorTest, InvalidLimits) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "posix/rlimits"; Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_NE(0u, offers->size()); TaskInfo task = createTask( offers.get()[0].slave_id(), offers.get()[0].resources(), "true"); ContainerInfo* container = task.mutable_container(); container->set_type(ContainerInfo::MESOS); // Set impossible limit soft > hard. RLimitInfo rlimitInfo; RLimitInfo::RLimit* rlimit = rlimitInfo.add_rlimits(); rlimit->set_type(RLimitInfo::RLimit::RLMT_CPU); rlimit->set_soft(100); rlimit->set_hard(1); container->mutable_rlimit_info()->CopyFrom(rlimitInfo); Future<TaskStatus> taskStatus; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&taskStatus)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(taskStatus); EXPECT_EQ(task.task_id(), taskStatus->task_id()); EXPECT_EQ(TASK_FAILED, taskStatus->state()); EXPECT_EQ(TaskStatus::REASON_EXECUTOR_TERMINATED, taskStatus->reason()); driver.stop(); driver.join(); }
// This test confirms that if a task exceeds configured resource // limits it is forcibly terminated. TEST_F(PosixRLimitsIsolatorTest, TaskExceedingLimit) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "posix/rlimits"; Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_NE(0u, offers->size()); // The task attempts to use an infinite amount of CPU time. TaskInfo task = createTask( offers.get()[0].slave_id(), offers.get()[0].resources(), "while true; do true; done"); ContainerInfo* container = task.mutable_container(); container->set_type(ContainerInfo::MESOS); // Limit the process to use maximally 1 second of CPU time. RLimitInfo rlimitInfo; RLimitInfo::RLimit* cpuLimit = rlimitInfo.add_rlimits(); cpuLimit->set_type(RLimitInfo::RLimit::RLMT_CPU); cpuLimit->set_soft(1); cpuLimit->set_hard(1); container->mutable_rlimit_info()->CopyFrom(rlimitInfo); Future<TaskStatus> statusRunning; Future<TaskStatus> statusFailed; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFailed)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(task.task_id(), statusRunning->task_id()); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); AWAIT_READY(statusFailed); EXPECT_EQ(task.task_id(), statusFailed->task_id()); EXPECT_EQ(TASK_FAILED, statusFailed->state()); driver.stop(); driver.join(); }
// This test confirms that setting no values for the soft and hard // limits implies an unlimited resource. TEST_F(PosixRLimitsIsolatorTest, UnsetLimits) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "posix/rlimits"; Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_NE(0u, offers->size()); TaskInfo task = createTask( offers.get()[0].slave_id(), offers.get()[0].resources(), "exit `ulimit -c | grep -q unlimited`"); // Force usage of C locale as we interpret a potentially translated // string in the task's command. mesos::Environment::Variable* locale = task.mutable_command()->mutable_environment()->add_variables(); locale->set_name("LC_ALL"); locale->set_value("C"); ContainerInfo* container = task.mutable_container(); container->set_type(ContainerInfo::MESOS); // Setting rlimit for core without soft or hard limit signifies // unlimited range. RLimitInfo rlimitInfo; RLimitInfo::RLimit* rlimit = rlimitInfo.add_rlimits(); rlimit->set_type(RLimitInfo::RLimit::RLMT_CORE); container->mutable_rlimit_info()->CopyFrom(rlimitInfo); Future<TaskStatus> statusRunning; Future<TaskStatus> statusFinal; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFinal)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(task.task_id(), statusRunning->task_id()); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); AWAIT_READY(statusFinal); EXPECT_EQ(task.task_id(), statusFinal->task_id()); EXPECT_EQ(TASK_FINISHED, statusFinal->state()); driver.stop(); driver.join(); }
// This test verifies that the environment secrets are resolved when launching a // task. TEST_F(EnvironmentSecretIsolatorTest, ResolveSecret) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); mesos::internal::slave::Flags flags = CreateSlaveFlags(); Fetcher fetcher(flags); Try<SecretResolver*> secretResolver = SecretResolver::create(); EXPECT_SOME(secretResolver); Try<MesosContainerizer*> containerizer = MesosContainerizer::create(flags, false, &fetcher, secretResolver.get()); EXPECT_SOME(containerizer); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), containerizer.get()); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<std::vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_FALSE(offers->empty()); const string commandString = strings::format( "env; test \"$%s\" = \"%s\"", SECRET_ENV_NAME, SECRET_VALUE).get(); CommandInfo command; command.set_value(commandString); // Request a secret. // TODO(kapil): Update createEnvironment() to support secrets. mesos::Environment::Variable *env = command.mutable_environment()->add_variables(); env->set_name(SECRET_ENV_NAME); env->set_type(mesos::Environment::Variable::SECRET); mesos::Secret* secret = env->mutable_secret(); secret->set_type(Secret::VALUE); secret->mutable_value()->set_data(SECRET_VALUE); TaskInfo task = createTask( offers.get()[0].slave_id(), Resources::parse("cpus:0.1;mem:32").get(), command); // NOTE: Successful tasks will output two status updates. Future<TaskStatus> statusRunning; Future<TaskStatus> statusFinished; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFinished)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning.get().state()); AWAIT_READY(statusFinished); EXPECT_EQ(TASK_FINISHED, statusFinished.get().state()); driver.stop(); driver.join(); }
// This test verifies that status update manager ignores // duplicate ACK for an earlier update when it is waiting // for an ACK for a later update. This could happen when the // duplicate ACK is for a retried update. TEST_F(StatusUpdateManagerTest, IgnoreDuplicateStatusUpdateAck) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); Try<PID<Slave> > slave = StartSlave(&exec); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); FrameworkID frameworkId; EXPECT_CALL(sched, registered(_, _, _)) .WillOnce(SaveArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); // Drop the first update, so that status update manager // resends the update. Future<StatusUpdateMessage> statusUpdateMessage = DROP_PROTOBUF(StatusUpdateMessage(), master.get(), _); Clock::pause(); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(statusUpdateMessage); StatusUpdate update = statusUpdateMessage.get().update(); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); // This is the ACK for the retried update. Future<Nothing> ack = FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement); Clock::advance(slave::STATUS_UPDATE_RETRY_INTERVAL_MIN); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); AWAIT_READY(ack); // Now send TASK_FINISHED update so that the status update manager // is waiting for its ACK, which it never gets because we drop the // update. DROP_PROTOBUFS(StatusUpdateMessage(), master.get(), _); Future<Nothing> update2 = FUTURE_DISPATCH(_, &Slave::_statusUpdate); TaskStatus status2 = status.get(); status2.set_state(TASK_FINISHED); execDriver->sendStatusUpdate(status2); AWAIT_READY(update2); // This is to catch the duplicate ack for TASK_RUNNING. Future<Nothing> duplicateAck = FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement); // Now send a duplicate ACK for the TASK_RUNNING update. process::dispatch( slave.get(), &Slave::statusUpdateAcknowledgement, master.get(), update.slave_id(), frameworkId, update.status().task_id(), update.uuid()); AWAIT_READY(duplicateAck); Clock::resume(); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); }
// Ensures that when a scheduler enables explicit acknowledgements // on the driver, there are no implicit acknowledgements sent, and // the call to 'acknowledgeStatusUpdate' sends the ack to the master. TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgements) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, false, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 16, "*")) .WillRepeatedly(Return()); // Ignore subsequent offers. Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); // Ensure no status update acknowledgements are sent from the driver // to the master until the explicit acknowledgement is sent. EXPECT_NO_FUTURE_CALLS( mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _ , master.get()->pid); EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.start(); AWAIT_READY(status); // Settle the clock to ensure driver finishes processing the status // update, we want to ensure that no implicit acknowledgement gets // sent. Clock::pause(); Clock::settle(); // Now send the acknowledgement. Future<mesos::scheduler::Call> acknowledgement = FUTURE_CALL( mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _, master.get()->pid); driver.acknowledgeStatusUpdate(status.get()); AWAIT_READY(acknowledgement); driver.stop(); driver.join(); }
// This test verifies that status update manager ignores // unexpected ACK for an earlier update when it is waiting // for an ACK for another update. We do this by dropping ACKs // for the original update and sending a random ACK to the slave. TEST_F(StatusUpdateManagerTest, IgnoreUnexpectedStatusUpdateAck) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); Try<PID<Slave> > slave = StartSlave(&exec); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); FrameworkID frameworkId; EXPECT_CALL(sched, registered(_, _, _)) .WillOnce(SaveArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<StatusUpdateMessage> statusUpdateMessage = FUTURE_PROTOBUF(StatusUpdateMessage(), master.get(), _); // Drop the ACKs, so that status update manager // retries the update. DROP_CALLS(mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _, master.get()); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(statusUpdateMessage); StatusUpdate update = statusUpdateMessage.get().update(); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); Future<Nothing> unexpectedAck = FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement); // Now send an ACK with a random UUID. process::dispatch( slave.get(), &Slave::statusUpdateAcknowledgement, master.get(), update.slave_id(), frameworkId, update.status().task_id(), UUID::random().toBytes()); AWAIT_READY(unexpectedAck); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); }
TEST_P(MemoryIsolatorTest, ROOT_MemUsage) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = GetParam(); Fetcher fetcher(flags); Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave( detector.get(), containerizer.get()); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); TaskInfo task = createTask(offers.get()[0], "sleep 120"); Future<TaskStatus> statusRunning; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers->size()); ContainerID containerId = *(containers->begin()); Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); // TODO(jieyu): Consider using a program that predictably increases // RSS so that we can set more meaningful expectation here. EXPECT_LT(0u, usage->mem_rss_bytes()); driver.stop(); driver.join(); }
// This test verifies that the slave and status update manager // properly handle duplicate terminal status updates, when the // second update is received after the ACK for the first update. // The proper behavior here is for the status update manager to // forward the duplicate update to the scheduler. TEST_F(StatusUpdateManagerTest, DuplicateTerminalUpdateAfterAck) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); slave::Flags flags = CreateSlaveFlags(); Try<PID<Slave> > slave = StartSlave(&exec, flags); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); FrameworkID frameworkId; EXPECT_CALL(sched, registered(_, _, _)) .WillOnce(SaveArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); // Send a terminal update right away. EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_FINISHED)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); Future<Nothing> _statusUpdateAcknowledgement = FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(status); EXPECT_EQ(TASK_FINISHED, status.get().state()); AWAIT_READY(_statusUpdateAcknowledgement); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&update)); Future<Nothing> _statusUpdateAcknowledgement2 = FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement); Clock::pause(); // Now send a TASK_KILLED update for the same task. TaskStatus status2 = status.get(); status2.set_state(TASK_KILLED); execDriver->sendStatusUpdate(status2); // Ensure the scheduler receives TASK_KILLED. AWAIT_READY(update); EXPECT_EQ(TASK_KILLED, update.get().state()); // Ensure the slave properly handles the ACK. // Clock::settle() ensures that the slave successfully // executes Slave::_statusUpdateAcknowledgement(). AWAIT_READY(_statusUpdateAcknowledgement2); Clock::settle(); Clock::resume(); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); }
// The purpose of this test is to ensure that when slaves are removed // from the master, and then attempt to re-register, we deny the // re-registration by sending a ShutdownMessage to the slave. // Why? Because during a network partition, the master will remove a // partitioned slave, thus sending its tasks to LOST. At this point, // when the partition is removed, the slave will attempt to // re-register with its running tasks. We've already notified // frameworks that these tasks were LOST, so we have to have the slave // slave shut down. TEST_F(PartitionTest, PartitionedSlaveReregistration) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); // Allow the master to PING the slave, but drop all PONG messages // from the slave. Note that we don't match on the master / slave // PIDs because it's actually the SlaveObserver Process that sends // the pings. Future<Message> ping = FUTURE_MESSAGE(Eq("PING"), _, _); DROP_MESSAGES(Eq("PONG"), _, _); MockExecutor exec(DEFAULT_EXECUTOR_ID); StandaloneMasterDetector detector(master.get()); Try<PID<Slave> > slave = StartSlave(&exec, &detector); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); driver.start(); AWAIT_READY(offers); ASSERT_NE(0u, offers.get().size()); // Launch a task. This is to ensure the task is killed by the slave, // during shutdown. TaskID taskId; taskId.set_value("1"); TaskInfo task; task.set_name(""); task.mutable_task_id()->MergeFrom(taskId); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); task.mutable_executor()->mutable_command()->set_value("sleep 60"); vector<TaskInfo> tasks; tasks.push_back(task); // Set up the expectations for launching the task. EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<TaskStatus> runningStatus; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&runningStatus)); Future<Nothing> statusUpdateAck = FUTURE_DISPATCH( slave.get(), &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), tasks); AWAIT_READY(runningStatus); EXPECT_EQ(TASK_RUNNING, runningStatus.get().state()); // Wait for the slave to have handled the acknowledgment prior // to pausing the clock. AWAIT_READY(statusUpdateAck); // Drop the first shutdown message from the master (simulated // partition), allow the second shutdown message to pass when // the slave re-registers. Future<ShutdownMessage> shutdownMessage = DROP_PROTOBUF(ShutdownMessage(), _, slave.get()); Future<TaskStatus> lostStatus; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&lostStatus)); Future<Nothing> slaveLost; EXPECT_CALL(sched, slaveLost(&driver, _)) .WillOnce(FutureSatisfy(&slaveLost)); Clock::pause(); // Now, induce a partition of the slave by having the master // timeout the slave. uint32_t pings = 0; while (true) { AWAIT_READY(ping); pings++; if (pings == master::MAX_SLAVE_PING_TIMEOUTS) { break; } ping = FUTURE_MESSAGE(Eq("PING"), _, _); Clock::advance(master::SLAVE_PING_TIMEOUT); Clock::settle(); } Clock::advance(master::SLAVE_PING_TIMEOUT); Clock::settle(); // The master will have notified the framework of the lost task. AWAIT_READY(lostStatus); EXPECT_EQ(TASK_LOST, lostStatus.get().state()); // Wait for the master to attempt to shut down the slave. AWAIT_READY(shutdownMessage); // The master will notify the framework that the slave was lost. AWAIT_READY(slaveLost); Clock::resume(); // We now complete the partition on the slave side as well. This // is done by simulating a master loss event which would normally // occur during a network partition. detector.appoint(None()); Future<Nothing> shutdown; EXPECT_CALL(exec, shutdown(_)) .WillOnce(FutureSatisfy(&shutdown)); shutdownMessage = FUTURE_PROTOBUF(ShutdownMessage(), _, slave.get()); // Have the slave re-register with the master. detector.appoint(master.get()); // Upon re-registration, the master will shutdown the slave. // The slave will then shut down the executor. AWAIT_READY(shutdownMessage); AWAIT_READY(shutdown); driver.stop(); driver.join(); Shutdown(); }
// This test verifies that the slave and status update manager // properly handle duplicate status updates, when the second // update with the same UUID is received before the ACK for the // first update. The proper behavior here is for the status update // manager to drop the duplicate update. TEST_F(StatusUpdateManagerTest, DuplicateUpdateBeforeAck) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); Try<PID<Slave> > slave = StartSlave(&exec); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); FrameworkID frameworkId; EXPECT_CALL(sched, registered(_, _, _)) .WillOnce(SaveArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); // Capture the first status update message. Future<StatusUpdateMessage> statusUpdateMessage = FUTURE_PROTOBUF(StatusUpdateMessage(), _, _); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); // Drop the first ACK from the scheduler to the slave. Future<StatusUpdateAcknowledgementMessage> statusUpdateAckMessage = DROP_PROTOBUF(StatusUpdateAcknowledgementMessage(), _, slave.get()); Clock::pause(); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(statusUpdateMessage); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); AWAIT_READY(statusUpdateAckMessage); Future<Nothing> __statusUpdate = FUTURE_DISPATCH(slave.get(), &Slave::__statusUpdate); // Now resend the TASK_RUNNING update. process::post(slave.get(), statusUpdateMessage.get()); // At this point the status update manager has handled // the duplicate status update. AWAIT_READY(__statusUpdate); // After we advance the clock, the status update manager should // retry the TASK_RUNNING update and the scheduler should receive // and acknowledge it. Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&update)); Clock::advance(slave::STATUS_UPDATE_RETRY_INTERVAL_MIN); Clock::settle(); // Ensure the scheduler receives TASK_FINISHED. AWAIT_READY(update); EXPECT_EQ(TASK_RUNNING, update.get().state()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); Clock::resume(); driver.stop(); driver.join(); Shutdown(); }
Servatrice::Servatrice(QSettings *_settings, QObject *parent) : Server(parent), dbMutex(QMutex::Recursive), settings(_settings), uptime(0), shutdownTimer(0) { pingClock = new QTimer(this); connect(pingClock, SIGNAL(timeout()), this, SIGNAL(pingClockTimeout())); pingClock->start(1000); ProtocolItem::initializeHash(); serverId = settings->value("server/id", 0).toInt(); int statusUpdateTime = settings->value("server/statusupdate").toInt(); statusUpdateClock = new QTimer(this); connect(statusUpdateClock, SIGNAL(timeout()), this, SLOT(statusUpdate())); if (statusUpdateTime != 0) { qDebug() << "Starting status update clock, interval " << statusUpdateTime << " ms"; statusUpdateClock->start(statusUpdateTime); } threaded = settings->value("server/threaded", false).toInt(); tcpServer = new Servatrice_TcpServer(this, threaded, this); int port = settings->value("server/port", 4747).toInt(); qDebug() << "Starting server on port" << port; if (tcpServer->listen(QHostAddress::Any, port)) qDebug() << "Server listening."; else qDebug() << "tcpServer->listen(): Error."; QString dbType = settings->value("database/type").toString(); dbPrefix = settings->value("database/prefix").toString(); if (dbType == "mysql") openDatabase(); int size = settings->beginReadArray("rooms"); for (int i = 0; i < size; ++i) { settings->setArrayIndex(i); QStringList gameTypes; int size2 = settings->beginReadArray("game_types"); for (int j = 0; j < size2; ++j) { settings->setArrayIndex(j); gameTypes.append(settings->value("name").toString()); } settings->endArray(); Server_Room *newRoom = new Server_Room( i, settings->value("name").toString(), settings->value("description").toString(), settings->value("autojoin").toBool(), settings->value("joinmessage").toString(), gameTypes, this ); addRoom(newRoom); } settings->endArray(); updateLoginMessage(); maxGameInactivityTime = settings->value("game/max_game_inactivity_time").toInt(); maxPlayerInactivityTime = settings->value("game/max_player_inactivity_time").toInt(); maxUsersPerAddress = settings->value("security/max_users_per_address").toInt(); messageCountingInterval = settings->value("security/message_counting_interval").toInt(); maxMessageCountPerInterval = settings->value("security/max_message_count_per_interval").toInt(); maxMessageSizePerInterval = settings->value("security/max_message_size_per_interval").toInt(); maxGamesPerUser = settings->value("security/max_games_per_user").toInt(); }
// This test verifies that if master receives a status update // for an already terminated task it forwards it without // changing the state of the task. TEST_F(StatusUpdateManagerTest, DuplicatedTerminalStatusUpdate) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); Try<PID<Slave>> slave = StartSlave(&exec); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); FrameworkID frameworkId; EXPECT_CALL(sched, registered(_, _, _)) .WillOnce(SaveArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); // Send a terminal update right away. EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_FINISHED)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); Future<Nothing> _statusUpdateAcknowledgement = FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(status); EXPECT_EQ(TASK_FINISHED, status.get().state()); AWAIT_READY(_statusUpdateAcknowledgement); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&update)); Future<Nothing> _statusUpdateAcknowledgement2 = FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement); Clock::pause(); // Now send a TASK_KILLED update for the same task. TaskStatus status2 = status.get(); status2.set_state(TASK_KILLED); execDriver->sendStatusUpdate(status2); // Ensure the scheduler receives TASK_KILLED. AWAIT_READY(update); EXPECT_EQ(TASK_KILLED, update.get().state()); // Ensure the slave properly handles the ACK. // Clock::settle() ensures that the slave successfully // executes Slave::_statusUpdateAcknowledgement(). AWAIT_READY(_statusUpdateAcknowledgement2); // Verify the latest task status. Future<process::http::Response> tasks = process::http::get(master.get(), "tasks"); AWAIT_EXPECT_RESPONSE_STATUS_EQ(process::http::OK().status, tasks); AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", tasks); Try<JSON::Object> parse = JSON::parse<JSON::Object>(tasks.get().body); ASSERT_SOME(parse); Result<JSON::String> state = parse.get().find<JSON::String>("tasks[0].state"); ASSERT_SOME_EQ(JSON::String("TASK_FINISHED"), state); Clock::resume(); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); }
// This test verifies that the master reconciles tasks that are // missing from a re-registering slave. In this case, we trigger // a race between the slave re-registration message and the launch // message. There should be no TASK_LOST. // This was motivated by MESOS-1696. TEST_F(MasterSlaveReconciliationTest, ReconcileRace) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), master.get()->pid, _); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); AWAIT_READY(slaveRegisteredMessage); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); // Since the agent may have retried registration, we want to // ensure that any duplicate registrations are flushed before // we appoint the master again. Otherwise, the agent may // receive a stale registration message. Clock::pause(); Clock::settle(); Clock::resume(); // Trigger a re-registration of the slave and capture the message // so that we can spoof a race with a launch task message. DROP_PROTOBUFS(ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid); Future<ReregisterSlaveMessage> reregisterSlaveMessage = DROP_PROTOBUF( ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid); detector.appoint(master.get()->pid); AWAIT_READY(reregisterSlaveMessage); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task; task.set_name("test task"); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); ExecutorDriver* executorDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&executorDriver)); // Leave the task in TASK_STAGING. Future<Nothing> launchTask; EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(FutureSatisfy(&launchTask)); EXPECT_CALL(sched, statusUpdate(&driver, _)) .Times(0); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(launchTask); // Send the stale re-registration message, which does not contain // the task we just launched. This will trigger a reconciliation // by the master. Future<SlaveReregisteredMessage> slaveReregisteredMessage = FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _); // Prevent this from being dropped per the DROP_PROTOBUFS above. FUTURE_PROTOBUF( ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid); process::post( slave.get()->pid, master.get()->pid, reregisterSlaveMessage.get()); AWAIT_READY(slaveReregisteredMessage); // Neither the master nor the slave should send a TASK_LOST // as part of the reconciliation. We check this by calling // Clock::settle() to flush all pending events. Clock::pause(); Clock::settle(); Clock::resume(); // Now send TASK_FINISHED and make sure it's the only message // received by the scheduler. Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); TaskStatus taskStatus; taskStatus.mutable_task_id()->CopyFrom(task.task_id()); taskStatus.set_state(TASK_FINISHED); executorDriver->sendStatusUpdate(taskStatus); AWAIT_READY(status); ASSERT_EQ(TASK_FINISHED, status.get().state()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
TEST_F(StatusUpdateManagerTest, CheckpointStatusUpdate) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); // Require flags to retrieve work_dir when recovering // the checkpointed data. slave::Flags flags = CreateSlaveFlags(); Try<PID<Slave> > slave = StartSlave(&exec, flags); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(_, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(frameworkId); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); EXPECT_CALL(exec, registered(_, _, _, _)) .Times(1); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); Future<Nothing> _statusUpdateAcknowledgement = FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); AWAIT_READY(_statusUpdateAcknowledgement); // Ensure that both the status update and its acknowledgement are // correctly checkpointed. Result<slave::state::State> state = slave::state::recover(slave::paths::getMetaRootDir(flags.work_dir), true); ASSERT_SOME(state); ASSERT_SOME(state.get().slave); ASSERT_TRUE(state.get().slave.get().frameworks.contains(frameworkId.get())); slave::state::FrameworkState frameworkState = state.get().slave.get().frameworks.get(frameworkId.get()).get(); ASSERT_EQ(1u, frameworkState.executors.size()); slave::state::ExecutorState executorState = frameworkState.executors.begin()->second; ASSERT_EQ(1u, executorState.runs.size()); slave::state::RunState runState = executorState.runs.begin()->second; ASSERT_EQ(1u, runState.tasks.size()); slave::state::TaskState taskState = runState.tasks.begin()->second; EXPECT_EQ(1u, taskState.updates.size()); EXPECT_EQ(1u, taskState.acks.size()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); }
// This test verifies that when the slave re-registers, the master // does not send TASK_LOST update for a task that has reached terminal // state but is waiting for an acknowledgement. TEST_F(MasterSlaveReconciliationTest, SlaveReregisterTerminalTask) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task; task.set_name("test task"); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); EXPECT_CALL(exec, registered(_, _, _, _)); // Send a terminal update right away. EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_FINISHED)); // Drop the status update from slave to the master, so that // the slave has a pending terminal update when it re-registers. DROP_PROTOBUF(StatusUpdateMessage(), _, master.get()->pid); Future<Nothing> _statusUpdate = FUTURE_DISPATCH(_, &Slave::_statusUpdate); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)) .WillRepeatedly(Return()); // Ignore retried update due to update framework. driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(_statusUpdate); Future<SlaveReregisteredMessage> slaveReregisteredMessage = FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _); // Simulate a spurious master change event (e.g., due to ZooKeeper // expiration) at the slave to force re-registration. detector.appoint(master.get()->pid); AWAIT_READY(slaveReregisteredMessage); // The master should not send a TASK_LOST after the slave // re-registers. We check this by calling Clock::settle() so that // the only update the scheduler receives is the retried // TASK_FINISHED update. // NOTE: The status update manager resends the status update when // it detects a new master. Clock::pause(); Clock::settle(); AWAIT_READY(status); ASSERT_EQ(TASK_FINISHED, status.get().state()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
/* ... the main entry point for the ftp process */ int main (int argc, char *argv[]) { void (*alarmHandler)(int); STIM stim; int i; int seconds; time_t ntime; struct tm locTime; long offset, retVal; long msOffset; FILE *pidfile; int runAsDaemon = TRUE; if (argc > 1) { if (!strcmp(argv[1], "-f")) { runAsDaemon = FALSE; } } memset (&ftpWork, 0, sizeof (ftpWork)); /* ... initialize some system stuff first */ retVal = ftpSysInit (&ftpWork); if (retVal == ERROR) { radMsgLogInit (PROC_NAME_FTP, FALSE, TRUE); radMsgLog (PRI_CATASTROPHIC, "ftp init failed!"); radMsgLogExit (); exit (1); } else if (retVal == ERROR_ABORT) { exit (2); } /* ... call the global radlib system init function */ if (radSystemInit (WVIEW_SYSTEM_ID) == ERROR) { radMsgLogInit (PROC_NAME_FTP, TRUE, TRUE); radMsgLog (PRI_CATASTROPHIC, "radSystemInit failed!"); radMsgLogExit (); exit (1); } /* ... call the radlib process init function */ if (radProcessInit (PROC_NAME_FTP, ftpWork.fifoFile, PROC_NUM_TIMERS_FTP, runAsDaemon, // TRUE for daemon msgHandler, evtHandler, NULL) == ERROR) { printf ("radProcessInit failed: %s", PROC_NAME_FTP); radSystemExit (WVIEW_SYSTEM_ID); exit (1); } ftpWork.myPid = getpid (); pidfile = fopen (ftpWork.pidFile, "w"); if (pidfile == NULL) { radMsgLog (PRI_CATASTROPHIC, "lock file create failed!"); radProcessExit (); radSystemExit (WVIEW_SYSTEM_ID); exit (1); } fprintf (pidfile, "%d", getpid ()); fclose (pidfile); alarmHandler = radProcessSignalGetHandler (SIGALRM); radProcessSignalCatchAll (FTPDefaultSigHandler); radProcessSignalCatch (SIGALRM, alarmHandler); radProcessSignalRelease(SIGABRT); ftpWork.timer = radTimerCreate (NULL, timerHandler, NULL); if (ftpWork.timer == NULL) { radMsgLog (PRI_HIGH, "radTimerCreate failed"); ftpSysExit (&ftpWork); radProcessExit (); radSystemExit (WVIEW_SYSTEM_ID); exit (1); } // ... initialize the ftp utilities retVal = ftpUtilsInit (&ftpWork.ftpData); if (retVal != OK) { if (retVal == ERROR) { radMsgLog (PRI_HIGH, "ftpUtilsInit failed"); } radTimerDelete (ftpWork.timer); ftpSysExit (&ftpWork); radProcessExit (); radSystemExit (WVIEW_SYSTEM_ID); exit (1); } else { ftpWork.ftpId = &ftpWork.ftpData; } if (statusInit(ftpWork.statusFile, ftpStatusLabels) == ERROR) { radMsgLog (PRI_HIGH, "ALARM status init failed - exiting..."); ftpSysExit (&ftpWork); radProcessExit (); radSystemExit (WVIEW_SYSTEM_ID); exit (1); } statusUpdate(STATUS_BOOTING); statusUpdateStat(FTP_STATS_RULES_DEFINED, radListGetNumberOfNodes(&ftpWork.ftpData.rules)); // ... start THE timer ntime = time (NULL); localtime_r (&ntime, &locTime); seconds = locTime.tm_sec - 15; // start at 15 secs past /* ... start the ftp timer to go off 1 min past the next 5 minute mark */ offset = locTime.tm_min % 5; if (offset) offset = 6 - offset; else offset = 1; radMsgLog (PRI_HIGH, "starting ftp timer for %d mins %d secs", ((seconds > 0) ? ((offset > 0) ? offset-1 : offset) : offset), (seconds > 0) ? 60 - seconds : (-1) * seconds); msOffset = radTimeGetMSSinceEpoch () % 1000; msOffset -= 250; // land on 250 ms mark ftpWork.msOffset = 0; radProcessTimerStart (ftpWork.timer, ((((offset * 60) - seconds) * 1000)) - msOffset); statusUpdate(STATUS_RUNNING); statusUpdateMessage("Normal operation"); while (! ftpWork.exiting) { /* ... wait on timers, events, file descriptors, msgs, everything! */ if (radProcessWait (0) == ERROR) { ftpWork.exiting = TRUE; } } statusUpdateMessage("exiting normally"); radMsgLog (PRI_STATUS, "exiting normally..."); statusUpdate(STATUS_SHUTDOWN); if (ftpWork.ftpId != NULL) { ftpUtilsExit (ftpWork.ftpId); } radTimerDelete (ftpWork.timer); ftpSysExit (&ftpWork); radProcessExit (); radSystemExit (WVIEW_SYSTEM_ID); exit (0); }
// This test verifies that a re-registering slave sends the terminal // unacknowledged tasks for a terminal executor. This is required // for the master to correctly reconcile its view with the slave's // view of tasks. This test drops a terminal update to the master // and then forces the slave to re-register. TEST_F(MasterSlaveReconciliationTest, SlaveReregisterTerminatedExecutor) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "*")) .WillRepeatedly(Return()); // Ignore subsequent offers. ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); Future<StatusUpdateAcknowledgementMessage> statusUpdateAcknowledgementMessage = FUTURE_PROTOBUF( StatusUpdateAcknowledgementMessage(), master.get()->pid, slave.get()->pid); driver.start(); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); // Make sure the acknowledgement reaches the slave. AWAIT_READY(statusUpdateAcknowledgementMessage); // Drop the TASK_FINISHED status update sent to the master. Future<StatusUpdateMessage> statusUpdateMessage = DROP_PROTOBUF(StatusUpdateMessage(), _, master.get()->pid); Future<ExitedExecutorMessage> executorExitedMessage = FUTURE_PROTOBUF(ExitedExecutorMessage(), _, _); TaskStatus finishedStatus; finishedStatus = status.get(); finishedStatus.set_state(TASK_FINISHED); execDriver->sendStatusUpdate(finishedStatus); // Ensure the update was sent. AWAIT_READY(statusUpdateMessage); EXPECT_CALL(sched, executorLost(&driver, DEFAULT_EXECUTOR_ID, _, _)); // Now kill the executor. containerizer.destroy(frameworkId.get(), DEFAULT_EXECUTOR_ID); Future<TaskStatus> status2; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status2)); // We drop the 'UpdateFrameworkMessage' from the master to slave to // stop the status update manager from retrying the update that was // already sent due to the new master detection. DROP_PROTOBUFS(UpdateFrameworkMessage(), _, _); detector.appoint(master.get()->pid); AWAIT_READY(status2); EXPECT_EQ(TASK_FINISHED, status2.get().state()); driver.stop(); driver.join(); }
bool InitDriveThread::method_resizePartitions() { int newStartOfRescuePartition = getFileContents("/sys/class/block/mmcblk0p1/start").trimmed().toInt(); int newSizeOfRescuePartition = sizeofBootFilesInKB()/1000 + 100; if (!umountSystemPartition()) { emit error(tr("Error unmounting system partition.")); return false; } if (!QFile::exists("/dev/mmcblk0p1")) { // SD card does not have a MBR. // Warn user that their SD card does not have an MBR and ask // if they would like us to create one for them QMessageBox::StandardButton answer; emit query(tr("Would you like NOOBS to create one for you?\nWARNING: This will erase all data on your SD card"), tr("Error: No MBR present on SD Card"), &answer); if(answer == QMessageBox::Yes) { emit statusUpdate(tr("Zeroing partition table")); if (!zeroMbr()) { emit error(tr("Error zero'ing MBR/GPT. SD card may be broken or advertising wrong capacity.")); return false; } // Create MBR containing single FAT partition emit statusUpdate(tr("Writing new MBR")); QProcess proc; proc.setProcessChannelMode(proc.MergedChannels); proc.start("/usr/sbin/parted /dev/mmcblk0 --script -- mktable msdos mkpartfs primary fat32 8192s -1"); proc.waitForFinished(-1); if (proc.exitCode() != 0) { // Warn user if we failed to create an MBR on their card emit error(tr("Error creating MBR")+"\n"+proc.readAll()); return false; } qDebug() << "Created missing MBR on SD card. parted output:" << proc.readAll(); // Advise user that their SD card has now been formatted // suitably for installing NOOBS and that they will have to // re-copy the files before rebooting emit error(tr("SD card has now been formatted ready for NOOBS installation. Please re-copy the NOOBS files onto the card and reboot")); return false; } else { emit error(tr("SD card has not been formatted correctly. Please reformat using the SD Association Formatting Tool and try again.")); return false; } } emit statusUpdate(tr("Removing partitions 2,3,4")); QFile f("/dev/mmcblk0"); f.open(f.ReadWrite); // Seek to partition entry 2 f.seek(462); // Zero out partition 2,3,4 to prevent parted complaining about invalid constraints f.write(QByteArray(16*3, '\0')); f.flush(); // Tell Linux to re-read the partition table ioctl(f.handle(), BLKRRPART); f.close(); QThread::msleep(500); emit statusUpdate(tr("Resizing FAT partition")); /* Relocating the start of the FAT partition is a write intensive operation * only move it when it is not aligned on a MiB boundary already */ if (newStartOfRescuePartition < 2048 || newStartOfRescuePartition % 2048 != 0) { newStartOfRescuePartition = 8192; /* 4 MiB */ } QString cmd = "/usr/sbin/parted --script /dev/mmcblk0 resize 1 "+QString::number(newStartOfRescuePartition)+"s "+QString::number(newSizeOfRescuePartition)+"M"; qDebug() << "Executing" << cmd; QProcess p; p.setProcessChannelMode(p.MergedChannels); p.start(cmd); p.closeWriteChannel(); p.waitForFinished(-1); if (p.exitCode() != 0) { emit error(tr("Error resizing existing FAT partition")+"\n"+p.readAll()); return false; } qDebug() << "parted done, output:" << p.readAll(); QThread::msleep(500); emit statusUpdate(tr("Creating extended partition")); mbr_table extended_mbr; QByteArray partitionTable; int startOfOurPartition = getFileContents("/sys/class/block/mmcblk0p1/start").trimmed().toInt(); int sizeOfOurPartition = getFileContents("/sys/class/block/mmcblk0p1/size").trimmed().toInt(); int startOfExtended = startOfOurPartition + sizeOfOurPartition; // Align on 4 MiB boundary startOfExtended += 8192-(startOfExtended % 8192); // BUGBUG - reserve space for WinIOT startOfExtended += 5000 * 2048; // int sizeOfDisk = sizeofSDCardInBlocks(); int sizeOfExtended = sizeOfDisk - startOfExtended; partitionTable = QByteArray::number(startOfOurPartition)+","+QByteArray::number(sizeOfOurPartition)+",0E\n"; /* FAT partition */ partitionTable += "0,0\n"; partitionTable += "0,0\n"; partitionTable += QByteArray::number(startOfExtended)+","+QByteArray::number(sizeOfExtended)+",X\n"; /* Extended partition with all remaining space */ qDebug() << "Writing partition table" << partitionTable; /* Write out extended partition table with settings logical partition */ memset(&extended_mbr, 0, sizeof extended_mbr); extended_mbr.part[0].starting_sector = EBR_PARTITION_OFFSET; extended_mbr.part[0].nr_of_sectors = SETTINGS_PARTITION_SIZE; extended_mbr.part[0].id = 0x83; extended_mbr.signature[0] = 0x55; extended_mbr.signature[1] = 0xAA; f.open(f.ReadWrite); f.seek(((qint64)startOfExtended)*512L); f.write((char *) &extended_mbr, sizeof(extended_mbr)); f.flush(); f.close(); /* Let sfdisk write a proper partition table */ cmd = QString("/sbin/sfdisk -uS /dev/mmcblk0"); QProcess proc; proc.setProcessChannelMode(proc.MergedChannels); proc.start(cmd); proc.write(partitionTable); proc.closeWriteChannel(); proc.waitForFinished(-1); if (proc.exitCode() != 0) { emit error(tr("Error creating extended partition")+"\n"+proc.readAll()); return false; } qDebug() << "sfdisk done, output:" << proc.readAll(); QThread::msleep(500); /* For reasons unknown Linux sometimes * only finds /dev/mmcblk0p2 and /dev/mmcblk0p1 goes missing */ if (!QFile::exists("/dev/mmcblk0p1")) { /* Probe again */ QProcess::execute("/usr/sbin/partprobe"); QThread::msleep(1500); } QProcess::execute("/sbin/mlabel p:RECOVERY"); emit statusUpdate(tr("Mounting FAT partition")); if (!mountSystemPartition()) { emit error(tr("Error mounting system partition.")); return false; } return true; }