char*
connectAndSendUserCommand(char c,
int *coordCmdStatus,
int *numPeers,
int *isRunning,
int *ckptInterval)
{
char *replyData = NULL;
int coordFd = createNewSocketToCoordinator(COORD_ANY);
if (coordFd == -1) {
*coordCmdStatus = CoordCmdStatus::ERROR_COORDINATOR_NOT_FOUND;
return replyData;
}
// Tell the coordinator to run given user command
DmtcpMessage msg(DMT_USER_CMD);
msg.coordCmd = c;
if (c == 'i') {
const char *interval = getenv(ENV_VAR_CKPT_INTR);
if (interval != NULL) {
msg.theCheckpointInterval = jalib::StringToInt(interval);
}
}
JASSERT(Util::writeAll(coordFd, &msg, sizeof(msg)) == sizeof(msg));
// The coordinator will violently close our socket...
if (c == 'q' || c == 'Q') {
*coordCmdStatus = CoordCmdStatus::NOERROR;
return replyData;
}
// Receive REPLY
DmtcpMessage reply;
reply.poison();
recvMsgFromCoordinatorRaw(coordFd, &reply, (void**)&replyData);
reply.assertValid();
JASSERT(reply.type == DMT_USER_CMD_RESULT);
if (coordCmdStatus != NULL) {
*coordCmdStatus = reply.coordCmdStatus;
}
if (numPeers != NULL) {
*numPeers = reply.numPeers;
}
if (isRunning != NULL) {
*isRunning = reply.isRunning;
}
if (ckptInterval != NULL) {
*ckptInterval = reply.theCheckpointInterval;
}
_real_close(coordFd);
return replyData;
}
void
DmtcpWorker::waitForSuspendMessage()
{
SharedData::resetBarrierInfo();
if (dmtcp_no_coordinator()) {
string shmFile = jalib::Filesystem::GetDeviceName(PROTECTED_SHM_FD);
JASSERT(!shmFile.empty());
unlink(shmFile.c_str());
CoordinatorAPI::waitForCheckpointCommand();
ProcessInfo::instance().numPeers(1);
ProcessInfo::instance().compGroup(SharedData::getCompId());
return;
}
if (ThreadSync::destroyDmtcpWorkerLockTryLock() != 0) {
JTRACE("User thread is performing exit()."
" ckpt thread exit()ing as well");
ckptThreadPerformExit();
}
if (exitInProgress()) {
ThreadSync::destroyDmtcpWorkerLockUnlock();
ckptThreadPerformExit();
}
JTRACE("waiting for SUSPEND message");
DmtcpMessage msg;
CoordinatorAPI::recvMsgFromCoordinator(&msg);
if (exitInProgress()) {
ThreadSync::destroyDmtcpWorkerLockUnlock();
ckptThreadPerformExit();
}
msg.assertValid();
if (msg.type == DMT_KILL_PEER) {
JTRACE("Received KILL message from coordinator, exiting");
_exit(0);
}
JASSERT(msg.type == DMT_DO_SUSPEND) (msg.type);
// Coordinator sends some computation information along with the SUSPEND
// message. Extracting that.
SharedData::updateGeneration(msg.compGroup.computationGeneration());
JASSERT(SharedData::getCompId() == msg.compGroup.upid())
(SharedData::getCompId()) (msg.compGroup);
_exitAfterCkpt = msg.exitAfterCkpt;
}
pid_t dmtcp::DmtcpCoordinatorAPI::getVirtualPidFromCoordinator()
{
connectToCoordinator();
DmtcpMessage msg(DMT_GET_VIRTUAL_PID);
_coordinatorSocket << msg;
DmtcpMessage reply;
reply.poison();
_coordinatorSocket >> reply;
reply.assertValid();
JASSERT(reply.type == DMT_GET_VIRTUAL_PID_RESULT) (reply.type);
JASSERT(reply.virtualPid != -1);
_coordinatorSocket.close();
return reply.virtualPid;
}
// On input, val points to a buffer in user memory and *val_len is the maximum
// size of that buffer (the memory allocated by user).
// On output, we copy data to val, and set *val_len to the actual buffer size
// (to the size of the data that we copied to the user buffer).
int CoordinatorAPI::sendQueryToCoordinator(const char *id,
const void *key,
uint32_t key_len,
void *val,
uint32_t *val_len)
{
DmtcpMessage msg (DMT_NAME_SERVICE_QUERY);
JWARNING(strlen(id) < sizeof(msg.nsid));
strncpy(msg.nsid, id, 8);
msg.keyLen = key_len;
msg.valLen = 0;
msg.extraBytes = key_len;
jalib::JSocket sock = _coordinatorSocket;
if (key == NULL || key_len == 0 || val == NULL || val_len == 0) {
return 0;
}
if (dmtcp_is_running_state()) {
if (!_nsSock.isValid()) {
_nsSock = createNewSocketToCoordinator(COORD_ANY);
JASSERT(_nsSock.isValid());
_nsSock.changeFd(PROTECTED_NS_FD);
DmtcpMessage m(DMT_NAME_SERVICE_WORKER);
_nsSock << m;
}
sock = _nsSock;
JASSERT(sock.isValid());
}
sock << msg;
sock.writeAll((const char *)key, key_len);
msg.poison();
sock >> msg;
msg.assertValid();
JASSERT(msg.type == DMT_NAME_SERVICE_QUERY_RESPONSE &&
msg.extraBytes == msg.valLen);
JASSERT (*val_len >= msg.valLen);
*val_len = msg.valLen;
if (*val_len > 0) {
sock.readAll((char*)val, *val_len);
}
return *val_len;
}
void
DmtcpWorker::acknowledgeSuspendMsg()
{
if (dmtcp_no_coordinator()) {
return;
}
JTRACE("Waiting for DMT_DO_CHECKPOINT message");
CoordinatorAPI::sendMsgToCoordinator(DmtcpMessage(DMT_OK));
DmtcpMessage msg;
CoordinatorAPI::recvMsgFromCoordinator(&msg);
msg.assertValid();
JASSERT(msg.type == DMT_COMPUTATION_INFO) (msg.type);
JTRACE("Computation information") (msg.compGroup) (msg.numPeers);
ProcessInfo::instance().compGroup(msg.compGroup);
ProcessInfo::instance().numPeers(msg.numPeers);
}
DmtcpMessage
sendRecvHandshake(int fd,
DmtcpMessage msg,
string progname,
UniquePid *compId)
{
if (dmtcp_virtual_to_real_pid) {
msg.realPid = dmtcp_virtual_to_real_pid(getpid());
} else {
msg.realPid = getpid();
}
msg.theCheckpointInterval = getCkptInterval();
string hostname = jalib::Filesystem::GetCurrentHostname();
size_t buflen = hostname.length() + progname.length() + 2;
char buf[buflen];
strcpy(buf, hostname.c_str());
strcpy(&buf[hostname.length() + 1], progname.c_str());
sendMsgToCoordinatorRaw(fd, msg, buf, buflen);
recvMsgFromCoordinatorRaw(fd, &msg);
msg.assertValid();
if (msg.type == DMT_KILL_PEER) {
JTRACE("Received KILL message from coordinator, exiting");
_real_exit(0);
}
if (msg.type == DMT_REJECT_NOT_RUNNING) {
JASSERT(false)
.Text("Connection rejected by the coordinator.\n"
"Reason: Current computation not in RUNNING state.\n"
" Is a checkpoint/restart in progress?");
} else if (msg.type == DMT_REJECT_WRONG_COMP) {
JASSERT(compId != NULL);
JASSERT(false) (*compId)
.Text("Connection rejected by the coordinator.\n"
" Reason: This process has a different computation group.");
}
// Coordinator also prints this, but its stderr may go to /dev/null
if (msg.type == DMT_REJECT_NOT_RESTARTING) {
string coordinatorHost = ""; // C++ magic code; "" to be invisibly replaced
int coordinatorPort;
getCoordHostAndPort(COORD_ANY, coordinatorHost, &coordinatorPort);
JNOTE ("\n\n*** Computation not in RESTARTING or CHECKPOINTED state."
"\n***Can't join the existing coordinator, as it is serving a"
"\n***different computation. Consider launching a new coordinator."
"\n***Consider, also, checking with: dmtcp_command --status")
(coordinatorPort);
}
JASSERT(msg.type == DMT_ACCEPT)(msg.type);
return msg;
}
int CoordinatorAPI::sendKeyValPairToCoordinator(const char *id,
const void *key,
uint32_t key_len,
const void *val,
uint32_t val_len,
int sync)
{
DmtcpMessage msg (DMT_REGISTER_NAME_SERVICE_DATA);
if (sync) {
msg.type = DMT_REGISTER_NAME_SERVICE_DATA_SYNC;
}
JWARNING(strlen(id) < sizeof(msg.nsid));
strncpy(msg.nsid, id, 8);
msg.keyLen = key_len;
msg.valLen = val_len;
msg.extraBytes = key_len + val_len;
jalib::JSocket sock = _coordinatorSocket;
if (dmtcp_is_running_state()) {
if (_nsSock.sockfd() == -1) {
_nsSock = createNewSocketToCoordinator(COORD_ANY);
JASSERT(_nsSock.isValid());
_nsSock.changeFd(PROTECTED_NS_FD);
DmtcpMessage m(DMT_NAME_SERVICE_WORKER);
_nsSock << m;
}
sock = _nsSock;
JASSERT(sock.isValid());
}
sock << msg;
sock.writeAll((const char *)key, key_len);
sock.writeAll((const char *)val, val_len);
if (sync) {
msg.poison();
sock >> msg;
JASSERT(msg.type == DMT_REGISTER_NAME_SERVICE_DATA_SYNC_RESPONSE)(msg.type);
}
return 1;
}
void waitForBarrier(const string& barrierId)
{
sendMsgToCoordinator(DmtcpMessage(DMT_OK));
JTRACE("waiting for DMT_BARRIER_RELEASED message");
char *extraData = NULL;
DmtcpMessage msg;
recvMsgFromCoordinator(&msg, (void**)&extraData);
msg.assertValid();
if (msg.type == DMT_KILL_PEER) {
JTRACE("Received KILL message from coordinator, exiting");
_exit(0);
}
JASSERT(msg.type == DMT_BARRIER_RELEASED) (msg.type);
JASSERT(extraData != NULL);
JASSERT(barrierId == extraData) (barrierId) (extraData);
JALLOC_FREE(extraData);
}
DmtcpMessage CoordinatorAPI::sendRecvHandshake(DmtcpMessage msg,
string progname,
UniquePid *compId)
{
if (dmtcp_virtual_to_real_pid) {
msg.realPid = dmtcp_virtual_to_real_pid(getpid());
} else {
msg.realPid = getpid();
}
msg.theCheckpointInterval = getCkptInterval();
string hostname = jalib::Filesystem::GetCurrentHostname();
msg.extraBytes = hostname.length() + 1 + progname.length() + 1;
_coordinatorSocket << msg;
_coordinatorSocket.writeAll(hostname.c_str(), hostname.length() + 1);
_coordinatorSocket.writeAll(progname.c_str(), progname.length() + 1);
msg.poison();
_coordinatorSocket >> msg;
msg.assertValid();
if (msg.type == DMT_KILL_PEER) {
JTRACE("Received KILL message from coordinator, exiting");
_real_exit (0);
}
if (msg.type == DMT_REJECT_NOT_RUNNING) {
JASSERT(false)
.Text("Connection rejected by the coordinator.\n"
"Reason: Current computation not in RUNNING state.\n"
" Is a checkpoint/restart in progress?");
} else if (msg.type == DMT_REJECT_WRONG_COMP) {
JASSERT(compId != NULL);
JASSERT(false) (*compId)
.Text("Connection rejected by the coordinator.\n"
" Reason: This process has a different computation group.");
}
JASSERT(msg.type == DMT_ACCEPT)(msg.type);
return msg;
}
void
DmtcpWorker::waitForSuspendMessage()
{
SharedData::resetBarrierInfo();
if (dmtcp_no_coordinator()) {
string shmFile = jalib::Filesystem::GetDeviceName(PROTECTED_SHM_FD);
JASSERT(!shmFile.empty());
unlink(shmFile.c_str());
CoordinatorAPI::waitForCheckpointCommand();
ProcessInfo::instance().numPeers(1);
ProcessInfo::instance().compGroup(SharedData::getCompId());
return;
}
JTRACE("waiting for SUSPEND message");
DmtcpMessage msg;
CoordinatorAPI::recvMsgFromCoordinator(&msg);
// Before validating message; make sure we are not exiting.
if (exitInProgress) {
ckptThreadPerformExit();
}
msg.assertValid();
JASSERT(msg.type == DMT_DO_SUSPEND) (msg.type);
// Coordinator sends some computation information along with the SUSPEND
// message. Extracting that.
SharedData::updateGeneration(msg.compGroup.computationGeneration());
JASSERT(SharedData::getCompId() == msg.compGroup.upid())
(SharedData::getCompId()) (msg.compGroup);
exitAfterCkpt = msg.exitAfterCkpt;
}
void dmtcp::ConnectionRewirer::onConnect ( const jalib::JSocket& sock, const struct sockaddr* /*remoteAddr*/,socklen_t /*remoteLen*/ )
{
jalib::JSocket remote = sock;
DmtcpMessage msg;
msg.poison();
remote >> msg;
msg.assertValid();
JASSERT ( msg.type == DMT_RESTORE_RECONNECTED ) ( msg.type ).Text ( "unexpected message" );
iterator i = _pendingIncoming.find ( msg.restorePid );
JASSERT ( i != _pendingIncoming.end() ) ( msg.restorePid )
.Text ( "got unexpected incoming restore request" );
const dmtcp::vector<int>& fds = i->second;
JASSERT ( fds.size() > 0 );
int fd0 = fds[0];
remote.changeFd ( fd0 );
JTRACE ( "restoring incoming connection" ) ( msg.restorePid ) ( fd0 ) ( fds.size() );
for ( size_t i = 1; i<fds.size(); ++i )
{
JTRACE ( "restoring extra fd" ) ( fd0 ) ( fds[i] );
JASSERT ( _real_dup2 ( fd0,fds[i] ) == fds[i] ) ( fd0 ) ( fds[i] ) ( msg.restorePid )
.Text ( "dup2() failed" );
}
_pendingIncoming.erase ( i );
if ( pendingCount() ==0 ) finishup();
#ifdef DEBUG
else debugPrint();
#endif
}
void DmtcpWorker::waitForCoordinatorMsg(string msgStr,
DmtcpMessageType type)
{
if (dmtcp_no_coordinator()) {
if (type == DMT_DO_SUSPEND) {
string shmFile = jalib::Filesystem::GetDeviceName(PROTECTED_SHM_FD);
JASSERT(!shmFile.empty());
unlink(shmFile.c_str());
CoordinatorAPI::instance().waitForCheckpointCommand();
ProcessInfo::instance().numPeers(1);
ProcessInfo::instance().compGroup(SharedData::getCompId());
}
return;
}
if (type == DMT_DO_SUSPEND) {
if (ThreadSync::destroyDmtcpWorkerLockTryLock() != 0) {
JTRACE("User thread is performing exit()."
" ckpt thread exit()ing as well");
ckptThreadPerformExit();
}
if (exitInProgress()) {
ThreadSync::destroyDmtcpWorkerLockUnlock();
ckptThreadPerformExit();
}
}
DmtcpMessage msg;
if (type == DMT_DO_SUSPEND) {
// Make a dummy syscall to inform superior of our status before we go into
// select. If ptrace is disabled, this call has no significant effect.
_real_syscall(DMTCP_FAKE_SYSCALL);
} else {
msg.type = DMT_OK;
msg.state = WorkerState::currentState();
CoordinatorAPI::instance().sendMsgToCoordinator(msg);
}
JTRACE("waiting for " + msgStr + " message");
CoordinatorAPI::instance().recvMsgFromCoordinator(&msg);
if (type == DMT_DO_SUSPEND && exitInProgress()) {
ThreadSync::destroyDmtcpWorkerLockUnlock();
ckptThreadPerformExit();
}
msg.assertValid();
if (msg.type == DMT_KILL_PEER) {
JTRACE("Received KILL message from coordinator, exiting");
_exit (0);
}
JASSERT(msg.type == type) (msg.type) (type);
// Coordinator sends some computation information along with the SUSPEND
// message. Extracting that.
if (type == DMT_DO_SUSPEND) {
SharedData::updateGeneration(msg.compGroup.computationGeneration());
JASSERT(SharedData::getCompId() == msg.compGroup.upid())
(SharedData::getCompId()) (msg.compGroup);
} else if (type == DMT_DO_FD_LEADER_ELECTION) {
JTRACE("Computation information") (msg.compGroup) (msg.numPeers);
ProcessInfo::instance().compGroup(msg.compGroup);
ProcessInfo::instance().numPeers(msg.numPeers);
}
}
void
waitForCheckpointCommand()
{
uint32_t ckptInterval = SharedData::getCkptInterval();
struct timeval tmptime = { 0, 0 };
long remaining = ckptInterval;
do {
struct timeval *timeout = NULL;
struct timeval start;
if (ckptInterval > 0) {
timeout = &tmptime;
timeout->tv_sec = remaining;
JASSERT(gettimeofday(&start, NULL) == 0) (JASSERT_ERRNO);
}
// This call to poll() does nothing and returns.
// But we want to find address of poll() using dlsym/libc before
// allowing the user thread to continue.
poll(NULL, 0, 0);
if (sem_launch_first_time) {
// Release user thread now that we've initialized the checkpoint thread.
// This code is reached if the --no-coordinator flag is used.
sem_post(&sem_launch);
sem_launch_first_time = false;
}
struct pollfd socketFd = {0};
socketFd.fd = coordinatorSocket;
socketFd.events = POLLIN;
uint64_t millis = timeout ? ((timeout->tv_sec * (uint64_t)1000) +
(timeout->tv_usec / 1000))
: -1;
int retval = poll(&socketFd, 1, millis);
if (retval == 0) { // timeout expired, time for checkpoint
JTRACE("Timeout expired, checkpointing now.");
return;
} else if (retval > 0) {
JASSERT(socketFd.revents & POLLIN);
JTRACE("Connect request on virtual coordinator socket.");
break;
}
JASSERT(errno == EINTR) (JASSERT_ERRNO); /* EINTR: a signal was caught */
if (ckptInterval > 0) {
struct timeval end;
JASSERT(gettimeofday(&end, NULL) == 0) (JASSERT_ERRNO);
remaining -= end.tv_sec - start.tv_sec;
// If the remaining time is negative, we can checkpoint now
if (remaining < 0) {
return;
}
}
} while (remaining > 0);
jalib::JSocket cmdSock(-1);
DmtcpMessage msg;
DmtcpMessage reply(DMT_USER_CMD_RESULT);
do {
cmdSock.close();
jalib::JServerSocket sock(coordinatorSocket);
cmdSock = sock.accept();
msg.poison();
JTRACE("Reading from incoming connection...");
cmdSock >> msg;
} while (!cmdSock.isValid());
JASSERT(msg.type == DMT_USER_CMD) (msg.type)
.Text("Unexpected connection.");
reply.coordCmdStatus = CoordCmdStatus::NOERROR;
bool exitWhenDone = false;
switch (msg.coordCmd) {
// case 'b': case 'B': // prefix blocking command, prior to checkpoint
// command
// JTRACE("blocking checkpoint beginning...");
// blockUntilDone = true;
// break;
case 's': case 'S':
JTRACE("Received status command");
reply.numPeers = 1;
reply.isRunning = 1;
break;
case 'c': case 'C':
JTRACE("checkpointing...");
break;
case 'k': case 'K':
case 'q': case 'Q':
JTRACE("Received KILL command from user, exiting");
exitWhenDone = true;
break;
default:
JTRACE("unhandled user command") (msg.coordCmd);
reply.coordCmdStatus = CoordCmdStatus::ERROR_INVALID_COMMAND;
}
cmdSock << reply;
cmdSock.close();
if (exitWhenDone) {
_real_exit(0);
}
}
void CoordinatorAPI::waitForCheckpointCommand()
{
uint32_t ckptInterval = SharedData::getCkptInterval();
struct timeval tmptime={0,0};
long remaining = ckptInterval;
do {
fd_set rfds;
struct timeval *timeout = NULL;
struct timeval start;
if (ckptInterval > 0) {
timeout = &tmptime;
timeout->tv_sec = remaining;
JASSERT(gettimeofday(&start, NULL) == 0) (JASSERT_ERRNO);
}
FD_ZERO(&rfds);
FD_SET(PROTECTED_COORD_FD, &rfds );
int retval = select(PROTECTED_COORD_FD+1, &rfds, NULL, NULL, timeout);
if (retval == 0) { // timeout expired, time for checkpoint
JTRACE("Timeout expired, checkpointing now.");
return;
} else if (retval > 0) {
JASSERT(FD_ISSET(PROTECTED_COORD_FD, &rfds));
JTRACE("Connect request on virtual coordinator socket.");
break;
}
JASSERT(errno == EINTR) (JASSERT_ERRNO);
if (ckptInterval > 0) {
struct timeval end;
JASSERT(gettimeofday(&end, NULL) == 0) (JASSERT_ERRNO);
remaining -= end.tv_sec - start.tv_sec;
// If the remaining time is negative, we can checkpoint now
if (remaining < 0) {
return;
}
}
} while (remaining > 0);
jalib::JSocket cmdSock(-1);
DmtcpMessage msg;
DmtcpMessage reply(DMT_USER_CMD_RESULT);
do {
cmdSock.close();
cmdSock = _coordinatorSocket.accept();
msg.poison();
JTRACE("Reading from incoming connection...");
cmdSock >> msg;
} while (!cmdSock.isValid());
JASSERT(msg.type == DMT_USER_CMD) (msg.type)
.Text("Unexpected connection.");
reply.coordCmdStatus = CoordCmdStatus::NOERROR;
bool exitWhenDone = false;
switch (msg.coordCmd) {
// case 'b': case 'B': // prefix blocking command, prior to checkpoint command
// JTRACE("blocking checkpoint beginning...");
// blockUntilDone = true;
// break;
case 's': case 'S':
JTRACE("Received status command");
reply.numPeers = 1;
reply.isRunning = 1;
break;
case 'c': case 'C':
JTRACE("checkpointing...");
break;
case 'k': case 'K':
case 'q': case 'Q':
JTRACE("Received KILL command from user, exiting");
exitWhenDone = true;
break;
default:
JTRACE("unhandled user command") (msg.coordCmd);
reply.coordCmdStatus = CoordCmdStatus::ERROR_INVALID_COMMAND;
}
cmdSock << reply;
cmdSock.close();
if (exitWhenDone) {
_real_exit(0);
}
return;
}
