char* connectAndSendUserCommand(char c, int *coordCmdStatus, int *numPeers, int *isRunning, int *ckptInterval) { char *replyData = NULL; int coordFd = createNewSocketToCoordinator(COORD_ANY); if (coordFd == -1) { *coordCmdStatus = CoordCmdStatus::ERROR_COORDINATOR_NOT_FOUND; return replyData; } // Tell the coordinator to run given user command DmtcpMessage msg(DMT_USER_CMD); msg.coordCmd = c; if (c == 'i') { const char *interval = getenv(ENV_VAR_CKPT_INTR); if (interval != NULL) { msg.theCheckpointInterval = jalib::StringToInt(interval); } } JASSERT(Util::writeAll(coordFd, &msg, sizeof(msg)) == sizeof(msg)); // The coordinator will violently close our socket... if (c == 'q' || c == 'Q') { *coordCmdStatus = CoordCmdStatus::NOERROR; return replyData; } // Receive REPLY DmtcpMessage reply; reply.poison(); recvMsgFromCoordinatorRaw(coordFd, &reply, (void**)&replyData); reply.assertValid(); JASSERT(reply.type == DMT_USER_CMD_RESULT); if (coordCmdStatus != NULL) { *coordCmdStatus = reply.coordCmdStatus; } if (numPeers != NULL) { *numPeers = reply.numPeers; } if (isRunning != NULL) { *isRunning = reply.isRunning; } if (ckptInterval != NULL) { *ckptInterval = reply.theCheckpointInterval; } _real_close(coordFd); return replyData; }
pid_t dmtcp::DmtcpCoordinatorAPI::getVirtualPidFromCoordinator() { connectToCoordinator(); DmtcpMessage msg(DMT_GET_VIRTUAL_PID); _coordinatorSocket << msg; DmtcpMessage reply; reply.poison(); _coordinatorSocket >> reply; reply.assertValid(); JASSERT(reply.type == DMT_GET_VIRTUAL_PID_RESULT) (reply.type); JASSERT(reply.virtualPid != -1); _coordinatorSocket.close(); return reply.virtualPid; }
// On input, val points to a buffer in user memory and *val_len is the maximum // size of that buffer (the memory allocated by user). // On output, we copy data to val, and set *val_len to the actual buffer size // (to the size of the data that we copied to the user buffer). int CoordinatorAPI::sendQueryToCoordinator(const char *id, const void *key, uint32_t key_len, void *val, uint32_t *val_len) { DmtcpMessage msg (DMT_NAME_SERVICE_QUERY); JWARNING(strlen(id) < sizeof(msg.nsid)); strncpy(msg.nsid, id, 8); msg.keyLen = key_len; msg.valLen = 0; msg.extraBytes = key_len; jalib::JSocket sock = _coordinatorSocket; if (key == NULL || key_len == 0 || val == NULL || val_len == 0) { return 0; } if (dmtcp_is_running_state()) { if (!_nsSock.isValid()) { _nsSock = createNewSocketToCoordinator(COORD_ANY); JASSERT(_nsSock.isValid()); _nsSock.changeFd(PROTECTED_NS_FD); DmtcpMessage m(DMT_NAME_SERVICE_WORKER); _nsSock << m; } sock = _nsSock; JASSERT(sock.isValid()); } sock << msg; sock.writeAll((const char *)key, key_len); msg.poison(); sock >> msg; msg.assertValid(); JASSERT(msg.type == DMT_NAME_SERVICE_QUERY_RESPONSE && msg.extraBytes == msg.valLen); JASSERT (*val_len >= msg.valLen); *val_len = msg.valLen; if (*val_len > 0) { sock.readAll((char*)val, *val_len); } return *val_len; }
int CoordinatorAPI::sendKeyValPairToCoordinator(const char *id, const void *key, uint32_t key_len, const void *val, uint32_t val_len, int sync) { DmtcpMessage msg (DMT_REGISTER_NAME_SERVICE_DATA); if (sync) { msg.type = DMT_REGISTER_NAME_SERVICE_DATA_SYNC; } JWARNING(strlen(id) < sizeof(msg.nsid)); strncpy(msg.nsid, id, 8); msg.keyLen = key_len; msg.valLen = val_len; msg.extraBytes = key_len + val_len; jalib::JSocket sock = _coordinatorSocket; if (dmtcp_is_running_state()) { if (_nsSock.sockfd() == -1) { _nsSock = createNewSocketToCoordinator(COORD_ANY); JASSERT(_nsSock.isValid()); _nsSock.changeFd(PROTECTED_NS_FD); DmtcpMessage m(DMT_NAME_SERVICE_WORKER); _nsSock << m; } sock = _nsSock; JASSERT(sock.isValid()); } sock << msg; sock.writeAll((const char *)key, key_len); sock.writeAll((const char *)val, val_len); if (sync) { msg.poison(); sock >> msg; JASSERT(msg.type == DMT_REGISTER_NAME_SERVICE_DATA_SYNC_RESPONSE)(msg.type); } return 1; }
DmtcpMessage CoordinatorAPI::sendRecvHandshake(DmtcpMessage msg, string progname, UniquePid *compId) { if (dmtcp_virtual_to_real_pid) { msg.realPid = dmtcp_virtual_to_real_pid(getpid()); } else { msg.realPid = getpid(); } msg.theCheckpointInterval = getCkptInterval(); string hostname = jalib::Filesystem::GetCurrentHostname(); msg.extraBytes = hostname.length() + 1 + progname.length() + 1; _coordinatorSocket << msg; _coordinatorSocket.writeAll(hostname.c_str(), hostname.length() + 1); _coordinatorSocket.writeAll(progname.c_str(), progname.length() + 1); msg.poison(); _coordinatorSocket >> msg; msg.assertValid(); if (msg.type == DMT_KILL_PEER) { JTRACE("Received KILL message from coordinator, exiting"); _real_exit (0); } if (msg.type == DMT_REJECT_NOT_RUNNING) { JASSERT(false) .Text("Connection rejected by the coordinator.\n" "Reason: Current computation not in RUNNING state.\n" " Is a checkpoint/restart in progress?"); } else if (msg.type == DMT_REJECT_WRONG_COMP) { JASSERT(compId != NULL); JASSERT(false) (*compId) .Text("Connection rejected by the coordinator.\n" " Reason: This process has a different computation group."); } JASSERT(msg.type == DMT_ACCEPT)(msg.type); return msg; }
void dmtcp::ConnectionRewirer::onConnect ( const jalib::JSocket& sock, const struct sockaddr* /*remoteAddr*/,socklen_t /*remoteLen*/ ) { jalib::JSocket remote = sock; DmtcpMessage msg; msg.poison(); remote >> msg; msg.assertValid(); JASSERT ( msg.type == DMT_RESTORE_RECONNECTED ) ( msg.type ).Text ( "unexpected message" ); iterator i = _pendingIncoming.find ( msg.restorePid ); JASSERT ( i != _pendingIncoming.end() ) ( msg.restorePid ) .Text ( "got unexpected incoming restore request" ); const dmtcp::vector<int>& fds = i->second; JASSERT ( fds.size() > 0 ); int fd0 = fds[0]; remote.changeFd ( fd0 ); JTRACE ( "restoring incoming connection" ) ( msg.restorePid ) ( fd0 ) ( fds.size() ); for ( size_t i = 1; i<fds.size(); ++i ) { JTRACE ( "restoring extra fd" ) ( fd0 ) ( fds[i] ); JASSERT ( _real_dup2 ( fd0,fds[i] ) == fds[i] ) ( fd0 ) ( fds[i] ) ( msg.restorePid ) .Text ( "dup2() failed" ); } _pendingIncoming.erase ( i ); if ( pendingCount() ==0 ) finishup(); #ifdef DEBUG else debugPrint(); #endif }
void waitForCheckpointCommand() { uint32_t ckptInterval = SharedData::getCkptInterval(); struct timeval tmptime = { 0, 0 }; long remaining = ckptInterval; do { struct timeval *timeout = NULL; struct timeval start; if (ckptInterval > 0) { timeout = &tmptime; timeout->tv_sec = remaining; JASSERT(gettimeofday(&start, NULL) == 0) (JASSERT_ERRNO); } // This call to poll() does nothing and returns. // But we want to find address of poll() using dlsym/libc before // allowing the user thread to continue. poll(NULL, 0, 0); if (sem_launch_first_time) { // Release user thread now that we've initialized the checkpoint thread. // This code is reached if the --no-coordinator flag is used. sem_post(&sem_launch); sem_launch_first_time = false; } struct pollfd socketFd = {0}; socketFd.fd = coordinatorSocket; socketFd.events = POLLIN; uint64_t millis = timeout ? ((timeout->tv_sec * (uint64_t)1000) + (timeout->tv_usec / 1000)) : -1; int retval = poll(&socketFd, 1, millis); if (retval == 0) { // timeout expired, time for checkpoint JTRACE("Timeout expired, checkpointing now."); return; } else if (retval > 0) { JASSERT(socketFd.revents & POLLIN); JTRACE("Connect request on virtual coordinator socket."); break; } JASSERT(errno == EINTR) (JASSERT_ERRNO); /* EINTR: a signal was caught */ if (ckptInterval > 0) { struct timeval end; JASSERT(gettimeofday(&end, NULL) == 0) (JASSERT_ERRNO); remaining -= end.tv_sec - start.tv_sec; // If the remaining time is negative, we can checkpoint now if (remaining < 0) { return; } } } while (remaining > 0); jalib::JSocket cmdSock(-1); DmtcpMessage msg; DmtcpMessage reply(DMT_USER_CMD_RESULT); do { cmdSock.close(); jalib::JServerSocket sock(coordinatorSocket); cmdSock = sock.accept(); msg.poison(); JTRACE("Reading from incoming connection..."); cmdSock >> msg; } while (!cmdSock.isValid()); JASSERT(msg.type == DMT_USER_CMD) (msg.type) .Text("Unexpected connection."); reply.coordCmdStatus = CoordCmdStatus::NOERROR; bool exitWhenDone = false; switch (msg.coordCmd) { // case 'b': case 'B': // prefix blocking command, prior to checkpoint // command // JTRACE("blocking checkpoint beginning..."); // blockUntilDone = true; // break; case 's': case 'S': JTRACE("Received status command"); reply.numPeers = 1; reply.isRunning = 1; break; case 'c': case 'C': JTRACE("checkpointing..."); break; case 'k': case 'K': case 'q': case 'Q': JTRACE("Received KILL command from user, exiting"); exitWhenDone = true; break; default: JTRACE("unhandled user command") (msg.coordCmd); reply.coordCmdStatus = CoordCmdStatus::ERROR_INVALID_COMMAND; } cmdSock << reply; cmdSock.close(); if (exitWhenDone) { _real_exit(0); } }
void CoordinatorAPI::waitForCheckpointCommand() { uint32_t ckptInterval = SharedData::getCkptInterval(); struct timeval tmptime={0,0}; long remaining = ckptInterval; do { fd_set rfds; struct timeval *timeout = NULL; struct timeval start; if (ckptInterval > 0) { timeout = &tmptime; timeout->tv_sec = remaining; JASSERT(gettimeofday(&start, NULL) == 0) (JASSERT_ERRNO); } FD_ZERO(&rfds); FD_SET(PROTECTED_COORD_FD, &rfds ); int retval = select(PROTECTED_COORD_FD+1, &rfds, NULL, NULL, timeout); if (retval == 0) { // timeout expired, time for checkpoint JTRACE("Timeout expired, checkpointing now."); return; } else if (retval > 0) { JASSERT(FD_ISSET(PROTECTED_COORD_FD, &rfds)); JTRACE("Connect request on virtual coordinator socket."); break; } JASSERT(errno == EINTR) (JASSERT_ERRNO); if (ckptInterval > 0) { struct timeval end; JASSERT(gettimeofday(&end, NULL) == 0) (JASSERT_ERRNO); remaining -= end.tv_sec - start.tv_sec; // If the remaining time is negative, we can checkpoint now if (remaining < 0) { return; } } } while (remaining > 0); jalib::JSocket cmdSock(-1); DmtcpMessage msg; DmtcpMessage reply(DMT_USER_CMD_RESULT); do { cmdSock.close(); cmdSock = _coordinatorSocket.accept(); msg.poison(); JTRACE("Reading from incoming connection..."); cmdSock >> msg; } while (!cmdSock.isValid()); JASSERT(msg.type == DMT_USER_CMD) (msg.type) .Text("Unexpected connection."); reply.coordCmdStatus = CoordCmdStatus::NOERROR; bool exitWhenDone = false; switch (msg.coordCmd) { // case 'b': case 'B': // prefix blocking command, prior to checkpoint command // JTRACE("blocking checkpoint beginning..."); // blockUntilDone = true; // break; case 's': case 'S': JTRACE("Received status command"); reply.numPeers = 1; reply.isRunning = 1; break; case 'c': case 'C': JTRACE("checkpointing..."); break; case 'k': case 'K': case 'q': case 'Q': JTRACE("Received KILL command from user, exiting"); exitWhenDone = true; break; default: JTRACE("unhandled user command") (msg.coordCmd); reply.coordCmdStatus = CoordCmdStatus::ERROR_INVALID_COMMAND; } cmdSock << reply; cmdSock.close(); if (exitWhenDone) { _real_exit(0); } return; }