extern "C" int close(int fd)
{
if (dmtcp::ProtectedFDs::isProtected(fd))
{
JTRACE("blocked attempt to close protected fd") (fd);
errno = EBADF;
return -1;
}
#ifdef EXTERNAL_SOCKET_HANDLING
dmtcp::ConnectionIdentifier conId;
if (dmtcp::WorkerState::currentState() == dmtcp::WorkerState::RUNNING &&
dmtcp::DmtcpWorker::waitingForExternalSocketsToClose() == true &&
dup2(fd,fd) != -1) {
conId = dmtcp::KernelDeviceToConnection::instance().retrieve(fd).id();
}
int rv = _real_close(fd);
if (rv == 0) {
processClose(conId);
}
#else
int rv = _real_close(fd);
#endif
return rv;
}
void dmtcp::ProcessInfo::restart()
{
JASSERT(mprotect((void*)_restoreBufAddr, _restoreBufLen, PROT_NONE) == 0)
((void*)_restoreBufAddr) (_restoreBufLen) (JASSERT_ERRNO);
restoreHeap();
// Update the ckptDir
string ckptDir = jalib::Filesystem::GetDeviceName(PROTECTED_CKPT_DIR_FD);
JASSERT(ckptDir.length() > 0);
_real_close(PROTECTED_CKPT_DIR_FD);
updateCkptDirFileSubdir(ckptDir);
if (_launchCWD != _ckptCWD) {
dmtcp::string rpath = "";
size_t llen = _launchCWD.length();
if (Util::strStartsWith(_ckptCWD.c_str(), _launchCWD.c_str()) &&
_ckptCWD[llen] == '/') {
// _launchCWD = "/A/B"; _ckptCWD = "/A/B/C" -> rpath = "./c"
rpath = "./" + _ckptCWD.substr(llen + 1);
if (chdir(rpath.c_str()) == 0) {
JTRACE("Changed cwd") (_launchCWD) (_ckptCWD) (_launchCWD + rpath);
} else {
JWARNING(chdir(_ckptCWD.c_str()) == 0) (_ckptCWD) (_launchCWD)
(JASSERT_ERRNO) .Text("Failed to change directory to _ckptCWD");
}
}
}
}
LIB_PRIVATE
void initSyncAddresses()
{
int mapsFd = -1;
ProcMapsArea area;
if (isProcessGDB()) {
return;
}
if (areasToNotLogLen > 0) {
return;
}
if ((mapsFd = _real_open("/proc/self/maps", O_RDONLY, S_IRUSR)) == -1) {
perror("open");
exit(1);
}
while (dmtcp::Util::readProcMapsLine(mapsFd, &area)) {
if ((area.prot & PROT_EXEC) != 0 && strlen(area.name) != 0 &&
shouldLogArea(area.name) == false) {
JASSERT(areasToNotLogLen < MAX_PROC_MAPS_AREAS) (areasToNotLogLen);
areasToNotLog[areasToNotLogLen++] = area;
}
}
_real_close(mapsFd);
for (int i = 1; i < areasToNotLogLen; i++) {
JASSERT(areasToNotLog[i].addr >= areasToNotLog[i-1].endAddr)
(areasToNotLog[i].name) (areasToNotLog[i].addr) (areasToNotLog[i].endAddr)
(areasToNotLog[i-1].name) (areasToNotLog[i-1].addr)
(areasToNotLog[i-1].endAddr)
.Text ("ERROR: Blacklisted library addresses not in ascending order.");
}
}
pid_t dmtcp::Util::getTracerPid(pid_t tid)
{
if (!dmtcp_real_to_virtual_pid) {
return 0;
}
char buf[512];
char *str;
static int tracerStrLen = strlen(TRACER_PID_STR);
int fd;
if (tid == -1) {
tid = gettid();
}
sprintf(buf, "/proc/%d/status", tid);
fd = _real_open(buf, O_RDONLY, 0);
JASSERT(fd != -1) (buf) (JASSERT_ERRNO);
readAll(fd, buf, sizeof buf);
_real_close(fd);
str = strstr(buf, TRACER_PID_STR);
JASSERT(str != NULL);
str += tracerStrLen;
while (*str == ' ' || *str == '\t') {
str++;
}
pid_t tracerPid = (pid_t) strtol(str, NULL, 10);
return tracerPid == 0 ? tracerPid : dmtcp_real_to_virtual_pid(tracerPid);
}
void dmtcp::ProcessInfo::refresh()
{
_pid = getpid();
_ppid = getppid();
_gid = getpgid(0);
_sid = getsid(0);
_fgid = -1;
dmtcp::string controllingTerm = jalib::Filesystem::GetControllingTerm();
if (!controllingTerm.empty()) {
int tfd = _real_open(controllingTerm.c_str(), O_RDONLY, 0);
if (tfd >= 0) {
_fgid = tcgetpgrp(tfd);
_real_close(tfd);
}
}
if (_ppid == 1) {
_isRootOfProcessTree = true;
}
_procname = jalib::Filesystem::GetProgramName();
_hostname = jalib::Filesystem::GetCurrentHostname();
_upid = UniquePid::ThisProcess();
_uppid = UniquePid::ParentProcess();
_noCoordinator = dmtcp_no_coordinator();
refreshChildTable();
refreshTidVector();
JTRACE("CHECK GROUP PID")(_gid)(_fgid)(_ppid)(_pid);
}
extern "C" int close(int fd)
{
if (dmtcp::ProtectedFDs::isProtected(fd)) {
JTRACE("blocked attempt to close protected fd") (fd);
errno = EBADF;
return -1;
}
return _real_close(fd);
}
extern "C" int close(int fd)
{
if (DMTCP_IS_PROTECTED_FD(fd)) {
JTRACE("blocked attempt to close protected fd") (fd);
errno = EBADF;
return -1;
}
return _real_close(fd);
}
char*
connectAndSendUserCommand(char c,
int *coordCmdStatus,
int *numPeers,
int *isRunning,
int *ckptInterval)
{
char *replyData = NULL;
int coordFd = createNewSocketToCoordinator(COORD_ANY);
if (coordFd == -1) {
*coordCmdStatus = CoordCmdStatus::ERROR_COORDINATOR_NOT_FOUND;
return replyData;
}
// Tell the coordinator to run given user command
DmtcpMessage msg(DMT_USER_CMD);
msg.coordCmd = c;
if (c == 'i') {
const char *interval = getenv(ENV_VAR_CKPT_INTR);
if (interval != NULL) {
msg.theCheckpointInterval = jalib::StringToInt(interval);
}
}
JASSERT(Util::writeAll(coordFd, &msg, sizeof(msg)) == sizeof(msg));
// The coordinator will violently close our socket...
if (c == 'q' || c == 'Q') {
*coordCmdStatus = CoordCmdStatus::NOERROR;
return replyData;
}
// Receive REPLY
DmtcpMessage reply;
reply.poison();
recvMsgFromCoordinatorRaw(coordFd, &reply, (void**)&replyData);
reply.assertValid();
JASSERT(reply.type == DMT_USER_CMD_RESULT);
if (coordCmdStatus != NULL) {
*coordCmdStatus = reply.coordCmdStatus;
}
if (numPeers != NULL) {
*numPeers = reply.numPeers;
}
if (isRunning != NULL) {
*isRunning = reply.isRunning;
}
if (ckptInterval != NULL) {
*ckptInterval = reply.theCheckpointInterval;
}
_real_close(coordFd);
return replyData;
}
void dmtcp_ProcessInfo_EventHook(DmtcpEvent_t event, DmtcpEventData_t *data)
{
switch (event) {
case DMTCP_EVENT_INIT:
dmtcp::ProcessInfo::instance().init();
break;
case DMTCP_EVENT_PRE_EXEC:
{
jalib::JBinarySerializeWriterRaw wr("", data->serializerInfo.fd);
dmtcp::ProcessInfo::instance().refresh();
dmtcp::ProcessInfo::instance().serialize(wr);
}
break;
case DMTCP_EVENT_POST_EXEC:
{
jalib::JBinarySerializeReaderRaw rd("", data->serializerInfo.fd);
dmtcp::ProcessInfo::instance().serialize(rd);
dmtcp::ProcessInfo::instance().postExec();
}
break;
case DMTCP_EVENT_DRAIN:
dmtcp::ProcessInfo::instance().refresh();
break;
case DMTCP_EVENT_RESTART:
fesetround(roundingMode);
dmtcp::ProcessInfo::instance().restart();
break;
case DMTCP_EVENT_REFILL:
if (data->refillInfo.isRestart) {
dmtcp::ProcessInfo::instance().restoreProcessGroupInfo();
}
break;
case DMTCP_EVENT_THREADS_SUSPEND:
roundingMode = fegetround();
break;
case DMTCP_EVENT_THREADS_RESUME:
if (data->refillInfo.isRestart) {
_real_close(PROTECTED_ENVIRON_FD);
}
break;
default:
break;
}
}
extern "C" int close(int fd)
{
if (dmtcp_is_protected_fd(fd)) {
JTRACE("blocked attempt to close protected fd") (fd);
errno = EBADF;
return -1;
}
DMTCP_DISABLE_CKPT();
int rv = _real_close(fd);
if (rv == 0 && dmtcp_is_running_state()) {
process_fd_event(SYS_close, fd);
}
DMTCP_ENABLE_CKPT();
return rv;
}
void resetOnFork(int sock)
{
JASSERT(Util::isValidFd(sock));
JASSERT(sock != PROTECTED_COORD_FD);
Util::changeFd(sock, PROTECTED_COORD_FD);
JASSERT(Util::isValidFd(coordinatorSocket));
JTRACE("Informing coordinator of new process") (UniquePid::ThisProcess());
DmtcpMessage msg(DMT_UPDATE_PROCESS_INFO_AFTER_FORK);
if (dmtcp_virtual_to_real_pid) {
msg.realPid = dmtcp_virtual_to_real_pid(getpid());
} else {
msg.realPid = getpid();
}
sendMsgToCoordinator(msg);
_real_close(nsSock);
nsSock = -1;
}
int findLibTorque_maps(dmtcp::string &libpath)
{
// /proc/self/maps looks like: "<start addr>-<end addr> <mode> <offset> <device> <inode> <libpath>
// we need to extract libpath
dmtcp::Util::ProcMapsArea area;
int ret = -1;
// we will search for first libpath and first libname
int fd = _real_open ( "/proc/self/maps", O_RDONLY);
if( fd < 0 ){
JTRACE("Cannot open /proc/self/maps file");
return -1;
}
while( dmtcp::Util::readProcMapsLine(fd, &area) ){
libpath = area.name;
JNOTE("Inspect new /proc/seft/maps line")(libpath);
if( libpath.size() == 0 ){
JNOTE("anonymous region, skip");
continue;
}
if( libpath.find("libtorque") != dmtcp::string::npos ){
// this is library path that contains libtorque. This is what we need
JTRACE("Torque PBS libpath")(libpath);
ret = 0;
break;
}else{
JNOTE("Not a libtorque region")(libpath);
}
}
_real_close(fd);
return ret;
}
void initLogsForRecordReplay()
{
if (isProcessGDB()) {
return;
}
global_log.initialize(RECORD_LOG_PATH, MAX_LOG_LENGTH);
if (read_data_fd == -1) {
int fd;
JASSERT(SYNC_IS_RECORD || SYNC_IS_REPLAY);
if (SYNC_IS_RECORD) {
fd = _real_open(RECORD_READ_DATA_LOG_PATH,
O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR);
} else if (SYNC_IS_REPLAY) {
fd = _real_open(RECORD_READ_DATA_LOG_PATH, O_RDONLY, 0);
} else {
JASSERT(false) .Text("Not Reached");
}
JASSERT(fd != -1);
read_data_fd = _real_dup2(fd, dmtcp_get_readlog_fd());
JASSERT(read_data_fd != -1);
_real_close(fd);
}
}
void dmtcp::ProcessInfo::refresh()
{
_pid = getpid();
_ppid = getppid();
_gid = getpgid(0);
_sid = getsid(0);
_fgid = -1;
// Try to open the controlling terminal
int tfd = _real_open("/dev/tty", O_RDWR);
if (tfd != -1) {
_fgid = tcgetpgrp(tfd);
_real_close(tfd);
}
if (_ppid == 1) {
_isRootOfProcessTree = true;
_uppid = UniquePid();
} else {
_uppid = UniquePid::ParentProcess();
}
_procname = jalib::Filesystem::GetProgramName();
_hostname = jalib::Filesystem::GetCurrentHostname();
_upid = UniquePid::ThisProcess();
_noCoordinator = dmtcp_no_coordinator();
char buf[PATH_MAX];
JASSERT(getcwd(buf, sizeof buf) != NULL);
_ckptCWD = buf;
_sessionIds.clear();
refreshChildTable();
JTRACE("CHECK GROUP PID")(_gid)(_fgid)(_ppid)(_pid);
}
void
startNewCoordinator(CoordinatorMode mode)
{
string host;
int port;
getCoordHostAndPort(mode, host, &port);
JASSERT(strcmp(host.c_str(), "localhost") == 0 ||
strcmp(host.c_str(), "127.0.0.1") == 0 ||
jalib::Filesystem::GetCurrentHostname() == host.c_str())
(host) (jalib::Filesystem::GetCurrentHostname())
.Text("Won't automatically start coordinator because DMTCP_HOST"
" is set to a remote host.");
// Create a socket and bind it to an unused port.
errno = 0;
jalib::JServerSocket coordinatorListenerSocket(jalib::JSockAddr::ANY,
port, 128);
JASSERT(coordinatorListenerSocket.isValid())
(coordinatorListenerSocket.port()) (JASSERT_ERRNO) (host) (port)
.Text("Failed to create socket to coordinator port."
"\nIf msg is \"Address already in use\","
" this may be an old coordinator."
"\nEither try again a few seconds or a minute later,"
"\nOr kill other coordinators on this host and port:"
"\n dmtcp_command ---coord-host XXX --coord-port XXX"
"\nOr specify --join-coordinator if joining existing computation.");
// Now dup the sockfd to
coordinatorListenerSocket.changeFd(PROTECTED_COORD_FD);
setCoordPort(coordinatorListenerSocket.port());
JTRACE("Starting a new coordinator automatically.")
(coordinatorListenerSocket.port());
if (fork() == 0) {
/* NOTE: This code assumes that dmtcp_launch (the current program)
* and dmtcp_coordinator are in the same directory. Namely,
* GetProgramDir() gets the dir of the current program (dmtcp_launch).
* Hence, if dmtcp_coordinator is in a different directory, then
* jalib::Filesystem::GetProgramDir() + "/dmtcp_coordinator"
* will not exist, and the child will fail.
*/
// We can't use Util::getPath() here since the SharedData has not been
// initialized yet.
string coordinator =
jalib::Filesystem::GetProgramDir() + "/dmtcp_coordinator";
char *modeStr = (char *)"--daemon";
char *args[] = {
(char *)coordinator.c_str(),
(char *)"--quiet",
/* If we wish to also suppress coordinator warnings, call --quiet twice */
(char *)"--exit-on-last",
modeStr,
NULL
};
execv(args[0], args);
JASSERT(false)(coordinator)(JASSERT_ERRNO).Text(
"exec(dmtcp_coordinator) failed");
} else {
int status;
_real_close(PROTECTED_COORD_FD);
JASSERT(wait(&status) > 0) (JASSERT_ERRNO);
}
}
void dmtcp::DmtcpCoordinatorAPI::startNewCoordinator(int modes, int isRestart)
{
int coordinatorStatus = -1;
//get location of coordinator
const char *coordinatorAddr = getenv ( ENV_VAR_NAME_HOST );
if(coordinatorAddr == NULL) coordinatorAddr = DEFAULT_HOST;
const char *coordinatorPortStr = getenv ( ENV_VAR_NAME_PORT );
dmtcp::string s = coordinatorAddr;
if(s != "localhost" && s != "127.0.0.1" &&
s != jalib::Filesystem::GetCurrentHostname()){
JASSERT(false)(s)(jalib::Filesystem::GetCurrentHostname())
.Text("Won't automatically start coordinator because DMTCP_HOST"
" is set to a remote host.");
_real_exit(DMTCP_FAIL_RC);
}
if ( modes & COORD_BATCH || modes & COORD_FORCE_NEW ) {
// Create a socket and bind it to an unused port.
jalib::JServerSocket coordinatorListenerSocket ( jalib::JSockAddr::ANY, 0 );
errno = 0;
JASSERT ( coordinatorListenerSocket.isValid() )
( coordinatorListenerSocket.port() ) ( JASSERT_ERRNO )
.Text ( "Failed to create listen socket."
"\nIf msg is \"Address already in use\", this may be an old coordinator."
"\nKill other coordinators and try again in a minute or so." );
// Now dup the sockfd to
coordinatorListenerSocket.changeFd(PROTECTED_COORD_FD);
dmtcp::string coordPort= jalib::XToString(coordinatorListenerSocket.port());
setenv ( ENV_VAR_NAME_PORT, coordPort.c_str(), 1 );
}
JTRACE("Starting a new coordinator automatically.") (coordinatorPortStr);
if(fork()==0){
dmtcp::string coordinator = jalib::Filesystem::FindHelperUtility("dmtcp_coordinator");
char *modeStr = (char *)"--background";
if ( modes & COORD_BATCH ) {
modeStr = (char *)"--batch";
}
char * args[] = {
(char*)coordinator.c_str(),
(char*)"--exit-on-last",
modeStr,
NULL
};
execv(args[0], args);
JASSERT(false)(coordinator)(JASSERT_ERRNO).Text("exec(dmtcp_coordinator) failed");
} else {
_real_close ( PROTECTED_COORD_FD );
}
errno = 0;
if ( modes & COORD_BATCH ) {
// FIXME: If running in batch Mode, we sleep here for 5 seconds to let
// the coordinator get started up. We need to fix this in future.
sleep(5);
} else {
JASSERT(wait(&coordinatorStatus)>0)(JASSERT_ERRNO);
JASSERT(WEXITSTATUS(coordinatorStatus) == 0)
.Text("Failed to start coordinator, port already in use. You may use a different port by running with \'-p 12345\'\n");
}
}
void
closeConnection()
{
_real_close(coordinatorSocket);
}
/* Close the "read log", which is done before a checkpoint is taken. */
void close_read_log()
{
_real_close(read_data_fd);
read_data_fd = -1;
}
void dmtcp::ProcessInfo::growStack()
{
/* Grow the stack to the stack limit */
struct rlimit rlim;
size_t stackSize;
const rlim_t eightMB = 8 * MB;
JASSERT(getrlimit(RLIMIT_STACK, &rlim) == 0) (JASSERT_ERRNO);
if (rlim.rlim_cur == RLIM_INFINITY) {
if (rlim.rlim_max == RLIM_INFINITY) {
stackSize = 8 * 1024 * 1024;
} else {
stackSize = MIN(rlim.rlim_max, eightMB);
}
} else {
stackSize = rlim.rlim_cur;
}
// Find the current stack area and heap
ProcMapsArea area;
bool flag = false;
size_t allocSize;
void *tmpbuf;
int fd = _real_open("/proc/self/maps", O_RDONLY);
JASSERT(fd != -1) (JASSERT_ERRNO);
while (Util::readProcMapsLine(fd, &area)) {
if (strcmp(area.name, "[heap]") == 0) {
// Record start of heap which will later be used to restore heap
_savedHeapStart = (unsigned long) area.addr;
}
if ((VA) &area >= area.addr && (VA) &area < area.endAddr) {
// Stack found
flag = true;
break;
}
}
_real_close(fd);
JTRACE("Original stack area") ((void*)area.addr) (area.size);
JASSERT(flag && area.addr != NULL);
// Grow the stack
{
allocSize = stackSize - area.size - 4095;
tmpbuf = alloca(allocSize);
JASSERT(tmpbuf != NULL) (JASSERT_ERRNO);
memset(tmpbuf, 0, allocSize);
}
#ifdef DEBUG
{
int fd = _real_open("/proc/self/maps", O_RDONLY);
JASSERT(fd != -1) (JASSERT_ERRNO);
while (Util::readProcMapsLine(fd, &area)) {
if ((VA)&area >= area.addr && (VA)&area < area.endAddr) { // Stack found
area = area;
break;
}
}
_real_close(fd);
JTRACE("New stack size") ((void*)area.addr) (area.size);
}
#endif
}