int
sendQueryAllToCoordinator(const char *id, void **buf, int *len)
{
DmtcpMessage msg(DMT_NAME_SERVICE_QUERY_ALL);
JWARNING(strlen(id) < sizeof(msg.nsid));
strncpy(msg.nsid, id, sizeof msg.nsid);
int sock = coordinatorSocket;
if (dmtcp_is_running_state()) {
if (nsSock == -1) {
nsSock = createNewSocketToCoordinator(COORD_ANY);
JASSERT(nsSock != -1);
nsSock = Util::changeFd(nsSock, PROTECTED_NS_FD);
JASSERT(nsSock == PROTECTED_NS_FD);
DmtcpMessage m(DMT_NAME_SERVICE_WORKER);
JASSERT(Util::writeAll(nsSock, &m, sizeof(m)) == sizeof(m));
}
sock = nsSock;
}
JASSERT(Util::writeAll(sock, &msg, sizeof(msg)) == sizeof(msg));
msg.poison();
JASSERT(Util::readAll(sock, &msg, sizeof(msg)) == sizeof(msg));
msg.assertValid();
JASSERT(msg.type == DMT_NAME_SERVICE_QUERY_ALL_RESPONSE &&
msg.extraBytes == msg.valLen);
/*
* We can't assume anything about the size of the user-specified buffer,
* so we read in in a safe, temporary buffer. This way there's no stale
* data on the socket for the next reader.
*/
void *tmp = JALLOC_HELPER_MALLOC(msg.extraBytes);
JASSERT (Util::readAll(sock, tmp, msg.extraBytes) == msg.extraBytes);
if (*len > 0) {
if ((size_t)*len < msg.extraBytes) {
JALLOC_HELPER_FREE(tmp);
errno = ERANGE;
return -1;
} else {
memcpy(*buf, tmp, msg.extraBytes);
*len = msg.extraBytes;
JALLOC_HELPER_FREE(tmp);
return 0;
}
} else if (*len == 0) {
// Caller must free this buffer
*buf = tmp;
*len = msg.extraBytes;
return 0;
}
JALLOC_HELPER_FREE(tmp);
errno = EINVAL;
return -1;
}
// Special short-lived processes from executables like /lib/libc.so.6
// and man setuid/setgid executables cannot be loaded with LD_PRELOAD.
// Since they're short-lived, we execute them while holding a lock
// delaying checkpointing.
static void execShortLivedProcessAndExit(const char *path, char *const argv[])
{
unsetenv("LD_PRELOAD"); // /lib/ld.so won't let us preload if exec'ing lib
const unsigned int bufSize = 100000;
char *buf = (char*)JALLOC_HELPER_MALLOC(bufSize);
memset(buf, 0, bufSize);
FILE *output;
if (argv[0] == NULL) {
output = _real_popen(path, "r");
} else {
dmtcp::string command = path;
for (int i = 1; argv[i] != NULL; i++)
command = command + " " + argv[i];
output = _real_popen(command.c_str(), "r");
}
int numRead = fread(buf, 1, bufSize, output);
numRead++, numRead--; // suppress unused-var warning
pclose(output); // /lib/libXXX process is now done; can checkpoint now
// FIXME: code currently allows wrapper to proceed without lock if
// it was busy because of a writer. The unlock will then fail below.
bool __wrapperExecutionLockAcquired = true; // needed for LOCK_UNLOCK macro
WRAPPER_EXECUTION_RELEASE_EXCL_LOCK();
// We are now the new /lib/libXXX process, and it's safe for DMTCP to ckpt us.
printf("%s", buf); // print buf, which is what /lib/libXXX would print
JALLOC_HELPER_FREE(buf);
exit(0);
}
// Invoked via __clone
LIB_PRIVATE
int clone_start(void *arg)
{
struct ThreadArg *threadArg = (struct ThreadArg*) arg;
int (*fn) (void *) = threadArg->fn;
void *thread_arg = threadArg->arg;
void *mtcpArg = threadArg->mtcpArg;
mtcpFuncPtrs.thread_start(mtcpArg);
// Free memory previously allocated through JALLOC_HELPER_MALLOC in __clone
JALLOC_HELPER_FREE(threadArg);
dmtcp_process_event(DMTCP_EVENT_THREAD_START, NULL);
/* Thread finished initialization. It's now safe for this thread to
* participate in checkpoint. Decrement the uninitializedThreadCount in
* DmtcpWorker.
*/
dmtcp::ThreadSync::decrementUninitializedThreadCount();
JTRACE ( "Calling user function" ) (gettid());
int ret = (*fn) ( thread_arg );
mtcpFuncPtrs.thread_return();
return ret;
}
// Invoked via pthread_create as start_routine
// On return, it calls mtcp_threadiszombie()
static void *pthread_start(void *arg)
{
struct ThreadArg *threadArg = (struct ThreadArg*) arg;
void *thread_arg = threadArg->arg;
void * (*pthread_fn) (void *) = threadArg->pthread_fn;
pid_t virtualTid = threadArg->virtualTid;
JASSERT(pthread_fn != 0x0);
JALLOC_HELPER_FREE(arg); // Was allocated in calling thread in pthread_create
mtcpFuncPtrs.fill_in_pthread_id(_real_gettid(), pthread_self());
dmtcp::ThreadSync::threadFinishedInitialization();
void *result = (*pthread_fn)(thread_arg);
JTRACE("Thread returned") (virtualTid);
WRAPPER_EXECUTION_DISABLE_CKPT();
mtcpFuncPtrs.threadiszombie();
/*
* This thread has finished its execution, do some cleanup on our part.
* erasing the virtualTid entry from virtualpidtable
* FIXME: What if the process gets checkpointed after erase() but before the
* thread actually exits?
*/
dmtcp::ProcessInfo::instance().eraseTid(virtualTid);
dmtcp_process_event(DMTCP_EVENT_PTHREAD_RETURN, NULL);
WRAPPER_EXECUTION_ENABLE_CKPT();
dmtcp::ThreadSync::unsetOkToGrabLock();
return result;
}
jalib::JBuffer&
jalib::JBuffer::operator=(const JBuffer &that)
{
JALLOC_HELPER_FREE(_buffer);
_buffer = 0;
_size = 0;
new (this)JBuffer(that);
return *this;
}
extern "C" int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine)(void*), void *arg)
{
int retval;
// We have to use DMTCP-specific memory allocator because using glibc:malloc
// can interfere with user threads.
// We use JALLOC_HELPER_FREE to free this memory in two places:
// 1. near the beginning of pthread_start (wrapper for start_routine),
// providing that the __clone call succeeds with no tid conflict.
// 2. if the call to _real_pthread_create fails, then free memory
// near the end of this function.
// We use MALLOC/FREE so that pthread_create() can be called again, without
// waiting for the new thread to give up the buffer in pthread_start().
struct ThreadArg *threadArg =
(struct ThreadArg *) JALLOC_HELPER_MALLOC (sizeof (struct ThreadArg));
threadArg->pthread_fn = start_routine;
threadArg->arg = arg;
/* pthread_create() should acquire the thread-creation lock. Not doing so can
* result in a deadlock in the following scenario:
* 1. user thread: pthread_create() - acquire wrapper-execution lock
* 2. ckpt-thread: SUSPEND msg received, wait on wrlock for wrapper-exection lock
* 3. user thread: __clone() - try to acquire wrapper-execution lock
*
* We also need to increment the uninitialized-thread-count so that it is
* safe to checkpoint the newly created thread.
*
* There is another possible deadlock situation if we do not grab the
* thread-creation lock:
* 1. user thread: pthread_create(): waiting on tbl_lock inside libpthread
* 2. ckpt-thread: SUSPEND msg received, wait on wrlock for wrapper-exec lock
* 3. uset thread: a. exiting after returning from user fn.
* b. grabs tbl_lock()
* c. tries to call free() to deallocate previously allocated
* space (stack etc.). The free() wrapper requires
* wrapper-exec lock, which is not available.
*/
bool threadCreationLockAcquired = dmtcp::ThreadSync::threadCreationLockLock();
dmtcp::ThreadSync::incrementUninitializedThreadCount();
retval = _real_pthread_create(thread, attr, pthread_start, threadArg);
if (threadCreationLockAcquired) {
dmtcp::ThreadSync::threadCreationLockUnlock();
}
if (retval != 0) { // if we failed to create new pthread
JALLOC_HELPER_FREE(threadArg);
dmtcp::ThreadSync::decrementUninitializedThreadCount();
}
return retval;
}
// Invoked via __clone
LIB_PRIVATE
int clone_start(void *arg)
{
struct ThreadArg *threadArg = (struct ThreadArg*) arg;
int (*fn) (void *) = threadArg->fn;
void *thread_arg = threadArg->arg;
pid_t virtualTid = threadArg -> virtualTid;
if (dmtcp_is_running_state()) {
dmtcpResetTid(virtualTid);
}
// Free memory previously allocated through JALLOC_HELPER_MALLOC in __clone
JALLOC_HELPER_FREE(threadArg);
dmtcp::VirtualPidTable::instance().updateMapping(virtualTid, _real_gettid());
JTRACE("Calling user function") (virtualTid);
return (*fn) ( thread_arg );
}
string
getCoordCkptDir(void)
{
// FIXME: Add a test for make-check.
char buf[PATH_MAX] = { 0 };
if (noCoordinator()) {
return "";
}
DmtcpMessage msg(DMT_GET_CKPT_DIR);
sendMsgToCoordinator(msg);
char *extraData = NULL;
recvMsgFromCoordinator(&msg, (void **)&extraData);
msg.assertValid();
JASSERT(msg.type == DMT_GET_CKPT_DIR_RESULT) (msg.type);
JASSERT(msg.extraBytes > 0 && msg.extraBytes < PATH_MAX);
strcpy(buf, extraData);
JALLOC_HELPER_FREE(extraData);
return buf;
}
static void *start_wrapper(void *arg)
{
dmtcp::ThreadInfo::postPthreadCreate();
JASSERT(my_clone_id != -1);
/*
This start function calls the user's start function. We need this so that we
gain control immediately after the user's start function terminates, but
before control goes back to libpthread. Libpthread will do some cleanup
involving a free() call and some low level locks. Since we can't control the
low level locks, we must implement our own lock: thread_transition_mutex.
*/
void *retval;
struct create_arg *createArg = (struct create_arg *)arg;
void *(*user_fnc) (void *) = createArg->fn;
void *thread_arg = createArg->thread_arg;
dmtcp::ThreadInfo::updateState(pthread_self(), createArg->attr,
createArg->userStack,
createArg->userDetachState);
JALLOC_HELPER_FREE(arg);
retval = (*user_fnc)(thread_arg);
JTRACE ( "User start function over." );
return retval;
}
jalib::JBuffer::~JBuffer()
{
JALLOC_HELPER_FREE(_buffer);
_buffer = 0;
_size = 0;
}
//need to forward user clone
extern "C" int __clone(int (*fn) (void *arg), void *child_stack, int flags,
void *arg, int *parent_tidptr, struct user_desc *newtls,
int *child_tidptr)
{
/*
* struct MtcpRestartThreadArg
*
* DMTCP requires the virtualTids of the threads being created during
* the RESTARTING phase. We use an MtcpRestartThreadArg structure to pass
* the virtualTid of the thread being created from MTCP to DMTCP.
*
* actual clone call: clone (fn, child_stack, flags, void *, ... )
* new clone call : clone (fn, child_stack, flags,
* (struct MtcpRestartThreadArg *), ...)
*
* DMTCP automatically extracts arg from this structure and passes that
* to the _real_clone call.
*
* IMPORTANT NOTE: While updating, this struct must be kept in sync
* with the struct of the same name in mtcp.c
*/
struct MtcpRestartThreadArg {
void * arg;
pid_t virtualTid;
} *mtcpRestartThreadArg;
pid_t virtualTid = -1;
if (!dmtcp_is_running_state()) {
mtcpRestartThreadArg = (struct MtcpRestartThreadArg *) arg;
arg = mtcpRestartThreadArg -> arg;
virtualTid = mtcpRestartThreadArg -> virtualTid;
if (virtualTid != VIRTUAL_TO_REAL_PID(virtualTid)) {
dmtcp::VirtualPidTable::instance().postRestart();
}
} else {
virtualTid = dmtcp::VirtualPidTable::instance().getNewVirtualTid();
dmtcp::VirtualPidTable::instance().writeVirtualTidToFileForPtrace(virtualTid);
}
// We have to use DMTCP-specific memory allocator because using glibc:malloc
// can interfere with user threads.
// We use JALLOC_HELPER_FREE to free this memory in two places:
// 1. later in this function in case of failure on call to __clone; and
// 2. near the beginnging of clone_start (wrapper for start_routine).
struct ThreadArg *threadArg =
(struct ThreadArg *) JALLOC_HELPER_MALLOC(sizeof (struct ThreadArg));
threadArg->fn = fn;
threadArg->arg = arg;
threadArg->virtualTid = virtualTid;
JTRACE("Calling libc:__clone");
pid_t tid = _real_clone(clone_start, child_stack, flags, threadArg,
parent_tidptr, newtls, child_tidptr);
if (dmtcp_is_running_state() && dmtcp::Util::isPtraced()) {
dmtcp::VirtualPidTable::instance().readVirtualTidFromFileForPtrace();
}
if (tid > 0) {
JTRACE("New thread created") (tid);
dmtcp::VirtualPidTable::instance().updateMapping(virtualTid, tid);
} else {
// Free memory previously allocated by calling JALLOC_HELPER_MALLOC
JALLOC_HELPER_FREE(threadArg);
virtualTid = tid;
}
return virtualTid;
}
