static void initialize_pthread_cancel()
{
/* libpthread defines pthread_cancel_init() which initializes some internal
* data structures for future use. The function is called whenever a threads
* is cancelled (pthread_exit(), return from start etc.). The function makes
* internal calls to dlopen() etc. If pthread_cancel_init() is not called
* prior to starting logging, it may cause log divergence as we do not have a
* mechanism to capture and serialize calls to libc internal dlopen().
*
* The fix is to create a short lived thread which would eventually make a
* call to pthread_cancel_init() while exiting thus solving the problem.
* Notice that we do not want to create this thread before creating the
* checkpoint thread as this could lead to races withing libmtcp.so. The
* safest place is to create the thread after checkpoint thread is created
* and before starting user main().
*/
if (!pthread_cancel_initialized) {
if (!isProcessGDB()) {
pthread_t pth;
JASSERT(_real_pthread_create(&pth, NULL, dummy_start_wrapper, NULL) == 0);
pthread_cancel(pth);
JASSERT(_real_pthread_join(pth, NULL) == 0);
dmtcp::ThreadInfo::destroyThread(pth);
pthread_cancel_initialized = true;
}
}
}
extern "C" int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine)(void*), void *arg)
{
/* Reap Existing Threads */
dmtcp::ThreadInfo::reapThreads();
int retval;
/* Here I've split apart WRAPPER_HEADER_RAW because we need to create the
* reaper thread even if the current mode is SYNC_IS_NOOP. */
void *return_addr = GET_RETURN_ADDRESS();
if (!dmtcp_is_running_state() ||
!validAddress(return_addr) || isProcessGDB()) {
retval = _real_pthread_create(thread, attr, start_routine, arg);
} else if (SYNC_IS_NOOP) {
struct create_arg *createArg =
(struct create_arg*) JALLOC_HELPER_MALLOC(sizeof(*createArg));
createArg->fn = start_routine;
createArg->thread_arg = arg;
createArg->userStack = (void*) -1; // Make sure it is non-NULL
if (attr != NULL) {
createArg->attr = *attr;
pthread_attr_getdetachstate(attr, &createArg->userDetachState);
} else {
createArg->userDetachState = PTHREAD_CREATE_JOINABLE;
}
dmtcp::ThreadInfo::prePthreadCreate();
retval = _real_pthread_create(thread, attr, start_wrapper, createArg);
if (retval != 0) {
dmtcp::ThreadInfo::postPthreadCreate();
}
} else {
retval = internal_pthread_create(thread, attr, start_routine, arg);
}
if (!pthread_cancel_initialized) {
initialize_pthread_cancel();
}
return retval;
}
extern "C" int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine)(void*), void *arg)
{
int retval;
// We have to use DMTCP-specific memory allocator because using glibc:malloc
// can interfere with user threads.
// We use JALLOC_HELPER_FREE to free this memory in two places:
// 1. near the beginning of pthread_start (wrapper for start_routine),
// providing that the __clone call succeeds with no tid conflict.
// 2. if the call to _real_pthread_create fails, then free memory
// near the end of this function.
// We use MALLOC/FREE so that pthread_create() can be called again, without
// waiting for the new thread to give up the buffer in pthread_start().
struct ThreadArg *threadArg =
(struct ThreadArg *) JALLOC_HELPER_MALLOC (sizeof (struct ThreadArg));
threadArg->pthread_fn = start_routine;
threadArg->arg = arg;
/* pthread_create() should acquire the thread-creation lock. Not doing so can
* result in a deadlock in the following scenario:
* 1. user thread: pthread_create() - acquire wrapper-execution lock
* 2. ckpt-thread: SUSPEND msg received, wait on wrlock for wrapper-exection lock
* 3. user thread: __clone() - try to acquire wrapper-execution lock
*
* We also need to increment the uninitialized-thread-count so that it is
* safe to checkpoint the newly created thread.
*
* There is another possible deadlock situation if we do not grab the
* thread-creation lock:
* 1. user thread: pthread_create(): waiting on tbl_lock inside libpthread
* 2. ckpt-thread: SUSPEND msg received, wait on wrlock for wrapper-exec lock
* 3. uset thread: a. exiting after returning from user fn.
* b. grabs tbl_lock()
* c. tries to call free() to deallocate previously allocated
* space (stack etc.). The free() wrapper requires
* wrapper-exec lock, which is not available.
*/
bool threadCreationLockAcquired = dmtcp::ThreadSync::threadCreationLockLock();
dmtcp::ThreadSync::incrementUninitializedThreadCount();
retval = _real_pthread_create(thread, attr, pthread_start, threadArg);
if (threadCreationLockAcquired) {
dmtcp::ThreadSync::threadCreationLockUnlock();
}
if (retval != 0) { // if we failed to create new pthread
JALLOC_HELPER_FREE(threadArg);
dmtcp::ThreadSync::decrementUninitializedThreadCount();
}
return retval;
}
/* Performs the _real version with log and replay. Does NOT check
shouldSynchronize() and shouldn't be called directly unless you know what
you're doing. */
static int internal_pthread_create(pthread_t *thread,
const pthread_attr_t *attr, void *(*start_routine)(void*), void *arg)
{
int retval = 0, was_detached = 0;
pthread_attr_t the_attr;
size_t stack_size;
void *stack_addr;
struct create_arg *createArg =
(struct create_arg*) JALLOC_HELPER_MALLOC(sizeof(*createArg));
createArg->fn = start_routine;
createArg->thread_arg = arg;
if (attr != NULL) {
pthread_attr_getstack(attr, &createArg->userStack, &stack_size);
pthread_attr_getdetachstate(attr, &createArg->userDetachState);
} else {
createArg->userStack = NULL;
createArg->userDetachState = PTHREAD_CREATE_JOINABLE;
}
log_entry_t my_entry = create_pthread_create_entry(my_clone_id,
pthread_create_event,
thread, attr,
start_routine, arg);
if (SYNC_IS_REPLAY) {
WRAPPER_REPLAY_START(pthread_create);
stack_addr = (void *)GET_FIELD(my_entry, pthread_create, stack_addr);
stack_size = GET_FIELD(my_entry, pthread_create, stack_size);
dmtcp::ThreadInfo::prePthreadCreate();
WRAPPER_REPLAY_END(pthread_create);
// Register a new thread with ThreadInfo.
pthread_t newPthreadId = GET_FIELD(my_entry, pthread_create, ret_thread);
// Set up thread stacks to how they were at record time.
pthread_attr_init(&the_attr);
setupThreadStack(&the_attr, attr, stack_size);
// Never let the user create a detached thread:
disableDetachState(&the_attr);
createArg->attr = the_attr;
retval = _real_pthread_create(thread, &the_attr,
start_wrapper, (void *)createArg);
pthread_attr_destroy(&the_attr);
} else if (SYNC_IS_RECORD) {
WRAPPER_LOG_RESERVE_SLOT(pthread_create);
dmtcp::ThreadInfo::prePthreadCreate();
pthread_attr_init(&the_attr);
// Possibly create a thread stack if the user has not provided one:
setupThreadStack(&the_attr, attr, 0);
// Never let the user create a detached thread:
disableDetachState(&the_attr);
createArg->attr = the_attr;
// MTCP may call jalib::malloc to allocate space for thread struct.
dmtcp::ThreadInfo::setOptionalEvent();
retval = _real_pthread_create(thread, &the_attr,
start_wrapper, (void *)createArg);
dmtcp::ThreadInfo::unsetOptionalEvent();
SET_RETVAL_ERRNO(my_entry, pthread_create, retval, errno);
// Log whatever stack we ended up using:
pthread_attr_getstack(&the_attr, &stack_addr, &stack_size);
pthread_attr_destroy(&the_attr);
SET_FIELD(my_entry, pthread_create, stack_addr);
SET_FIELD(my_entry, pthread_create, stack_size);
SET_FIELD2(my_entry, pthread_create, ret_thread, *thread);
// Log annotation on the fly.
WRAPPER_LOG_UPDATE_ENTRY(pthread_create);
}
if (retval != 0) {
dmtcp::ThreadInfo::postPthreadCreate();
}
return retval;
}
