static void initialize_pthread_cancel() { /* libpthread defines pthread_cancel_init() which initializes some internal * data structures for future use. The function is called whenever a threads * is cancelled (pthread_exit(), return from start etc.). The function makes * internal calls to dlopen() etc. If pthread_cancel_init() is not called * prior to starting logging, it may cause log divergence as we do not have a * mechanism to capture and serialize calls to libc internal dlopen(). * * The fix is to create a short lived thread which would eventually make a * call to pthread_cancel_init() while exiting thus solving the problem. * Notice that we do not want to create this thread before creating the * checkpoint thread as this could lead to races withing libmtcp.so. The * safest place is to create the thread after checkpoint thread is created * and before starting user main(). */ if (!pthread_cancel_initialized) { if (!isProcessGDB()) { pthread_t pth; JASSERT(_real_pthread_create(&pth, NULL, dummy_start_wrapper, NULL) == 0); pthread_cancel(pth); JASSERT(_real_pthread_join(pth, NULL) == 0); dmtcp::ThreadInfo::destroyThread(pth); pthread_cancel_initialized = true; } } }
extern "C" int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void*), void *arg) { /* Reap Existing Threads */ dmtcp::ThreadInfo::reapThreads(); int retval; /* Here I've split apart WRAPPER_HEADER_RAW because we need to create the * reaper thread even if the current mode is SYNC_IS_NOOP. */ void *return_addr = GET_RETURN_ADDRESS(); if (!dmtcp_is_running_state() || !validAddress(return_addr) || isProcessGDB()) { retval = _real_pthread_create(thread, attr, start_routine, arg); } else if (SYNC_IS_NOOP) { struct create_arg *createArg = (struct create_arg*) JALLOC_HELPER_MALLOC(sizeof(*createArg)); createArg->fn = start_routine; createArg->thread_arg = arg; createArg->userStack = (void*) -1; // Make sure it is non-NULL if (attr != NULL) { createArg->attr = *attr; pthread_attr_getdetachstate(attr, &createArg->userDetachState); } else { createArg->userDetachState = PTHREAD_CREATE_JOINABLE; } dmtcp::ThreadInfo::prePthreadCreate(); retval = _real_pthread_create(thread, attr, start_wrapper, createArg); if (retval != 0) { dmtcp::ThreadInfo::postPthreadCreate(); } } else { retval = internal_pthread_create(thread, attr, start_routine, arg); } if (!pthread_cancel_initialized) { initialize_pthread_cancel(); } return retval; }
extern "C" int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void*), void *arg) { int retval; // We have to use DMTCP-specific memory allocator because using glibc:malloc // can interfere with user threads. // We use JALLOC_HELPER_FREE to free this memory in two places: // 1. near the beginning of pthread_start (wrapper for start_routine), // providing that the __clone call succeeds with no tid conflict. // 2. if the call to _real_pthread_create fails, then free memory // near the end of this function. // We use MALLOC/FREE so that pthread_create() can be called again, without // waiting for the new thread to give up the buffer in pthread_start(). struct ThreadArg *threadArg = (struct ThreadArg *) JALLOC_HELPER_MALLOC (sizeof (struct ThreadArg)); threadArg->pthread_fn = start_routine; threadArg->arg = arg; /* pthread_create() should acquire the thread-creation lock. Not doing so can * result in a deadlock in the following scenario: * 1. user thread: pthread_create() - acquire wrapper-execution lock * 2. ckpt-thread: SUSPEND msg received, wait on wrlock for wrapper-exection lock * 3. user thread: __clone() - try to acquire wrapper-execution lock * * We also need to increment the uninitialized-thread-count so that it is * safe to checkpoint the newly created thread. * * There is another possible deadlock situation if we do not grab the * thread-creation lock: * 1. user thread: pthread_create(): waiting on tbl_lock inside libpthread * 2. ckpt-thread: SUSPEND msg received, wait on wrlock for wrapper-exec lock * 3. uset thread: a. exiting after returning from user fn. * b. grabs tbl_lock() * c. tries to call free() to deallocate previously allocated * space (stack etc.). The free() wrapper requires * wrapper-exec lock, which is not available. */ bool threadCreationLockAcquired = dmtcp::ThreadSync::threadCreationLockLock(); dmtcp::ThreadSync::incrementUninitializedThreadCount(); retval = _real_pthread_create(thread, attr, pthread_start, threadArg); if (threadCreationLockAcquired) { dmtcp::ThreadSync::threadCreationLockUnlock(); } if (retval != 0) { // if we failed to create new pthread JALLOC_HELPER_FREE(threadArg); dmtcp::ThreadSync::decrementUninitializedThreadCount(); } return retval; }
/* Performs the _real version with log and replay. Does NOT check shouldSynchronize() and shouldn't be called directly unless you know what you're doing. */ static int internal_pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void*), void *arg) { int retval = 0, was_detached = 0; pthread_attr_t the_attr; size_t stack_size; void *stack_addr; struct create_arg *createArg = (struct create_arg*) JALLOC_HELPER_MALLOC(sizeof(*createArg)); createArg->fn = start_routine; createArg->thread_arg = arg; if (attr != NULL) { pthread_attr_getstack(attr, &createArg->userStack, &stack_size); pthread_attr_getdetachstate(attr, &createArg->userDetachState); } else { createArg->userStack = NULL; createArg->userDetachState = PTHREAD_CREATE_JOINABLE; } log_entry_t my_entry = create_pthread_create_entry(my_clone_id, pthread_create_event, thread, attr, start_routine, arg); if (SYNC_IS_REPLAY) { WRAPPER_REPLAY_START(pthread_create); stack_addr = (void *)GET_FIELD(my_entry, pthread_create, stack_addr); stack_size = GET_FIELD(my_entry, pthread_create, stack_size); dmtcp::ThreadInfo::prePthreadCreate(); WRAPPER_REPLAY_END(pthread_create); // Register a new thread with ThreadInfo. pthread_t newPthreadId = GET_FIELD(my_entry, pthread_create, ret_thread); // Set up thread stacks to how they were at record time. pthread_attr_init(&the_attr); setupThreadStack(&the_attr, attr, stack_size); // Never let the user create a detached thread: disableDetachState(&the_attr); createArg->attr = the_attr; retval = _real_pthread_create(thread, &the_attr, start_wrapper, (void *)createArg); pthread_attr_destroy(&the_attr); } else if (SYNC_IS_RECORD) { WRAPPER_LOG_RESERVE_SLOT(pthread_create); dmtcp::ThreadInfo::prePthreadCreate(); pthread_attr_init(&the_attr); // Possibly create a thread stack if the user has not provided one: setupThreadStack(&the_attr, attr, 0); // Never let the user create a detached thread: disableDetachState(&the_attr); createArg->attr = the_attr; // MTCP may call jalib::malloc to allocate space for thread struct. dmtcp::ThreadInfo::setOptionalEvent(); retval = _real_pthread_create(thread, &the_attr, start_wrapper, (void *)createArg); dmtcp::ThreadInfo::unsetOptionalEvent(); SET_RETVAL_ERRNO(my_entry, pthread_create, retval, errno); // Log whatever stack we ended up using: pthread_attr_getstack(&the_attr, &stack_addr, &stack_size); pthread_attr_destroy(&the_attr); SET_FIELD(my_entry, pthread_create, stack_addr); SET_FIELD(my_entry, pthread_create, stack_size); SET_FIELD2(my_entry, pthread_create, ret_thread, *thread); // Log annotation on the fly. WRAPPER_LOG_UPDATE_ENTRY(pthread_create); } if (retval != 0) { dmtcp::ThreadInfo::postPthreadCreate(); } return retval; }