/* Init TLS for the initial thread. Called by the linker _before_ libc is mapped
* in memory. Beware: all writes to libc globals from this function will
* apply to linker-private copies and will not be visible from libc later on.
*
* Note: this function creates a pthread_internal_t for the initial thread and
* stores the pointer in TLS, but does not add it to pthread's thread list. This
* has to be done later from libc itself (see __libc_init_common).
*
* This function also stores a pointer to the kernel argument block in a TLS slot to be
* picked up by the libc constructor.
*/
void __libc_init_tls(KernelArgumentBlock& args) {
__libc_auxv = args.auxv;
static void* tls[BIONIC_TLS_SLOTS];
static pthread_internal_t main_thread;
main_thread.tls = tls;
// Tell the kernel to clear our tid field when we exit, so we're like any other pthread.
// As a side-effect, this tells us our pid (which is the same as the main thread's tid).
main_thread.tid = __set_tid_address(&main_thread.tid);
main_thread.set_cached_pid(main_thread.tid);
// Work out the extent of the main thread's stack.
uintptr_t stack_top = (__get_sp() & ~(PAGE_SIZE - 1)) + PAGE_SIZE;
size_t stack_size = get_main_thread_stack_size();
void* stack_bottom = reinterpret_cast<void*>(stack_top - stack_size);
// We don't want to free the main thread's stack even when the main thread exits
// because things like environment variables with global scope live on it.
pthread_attr_init(&main_thread.attr);
pthread_attr_setstack(&main_thread.attr, stack_bottom, stack_size);
main_thread.attr.flags = PTHREAD_ATTR_FLAG_USER_ALLOCATED_STACK | PTHREAD_ATTR_FLAG_MAIN_THREAD;
__init_thread(&main_thread, false);
__init_tls(&main_thread);
__set_tls(main_thread.tls);
tls[TLS_SLOT_BIONIC_PREINIT] = &args;
__init_alternate_signal_stack(&main_thread);
}
extern "C" void __libc_init_main_thread_late() {
__init_bionic_tls_ptrs(__get_bionic_tcb(), __allocate_temp_bionic_tls());
// Tell the kernel to clear our tid field when we exit, so we're like any other pthread.
// For threads created by pthread_create, this setup happens during the clone syscall (i.e.
// CLONE_CHILD_CLEARTID).
__set_tid_address(&main_thread.tid);
pthread_attr_init(&main_thread.attr);
// We don't want to explicitly set the main thread's scheduler attributes (http://b/68328561).
pthread_attr_setinheritsched(&main_thread.attr, PTHREAD_INHERIT_SCHED);
// The main thread has no guard page.
pthread_attr_setguardsize(&main_thread.attr, 0);
// User code should never see this; we'll compute it when asked.
pthread_attr_setstacksize(&main_thread.attr, 0);
// The TLS stack guard is set from the global, so ensure that we've initialized the global
// before we initialize the TLS. Dynamic executables will initialize their copy of the global
// stack protector from the one in the main thread's TLS.
__libc_safe_arc4random_buf(&__stack_chk_guard, sizeof(__stack_chk_guard));
__init_tcb_stack_guard(__get_bionic_tcb());
__init_thread(&main_thread);
__init_additional_stacks(&main_thread);
}
void __libc_init_main_thread(KernelArgumentBlock& args) {
__libc_auxv = args.auxv;
#if defined(__i386__)
__libc_init_sysinfo(args);
#endif
static pthread_internal_t main_thread;
// The -fstack-protector implementation uses TLS, so make sure that's
// set up before we call any function that might get a stack check inserted.
// TLS also needs to be set up before errno (and therefore syscalls) can be used.
__set_tls(main_thread.tls);
__init_tls(&main_thread);
// Tell the kernel to clear our tid field when we exit, so we're like any other pthread.
// As a side-effect, this tells us our pid (which is the same as the main thread's tid).
main_thread.tid = __set_tid_address(&main_thread.tid);
main_thread.set_cached_pid(main_thread.tid);
// We don't want to free the main thread's stack even when the main thread exits
// because things like environment variables with global scope live on it.
// We also can't free the pthread_internal_t itself, since that lives on the main
// thread's stack rather than on the heap.
// The main thread has no mmap allocated space for stack or pthread_internal_t.
main_thread.mmap_size = 0;
pthread_attr_init(&main_thread.attr);
main_thread.attr.guard_size = 0; // The main thread has no guard page.
main_thread.attr.stack_size = 0; // User code should never see this; we'll compute it when asked.
// TODO: the main thread's sched_policy and sched_priority need to be queried.
// The TLS stack guard is set from the global, so ensure that we've initialized the global
// before we initialize the TLS. Dynamic executables will initialize their copy of the global
// stack protector from the one in the main thread's TLS.
__libc_init_global_stack_chk_guard(args);
__init_thread_stack_guard(&main_thread);
__init_thread(&main_thread);
// Store a pointer to the kernel argument block in a TLS slot to be
// picked up by the libc constructor.
main_thread.tls[TLS_SLOT_BIONIC_PREINIT] = &args;
__init_alternate_signal_stack(&main_thread);
}
int pthread_create(pthread_t* thread_out, pthread_attr_t const* attr,
void* (*start_routine)(void*), void* arg) {
ErrnoRestorer errno_restorer;
// Inform the rest of the C library that at least one thread was created.
__isthreaded = 1;
pthread_internal_t* thread = reinterpret_cast<pthread_internal_t*>(calloc(sizeof(*thread), 1));
if (thread == NULL) {
__libc_format_log(ANDROID_LOG_WARN, "libc", "pthread_create failed: couldn't allocate thread");
return EAGAIN;
}
if (attr == NULL) {
pthread_attr_init(&thread->attr);
} else {
thread->attr = *attr;
attr = NULL; // Prevent misuse below.
}
// Make sure the stack size and guard size are multiples of PAGE_SIZE.
thread->attr.stack_size = BIONIC_ALIGN(thread->attr.stack_size, PAGE_SIZE);
thread->attr.guard_size = BIONIC_ALIGN(thread->attr.guard_size, PAGE_SIZE);
if (thread->attr.stack_base == NULL) {
// The caller didn't provide a stack, so allocate one.
thread->attr.stack_base = __create_thread_stack(thread);
if (thread->attr.stack_base == NULL) {
free(thread);
return EAGAIN;
}
} else {
// The caller did provide a stack, so remember we're not supposed to free it.
thread->attr.flags |= PTHREAD_ATTR_FLAG_USER_ALLOCATED_STACK;
}
// Make room for the TLS area.
// The child stack is the same address, just growing in the opposite direction.
// At offsets >= 0, we have the TLS slots.
// At offsets < 0, we have the child stack.
thread->tls = reinterpret_cast<void**>(reinterpret_cast<uint8_t*>(thread->attr.stack_base) +
thread->attr.stack_size - BIONIC_TLS_SLOTS * sizeof(void*));
void* child_stack = thread->tls;
__init_tls(thread);
// Create a mutex for the thread in TLS to wait on once it starts so we can keep
// it from doing anything until after we notify the debugger about it
//
// This also provides the memory barrier we need to ensure that all
// memory accesses previously performed by this thread are visible to
// the new thread.
pthread_mutex_init(&thread->startup_handshake_mutex, NULL);
pthread_mutex_lock(&thread->startup_handshake_mutex);
thread->start_routine = start_routine;
thread->start_routine_arg = arg;
thread->set_cached_pid(getpid());
int flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM |
CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID;
void* tls = thread->tls;
#if defined(__i386__)
// On x86 (but not x86-64), CLONE_SETTLS takes a pointer to a struct user_desc rather than
// a pointer to the TLS itself.
user_desc tls_descriptor;
__init_user_desc(&tls_descriptor, false, tls);
tls = &tls_descriptor;
#endif
int rc = clone(__pthread_start, child_stack, flags, thread, &(thread->tid), tls, &(thread->tid));
if (rc == -1) {
int clone_errno = errno;
// We don't have to unlock the mutex at all because clone(2) failed so there's no child waiting to
// be unblocked, but we're about to unmap the memory the mutex is stored in, so this serves as a
// reminder that you can't rewrite this function to use a ScopedPthreadMutexLocker.
pthread_mutex_unlock(&thread->startup_handshake_mutex);
if ((thread->attr.flags & PTHREAD_ATTR_FLAG_USER_ALLOCATED_STACK) == 0) {
munmap(thread->attr.stack_base, thread->attr.stack_size);
}
free(thread);
__libc_format_log(ANDROID_LOG_WARN, "libc", "pthread_create failed: clone failed: %s", strerror(errno));
return clone_errno;
}
int init_errno = __init_thread(thread, true);
if (init_errno != 0) {
// Mark the thread detached and replace its start_routine with a no-op.
// Letting the thread run is the easiest way to clean up its resources.
thread->attr.flags |= PTHREAD_ATTR_FLAG_DETACHED;
thread->start_routine = __do_nothing;
pthread_mutex_unlock(&thread->startup_handshake_mutex);
return init_errno;
}
// Publish the pthread_t and unlock the mutex to let the new thread start running.
*thread_out = reinterpret_cast<pthread_t>(thread);
pthread_mutex_unlock(&thread->startup_handshake_mutex);
return 0;
}
int pthread_create(pthread_t* thread_out, pthread_attr_t const* attr,
void* (*start_routine)(void*), void* arg) {
ErrnoRestorer errno_restorer;
// Inform the rest of the C library that at least one thread was created.
__isthreaded = 1;
pthread_attr_t thread_attr;
if (attr == NULL) {
pthread_attr_init(&thread_attr);
} else {
thread_attr = *attr;
attr = NULL; // Prevent misuse below.
}
pthread_internal_t* thread = NULL;
void* child_stack = NULL;
int result = __allocate_thread(&thread_attr, &thread, &child_stack);
if (result != 0) {
return result;
}
// Create a lock for the thread to wait on once it starts so we can keep
// it from doing anything until after we notify the debugger about it
//
// This also provides the memory barrier we need to ensure that all
// memory accesses previously performed by this thread are visible to
// the new thread.
thread->startup_handshake_lock.init(false);
thread->startup_handshake_lock.lock();
thread->start_routine = start_routine;
thread->start_routine_arg = arg;
thread->set_cached_pid(getpid());
int flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM |
CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID;
void* tls = reinterpret_cast<void*>(thread->tls);
#if defined(__i386__)
// On x86 (but not x86-64), CLONE_SETTLS takes a pointer to a struct user_desc rather than
// a pointer to the TLS itself.
user_desc tls_descriptor;
__init_user_desc(&tls_descriptor, false, tls);
tls = &tls_descriptor;
#endif
int rc = clone(__pthread_start, child_stack, flags, thread, &(thread->tid), tls, &(thread->tid));
if (rc == -1) {
int clone_errno = errno;
// We don't have to unlock the mutex at all because clone(2) failed so there's no child waiting to
// be unblocked, but we're about to unmap the memory the mutex is stored in, so this serves as a
// reminder that you can't rewrite this function to use a ScopedPthreadMutexLocker.
thread->startup_handshake_lock.unlock();
if (thread->mmap_size != 0) {
munmap(thread->attr.stack_base, thread->mmap_size);
}
__libc_format_log(ANDROID_LOG_WARN, "libc", "pthread_create failed: clone failed: %s", strerror(errno));
return clone_errno;
}
int init_errno = __init_thread(thread);
if (init_errno != 0) {
// Mark the thread detached and replace its start_routine with a no-op.
// Letting the thread run is the easiest way to clean up its resources.
atomic_store(&thread->join_state, THREAD_DETACHED);
__pthread_internal_add(thread);
thread->start_routine = __do_nothing;
thread->startup_handshake_lock.unlock();
return init_errno;
}
// Publish the pthread_t and unlock the mutex to let the new thread start running.
*thread_out = __pthread_internal_add(thread);
thread->startup_handshake_lock.unlock();
return 0;
}
int pthread_create(pthread_t* thread_out, pthread_attr_t const* attr,
void* (*start_routine)(void*), void* arg) {
ErrnoRestorer errno_restorer;
// Inform the rest of the C library that at least one thread
// was created. This will enforce certain functions to acquire/release
// locks (e.g. atexit()) to protect shared global structures.
// This works because pthread_create() is not called by the C library
// initialization routine that sets up the main thread's data structures.
__isthreaded = 1;
pthread_internal_t* thread = reinterpret_cast<pthread_internal_t*>(calloc(sizeof(*thread), 1));
if (thread == NULL) {
__libc_format_log(ANDROID_LOG_WARN, "libc", "pthread_create failed: couldn't allocate thread");
return EAGAIN;
}
if (attr == NULL) {
pthread_attr_init(&thread->attr);
} else {
thread->attr = *attr;
attr = NULL; // Prevent misuse below.
}
// Make sure the stack size and guard size are multiples of PAGE_SIZE.
thread->attr.stack_size = (thread->attr.stack_size + (PAGE_SIZE-1)) & ~(PAGE_SIZE-1);
thread->attr.guard_size = (thread->attr.guard_size + (PAGE_SIZE-1)) & ~(PAGE_SIZE-1);
if (thread->attr.stack_base == NULL) {
// The caller didn't provide a stack, so allocate one.
thread->attr.stack_base = __create_thread_stack(thread);
if (thread->attr.stack_base == NULL) {
free(thread);
return EAGAIN;
}
} else {
// The caller did provide a stack, so remember we're not supposed to free it.
thread->attr.flags |= PTHREAD_ATTR_FLAG_USER_ALLOCATED_STACK;
}
// Make room for the TLS area.
// The child stack is the same address, just growing in the opposite direction.
// At offsets >= 0, we have the TLS slots.
// At offsets < 0, we have the child stack.
thread->tls = (void**)((uint8_t*)(thread->attr.stack_base) + thread->attr.stack_size - BIONIC_TLS_SLOTS * sizeof(void*));
void* child_stack = thread->tls;
// Create a mutex for the thread in TLS to wait on once it starts so we can keep
// it from doing anything until after we notify the debugger about it
//
// This also provides the memory barrier we need to ensure that all
// memory accesses previously performed by this thread are visible to
// the new thread.
pthread_mutex_t* start_mutex = (pthread_mutex_t*) &thread->tls[TLS_SLOT_START_MUTEX];
pthread_mutex_init(start_mutex, NULL);
pthread_mutex_lock(start_mutex);
thread->tls[TLS_SLOT_THREAD_ID] = thread;
thread->start_routine = start_routine;
thread->start_routine_arg = arg;
int flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM |
CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID;
#if defined(__i386__)
// On x86 (but not x86-64), CLONE_SETTLS takes a pointer to a struct user_desc rather than
// a pointer to the TLS itself. Rather than try to deal with that here, we just let x86 set
// the TLS manually in __init_tls, like all architectures used to.
flags &= ~CLONE_SETTLS;
#endif
int rc = __bionic_clone(flags, child_stack, &(thread->tid), thread->tls, &(thread->tid), __pthread_start, thread);
if (rc == -1) {
int clone_errno = errno;
// We don't have to unlock the mutex at all because clone(2) failed so there's no child waiting to
// be unblocked, but we're about to unmap the memory the mutex is stored in, so this serves as a
// reminder that you can't rewrite this function to use a ScopedPthreadMutexLocker.
pthread_mutex_unlock(start_mutex);
if ((thread->attr.flags & PTHREAD_ATTR_FLAG_USER_ALLOCATED_STACK) == 0) {
munmap(thread->attr.stack_base, thread->attr.stack_size);
}
free(thread);
__libc_format_log(ANDROID_LOG_WARN, "libc", "pthread_create failed: clone failed: %s", strerror(errno));
return clone_errno;
}
int init_errno = __init_thread(thread, true);
if (init_errno != 0) {
// Mark the thread detached and replace its start_routine with a no-op.
// Letting the thread run is the easiest way to clean up its resources.
thread->attr.flags |= PTHREAD_ATTR_FLAG_DETACHED;
thread->start_routine = __do_nothing;
pthread_mutex_unlock(start_mutex);
return init_errno;
}
// Notify any debuggers about the new thread.
{
ScopedPthreadMutexLocker debugger_locker(&gDebuggerNotificationLock);
_thread_created_hook(thread->tid);
}
// Publish the pthread_t and unlock the mutex to let the new thread start running.
*thread_out = reinterpret_cast<pthread_t>(thread);
pthread_mutex_unlock(start_mutex);
return 0;
}
