static #endif void __init_libc(char **envp, char *pn) { size_t i, *auxv, aux[AUX_CNT] = { 0 }; __environ = envp; for (i=0; envp[i]; i++); libc.auxv = auxv = (void *)(envp+i+1); for (i=0; auxv[i]; i+=2) if (auxv[i]<AUX_CNT) aux[auxv[i]] = auxv[i+1]; __hwcap = aux[AT_HWCAP]; __sysinfo = aux[AT_SYSINFO]; libc.page_size = aux[AT_PAGESZ]; if (pn) { __progname = __progname_full = pn; for (i=0; pn[i]; i++) if (pn[i]=='/') __progname = pn+i+1; } __init_tls(aux); __init_ssp((void *)aux[AT_RANDOM]); if (aux[AT_UID]==aux[AT_EUID] && aux[AT_GID]==aux[AT_EGID] && !aux[AT_SECURE]) return; struct pollfd pfd[3] = { {.fd=0}, {.fd=1}, {.fd=2} };
/* Init TLS for the initial thread. Called by the linker _before_ libc is mapped * in memory. Beware: all writes to libc globals from this function will * apply to linker-private copies and will not be visible from libc later on. * * Note: this function creates a pthread_internal_t for the initial thread and * stores the pointer in TLS, but does not add it to pthread's thread list. This * has to be done later from libc itself (see __libc_init_common). * * This function also stores a pointer to the kernel argument block in a TLS slot to be * picked up by the libc constructor. */ void __libc_init_tls(KernelArgumentBlock& args) { __libc_auxv = args.auxv; static void* tls[BIONIC_TLS_SLOTS]; static pthread_internal_t main_thread; main_thread.tls = tls; // Tell the kernel to clear our tid field when we exit, so we're like any other pthread. // As a side-effect, this tells us our pid (which is the same as the main thread's tid). main_thread.tid = __set_tid_address(&main_thread.tid); main_thread.set_cached_pid(main_thread.tid); // Work out the extent of the main thread's stack. uintptr_t stack_top = (__get_sp() & ~(PAGE_SIZE - 1)) + PAGE_SIZE; size_t stack_size = get_main_thread_stack_size(); void* stack_bottom = reinterpret_cast<void*>(stack_top - stack_size); // We don't want to free the main thread's stack even when the main thread exits // because things like environment variables with global scope live on it. pthread_attr_init(&main_thread.attr); pthread_attr_setstack(&main_thread.attr, stack_bottom, stack_size); main_thread.attr.flags = PTHREAD_ATTR_FLAG_USER_ALLOCATED_STACK | PTHREAD_ATTR_FLAG_MAIN_THREAD; __init_thread(&main_thread, false); __init_tls(&main_thread); __set_tls(main_thread.tls); tls[TLS_SLOT_BIONIC_PREINIT] = &args; __init_alternate_signal_stack(&main_thread); }
void __libc_init_common(uintptr_t *elfdata) { int argc = *elfdata; char** argv = (char**)(elfdata + 1); char** envp = argv + argc + 1; pthread_attr_t thread_attr; static pthread_internal_t thread; static void* tls_area[BIONIC_TLS_SLOTS]; /* setup pthread runtime and maint thread descriptor */ unsigned stacktop = (__get_sp() & ~(PAGE_SIZE - 1)) + PAGE_SIZE; unsigned stacksize = 128 * 1024; unsigned stackbottom = stacktop - stacksize; pthread_attr_init(&thread_attr); pthread_attr_setstack(&thread_attr, (void*)stackbottom, stacksize); _init_thread(&thread, gettid(), &thread_attr, (void*)stackbottom); __init_tls(tls_area, &thread); /* clear errno - requires TLS area */ errno = 0; /* set program name */ __progname = argv[0] ? argv[0] : "<unknown>"; /* setup environment pointer */ environ = envp; /* setup system properties - requires environment */ __system_properties_init(); }
/* Init TLS for the initial thread. Called by the linker _before_ libc is mapped * in memory. Beware: all writes to libc globals from this function will * apply to linker-private copies and will not be visible from libc later on. * * Note: this function creates a pthread_internal_t for the initial thread and * stores the pointer in TLS, but does not add it to pthread's gThreadList. This * has to be done later from libc itself (see __libc_init_common). * * This function also stores the elf_data argument in a specific TLS slot to be later * picked up by the libc constructor. */ void __libc_init_tls(unsigned** elf_data) { unsigned stack_top = (__get_sp() & ~(PAGE_SIZE - 1)) + PAGE_SIZE; unsigned stack_size = 128 * 1024; unsigned stack_bottom = stack_top - stack_size; pthread_attr_t thread_attr; pthread_attr_init(&thread_attr); pthread_attr_setstack(&thread_attr, (void*) stack_bottom, stack_size); static pthread_internal_t thread; _init_thread(&thread, gettid(), &thread_attr, (void*) stack_bottom, false); static void* tls_area[BIONIC_TLS_SLOTS]; __init_tls(tls_area, &thread); tls_area[TLS_SLOT_BIONIC_PREINIT] = elf_data; }
/* Init TLS for the initial thread. Called by the linker _before_ libc is mapped * in memory. Beware: all writes to libc globals from this function will * apply to linker-private copies and will not be visible from libc later on. * * Note: this function creates a pthread_internal_t for the initial thread and * stores the pointer in TLS, but does not add it to pthread's gThreadList. This * has to be done later from libc itself (see __libc_init_common). * * This function also stores a pointer to the kernel argument block in a TLS slot to be * picked up by the libc constructor. */ void __libc_init_tls(KernelArgumentBlock& args) { __libc_auxv = args.auxv; uintptr_t stack_top = (__get_sp() & ~(PAGE_SIZE - 1)) + PAGE_SIZE; size_t stack_size = get_main_thread_stack_size(); uintptr_t stack_bottom = stack_top - stack_size; static void* tls[BIONIC_TLS_SLOTS]; static pthread_internal_t thread; thread.tid = gettid(); thread.tls = tls; pthread_attr_init(&thread.attr); pthread_attr_setstack(&thread.attr, (void*) stack_bottom, stack_size); _init_thread(&thread, false); __init_tls(&thread); tls[TLS_SLOT_BIONIC_PREINIT] = &args; }
void __init_libc(char **envp, char *pn) { size_t i, *auxv, aux[AUX_CNT] = { 0 }; __environ = envp; for (i=0; envp[i]; i++); libc.auxv = auxv = (void *)(envp+i+1); for (i=0; auxv[i]; i+=2) if (auxv[i]<AUX_CNT) aux[auxv[i]] = auxv[i+1]; __hwcap = aux[AT_HWCAP]; __sysinfo = aux[AT_SYSINFO]; if (pn) { __progname = __progname_full = pn; for (i=0; pn[i]; i++) if (pn[i]=='/') __progname = pn+i+1; } __init_tls(aux); __init_security(aux); }
static int __allocate_thread(pthread_attr_t* attr, pthread_internal_t** threadp, void** child_stack) { size_t mmap_size; uint8_t* stack_top; if (attr->stack_base == NULL) { // The caller didn't provide a stack, so allocate one. // Make sure the stack size and guard size are multiples of PAGE_SIZE. mmap_size = BIONIC_ALIGN(attr->stack_size + sizeof(pthread_internal_t), PAGE_SIZE); attr->guard_size = BIONIC_ALIGN(attr->guard_size, PAGE_SIZE); attr->stack_base = __create_thread_mapped_space(mmap_size, attr->guard_size); if (attr->stack_base == NULL) { return EAGAIN; } stack_top = reinterpret_cast<uint8_t*>(attr->stack_base) + mmap_size; } else { // Remember the mmap size is zero and we don't need to free it. mmap_size = 0; stack_top = reinterpret_cast<uint8_t*>(attr->stack_base) + attr->stack_size; } // Mapped space(or user allocated stack) is used for: // pthread_internal_t // thread stack (including guard page) // To safely access the pthread_internal_t and thread stack, we need to find a 16-byte aligned boundary. stack_top = reinterpret_cast<uint8_t*>( (reinterpret_cast<uintptr_t>(stack_top) - sizeof(pthread_internal_t)) & ~0xf); pthread_internal_t* thread = reinterpret_cast<pthread_internal_t*>(stack_top); if (mmap_size == 0) { // If thread was not allocated by mmap(), it may not have been cleared to zero. // So assume the worst and zero it. memset(thread, 0, sizeof(pthread_internal_t)); } attr->stack_size = stack_top - reinterpret_cast<uint8_t*>(attr->stack_base); thread->mmap_size = mmap_size; thread->attr = *attr; __init_tls(thread); *threadp = thread; *child_stack = stack_top; return 0; }
void __libc_init_main_thread(KernelArgumentBlock& args) { __libc_auxv = args.auxv; #if defined(__i386__) __libc_init_sysinfo(args); #endif static pthread_internal_t main_thread; // The -fstack-protector implementation uses TLS, so make sure that's // set up before we call any function that might get a stack check inserted. // TLS also needs to be set up before errno (and therefore syscalls) can be used. __set_tls(main_thread.tls); __init_tls(&main_thread); // Tell the kernel to clear our tid field when we exit, so we're like any other pthread. // As a side-effect, this tells us our pid (which is the same as the main thread's tid). main_thread.tid = __set_tid_address(&main_thread.tid); main_thread.set_cached_pid(main_thread.tid); // We don't want to free the main thread's stack even when the main thread exits // because things like environment variables with global scope live on it. // We also can't free the pthread_internal_t itself, since that lives on the main // thread's stack rather than on the heap. // The main thread has no mmap allocated space for stack or pthread_internal_t. main_thread.mmap_size = 0; pthread_attr_init(&main_thread.attr); main_thread.attr.guard_size = 0; // The main thread has no guard page. main_thread.attr.stack_size = 0; // User code should never see this; we'll compute it when asked. // TODO: the main thread's sched_policy and sched_priority need to be queried. // The TLS stack guard is set from the global, so ensure that we've initialized the global // before we initialize the TLS. Dynamic executables will initialize their copy of the global // stack protector from the one in the main thread's TLS. __libc_init_global_stack_chk_guard(args); __init_thread_stack_guard(&main_thread); __init_thread(&main_thread); // Store a pointer to the kernel argument block in a TLS slot to be // picked up by the libc constructor. main_thread.tls[TLS_SLOT_BIONIC_PREINIT] = &args; __init_alternate_signal_stack(&main_thread); }
// This trampoline is called from the assembly _pthread_clone() function. extern "C" void __thread_entry(void* (*func)(void*), void* arg, void** tls) { // Wait for our creating thread to release us. This lets it have time to // notify gdb about this thread before we start doing anything. // This also provides the memory barrier needed to ensure that all memory // accesses previously made by the creating thread are visible to us. pthread_mutex_t* start_mutex = (pthread_mutex_t*) &tls[TLS_SLOT_SELF]; pthread_mutex_lock(start_mutex); pthread_mutex_destroy(start_mutex); pthread_internal_t* thread = (pthread_internal_t*) tls[TLS_SLOT_THREAD_ID]; thread->tls = tls; __init_tls(thread); if ((thread->internal_flags & kPthreadInitFailed) != 0) { pthread_exit(NULL); } void* result = func(arg); pthread_exit(result); }
static int __pthread_start(void* arg) { pthread_internal_t* thread = reinterpret_cast<pthread_internal_t*>(arg); // Wait for our creating thread to release us. This lets it have time to // notify gdb about this thread before we start doing anything. // This also provides the memory barrier needed to ensure that all memory // accesses previously made by the creating thread are visible to us. pthread_mutex_t* start_mutex = (pthread_mutex_t*) &thread->tls[TLS_SLOT_START_MUTEX]; pthread_mutex_lock(start_mutex); pthread_mutex_destroy(start_mutex); __init_tls(thread); __init_alternate_signal_stack(thread); void* result = thread->start_routine(thread->start_routine_arg); pthread_exit(result); return 0; }
int pthread_create(pthread_t* thread_out, pthread_attr_t const* attr, void* (*start_routine)(void*), void* arg) { ErrnoRestorer errno_restorer; // Inform the rest of the C library that at least one thread was created. __isthreaded = 1; pthread_internal_t* thread = reinterpret_cast<pthread_internal_t*>(calloc(sizeof(*thread), 1)); if (thread == NULL) { __libc_format_log(ANDROID_LOG_WARN, "libc", "pthread_create failed: couldn't allocate thread"); return EAGAIN; } if (attr == NULL) { pthread_attr_init(&thread->attr); } else { thread->attr = *attr; attr = NULL; // Prevent misuse below. } // Make sure the stack size and guard size are multiples of PAGE_SIZE. thread->attr.stack_size = BIONIC_ALIGN(thread->attr.stack_size, PAGE_SIZE); thread->attr.guard_size = BIONIC_ALIGN(thread->attr.guard_size, PAGE_SIZE); if (thread->attr.stack_base == NULL) { // The caller didn't provide a stack, so allocate one. thread->attr.stack_base = __create_thread_stack(thread); if (thread->attr.stack_base == NULL) { free(thread); return EAGAIN; } } else { // The caller did provide a stack, so remember we're not supposed to free it. thread->attr.flags |= PTHREAD_ATTR_FLAG_USER_ALLOCATED_STACK; } // Make room for the TLS area. // The child stack is the same address, just growing in the opposite direction. // At offsets >= 0, we have the TLS slots. // At offsets < 0, we have the child stack. thread->tls = reinterpret_cast<void**>(reinterpret_cast<uint8_t*>(thread->attr.stack_base) + thread->attr.stack_size - BIONIC_TLS_SLOTS * sizeof(void*)); void* child_stack = thread->tls; __init_tls(thread); // Create a mutex for the thread in TLS to wait on once it starts so we can keep // it from doing anything until after we notify the debugger about it // // This also provides the memory barrier we need to ensure that all // memory accesses previously performed by this thread are visible to // the new thread. pthread_mutex_init(&thread->startup_handshake_mutex, NULL); pthread_mutex_lock(&thread->startup_handshake_mutex); thread->start_routine = start_routine; thread->start_routine_arg = arg; thread->set_cached_pid(getpid()); int flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID; void* tls = thread->tls; #if defined(__i386__) // On x86 (but not x86-64), CLONE_SETTLS takes a pointer to a struct user_desc rather than // a pointer to the TLS itself. user_desc tls_descriptor; __init_user_desc(&tls_descriptor, false, tls); tls = &tls_descriptor; #endif int rc = clone(__pthread_start, child_stack, flags, thread, &(thread->tid), tls, &(thread->tid)); if (rc == -1) { int clone_errno = errno; // We don't have to unlock the mutex at all because clone(2) failed so there's no child waiting to // be unblocked, but we're about to unmap the memory the mutex is stored in, so this serves as a // reminder that you can't rewrite this function to use a ScopedPthreadMutexLocker. pthread_mutex_unlock(&thread->startup_handshake_mutex); if ((thread->attr.flags & PTHREAD_ATTR_FLAG_USER_ALLOCATED_STACK) == 0) { munmap(thread->attr.stack_base, thread->attr.stack_size); } free(thread); __libc_format_log(ANDROID_LOG_WARN, "libc", "pthread_create failed: clone failed: %s", strerror(errno)); return clone_errno; } int init_errno = __init_thread(thread, true); if (init_errno != 0) { // Mark the thread detached and replace its start_routine with a no-op. // Letting the thread run is the easiest way to clean up its resources. thread->attr.flags |= PTHREAD_ATTR_FLAG_DETACHED; thread->start_routine = __do_nothing; pthread_mutex_unlock(&thread->startup_handshake_mutex); return init_errno; } // Publish the pthread_t and unlock the mutex to let the new thread start running. *thread_out = reinterpret_cast<pthread_t>(thread); pthread_mutex_unlock(&thread->startup_handshake_mutex); return 0; }