/* Final stages. Compute atan(x) by multiple precision arithmetic */
static double
atanMp (double x, const int pr[])
{
mp_no mpx, mpy, mpy2, mperr, mpt1, mpy1;
double y1, y2;
int i, p;
for (i = 0; i < M; i++)
{
p = pr[i];
__dbl_mp (x, &mpx, p);
__mpatan (&mpx, &mpy, p);
__dbl_mp (u9[i].d, &mpt1, p);
__mul (&mpy, &mpt1, &mperr, p);
__add (&mpy, &mperr, &mpy1, p);
__sub (&mpy, &mperr, &mpy2, p);
__mp_dbl (&mpy1, &y1, p);
__mp_dbl (&mpy2, &y2, p);
if (y1 == y2)
{
LIBC_PROBE (slowatan, 3, &p, &x, &y1);
return y1;
}
}
LIBC_PROBE (slowatan_inexact, 3, &p, &x, &y1);
return y1; /*if impossible to do exact computing */
}
/* Lock and return an arena that can be reused for memory allocation.
Avoid AVOID_ARENA as we have already failed to allocate memory in
it and it is currently locked. */
static mstate
reused_arena (mstate avoid_arena)
{
mstate result;
/* FIXME: Access to next_to_use suffers from data races. */
static mstate next_to_use;
if (next_to_use == NULL)
next_to_use = &main_arena;
/* Iterate over all arenas (including those linked from
free_list). */
result = next_to_use;
do
{
if (!__libc_lock_trylock (result->mutex))
goto out;
/* FIXME: This is a data race, see _int_new_arena. */
result = result->next;
}
while (result != next_to_use);
/* Avoid AVOID_ARENA as we have already failed to allocate memory
in that arena and it is currently locked. */
if (result == avoid_arena)
result = result->next;
/* No arena available without contention. Wait for the next in line. */
LIBC_PROBE (memory_arena_reuse_wait, 3, &result->mutex, result, avoid_arena);
__libc_lock_lock (result->mutex);
out:
/* Attach the arena to the current thread. */
{
/* Update the arena thread attachment counters. */
mstate replaced_arena = thread_arena;
__libc_lock_lock (free_list_lock);
detach_arena (replaced_arena);
/* We may have picked up an arena on the free list. We need to
preserve the invariant that no arena on the free list has a
positive attached_threads counter (otherwise,
arena_thread_freeres cannot use the counter to determine if the
arena needs to be put on the free list). We unconditionally
remove the selected arena from the free list. The caller of
reused_arena checked the free list and observed it to be empty,
so the list is very short. */
remove_from_free_list (result);
++result->attached_threads;
__libc_lock_unlock (free_list_lock);
}
LIBC_PROBE (memory_arena_reuse, 2, result, avoid_arena);
thread_arena = result;
next_to_use = result->next;
return result;
}
/* Stage 3: Perform a multi-Precision computation */
static double
SECTION
atan2Mp (double x, double y, const int pr[])
{
double z1, z2;
int i, p;
mp_no mpx, mpy, mpz, mpz1, mpz2, mperr, mpt1;
for (i = 0; i < MM; i++)
{
p = pr[i];
__dbl_mp (x, &mpx, p);
__dbl_mp (y, &mpy, p);
__mpatan2 (&mpy, &mpx, &mpz, p);
__dbl_mp (ud[i].d, &mpt1, p);
__mul (&mpz, &mpt1, &mperr, p);
__add (&mpz, &mperr, &mpz1, p);
__sub (&mpz, &mperr, &mpz2, p);
__mp_dbl (&mpz1, &z1, p);
__mp_dbl (&mpz2, &z2, p);
if (z1 == z2)
{
LIBC_PROBE (slowatan2, 4, &p, &x, &y, &z1);
return z1;
}
}
LIBC_PROBE (slowatan2_inexact, 4, &p, &x, &y, &z1);
return z1; /*if impossible to do exact computing */
}
/* Lock and return an arena that can be reused for memory allocation.
Avoid AVOID_ARENA as we have already failed to allocate memory in
it and it is currently locked. */
static mstate
reused_arena (mstate avoid_arena)
{
mstate result;
static mstate next_to_use;
if (next_to_use == NULL)
next_to_use = &main_arena;
result = next_to_use;
do
{
if (!arena_is_corrupt (result) && !mutex_trylock (&result->mutex))
goto out;
result = result->next;
}
while (result != next_to_use);
/* Avoid AVOID_ARENA as we have already failed to allocate memory
in that arena and it is currently locked. */
if (result == avoid_arena)
result = result->next;
/* Make sure that the arena we get is not corrupted. */
mstate begin = result;
while (arena_is_corrupt (result) || result == avoid_arena)
{
result = result->next;
if (result == begin)
break;
}
/* We could not find any arena that was either not corrupted or not the one
we wanted to avoid. */
if (result == begin || result == avoid_arena)
return NULL;
/* No arena available without contention. Wait for the next in line. */
LIBC_PROBE (memory_arena_reuse_wait, 3, &result->mutex, result, avoid_arena);
(void) mutex_lock (&result->mutex);
out:
LIBC_PROBE (memory_arena_reuse, 2, result, avoid_arena);
tsd_setspecific (arena_key, (void *) result);
next_to_use = result->next;
return result;
}
/* See __pthread_cond_wait for details. */
int
__pthread_cond_init (pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
{
ASSERT_TYPE_SIZE (pthread_cond_t, __SIZEOF_PTHREAD_COND_T);
struct pthread_condattr *icond_attr = (struct pthread_condattr *) cond_attr;
memset (cond, 0, sizeof (pthread_cond_t));
/* Update the pretty printers if the internal representation of icond_attr
is changed. */
/* Iff not equal to ~0l, this is a PTHREAD_PROCESS_PRIVATE condvar. */
if (icond_attr != NULL && (icond_attr->value & 1) != 0)
cond->__data.__wrefs |= __PTHREAD_COND_SHARED_MASK;
int clockid = (icond_attr != NULL
? ((icond_attr->value >> 1) & ((1 << COND_CLOCK_BITS) - 1))
: CLOCK_REALTIME);
/* If 0, CLOCK_REALTIME is used; CLOCK_MONOTONIC otherwise. */
if (clockid != CLOCK_REALTIME)
cond->__data.__wrefs |= __PTHREAD_COND_CLOCK_MONOTONIC_MASK;
LIBC_PROBE (cond_init, 2, cond, cond_attr);
return 0;
}
static int
grow_heap (heap_info *h, long diff)
{
size_t pagesize = GLRO (dl_pagesize);
long new_size;
diff = ALIGN_UP (diff, pagesize);
new_size = (long) h->size + diff;
if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE)
return -1;
if ((unsigned long) new_size > h->mprotect_size)
{
if (__mprotect ((char *) h + h->mprotect_size,
(unsigned long) new_size - h->mprotect_size,
PROT_READ | PROT_WRITE) != 0)
return -2;
h->mprotect_size = new_size;
}
h->size = new_size;
LIBC_PROBE (memory_heap_more, 2, h, h->size);
return 0;
}
/* Receive double x and two double results of cos(x) and return result which is
more accurate, computing cos(x) with multi precision routine c32. */
double
SECTION
__cos32 (double x, double res, double res1)
{
int p;
mp_no a, b, c;
p = 32;
__dbl_mp (res, &a, p);
__dbl_mp (0.5 * (res1 - res), &b, p);
__add (&a, &b, &c, p);
if (x > 2.4)
{
__sub (&pi, &c, &a, p);
__c32 (&a, &b, &c, p);
b.d[0] = -b.d[0];
}
else if (x > 0.8)
{
__sub (&hp, &c, &a, p);
__c32 (&a, &c, &b, p);
}
else
__c32 (&c, &b, &a, p); /* b=cos(0.5*(res+res1)) */
__dbl_mp (x, &c, p); /* c = x */
__sub (&b, &c, &a, p);
/* if a > 0 return max (res, res1), otherwise return min (res, res1). */
if ((a.d[0] > 0 && res <= res1) || (a.d[0] <= 0 && res >= res1))
res = res1;
LIBC_PROBE (slowacos, 2, &res, &x);
return res;
}
/* Remove an arena from free_list. */
static mstate
get_free_list (void)
{
mstate replaced_arena = thread_arena;
mstate result = free_list;
if (result != NULL)
{
__libc_lock_lock (free_list_lock);
result = free_list;
if (result != NULL)
{
free_list = result->next_free;
/* The arena will be attached to this thread. */
assert (result->attached_threads == 0);
result->attached_threads = 1;
detach_arena (replaced_arena);
}
__libc_lock_unlock (free_list_lock);
if (result != NULL)
{
LIBC_PROBE (memory_arena_reuse_free_list, 1, result);
__libc_lock_lock (result->mutex);
thread_arena = result;
}
}
return result;
}
/* Unlock RWLOCK. */
int
__pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
{
LIBC_PROBE (rwlock_unlock, 1, rwlock);
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
if (rwlock->__data.__writer)
rwlock->__data.__writer = 0;
else
--rwlock->__data.__nr_readers;
if (rwlock->__data.__nr_readers == 0)
{
if (rwlock->__data.__nr_writers_queued)
{
++rwlock->__data.__writer_wakeup;
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
lll_futex_wake (&rwlock->__data.__writer_wakeup, 1,
rwlock->__data.__shared);
return 0;
}
else if (rwlock->__data.__nr_readers_queued)
{
++rwlock->__data.__readers_wakeup;
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
rwlock->__data.__shared);
return 0;
}
}
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
return 0;
}
static int
shrink_heap (heap_info *h, long diff)
{
long new_size;
new_size = (long) h->size - diff;
if (new_size < (long) sizeof (*h))
return -1;
/* Try to re-map the extra heap space freshly to save memory, and make it
inaccessible. See malloc-sysdep.h to know when this is true. */
if (__glibc_unlikely (check_may_shrink_heap ()))
{
if ((char *) MMAP ((char *) h + new_size, diff, PROT_NONE,
MAP_FIXED) == (char *) MAP_FAILED)
return -2;
h->mprotect_size = new_size;
}
else
__madvise ((char *) h + new_size, diff, MADV_DONTNEED);
/*fprintf(stderr, "shrink %p %08lx\n", h, new_size);*/
h->size = new_size;
LIBC_PROBE (memory_heap_less, 2, h, h->size);
return 0;
}
/*Converting from double precision to Multi-precision and calculating e^x */
double
SECTION
__slowexp (double x)
{
#ifndef USE_LONG_DOUBLE_FOR_MP
double w, z, res, eps = 3.0e-26;
int p;
mp_no mpx, mpy, mpz, mpw, mpeps, mpcor;
/* Use the multiple precision __MPEXP function to compute the exponential
First at 144 bits and if it is not accurate enough, at 768 bits. */
p = 6;
__dbl_mp (x, &mpx, p);
__mpexp (&mpx, &mpy, p);
__dbl_mp (eps, &mpeps, p);
__mul (&mpeps, &mpy, &mpcor, p);
__add (&mpy, &mpcor, &mpw, p);
__sub (&mpy, &mpcor, &mpz, p);
__mp_dbl (&mpw, &w, p);
__mp_dbl (&mpz, &z, p);
if (w == z)
{
/* Track how often we get to the slow exp code plus
its input/output values. */
LIBC_PROBE (slowexp_p6, 2, &x, &w);
return w;
}
else
{
p = 32;
__dbl_mp (x, &mpx, p);
__mpexp (&mpx, &mpy, p);
__mp_dbl (&mpy, &res, p);
/* Track how often we get to the uber-slow exp code plus
its input/output values. */
LIBC_PROBE (slowexp_p32, 2, &x, &res);
return res;
}
#else
return (double) __ieee754_expl((long double)x);
#endif
}
/* Compute cos() of double-length number (X + DX) as Multi Precision number and
return result as double. If REDUCE_RANGE is true, X is assumed to be the
original input and DX is ignored. */
double
SECTION
__mpcos (double x, double dx, bool reduce_range)
{
double y;
mp_no a, b, c, s;
int n;
int p = 32;
if (reduce_range)
{
n = __mpranred (x, &a, p); /* n is 0, 1, 2 or 3. */
__c32 (&a, &c, &s, p);
}
else
{
n = -1;
__dbl_mp (x, &b, p);
__dbl_mp (dx, &c, p);
__add (&b, &c, &a, p);
if (x > 0.8)
{
__sub (&hp, &a, &b, p);
__c32 (&b, &s, &c, p);
}
else
__c32 (&a, &c, &s, p); /* a = cos(x+dx) */
}
/* Convert result based on which quarter of unit circle y is in. */
switch (n)
{
case 1:
__mp_dbl (&s, &y, p);
y = -y;
break;
case 3:
__mp_dbl (&s, &y, p);
break;
case 2:
__mp_dbl (&c, &y, p);
y = -y;
break;
/* Quadrant not set, so the result must be cos (X + DX), which is also
stored in C. */
case 0:
default:
__mp_dbl (&c, &y, p);
}
LIBC_PROBE (slowcos, 3, &x, &dx, &y);
return y;
}
int
__pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
{
int result = 0;
LIBC_PROBE (rdlock_entry, 1, rwlock);
if (ELIDE_LOCK (rwlock->__data.__rwelision,
rwlock->__data.__lock == 0
&& rwlock->__data.__writer == 0
&& rwlock->__data.__nr_readers == 0))
return 0;
/* Make sure we are alone. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
/* Get the rwlock if there is no writer... */
if (rwlock->__data.__writer == 0
/* ...and if either no writer is waiting or we prefer readers. */
&& (!rwlock->__data.__nr_writers_queued
|| PTHREAD_RWLOCK_PREFER_READER_P (rwlock)))
{
/* Increment the reader counter. Avoid overflow. */
if (__glibc_unlikely (++rwlock->__data.__nr_readers == 0))
{
/* Overflow on number of readers. */
--rwlock->__data.__nr_readers;
result = EAGAIN;
}
else
LIBC_PROBE (rdlock_acquire_read, 1, rwlock);
/* We are done, free the lock. */
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
return result;
}
return __pthread_rwlock_rdlock_slow (rwlock);
}
/*Converting from double precision to Multi-precision and calculating e^x */
double __slowexp(double x) {
double w,z,res,eps=3.0e-26;
#if 0
double y;
#endif
int p;
#if 0
int orig,i;
#endif
mp_no mpx, mpy, mpz,mpw,mpeps,mpcor;
p=6;
__dbl_mp(x,&mpx,p); /* Convert a double precision number x */
/* into a multiple precision number mpx with prec. p. */
__mpexp(&mpx, &mpy, p); /* Multi-Precision exponential function */
__dbl_mp(eps,&mpeps,p);
__mul(&mpeps,&mpy,&mpcor,p);
__add(&mpy,&mpcor,&mpw,p);
__sub(&mpy,&mpcor,&mpz,p);
__mp_dbl(&mpw, &w, p);
__mp_dbl(&mpz, &z, p);
if (w == z) {
/* Track how often we get to the slow exp code plus
its input/output values. */
LIBC_PROBE (slowexp_p6, 2, &x, &w);
return w;
}
else { /* if calculating is not exactly */
p = 32;
__dbl_mp(x,&mpx,p);
__mpexp(&mpx, &mpy, p);
__mp_dbl(&mpy, &res, p);
/* Track how often we get to the uber-slow exp code plus
its input/output values. */
LIBC_PROBE (slowexp_p32, 2, &x, &res);
return res;
}
}
int
__pthread_cond_signal (pthread_cond_t *cond)
{
int pshared = (cond->__data.__mutex == (void *) ~0l)
? LLL_SHARED : LLL_PRIVATE;
LIBC_PROBE (cond_signal, 1, cond);
/* Make sure we are alone. */
lll_lock (cond->__data.__lock, pshared);
/* Are there any waiters to be woken? */
if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
{
/* Yes. Mark one of them as woken. */
++cond->__data.__wakeup_seq;
++cond->__data.__futex;
#if (defined lll_futex_cmp_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
pthread_mutex_t *mut = cond->__data.__mutex;
if (USE_REQUEUE_PI (mut)
/* This can only really fail with a ENOSYS, since nobody can modify
futex while we have the cond_lock. */
&& lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, 0,
&mut->__data.__lock,
cond->__data.__futex, pshared) == 0)
{
lll_unlock (cond->__data.__lock, pshared);
return 0;
}
else
#endif
/* Wake one. */
if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex,
1, 1,
&cond->__data.__lock,
pshared), 0))
return 0;
/* Fallback if neither of them work. */
lll_futex_wake (&cond->__data.__futex, 1, pshared);
}
/* We are done. */
lll_unlock (cond->__data.__lock, pshared);
return 0;
}
int
__pthread_mutex_destroy (pthread_mutex_t *mutex)
{
LIBC_PROBE (mutex_destroy, 1, mutex);
if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0
&& mutex->__data.__nusers != 0)
return EBUSY;
/* Set to an invalid value. */
mutex->__data.__kind = -1;
return 0;
}
static mstate
_int_new_arena (size_t size)
{
mstate a;
heap_info *h;
char *ptr;
unsigned long misalign;
h = new_heap (size + (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT),
mp_.top_pad);
if (!h)
{
/* Maybe size is too large to fit in a single heap. So, just try
to create a minimally-sized arena and let _int_malloc() attempt
to deal with the large request via mmap_chunk(). */
h = new_heap (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT, mp_.top_pad);
if (!h)
return 0;
}
a = h->ar_ptr = (mstate) (h + 1);
malloc_init_state (a);
/*a->next = NULL;*/
a->system_mem = a->max_system_mem = h->size;
arena_mem += h->size;
/* Set up the top chunk, with proper alignment. */
ptr = (char *) (a + 1);
misalign = (unsigned long) chunk2mem (ptr) & MALLOC_ALIGN_MASK;
if (misalign > 0)
ptr += MALLOC_ALIGNMENT - misalign;
top (a) = (mchunkptr) ptr;
set_head (top (a), (((char *) h + h->size) - ptr) | PREV_INUSE);
LIBC_PROBE (memory_arena_new, 2, a, size);
tsd_setspecific (arena_key, (void *) a);
mutex_init (&a->mutex);
(void) mutex_lock (&a->mutex);
(void) mutex_lock (&list_lock);
/* Add the new arena to the global list. */
a->next = main_arena.next;
atomic_write_barrier ();
main_arena.next = a;
(void) mutex_unlock (&list_lock);
return a;
}
/* If we don't have the main arena, then maybe the failure is due to running
out of mmapped areas, so we can try allocating on the main arena.
Otherwise, it is likely that sbrk() has failed and there is still a chance
to mmap(), so try one of the other arenas. */
static mstate
arena_get_retry (mstate ar_ptr, size_t bytes)
{
LIBC_PROBE (memory_arena_retry, 2, bytes, ar_ptr);
if (ar_ptr != &main_arena)
{
__libc_lock_unlock (ar_ptr->mutex);
ar_ptr = &main_arena;
__libc_lock_lock (ar_ptr->mutex);
}
else
{
__libc_lock_unlock (ar_ptr->mutex);
ar_ptr = arena_get2 (bytes, ar_ptr);
}
return ar_ptr;
}
/* Unlock RWLOCK. */
int
__pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
{
int futex_shared =
rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
LIBC_PROBE (rwlock_unlock, 1, rwlock);
if (ELIDE_UNLOCK (rwlock->__data.__writer == 0
&& rwlock->__data.__nr_readers == 0))
return 0;
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
if (rwlock->__data.__writer)
rwlock->__data.__writer = 0;
else
--rwlock->__data.__nr_readers;
/* If there are still readers present, we do not yet need to wake writers
nor are responsible to wake any readers. */
if (rwlock->__data.__nr_readers == 0)
{
/* Note that if there is a blocked writer, we effectively make it
responsible for waking any readers because we don't wake readers in
this case. */
if (rwlock->__data.__nr_writers_queued)
{
++rwlock->__data.__writer_wakeup;
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
futex_wake (&rwlock->__data.__writer_wakeup, 1, futex_shared);
return 0;
}
else if (rwlock->__data.__nr_readers_queued)
{
++rwlock->__data.__readers_wakeup;
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
futex_shared);
return 0;
}
}
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
return 0;
}
/* If we don't have the main arena, then maybe the failure is due to running
out of mmapped areas, so we can try allocating on the main arena.
Otherwise, it is likely that sbrk() has failed and there is still a chance
to mmap(), so try one of the other arenas. */
static mstate
arena_get_retry (mstate ar_ptr, size_t bytes)
{
LIBC_PROBE (memory_arena_retry, 2, bytes, ar_ptr);
if (ar_ptr != &main_arena)
{
(void) mutex_unlock (&ar_ptr->mutex);
ar_ptr = &main_arena;
(void) mutex_lock (&ar_ptr->mutex);
}
else
{
/* Grab ar_ptr->next prior to releasing its lock. */
mstate prev = ar_ptr->next ? ar_ptr : 0;
(void) mutex_unlock (&ar_ptr->mutex);
ar_ptr = arena_get2 (prev, bytes, ar_ptr);
}
return ar_ptr;
}
int
__pthread_mutex_lock (pthread_mutex_t *mutex)
{
assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
LIBC_PROBE (mutex_entry, 1, mutex);
if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
| PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
return __pthread_mutex_lock_full (mutex);
if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
{
FORCE_ELISION (mutex, goto elision);
simple:
/* Normal mutex. */
LLL_MUTEX_LOCK (mutex);
assert (mutex->__data.__owner == 0);
}
/* If we don't have the main arena, then maybe the failure is due to running
out of mmapped areas, so we can try allocating on the main arena.
Otherwise, it is likely that sbrk() has failed and there is still a chance
to mmap(), so try one of the other arenas. */
static mstate
arena_get_retry (mstate ar_ptr, size_t bytes)
{
LIBC_PROBE (memory_arena_retry, 2, bytes, ar_ptr);
if (ar_ptr != &main_arena)
{
__libc_lock_unlock (ar_ptr->mutex);
/* Don't touch the main arena if it is corrupt. */
if (arena_is_corrupt (&main_arena))
return NULL;
ar_ptr = &main_arena;
__libc_lock_lock (ar_ptr->mutex);
}
else
{
__libc_lock_unlock (ar_ptr->mutex);
ar_ptr = arena_get2 (bytes, ar_ptr);
}
return ar_ptr;
}
static mstate
get_free_list (void)
{
mstate result = free_list;
if (result != NULL)
{
(void) mutex_lock (&list_lock);
result = free_list;
if (result != NULL)
free_list = result->next_free;
(void) mutex_unlock (&list_lock);
if (result != NULL)
{
LIBC_PROBE (memory_arena_reuse_free_list, 1, result);
(void) mutex_lock (&result->mutex);
tsd_setspecific (arena_key, (void *) result);
}
}
return result;
}
double
SECTION
__ieee754_log (double x)
{
#define M 4
static const int pr[M] = { 8, 10, 18, 32 };
int i, j, n, ux, dx, p;
double dbl_n, u, p0, q, r0, w, nln2a, luai, lubi, lvaj, lvbj,
sij, ssij, ttij, A, B, B0, y, y1, y2, polI, polII, sa, sb,
t1, t2, t7, t8, t, ra, rb, ww,
a0, aa0, s1, s2, ss2, s3, ss3, a1, aa1, a, aa, b, bb, c;
#ifndef DLA_FMS
double t3, t4, t5, t6;
#endif
number num;
mp_no mpx, mpy, mpy1, mpy2, mperr;
#include "ulog.tbl"
#include "ulog.h"
/* Treating special values of x ( x<=0, x=INF, x=NaN etc.). */
num.d = x;
ux = num.i[HIGH_HALF];
dx = num.i[LOW_HALF];
n = 0;
if (__glibc_unlikely (ux < 0x00100000))
{
if (__glibc_unlikely (((ux & 0x7fffffff) | dx) == 0))
return MHALF / 0.0; /* return -INF */
if (__glibc_unlikely (ux < 0))
return (x - x) / 0.0; /* return NaN */
n -= 54;
x *= two54.d; /* scale x */
num.d = x;
}
if (__glibc_unlikely (ux >= 0x7ff00000))
return x + x; /* INF or NaN */
/* Regular values of x */
w = x - 1;
if (__glibc_likely (ABS (w) > U03))
goto case_03;
/*--- Stage I, the case abs(x-1) < 0.03 */
t8 = MHALF * w;
EMULV (t8, w, a, aa, t1, t2, t3, t4, t5);
EADD (w, a, b, bb);
/* Evaluate polynomial II */
polII = b7.d + w * b8.d;
polII = b6.d + w * polII;
polII = b5.d + w * polII;
polII = b4.d + w * polII;
polII = b3.d + w * polII;
polII = b2.d + w * polII;
polII = b1.d + w * polII;
polII = b0.d + w * polII;
polII *= w * w * w;
c = (aa + bb) + polII;
/* End stage I, case abs(x-1) < 0.03 */
if ((y = b + (c + b * E2)) == b + (c - b * E2))
return y;
/*--- Stage II, the case abs(x-1) < 0.03 */
a = d19.d + w * d20.d;
a = d18.d + w * a;
a = d17.d + w * a;
a = d16.d + w * a;
a = d15.d + w * a;
a = d14.d + w * a;
a = d13.d + w * a;
a = d12.d + w * a;
a = d11.d + w * a;
EMULV (w, a, s2, ss2, t1, t2, t3, t4, t5);
ADD2 (d10.d, dd10.d, s2, ss2, s3, ss3, t1, t2);
MUL2 (w, 0, s3, ss3, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (d9.d, dd9.d, s2, ss2, s3, ss3, t1, t2);
MUL2 (w, 0, s3, ss3, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (d8.d, dd8.d, s2, ss2, s3, ss3, t1, t2);
MUL2 (w, 0, s3, ss3, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (d7.d, dd7.d, s2, ss2, s3, ss3, t1, t2);
MUL2 (w, 0, s3, ss3, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (d6.d, dd6.d, s2, ss2, s3, ss3, t1, t2);
MUL2 (w, 0, s3, ss3, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (d5.d, dd5.d, s2, ss2, s3, ss3, t1, t2);
MUL2 (w, 0, s3, ss3, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (d4.d, dd4.d, s2, ss2, s3, ss3, t1, t2);
MUL2 (w, 0, s3, ss3, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (d3.d, dd3.d, s2, ss2, s3, ss3, t1, t2);
MUL2 (w, 0, s3, ss3, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (d2.d, dd2.d, s2, ss2, s3, ss3, t1, t2);
MUL2 (w, 0, s3, ss3, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
MUL2 (w, 0, s2, ss2, s3, ss3, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (w, 0, s3, ss3, b, bb, t1, t2);
/* End stage II, case abs(x-1) < 0.03 */
if ((y = b + (bb + b * E4)) == b + (bb - b * E4))
return y;
goto stage_n;
/*--- Stage I, the case abs(x-1) > 0.03 */
case_03:
/* Find n,u such that x = u*2**n, 1/sqrt(2) < u < sqrt(2) */
n += (num.i[HIGH_HALF] >> 20) - 1023;
num.i[HIGH_HALF] = (num.i[HIGH_HALF] & 0x000fffff) | 0x3ff00000;
if (num.d > SQRT_2)
{
num.d *= HALF;
n++;
}
u = num.d;
dbl_n = (double) n;
/* Find i such that ui=1+(i-75)/2**8 is closest to u (i= 0,1,2,...,181) */
num.d += h1.d;
i = (num.i[HIGH_HALF] & 0x000fffff) >> 12;
/* Find j such that vj=1+(j-180)/2**16 is closest to v=u/ui (j= 0,...,361) */
num.d = u * Iu[i].d + h2.d;
j = (num.i[HIGH_HALF] & 0x000fffff) >> 4;
/* Compute w=(u-ui*vj)/(ui*vj) */
p0 = (1 + (i - 75) * DEL_U) * (1 + (j - 180) * DEL_V);
q = u - p0;
r0 = Iu[i].d * Iv[j].d;
w = q * r0;
/* Evaluate polynomial I */
polI = w + (a2.d + a3.d * w) * w * w;
/* Add up everything */
nln2a = dbl_n * LN2A;
luai = Lu[i][0].d;
lubi = Lu[i][1].d;
lvaj = Lv[j][0].d;
lvbj = Lv[j][1].d;
EADD (luai, lvaj, sij, ssij);
EADD (nln2a, sij, A, ttij);
B0 = (((lubi + lvbj) + ssij) + ttij) + dbl_n * LN2B;
B = polI + B0;
/* End stage I, case abs(x-1) >= 0.03 */
if ((y = A + (B + E1)) == A + (B - E1))
return y;
/*--- Stage II, the case abs(x-1) > 0.03 */
/* Improve the accuracy of r0 */
EMULV (p0, r0, sa, sb, t1, t2, t3, t4, t5);
t = r0 * ((1 - sa) - sb);
EADD (r0, t, ra, rb);
/* Compute w */
MUL2 (q, 0, ra, rb, w, ww, t1, t2, t3, t4, t5, t6, t7, t8);
EADD (A, B0, a0, aa0);
/* Evaluate polynomial III */
s1 = (c3.d + (c4.d + c5.d * w) * w) * w;
EADD (c2.d, s1, s2, ss2);
MUL2 (s2, ss2, w, ww, s3, ss3, t1, t2, t3, t4, t5, t6, t7, t8);
MUL2 (s3, ss3, w, ww, s2, ss2, t1, t2, t3, t4, t5, t6, t7, t8);
ADD2 (s2, ss2, w, ww, s3, ss3, t1, t2);
ADD2 (s3, ss3, a0, aa0, a1, aa1, t1, t2);
/* End stage II, case abs(x-1) >= 0.03 */
if ((y = a1 + (aa1 + E3)) == a1 + (aa1 - E3))
return y;
/* Final stages. Use multi-precision arithmetic. */
stage_n:
for (i = 0; i < M; i++)
{
p = pr[i];
__dbl_mp (x, &mpx, p);
__dbl_mp (y, &mpy, p);
__mplog (&mpx, &mpy, p);
__dbl_mp (e[i].d, &mperr, p);
__add (&mpy, &mperr, &mpy1, p);
__sub (&mpy, &mperr, &mpy2, p);
__mp_dbl (&mpy1, &y1, p);
__mp_dbl (&mpy2, &y2, p);
if (y1 == y2)
{
LIBC_PROBE (slowlog, 3, &p, &x, &y1);
return y1;
}
}
LIBC_PROBE (slowlog_inexact, 3, &p, &x, &y1);
return y1;
}
static mstate
_int_new_arena (size_t size)
{
mstate a;
heap_info *h;
char *ptr;
unsigned long misalign;
h = new_heap (size + (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT),
mp_.top_pad);
if (!h)
{
/* Maybe size is too large to fit in a single heap. So, just try
to create a minimally-sized arena and let _int_malloc() attempt
to deal with the large request via mmap_chunk(). */
h = new_heap (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT, mp_.top_pad);
if (!h)
return 0;
}
a = h->ar_ptr = (mstate) (h + 1);
malloc_init_state (a);
a->attached_threads = 1;
/*a->next = NULL;*/
a->system_mem = a->max_system_mem = h->size;
/* Set up the top chunk, with proper alignment. */
ptr = (char *) (a + 1);
misalign = (unsigned long) chunk2mem (ptr) & MALLOC_ALIGN_MASK;
if (misalign > 0)
ptr += MALLOC_ALIGNMENT - misalign;
top (a) = (mchunkptr) ptr;
set_head (top (a), (((char *) h + h->size) - ptr) | PREV_INUSE);
LIBC_PROBE (memory_arena_new, 2, a, size);
mstate replaced_arena = thread_arena;
thread_arena = a;
__libc_lock_init (a->mutex);
__libc_lock_lock (list_lock);
/* Add the new arena to the global list. */
a->next = main_arena.next;
/* FIXME: The barrier is an attempt to synchronize with read access
in reused_arena, which does not acquire list_lock while
traversing the list. */
atomic_write_barrier ();
main_arena.next = a;
__libc_lock_unlock (list_lock);
__libc_lock_lock (free_list_lock);
detach_arena (replaced_arena);
__libc_lock_unlock (free_list_lock);
/* Lock this arena. NB: Another thread may have been attached to
this arena because the arena is now accessible from the
main_arena.next list and could have been picked by reused_arena.
This can only happen for the last arena created (before the arena
limit is reached). At this point, some arena has to be attached
to two threads. We could acquire the arena lock before list_lock
to make it less likely that reused_arena picks this new arena,
but this could result in a deadlock with
__malloc_fork_lock_parent. */
__libc_lock_lock (a->mutex);
return a;
}
internal_function
new_heap (size_t size, size_t top_pad)
{
size_t pagesize = GLRO (dl_pagesize);
char *p1, *p2;
unsigned long ul;
heap_info *h;
if (size + top_pad < HEAP_MIN_SIZE)
size = HEAP_MIN_SIZE;
else if (size + top_pad <= HEAP_MAX_SIZE)
size += top_pad;
else if (size > HEAP_MAX_SIZE)
return 0;
else
size = HEAP_MAX_SIZE;
size = ALIGN_UP (size, pagesize);
/* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed.
No swap space needs to be reserved for the following large
mapping (on Linux, this is the case for all non-writable mappings
anyway). */
p2 = MAP_FAILED;
if (aligned_heap_area)
{
p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE,
MAP_NORESERVE);
aligned_heap_area = NULL;
if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)))
{
__munmap (p2, HEAP_MAX_SIZE);
p2 = MAP_FAILED;
}
}
if (p2 == MAP_FAILED)
{
p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE);
if (p1 != MAP_FAILED)
{
p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1))
& ~(HEAP_MAX_SIZE - 1));
ul = p2 - p1;
if (ul)
__munmap (p1, ul);
else
aligned_heap_area = p2 + HEAP_MAX_SIZE;
__munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul);
}
else
{
/* Try to take the chance that an allocation of only HEAP_MAX_SIZE
is already aligned. */
p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE);
if (p2 == MAP_FAILED)
return 0;
if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))
{
__munmap (p2, HEAP_MAX_SIZE);
return 0;
}
}
}
if (__mprotect (p2, size, PROT_READ | PROT_WRITE) != 0)
{
__munmap (p2, HEAP_MAX_SIZE);
return 0;
}
h = (heap_info *) p2;
h->size = size;
h->mprotect_size = size;
LIBC_PROBE (memory_heap_new, 2, h, h->size);
return h;
}
static int
internal_function
heap_trim (heap_info *heap, size_t pad)
{
mstate ar_ptr = heap->ar_ptr;
unsigned long pagesz = GLRO (dl_pagesize);
mchunkptr top_chunk = top (ar_ptr), p, bck, fwd;
heap_info *prev_heap;
long new_size, top_size, top_area, extra, prev_size, misalign;
/* Can this heap go away completely? */
while (top_chunk == chunk_at_offset (heap, sizeof (*heap)))
{
prev_heap = heap->prev;
prev_size = prev_heap->size - (MINSIZE - 2 * SIZE_SZ);
p = chunk_at_offset (prev_heap, prev_size);
/* fencepost must be properly aligned. */
misalign = ((long) p) & MALLOC_ALIGN_MASK;
p = chunk_at_offset (prev_heap, prev_size - misalign);
assert (p->size == (0 | PREV_INUSE)); /* must be fencepost */
p = prev_chunk (p);
new_size = chunksize (p) + (MINSIZE - 2 * SIZE_SZ) + misalign;
assert (new_size > 0 && new_size < (long) (2 * MINSIZE));
if (!prev_inuse (p))
new_size += p->prev_size;
assert (new_size > 0 && new_size < HEAP_MAX_SIZE);
if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz)
break;
ar_ptr->system_mem -= heap->size;
arena_mem -= heap->size;
LIBC_PROBE (memory_heap_free, 2, heap, heap->size);
delete_heap (heap);
heap = prev_heap;
if (!prev_inuse (p)) /* consolidate backward */
{
p = prev_chunk (p);
unlink (ar_ptr, p, bck, fwd);
}
assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0);
assert (((char *) p + new_size) == ((char *) heap + heap->size));
top (ar_ptr) = top_chunk = p;
set_head (top_chunk, new_size | PREV_INUSE);
/*check_chunk(ar_ptr, top_chunk);*/
}
/* Uses similar logic for per-thread arenas as the main arena with systrim
by preserving the top pad and at least a page. */
top_size = chunksize (top_chunk);
top_area = top_size - MINSIZE - 1;
if (top_area <= pad)
return 0;
extra = ALIGN_DOWN(top_area - pad, pagesz);
if ((unsigned long) extra < mp_.trim_threshold)
return 0;
/* Try to shrink. */
if (shrink_heap (heap, extra) != 0)
return 0;
ar_ptr->system_mem -= extra;
arena_mem -= extra;
/* Success. Adjust top accordingly. */
set_head (top_chunk, (top_size - extra) | PREV_INUSE);
/*check_chunk(ar_ptr, top_chunk);*/
return 1;
}
__pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock)
{
int result = 0;
/* Lock is taken in caller. */
while (1)
{
/* Make sure we are not holding the rwlock as a writer. This is
a deadlock situation we recognize and report. */
if (__builtin_expect (rwlock->__data.__writer
== THREAD_GETMEM (THREAD_SELF, tid), 0))
{
result = EDEADLK;
break;
}
/* Remember that we are a reader. */
if (__glibc_unlikely (++rwlock->__data.__nr_readers_queued == 0))
{
/* Overflow on number of queued readers. */
--rwlock->__data.__nr_readers_queued;
result = EAGAIN;
break;
}
int waitval = rwlock->__data.__readers_wakeup;
/* Free the lock. */
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
/* Wait for the writer to finish. */
lll_futex_wait (&rwlock->__data.__readers_wakeup, waitval,
rwlock->__data.__shared);
/* Get the lock. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
--rwlock->__data.__nr_readers_queued;
/* Get the rwlock if there is no writer... */
if (rwlock->__data.__writer == 0
/* ...and if either no writer is waiting or we prefer readers. */
&& (!rwlock->__data.__nr_writers_queued
|| PTHREAD_RWLOCK_PREFER_READER_P (rwlock)))
{
/* Increment the reader counter. Avoid overflow. */
if (__glibc_unlikely (++rwlock->__data.__nr_readers == 0))
{
/* Overflow on number of readers. */
--rwlock->__data.__nr_readers;
result = EAGAIN;
}
else
LIBC_PROBE (rdlock_acquire_read, 1, rwlock);
break;
}
}
/* We are done, free the lock. */
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
return result;
}
