/* Note: this is VG_, not ML_. */
SysRes VG_(am_do_mmap_NO_NOTIFY)( Addr start, SizeT length, UInt prot,
UInt flags, Int fd, Off64T offset)
{
SysRes res;
aspacem_assert(VG_IS_PAGE_ALIGNED(offset));
# if defined(VGP_x86_linux) || defined(VGP_ppc32_linux) \
|| defined(VGP_arm_linux)
/* mmap2 uses 4096 chunks even if actual page size is bigger. */
aspacem_assert((offset % 4096) == 0);
res = VG_(do_syscall6)(__NR_mmap2, (UWord)start, length,
prot, flags, fd, offset / 4096);
# elif defined(VGP_amd64_linux) || defined(VGP_ppc64_linux) \
|| defined(VGP_s390x_linux) || defined(VGP_mips32_linux)
res = VG_(do_syscall6)(__NR_mmap, (UWord)start, length,
prot, flags, fd, offset);
# elif defined(VGP_x86_darwin)
if (fd == 0 && (flags & VKI_MAP_ANONYMOUS)) {
fd = -1; // MAP_ANON with fd==0 is EINVAL
}
res = VG_(do_syscall7)(__NR_mmap, (UWord)start, length,
prot, flags, fd, offset & 0xffffffff, offset >> 32);
# elif defined(VGP_amd64_darwin)
if (fd == 0 && (flags & VKI_MAP_ANONYMOUS)) {
fd = -1; // MAP_ANON with fd==0 is EINVAL
}
res = VG_(do_syscall6)(__NR_mmap, (UWord)start, length,
prot, flags, (UInt)fd, offset);
# else
# error Unknown platform
# endif
return res;
}
static void
put_slotsize(UInt ix, UInt size)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
aspacem_assert(size <= max_slotsize);
segnames[ix - 1] = size & 0xff;
segnames[ix - 2] = size >> 8;
}
static void
put_refcount(UInt ix, UInt rc)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
aspacem_assert(rc <= max_refcount);
// rc <= max_refcount ensures that the F-bit is zero
segnames[ix - 3] = rc & 0xff;
segnames[ix - 4] = rc >> 8;
}
static UInt
get_refcount(UInt ix)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
// must not be a free slot
aspacem_assert(! is_freeslot(ix));
// Avoid unexpected sign extension
const UChar *unames = (const UChar *)segnames;
return (unames[ix - 4] << 8) | unames[ix - 3];
}
static void
put_slotindex(UInt ix, UInt slotindex)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
if (slotindex != 0)
aspacem_assert(slotindex >= overhead && slotindex <= segnames_used);
slotindex |= fbit_mask << 24;
segnames[ix - 1] = slotindex & 0xFF; slotindex >>= 8;
segnames[ix - 2] = slotindex & 0xFF; slotindex >>= 8;
segnames[ix - 3] = slotindex & 0xFF; slotindex >>= 8;
segnames[ix - 4] = slotindex & 0xFF;
}
static UInt
get_slotindex(UInt ix)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
aspacem_assert(is_freeslot(ix));
// Avoid unexpected sign extension
const UChar *unames = (const UChar *)segnames;
UInt slotindex = 0;
slotindex |= unames[ix - 4]; slotindex <<= 8;
slotindex |= unames[ix - 3]; slotindex <<= 8;
slotindex |= unames[ix - 2]; slotindex <<= 8;
slotindex |= unames[ix - 1];
return slotindex & max_slotindex; // removes the F-bit
}
static void
inc_refcount(UInt ix)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
UInt rc = get_refcount(ix);
if (rc != max_refcount)
put_refcount(ix, rc + 1);
}
static void
put_sentinel(UInt ix)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
put_refcount(ix, 0);
put_slotsize(ix, 0);
}
/* Initialise the string table for segment names. It contains an empty
string which is not referenced. */
void
ML_(am_segnames_init)(void)
{
aspacem_assert(sizeof segnames >= overhead);
segnames_used = overhead;
put_sentinel(segnames_used);
}
/* Note: this is VG_, not ML_. */
SysRes VG_(am_do_mmap_NO_NOTIFY)( Addr start, SizeT length, UInt prot,
UInt flags, UInt fd, Off64T offset)
{
SysRes res;
aspacem_assert(VG_IS_PAGE_ALIGNED(offset));
# if defined(VGP_x86_linux) || defined(VGP_ppc32_linux)
/* mmap2 uses 4096 chunks even if actual page size is bigger. */
aspacem_assert((offset % 4096) == 0);
res = VG_(do_syscall6)(__NR_mmap2, (UWord)start, length,
prot, flags, fd, offset / 4096);
# elif defined(VGP_amd64_linux) || defined(VGP_ppc64_linux) \
|| defined(VGP_ppc32_aix5) || defined(VGP_ppc64_aix5)
res = VG_(do_syscall6)(__NR_mmap, (UWord)start, length,
prot, flags, fd, offset);
# else
# error Unknown platform
# endif
return res;
}
Int ML_(am_getpid)( void )
{
#if defined(VGO_l4re)
VG_(unimplemented)((char*)__func__);
#else
SysRes sres = VG_(do_syscall0)(__NR_getpid);
aspacem_assert(!sr_isError(sres));
return sr_Res(sres);
#endif
}
void ML_(am_exit)( Int status )
{
# if defined(VGO_linux)
(void)VG_(do_syscall1)(__NR_exit_group, status);
# endif
(void)VG_(do_syscall1)(__NR_exit, status);
*(volatile Int *)0 = 'x';
aspacem_assert(2+2 == 5);
}
void ML_(am_exit)( Int status )
{
# if defined(VGO_linux)
(void)VG_(do_syscall1)(__NR_exit_group, status);
# endif
(void)VG_(do_syscall1)(__NR_exit, status);
/* Why are we still alive here? */
/*NOTREACHED*/
*(volatile Int *)0 = 'x';
aspacem_assert(2+2 == 5);
}
static UInt
get_slotsize(UInt ix)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
// Avoid unexpected sign extension
const UChar *unames = (const UChar *)segnames;
if (is_freeslot(ix))
return (unames[ix] << 8) | unames[ix+1];
else
return (unames[ix - 2] << 8) | unames[ix - 1];
}
static void
dec_refcount(UInt ix)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
UInt rc = get_refcount(ix);
aspacem_assert(rc > 0);
if (rc != max_refcount) {
--rc;
if (rc != 0) {
put_refcount(ix, rc);
} else {
UInt size = get_slotsize(ix);
/* Chain this slot in the freelist */
put_slotindex(ix, freeslot_chain);
get_slotindex(ix);
put_slotsize(ix + slotsize_size, size);
get_slotindex(ix);
freeslot_chain = ix;
--num_segnames;
if (0) VG_(am_show_nsegments)(0, "AFTER DECREASE rc -> 0");
}
}
}
/* Returns a sequence number for the fnIdx position in segnames.
Used in aspacemgr debug output to associate a segment with
a segment name. */
Int
ML_(am_segname_get_seqnr)(Int fnIdx)
{
SizeT ix, size;
Int seqnr = -1;
if (fnIdx == -1) return -1; // shortcut
for (ix = overhead; (size = get_slotsize(ix)) != 0; ix += size + overhead) {
seqnr++;
if (ix == fnIdx)
return seqnr;
}
// We should always find the given index; something's busted
aspacem_assert(0);
return -1;
}
Int ML_(am_getpid)( void )
{
SysRes sres = VG_(do_syscall0)(__NR_getpid);
aspacem_assert(!sr_isError(sres));
return sr_Res(sres);
}
VgStack* VG_(am_alloc_VgStack)( /*OUT*/Addr* initial_sp )
{
Int szB;
SysRes sres;
VgStack* stack;
UInt* p;
Int i;
/* Allocate the stack. */
szB = VG_STACK_GUARD_SZB
+ VG_STACK_ACTIVE_SZB + VG_STACK_GUARD_SZB;
#if !defined(VGPV_ppc64_linux_bgq)
sres = VG_(am_mmap_anon_float_valgrind)( szB );
if (sr_isError(sres))
return NULL;
stack = (VgStack*)(AddrH)sr_Res(sres);
aspacem_assert(VG_IS_PAGE_ALIGNED(szB));
aspacem_assert(VG_IS_PAGE_ALIGNED(stack));
/* Protect the guard areas. */
sres = local_do_mprotect_NO_NOTIFY(
(Addr) &stack[0],
VG_STACK_GUARD_SZB, VKI_PROT_NONE
);
if (sr_isError(sres)) goto protect_failed;
VG_(am_notify_mprotect)(
(Addr) &stack->bytes[0],
VG_STACK_GUARD_SZB, VKI_PROT_NONE
);
sres = local_do_mprotect_NO_NOTIFY(
(Addr) &stack->bytes[VG_STACK_GUARD_SZB + VG_STACK_ACTIVE_SZB],
VG_STACK_GUARD_SZB, VKI_PROT_NONE
);
if (sr_isError(sres)) goto protect_failed;
VG_(am_notify_mprotect)(
(Addr) &stack->bytes[VG_STACK_GUARD_SZB + VG_STACK_ACTIVE_SZB],
VG_STACK_GUARD_SZB, VKI_PROT_NONE
);
#else
{ sres = VG_(am_mmap_anon_float_valgrind)( szB );
if (sr_isError(sres))
return NULL;
stack = (VgStack*)sr_Res(sres);
}
#endif
/* Looks good. Fill the active area with junk so we can later
tell how much got used. */
p = (UInt*)&stack->bytes[VG_STACK_GUARD_SZB];
for (i = 0; i < VG_STACK_ACTIVE_SZB/sizeof(UInt); i++)
p[i] = 0xDEADBEEF;
*initial_sp = (Addr)&stack->bytes[VG_STACK_GUARD_SZB + VG_STACK_ACTIVE_SZB];
*initial_sp -= 8;
*initial_sp &= ~((Addr)0x1F); /* 32-align it */
VG_(debugLog)( 1,"aspacem","allocated thread stack at 0x%llx size %d\n",
(ULong)(Addr)stack, szB);
ML_(am_do_sanity_check)();
return stack;
protect_failed:
/* The stack was allocated, but we can't protect it. Unmap it and
return NULL (failure). */
(void)ML_(am_do_munmap_NO_NOTIFY)( (Addr)stack, szB );
ML_(am_do_sanity_check)();
return NULL;
}
VgStack* VG_(am_alloc_VgStack)( Addr* initial_sp )
{
Int szB;
SysRes sres;
VgStack* stack;
UInt* p;
Int i;
szB = VG_STACK_GUARD_SZB
+ VG_STACK_ACTIVE_SZB + VG_STACK_GUARD_SZB;
sres = VG_(am_mmap_anon_float_valgrind)( szB );
if (sr_isError(sres))
return NULL;
stack = (VgStack*)(AddrH)sr_Res(sres);
aspacem_assert(VG_IS_PAGE_ALIGNED(szB));
aspacem_assert(VG_IS_PAGE_ALIGNED(stack));
sres = local_do_mprotect_NO_NOTIFY(
(Addr) &stack[0],
VG_STACK_GUARD_SZB, VKI_PROT_NONE
);
if (sr_isError(sres)) goto protect_failed;
VG_(am_notify_mprotect)(
(Addr) &stack->bytes[0],
VG_STACK_GUARD_SZB, VKI_PROT_NONE
);
sres = local_do_mprotect_NO_NOTIFY(
(Addr) &stack->bytes[VG_STACK_GUARD_SZB + VG_STACK_ACTIVE_SZB],
VG_STACK_GUARD_SZB, VKI_PROT_NONE
);
if (sr_isError(sres)) goto protect_failed;
VG_(am_notify_mprotect)(
(Addr) &stack->bytes[VG_STACK_GUARD_SZB + VG_STACK_ACTIVE_SZB],
VG_STACK_GUARD_SZB, VKI_PROT_NONE
);
p = (UInt*)&stack->bytes[VG_STACK_GUARD_SZB];
for (i = 0; i < VG_STACK_ACTIVE_SZB/sizeof(UInt); i++)
p[i] = 0xDEADBEEF;
*initial_sp = (Addr)&stack->bytes[VG_STACK_GUARD_SZB + VG_STACK_ACTIVE_SZB];
*initial_sp -= 8;
*initial_sp &= ~((Addr)0x1F);
VG_(debugLog)( 1,"aspacem","allocated thread stack at 0x%llx size %d\n",
(ULong)(Addr)stack, szB);
ML_(am_do_sanity_check)();
return stack;
protect_failed:
(void)ML_(am_do_munmap_NO_NOTIFY)( (Addr)stack, szB );
ML_(am_do_sanity_check)();
return NULL;
}
/* Searches the string table to find an index for the given name.
If none is found, an index is allocated and the name stored.
If running ouf of memory, return -1. */
Int
ML_(am_allocate_segname)(const HChar *name)
{
UInt len, ix, size, next_freeslot;
aspacem_assert(name);
if (0) VG_(debugLog)(0, "aspacem", "allocate_segname %s\n", name);
len = VG_(strlen)(name);
/* First see if we already have the name. */
for (ix = overhead; (size = get_slotsize(ix)) != 0; ix += size + overhead) {
if (is_freeslot(ix)) continue;
if (VG_(strcmp)(name, segnames + ix) == 0) {
inc_refcount(ix);
return ix;
}
}
/* Is there a free slot in the string table from a previously "freed"
segment name ? */
Int prev;
for (prev = -1, ix = freeslot_chain; ix != end_of_chain;
prev = ix, ix = next_freeslot) {
next_freeslot = get_slotindex(ix); // next in chain
size = get_slotsize(ix);
if (size >= len + 1) {
/* Note, if the size of the slot is a lot larger than the length
of the string we're about to store in it, we could split the
slot into two. But that complicates matters and as we're not
doing any coalescing of adjacent free slots this could lead to
fragmentation. */
if (prev == -1)
freeslot_chain = next_freeslot;
else
put_slotindex(prev, next_freeslot);
put_refcount(ix, 1);
put_slotsize(ix, size);
VG_(strcpy)(segnames + ix, name);
++num_segnames;
return ix;
}
}
/* We need to add a new name. */
/* Note, that we need at least two bytes in the payload. The reason is
that the payload area will be used to store the size of the slot when
the slot is on the freelist. */
if (len == 0) len = 1;
/* Is there enough room in the string table? The OVERHEAD is for the
sentinel following the payload of new slot. */
SizeT need = len + 1 + overhead;
if (need > (sizeof segnames) - segnames_used) {
return -1;
}
++num_segnames;
++num_slots;
/* copy it in */
ix = segnames_used;
put_refcount(ix, 1);
put_slotsize(ix, len + 1);
VG_(strcpy)(segnames + ix, name);
segnames_used += need;
/* Add sentinel at end of segment name list */
put_sentinel(segnames_used);
return ix;
}
Int ML_(am_getpid)( void )
{
SysRes sres = VG_(do_syscall0)(__NR_getpid);
aspacem_assert(!sres.isError);
return sres.res;
}
static Bool
is_freeslot(UInt ix)
{
aspacem_assert(ix >= overhead && ix <= segnames_used);
return (segnames[ix - 4] & fbit_mask) != 0;
}
/* Note: this is VG_, not ML_. */
SysRes VG_(am_do_mmap_NO_NOTIFY)( Addr start, SizeT length, UInt prot,
UInt flags, Int fd, Off64T offset)
{
#define DEBUG_MYSELF 0
#if defined(VGO_l4re)
void *val;
#endif
SysRes res;
aspacem_assert(VG_IS_PAGE_ALIGNED(offset));
# if defined(VGP_x86_linux) || defined(VGP_ppc32_linux) \
|| defined(VGP_arm_linux)
/* mmap2 uses 4096 chunks even if actual page size is bigger. */
aspacem_assert((offset % 4096) == 0);
res = VG_(do_syscall6)(__NR_mmap2, (UWord)start, length,
prot, flags, fd, offset / 4096);
# elif defined(VGP_amd64_linux) || defined(VGP_ppc64_linux) \
|| defined(VGP_ppc32_aix5) || defined(VGP_ppc64_aix5) \
|| defined(VGP_s390x_linux)
res = VG_(do_syscall6)(__NR_mmap, (UWord)start, length,
prot, flags, fd, offset);
# elif defined(VGP_x86_darwin)
if (fd == 0 && (flags & VKI_MAP_ANONYMOUS)) {
fd = -1; // MAP_ANON with fd==0 is EINVAL
}
res = VG_(do_syscall7)(__NR_mmap, (UWord)start, length,
prot, flags, fd, offset & 0xffffffff, offset >> 32);
# elif defined(VGP_amd64_darwin)
if (fd == 0 && (flags & VKI_MAP_ANONYMOUS)) {
fd = -1; // MAP_ANON with fd==0 is EINVAL
}
res = VG_(do_syscall6)(__NR_mmap, (UWord)start, length,
prot, flags, (UInt)fd, offset);
# elif defined(VGO_l4re)
#if DEBUG_MYSELF
VG_(am_show_nsegments)(0,"before mmap");
l4re_rm_show_lists();
VG_(debugLog)(1, "aspacem", "\033[34mmmap(start=%p, length=%d (0x%x), fd=%d,\033[0m\n",(void *)start, (int) length, (int)length, fd);
VG_(debugLog)(1, "aspacem", "\033[34moffset=%ld (0x%lx), prot=%d=%c%c%c, flags=%d=%s%s%s%s)\033[0m\n",
(long)offset, (long) offset,
prot,
prot & VKI_PROT_READ ? 'r' : '-',
prot & VKI_PROT_WRITE ? 'w' : '-',
prot & VKI_PROT_EXEC ? 'x' : '-',
flags,
flags & VKI_MAP_SHARED ? "MAP_SHARED " : "",
flags & VKI_MAP_PRIVATE ? "MAP_PRIVATE " : "",
flags & VKI_MAP_FIXED ? "MAP_FIXED " : "",
flags & VKI_MAP_ANONYMOUS ? "MAP_ANONYMOUS " : "");
#endif
val = mmap((void *)start, VG_PGROUNDUP(length), prot, flags, fd, offset);
#if DEBUG_MYSELF
VG_(debugLog)(1, "aspacem", "\033[34;1mmmap returns %p\n", val);
VG_(am_show_nsegments)(0,"after mmap");
l4re_rm_show_lists();
#endif
res._isError = (val == (void *)-1);
if (sr_isError(res)) {
res._err = - (int)val;
res._res = -1;
VG_(debugLog)(1, "aspacem", "mmap failed\n");
enter_kdebug("ERROR: mmap failed");
} else {
res._err = 0;
res._res = (int)val;
}
# else
# error Unknown platform
# endif
return res;
}
