void ffi_call(/*@dependent@*/ ffi_cif *cif,
void (*fn)(),
/*@out@*/ void *rvalue,
/*@dependent@*/ void **avalue)
{
extended_cif ecif;
int dummy;
ecif.cif = cif;
ecif.avalue = avalue;
/* If the return value is a struct and we don't have a return */
/* value address then we need to make one */
if ((rvalue == NULL) &&
(examine_argument (cif->rtype, 1, &dummy, &dummy) == 0))
{
/*@-sysunrecog@*/
ecif.rvalue = alloca(cif->rtype->size);
/*@=sysunrecog@*/
}
else
ecif.rvalue = rvalue;
/* Stack must always be 16byte aligned. Make it so. */
cif->bytes = ALIGN(cif->bytes, 16);
switch (cif->abi)
{
case FFI_SYSV:
/* Calling 32bit code from 64bit is not possible */
FFI_ASSERT(0);
break;
case FFI_UNIX64:
/*@-usedef@*/
ffi_call_UNIX64 (ffi_prep_args, ffi_fill_return_value, &ecif,
cif->bytes, ecif.rvalue, fn);
/*@=usedef@*/
break;
default:
FFI_ASSERT(0);
break;
}
}
int
ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue,
struct register_args *reg_args, char *argp)
{
ffi_cif *cif;
void **avalue;
ffi_type **arg_types;
long i, avn;
int gprcount, ssecount, ngpr, nsse;
int ret;
cif = closure->cif;
avalue = alloca(cif->nargs * sizeof(void *));
gprcount = ssecount = 0;
ret = cif->rtype->type;
if (ret != FFI_TYPE_VOID)
{
enum x86_64_reg_class classes[MAX_CLASSES];
int n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
if (n == 0)
{
/* The return value goes in memory. Arrange for the closure
return value to go directly back to the original caller. */
rvalue = (void *) reg_args->gpr[gprcount++];
/* We don't have to do anything in asm for the return. */
ret = FFI_TYPE_VOID;
}
else if (ret == FFI_TYPE_STRUCT && n == 2)
{
/* Mark which register the second word of the structure goes in. */
_Bool sse0 = SSE_CLASS_P (classes[0]);
_Bool sse1 = SSE_CLASS_P (classes[1]);
if (!sse0 && sse1)
ret |= 1 << 8;
else if (sse0 && !sse1)
ret |= 1 << 9;
}
}
avn = cif->nargs;
arg_types = cif->arg_types;
for (i = 0; i < avn; ++i)
{
enum x86_64_reg_class classes[MAX_CLASSES];
int n;
n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
if (n == 0
|| gprcount + ngpr > MAX_GPR_REGS
|| ssecount + nsse > MAX_SSE_REGS)
{
long align = arg_types[i]->alignment;
/* Stack arguments are *always* at least 8 byte aligned. */
if (align < 8)
align = 8;
/* Pass this argument in memory. */
argp = (void *) ALIGN (argp, align);
avalue[i] = argp;
argp += arg_types[i]->size;
}
/* If the argument is in a single register, or two consecutive
integer registers, then we can use that address directly. */
else if (n == 1
|| (n == 2 && !(SSE_CLASS_P (classes[0])
|| SSE_CLASS_P (classes[1]))))
{
/* The argument is in a single register. */
if (SSE_CLASS_P (classes[0]))
{
avalue[i] = ®_args->sse[ssecount];
ssecount += n;
}
else
{
avalue[i] = ®_args->gpr[gprcount];
gprcount += n;
}
}
/* Otherwise, allocate space to make them consecutive. */
else
{
char *a = alloca (64);
int j;
avalue[i] = a;
for (j = 0; j < n; j++, a += 8)
{
if (SSE_CLASS_P (classes[j]))
memcpy (a, ®_args->sse[ssecount++], 8);
else
memcpy (a, ®_args->gpr[gprcount++], 8);
}
}
}
/* Invoke the closure. */
closure->fun (cif, rvalue, avalue, closure->user_data);
/* Tell assembly how to perform return type promotions. */
return ret;
}
void
ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
{
enum x86_64_reg_class classes[MAX_CLASSES];
char *stack, *argp;
ffi_type **arg_types;
int gprcount, ssecount, ngpr, nsse, i, avn;
_Bool ret_in_memory;
struct register_args *reg_args;
/* Can't call 32-bit mode from 64-bit mode. */
FFI_ASSERT (cif->abi == FFI_UNIX64);
/* If the return value is a struct and we don't have a return value
address then we need to make one. Note the setting of flags to
VOID above in ffi_prep_cif_machdep. */
ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT
&& (cif->flags & 0xff) == FFI_TYPE_VOID);
if (rvalue == NULL && ret_in_memory)
rvalue = alloca (cif->rtype->size);
/* Allocate the space for the arguments, plus 4 words of temp space. */
stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
reg_args = (struct register_args *) stack;
argp = stack + sizeof (struct register_args);
gprcount = ssecount = 0;
/* If the return value is passed in memory, add the pointer as the
first integer argument. */
if (ret_in_memory)
reg_args->gpr[gprcount++] = (long) rvalue;
avn = cif->nargs;
arg_types = cif->arg_types;
for (i = 0; i < avn; ++i)
{
size_t size = arg_types[i]->size;
int n;
n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
if (n == 0
|| gprcount + ngpr > MAX_GPR_REGS
|| ssecount + nsse > MAX_SSE_REGS)
{
long align = arg_types[i]->alignment;
/* Stack arguments are *always* at least 8 byte aligned. */
if (align < 8)
align = 8;
/* Pass this argument in memory. */
argp = (void *) ALIGN (argp, align);
memcpy (argp, avalue[i], size);
argp += size;
}
else
{
/* The argument is passed entirely in registers. */
char *a = (char *) avalue[i];
int j;
for (j = 0; j < n; j++, a += 8, size -= 8)
{
switch (classes[j])
{
case X86_64_INTEGER_CLASS:
case X86_64_INTEGERSI_CLASS:
reg_args->gpr[gprcount] = 0;
memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8);
gprcount++;
break;
case X86_64_SSE_CLASS:
case X86_64_SSEDF_CLASS:
reg_args->sse[ssecount].m[0] = *(UINT64 *) a;
reg_args->sse[ssecount].m[1] = 0;
reg_args->sse[ssecount].m[2] = 0;
reg_args->sse[ssecount].m[3] = 0;
reg_args->sse[ssecount].m[4] = 0;
reg_args->sse[ssecount].m[5] = 0;
reg_args->sse[ssecount].m[6] = 0;
reg_args->sse[ssecount].m[7] = 0;
ssecount++;
break;
case X86_64_SSESF_CLASS:
reg_args->sse[ssecount].m[0] = *(UINT32 *) a;
reg_args->sse[ssecount].m[1] = 0;
reg_args->sse[ssecount].m[2] = 0;
reg_args->sse[ssecount].m[3] = 0;
reg_args->sse[ssecount].m[4] = 0;
reg_args->sse[ssecount].m[5] = 0;
reg_args->sse[ssecount].m[6] = 0;
reg_args->sse[ssecount].m[7] = 0;
ssecount++;
break;
default:
abort();
}
}
}
}
ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
cif->flags, rvalue, fn, ssecount);
}
ffi_status
ffi_prep_cif_machdep (ffi_cif *cif)
{
int gprcount, ssecount, i, avn, n, ngpr, nsse, flags;
enum x86_64_reg_class classes[MAX_CLASSES];
size_t bytes;
gprcount = ssecount = 0;
flags = cif->rtype->type;
if (flags != FFI_TYPE_VOID)
{
n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
if (n == 0)
{
/* The return value is passed in memory. A pointer to that
memory is the first argument. Allocate a register for it. */
gprcount++;
/* We don't have to do anything in asm for the return. */
flags = FFI_TYPE_VOID;
}
else if (flags == FFI_TYPE_STRUCT)
{
/* Mark which registers the result appears in. */
_Bool sse0 = SSE_CLASS_P (classes[0]);
_Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
if (sse0 && !sse1)
flags |= 1 << 8;
else if (!sse0 && sse1)
flags |= 1 << 9;
else if (sse0 && sse1)
flags |= 1 << 10;
/* Mark the true size of the structure. */
flags |= cif->rtype->size << 12;
}
}
/* Go over all arguments and determine the way they should be passed.
If it's in a register and there is space for it, let that be so. If
not, add it's size to the stack byte count. */
for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
{
if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
|| gprcount + ngpr > MAX_GPR_REGS
|| ssecount + nsse > MAX_SSE_REGS)
{
long align = cif->arg_types[i]->alignment;
if (align < 8)
align = 8;
bytes = ALIGN(bytes, align);
bytes += cif->arg_types[i]->size;
}
else
{
gprcount += ngpr;
ssecount += nsse;
}
}
if (ssecount)
flags |= 1 << 11;
cif->flags = flags;
cif->bytes = bytes;
return FFI_OK;
}
/* Perform machine dependent cif processing. */
ffi_status
ffi_prep_cif_machdep (ffi_cif *cif)
{
int gprcount, ssecount, i, g, s;
gprcount = ssecount = 0;
/* Reset the byte count. We handle this size estimation here. */
cif->bytes = 0;
/* If the return value should be passed in memory, pass the pointer
as the first argument. The actual memory isn't allocated here. */
if (cif->rtype->type != FFI_TYPE_VOID
&& examine_argument (cif->rtype, 1, &g, &s) == 0)
gprcount = 1;
/* Go over all arguments and determine the way they should be passed.
If it's in a register and there is space for it, let that be so. If
not, add it's size to the stack byte count. */
for (i=0; i<cif->nargs; i++)
{
if (examine_argument (cif->arg_types[i], 0, &g, &s) == 0
|| gprcount + g > MAX_GPR_REGS || ssecount + s > MAX_SSE_REGS)
{
/* This is passed in memory. First align to the basic type. */
cif->bytes = ALIGN(cif->bytes, cif->arg_types[i]->alignment);
/* Stack arguments are *always* at least 8 byte aligned. */
cif->bytes = ALIGN(cif->bytes, 8);
/* Now add the size of this argument. */
cif->bytes += cif->arg_types[i]->size;
}
else
{
gprcount += g;
ssecount += s;
}
}
/* Set the flag for the closures return. */
switch (cif->rtype->type)
{
case FFI_TYPE_VOID:
case FFI_TYPE_STRUCT:
case FFI_TYPE_SINT64:
case FFI_TYPE_FLOAT:
case FFI_TYPE_DOUBLE:
case FFI_TYPE_LONGDOUBLE:
cif->flags = (unsigned) cif->rtype->type;
break;
case FFI_TYPE_UINT64:
cif->flags = FFI_TYPE_SINT64;
break;
default:
cif->flags = FFI_TYPE_INT;
break;
}
return FFI_OK;
}
/*@-exportheader@*/
void
ffi_prep_args (stackLayout *stack, extended_cif *ecif)
/*@=exportheader@*/
{
int gprcount, ssecount, i, g, s;
void **p_argv;
void *argp = &stack->argspace;
ffi_type **p_arg;
/* First check if the return value should be passed in memory. If so,
pass the pointer as the first argument. */
gprcount = ssecount = 0;
if (ecif->cif->rtype->type != FFI_TYPE_VOID
&& examine_argument (ecif->cif->rtype, 1, &g, &s) == 0)
stack->gpr[gprcount++] = (long) ecif->rvalue;
for (i=ecif->cif->nargs, p_arg=ecif->cif->arg_types, p_argv = ecif->avalue;
i!=0; i--, p_arg++, p_argv++)
{
int in_register = 0;
switch ((*p_arg)->type)
{
case FFI_TYPE_SINT8:
case FFI_TYPE_SINT16:
case FFI_TYPE_SINT32:
case FFI_TYPE_SINT64:
case FFI_TYPE_UINT8:
case FFI_TYPE_UINT16:
case FFI_TYPE_UINT32:
case FFI_TYPE_UINT64:
case FFI_TYPE_POINTER:
if (gprcount < MAX_GPR_REGS)
{
stack->gpr[gprcount] = 0;
stack->gpr[gprcount++] = *(long long *)(*p_argv);
in_register = 1;
}
break;
case FFI_TYPE_FLOAT:
if (ssecount < MAX_SSE_REGS)
{
float2sse (*(float *)(*p_argv), &stack->sse[ssecount++]);
in_register = 1;
}
break;
case FFI_TYPE_DOUBLE:
if (ssecount < MAX_SSE_REGS)
{
double2sse (*(double *)(*p_argv), &stack->sse[ssecount++]);
in_register = 1;
}
break;
}
if (in_register)
continue;
/* Either all places in registers where filled, or this is a
type that potentially goes into a memory slot. */
if (examine_argument (*p_arg, 0, &g, &s) == 0
|| gprcount + g > MAX_GPR_REGS || ssecount + s > MAX_SSE_REGS)
{
/* Pass this argument in memory. */
argp = (void *)ALIGN(argp, (*p_arg)->alignment);
memcpy (argp, *p_argv, (*p_arg)->size);
argp += (*p_arg)->size;
}
else
{
/* All easy cases are eliminated. Now fire the big guns. */
enum x86_64_reg_class classes[MAX_CLASSES];
int offset = 0, j, num;
void *a;
num = classify_argument (*p_arg, classes, &offset);
for (j=0, a=*p_argv; j<num; j++, a+=8)
{
switch (classes[j])
{
case X86_64_INTEGER_CLASS:
case X86_64_INTEGERSI_CLASS:
stack->gpr[gprcount++] = *(long long *)a;
break;
case X86_64_SSE_CLASS:
floatfloat2sse (a, &stack->sse[ssecount++]);
break;
case X86_64_SSESF_CLASS:
float2sse (*(float *)a, &stack->sse[ssecount++]);
break;
case X86_64_SSEDF_CLASS:
double2sse (*(double *)a, &stack->sse[ssecount++]);
break;
default:
abort();
}
}
}
}
}
