void ffi_call(/*@dependent@*/ ffi_cif *cif, void (*fn)(), /*@out@*/ void *rvalue, /*@dependent@*/ void **avalue) { extended_cif ecif; int dummy; ecif.cif = cif; ecif.avalue = avalue; /* If the return value is a struct and we don't have a return */ /* value address then we need to make one */ if ((rvalue == NULL) && (examine_argument (cif->rtype, 1, &dummy, &dummy) == 0)) { /*@-sysunrecog@*/ ecif.rvalue = alloca(cif->rtype->size); /*@=sysunrecog@*/ } else ecif.rvalue = rvalue; /* Stack must always be 16byte aligned. Make it so. */ cif->bytes = ALIGN(cif->bytes, 16); switch (cif->abi) { case FFI_SYSV: /* Calling 32bit code from 64bit is not possible */ FFI_ASSERT(0); break; case FFI_UNIX64: /*@-usedef@*/ ffi_call_UNIX64 (ffi_prep_args, ffi_fill_return_value, &ecif, cif->bytes, ecif.rvalue, fn); /*@=usedef@*/ break; default: FFI_ASSERT(0); break; } }
int ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue, struct register_args *reg_args, char *argp) { ffi_cif *cif; void **avalue; ffi_type **arg_types; long i, avn; int gprcount, ssecount, ngpr, nsse; int ret; cif = closure->cif; avalue = alloca(cif->nargs * sizeof(void *)); gprcount = ssecount = 0; ret = cif->rtype->type; if (ret != FFI_TYPE_VOID) { enum x86_64_reg_class classes[MAX_CLASSES]; int n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse); if (n == 0) { /* The return value goes in memory. Arrange for the closure return value to go directly back to the original caller. */ rvalue = (void *) reg_args->gpr[gprcount++]; /* We don't have to do anything in asm for the return. */ ret = FFI_TYPE_VOID; } else if (ret == FFI_TYPE_STRUCT && n == 2) { /* Mark which register the second word of the structure goes in. */ _Bool sse0 = SSE_CLASS_P (classes[0]); _Bool sse1 = SSE_CLASS_P (classes[1]); if (!sse0 && sse1) ret |= 1 << 8; else if (sse0 && !sse1) ret |= 1 << 9; } } avn = cif->nargs; arg_types = cif->arg_types; for (i = 0; i < avn; ++i) { enum x86_64_reg_class classes[MAX_CLASSES]; int n; n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse); if (n == 0 || gprcount + ngpr > MAX_GPR_REGS || ssecount + nsse > MAX_SSE_REGS) { long align = arg_types[i]->alignment; /* Stack arguments are *always* at least 8 byte aligned. */ if (align < 8) align = 8; /* Pass this argument in memory. */ argp = (void *) ALIGN (argp, align); avalue[i] = argp; argp += arg_types[i]->size; } /* If the argument is in a single register, or two consecutive integer registers, then we can use that address directly. */ else if (n == 1 || (n == 2 && !(SSE_CLASS_P (classes[0]) || SSE_CLASS_P (classes[1])))) { /* The argument is in a single register. */ if (SSE_CLASS_P (classes[0])) { avalue[i] = ®_args->sse[ssecount]; ssecount += n; } else { avalue[i] = ®_args->gpr[gprcount]; gprcount += n; } } /* Otherwise, allocate space to make them consecutive. */ else { char *a = alloca (64); int j; avalue[i] = a; for (j = 0; j < n; j++, a += 8) { if (SSE_CLASS_P (classes[j])) memcpy (a, ®_args->sse[ssecount++], 8); else memcpy (a, ®_args->gpr[gprcount++], 8); } } } /* Invoke the closure. */ closure->fun (cif, rvalue, avalue, closure->user_data); /* Tell assembly how to perform return type promotions. */ return ret; }
void ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) { enum x86_64_reg_class classes[MAX_CLASSES]; char *stack, *argp; ffi_type **arg_types; int gprcount, ssecount, ngpr, nsse, i, avn; _Bool ret_in_memory; struct register_args *reg_args; /* Can't call 32-bit mode from 64-bit mode. */ FFI_ASSERT (cif->abi == FFI_UNIX64); /* If the return value is a struct and we don't have a return value address then we need to make one. Note the setting of flags to VOID above in ffi_prep_cif_machdep. */ ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT && (cif->flags & 0xff) == FFI_TYPE_VOID); if (rvalue == NULL && ret_in_memory) rvalue = alloca (cif->rtype->size); /* Allocate the space for the arguments, plus 4 words of temp space. */ stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8); reg_args = (struct register_args *) stack; argp = stack + sizeof (struct register_args); gprcount = ssecount = 0; /* If the return value is passed in memory, add the pointer as the first integer argument. */ if (ret_in_memory) reg_args->gpr[gprcount++] = (long) rvalue; avn = cif->nargs; arg_types = cif->arg_types; for (i = 0; i < avn; ++i) { size_t size = arg_types[i]->size; int n; n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse); if (n == 0 || gprcount + ngpr > MAX_GPR_REGS || ssecount + nsse > MAX_SSE_REGS) { long align = arg_types[i]->alignment; /* Stack arguments are *always* at least 8 byte aligned. */ if (align < 8) align = 8; /* Pass this argument in memory. */ argp = (void *) ALIGN (argp, align); memcpy (argp, avalue[i], size); argp += size; } else { /* The argument is passed entirely in registers. */ char *a = (char *) avalue[i]; int j; for (j = 0; j < n; j++, a += 8, size -= 8) { switch (classes[j]) { case X86_64_INTEGER_CLASS: case X86_64_INTEGERSI_CLASS: reg_args->gpr[gprcount] = 0; memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8); gprcount++; break; case X86_64_SSE_CLASS: case X86_64_SSEDF_CLASS: reg_args->sse[ssecount].m[0] = *(UINT64 *) a; reg_args->sse[ssecount].m[1] = 0; reg_args->sse[ssecount].m[2] = 0; reg_args->sse[ssecount].m[3] = 0; reg_args->sse[ssecount].m[4] = 0; reg_args->sse[ssecount].m[5] = 0; reg_args->sse[ssecount].m[6] = 0; reg_args->sse[ssecount].m[7] = 0; ssecount++; break; case X86_64_SSESF_CLASS: reg_args->sse[ssecount].m[0] = *(UINT32 *) a; reg_args->sse[ssecount].m[1] = 0; reg_args->sse[ssecount].m[2] = 0; reg_args->sse[ssecount].m[3] = 0; reg_args->sse[ssecount].m[4] = 0; reg_args->sse[ssecount].m[5] = 0; reg_args->sse[ssecount].m[6] = 0; reg_args->sse[ssecount].m[7] = 0; ssecount++; break; default: abort(); } } } } ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args), cif->flags, rvalue, fn, ssecount); }
ffi_status ffi_prep_cif_machdep (ffi_cif *cif) { int gprcount, ssecount, i, avn, n, ngpr, nsse, flags; enum x86_64_reg_class classes[MAX_CLASSES]; size_t bytes; gprcount = ssecount = 0; flags = cif->rtype->type; if (flags != FFI_TYPE_VOID) { n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse); if (n == 0) { /* The return value is passed in memory. A pointer to that memory is the first argument. Allocate a register for it. */ gprcount++; /* We don't have to do anything in asm for the return. */ flags = FFI_TYPE_VOID; } else if (flags == FFI_TYPE_STRUCT) { /* Mark which registers the result appears in. */ _Bool sse0 = SSE_CLASS_P (classes[0]); _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]); if (sse0 && !sse1) flags |= 1 << 8; else if (!sse0 && sse1) flags |= 1 << 9; else if (sse0 && sse1) flags |= 1 << 10; /* Mark the true size of the structure. */ flags |= cif->rtype->size << 12; } } /* Go over all arguments and determine the way they should be passed. If it's in a register and there is space for it, let that be so. If not, add it's size to the stack byte count. */ for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++) { if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0 || gprcount + ngpr > MAX_GPR_REGS || ssecount + nsse > MAX_SSE_REGS) { long align = cif->arg_types[i]->alignment; if (align < 8) align = 8; bytes = ALIGN(bytes, align); bytes += cif->arg_types[i]->size; } else { gprcount += ngpr; ssecount += nsse; } } if (ssecount) flags |= 1 << 11; cif->flags = flags; cif->bytes = bytes; return FFI_OK; }
/* Perform machine dependent cif processing. */ ffi_status ffi_prep_cif_machdep (ffi_cif *cif) { int gprcount, ssecount, i, g, s; gprcount = ssecount = 0; /* Reset the byte count. We handle this size estimation here. */ cif->bytes = 0; /* If the return value should be passed in memory, pass the pointer as the first argument. The actual memory isn't allocated here. */ if (cif->rtype->type != FFI_TYPE_VOID && examine_argument (cif->rtype, 1, &g, &s) == 0) gprcount = 1; /* Go over all arguments and determine the way they should be passed. If it's in a register and there is space for it, let that be so. If not, add it's size to the stack byte count. */ for (i=0; i<cif->nargs; i++) { if (examine_argument (cif->arg_types[i], 0, &g, &s) == 0 || gprcount + g > MAX_GPR_REGS || ssecount + s > MAX_SSE_REGS) { /* This is passed in memory. First align to the basic type. */ cif->bytes = ALIGN(cif->bytes, cif->arg_types[i]->alignment); /* Stack arguments are *always* at least 8 byte aligned. */ cif->bytes = ALIGN(cif->bytes, 8); /* Now add the size of this argument. */ cif->bytes += cif->arg_types[i]->size; } else { gprcount += g; ssecount += s; } } /* Set the flag for the closures return. */ switch (cif->rtype->type) { case FFI_TYPE_VOID: case FFI_TYPE_STRUCT: case FFI_TYPE_SINT64: case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: cif->flags = (unsigned) cif->rtype->type; break; case FFI_TYPE_UINT64: cif->flags = FFI_TYPE_SINT64; break; default: cif->flags = FFI_TYPE_INT; break; } return FFI_OK; }
/*@-exportheader@*/ void ffi_prep_args (stackLayout *stack, extended_cif *ecif) /*@=exportheader@*/ { int gprcount, ssecount, i, g, s; void **p_argv; void *argp = &stack->argspace; ffi_type **p_arg; /* First check if the return value should be passed in memory. If so, pass the pointer as the first argument. */ gprcount = ssecount = 0; if (ecif->cif->rtype->type != FFI_TYPE_VOID && examine_argument (ecif->cif->rtype, 1, &g, &s) == 0) stack->gpr[gprcount++] = (long) ecif->rvalue; for (i=ecif->cif->nargs, p_arg=ecif->cif->arg_types, p_argv = ecif->avalue; i!=0; i--, p_arg++, p_argv++) { int in_register = 0; switch ((*p_arg)->type) { case FFI_TYPE_SINT8: case FFI_TYPE_SINT16: case FFI_TYPE_SINT32: case FFI_TYPE_SINT64: case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: case FFI_TYPE_UINT32: case FFI_TYPE_UINT64: case FFI_TYPE_POINTER: if (gprcount < MAX_GPR_REGS) { stack->gpr[gprcount] = 0; stack->gpr[gprcount++] = *(long long *)(*p_argv); in_register = 1; } break; case FFI_TYPE_FLOAT: if (ssecount < MAX_SSE_REGS) { float2sse (*(float *)(*p_argv), &stack->sse[ssecount++]); in_register = 1; } break; case FFI_TYPE_DOUBLE: if (ssecount < MAX_SSE_REGS) { double2sse (*(double *)(*p_argv), &stack->sse[ssecount++]); in_register = 1; } break; } if (in_register) continue; /* Either all places in registers where filled, or this is a type that potentially goes into a memory slot. */ if (examine_argument (*p_arg, 0, &g, &s) == 0 || gprcount + g > MAX_GPR_REGS || ssecount + s > MAX_SSE_REGS) { /* Pass this argument in memory. */ argp = (void *)ALIGN(argp, (*p_arg)->alignment); memcpy (argp, *p_argv, (*p_arg)->size); argp += (*p_arg)->size; } else { /* All easy cases are eliminated. Now fire the big guns. */ enum x86_64_reg_class classes[MAX_CLASSES]; int offset = 0, j, num; void *a; num = classify_argument (*p_arg, classes, &offset); for (j=0, a=*p_argv; j<num; j++, a+=8) { switch (classes[j]) { case X86_64_INTEGER_CLASS: case X86_64_INTEGERSI_CLASS: stack->gpr[gprcount++] = *(long long *)a; break; case X86_64_SSE_CLASS: floatfloat2sse (a, &stack->sse[ssecount++]); break; case X86_64_SSESF_CLASS: float2sse (*(float *)a, &stack->sse[ssecount++]); break; case X86_64_SSEDF_CLASS: double2sse (*(double *)a, &stack->sse[ssecount++]); break; default: abort(); } } } } }