MonoContinuationRestore mono_tasklets_arch_restore (void) { static guint8* saved = NULL; guint8 *code, *start; if (saved) return (MonoContinuationRestore)saved; code = start = mono_global_codeman_reserve (48); /* the signature is: restore (MonoContinuation *cont, int state, MonoLMF **lmf_addr) */ /* put cont in edx */ x86_mov_reg_membase (code, X86_EDX, X86_ESP, 4, 4); /* state in eax, so it's setup as the return value */ x86_mov_reg_membase (code, X86_EAX, X86_ESP, 8, 4); /* lmf_addr in ebx */ x86_mov_reg_membase(code, X86_EBX, X86_ESP, 0x0C, 4); /* setup the copy of the stack */ x86_mov_reg_membase (code, X86_ECX, X86_EDX, MONO_STRUCT_OFFSET (MonoContinuation, stack_used_size), 4); x86_shift_reg_imm (code, X86_SHR, X86_ECX, 2); x86_cld (code); x86_mov_reg_membase (code, X86_ESI, X86_EDX, MONO_STRUCT_OFFSET (MonoContinuation, saved_stack), 4); x86_mov_reg_membase (code, X86_EDI, X86_EDX, MONO_STRUCT_OFFSET (MonoContinuation, return_sp), 4); x86_prefix (code, X86_REP_PREFIX); x86_movsl (code); /* now restore the registers from the LMF */ x86_mov_reg_membase (code, X86_ECX, X86_EDX, MONO_STRUCT_OFFSET (MonoContinuation, lmf), 4); x86_mov_reg_membase (code, X86_EBP, X86_ECX, MONO_STRUCT_OFFSET (MonoLMF, ebp), 4); x86_mov_reg_membase (code, X86_ESP, X86_ECX, MONO_STRUCT_OFFSET (MonoLMF, esp), 4); /* restore the lmf chain */ /*x86_mov_reg_membase (code, X86_ECX, X86_ESP, 12, 4); x86_mov_membase_reg (code, X86_ECX, 0, X86_EDX, 4);*/ x86_jump_membase (code, X86_EDX, MONO_STRUCT_OFFSET (MonoContinuation, return_ip)); mono_arch_flush_icache (start, code - start); MONO_PROFILER_RAISE (jit_code_buffer, (start, code - start, MONO_PROFILER_CODE_BUFFER_EXCEPTION_HANDLING, NULL)); g_assert ((code - start) <= 48); saved = start; return (MonoContinuationRestore)saved; }
MonoContinuationRestore mono_tasklets_arch_restore (void) { static guint8* saved = NULL; guint8 *code, *start; #ifdef __native_client_codegen__ g_print("mono_tasklets_arch_restore needs to be aligned for Native Client\n"); #endif if (saved) return (MonoContinuationRestore)saved; code = start = mono_global_codeman_reserve (48); /* the signature is: restore (MonoContinuation *cont, int state, MonoLMF **lmf_addr) */ /* put cont in edx */ x86_mov_reg_membase (code, X86_EDX, X86_ESP, 4, 4); /* state in eax, so it's setup as the return value */ x86_mov_reg_membase (code, X86_EAX, X86_ESP, 8, 4); /* setup the copy of the stack */ x86_mov_reg_membase (code, X86_ECX, X86_EDX, G_STRUCT_OFFSET (MonoContinuation, stack_used_size), 4); x86_shift_reg_imm (code, X86_SHR, X86_ECX, 2); x86_cld (code); x86_mov_reg_membase (code, X86_ESI, X86_EDX, G_STRUCT_OFFSET (MonoContinuation, saved_stack), 4); x86_mov_reg_membase (code, X86_EDI, X86_EDX, G_STRUCT_OFFSET (MonoContinuation, return_sp), 4); x86_prefix (code, X86_REP_PREFIX); x86_movsl (code); /* now restore the registers from the LMF */ x86_mov_reg_membase (code, X86_ECX, X86_EDX, G_STRUCT_OFFSET (MonoContinuation, lmf), 4); x86_mov_reg_membase (code, X86_EBX, X86_ECX, G_STRUCT_OFFSET (MonoLMF, ebx), 4); x86_mov_reg_membase (code, X86_EBP, X86_ECX, G_STRUCT_OFFSET (MonoLMF, ebp), 4); x86_mov_reg_membase (code, X86_ESI, X86_ECX, G_STRUCT_OFFSET (MonoLMF, esi), 4); x86_mov_reg_membase (code, X86_EDI, X86_ECX, G_STRUCT_OFFSET (MonoLMF, edi), 4); /* restore the lmf chain */ /*x86_mov_reg_membase (code, X86_ECX, X86_ESP, 12, 4); x86_mov_membase_reg (code, X86_ECX, 0, X86_EDX, 4);*/ x86_jump_membase (code, X86_EDX, G_STRUCT_OFFSET (MonoContinuation, return_ip)); g_assert ((code - start) <= 48); saved = start; return (MonoContinuationRestore)saved; }
/* * get_throw_trampoline: * * Generate a call to mono_x86_throw_exception/ * mono_x86_throw_corlib_exception. * If LLVM is true, generate code which assumes the caller is LLVM generated code, * which doesn't push the arguments. */ static guint8* get_throw_trampoline (const char *name, gboolean rethrow, gboolean llvm, gboolean corlib, gboolean llvm_abs, gboolean resume_unwind, MonoTrampInfo **info, gboolean aot) { guint8 *start, *code, *labels [16]; int i, stack_size, stack_offset, arg_offsets [5], regs_offset; MonoJumpInfo *ji = NULL; GSList *unwind_ops = NULL; guint kMaxCodeSize = 192; start = code = mono_global_codeman_reserve (kMaxCodeSize); stack_size = 128; /* * On apple, the stack is misaligned by the pushing of the return address. */ if (!llvm && corlib) /* On OSX, we don't generate alignment code to save space */ stack_size += 4; else stack_size += MONO_ARCH_FRAME_ALIGNMENT - 4; /* * The stack looks like this: * <pc offset> (only if corlib is TRUE) * <exception object>/<type token> * <return addr> <- esp (unaligned on apple) */ unwind_ops = mono_arch_get_cie_program (); /* Alloc frame */ x86_alu_reg_imm (code, X86_SUB, X86_ESP, stack_size); mono_add_unwind_op_def_cfa_offset (unwind_ops, code, start, stack_size + 4); arg_offsets [0] = 0; arg_offsets [1] = 4; arg_offsets [2] = 8; arg_offsets [3] = 12; regs_offset = 16; /* Save registers */ for (i = 0; i < X86_NREG; ++i) if (i != X86_ESP) x86_mov_membase_reg (code, X86_ESP, regs_offset + (i * 4), i, 4); /* Calculate the offset between the current sp and the sp of the caller */ if (llvm) { /* LLVM doesn't push the arguments */ stack_offset = stack_size + 4; } else { if (corlib) { /* Two arguments */ stack_offset = stack_size + 4 + 8; #ifdef __APPLE__ /* We don't generate stack alignment code on osx to save space */ #endif } else { /* One argument + stack alignment */ stack_offset = stack_size + 4 + 4; #ifdef __APPLE__ /* Pop the alignment added by OP_THROW too */ stack_offset += MONO_ARCH_FRAME_ALIGNMENT - 4; #else if (mono_do_x86_stack_align) stack_offset += MONO_ARCH_FRAME_ALIGNMENT - 4; #endif } } /* Save ESP */ x86_lea_membase (code, X86_EAX, X86_ESP, stack_offset); x86_mov_membase_reg (code, X86_ESP, regs_offset + (X86_ESP * 4), X86_EAX, 4); /* Clear fp stack */ labels [0] = code; x86_fnstsw (code); x86_shift_reg_imm (code, X86_SHR, X86_EAX, 11); x86_alu_reg_imm (code, X86_AND, X86_EAX, 7); x86_alu_reg_imm (code, X86_CMP, X86_EAX, 0); labels [1] = code; x86_branch8 (code, X86_CC_EQ, 0, FALSE); x86_fstp (code, 0); x86_jump_code (code, labels [0]); mono_x86_patch (labels [1], code); /* Set arg1 == regs */ x86_lea_membase (code, X86_EAX, X86_ESP, regs_offset); x86_mov_membase_reg (code, X86_ESP, arg_offsets [0], X86_EAX, 4); /* Set arg2 == exc/ex_token_index */ if (resume_unwind) x86_mov_reg_imm (code, X86_EAX, 0); else x86_mov_reg_membase (code, X86_EAX, X86_ESP, stack_size + 4, 4); x86_mov_membase_reg (code, X86_ESP, arg_offsets [1], X86_EAX, 4); /* Set arg3 == eip */ if (llvm_abs) x86_alu_reg_reg (code, X86_XOR, X86_EAX, X86_EAX); else x86_mov_reg_membase (code, X86_EAX, X86_ESP, stack_size, 4); x86_mov_membase_reg (code, X86_ESP, arg_offsets [2], X86_EAX, 4); /* Set arg4 == rethrow/pc_offset */ if (resume_unwind) { x86_mov_membase_imm (code, X86_ESP, arg_offsets [3], 0, 4); } else if (corlib) { x86_mov_reg_membase (code, X86_EAX, X86_ESP, stack_size + 8, 4); if (llvm_abs) { /* * The caller is LLVM code which passes the absolute address not a pc offset, * so compensate by passing 0 as 'ip' and passing the negated abs address as * the pc offset. */ x86_neg_reg (code, X86_EAX); } x86_mov_membase_reg (code, X86_ESP, arg_offsets [3], X86_EAX, 4); } else { x86_mov_membase_imm (code, X86_ESP, arg_offsets [3], rethrow, 4); } /* Make the call */ if (aot) { // This can be called from runtime code, which can't guarantee that // ebx contains the got address. // So emit the got address loading code too code = mono_arch_emit_load_got_addr (start, code, NULL, &ji); code = mono_arch_emit_load_aotconst (start, code, &ji, MONO_PATCH_INFO_JIT_ICALL_ADDR, corlib ? "mono_x86_throw_corlib_exception" : "mono_x86_throw_exception"); x86_call_reg (code, X86_EAX); } else { x86_call_code (code, resume_unwind ? (gpointer)(mono_x86_resume_unwind) : (corlib ? (gpointer)mono_x86_throw_corlib_exception : (gpointer)mono_x86_throw_exception)); } x86_breakpoint (code); g_assert ((code - start) < kMaxCodeSize); if (info) *info = mono_tramp_info_create (name, start, code - start, ji, unwind_ops); else { GSList *l; for (l = unwind_ops; l; l = l->next) g_free (l->data); g_slist_free (unwind_ops); } mono_arch_flush_icache (start, code - start); mono_profiler_code_buffer_new (start, code - start, MONO_PROFILER_CODE_BUFFER_EXCEPTION_HANDLING, NULL); return start; }
gpointer mono_arch_get_gsharedvt_trampoline (MonoTrampInfo **info, gboolean aot) { guint8 *code, *buf; int buf_len, cfa_offset; GSList *unwind_ops = NULL; MonoJumpInfo *ji = NULL; guint8 *br_out, *br [16]; int info_offset, mrgctx_offset; buf_len = 320; buf = code = mono_global_codeman_reserve (buf_len); /* * This trampoline is responsible for marshalling calls between normal code and gsharedvt code. The * caller is a normal or gshared method which uses the signature of the inflated method to make the call, while * the callee is a gsharedvt method which has a signature which uses valuetypes in place of type parameters, i.e. * caller: * foo<bool> (bool b) * callee: * T=<type used to represent vtype type arguments, currently TypedByRef> * foo<T> (T b) * The trampoline is responsible for marshalling the arguments and marshalling the result back. To simplify * things, we create our own stack frame, and do most of the work in a C function, which receives a * GSharedVtCallInfo structure as an argument. The structure should contain information to execute the C function to * be as fast as possible. The argument is received in EAX from a gsharedvt trampoline. So the real * call sequence looks like this: * caller -> gsharedvt trampoline -> gsharevt in trampoline -> start_gsharedvt_call * FIXME: Optimize this. */ cfa_offset = sizeof (gpointer); mono_add_unwind_op_def_cfa (unwind_ops, code, buf, X86_ESP, cfa_offset); mono_add_unwind_op_offset (unwind_ops, code, buf, X86_NREG, -cfa_offset); x86_push_reg (code, X86_EBP); cfa_offset += sizeof (gpointer); mono_add_unwind_op_def_cfa_offset (unwind_ops, code, buf, cfa_offset); mono_add_unwind_op_offset (unwind_ops, code, buf, X86_EBP, - cfa_offset); x86_mov_reg_reg (code, X86_EBP, X86_ESP, sizeof (gpointer)); mono_add_unwind_op_def_cfa_reg (unwind_ops, code, buf, X86_EBP); /* Alloc stack frame/align stack */ x86_alu_reg_imm (code, X86_SUB, X86_ESP, 8); info_offset = -4; mrgctx_offset = - 8; /* The info struct is put into EAX by the gsharedvt trampoline */ /* Save info struct addr */ x86_mov_membase_reg (code, X86_EBP, info_offset, X86_EAX, 4); /* Save rgctx */ x86_mov_membase_reg (code, X86_EBP, mrgctx_offset, MONO_ARCH_RGCTX_REG, 4); /* Allocate stack area used to pass arguments to the method */ x86_mov_reg_membase (code, X86_EAX, X86_EAX, MONO_STRUCT_OFFSET (GSharedVtCallInfo, stack_usage), sizeof (gpointer)); x86_alu_reg_reg (code, X86_SUB, X86_ESP, X86_EAX); #if 0 /* Stack alignment check */ x86_mov_reg_reg (code, X86_ECX, X86_ESP, 4); x86_alu_reg_imm (code, X86_AND, X86_ECX, MONO_ARCH_FRAME_ALIGNMENT - 1); x86_alu_reg_imm (code, X86_CMP, X86_ECX, 0); x86_branch_disp (code, X86_CC_EQ, 3, FALSE); x86_breakpoint (code); #endif /* ecx = caller argument area */ x86_mov_reg_reg (code, X86_ECX, X86_EBP, 4); x86_alu_reg_imm (code, X86_ADD, X86_ECX, 8); /* eax = callee argument area */ x86_mov_reg_reg (code, X86_EAX, X86_ESP, 4); /* Call start_gsharedvt_call */ /* Arg 4 */ x86_push_membase (code, X86_EBP, mrgctx_offset); /* Arg3 */ x86_push_reg (code, X86_EAX); /* Arg2 */ x86_push_reg (code, X86_ECX); /* Arg1 */ x86_push_membase (code, X86_EBP, info_offset); if (aot) { code = mono_arch_emit_load_aotconst (buf, code, &ji, MONO_PATCH_INFO_JIT_ICALL_ADDR, "mono_x86_start_gsharedvt_call"); x86_call_reg (code, X86_EAX); } else { x86_call_code (code, mono_x86_start_gsharedvt_call); } x86_alu_reg_imm (code, X86_ADD, X86_ESP, 4 * 4); /* The address to call is in eax */ /* The stack is now setup for the real call */ /* Load info struct */ x86_mov_reg_membase (code, X86_ECX, X86_EBP, info_offset, 4); /* Load rgctx */ x86_mov_reg_membase (code, MONO_ARCH_RGCTX_REG, X86_EBP, mrgctx_offset, sizeof (gpointer)); /* Make the call */ x86_call_reg (code, X86_EAX); /* The return value is either in registers, or stored to an area beginning at sp [info->vret_slot] */ /* EAX/EDX might contain the return value, only ECX is free */ /* Load info struct */ x86_mov_reg_membase (code, X86_ECX, X86_EBP, info_offset, 4); /* Branch to the in/out handling code */ x86_alu_membase_imm (code, X86_CMP, X86_ECX, MONO_STRUCT_OFFSET (GSharedVtCallInfo, gsharedvt_in), 1); br_out = code; x86_branch32 (code, X86_CC_NE, 0, TRUE); /* * IN CASE */ /* Load ret marshal type */ x86_mov_reg_membase (code, X86_ECX, X86_ECX, MONO_STRUCT_OFFSET (GSharedVtCallInfo, ret_marshal), 4); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_NONE); br [0] = code; x86_branch8 (code, X86_CC_NE, 0, TRUE); /* Normal return, no marshalling required */ x86_leave (code); x86_ret (code); /* Return value marshalling */ x86_patch (br [0], code); /* Load info struct */ x86_mov_reg_membase (code, X86_EAX, X86_EBP, info_offset, 4); /* Load 'vret_slot' */ x86_mov_reg_membase (code, X86_EAX, X86_EAX, MONO_STRUCT_OFFSET (GSharedVtCallInfo, vret_slot), 4); /* Compute ret area address */ x86_shift_reg_imm (code, X86_SHL, X86_EAX, 2); x86_alu_reg_reg (code, X86_ADD, X86_EAX, X86_ESP); /* The callee does a ret $4, so sp is off by 4 */ x86_alu_reg_imm (code, X86_SUB, X86_EAX, sizeof (gpointer)); /* Branch to specific marshalling code */ // FIXME: Move the I4 case to the top */ x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_DOUBLE_FPSTACK); br [1] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_FLOAT_FPSTACK); br [2] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_STACK_POP); br [3] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_I1); br [4] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_U1); br [5] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_I2); br [6] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_U2); br [7] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); /* IREGS case */ /* Load both eax and edx for simplicity */ x86_mov_reg_membase (code, X86_EDX, X86_EAX, sizeof (gpointer), sizeof (gpointer)); x86_mov_reg_membase (code, X86_EAX, X86_EAX, 0, sizeof (gpointer)); x86_leave (code); x86_ret (code); /* DOUBLE_FPSTACK case */ x86_patch (br [1], code); x86_fld_membase (code, X86_EAX, 0, TRUE); x86_jump8 (code, 0); x86_leave (code); x86_ret (code); /* FLOAT_FPSTACK case */ x86_patch (br [2], code); x86_fld_membase (code, X86_EAX, 0, FALSE); x86_leave (code); x86_ret (code); /* STACK_POP case */ x86_patch (br [3], code); x86_leave (code); x86_ret_imm (code, 4); /* I1 case */ x86_patch (br [4], code); x86_widen_membase (code, X86_EAX, X86_EAX, 0, TRUE, FALSE); x86_leave (code); x86_ret (code); /* U1 case */ x86_patch (br [5], code); x86_widen_membase (code, X86_EAX, X86_EAX, 0, FALSE, FALSE); x86_leave (code); x86_ret (code); /* I2 case */ x86_patch (br [6], code); x86_widen_membase (code, X86_EAX, X86_EAX, 0, TRUE, TRUE); x86_leave (code); x86_ret (code); /* U2 case */ x86_patch (br [7], code); x86_widen_membase (code, X86_EAX, X86_EAX, 0, FALSE, TRUE); x86_leave (code); x86_ret (code); /* * OUT CASE */ x86_patch (br_out, code); /* Load ret marshal type into ECX */ x86_mov_reg_membase (code, X86_ECX, X86_ECX, MONO_STRUCT_OFFSET (GSharedVtCallInfo, ret_marshal), 4); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_NONE); br [0] = code; x86_branch8 (code, X86_CC_NE, 0, TRUE); /* Normal return, no marshalling required */ x86_leave (code); x86_ret (code); /* Return value marshalling */ x86_patch (br [0], code); /* EAX might contain the return value */ // FIXME: Use moves x86_push_reg (code, X86_EAX); /* Load info struct */ x86_mov_reg_membase (code, X86_EAX, X86_EBP, info_offset, 4); /* Load 'vret_arg_slot' */ x86_mov_reg_membase (code, X86_EAX, X86_EAX, MONO_STRUCT_OFFSET (GSharedVtCallInfo, vret_arg_slot), 4); /* Compute ret area address in the caller frame in EAX */ x86_shift_reg_imm (code, X86_SHL, X86_EAX, 2); x86_alu_reg_reg (code, X86_ADD, X86_EAX, X86_EBP); x86_alu_reg_imm (code, X86_ADD, X86_EAX, 8); x86_mov_reg_membase (code, X86_EAX, X86_EAX, 0, sizeof (gpointer)); /* Branch to specific marshalling code */ x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_DOUBLE_FPSTACK); br [1] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_FLOAT_FPSTACK); br [2] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_STACK_POP); br [3] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); x86_alu_reg_imm (code, X86_CMP, X86_ECX, GSHAREDVT_RET_IREGS); br [4] = code; x86_branch8 (code, X86_CC_E, 0, TRUE); /* IREG case */ x86_mov_reg_reg (code, X86_ECX, X86_EAX, sizeof (gpointer)); x86_pop_reg (code, X86_EAX); x86_mov_membase_reg (code, X86_ECX, 0, X86_EAX, sizeof (gpointer)); x86_leave (code); x86_ret_imm (code, 4); /* IREGS case */ x86_patch (br [4], code); x86_mov_reg_reg (code, X86_ECX, X86_EAX, sizeof (gpointer)); x86_pop_reg (code, X86_EAX); x86_mov_membase_reg (code, X86_ECX, sizeof (gpointer), X86_EDX, sizeof (gpointer)); x86_mov_membase_reg (code, X86_ECX, 0, X86_EAX, sizeof (gpointer)); x86_leave (code); x86_ret_imm (code, 4); /* DOUBLE_FPSTACK case */ x86_alu_reg_imm (code, X86_ADD, X86_ESP, 4); x86_patch (br [1], code); x86_fst_membase (code, X86_EAX, 0, TRUE, TRUE); x86_jump8 (code, 0); x86_leave (code); x86_ret_imm (code, 4); /* FLOAT_FPSTACK case */ x86_alu_reg_imm (code, X86_ADD, X86_ESP, 4); x86_patch (br [2], code); x86_fst_membase (code, X86_EAX, 0, FALSE, TRUE); x86_leave (code); x86_ret_imm (code, 4); /* STACK_POP case */ x86_patch (br [3], code); x86_leave (code); x86_ret_imm (code, 4); g_assert ((code - buf) < buf_len); if (info) *info = mono_tramp_info_create ("gsharedvt_trampoline", buf, code - buf, ji, unwind_ops); mono_arch_flush_icache (buf, code - buf); return buf; }
/* * Store a small structure from registers to a pointer. The base * register must not be either "reg" or "other_reg". */ static unsigned char *store_small_struct (unsigned char *inst, int reg, int other_reg, int base_reg, jit_nint offset, jit_nint size, int preserve) { switch(size) { case 1: { inst = mov_membase_reg_byte(inst, base_reg, offset, reg); } break; case 2: { x86_mov_membase_reg(inst, base_reg, offset, reg, 2); } break; case 3: { if(preserve) { x86_push_reg(inst, reg); } x86_mov_membase_reg(inst, base_reg, offset, reg, 2); x86_shift_reg_imm(inst, reg, X86_SHR, 16); inst = mov_membase_reg_byte(inst, base_reg, offset + 2, reg); if(preserve) { x86_pop_reg(inst, reg); } } break; case 4: { x86_mov_membase_reg(inst, base_reg, offset, reg, 4); } break; case 5: { x86_mov_membase_reg(inst, base_reg, offset, reg, 4); inst = mov_membase_reg_byte(inst, base_reg, offset + 4, other_reg); } break; case 6: { x86_mov_membase_reg(inst, base_reg, offset, reg, 4); x86_mov_membase_reg(inst, base_reg, offset + 4, other_reg, 2); } break; case 7: { if(preserve) { x86_push_reg(inst, other_reg); } x86_mov_membase_reg(inst, base_reg, offset, reg, 4); x86_mov_membase_reg(inst, base_reg, offset + 4, other_reg, 2); x86_shift_reg_imm(inst, other_reg, X86_SHR, 16); inst = mov_membase_reg_byte(inst, base_reg, offset + 6, other_reg); if(preserve) { x86_pop_reg(inst, other_reg); } } break; case 8: { x86_mov_membase_reg(inst, base_reg, offset, reg, 4); x86_mov_membase_reg(inst, base_reg, offset + 4, other_reg, 4); } break; } return inst; }
transform_func_t ffts_generate_func_code(ffts_plan_t *p, size_t N, size_t leaf_N, int sign) { uint32_t offsets[8] = {0, 4*N, 2*N, 6*N, N, 5*N, 7*N, 3*N}; uint32_t offsets_o[8] = {0, 4*N, 2*N, 6*N, 7*N, 3*N, N, 5*N}; int32_t pAddr = 0; int32_t pN = 0; int32_t pLUT = 0; insns_t *fp; insns_t *start; insns_t *x_4_addr; insns_t *x_8_addr; uint32_t loop_count; int count; ptrdiff_t len; size_t *ps; size_t *pps; count = ffts_tree_count(N, leaf_N, 0) + 1; ps = pps = malloc(2 * count * sizeof(*ps)); if (!ps) { return NULL; } ffts_elaborate_tree(&pps, N, leaf_N, 0); pps[0] = 0; pps[1] = 0; pps = ps; #ifdef HAVE_SSE if (sign < 0) { p->constants = (const void*) sse_constants; } else { p->constants = (const void*) sse_constants_inv; } #endif fp = (insns_t*) p->transform_base; /* generate base cases */ x_4_addr = generate_size4_base_case(&fp, sign); x_8_addr = generate_size8_base_case(&fp, sign); #ifdef __arm__ start = generate_prologue(&fp, p); #ifdef HAVE_NEON memcpy(fp, neon_ee, neon_oo - neon_ee); if (sign < 0) { fp[33] ^= 0x00200000; fp[37] ^= 0x00200000; fp[38] ^= 0x00200000; fp[39] ^= 0x00200000; fp[40] ^= 0x00200000; fp[41] ^= 0x00200000; fp[44] ^= 0x00200000; fp[45] ^= 0x00200000; fp[46] ^= 0x00200000; fp[47] ^= 0x00200000; fp[48] ^= 0x00200000; fp[57] ^= 0x00200000; } fp += (neon_oo - neon_ee) / 4; #else memcpy(fp, vfp_e, vfp_o - vfp_e); if (sign > 0) { fp[64] ^= 0x00000040; fp[65] ^= 0x00000040; fp[68] ^= 0x00000040; fp[75] ^= 0x00000040; fp[76] ^= 0x00000040; fp[79] ^= 0x00000040; fp[80] ^= 0x00000040; fp[83] ^= 0x00000040; fp[84] ^= 0x00000040; fp[87] ^= 0x00000040; fp[91] ^= 0x00000040; fp[93] ^= 0x00000040; } fp += (vfp_o - vfp_e) / 4; #endif #else /* generate functions */ start = generate_prologue(&fp, p); loop_count = 4 * p->i0; generate_leaf_init(&fp, loop_count); if (ffts_ctzl(N) & 1) { generate_leaf_ee(&fp, offsets, p->i1 ? 6 : 0); if (p->i1) { loop_count += 4 * p->i1; generate_leaf_oo(&fp, loop_count, offsets_o, 7); } loop_count += 4; generate_leaf_oe(&fp, offsets_o); } else { generate_leaf_ee(&fp, offsets, N >= 256 ? 2 : 8); loop_count += 4; generate_leaf_eo(&fp, offsets); if (p->i1) { loop_count += 4 * p->i1; generate_leaf_oo(&fp, loop_count, offsets_o, N >= 256 ? 4 : 7); } } if (p->i1) { uint32_t offsets_oe[8] = {7*N, 3*N, N, 5*N, 0, 4*N, 6*N, 2*N}; loop_count += 4 * p->i1; /* align loop/jump destination */ #ifdef _M_X64 x86_mov_reg_imm(fp, X86_EBX, loop_count); #else x86_mov_reg_imm(fp, X86_ECX, loop_count); ffts_align_mem16(&fp, 9); #endif generate_leaf_ee(&fp, offsets_oe, 0); } generate_transform_init(&fp); /* generate subtransform calls */ count = 2; while (pps[0]) { size_t ws_is; if (!pN) { #ifdef _M_X64 x86_mov_reg_imm(fp, X86_EBX, pps[0]); #else x86_mov_reg_imm(fp, X86_ECX, pps[0] / 4); #endif } else { int offset = (4 * pps[1]) - pAddr; if (offset) { #ifdef _M_X64 x64_alu_reg_imm_size(fp, X86_ADD, X64_R8, offset, 8); #else x64_alu_reg_imm_size(fp, X86_ADD, X64_RDX, offset, 8); #endif } if (pps[0] > leaf_N && pps[0] - pN) { int factor = ffts_ctzl(pps[0]) - ffts_ctzl(pN); #ifdef _M_X64 if (factor > 0) { x86_shift_reg_imm(fp, X86_SHL, X86_EBX, factor); } else { x86_shift_reg_imm(fp, X86_SHR, X86_EBX, -factor); } #else if (factor > 0) { x86_shift_reg_imm(fp, X86_SHL, X86_ECX, factor); } else { x86_shift_reg_imm(fp, X86_SHR, X86_ECX, -factor); } #endif } } ws_is = 8 * p->ws_is[ffts_ctzl(pps[0] / leaf_N) - 1]; if (ws_is != pLUT) { int offset = (int) (ws_is - pLUT); #ifdef _M_X64 x64_alu_reg_imm_size(fp, X86_ADD, X64_R9, offset, 8); #else x64_alu_reg_imm_size(fp, X86_ADD, X64_R8, offset, 8); #endif } if (pps[0] == 2 * leaf_N) { x64_call_code(fp, x_4_addr); } else { x64_call_code(fp, x_8_addr); } pAddr = 4 * pps[1]; if (pps[0] > leaf_N) { pN = pps[0]; } pLUT = ws_is;//LUT_offset(pps[0], leafN); //fprintf(stderr, "LUT offset for %d is %d\n", pN, pLUT); count += 4; pps += 2; } #endif #ifdef __arm__ #ifdef HAVE_NEON if (ffts_ctzl(N) & 1) { ADDI(&fp, 2, 7, 0); ADDI(&fp, 7, 9, 0); ADDI(&fp, 9, 2, 0); ADDI(&fp, 2, 8, 0); ADDI(&fp, 8, 10, 0); ADDI(&fp, 10, 2, 0); if(p->i1) { MOVI(&fp, 11, p->i1); memcpy(fp, neon_oo, neon_eo - neon_oo); if(sign < 0) { fp[12] ^= 0x00200000; fp[13] ^= 0x00200000; fp[14] ^= 0x00200000; fp[15] ^= 0x00200000; fp[27] ^= 0x00200000; fp[29] ^= 0x00200000; fp[30] ^= 0x00200000; fp[31] ^= 0x00200000; fp[46] ^= 0x00200000; fp[47] ^= 0x00200000; fp[48] ^= 0x00200000; fp[57] ^= 0x00200000; } fp += (neon_eo - neon_oo) / 4; } *fp = LDRI(11, 1, ((uint32_t)&p->oe_ws) - ((uint32_t)p)); fp++; memcpy(fp, neon_oe, neon_end - neon_oe); if(sign < 0) { fp[19] ^= 0x00200000; fp[20] ^= 0x00200000; fp[22] ^= 0x00200000; fp[23] ^= 0x00200000; fp[37] ^= 0x00200000; fp[38] ^= 0x00200000; fp[40] ^= 0x00200000; fp[41] ^= 0x00200000; fp[64] ^= 0x00200000; fp[65] ^= 0x00200000; fp[66] ^= 0x00200000; fp[67] ^= 0x00200000; } fp += (neon_end - neon_oe) / 4; } else { *fp = LDRI(11, 1, ((uint32_t)&p->eo_ws) - ((uint32_t)p)); fp++; memcpy(fp, neon_eo, neon_oe - neon_eo); if(sign < 0) { fp[10] ^= 0x00200000; fp[11] ^= 0x00200000; fp[13] ^= 0x00200000; fp[14] ^= 0x00200000; fp[31] ^= 0x00200000; fp[33] ^= 0x00200000; fp[34] ^= 0x00200000; fp[35] ^= 0x00200000; fp[59] ^= 0x00200000; fp[60] ^= 0x00200000; fp[61] ^= 0x00200000; fp[62] ^= 0x00200000; } fp += (neon_oe - neon_eo) / 4; ADDI(&fp, 2, 7, 0); ADDI(&fp, 7, 9, 0); ADDI(&fp, 9, 2, 0); ADDI(&fp, 2, 8, 0); ADDI(&fp, 8, 10, 0); ADDI(&fp, 10, 2, 0); if(p->i1) { MOVI(&fp, 11, p->i1); memcpy(fp, neon_oo, neon_eo - neon_oo); if(sign < 0) { fp[12] ^= 0x00200000; fp[13] ^= 0x00200000; fp[14] ^= 0x00200000; fp[15] ^= 0x00200000; fp[27] ^= 0x00200000; fp[29] ^= 0x00200000; fp[30] ^= 0x00200000; fp[31] ^= 0x00200000; fp[46] ^= 0x00200000; fp[47] ^= 0x00200000; fp[48] ^= 0x00200000; fp[57] ^= 0x00200000; } fp += (neon_eo - neon_oo) / 4; } } if(p->i1) { ADDI(&fp, 2, 3, 0); ADDI(&fp, 3, 7, 0); ADDI(&fp, 7, 2, 0); ADDI(&fp, 2, 4, 0); ADDI(&fp, 4, 8, 0); ADDI(&fp, 8, 2, 0); ADDI(&fp, 2, 5, 0); ADDI(&fp, 5, 9, 0); ADDI(&fp, 9, 2, 0); ADDI(&fp, 2, 6, 0); ADDI(&fp, 6, 10, 0); ADDI(&fp, 10, 2, 0); ADDI(&fp, 2, 9, 0); ADDI(&fp, 9, 10, 0); ADDI(&fp, 10, 2, 0); *fp = LDRI(2, 1, ((uint32_t)&p->ee_ws) - ((uint32_t)p)); fp++; MOVI(&fp, 11, p->i1); memcpy(fp, neon_ee, neon_oo - neon_ee); if(sign < 0) { fp[33] ^= 0x00200000; fp[37] ^= 0x00200000; fp[38] ^= 0x00200000; fp[39] ^= 0x00200000; fp[40] ^= 0x00200000; fp[41] ^= 0x00200000; fp[44] ^= 0x00200000; fp[45] ^= 0x00200000; fp[46] ^= 0x00200000; fp[47] ^= 0x00200000; fp[48] ^= 0x00200000; fp[57] ^= 0x00200000; } fp += (neon_oo - neon_ee) / 4; } #else ADDI(&fp, 2, 7, 0); ADDI(&fp, 7, 9, 0); ADDI(&fp, 9, 2, 0); ADDI(&fp, 2, 8, 0); ADDI(&fp, 8, 10, 0); ADDI(&fp, 10, 2, 0); MOVI(&fp, 11, (p->i1>0) ? p->i1 : 1); memcpy(fp, vfp_o, vfp_x4 - vfp_o); if(sign > 0) { fp[22] ^= 0x00000040; fp[24] ^= 0x00000040; fp[25] ^= 0x00000040; fp[26] ^= 0x00000040; fp[62] ^= 0x00000040; fp[64] ^= 0x00000040; fp[65] ^= 0x00000040; fp[66] ^= 0x00000040; } fp += (vfp_x4 - vfp_o) / 4; ADDI(&fp, 2, 3, 0); ADDI(&fp, 3, 7, 0); ADDI(&fp, 7, 2, 0); ADDI(&fp, 2, 4, 0); ADDI(&fp, 4, 8, 0); ADDI(&fp, 8, 2, 0); ADDI(&fp, 2, 5, 0); ADDI(&fp, 5, 9, 0); ADDI(&fp, 9, 2, 0); ADDI(&fp, 2, 6, 0); ADDI(&fp, 6, 10, 0); ADDI(&fp, 10, 2, 0); ADDI(&fp, 2, 9, 0); ADDI(&fp, 9, 10, 0); ADDI(&fp, 10, 2, 0); *fp = LDRI(2, 1, ((uint32_t)&p->ee_ws) - ((uint32_t)p)); fp++; MOVI(&fp, 11, (p->i2>0) ? p->i2 : 1); memcpy(fp, vfp_e, vfp_o - vfp_e); if(sign > 0) { fp[64] ^= 0x00000040; fp[65] ^= 0x00000040; fp[68] ^= 0x00000040; fp[75] ^= 0x00000040; fp[76] ^= 0x00000040; fp[79] ^= 0x00000040; fp[80] ^= 0x00000040; fp[83] ^= 0x00000040; fp[84] ^= 0x00000040; fp[87] ^= 0x00000040; fp[91] ^= 0x00000040; fp[93] ^= 0x00000040; } fp += (vfp_o - vfp_e) / 4; #endif *fp = LDRI(2, 1, ((uint32_t)&p->ws) - ((uint32_t)p)); fp++; // load offsets into r12 //ADDI(&fp, 2, 1, 0); MOVI(&fp, 1, 0); // args: r0 - out // r1 - N // r2 - ws // ADDI(&fp, 3, 1, 0); // put N into r3 for counter count = 2; while(pps[0]) { // fprintf(stderr, "size %zu at %zu - diff %zu\n", pps[0], pps[1]*4, (pps[1]*4) - pAddr); if(!pN) { MOVI(&fp, 1, pps[0]); } else { if((pps[1]*4)-pAddr) ADDI(&fp, 0, 0, (pps[1] * 4)- pAddr); if(pps[0] - pN) ADDI(&fp, 1, 1, pps[0] - pN); } if (p->ws_is[ffts_ctzl(pps[0]/leaf_N)-1]*8 - pLUT) { ADDI(&fp, 2, 2, p->ws_is[ffts_ctzl(pps[0]/leaf_N)-1]*8 - pLUT); } if(pps[0] == 2 * leaf_N) { *fp = BL(fp+2, x_4_addr); fp++; } else if(!pps[2]) { //uint32_t *x_8_t_addr = fp; #ifdef HAVE_NEON memcpy(fp, neon_x8_t, neon_ee - neon_x8_t); if(sign < 0) { fp[31] ^= 0x00200000; fp[32] ^= 0x00200000; fp[33] ^= 0x00200000; fp[34] ^= 0x00200000; fp[65] ^= 0x00200000; fp[66] ^= 0x00200000; fp[70] ^= 0x00200000; fp[74] ^= 0x00200000; fp[97] ^= 0x00200000; fp[98] ^= 0x00200000; fp[102] ^= 0x00200000; fp[104] ^= 0x00200000; } fp += (neon_ee - neon_x8_t) / 4; //*fp++ = BL(fp+2, x_8_t_addr); #else *fp = BL(fp+2, x_8_addr); fp++; #endif } else { *fp = BL(fp+2, x_8_addr); fp++; } pAddr = pps[1] * 4; pN = pps[0]; pLUT = p->ws_is[ffts_ctzl(pps[0]/leaf_N)-1]*8;//LUT_offset(pps[0], leafN); // fprintf(stderr, "LUT offset for %d is %d\n", pN, pLUT); count += 4; pps += 2; } *fp++ = 0xecbd8b10; *fp++ = POP_LR(); count++; #else generate_epilogue(&fp); #endif // *fp++ = B(14); count++; //for(int i=0;i<(neon_x8 - neon_x4)/4;i++) // fprintf(stderr, "%08x\n", x_4_addr[i]); //fprintf(stderr, "\n"); //for(int i=0;i<count;i++) //fprintf(stderr, "size of transform %u = %d\n", N, (fp - x_8_addr) * sizeof(*fp)); free(ps); #if defined(_MSC_VER) #pragma warning(push) /* disable type cast warning from data pointer to function pointer */ #pragma warning(disable : 4055) #endif return (transform_func_t) start; #if defined(_MSC_VER) #pragma warning(pop) #endif }