void MethodHandles::verify_klass(MacroAssembler* _masm,
Register obj, SystemDictionary::WKID klass_id,
const char* error_message) {
InstanceKlass** klass_addr = SystemDictionary::well_known_klass_addr(klass_id);
KlassHandle klass = SystemDictionary::well_known_klass(klass_id);
Register temp = rscratch2;
Register temp2 = rscratch1; // used by MacroAssembler::cmpptr
Label L_ok, L_bad;
BLOCK_COMMENT("verify_klass {");
__ verify_oop(obj);
__ cbz(obj, L_bad);
__ push(RegSet::of(temp, temp2), sp);
__ load_klass(temp, obj);
__ cmpptr(temp, ExternalAddress((address) klass_addr));
__ br(Assembler::EQ, L_ok);
intptr_t super_check_offset = klass->super_check_offset();
__ ldr(temp, Address(temp, super_check_offset));
__ cmpptr(temp, ExternalAddress((address) klass_addr));
__ br(Assembler::EQ, L_ok);
__ pop(RegSet::of(temp, temp2), sp);
__ bind(L_bad);
__ stop(error_message);
__ BIND(L_ok);
__ pop(RegSet::of(temp, temp2), sp);
BLOCK_COMMENT("} verify_klass");
}
void MethodHandles::verify_klass(MacroAssembler* _masm,
Register obj, SystemDictionary::WKID klass_id,
const char* error_message) {
Klass** klass_addr = SystemDictionary::well_known_klass_addr(klass_id);
KlassHandle klass = SystemDictionary::well_known_klass(klass_id);
Register temp = rdi;
Register temp2 = noreg;
LP64_ONLY(temp2 = rscratch1); // used by MacroAssembler::cmpptr
Label L_ok, L_bad;
BLOCK_COMMENT("verify_klass {");
__ verify_oop(obj);
__ testptr(obj, obj);
__ jcc(Assembler::zero, L_bad);
__ push(temp); if (temp2 != noreg) __ push(temp2);
#define UNPUSH { if (temp2 != noreg) __ pop(temp2); __ pop(temp); }
__ load_klass(temp, obj);
__ cmpptr(temp, ExternalAddress((address) klass_addr));
__ jcc(Assembler::equal, L_ok);
intptr_t super_check_offset = klass->super_check_offset();
__ movptr(temp, Address(temp, super_check_offset));
__ cmpptr(temp, ExternalAddress((address) klass_addr));
__ jcc(Assembler::equal, L_ok);
UNPUSH;
__ bind(L_bad);
__ STOP(error_message);
__ BIND(L_ok);
UNPUSH;
BLOCK_COMMENT("} verify_klass");
}
VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
const int code_length = VtableStub::pd_code_size_limit(true);
VtableStub* s = new(code_length) VtableStub(true, vtable_index);
// Can be NULL if there is no free space in the code cache.
if (s == NULL) {
return NULL;
}
ResourceMark rm;
CodeBuffer cb(s->entry_point(), code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
assert(VtableStub::receiver_location() == R0->as_VMReg(), "receiver expected in R0");
const Register tmp = Rtemp; // Rtemp OK, should be free at call sites
address npe_addr = __ pc();
__ load_klass(tmp, R0);
{
int entry_offset = in_bytes(Klass::vtable_start_offset()) + vtable_index * vtableEntry::size_in_bytes();
int method_offset = vtableEntry::method_offset_in_bytes() + entry_offset;
assert ((method_offset & (wordSize - 1)) == 0, "offset should be aligned");
int offset_mask = AARCH64_ONLY(0xfff << LogBytesPerWord) NOT_AARCH64(0xfff);
if (method_offset & ~offset_mask) {
__ add(tmp, tmp, method_offset & ~offset_mask);
}
__ ldr(Rmethod, Address(tmp, method_offset & offset_mask));
}
address ame_addr = __ pc();
#ifdef AARCH64
__ ldr(tmp, Address(Rmethod, Method::from_compiled_offset()));
__ br(tmp);
#else
__ ldr(PC, Address(Rmethod, Method::from_compiled_offset()));
#endif // AARCH64
masm->flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("vtable #%d at " PTR_FORMAT "[%d] left over: %d",
vtable_index, p2i(s->entry_point()),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
// FIXME ARM: need correct 'slop' - below is x86 code
// shut the door on sizing bugs
//int slop = 8; // 32-bit offset is this much larger than a 13-bit one
//assert(vtable_index > 10 || __ pc() + slop <= s->code_end(), "room for 32-bit offset");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
Label L;
const Register temp_reg = G3_scratch;
// Note: needs more testing of out-of-line vs. inline slow case
verify_oop(receiver);
load_klass(receiver, temp_reg);
cmp_and_brx_short(temp_reg, iCache, Assembler::equal, Assembler::pt, L);
AddressLiteral ic_miss(SharedRuntime::get_ic_miss_stub());
jump_to(ic_miss, temp_reg);
delayed()->nop();
align(CodeEntryAlignment);
bind(L);
}
void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
const Register temp_reg = R12_scratch2;
verify_oop(receiver);
load_klass(temp_reg, receiver);
if (TrapBasedICMissChecks) {
trap_ic_miss_check(temp_reg, iCache);
} else {
Label L;
cmpd(CCR0, temp_reg, iCache);
beq(CCR0, L);
//load_const_optimized(temp_reg, SharedRuntime::get_ic_miss_stub(), R0);
calculate_address_from_global_toc(temp_reg, SharedRuntime::get_ic_miss_stub(), true, true, false);
mtctr(temp_reg);
bctr();
align(32, 12);
bind(L);
}
}
void MethodHandles::verify_klass(MacroAssembler* _masm,
Register obj_reg, SystemDictionary::WKID klass_id,
Register temp_reg, Register temp2_reg,
const char* error_message) {
Klass** klass_addr = SystemDictionary::well_known_klass_addr(klass_id);
KlassHandle klass = SystemDictionary::well_known_klass(klass_id);
Label L_ok, L_bad;
BLOCK_COMMENT("verify_klass {");
__ verify_oop(obj_reg);
__ cmpdi(CCR0, obj_reg, 0);
__ beq(CCR0, L_bad);
__ load_klass(temp_reg, obj_reg);
__ load_const_optimized(temp2_reg, (address) klass_addr);
__ ld(temp2_reg, 0, temp2_reg);
__ cmpd(CCR0, temp_reg, temp2_reg);
__ beq(CCR0, L_ok);
__ ld(temp_reg, klass->super_check_offset(), temp_reg);
__ cmpd(CCR0, temp_reg, temp2_reg);
__ beq(CCR0, L_ok);
__ BIND(L_bad);
__ stop(error_message);
__ BIND(L_ok);
BLOCK_COMMENT("} verify_klass");
}
VtableStub* VtableStubs::create_itable_stub(int itable_index) {
// Note well: pd_code_size_limit is the absolute minimum we can get
// away with. If you add code here, bump the code stub size
// returned by pd_code_size_limit!
const int code_length = VtableStub::pd_code_size_limit(false);
VtableStub* s = new(code_length) VtableStub(false, itable_index);
ResourceMark rm;
CodeBuffer cb(s->entry_point(), code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
#ifndef PRODUCT
if (CountCompiledCalls) {
__ lea(r10, ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
__ incrementw(Address(r10));
}
#endif
// Entry arguments:
// rscratch2: Interface
// j_rarg0: Receiver
// Free registers (non-args) are r0 (interface), rmethod
// get receiver (need to skip return address on top of stack)
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// get receiver klass (also an implicit null-check)
address npe_addr = __ pc();
// Most registers are in use; we'll use r0, rmethod, r10, r11
__ load_klass(r10, j_rarg0);
Label throw_icce;
// Get Method* and entrypoint for compiler
__ lookup_interface_method(// inputs: rec. class, interface, itable index
r10, rscratch2, itable_index,
// outputs: method, scan temp. reg
rmethod, r11,
throw_icce);
// method (rmethod): Method*
// j_rarg0: receiver
#ifdef ASSERT
if (DebugVtables) {
Label L2;
__ cbz(rmethod, L2);
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ cbnz(rscratch1, L2);
__ stop("compiler entrypoint is null");
__ bind(L2);
}
#endif // ASSERT
// rmethod: Method*
// j_rarg0: receiver
address ame_addr = __ pc();
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ br(rscratch1);
__ bind(throw_icce);
__ far_jump(RuntimeAddress(StubRoutines::throw_IncompatibleClassChangeError_entry()));
__ flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("itable #%d at " PTR_FORMAT "[%d] left over: %d",
itable_index, p2i(s->entry_point()),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
void G1UnsafeGetObjSATBBarrierStub::emit_code(LIR_Assembler* ce) {
// At this point we know that offset == referent_offset.
//
// So we might have to emit:
// if (src == null) goto continuation.
//
// and we definitely have to emit:
// if (klass(src).reference_type == REF_NONE) goto continuation
// if (!marking_active) goto continuation
// if (pre_val == null) goto continuation
// call pre_barrier(pre_val)
// goto continuation
//
__ bind(_entry);
assert(src()->is_register(), "sanity");
Register src_reg = src()->as_register();
if (gen_src_check()) {
// The original src operand was not a constant.
// Generate src == null?
if (__ is_in_wdisp16_range(_continuation)) {
__ br_null(src_reg, /*annul*/false, Assembler::pt, _continuation);
} else {
__ cmp(src_reg, G0);
__ brx(Assembler::equal, false, Assembler::pt, _continuation);
}
__ delayed()->nop();
}
// Generate src->_klass->_reference_type() == REF_NONE)?
assert(tmp()->is_register(), "sanity");
Register tmp_reg = tmp()->as_register();
__ load_klass(src_reg, tmp_reg);
Address ref_type_adr(tmp_reg, instanceKlass::reference_type_offset());
__ ldub(ref_type_adr, tmp_reg);
// _reference_type field is of type ReferenceType (enum)
assert(REF_NONE == 0, "check this code");
__ cmp_zero_and_br(Assembler::equal, tmp_reg, _continuation, /*annul*/false, Assembler::pt);
__ delayed()->nop();
// Is marking active?
assert(thread()->is_register(), "precondition");
Register thread_reg = thread()->as_pointer_register();
Address in_progress(thread_reg, in_bytes(JavaThread::satb_mark_queue_offset() +
PtrQueue::byte_offset_of_active()));
if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {
__ ld(in_progress, tmp_reg);
} else {
assert(in_bytes(PtrQueue::byte_width_of_active()) == 1, "Assumption");
__ ldsb(in_progress, tmp_reg);
}
__ cmp_zero_and_br(Assembler::equal, tmp_reg, _continuation, /*annul*/false, Assembler::pt);
__ delayed()->nop();
// val == null?
assert(val()->is_register(), "Precondition.");
Register val_reg = val()->as_register();
if (__ is_in_wdisp16_range(_continuation)) {
__ br_null(val_reg, /*annul*/false, Assembler::pt, _continuation);
} else {
__ cmp(val_reg, G0);
__ brx(Assembler::equal, false, Assembler::pt, _continuation);
}
__ delayed()->nop();
__ call(Runtime1::entry_for(Runtime1::Runtime1::g1_pre_barrier_slow_id));
__ delayed()->mov(val_reg, G4);
__ br(Assembler::always, false, Assembler::pt, _continuation);
__ delayed()->nop();
}
OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
OopMapSet* oop_maps = NULL;
// for better readability
const bool must_gc_arguments = true;
const bool dont_gc_arguments = false;
// stub code & info for the different stubs
switch (id) {
case forward_exception_id:
{
oop_maps = generate_handle_exception(id, sasm);
}
break;
case new_instance_id:
case fast_new_instance_id:
case fast_new_instance_init_check_id:
{
Register G5_klass = G5; // Incoming
Register O0_obj = O0; // Outgoing
if (id == new_instance_id) {
__ set_info("new_instance", dont_gc_arguments);
} else if (id == fast_new_instance_id) {
__ set_info("fast new_instance", dont_gc_arguments);
} else {
assert(id == fast_new_instance_init_check_id, "bad StubID");
__ set_info("fast new_instance init check", dont_gc_arguments);
}
if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
UseTLAB && FastTLABRefill) {
Label slow_path;
Register G1_obj_size = G1;
Register G3_t1 = G3;
Register G4_t2 = G4;
assert_different_registers(G5_klass, G1_obj_size, G3_t1, G4_t2);
// Push a frame since we may do dtrace notification for the
// allocation which requires calling out and we don't want
// to stomp the real return address.
__ save_frame(0);
if (id == fast_new_instance_init_check_id) {
// make sure the klass is initialized
__ ldub(G5_klass, in_bytes(InstanceKlass::init_state_offset()), G3_t1);
__ cmp_and_br_short(G3_t1, InstanceKlass::fully_initialized, Assembler::notEqual, Assembler::pn, slow_path);
}
#ifdef ASSERT
// assert object can be fast path allocated
{
Label ok, not_ok;
__ ld(G5_klass, in_bytes(Klass::layout_helper_offset()), G1_obj_size);
// make sure it's an instance (LH > 0)
__ cmp_and_br_short(G1_obj_size, 0, Assembler::lessEqual, Assembler::pn, not_ok);
__ btst(Klass::_lh_instance_slow_path_bit, G1_obj_size);
__ br(Assembler::zero, false, Assembler::pn, ok);
__ delayed()->nop();
__ bind(not_ok);
__ stop("assert(can be fast path allocated)");
__ should_not_reach_here();
__ bind(ok);
}
#endif // ASSERT
// if we got here then the TLAB allocation failed, so try
// refilling the TLAB or allocating directly from eden.
Label retry_tlab, try_eden;
__ tlab_refill(retry_tlab, try_eden, slow_path); // preserves G5_klass
__ bind(retry_tlab);
// get the instance size
__ ld(G5_klass, in_bytes(Klass::layout_helper_offset()), G1_obj_size);
__ tlab_allocate(O0_obj, G1_obj_size, 0, G3_t1, slow_path);
__ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2);
__ verify_oop(O0_obj);
__ mov(O0, I0);
__ ret();
__ delayed()->restore();
__ bind(try_eden);
// get the instance size
__ ld(G5_klass, in_bytes(Klass::layout_helper_offset()), G1_obj_size);
__ eden_allocate(O0_obj, G1_obj_size, 0, G3_t1, G4_t2, slow_path);
__ incr_allocated_bytes(G1_obj_size, G3_t1, G4_t2);
__ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2);
__ verify_oop(O0_obj);
__ mov(O0, I0);
__ ret();
__ delayed()->restore();
__ bind(slow_path);
// pop this frame so generate_stub_call can push it's own
__ restore();
}
oop_maps = generate_stub_call(sasm, I0, CAST_FROM_FN_PTR(address, new_instance), G5_klass);
// I0->O0: new instance
}
break;
case counter_overflow_id:
// G4 contains bci, G5 contains method
oop_maps = generate_stub_call(sasm, noreg, CAST_FROM_FN_PTR(address, counter_overflow), G4, G5);
break;
case new_type_array_id:
case new_object_array_id:
{
Register G5_klass = G5; // Incoming
Register G4_length = G4; // Incoming
Register O0_obj = O0; // Outgoing
Address klass_lh(G5_klass, Klass::layout_helper_offset());
assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
assert(Klass::_lh_header_size_mask == 0xFF, "bytewise");
// Use this offset to pick out an individual byte of the layout_helper:
const int klass_lh_header_size_offset = ((BytesPerInt - 1) // 3 - 2 selects byte {0,1,0,0}
- Klass::_lh_header_size_shift / BitsPerByte);
if (id == new_type_array_id) {
__ set_info("new_type_array", dont_gc_arguments);
} else {
__ set_info("new_object_array", dont_gc_arguments);
}
#ifdef ASSERT
// assert object type is really an array of the proper kind
{
Label ok;
Register G3_t1 = G3;
__ ld(klass_lh, G3_t1);
__ sra(G3_t1, Klass::_lh_array_tag_shift, G3_t1);
int tag = ((id == new_type_array_id)
? Klass::_lh_array_tag_type_value
: Klass::_lh_array_tag_obj_value);
__ cmp_and_brx_short(G3_t1, tag, Assembler::equal, Assembler::pt, ok);
__ stop("assert(is an array klass)");
__ should_not_reach_here();
__ bind(ok);
}
#endif // ASSERT
if (UseTLAB && FastTLABRefill) {
Label slow_path;
Register G1_arr_size = G1;
Register G3_t1 = G3;
Register O1_t2 = O1;
assert_different_registers(G5_klass, G4_length, G1_arr_size, G3_t1, O1_t2);
// check that array length is small enough for fast path
__ set(C1_MacroAssembler::max_array_allocation_length, G3_t1);
__ cmp_and_br_short(G4_length, G3_t1, Assembler::greaterUnsigned, Assembler::pn, slow_path);
// if we got here then the TLAB allocation failed, so try
// refilling the TLAB or allocating directly from eden.
Label retry_tlab, try_eden;
__ tlab_refill(retry_tlab, try_eden, slow_path); // preserves G4_length and G5_klass
__ bind(retry_tlab);
// get the allocation size: (length << (layout_helper & 0x1F)) + header_size
__ ld(klass_lh, G3_t1);
__ sll(G4_length, G3_t1, G1_arr_size);
__ srl(G3_t1, Klass::_lh_header_size_shift, G3_t1);
__ and3(G3_t1, Klass::_lh_header_size_mask, G3_t1);
__ add(G1_arr_size, G3_t1, G1_arr_size);
__ add(G1_arr_size, MinObjAlignmentInBytesMask, G1_arr_size); // align up
__ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size);
__ tlab_allocate(O0_obj, G1_arr_size, 0, G3_t1, slow_path); // preserves G1_arr_size
__ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2);
__ ldub(klass_lh, G3_t1, klass_lh_header_size_offset);
__ sub(G1_arr_size, G3_t1, O1_t2); // body length
__ add(O0_obj, G3_t1, G3_t1); // body start
__ initialize_body(G3_t1, O1_t2);
__ verify_oop(O0_obj);
__ retl();
__ delayed()->nop();
__ bind(try_eden);
// get the allocation size: (length << (layout_helper & 0x1F)) + header_size
__ ld(klass_lh, G3_t1);
__ sll(G4_length, G3_t1, G1_arr_size);
__ srl(G3_t1, Klass::_lh_header_size_shift, G3_t1);
__ and3(G3_t1, Klass::_lh_header_size_mask, G3_t1);
__ add(G1_arr_size, G3_t1, G1_arr_size);
__ add(G1_arr_size, MinObjAlignmentInBytesMask, G1_arr_size);
__ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size);
__ eden_allocate(O0_obj, G1_arr_size, 0, G3_t1, O1_t2, slow_path); // preserves G1_arr_size
__ incr_allocated_bytes(G1_arr_size, G3_t1, O1_t2);
__ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2);
__ ldub(klass_lh, G3_t1, klass_lh_header_size_offset);
__ sub(G1_arr_size, G3_t1, O1_t2); // body length
__ add(O0_obj, G3_t1, G3_t1); // body start
__ initialize_body(G3_t1, O1_t2);
__ verify_oop(O0_obj);
__ retl();
__ delayed()->nop();
__ bind(slow_path);
}
if (id == new_type_array_id) {
oop_maps = generate_stub_call(sasm, I0, CAST_FROM_FN_PTR(address, new_type_array), G5_klass, G4_length);
} else {
oop_maps = generate_stub_call(sasm, I0, CAST_FROM_FN_PTR(address, new_object_array), G5_klass, G4_length);
}
// I0 -> O0: new array
}
break;
case new_multi_array_id:
{ // O0: klass
// O1: rank
// O2: address of 1st dimension
__ set_info("new_multi_array", dont_gc_arguments);
oop_maps = generate_stub_call(sasm, I0, CAST_FROM_FN_PTR(address, new_multi_array), I0, I1, I2);
// I0 -> O0: new multi array
}
break;
case register_finalizer_id:
{
__ set_info("register_finalizer", dont_gc_arguments);
// load the klass and check the has finalizer flag
Label register_finalizer;
Register t = O1;
__ load_klass(O0, t);
__ ld(t, in_bytes(Klass::access_flags_offset()), t);
__ set(JVM_ACC_HAS_FINALIZER, G3);
__ andcc(G3, t, G0);
__ br(Assembler::notZero, false, Assembler::pt, register_finalizer);
__ delayed()->nop();
// do a leaf return
__ retl();
__ delayed()->nop();
__ bind(register_finalizer);
OopMap* oop_map = save_live_registers(sasm);
int call_offset = __ call_RT(noreg, noreg,
CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), I0);
oop_maps = new OopMapSet();
oop_maps->add_gc_map(call_offset, oop_map);
// Now restore all the live registers
restore_live_registers(sasm);
__ ret();
__ delayed()->restore();
}
break;
case throw_range_check_failed_id:
{ __ set_info("range_check_failed", dont_gc_arguments); // arguments will be discarded
// G4: index
oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
}
break;
case throw_index_exception_id:
{ __ set_info("index_range_check_failed", dont_gc_arguments); // arguments will be discarded
// G4: index
oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
}
break;
case throw_div0_exception_id:
{ __ set_info("throw_div0_exception", dont_gc_arguments);
oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
}
break;
case throw_null_pointer_exception_id:
{ __ set_info("throw_null_pointer_exception", dont_gc_arguments);
oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
}
break;
case handle_exception_id:
{ __ set_info("handle_exception", dont_gc_arguments);
oop_maps = generate_handle_exception(id, sasm);
}
break;
case handle_exception_from_callee_id:
{ __ set_info("handle_exception_from_callee", dont_gc_arguments);
oop_maps = generate_handle_exception(id, sasm);
}
break;
case unwind_exception_id:
{
// O0: exception
// I7: address of call to this method
__ set_info("unwind_exception", dont_gc_arguments);
__ mov(Oexception, Oexception->after_save());
__ add(I7, frame::pc_return_offset, Oissuing_pc->after_save());
__ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address),
G2_thread, Oissuing_pc->after_save());
__ verify_not_null_oop(Oexception->after_save());
// Restore SP from L7 if the exception PC is a method handle call site.
__ mov(O0, G5); // Save the target address.
__ lduw(Address(G2_thread, JavaThread::is_method_handle_return_offset()), L0);
__ tst(L0); // Condition codes are preserved over the restore.
__ restore();
__ jmp(G5, 0);
__ delayed()->movcc(Assembler::notZero, false, Assembler::icc, L7_mh_SP_save, SP); // Restore SP if required.
}
break;
case throw_array_store_exception_id:
{
__ set_info("throw_array_store_exception", dont_gc_arguments);
oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
}
break;
case throw_class_cast_exception_id:
{
// G4: object
__ set_info("throw_class_cast_exception", dont_gc_arguments);
oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
}
break;
case throw_incompatible_class_change_error_id:
{
__ set_info("throw_incompatible_class_cast_exception", dont_gc_arguments);
oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
}
break;
case slow_subtype_check_id:
{ // Support for uint StubRoutine::partial_subtype_check( Klass sub, Klass super );
// Arguments :
//
// ret : G3
// sub : G3, argument, destroyed
// super: G1, argument, not changed
// raddr: O7, blown by call
Label miss;
__ save_frame(0); // Blow no registers!
__ check_klass_subtype_slow_path(G3, G1, L0, L1, L2, L4, NULL, &miss);
__ mov(1, G3);
__ ret(); // Result in G5 is 'true'
__ delayed()->restore(); // free copy or add can go here
__ bind(miss);
__ mov(0, G3);
__ ret(); // Result in G5 is 'false'
__ delayed()->restore(); // free copy or add can go here
}
case monitorenter_nofpu_id:
case monitorenter_id:
{ // G4: object
// G5: lock address
__ set_info("monitorenter", dont_gc_arguments);
int save_fpu_registers = (id == monitorenter_id);
// make a frame and preserve the caller's caller-save registers
OopMap* oop_map = save_live_registers(sasm, save_fpu_registers);
int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), G4, G5);
oop_maps = new OopMapSet();
oop_maps->add_gc_map(call_offset, oop_map);
restore_live_registers(sasm, save_fpu_registers);
__ ret();
__ delayed()->restore();
}
break;
case monitorexit_nofpu_id:
case monitorexit_id:
{ // G4: lock address
// note: really a leaf routine but must setup last java sp
// => use call_RT for now (speed can be improved by
// doing last java sp setup manually)
__ set_info("monitorexit", dont_gc_arguments);
int save_fpu_registers = (id == monitorexit_id);
// make a frame and preserve the caller's caller-save registers
OopMap* oop_map = save_live_registers(sasm, save_fpu_registers);
int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), G4);
oop_maps = new OopMapSet();
oop_maps->add_gc_map(call_offset, oop_map);
restore_live_registers(sasm, save_fpu_registers);
__ ret();
__ delayed()->restore();
}
break;
case deoptimize_id:
{
__ set_info("deoptimize", dont_gc_arguments);
OopMap* oop_map = save_live_registers(sasm);
int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize));
oop_maps = new OopMapSet();
oop_maps->add_gc_map(call_offset, oop_map);
restore_live_registers(sasm);
DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
assert(deopt_blob != NULL, "deoptimization blob must have been created");
AddressLiteral dest(deopt_blob->unpack_with_reexecution());
__ jump_to(dest, O0);
__ delayed()->restore();
}
break;
case access_field_patching_id:
{ __ set_info("access_field_patching", dont_gc_arguments);
oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
}
break;
case load_klass_patching_id:
{ __ set_info("load_klass_patching", dont_gc_arguments);
oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
}
break;
case load_mirror_patching_id:
{ __ set_info("load_mirror_patching", dont_gc_arguments);
oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
}
break;
case dtrace_object_alloc_id:
{ // O0: object
__ set_info("dtrace_object_alloc", dont_gc_arguments);
// we can't gc here so skip the oopmap but make sure that all
// the live registers get saved.
save_live_registers(sasm);
__ save_thread(L7_thread_cache);
__ call(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc),
relocInfo::runtime_call_type);
__ delayed()->mov(I0, O0);
__ restore_thread(L7_thread_cache);
restore_live_registers(sasm);
__ ret();
__ delayed()->restore();
}
break;
#if INCLUDE_ALL_GCS
case g1_pre_barrier_slow_id:
{ // G4: previous value of memory
BarrierSet* bs = Universe::heap()->barrier_set();
if (bs->kind() != BarrierSet::G1SATBCTLogging) {
__ save_frame(0);
__ set((int)id, O1);
__ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), I0);
__ should_not_reach_here();
break;
}
__ set_info("g1_pre_barrier_slow_id", dont_gc_arguments);
Register pre_val = G4;
Register tmp = G1_scratch;
Register tmp2 = G3_scratch;
Label refill, restart;
bool with_frame = false; // I don't know if we can do with-frame.
int satb_q_index_byte_offset =
in_bytes(JavaThread::satb_mark_queue_offset() +
PtrQueue::byte_offset_of_index());
int satb_q_buf_byte_offset =
in_bytes(JavaThread::satb_mark_queue_offset() +
PtrQueue::byte_offset_of_buf());
__ bind(restart);
// Load the index into the SATB buffer. PtrQueue::_index is a
// size_t so ld_ptr is appropriate
__ ld_ptr(G2_thread, satb_q_index_byte_offset, tmp);
// index == 0?
__ cmp_and_brx_short(tmp, G0, Assembler::equal, Assembler::pn, refill);
__ ld_ptr(G2_thread, satb_q_buf_byte_offset, tmp2);
__ sub(tmp, oopSize, tmp);
__ st_ptr(pre_val, tmp2, tmp); // [_buf + index] := <address_of_card>
// Use return-from-leaf
__ retl();
__ delayed()->st_ptr(tmp, G2_thread, satb_q_index_byte_offset);
__ bind(refill);
__ save_frame(0);
__ mov(pre_val, L0);
__ mov(tmp, L1);
__ mov(tmp2, L2);
__ call_VM_leaf(L7_thread_cache,
CAST_FROM_FN_PTR(address,
SATBMarkQueueSet::handle_zero_index_for_thread),
G2_thread);
__ mov(L0, pre_val);
__ mov(L1, tmp);
__ mov(L2, tmp2);
__ br(Assembler::always, /*annul*/false, Assembler::pt, restart);
__ delayed()->restore();
}
break;
case g1_post_barrier_slow_id:
{
BarrierSet* bs = Universe::heap()->barrier_set();
if (bs->kind() != BarrierSet::G1SATBCTLogging) {
__ save_frame(0);
__ set((int)id, O1);
__ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), I0);
__ should_not_reach_here();
break;
}
__ set_info("g1_post_barrier_slow_id", dont_gc_arguments);
Register addr = G4;
Register cardtable = G5;
Register tmp = G1_scratch;
Register tmp2 = G3_scratch;
jbyte* byte_map_base = ((CardTableModRefBS*)bs)->byte_map_base;
Label not_already_dirty, restart, refill;
#ifdef _LP64
__ srlx(addr, CardTableModRefBS::card_shift, addr);
#else
__ srl(addr, CardTableModRefBS::card_shift, addr);
#endif
AddressLiteral rs(byte_map_base);
__ set(rs, cardtable); // cardtable := <card table base>
__ ldub(addr, cardtable, tmp); // tmp := [addr + cardtable]
assert(CardTableModRefBS::dirty_card_val() == 0, "otherwise check this code");
__ cmp_and_br_short(tmp, G0, Assembler::notEqual, Assembler::pt, not_already_dirty);
// We didn't take the branch, so we're already dirty: return.
// Use return-from-leaf
__ retl();
__ delayed()->nop();
// Not dirty.
__ bind(not_already_dirty);
// Get cardtable + tmp into a reg by itself
__ add(addr, cardtable, tmp2);
// First, dirty it.
__ stb(G0, tmp2, 0); // [cardPtr] := 0 (i.e., dirty).
Register tmp3 = cardtable;
Register tmp4 = tmp;
// these registers are now dead
addr = cardtable = tmp = noreg;
int dirty_card_q_index_byte_offset =
in_bytes(JavaThread::dirty_card_queue_offset() +
PtrQueue::byte_offset_of_index());
int dirty_card_q_buf_byte_offset =
in_bytes(JavaThread::dirty_card_queue_offset() +
PtrQueue::byte_offset_of_buf());
__ bind(restart);
// Get the index into the update buffer. PtrQueue::_index is
// a size_t so ld_ptr is appropriate here.
__ ld_ptr(G2_thread, dirty_card_q_index_byte_offset, tmp3);
// index == 0?
__ cmp_and_brx_short(tmp3, G0, Assembler::equal, Assembler::pn, refill);
__ ld_ptr(G2_thread, dirty_card_q_buf_byte_offset, tmp4);
__ sub(tmp3, oopSize, tmp3);
__ st_ptr(tmp2, tmp4, tmp3); // [_buf + index] := <address_of_card>
// Use return-from-leaf
__ retl();
__ delayed()->st_ptr(tmp3, G2_thread, dirty_card_q_index_byte_offset);
__ bind(refill);
__ save_frame(0);
__ mov(tmp2, L0);
__ mov(tmp3, L1);
__ mov(tmp4, L2);
__ call_VM_leaf(L7_thread_cache,
CAST_FROM_FN_PTR(address,
DirtyCardQueueSet::handle_zero_index_for_thread),
G2_thread);
__ mov(L0, tmp2);
__ mov(L1, tmp3);
__ mov(L2, tmp4);
__ br(Assembler::always, /*annul*/false, Assembler::pt, restart);
__ delayed()->restore();
}
break;
#endif // INCLUDE_ALL_GCS
case predicate_failed_trap_id:
{
__ set_info("predicate_failed_trap", dont_gc_arguments);
OopMap* oop_map = save_live_registers(sasm);
int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
oop_maps = new OopMapSet();
oop_maps->add_gc_map(call_offset, oop_map);
DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
assert(deopt_blob != NULL, "deoptimization blob must have been created");
restore_live_registers(sasm);
AddressLiteral dest(deopt_blob->unpack_with_reexecution());
__ jump_to(dest, O0);
__ delayed()->restore();
}
break;
default:
{ __ set_info("unimplemented entry", dont_gc_arguments);
__ save_frame(0);
__ set((int)id, O1);
__ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), O1);
__ should_not_reach_here();
}
break;
}
return oop_maps;
}
VtableStub* VtableStubs::create_itable_stub(int itable_index) {
// Note well: pd_code_size_limit is the absolute minimum we can get
// away with. If you add code here, bump the code stub size
// returned by pd_code_size_limit!
const int amd64_code_length = VtableStub::pd_code_size_limit(false);
VtableStub* s = new(amd64_code_length) VtableStub(false, itable_index);
ResourceMark rm;
CodeBuffer cb(s->entry_point(), amd64_code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
#ifndef PRODUCT
if (CountCompiledCalls) {
__ incrementl(ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
}
#endif
// Entry arguments:
// rax: Interface
// j_rarg0: Receiver
// Free registers (non-args) are rax (interface), rbx
// get receiver (need to skip return address on top of stack)
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// get receiver klass (also an implicit null-check)
address npe_addr = __ pc();
// Most registers are in use; we'll use rax, rbx, r10, r11
// (various calling sequences use r[cd]x, r[sd]i, r[89]; stay away from them)
__ load_klass(r10, j_rarg0);
// If we take a trap while this arg is on the stack we will not
// be able to walk the stack properly. This is not an issue except
// when there are mistakes in this assembly code that could generate
// a spurious fault. Ask me how I know...
const Register method = rbx;
Label throw_icce;
// Get methodOop and entrypoint for compiler
__ lookup_interface_method(// inputs: rec. class, interface, itable index
r10, rax, itable_index,
// outputs: method, scan temp. reg
method, r11,
throw_icce);
// method (rbx): methodOop
// j_rarg0: receiver
#ifdef ASSERT
if (DebugVtables) {
Label L2;
__ cmpptr(method, (int32_t)NULL_WORD);
__ jcc(Assembler::equal, L2);
__ cmpptr(Address(method, methodOopDesc::from_compiled_offset()), (int32_t)NULL_WORD);
__ jcc(Assembler::notZero, L2);
__ stop("compiler entrypoint is null");
__ bind(L2);
}
#endif // ASSERT
// rbx: methodOop
// j_rarg0: receiver
address ame_addr = __ pc();
__ jmp(Address(method, methodOopDesc::from_compiled_offset()));
__ bind(throw_icce);
__ jump(RuntimeAddress(StubRoutines::throw_IncompatibleClassChangeError_entry()));
__ flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("itable #%d at "PTR_FORMAT"[%d] left over: %d",
itable_index, s->entry_point(),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
// shut the door on sizing bugs
int slop = 3; // 32-bit offset is this much larger than an 8-bit one
assert(itable_index > 10 || __ pc() + slop <= s->code_end(), "room for 32-bit offset");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
const int amd64_code_length = VtableStub::pd_code_size_limit(true);
VtableStub* s = new(amd64_code_length) VtableStub(true, vtable_index);
ResourceMark rm;
CodeBuffer cb(s->entry_point(), amd64_code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
#ifndef PRODUCT
if (CountCompiledCalls) {
__ incrementl(ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
}
#endif
// get receiver (need to skip return address on top of stack)
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// Free registers (non-args) are rax, rbx
// get receiver klass
address npe_addr = __ pc();
__ load_klass(rax, j_rarg0);
#ifndef PRODUCT
if (DebugVtables) {
Label L;
// check offset vs vtable length
__ cmpl(Address(rax, instanceKlass::vtable_length_offset() * wordSize),
vtable_index * vtableEntry::size());
__ jcc(Assembler::greater, L);
__ movl(rbx, vtable_index);
__ call_VM(noreg,
CAST_FROM_FN_PTR(address, bad_compiled_vtable_index), j_rarg0, rbx);
__ bind(L);
}
#endif // PRODUCT
// load methodOop and target address
const Register method = rbx;
__ lookup_virtual_method(rax, vtable_index, method);
if (DebugVtables) {
Label L;
__ cmpptr(method, (int32_t)NULL_WORD);
__ jcc(Assembler::equal, L);
__ cmpptr(Address(method, methodOopDesc::from_compiled_offset()), (int32_t)NULL_WORD);
__ jcc(Assembler::notZero, L);
__ stop("Vtable entry is NULL");
__ bind(L);
}
// rax: receiver klass
// rbx: methodOop
// rcx: receiver
address ame_addr = __ pc();
__ jmp( Address(rbx, methodOopDesc::from_compiled_offset()));
__ flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("vtable #%d at "PTR_FORMAT"[%d] left over: %d",
vtable_index, s->entry_point(),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
// shut the door on sizing bugs
int slop = 3; // 32-bit offset is this much larger than an 8-bit one
assert(vtable_index > 10 || __ pc() + slop <= s->code_end(), "room for 32-bit offset");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
// NOTE: %%%% if any change is made to this stub make sure that the function
// pd_code_size_limit is changed to ensure the correct size for VtableStub
VtableStub* VtableStubs::create_itable_stub(int itable_index) {
const int sparc_code_length = VtableStub::pd_code_size_limit(false);
VtableStub* s = new(sparc_code_length) VtableStub(false, itable_index);
// Can be NULL if there is no free space in the code cache.
if (s == NULL) {
return NULL;
}
ResourceMark rm;
CodeBuffer cb(s->entry_point(), sparc_code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
Register G3_Klass = G3_scratch;
Register G5_interface = G5; // Passed in as an argument
Label search;
// Entry arguments:
// G5_interface: Interface
// O0: Receiver
assert(VtableStub::receiver_location() == O0->as_VMReg(), "receiver expected in O0");
// get receiver klass (also an implicit null-check)
address npe_addr = __ pc();
__ load_klass(O0, G3_Klass);
// Push a new window to get some temp registers. This chops the head of all
// my 64-bit %o registers in the LION build, but this is OK because no longs
// are passed in the %o registers. Instead, longs are passed in G1 and G4
// and so those registers are not available here.
__ save(SP,-frame::register_save_words*wordSize,SP);
#ifndef PRODUCT
if (CountCompiledCalls) {
__ inc_counter(SharedRuntime::nof_megamorphic_calls_addr(), L0, L1);
}
#endif /* PRODUCT */
Label throw_icce;
Register L5_method = L5;
__ lookup_interface_method(// inputs: rec. class, interface, itable index
G3_Klass, G5_interface, itable_index,
// outputs: method, scan temp. reg
L5_method, L2, L3,
throw_icce);
#ifndef PRODUCT
if (DebugVtables) {
Label L01;
__ br_notnull_short(L5_method, Assembler::pt, L01);
__ stop("Method* is null");
__ bind(L01);
}
#endif
// If the following load is through a NULL pointer, we'll take an OS
// exception that should translate into an AbstractMethodError. We need the
// window count to be correct at that time.
__ restore(L5_method, 0, G5_method);
// Restore registers *before* the AME point.
address ame_addr = __ pc(); // if the vtable entry is null, the method is abstract
__ ld_ptr(G5_method, in_bytes(Method::from_compiled_offset()), G3_scratch);
// G5_method: Method*
// O0: Receiver
// G3_scratch: entry point
__ JMP(G3_scratch, 0);
__ delayed()->nop();
__ bind(throw_icce);
AddressLiteral icce(StubRoutines::throw_IncompatibleClassChangeError_entry());
__ jump_to(icce, G3_scratch);
__ delayed()->restore();
masm->flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("itable #%d at " PTR_FORMAT "[%d] left over: %d",
itable_index, p2i(s->entry_point()),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
// shut the door on sizing bugs
int slop = 2*BytesPerInstWord; // 32-bit offset is this much larger than a 13-bit one
assert(itable_index > 10 || __ pc() + slop <= s->code_end(), "room for sethi;add");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
// Used by compiler only; may use only caller saved, non-argument registers
// NOTE: %%%% if any change is made to this stub make sure that the function
// pd_code_size_limit is changed to ensure the correct size for VtableStub
VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
const int sparc_code_length = VtableStub::pd_code_size_limit(true);
VtableStub* s = new(sparc_code_length) VtableStub(true, vtable_index);
// Can be NULL if there is no free space in the code cache.
if (s == NULL) {
return NULL;
}
ResourceMark rm;
CodeBuffer cb(s->entry_point(), sparc_code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
#ifndef PRODUCT
if (CountCompiledCalls) {
__ inc_counter(SharedRuntime::nof_megamorphic_calls_addr(), G5, G3_scratch);
}
#endif /* PRODUCT */
assert(VtableStub::receiver_location() == O0->as_VMReg(), "receiver expected in O0");
// get receiver klass
address npe_addr = __ pc();
__ load_klass(O0, G3_scratch);
// set Method* (in case of interpreted method), and destination address
#ifndef PRODUCT
if (DebugVtables) {
Label L;
// check offset vs vtable length
__ ld(G3_scratch, in_bytes(Klass::vtable_length_offset()), G5);
__ cmp_and_br_short(G5, vtable_index*vtableEntry::size(), Assembler::greaterUnsigned, Assembler::pt, L);
__ set(vtable_index, O2);
__ call_VM(noreg, CAST_FROM_FN_PTR(address, bad_compiled_vtable_index), O0, O2);
__ bind(L);
}
#endif
__ lookup_virtual_method(G3_scratch, vtable_index, G5_method);
#ifndef PRODUCT
if (DebugVtables) {
Label L;
__ br_notnull_short(G5_method, Assembler::pt, L);
__ stop("Vtable entry is ZERO");
__ bind(L);
}
#endif
address ame_addr = __ pc(); // if the vtable entry is null, the method is abstract
// NOTE: for vtable dispatches, the vtable entry will never be null.
__ ld_ptr(G5_method, in_bytes(Method::from_compiled_offset()), G3_scratch);
// jump to target (either compiled code or c2iadapter)
__ JMP(G3_scratch, 0);
// load Method* (in case we call c2iadapter)
__ delayed()->nop();
masm->flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("vtable #%d at " PTR_FORMAT "[%d] left over: %d",
vtable_index, p2i(s->entry_point()),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
// shut the door on sizing bugs
int slop = 2*BytesPerInstWord; // 32-bit offset is this much larger than a 13-bit one
assert(vtable_index > 10 || __ pc() + slop <= s->code_end(), "room for sethi;add");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
VtableStub* VtableStubs::create_itable_stub(int vtable_index) {
const int code_length = VtableStub::pd_code_size_limit(false);
VtableStub *s = new(code_length) VtableStub(false, vtable_index);
if (s == NULL) { // Indicates OOM in the code cache.
return NULL;
}
ResourceMark rm;
CodeBuffer cb(s->entry_point(), code_length);
MacroAssembler *masm = new MacroAssembler(&cb);
address start_pc;
int padding_bytes = 0;
#if (!defined(PRODUCT) && defined(COMPILER2))
if (CountCompiledCalls) {
// Count unused bytes
// worst case actual size
padding_bytes += __ load_const_size() - __ load_const_optimized_rtn_len(Z_R1_scratch, (long)SharedRuntime::nof_megamorphic_calls_addr(), true);
// Use generic emitter for direct memory increment.
// Use Z_tmp_1 as scratch register for generic emitter.
__ add2mem_32((Z_R1_scratch), 1, Z_tmp_1);
}
#endif
assert(VtableStub::receiver_location() == Z_R2->as_VMReg(), "receiver expected in Z_ARG1");
// Entry arguments:
// Z_method: Interface
// Z_ARG1: Receiver
const Register rcvr_klass = Z_tmp_1; // Used to compute itable_entry_addr.
// Use extra reg to avoid re-load.
const Register vtable_len = Z_tmp_2; // Used to compute itable_entry_addr.
const Register itable_entry_addr = Z_R1_scratch;
const Register itable_interface = Z_R0_scratch;
// Get receiver klass.
// Must do an explicit check if implicit checks are disabled.
address npe_addr = __ pc(); // npe == NULL ptr exception
__ null_check(Z_ARG1, Z_R1_scratch, oopDesc::klass_offset_in_bytes());
__ load_klass(rcvr_klass, Z_ARG1);
// Load start of itable entries into itable_entry.
__ z_llgf(vtable_len, Address(rcvr_klass, InstanceKlass::vtable_length_offset()));
__ z_sllg(vtable_len, vtable_len, exact_log2(vtableEntry::size_in_bytes()));
// Loop over all itable entries until desired interfaceOop(Rinterface) found.
const int vtable_base_offset = in_bytes(InstanceKlass::vtable_start_offset());
// Count unused bytes.
start_pc = __ pc();
__ add2reg_with_index(itable_entry_addr, vtable_base_offset + itableOffsetEntry::interface_offset_in_bytes(), rcvr_klass, vtable_len);
padding_bytes += 20 - (__ pc() - start_pc);
const int itable_offset_search_inc = itableOffsetEntry::size() * wordSize;
Label search;
__ bind(search);
// Handle IncompatibleClassChangeError in itable stubs.
// If the entry is NULL then we've reached the end of the table
// without finding the expected interface, so throw an exception.
NearLabel throw_icce;
__ load_and_test_long(itable_interface, Address(itable_entry_addr));
__ z_bre(throw_icce); // Throw the exception out-of-line.
// Count unused bytes.
start_pc = __ pc();
__ add2reg(itable_entry_addr, itable_offset_search_inc);
padding_bytes += 20 - (__ pc() - start_pc);
__ z_cgr(itable_interface, Z_method);
__ z_brne(search);
// Entry found. Itable_entry_addr points to the subsequent entry (itable_offset_search_inc too far).
// Get offset of vtable for interface.
const Register vtable_offset = Z_R1_scratch;
const Register itable_method = rcvr_klass; // Calculated before.
const int vtable_offset_offset = (itableOffsetEntry::offset_offset_in_bytes() -
itableOffsetEntry::interface_offset_in_bytes()) -
itable_offset_search_inc;
__ z_llgf(vtable_offset, vtable_offset_offset, itable_entry_addr);
// Compute itableMethodEntry and get method and entry point for compiler.
const int method_offset = (itableMethodEntry::size() * wordSize * vtable_index) +
itableMethodEntry::method_offset_in_bytes();
__ z_lg(Z_method, method_offset, vtable_offset, itable_method);
#ifndef PRODUCT
if (DebugVtables) {
Label ok1;
__ z_ltgr(Z_method, Z_method);
__ z_brne(ok1);
__ stop("method is null",103);
__ bind(ok1);
}
#endif
address ame_addr = __ pc();
// Must do an explicit check if implicit checks are disabled.
if (!ImplicitNullChecks) {
__ compare64_and_branch(Z_method, (intptr_t) 0, Assembler::bcondEqual, throw_icce);
}
__ z_lg(Z_R1_scratch, in_bytes(Method::from_compiled_offset()), Z_method);
__ z_br(Z_R1_scratch);
// Handle IncompatibleClassChangeError in itable stubs.
__ bind(throw_icce);
// Count unused bytes
// worst case actual size
// We force resolving of the call site by jumping to
// the "handle wrong method" stub, and so let the
// interpreter runtime do all the dirty work.
padding_bytes += __ load_const_size() - __ load_const_optimized_rtn_len(Z_R1_scratch, (long)SharedRuntime::get_handle_wrong_method_stub(), true);
__ z_br(Z_R1_scratch);
masm->flush();
s->set_exception_points(npe_addr, ame_addr);
return s;
}
void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
vmIntrinsics::ID iid,
Register receiver_reg,
Register member_reg,
bool for_compiler_entry) {
assert(is_signature_polymorphic(iid), "expected invoke iid");
Register temp1 = (for_compiler_entry ? R25_tmp5 : R7);
Register temp2 = (for_compiler_entry ? R22_tmp2 : R8);
Register temp3 = (for_compiler_entry ? R23_tmp3 : R9);
Register temp4 = (for_compiler_entry ? R24_tmp4 : R10);
if (receiver_reg != noreg) assert_different_registers(temp1, temp2, temp3, temp4, receiver_reg);
if (member_reg != noreg) assert_different_registers(temp1, temp2, temp3, temp4, member_reg);
if (iid == vmIntrinsics::_invokeBasic) {
// indirect through MH.form.vmentry.vmtarget
jump_to_lambda_form(_masm, receiver_reg, R19_method, temp1, temp2, for_compiler_entry);
} else {
// The method is a member invoker used by direct method handles.
if (VerifyMethodHandles) {
// make sure the trailing argument really is a MemberName (caller responsibility)
verify_klass(_masm, member_reg, SystemDictionary::WK_KLASS_ENUM_NAME(MemberName_klass),
temp1, temp2,
"MemberName required for invokeVirtual etc.");
}
Register temp1_recv_klass = temp1;
if (iid != vmIntrinsics::_linkToStatic) {
__ verify_oop(receiver_reg);
if (iid == vmIntrinsics::_linkToSpecial) {
// Don't actually load the klass; just null-check the receiver.
__ null_check_throw(receiver_reg, -1, temp1, EXCEPTION_ENTRY);
} else {
// load receiver klass itself
__ null_check_throw(receiver_reg, oopDesc::klass_offset_in_bytes(), temp1, EXCEPTION_ENTRY);
__ load_klass(temp1_recv_klass, receiver_reg);
__ verify_klass_ptr(temp1_recv_klass);
}
BLOCK_COMMENT("check_receiver {");
// The receiver for the MemberName must be in receiver_reg.
// Check the receiver against the MemberName.clazz
if (VerifyMethodHandles && iid == vmIntrinsics::_linkToSpecial) {
// Did not load it above...
__ load_klass(temp1_recv_klass, receiver_reg);
__ verify_klass_ptr(temp1_recv_klass);
}
if (VerifyMethodHandles && iid != vmIntrinsics::_linkToInterface) {
Label L_ok;
Register temp2_defc = temp2;
__ load_heap_oop_not_null(temp2_defc, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes()), member_reg, temp3);
load_klass_from_Class(_masm, temp2_defc, temp3, temp4);
__ verify_klass_ptr(temp2_defc);
__ check_klass_subtype(temp1_recv_klass, temp2_defc, temp3, temp4, L_ok);
// If we get here, the type check failed!
__ stop("receiver class disagrees with MemberName.clazz");
__ BIND(L_ok);
}
BLOCK_COMMENT("} check_receiver");
}
if (iid == vmIntrinsics::_linkToSpecial ||
iid == vmIntrinsics::_linkToStatic) {
DEBUG_ONLY(temp1_recv_klass = noreg); // these guys didn't load the recv_klass
}
// Live registers at this point:
// member_reg - MemberName that was the trailing argument
// temp1_recv_klass - klass of stacked receiver, if needed
// O5_savedSP - interpreter linkage (if interpreted)
// O0..O5 - compiler arguments (if compiled)
Label L_incompatible_class_change_error;
switch (iid) {
case vmIntrinsics::_linkToSpecial:
if (VerifyMethodHandles) {
verify_ref_kind(_masm, JVM_REF_invokeSpecial, member_reg, temp2);
}
__ ld(R19_method, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes()), member_reg);
break;
case vmIntrinsics::_linkToStatic:
if (VerifyMethodHandles) {
verify_ref_kind(_masm, JVM_REF_invokeStatic, member_reg, temp2);
}
__ ld(R19_method, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes()), member_reg);
break;
case vmIntrinsics::_linkToVirtual:
{
// same as TemplateTable::invokevirtual,
// minus the CP setup and profiling:
if (VerifyMethodHandles) {
verify_ref_kind(_masm, JVM_REF_invokeVirtual, member_reg, temp2);
}
// pick out the vtable index from the MemberName, and then we can discard it:
Register temp2_index = temp2;
__ ld(temp2_index, NONZERO(java_lang_invoke_MemberName::vmindex_offset_in_bytes()), member_reg);
if (VerifyMethodHandles) {
Label L_index_ok;
__ cmpdi(CCR1, temp2_index, 0);
__ bge(CCR1, L_index_ok);
__ stop("no virtual index");
__ BIND(L_index_ok);
}
// Note: The verifier invariants allow us to ignore MemberName.clazz and vmtarget
// at this point. And VerifyMethodHandles has already checked clazz, if needed.
// get target Method* & entry point
__ lookup_virtual_method(temp1_recv_klass, temp2_index, R19_method);
break;
}
case vmIntrinsics::_linkToInterface:
{
// same as TemplateTable::invokeinterface
// (minus the CP setup and profiling, with different argument motion)
if (VerifyMethodHandles) {
verify_ref_kind(_masm, JVM_REF_invokeInterface, member_reg, temp2);
}
Register temp2_intf = temp2;
__ load_heap_oop_not_null(temp2_intf, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes()), member_reg, temp3);
load_klass_from_Class(_masm, temp2_intf, temp3, temp4);
__ verify_klass_ptr(temp2_intf);
Register vtable_index = R19_method;
__ ld(vtable_index, NONZERO(java_lang_invoke_MemberName::vmindex_offset_in_bytes()), member_reg);
if (VerifyMethodHandles) {
Label L_index_ok;
__ cmpdi(CCR1, vtable_index, 0);
__ bge(CCR1, L_index_ok);
__ stop("invalid vtable index for MH.invokeInterface");
__ BIND(L_index_ok);
}
// given intf, index, and recv klass, dispatch to the implementation method
__ lookup_interface_method(temp1_recv_klass, temp2_intf,
// note: next two args must be the same:
vtable_index, R19_method,
temp3, temp4,
L_incompatible_class_change_error);
break;
}
default:
fatal(err_msg_res("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)));
break;
}
// Live at this point:
// R19_method
// O5_savedSP (if interpreted)
// After figuring out which concrete method to call, jump into it.
// Note that this works in the interpreter with no data motion.
// But the compiled version will require that rcx_recv be shifted out.
__ verify_method_ptr(R19_method);
jump_from_method_handle(_masm, R19_method, temp1, temp2, for_compiler_entry);
if (iid == vmIntrinsics::_linkToInterface) {
__ BIND(L_incompatible_class_change_error);
__ load_const_optimized(temp1, StubRoutines::throw_IncompatibleClassChangeError_entry());
__ mtctr(temp1);
__ bctr();
}
}
}
// Used by compiler only; may use only caller saved, non-argument registers.
VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
const int code_length = VtableStub::pd_code_size_limit(true);
VtableStub *s = new(code_length) VtableStub(true, vtable_index);
if (s == NULL) { // Indicates OOM In the code cache.
return NULL;
}
ResourceMark rm;
CodeBuffer cb(s->entry_point(), code_length);
MacroAssembler *masm = new MacroAssembler(&cb);
address start_pc;
int padding_bytes = 0;
#if (!defined(PRODUCT) && defined(COMPILER2))
if (CountCompiledCalls) {
// Count unused bytes
// worst case actual size
padding_bytes += __ load_const_size() - __ load_const_optimized_rtn_len(Z_R1_scratch, (long)SharedRuntime::nof_megamorphic_calls_addr(), true);
// Use generic emitter for direct memory increment.
// Abuse Z_method as scratch register for generic emitter.
// It is loaded further down anyway before it is first used.
__ add2mem_32(Address(Z_R1_scratch), 1, Z_method);
}
#endif
assert(VtableStub::receiver_location() == Z_R2->as_VMReg(), "receiver expected in Z_ARG1");
// Get receiver klass.
// Must do an explicit check if implicit checks are disabled.
address npe_addr = __ pc(); // npe == NULL ptr exception
__ null_check(Z_ARG1, Z_R1_scratch, oopDesc::klass_offset_in_bytes());
const Register rcvr_klass = Z_R1_scratch;
__ load_klass(rcvr_klass, Z_ARG1);
// Set method (in case of interpreted method), and destination address.
int entry_offset = in_bytes(InstanceKlass::vtable_start_offset()) +
vtable_index * vtableEntry::size_in_bytes();
#ifndef PRODUCT
if (DebugVtables) {
Label L;
// Check offset vs vtable length.
const Register vtable_idx = Z_R0_scratch;
// Count unused bytes.
// worst case actual size
padding_bytes += __ load_const_size() - __ load_const_optimized_rtn_len(vtable_idx, vtable_index*vtableEntry::size_in_bytes(), true);
assert(Immediate::is_uimm12(in_bytes(InstanceKlass::vtable_length_offset())), "disp to large");
__ z_cl(vtable_idx, in_bytes(InstanceKlass::vtable_length_offset()), rcvr_klass);
__ z_brl(L);
__ z_lghi(Z_ARG3, vtable_index); // Debug code, don't optimize.
__ call_VM(noreg, CAST_FROM_FN_PTR(address, bad_compiled_vtable_index), Z_ARG1, Z_ARG3, false);
// Count unused bytes (assume worst case here).
padding_bytes += 12;
__ bind(L);
}
#endif
int v_off = entry_offset + vtableEntry::method_offset_in_bytes();
// Duplicate safety code from enc_class Java_Dynamic_Call_dynTOC.
if (Displacement::is_validDisp(v_off)) {
__ z_lg(Z_method/*method oop*/, v_off, rcvr_klass/*class oop*/);
// Account for the load_const in the else path.
padding_bytes += __ load_const_size();
} else {
// Worse case, offset does not fit in displacement field.
__ load_const(Z_method, v_off); // Z_method temporarily holds the offset value.
__ z_lg(Z_method/*method oop*/, 0, Z_method/*method offset*/, rcvr_klass/*class oop*/);
}
#ifndef PRODUCT
if (DebugVtables) {
Label L;
__ z_ltgr(Z_method, Z_method);
__ z_brne(L);
__ stop("Vtable entry is ZERO",102);
__ bind(L);
}
#endif
address ame_addr = __ pc(); // ame = abstract method error
// Must do an explicit check if implicit checks are disabled.
__ null_check(Z_method, Z_R1_scratch, in_bytes(Method::from_compiled_offset()));
__ z_lg(Z_R1_scratch, in_bytes(Method::from_compiled_offset()), Z_method);
__ z_br(Z_R1_scratch);
masm->flush();
s->set_exception_points(npe_addr, ame_addr);
return s;
}
void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
vmIntrinsics::ID iid,
Register receiver_reg,
Register member_reg,
bool for_compiler_entry) {
assert(is_signature_polymorphic(iid), "expected invoke iid");
// temps used in this code are not used in *either* compiled or interpreted calling sequences
Register temp1 = r10;
Register temp2 = r11;
Register temp3 = r14; // r13 is live by this point: it contains the sender SP
if (for_compiler_entry) {
assert(receiver_reg == (iid == vmIntrinsics::_linkToStatic ? noreg : j_rarg0), "only valid assignment");
assert_different_registers(temp1, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7);
assert_different_registers(temp2, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7);
assert_different_registers(temp3, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7);
}
assert_different_registers(temp1, temp2, temp3, receiver_reg);
assert_different_registers(temp1, temp2, temp3, member_reg);
if (iid == vmIntrinsics::_invokeBasic) {
// indirect through MH.form.vmentry.vmtarget
jump_to_lambda_form(_masm, receiver_reg, rmethod, temp1, for_compiler_entry);
} else {
// The method is a member invoker used by direct method handles.
if (VerifyMethodHandles) {
// make sure the trailing argument really is a MemberName (caller responsibility)
verify_klass(_masm, member_reg, SystemDictionary::WK_KLASS_ENUM_NAME(java_lang_invoke_MemberName),
"MemberName required for invokeVirtual etc.");
}
Address member_clazz( member_reg, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes()));
Address member_vmindex( member_reg, NONZERO(java_lang_invoke_MemberName::vmindex_offset_in_bytes()));
Address member_vmtarget( member_reg, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes()));
Register temp1_recv_klass = temp1;
if (iid != vmIntrinsics::_linkToStatic) {
__ verify_oop(receiver_reg);
if (iid == vmIntrinsics::_linkToSpecial) {
// Don't actually load the klass; just null-check the receiver.
__ null_check(receiver_reg);
} else {
// load receiver klass itself
__ null_check(receiver_reg, oopDesc::klass_offset_in_bytes());
__ load_klass(temp1_recv_klass, receiver_reg);
__ verify_klass_ptr(temp1_recv_klass);
}
BLOCK_COMMENT("check_receiver {");
// The receiver for the MemberName must be in receiver_reg.
// Check the receiver against the MemberName.clazz
if (VerifyMethodHandles && iid == vmIntrinsics::_linkToSpecial) {
// Did not load it above...
__ load_klass(temp1_recv_klass, receiver_reg);
__ verify_klass_ptr(temp1_recv_klass);
}
if (VerifyMethodHandles && iid != vmIntrinsics::_linkToInterface) {
Label L_ok;
Register temp2_defc = temp2;
__ load_heap_oop(temp2_defc, member_clazz);
load_klass_from_Class(_masm, temp2_defc);
__ verify_klass_ptr(temp2_defc);
__ check_klass_subtype(temp1_recv_klass, temp2_defc, temp3, L_ok);
// If we get here, the type check failed!
__ hlt(0);
// __ STOP("receiver class disagrees with MemberName.clazz");
__ bind(L_ok);
}
BLOCK_COMMENT("} check_receiver");
}
if (iid == vmIntrinsics::_linkToSpecial ||
iid == vmIntrinsics::_linkToStatic) {
DEBUG_ONLY(temp1_recv_klass = noreg); // these guys didn't load the recv_klass
}
// Live registers at this point:
// member_reg - MemberName that was the trailing argument
// temp1_recv_klass - klass of stacked receiver, if needed
// r13 - interpreter linkage (if interpreted) ??? FIXME
// r1 ... r0 - compiler arguments (if compiled)
Label L_incompatible_class_change_error;
switch (iid) {
case vmIntrinsics::_linkToSpecial:
if (VerifyMethodHandles) {
verify_ref_kind(_masm, JVM_REF_invokeSpecial, member_reg, temp3);
}
__ ldr(rmethod, member_vmtarget);
break;
case vmIntrinsics::_linkToStatic:
if (VerifyMethodHandles) {
verify_ref_kind(_masm, JVM_REF_invokeStatic, member_reg, temp3);
}
__ ldr(rmethod, member_vmtarget);
break;
case vmIntrinsics::_linkToVirtual:
{
// same as TemplateTable::invokevirtual,
// minus the CP setup and profiling:
if (VerifyMethodHandles) {
verify_ref_kind(_masm, JVM_REF_invokeVirtual, member_reg, temp3);
}
// pick out the vtable index from the MemberName, and then we can discard it:
Register temp2_index = temp2;
__ ldr(temp2_index, member_vmindex);
if (VerifyMethodHandles) {
Label L_index_ok;
__ cmpw(temp2_index, 0U);
__ br(Assembler::GE, L_index_ok);
__ hlt(0);
__ BIND(L_index_ok);
}
// Note: The verifier invariants allow us to ignore MemberName.clazz and vmtarget
// at this point. And VerifyMethodHandles has already checked clazz, if needed.
// get target Method* & entry point
__ lookup_virtual_method(temp1_recv_klass, temp2_index, rmethod);
break;
}
case vmIntrinsics::_linkToInterface:
{
// same as TemplateTable::invokeinterface
// (minus the CP setup and profiling, with different argument motion)
if (VerifyMethodHandles) {
verify_ref_kind(_masm, JVM_REF_invokeInterface, member_reg, temp3);
}
Register temp3_intf = temp3;
__ load_heap_oop(temp3_intf, member_clazz);
load_klass_from_Class(_masm, temp3_intf);
__ verify_klass_ptr(temp3_intf);
Register rindex = rmethod;
__ ldr(rindex, member_vmindex);
if (VerifyMethodHandles) {
Label L;
__ cmpw(rindex, 0U);
__ br(Assembler::GE, L);
__ hlt(0);
__ bind(L);
}
// given intf, index, and recv klass, dispatch to the implementation method
__ lookup_interface_method(temp1_recv_klass, temp3_intf,
// note: next two args must be the same:
rindex, rmethod,
temp2,
L_incompatible_class_change_error);
break;
}
default:
fatal("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid));
break;
}
// live at this point: rmethod, r13 (if interpreted)
// After figuring out which concrete method to call, jump into it.
// Note that this works in the interpreter with no data motion.
// But the compiled version will require that r2_recv be shifted out.
__ verify_method_ptr(rmethod);
jump_from_method_handle(_masm, rmethod, temp1, for_compiler_entry);
if (iid == vmIntrinsics::_linkToInterface) {
__ bind(L_incompatible_class_change_error);
__ far_jump(RuntimeAddress(StubRoutines::throw_IncompatibleClassChangeError_entry()));
}
}
}
VtableStub* VtableStubs::create_itable_stub(int itable_index) {
const int code_length = VtableStub::pd_code_size_limit(false);
VtableStub* s = new(code_length) VtableStub(false, itable_index);
// Can be NULL if there is no free space in the code cache.
if (s == NULL) {
return NULL;
}
ResourceMark rm;
CodeBuffer cb(s->entry_point(), code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
assert(VtableStub::receiver_location() == R0->as_VMReg(), "receiver expected in R0");
// R0-R3 / R0-R7 registers hold the arguments and cannot be spoiled
const Register Rclass = AARCH64_ONLY(R9) NOT_AARCH64(R4);
const Register Rlength = AARCH64_ONLY(R10) NOT_AARCH64(R5);
const Register Rscan = AARCH64_ONLY(R11) NOT_AARCH64(R6);
const Register tmp = Rtemp;
assert_different_registers(Ricklass, Rclass, Rlength, Rscan, tmp);
// Calculate the start of itable (itable goes after vtable)
const int scale = exact_log2(vtableEntry::size_in_bytes());
address npe_addr = __ pc();
__ load_klass(Rclass, R0);
__ ldr_s32(Rlength, Address(Rclass, Klass::vtable_length_offset()));
__ add(Rscan, Rclass, in_bytes(Klass::vtable_start_offset()));
__ add(Rscan, Rscan, AsmOperand(Rlength, lsl, scale));
// Search through the itable for an interface equal to incoming Ricklass
// itable looks like [intface][offset][intface][offset][intface][offset]
const int entry_size = itableOffsetEntry::size() * HeapWordSize;
assert(itableOffsetEntry::interface_offset_in_bytes() == 0, "not added for convenience");
Label loop;
__ bind(loop);
__ ldr(tmp, Address(Rscan, entry_size, post_indexed));
#ifdef AARCH64
Label found;
__ cmp(tmp, Ricklass);
__ b(found, eq);
__ cbnz(tmp, loop);
#else
__ cmp(tmp, Ricklass); // set ZF and CF if interface is found
__ cmn(tmp, 0, ne); // check if tmp == 0 and clear CF if it is
__ b(loop, ne);
#endif // AARCH64
assert(StubRoutines::throw_IncompatibleClassChangeError_entry() != NULL, "Check initialization order");
#ifdef AARCH64
__ jump(StubRoutines::throw_IncompatibleClassChangeError_entry(), relocInfo::runtime_call_type, tmp);
__ bind(found);
#else
// CF == 0 means we reached the end of itable without finding icklass
__ jump(StubRoutines::throw_IncompatibleClassChangeError_entry(), relocInfo::runtime_call_type, noreg, cc);
#endif // !AARCH64
// Interface found at previous position of Rscan, now load the method oop
__ ldr_s32(tmp, Address(Rscan, itableOffsetEntry::offset_offset_in_bytes() - entry_size));
{
const int method_offset = itableMethodEntry::size() * HeapWordSize * itable_index +
itableMethodEntry::method_offset_in_bytes();
__ add_slow(Rmethod, Rclass, method_offset);
}
__ ldr(Rmethod, Address(Rmethod, tmp));
address ame_addr = __ pc();
#ifdef AARCH64
__ ldr(tmp, Address(Rmethod, Method::from_compiled_offset()));
__ br(tmp);
#else
__ ldr(PC, Address(Rmethod, Method::from_compiled_offset()));
#endif // AARCH64
masm->flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("itable #%d at " PTR_FORMAT "[%d] left over: %d",
itable_index, p2i(s->entry_point()),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
// FIXME ARM: need correct 'slop' - below is x86 code
// shut the door on sizing bugs
//int slop = 8; // 32-bit offset is this much larger than a 13-bit one
//assert(itable_index > 10 || __ pc() + slop <= s->code_end(), "room for 32-bit offset");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
const int aarch64_code_length = VtableStub::pd_code_size_limit(true);
VtableStub* s = new(aarch64_code_length) VtableStub(true, vtable_index);
ResourceMark rm;
CodeBuffer cb(s->entry_point(), aarch64_code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
#ifndef PRODUCT
if (CountCompiledCalls) {
__ lea(r19, ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
__ incrementw(Address(r19));
}
#endif
// get receiver (need to skip return address on top of stack)
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// get receiver klass
address npe_addr = __ pc();
__ load_klass(r19, j_rarg0);
#ifndef PRODUCT
if (DebugVtables) {
Label L;
// check offset vs vtable length
__ ldrw(rscratch1, Address(r19, Klass::vtable_length_offset()));
__ cmpw(rscratch1, vtable_index * vtableEntry::size());
__ br(Assembler::GT, L);
__ enter();
__ mov(r2, vtable_index);
__ call_VM(noreg,
CAST_FROM_FN_PTR(address, bad_compiled_vtable_index), j_rarg0, r2);
__ leave();
__ bind(L);
}
#endif // PRODUCT
__ lookup_virtual_method(r19, vtable_index, rmethod);
if (DebugVtables) {
Label L;
__ cbz(rmethod, L);
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ cbnz(rscratch1, L);
__ stop("Vtable entry is NULL");
__ bind(L);
}
// r0: receiver klass
// rmethod: Method*
// r2: receiver
address ame_addr = __ pc();
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ br(rscratch1);
__ flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("vtable #%d at " PTR_FORMAT "[%d] left over: %d",
vtable_index, p2i(s->entry_point()),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
//------------------------------------------------------------------------------
// MethodHandles::generate_method_handle_stub
//
// Generate an "entry" field for a method handle.
// This determines how the method handle will respond to calls.
void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHandles::EntryKind ek) {
// Here is the register state during an interpreted call,
// as set up by generate_method_handle_interpreter_entry():
// - rbx: garbage temp (was MethodHandle.invoke methodOop, unused)
// - rcx: receiver method handle
// - rax: method handle type (only used by the check_mtype entry point)
// - rsi/r13: sender SP (must preserve; see prepare_to_jump_from_interpreted)
// - rdx: garbage temp, can blow away
const Register rcx_recv = rcx;
const Register rax_argslot = rax;
const Register rbx_temp = rbx;
const Register rdx_temp = rdx;
// This guy is set up by prepare_to_jump_from_interpreted (from interpreted calls)
// and gen_c2i_adapter (from compiled calls):
const Register saved_last_sp = LP64_ONLY(r13) NOT_LP64(rsi);
// Argument registers for _raise_exception.
// 32-bit: Pass first two oop/int args in registers ECX and EDX.
const Register rarg0_code = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
const Register rarg1_actual = LP64_ONLY(j_rarg1) NOT_LP64(rdx);
const Register rarg2_required = LP64_ONLY(j_rarg2) NOT_LP64(rdi);
assert_different_registers(rarg0_code, rarg1_actual, rarg2_required, saved_last_sp);
guarantee(java_lang_invoke_MethodHandle::vmentry_offset_in_bytes() != 0, "must have offsets");
// some handy addresses
Address rbx_method_fie( rbx, methodOopDesc::from_interpreted_offset() );
Address rbx_method_fce( rbx, methodOopDesc::from_compiled_offset() );
Address rcx_mh_vmtarget( rcx_recv, java_lang_invoke_MethodHandle::vmtarget_offset_in_bytes() );
Address rcx_dmh_vmindex( rcx_recv, java_lang_invoke_DirectMethodHandle::vmindex_offset_in_bytes() );
Address rcx_bmh_vmargslot( rcx_recv, java_lang_invoke_BoundMethodHandle::vmargslot_offset_in_bytes() );
Address rcx_bmh_argument( rcx_recv, java_lang_invoke_BoundMethodHandle::argument_offset_in_bytes() );
Address rcx_amh_vmargslot( rcx_recv, java_lang_invoke_AdapterMethodHandle::vmargslot_offset_in_bytes() );
Address rcx_amh_argument( rcx_recv, java_lang_invoke_AdapterMethodHandle::argument_offset_in_bytes() );
Address rcx_amh_conversion( rcx_recv, java_lang_invoke_AdapterMethodHandle::conversion_offset_in_bytes() );
Address vmarg; // __ argument_address(vmargslot)
const int java_mirror_offset = klassOopDesc::klass_part_offset_in_bytes() + Klass::java_mirror_offset_in_bytes();
if (have_entry(ek)) {
__ nop(); // empty stubs make SG sick
return;
}
address interp_entry = __ pc();
trace_method_handle(_masm, entry_name(ek));
BLOCK_COMMENT(entry_name(ek));
switch ((int) ek) {
case _raise_exception:
{
// Not a real MH entry, but rather shared code for raising an
// exception. Since we use the compiled entry, arguments are
// expected in compiler argument registers.
assert(raise_exception_method(), "must be set");
assert(raise_exception_method()->from_compiled_entry(), "method must be linked");
const Register rdi_pc = rax;
__ pop(rdi_pc); // caller PC
__ mov(rsp, saved_last_sp); // cut the stack back to where the caller started
Register rbx_method = rbx_temp;
Label L_no_method;
// FIXME: fill in _raise_exception_method with a suitable java.lang.invoke method
__ movptr(rbx_method, ExternalAddress((address) &_raise_exception_method));
__ testptr(rbx_method, rbx_method);
__ jccb(Assembler::zero, L_no_method);
const int jobject_oop_offset = 0;
__ movptr(rbx_method, Address(rbx_method, jobject_oop_offset)); // dereference the jobject
__ testptr(rbx_method, rbx_method);
__ jccb(Assembler::zero, L_no_method);
__ verify_oop(rbx_method);
NOT_LP64(__ push(rarg2_required));
__ push(rdi_pc); // restore caller PC
__ jmp(rbx_method_fce); // jump to compiled entry
// Do something that is at least causes a valid throw from the interpreter.
__ bind(L_no_method);
__ push(rarg2_required);
__ push(rarg1_actual);
__ jump(ExternalAddress(Interpreter::throw_WrongMethodType_entry()));
}
break;
case _invokestatic_mh:
case _invokespecial_mh:
{
Register rbx_method = rbx_temp;
__ load_heap_oop(rbx_method, rcx_mh_vmtarget); // target is a methodOop
__ verify_oop(rbx_method);
// same as TemplateTable::invokestatic or invokespecial,
// minus the CP setup and profiling:
if (ek == _invokespecial_mh) {
// Must load & check the first argument before entering the target method.
__ load_method_handle_vmslots(rax_argslot, rcx_recv, rdx_temp);
__ movptr(rcx_recv, __ argument_address(rax_argslot, -1));
__ null_check(rcx_recv);
__ verify_oop(rcx_recv);
}
__ jmp(rbx_method_fie);
}
break;
case _invokevirtual_mh:
{
// same as TemplateTable::invokevirtual,
// minus the CP setup and profiling:
// pick out the vtable index and receiver offset from the MH,
// and then we can discard it:
__ load_method_handle_vmslots(rax_argslot, rcx_recv, rdx_temp);
Register rbx_index = rbx_temp;
__ movl(rbx_index, rcx_dmh_vmindex);
// Note: The verifier allows us to ignore rcx_mh_vmtarget.
__ movptr(rcx_recv, __ argument_address(rax_argslot, -1));
__ null_check(rcx_recv, oopDesc::klass_offset_in_bytes());
// get receiver klass
Register rax_klass = rax_argslot;
__ load_klass(rax_klass, rcx_recv);
__ verify_oop(rax_klass);
// get target methodOop & entry point
const int base = instanceKlass::vtable_start_offset() * wordSize;
assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
Address vtable_entry_addr(rax_klass,
rbx_index, Address::times_ptr,
base + vtableEntry::method_offset_in_bytes());
Register rbx_method = rbx_temp;
__ movptr(rbx_method, vtable_entry_addr);
__ verify_oop(rbx_method);
__ jmp(rbx_method_fie);
}
break;
case _invokeinterface_mh:
{
// same as TemplateTable::invokeinterface,
// minus the CP setup and profiling:
// pick out the interface and itable index from the MH.
__ load_method_handle_vmslots(rax_argslot, rcx_recv, rdx_temp);
Register rdx_intf = rdx_temp;
Register rbx_index = rbx_temp;
__ load_heap_oop(rdx_intf, rcx_mh_vmtarget);
__ movl(rbx_index, rcx_dmh_vmindex);
__ movptr(rcx_recv, __ argument_address(rax_argslot, -1));
__ null_check(rcx_recv, oopDesc::klass_offset_in_bytes());
// get receiver klass
Register rax_klass = rax_argslot;
__ load_klass(rax_klass, rcx_recv);
__ verify_oop(rax_klass);
Register rdi_temp = rdi;
Register rbx_method = rbx_index;
// get interface klass
Label no_such_interface;
__ verify_oop(rdx_intf);
__ lookup_interface_method(rax_klass, rdx_intf,
// note: next two args must be the same:
rbx_index, rbx_method,
rdi_temp,
no_such_interface);
__ verify_oop(rbx_method);
__ jmp(rbx_method_fie);
__ hlt();
__ bind(no_such_interface);
// Throw an exception.
// For historical reasons, it will be IncompatibleClassChangeError.
__ mov(rbx_temp, rcx_recv); // rarg2_required might be RCX
assert_different_registers(rarg2_required, rbx_temp);
__ movptr(rarg2_required, Address(rdx_intf, java_mirror_offset)); // required interface
__ mov( rarg1_actual, rbx_temp); // bad receiver
__ movl( rarg0_code, (int) Bytecodes::_invokeinterface); // who is complaining?
__ jump(ExternalAddress(from_interpreted_entry(_raise_exception)));
}
break;
case _bound_ref_mh:
case _bound_int_mh:
case _bound_long_mh:
case _bound_ref_direct_mh:
case _bound_int_direct_mh:
case _bound_long_direct_mh:
{
bool direct_to_method = (ek >= _bound_ref_direct_mh);
BasicType arg_type = T_ILLEGAL;
int arg_mask = _INSERT_NO_MASK;
int arg_slots = -1;
get_ek_bound_mh_info(ek, arg_type, arg_mask, arg_slots);
// make room for the new argument:
__ movl(rax_argslot, rcx_bmh_vmargslot);
__ lea(rax_argslot, __ argument_address(rax_argslot));
insert_arg_slots(_masm, arg_slots * stack_move_unit(), arg_mask, rax_argslot, rbx_temp, rdx_temp);
// store bound argument into the new stack slot:
__ load_heap_oop(rbx_temp, rcx_bmh_argument);
if (arg_type == T_OBJECT) {
__ movptr(Address(rax_argslot, 0), rbx_temp);
} else {
Address prim_value_addr(rbx_temp, java_lang_boxing_object::value_offset_in_bytes(arg_type));
const int arg_size = type2aelembytes(arg_type);
__ load_sized_value(rdx_temp, prim_value_addr, arg_size, is_signed_subword_type(arg_type), rbx_temp);
__ store_sized_value(Address(rax_argslot, 0), rdx_temp, arg_size, rbx_temp);
}
if (direct_to_method) {
Register rbx_method = rbx_temp;
__ load_heap_oop(rbx_method, rcx_mh_vmtarget);
__ verify_oop(rbx_method);
__ jmp(rbx_method_fie);
} else {
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
__ verify_oop(rcx_recv);
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
}
}
break;
case _adapter_retype_only:
case _adapter_retype_raw:
// immediately jump to the next MH layer:
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
__ verify_oop(rcx_recv);
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
// This is OK when all parameter types widen.
// It is also OK when a return type narrows.
break;
case _adapter_check_cast:
{
// temps:
Register rbx_klass = rbx_temp; // interesting AMH data
// check a reference argument before jumping to the next layer of MH:
__ movl(rax_argslot, rcx_amh_vmargslot);
vmarg = __ argument_address(rax_argslot);
// What class are we casting to?
__ load_heap_oop(rbx_klass, rcx_amh_argument); // this is a Class object!
__ load_heap_oop(rbx_klass, Address(rbx_klass, java_lang_Class::klass_offset_in_bytes()));
Label done;
__ movptr(rdx_temp, vmarg);
__ testptr(rdx_temp, rdx_temp);
__ jcc(Assembler::zero, done); // no cast if null
__ load_klass(rdx_temp, rdx_temp);
// live at this point:
// - rbx_klass: klass required by the target method
// - rdx_temp: argument klass to test
// - rcx_recv: adapter method handle
__ check_klass_subtype(rdx_temp, rbx_klass, rax_argslot, done);
// If we get here, the type check failed!
// Call the wrong_method_type stub, passing the failing argument type in rax.
Register rax_mtype = rax_argslot;
__ movl(rax_argslot, rcx_amh_vmargslot); // reload argslot field
__ movptr(rdx_temp, vmarg);
assert_different_registers(rarg2_required, rdx_temp);
__ load_heap_oop(rarg2_required, rcx_amh_argument); // required class
__ mov( rarg1_actual, rdx_temp); // bad object
__ movl( rarg0_code, (int) Bytecodes::_checkcast); // who is complaining?
__ jump(ExternalAddress(from_interpreted_entry(_raise_exception)));
__ bind(done);
// get the new MH:
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
}
break;
case _adapter_prim_to_prim:
case _adapter_ref_to_prim:
// handled completely by optimized cases
__ stop("init_AdapterMethodHandle should not issue this");
break;
case _adapter_opt_i2i: // optimized subcase of adapt_prim_to_prim
//case _adapter_opt_f2i: // optimized subcase of adapt_prim_to_prim
case _adapter_opt_l2i: // optimized subcase of adapt_prim_to_prim
case _adapter_opt_unboxi: // optimized subcase of adapt_ref_to_prim
{
// perform an in-place conversion to int or an int subword
__ movl(rax_argslot, rcx_amh_vmargslot);
vmarg = __ argument_address(rax_argslot);
switch (ek) {
case _adapter_opt_i2i:
__ movl(rdx_temp, vmarg);
break;
case _adapter_opt_l2i:
{
// just delete the extra slot; on a little-endian machine we keep the first
__ lea(rax_argslot, __ argument_address(rax_argslot, 1));
remove_arg_slots(_masm, -stack_move_unit(),
rax_argslot, rbx_temp, rdx_temp);
vmarg = Address(rax_argslot, -Interpreter::stackElementSize);
__ movl(rdx_temp, vmarg);
}
break;
case _adapter_opt_unboxi:
{
// Load the value up from the heap.
__ movptr(rdx_temp, vmarg);
int value_offset = java_lang_boxing_object::value_offset_in_bytes(T_INT);
#ifdef ASSERT
for (int bt = T_BOOLEAN; bt < T_INT; bt++) {
if (is_subword_type(BasicType(bt)))
assert(value_offset == java_lang_boxing_object::value_offset_in_bytes(BasicType(bt)), "");
}
#endif
__ null_check(rdx_temp, value_offset);
__ movl(rdx_temp, Address(rdx_temp, value_offset));
// We load this as a word. Because we are little-endian,
// the low bits will be correct, but the high bits may need cleaning.
// The vminfo will guide us to clean those bits.
}
break;
default:
ShouldNotReachHere();
}
// Do the requested conversion and store the value.
Register rbx_vminfo = rbx_temp;
__ movl(rbx_vminfo, rcx_amh_conversion);
assert(CONV_VMINFO_SHIFT == 0, "preshifted");
// get the new MH:
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
// (now we are done with the old MH)
// original 32-bit vmdata word must be of this form:
// | MBZ:6 | signBitCount:8 | srcDstTypes:8 | conversionOp:8 |
__ xchgptr(rcx, rbx_vminfo); // free rcx for shifts
__ shll(rdx_temp /*, rcx*/);
Label zero_extend, done;
__ testl(rcx, CONV_VMINFO_SIGN_FLAG);
__ jccb(Assembler::zero, zero_extend);
// this path is taken for int->byte, int->short
__ sarl(rdx_temp /*, rcx*/);
__ jmpb(done);
__ bind(zero_extend);
// this is taken for int->char
__ shrl(rdx_temp /*, rcx*/);
__ bind(done);
__ movl(vmarg, rdx_temp); // Store the value.
__ xchgptr(rcx, rbx_vminfo); // restore rcx_recv
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
}
break;
case _adapter_opt_i2l: // optimized subcase of adapt_prim_to_prim
case _adapter_opt_unboxl: // optimized subcase of adapt_ref_to_prim
{
// perform an in-place int-to-long or ref-to-long conversion
__ movl(rax_argslot, rcx_amh_vmargslot);
// on a little-endian machine we keep the first slot and add another after
__ lea(rax_argslot, __ argument_address(rax_argslot, 1));
insert_arg_slots(_masm, stack_move_unit(), _INSERT_INT_MASK,
rax_argslot, rbx_temp, rdx_temp);
Address vmarg1(rax_argslot, -Interpreter::stackElementSize);
Address vmarg2 = vmarg1.plus_disp(Interpreter::stackElementSize);
switch (ek) {
case _adapter_opt_i2l:
{
#ifdef _LP64
__ movslq(rdx_temp, vmarg1); // Load sign-extended
__ movq(vmarg1, rdx_temp); // Store into first slot
#else
__ movl(rdx_temp, vmarg1);
__ sarl(rdx_temp, BitsPerInt - 1); // __ extend_sign()
__ movl(vmarg2, rdx_temp); // store second word
#endif
}
break;
case _adapter_opt_unboxl:
{
// Load the value up from the heap.
__ movptr(rdx_temp, vmarg1);
int value_offset = java_lang_boxing_object::value_offset_in_bytes(T_LONG);
assert(value_offset == java_lang_boxing_object::value_offset_in_bytes(T_DOUBLE), "");
__ null_check(rdx_temp, value_offset);
#ifdef _LP64
__ movq(rbx_temp, Address(rdx_temp, value_offset));
__ movq(vmarg1, rbx_temp);
#else
__ movl(rbx_temp, Address(rdx_temp, value_offset + 0*BytesPerInt));
__ movl(rdx_temp, Address(rdx_temp, value_offset + 1*BytesPerInt));
__ movl(vmarg1, rbx_temp);
__ movl(vmarg2, rdx_temp);
#endif
}
break;
default:
ShouldNotReachHere();
}
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
}
break;
case _adapter_opt_f2d: // optimized subcase of adapt_prim_to_prim
case _adapter_opt_d2f: // optimized subcase of adapt_prim_to_prim
{
// perform an in-place floating primitive conversion
__ movl(rax_argslot, rcx_amh_vmargslot);
__ lea(rax_argslot, __ argument_address(rax_argslot, 1));
if (ek == _adapter_opt_f2d) {
insert_arg_slots(_masm, stack_move_unit(), _INSERT_INT_MASK,
rax_argslot, rbx_temp, rdx_temp);
}
Address vmarg(rax_argslot, -Interpreter::stackElementSize);
#ifdef _LP64
if (ek == _adapter_opt_f2d) {
__ movflt(xmm0, vmarg);
__ cvtss2sd(xmm0, xmm0);
__ movdbl(vmarg, xmm0);
} else {
__ movdbl(xmm0, vmarg);
__ cvtsd2ss(xmm0, xmm0);
__ movflt(vmarg, xmm0);
}
#else //_LP64
if (ek == _adapter_opt_f2d) {
__ fld_s(vmarg); // load float to ST0
__ fstp_s(vmarg); // store single
} else {
__ fld_d(vmarg); // load double to ST0
__ fstp_s(vmarg); // store single
}
#endif //_LP64
if (ek == _adapter_opt_d2f) {
remove_arg_slots(_masm, -stack_move_unit(),
rax_argslot, rbx_temp, rdx_temp);
}
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
}
break;
case _adapter_prim_to_ref:
__ unimplemented(entry_name(ek)); // %%% FIXME: NYI
break;
case _adapter_swap_args:
case _adapter_rot_args:
// handled completely by optimized cases
__ stop("init_AdapterMethodHandle should not issue this");
break;
case _adapter_opt_swap_1:
case _adapter_opt_swap_2:
case _adapter_opt_rot_1_up:
case _adapter_opt_rot_1_down:
case _adapter_opt_rot_2_up:
case _adapter_opt_rot_2_down:
{
int swap_bytes = 0, rotate = 0;
get_ek_adapter_opt_swap_rot_info(ek, swap_bytes, rotate);
// 'argslot' is the position of the first argument to swap
__ movl(rax_argslot, rcx_amh_vmargslot);
__ lea(rax_argslot, __ argument_address(rax_argslot));
// 'vminfo' is the second
Register rbx_destslot = rbx_temp;
__ movl(rbx_destslot, rcx_amh_conversion);
assert(CONV_VMINFO_SHIFT == 0, "preshifted");
__ andl(rbx_destslot, CONV_VMINFO_MASK);
__ lea(rbx_destslot, __ argument_address(rbx_destslot));
DEBUG_ONLY(verify_argslot(_masm, rbx_destslot, "swap point must fall within current frame"));
if (!rotate) {
for (int i = 0; i < swap_bytes; i += wordSize) {
__ movptr(rdx_temp, Address(rax_argslot , i));
__ push(rdx_temp);
__ movptr(rdx_temp, Address(rbx_destslot, i));
__ movptr(Address(rax_argslot, i), rdx_temp);
__ pop(rdx_temp);
__ movptr(Address(rbx_destslot, i), rdx_temp);
}
} else {
// push the first chunk, which is going to get overwritten
for (int i = swap_bytes; (i -= wordSize) >= 0; ) {
__ movptr(rdx_temp, Address(rax_argslot, i));
__ push(rdx_temp);
}
if (rotate > 0) {
// rotate upward
__ subptr(rax_argslot, swap_bytes);
#ifdef ASSERT
{
// Verify that argslot > destslot, by at least swap_bytes.
Label L_ok;
__ cmpptr(rax_argslot, rbx_destslot);
__ jccb(Assembler::aboveEqual, L_ok);
__ stop("source must be above destination (upward rotation)");
__ bind(L_ok);
}
#endif
// work argslot down to destslot, copying contiguous data upwards
// pseudo-code:
// rax = src_addr - swap_bytes
// rbx = dest_addr
// while (rax >= rbx) *(rax + swap_bytes) = *(rax + 0), rax--;
Label loop;
__ bind(loop);
__ movptr(rdx_temp, Address(rax_argslot, 0));
__ movptr(Address(rax_argslot, swap_bytes), rdx_temp);
__ addptr(rax_argslot, -wordSize);
__ cmpptr(rax_argslot, rbx_destslot);
__ jccb(Assembler::aboveEqual, loop);
} else {
__ addptr(rax_argslot, swap_bytes);
#ifdef ASSERT
{
// Verify that argslot < destslot, by at least swap_bytes.
Label L_ok;
__ cmpptr(rax_argslot, rbx_destslot);
__ jccb(Assembler::belowEqual, L_ok);
__ stop("source must be below destination (downward rotation)");
__ bind(L_ok);
}
#endif
// work argslot up to destslot, copying contiguous data downwards
// pseudo-code:
// rax = src_addr + swap_bytes
// rbx = dest_addr
// while (rax <= rbx) *(rax - swap_bytes) = *(rax + 0), rax++;
Label loop;
__ bind(loop);
__ movptr(rdx_temp, Address(rax_argslot, 0));
__ movptr(Address(rax_argslot, -swap_bytes), rdx_temp);
__ addptr(rax_argslot, wordSize);
__ cmpptr(rax_argslot, rbx_destslot);
__ jccb(Assembler::belowEqual, loop);
}
// pop the original first chunk into the destination slot, now free
for (int i = 0; i < swap_bytes; i += wordSize) {
__ pop(rdx_temp);
__ movptr(Address(rbx_destslot, i), rdx_temp);
}
}
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
}
break;
case _adapter_dup_args:
{
// 'argslot' is the position of the first argument to duplicate
__ movl(rax_argslot, rcx_amh_vmargslot);
__ lea(rax_argslot, __ argument_address(rax_argslot));
// 'stack_move' is negative number of words to duplicate
Register rdx_stack_move = rdx_temp;
__ movl2ptr(rdx_stack_move, rcx_amh_conversion);
__ sarptr(rdx_stack_move, CONV_STACK_MOVE_SHIFT);
int argslot0_num = 0;
Address argslot0 = __ argument_address(RegisterOrConstant(argslot0_num));
assert(argslot0.base() == rsp, "");
int pre_arg_size = argslot0.disp();
assert(pre_arg_size % wordSize == 0, "");
assert(pre_arg_size > 0, "must include PC");
// remember the old rsp+1 (argslot[0])
Register rbx_oldarg = rbx_temp;
__ lea(rbx_oldarg, argslot0);
// move rsp down to make room for dups
__ lea(rsp, Address(rsp, rdx_stack_move, Address::times_ptr));
// compute the new rsp+1 (argslot[0])
Register rdx_newarg = rdx_temp;
__ lea(rdx_newarg, argslot0);
__ push(rdi); // need a temp
// (preceding push must be done after arg addresses are taken!)
// pull down the pre_arg_size data (PC)
for (int i = -pre_arg_size; i < 0; i += wordSize) {
__ movptr(rdi, Address(rbx_oldarg, i));
__ movptr(Address(rdx_newarg, i), rdi);
}
// copy from rax_argslot[0...] down to new_rsp[1...]
// pseudo-code:
// rbx = old_rsp+1
// rdx = new_rsp+1
// rax = argslot
// while (rdx < rbx) *rdx++ = *rax++
Label loop;
__ bind(loop);
__ movptr(rdi, Address(rax_argslot, 0));
__ movptr(Address(rdx_newarg, 0), rdi);
__ addptr(rax_argslot, wordSize);
__ addptr(rdx_newarg, wordSize);
__ cmpptr(rdx_newarg, rbx_oldarg);
__ jccb(Assembler::less, loop);
__ pop(rdi); // restore temp
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
}
break;
case _adapter_drop_args:
{
// 'argslot' is the position of the first argument to nuke
__ movl(rax_argslot, rcx_amh_vmargslot);
__ lea(rax_argslot, __ argument_address(rax_argslot));
__ push(rdi); // need a temp
// (must do previous push after argslot address is taken)
// 'stack_move' is number of words to drop
Register rdi_stack_move = rdi;
__ movl2ptr(rdi_stack_move, rcx_amh_conversion);
__ sarptr(rdi_stack_move, CONV_STACK_MOVE_SHIFT);
remove_arg_slots(_masm, rdi_stack_move,
rax_argslot, rbx_temp, rdx_temp);
__ pop(rdi); // restore temp
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
}
break;
case _adapter_collect_args:
__ unimplemented(entry_name(ek)); // %%% FIXME: NYI
break;
case _adapter_spread_args:
// handled completely by optimized cases
__ stop("init_AdapterMethodHandle should not issue this");
break;
case _adapter_opt_spread_0:
case _adapter_opt_spread_1:
case _adapter_opt_spread_more:
{
// spread an array out into a group of arguments
int length_constant = get_ek_adapter_opt_spread_info(ek);
// find the address of the array argument
__ movl(rax_argslot, rcx_amh_vmargslot);
__ lea(rax_argslot, __ argument_address(rax_argslot));
// grab some temps
{ __ push(rsi); __ push(rdi); }
// (preceding pushes must be done after argslot address is taken!)
#define UNPUSH_RSI_RDI \
{ __ pop(rdi); __ pop(rsi); }
// arx_argslot points both to the array and to the first output arg
vmarg = Address(rax_argslot, 0);
// Get the array value.
Register rsi_array = rsi;
Register rdx_array_klass = rdx_temp;
BasicType elem_type = T_OBJECT;
int length_offset = arrayOopDesc::length_offset_in_bytes();
int elem0_offset = arrayOopDesc::base_offset_in_bytes(elem_type);
__ movptr(rsi_array, vmarg);
Label skip_array_check;
if (length_constant == 0) {
__ testptr(rsi_array, rsi_array);
__ jcc(Assembler::zero, skip_array_check);
}
__ null_check(rsi_array, oopDesc::klass_offset_in_bytes());
__ load_klass(rdx_array_klass, rsi_array);
// Check the array type.
Register rbx_klass = rbx_temp;
__ load_heap_oop(rbx_klass, rcx_amh_argument); // this is a Class object!
__ load_heap_oop(rbx_klass, Address(rbx_klass, java_lang_Class::klass_offset_in_bytes()));
Label ok_array_klass, bad_array_klass, bad_array_length;
__ check_klass_subtype(rdx_array_klass, rbx_klass, rdi, ok_array_klass);
// If we get here, the type check failed!
__ jmp(bad_array_klass);
__ bind(ok_array_klass);
// Check length.
if (length_constant >= 0) {
__ cmpl(Address(rsi_array, length_offset), length_constant);
} else {
Register rbx_vminfo = rbx_temp;
__ movl(rbx_vminfo, rcx_amh_conversion);
assert(CONV_VMINFO_SHIFT == 0, "preshifted");
__ andl(rbx_vminfo, CONV_VMINFO_MASK);
__ cmpl(rbx_vminfo, Address(rsi_array, length_offset));
}
__ jcc(Assembler::notEqual, bad_array_length);
Register rdx_argslot_limit = rdx_temp;
// Array length checks out. Now insert any required stack slots.
if (length_constant == -1) {
// Form a pointer to the end of the affected region.
__ lea(rdx_argslot_limit, Address(rax_argslot, Interpreter::stackElementSize));
// 'stack_move' is negative number of words to insert
Register rdi_stack_move = rdi;
__ movl2ptr(rdi_stack_move, rcx_amh_conversion);
__ sarptr(rdi_stack_move, CONV_STACK_MOVE_SHIFT);
Register rsi_temp = rsi_array; // spill this
insert_arg_slots(_masm, rdi_stack_move, -1,
rax_argslot, rbx_temp, rsi_temp);
// reload the array (since rsi was killed)
__ movptr(rsi_array, vmarg);
} else if (length_constant > 1) {
int arg_mask = 0;
int new_slots = (length_constant - 1);
for (int i = 0; i < new_slots; i++) {
arg_mask <<= 1;
arg_mask |= _INSERT_REF_MASK;
}
insert_arg_slots(_masm, new_slots * stack_move_unit(), arg_mask,
rax_argslot, rbx_temp, rdx_temp);
} else if (length_constant == 1) {
// no stack resizing required
} else if (length_constant == 0) {
remove_arg_slots(_masm, -stack_move_unit(),
rax_argslot, rbx_temp, rdx_temp);
}
// Copy from the array to the new slots.
// Note: Stack change code preserves integrity of rax_argslot pointer.
// So even after slot insertions, rax_argslot still points to first argument.
if (length_constant == -1) {
// [rax_argslot, rdx_argslot_limit) is the area we are inserting into.
Register rsi_source = rsi_array;
__ lea(rsi_source, Address(rsi_array, elem0_offset));
Label loop;
__ bind(loop);
__ movptr(rbx_temp, Address(rsi_source, 0));
__ movptr(Address(rax_argslot, 0), rbx_temp);
__ addptr(rsi_source, type2aelembytes(elem_type));
__ addptr(rax_argslot, Interpreter::stackElementSize);
__ cmpptr(rax_argslot, rdx_argslot_limit);
__ jccb(Assembler::less, loop);
} else if (length_constant == 0) {
__ bind(skip_array_check);
// nothing to copy
} else {
int elem_offset = elem0_offset;
int slot_offset = 0;
for (int index = 0; index < length_constant; index++) {
__ movptr(rbx_temp, Address(rsi_array, elem_offset));
__ movptr(Address(rax_argslot, slot_offset), rbx_temp);
elem_offset += type2aelembytes(elem_type);
slot_offset += Interpreter::stackElementSize;
}
}
// Arguments are spread. Move to next method handle.
UNPUSH_RSI_RDI;
__ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
__ jump_to_method_handle_entry(rcx_recv, rdx_temp);
__ bind(bad_array_klass);
UNPUSH_RSI_RDI;
assert(!vmarg.uses(rarg2_required), "must be different registers");
__ movptr(rarg2_required, Address(rdx_array_klass, java_mirror_offset)); // required type
__ movptr(rarg1_actual, vmarg); // bad array
__ movl( rarg0_code, (int) Bytecodes::_aaload); // who is complaining?
__ jump(ExternalAddress(from_interpreted_entry(_raise_exception)));
__ bind(bad_array_length);
UNPUSH_RSI_RDI;
assert(!vmarg.uses(rarg2_required), "must be different registers");
__ mov (rarg2_required, rcx_recv); // AMH requiring a certain length
__ movptr(rarg1_actual, vmarg); // bad array
__ movl( rarg0_code, (int) Bytecodes::_arraylength); // who is complaining?
__ jump(ExternalAddress(from_interpreted_entry(_raise_exception)));
#undef UNPUSH_RSI_RDI
}
break;
case _adapter_flyby:
case _adapter_ricochet:
__ unimplemented(entry_name(ek)); // %%% FIXME: NYI
break;
default: ShouldNotReachHere();
}
__ hlt();
address me_cookie = MethodHandleEntry::start_compiled_entry(_masm, interp_entry);
__ unimplemented(entry_name(ek)); // %%% FIXME: NYI
init_entry(ek, MethodHandleEntry::finish_compiled_entry(_masm, me_cookie));
}
