void MethodHandles::jump_from_method_handle(MacroAssembler* _masm, Register method, Register temp,
bool for_compiler_entry) {
assert(method == rmethod, "interpreter calling convention");
Label L_no_such_method;
__ cbz(rmethod, L_no_such_method);
__ verify_method_ptr(method);
if (!for_compiler_entry && JvmtiExport::can_post_interpreter_events()) {
Label run_compiled_code;
// JVMTI events, such as single-stepping, are implemented partly by avoiding running
// compiled code in threads for which the event is enabled. Check here for
// interp_only_mode if these events CAN be enabled.
__ ldrb(rscratch1, Address(rthread, JavaThread::interp_only_mode_offset()));
__ cbnz(rscratch1, run_compiled_code);
__ ldr(rscratch1, Address(method, Method::interpreter_entry_offset()));
__ br(rscratch1);
__ BIND(run_compiled_code);
}
const ByteSize entry_offset = for_compiler_entry ? Method::from_compiled_offset() :
Method::from_interpreted_offset();
__ ldr(rscratch1,Address(method, entry_offset));
__ br(rscratch1);
__ bind(L_no_such_method);
__ far_jump(RuntimeAddress(StubRoutines::throw_AbstractMethodError_entry()));
}
void MethodHandles::verify_klass(MacroAssembler* _masm,
Register obj, SystemDictionary::WKID klass_id,
const char* error_message) {
InstanceKlass** klass_addr = SystemDictionary::well_known_klass_addr(klass_id);
KlassHandle klass = SystemDictionary::well_known_klass(klass_id);
Register temp = rscratch2;
Register temp2 = rscratch1; // used by MacroAssembler::cmpptr
Label L_ok, L_bad;
BLOCK_COMMENT("verify_klass {");
__ verify_oop(obj);
__ cbz(obj, L_bad);
__ push(RegSet::of(temp, temp2), sp);
__ load_klass(temp, obj);
__ cmpptr(temp, ExternalAddress((address) klass_addr));
__ br(Assembler::EQ, L_ok);
intptr_t super_check_offset = klass->super_check_offset();
__ ldr(temp, Address(temp, super_check_offset));
__ cmpptr(temp, ExternalAddress((address) klass_addr));
__ br(Assembler::EQ, L_ok);
__ pop(RegSet::of(temp, temp2), sp);
__ bind(L_bad);
__ stop(error_message);
__ BIND(L_ok);
__ pop(RegSet::of(temp, temp2), sp);
BLOCK_COMMENT("} verify_klass");
}
void Assembler::cbz(const Register& rt, Label* label) {
// Flush the instruction buffer if necessary before getting an offset.
BufferOffset branch = b(0);
Instruction* ins = getInstructionAt(branch);
// Encode the relative offset.
cbz(ins, rt, LinkAndGetInstructionOffsetTo(branch, label));
}
void G1PostBarrierStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
assert(addr()->is_register(), "Precondition.");
assert(new_val()->is_register(), "Precondition.");
Register new_val_reg = new_val()->as_register();
__ cbz(new_val_reg, _continuation);
ce->store_parameter(addr()->as_pointer_register(), 0);
__ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::g1_post_barrier_slow_id)));
__ b(_continuation);
}
void G1PreBarrierStub::emit_code(LIR_Assembler* ce) {
// At this point we know that marking is in progress.
// If do_load() is true then we have to emit the
// load of the previous value; otherwise it has already
// been loaded into _pre_val.
__ bind(_entry);
assert(pre_val()->is_register(), "Precondition.");
Register pre_val_reg = pre_val()->as_register();
if (do_load()) {
ce->mem2reg(addr(), pre_val(), T_OBJECT, patch_code(), info(), false /*wide*/, false /*unaligned*/);
}
__ cbz(pre_val_reg, _continuation);
ce->store_parameter(pre_val()->as_register(), 0);
__ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::g1_pre_barrier_slow_id)));
__ b(_continuation);
}
VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
const int aarch64_code_length = VtableStub::pd_code_size_limit(true);
VtableStub* s = new(aarch64_code_length) VtableStub(true, vtable_index);
ResourceMark rm;
CodeBuffer cb(s->entry_point(), aarch64_code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
#ifndef PRODUCT
if (CountCompiledCalls) {
__ lea(r19, ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
__ incrementw(Address(r19));
}
#endif
// get receiver (need to skip return address on top of stack)
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// get receiver klass
address npe_addr = __ pc();
__ load_klass(r19, j_rarg0);
#ifndef PRODUCT
if (DebugVtables) {
Label L;
// check offset vs vtable length
__ ldrw(rscratch1, Address(r19, Klass::vtable_length_offset()));
__ cmpw(rscratch1, vtable_index * vtableEntry::size());
__ br(Assembler::GT, L);
__ enter();
__ mov(r2, vtable_index);
__ call_VM(noreg,
CAST_FROM_FN_PTR(address, bad_compiled_vtable_index), j_rarg0, r2);
__ leave();
__ bind(L);
}
#endif // PRODUCT
__ lookup_virtual_method(r19, vtable_index, rmethod);
if (DebugVtables) {
Label L;
__ cbz(rmethod, L);
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ cbnz(rscratch1, L);
__ stop("Vtable entry is NULL");
__ bind(L);
}
// r0: receiver klass
// rmethod: Method*
// r2: receiver
address ame_addr = __ pc();
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ br(rscratch1);
__ flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("vtable #%d at " PTR_FORMAT "[%d] left over: %d",
vtable_index, p2i(s->entry_point()),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
VtableStub* VtableStubs::create_itable_stub(int itable_index) {
// Note well: pd_code_size_limit is the absolute minimum we can get
// away with. If you add code here, bump the code stub size
// returned by pd_code_size_limit!
const int code_length = VtableStub::pd_code_size_limit(false);
VtableStub* s = new(code_length) VtableStub(false, itable_index);
ResourceMark rm;
CodeBuffer cb(s->entry_point(), code_length);
MacroAssembler* masm = new MacroAssembler(&cb);
#ifndef PRODUCT
if (CountCompiledCalls) {
__ lea(r10, ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
__ incrementw(Address(r10));
}
#endif
// Entry arguments:
// rscratch2: Interface
// j_rarg0: Receiver
// Free registers (non-args) are r0 (interface), rmethod
// get receiver (need to skip return address on top of stack)
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// get receiver klass (also an implicit null-check)
address npe_addr = __ pc();
// Most registers are in use; we'll use r0, rmethod, r10, r11
__ load_klass(r10, j_rarg0);
Label throw_icce;
// Get Method* and entrypoint for compiler
__ lookup_interface_method(// inputs: rec. class, interface, itable index
r10, rscratch2, itable_index,
// outputs: method, scan temp. reg
rmethod, r11,
throw_icce);
// method (rmethod): Method*
// j_rarg0: receiver
#ifdef ASSERT
if (DebugVtables) {
Label L2;
__ cbz(rmethod, L2);
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ cbnz(rscratch1, L2);
__ stop("compiler entrypoint is null");
__ bind(L2);
}
#endif // ASSERT
// rmethod: Method*
// j_rarg0: receiver
address ame_addr = __ pc();
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ br(rscratch1);
__ bind(throw_icce);
__ far_jump(RuntimeAddress(StubRoutines::throw_IncompatibleClassChangeError_entry()));
__ flush();
if (PrintMiscellaneous && (WizardMode || Verbose)) {
tty->print_cr("itable #%d at " PTR_FORMAT "[%d] left over: %d",
itable_index, p2i(s->entry_point()),
(int)(s->code_end() - s->entry_point()),
(int)(s->code_end() - __ pc()));
}
guarantee(__ pc() <= s->code_end(), "overflowed buffer");
s->set_exception_points(npe_addr, ame_addr);
return s;
}
static void SubstrateHookFunctionThumb(SubstrateProcessRef process, void *symbol, void *replace, void **result) {
if (symbol == NULL)
return;
uint16_t *area(reinterpret_cast<uint16_t *>(symbol));
unsigned align((reinterpret_cast<uintptr_t>(area) & 0x2) == 0 ? 0 : 1);
uint16_t *thumb(area + align);
uint32_t *arm(reinterpret_cast<uint32_t *>(thumb + 2));
uint16_t *trail(reinterpret_cast<uint16_t *>(arm + 2));
if (
(align == 0 || area[0] == T$nop) &&
thumb[0] == T$bx(A$pc) &&
thumb[1] == T$nop &&
arm[0] == A$ldr_rd_$rn_im$(A$pc, A$pc, 4 - 8)
) {
if (result != NULL)
*result = reinterpret_cast<void *>(arm[1]);
SubstrateHookMemory code(process, arm + 1, sizeof(uint32_t) * 1);
arm[1] = reinterpret_cast<uint32_t>(replace);
return;
}
size_t required((trail - area) * sizeof(uint16_t));
size_t used(0);
while (used < required)
used += MSGetInstructionWidthThumb(reinterpret_cast<uint8_t *>(area) + used);
used = (used + sizeof(uint16_t) - 1) / sizeof(uint16_t) * sizeof(uint16_t);
size_t blank((used - required) / sizeof(uint16_t));
uint16_t backup[used / sizeof(uint16_t)];
memcpy(backup, area, used);
if (MSDebug) {
char name[16];
sprintf(name, "%p", area);
MSLogHexEx(area, used + sizeof(uint16_t), 2, name);
}
if (result != NULL) {
size_t length(used);
for (unsigned offset(0); offset != used / sizeof(uint16_t); ++offset)
if (T$pcrel$ldr(backup[offset]))
length += 3 * sizeof(uint16_t);
else if (T$pcrel$b(backup[offset]))
length += 6 * sizeof(uint16_t);
else if (T2$pcrel$b(backup + offset)) {
length += 5 * sizeof(uint16_t);
++offset;
} else if (T$pcrel$bl(backup + offset)) {
length += 5 * sizeof(uint16_t);
++offset;
} else if (T$pcrel$cbz(backup[offset])) {
length += 16 * sizeof(uint16_t);
} else if (T$pcrel$ldrw(backup[offset])) {
length += 4 * sizeof(uint16_t);
++offset;
} else if (T$pcrel$add(backup[offset]))
length += 6 * sizeof(uint16_t);
else if (T$32bit$i(backup[offset]))
++offset;
unsigned pad((length & 0x2) == 0 ? 0 : 1);
length += (pad + 2) * sizeof(uint16_t) + 2 * sizeof(uint32_t);
uint16_t *buffer(reinterpret_cast<uint16_t *>(mmap(
NULL, length, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0
)));
if (buffer == MAP_FAILED) {
MSLog(MSLogLevelError, "MS:Error:mmap() = %d", errno);
*result = NULL;
return;
}
if (false) fail: {
munmap(buffer, length);
*result = NULL;
return;
}
size_t start(pad), end(length / sizeof(uint16_t));
uint32_t *trailer(reinterpret_cast<uint32_t *>(buffer + end));
for (unsigned offset(0); offset != used / sizeof(uint16_t); ++offset) {
if (T$pcrel$ldr(backup[offset])) {
union {
uint16_t value;
struct {
uint16_t immediate : 8;
uint16_t rd : 3;
uint16_t : 5;
};
} bits = {backup[offset+0]};
buffer[start+0] = T$ldr_rd_$pc_im_4$(bits.rd, T$Label(start+0, end-2) / 4);
buffer[start+1] = T$ldr_rd_$rn_im_4$(bits.rd, bits.rd, 0);
// XXX: this code "works", but is "wrong": the mechanism is more complex than this
*--trailer = ((reinterpret_cast<uint32_t>(area + offset) + 4) & ~0x2) + bits.immediate * 4;
start += 2;
end -= 2;
} else if (T$pcrel$b(backup[offset])) {
union {
uint16_t value;
struct {
uint16_t imm8 : 8;
uint16_t cond : 4;
uint16_t /*1101*/ : 4;
};
} bits = {backup[offset+0]};
intptr_t jump(bits.imm8 << 1);
jump |= 1;
jump <<= 23;
jump >>= 23;
buffer[start+0] = T$b$_$im(bits.cond, (end-6 - (start+0)) * 2 - 4);
*--trailer = reinterpret_cast<uint32_t>(area + offset) + 4 + jump;
*--trailer = A$ldr_rd_$rn_im$(A$pc, A$pc, 4 - 8);
*--trailer = T$nop << 16 | T$bx(A$pc);
start += 1;
end -= 6;
} else if (T2$pcrel$b(backup + offset)) {
union {
uint16_t value;
struct {
uint16_t imm6 : 6;
uint16_t cond : 4;
uint16_t s : 1;
uint16_t : 5;
};
} bits = {backup[offset+0]};
union {
uint16_t value;
struct {
uint16_t imm11 : 11;
uint16_t j2 : 1;
uint16_t a : 1;
uint16_t j1 : 1;
uint16_t : 2;
};
} exts = {backup[offset+1]};
intptr_t jump(1);
jump |= exts.imm11 << 1;
jump |= bits.imm6 << 12;
if (exts.a) {
jump |= bits.s << 24;
jump |= (~(bits.s ^ exts.j1) & 0x1) << 23;
jump |= (~(bits.s ^ exts.j2) & 0x1) << 22;
jump |= bits.cond << 18;
jump <<= 7;
jump >>= 7;
} else {
