void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) { assert_different_registers(obj, klass, len); if (UseBiasedLocking && !len->is_valid()) { assert_different_registers(obj, klass, len, t1, t2); movptr(t1, Address(klass, Klass::prototype_header_offset())); movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1); } else { // This assumes that all prototype bits fit in an int32_t movptr(Address(obj, oopDesc::mark_offset_in_bytes ()), (int32_t)(intptr_t)markOopDesc::prototype()); } #ifdef _LP64 if (UseCompressedClassPointers) { // Take care not to kill klass movptr(t1, klass); encode_klass_not_null(t1); movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1); } else #endif { movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass); } if (len->is_valid()) { movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len); } #ifdef _LP64 else if (UseCompressedClassPointers) { xorptr(t1, t1); store_klass_gap(obj, t1); } #endif }
void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rbox, Label& slow_case) { assert_different_registers(Rmark, Roop, Rbox); Label done; Address mark_addr(Roop, oopDesc::mark_offset_in_bytes()); assert(mark_addr.disp() == 0, "cas must take a zero displacement"); if (UseBiasedLocking) { // load the object out of the BasicObjectLock ld_ptr(Rbox, BasicObjectLock::obj_offset_in_bytes(), Roop); verify_oop(Roop); biased_locking_exit(mark_addr, Rmark, done); } // Test first it it is a fast recursive unlock ld_ptr(Rbox, BasicLock::displaced_header_offset_in_bytes(), Rmark); br_null_short(Rmark, Assembler::pt, done); if (!UseBiasedLocking) { // load object ld_ptr(Rbox, BasicObjectLock::obj_offset_in_bytes(), Roop); verify_oop(Roop); } // Check if it is still a light weight lock, this is is true if we see // the stack address of the basicLock in the markOop of the object cas_ptr(mark_addr.base(), Rbox, Rmark); cmp(Rbox, Rmark); brx(Assembler::notEqual, false, Assembler::pn, slow_case); delayed()->nop(); // Done bind(done); }
void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) { const int aligned_mask = BytesPerWord -1; const int hdr_offset = oopDesc::mark_offset_in_bytes(); assert_different_registers(hdr, obj, disp_hdr); NearLabel done; if (UseBiasedLocking) { // Load object. z_lg(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); biased_locking_exit(obj, hdr, done); } // Load displaced header. z_ltg(hdr, Address(disp_hdr, (intptr_t)0)); // If the loaded hdr is NULL we had recursive locking, and we are done. z_bre(done); if (!UseBiasedLocking) { // Load object. z_lg(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); } verify_oop(obj); // Test if object header is pointing to the displaced header, and if so, restore // the displaced header in the object. If the object header is not pointing to // the displaced header, get the object header instead. z_csg(disp_hdr, hdr, hdr_offset, obj); // If the object header was not pointing to the displaced header, // we do unlocking via runtime call. branch_optimized(Assembler::bcondNotEqual, slow_case); // done bind(done); }
void C1_MacroAssembler::allocate_object( Register obj, // result: pointer to object after successful allocation Register t1, // temp register Register t2, // temp register, must be a global register for try_allocate Register t3, // temp register int hdr_size, // object header size in words int obj_size, // object size in words Register klass, // object klass Label& slow_case // continuation point if fast allocation fails ) { assert_different_registers(obj, t1, t2, t3, klass); assert(klass == G5, "must be G5"); // allocate space & initialize header if (!is_simm13(obj_size * wordSize)) { // would need to use extra register to load // object size => go the slow case for now ba(slow_case); delayed()->nop(); return; } try_allocate(obj, noreg, obj_size * wordSize, t2, t3, slow_case); initialize_object(obj, klass, noreg, obj_size * HeapWordSize, t1, t2); }
void C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) { const int hdr_offset = oopDesc::mark_offset_in_bytes(); assert_different_registers(hdr, obj, disp_hdr); NearLabel done; verify_oop(obj); // Load object header. z_lg(hdr, Address(obj, hdr_offset)); // Save object being locked into the BasicObjectLock... z_stg(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); if (UseBiasedLocking) { biased_locking_enter(obj, hdr, Z_R1_scratch, Z_R0_scratch, done, &slow_case); } // and mark it as unlocked. z_oill(hdr, markOopDesc::unlocked_value); // Save unlocked object header into the displaced header location on the stack. z_stg(hdr, Address(disp_hdr, (intptr_t)0)); // Test if object header is still the same (i.e. unlocked), and if so, store the // displaced header address in the object header. If it is not the same, get the // object header instead. z_csg(hdr, disp_hdr, hdr_offset, obj); // If the object header was the same, we're done. if (PrintBiasedLockingStatistics) { Unimplemented(); #if 0 cond_inc32(Assembler::equal, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr())); #endif } branch_optimized(Assembler::bcondEqual, done); // If the object header was not the same, it is now in the hdr register. // => Test if it is a stack pointer into the same stack (recursive locking), i.e.: // // 1) (hdr & markOopDesc::lock_mask_in_place) == 0 // 2) rsp <= hdr // 3) hdr <= rsp + page_size // // These 3 tests can be done by evaluating the following expression: // // (hdr - Z_SP) & (~(page_size-1) | markOopDesc::lock_mask_in_place) // // assuming both the stack pointer and page_size have their least // significant 2 bits cleared and page_size is a power of 2 z_sgr(hdr, Z_SP); load_const_optimized(Z_R0_scratch, (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place)); z_ngr(hdr, Z_R0_scratch); // AND sets CC (result eq/ne 0). // For recursive locking, the result is zero. => Save it in the displaced header // location (NULL in the displaced hdr location indicates recursive locking). z_stg(hdr, Address(disp_hdr, (intptr_t)0)); // Otherwise we don't care about the result and handle locking via runtime call. branch_optimized(Assembler::bcondNotZero, slow_case); // done bind(done); }
// Helper to insert argument slots into the stack. // arg_slots must be a multiple of stack_move_unit() and <= 0 void MethodHandles::insert_arg_slots(MacroAssembler* _masm, RegisterOrConstant arg_slots, int arg_mask, Register rax_argslot, Register rbx_temp, Register rdx_temp, Register temp3_reg) { assert(temp3_reg == noreg, "temp3 not required"); assert_different_registers(rax_argslot, rbx_temp, rdx_temp, (!arg_slots.is_register() ? rsp : arg_slots.as_register())); #ifdef ASSERT verify_argslot(_masm, rax_argslot, "insertion point must fall within current frame"); if (arg_slots.is_register()) { Label L_ok, L_bad; __ cmpptr(arg_slots.as_register(), (int32_t) NULL_WORD); __ jccb(Assembler::greater, L_bad); __ testl(arg_slots.as_register(), -stack_move_unit() - 1); __ jccb(Assembler::zero, L_ok); __ bind(L_bad); __ stop("assert arg_slots <= 0 and clear low bits"); __ bind(L_ok); } else { assert(arg_slots.as_constant() <= 0, ""); assert(arg_slots.as_constant() % -stack_move_unit() == 0, ""); } #endif //ASSERT #ifdef _LP64 if (arg_slots.is_register()) { // clean high bits of stack motion register (was loaded as an int) __ movslq(arg_slots.as_register(), arg_slots.as_register()); } #endif // Make space on the stack for the inserted argument(s). // Then pull down everything shallower than rax_argslot. // The stacked return address gets pulled down with everything else. // That is, copy [rsp, argslot) downward by -size words. In pseudo-code: // rsp -= size; // for (rdx = rsp + size; rdx < argslot; rdx++) // rdx[-size] = rdx[0] // argslot -= size; BLOCK_COMMENT("insert_arg_slots {"); __ mov(rdx_temp, rsp); // source pointer for copy __ lea(rsp, Address(rsp, arg_slots, Address::times_ptr)); { Label loop; __ BIND(loop); // pull one word down each time through the loop __ movptr(rbx_temp, Address(rdx_temp, 0)); __ movptr(Address(rdx_temp, arg_slots, Address::times_ptr), rbx_temp); __ addptr(rdx_temp, wordSize); __ cmpptr(rdx_temp, rax_argslot); __ jccb(Assembler::less, loop); } // Now move the argslot down, to point to the opened-up space. __ lea(rax_argslot, Address(rax_argslot, arg_slots, Address::times_ptr)); BLOCK_COMMENT("} insert_arg_slots"); }
void C1_MacroAssembler::initialize_body(Register base, Register index) { assert_different_registers(base, index); Label loop; bind(loop); subcc(index, HeapWordSize, index); brx(Assembler::greaterEqual, true, Assembler::pt, loop); delayed()->st_ptr(G0, base, index); }
void C1_MacroAssembler::allocate_array( Register obj, // result: Pointer to array after successful allocation. Register len, // array length Register t1, // temp register Register t2, // temp register int hdr_size, // object header size in words int elt_size, // element size in bytes Register klass, // object klass Label& slow_case // Continuation point if fast allocation fails. ) { assert_different_registers(obj, len, t1, t2, klass); // Determine alignment mask. assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work"); // Check for negative or excessive length. compareU64_and_branch(len, (int32_t)max_array_allocation_length, bcondHigh, slow_case); // Compute array size. // Note: If 0 <= len <= max_length, len*elt_size + header + alignment is // smaller or equal to the largest integer. Also, since top is always // aligned, we can do the alignment here instead of at the end address // computation. const Register arr_size = t2; switch (elt_size) { case 1: lgr_if_needed(arr_size, len); break; case 2: z_sllg(arr_size, len, 1); break; case 4: z_sllg(arr_size, len, 2); break; case 8: z_sllg(arr_size, len, 3); break; default: ShouldNotReachHere(); } add2reg(arr_size, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment. z_nill(arr_size, (~MinObjAlignmentInBytesMask) & 0xffff); // Align array size. try_allocate(obj, arr_size, 0, t1, slow_case); initialize_header(obj, klass, len, noreg, t1); // Clear rest of allocated space. Label done; Register object_fields = t1; Register Rzero = Z_R1_scratch; z_aghi(arr_size, -(hdr_size * BytesPerWord)); z_bre(done); // Jump if size of fields is zero. z_la(object_fields, hdr_size * BytesPerWord, obj); z_xgr(Rzero, Rzero); initialize_body(object_fields, arr_size, Rzero); bind(done); // Dtrace support is unimplemented. // if (CURRENT_ENV->dtrace_alloc_probes()) { // assert(obj == rax, "must be"); // call(RuntimeAddress(Runtime1::entry_for (Runtime1::dtrace_object_alloc_id))); // } verify_oop(obj); }
void C1_MacroAssembler::allocate_object(Register obj, Register t1, Register t2, int header_size, int object_size, Register klass, Label& slow_case) { assert(obj == rax, "obj must be in rax, for cmpxchg"); assert_different_registers(obj, t1, t2); // XXX really? assert(header_size >= 0 && object_size >= header_size, "illegal sizes"); try_allocate(obj, noreg, object_size * BytesPerWord, t1, t2, slow_case); initialize_object(obj, klass, noreg, object_size * HeapWordSize, t1, t2, UseTLAB); }
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register Rzero, Register t1) { assert_different_registers(obj, klass, len, t1, Rzero); if (UseBiasedLocking && !len->is_valid()) { assert_different_registers(obj, klass, len, t1); z_lg(t1, Address(klass, Klass::prototype_header_offset())); } else { // This assumes that all prototype bits fit in an int32_t. load_const_optimized(t1, (intx)markOopDesc::prototype()); } z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes())); if (len->is_valid()) { // Length will be in the klass gap, if one exists. z_st(len, Address(obj, arrayOopDesc::length_offset_in_bytes())); } else if (UseCompressedClassPointers) { store_klass_gap(Rzero, obj); // Zero klass gap for compressed oops. } store_klass(klass, obj, t1); }
void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox, Register Rscratch, Label& slow_case) { assert_different_registers(Rmark, Roop, Rbox, Rscratch); Label done, cas_failed, slow_int; // The following move must be the first instruction of emitted since debug // information may be generated for it. // Load object header. ld(Rmark, oopDesc::mark_offset_in_bytes(), Roop); verify_oop(Roop); // Save object being locked into the BasicObjectLock... std(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox); if (UseBiasedLocking) { biased_locking_enter(CCR0, Roop, Rmark, Rscratch, R0, done, &slow_int); } // ... and mark it unlocked. ori(Rmark, Rmark, markOopDesc::unlocked_value); // Save unlocked object header into the displaced header location on the stack. std(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox); // Compare object markOop with Rmark and if equal exchange Rscratch with object markOop. assert(oopDesc::mark_offset_in_bytes() == 0, "cas must take a zero displacement"); cmpxchgd(/*flag=*/CCR0, /*current_value=*/Rscratch, /*compare_value=*/Rmark, /*exchange_value=*/Rbox, /*where=*/Roop/*+0==mark_offset_in_bytes*/, MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq, MacroAssembler::cmpxchgx_hint_acquire_lock(), noreg, &cas_failed, /*check without membar and ldarx first*/true); // If compare/exchange succeeded we found an unlocked object and we now have locked it // hence we are done. b(done); bind(slow_int); b(slow_case); // far bind(cas_failed); // We did not find an unlocked object so see if this is a recursive case. sub(Rscratch, Rscratch, R1_SP); load_const_optimized(R0, (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place)); and_(R0/*==0?*/, Rscratch, R0); std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), Rbox); bne(CCR0, slow_int); bind(done); }
void C1_MacroAssembler::initialize_body(Register objectFields, Register len_in_bytes, Register Rzero) { Label done; assert_different_registers(objectFields, len_in_bytes, Rzero); // Initialize object fields. // See documentation for MVCLE instruction!!! assert(objectFields->encoding()%2==0, "objectFields must be an even register"); assert(len_in_bytes->encoding() == (objectFields->encoding()+1), "objectFields and len_in_bytes must be a register pair"); assert(Rzero->encoding()%2==1, "Rzero must be an odd register"); // Use Rzero as src length, then mvcle will copy nothing // and fill the object with the padding value 0. move_long_ext(objectFields, as_Register(Rzero->encoding()-1), 0); bind(done); }
void C1_MacroAssembler::allocate_object( Register obj, // Result: pointer to object after successful allocation. Register t1, // temp register Register t2, // temp register: Must be a global register for try_allocate. int hdr_size, // object header size in words int obj_size, // object size in words Register klass, // object klass Label& slow_case // Continuation point if fast allocation fails. ) { assert_different_registers(obj, t1, t2, klass); // Allocate space and initialize header. try_allocate(obj, noreg, obj_size * wordSize, t1, slow_case); initialize_object(obj, klass, noreg, obj_size * HeapWordSize, t1, t2); }
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) { assert_different_registers(obj, klass, len, t1, t2); if (UseBiasedLocking && !len->is_valid()) { ld(t1, in_bytes(Klass::prototype_header_offset()), klass); } else { load_const_optimized(t1, (intx)markOopDesc::prototype()); } std(t1, oopDesc::mark_offset_in_bytes(), obj); store_klass(obj, klass); if (len->is_valid()) { stw(len, arrayOopDesc::length_offset_in_bytes(), obj); } else if (UseCompressedClassPointers) { // Otherwise length is in the class gap. store_klass_gap(obj); } }
void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox, Register Rscratch, Label& slow_case) { assert_different_registers(Rmark, Roop, Rbox, Rscratch); Label done; Address mark_addr(Roop, oopDesc::mark_offset_in_bytes()); // The following move must be the first instruction of emitted since debug // information may be generated for it. // Load object header ld_ptr(mark_addr, Rmark); verify_oop(Roop); // save object being locked into the BasicObjectLock st_ptr(Roop, Rbox, BasicObjectLock::obj_offset_in_bytes()); if (UseBiasedLocking) { biased_locking_enter(Roop, Rmark, Rscratch, done, &slow_case); } // Save Rbox in Rscratch to be used for the cas operation mov(Rbox, Rscratch); // and mark it unlocked or3(Rmark, markOopDesc::unlocked_value, Rmark); // save unlocked object header into the displaced header location on the stack st_ptr(Rmark, Rbox, BasicLock::displaced_header_offset_in_bytes()); // compare object markOop with Rmark and if equal exchange Rscratch with object markOop assert(mark_addr.disp() == 0, "cas must take a zero displacement"); cas_ptr(mark_addr.base(), Rmark, Rscratch); // if compare/exchange succeeded we found an unlocked object and we now have locked it // hence we are done cmp(Rmark, Rscratch); brx(Assembler::equal, false, Assembler::pt, done); delayed()->sub(Rscratch, SP, Rscratch); //pull next instruction into delay slot // we did not find an unlocked object so see if this is a recursive case // sub(Rscratch, SP, Rscratch); assert(os::vm_page_size() > 0xfff, "page size too small - change the constant"); andcc(Rscratch, 0xfffff003, Rscratch); brx(Assembler::notZero, false, Assembler::pn, slow_case); delayed()->st_ptr(Rscratch, Rbox, BasicLock::displaced_header_offset_in_bytes()); bind(done); }
void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rbox, Label& slow_case) { assert_different_registers(Rmark, Roop, Rbox); Label slow_int, done; Address mark_addr(Roop, oopDesc::mark_offset_in_bytes()); assert(mark_addr.disp() == 0, "cas must take a zero displacement"); if (UseBiasedLocking) { // Load the object out of the BasicObjectLock. ld(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox); verify_oop(Roop); biased_locking_exit(CCR0, Roop, R0, done); } // Test first it it is a fast recursive unlock. ld(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox); cmpdi(CCR0, Rmark, 0); beq(CCR0, done); if (!UseBiasedLocking) { // Load object. ld(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox); verify_oop(Roop); } // Check if it is still a light weight lock, this is is true if we see // the stack address of the basicLock in the markOop of the object. cmpxchgd(/*flag=*/CCR0, /*current_value=*/R0, /*compare_value=*/Rbox, /*exchange_value=*/Rmark, /*where=*/Roop, MacroAssembler::MemBarRel, MacroAssembler::cmpxchgx_hint_release_lock(), noreg, &slow_int); b(done); bind(slow_int); b(slow_case); // far // Done bind(done); }
void MethodHandles::jump_from_method_handle(MacroAssembler* _masm, Register method, Register target, Register temp, bool for_compiler_entry) { Label L_no_such_method; assert(method == R19_method, "interpreter calling convention"); assert_different_registers(method, target, temp); if (!for_compiler_entry && JvmtiExport::can_post_interpreter_events()) { Label run_compiled_code; // JVMTI events, such as single-stepping, are implemented partly by avoiding running // compiled code in threads for which the event is enabled. Check here for // interp_only_mode if these events CAN be enabled. __ verify_thread(); __ lwz(temp, in_bytes(JavaThread::interp_only_mode_offset()), R16_thread); __ cmplwi(CCR0, temp, 0); __ beq(CCR0, run_compiled_code); // Null method test is replicated below in compiled case, // it might be able to address across the verify_thread() __ cmplwi(CCR0, R19_method, 0); __ beq(CCR0, L_no_such_method); __ ld(target, in_bytes(Method::interpreter_entry_offset()), R19_method); __ mtctr(target); __ bctr(); __ BIND(run_compiled_code); } // Compiled case, either static or fall-through from runtime conditional __ cmplwi(CCR0, R19_method, 0); __ beq(CCR0, L_no_such_method); const ByteSize entry_offset = for_compiler_entry ? Method::from_compiled_offset() : Method::from_interpreted_offset(); __ ld(target, in_bytes(entry_offset), R19_method); __ mtctr(target); __ bctr(); __ bind(L_no_such_method); assert(StubRoutines::throw_AbstractMethodError_entry() != NULL, "not yet generated!"); __ load_const_optimized(target, StubRoutines::throw_AbstractMethodError_entry()); __ mtctr(target); __ bctr(); }
void MethodHandles::jump_to_lambda_form(MacroAssembler* _masm, Register recv, Register method_temp, Register temp2, bool for_compiler_entry) { BLOCK_COMMENT("jump_to_lambda_form {"); // This is the initial entry point of a lazy method handle. // After type checking, it picks up the invoker from the LambdaForm. assert_different_registers(recv, method_temp, temp2); assert(recv != noreg, "required register"); assert(method_temp == rmethod, "required register for loading method"); //NOT_PRODUCT({ FlagSetting fs(TraceMethodHandles, true); trace_method_handle(_masm, "LZMH"); }); // Load the invoker, as MH -> MH.form -> LF.vmentry __ verify_oop(recv); __ load_heap_oop(method_temp, Address(recv, NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes()))); __ verify_oop(method_temp); __ load_heap_oop(method_temp, Address(method_temp, NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes()))); __ verify_oop(method_temp); // the following assumes that a Method* is normally compressed in the vmtarget field: __ ldr(method_temp, Address(method_temp, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes()))); if (VerifyMethodHandles && !for_compiler_entry) { // make sure recv is already on stack __ ldr(temp2, Address(method_temp, Method::const_offset())); __ load_sized_value(temp2, Address(temp2, ConstMethod::size_of_parameters_offset()), sizeof(u2), /*is_signed*/ false); // assert(sizeof(u2) == sizeof(Method::_size_of_parameters), ""); Label L; __ ldr(rscratch1, __ argument_address(temp2, -1)); __ cmp(recv, rscratch1); __ br(Assembler::EQ, L); __ ldr(r0, __ argument_address(temp2, -1)); __ hlt(0); __ BIND(L); } jump_from_method_handle(_masm, method_temp, temp2, for_compiler_entry); BLOCK_COMMENT("} jump_to_lambda_form"); }
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) { assert_different_registers(obj, klass, len, t1, t2); if (UseBiasedLocking && !len->is_valid()) { ld_ptr(klass, in_bytes(Klass::prototype_header_offset()), t1); } else { set((intx)markOopDesc::prototype(), t1); } st_ptr(t1, obj, oopDesc::mark_offset_in_bytes()); if (UseCompressedKlassPointers) { // Save klass mov(klass, t1); encode_klass_not_null(t1); stw(t1, obj, oopDesc::klass_offset_in_bytes()); } else { st_ptr(klass, obj, oopDesc::klass_offset_in_bytes()); } if (len->is_valid()) { st(len, obj, arrayOopDesc::length_offset_in_bytes()); } else if (UseCompressedKlassPointers) { // otherwise length is in the class gap store_klass_gap(G0, obj); } }
void MethodHandles::jump_to_lambda_form(MacroAssembler* _masm, Register recv, Register method_temp, Register temp2, Register temp3, bool for_compiler_entry) { BLOCK_COMMENT("jump_to_lambda_form {"); // This is the initial entry point of a lazy method handle. // After type checking, it picks up the invoker from the LambdaForm. assert_different_registers(recv, method_temp, temp2); // temp3 is only passed on assert(method_temp == R19_method, "required register for loading method"); // Load the invoker, as MH -> MH.form -> LF.vmentry __ verify_oop(recv); __ load_heap_oop_not_null(method_temp, NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes()), recv, temp2); __ verify_oop(method_temp); __ load_heap_oop_not_null(method_temp, NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes()), method_temp, temp2); __ verify_oop(method_temp); // The following assumes that a Method* is normally compressed in the vmtarget field: __ ld(method_temp, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes()), method_temp); if (VerifyMethodHandles && !for_compiler_entry) { // Make sure recv is already on stack. __ ld(temp2, in_bytes(Method::const_offset()), method_temp); __ load_sized_value(temp2, in_bytes(ConstMethod::size_of_parameters_offset()), temp2, sizeof(u2), /*is_signed*/ false); // assert(sizeof(u2) == sizeof(ConstMethod::_size_of_parameters), ""); Label L; __ ld(temp2, __ argument_offset(temp2, temp2, 0), CC_INTERP_ONLY(R17_tos) NOT_CC_INTERP(R15_esp)); __ cmpd(CCR1, temp2, recv); __ beq(CCR1, L); __ stop("receiver not on stack"); __ BIND(L); } jump_from_method_handle(_masm, method_temp, temp2, temp3, for_compiler_entry); BLOCK_COMMENT("} jump_to_lambda_form"); }
void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm, vmIntrinsics::ID iid, Register receiver_reg, Register member_reg, bool for_compiler_entry) { assert(is_signature_polymorphic(iid), "expected invoke iid"); Register temp1 = (for_compiler_entry ? R25_tmp5 : R7); Register temp2 = (for_compiler_entry ? R22_tmp2 : R8); Register temp3 = (for_compiler_entry ? R23_tmp3 : R9); Register temp4 = (for_compiler_entry ? R24_tmp4 : R10); if (receiver_reg != noreg) assert_different_registers(temp1, temp2, temp3, temp4, receiver_reg); if (member_reg != noreg) assert_different_registers(temp1, temp2, temp3, temp4, member_reg); if (iid == vmIntrinsics::_invokeBasic) { // indirect through MH.form.vmentry.vmtarget jump_to_lambda_form(_masm, receiver_reg, R19_method, temp1, temp2, for_compiler_entry); } else { // The method is a member invoker used by direct method handles. if (VerifyMethodHandles) { // make sure the trailing argument really is a MemberName (caller responsibility) verify_klass(_masm, member_reg, SystemDictionary::WK_KLASS_ENUM_NAME(MemberName_klass), temp1, temp2, "MemberName required for invokeVirtual etc."); } Register temp1_recv_klass = temp1; if (iid != vmIntrinsics::_linkToStatic) { __ verify_oop(receiver_reg); if (iid == vmIntrinsics::_linkToSpecial) { // Don't actually load the klass; just null-check the receiver. __ null_check_throw(receiver_reg, -1, temp1, EXCEPTION_ENTRY); } else { // load receiver klass itself __ null_check_throw(receiver_reg, oopDesc::klass_offset_in_bytes(), temp1, EXCEPTION_ENTRY); __ load_klass(temp1_recv_klass, receiver_reg); __ verify_klass_ptr(temp1_recv_klass); } BLOCK_COMMENT("check_receiver {"); // The receiver for the MemberName must be in receiver_reg. // Check the receiver against the MemberName.clazz if (VerifyMethodHandles && iid == vmIntrinsics::_linkToSpecial) { // Did not load it above... __ load_klass(temp1_recv_klass, receiver_reg); __ verify_klass_ptr(temp1_recv_klass); } if (VerifyMethodHandles && iid != vmIntrinsics::_linkToInterface) { Label L_ok; Register temp2_defc = temp2; __ load_heap_oop_not_null(temp2_defc, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes()), member_reg, temp3); load_klass_from_Class(_masm, temp2_defc, temp3, temp4); __ verify_klass_ptr(temp2_defc); __ check_klass_subtype(temp1_recv_klass, temp2_defc, temp3, temp4, L_ok); // If we get here, the type check failed! __ stop("receiver class disagrees with MemberName.clazz"); __ BIND(L_ok); } BLOCK_COMMENT("} check_receiver"); } if (iid == vmIntrinsics::_linkToSpecial || iid == vmIntrinsics::_linkToStatic) { DEBUG_ONLY(temp1_recv_klass = noreg); // these guys didn't load the recv_klass } // Live registers at this point: // member_reg - MemberName that was the trailing argument // temp1_recv_klass - klass of stacked receiver, if needed // O5_savedSP - interpreter linkage (if interpreted) // O0..O5 - compiler arguments (if compiled) Label L_incompatible_class_change_error; switch (iid) { case vmIntrinsics::_linkToSpecial: if (VerifyMethodHandles) { verify_ref_kind(_masm, JVM_REF_invokeSpecial, member_reg, temp2); } __ ld(R19_method, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes()), member_reg); break; case vmIntrinsics::_linkToStatic: if (VerifyMethodHandles) { verify_ref_kind(_masm, JVM_REF_invokeStatic, member_reg, temp2); } __ ld(R19_method, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes()), member_reg); break; case vmIntrinsics::_linkToVirtual: { // same as TemplateTable::invokevirtual, // minus the CP setup and profiling: if (VerifyMethodHandles) { verify_ref_kind(_masm, JVM_REF_invokeVirtual, member_reg, temp2); } // pick out the vtable index from the MemberName, and then we can discard it: Register temp2_index = temp2; __ ld(temp2_index, NONZERO(java_lang_invoke_MemberName::vmindex_offset_in_bytes()), member_reg); if (VerifyMethodHandles) { Label L_index_ok; __ cmpdi(CCR1, temp2_index, 0); __ bge(CCR1, L_index_ok); __ stop("no virtual index"); __ BIND(L_index_ok); } // Note: The verifier invariants allow us to ignore MemberName.clazz and vmtarget // at this point. And VerifyMethodHandles has already checked clazz, if needed. // get target Method* & entry point __ lookup_virtual_method(temp1_recv_klass, temp2_index, R19_method); break; } case vmIntrinsics::_linkToInterface: { // same as TemplateTable::invokeinterface // (minus the CP setup and profiling, with different argument motion) if (VerifyMethodHandles) { verify_ref_kind(_masm, JVM_REF_invokeInterface, member_reg, temp2); } Register temp2_intf = temp2; __ load_heap_oop_not_null(temp2_intf, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes()), member_reg, temp3); load_klass_from_Class(_masm, temp2_intf, temp3, temp4); __ verify_klass_ptr(temp2_intf); Register vtable_index = R19_method; __ ld(vtable_index, NONZERO(java_lang_invoke_MemberName::vmindex_offset_in_bytes()), member_reg); if (VerifyMethodHandles) { Label L_index_ok; __ cmpdi(CCR1, vtable_index, 0); __ bge(CCR1, L_index_ok); __ stop("invalid vtable index for MH.invokeInterface"); __ BIND(L_index_ok); } // given intf, index, and recv klass, dispatch to the implementation method __ lookup_interface_method(temp1_recv_klass, temp2_intf, // note: next two args must be the same: vtable_index, R19_method, temp3, temp4, L_incompatible_class_change_error); break; } default: fatal(err_msg_res("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid))); break; } // Live at this point: // R19_method // O5_savedSP (if interpreted) // After figuring out which concrete method to call, jump into it. // Note that this works in the interpreter with no data motion. // But the compiled version will require that rcx_recv be shifted out. __ verify_method_ptr(R19_method); jump_from_method_handle(_masm, R19_method, temp1, temp2, for_compiler_entry); if (iid == vmIntrinsics::_linkToInterface) { __ BIND(L_incompatible_class_change_error); __ load_const_optimized(temp1, StubRoutines::throw_IncompatibleClassChangeError_entry()); __ mtctr(temp1); __ bctr(); } } }
void C1_MacroAssembler::allocate_array( Register obj, // result: pointer to array after successful allocation Register len, // array length Register t1, // temp register Register t2, // temp register Register t3, // temp register int hdr_size, // object header size in words int elt_size, // element size in bytes Register klass, // object klass Label& slow_case // continuation point if fast allocation fails ) { assert_different_registers(obj, len, t1, t2, t3, klass); assert(klass == G5, "must be G5"); assert(t1 == G1, "must be G1"); // determine alignment mask assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work"); // check for negative or excessive length // note: the maximum length allowed is chosen so that arrays of any // element size with this length are always smaller or equal // to the largest integer (i.e., array size computation will // not overflow) set(max_array_allocation_length, t1); cmp(len, t1); br(Assembler::greaterUnsigned, false, Assembler::pn, slow_case); // compute array size // note: if 0 <= len <= max_length, len*elt_size + header + alignment is // smaller or equal to the largest integer; also, since top is always // aligned, we can do the alignment here instead of at the end address // computation const Register arr_size = t1; switch (elt_size) { case 1: delayed()->mov(len, arr_size); break; case 2: delayed()->sll(len, 1, arr_size); break; case 4: delayed()->sll(len, 2, arr_size); break; case 8: delayed()->sll(len, 3, arr_size); break; default: ShouldNotReachHere(); } add(arr_size, hdr_size * wordSize + MinObjAlignmentInBytesMask, arr_size); // add space for header & alignment and3(arr_size, ~MinObjAlignmentInBytesMask, arr_size); // align array size // allocate space & initialize header if (UseTLAB) { tlab_allocate(obj, arr_size, 0, t2, slow_case); } else { eden_allocate(obj, arr_size, 0, t2, t3, slow_case); } initialize_header(obj, klass, len, t2, t3); // initialize body const Register base = t2; const Register index = t3; add(obj, hdr_size * wordSize, base); // compute address of first element sub(arr_size, hdr_size * wordSize, index); // compute index = number of words to clear initialize_body(base, index); if (CURRENT_ENV->dtrace_alloc_probes()) { assert(obj == O0, "must be"); call(CAST_FROM_FN_PTR(address, Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)), relocInfo::runtime_call_type); delayed()->nop(); } verify_oop(obj); }
void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm, vmIntrinsics::ID iid, Register receiver_reg, Register member_reg, bool for_compiler_entry) { assert(is_signature_polymorphic(iid), "expected invoke iid"); // temps used in this code are not used in *either* compiled or interpreted calling sequences Register temp1 = r10; Register temp2 = r11; Register temp3 = r14; // r13 is live by this point: it contains the sender SP if (for_compiler_entry) { assert(receiver_reg == (iid == vmIntrinsics::_linkToStatic ? noreg : j_rarg0), "only valid assignment"); assert_different_registers(temp1, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7); assert_different_registers(temp2, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7); assert_different_registers(temp3, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7); } assert_different_registers(temp1, temp2, temp3, receiver_reg); assert_different_registers(temp1, temp2, temp3, member_reg); if (iid == vmIntrinsics::_invokeBasic) { // indirect through MH.form.vmentry.vmtarget jump_to_lambda_form(_masm, receiver_reg, rmethod, temp1, for_compiler_entry); } else { // The method is a member invoker used by direct method handles. if (VerifyMethodHandles) { // make sure the trailing argument really is a MemberName (caller responsibility) verify_klass(_masm, member_reg, SystemDictionary::WK_KLASS_ENUM_NAME(java_lang_invoke_MemberName), "MemberName required for invokeVirtual etc."); } Address member_clazz( member_reg, NONZERO(java_lang_invoke_MemberName::clazz_offset_in_bytes())); Address member_vmindex( member_reg, NONZERO(java_lang_invoke_MemberName::vmindex_offset_in_bytes())); Address member_vmtarget( member_reg, NONZERO(java_lang_invoke_MemberName::vmtarget_offset_in_bytes())); Register temp1_recv_klass = temp1; if (iid != vmIntrinsics::_linkToStatic) { __ verify_oop(receiver_reg); if (iid == vmIntrinsics::_linkToSpecial) { // Don't actually load the klass; just null-check the receiver. __ null_check(receiver_reg); } else { // load receiver klass itself __ null_check(receiver_reg, oopDesc::klass_offset_in_bytes()); __ load_klass(temp1_recv_klass, receiver_reg); __ verify_klass_ptr(temp1_recv_klass); } BLOCK_COMMENT("check_receiver {"); // The receiver for the MemberName must be in receiver_reg. // Check the receiver against the MemberName.clazz if (VerifyMethodHandles && iid == vmIntrinsics::_linkToSpecial) { // Did not load it above... __ load_klass(temp1_recv_klass, receiver_reg); __ verify_klass_ptr(temp1_recv_klass); } if (VerifyMethodHandles && iid != vmIntrinsics::_linkToInterface) { Label L_ok; Register temp2_defc = temp2; __ load_heap_oop(temp2_defc, member_clazz); load_klass_from_Class(_masm, temp2_defc); __ verify_klass_ptr(temp2_defc); __ check_klass_subtype(temp1_recv_klass, temp2_defc, temp3, L_ok); // If we get here, the type check failed! __ hlt(0); // __ STOP("receiver class disagrees with MemberName.clazz"); __ bind(L_ok); } BLOCK_COMMENT("} check_receiver"); } if (iid == vmIntrinsics::_linkToSpecial || iid == vmIntrinsics::_linkToStatic) { DEBUG_ONLY(temp1_recv_klass = noreg); // these guys didn't load the recv_klass } // Live registers at this point: // member_reg - MemberName that was the trailing argument // temp1_recv_klass - klass of stacked receiver, if needed // r13 - interpreter linkage (if interpreted) ??? FIXME // r1 ... r0 - compiler arguments (if compiled) Label L_incompatible_class_change_error; switch (iid) { case vmIntrinsics::_linkToSpecial: if (VerifyMethodHandles) { verify_ref_kind(_masm, JVM_REF_invokeSpecial, member_reg, temp3); } __ ldr(rmethod, member_vmtarget); break; case vmIntrinsics::_linkToStatic: if (VerifyMethodHandles) { verify_ref_kind(_masm, JVM_REF_invokeStatic, member_reg, temp3); } __ ldr(rmethod, member_vmtarget); break; case vmIntrinsics::_linkToVirtual: { // same as TemplateTable::invokevirtual, // minus the CP setup and profiling: if (VerifyMethodHandles) { verify_ref_kind(_masm, JVM_REF_invokeVirtual, member_reg, temp3); } // pick out the vtable index from the MemberName, and then we can discard it: Register temp2_index = temp2; __ ldr(temp2_index, member_vmindex); if (VerifyMethodHandles) { Label L_index_ok; __ cmpw(temp2_index, 0U); __ br(Assembler::GE, L_index_ok); __ hlt(0); __ BIND(L_index_ok); } // Note: The verifier invariants allow us to ignore MemberName.clazz and vmtarget // at this point. And VerifyMethodHandles has already checked clazz, if needed. // get target Method* & entry point __ lookup_virtual_method(temp1_recv_klass, temp2_index, rmethod); break; } case vmIntrinsics::_linkToInterface: { // same as TemplateTable::invokeinterface // (minus the CP setup and profiling, with different argument motion) if (VerifyMethodHandles) { verify_ref_kind(_masm, JVM_REF_invokeInterface, member_reg, temp3); } Register temp3_intf = temp3; __ load_heap_oop(temp3_intf, member_clazz); load_klass_from_Class(_masm, temp3_intf); __ verify_klass_ptr(temp3_intf); Register rindex = rmethod; __ ldr(rindex, member_vmindex); if (VerifyMethodHandles) { Label L; __ cmpw(rindex, 0U); __ br(Assembler::GE, L); __ hlt(0); __ bind(L); } // given intf, index, and recv klass, dispatch to the implementation method __ lookup_interface_method(temp1_recv_klass, temp3_intf, // note: next two args must be the same: rindex, rmethod, temp2, L_incompatible_class_change_error); break; } default: fatal("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)); break; } // live at this point: rmethod, r13 (if interpreted) // After figuring out which concrete method to call, jump into it. // Note that this works in the interpreter with no data motion. // But the compiled version will require that r2_recv be shifted out. __ verify_method_ptr(rmethod); jump_from_method_handle(_masm, rmethod, temp1, for_compiler_entry); if (iid == vmIntrinsics::_linkToInterface) { __ bind(L_incompatible_class_change_error); __ far_jump(RuntimeAddress(StubRoutines::throw_IncompatibleClassChangeError_entry())); } } }
void MacroAssembler::atomic_cas_bool(Register oldval, Register newval, Register base, int offset, Register tmpreg) { if (VM_Version::supports_ldrex()) { Register tmp_reg; if (tmpreg == noreg) { push(LR); tmp_reg = LR; } else { tmp_reg = tmpreg; } assert_different_registers(tmp_reg, oldval, newval, base); Label loop; bind(loop); ldrex(tmp_reg, Address(base, offset)); subs(tmp_reg, tmp_reg, oldval); strex(tmp_reg, newval, Address(base, offset), eq); cmp(tmp_reg, 1, eq); b(loop, eq); cmp(tmp_reg, 0); if (tmpreg == noreg) { pop(tmp_reg); } } else if (VM_Version::supports_kuser_cmpxchg32()) { // On armv5 platforms we must use the Linux kernel helper // function for atomic cas operations since ldrex/strex is // not supported. // // This is a special routine at a fixed address 0xffff0fc0 with // with these arguments and results // // input: // r0 = oldval, r1 = newval, r2 = ptr, lr = return adress // output: // r0 = 0 carry set on success // r0 != 0 carry clear on failure // // r3, ip and flags are clobbered // Label loop; push(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR)); Register tmp_reg = LR; // ignore the argument assert_different_registers(tmp_reg, oldval, newval, base); // Shuffle registers for kernel call if (oldval != R0) { if (newval == R0) { mov(tmp_reg, newval); newval = tmp_reg; } if (base == R0) { mov(tmp_reg, base); base = tmp_reg; } mov(R0, oldval); } if(newval != R1) { if(base == R1) { if(newval == R2) { mov(tmp_reg, base); base = tmp_reg; } else { mov(R2, base); base = R2; } } mov(R1, newval); } if (base != R2) mov(R2, base); if (offset != 0) add(R2, R2, offset); mvn(R3, 0xf000); mov(LR, PC); sub(PC, R3, 0x3f); cmp (R0, 0); pop(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR)); } else { // Should never run on a platform so old that it does not have kernel helper stop("Atomic cmpxchg32 unsupported on this platform"); } }
OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { OopMapSet* oop_maps = NULL; // for better readability const bool must_gc_arguments = true; const bool dont_gc_arguments = false; // stub code & info for the different stubs switch (id) { case forward_exception_id: { oop_maps = generate_handle_exception(id, sasm); } break; case new_instance_id: case fast_new_instance_id: case fast_new_instance_init_check_id: { Register G5_klass = G5; // Incoming Register O0_obj = O0; // Outgoing if (id == new_instance_id) { __ set_info("new_instance", dont_gc_arguments); } else if (id == fast_new_instance_id) { __ set_info("fast new_instance", dont_gc_arguments); } else { assert(id == fast_new_instance_init_check_id, "bad StubID"); __ set_info("fast new_instance init check", dont_gc_arguments); } if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) && UseTLAB && FastTLABRefill) { Label slow_path; Register G1_obj_size = G1; Register G3_t1 = G3; Register G4_t2 = G4; assert_different_registers(G5_klass, G1_obj_size, G3_t1, G4_t2); // Push a frame since we may do dtrace notification for the // allocation which requires calling out and we don't want // to stomp the real return address. __ save_frame(0); if (id == fast_new_instance_init_check_id) { // make sure the klass is initialized __ ldub(G5_klass, in_bytes(InstanceKlass::init_state_offset()), G3_t1); __ cmp_and_br_short(G3_t1, InstanceKlass::fully_initialized, Assembler::notEqual, Assembler::pn, slow_path); } #ifdef ASSERT // assert object can be fast path allocated { Label ok, not_ok; __ ld(G5_klass, in_bytes(Klass::layout_helper_offset()), G1_obj_size); // make sure it's an instance (LH > 0) __ cmp_and_br_short(G1_obj_size, 0, Assembler::lessEqual, Assembler::pn, not_ok); __ btst(Klass::_lh_instance_slow_path_bit, G1_obj_size); __ br(Assembler::zero, false, Assembler::pn, ok); __ delayed()->nop(); __ bind(not_ok); __ stop("assert(can be fast path allocated)"); __ should_not_reach_here(); __ bind(ok); } #endif // ASSERT // if we got here then the TLAB allocation failed, so try // refilling the TLAB or allocating directly from eden. Label retry_tlab, try_eden; __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves G5_klass __ bind(retry_tlab); // get the instance size __ ld(G5_klass, in_bytes(Klass::layout_helper_offset()), G1_obj_size); __ tlab_allocate(O0_obj, G1_obj_size, 0, G3_t1, slow_path); __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); __ mov(O0, I0); __ ret(); __ delayed()->restore(); __ bind(try_eden); // get the instance size __ ld(G5_klass, in_bytes(Klass::layout_helper_offset()), G1_obj_size); __ eden_allocate(O0_obj, G1_obj_size, 0, G3_t1, G4_t2, slow_path); __ incr_allocated_bytes(G1_obj_size, G3_t1, G4_t2); __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); __ mov(O0, I0); __ ret(); __ delayed()->restore(); __ bind(slow_path); // pop this frame so generate_stub_call can push it's own __ restore(); } oop_maps = generate_stub_call(sasm, I0, CAST_FROM_FN_PTR(address, new_instance), G5_klass); // I0->O0: new instance } break; case counter_overflow_id: // G4 contains bci, G5 contains method oop_maps = generate_stub_call(sasm, noreg, CAST_FROM_FN_PTR(address, counter_overflow), G4, G5); break; case new_type_array_id: case new_object_array_id: { Register G5_klass = G5; // Incoming Register G4_length = G4; // Incoming Register O0_obj = O0; // Outgoing Address klass_lh(G5_klass, Klass::layout_helper_offset()); assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); assert(Klass::_lh_header_size_mask == 0xFF, "bytewise"); // Use this offset to pick out an individual byte of the layout_helper: const int klass_lh_header_size_offset = ((BytesPerInt - 1) // 3 - 2 selects byte {0,1,0,0} - Klass::_lh_header_size_shift / BitsPerByte); if (id == new_type_array_id) { __ set_info("new_type_array", dont_gc_arguments); } else { __ set_info("new_object_array", dont_gc_arguments); } #ifdef ASSERT // assert object type is really an array of the proper kind { Label ok; Register G3_t1 = G3; __ ld(klass_lh, G3_t1); __ sra(G3_t1, Klass::_lh_array_tag_shift, G3_t1); int tag = ((id == new_type_array_id) ? Klass::_lh_array_tag_type_value : Klass::_lh_array_tag_obj_value); __ cmp_and_brx_short(G3_t1, tag, Assembler::equal, Assembler::pt, ok); __ stop("assert(is an array klass)"); __ should_not_reach_here(); __ bind(ok); } #endif // ASSERT if (UseTLAB && FastTLABRefill) { Label slow_path; Register G1_arr_size = G1; Register G3_t1 = G3; Register O1_t2 = O1; assert_different_registers(G5_klass, G4_length, G1_arr_size, G3_t1, O1_t2); // check that array length is small enough for fast path __ set(C1_MacroAssembler::max_array_allocation_length, G3_t1); __ cmp_and_br_short(G4_length, G3_t1, Assembler::greaterUnsigned, Assembler::pn, slow_path); // if we got here then the TLAB allocation failed, so try // refilling the TLAB or allocating directly from eden. Label retry_tlab, try_eden; __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves G4_length and G5_klass __ bind(retry_tlab); // get the allocation size: (length << (layout_helper & 0x1F)) + header_size __ ld(klass_lh, G3_t1); __ sll(G4_length, G3_t1, G1_arr_size); __ srl(G3_t1, Klass::_lh_header_size_shift, G3_t1); __ and3(G3_t1, Klass::_lh_header_size_mask, G3_t1); __ add(G1_arr_size, G3_t1, G1_arr_size); __ add(G1_arr_size, MinObjAlignmentInBytesMask, G1_arr_size); // align up __ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size); __ tlab_allocate(O0_obj, G1_arr_size, 0, G3_t1, slow_path); // preserves G1_arr_size __ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2); __ ldub(klass_lh, G3_t1, klass_lh_header_size_offset); __ sub(G1_arr_size, G3_t1, O1_t2); // body length __ add(O0_obj, G3_t1, G3_t1); // body start __ initialize_body(G3_t1, O1_t2); __ verify_oop(O0_obj); __ retl(); __ delayed()->nop(); __ bind(try_eden); // get the allocation size: (length << (layout_helper & 0x1F)) + header_size __ ld(klass_lh, G3_t1); __ sll(G4_length, G3_t1, G1_arr_size); __ srl(G3_t1, Klass::_lh_header_size_shift, G3_t1); __ and3(G3_t1, Klass::_lh_header_size_mask, G3_t1); __ add(G1_arr_size, G3_t1, G1_arr_size); __ add(G1_arr_size, MinObjAlignmentInBytesMask, G1_arr_size); __ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size); __ eden_allocate(O0_obj, G1_arr_size, 0, G3_t1, O1_t2, slow_path); // preserves G1_arr_size __ incr_allocated_bytes(G1_arr_size, G3_t1, O1_t2); __ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2); __ ldub(klass_lh, G3_t1, klass_lh_header_size_offset); __ sub(G1_arr_size, G3_t1, O1_t2); // body length __ add(O0_obj, G3_t1, G3_t1); // body start __ initialize_body(G3_t1, O1_t2); __ verify_oop(O0_obj); __ retl(); __ delayed()->nop(); __ bind(slow_path); } if (id == new_type_array_id) { oop_maps = generate_stub_call(sasm, I0, CAST_FROM_FN_PTR(address, new_type_array), G5_klass, G4_length); } else { oop_maps = generate_stub_call(sasm, I0, CAST_FROM_FN_PTR(address, new_object_array), G5_klass, G4_length); } // I0 -> O0: new array } break; case new_multi_array_id: { // O0: klass // O1: rank // O2: address of 1st dimension __ set_info("new_multi_array", dont_gc_arguments); oop_maps = generate_stub_call(sasm, I0, CAST_FROM_FN_PTR(address, new_multi_array), I0, I1, I2); // I0 -> O0: new multi array } break; case register_finalizer_id: { __ set_info("register_finalizer", dont_gc_arguments); // load the klass and check the has finalizer flag Label register_finalizer; Register t = O1; __ load_klass(O0, t); __ ld(t, in_bytes(Klass::access_flags_offset()), t); __ set(JVM_ACC_HAS_FINALIZER, G3); __ andcc(G3, t, G0); __ br(Assembler::notZero, false, Assembler::pt, register_finalizer); __ delayed()->nop(); // do a leaf return __ retl(); __ delayed()->nop(); __ bind(register_finalizer); OopMap* oop_map = save_live_registers(sasm); int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), I0); oop_maps = new OopMapSet(); oop_maps->add_gc_map(call_offset, oop_map); // Now restore all the live registers restore_live_registers(sasm); __ ret(); __ delayed()->restore(); } break; case throw_range_check_failed_id: { __ set_info("range_check_failed", dont_gc_arguments); // arguments will be discarded // G4: index oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true); } break; case throw_index_exception_id: { __ set_info("index_range_check_failed", dont_gc_arguments); // arguments will be discarded // G4: index oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true); } break; case throw_div0_exception_id: { __ set_info("throw_div0_exception", dont_gc_arguments); oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false); } break; case throw_null_pointer_exception_id: { __ set_info("throw_null_pointer_exception", dont_gc_arguments); oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false); } break; case handle_exception_id: { __ set_info("handle_exception", dont_gc_arguments); oop_maps = generate_handle_exception(id, sasm); } break; case handle_exception_from_callee_id: { __ set_info("handle_exception_from_callee", dont_gc_arguments); oop_maps = generate_handle_exception(id, sasm); } break; case unwind_exception_id: { // O0: exception // I7: address of call to this method __ set_info("unwind_exception", dont_gc_arguments); __ mov(Oexception, Oexception->after_save()); __ add(I7, frame::pc_return_offset, Oissuing_pc->after_save()); __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), G2_thread, Oissuing_pc->after_save()); __ verify_not_null_oop(Oexception->after_save()); // Restore SP from L7 if the exception PC is a method handle call site. __ mov(O0, G5); // Save the target address. __ lduw(Address(G2_thread, JavaThread::is_method_handle_return_offset()), L0); __ tst(L0); // Condition codes are preserved over the restore. __ restore(); __ jmp(G5, 0); __ delayed()->movcc(Assembler::notZero, false, Assembler::icc, L7_mh_SP_save, SP); // Restore SP if required. } break; case throw_array_store_exception_id: { __ set_info("throw_array_store_exception", dont_gc_arguments); oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true); } break; case throw_class_cast_exception_id: { // G4: object __ set_info("throw_class_cast_exception", dont_gc_arguments); oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true); } break; case throw_incompatible_class_change_error_id: { __ set_info("throw_incompatible_class_cast_exception", dont_gc_arguments); oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false); } break; case slow_subtype_check_id: { // Support for uint StubRoutine::partial_subtype_check( Klass sub, Klass super ); // Arguments : // // ret : G3 // sub : G3, argument, destroyed // super: G1, argument, not changed // raddr: O7, blown by call Label miss; __ save_frame(0); // Blow no registers! __ check_klass_subtype_slow_path(G3, G1, L0, L1, L2, L4, NULL, &miss); __ mov(1, G3); __ ret(); // Result in G5 is 'true' __ delayed()->restore(); // free copy or add can go here __ bind(miss); __ mov(0, G3); __ ret(); // Result in G5 is 'false' __ delayed()->restore(); // free copy or add can go here } case monitorenter_nofpu_id: case monitorenter_id: { // G4: object // G5: lock address __ set_info("monitorenter", dont_gc_arguments); int save_fpu_registers = (id == monitorenter_id); // make a frame and preserve the caller's caller-save registers OopMap* oop_map = save_live_registers(sasm, save_fpu_registers); int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), G4, G5); oop_maps = new OopMapSet(); oop_maps->add_gc_map(call_offset, oop_map); restore_live_registers(sasm, save_fpu_registers); __ ret(); __ delayed()->restore(); } break; case monitorexit_nofpu_id: case monitorexit_id: { // G4: lock address // note: really a leaf routine but must setup last java sp // => use call_RT for now (speed can be improved by // doing last java sp setup manually) __ set_info("monitorexit", dont_gc_arguments); int save_fpu_registers = (id == monitorexit_id); // make a frame and preserve the caller's caller-save registers OopMap* oop_map = save_live_registers(sasm, save_fpu_registers); int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), G4); oop_maps = new OopMapSet(); oop_maps->add_gc_map(call_offset, oop_map); restore_live_registers(sasm, save_fpu_registers); __ ret(); __ delayed()->restore(); } break; case deoptimize_id: { __ set_info("deoptimize", dont_gc_arguments); OopMap* oop_map = save_live_registers(sasm); int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize)); oop_maps = new OopMapSet(); oop_maps->add_gc_map(call_offset, oop_map); restore_live_registers(sasm); DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); assert(deopt_blob != NULL, "deoptimization blob must have been created"); AddressLiteral dest(deopt_blob->unpack_with_reexecution()); __ jump_to(dest, O0); __ delayed()->restore(); } break; case access_field_patching_id: { __ set_info("access_field_patching", dont_gc_arguments); oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching)); } break; case load_klass_patching_id: { __ set_info("load_klass_patching", dont_gc_arguments); oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching)); } break; case load_mirror_patching_id: { __ set_info("load_mirror_patching", dont_gc_arguments); oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching)); } break; case dtrace_object_alloc_id: { // O0: object __ set_info("dtrace_object_alloc", dont_gc_arguments); // we can't gc here so skip the oopmap but make sure that all // the live registers get saved. save_live_registers(sasm); __ save_thread(L7_thread_cache); __ call(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), relocInfo::runtime_call_type); __ delayed()->mov(I0, O0); __ restore_thread(L7_thread_cache); restore_live_registers(sasm); __ ret(); __ delayed()->restore(); } break; #if INCLUDE_ALL_GCS case g1_pre_barrier_slow_id: { // G4: previous value of memory BarrierSet* bs = Universe::heap()->barrier_set(); if (bs->kind() != BarrierSet::G1SATBCTLogging) { __ save_frame(0); __ set((int)id, O1); __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), I0); __ should_not_reach_here(); break; } __ set_info("g1_pre_barrier_slow_id", dont_gc_arguments); Register pre_val = G4; Register tmp = G1_scratch; Register tmp2 = G3_scratch; Label refill, restart; bool with_frame = false; // I don't know if we can do with-frame. int satb_q_index_byte_offset = in_bytes(JavaThread::satb_mark_queue_offset() + PtrQueue::byte_offset_of_index()); int satb_q_buf_byte_offset = in_bytes(JavaThread::satb_mark_queue_offset() + PtrQueue::byte_offset_of_buf()); __ bind(restart); // Load the index into the SATB buffer. PtrQueue::_index is a // size_t so ld_ptr is appropriate __ ld_ptr(G2_thread, satb_q_index_byte_offset, tmp); // index == 0? __ cmp_and_brx_short(tmp, G0, Assembler::equal, Assembler::pn, refill); __ ld_ptr(G2_thread, satb_q_buf_byte_offset, tmp2); __ sub(tmp, oopSize, tmp); __ st_ptr(pre_val, tmp2, tmp); // [_buf + index] := <address_of_card> // Use return-from-leaf __ retl(); __ delayed()->st_ptr(tmp, G2_thread, satb_q_index_byte_offset); __ bind(refill); __ save_frame(0); __ mov(pre_val, L0); __ mov(tmp, L1); __ mov(tmp2, L2); __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, SATBMarkQueueSet::handle_zero_index_for_thread), G2_thread); __ mov(L0, pre_val); __ mov(L1, tmp); __ mov(L2, tmp2); __ br(Assembler::always, /*annul*/false, Assembler::pt, restart); __ delayed()->restore(); } break; case g1_post_barrier_slow_id: { BarrierSet* bs = Universe::heap()->barrier_set(); if (bs->kind() != BarrierSet::G1SATBCTLogging) { __ save_frame(0); __ set((int)id, O1); __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), I0); __ should_not_reach_here(); break; } __ set_info("g1_post_barrier_slow_id", dont_gc_arguments); Register addr = G4; Register cardtable = G5; Register tmp = G1_scratch; Register tmp2 = G3_scratch; jbyte* byte_map_base = ((CardTableModRefBS*)bs)->byte_map_base; Label not_already_dirty, restart, refill; #ifdef _LP64 __ srlx(addr, CardTableModRefBS::card_shift, addr); #else __ srl(addr, CardTableModRefBS::card_shift, addr); #endif AddressLiteral rs(byte_map_base); __ set(rs, cardtable); // cardtable := <card table base> __ ldub(addr, cardtable, tmp); // tmp := [addr + cardtable] assert(CardTableModRefBS::dirty_card_val() == 0, "otherwise check this code"); __ cmp_and_br_short(tmp, G0, Assembler::notEqual, Assembler::pt, not_already_dirty); // We didn't take the branch, so we're already dirty: return. // Use return-from-leaf __ retl(); __ delayed()->nop(); // Not dirty. __ bind(not_already_dirty); // Get cardtable + tmp into a reg by itself __ add(addr, cardtable, tmp2); // First, dirty it. __ stb(G0, tmp2, 0); // [cardPtr] := 0 (i.e., dirty). Register tmp3 = cardtable; Register tmp4 = tmp; // these registers are now dead addr = cardtable = tmp = noreg; int dirty_card_q_index_byte_offset = in_bytes(JavaThread::dirty_card_queue_offset() + PtrQueue::byte_offset_of_index()); int dirty_card_q_buf_byte_offset = in_bytes(JavaThread::dirty_card_queue_offset() + PtrQueue::byte_offset_of_buf()); __ bind(restart); // Get the index into the update buffer. PtrQueue::_index is // a size_t so ld_ptr is appropriate here. __ ld_ptr(G2_thread, dirty_card_q_index_byte_offset, tmp3); // index == 0? __ cmp_and_brx_short(tmp3, G0, Assembler::equal, Assembler::pn, refill); __ ld_ptr(G2_thread, dirty_card_q_buf_byte_offset, tmp4); __ sub(tmp3, oopSize, tmp3); __ st_ptr(tmp2, tmp4, tmp3); // [_buf + index] := <address_of_card> // Use return-from-leaf __ retl(); __ delayed()->st_ptr(tmp3, G2_thread, dirty_card_q_index_byte_offset); __ bind(refill); __ save_frame(0); __ mov(tmp2, L0); __ mov(tmp3, L1); __ mov(tmp4, L2); __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, DirtyCardQueueSet::handle_zero_index_for_thread), G2_thread); __ mov(L0, tmp2); __ mov(L1, tmp3); __ mov(L2, tmp4); __ br(Assembler::always, /*annul*/false, Assembler::pt, restart); __ delayed()->restore(); } break; #endif // INCLUDE_ALL_GCS case predicate_failed_trap_id: { __ set_info("predicate_failed_trap", dont_gc_arguments); OopMap* oop_map = save_live_registers(sasm); int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap)); oop_maps = new OopMapSet(); oop_maps->add_gc_map(call_offset, oop_map); DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); assert(deopt_blob != NULL, "deoptimization blob must have been created"); restore_live_registers(sasm); AddressLiteral dest(deopt_blob->unpack_with_reexecution()); __ jump_to(dest, O0); __ delayed()->restore(); } break; default: { __ set_info("unimplemented entry", dont_gc_arguments); __ save_frame(0); __ set((int)id, O1); __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), O1); __ should_not_reach_here(); } break; } return oop_maps; }
address InterpreterGenerator::generate_normal_entry(bool synchronized) { assert_different_registers(Rmethod, Rlocals, Rthread, Rstate, Rmonitor); Label re_dispatch; Label call_interpreter; Label call_method; Label call_non_interpreted_method; Label return_with_exception; Label return_from_method; Label resume_interpreter; Label return_to_initial_caller; Label more_monitors; Label throwing_exception; // We use the same code for synchronized and not if (normal_entry) return normal_entry; address start = __ pc(); // There are two ways in which we can arrive at this entry. // There is the special case where a normal interpreted method // calls another normal interpreted method, and there is the // general case of when we enter from somewhere else: from // call_stub, from C1 or C2, or from a fast accessor which // deferred. In the special case we're already in frame manager // code: we arrive at re_dispatch with Rstate containing the // previous interpreter state. In the general case we arrive // at start with no previous interpreter state so we set Rstate // to NULL to indicate this. __ bind (fast_accessor_slow_entry_path); __ load (Rstate, 0); __ bind (re_dispatch); // Adjust the caller's stack frame to accomodate any additional // local variables we have contiguously with our parameters. generate_adjust_callers_stack(); // Allocate and initialize our stack frame. generate_compute_interpreter_state(false); // Call the interpreter ============================================== __ bind (call_interpreter); // We can setup the frame anchor with everything we want at // this point as we are thread_in_Java and no safepoints can // occur until we go to vm mode. We do have to clear flags // on return from vm but that is it __ set_last_Java_frame (); // Call interpreter address interpreter = JvmtiExport::can_post_interpreter_events() ? CAST_FROM_FN_PTR(address, BytecodeInterpreter::runWithChecks) : CAST_FROM_FN_PTR(address, BytecodeInterpreter::run); __ mr (r3, Rstate); __ call (interpreter); __ fixup_after_potential_safepoint (); // Clear the frame anchor __ reset_last_Java_frame (); // Examine the message from the interpreter to decide what to do __ lwz (r4, STATE(_msg)); __ compare (r4, BytecodeInterpreter::call_method); __ beq (call_method); __ compare (r4, BytecodeInterpreter::return_from_method); __ beq (return_from_method); __ compare (r4, BytecodeInterpreter::more_monitors); __ beq (more_monitors); __ compare (r4, BytecodeInterpreter::throwing_exception); __ beq (throwing_exception); __ load (r3, (intptr_t) "error: bad message from interpreter: %d\n"); __ call (CAST_FROM_FN_PTR(address, printf)); __ should_not_reach_here (__FILE__, __LINE__); // Handle a call_method message ====================================== __ bind (call_method); __ load (Rmethod, STATE(_result._to_call._callee)); __ verify_oop(Rmethod); __ load (Rlocals, STATE(_stack)); __ lhz (r0, Address(Rmethod, methodOopDesc::size_of_parameters_offset())); __ shift_left (r0, r0, LogBytesPerWord); __ add (Rlocals, Rlocals, r0); __ load (r0, STATE(_result._to_call._callee_entry_point)); __ load (r3, (intptr_t) start); __ compare (r0, r3); __ bne (call_non_interpreted_method); // Interpreted methods are intercepted and re-dispatched ----------- __ load (r0, CAST_FROM_FN_PTR(intptr_t, RecursiveInterpreterActivation)); __ mtlr (r0); __ b (re_dispatch); // Non-interpreted methods are dispatched normally ----------------- __ bind (call_non_interpreted_method); __ mtctr (r0); __ bctrl (); // Restore Rstate __ load (Rstate, Address(r1, StackFrame::back_chain_offset * wordSize)); __ subi (Rstate, Rstate, sizeof(BytecodeInterpreter)); // Check for pending exceptions __ load (r0, Address(Rthread, Thread::pending_exception_offset())); __ compare (r0, 0); __ bne (return_with_exception); // Convert the result and resume generate_convert_result(CppInterpreter::_tosca_to_stack); __ b (resume_interpreter); // Handle a return_from_method message =============================== __ bind (return_from_method); __ load (r0, STATE(_prev_link)); __ compare (r0, 0); __ beq (return_to_initial_caller); // "Return" from a re-dispatch ------------------------------------- generate_convert_result(CppInterpreter::_stack_to_stack); generate_unwind_interpreter_state(); // Resume the interpreter __ bind (resume_interpreter); __ store (Rlocals, STATE(_stack)); __ load (Rlocals, STATE(_locals)); __ load (Rmethod, STATE(_method)); __ verify_oop(Rmethod); __ load (r0, BytecodeInterpreter::method_resume); __ stw (r0, STATE(_msg)); __ b (call_interpreter); // Return to the initial caller (call_stub etc) -------------------- __ bind (return_to_initial_caller); generate_convert_result(CppInterpreter::_stack_to_native_abi); generate_unwind_interpreter_state(); __ blr (); // Handle a more_monitors message ==================================== __ bind (more_monitors); generate_more_monitors(); __ load (r0, BytecodeInterpreter::got_monitors); __ stw (r0, STATE(_msg)); __ b (call_interpreter); // Handle a throwing_exception message =============================== __ bind (throwing_exception); // Check we actually have an exception #ifdef ASSERT { Label ok; __ load (r0, Address(Rthread, Thread::pending_exception_offset())); __ compare (r0, 0); __ bne (ok); __ should_not_reach_here (__FILE__, __LINE__); __ bind (ok); } #endif // Return to wherever generate_unwind_interpreter_state(); __ bind (return_with_exception); __ compare (Rstate, 0); __ bne (resume_interpreter); __ blr (); normal_entry = start; return start; }
address InterpreterGenerator::generate_native_entry(bool synchronized) { const Register handler = r14; const Register function = r15; assert_different_registers(Rmethod, Rlocals, Rthread, Rstate, Rmonitor, handler, function); // We use the same code for synchronized and not if (native_entry) return native_entry; address start = __ pc(); // Allocate and initialize our stack frame. __ load (Rstate, 0); generate_compute_interpreter_state(true); // Make sure method is native and not abstract #ifdef ASSERT { Label ok; __ lwz (r0, Address(Rmethod, methodOopDesc::access_flags_offset())); __ andi_ (r0, r0, JVM_ACC_NATIVE | JVM_ACC_ABSTRACT); __ compare (r0, JVM_ACC_NATIVE); __ beq (ok); __ should_not_reach_here (__FILE__, __LINE__); __ bind (ok); } #endif // Lock if necessary Label not_synchronized_1; __ bne (CRsync, not_synchronized_1); __ lock_object (Rmonitor); __ bind (not_synchronized_1); // Get signature handler const Address signature_handler_addr( Rmethod, methodOopDesc::signature_handler_offset()); Label return_to_caller, got_signature_handler; __ load (handler, signature_handler_addr); __ compare (handler, 0); __ bne (got_signature_handler); __ call_VM (noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), Rmethod, CALL_VM_NO_EXCEPTION_CHECKS); __ load (r0, Address(Rthread, Thread::pending_exception_offset())); __ compare (r0, 0); __ bne (return_to_caller); __ load (handler, signature_handler_addr); __ bind (got_signature_handler); // Get the native function entry point const Address native_function_addr( Rmethod, methodOopDesc::native_function_offset()); Label got_function; __ load (function, native_function_addr); #ifdef ASSERT { // InterpreterRuntime::prepare_native_call() sets the mirror // handle and native function address first and the signature // handler last, so function should always be set here. Label ok; __ compare (function, 0); __ bne (ok); __ should_not_reach_here (__FILE__, __LINE__); __ bind (ok); } #endif // Call signature handler __ mtctr (handler); __ bctrl (); __ mr (handler, r0); // Pass JNIEnv __ la (r3, Address(Rthread, JavaThread::jni_environment_offset())); // Pass mirror handle if static const Address oop_temp_addr = STATE(_oop_temp); Label not_static; __ bne (CRstatic, not_static); __ get_mirror_handle (r4); __ store (r4, oop_temp_addr); __ la (r4, oop_temp_addr); __ bind (not_static); // Set up the Java frame anchor __ set_last_Java_frame (); // Change the thread state to native const Address thread_state_addr(Rthread, JavaThread::thread_state_offset()); #ifdef ASSERT { Label ok; __ lwz (r0, thread_state_addr); __ compare (r0, _thread_in_Java); __ beq (ok); __ should_not_reach_here (__FILE__, __LINE__); __ bind (ok); } #endif __ load (r0, _thread_in_native); __ stw (r0, thread_state_addr); // Make the call __ call (function); __ fixup_after_potential_safepoint (); // The result will be in r3 (and maybe r4 on 32-bit) or f1. // Wherever it is, we need to store it before calling anything const Register r3_save = r16; #ifdef PPC32 const Register r4_save = r17; #endif const FloatRegister f1_save = f14; __ mr (r3_save, r3); #ifdef PPC32 __ mr (r4_save, r4); #endif __ fmr (f1_save, f1); // Switch thread to "native transition" state before reading the // synchronization state. This additional state is necessary // because reading and testing the synchronization state is not // atomic with respect to garbage collection. __ load (r0, _thread_in_native_trans); __ stw (r0, thread_state_addr); // Ensure the new state is visible to the VM thread. if(os::is_MP()) { if (UseMembar) __ sync (); else __ serialize_memory (r3, r4); } // Check for safepoint operation in progress and/or pending // suspend requests. We use a leaf call in order to leave // the last_Java_frame setup undisturbed. Label block, no_block; __ load (r3, (intptr_t) SafepointSynchronize::address_of_state()); __ lwz (r0, Address(r3, 0)); __ compare (r0, SafepointSynchronize::_not_synchronized); __ bne (block); __ lwz (r0, Address(Rthread, JavaThread::suspend_flags_offset())); __ compare (r0, 0); __ beq (no_block); __ bind (block); __ call_VM_leaf ( CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)); __ fixup_after_potential_safepoint (); __ bind (no_block); // Change the thread state __ load (r0, _thread_in_Java); __ stw (r0, thread_state_addr); // Reset the frame anchor __ reset_last_Java_frame (); // If the result was an OOP then unbox it and store it in the frame // (where it will be safe from garbage collection) before we release // the handle it might be protected by Label non_oop, store_oop; __ load (r0, (intptr_t) AbstractInterpreter::result_handler(T_OBJECT)); __ compare (r0, handler); __ bne (non_oop); __ compare (r3_save, 0); __ beq (store_oop); __ load (r3_save, Address(r3_save, 0)); __ bind (store_oop); __ store (r3_save, STATE(_oop_temp)); __ bind (non_oop); // Reset handle block __ load (r3, Address(Rthread, JavaThread::active_handles_offset())); __ load (r0, 0); __ stw (r0, Address(r3, JNIHandleBlock::top_offset_in_bytes())); // If there is an exception we skip the result handler and return. // Note that this also skips unlocking which seems totally wrong, // but apparently this is what the asm interpreter does so we do // too. __ load (r0, Address(Rthread, Thread::pending_exception_offset())); __ compare (r0, 0); __ bne (return_to_caller); // Unlock if necessary Label not_synchronized_2; __ bne (CRsync, not_synchronized_2); __ unlock_object (Rmonitor); __ bind (not_synchronized_2); // Restore saved result and call the result handler __ mr (r3, r3_save); #ifdef PPC32 __ mr (r4, r4_save); #endif __ fmr (f1, f1_save); __ mtctr (handler); __ bctrl (); // Unwind the current activation and return __ bind (return_to_caller); generate_unwind_interpreter_state(); __ blr (); native_entry = start; return start; }
void MacroAssembler::atomic_cas(Register temp1, Register temp2, Register oldval, Register newval, Register base, int offset) { if (temp1 != R0) { // try to read the previous value directly in R0 if (temp2 == R0) { // R0 declared free temp2 = temp1; temp1 = R0; } else if ((oldval != R0) && (newval != R0) && (base != R0)) { // free, and scratched on return temp1 = R0; } } if (VM_Version::supports_ldrex()) { Label loop; assert_different_registers(temp1, temp2, oldval, newval, base); bind(loop); ldrex(temp1, Address(base, offset)); cmp(temp1, oldval); strex(temp2, newval, Address(base, offset), eq); cmp(temp2, 1, eq); b(loop, eq); if (temp1 != R0) { mov(R0, temp1); } } else if (VM_Version::supports_kuser_cmpxchg32()) { // On armv5 platforms we must use the Linux kernel helper // function for atomic cas operations since ldrex/strex is // not supported. // // This is a special routine at a fixed address 0xffff0fc0 // // input: // r0 = oldval, r1 = newval, r2 = ptr, lr = return adress // output: // r0 = 0 carry set on success // r0 != 0 carry clear on failure // // r3, ip and flags are clobbered // Label done; Label loop; push(RegisterSet(R1, R4) | RegisterSet(R12) | RegisterSet(LR)); if ( oldval != R0 || newval != R1 || base != R2 ) { push(oldval); push(newval); push(base); pop(R2); pop(R1); pop(R0); } if (offset != 0) { add(R2, R2, offset); } mov(R4, R0); bind(loop); ldr(R0, Address(R2)); cmp(R0, R4); b(done, ne); mvn(R12, 0xf000); mov(LR, PC); sub(PC, R12, 0x3f); b(loop, cc); mov(R0, R4); bind(done); pop(RegisterSet(R1, R4) | RegisterSet(R12) | RegisterSet(LR)); } else { // Should never run on a platform so old that it does not have kernel helper stop("Atomic cmpxchg32 unsupported on this platform"); } }
inline void MacroAssembler::call_stub_and_return_to(Register function_entry, Register return_pc) { assert_different_registers(function_entry, return_pc); mtlr(return_pc); mtctr(function_entry); bctr(); }
void MacroAssembler::atomic_cas64(Register memval_lo, Register memval_hi, Register result, Register oldval_lo, Register oldval_hi, Register newval_lo, Register newval_hi, Register base, int offset) { if (VM_Version::supports_ldrexd()) { Label loop; assert_different_registers(memval_lo, memval_hi, result, oldval_lo, oldval_hi, newval_lo, newval_hi, base); assert(memval_hi == memval_lo + 1 && memval_lo < R9, "cmpxchg_long: illegal registers"); assert(oldval_hi == oldval_lo + 1 && oldval_lo < R9, "cmpxchg_long: illegal registers"); assert(newval_hi == newval_lo + 1 && newval_lo < R9, "cmpxchg_long: illegal registers"); assert(result != R10, "cmpxchg_long: illegal registers"); assert(base != R10, "cmpxchg_long: illegal registers"); mov(result, 0); bind(loop); ldrexd(memval_lo, Address(base, offset)); cmp(memval_lo, oldval_lo); cmp(memval_hi, oldval_hi, eq); strexd(result, newval_lo, Address(base, offset), eq); rsbs(result, result, 1, eq); b(loop, eq); } else if (VM_Version::supports_kuser_cmpxchg64()) { // On armv5 platforms we must use the Linux kernel helper // function for atomic cas64 operations since ldrexd/strexd is // not supported. // // This is a special routine at a fixed address 0xffff0f60 // // input: // r0 = (long long *)oldval, r1 = (long long *)newval, // r2 = ptr, lr = return adress // output: // r0 = 0 carry set on success // r0 != 0 carry clear on failure // // r3, and flags are clobbered // Label done; Label loop; if (result != R12) { push(R12); } push(RegisterSet(R10) | RegisterSet(LR)); mov(R10, SP); // Save SP bic(SP, SP, StackAlignmentInBytes - 1); // align stack push(RegisterSet(oldval_lo, oldval_hi)); push(RegisterSet(newval_lo, newval_hi)); if ((offset != 0) || (base != R12)) { add(R12, base, offset); } push(RegisterSet(R0, R3)); bind(loop); ldrd(memval_lo, Address(R12)); //current ldrd(oldval_lo, Address(SP, 24)); cmp(memval_lo, oldval_lo); cmp(memval_hi, oldval_hi, eq); pop(RegisterSet(R0, R3), ne); mov(result, 0, ne); b(done, ne); // Setup for kernel call mov(R2, R12); add(R0, SP, 24); // R0 == &oldval_lo add(R1, SP, 16); // R1 == &newval_lo mvn(R3, 0xf000); // call kernel helper at 0xffff0f60 mov(LR, PC); sub(PC, R3, 0x9f); b(loop, cc); // if Carry clear then oldval != current // try again. Otherwise, return oldval // Here on success pop(RegisterSet(R0, R3)); mov(result, 1); ldrd(memval_lo, Address(SP, 8)); bind(done); pop(RegisterSet(newval_lo, newval_hi)); pop(RegisterSet(oldval_lo, oldval_hi)); mov(SP, R10); // restore SP pop(RegisterSet(R10) | RegisterSet(LR)); if (result != R12) { pop(R12); } } else { stop("Atomic cmpxchg64 unsupported on this platform"); } }