void C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
const int hdr_offset = oopDesc::mark_offset_in_bytes();
assert_different_registers(hdr, obj, disp_hdr);
NearLabel done;
verify_oop(obj);
// Load object header.
z_lg(hdr, Address(obj, hdr_offset));
// Save object being locked into the BasicObjectLock...
z_stg(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
if (UseBiasedLocking) {
biased_locking_enter(obj, hdr, Z_R1_scratch, Z_R0_scratch, done, &slow_case);
}
// and mark it as unlocked.
z_oill(hdr, markOopDesc::unlocked_value);
// Save unlocked object header into the displaced header location on the stack.
z_stg(hdr, Address(disp_hdr, (intptr_t)0));
// Test if object header is still the same (i.e. unlocked), and if so, store the
// displaced header address in the object header. If it is not the same, get the
// object header instead.
z_csg(hdr, disp_hdr, hdr_offset, obj);
// If the object header was the same, we're done.
if (PrintBiasedLockingStatistics) {
Unimplemented();
#if 0
cond_inc32(Assembler::equal,
ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
#endif
}
branch_optimized(Assembler::bcondEqual, done);
// If the object header was not the same, it is now in the hdr register.
// => Test if it is a stack pointer into the same stack (recursive locking), i.e.:
//
// 1) (hdr & markOopDesc::lock_mask_in_place) == 0
// 2) rsp <= hdr
// 3) hdr <= rsp + page_size
//
// These 3 tests can be done by evaluating the following expression:
//
// (hdr - Z_SP) & (~(page_size-1) | markOopDesc::lock_mask_in_place)
//
// assuming both the stack pointer and page_size have their least
// significant 2 bits cleared and page_size is a power of 2
z_sgr(hdr, Z_SP);
load_const_optimized(Z_R0_scratch, (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
z_ngr(hdr, Z_R0_scratch); // AND sets CC (result eq/ne 0).
// For recursive locking, the result is zero. => Save it in the displaced header
// location (NULL in the displaced hdr location indicates recursive locking).
z_stg(hdr, Address(disp_hdr, (intptr_t)0));
// Otherwise we don't care about the result and handle locking via runtime call.
branch_optimized(Assembler::bcondNotZero, slow_case);
// done
bind(done);
}
void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox, Register Rscratch, Label& slow_case) {
assert_different_registers(Rmark, Roop, Rbox, Rscratch);
Label done, cas_failed, slow_int;
// The following move must be the first instruction of emitted since debug
// information may be generated for it.
// Load object header.
ld(Rmark, oopDesc::mark_offset_in_bytes(), Roop);
verify_oop(Roop);
// Save object being locked into the BasicObjectLock...
std(Roop, BasicObjectLock::obj_offset_in_bytes(), Rbox);
if (UseBiasedLocking) {
biased_locking_enter(CCR0, Roop, Rmark, Rscratch, R0, done, &slow_int);
}
// ... and mark it unlocked.
ori(Rmark, Rmark, markOopDesc::unlocked_value);
// Save unlocked object header into the displaced header location on the stack.
std(Rmark, BasicLock::displaced_header_offset_in_bytes(), Rbox);
// Compare object markOop with Rmark and if equal exchange Rscratch with object markOop.
assert(oopDesc::mark_offset_in_bytes() == 0, "cas must take a zero displacement");
cmpxchgd(/*flag=*/CCR0,
/*current_value=*/Rscratch,
/*compare_value=*/Rmark,
/*exchange_value=*/Rbox,
/*where=*/Roop/*+0==mark_offset_in_bytes*/,
MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
MacroAssembler::cmpxchgx_hint_acquire_lock(),
noreg,
&cas_failed,
/*check without membar and ldarx first*/true);
// If compare/exchange succeeded we found an unlocked object and we now have locked it
// hence we are done.
b(done);
bind(slow_int);
b(slow_case); // far
bind(cas_failed);
// We did not find an unlocked object so see if this is a recursive case.
sub(Rscratch, Rscratch, R1_SP);
load_const_optimized(R0, (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place));
and_(R0/*==0?*/, Rscratch, R0);
std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), Rbox);
bne(CCR0, slow_int);
bind(done);
}
void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox, Register Rscratch, Label& slow_case) {
assert_different_registers(Rmark, Roop, Rbox, Rscratch);
Label done;
Address mark_addr(Roop, oopDesc::mark_offset_in_bytes());
// The following move must be the first instruction of emitted since debug
// information may be generated for it.
// Load object header
ld_ptr(mark_addr, Rmark);
verify_oop(Roop);
// save object being locked into the BasicObjectLock
st_ptr(Roop, Rbox, BasicObjectLock::obj_offset_in_bytes());
if (UseBiasedLocking) {
biased_locking_enter(Roop, Rmark, Rscratch, done, &slow_case);
}
// Save Rbox in Rscratch to be used for the cas operation
mov(Rbox, Rscratch);
// and mark it unlocked
or3(Rmark, markOopDesc::unlocked_value, Rmark);
// save unlocked object header into the displaced header location on the stack
st_ptr(Rmark, Rbox, BasicLock::displaced_header_offset_in_bytes());
// compare object markOop with Rmark and if equal exchange Rscratch with object markOop
assert(mark_addr.disp() == 0, "cas must take a zero displacement");
cas_ptr(mark_addr.base(), Rmark, Rscratch);
// if compare/exchange succeeded we found an unlocked object and we now have locked it
// hence we are done
cmp(Rmark, Rscratch);
brx(Assembler::equal, false, Assembler::pt, done);
delayed()->sub(Rscratch, SP, Rscratch); //pull next instruction into delay slot
// we did not find an unlocked object so see if this is a recursive case
// sub(Rscratch, SP, Rscratch);
assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
andcc(Rscratch, 0xfffff003, Rscratch);
brx(Assembler::notZero, false, Assembler::pn, slow_case);
delayed()->st_ptr(Rscratch, Rbox, BasicLock::displaced_header_offset_in_bytes());
bind(done);
}
int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
Label done;
int null_check_offset = -1;
verify_oop(obj);
// save object being locked into the BasicObjectLock
movptr(Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()), obj);
if (UseBiasedLocking) {
assert(scratch != noreg, "should have scratch register at this point");
null_check_offset = biased_locking_enter(disp_hdr, obj, hdr, scratch, false, done, &slow_case);
} else {
null_check_offset = offset();
}
// Load object header
movptr(hdr, Address(obj, hdr_offset));
// and mark it as unlocked
orptr(hdr, markOopDesc::unlocked_value);
// save unlocked object header into the displaced header location on the stack
movptr(Address(disp_hdr, 0), hdr);
// test if object header is still the same (i.e. unlocked), and if so, store the
// displaced header address in the object header - if it is not the same, get the
// object header instead
if (os::is_MP()) MacroAssembler::lock(); // must be immediately before cmpxchg!
cmpxchgptr(disp_hdr, Address(obj, hdr_offset));
// if the object header was the same, we're done
if (PrintBiasedLockingStatistics) {
cond_inc32(Assembler::equal,
ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
}
jcc(Assembler::equal, done);
// if the object header was not the same, it is now in the hdr register
// => test if it is a stack pointer into the same stack (recursive locking), i.e.:
//
// 1) (hdr & aligned_mask) == 0
// 2) rsp <= hdr
// 3) hdr <= rsp + page_size
//
// these 3 tests can be done by evaluating the following expression:
//
// (hdr - rsp) & (aligned_mask - page_size)
//
// assuming both the stack pointer and page_size have their least
// significant 2 bits cleared and page_size is a power of 2
subptr(hdr, rsp);
andptr(hdr, aligned_mask - os::vm_page_size());
// for recursive locking, the result is zero => save it in the displaced header
// location (NULL in the displaced hdr location indicates recursive locking)
movptr(Address(disp_hdr, 0), hdr);
// otherwise we don't care about the result and handle locking via runtime call
jcc(Assembler::notZero, slow_case);
// done
bind(done);
return null_check_offset;
}
