void test_specialize_transforms_ivars_to_slots() {
CompiledCode* code = CompiledCode::create(state);
Tuple* tup = Tuple::from(state, 1, state->symbol("@blah"));
code->literals(state, tup);
InstructionSequence* iseq = InstructionSequence::create(state, 3);
iseq->opcodes()->put(state, 0, Fixnum::from(InstructionSequence::insn_push_ivar));
iseq->opcodes()->put(state, 1, Fixnum::from(0));
iseq->opcodes()->put(state, 2, Fixnum::from(InstructionSequence::insn_push_nil));
code->iseq(state, iseq);
MachineCode* mcode = new MachineCode(state, code);
Object::Info ti(ObjectType);
ti.slots[state->symbol("@blah")->index()] = 5;
ti.slot_locations.resize(6);
ti.slot_locations[5] = 33;
mcode->specialize(state, code, &ti);
TS_ASSERT_EQUALS(mcode->total, 3U);
TS_ASSERT_EQUALS(mcode->opcodes[0], static_cast<unsigned int>(InstructionSequence::insn_push_my_offset));
TS_ASSERT_EQUALS(mcode->opcodes[1], 33U);
TS_ASSERT_EQUALS(mcode->opcodes[2], static_cast<unsigned int>(InstructionSequence::insn_push_nil));
}
void CompiledCode::add_specialized(uint32_t class_id, uint32_t serial_id, executor exec,
jit::RuntimeDataHolder* rd)
{
if(!machine_code_) rubinius::bug("specializing with no backend");
MachineCode* v = machine_code_;
for(int i = 0; i < MachineCode::cMaxSpecializations; i++) {
uint32_t id = v->specializations[i].class_data.f.class_id;
if(id == 0 || id == class_id) {
v->specializations[i].class_data.f.class_id = class_id;
v->specializations[i].class_data.f.serial_id = serial_id;
v->specializations[i].execute = exec;
v->specializations[i].jit_data = rd;
v->set_execute_status(MachineCode::eJIT);
if(primitive()->nil_p()) {
execute = specialized_executor;
}
return;
}
}
// No room for the specialization, this is bad.
std::cerr << "No room for specialization!\n";
}
BlockEnvironment* BlockEnvironment::under_call_frame(STATE, GCToken gct,
CompiledCode* ccode, MachineCode* caller,
CallFrame* call_frame)
{
MachineCode* mcode = ccode->machine_code();
if(!mcode) {
OnStack<1> os(state, ccode);
state->set_call_frame(call_frame);
mcode = ccode->internalize(state, gct);
if(!mcode) {
Exception::internal_error(state, call_frame, "invalid bytecode method");
return 0;
}
}
mcode->set_parent(caller);
BlockEnvironment* be = state->new_object_dirty<BlockEnvironment>(G(blokenv));
be->scope(state, call_frame->promote_scope(state));
be->top_scope(state, call_frame->top_scope(state));
be->compiled_code(state, ccode);
be->module(state, call_frame->module());
be->metadata_container(state, nil<Tuple>());
return be;
}
void CompiledCode::add_specialized(int spec_id, executor exec,
jit::RuntimeDataHolder* rd)
{
if(!machine_code_) rubinius::bug("specializing with no backend");
MachineCode* v = machine_code_;
// Must happen only on the first specialization
if(!v->unspecialized) {
if(execute == specialized_executor) {
rubinius::bug("cant setup unspecialized from specialized");
}
v->unspecialized = execute;
}
for(int i = 0; i < MachineCode::cMaxSpecializations; i++) {
int id = v->specializations[i].class_id;
if(id == 0 || id == spec_id) {
v->specializations[i].class_id = spec_id;
v->specializations[i].execute = exec;
v->specializations[i].jit_data = rd;
v->set_execute_status(MachineCode::eJIT);
execute = specialized_executor;
return;
}
}
// No room for the specialization, this is bad.
std::cerr << "No room for specialization!\n";
}
void CompiledCode::Info::mark(Object* obj, memory::ObjectMark& mark) {
auto_mark(obj, mark);
mark_inliners(obj, mark);
CompiledCode* code = as<CompiledCode>(obj);
if(!code->machine_code()) return;
MachineCode* mcode = code->machine_code();
mcode->set_mark();
for(int i = 0; i < MachineCode::cMaxSpecializations; i++) {
// TODO: JIT
}
for(size_t i = 0; i < mcode->references_count(); i++) {
if(size_t ip = mcode->references()[i]) {
Object* ref = reinterpret_cast<Object*>(mcode->opcodes[ip]);
if(Object* updated_ref = mark.call(ref)) {
mcode->opcodes[ip] = reinterpret_cast<intptr_t>(updated_ref);
mark.just_set(code, updated_ref);
}
}
}
}
Tuple* CompiledCode::constant_caches(STATE) {
CompiledCode* self = this;
OnStack<1> os(state, self);
if(self->machine_code() == NULL) {
if(!self->internalize(state)) return force_as<Tuple>(Primitives::failure());
}
MachineCode* mcode = self->machine_code();
return mcode->constant_caches(state);
}
Tuple* CompiledCode::constant_caches(STATE, CallFrame* calling_environment) {
GCTokenImpl gct;
CompiledCode* self = this;
OnStack<1> os(state, self);
if(self->machine_code_ == NULL) {
if(!self->internalize(state, gct, calling_environment)) return force_as<Tuple>(Primitives::failure());
}
MachineCode* mcode = self->machine_code_;
return mcode->constant_caches(state);
}
void CompiledCode::Info::mark(Object* obj, ObjectMark& mark) {
auto_mark(obj, mark);
mark_inliners(obj, mark);
CompiledCode* code = as<CompiledCode>(obj);
if(!code->machine_code_) return;
MachineCode* mcode = code->machine_code_;
mcode->set_mark();
Object* tmp;
#ifdef ENABLE_LLVM
if(code->jit_data()) {
code->jit_data()->set_mark();
code->jit_data()->mark_all(code, mark);
}
for(int i = 0; i < MachineCode::cMaxSpecializations; i++) {
if(mcode->specializations[i].jit_data) {
mcode->specializations[i].jit_data->set_mark();
mcode->specializations[i].jit_data->mark_all(code, mark);
}
}
#endif
for(size_t i = 0; i < mcode->inline_cache_count(); i++) {
InlineCache* cache = &mcode->caches[i];
for(int j = 0; j < cTrackedICHits; ++j) {
MethodCacheEntry* mce = cache->cache_[j].entry();
if(mce) {
tmp = mark.call(mce);
if(tmp) {
cache->cache_[j].assign(static_cast<MethodCacheEntry*>(tmp));
mark.just_set(obj, tmp);
}
}
}
if(cache->call_unit_) {
tmp = mark.call(cache->call_unit_);
if(tmp) {
cache->call_unit_ = static_cast<CallUnit*>(tmp);
mark.just_set(obj, tmp);
}
}
}
}
MachineCode* CompiledCode::internalize(STATE, GCToken gct,
const char** reason, int* ip)
{
MachineCode* mcode = machine_code_;
atomic::memory_barrier();
if(mcode) return mcode;
CompiledCode* self = this;
OnStack<1> os(state, self);
self->hard_lock(state, gct);
mcode = self->machine_code_;
if(!mcode) {
{
BytecodeVerification bv(self);
if(!bv.verify(state)) {
if(reason) *reason = bv.failure_reason();
if(ip) *ip = bv.failure_ip();
std::cerr << "Error validating bytecode: " << bv.failure_reason() << "\n";
return 0;
}
}
mcode = new MachineCode(state, self);
if(self->resolve_primitive(state)) {
mcode->fallback = execute;
} else {
mcode->setup_argument_handler();
}
// We need to have an explicit memory barrier here, because we need to
// be sure that mcode is completely initialized before it's set.
// Otherwise another thread might see a partially initialized
// MachineCode.
atomic::write(&self->machine_code_, mcode);
set_executor(mcode->fallback);
}
self->hard_unlock(state, gct);
return mcode;
}
void test_validate_ip() {
CompiledCode* code = CompiledCode::create(state);
Tuple* tup = Tuple::from(state, 1, state->symbol("@blah"));
code->literals(state, tup);
InstructionSequence* iseq = InstructionSequence::create(state, 3);
iseq->opcodes()->put(state, 0, Fixnum::from(InstructionSequence::insn_push_ivar));
iseq->opcodes()->put(state, 1, Fixnum::from(0));
iseq->opcodes()->put(state, 2, Fixnum::from(InstructionSequence::insn_push_nil));
code->iseq(state, iseq);
MachineCode* mcode = new MachineCode(state, code);
TS_ASSERT_EQUALS(mcode->validate_ip(state, 0), true);
TS_ASSERT_EQUALS(mcode->validate_ip(state, 1), false);
TS_ASSERT_EQUALS(mcode->validate_ip(state, 2), true);
}
void CompiledCode::add_specialized(STATE, uint32_t class_id, uint32_t serial_id, executor exec,
jit::RuntimeDataHolder* rd)
{
if(!machine_code_) {
utilities::logger::error("specializing with no backend");
return;
}
MachineCode* v = machine_code_;
int i;
for(i = 0; i < MachineCode::cMaxSpecializations; i++) {
uint32_t id = v->specializations[i].class_data.f.class_id;
if(id == 0 || id == class_id) break;
}
/* We have fixed space for specializations. If we exceed this, overwrite
* the first one. This should be converted to some sort of LRU cache.
*/
if(i == MachineCode::cMaxSpecializations) {
std::ostringstream msg;
msg << "Specialization space exceeded for " <<
machine_code_->name()->cpp_str(state);
utilities::logger::warn(msg.str().c_str());
i = 0;
}
v->specializations[i].class_data.f.class_id = class_id;
v->specializations[i].class_data.f.serial_id = serial_id;
v->specializations[i].execute = exec;
v->specializations[i].jit_data = rd;
v->set_execute_status(MachineCode::eJIT);
if(primitive()->nil_p()) {
execute = specialized_executor;
}
}
Location* Location::create(STATE, CallFrame* call_frame,
bool include_variables)
{
if(NativeMethodFrame* nmf = call_frame->native_method_frame()) {
return create(state, nmf);
}
Location* loc = state->new_object_dirty<Location>(G(location));
loc->method_module(state, call_frame->module());
loc->receiver(state, call_frame->self());
loc->method(state, call_frame->compiled_code);
loc->ip(state, Fixnum::from(call_frame->ip()));
loc->flags(state, Fixnum::from(0));
if(call_frame->is_block_p(state)) {
loc->name(state, call_frame->top_scope(state)->method()->name());
loc->set_is_block(state);
} else {
loc->name(state, call_frame->name());
}
MachineCode* mcode = call_frame->compiled_code->machine_code();
if(mcode && mcode->jitted_p()) {
loc->set_is_jit(state);
}
if(include_variables) {
// Use promote_scope because it can figure out of the generated
// VariableScope should be isolated by default (true atm for JITd
// frames)
loc->variables(state, call_frame->promote_scope(state));
} else {
loc->variables(state, nil<VariableScope>());
}
loc->constant_scope(state, call_frame->constant_scope());
return loc;
}
MachineCode* CompiledCode::internalize(STATE) {
timer::StopWatch<timer::microseconds> timer(
state->vm()->metrics().machine.bytecode_internalizer_us);
atomic::memory_barrier();
MachineCode* mcode = machine_code();
if(mcode) return mcode;
{
BytecodeVerifier bytecode_verifier(this);
bytecode_verifier.verify(state);
}
mcode = new MachineCode(state, this);
if(resolve_primitive(state)) {
mcode->fallback = execute;
} else {
mcode->setup_argument_handler();
}
/* There is a race here because another Thread may have run this
* CompiledCode instance and internalized it. We attempt to store our
* version assuming that we are the only ones to do so and throw away our
* work if someone else has beat us to it.
*/
MachineCode** mcode_ptr = &_machine_code_;
if(atomic::compare_and_swap(reinterpret_cast<void**>(mcode_ptr), 0, mcode)) {
set_executor(mcode->fallback);
return mcode;
} else {
return machine_code();
}
}
void CompiledCode::Info::mark(Object* obj, ObjectMark& mark) {
auto_mark(obj, mark);
mark_inliners(obj, mark);
CompiledCode* code = as<CompiledCode>(obj);
if(!code->machine_code_) return;
MachineCode* mcode = code->machine_code_;
mcode->set_mark();
#ifdef ENABLE_LLVM
if(code->jit_data()) {
code->jit_data()->set_mark();
code->jit_data()->mark_all(code, mark);
}
for(int i = 0; i < MachineCode::cMaxSpecializations; i++) {
if(mcode->specializations[i].jit_data) {
mcode->specializations[i].jit_data->set_mark();
mcode->specializations[i].jit_data->mark_all(code, mark);
}
}
#endif
for(size_t i = 0; i < mcode->call_site_count(); i++) {
size_t index = mcode->call_site_offsets()[i];
Object* old_cache = reinterpret_cast<Object*>(mcode->opcodes[index + 1]);
Object* new_cache = mark.call(old_cache);
if(new_cache != old_cache) {
mcode->opcodes[index + 1] = reinterpret_cast<intptr_t>(new_cache);
mark.just_set(code, new_cache);
}
}
for(size_t i = 0; i < mcode->constant_cache_count(); i++) {
size_t index = mcode->constant_cache_offsets()[i];
Object* old_cache = reinterpret_cast<Object*>(mcode->opcodes[index + 1]);
Object* new_cache = mark.call(old_cache);
if(new_cache != old_cache) {
mcode->opcodes[index + 1] = reinterpret_cast<intptr_t>(new_cache);
mark.just_set(code, new_cache);
}
}
}
CallFrame* LLVMState::find_candidate(STATE, CompiledCode* start, CallFrame* call_frame) {
if(!config_.jit_inline_generic) {
return call_frame;
}
int depth = config().jit_limit_search;
if(!start) {
throw CompileError("find_candidate: null start");
}
if(!call_frame) {
throw CompileError("find_candidate: null call frame");
}
// if(!start) {
// start = call_frame->compiled_code;
// call_frame = call_frame->previous;
// depth--;
// }
if(debug_search) {
std::cout << "> call_count: " << call_frame->compiled_code->machine_code()->call_count
<< " size: " << call_frame->compiled_code->machine_code()->total
<< " sends: " << call_frame->compiled_code->machine_code()->call_site_count()
<< std::endl;
call_frame->print_backtrace(state, 1);
}
if(start->machine_code()->total > (size_t)config_.jit_limit_inline_method) {
if(debug_search) {
std::cout << "JIT: STOP. reason: trigger method isn't small: "
<< start->machine_code()->total << " > "
<< config_.jit_limit_inline_method
<< std::endl;
}
return call_frame;
}
MachineCode* mcode = start->machine_code();
if(mcode->required_args != mcode->total_args) {
if(debug_search) {
std::cout << "JIT: STOP. reason: trigger method req_args != total_args" << std::endl;
}
return call_frame;
}
if(mcode->no_inline_p()) {
if(debug_search) {
std::cout << "JIT: STOP. reason: trigger method no_inline_p() = true" << std::endl;
}
return call_frame;
}
CallFrame* callee = call_frame;
call_frame = call_frame->previous;
if(!call_frame) return callee;
// Now start looking at callers.
while(depth-- > 0) {
CompiledCode* cur = call_frame->compiled_code;
if(!cur) {
if(debug_search) {
std::cout << "JIT: STOP. reason: synthetic CallFrame hit" << std::endl;
}
return callee;
}
MachineCode* mcode = cur->machine_code();
if(debug_search) {
std::cout << "> call_count: " << mcode->call_count
<< " size: " << mcode->total
<< " sends: " << mcode->call_site_count()
<< std::endl;
call_frame->print_backtrace(state, 1);
}
/*
if(call_frame->block_p()
|| mcode->required_args != mcode->total_args // has a splat
|| mcode->call_count < 200 // not called much
|| mcode->jitted() // already jitted
|| mcode->parent() // is a block
) return callee;
*/
if(mcode->required_args != mcode->total_args) {
if(debug_search) {
std::cout << "JIT: STOP. reason: req_args != total_args" << std::endl;
}
return callee;
}
if(mcode->call_count < config_.jit_threshold_inline) {
if(debug_search) {
std::cout << "JIT: STOP. reason: call_count too small: "
<< mcode->call_count << " < "
<< config_.jit_threshold_inline << std::endl;
}
return callee;
}
if(mcode->jitted_p()) {
if(debug_search) {
std::cout << "JIT: STOP. reason: already jitted" << std::endl;
}
return callee;
}
if(mcode->no_inline_p()) {
if(debug_search) {
std::cout << "JIT: STOP. reason: no_inline_p() = true" << std::endl;
}
return callee;
}
if(call_frame->jitted_p() || call_frame->inline_method_p()) {
return callee;
}
if(mcode->call_site_count() > eMaxInlineSendCount) {
if(debug_search) {
std::cout << "JIT: STOP. reason: high send count" << std::endl;
}
return call_frame;
}
// if(mcode->required_args != mcode->total_args // has a splat
// || mcode->call_count < 200 // not called much
// || mcode->jitted() // already jitted
// || !mcode->no_inline_p() // method marked as not inlineable
// ) return callee;
CallFrame* prev = call_frame->previous;
if(!prev) {
if(debug_search) {
std::cout << "JIT: STOP. reason: toplevel method" << std::endl;
}
return call_frame;
}
// if(cur->machine_code()->total > SMALL_METHOD_SIZE) {
// if(debug_search) {
// std::cout << "JIT: STOP. reason: big method: "
// << cur->machine_code()->total << " > "
// << SMALL_METHOD_SIZE
// << "\n";
// }
// return call_frame;
// }
// if(!next || cur->machine_code()->total > SMALL_METHOD_SIZE) return call_frame;
callee = call_frame;
call_frame = prev;
}
return callee;
}
Object* MachineCode::uncommon_interpreter(STATE,
MachineCode* const mcode,
CallFrame* const call_frame,
int32_t entry_ip,
native_int sp,
CallFrame* const method_call_frame,
jit::RuntimeDataHolder* rd,
UnwindInfoSet& thread_unwinds)
{
MachineCode* mc = method_call_frame->compiled_code->machine_code();
if(++mc->uncommon_count > state->shared().config.jit_deoptimize_threshold) {
if(state->shared().config.jit_uncommon_print) {
std::cerr << "[[[ Deoptimizing uncommon method ]]]\n";
call_frame->print_backtrace(state);
std::cerr << "Method Call Frame:\n";
method_call_frame->print_backtrace(state);
}
mc->uncommon_count = 0;
mc->deoptimize(state, method_call_frame->compiled_code, rd);
}
#include "vm/gen/instruction_locations.hpp"
opcode* stream = mcode->opcodes;
InterpreterState is;
GCTokenImpl gct;
Object** stack_ptr = call_frame->stk + sp;
UnwindInfoSet unwinds(thread_unwinds);
continue_to_run:
try {
#undef DISPATCH
#define DISPATCH goto *insn_locations[stream[call_frame->inc_ip()]];
#undef next_int
#undef cache_ip
#undef flush_ip
#define next_int ((opcode)(stream[call_frame->inc_ip()]))
#define cache_ip(which)
#define flush_ip()
#include "vm/gen/instruction_implementations.hpp"
} catch(TypeError& e) {
flush_ip();
Exception* exc =
Exception::make_type_error(state, e.type, e.object, e.reason);
exc->locations(state, Location::from_call_stack(state, call_frame));
state->raise_exception(exc);
call_frame->scope->flush_to_heap(state);
return NULL;
} catch(const RubyException& exc) {
exc.exception->locations(state,
Location::from_call_stack(state, call_frame));
state->raise_exception(exc.exception);
return NULL;
}
// No reason to be here!
rubinius::bug("Control flow error in interpreter");
exception:
VMThreadState* th = state->vm()->thread_state();
//
switch(th->raise_reason()) {
case cException:
if(unwinds.has_unwinds()) {
UnwindInfo info = unwinds.pop();
stack_position(info.stack_depth);
call_frame->set_ip(info.target_ip);
cache_ip(info.target_ip);
goto continue_to_run;
} else {
call_frame->scope->flush_to_heap(state);
return NULL;
}
case cBreak:
// If we're trying to break to here, we're done!
if(th->destination_scope() == call_frame->scope->on_heap()) {
stack_push(th->raise_value());
th->clear_break();
goto continue_to_run;
// Don't return here, because we want to loop back to the top
// and keep running this method.
}
// Otherwise, fall through and run the unwinds
case cReturn:
case cCatchThrow:
case cThreadKill:
// Otherwise, we're doing a long return/break unwind through
// here. We need to run ensure blocks.
while(unwinds.has_unwinds()) {
UnwindInfo info = unwinds.pop();
if(info.for_ensure()) {
stack_position(info.stack_depth);
call_frame->set_ip(info.target_ip);
cache_ip(info.target_ip);
// Don't reset ep here, we're still handling the return/break.
goto continue_to_run;
}
}
// Ok, no ensures to run.
if(th->raise_reason() == cReturn) {
call_frame->scope->flush_to_heap(state);
// If we're trying to return to here, we're done!
if(th->destination_scope() == call_frame->scope->on_heap()) {
Object* val = th->raise_value();
th->clear_return();
return val;
} else {
// Give control of this exception to the caller.
return NULL;
}
} else { // It's cBreak thats not for us!
call_frame->scope->flush_to_heap(state);
// Give control of this exception to the caller.
return NULL;
}
case cExit:
call_frame->scope->flush_to_heap(state);
return NULL;
default:
break;
} // switch
rubinius::bug("Control flow error in interpreter");
return NULL;
}
r_mint Env::machine_code_id(rmachine_code code) {
MachineCode* mcode = i(code);
return (mcode->method_id() << 1) | 1;
}
