Sint erts_complete_off_heap_message_queue_change(Process *c_p) { int reds = 1; ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); /* * This job was first initiated when the process changed to off heap * message queue management. Since then ERTS_PSFLG_OFF_HEAP_MSGQ * has been set. However, the management state might have been changed * again (multiple times) since then. Check users last requested state * (the flags F_OFF_HEAP_MSGQ, and F_ON_HEAP_MSGQ), and make the state * consistent with that. */ if (!(c_p->flags & F_OFF_HEAP_MSGQ)) erts_smp_atomic32_read_band_nob(&c_p->state, ~ERTS_PSFLG_OFF_HEAP_MSGQ); else { reds += 2; erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); reds += erts_move_messages_off_heap(c_p); } c_p->flags &= ~F_OFF_HEAP_MSGQ_CHNG; return reds; }
static void init_shared_memory(int argc, char **argv) { #ifdef HYBRID int arg_size = 0; global_heap_sz = erts_next_heap_size(global_heap_sz,0); /* Make sure arguments will fit on the heap, no one else will check! */ while (argc--) arg_size += 2 + strlen(argv[argc]); if (global_heap_sz < arg_size) global_heap_sz = erts_next_heap_size(arg_size,1); #ifndef INCREMENTAL global_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm) * global_heap_sz); global_hend = global_heap + global_heap_sz; global_htop = global_heap; global_high_water = global_heap; global_old_hend = global_old_htop = global_old_heap = NULL; #endif global_gen_gcs = 0; global_max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); global_gc_flags = erts_default_process_flags; erts_global_offheap.mso = NULL; #ifndef HYBRID /* FIND ME! */ erts_global_offheap.funs = NULL; #endif erts_global_offheap.overhead = 0; #endif #ifdef INCREMENTAL erts_init_incgc(); #endif }
Sint erts_move_messages_off_heap(Process *c_p) { int reds = 1; /* * Move all messages off heap. This *only* occurs when the * process had off heap message disabled and just enabled * it... */ ErtsMessage *mp; reds += c_p->msg.len / 10; ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); for (mp = c_p->msg.first; mp; mp = mp->next) { Uint msg_sz, token_sz; #ifdef USE_VM_PROBES Uint utag_sz; #endif Eterm *hp; ErlHeapFragment *hfrag; if (mp->data.attached) continue; if (is_immed(ERL_MESSAGE_TERM(mp)) #ifdef USE_VM_PROBES && is_immed(ERL_MESSAGE_DT_UTAG(mp)) #endif && is_not_immed(ERL_MESSAGE_TOKEN(mp))) continue; /* * The message refers into the heap. Copy the message * from the heap into a heap fragment and attach * it to the message... */ msg_sz = size_object(ERL_MESSAGE_TERM(mp)); #ifdef USE_VM_PROBES utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp)); #endif token_sz = size_object(ERL_MESSAGE_TOKEN(mp)); hfrag = new_message_buffer(msg_sz #ifdef USE_VM_PROBES + utag_sz #endif + token_sz); hp = hfrag->mem; if (is_not_immed(ERL_MESSAGE_TERM(mp))) ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp), msg_sz, &hp, &hfrag->off_heap); if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), token_sz, &hp, &hfrag->off_heap); #ifdef USE_VM_PROBES if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp))) ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp), utag_sz, &hp, &hfrag->off_heap); #endif mp->data.heap_frag = hfrag; reds += 1; } return reds; }
Sint erts_send_message(Process* sender, Process* receiver, ErtsProcLocks *receiver_locks, Eterm message, unsigned flags) { Uint msize; ErtsMessage* mp; ErlOffHeap *ohp; Eterm token = NIL; Sint res = 0; #ifdef USE_VM_PROBES DTRACE_CHARBUF(sender_name, 64); DTRACE_CHARBUF(receiver_name, 64); Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; Eterm utag = NIL; #endif erts_aint32_t receiver_state; #ifdef SHCOPY_SEND erts_shcopy_t info; #endif BM_STOP_TIMER(system); BM_MESSAGE(message,sender,receiver); BM_START_TIMER(send); #ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send)) { erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->common.id); } #endif receiver_state = erts_smp_atomic32_read_nob(&receiver->state); if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { Eterm* hp; Eterm stoken = SEQ_TRACE_TOKEN(sender); Uint seq_trace_size = 0; #ifdef USE_VM_PROBES Uint dt_utag_size = 0; #endif BM_SWAP_TIMER(send,size); /* SHCOPY corrupts the heap between * copy_shared_calculate, and * copy_shared_perform. (it inserts move_markers like the gc). * Make sure we don't use the heap between those instances. */ if (have_seqtrace(stoken)) { seq_trace_update_send(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); seq_trace_size = 6; /* TUPLE5 */ } #ifdef USE_VM_PROBES if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { dt_utag_size = size_object(DT_UTAG(sender)); } else if (stoken == am_have_dt_utag ) { stoken = NIL; } #endif #ifdef SHCOPY_SEND INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else msize = size_object(message); #endif BM_SWAP_TIMER(size,send); mp = erts_alloc_message_heap_state(receiver, &receiver_state, receiver_locks, (msize #ifdef USE_VM_PROBES + dt_utag_size #endif + seq_trace_size), &hp, &ohp); BM_SWAP_TIMER(send,copy); #ifdef SHCOPY_SEND if (is_not_immed(message)) message = copy_shared_perform(message, msize, &info, &hp, ohp); DESTROY_SHCOPY(info); #else if (is_not_immed(message)) message = copy_struct(message, msize, &hp, ohp); #endif if (is_immed(stoken)) token = stoken; else token = copy_struct(stoken, seq_trace_size, &hp, ohp); #ifdef USE_VM_PROBES if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { if (is_immed(DT_UTAG(sender))) utag = DT_UTAG(sender); else utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, ohp); #ifdef DTRACE_TAG_HARDDEBUG erts_fprintf(stderr, "Dtrace -> (%T) Spreading tag (%T) with " "message %T!\r\n",sender->common.id, utag, message); #endif } #endif BM_MESSAGE_COPIED(msize); BM_SWAP_TIMER(copy,send); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_send)) { if (have_seqtrace(stoken)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken)); } DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); } #endif } else { Eterm *hp; if (receiver == sender && !(receiver_state & ERTS_PSFLG_OFF_HEAP_MSGQ)) { mp = erts_alloc_message(0, NULL); msize = 0; } else { BM_SWAP_TIMER(send,size); #ifdef SHCOPY_SEND INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else msize = size_object(message); #endif BM_SWAP_TIMER(size,send); mp = erts_alloc_message_heap_state(receiver, &receiver_state, receiver_locks, msize, &hp, &ohp); BM_SWAP_TIMER(send,copy); #ifdef SHCOPY_SEND if (is_not_immed(message)) message = copy_shared_perform(message, msize, &info, &hp, ohp); DESTROY_SHCOPY(info); #else if (is_not_immed(message)) message = copy_struct(message, msize, &hp, ohp); #endif BM_MESSAGE_COPIED(msz); BM_SWAP_TIMER(copy,send); } DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); } res = queue_message(sender, receiver, &receiver_state, receiver_locks, mp, message, token #ifdef USE_VM_PROBES , utag #endif ); BM_SWAP_TIMER(send,system); return res; }
ErtsMessage * erts_try_alloc_message_on_heap(Process *pp, erts_aint32_t *psp, ErtsProcLocks *plp, Uint sz, Eterm **hpp, ErlOffHeap **ohpp, int *on_heap_p) { #ifdef ERTS_SMP int locked_main = 0; #endif ErtsMessage *mp; ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ)); if ( #if defined(ERTS_SMP) *plp & ERTS_PROC_LOCK_MAIN #else 1 #endif ) { #ifdef ERTS_SMP try_on_heap: #endif if ((*psp & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) || (pp->flags & F_DISABLE_GC) || HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) { /* * The heap is either potentially in an inconsistent * state, or not large enough. */ #ifdef ERTS_SMP if (locked_main) { *plp &= ~ERTS_PROC_LOCK_MAIN; erts_smp_proc_unlock(pp, ERTS_PROC_LOCK_MAIN); } #endif goto in_message_fragment; } *hpp = HEAP_TOP(pp); HEAP_TOP(pp) = *hpp + sz; *ohpp = &MSO(pp); mp = erts_alloc_message(0, NULL); mp->data.attached = NULL; *on_heap_p = !0; } #ifdef ERTS_SMP else if (erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) { locked_main = 1; *psp = erts_smp_atomic32_read_nob(&pp->state); *plp |= ERTS_PROC_LOCK_MAIN; goto try_on_heap; } #endif else { in_message_fragment: if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) { mp = erts_alloc_message(sz, hpp); *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; } else { mp = erts_alloc_message(0, NULL); if (!sz) { *hpp = NULL; *ohpp = NULL; } else { ErlHeapFragment *bp; bp = new_message_buffer(sz); *hpp = &bp->mem[0]; mp->data.heap_frag = bp; *ohpp = &bp->off_heap; } } *on_heap_p = 0; } return mp; }
/* Add a message last in message queue */ static Sint queue_message(Process *c_p, Process* receiver, erts_aint32_t *receiver_state, ErtsProcLocks *receiver_locks, ErtsMessage* mp, Eterm message, Eterm seq_trace_token #ifdef USE_VM_PROBES , Eterm dt_utag #endif ) { Sint res; int locked_msgq = 0; erts_aint32_t state; ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver)); #ifdef ERTS_SMP if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; if (receiver_state) state = *receiver_state; else state = erts_smp_atomic32_read_nob(&receiver->state); if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) goto exiting; if (*receiver_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); need_locks |= ERTS_PROC_LOCK_STATUS; } erts_smp_proc_lock(receiver, need_locks); } locked_msgq = 1; } #endif state = erts_smp_atomic32_read_nob(&receiver->state); if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { #ifdef ERTS_SMP exiting: #endif /* Drop message if receiver is exiting or has a pending exit... */ if (locked_msgq) erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); erts_cleanup_messages(mp); return 0; } ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = dt_utag; #endif res = receiver->msg.len; #ifdef ERTS_SMP if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order * to ensure that the 'in queue' doesn't contain * references into the heap. By ensuring this, * we don't need to include the 'in queue' in * the root set when garbage collecting. */ res += receiver->msg_inq.len; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); } else #endif { LINK_MESSAGE(receiver, mp); } #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; dtrace_proc_str(receiver, receiver_name); if (seq_trace_token != NIL && is_tuple(seq_trace_token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token)); } DTRACE6(message_queued, receiver_name, size_object(message), receiver->msg.len, tok_label, tok_lastcnt, tok_serial); } #endif if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) trace_receive(receiver, message); if (locked_msgq) erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); erts_proc_notify_new_message(receiver, #ifdef ERTS_SMP *receiver_locks #else 0 #endif ); #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); #endif return res; }
ErtsMessage * erts_factory_message_create(ErtsHeapFactory* factory, Process *proc, ErtsProcLocks *proc_locksp, Uint sz) { Eterm *hp; ErlOffHeap *ohp; ErtsMessage *msgp; int on_heap; erts_aint32_t state; state = erts_smp_atomic32_read_nob(&proc->state); if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) { msgp = erts_alloc_message(sz, &hp); ohp = sz == 0 ? NULL : &msgp->hfrag.off_heap; on_heap = 0; } else { msgp = erts_try_alloc_message_on_heap(proc, &state, proc_locksp, sz, &hp, &ohp, &on_heap); } if (on_heap) { ERTS_SMP_ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN); ASSERT(ohp == &proc->off_heap); factory->mode = FACTORY_HALLOC; factory->p = proc; factory->heap_frags_saved = proc->mbuf; factory->heap_frags_saved_used = proc->mbuf ? proc->mbuf->used_size : 0; } else { factory->mode = FACTORY_MESSAGE; factory->p = NULL; factory->heap_frags_saved = NULL; factory->heap_frags_saved_used = 0; if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) { ASSERT(!msgp->hfrag.next); factory->heap_frags = NULL; } else { ASSERT(!msgp->data.heap_frag || !msgp->data.heap_frag->next); factory->heap_frags = msgp->data.heap_frag; } } factory->hp_start = hp; factory->hp = hp; factory->hp_end = hp + sz; factory->message = msgp; factory->off_heap = ohp; factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; if (ohp) { factory->off_heap_saved.first = ohp->first; factory->off_heap_saved.overhead = ohp->overhead; } else { factory->off_heap_saved.first = NULL; factory->off_heap_saved.overhead = 0; } ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); return msgp; }
Eterm erts_change_message_queue_management(Process *c_p, Eterm new_state) { Eterm res; #ifdef DEBUG if (c_p->flags & F_OFF_HEAP_MSGQ) { ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); } else { if (c_p->flags & F_OFF_HEAP_MSGQ_CHNG) { ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); } else { ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ)); } } #endif switch (c_p->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) { case F_OFF_HEAP_MSGQ: res = am_off_heap; switch (new_state) { case am_off_heap: break; case am_on_heap: c_p->flags |= F_ON_HEAP_MSGQ; erts_smp_atomic32_read_bor_nob(&c_p->state, ERTS_PSFLG_ON_HEAP_MSGQ); /* fall through */ case am_mixed: c_p->flags &= ~F_OFF_HEAP_MSGQ; /* * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ * if a off heap change is ongoing. It will be adjusted * when the change completes... */ if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) { /* Safe to clear ERTS_PSFLG_OFF_HEAP_MSGQ... */ erts_smp_atomic32_read_band_nob(&c_p->state, ~ERTS_PSFLG_OFF_HEAP_MSGQ); } break; default: res = THE_NON_VALUE; /* badarg */ break; } break; case F_ON_HEAP_MSGQ: res = am_on_heap; switch (new_state) { case am_on_heap: break; case am_mixed: c_p->flags &= ~F_ON_HEAP_MSGQ; erts_smp_atomic32_read_band_nob(&c_p->state, ~ERTS_PSFLG_ON_HEAP_MSGQ); break; case am_off_heap: c_p->flags &= ~F_ON_HEAP_MSGQ; erts_smp_atomic32_read_band_nob(&c_p->state, ~ERTS_PSFLG_ON_HEAP_MSGQ); goto change_to_off_heap; default: res = THE_NON_VALUE; /* badarg */ break; } break; case 0: res = am_mixed; switch (new_state) { case am_mixed: break; case am_on_heap: c_p->flags |= F_ON_HEAP_MSGQ; erts_smp_atomic32_read_bor_nob(&c_p->state, ERTS_PSFLG_ON_HEAP_MSGQ); break; case am_off_heap: goto change_to_off_heap; default: res = THE_NON_VALUE; /* badarg */ break; } break; default: res = am_error; ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); break; } return res; change_to_off_heap: c_p->flags |= F_OFF_HEAP_MSGQ; /* * We do not have to schedule a change if * we have an ongoing off heap change... */ if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) { ErtsChangeOffHeapMessageQueue *cohmq; /* * Need to set ERTS_PSFLG_OFF_HEAP_MSGQ and wait * thread progress before completing the change in * order to ensure that all senders observe that * messages should be passed off heap. When the * change has completed, GC does not need to inspect * the message queue at all. */ erts_smp_atomic32_read_bor_nob(&c_p->state, ERTS_PSFLG_OFF_HEAP_MSGQ); c_p->flags |= F_OFF_HEAP_MSGQ_CHNG; cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG, sizeof(ErtsChangeOffHeapMessageQueue)); cohmq->pid = c_p->common.id; erts_schedule_thr_prgr_later_op(change_off_heap_msgq, (void *) cohmq, &cohmq->lop); } return res; }
//将消息放入目标进程的消息队列中 static Sint queue_message(Process *c_p, Process* receiver, ErtsProcLocks *receiver_locks, erts_aint32_t *receiver_state, ErlHeapFragment* bp, Eterm message, Eterm seq_trace_token #ifdef USE_VM_PROBES , Eterm dt_utag #endif ) { Sint res; ErlMessage* mp; int locked_msgq = 0; erts_aint_t state; #ifndef ERTS_SMP ASSERT(bp != NULL || receiver->mbuf == NULL); #endif ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver)); mp = message_alloc(); if (receiver_state){ state = *receiver_state; }else{ state = erts_smp_atomic32_read_acqb(&receiver->state); } #ifdef ERTS_SMP //如果目标进程处在退出的状态,直接进入退出清理 if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)){ goto exiting; } //如果当前没有目标Erlang进程的消息队列锁 //尝试获取消息队列的锁 if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; if (*receiver_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); need_locks |= ERTS_PROC_LOCK_STATUS; } erts_smp_proc_lock(receiver, need_locks); } locked_msgq = 1; state = erts_smp_atomic32_read_nob(&receiver->state); //获取目标Erlang进程的状态 if (receiver_state){ *receiver_state = state; } } #endif //Erlang进程处于退出的状态 //直接释放目标进程的锁,和当前消息 if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { #ifdef ERTS_SMP exiting: #endif /* Drop message if receiver is exiting or has a pending exit... */ if (locked_msgq){ erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); } if (bp){ free_message_buffer(bp); } message_free(mp); return 0; } //从此处可以看出,放入目标进程的消息,只是对当前进程的消息的一个指针应用 //那我们需要找出,message是如何被处理的,何时增加的引用计数 ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = dt_utag; #endif mp->next = NULL; mp->data.heap_frag = bp; #ifndef ERTS_SMP res = receiver->msg.len; #else res = receiver->msg_inq.len; //把消息attach的目标Erlang进程的消息队列或者private //的消息队列 if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order * to ensure that the 'in queue' doesn't contain * references into the heap. By ensuring this, * we don't need to include the 'in queue' in * the root set when garbage collecting. */ //这个时候可以操作msg_inq和msg res += receiver->msg.len; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); } else #endif { //这个时候只操作msg_inq LINK_MESSAGE(receiver, mp); } #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; dtrace_proc_str(receiver, receiver_name); if (seq_trace_token != NIL && is_tuple(seq_trace_token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token)); } DTRACE6(message_queued, receiver_name, size_object(message), receiver->msg.len, tok_label, tok_lastcnt, tok_serial); } #endif if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)){ trace_receive(receiver, message); } if (locked_msgq){ erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); } erts_proc_notify_new_message(receiver, #ifdef ERTS_SMP *receiver_locks #else 0 #endif ); #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); #endif return res; }
Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) { unsigned result; Eterm reds_in = p->def_arg_reg[5]; /* * Process is in the normal case scheduled out when reduction * count reach zero. When "save calls" is enabled reduction * count is subtracted with CONTEXT_REDS, i.e. initial reduction * count will be zero or less and process is scheduled out * when -CONTEXT_REDS is reached. * * HiPE does not support the "save calls" feature, so we switch * to using a positive reduction counter when executing in * hipe mode, but need to restore the "save calls" when * returning to beam. We also need to hide the save calls buffer * from BIFs. We do that by moving the saved calls buf to * suspended saved calls buf. * * Beam has initial reduction count in stored in p->def_arg_reg[5]. * * Beam expects -neg_o_reds to be found in p->def_arg_reg[4] * on return to beam. */ { struct saved_calls *scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); if (scb) { reds_in += CONTEXT_REDS; p->fcalls += CONTEXT_REDS; ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, scb); } } p->flags |= F_HIPE_MODE; /* inform bifs where we are comming from... */ p->i = NULL; /* Set current_function to undefined. stdlib hibernate tests rely on it. */ p->current = NULL; DPRINTF("cmd == %#x (%s)", cmd, code_str(cmd)); HIPE_CHECK_PCB(p); p->arity = 0; switch (cmd & 0xFF) { case HIPE_MODE_SWITCH_CMD_CALL: { /* BEAM calls a native code function */ unsigned arity = cmd >> 8; /* p->hipe.u.ncallee set in beam_emu */ if (p->cp == hipe_beam_pc_return) { /* Native called BEAM, which now tailcalls native. */ hipe_pop_beam_trap_frame(p); result = hipe_tailcall_to_native(p, arity, reg); break; } DPRINTF("calling %#lx/%u", (long)p->hipe.u.ncallee, arity); result = hipe_call_to_native(p, arity, reg); break; } case HIPE_MODE_SWITCH_CMD_CALL_CLOSURE: { /* BEAM calls a native code closure */ unsigned arity = cmd >> 8; /* #formals + #fvs (closure not counted) */ Eterm fun; ErlFunThing *funp; /* drop the fvs, move the closure, correct arity */ fun = reg[arity]; HIPE_ASSERT(is_fun(fun)); funp = (ErlFunThing*)fun_val(fun); HIPE_ASSERT(funp->num_free <= arity); arity -= funp->num_free; /* arity == #formals */ reg[arity] = fun; ++arity; /* correct for having added the closure */ /* HIPE_ASSERT(p->hipe.u.ncallee == (void(*)(void))funp->native_address); */ /* just like a normal call from now on */ /* p->hipe.u.ncallee set in beam_emu */ if (p->cp == hipe_beam_pc_return) { /* Native called BEAM, which now tailcalls native. */ hipe_pop_beam_trap_frame(p); result = hipe_tailcall_to_native(p, arity, reg); break; } DPRINTF("calling %#lx/%u", (long)p->hipe.u.ncallee, arity); result = hipe_call_to_native(p, arity, reg); break; } case HIPE_MODE_SWITCH_CMD_THROW: { /* BEAM just executed hipe_beam_pc_throw[] */ /* Native called BEAM, which now throws an exception back to native. */ DPRINTF("beam throws freason %#lx fvalue %#lx", p->freason, p->fvalue); hipe_pop_beam_trap_frame(p); do_throw_to_native: p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(p->freason)]; hipe_find_handler(p); result = hipe_throw_to_native(p); break; } case HIPE_MODE_SWITCH_CMD_RETURN: { /* BEAM just executed hipe_beam_pc_return[] */ /* Native called BEAM, which now returns back to native. */ /* pop trap frame off estack */ hipe_pop_beam_trap_frame(p); p->def_arg_reg[0] = reg[0]; result = hipe_return_to_native(p); break; } do_resume: case HIPE_MODE_SWITCH_CMD_RESUME: { /* BEAM just executed hipe_beam_pc_resume[] */ /* BEAM called native, which suspended. */ if (p->flags & F_TIMO) { /* XXX: The process will immediately execute 'clear_timeout', repeating these two statements. Remove them? */ p->flags &= ~F_TIMO; JOIN_MESSAGE(p); p->def_arg_reg[0] = 0; /* make_small(0)? */ } else p->def_arg_reg[0] = 1; /* make_small(1)? */ result = hipe_return_to_native(p); break; } default: erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: cmd %#x\r\n", cmd); } do_return_from_native: DPRINTF("result == %#x (%s)", result, code_str(result)); HIPE_CHECK_PCB(p); switch (result) { case HIPE_MODE_SWITCH_RES_RETURN: { hipe_return_from_native(p); reg[0] = p->def_arg_reg[0]; DPRINTF("returning with r(0) == %#lx", reg[0]); break; } case HIPE_MODE_SWITCH_RES_THROW: { DPRINTF("native throws freason %#lx fvalue %#lx", p->freason, p->fvalue); hipe_throw_from_native(p); break; } case HIPE_MODE_SWITCH_RES_TRAP: { /* * Native code called a BIF, which "failed" with a TRAP to BEAM. * Prior to returning, the BIF stored (see BIF_TRAP<N>): * the callee's address in p->i * the callee's parameters in reg[0..2] * the callee's arity in p->arity (for BEAM gc purposes) * * We need to remove the BIF's parameters from the native * stack: to this end hipe_${ARCH}_glue.S stores the BIF's * arity in p->hipe.narity. * * If the BIF emptied the stack (typically hibernate), p->hipe.nstack * is NULL and there is no need to get rid of stacked parameters. */ unsigned int i, is_recursive = 0; if (p->hipe.nstack != NULL) { ASSERT(p->hipe.nsp != NULL); is_recursive = hipe_trap_from_native_is_recursive(p); } else { /* Some architectures (risc) need this re-reset of nsp as the * BIF wrapper do not detect stack change and causes an obsolete * stack pointer to be saved in p->hipe.nsp before return to us. */ p->hipe.nsp = NULL; } /* Schedule next process if current process was hibernated or is waiting for messages */ if (p->flags & F_HIBERNATE_SCHED) { p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } if (!(erts_smp_atomic32_read_acqb(&p->state) & ERTS_PSFLG_ACTIVE)) { for (i = 0; i < p->arity; ++i) p->arg_reg[i] = reg[i]; goto do_schedule; } if (is_recursive) hipe_push_beam_trap_frame(p, reg, p->arity); result = HIPE_MODE_SWITCH_RES_CALL_BEAM; break; } case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: { /* Native code calls or tailcalls BEAM. * * p->hipe.u.callee_exp is the callee's export entry * p->arity is the callee's arity * p->def_arg_reg[] contains the register parameters * p->hipe.nsp[] contains the stacked parameters */ if (hipe_call_from_native_is_recursive(p, reg)) { /* BEAM called native, which now calls BEAM */ hipe_push_beam_trap_frame(p, reg, p->arity); } break; } case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: { /* Native code calls or tailcalls a closure in BEAM * * In native code a call to a closure of arity n looks like * F(A1, ..., AN, Closure), * The BEAM code for a closure expects to get: * F(A1, ..., AN, FV1, ..., FVM, Closure) * (Where Ai is argument i and FVj is free variable j) * * p->hipe.u.closure contains the closure * p->def_arg_reg[] contains the register parameters * p->hipe.nsp[] contains the stacked parameters */ ErlFunThing *closure; unsigned num_free, arity, i, is_recursive; HIPE_ASSERT(is_fun(p->hipe.u.closure)); closure = (ErlFunThing*)fun_val(p->hipe.u.closure); num_free = closure->num_free; arity = closure->fe->arity; /* Store the arity in p->arity for the stack popping. */ /* Note: we already have the closure so only need to move arity values to reg[]. However, there are arity+1 parameters in the native code state that need to be removed. */ p->arity = arity+1; /* +1 for the closure */ /* Get parameters, don't do GC just yet. */ is_recursive = hipe_call_from_native_is_recursive(p, reg); if ((Sint)closure->fe->address[-1] < 0) { /* Unloaded. Let beam_emu.c:call_fun() deal with it. */ result = HIPE_MODE_SWITCH_RES_CALL_CLOSURE; } else { /* The BEAM code is present. Prepare to call it. */ /* Append the free vars after the actual parameters. */ for (i = 0; i < num_free; ++i) reg[arity+i] = closure->env[i]; /* Update arity to reflect the new parameters. */ arity += i; /* Make a call to the closure's BEAM code. */ p->i = closure->fe->address; /* Change result code to the faster plain CALL type. */ result = HIPE_MODE_SWITCH_RES_CALL_BEAM; } /* Append the closure as the last parameter. Don't increment arity. */ reg[arity] = p->hipe.u.closure; if (is_recursive) { /* BEAM called native, which now calls BEAM. Need to put a trap-frame on the beam stack. This may cause GC, which is safe now that the arguments, free vars, and most importantly the closure, all are in reg[]. */ hipe_push_beam_trap_frame(p, reg, arity+1); } break; } case HIPE_MODE_SWITCH_RES_SUSPEND: { p->i = hipe_beam_pc_resume; p->arity = 0; goto do_schedule; } case HIPE_MODE_SWITCH_RES_WAIT: case HIPE_MODE_SWITCH_RES_WAIT_TIMEOUT: { /* same semantics, different debug trace messages */ #ifdef ERTS_SMP /* XXX: BEAM has different entries for the locked and unlocked cases. HiPE doesn't, so we must check dynamically. */ if (p->hipe_smp.have_receive_locks) p->hipe_smp.have_receive_locks = 0; else erts_smp_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); #endif p->i = hipe_beam_pc_resume; p->arity = 0; erts_smp_atomic32_read_band_relb(&p->state, ~ERTS_PSFLG_ACTIVE); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); do_schedule: { struct saved_calls *scb; scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, NULL); if (scb) ERTS_PROC_SET_SAVED_CALLS_BUF(p, scb); /* The process may have died while it was executing, if so we return out from native code to the interpreter */ if (erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING) p->i = beam_exit; #ifdef DEBUG ASSERT(p->debug_reds_in == reds_in); #endif p->flags &= ~F_HIPE_MODE; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p); p = erts_schedule(NULL, p, reds_in - p->fcalls); ERTS_SMP_REQ_PROC_MAIN_LOCK(p); ASSERT(!(p->flags & F_HIPE_MODE)); #ifdef ERTS_SMP p->hipe_smp.have_receive_locks = 0; reg = p->scheduler_data->x_reg_array; #endif } { Eterm *argp; int i; argp = p->arg_reg; for (i = p->arity; --i >= 0;) reg[i] = argp[i]; } { struct saved_calls *scb; reds_in = p->fcalls; p->def_arg_reg[5] = reds_in; #ifdef DEBUG p->debug_reds_in = reds_in; #endif if (p->i == hipe_beam_pc_resume) { p->flags |= F_HIPE_MODE; /* inform bifs where we are comming from... */ scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); if (scb) ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, scb); p->i = NULL; p->arity = 0; goto do_resume; } scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); if (!scb) p->def_arg_reg[4] = 0; else { p->def_arg_reg[4] = CONTEXT_REDS; p->fcalls = -CONTEXT_REDS + reds_in; } } HIPE_CHECK_PCB(p); result = HIPE_MODE_SWITCH_RES_CALL_BEAM; p->def_arg_reg[3] = result; return p; } case HIPE_MODE_SWITCH_RES_APPLY: { Eterm mfa[3], args; unsigned int arity; void *address; hipe_pop_params(p, 3, &mfa[0]); /* Unroll the arglist onto reg[]. */ args = mfa[2]; arity = 0; while (is_list(args)) { if (arity < 255) { reg[arity++] = CAR(list_val(args)); args = CDR(list_val(args)); } else goto do_apply_fail; } if (is_not_nil(args)) goto do_apply_fail; /* find a native code entry point for {M,F,A} for a remote call */ address = hipe_get_remote_na(mfa[0], mfa[1], arity); if (!address) goto do_apply_fail; p->hipe.u.ncallee = (void(*)(void)) address; result = hipe_tailcall_to_native(p, arity, reg); goto do_return_from_native; do_apply_fail: p->freason = BADARG; goto do_throw_to_native; } default: erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %#x\r\n", result); } { struct saved_calls *scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, NULL); if (!scb) p->def_arg_reg[4] = 0; else { p->def_arg_reg[4] = CONTEXT_REDS; p->fcalls -= CONTEXT_REDS; ERTS_PROC_SET_SAVED_CALLS_BUF(p, scb); } } HIPE_CHECK_PCB(p); p->def_arg_reg[3] = result; #if NR_ARG_REGS > 5 /* * When NR_ARG_REGS > 5, we need to protect the process' input * reduction count (which BEAM stores in def_arg_reg[5]) from * being clobbered by the arch glue code. */ p->def_arg_reg[5] = reds_in; #endif p->flags &= ~F_HIPE_MODE; return p; }