Sint erts_complete_off_heap_message_queue_change(Process *c_p) { int reds = 1; ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); /* * This job was first initiated when the process changed to off heap * message queue management. Since then ERTS_PSFLG_OFF_HEAP_MSGQ * has been set. However, the management state might have been changed * again (multiple times) since then. Check users last requested state * (the flags F_OFF_HEAP_MSGQ, and F_ON_HEAP_MSGQ), and make the state * consistent with that. */ if (!(c_p->flags & F_OFF_HEAP_MSGQ)) erts_smp_atomic32_read_band_nob(&c_p->state, ~ERTS_PSFLG_OFF_HEAP_MSGQ); else { reds += 2; erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); reds += erts_move_messages_off_heap(c_p); } c_p->flags &= ~F_OFF_HEAP_MSGQ_CHNG; return reds; }
static ERTS_INLINE void notify_new_message(Process *receiver) { ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(receiver)); switch (receiver->status) { case P_GARBING: switch (receiver->gcstatus) { case P_SUSPENDED: goto suspended; case P_WAITING: goto waiting; default: break; } break; case P_SUSPENDED: suspended: receiver->rstatus = P_RUNABLE; break; case P_WAITING: waiting: erts_add_to_runq(receiver); break; default: break; } }
int erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return set_break(mfa, specified, match_spec, (Uint) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); }
int erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); }
static ERTS_INLINE Port * pop_port(ErtsRunQueue *runq) { Port *pp = runq->ports.start; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); if (!pp) { ASSERT(!runq->ports.end); } else { runq->ports.start = runq->ports.start->sched.next; if (runq->ports.start) runq->ports.start->sched.prev = NULL; else { ASSERT(runq->ports.end == pp); runq->ports.end = NULL; } ASSERT(runq->ports.info.len > 0); runq->ports.info.len--; ASSERT(runq->len > 0); runq->len--; } ASSERT(runq->ports.start || !runq->ports.end); ASSERT(runq->ports.end || !runq->ports.start); return pp; }
int erts_clear_module_break(Module *modp) { BeamCodeHeader* code_hdr; Uint n; Uint i; ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); code_hdr = modp->curr.code_hdr; if (!code_hdr) { return 0; } n = (Uint)(UWord) code_hdr->num_functions; for (i = 0; i < n; ++i) { ErtsCodeInfo *ci = code_hdr->functions[i]; if (erts_is_function_native(ci)) continue; clear_function_break(ci, ERTS_BPF_ALL); } erts_commit_staged_bp(); for (i = 0; i < n; ++i) { ErtsCodeInfo *ci = code_hdr->functions[i]; if (erts_is_function_native(ci)) continue; uninstall_breakpoint(ci); consolidate_bp_data(modp, ci, 1); ASSERT(ci->u.gen_bp == NULL); } return n; }
static void * signal_dispatcher_thread_func(void *unused) { #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_set_thread_name("signal_dispatcher"); #endif while (1) { union {int signum; char buf[4];} sb; Eterm signal; int res, i = 0; /* Block on read() waiting for a signal notification to arrive... */ do { res = read(sig_notify_fds[0], (void *) &sb.buf[i], sizeof(int) - i); i += res > 0 ? res : 0; } while ((i < sizeof(int) && res >= 0) || (res < 0 && errno == EINTR)); if (res < 0) { erts_exit(ERTS_ABORT_EXIT, "signal-dispatcher thread got unexpected error: %s (%d)\n", erl_errno_id(errno), errno); } /* * NOTE 1: The signal dispatcher thread should not do work * that takes a substantial amount of time (except * perhaps in test and debug builds). It needs to * be responsive, i.e, it should only dispatch work * to other threads. * * NOTE 2: The signal dispatcher thread is not a blockable * thread (i.e., not a thread managed by the * erl_thr_progress module). This is intentional. * We want to be able to interrupt writing of a crash * dump by hitting C-c twice. Since it isn't a * blockable thread it is important that it doesn't * change the state of any data that a blocking thread * expects to have exclusive access to (unless the * signal dispatcher itself explicitly is blocking all * blockable threads). */ switch (sb.signum) { case 0: continue; case SIGINT: break_requested(); break; default: if ((signal = signum_to_signalterm(sb.signum)) == am_error) { erts_exit(ERTS_ABORT_EXIT, "signal-dispatcher thread received unknown " "signal notification: '%d'\n", sb.signum); } signal_notify_requested(signal); } ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); } return NULL; }
char *getenv_string(GETENV_STATE *state) { ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx)); if (state->next_string[0] == L'\0') { return NULL; } else { WCHAR *res = state->next_string; state->next_string += wcslen(res) + 1; return (char *) res; } }
char *getenv_string(GETENV_STATE *state) { ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx)); if (state->next_string[0] == '\0') return NULL; else { char *res = state->next_string; state->next_string += sys_strlen(res) + 1; return res; } }
void erts_consolidate_bif_bp_data(void) { int i; ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < BIF_SIZE; i++) { Export *ep = bif_export[i]; consolidate_bp_data(0, ep->code+3, 0); } }
ErtsMigrateResult erts_port_migrate(Port *prt, int *prt_locked, ErtsRunQueue *from_rq, int *from_locked, ErtsRunQueue *to_rq, int *to_locked) { ERTS_SMP_LC_ASSERT(*from_locked); ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); ASSERT(!erts_common_run_queue); if (!*from_locked || !*to_locked) { if (from_rq < to_rq) { if (!*to_locked) { if (!*from_locked) erts_smp_runq_lock(from_rq); erts_smp_runq_lock(to_rq); } else if (erts_smp_runq_trylock(from_rq) == EBUSY) { erts_smp_runq_unlock(to_rq); erts_smp_runq_lock(from_rq); erts_smp_runq_lock(to_rq); } } else { if (!*from_locked) { if (!*to_locked) erts_smp_runq_lock(to_rq); erts_smp_runq_lock(from_rq); } else if (erts_smp_runq_trylock(to_rq) == EBUSY) { erts_smp_runq_unlock(from_rq); erts_smp_runq_lock(to_rq); erts_smp_runq_lock(from_rq); } } *to_locked = *from_locked = 1; } ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); /* Refuse to migrate to a suspended run queue */ if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED) return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue)) return ERTS_MIGRATE_FAILED_RUNQ_CHANGED; if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt)) return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; dequeue_port(from_rq, prt); erts_smp_atomic_set(&prt->run_queue, (long) to_rq); enqueue_port(to_rq, prt); erts_smp_notify_inc_runq(to_rq); return ERTS_MIGRATE_SUCCESS; }
void erts_clear_export_break(Module* modp, BeamInstr* pc) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); clear_function_break(pc, ERTS_BPF_ALL); erts_commit_staged_bp(); *pc = (BeamInstr) 0; consolidate_bp_data(modp, pc, 0); ASSERT(pc[-4] == 0); }
void erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); clear_function_break(ci, ERTS_BPF_ALL); erts_commit_staged_bp(); *erts_codeinfo_to_code(ci) = (BeamInstr) 0; consolidate_bp_data(modp, ci, 0); ASSERT(ci->u.gen_bp == NULL); }
Module* erts_put_module(Eterm mod) { Module e; int index; ASSERT(is_atom(mod)); ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0)); e.module = atom_val(mod); index = index_put(&module_table, (void*) &e); return (Module*) erts_index_lookup(&module_table, index); }
void erts_set_dist_entry_not_connected(DistEntry *dep) { ERTS_SMP_LC_ASSERT(erts_lc_is_dist_entry_locked(dep)); erts_smp_mtx_lock(&erts_dist_table_mtx); ASSERT(dep != erts_this_dist_entry); ASSERT(is_internal_port(dep->cid)); if(dep->flags & DFLAG_PUBLISHED) { if(dep->prev) { ASSERT(is_in_de_list(dep, erts_visible_dist_entries)); dep->prev->next = dep->next; } else { ASSERT(erts_visible_dist_entries == dep); erts_visible_dist_entries = dep->next; } ASSERT(erts_no_of_visible_dist_entries > 0); erts_no_of_visible_dist_entries--; } else { if(dep->prev) { ASSERT(is_in_de_list(dep, erts_hidden_dist_entries)); dep->prev->next = dep->next; } else { ASSERT(erts_hidden_dist_entries == dep); erts_hidden_dist_entries = dep->next; } ASSERT(erts_no_of_hidden_dist_entries > 0); erts_no_of_hidden_dist_entries--; } if(dep->next) dep->next->prev = dep->prev; dep->status &= ~ERTS_DE_SFLG_CONNECTED; dep->flags = 0; dep->prev = NULL; dep->cid = NIL; dep->next = erts_not_connected_dist_entries; if(erts_not_connected_dist_entries) { ASSERT(erts_not_connected_dist_entries->prev == NULL); erts_not_connected_dist_entries->prev = dep; } erts_not_connected_dist_entries = dep; erts_no_of_not_connected_dist_entries++; erts_smp_mtx_unlock(&erts_dist_table_mtx); }
void erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) { ERTS_SMP_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); ASSERT(dep != erts_this_dist_entry); ASSERT(is_nil(dep->cid)); ASSERT(is_internal_port(cid)); if(dep->prev) { ASSERT(is_in_de_list(dep, erts_not_connected_dist_entries)); dep->prev->next = dep->next; } else { ASSERT(erts_not_connected_dist_entries == dep); erts_not_connected_dist_entries = dep->next; } if(dep->next) dep->next->prev = dep->prev; ASSERT(erts_no_of_not_connected_dist_entries > 0); erts_no_of_not_connected_dist_entries--; dep->status |= ERTS_DE_SFLG_CONNECTED; dep->flags = flags; dep->cid = cid; dep->connection_id++; dep->connection_id &= ERTS_DIST_EXT_CON_ID_MASK; dep->prev = NULL; if(flags & DFLAG_PUBLISHED) { dep->next = erts_visible_dist_entries; if(erts_visible_dist_entries) { ASSERT(erts_visible_dist_entries->prev == NULL); erts_visible_dist_entries->prev = dep; } erts_visible_dist_entries = dep; erts_no_of_visible_dist_entries++; } else { dep->next = erts_hidden_dist_entries; if(erts_hidden_dist_entries) { ASSERT(erts_hidden_dist_entries->prev == NULL); erts_hidden_dist_entries->prev = dep; } erts_hidden_dist_entries = dep; erts_no_of_hidden_dist_entries++; } erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); }
void erts_consolidate_bp_data(BpFunctions* f, int local) { BpFunction* fs = f->matching; Uint i; Uint n = f->matched; ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < n; i++) { consolidate_bp_data(fs[i].mod, fs[i].pc, local); } }
BIF_RETTYPE make_ref_0(BIF_ALIST_0) { BIF_RETTYPE res; Eterm* hp; ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); hp = HAlloc(BIF_P, REF_THING_SIZE); res = erts_sched_make_ref_in_buffer(ERTS_PROC_GET_SCHDATA(BIF_P), hp); BIF_RET(res); }
Eterm erts_make_ref(Process *c_p) { Eterm* hp; Uint32 ref[ERTS_MAX_REF_NUMBERS]; ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); hp = HAlloc(c_p, REF_THING_SIZE); make_ref_in_array(ref); write_ref_thing(hp, ref[0], ref[1], ref[2]); return make_internal_ref(hp); }
void erts_clear_mtrace_bif(Uint *pc) { BpDataTrace *bdt; ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); bdt = (BpDataTrace *) pc[-4]; if (bdt) { if (bdt->match_spec) { MatchSetUnref(bdt->match_spec); } Free(bdt); } pc[-4] = (Uint) NULL; }
char *getenv_string(GETENV_STATE *state0) { char **state = (char **) *state0; char *cp; ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx)); if (state == NULL) state = environ; cp = *state++; *state0 = (GETENV_STATE) state; return cp; }
static void handle_remaining_tasks(ErtsRunQueue *runq, Port *pp) { int i; ErtsPortTask *ptp; ErtsPortTaskQueue *ptqps[] = {pp->sched.exe_taskq, pp->sched.taskq}; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); for (i = 0; i < sizeof(ptqps)/sizeof(ErtsPortTaskQueue *); i++) { if (!ptqps[i]) continue; ptp = pop_task(ptqps[i]); while (ptp) { reset_handle(ptp); erts_smp_runq_unlock(runq); switch (ptp->type) { case ERTS_PORT_TASK_FREE: case ERTS_PORT_TASK_TIMEOUT: break; case ERTS_PORT_TASK_INPUT: erts_stale_drv_select(pp->id, ptp->event, DO_READ, 1); break; case ERTS_PORT_TASK_OUTPUT: erts_stale_drv_select(pp->id, ptp->event, DO_WRITE, 1); break; case ERTS_PORT_TASK_EVENT: erts_stale_drv_select(pp->id, ptp->event, 0, 1); break; case ERTS_PORT_TASK_DIST_CMD: break; default: erl_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); } port_task_free(ptp); erts_smp_runq_lock(runq); ptp = pop_task(ptqps[i]); } } ASSERT(!pp->sched.taskq || !pp->sched.taskq->first); }
static void node_table_free(void *venp) { ErlNode *enp = (ErlNode *) venp; ERTS_SMP_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking()); erts_deref_dist_entry(enp->dist_entry); #ifdef DEBUG sys_memset(venp, 0x55, sizeof(ErlNode)); #endif erts_free(ERTS_ALC_T_NODE_ENTRY, venp); ASSERT(node_entries > 0); node_entries--; }
void erts_set_mtrace_bif(Uint *pc, Binary *match_spec, Eterm tracer_pid) { BpDataTrace *bdt; ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); bdt = (BpDataTrace *) pc[-4]; if (bdt) { MatchSetUnref(bdt->match_spec); MatchSetRef(match_spec); bdt->match_spec = match_spec; bdt->tracer_pid = tracer_pid; } else { bdt = Alloc(sizeof(BpDataTrace)); BpInit((BpData *) bdt, 0); MatchSetRef(match_spec); bdt->match_spec = match_spec; bdt->tracer_pid = tracer_pid; pc[-4] = (Uint) bdt; } }
void erts_release_code_write_permission(void) { erts_smp_mtx_lock(&code_write_permission_mtx); ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); while (code_write_queue != NULL) { /* unleash the entire herd */ struct code_write_queue_item* qitem = code_write_queue; erts_smp_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS); if (!ERTS_PROC_IS_EXITING(qitem->p)) { erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS); } erts_smp_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); code_write_queue = qitem->next; erts_proc_dec_refc(qitem->p); erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem); } code_writing_process = NULL; #ifdef ERTS_ENABLE_LOCK_CHECK erts_tsd_set(has_code_write_permission, (void *) 0); #endif erts_smp_mtx_unlock(&code_write_permission_mtx); }
static void delete_code(Module* modp) { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = make_atom(modp->module); int i; for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->code[0] == module)) { if (ep->addressv[code_ix] == ep->code+3) { if (ep->code[3] == (BeamInstr) em_apply_bif) { continue; } else if (ep->code[3] == (BeamInstr) em_call_traced_function) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); --modp->curr.num_traced_exports; MatchSetUnref(ep->match_prog_set); ep->match_prog_set = NULL; } else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler || !erts_initialized); } ep->addressv[code_ix] = ep->code+3; ep->code[3] = (BeamInstr) em_call_error_handler; ep->code[4] = 0; ASSERT(ep->match_prog_set == NULL); } } ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); modp->old = modp->curr; modp->curr.code = NULL; modp->curr.code_length = 0; modp->curr.catches = BEAM_CATCHES_NIL; modp->curr.nif = NULL; }
void erts_set_this_node(Eterm sysname, Uint creation) { ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2)); if (erts_refc_dectest(&erts_this_node->refc, 0) == 0) try_delete_node(erts_this_node); if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0) try_delete_dist_entry(erts_this_dist_entry); erts_this_node = NULL; /* to make sure refc is bumped for this node */ erts_this_node = erts_find_or_insert_node(sysname, creation); erts_this_dist_entry = erts_this_node->dist_entry; erts_refc_inc(&erts_this_dist_entry->refc, 2); erts_this_node_sysname = erts_this_node_sysname_BUFFER; erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), "%T", sysname); }
static void delete_code(Module* modp) { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = make_atom(modp->module); int i; for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->info.mfa.module == module)) { if (ep->addressv[code_ix] == ep->beam) { if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } else if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); DBG_TRACE_MFA_P(&ep->info.mfa, "export trace cleared, code_ix=%d", code_ix); erts_clear_export_break(modp, &ep->info); } else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler || !erts_initialized); } ep->addressv[code_ix] = ep->beam; ep->beam[0] = (BeamInstr) em_call_error_handler; ep->beam[1] = 0; DBG_TRACE_MFA_P(&ep->info.mfa, "export invalidation, code_ix=%d", code_ix); } } ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); modp->old = modp->curr; erts_module_instance_init(&modp->curr); }
int erts_clear_module_break(Module *modp) { BeamInstr** code_base; Uint n; Uint i; ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); code_base = (BeamInstr **) modp->curr.code; if (code_base == NULL) { return 0; } n = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { BeamInstr* pc; pc = code_base[MI_FUNCTIONS+i] + 5; if (erts_is_native_break(pc)) { continue; } clear_function_break(pc, ERTS_BPF_ALL); } erts_commit_staged_bp(); for (i = 0; i < n; ++i) { BeamInstr* pc; pc = code_base[MI_FUNCTIONS+i] + 5; if (erts_is_native_break(pc)) { continue; } uninstall_breakpoint(pc); consolidate_bp_data(modp, pc, 1); ASSERT(pc[-4] == 0); } return n; }
int erts_clear_module_break(Module *modp) { BeamCodeHeader* code_hdr; Uint n; Uint i; ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); code_hdr = modp->curr.code_hdr; if (!code_hdr) { return 0; } n = (Uint)(UWord) code_hdr->num_functions; for (i = 0; i < n; ++i) { BeamInstr* pc; pc = code_hdr->functions[i] + 5; if (erts_is_native_break(pc)) { continue; } clear_function_break(pc, ERTS_BPF_ALL); } erts_commit_staged_bp(); for (i = 0; i < n; ++i) { BeamInstr* pc; pc = code_hdr->functions[i] + 5; if (erts_is_native_break(pc)) { continue; } uninstall_breakpoint(pc); consolidate_bp_data(modp, pc, 1); ASSERT(pc[-4] == 0); } return n; }