void
erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp)
{
Eterm* htop = HEAP_TOP(proc);
link_mbuf_to_proc(proc, bp);
if (htop < HEAP_LIMIT(proc)) {
*htop = make_pos_bignum_header(HEAP_LIMIT(proc)-htop-1);
HEAP_TOP(proc) = HEAP_LIMIT(proc);
}
}
static BIF_RETTYPE lists_reverse_onheap(Process *c_p,
Eterm list_in,
Eterm tail_in)
{
static const Uint CELLS_PER_RED = 60;
Eterm *alloc_start, *alloc_top, *alloc_end;
Uint cells_left, max_cells;
Eterm list, tail;
list = list_in;
tail = tail_in;
cells_left = max_cells = CELLS_PER_RED * (1 + ERTS_BIF_REDS_LEFT(c_p));
ASSERT(HEAP_LIMIT(c_p) >= HEAP_TOP(c_p) + 2);
alloc_start = HEAP_TOP(c_p);
alloc_end = HEAP_LIMIT(c_p) - 2;
alloc_top = alloc_start;
/* Don't process more cells than we have reductions for. */
alloc_end = MIN(alloc_top + (cells_left * 2), alloc_end);
while (alloc_top < alloc_end && is_list(list)) {
Eterm *pair = list_val(list);
tail = CONS(alloc_top, CAR(pair), tail);
list = CDR(pair);
alloc_top += 2;
}
cells_left -= (alloc_top - alloc_start) / 2;
HEAP_TOP(c_p) = alloc_top;
ASSERT(cells_left >= 0 && cells_left <= max_cells);
BUMP_REDS(c_p, (max_cells - cells_left) / CELLS_PER_RED);
if (is_nil(list)) {
BIF_RET(tail);
} else if (is_list(list)) {
ASSERT(is_list(tail));
if (cells_left > CELLS_PER_RED) {
return lists_reverse_alloc(c_p, list, tail);
}
BUMP_ALL_REDS(c_p);
BIF_TRAP2(bif_export[BIF_lists_reverse_2], c_p, list, tail);
}
BIF_ERROR(c_p, BADARG);
}
/* Flush out our cached heap pointers to allow an ordinary HAlloc
*/
static void flush_env(ErlNifEnv* env)
{
if (env->heap_frag == NULL) {
ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
ASSERT(env->hp >= HEAP_TOP(env->proc));
ASSERT(env->hp <= HEAP_LIMIT(env->proc));
HEAP_TOP(env->proc) = env->hp;
}
else {
ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
env->heap_frag->used_size = env->hp - env->heap_frag->mem;
ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
}
}
ErlNifEnv* enif_alloc_env(void)
{
struct enif_msg_environment_t* msg_env =
erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t));
Eterm* phony_heap = (Eterm*) msg_env; /* dummy non-NULL ptr */
msg_env->env.hp = phony_heap;
msg_env->env.hp_end = phony_heap;
msg_env->env.heap_frag = NULL;
msg_env->env.mod_nif = NULL;
msg_env->env.tmp_obj_list = NULL;
msg_env->env.proc = &msg_env->phony_proc;
memset(&msg_env->phony_proc, 0, sizeof(Process));
HEAP_START(&msg_env->phony_proc) = phony_heap;
HEAP_TOP(&msg_env->phony_proc) = phony_heap;
HEAP_LIMIT(&msg_env->phony_proc) = phony_heap;
HEAP_END(&msg_env->phony_proc) = phony_heap;
MBUF(&msg_env->phony_proc) = NULL;
msg_env->phony_proc.id = ERTS_INVALID_PID;
#ifdef FORCE_HEAP_FRAGS
msg_env->phony_proc.space_verified = 0;
msg_env->phony_proc.space_verified_from = NULL;
#endif
return &msg_env->env;
}
/* Restore cached heap pointers to allow alloc_heap again.
*/
static void cache_env(ErlNifEnv* env)
{
if (env->heap_frag == NULL) {
ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
ASSERT(env->hp <= HEAP_TOP(env->proc));
ASSERT(env->hp <= HEAP_LIMIT(env->proc));
env->hp = HEAP_TOP(env->proc);
}
else {
ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
env->heap_frag = MBUF(env->proc);
ASSERT(env->heap_frag != NULL);
env->hp = env->heap_frag->mem + env->heap_frag->used_size;
env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size;
}
}
void erts_post_nif(ErlNifEnv* env)
{
erts_unblock_fpe(env->fpe_was_unmasked);
if (env->heap_frag == NULL) {
ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
ASSERT(env->hp >= HEAP_TOP(env->proc));
ASSERT(env->hp <= HEAP_LIMIT(env->proc));
HEAP_TOP(env->proc) = env->hp;
}
else {
ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
env->heap_frag->used_size = env->hp - env->heap_frag->mem;
ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
}
free_tmp_objs(env);
}
void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif)
{
env->mod_nif = mod_nif;
env->proc = p;
env->hp = HEAP_TOP(p);
env->hp_end = HEAP_LIMIT(p);
env->heap_frag = NULL;
env->fpe_was_unmasked = erts_block_fpe();
env->tmp_obj_list = NULL;
}
void erts_factory_proc_init(ErtsHeapFactory* factory, Process* p)
{
/* This function does not use HAlloc to allocate on the heap
as we do not want to use INIT_HEAP_MEM on the allocated
heap as that completely destroys the DEBUG emulators
performance. */
ErlHeapFragment *bp = p->mbuf;
factory->mode = FACTORY_HALLOC;
factory->p = p;
factory->hp_start = HEAP_TOP(p);
factory->hp = factory->hp_start;
factory->hp_end = HEAP_LIMIT(p);
factory->off_heap = &p->off_heap;
factory->message = NULL;
factory->off_heap_saved.first = p->off_heap.first;
factory->off_heap_saved.overhead = p->off_heap.overhead;
factory->heap_frags_saved = bp;
factory->heap_frags_saved_used = bp ? bp->used_size : 0;
factory->heap_frags = NULL; /* not used */
factory->alloc_type = 0; /* not used */
HEAP_TOP(p) = HEAP_LIMIT(p);
}
static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp)
{
env->hp = hp;
if (env->heap_frag == NULL) {
ASSERT(HEAP_LIMIT(env->proc) == env->hp_end);
HEAP_TOP(env->proc) = env->hp;
}
else {
env->heap_frag->used_size = hp - env->heap_frag->mem;
ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
}
hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ);
env->heap_frag = MBUF(env->proc);
env->hp = hp + need;
env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size;
return hp;
}
ErtsMessage *
erts_try_alloc_message_on_heap(Process *pp,
erts_aint32_t *psp,
ErtsProcLocks *plp,
Uint sz,
Eterm **hpp,
ErlOffHeap **ohpp,
int *on_heap_p)
{
int locked_main = 0;
ErtsMessage *mp;
ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ));
if ((*psp) & ERTS_PSFLGS_VOLATILE_HEAP)
goto in_message_fragment;
else if (
*plp & ERTS_PROC_LOCK_MAIN
) {
try_on_heap:
if (((*psp) & ERTS_PSFLGS_VOLATILE_HEAP)
|| (pp->flags & F_DISABLE_GC)
|| HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) {
/*
* The heap is either potentially in an inconsistent
* state, or not large enough.
*/
if (locked_main) {
*plp &= ~ERTS_PROC_LOCK_MAIN;
erts_proc_unlock(pp, ERTS_PROC_LOCK_MAIN);
}
goto in_message_fragment;
}
*hpp = HEAP_TOP(pp);
HEAP_TOP(pp) = *hpp + sz;
*ohpp = &MSO(pp);
mp = erts_alloc_message(0, NULL);
mp->data.attached = NULL;
*on_heap_p = !0;
}
else if (pp && erts_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) {
locked_main = 1;
*psp = erts_atomic32_read_nob(&pp->state);
*plp |= ERTS_PROC_LOCK_MAIN;
goto try_on_heap;
}
else {
in_message_fragment:
if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) {
mp = erts_alloc_message(sz, hpp);
*ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
}
else {
mp = erts_alloc_message(0, NULL);
if (!sz) {
*hpp = NULL;
*ohpp = NULL;
}
else {
ErlHeapFragment *bp;
bp = new_message_buffer(sz);
*hpp = &bp->mem[0];
mp->data.heap_frag = bp;
*ohpp = &bp->off_heap;
}
}
*on_heap_p = 0;
}
return mp;
}
void erts_factory_undo(ErtsHeapFactory* factory)
{
ErlHeapFragment* bp;
struct erl_off_heap_header *hdr, **hdr_nextp;
switch (factory->mode) {
case FACTORY_HALLOC:
case FACTORY_STATIC:
/* Cleanup off-heap
*/
hdr_nextp = NULL;
for (hdr = factory->off_heap->first;
hdr != factory->off_heap_saved.first;
hdr = hdr->next) {
hdr_nextp = &hdr->next;
}
if (hdr_nextp != NULL) {
*hdr_nextp = NULL;
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = factory->off_heap_saved.first;
factory->off_heap->overhead = factory->off_heap_saved.overhead;
}
if (factory->mode == FACTORY_HALLOC) {
/* Free heap frags
*/
bp = factory->p->mbuf;
if (bp != factory->heap_frags_saved) {
do {
ErlHeapFragment *next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
} while (bp != factory->heap_frags_saved);
factory->p->mbuf = bp;
}
/* Rollback heap top
*/
if (HEAP_START(factory->p) <= factory->hp_start
&& factory->hp_start <= HEAP_LIMIT(factory->p)) {
HEAP_TOP(factory->p) = factory->hp_start;
}
/* Fix last heap frag */
if (factory->heap_frags_saved) {
ASSERT(factory->heap_frags_saved == factory->p->mbuf);
if (factory->hp_start != factory->heap_frags_saved->mem)
factory->heap_frags_saved->used_size = factory->heap_frags_saved_used;
else {
factory->p->mbuf = factory->p->mbuf->next;
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved,
ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size));
}
}
}
break;
case FACTORY_MESSAGE:
if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG)
factory->message->hfrag.next = factory->heap_frags;
else
factory->message->data.heap_frag = factory->heap_frags;
erts_cleanup_messages(factory->message);
break;
case FACTORY_TMP:
case FACTORY_HEAP_FRAGS:
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = NULL;
bp = factory->heap_frags;
while (bp != NULL) {
ErlHeapFragment* next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
}
break;
case FACTORY_CLOSED: break;
default:
ASSERT(!"Invalid factory mode");
}
factory->mode = FACTORY_CLOSED;
#ifdef DEBUG
factory->p = NULL;
factory->hp = NULL;
factory->heap_frags = NULL;
#endif
}
void erts_factory_proc_init(ErtsHeapFactory* factory,
Process* p)
{
erts_factory_proc_prealloc_init(factory, p, HEAP_LIMIT(p) - HEAP_TOP(p));
}
void erts_factory_undo(ErtsHeapFactory* factory)
{
ErlHeapFragment* bp;
struct erl_off_heap_header *hdr, **hdr_nextp;
switch (factory->mode) {
case FACTORY_HALLOC:
case FACTORY_STATIC:
/* Cleanup off-heap
*/
hdr_nextp = NULL;
for (hdr = factory->off_heap->first;
hdr != factory->off_heap_saved.first;
hdr = hdr->next) {
hdr_nextp = &hdr->next;
}
if (hdr_nextp != NULL) {
*hdr_nextp = NULL;
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = factory->off_heap_saved.first;
factory->off_heap->overhead = factory->off_heap_saved.overhead;
}
if (factory->mode == FACTORY_HALLOC) {
/* Free heap frags
*/
bp = factory->p->mbuf;
if (bp != factory->heap_frags_saved) {
do {
ErlHeapFragment *next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
} while (bp != factory->heap_frags_saved);
factory->p->mbuf = bp;
}
/* Rollback heap top
*/
if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */
ASSERT(factory->hp_start >= HEAP_START(factory->p));
ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p));
HEAP_TOP(factory->p) = factory->hp_start;
}
else {
ASSERT(factory->heap_frags_saved == factory->p->mbuf);
if (factory->hp_start == factory->heap_frags_saved->mem) {
factory->p->mbuf = factory->p->mbuf->next;
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved,
ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size));
}
else if (factory->hp_start != factory->hp_end) {
unsigned remains = factory->hp_start - factory->heap_frags_saved->mem;
ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size);
factory->heap_frags_saved->used_size = remains;
}
}
}
break;
case FACTORY_TMP:
case FACTORY_HEAP_FRAGS:
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = NULL;
bp = factory->heap_frags;
while (bp != NULL) {
ErlHeapFragment* next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
}
break;
case FACTORY_CLOSED: break;
default:
ASSERT(!"Invalid factory mode");
}
factory->mode = FACTORY_CLOSED;
#ifdef DEBUG
factory->p = NULL;
factory->hp = NULL;
factory->heap_frags = NULL;
#endif
}