/* Allocate message buffer (size in words) */
ErlHeapFragment*
new_message_buffer(Uint size)
{
ErlHeapFragment* bp;
bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG,
ERTS_HEAP_FRAG_SIZE(size));
ERTS_INIT_HEAP_FRAG(bp, size);
return bp;
}
/* Allocate message buffer (size in words) */
ErlHeapFragment*
new_message_buffer(Uint size)
{
ErlHeapFragment* bp;
bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG,
ERTS_HEAP_FRAG_SIZE(size));
ERTS_INIT_HEAP_FRAG(bp, size, size);
VERBOSE(DEBUG_SHCOPY, ("[pid=%T] new message buffer %p\n", erts_get_current_pid(), bp->mem));
return bp;
}
void
free_message_buffer(ErlHeapFragment* bp)
{
ASSERT(bp != NULL);
do {
ErlHeapFragment* next_bp = bp->next;
erts_cleanup_offheap(&bp->off_heap);
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
}while (bp != NULL);
}
static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra)
{
ErlHeapFragment* bp;
switch (factory->mode) {
case FACTORY_HALLOC:
HRelease(factory->p, factory->hp_end, factory->hp);
factory->hp = HAllocX(factory->p, need, xtra);
factory->hp_end = factory->hp + need;
return;
case FACTORY_MESSAGE:
if (!factory->heap_frags) {
ASSERT(factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG);
bp = &factory->message->hfrag;
}
else {
/* Fall through */
case FACTORY_HEAP_FRAGS:
case FACTORY_TMP:
bp = factory->heap_frags;
}
if (bp) {
ASSERT(factory->hp > bp->mem);
ASSERT(factory->hp <= factory->hp_end);
ASSERT(factory->hp_end == bp->mem + bp->alloc_size);
bp->used_size = factory->hp - bp->mem;
}
bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type,
ERTS_HEAP_FRAG_SIZE(need+xtra));
bp->next = factory->heap_frags;
factory->heap_frags = bp;
bp->alloc_size = need + xtra;
bp->used_size = need;
bp->off_heap.first = NULL;
bp->off_heap.overhead = 0;
factory->hp = bp->mem;
factory->hp_end = bp->mem + bp->alloc_size;
return;
case FACTORY_STATIC:
case FACTORY_CLOSED:
default:
ASSERT(!"Invalid factory mode");
}
}
ErlHeapFragment*
erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
Eterm *brefs, Uint brefs_size)
{
#ifdef DEBUG
int i;
#endif
#ifdef HARD_DEBUG
ErlHeapFragment *dbg_bp;
Eterm *dbg_brefs;
Uint dbg_size;
Uint dbg_tot_size;
Eterm *dbg_hp;
#endif
ErlHeapFragment* nbp;
#ifdef DEBUG
{
Uint off_sz = size < bp->used_size ? size : bp->used_size;
for (i = 0; i < brefs_size; i++) {
Eterm *ptr;
if (is_immed(brefs[i]))
continue;
ptr = ptr_val(brefs[i]);
ASSERT(&bp->mem[0] <= ptr && ptr < &bp->mem[0] + off_sz);
}
}
#endif
if (size >= (bp->used_size - bp->used_size / 16)) {
bp->used_size = size;
return bp;
}
#ifdef HARD_DEBUG
dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size);
dbg_bp = new_message_buffer(bp->used_size);
dbg_hp = dbg_bp->mem;
dbg_tot_size = 0;
for (i = 0; i < brefs_size; i++) {
dbg_size = size_object(brefs[i]);
dbg_tot_size += dbg_size;
dbg_brefs[i] = copy_struct(brefs[i], dbg_size, &dbg_hp,
&dbg_bp->off_heap);
}
ASSERT(dbg_tot_size == (size < bp->used_size ? size : bp->used_size));
#endif
nbp = (ErlHeapFragment*) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP_FRAG,
(void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size),
ERTS_HEAP_FRAG_SIZE(size));
if (bp != nbp) {
Uint off_sz = size < nbp->used_size ? size : nbp->used_size;
Eterm *sp = &bp->mem[0];
Eterm *ep = sp + off_sz;
Sint offs = &nbp->mem[0] - sp;
erts_offset_off_heap(&nbp->off_heap, offs, sp, ep);
erts_offset_heap(&nbp->mem[0], off_sz, offs, sp, ep);
if (brefs && brefs_size)
erts_offset_heap_ptr(brefs, brefs_size, offs, sp, ep);
#ifdef DEBUG
for (i = 0; i < brefs_size; i++) {
Eterm *ptr;
if (is_immed(brefs[i]))
continue;
ptr = ptr_val(brefs[i]);
ASSERT(&nbp->mem[0] <= ptr && ptr < &nbp->mem[0] + off_sz);
}
#endif
}
nbp->alloc_size = size;
nbp->used_size = size;
#ifdef HARD_DEBUG
for (i = 0; i < brefs_size; i++)
ASSERT(eq(dbg_brefs[i], brefs[i]));
free_message_buffer(dbg_bp);
erts_free(ERTS_ALC_T_UNDEF, dbg_brefs);
#endif
return nbp;
}
void erts_factory_undo(ErtsHeapFactory* factory)
{
ErlHeapFragment* bp;
struct erl_off_heap_header *hdr, **hdr_nextp;
switch (factory->mode) {
case FACTORY_HALLOC:
case FACTORY_STATIC:
/* Cleanup off-heap
*/
hdr_nextp = NULL;
for (hdr = factory->off_heap->first;
hdr != factory->off_heap_saved.first;
hdr = hdr->next) {
hdr_nextp = &hdr->next;
}
if (hdr_nextp != NULL) {
*hdr_nextp = NULL;
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = factory->off_heap_saved.first;
factory->off_heap->overhead = factory->off_heap_saved.overhead;
}
if (factory->mode == FACTORY_HALLOC) {
/* Free heap frags
*/
bp = factory->p->mbuf;
if (bp != factory->heap_frags_saved) {
do {
ErlHeapFragment *next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
} while (bp != factory->heap_frags_saved);
factory->p->mbuf = bp;
}
/* Rollback heap top
*/
if (HEAP_START(factory->p) <= factory->hp_start
&& factory->hp_start <= HEAP_LIMIT(factory->p)) {
HEAP_TOP(factory->p) = factory->hp_start;
}
/* Fix last heap frag */
if (factory->heap_frags_saved) {
ASSERT(factory->heap_frags_saved == factory->p->mbuf);
if (factory->hp_start != factory->heap_frags_saved->mem)
factory->heap_frags_saved->used_size = factory->heap_frags_saved_used;
else {
factory->p->mbuf = factory->p->mbuf->next;
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved,
ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size));
}
}
}
break;
case FACTORY_MESSAGE:
if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG)
factory->message->hfrag.next = factory->heap_frags;
else
factory->message->data.heap_frag = factory->heap_frags;
erts_cleanup_messages(factory->message);
break;
case FACTORY_TMP:
case FACTORY_HEAP_FRAGS:
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = NULL;
bp = factory->heap_frags;
while (bp != NULL) {
ErlHeapFragment* next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
}
break;
case FACTORY_CLOSED: break;
default:
ASSERT(!"Invalid factory mode");
}
factory->mode = FACTORY_CLOSED;
#ifdef DEBUG
factory->p = NULL;
factory->hp = NULL;
factory->heap_frags = NULL;
#endif
}
void erts_factory_undo(ErtsHeapFactory* factory)
{
ErlHeapFragment* bp;
struct erl_off_heap_header *hdr, **hdr_nextp;
switch (factory->mode) {
case FACTORY_HALLOC:
case FACTORY_STATIC:
/* Cleanup off-heap
*/
hdr_nextp = NULL;
for (hdr = factory->off_heap->first;
hdr != factory->off_heap_saved.first;
hdr = hdr->next) {
hdr_nextp = &hdr->next;
}
if (hdr_nextp != NULL) {
*hdr_nextp = NULL;
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = factory->off_heap_saved.first;
factory->off_heap->overhead = factory->off_heap_saved.overhead;
}
if (factory->mode == FACTORY_HALLOC) {
/* Free heap frags
*/
bp = factory->p->mbuf;
if (bp != factory->heap_frags_saved) {
do {
ErlHeapFragment *next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
} while (bp != factory->heap_frags_saved);
factory->p->mbuf = bp;
}
/* Rollback heap top
*/
if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */
ASSERT(factory->hp_start >= HEAP_START(factory->p));
ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p));
HEAP_TOP(factory->p) = factory->hp_start;
}
else {
ASSERT(factory->heap_frags_saved == factory->p->mbuf);
if (factory->hp_start == factory->heap_frags_saved->mem) {
factory->p->mbuf = factory->p->mbuf->next;
ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved,
ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size));
}
else if (factory->hp_start != factory->hp_end) {
unsigned remains = factory->hp_start - factory->heap_frags_saved->mem;
ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size);
factory->heap_frags_saved->used_size = remains;
}
}
}
break;
case FACTORY_TMP:
case FACTORY_HEAP_FRAGS:
erts_cleanup_offheap(factory->off_heap);
factory->off_heap->first = NULL;
bp = factory->heap_frags;
while (bp != NULL) {
ErlHeapFragment* next_bp = bp->next;
ASSERT(bp->off_heap.first == NULL);
ERTS_HEAP_FREE(factory->alloc_type, (void *) bp,
ERTS_HEAP_FRAG_SIZE(bp->alloc_size));
bp = next_bp;
}
break;
case FACTORY_CLOSED: break;
default:
ASSERT(!"Invalid factory mode");
}
factory->mode = FACTORY_CLOSED;
#ifdef DEBUG
factory->p = NULL;
factory->hp = NULL;
factory->heap_frags = NULL;
#endif
}
void erts_reserve_heap__(ErtsHeapFactory* factory, Uint need, Uint xtra)
{
/* internal... */
ErlHeapFragment* bp;
switch (factory->mode) {
case FACTORY_HALLOC:
HRelease(factory->p, factory->hp_end, factory->hp);
factory->hp = HAllocX(factory->p, need, xtra);
factory->hp_end = factory->hp + need;
return;
case FACTORY_MESSAGE: {
int replace_oh;
int replace_msg_hfrag;
if (!factory->heap_frags) {
ASSERT(factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG);
bp = &factory->message->hfrag;
}
else {
/* Fall through */
case FACTORY_HEAP_FRAGS:
case FACTORY_TMP:
bp = factory->heap_frags;
}
replace_oh = 0;
replace_msg_hfrag = 0;
if (bp) {
ASSERT(factory->hp >= bp->mem);
ASSERT(factory->hp <= factory->hp_end);
ASSERT(factory->hp_end == bp->mem + bp->alloc_size);
bp->used_size = factory->hp - bp->mem;
if (!bp->used_size && factory->heap_frags) {
factory->heap_frags = bp->next;
bp->next = NULL;
ASSERT(!bp->off_heap.first);
if (factory->off_heap == &bp->off_heap)
replace_oh = !0;
if (factory->message && factory->message->data.heap_frag == bp)
replace_msg_hfrag = !0;
free_message_buffer(bp);
}
}
bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type,
ERTS_HEAP_FRAG_SIZE(need+xtra));
bp->next = factory->heap_frags;
factory->heap_frags = bp;
bp->alloc_size = need + xtra;
bp->used_size = need + xtra;
bp->off_heap.first = NULL;
bp->off_heap.overhead = 0;
if (replace_oh) {
factory->off_heap = &bp->off_heap;
factory->off_heap_saved.first = factory->off_heap->first;
factory->off_heap_saved.overhead = factory->off_heap->overhead;
}
if (replace_msg_hfrag)
factory->message->data.heap_frag = bp;
factory->hp = bp->mem;
factory->hp_end = bp->mem + bp->alloc_size;
return;
}
case FACTORY_STATIC:
case FACTORY_CLOSED:
default:
ASSERT(!"Invalid factory mode");
}
}