// Free n objects from a span s back into the central free list c.
// Called during sweep.
// Returns true if the span was returned to heap.
bool
runtime_MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end)
{
int32 size;
runtime_lock(c);
// Move to nonempty if necessary.
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
// Add the objects back to s's free list.
end->next = s->freelist;
s->freelist = start;
s->ref -= n;
c->nfree += n;
if(s->ref != 0) {
runtime_unlock(c);
return false;
}
// s is completely freed, return it to the heap.
size = runtime_class_to_size[c->sizeclass];
runtime_MSpanList_Remove(s);
s->needzero = 1;
s->freelist = nil;
c->nfree -= (s->npages << PageShift) / size;
runtime_unlock(c);
runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
runtime_MHeap_Free(&runtime_mheap, s, 0);
return true;
}
// Sweeps spans in list until reclaims at least npages into heap.
// Returns the actual number of pages reclaimed.
static uintptr
MHeap_ReclaimList(MHeap *h, MSpan *list, uintptr npages)
{
MSpan *s;
uintptr n;
uint32 sg;
n = 0;
sg = runtime_mheap.sweepgen;
retry:
for(s = list->next; s != list; s = s->next) {
if(s->sweepgen == sg-2 && runtime_cas(&s->sweepgen, sg-2, sg-1)) {
runtime_MSpanList_Remove(s);
// swept spans are at the end of the list
runtime_MSpanList_InsertBack(list, s);
runtime_unlock(h);
n += runtime_MSpan_Sweep(s);
runtime_lock(h);
if(n >= npages)
return n;
// the span could have been moved elsewhere
goto retry;
}
if(s->sweepgen == sg-1) {
// the span is being sweept by background sweeper, skip
continue;
}
// already swept empty span,
// all subsequent ones must also be either swept or in process of sweeping
break;
}
return n;
}
static MSpan*
MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass)
{
uintptr n;
MSpan *s, *t;
PageID p;
// To prevent excessive heap growth, before allocating n pages
// we need to sweep and reclaim at least n pages.
if(!h->sweepdone)
MHeap_Reclaim(h, npage);
// Try in fixed-size lists up to max.
for(n=npage; n < nelem(h->free); n++) {
if(!runtime_MSpanList_IsEmpty(&h->free[n])) {
s = h->free[n].next;
goto HaveSpan;
}
}
// Best fit in list of large spans.
if((s = MHeap_AllocLarge(h, npage)) == nil) {
if(!MHeap_Grow(h, npage))
return nil;
if((s = MHeap_AllocLarge(h, npage)) == nil)
return nil;
}
HaveSpan:
// Mark span in use.
if(s->state != MSpanFree)
runtime_throw("MHeap_AllocLocked - MSpan not free");
if(s->npages < npage)
runtime_throw("MHeap_AllocLocked - bad npages");
runtime_MSpanList_Remove(s);
runtime_atomicstore(&s->sweepgen, h->sweepgen);
s->state = MSpanInUse;
mstats.heap_idle -= s->npages<<PageShift;
mstats.heap_released -= s->npreleased<<PageShift;
if(s->npreleased > 0)
runtime_SysUsed((void*)(s->start<<PageShift), s->npages<<PageShift);
s->npreleased = 0;
if(s->npages > npage) {
// Trim extra and put it back in the heap.
t = runtime_FixAlloc_Alloc(&h->spanalloc);
runtime_MSpan_Init(t, s->start + npage, s->npages - npage);
s->npages = npage;
p = t->start;
p -= ((uintptr)h->arena_start>>PageShift);
if(p > 0)
h->spans[p-1] = s;
h->spans[p] = t;
h->spans[p+t->npages-1] = t;
t->needzero = s->needzero;
runtime_atomicstore(&t->sweepgen, h->sweepgen);
t->state = MSpanInUse;
MHeap_FreeLocked(h, t);
t->unusedsince = s->unusedsince; // preserve age
}
// Free n objects from a span s back into the central free list c.
// Called during sweep.
// Returns true if the span was returned to heap. Sets sweepgen to
// the latest generation.
bool
runtime_MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end)
{
if(s->incache)
runtime_throw("freespan into cached span");
runtime_lock(&c->lock);
// Move to nonempty if necessary.
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
// Add the objects back to s's free list.
end->next = s->freelist;
s->freelist = start;
s->ref -= n;
c->nfree += n;
// delay updating sweepgen until here. This is the signal that
// the span may be used in an MCache, so it must come after the
// linked list operations above (actually, just after the
// lock of c above.)
runtime_atomicstore(&s->sweepgen, runtime_mheap.sweepgen);
if(s->ref != 0) {
runtime_unlock(&c->lock);
return false;
}
// s is completely freed, return it to the heap.
MCentral_ReturnToHeap(c, s); // unlocks c
return true;
}
// Return span from an MCache.
void
runtime_MCentral_UncacheSpan(MCentral *c, MSpan *s)
{
MLink *v;
int32 cap, n;
runtime_lock(&c->lock);
s->incache = false;
// Move any explicitly freed items from the freebuf to the freelist.
while((v = s->freebuf) != nil) {
s->freebuf = v->next;
runtime_markfreed(v);
v->next = s->freelist;
s->freelist = v;
s->ref--;
}
if(s->ref == 0) {
// Free back to heap. Unlikely, but possible.
MCentral_ReturnToHeap(c, s); // unlocks c
return;
}
cap = (s->npages << PageShift) / s->elemsize;
n = cap - s->ref;
if(n > 0) {
c->nfree += n;
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
runtime_unlock(&c->lock);
}
static MSpan*
MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass)
{
uintptr n;
MSpan *s, *t;
PageID p;
// Try in fixed-size lists up to max.
for(n=npage; n < nelem(h->free); n++) {
if(!runtime_MSpanList_IsEmpty(&h->free[n])) {
s = h->free[n].next;
goto HaveSpan;
}
}
// Best fit in list of large spans.
if((s = MHeap_AllocLarge(h, npage)) == nil) {
if(!MHeap_Grow(h, npage))
return nil;
if((s = MHeap_AllocLarge(h, npage)) == nil)
return nil;
}
HaveSpan:
// Mark span in use.
if(s->state != MSpanFree)
runtime_throw("MHeap_AllocLocked - MSpan not free");
if(s->npages < npage)
runtime_throw("MHeap_AllocLocked - bad npages");
runtime_MSpanList_Remove(s);
s->state = MSpanInUse;
mstats.heap_idle -= s->npages<<PageShift;
mstats.heap_released -= s->npreleased<<PageShift;
s->npreleased = 0;
if(s->npages > npage) {
// Trim extra and put it back in the heap.
t = runtime_FixAlloc_Alloc(&h->spanalloc);
mstats.mspan_inuse = h->spanalloc.inuse;
mstats.mspan_sys = h->spanalloc.sys;
runtime_MSpan_Init(t, s->start + npage, s->npages - npage);
s->npages = npage;
p = t->start;
if(sizeof(void*) == 8)
p -= ((uintptr)h->arena_start>>PageShift);
if(p > 0)
h->map[p-1] = s;
h->map[p] = t;
h->map[p+t->npages-1] = t;
*(uintptr*)(t->start<<PageShift) = *(uintptr*)(s->start<<PageShift); // copy "needs zeroing" mark
t->state = MSpanInUse;
MHeap_FreeLocked(h, t);
}
// Helper: free one object back into the central free list.
static void
MCentral_Free(MCentral *c, void *v)
{
MSpan *s;
MLink *p;
int32 size;
// Find span for v.
s = runtime_MHeap_Lookup(&runtime_mheap, v);
if(s == nil || s->ref == 0)
runtime_throw("invalid free");
// Move to nonempty if necessary.
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
// Add v back to s's free list.
p = v;
p->next = s->freelist;
s->freelist = p;
c->nfree++;
// If s is completely freed, return it to the heap.
if(--s->ref == 0) {
size = runtime_class_to_size[c->sizeclass];
runtime_MSpanList_Remove(s);
runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
*(uintptr*)(s->start<<PageShift) = 1; // needs zeroing
s->freelist = nil;
c->nfree -= (s->npages << PageShift) / size;
runtime_unlock(c);
runtime_MHeap_Free(&runtime_mheap, s, 0);
runtime_lock(c);
}
}
// Return s to the heap. s must be unused (s->ref == 0). Unlocks c.
static void
MCentral_ReturnToHeap(MCentral *c, MSpan *s)
{
int32 size;
size = runtime_class_to_size[c->sizeclass];
runtime_MSpanList_Remove(s);
s->needzero = 1;
s->freelist = nil;
if(s->ref != 0)
runtime_throw("ref wrong");
c->nfree -= (s->npages << PageShift) / size;
runtime_unlock(&c->lock);
runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
runtime_MHeap_Free(&runtime_mheap, s, 0);
}
// Helper: allocate one object from the central free list.
static void*
MCentral_Alloc(MCentral *c)
{
MSpan *s;
MLink *v;
if(runtime_MSpanList_IsEmpty(&c->nonempty))
return nil;
s = c->nonempty.next;
s->ref++;
v = s->freelist;
s->freelist = v->next;
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->empty, s);
}
return v;
}
// Helper: free one object back into the central free list.
// Caller must hold lock on c on entry. Holds lock on exit.
static void
MCentral_Free(MCentral *c, MLink *v)
{
MSpan *s;
// Find span for v.
s = runtime_MHeap_Lookup(&runtime_mheap, v);
if(s == nil || s->ref == 0)
runtime_throw("invalid free");
if(s->sweepgen != runtime_mheap.sweepgen)
runtime_throw("free into unswept span");
// If the span is currently being used unsynchronized by an MCache,
// we can't modify the freelist. Add to the freebuf instead. The
// items will get moved to the freelist when the span is returned
// by the MCache.
if(s->incache) {
v->next = s->freebuf;
s->freebuf = v;
return;
}
// Move span to nonempty if necessary.
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
// Add the object to span's free list.
runtime_markfreed(v);
v->next = s->freelist;
s->freelist = v;
s->ref--;
c->nfree++;
// If s is completely freed, return it to the heap.
if(s->ref == 0) {
MCentral_ReturnToHeap(c, s); // unlocks c
runtime_lock(&c->lock);
}
}
// Allocate a list of objects from the central free list.
// Return the number of objects allocated.
// The objects are linked together by their first words.
// On return, *pfirst points at the first object.
int32
runtime_MCentral_AllocList(MCentral *c, MLink **pfirst)
{
MSpan *s;
int32 cap, n;
uint32 sg;
runtime_lock(c);
sg = runtime_mheap.sweepgen;
retry:
for(s = c->nonempty.next; s != &c->nonempty; s = s->next) {
if(s->sweepgen == sg-2 && runtime_cas(&s->sweepgen, sg-2, sg-1)) {
runtime_unlock(c);
runtime_MSpan_Sweep(s);
runtime_lock(c);
// the span could have been moved to heap, retry
goto retry;
}
if(s->sweepgen == sg-1) {
// the span is being swept by background sweeper, skip
continue;
}
// we have a nonempty span that does not require sweeping, allocate from it
goto havespan;
}
for(s = c->empty.next; s != &c->empty; s = s->next) {
if(s->sweepgen == sg-2 && runtime_cas(&s->sweepgen, sg-2, sg-1)) {
// we have an empty span that requires sweeping,
// sweep it and see if we can free some space in it
runtime_MSpanList_Remove(s);
// swept spans are at the end of the list
runtime_MSpanList_InsertBack(&c->empty, s);
runtime_unlock(c);
runtime_MSpan_Sweep(s);
runtime_lock(c);
// the span could be moved to nonempty or heap, retry
goto retry;
}
if(s->sweepgen == sg-1) {
// the span is being swept by background sweeper, skip
continue;
}
// already swept empty span,
// all subsequent ones must also be either swept or in process of sweeping
break;
}
// Replenish central list if empty.
if(!MCentral_Grow(c)) {
runtime_unlock(c);
*pfirst = nil;
return 0;
}
s = c->nonempty.next;
havespan:
cap = (s->npages << PageShift) / s->elemsize;
n = cap - s->ref;
*pfirst = s->freelist;
s->freelist = nil;
s->ref += n;
c->nfree -= n;
runtime_MSpanList_Remove(s);
runtime_MSpanList_InsertBack(&c->empty, s);
runtime_unlock(c);
return n;
}
static MSpan*
MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass)
{
uintptr n;
MSpan *s, *t;
PageID p;
// Try in fixed-size lists up to max.
for(n=npage; n < nelem(h->free); n++) {
if(!runtime_MSpanList_IsEmpty(&h->free[n])) {
s = h->free[n].next;
goto HaveSpan;
}
}
// Best fit in list of large spans.
if((s = MHeap_AllocLarge(h, npage)) == nil) {
if(!MHeap_Grow(h, npage))
return nil;
if((s = MHeap_AllocLarge(h, npage)) == nil)
return nil;
}
HaveSpan:
// Mark span in use.
if(s->state != MSpanFree)
runtime_throw("MHeap_AllocLocked - MSpan not free");
if(s->npages < npage)
runtime_throw("MHeap_AllocLocked - bad npages");
runtime_MSpanList_Remove(s);
s->state = MSpanInUse;
mstats.heap_idle -= s->npages<<PageShift;
mstats.heap_released -= s->npreleased<<PageShift;
if(s->npreleased > 0) {
// We have called runtime_SysUnused with these pages, and on
// Unix systems it called madvise. At this point at least
// some BSD-based kernels will return these pages either as
// zeros or with the old data. For our caller, the first word
// in the page indicates whether the span contains zeros or
// not (this word was set when the span was freed by
// MCentral_Free or runtime_MCentral_FreeSpan). If the first
// page in the span is returned as zeros, and some subsequent
// page is returned with the old data, then we will be
// returning a span that is assumed to be all zeros, but the
// actual data will not be all zeros. Avoid that problem by
// explicitly marking the span as not being zeroed, just in
// case. The beadbead constant we use here means nothing, it
// is just a unique constant not seen elsewhere in the
// runtime, as a clue in case it turns up unexpectedly in
// memory or in a stack trace.
*(uintptr*)(s->start<<PageShift) = (uintptr)0xbeadbeadbeadbeadULL;
}
s->npreleased = 0;
if(s->npages > npage) {
// Trim extra and put it back in the heap.
t = runtime_FixAlloc_Alloc(&h->spanalloc);
mstats.mspan_inuse = h->spanalloc.inuse;
mstats.mspan_sys = h->spanalloc.sys;
runtime_MSpan_Init(t, s->start + npage, s->npages - npage);
s->npages = npage;
p = t->start;
p -= ((uintptr)h->arena_start>>PageShift);
if(p > 0)
h->map[p-1] = s;
h->map[p] = t;
h->map[p+t->npages-1] = t;
*(uintptr*)(t->start<<PageShift) = *(uintptr*)(s->start<<PageShift); // copy "needs zeroing" mark
t->state = MSpanInUse;
MHeap_FreeLocked(h, t);
t->unusedsince = s->unusedsince; // preserve age
}
