// Free n objects from a span s back into the central free list c.
// Called during sweep.
// Returns true if the span was returned to heap.
bool
runtime_MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end)
{
int32 size;
runtime_lock(c);
// Move to nonempty if necessary.
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
// Add the objects back to s's free list.
end->next = s->freelist;
s->freelist = start;
s->ref -= n;
c->nfree += n;
if(s->ref != 0) {
runtime_unlock(c);
return false;
}
// s is completely freed, return it to the heap.
size = runtime_class_to_size[c->sizeclass];
runtime_MSpanList_Remove(s);
s->needzero = 1;
s->freelist = nil;
c->nfree -= (s->npages << PageShift) / size;
runtime_unlock(c);
runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
runtime_MHeap_Free(&runtime_mheap, s, 0);
return true;
}
// Free n objects from a span s back into the central free list c.
// Called during sweep.
// Returns true if the span was returned to heap. Sets sweepgen to
// the latest generation.
bool
runtime_MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end)
{
if(s->incache)
runtime_throw("freespan into cached span");
runtime_lock(&c->lock);
// Move to nonempty if necessary.
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
// Add the objects back to s's free list.
end->next = s->freelist;
s->freelist = start;
s->ref -= n;
c->nfree += n;
// delay updating sweepgen until here. This is the signal that
// the span may be used in an MCache, so it must come after the
// linked list operations above (actually, just after the
// lock of c above.)
runtime_atomicstore(&s->sweepgen, runtime_mheap.sweepgen);
if(s->ref != 0) {
runtime_unlock(&c->lock);
return false;
}
// s is completely freed, return it to the heap.
MCentral_ReturnToHeap(c, s); // unlocks c
return true;
}
// Return span from an MCache.
void
runtime_MCentral_UncacheSpan(MCentral *c, MSpan *s)
{
MLink *v;
int32 cap, n;
runtime_lock(&c->lock);
s->incache = false;
// Move any explicitly freed items from the freebuf to the freelist.
while((v = s->freebuf) != nil) {
s->freebuf = v->next;
runtime_markfreed(v);
v->next = s->freelist;
s->freelist = v;
s->ref--;
}
if(s->ref == 0) {
// Free back to heap. Unlikely, but possible.
MCentral_ReturnToHeap(c, s); // unlocks c
return;
}
cap = (s->npages << PageShift) / s->elemsize;
n = cap - s->ref;
if(n > 0) {
c->nfree += n;
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
runtime_unlock(&c->lock);
}
// Helper: allocate one object from the central free list.
static void*
MCentral_Alloc(MCentral *c)
{
MSpan *s;
MLink *v;
if(runtime_MSpanList_IsEmpty(&c->nonempty))
return nil;
s = c->nonempty.next;
s->ref++;
v = s->freelist;
s->freelist = v->next;
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->empty, s);
}
return v;
}
// Helper: free one object back into the central free list.
// Caller must hold lock on c on entry. Holds lock on exit.
static void
MCentral_Free(MCentral *c, MLink *v)
{
MSpan *s;
// Find span for v.
s = runtime_MHeap_Lookup(&runtime_mheap, v);
if(s == nil || s->ref == 0)
runtime_throw("invalid free");
if(s->sweepgen != runtime_mheap.sweepgen)
runtime_throw("free into unswept span");
// If the span is currently being used unsynchronized by an MCache,
// we can't modify the freelist. Add to the freebuf instead. The
// items will get moved to the freelist when the span is returned
// by the MCache.
if(s->incache) {
v->next = s->freebuf;
s->freebuf = v;
return;
}
// Move span to nonempty if necessary.
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
// Add the object to span's free list.
runtime_markfreed(v);
v->next = s->freelist;
s->freelist = v;
s->ref--;
c->nfree++;
// If s is completely freed, return it to the heap.
if(s->ref == 0) {
MCentral_ReturnToHeap(c, s); // unlocks c
runtime_lock(&c->lock);
}
}
// Helper: free one object back into the central free list.
static void
MCentral_Free(MCentral *c, void *v)
{
MSpan *s;
MLink *p;
int32 size;
// Find span for v.
s = runtime_MHeap_Lookup(&runtime_mheap, v);
if(s == nil || s->ref == 0)
runtime_throw("invalid free");
// Move to nonempty if necessary.
if(s->freelist == nil) {
runtime_MSpanList_Remove(s);
runtime_MSpanList_Insert(&c->nonempty, s);
}
// Add v back to s's free list.
p = v;
p->next = s->freelist;
s->freelist = p;
c->nfree++;
// If s is completely freed, return it to the heap.
if(--s->ref == 0) {
size = runtime_class_to_size[c->sizeclass];
runtime_MSpanList_Remove(s);
runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
*(uintptr*)(s->start<<PageShift) = 1; // needs zeroing
s->freelist = nil;
c->nfree -= (s->npages << PageShift) / size;
runtime_unlock(c);
runtime_MHeap_Free(&runtime_mheap, s, 0);
runtime_lock(c);
}
}
// Fetch a new span from the heap and
// carve into objects for the free list.
static bool
MCentral_Grow(MCentral *c)
{
int32 i, n, npages;
uintptr size;
MLink **tailp, *v;
byte *p;
MSpan *s;
runtime_unlock(c);
runtime_MGetSizeClassInfo(c->sizeclass, &size, &npages, &n);
s = runtime_MHeap_Alloc(&runtime_mheap, npages, c->sizeclass, 0);
if(s == nil) {
// TODO(rsc): Log out of memory
runtime_lock(c);
return false;
}
// Carve span into sequence of blocks.
tailp = &s->freelist;
p = (byte*)(s->start << PageShift);
s->limit = p + size*n;
for(i=0; i<n; i++) {
v = (MLink*)p;
*tailp = v;
tailp = &v->next;
p += size;
}
*tailp = nil;
runtime_markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift));
runtime_lock(c);
c->nfree += n;
runtime_MSpanList_Insert(&c->nonempty, s);
return true;
}
