// Free n objects from a span s back into the central free list c.
// Called during sweep.
// Returns true if the span was returned to heap. Sets sweepgen to
// the latest generation.
bool
runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end)
{
if(s->incache)
runtime·throw("freespan into cached span");
runtime·lock(c);
// Move to nonempty if necessary.
if(s->freelist == nil) {
runtime·MSpanList_Remove(s);
runtime·MSpanList_Insert(&c->nonempty, s);
}
// Add the objects back to s's free list.
end->next = s->freelist;
s->freelist = start;
s->ref -= n;
c->nfree += n;
// delay updating sweepgen until here. This is the signal that
// the span may be used in an MCache, so it must come after the
// linked list operations above (actually, just after the
// lock of c above.)
runtime·atomicstore(&s->sweepgen, runtime·mheap.sweepgen);
if(s->ref != 0) {
runtime·unlock(c);
return false;
}
// s is completely freed, return it to the heap.
MCentral_ReturnToHeap(c, s); // unlocks c
return true;
}
// Return span from an MCache.
void
runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s)
{
MLink *v;
int32 cap, n;
runtime·lock(c);
s->incache = false;
// Move any explicitly freed items from the freebuf to the freelist.
while((v = s->freebuf) != nil) {
s->freebuf = v->next;
runtime·markfreed(v);
v->next = s->freelist;
s->freelist = v;
s->ref--;
}
if(s->ref == 0) {
// Free back to heap. Unlikely, but possible.
MCentral_ReturnToHeap(c, s); // unlocks c
return;
}
cap = (s->npages << PageShift) / s->elemsize;
n = cap - s->ref;
if(n > 0) {
c->nfree += n;
runtime·MSpanList_Remove(s);
runtime·MSpanList_Insert(&c->nonempty, s);
}
runtime·unlock(c);
}
// Free n objects from a span s back into the central free list c.
// Called from GC.
void
runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end)
{
int32 size;
runtime·lock(c);
// Move to nonempty if necessary.
if(s->freelist == nil) {
runtime·MSpanList_Remove(s);
runtime·MSpanList_Insert(&c->nonempty, s);
}
// Add the objects back to s's free list.
end->next = s->freelist;
s->freelist = start;
s->ref -= n;
c->nfree += n;
// If s is completely freed, return it to the heap.
if(s->ref == 0) {
size = runtime·class_to_size[c->sizeclass];
runtime·MSpanList_Remove(s);
*(uintptr*)(s->start<<PageShift) = 1; // needs zeroing
s->freelist = nil;
c->nfree -= (s->npages << PageShift) / size;
runtime·unlock(c);
runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
runtime·MHeap_Free(runtime·mheap, s, 0);
} else {
runtime·unlock(c);
}
}
static void
MHeap_FreeLocked(MHeap *h, MSpan *s)
{
MSpan *t;
if(s->state != MSpanInUse || s->ref != 0) {
printf("MHeap_FreeLocked - span %p ptr %p state %d ref %d\n", s, s->start<<PageShift, s->state, s->ref);
throw("MHeap_FreeLocked - invalid free");
}
s->state = MSpanFree;
MSpanList_Remove(s);
// Coalesce with earlier, later spans.
if((t = MHeapMap_Get(&h->map, s->start - 1)) != nil && t->state != MSpanInUse) {
s->start = t->start;
s->npages += t->npages;
MHeapMap_Set(&h->map, s->start, s);
MSpanList_Remove(t);
t->state = MSpanDead;
FixAlloc_Free(&h->spanalloc, t);
mstats.mspan_inuse = h->spanalloc.inuse;
mstats.mspan_sys = h->spanalloc.sys;
}
if((t = MHeapMap_Get(&h->map, s->start + s->npages)) != nil && t->state != MSpanInUse) {
s->npages += t->npages;
MHeapMap_Set(&h->map, s->start + s->npages - 1, s);
MSpanList_Remove(t);
t->state = MSpanDead;
FixAlloc_Free(&h->spanalloc, t);
mstats.mspan_inuse = h->spanalloc.inuse;
mstats.mspan_sys = h->spanalloc.sys;
}
// Insert s into appropriate list.
if(s->npages < nelem(h->free))
MSpanList_Insert(&h->free[s->npages], s);
else
MSpanList_Insert(&h->large, s);
// TODO(rsc): IncrementalScavenge() to return memory to OS.
}
// Allocate up to n objects from the central free list.
// Return the number of objects allocated.
// The objects are linked together by their first words.
// On return, *pstart points at the first object.
int32
runtime·MCentral_AllocList(MCentral *c, int32 n, MLink **pfirst)
{
MSpan *s;
MLink *first, *last;
int32 cap, avail, i;
runtime·lock(c);
// Replenish central list if empty.
if(runtime·MSpanList_IsEmpty(&c->nonempty)) {
if(!MCentral_Grow(c)) {
runtime·unlock(c);
*pfirst = nil;
return 0;
}
}
s = c->nonempty.next;
cap = (s->npages << PageShift) / s->elemsize;
avail = cap - s->ref;
if(avail < n)
n = avail;
// First one is guaranteed to work, because we just grew the list.
first = s->freelist;
last = first;
for(i=1; i<n; i++) {
last = last->next;
}
s->freelist = last->next;
last->next = nil;
s->ref += n;
c->nfree -= n;
if(n == avail) {
if(s->freelist != nil || s->ref != cap) {
runtime·throw("invalid freelist");
}
runtime·MSpanList_Remove(s);
runtime·MSpanList_Insert(&c->empty, s);
}
runtime·unlock(c);
*pfirst = first;
return n;
}
// Helper: allocate one object from the central free list.
static void*
MCentral_Alloc(MCentral *c)
{
MSpan *s;
MLink *v;
if(runtime·MSpanList_IsEmpty(&c->nonempty))
return nil;
s = c->nonempty.next;
s->ref++;
v = s->freelist;
s->freelist = v->next;
if(s->freelist == nil) {
runtime·MSpanList_Remove(s);
runtime·MSpanList_Insert(&c->empty, s);
}
return v;
}
// Return span from an MCache.
void
runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s)
{
int32 cap, n;
runtime·lock(&c->lock);
s->incache = false;
if(s->ref == 0)
runtime·throw("uncaching full span");
cap = (s->npages << PageShift) / s->elemsize;
n = cap - s->ref;
if(n > 0) {
runtime·MSpanList_Remove(s);
runtime·MSpanList_Insert(&c->nonempty, s);
}
runtime·unlock(&c->lock);
}
// Helper: free one object back into the central free list.
// Caller must hold lock on c on entry. Holds lock on exit.
static void
MCentral_Free(MCentral *c, MLink *v)
{
MSpan *s;
// Find span for v.
s = runtime·MHeap_Lookup(&runtime·mheap, v);
if(s == nil || s->ref == 0)
runtime·throw("invalid free");
if(s->sweepgen != runtime·mheap.sweepgen)
runtime·throw("free into unswept span");
// If the span is currently being used unsynchronized by an MCache,
// we can't modify the freelist. Add to the freebuf instead. The
// items will get moved to the freelist when the span is returned
// by the MCache.
if(s->incache) {
v->next = s->freebuf;
s->freebuf = v;
return;
}
// Move span to nonempty if necessary.
if(s->freelist == nil) {
runtime·MSpanList_Remove(s);
runtime·MSpanList_Insert(&c->nonempty, s);
}
// Add the object to span's free list.
runtime·markfreed(v);
v->next = s->freelist;
s->freelist = v;
s->ref--;
c->nfree++;
// If s is completely freed, return it to the heap.
if(s->ref == 0) {
MCentral_ReturnToHeap(c, s); // unlocks c
runtime·lock(c);
}
}
// Helper: free one object back into the central free list.
static void
MCentral_Free(MCentral *c, void *v)
{
MSpan *s;
MLink *p;
int32 size;
// Find span for v.
s = runtime·MHeap_Lookup(runtime·mheap, v);
if(s == nil || s->ref == 0)
runtime·throw("invalid free");
// Move to nonempty if necessary.
if(s->freelist == nil) {
runtime·MSpanList_Remove(s);
runtime·MSpanList_Insert(&c->nonempty, s);
}
// Add v back to s's free list.
p = v;
p->next = s->freelist;
s->freelist = p;
c->nfree++;
// If s is completely freed, return it to the heap.
if(--s->ref == 0) {
size = runtime·class_to_size[c->sizeclass];
runtime·MSpanList_Remove(s);
runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
*(uintptr*)(s->start<<PageShift) = 1; // needs zeroing
s->freelist = nil;
c->nfree -= (s->npages << PageShift) / size;
runtime·unlock(c);
runtime·MHeap_Free(runtime·mheap, s, 0);
runtime·lock(c);
}
}
// Fetch a new span from the heap and
// carve into objects for the free list.
static bool
MCentral_Grow(MCentral *c)
{
uintptr size, npages, i, n;
MLink **tailp, *v;
byte *p;
MSpan *s;
runtime·unlock(&c->lock);
npages = runtime·class_to_allocnpages[c->sizeclass];
size = runtime·class_to_size[c->sizeclass];
n = (npages << PageShift) / size;
s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1);
if(s == nil) {
// TODO(rsc): Log out of memory
runtime·lock(&c->lock);
return false;
}
// Carve span into sequence of blocks.
tailp = &s->freelist;
p = (byte*)(s->start << PageShift);
s->limit = p + size*n;
for(i=0; i<n; i++) {
v = (MLink*)p;
*tailp = v;
tailp = &v->next;
p += size;
}
*tailp = nil;
runtime·markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift));
runtime·lock(&c->lock);
runtime·MSpanList_Insert(&c->nonempty, s);
return true;
}
// Fetch a new span from the heap and
// carve into objects for the free list.
static bool
MCentral_Grow(MCentral *c)
{
int32 i, n, npages;
uintptr size;
MLink **tailp, *v;
byte *p;
MSpan *s;
runtime·unlock(c);
runtime·MGetSizeClassInfo(c->sizeclass, &size, &npages, &n);
s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1);
if(s == nil) {
// TODO(rsc): Log out of memory
runtime·lock(c);
return false;
}
// Carve span into sequence of blocks.
tailp = &s->freelist;
p = (byte*)(s->start << PageShift);
s->limit = p + size*n;
for(i=0; i<n; i++) {
v = (MLink*)p;
*tailp = v;
tailp = &v->next;
p += size;
}
*tailp = nil;
runtime·markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift));
runtime·lock(c);
c->nfree += n;
runtime·MSpanList_Insert(&c->nonempty, s);
return true;
}
