// Free n objects from a span s back into the central free list c. // Called during sweep. // Returns true if the span was returned to heap. bool runtime_MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end) { int32 size; runtime_lock(c); // Move to nonempty if necessary. if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } // Add the objects back to s's free list. end->next = s->freelist; s->freelist = start; s->ref -= n; c->nfree += n; if(s->ref != 0) { runtime_unlock(c); return false; } // s is completely freed, return it to the heap. size = runtime_class_to_size[c->sizeclass]; runtime_MSpanList_Remove(s); s->needzero = 1; s->freelist = nil; c->nfree -= (s->npages << PageShift) / size; runtime_unlock(c); runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); runtime_MHeap_Free(&runtime_mheap, s, 0); return true; }
// Free n objects from a span s back into the central free list c. // Called during sweep. // Returns true if the span was returned to heap. Sets sweepgen to // the latest generation. bool runtime_MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end) { if(s->incache) runtime_throw("freespan into cached span"); runtime_lock(&c->lock); // Move to nonempty if necessary. if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } // Add the objects back to s's free list. end->next = s->freelist; s->freelist = start; s->ref -= n; c->nfree += n; // delay updating sweepgen until here. This is the signal that // the span may be used in an MCache, so it must come after the // linked list operations above (actually, just after the // lock of c above.) runtime_atomicstore(&s->sweepgen, runtime_mheap.sweepgen); if(s->ref != 0) { runtime_unlock(&c->lock); return false; } // s is completely freed, return it to the heap. MCentral_ReturnToHeap(c, s); // unlocks c return true; }
// Return span from an MCache. void runtime_MCentral_UncacheSpan(MCentral *c, MSpan *s) { MLink *v; int32 cap, n; runtime_lock(&c->lock); s->incache = false; // Move any explicitly freed items from the freebuf to the freelist. while((v = s->freebuf) != nil) { s->freebuf = v->next; runtime_markfreed(v); v->next = s->freelist; s->freelist = v; s->ref--; } if(s->ref == 0) { // Free back to heap. Unlikely, but possible. MCentral_ReturnToHeap(c, s); // unlocks c return; } cap = (s->npages << PageShift) / s->elemsize; n = cap - s->ref; if(n > 0) { c->nfree += n; runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } runtime_unlock(&c->lock); }
// Helper: allocate one object from the central free list. static void* MCentral_Alloc(MCentral *c) { MSpan *s; MLink *v; if(runtime_MSpanList_IsEmpty(&c->nonempty)) return nil; s = c->nonempty.next; s->ref++; v = s->freelist; s->freelist = v->next; if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->empty, s); } return v; }
// Helper: free one object back into the central free list. // Caller must hold lock on c on entry. Holds lock on exit. static void MCentral_Free(MCentral *c, MLink *v) { MSpan *s; // Find span for v. s = runtime_MHeap_Lookup(&runtime_mheap, v); if(s == nil || s->ref == 0) runtime_throw("invalid free"); if(s->sweepgen != runtime_mheap.sweepgen) runtime_throw("free into unswept span"); // If the span is currently being used unsynchronized by an MCache, // we can't modify the freelist. Add to the freebuf instead. The // items will get moved to the freelist when the span is returned // by the MCache. if(s->incache) { v->next = s->freebuf; s->freebuf = v; return; } // Move span to nonempty if necessary. if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } // Add the object to span's free list. runtime_markfreed(v); v->next = s->freelist; s->freelist = v; s->ref--; c->nfree++; // If s is completely freed, return it to the heap. if(s->ref == 0) { MCentral_ReturnToHeap(c, s); // unlocks c runtime_lock(&c->lock); } }
// Helper: free one object back into the central free list. static void MCentral_Free(MCentral *c, void *v) { MSpan *s; MLink *p; int32 size; // Find span for v. s = runtime_MHeap_Lookup(&runtime_mheap, v); if(s == nil || s->ref == 0) runtime_throw("invalid free"); // Move to nonempty if necessary. if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } // Add v back to s's free list. p = v; p->next = s->freelist; s->freelist = p; c->nfree++; // If s is completely freed, return it to the heap. if(--s->ref == 0) { size = runtime_class_to_size[c->sizeclass]; runtime_MSpanList_Remove(s); runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); *(uintptr*)(s->start<<PageShift) = 1; // needs zeroing s->freelist = nil; c->nfree -= (s->npages << PageShift) / size; runtime_unlock(c); runtime_MHeap_Free(&runtime_mheap, s, 0); runtime_lock(c); } }
// Fetch a new span from the heap and // carve into objects for the free list. static bool MCentral_Grow(MCentral *c) { int32 i, n, npages; uintptr size; MLink **tailp, *v; byte *p; MSpan *s; runtime_unlock(c); runtime_MGetSizeClassInfo(c->sizeclass, &size, &npages, &n); s = runtime_MHeap_Alloc(&runtime_mheap, npages, c->sizeclass, 0); if(s == nil) { // TODO(rsc): Log out of memory runtime_lock(c); return false; } // Carve span into sequence of blocks. tailp = &s->freelist; p = (byte*)(s->start << PageShift); s->limit = p + size*n; for(i=0; i<n; i++) { v = (MLink*)p; *tailp = v; tailp = &v->next; p += size; } *tailp = nil; runtime_markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift)); runtime_lock(c); c->nfree += n; runtime_MSpanList_Insert(&c->nonempty, s); return true; }