// Free n objects from a span s back into the central free list c. // Called during sweep. // Returns true if the span was returned to heap. Sets sweepgen to // the latest generation. bool runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end) { if(s->incache) runtime·throw("freespan into cached span"); runtime·lock(c); // Move to nonempty if necessary. if(s->freelist == nil) { runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->nonempty, s); } // Add the objects back to s's free list. end->next = s->freelist; s->freelist = start; s->ref -= n; c->nfree += n; // delay updating sweepgen until here. This is the signal that // the span may be used in an MCache, so it must come after the // linked list operations above (actually, just after the // lock of c above.) runtime·atomicstore(&s->sweepgen, runtime·mheap.sweepgen); if(s->ref != 0) { runtime·unlock(c); return false; } // s is completely freed, return it to the heap. MCentral_ReturnToHeap(c, s); // unlocks c return true; }
// Return span from an MCache. void runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s) { MLink *v; int32 cap, n; runtime·lock(c); s->incache = false; // Move any explicitly freed items from the freebuf to the freelist. while((v = s->freebuf) != nil) { s->freebuf = v->next; runtime·markfreed(v); v->next = s->freelist; s->freelist = v; s->ref--; } if(s->ref == 0) { // Free back to heap. Unlikely, but possible. MCentral_ReturnToHeap(c, s); // unlocks c return; } cap = (s->npages << PageShift) / s->elemsize; n = cap - s->ref; if(n > 0) { c->nfree += n; runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->nonempty, s); } runtime·unlock(c); }
// Free n objects from a span s back into the central free list c. // Called from GC. void runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end) { int32 size; runtime·lock(c); // Move to nonempty if necessary. if(s->freelist == nil) { runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->nonempty, s); } // Add the objects back to s's free list. end->next = s->freelist; s->freelist = start; s->ref -= n; c->nfree += n; // If s is completely freed, return it to the heap. if(s->ref == 0) { size = runtime·class_to_size[c->sizeclass]; runtime·MSpanList_Remove(s); *(uintptr*)(s->start<<PageShift) = 1; // needs zeroing s->freelist = nil; c->nfree -= (s->npages << PageShift) / size; runtime·unlock(c); runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); runtime·MHeap_Free(runtime·mheap, s, 0); } else { runtime·unlock(c); } }
static void MHeap_FreeLocked(MHeap *h, MSpan *s) { MSpan *t; if(s->state != MSpanInUse || s->ref != 0) { printf("MHeap_FreeLocked - span %p ptr %p state %d ref %d\n", s, s->start<<PageShift, s->state, s->ref); throw("MHeap_FreeLocked - invalid free"); } s->state = MSpanFree; MSpanList_Remove(s); // Coalesce with earlier, later spans. if((t = MHeapMap_Get(&h->map, s->start - 1)) != nil && t->state != MSpanInUse) { s->start = t->start; s->npages += t->npages; MHeapMap_Set(&h->map, s->start, s); MSpanList_Remove(t); t->state = MSpanDead; FixAlloc_Free(&h->spanalloc, t); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; } if((t = MHeapMap_Get(&h->map, s->start + s->npages)) != nil && t->state != MSpanInUse) { s->npages += t->npages; MHeapMap_Set(&h->map, s->start + s->npages - 1, s); MSpanList_Remove(t); t->state = MSpanDead; FixAlloc_Free(&h->spanalloc, t); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; } // Insert s into appropriate list. if(s->npages < nelem(h->free)) MSpanList_Insert(&h->free[s->npages], s); else MSpanList_Insert(&h->large, s); // TODO(rsc): IncrementalScavenge() to return memory to OS. }
// Allocate up to n objects from the central free list. // Return the number of objects allocated. // The objects are linked together by their first words. // On return, *pstart points at the first object. int32 runtime·MCentral_AllocList(MCentral *c, int32 n, MLink **pfirst) { MSpan *s; MLink *first, *last; int32 cap, avail, i; runtime·lock(c); // Replenish central list if empty. if(runtime·MSpanList_IsEmpty(&c->nonempty)) { if(!MCentral_Grow(c)) { runtime·unlock(c); *pfirst = nil; return 0; } } s = c->nonempty.next; cap = (s->npages << PageShift) / s->elemsize; avail = cap - s->ref; if(avail < n) n = avail; // First one is guaranteed to work, because we just grew the list. first = s->freelist; last = first; for(i=1; i<n; i++) { last = last->next; } s->freelist = last->next; last->next = nil; s->ref += n; c->nfree -= n; if(n == avail) { if(s->freelist != nil || s->ref != cap) { runtime·throw("invalid freelist"); } runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->empty, s); } runtime·unlock(c); *pfirst = first; return n; }
// Helper: allocate one object from the central free list. static void* MCentral_Alloc(MCentral *c) { MSpan *s; MLink *v; if(runtime·MSpanList_IsEmpty(&c->nonempty)) return nil; s = c->nonempty.next; s->ref++; v = s->freelist; s->freelist = v->next; if(s->freelist == nil) { runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->empty, s); } return v; }
// Return span from an MCache. void runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s) { int32 cap, n; runtime·lock(&c->lock); s->incache = false; if(s->ref == 0) runtime·throw("uncaching full span"); cap = (s->npages << PageShift) / s->elemsize; n = cap - s->ref; if(n > 0) { runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->nonempty, s); } runtime·unlock(&c->lock); }
// Helper: free one object back into the central free list. // Caller must hold lock on c on entry. Holds lock on exit. static void MCentral_Free(MCentral *c, MLink *v) { MSpan *s; // Find span for v. s = runtime·MHeap_Lookup(&runtime·mheap, v); if(s == nil || s->ref == 0) runtime·throw("invalid free"); if(s->sweepgen != runtime·mheap.sweepgen) runtime·throw("free into unswept span"); // If the span is currently being used unsynchronized by an MCache, // we can't modify the freelist. Add to the freebuf instead. The // items will get moved to the freelist when the span is returned // by the MCache. if(s->incache) { v->next = s->freebuf; s->freebuf = v; return; } // Move span to nonempty if necessary. if(s->freelist == nil) { runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->nonempty, s); } // Add the object to span's free list. runtime·markfreed(v); v->next = s->freelist; s->freelist = v; s->ref--; c->nfree++; // If s is completely freed, return it to the heap. if(s->ref == 0) { MCentral_ReturnToHeap(c, s); // unlocks c runtime·lock(c); } }
// Helper: free one object back into the central free list. static void MCentral_Free(MCentral *c, void *v) { MSpan *s; MLink *p; int32 size; // Find span for v. s = runtime·MHeap_Lookup(runtime·mheap, v); if(s == nil || s->ref == 0) runtime·throw("invalid free"); // Move to nonempty if necessary. if(s->freelist == nil) { runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->nonempty, s); } // Add v back to s's free list. p = v; p->next = s->freelist; s->freelist = p; c->nfree++; // If s is completely freed, return it to the heap. if(--s->ref == 0) { size = runtime·class_to_size[c->sizeclass]; runtime·MSpanList_Remove(s); runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); *(uintptr*)(s->start<<PageShift) = 1; // needs zeroing s->freelist = nil; c->nfree -= (s->npages << PageShift) / size; runtime·unlock(c); runtime·MHeap_Free(runtime·mheap, s, 0); runtime·lock(c); } }
// Fetch a new span from the heap and // carve into objects for the free list. static bool MCentral_Grow(MCentral *c) { uintptr size, npages, i, n; MLink **tailp, *v; byte *p; MSpan *s; runtime·unlock(&c->lock); npages = runtime·class_to_allocnpages[c->sizeclass]; size = runtime·class_to_size[c->sizeclass]; n = (npages << PageShift) / size; s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1); if(s == nil) { // TODO(rsc): Log out of memory runtime·lock(&c->lock); return false; } // Carve span into sequence of blocks. tailp = &s->freelist; p = (byte*)(s->start << PageShift); s->limit = p + size*n; for(i=0; i<n; i++) { v = (MLink*)p; *tailp = v; tailp = &v->next; p += size; } *tailp = nil; runtime·markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift)); runtime·lock(&c->lock); runtime·MSpanList_Insert(&c->nonempty, s); return true; }
// Fetch a new span from the heap and // carve into objects for the free list. static bool MCentral_Grow(MCentral *c) { int32 i, n, npages; uintptr size; MLink **tailp, *v; byte *p; MSpan *s; runtime·unlock(c); runtime·MGetSizeClassInfo(c->sizeclass, &size, &npages, &n); s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1); if(s == nil) { // TODO(rsc): Log out of memory runtime·lock(c); return false; } // Carve span into sequence of blocks. tailp = &s->freelist; p = (byte*)(s->start << PageShift); s->limit = p + size*n; for(i=0; i<n; i++) { v = (MLink*)p; *tailp = v; tailp = &v->next; p += size; } *tailp = nil; runtime·markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift)); runtime·lock(c); c->nfree += n; runtime·MSpanList_Insert(&c->nonempty, s); return true; }