// Free n objects from a span s back into the central free list c. // Called during sweep. // Returns true if the span was returned to heap. bool runtime_MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end) { int32 size; runtime_lock(c); // Move to nonempty if necessary. if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } // Add the objects back to s's free list. end->next = s->freelist; s->freelist = start; s->ref -= n; c->nfree += n; if(s->ref != 0) { runtime_unlock(c); return false; } // s is completely freed, return it to the heap. size = runtime_class_to_size[c->sizeclass]; runtime_MSpanList_Remove(s); s->needzero = 1; s->freelist = nil; c->nfree -= (s->npages << PageShift) / size; runtime_unlock(c); runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); runtime_MHeap_Free(&runtime_mheap, s, 0); return true; }
// Sweeps spans in list until reclaims at least npages into heap. // Returns the actual number of pages reclaimed. static uintptr MHeap_ReclaimList(MHeap *h, MSpan *list, uintptr npages) { MSpan *s; uintptr n; uint32 sg; n = 0; sg = runtime_mheap.sweepgen; retry: for(s = list->next; s != list; s = s->next) { if(s->sweepgen == sg-2 && runtime_cas(&s->sweepgen, sg-2, sg-1)) { runtime_MSpanList_Remove(s); // swept spans are at the end of the list runtime_MSpanList_InsertBack(list, s); runtime_unlock(h); n += runtime_MSpan_Sweep(s); runtime_lock(h); if(n >= npages) return n; // the span could have been moved elsewhere goto retry; } if(s->sweepgen == sg-1) { // the span is being sweept by background sweeper, skip continue; } // already swept empty span, // all subsequent ones must also be either swept or in process of sweeping break; } return n; }
static MSpan* MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass) { uintptr n; MSpan *s, *t; PageID p; // To prevent excessive heap growth, before allocating n pages // we need to sweep and reclaim at least n pages. if(!h->sweepdone) MHeap_Reclaim(h, npage); // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { if(!runtime_MSpanList_IsEmpty(&h->free[n])) { s = h->free[n].next; goto HaveSpan; } } // Best fit in list of large spans. if((s = MHeap_AllocLarge(h, npage)) == nil) { if(!MHeap_Grow(h, npage)) return nil; if((s = MHeap_AllocLarge(h, npage)) == nil) return nil; } HaveSpan: // Mark span in use. if(s->state != MSpanFree) runtime_throw("MHeap_AllocLocked - MSpan not free"); if(s->npages < npage) runtime_throw("MHeap_AllocLocked - bad npages"); runtime_MSpanList_Remove(s); runtime_atomicstore(&s->sweepgen, h->sweepgen); s->state = MSpanInUse; mstats.heap_idle -= s->npages<<PageShift; mstats.heap_released -= s->npreleased<<PageShift; if(s->npreleased > 0) runtime_SysUsed((void*)(s->start<<PageShift), s->npages<<PageShift); s->npreleased = 0; if(s->npages > npage) { // Trim extra and put it back in the heap. t = runtime_FixAlloc_Alloc(&h->spanalloc); runtime_MSpan_Init(t, s->start + npage, s->npages - npage); s->npages = npage; p = t->start; p -= ((uintptr)h->arena_start>>PageShift); if(p > 0) h->spans[p-1] = s; h->spans[p] = t; h->spans[p+t->npages-1] = t; t->needzero = s->needzero; runtime_atomicstore(&t->sweepgen, h->sweepgen); t->state = MSpanInUse; MHeap_FreeLocked(h, t); t->unusedsince = s->unusedsince; // preserve age }
// Free n objects from a span s back into the central free list c. // Called during sweep. // Returns true if the span was returned to heap. Sets sweepgen to // the latest generation. bool runtime_MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *end) { if(s->incache) runtime_throw("freespan into cached span"); runtime_lock(&c->lock); // Move to nonempty if necessary. if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } // Add the objects back to s's free list. end->next = s->freelist; s->freelist = start; s->ref -= n; c->nfree += n; // delay updating sweepgen until here. This is the signal that // the span may be used in an MCache, so it must come after the // linked list operations above (actually, just after the // lock of c above.) runtime_atomicstore(&s->sweepgen, runtime_mheap.sweepgen); if(s->ref != 0) { runtime_unlock(&c->lock); return false; } // s is completely freed, return it to the heap. MCentral_ReturnToHeap(c, s); // unlocks c return true; }
// Return span from an MCache. void runtime_MCentral_UncacheSpan(MCentral *c, MSpan *s) { MLink *v; int32 cap, n; runtime_lock(&c->lock); s->incache = false; // Move any explicitly freed items from the freebuf to the freelist. while((v = s->freebuf) != nil) { s->freebuf = v->next; runtime_markfreed(v); v->next = s->freelist; s->freelist = v; s->ref--; } if(s->ref == 0) { // Free back to heap. Unlikely, but possible. MCentral_ReturnToHeap(c, s); // unlocks c return; } cap = (s->npages << PageShift) / s->elemsize; n = cap - s->ref; if(n > 0) { c->nfree += n; runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } runtime_unlock(&c->lock); }
static MSpan* MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass) { uintptr n; MSpan *s, *t; PageID p; // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { if(!runtime_MSpanList_IsEmpty(&h->free[n])) { s = h->free[n].next; goto HaveSpan; } } // Best fit in list of large spans. if((s = MHeap_AllocLarge(h, npage)) == nil) { if(!MHeap_Grow(h, npage)) return nil; if((s = MHeap_AllocLarge(h, npage)) == nil) return nil; } HaveSpan: // Mark span in use. if(s->state != MSpanFree) runtime_throw("MHeap_AllocLocked - MSpan not free"); if(s->npages < npage) runtime_throw("MHeap_AllocLocked - bad npages"); runtime_MSpanList_Remove(s); s->state = MSpanInUse; mstats.heap_idle -= s->npages<<PageShift; mstats.heap_released -= s->npreleased<<PageShift; s->npreleased = 0; if(s->npages > npage) { // Trim extra and put it back in the heap. t = runtime_FixAlloc_Alloc(&h->spanalloc); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; runtime_MSpan_Init(t, s->start + npage, s->npages - npage); s->npages = npage; p = t->start; if(sizeof(void*) == 8) p -= ((uintptr)h->arena_start>>PageShift); if(p > 0) h->map[p-1] = s; h->map[p] = t; h->map[p+t->npages-1] = t; *(uintptr*)(t->start<<PageShift) = *(uintptr*)(s->start<<PageShift); // copy "needs zeroing" mark t->state = MSpanInUse; MHeap_FreeLocked(h, t); }
// Helper: free one object back into the central free list. static void MCentral_Free(MCentral *c, void *v) { MSpan *s; MLink *p; int32 size; // Find span for v. s = runtime_MHeap_Lookup(&runtime_mheap, v); if(s == nil || s->ref == 0) runtime_throw("invalid free"); // Move to nonempty if necessary. if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } // Add v back to s's free list. p = v; p->next = s->freelist; s->freelist = p; c->nfree++; // If s is completely freed, return it to the heap. if(--s->ref == 0) { size = runtime_class_to_size[c->sizeclass]; runtime_MSpanList_Remove(s); runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); *(uintptr*)(s->start<<PageShift) = 1; // needs zeroing s->freelist = nil; c->nfree -= (s->npages << PageShift) / size; runtime_unlock(c); runtime_MHeap_Free(&runtime_mheap, s, 0); runtime_lock(c); } }
// Return s to the heap. s must be unused (s->ref == 0). Unlocks c. static void MCentral_ReturnToHeap(MCentral *c, MSpan *s) { int32 size; size = runtime_class_to_size[c->sizeclass]; runtime_MSpanList_Remove(s); s->needzero = 1; s->freelist = nil; if(s->ref != 0) runtime_throw("ref wrong"); c->nfree -= (s->npages << PageShift) / size; runtime_unlock(&c->lock); runtime_unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); runtime_MHeap_Free(&runtime_mheap, s, 0); }
// Helper: allocate one object from the central free list. static void* MCentral_Alloc(MCentral *c) { MSpan *s; MLink *v; if(runtime_MSpanList_IsEmpty(&c->nonempty)) return nil; s = c->nonempty.next; s->ref++; v = s->freelist; s->freelist = v->next; if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->empty, s); } return v; }
// Helper: free one object back into the central free list. // Caller must hold lock on c on entry. Holds lock on exit. static void MCentral_Free(MCentral *c, MLink *v) { MSpan *s; // Find span for v. s = runtime_MHeap_Lookup(&runtime_mheap, v); if(s == nil || s->ref == 0) runtime_throw("invalid free"); if(s->sweepgen != runtime_mheap.sweepgen) runtime_throw("free into unswept span"); // If the span is currently being used unsynchronized by an MCache, // we can't modify the freelist. Add to the freebuf instead. The // items will get moved to the freelist when the span is returned // by the MCache. if(s->incache) { v->next = s->freebuf; s->freebuf = v; return; } // Move span to nonempty if necessary. if(s->freelist == nil) { runtime_MSpanList_Remove(s); runtime_MSpanList_Insert(&c->nonempty, s); } // Add the object to span's free list. runtime_markfreed(v); v->next = s->freelist; s->freelist = v; s->ref--; c->nfree++; // If s is completely freed, return it to the heap. if(s->ref == 0) { MCentral_ReturnToHeap(c, s); // unlocks c runtime_lock(&c->lock); } }
// Allocate a list of objects from the central free list. // Return the number of objects allocated. // The objects are linked together by their first words. // On return, *pfirst points at the first object. int32 runtime_MCentral_AllocList(MCentral *c, MLink **pfirst) { MSpan *s; int32 cap, n; uint32 sg; runtime_lock(c); sg = runtime_mheap.sweepgen; retry: for(s = c->nonempty.next; s != &c->nonempty; s = s->next) { if(s->sweepgen == sg-2 && runtime_cas(&s->sweepgen, sg-2, sg-1)) { runtime_unlock(c); runtime_MSpan_Sweep(s); runtime_lock(c); // the span could have been moved to heap, retry goto retry; } if(s->sweepgen == sg-1) { // the span is being swept by background sweeper, skip continue; } // we have a nonempty span that does not require sweeping, allocate from it goto havespan; } for(s = c->empty.next; s != &c->empty; s = s->next) { if(s->sweepgen == sg-2 && runtime_cas(&s->sweepgen, sg-2, sg-1)) { // we have an empty span that requires sweeping, // sweep it and see if we can free some space in it runtime_MSpanList_Remove(s); // swept spans are at the end of the list runtime_MSpanList_InsertBack(&c->empty, s); runtime_unlock(c); runtime_MSpan_Sweep(s); runtime_lock(c); // the span could be moved to nonempty or heap, retry goto retry; } if(s->sweepgen == sg-1) { // the span is being swept by background sweeper, skip continue; } // already swept empty span, // all subsequent ones must also be either swept or in process of sweeping break; } // Replenish central list if empty. if(!MCentral_Grow(c)) { runtime_unlock(c); *pfirst = nil; return 0; } s = c->nonempty.next; havespan: cap = (s->npages << PageShift) / s->elemsize; n = cap - s->ref; *pfirst = s->freelist; s->freelist = nil; s->ref += n; c->nfree -= n; runtime_MSpanList_Remove(s); runtime_MSpanList_InsertBack(&c->empty, s); runtime_unlock(c); return n; }
static MSpan* MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass) { uintptr n; MSpan *s, *t; PageID p; // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { if(!runtime_MSpanList_IsEmpty(&h->free[n])) { s = h->free[n].next; goto HaveSpan; } } // Best fit in list of large spans. if((s = MHeap_AllocLarge(h, npage)) == nil) { if(!MHeap_Grow(h, npage)) return nil; if((s = MHeap_AllocLarge(h, npage)) == nil) return nil; } HaveSpan: // Mark span in use. if(s->state != MSpanFree) runtime_throw("MHeap_AllocLocked - MSpan not free"); if(s->npages < npage) runtime_throw("MHeap_AllocLocked - bad npages"); runtime_MSpanList_Remove(s); s->state = MSpanInUse; mstats.heap_idle -= s->npages<<PageShift; mstats.heap_released -= s->npreleased<<PageShift; if(s->npreleased > 0) { // We have called runtime_SysUnused with these pages, and on // Unix systems it called madvise. At this point at least // some BSD-based kernels will return these pages either as // zeros or with the old data. For our caller, the first word // in the page indicates whether the span contains zeros or // not (this word was set when the span was freed by // MCentral_Free or runtime_MCentral_FreeSpan). If the first // page in the span is returned as zeros, and some subsequent // page is returned with the old data, then we will be // returning a span that is assumed to be all zeros, but the // actual data will not be all zeros. Avoid that problem by // explicitly marking the span as not being zeroed, just in // case. The beadbead constant we use here means nothing, it // is just a unique constant not seen elsewhere in the // runtime, as a clue in case it turns up unexpectedly in // memory or in a stack trace. *(uintptr*)(s->start<<PageShift) = (uintptr)0xbeadbeadbeadbeadULL; } s->npreleased = 0; if(s->npages > npage) { // Trim extra and put it back in the heap. t = runtime_FixAlloc_Alloc(&h->spanalloc); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; runtime_MSpan_Init(t, s->start + npage, s->npages - npage); s->npages = npage; p = t->start; p -= ((uintptr)h->arena_start>>PageShift); if(p > 0) h->map[p-1] = s; h->map[p] = t; h->map[p+t->npages-1] = t; *(uintptr*)(t->start<<PageShift) = *(uintptr*)(s->start<<PageShift); // copy "needs zeroing" mark t->state = MSpanInUse; MHeap_FreeLocked(h, t); t->unusedsince = s->unusedsince; // preserve age }