// Allocate up to n objects from the central free list. // Return the number of objects allocated. // The objects are linked together by their first words. // On return, *pstart points at the first object and *pend at the last. int32 runtime·MCentral_AllocList(MCentral *c, int32 n, MLink **pfirst) { MLink *first, *last, *v; int32 i; runtime·lock(c); // Replenish central list if empty. if(runtime·MSpanList_IsEmpty(&c->nonempty)) { if(!MCentral_Grow(c)) { runtime·unlock(c); *pfirst = nil; return 0; } } // Copy from list, up to n. // First one is guaranteed to work, because we just grew the list. first = MCentral_Alloc(c); last = first; for(i=1; i<n && (v = MCentral_Alloc(c)) != nil; i++) { last->next = v; last = v; } last->next = nil; c->nfree -= i; runtime·unlock(c); *pfirst = first; return i; }
// Allocates a span of the given size. h must be locked. // The returned span has been removed from the // free list, but its state is still MSpanFree. static MSpan* MHeap_AllocSpanLocked(MHeap *h, uintptr npage) { uintptr n; MSpan *s, *t; PageID p; // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { if(!runtime·MSpanList_IsEmpty(&h->free[n])) { s = h->free[n].next; goto HaveSpan; } } // Best fit in list of large spans. if((s = MHeap_AllocLarge(h, npage)) == nil) { if(!MHeap_Grow(h, npage)) return nil; if((s = MHeap_AllocLarge(h, npage)) == nil) return nil; } HaveSpan: // Mark span in use. if(s->state != MSpanFree) runtime·throw("MHeap_AllocLocked - MSpan not free"); if(s->npages < npage) runtime·throw("MHeap_AllocLocked - bad npages"); runtime·MSpanList_Remove(s); if(s->next != nil || s->prev != nil) runtime·throw("still in list"); if(s->npreleased > 0) { runtime·SysUsed((void*)(s->start<<PageShift), s->npages<<PageShift); mstats.heap_released -= s->npreleased<<PageShift; s->npreleased = 0; } if(s->npages > npage) { // Trim extra and put it back in the heap. t = runtime·FixAlloc_Alloc(&h->spanalloc); runtime·MSpan_Init(t, s->start + npage, s->npages - npage); s->npages = npage; p = t->start; p -= ((uintptr)h->arena_start>>PageShift); if(p > 0) h->spans[p-1] = s; h->spans[p] = t; h->spans[p+t->npages-1] = t; t->needzero = s->needzero; s->state = MSpanStack; // prevent coalescing with s t->state = MSpanStack; MHeap_FreeSpanLocked(h, t); t->unusedsince = s->unusedsince; // preserve age (TODO: wrong: t is possibly merged and/or deallocated at this point) s->state = MSpanFree; }
static MSpan* MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass) { uintptr n; MSpan *s, *t; PageID p; // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { if(!runtime·MSpanList_IsEmpty(&h->free[n])) { s = h->free[n].next; goto HaveSpan; } } // Best fit in list of large spans. if((s = MHeap_AllocLarge(h, npage)) == nil) { if(!MHeap_Grow(h, npage)) return nil; if((s = MHeap_AllocLarge(h, npage)) == nil) return nil; } HaveSpan: // Mark span in use. if(s->state != MSpanFree) runtime·throw("MHeap_AllocLocked - MSpan not free"); if(s->npages < npage) runtime·throw("MHeap_AllocLocked - bad npages"); runtime·MSpanList_Remove(s); s->state = MSpanInUse; mstats.heap_idle -= s->npages<<PageShift; mstats.heap_released -= s->npreleased<<PageShift; s->npreleased = 0; if(s->npages > npage) { // Trim extra and put it back in the heap. t = runtime·FixAlloc_Alloc(&h->spanalloc); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; runtime·MSpan_Init(t, s->start + npage, s->npages - npage); s->npages = npage; p = t->start; if(sizeof(void*) == 8) p -= ((uintptr)h->arena_start>>PageShift); if(p > 0) h->map[p-1] = s; h->map[p] = t; h->map[p+t->npages-1] = t; *(uintptr*)(t->start<<PageShift) = *(uintptr*)(s->start<<PageShift); // copy "needs zeroing" mark t->state = MSpanInUse; MHeap_FreeLocked(h, t); }
static MSpan* MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass) { uintptr n; MSpan *s, *t; // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { if(!MSpanList_IsEmpty(&h->free[n])) { s = h->free[n].next; goto HaveSpan; } } // Best fit in list of large spans. if((s = MHeap_AllocLarge(h, npage)) == nil) { if(!MHeap_Grow(h, npage)) return nil; if((s = MHeap_AllocLarge(h, npage)) == nil) return nil; } HaveSpan: // Mark span in use. if(s->state != MSpanFree) throw("MHeap_AllocLocked - MSpan not free"); if(s->npages < npage) throw("MHeap_AllocLocked - bad npages"); MSpanList_Remove(s); s->state = MSpanInUse; if(s->npages > npage) { // Trim extra and put it back in the heap. t = FixAlloc_Alloc(&h->spanalloc); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; MSpan_Init(t, s->start + npage, s->npages - npage); s->npages = npage; MHeapMap_Set(&h->map, t->start - 1, s); MHeapMap_Set(&h->map, t->start, t); MHeapMap_Set(&h->map, t->start + t->npages - 1, t); t->state = MSpanInUse; MHeap_FreeLocked(h, t); } // Record span info, because gc needs to be // able to map interior pointer to containing span. s->sizeclass = sizeclass; for(n=0; n<npage; n++) MHeapMap_Set(&h->map, s->start+n, s); return s; }
// Allocate up to n objects from the central free list. // Return the number of objects allocated. // The objects are linked together by their first words. // On return, *pstart points at the first object. int32 runtime·MCentral_AllocList(MCentral *c, int32 n, MLink **pfirst) { MSpan *s; MLink *first, *last; int32 cap, avail, i; runtime·lock(c); // Replenish central list if empty. if(runtime·MSpanList_IsEmpty(&c->nonempty)) { if(!MCentral_Grow(c)) { runtime·unlock(c); *pfirst = nil; return 0; } } s = c->nonempty.next; cap = (s->npages << PageShift) / s->elemsize; avail = cap - s->ref; if(avail < n) n = avail; // First one is guaranteed to work, because we just grew the list. first = s->freelist; last = first; for(i=1; i<n; i++) { last = last->next; } s->freelist = last->next; last->next = nil; s->ref += n; c->nfree -= n; if(n == avail) { if(s->freelist != nil || s->ref != cap) { runtime·throw("invalid freelist"); } runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->empty, s); } runtime·unlock(c); *pfirst = first; return n; }
// Helper: allocate one object from the central free list. static void* MCentral_Alloc(MCentral *c) { MSpan *s; MLink *v; if(runtime·MSpanList_IsEmpty(&c->nonempty)) return nil; s = c->nonempty.next; s->ref++; v = s->freelist; s->freelist = v->next; if(s->freelist == nil) { runtime·MSpanList_Remove(s); runtime·MSpanList_Insert(&c->empty, s); } return v; }
static MSpan* MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass) { uintptr n; MSpan *s, *t; PageID p; // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { if(!runtime·MSpanList_IsEmpty(&h->free[n])) { s = h->free[n].next; goto HaveSpan; } } // Best fit in list of large spans. if((s = MHeap_AllocLarge(h, npage)) == nil) { if(!MHeap_Grow(h, npage)) return nil; if((s = MHeap_AllocLarge(h, npage)) == nil) return nil; } HaveSpan: // Mark span in use. if(s->state != MSpanFree) runtime·throw("MHeap_AllocLocked - MSpan not free"); if(s->npages < npage) runtime·throw("MHeap_AllocLocked - bad npages"); runtime·MSpanList_Remove(s); s->state = MSpanInUse; mstats.heap_idle -= s->npages<<PageShift; mstats.heap_released -= s->npreleased<<PageShift; if(s->npreleased > 0) { // We have called runtime·SysUnused with these pages, and on // Unix systems it called madvise. At this point at least // some BSD-based kernels will return these pages either as // zeros or with the old data. For our caller, the first word // in the page indicates whether the span contains zeros or // not (this word was set when the span was freed by // MCentral_Free or runtime·MCentral_FreeSpan). If the first // page in the span is returned as zeros, and some subsequent // page is returned with the old data, then we will be // returning a span that is assumed to be all zeros, but the // actual data will not be all zeros. Avoid that problem by // explicitly marking the span as not being zeroed, just in // case. The beadbead constant we use here means nothing, it // is just a unique constant not seen elsewhere in the // runtime, as a clue in case it turns up unexpectedly in // memory or in a stack trace. *(uintptr*)(s->start<<PageShift) = (uintptr)0xbeadbeadbeadbeadULL; } s->npreleased = 0; if(s->npages > npage) { // Trim extra and put it back in the heap. t = runtime·FixAlloc_Alloc(&h->spanalloc); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; runtime·MSpan_Init(t, s->start + npage, s->npages - npage); s->npages = npage; p = t->start; if(sizeof(void*) == 8) p -= ((uintptr)h->arena_start>>PageShift); if(p > 0) h->map[p-1] = s; h->map[p] = t; h->map[p+t->npages-1] = t; *(uintptr*)(t->start<<PageShift) = *(uintptr*)(s->start<<PageShift); // copy "needs zeroing" mark t->state = MSpanInUse; MHeap_FreeLocked(h, t); t->unusedsince = s->unusedsince; // preserve age }