/* MVFFBufferFill -- Fill the buffer * * Fill it with the largest block we can find. */ static Res MVFFBufferFill(Addr *baseReturn, Addr *limitReturn, Pool pool, Buffer buffer, Size size, Bool withReservoirPermit) { Res res; MVFF mvff; Addr base, limit; Bool foundBlock; Seg seg = NULL; AVER(baseReturn != NULL); AVER(limitReturn != NULL); AVERT(Pool, pool); mvff = Pool2MVFF(pool); AVERT(MVFF, mvff); AVERT(Buffer, buffer); AVER(size > 0); AVER(SizeIsAligned(size, PoolAlignment(pool))); AVERT(Bool, withReservoirPermit); /* Hoping the largest is big enough, delete it and return if small. */ foundBlock = CBSFindLargest(&base, &limit, CBSOfMVFF(mvff), CBSFindDeleteENTIRE); if (foundBlock && AddrOffset(base, limit) < size) { foundBlock = FALSE; res = CBSInsert(CBSOfMVFF(mvff), base, limit); AVER(res == ResOK); } if (!foundBlock) { res = MVFFAddSeg(&seg, mvff, size, withReservoirPermit); if (res != ResOK) return res; foundBlock = CBSFindLargest(&base, &limit, CBSOfMVFF(mvff), CBSFindDeleteENTIRE); AVER(foundBlock); /* We will find the new segment. */ } AVER(AddrOffset(base, limit) >= size); mvff->free -= AddrOffset(base, limit); *baseReturn = base; *limitReturn = limit; return ResOK; }
static Bool MVFFFindLargest(Range range, Range oldRange, MVFF mvff, Size size, FindDelete findDelete) { AVER(range != NULL); AVER(oldRange != NULL); AVERT(MVFF, mvff); AVER(size > 0); AVERT(FindDelete, findDelete); FreelistFlushToCBS(FreelistOfMVFF(mvff), CBSOfMVFF(mvff)); if (CBSFindLargest(range, oldRange, CBSOfMVFF(mvff), size, findDelete)) return TRUE; if (FreelistFindLargest(range, oldRange, FreelistOfMVFF(mvff), size, findDelete)) return TRUE; return FALSE; }
/* MVFFFindFirstFree -- Finds the first (or last) suitable free block * * Finds a free block of the given (pool aligned) size, according * to a first (or last) fit policy controlled by the MVFF fields * firstFit, slotHigh (for whether to allocate the top or bottom * portion of a larger block). * * Will return FALSE if the free list has no large enough block. * In particular, will not attempt to allocate a new segment. */ static Bool MVFFFindFirstFree(Addr *baseReturn, Addr *limitReturn, MVFF mvff, Size size) { Bool foundBlock; FindDelete findDelete; RangeStruct range, oldRange; AVER(baseReturn != NULL); AVER(limitReturn != NULL); AVERT(MVFF, mvff); AVER(size > 0); AVER(SizeIsAligned(size, PoolAlignment(MVFF2Pool(mvff)))); FreelistFlushToCBS(FreelistOfMVFF(mvff), CBSOfMVFF(mvff)); findDelete = mvff->slotHigh ? FindDeleteHIGH : FindDeleteLOW; foundBlock = (mvff->firstFit ? CBSFindFirst : CBSFindLast) (&range, &oldRange, CBSOfMVFF(mvff), size, findDelete); if (!foundBlock) { /* Failed to find a block in the CBS: try the emergency free list * as well. */ foundBlock = (mvff->firstFit ? FreelistFindFirst : FreelistFindLast) (&range, &oldRange, FreelistOfMVFF(mvff), size, findDelete); } if (foundBlock) { *baseReturn = RangeBase(&range); *limitReturn = RangeLimit(&range); mvff->free -= size; } return foundBlock; }
/* MVFFFreeSegs -- Free segments from given range * * Given a free range, attempts to find entire segments within * it, and returns them to the arena, updating total size counter. * * This is usually called immediately after MVFFAddToFreeList. * It is not combined with MVFFAddToFreeList because the latter * is also called when new segments are added under MVFFAlloc. */ static void MVFFFreeSegs(MVFF mvff, Addr base, Addr limit) { Seg seg = NULL; /* suppress "may be used uninitialized" */ Arena arena; Bool b; Addr segLimit; /* limit of the current segment when iterating */ Addr segBase; /* base of the current segment when iterating */ Res res; AVERT(MVFF, mvff); AVER(base < limit); /* Could profitably AVER that the given range is free, */ /* but the CBS doesn't provide that facility. */ if (AddrOffset(base, limit) < mvff->minSegSize) return; /* not large enough for entire segments */ arena = PoolArena(MVFF2Pool(mvff)); b = SegOfAddr(&seg, arena, base); AVER(b); segBase = SegBase(seg); segLimit = SegLimit(seg); while(segLimit <= limit) { /* segment ends in range */ if (segBase >= base) { /* segment starts in range */ /* Must remove from free list first, in case free list */ /* is using inline data structures. */ res = CBSDelete(CBSOfMVFF(mvff), segBase, segLimit); AVER(res == ResOK); mvff->free -= AddrOffset(segBase, segLimit); mvff->total -= AddrOffset(segBase, segLimit); SegFree(seg); } /* Avoid calling SegNext if the next segment would fail */ /* the loop test, mainly because there might not be a */ /* next segment. */ if (segLimit == limit) /* segment ends at end of range */ break; b = SegNext(&seg, arena, segBase); AVER(b); segBase = SegBase(seg); segLimit = SegLimit(seg); } return; }
/* MVFFAddToFreeList -- Add given range to free list * * Updates MVFF counters for additional free space. Returns maximally * coalesced range containing given range. Does not attempt to free * segments (see MVFFFreeSegs). Cannot(!) fail. */ static void MVFFAddToFreeList(Addr *baseIO, Addr *limitIO, MVFF mvff) { Res res; Addr base, limit; AVER(baseIO != NULL); AVER(limitIO != NULL); AVERT(MVFF, mvff); base = *baseIO; limit = *limitIO; AVER(limit > base); res = CBSInsertReturningRange(baseIO, limitIO, CBSOfMVFF(mvff), base, limit); AVER(res == ResOK); mvff->free += AddrOffset(base, limit); return; }
static void MVFFFinish(Pool pool) { MVFF mvff; Arena arena; Seg seg; Ring ring, node, nextNode; AVERT(Pool, pool); mvff = Pool2MVFF(pool); AVERT(MVFF, mvff); /* Do this first, because the free list can use the segs in an emergency. */ CBSFinish(CBSOfMVFF(mvff)); ring = PoolSegRing(pool); RING_FOR(node, ring, nextNode) { seg = SegOfPoolRing(node); AVER(SegPool(seg) == pool); SegFree(seg); }
/* MVFFFindFirstFree -- Finds the first (or last) suitable free block * * Finds a free block of the given (pool aligned) size, according * to a first (or last) fit policy controlled by the MVFF fields * firstFit, slotHigh (for whether to allocate the top or bottom * portion of a larger block). * * Will return FALSE if the free list has no large enough block. * In particular, will not attempt to allocate a new segment. */ static Bool MVFFFindFirstFree(Addr *baseReturn, Addr *limitReturn, MVFF mvff, Size size) { Bool foundBlock; CBSFindDelete findDelete; AVER(baseReturn != NULL); AVER(limitReturn != NULL); AVERT(MVFF, mvff); AVER(size > 0); AVER(SizeIsAligned(size, PoolAlignment(MVFF2Pool(mvff)))); findDelete = mvff->slotHigh ? CBSFindDeleteHIGH : CBSFindDeleteLOW; foundBlock = (mvff->firstFit ? CBSFindFirst : CBSFindLast) (baseReturn, limitReturn, CBSOfMVFF(mvff), size, findDelete); if (foundBlock) mvff->free -= size; return foundBlock; }
/* MVFFAddToFreeList -- Add given range to free list * * Updates MVFF counters for additional free space. Returns maximally * coalesced range containing given range. Does not attempt to free * segments (see MVFFFreeSegs). */ static Res MVFFAddToFreeList(Addr *baseIO, Addr *limitIO, MVFF mvff) { Res res; RangeStruct range, newRange; AVER(baseIO != NULL); AVER(limitIO != NULL); AVERT(MVFF, mvff); RangeInit(&range, *baseIO, *limitIO); res = CBSInsert(&newRange, CBSOfMVFF(mvff), &range); if (ResIsAllocFailure(res)) { /* CBS ran out of memory for splay nodes: add range to emergency * free list instead. */ res = FreelistInsert(&newRange, FreelistOfMVFF(mvff), &range); } if (res == ResOK) { mvff->free += RangeSize(&range); *baseIO = RangeBase(&newRange); *limitIO = RangeLimit(&newRange); } return res; }
static Res MVFFInit(Pool pool, ArgList args) { Size extendBy = MVFF_EXTEND_BY_DEFAULT; Size avgSize = MVFF_AVG_SIZE_DEFAULT; Size align = MVFF_ALIGN_DEFAULT; Bool slotHigh = MVFF_SLOT_HIGH_DEFAULT; Bool arenaHigh = MVFF_ARENA_HIGH_DEFAULT; Bool firstFit = MVFF_FIRST_FIT_DEFAULT; MVFF mvff; Arena arena; Res res; void *p; ArgStruct arg; AVERT(Pool, pool); arena = PoolArena(pool); /* .arg: class-specific additional arguments; see */ /* <design/poolmvff/#method.init> */ /* .arg.check: we do the same checks here and in MVFFCheck */ /* except for arenaHigh, which is stored only in the segPref. */ if (ArgPick(&arg, args, MPS_KEY_EXTEND_BY)) extendBy = arg.val.size; if (ArgPick(&arg, args, MPS_KEY_MEAN_SIZE)) avgSize = arg.val.size; if (ArgPick(&arg, args, MPS_KEY_ALIGN)) align = arg.val.align; if (ArgPick(&arg, args, MPS_KEY_MVFF_SLOT_HIGH)) slotHigh = arg.val.b; if (ArgPick(&arg, args, MPS_KEY_MVFF_ARENA_HIGH)) arenaHigh = arg.val.b; if (ArgPick(&arg, args, MPS_KEY_MVFF_FIRST_FIT)) firstFit = arg.val.b; AVER(extendBy > 0); /* .arg.check */ AVER(avgSize > 0); /* .arg.check */ AVER(avgSize <= extendBy); /* .arg.check */ AVER(SizeIsAligned(align, MPS_PF_ALIGN)); AVERT(Bool, slotHigh); AVERT(Bool, arenaHigh); AVERT(Bool, firstFit); mvff = Pool2MVFF(pool); mvff->extendBy = extendBy; if (extendBy < ArenaAlign(arena)) mvff->minSegSize = ArenaAlign(arena); else mvff->minSegSize = extendBy; mvff->avgSize = avgSize; pool->alignment = align; mvff->slotHigh = slotHigh; mvff->firstFit = firstFit; res = ControlAlloc(&p, arena, sizeof(SegPrefStruct), FALSE); if (res != ResOK) return res; mvff->segPref = (SegPref)p; SegPrefInit(mvff->segPref); SegPrefExpress(mvff->segPref, arenaHigh ? SegPrefHigh : SegPrefLow, NULL); mvff->total = 0; mvff->free = 0; res = FreelistInit(FreelistOfMVFF(mvff), align); if (res != ResOK) goto failInit; res = CBSInit(CBSOfMVFF(mvff), arena, (void *)mvff, align, /* fastFind */ TRUE, /* zoned */ FALSE, args); if (res != ResOK) goto failInit; mvff->sig = MVFFSig; AVERT(MVFF, mvff); EVENT8(PoolInitMVFF, pool, arena, extendBy, avgSize, align, BOOL(slotHigh), BOOL(arenaHigh), BOOL(firstFit)); return ResOK; failInit: ControlFree(arena, p, sizeof(SegPrefStruct)); return res; }
/* MVFFFreeSegs -- Free segments from given range * * Given a free range, attempts to find entire segments within * it, and returns them to the arena, updating total size counter. * * This is usually called immediately after MVFFAddToFreeList. * It is not combined with MVFFAddToFreeList because the latter * is also called when new segments are added under MVFFAlloc. */ static void MVFFFreeSegs(MVFF mvff, Addr base, Addr limit) { Seg seg = NULL; /* suppress "may be used uninitialized" */ Arena arena; Bool b; Addr segLimit; /* limit of the current segment when iterating */ Addr segBase; /* base of the current segment when iterating */ Res res; AVERT(MVFF, mvff); AVER(base < limit); /* Could profitably AVER that the given range is free, */ /* but the CBS doesn't provide that facility. */ if (AddrOffset(base, limit) < mvff->minSegSize) return; /* not large enough for entire segments */ arena = PoolArena(MVFF2Pool(mvff)); b = SegOfAddr(&seg, arena, base); AVER(b); segBase = SegBase(seg); segLimit = SegLimit(seg); while(segLimit <= limit) { /* segment ends in range */ if (segBase >= base) { /* segment starts in range */ RangeStruct range, oldRange; RangeInit(&range, segBase, segLimit); res = CBSDelete(&oldRange, CBSOfMVFF(mvff), &range); if (res == ResOK) { mvff->free -= RangeSize(&range); } else if (ResIsAllocFailure(res)) { /* CBS ran out of memory for splay nodes, which must mean that * there were fragments on both sides: see * <design/cbs/#function.cbs.delete.fail>. Handle this by * deleting the whole of oldRange (which requires no * allocation) and re-inserting the fragments. */ RangeStruct oldRange2; res = CBSDelete(&oldRange2, CBSOfMVFF(mvff), &oldRange); AVER(res == ResOK); AVER(RangesEqual(&oldRange2, &oldRange)); mvff->free -= RangeSize(&oldRange); AVER(RangeBase(&oldRange) != segBase); { Addr leftBase = RangeBase(&oldRange); Addr leftLimit = segBase; res = MVFFAddToFreeList(&leftBase, &leftLimit, mvff); } AVER(RangeLimit(&oldRange) != segLimit); { Addr rightBase = segLimit; Addr rightLimit = RangeLimit(&oldRange); res = MVFFAddToFreeList(&rightBase, &rightLimit, mvff); } } else if (res == ResFAIL) { /* Not found in the CBS: must be found in the Freelist. */ res = FreelistDelete(&oldRange, FreelistOfMVFF(mvff), &range); AVER(res == ResOK); mvff->free -= RangeSize(&range); } AVER(res == ResOK); AVER(RangesNest(&oldRange, &range)); /* Can't free the segment earlier, because if it was on the * Freelist rather than the CBS then it likely contains data * that needs to be read in order to update the Freelist. */ SegFree(seg); mvff->total -= RangeSize(&range); } /* Avoid calling SegNext if the next segment would fail */ /* the loop test, mainly because there might not be a */ /* next segment. */ if (segLimit == limit) /* segment ends at end of range */ break; b = SegFindAboveAddr(&seg, arena, segBase); AVER(b); segBase = SegBase(seg); segLimit = SegLimit(seg); } return; }
static Res MVFFInit(Pool pool, va_list arg) { Size extendBy, avgSize, align; Bool slotHigh, arenaHigh, firstFit; MVFF mvff; Arena arena; Res res; void *p; ZoneSet zones; AVERT(Pool, pool); /* .arg: class-specific additional arguments; see */ /* <design/poolmvff/#method.init> */ /* .arg.check: we do the same checks here and in MVFFCheck */ /* except for arenaHigh, which is stored only in the segPref. */ extendBy = va_arg(arg, Size); avgSize = va_arg(arg, Size); align = va_arg(arg, Size); slotHigh = va_arg(arg, Bool); arenaHigh = va_arg(arg, Bool); firstFit = va_arg(arg, Bool); AVER(extendBy > 0); /* .arg.check */ AVER(avgSize > 0); /* .arg.check */ AVER(avgSize <= extendBy); /* .arg.check */ AVER(BoolCheck(slotHigh)); AVER(BoolCheck(arenaHigh)); AVER(BoolCheck(firstFit)); mvff = Pool2MVFF(pool); arena = PoolArena(pool); mvff->extendBy = extendBy; if (extendBy < ArenaAlign(arena)) mvff->minSegSize = ArenaAlign(arena); else mvff->minSegSize = extendBy; mvff->avgSize = avgSize; pool->alignment = align; mvff->slotHigh = slotHigh; mvff->firstFit = firstFit; res = ControlAlloc(&p, arena, sizeof(SegPrefStruct), FALSE); if (res != ResOK) return res; mvff->segPref = (SegPref)p; *mvff->segPref = *SegPrefDefault(); SegPrefExpress(mvff->segPref, arenaHigh ? SegPrefHigh : SegPrefLow, NULL); /* If using zoneset placement, just put it apart from the others. */ zones = ZoneSetComp(ArenaDefaultZONESET); SegPrefExpress(mvff->segPref, SegPrefZoneSet, (void *)&zones); mvff->total = 0; mvff->free = 0; res = CBSInit(arena, CBSOfMVFF(mvff), (void *)mvff, NULL, NULL, NULL, NULL, mvff->extendBy, align, TRUE, TRUE); if (res != ResOK) goto failInit; mvff->sig = MVFFSig; AVERT(MVFF, mvff); EVENT8(PoolInitMVFF, pool, arena, extendBy, avgSize, align, slotHigh, arenaHigh, firstFit); return ResOK; failInit: ControlFree(arena, p, sizeof(SegPrefStruct)); return res; }