Bool RangesNest(Range outer, Range inner)
{
AVERT(Range, outer);
AVERT(Range, inner);
return RangeBase(outer) <= RangeBase(inner)
&& RangeLimit(inner) <= RangeLimit(outer);
}
Bool RangesEqual(Range range1, Range range2)
{
AVERT(Range, range1);
AVERT(Range, range2);
return RangeBase(range1) == RangeBase(range2)
&& RangeLimit(range1) == RangeLimit(range2);
}
Bool RangesOverlap(Range range1, Range range2)
{
AVERT(Range, range1);
AVERT(Range, range2);
return RangeBase(range1) < RangeLimit(range2)
&& RangeBase(range2) < RangeLimit(range1);
}
static Bool checkCallback(Range range, void *closureP, Size closureS)
{
Addr base, limit;
CheckFBMClosure cl = (CheckFBMClosure)closureP;
UNUSED(closureS);
Insist(cl != NULL);
base = RangeBase(range);
limit = RangeLimit(range);
if (base > cl->oldLimit) {
Insist(BTIsSetRange(cl->state->allocTable,
indexOfAddr(cl->state, cl->oldLimit),
indexOfAddr(cl->state, base)));
} else { /* must be at start of table */
Insist(base == cl->oldLimit);
Insist(cl->oldLimit == cl->state->block);
}
Insist(BTIsResRange(cl->state->allocTable,
indexOfAddr(cl->state, base),
indexOfAddr(cl->state, limit)));
cl->oldLimit = limit;
return TRUE;
}
Res RangeDescribe(Range range, mps_lib_FILE *stream, Count depth)
{
Res res;
AVERT(Range, range);
AVER(stream != NULL);
res = WriteF(stream, depth,
"Range $P {\n", (WriteFP)range,
" base: $P\n", (WriteFP)RangeBase(range),
" limit: $P\n", (WriteFP)RangeLimit(range),
" size: $U\n", (WriteFU)RangeSize(range),
"} Range $P\n", (WriteFP)range, NULL);
if (res != ResOK) {
return res;
}
return ResOK;
}
/* MVFFFindFirstFree -- Finds the first (or last) suitable free block
*
* Finds a free block of the given (pool aligned) size, according
* to a first (or last) fit policy controlled by the MVFF fields
* firstFit, slotHigh (for whether to allocate the top or bottom
* portion of a larger block).
*
* Will return FALSE if the free list has no large enough block.
* In particular, will not attempt to allocate a new segment.
*/
static Bool MVFFFindFirstFree(Addr *baseReturn, Addr *limitReturn,
MVFF mvff, Size size)
{
Bool foundBlock;
FindDelete findDelete;
RangeStruct range, oldRange;
AVER(baseReturn != NULL);
AVER(limitReturn != NULL);
AVERT(MVFF, mvff);
AVER(size > 0);
AVER(SizeIsAligned(size, PoolAlignment(MVFF2Pool(mvff))));
FreelistFlushToCBS(FreelistOfMVFF(mvff), CBSOfMVFF(mvff));
findDelete = mvff->slotHigh ? FindDeleteHIGH : FindDeleteLOW;
foundBlock =
(mvff->firstFit ? CBSFindFirst : CBSFindLast)
(&range, &oldRange, CBSOfMVFF(mvff), size, findDelete);
if (!foundBlock) {
/* Failed to find a block in the CBS: try the emergency free list
* as well. */
foundBlock =
(mvff->firstFit ? FreelistFindFirst : FreelistFindLast)
(&range, &oldRange, FreelistOfMVFF(mvff), size, findDelete);
}
if (foundBlock) {
*baseReturn = RangeBase(&range);
*limitReturn = RangeLimit(&range);
mvff->free -= size;
}
return foundBlock;
}
/* MVFFBufferFill -- Fill the buffer
*
* Fill it with the largest block we can find.
*/
static Res MVFFBufferFill(Addr *baseReturn, Addr *limitReturn,
Pool pool, Buffer buffer, Size size,
Bool withReservoirPermit)
{
Res res;
MVFF mvff;
RangeStruct range, oldRange;
Bool found;
Seg seg = NULL;
AVER(baseReturn != NULL);
AVER(limitReturn != NULL);
AVERT(Pool, pool);
mvff = Pool2MVFF(pool);
AVERT(MVFF, mvff);
AVERT(Buffer, buffer);
AVER(size > 0);
AVER(SizeIsAligned(size, PoolAlignment(pool)));
AVERT(Bool, withReservoirPermit);
found = MVFFFindLargest(&range, &oldRange, mvff, size, FindDeleteENTIRE);
if (!found) {
/* Add a new segment to the free list and try again. */
res = MVFFAddSeg(&seg, mvff, size, withReservoirPermit);
if (res != ResOK)
return res;
found = MVFFFindLargest(&range, &oldRange, mvff, size, FindDeleteENTIRE);
}
AVER(found);
AVER(RangeSize(&range) >= size);
mvff->free -= RangeSize(&range);
*baseReturn = RangeBase(&range);
*limitReturn = RangeLimit(&range);
return ResOK;
}
/* MVFFAddToFreeList -- Add given range to free list
*
* Updates MVFF counters for additional free space. Returns maximally
* coalesced range containing given range. Does not attempt to free
* segments (see MVFFFreeSegs).
*/
static Res MVFFAddToFreeList(Addr *baseIO, Addr *limitIO, MVFF mvff) {
Res res;
RangeStruct range, newRange;
AVER(baseIO != NULL);
AVER(limitIO != NULL);
AVERT(MVFF, mvff);
RangeInit(&range, *baseIO, *limitIO);
res = CBSInsert(&newRange, CBSOfMVFF(mvff), &range);
if (ResIsAllocFailure(res)) {
/* CBS ran out of memory for splay nodes: add range to emergency
* free list instead. */
res = FreelistInsert(&newRange, FreelistOfMVFF(mvff), &range);
}
if (res == ResOK) {
mvff->free += RangeSize(&range);
*baseIO = RangeBase(&newRange);
*limitIO = RangeLimit(&newRange);
}
return res;
}
void RangeCopy(Range to, Range from)
{
AVERT(Range, from);
RangeInit(to, RangeBase(from), RangeLimit(from));
}
void (RangeSetBase)(Range range, Addr addr)
{
AVERT(Range, range);
AVER(addr >= RangeBase(range));
RangeSetBase(range, addr);
}
static void allocate(FBMState state, Addr base, Addr limit)
{
Res res;
Index ib, il; /* Indexed for base and limit */
Bool isFree;
RangeStruct range, oldRange;
Addr outerBase, outerLimit; /* interval containing [ib, il) */
ib = indexOfAddr(state, base);
il = indexOfAddr(state, limit);
isFree = BTIsResRange(state->allocTable, ib, il);
NAllocateTried++;
if (isFree) {
Size left, right, total; /* Sizes of block and two fragments */
outerBase =
addrOfIndex(state, lastEdge(state->allocTable, ArraySize, ib));
outerLimit =
addrOfIndex(state, nextEdge(state->allocTable, ArraySize, il - 1));
left = AddrOffset(outerBase, base);
right = AddrOffset(limit, outerLimit);
total = AddrOffset(outerBase, outerLimit);
/* TODO: check these values */
UNUSED(left);
UNUSED(right);
UNUSED(total);
} else {
outerBase = outerLimit = NULL;
}
RangeInit(&range, base, limit);
switch (state->type) {
case FBMTypeCBS:
res = CBSDelete(&oldRange, state->the.cbs, &range);
break;
case FBMTypeFreelist:
res = FreelistDelete(&oldRange, state->the.fl, &range);
break;
default:
fail();
return;
}
if (verbose) {
printf("allocate: [%p,%p) -- %s\n",
(void *)base, (void *)limit, isFree ? "succeed" : "fail");
describe(state);
}
if (!isFree) {
die_expect((mps_res_t)res, MPS_RES_FAIL,
"Succeeded in deleting allocated block");
} else { /* isFree */
die_expect((mps_res_t)res, MPS_RES_OK,
"failed to delete free block");
Insist(RangeBase(&oldRange) == outerBase);
Insist(RangeLimit(&oldRange) == outerLimit);
NAllocateSucceeded++;
BTSetRange(state->allocTable, ib, il);
}
}
/* MVFFFreeSegs -- Free segments from given range
*
* Given a free range, attempts to find entire segments within
* it, and returns them to the arena, updating total size counter.
*
* This is usually called immediately after MVFFAddToFreeList.
* It is not combined with MVFFAddToFreeList because the latter
* is also called when new segments are added under MVFFAlloc.
*/
static void MVFFFreeSegs(MVFF mvff, Addr base, Addr limit)
{
Seg seg = NULL; /* suppress "may be used uninitialized" */
Arena arena;
Bool b;
Addr segLimit; /* limit of the current segment when iterating */
Addr segBase; /* base of the current segment when iterating */
Res res;
AVERT(MVFF, mvff);
AVER(base < limit);
/* Could profitably AVER that the given range is free, */
/* but the CBS doesn't provide that facility. */
if (AddrOffset(base, limit) < mvff->minSegSize)
return; /* not large enough for entire segments */
arena = PoolArena(MVFF2Pool(mvff));
b = SegOfAddr(&seg, arena, base);
AVER(b);
segBase = SegBase(seg);
segLimit = SegLimit(seg);
while(segLimit <= limit) { /* segment ends in range */
if (segBase >= base) { /* segment starts in range */
RangeStruct range, oldRange;
RangeInit(&range, segBase, segLimit);
res = CBSDelete(&oldRange, CBSOfMVFF(mvff), &range);
if (res == ResOK) {
mvff->free -= RangeSize(&range);
} else if (ResIsAllocFailure(res)) {
/* CBS ran out of memory for splay nodes, which must mean that
* there were fragments on both sides: see
* <design/cbs/#function.cbs.delete.fail>. Handle this by
* deleting the whole of oldRange (which requires no
* allocation) and re-inserting the fragments. */
RangeStruct oldRange2;
res = CBSDelete(&oldRange2, CBSOfMVFF(mvff), &oldRange);
AVER(res == ResOK);
AVER(RangesEqual(&oldRange2, &oldRange));
mvff->free -= RangeSize(&oldRange);
AVER(RangeBase(&oldRange) != segBase);
{
Addr leftBase = RangeBase(&oldRange);
Addr leftLimit = segBase;
res = MVFFAddToFreeList(&leftBase, &leftLimit, mvff);
}
AVER(RangeLimit(&oldRange) != segLimit);
{
Addr rightBase = segLimit;
Addr rightLimit = RangeLimit(&oldRange);
res = MVFFAddToFreeList(&rightBase, &rightLimit, mvff);
}
} else if (res == ResFAIL) {
/* Not found in the CBS: must be found in the Freelist. */
res = FreelistDelete(&oldRange, FreelistOfMVFF(mvff), &range);
AVER(res == ResOK);
mvff->free -= RangeSize(&range);
}
AVER(res == ResOK);
AVER(RangesNest(&oldRange, &range));
/* Can't free the segment earlier, because if it was on the
* Freelist rather than the CBS then it likely contains data
* that needs to be read in order to update the Freelist. */
SegFree(seg);
mvff->total -= RangeSize(&range);
}
/* Avoid calling SegNext if the next segment would fail */
/* the loop test, mainly because there might not be a */
/* next segment. */
if (segLimit == limit) /* segment ends at end of range */
break;
b = SegFindAboveAddr(&seg, arena, segBase);
AVER(b);
segBase = SegBase(seg);
segLimit = SegLimit(seg);
}
return;
}
static void find(FBMState state, Size size, Bool high, FindDelete findDelete)
{
Bool expected, found;
Index expectedBase, expectedLimit;
RangeStruct foundRange, oldRange;
Addr remainderBase, remainderLimit;
Addr origBase, origLimit;
Size oldSize, newSize;
origBase = origLimit = NULL;
expected = (high ? BTFindLongResRangeHigh : BTFindLongResRange)
(&expectedBase, &expectedLimit, state->allocTable,
(Index)0, (Index)ArraySize, (Count)size);
if (expected) {
oldSize = (expectedLimit - expectedBase) * state->align;
remainderBase = origBase = addrOfIndex(state, expectedBase);
remainderLimit = origLimit = addrOfIndex(state, expectedLimit);
switch(findDelete) {
case FindDeleteNONE: {
/* do nothing */
} break;
case FindDeleteENTIRE: {
remainderBase = remainderLimit;
} break;
case FindDeleteLOW: {
expectedLimit = expectedBase + size;
remainderBase = addrOfIndex(state, expectedLimit);
} break;
case FindDeleteHIGH: {
expectedBase = expectedLimit - size;
remainderLimit = addrOfIndex(state, expectedBase);
} break;
}
if (findDelete != FindDeleteNONE) {
newSize = AddrOffset(remainderBase, remainderLimit);
}
/* TODO: check these values */
UNUSED(oldSize);
UNUSED(newSize);
}
switch (state->type) {
case FBMTypeCBS:
found = (high ? CBSFindLast : CBSFindFirst)
(&foundRange, &oldRange, state->the.cbs, size * state->align, findDelete);
break;
case FBMTypeFreelist:
found = (high ? FreelistFindLast : FreelistFindFirst)
(&foundRange, &oldRange, state->the.fl, size * state->align, findDelete);
break;
default:
fail();
return;
}
if (verbose) {
printf("find %s %lu: ", high ? "last" : "first",
(unsigned long)(size * state->align));
if (expected) {
printf("expecting [%p,%p)\n",
(void *)addrOfIndex(state, expectedBase),
(void *)addrOfIndex(state, expectedLimit));
} else {
printf("expecting this not to be found\n");
}
if (found) {
printf(" found [%p,%p)\n", (void *)RangeBase(&foundRange),
(void *)RangeLimit(&foundRange));
} else {
printf(" not found\n");
}
}
Insist(found == expected);
if (found) {
Insist(expectedBase == indexOfAddr(state, RangeBase(&foundRange)));
Insist(expectedLimit == indexOfAddr(state, RangeLimit(&foundRange)));
if (findDelete != FindDeleteNONE) {
Insist(RangeBase(&oldRange) == origBase);
Insist(RangeLimit(&oldRange) == origLimit);
BTSetRange(state->allocTable, expectedBase, expectedLimit);
}
}
return;
}
static void deallocate(FBMState state, Addr base, Addr limit)
{
Res res;
Index ib, il;
Bool isAllocated;
Addr outerBase = base, outerLimit = limit; /* interval containing [ib, il) */
RangeStruct range, freeRange; /* interval returned by the manager */
ib = indexOfAddr(state, base);
il = indexOfAddr(state, limit);
isAllocated = BTIsSetRange(state->allocTable, ib, il);
NDeallocateTried++;
if (isAllocated) {
Size left, right, total; /* Sizes of block and two fragments */
/* Find the free blocks adjacent to the allocated block */
if (ib > 0 && !BTGet(state->allocTable, ib - 1)) {
outerBase =
addrOfIndex(state, lastEdge(state->allocTable, ArraySize, ib - 1));
} else {
outerBase = base;
}
if (il < ArraySize && !BTGet(state->allocTable, il)) {
outerLimit =
addrOfIndex(state, nextEdge(state->allocTable, ArraySize, il));
} else {
outerLimit = limit;
}
left = AddrOffset(outerBase, base);
right = AddrOffset(limit, outerLimit);
total = AddrOffset(outerBase, outerLimit);
/* TODO: check these values */
UNUSED(left);
UNUSED(right);
UNUSED(total);
}
RangeInit(&range, base, limit);
switch (state->type) {
case FBMTypeCBS:
res = CBSInsert(&freeRange, state->the.cbs, &range);
break;
case FBMTypeFreelist:
res = FreelistInsert(&freeRange, state->the.fl, &range);
break;
default:
fail();
return;
}
if (verbose) {
printf("deallocate: [%p,%p) -- %s\n",
(void *)base, (void *)limit, isAllocated ? "succeed" : "fail");
describe(state);
}
if (!isAllocated) {
die_expect((mps_res_t)res, MPS_RES_FAIL,
"succeeded in inserting non-allocated block");
} else { /* isAllocated */
die_expect((mps_res_t)res, MPS_RES_OK,
"failed to insert allocated block");
NDeallocateSucceeded++;
BTResRange(state->allocTable, ib, il);
Insist(RangeBase(&freeRange) == outerBase);
Insist(RangeLimit(&freeRange) == outerLimit);
}
}
Bool RangeIsAligned(Range range, Align alignment)
{
AVERT(Range, range);
return AddrIsAligned(RangeBase(range), alignment)
&& AddrIsAligned(RangeLimit(range), alignment);
}
Addr (RangeBase)(Range range)
{
AVERT(Range, range);
return RangeBase(range);
}
}
}
Res FreelistInsert(Range rangeReturn, Freelist fl, Range range)
{
FreelistBlock prev, cur, next, new;
Addr base, limit;
Bool coalesceLeft, coalesceRight;
AVER(rangeReturn != NULL);
AVERT(Freelist, fl);
AVERT(Range, range);
AVER(RangeIsAligned(range, fl->alignment));
base = RangeBase(range);
limit = RangeLimit(range);
prev = NULL;
cur = fl->list;
while (cur) {
if (base < FreelistBlockLimit(fl, cur) && FreelistBlockBase(cur) < limit)
return ResFAIL; /* range overlaps with cur */
if (limit <= FreelistBlockBase(cur))
break;
next = FreelistBlockNext(cur);
if (next)
/* Isolated range invariant (design.mps.freelist.impl.invariant). */
AVER(FreelistBlockLimit(fl, cur) < FreelistBlockBase(next));
prev = cur;
cur = next;
