LargeObject FreeList::take(Owner owner, size_t alignment, size_t size, size_t unalignedSize)
{
BASSERT(isPowerOfTwo(alignment));
size_t alignmentMask = alignment - 1;
LargeObject candidate;
size_t candidateIndex;
size_t begin = m_vector.size() > freeListSearchDepth ? m_vector.size() - freeListSearchDepth : 0;
for (size_t i = begin; i < m_vector.size(); ++i) {
LargeObject largeObject(LargeObject::DoNotValidate, m_vector[i].begin());
if (!largeObject.isValidAndFree(owner, m_vector[i].size())) {
m_vector.pop(i--);
continue;
}
if (largeObject.size() < size)
continue;
if (test(largeObject.begin(), alignmentMask) && largeObject.size() < unalignedSize)
continue;
if (!!candidate && candidate.begin() < largeObject.begin())
continue;
candidate = largeObject;
candidateIndex = i;
}
if (!!candidate)
m_vector.pop(candidateIndex);
return candidate;
}
LargeObject FreeList::take(Owner owner, size_t size)
{
LargeObject candidate;
size_t candidateIndex;
size_t begin = m_vector.size() > freeListSearchDepth ? m_vector.size() - freeListSearchDepth : 0;
for (size_t i = begin; i < m_vector.size(); ++i) {
LargeObject largeObject(LargeObject::DoNotValidate, m_vector[i].begin());
if (!largeObject.isValidAndFree(owner, m_vector[i].size())) {
m_vector.pop(i--);
continue;
}
if (largeObject.size() < size)
continue;
if (!!candidate && candidate.begin() < largeObject.begin())
continue;
candidate = largeObject;
candidateIndex = i;
}
if (!!candidate)
m_vector.pop(candidateIndex);
return candidate;
}
void FreeList::removeInvalidAndDuplicateEntries(Owner owner)
{
for (size_t i = 0; i < m_vector.size(); ++i) {
LargeObject largeObject(LargeObject::DoNotValidate, m_vector[i].begin());
if (!largeObject.isValidAndFree(owner, m_vector[i].size())) {
m_vector.pop(i--);
continue;
}
largeObject.setMarked(false);
}
for (size_t i = 0; i < m_vector.size(); ++i) {
LargeObject largeObject(LargeObject::DoNotValidate, m_vector[i].begin());
if (largeObject.isMarked()) {
m_vector.pop(i--);
continue;
}
largeObject.setMarked(true);
}
}
void* Allocator::reallocate(void* object, size_t newSize)
{
if (!m_isBmallocEnabled)
return realloc(object, newSize);
size_t oldSize = 0;
switch (objectType(object)) {
case ObjectType::Small: {
size_t sizeClass = Object(object).page()->sizeClass();
oldSize = objectSize(sizeClass);
break;
}
case ObjectType::Large: {
std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
LargeObject largeObject(object);
oldSize = largeObject.size();
if (newSize < oldSize && newSize > smallMax) {
if (oldSize - newSize >= largeMin) {
newSize = roundUpToMultipleOf<largeAlignment>(newSize);
PerProcess<Heap>::getFastCase()->shrinkLarge(lock, largeObject, newSize);
return object;
}
}
break;
}
case ObjectType::XLarge: {
BASSERT(objectType(nullptr) == ObjectType::XLarge);
if (!object)
break;
std::unique_lock<StaticMutex> lock(PerProcess<Heap>::mutex());
oldSize = PerProcess<Heap>::getFastCase()->xLargeSize(lock, object);
if (newSize < oldSize && newSize > largeMax) {
PerProcess<Heap>::getFastCase()->shrinkXLarge(lock, Range(object, oldSize), newSize);
return object;
}
break;
}
}
void* result = allocate(newSize);
size_t copySize = std::min(oldSize, newSize);
memcpy(result, object, copySize);
m_deallocator.deallocate(object);
return result;
}
LargeObject FreeList::takeGreedy(Owner owner)
{
for (size_t i = 0; i < m_vector.size(); ++i) {
LargeObject largeObject(LargeObject::DoNotValidate, m_vector[i].begin());
if (!largeObject.isValidAndFree(owner, m_vector[i].size())) {
m_vector.pop(i--);
continue;
}
m_vector.pop(i--);
return largeObject;
}
return LargeObject();
}
void* Allocator::reallocate(void* object, size_t newSize)
{
if (!m_isBmallocEnabled)
return realloc(object, newSize);
void* result = allocate(newSize);
if (!object)
return result;
size_t oldSize = 0;
switch (objectType(object)) {
case Small: {
SmallPage* page = SmallPage::get(SmallLine::get(object));
oldSize = objectSize(page->sizeClass());
break;
}
case Medium: {
MediumPage* page = MediumPage::get(MediumLine::get(object));
oldSize = objectSize(page->sizeClass());
break;
}
case Large: {
std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
LargeObject largeObject(object);
oldSize = largeObject.size();
break;
}
case XLarge: {
std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
Range range = PerProcess<Heap>::getFastCase()->findXLarge(lock, object);
RELEASE_BASSERT(range);
oldSize = range.size();
break;
}
}
size_t copySize = std::min(oldSize, newSize);
memcpy(result, object, copySize);
m_deallocator.deallocate(object);
return result;
}
void Heap::deallocateLarge(std::lock_guard<StaticMutex>& lock, void* object)
{
LargeObject largeObject(object);
deallocateLarge(lock, largeObject);
}
