void Heap::Free(void *ptr)
{
if(!ptr) return;
LLOG("Free " << ptr);
if((((dword)(uintptr_t)ptr) & 8) == 0) {
Page *page = (Page *)((uintptr_t)ptr & ~(uintptr_t)4095);
int k = page->klass;
LLOG("Small free page: " << (void *)page << ", k: " << k << ", ksz: " << Ksz(k));
ASSERT((4096 - ((uintptr_t)ptr & (uintptr_t)4095)) % Ksz(k) == 0);
#ifdef _MULTITHREADED
if(page->heap != this) { // freeing page allocated in different thread
page->heap->RemoteFree(ptr); // add to original heap's list of free pages to be properly freed later
return;
}
#endif
DbgFreeFillK(ptr, k);
if(cachen[k]) {
cachen[k]--;
FreeLink *l = (FreeLink *)ptr;
l->next = cache[k];
cache[k] = l;
return;
}
FreeK(ptr, page, k);
}
else
LFree(ptr);
}
void Heap::SmallFreeDirect(void *ptr)
{ // does not need to check for target heap or small vs large
LLOG("Free Direct " << ptr);
Page *page = GetPage(ptr);
ASSERT(page->heap == this);
int k = page->klass;
LLOG("Small free page: " << (void *)page << ", k: " << k << ", ksz: " << Ksz(k));
ASSERT((4096 - ((uintptr_t)ptr & (uintptr_t)4095)) % Ksz(k) == 0);
DbgFreeFillK(ptr, k);
FreeK(ptr, page, k);
}
void Heap::FreeDirect(void *ptr)
{
LLOG("Free Direct " << ptr);
if((((dword)(uintptr_t)ptr) & 8) == 0) {
Page *page = (Page *)((uintptr_t)ptr & ~(uintptr_t)4095);
int k = page->klass;
LLOG("Small free page: " << (void *)page << ", k: " << k << ", ksz: " << Ksz(k));
ASSERT((4096 - ((uintptr_t)ptr & (uintptr_t)4095)) % Ksz(k) == 0);
DbgFreeFillK(ptr, k);
FreeK(ptr, page, k);
}
else
LFree(ptr);
}
void Heap::Init()
{
if(initialized)
return;
LLOG("Init heap " << (void *)this);
for(int i = 0; i < NKLASS; i++) {
empty[i] = NULL;
full[i]->LinkSelf();
work[i]->LinkSelf();
work[i]->freelist = NULL;
work[i]->klass = i;
cachen[i] = 3500 / Ksz(i);
}
ASSERT(sizeof(Header) == 16);
ASSERT(sizeof(DLink) <= 16);
ASSERT(sizeof(BigHdr) + sizeof(Header) < BIGHDRSZ);
GlobalLInit();
for(int i = 0; i < LBINS; i++)
freebin[i]->LinkSelf();
large->LinkSelf();
lcount = 0;
if(this != &aux && !aux.work[0]->next) {
Mutex::Lock __(mutex);
aux.Init();
}
initialized = true;
PROFILEMT(mutex);
}
Heap::Page *Heap::WorkPage(int k) // get a new empty workpage
{
LLOG("AllocK - next work not available " << k << " empty: " << (void *)empty[k]);
Page *page = empty[k]; // hot empty page of the same klass
empty[k] = NULL;
if(!page) { // try to reacquire pages freed remotely
LLOG("AllocK - trying FreeRemote");
FreeRemote();
if(work[k]->freelist) { // partially free page found
LLOG("AllocK - work available after FreeRemote " << k);
return work[k];
}
page = empty[k]; // hot empty page
empty[k] = NULL;
}
if(!page)
for(int i = 0; i < NKLASS; i++) // Try hot local page of different klass
if(empty[i]) {
LLOG("AllocK - free page available for reformatting " << k);
page = empty[i];
empty[i] = NULL;
page->Format(k);
break;
}
if(!page) { // Attempt to find page in global storage of free pages
Mutex::Lock __(mutex);
aux.FreeRemoteRaw();
if(aux.work[k]->next != aux.work[k]) { // Try page of the same klass first
page = aux.work[k]->next;
page->Unlink();
page->heap = this;
LLOG("AllocK - adopting aux page " << k << " page: " << (void *)page << ", free " << (void *)page->freelist);
}
if(!page && aux.empty[k]) { // Try hot empty page of the same klass
page = aux.empty[k];
aux.empty[k] = page->next;
LLOG("AllocK - empty aux page available of the same format " << k << " page: " << (void *)page << ", free " << (void *)page->freelist);
}
if(!page)
for(int i = 0; i < NKLASS; i++) // Finally try to to find hot page of different klass
if(aux.empty[i]) {
page = aux.empty[i];
aux.empty[i] = page->next;
page->Format(k);
LLOG("AllocK - empty aux page available for reformatting " << k << " page: " << (void *)page << ", free " << (void *)page->freelist);
break;
}
if(!page) { // Not free memory was found, ask system for the new page
page = (Page *)AllocRaw4KB(Ksz(k));
LLOG("AllocK - allocated new system page " << (void *)page << " " << k);
page->Format(k);
}
page->heap = this;
}
page->Link(work[k]);
ASSERT(page->klass == k);
return page;
}
bool Heap::TryRealloc(void *ptr, size_t newsize)
{
if(!ptr) return 0;
LLOG("GetBlockSize " << ptr);
if(IsSmall(ptr)) {
Page *page = GetPage(ptr);
int k = page->klass;
return newsize <= (size_t)Ksz(k);
}
return LTryRealloc(ptr, newsize);
}
size_t Heap::GetBlockSize(void *ptr)
{
if(!ptr) return 0;
LLOG("GetBlockSize " << ptr);
if(IsSmall(ptr)) {
Page *page = GetPage(ptr);
int k = page->klass;
return Ksz(k);
}
return LGetBlockSize(ptr);
}
bool Heap::TryRealloc(void *ptr, size_t newsize)
{
if(!ptr) return 0;
LLOG("GetBlockSize " << ptr);
if((((dword)(uintptr_t)ptr) & 8) == 0) {
Page *page = (Page *)((uintptr_t)ptr & ~(uintptr_t)4095);
int k = page->klass;
return newsize <= (size_t)Ksz(k);
}
return LTryRealloc(ptr, newsize);
}
size_t Heap::GetBlockSize(void *ptr)
{
if(!ptr) return 0;
LLOG("GetBlockSize " << ptr);
if((((dword)(uintptr_t)ptr) & 8) == 0) {
Page *page = (Page *)((uintptr_t)ptr & ~(uintptr_t)4095);
int k = page->klass;
return Ksz(k);
}
return LGetBlockSize(ptr);
}
force_inline
void Heap::Free(void *ptr, Page *page, int k)
{
LLOG("Small free page: " << (void *)page << ", k: " << k << ", ksz: " << Ksz(k));
ASSERT((4096 - ((uintptr_t)ptr & (uintptr_t)4095)) % Ksz(k) == 0);
#ifdef _MULTITHREADED
if(page->heap != this) { // freeing page allocated in different thread
RemoteFree(ptr, Ksz(k)); // add to originating heap's list of free pages to be properly freed later
return;
}
#endif
DbgFreeFillK(ptr, k);
if(cachen[k]) {
cachen[k]--;
FreeLink *l = (FreeLink *)ptr;
l->next = cache[k];
cache[k] = l;
return;
}
FreeK(ptr, page, k);
}
void *MSmall::CheckFree(void *p, int k)
{
Page *page = (Page *)((uintptr_t)p & ~(uintptr_t)4095);
ASSERT((byte *)page + sizeof(Page) <= (byte *)p && (byte *)p < (byte *)page + 4096);
ASSERT((4096 - ((uintptr_t)p & (uintptr_t)4095)) % Ksz(k) == 0);
ASSERT(page->klass == k);
#ifdef CPU_64
FreeCheck((dword *)p + 2, (Ksz(k) >> 2) - 2);
#else
FreeCheck((dword *)p + 1, (Ksz(k) >> 2) - 1);
#endif
return p;
}
inline
void Heap::Free48(void *ptr)
{
Page *page = (Page *)((uintptr_t)ptr & ~(uintptr_t)4095);
LLOG("Small free page: " << (void *)page << ", k: " << k << ", ksz: " << Ksz(k));
ASSERT((4096 - ((uintptr_t)ptr & (uintptr_t)4095)) % Ksz(2) == 0);
#ifdef _MULTITHREADED
if(page->heap != this) {
page->heap->RemoteFree(ptr);
return;
}
#endif
DbgFreeFillK(ptr, 2);
if(cachen[2]) {
cachen[2]--;
FreeLink *l = (FreeLink *)ptr;
l->next = cache[2];
cache[2] = l;
return;
}
FreeK(ptr, page, 2);
}
inline void MSmall::Page::Format(int k)
{
// RTIMING("Format");
LOG("Formatting " << (void *)this << " to " << k);
klass = k;
sz = Ksz(k);
count = sKount[k];
ASSERT(count == (4096 - HEADERSZ) / sz);
active = 0;
free = Begin() + sFreem[k];
ASSERT(free == Begin() + count * sz - sz);
freelist = NULL;
#ifdef HEAPDBG
FreeFill((dword *)Begin(), (4096 - HEADERSZ) / 4);
#endif
}
void MSmall::Init()
{
for(int i = 0; i < 16; i++) {
int sz = Ksz(i);
sKount[i] = (4096 - HEADERSZ) / sz;
sFreem[i] = sKount[i] * sz - sz;
}
for(int i = 0; i < 16; i++) {
empty[i]->LinkSelf();
full[i]->LinkSelf();
work[i]->LinkSelf();
work[i]->freelist = NULL;
work[i]->free = (byte *)work[i];
work[i]->klass = i;
}
emptypages = 0;
}
NAMESPACE_UPP
#ifdef UPP_HEAP
#include "HeapImp.h"
#define LLOG(x) // LOG((void *)this << ' ' << x)
inline void Heap::Page::Format(int k)
{
DbgFreeFill(Begin(), End() - Begin());
klass = k;
active = 0;
int sz = Ksz(k);
byte *ptr = End() - sz;
byte *b = Begin();
FreeLink *l = NULL;
while(ptr >= b) {
((FreeLink *)ptr)->next = l;
l = (FreeLink *)ptr;
ptr -= sz;
}
freelist = l;
}