/*
* Retrieve a new entry from the freelist
*
* Returns NULL if no free entries remain
*/
static CacheEntry *
Cache_GetFreeElement(Cache *cache)
{
CacheHdr *cacheHdr = cache->cacheHdr;
/* Must lock to touch freeList */
SpinLockAcquire(&cacheHdr->spinlock);
if (cacheHdr->freeList == NULL)
{
SpinLockRelease(&cacheHdr->spinlock);
return NULL;
}
CacheEntry *newEntry = cacheHdr->freeList;
Assert(newEntry->state == CACHE_ENTRY_FREE);
cacheHdr->freeList = cacheHdr->freeList->nextEntry;
Cache_UpdatePerfCounter(&cacheHdr->cacheStats.noFreeEntries, -1 /* delta */ );
Cache_UpdatePerfCounter(&cacheHdr->cacheStats.noAcquiredEntries, 1 /* delta */);
SpinLockRelease(&cacheHdr->spinlock);
return newEntry;
}
/*
* Mark an entry for removal from the cache.
* The entry is not immediately deleted, as there is at least one client
* using it.
* The entry will not be found using look-up operations after this step.
* The entry will physically be removed once all using clients release it.
*
* This function is not synchronized. Multiple clients can mark an entry
* deleted.
*/
void
Cache_Remove(Cache *cache, CacheEntry *entry)
{
Assert(NULL != entry);
#ifdef USE_ASSERT_CHECKING
int32 casResult =
#endif
compare_and_swap_32(&entry->state, CACHE_ENTRY_CACHED, CACHE_ENTRY_DELETED);
Assert(casResult == 1);
Cache_UpdatePerfCounter(&cache->cacheHdr->cacheStats.noCachedEntries, -1 /* delta */);
Cache_UpdatePerfCounter(&cache->cacheHdr->cacheStats.noDeletedEntries, 1 /* delta */);
}
/*
* Decrement the pinCount of a CacheEntry.
* This function is not synchronized, the caller must ensure it is holding
* a lock covering the entry.
*/
static uint32
Cache_EntryDecRef(Cache *cache, CacheEntry *entry)
{
Assert(NULL != entry);
Assert(entry->pinCount >= 1);
CACHE_ASSERT_VALID(entry);
entry->pinCount--;
if (0 == entry->pinCount)
{
Cache_UpdatePerfCounter(&cache->cacheHdr->cacheStats.noPinnedEntries, -1 /* delta */);
}
return entry->pinCount;
}
/*
* Increment the pinCount of a CacheEntry.
* This function is not synchronized, the caller must ensure it is holding
* a lock covering the entry.
*/
static uint32
Cache_EntryAddRef(Cache *cache, CacheEntry *entry)
{
Assert(NULL != entry);
Assert(entry->pinCount < UINT32_MAX);
CACHE_ASSERT_VALID(entry);
entry->pinCount++;
if (1 == entry->pinCount)
{
Cache_UpdatePerfCounter(&cache->cacheHdr->cacheStats.noPinnedEntries, 1 /* delta */);
}
return entry->pinCount;
}
/*
* Link an entry back in the cache freelist
*
* The entry must be already marked as free by the caller.
*/
void
Cache_AddToFreelist(Cache *cache, CacheEntry *entry)
{
Assert(NULL != cache);
Assert(NULL != entry);
CACHE_ASSERT_WIPED(entry);
Assert(entry->state == CACHE_ENTRY_FREE);
CacheHdr *cacheHdr = cache->cacheHdr;
/* Must lock to touch freeList */
SpinLockAcquire(&cacheHdr->spinlock);
entry->nextEntry = cacheHdr->freeList;
cacheHdr->freeList = entry;
Cache_UpdatePerfCounter(&cacheHdr->cacheStats.noFreeEntries, 1 /* delta */);
SpinLockRelease(&cacheHdr->spinlock);
}
/*
* Run cache eviction algorithm
*
* It will try to evict enough entries to add up to evictSize. Returns the
* actual accumulated size of the entries evicted
*/
int64
Cache_Evict(Cache *cache, int64 evictRequestSize)
{
Assert(NULL != cache);
Assert(evictRequestSize > 0);
Cache_TimedOperationStart();
int64 evictedSize = 0;
uint32 unsuccessfulLoops = 0;
bool foundVictim = false;
uint32 decAmount = cache->cacheHdr->policyContext.utilityDecrement;
Cache_Stats *cacheStats = &cache->cacheHdr->cacheStats;
while (true)
{
bool wraparound = false;
int32 entryIdx = Cache_NextClockHand(cache, &wraparound);
Assert(entryIdx < cache->cacheHdr->nEntries);
Cache_UpdatePerfCounter(&cacheStats->noEntriesScanned,1 /* delta */);
if (wraparound)
{
unsuccessfulLoops++;
Cache_UpdatePerfCounter(&cacheStats->noWraparound, 1 /* delta */);
if (!foundVictim)
{
/*
* We looped around and did not manage to evict any entries.
* Double the amount we decrement eviction candidate's utility by.
* This makes the eviction algorithm look for a victim more aggressively
*/
if (decAmount <= CACHE_MAX_UTILITY / 2)
{
decAmount = 2 * decAmount;
}
else
{
decAmount = CACHE_MAX_UTILITY;
}
}
foundVictim = false;
if (unsuccessfulLoops > cache->cacheHdr->policyContext.maxClockLoops)
{
/* Can't find any cached and unused entries candidates for evictions, even after looping around
* maxClockLoops times. Give up looking for victims. */
Cache_TimedOperationRecord(&cacheStats->timeEvictions, &cacheStats->maxTimeEvict);
break;
}
}
CacheEntry *crtEntry = Cache_GetEntryByIndex(cache->cacheHdr, entryIdx);
if (crtEntry->state != CACHE_ENTRY_CACHED)
{
/* Not interested in free/acquired/deleted entries. Go back and advance clock hand */
continue;
}
CacheAnchor *anchor = (CacheAnchor *) SyncHTLookup(cache->syncHashtable, &crtEntry->hashvalue);
if (NULL == anchor)
{
/* There's no anchor for this entry, someone might have snatched it in the meantime */
continue;
}
SpinLockAcquire(&anchor->spinlock);
if (crtEntry->state != CACHE_ENTRY_CACHED)
{
/* Someone freed this entry in the meantime, before we got a chance to acquire the anchor lock */
SpinLockRelease(&anchor->spinlock);
SyncHTRelease(cache->syncHashtable, (void *) anchor);
continue;
}
/* Ok, did all the checks, this entry must be valid now */
CACHE_ASSERT_VALID(crtEntry);
if (crtEntry->pinCount > 0)
{
/* Entry is in use and can't be evicted. Go back and advance clock hand */
SpinLockRelease(&anchor->spinlock);
SyncHTRelease(cache->syncHashtable, (void *) anchor);
continue;
}
/* Decrement utility */
gp_atomic_dec_positive_32(&crtEntry->utility, decAmount);
/* Just decremented someone's utility. Reset our unsuccessful loops counter */
unsuccessfulLoops = 0;
if (crtEntry->utility > 0)
{
/* Entry has non-zero utility, we shouldn't evict it. Go back and advance clock hand */
SpinLockRelease(&anchor->spinlock);
SyncHTRelease(cache->syncHashtable, (void *) anchor);
continue;
}
/* Found our victim */
Assert(0 == crtEntry->pinCount);
CACHE_ASSERT_VALID(crtEntry);
Assert(crtEntry->utility == 0);
#if USE_ASSERT_CHECKING
int32 casResult =
#endif
compare_and_swap_32(&crtEntry->state, CACHE_ENTRY_CACHED, CACHE_ENTRY_DELETED);
Assert(1 == casResult);
SpinLockRelease(&anchor->spinlock);
foundVictim = true;
evictedSize += crtEntry->size;
/* Don't update noFreeEntries yet. It will be done in Cache_AddToFreelist */
Cache_UpdatePerfCounter(&cacheStats->noCachedEntries, -1 /* delta */);
/* Unlink entry from the anchor chain */
SpinLockAcquire(&anchor->spinlock);
Cache_UnlinkEntry(cache, anchor, crtEntry);
SpinLockRelease(&anchor->spinlock);
SyncHTRelease(cache->syncHashtable, (void *) anchor);
if (NULL != cache->cleanupEntry)
{
/* Call client-side cleanup for entry */
cache->cleanupEntry(CACHE_ENTRY_PAYLOAD(crtEntry));
}
Cache_LockEntry(cache, crtEntry);
Assert(crtEntry->state == CACHE_ENTRY_DELETED);
crtEntry->state = CACHE_ENTRY_FREE;
#if USE_ASSERT_CHECKING
Cache_MemsetPayload(cache, crtEntry);
#endif
Cache_UnlockEntry(cache, crtEntry);
Cache_AddToFreelist(cache, crtEntry);
Cache_UpdatePerfCounter(&cacheStats->noEvicts, 1 /* delta */);
Cache_TimedOperationRecord(&cacheStats->timeEvictions, &cacheStats->maxTimeEvict);
if (evictedSize >= evictRequestSize)
{
/* We evicted as much as requested */
break;
}
Cache_TimedOperationStart();
}
return evictedSize;
}
/*
* Internal version of the CacheRelease function
*
* Unregisters the entry from the cleanup list if requested.
*/
static void
Cache_ReleaseCached(Cache *cache, CacheEntry *entry, bool unregisterCleanup)
{
Assert(NULL != cache);
Assert(NULL != entry);
Assert(CACHE_ENTRY_CACHED == entry->state || CACHE_ENTRY_DELETED == entry->state);
Cache_ComputeEntryHashcode(cache, entry);
volatile CacheAnchor *anchor = SyncHTLookup(cache->syncHashtable, &entry->hashvalue);
Assert(anchor != NULL);
/* Acquire anchor lock to touch the entry */
SpinLockAcquire(&anchor->spinlock);
Cache_LockEntry(cache, entry);
uint32 pinCount = Cache_EntryDecRef(cache, entry);
bool deleteEntry = false;
if (pinCount == 0 && entry->state == CACHE_ENTRY_DELETED)
{
/* Delete the cache entry if pin-count = 0 and it is marked for deletion */
Cache_UnlinkEntry(cache, (CacheAnchor *) anchor, entry);
deleteEntry = true;
Cache_UpdatePerfCounter(&cache->cacheHdr->cacheStats.noDeletedEntries, -1 /* delta */);
}
Cache_UnlockEntry(cache, entry);
SpinLockRelease(&anchor->spinlock);
/*
* Releasing anchor to hashtable.
* Ignoring 'removed' return value, both values are valid
*/
SyncHTRelease(cache->syncHashtable, (void *) anchor);
/* If requested, unregister entry from the cleanup list */
if (unregisterCleanup)
{
Cache_UnregisterCleanup(cache, entry);
}
if (deleteEntry)
{
if (NULL != cache->cleanupEntry)
{
PG_TRY();
{
/* Call client-specific cleanup function before removing entry from cache */
cache->cleanupEntry(CACHE_ENTRY_PAYLOAD(entry));
}
PG_CATCH();
{
/* Grab entry lock to ensure exclusive access to it while we're touching it */
Cache_LockEntry(cache, entry);
Assert(CACHE_ENTRY_DELETED == entry->state);
entry->state = CACHE_ENTRY_FREE;
#ifdef USE_ASSERT_CHECKING
Cache_MemsetPayload(cache, entry);
#endif
Cache_UnlockEntry(cache, entry);
/* Link entry back in the freelist */
Cache_AddToFreelist(cache, entry);
PG_RE_THROW();
}
PG_END_TRY();
}
/* Grab entry lock to ensure exclusive access to it while we're touching it */
Cache_LockEntry(cache, entry);
entry->state = CACHE_ENTRY_FREE;
#ifdef USE_ASSERT_CHECKING
Cache_MemsetPayload(cache, entry);
#endif
Cache_UnlockEntry(cache, entry);
/* Link entry back in the freelist */
Cache_AddToFreelist(cache, entry);
}
}
/*
* Look up an exact match for a cache entry
*
* Returns the matching cache entry if found, NULL otherwise
*/
CacheEntry *
Cache_Lookup(Cache *cache, CacheEntry *entry)
{
Assert(NULL != cache);
Assert(NULL != entry);
Cache_TimedOperationStart();
Cache_UpdatePerfCounter(&cache->cacheHdr->cacheStats.noLookups, 1 /* delta */);
/* Advance the clock for the replacement policy */
Cache_AdvanceClock(cache);
Cache_ComputeEntryHashcode(cache, entry);
volatile CacheAnchor *anchor = SyncHTLookup(cache->syncHashtable, &entry->hashvalue);
if (NULL == anchor)
{
/* No matching anchor found, there can't be a matching element in the cache */
Cache_TimedOperationRecord(&cache->cacheHdr->cacheStats.timeLookups,
&cache->cacheHdr->cacheStats.maxTimeLookup);
return NULL;
}
/* Acquire anchor lock to touch the chain */
SpinLockAcquire(&anchor->spinlock);
CacheEntry *crtEntry = anchor->firstEntry;
while (true)
{
while (NULL != crtEntry && crtEntry->state == CACHE_ENTRY_DELETED)
{
/* Skip over deleted entries */
crtEntry = crtEntry->nextEntry;
}
if (NULL == crtEntry)
{
/* No valid entries found in the chain */
SpinLockRelease(&anchor->spinlock);
Cache_TimedOperationRecord(&cache->cacheHdr->cacheStats.timeLookups,
&cache->cacheHdr->cacheStats.maxTimeLookup);
return NULL;
}
/* Found a valid entry. AddRef it and test to see if it matches */
Cache_EntryAddRef(cache, crtEntry);
SpinLockRelease(&anchor->spinlock);
/* Register it for cleanup in case we get an error while testing for equality */
Cache_RegisterCleanup(cache, crtEntry, true /* isCachedEntry */);
Cache_UpdatePerfCounter(&cache->cacheHdr->cacheStats.noCompares, 1 /* delta */);
if(cache->equivalentEntries(CACHE_ENTRY_PAYLOAD(entry),
CACHE_ENTRY_PAYLOAD(crtEntry)))
{
/* Found the match, we're done */
Cache_TouchEntry(cache, crtEntry);
Cache_UpdatePerfCounter(&cache->cacheHdr->cacheStats.noCacheHits, 1 /* delta */);
break;
}
/* Unregister it from cleanup since it wasn't the one */
Cache_UnregisterCleanup(cache, crtEntry);
SpinLockAcquire(&anchor->spinlock);
Cache_EntryDecRef(cache, crtEntry);
crtEntry = crtEntry->nextEntry;
}
/* ignoring return value, both values are valid */
SyncHTRelease(cache->syncHashtable, (void *) anchor);
Cache_TimedOperationRecord(&cache->cacheHdr->cacheStats.timeLookups,
&cache->cacheHdr->cacheStats.maxTimeLookup);
return crtEntry;
}
/*
* Inserts a previously acquired entry in the cache.
*
* This function should never fail.
*/
void
Cache_Insert(Cache *cache, CacheEntry *entry)
{
Assert(NULL != cache);
Assert(NULL != entry);
Cache_Stats *cacheStats = &cache->cacheHdr->cacheStats;
Cache_TimedOperationStart();
Cache_UpdatePerfCounter(&cacheStats->noInserts, 1 /* delta */);
Cache_UpdatePerfCounter(&cacheStats->noCachedEntries, 1 /* delta */);
Cache_UpdatePerfCounter(&cacheStats->noAcquiredEntries, -1 /* delta */);
Cache_UpdatePerfCounter64(&cacheStats->totalEntrySize, entry->size);
Cache_ComputeEntryHashcode(cache, entry);
/* Look up or insert anchor element for this entry */
bool existing = false;
volatile CacheAnchor *anchor = SyncHTInsert(cache->syncHashtable, &entry->hashvalue, &existing);
/*
* This should never happen since the SyncHT has as many entries as the SharedCache,
* and we'll run out of SharedCache entries before we fill up the SyncHT
*/
insist_log(NULL != anchor, "Could not insert in the cache: SyncHT full");
/* Acquire anchor lock to touch the chain */
SpinLockAcquire(&anchor->spinlock);
if (NULL == anchor->firstEntry)
{
Assert(NULL == anchor->lastEntry);
anchor->firstEntry = anchor->lastEntry = entry;
}
else
{
Assert(NULL != anchor->lastEntry);
anchor->lastEntry->nextEntry = entry;
anchor->lastEntry = entry;
}
entry->nextEntry = NULL;
Cache_EntryAddRef(cache, entry);
#ifdef USE_ASSERT_CHECKING
int32 casResult =
#endif
compare_and_swap_32(&entry->state, CACHE_ENTRY_ACQUIRED, CACHE_ENTRY_CACHED);
Assert(1 == casResult);
Assert(NULL != anchor->firstEntry && NULL != anchor->lastEntry);
SpinLockRelease(&anchor->spinlock);
#ifdef USE_ASSERT_CHECKING
bool deleted =
#endif
SyncHTRelease(cache->syncHashtable, (void *) anchor);
Assert(!deleted);
Cache_TimedOperationRecord(&cacheStats->timeInserts,
&cacheStats->maxTimeInsert);
}
/*
* Return a previously acquired entry to the cache freelist.
* Calls the client-specific cleanup before returning to the freelist.
*
* Unregisters the entry from the cleanup list if requested.
*/
static void
Cache_ReleaseAcquired(Cache *cache, CacheEntry *entry, bool unregisterCleanup)
{
Assert(NULL != cache);
Assert(NULL != entry);
Assert(CACHE_ENTRY_ACQUIRED == entry->state);
/* If a user-specified cleanupEntry function is defined, call it now */
if (NULL != cache->cleanupEntry)
{
PG_TRY();
{
/* Call client-specific cleanup function before removing entry from cache */
cache->cleanupEntry(CACHE_ENTRY_PAYLOAD(entry));
}
PG_CATCH();
{
/* Unregister entry from the cleanup list if requested */
if (unregisterCleanup)
{
Cache_UnregisterCleanup(cache, entry);
}
/* Grab entry lock to ensure exclusive access to it while we're touching it */
Cache_LockEntry(cache, entry);
Assert(CACHE_ENTRY_ACQUIRED == entry->state);
/* No need for atomic operations as long as we hold the entry lock */
entry->state = CACHE_ENTRY_FREE;
#ifdef USE_ASSERT_CHECKING
Cache_MemsetPayload(cache, entry);
#endif
Cache_UnlockEntry(cache, entry);
/* Link entry back in the freelist */
Cache_AddToFreelist(cache, entry);
PG_RE_THROW();
}
PG_END_TRY();
}
/* Unregister entry from the cleanup list if requested */
if (unregisterCleanup)
{
Cache_UnregisterCleanup(cache, entry);
}
/* Grab entry lock to ensure exclusive access to it while we're touching it */
Cache_LockEntry(cache, entry);
Assert(CACHE_ENTRY_ACQUIRED == entry->state);
/* No need for atomic operations as long as we hold the entry lock */
entry->state = CACHE_ENTRY_FREE;
#ifdef USE_ASSERT_CHECKING
Cache_MemsetPayload(cache, entry);
#endif
Cache_UnlockEntry(cache, entry);
Cache_AddToFreelist(cache, entry);
Cache_UpdatePerfCounter(&cache->cacheHdr->cacheStats.noAcquiredEntries, -1 /* delta */ );
}
