/*
* StrategySyncStart -- tell BufferSync where to start syncing
*
* The result is the buffer index of the best buffer to sync first.
* BufferSync() will proceed circularly around the buffer array from there.
*
* In addition, we return the completed-pass count (which is effectively
* the higher-order bits of nextVictimBuffer) and the count of recent buffer
* allocs if non-NULL pointers are passed. The alloc count is reset after
* being read.
*/
int
StrategySyncStart(uint32 *complete_passes, uint32 *num_buf_alloc)
{
uint32 nextVictimBuffer;
int result;
SpinLockAcquire(&StrategyControl->buffer_strategy_lock);
nextVictimBuffer = pg_atomic_read_u32(&StrategyControl->nextVictimBuffer);
result = nextVictimBuffer % NBuffers;
if (complete_passes)
{
*complete_passes = StrategyControl->completePasses;
/*
* Additionally add the number of wraparounds that happened before
* completePasses could be incremented. C.f. ClockSweepTick().
*/
*complete_passes += nextVictimBuffer / NBuffers;
}
if (num_buf_alloc)
{
*num_buf_alloc = pg_atomic_exchange_u32(&StrategyControl->numBufferAllocs, 0);
}
SpinLockRelease(&StrategyControl->buffer_strategy_lock);
return result;
}
/*
* MarkLocalBufferDirty -
* mark a local buffer dirty
*/
void
MarkLocalBufferDirty(Buffer buffer)
{
int bufid;
BufferDesc *bufHdr;
uint32 buf_state;
Assert(BufferIsLocal(buffer));
#ifdef LBDEBUG
fprintf(stderr, "LB DIRTY %d\n", buffer);
#endif
bufid = -(buffer + 1);
Assert(LocalRefCount[bufid] > 0);
bufHdr = GetLocalBufferDescriptor(bufid);
buf_state = pg_atomic_read_u32(&bufHdr->state);
if (!(buf_state & BM_DIRTY))
pgBufferUsage.local_blks_dirtied++;
buf_state |= BM_DIRTY;
pg_atomic_write_u32(&bufHdr->state, buf_state);
}
/*
* DropRelFileNodeLocalBuffers
* This function removes from the buffer pool all the pages of the
* specified relation that have block numbers >= firstDelBlock.
* (In particular, with firstDelBlock = 0, all pages are removed.)
* Dirty pages are simply dropped, without bothering to write them
* out first. Therefore, this is NOT rollback-able, and so should be
* used only with extreme caution!
*
* See DropRelFileNodeBuffers in bufmgr.c for more notes.
*/
void
DropRelFileNodeLocalBuffers(RelFileNode rnode, ForkNumber forkNum,
BlockNumber firstDelBlock)
{
int i;
for (i = 0; i < NLocBuffer; i++)
{
BufferDesc *bufHdr = GetLocalBufferDescriptor(i);
LocalBufferLookupEnt *hresult;
uint32 buf_state;
buf_state = pg_atomic_read_u32(&bufHdr->state);
if ((buf_state & BM_TAG_VALID) &&
RelFileNodeEquals(bufHdr->tag.rnode, rnode) &&
bufHdr->tag.forkNum == forkNum &&
bufHdr->tag.blockNum >= firstDelBlock)
{
if (LocalRefCount[i] != 0)
elog(ERROR, "block %u of %s is still referenced (local %u)",
bufHdr->tag.blockNum,
relpathbackend(bufHdr->tag.rnode, MyBackendId,
bufHdr->tag.forkNum),
LocalRefCount[i]);
/* Remove entry from hashtable */
hresult = (LocalBufferLookupEnt *)
hash_search(LocalBufHash, (void *) &bufHdr->tag,
HASH_REMOVE, NULL);
if (!hresult) /* shouldn't happen */
elog(ERROR, "local buffer hash table corrupted");
/* Mark buffer invalid */
CLEAR_BUFFERTAG(bufHdr->tag);
buf_state &= ~BUF_FLAG_MASK;
buf_state &= ~BUF_USAGECOUNT_MASK;
pg_atomic_write_u32(&bufHdr->state, buf_state);
}
}
}
static void
test_atomic_uint32(void)
{
pg_atomic_uint32 var;
uint32 expected;
int i;
pg_atomic_init_u32(&var, 0);
if (pg_atomic_read_u32(&var) != 0)
elog(ERROR, "atomic_read_u32() #1 wrong");
pg_atomic_write_u32(&var, 3);
if (pg_atomic_read_u32(&var) != 3)
elog(ERROR, "atomic_read_u32() #2 wrong");
if (pg_atomic_fetch_add_u32(&var, 1) != 3)
elog(ERROR, "atomic_fetch_add_u32() #1 wrong");
if (pg_atomic_fetch_sub_u32(&var, 1) != 4)
elog(ERROR, "atomic_fetch_sub_u32() #1 wrong");
if (pg_atomic_sub_fetch_u32(&var, 3) != 0)
elog(ERROR, "atomic_sub_fetch_u32() #1 wrong");
if (pg_atomic_add_fetch_u32(&var, 10) != 10)
elog(ERROR, "atomic_add_fetch_u32() #1 wrong");
if (pg_atomic_exchange_u32(&var, 5) != 10)
elog(ERROR, "pg_atomic_exchange_u32() #1 wrong");
if (pg_atomic_exchange_u32(&var, 0) != 5)
elog(ERROR, "pg_atomic_exchange_u32() #0 wrong");
/* test around numerical limits */
if (pg_atomic_fetch_add_u32(&var, INT_MAX) != 0)
elog(ERROR, "pg_atomic_fetch_add_u32() #2 wrong");
if (pg_atomic_fetch_add_u32(&var, INT_MAX) != INT_MAX)
elog(ERROR, "pg_atomic_add_fetch_u32() #3 wrong");
pg_atomic_fetch_add_u32(&var, 1); /* top up to UINT_MAX */
if (pg_atomic_read_u32(&var) != UINT_MAX)
elog(ERROR, "atomic_read_u32() #2 wrong");
if (pg_atomic_fetch_sub_u32(&var, INT_MAX) != UINT_MAX)
elog(ERROR, "pg_atomic_fetch_sub_u32() #2 wrong");
if (pg_atomic_read_u32(&var) != (uint32) INT_MAX + 1)
elog(ERROR, "atomic_read_u32() #3 wrong: %u", pg_atomic_read_u32(&var));
expected = pg_atomic_sub_fetch_u32(&var, INT_MAX);
if (expected != 1)
elog(ERROR, "pg_atomic_sub_fetch_u32() #3 wrong: %u", expected);
pg_atomic_sub_fetch_u32(&var, 1);
/* fail exchange because of old expected */
expected = 10;
if (pg_atomic_compare_exchange_u32(&var, &expected, 1))
elog(ERROR, "atomic_compare_exchange_u32() changed value spuriously");
/* CAS is allowed to fail due to interrupts, try a couple of times */
for (i = 0; i < 1000; i++)
{
expected = 0;
if (!pg_atomic_compare_exchange_u32(&var, &expected, 1))
break;
}
if (i == 1000)
elog(ERROR, "atomic_compare_exchange_u32() never succeeded");
if (pg_atomic_read_u32(&var) != 1)
elog(ERROR, "atomic_compare_exchange_u32() didn't set value properly");
pg_atomic_write_u32(&var, 0);
/* try setting flagbits */
if (pg_atomic_fetch_or_u32(&var, 1) & 1)
elog(ERROR, "pg_atomic_fetch_or_u32() #1 wrong");
if (!(pg_atomic_fetch_or_u32(&var, 2) & 1))
elog(ERROR, "pg_atomic_fetch_or_u32() #2 wrong");
if (pg_atomic_read_u32(&var) != 3)
elog(ERROR, "invalid result after pg_atomic_fetch_or_u32()");
/* try clearing flagbits */
if ((pg_atomic_fetch_and_u32(&var, ~2) & 3) != 3)
elog(ERROR, "pg_atomic_fetch_and_u32() #1 wrong");
if (pg_atomic_fetch_and_u32(&var, ~1) != 1)
elog(ERROR, "pg_atomic_fetch_and_u32() #2 wrong: is %u",
pg_atomic_read_u32(&var));
/* no bits set anymore */
if (pg_atomic_fetch_and_u32(&var, ~0) != 0)
elog(ERROR, "pg_atomic_fetch_and_u32() #3 wrong");
}
/*
* Perform garbage collection (if required) of file
* @param map_path path to file map file (*.map).
*/
static bool cfs_gc_file(char* map_path)
{
int md = open(map_path, O_RDWR|PG_BINARY, 0);
FileMap* map;
uint32 physSize;
uint32 usedSize;
uint32 virtSize;
int suf = strlen(map_path)-4;
int fd = -1, fd2 = -1, md2 = -1;
bool succeed = true;
if (md < 0) {
elog(LOG, "Failed to open map file %s: %m", map_path);
return false;
}
map = cfs_mmap(md);
if (map == MAP_FAILED) {
elog(LOG, "Failed to map file %s: %m", map_path);
close(md);
return false;
}
usedSize = pg_atomic_read_u32(&map->usedSize);
physSize = pg_atomic_read_u32(&map->physSize);
virtSize = pg_atomic_read_u32(&map->virtSize);
if ((physSize - usedSize)*100 > physSize*cfs_gc_threshold) /* do we need to perform defragmentation? */
{
long delay = CFS_LOCK_MIN_TIMEOUT;
char* file_path = (char*)palloc(suf+1);
char* map_bck_path = (char*)palloc(suf+10);
char* file_bck_path = (char*)palloc(suf+5);
FileMap* newMap = (FileMap*)palloc0(sizeof(FileMap));
uint32 newSize = 0;
inode_t** inodes = (inode_t**)palloc(RELSEG_SIZE*sizeof(inode_t*));
bool remove_backups = true;
int n_pages = virtSize / BLCKSZ;
TimestampTz startTime, endTime;
long secs;
int usecs;
int i;
startTime = GetCurrentTimestamp();
memcpy(file_path, map_path, suf);
file_path[suf] = '\0';
strcat(strcpy(map_bck_path, map_path), ".bck");
strcat(strcpy(file_bck_path, file_path), ".bck");
while (true) {
uint32 access_count = 0;
if (pg_atomic_compare_exchange_u32(&map->lock, &access_count, CFS_GC_LOCK)) {
break;
}
if (access_count >= CFS_GC_LOCK) {
/* Uhhh... looks like last GC was interrupted.
* Try to recover file
*/
if (access(file_bck_path, R_OK) != 0) {
/* There is no backup file: new map should be constructed */
md2 = open(map_bck_path, O_RDWR|PG_BINARY, 0);
if (md2 >= 0) {
/* Recover map */
if (!cfs_read_file(md2, newMap, sizeof(FileMap))) {
elog(LOG, "Failed to read file %s: %m", map_bck_path);
goto Cleanup;
}
close(md2);
md2 = -1;
newSize = pg_atomic_read_u32(&newMap->usedSize);
remove_backups = false;
goto ReplaceMap;
}
} else {
/* Presence of backup file means that we still have unchanged data and map files.
* Just remove backup files, grab lock and continue processing
*/
unlink(file_bck_path);
unlink(map_bck_path);
break;
}
}
pg_usleep(delay);
if (delay < CFS_LOCK_MAX_TIMEOUT) {
delay *= 2;
}
}
md2 = open(map_bck_path, O_CREAT|O_RDWR|PG_BINARY|O_TRUNC, 0600);
if (md2 < 0) {
goto Cleanup;
}
for (i = 0; i < n_pages; i++) {
newMap->inodes[i] = map->inodes[i];
inodes[i] = &newMap->inodes[i];
}
/* sort inodes by offset to improve read locality */
qsort(inodes, n_pages, sizeof(inode_t*), cfs_cmp_page_offs);
fd = open(file_path, O_RDWR|PG_BINARY, 0);
if (fd < 0) {
goto Cleanup;
}
fd2 = open(file_bck_path, O_CREAT|O_RDWR|PG_BINARY|O_TRUNC, 0600);
if (fd2 < 0) {
goto Cleanup;
}
for (i = 0; i < n_pages; i++) {
int size = CFS_INODE_SIZE(*inodes[i]);
if (size != 0) {
char block[BLCKSZ];
off_t rc PG_USED_FOR_ASSERTS_ONLY;
uint32 offs = CFS_INODE_OFFS(*inodes[i]);
Assert(size <= BLCKSZ);
rc = lseek(fd, offs, SEEK_SET);
Assert(rc == offs);
if (!cfs_read_file(fd, block, size)) {
elog(LOG, "Failed to read file %s: %m", file_path);
goto Cleanup;
}
if (!cfs_write_file(fd2, block, size)) {
elog(LOG, "Failed to write file %s: %m", file_bck_path);
goto Cleanup;
}
offs = newSize;
newSize += size;
*inodes[i] = CFS_INODE(size, offs);
}
}
pg_atomic_write_u32(&map->usedSize, newSize);
if (close(fd) < 0) {
elog(LOG, "Failed to close file %s: %m", file_path);
goto Cleanup;
}
fd = -1;
/* Persist copy of data file */
if (pg_fsync(fd2) < 0) {
elog(LOG, "Failed to sync file %s: %m", file_bck_path);
goto Cleanup;
}
if (close(fd2) < 0) {
elog(LOG, "Failed to close file %s: %m", file_bck_path);
goto Cleanup;
}
fd2 = -1;
/* Persist copy of map file */
if (!cfs_write_file(md2, &newMap, sizeof(newMap))) {
elog(LOG, "Failed to write file %s: %m", map_bck_path);
goto Cleanup;
}
if (pg_fsync(md2) < 0) {
elog(LOG, "Failed to sync file %s: %m", map_bck_path);
goto Cleanup;
}
if (close(md2) < 0) {
elog(LOG, "Failed to close file %s: %m", map_bck_path);
goto Cleanup;
}
md2 = -1;
/* Persist map with CFS_GC_LOCK set: in case of crash we will know that map may be changed by GC */
if (cfs_msync(map) < 0) {
elog(LOG, "Failed to sync map %s: %m", map_path);
goto Cleanup;
}
if (pg_fsync(md) < 0) {
elog(LOG, "Failed to sync file %s: %m", map_path);
goto Cleanup;
}
/*
* Now all information necessary for recovery is stored.
* We are ready to replace existed file with defragmented one.
* Use rename and rely on file system to provide atomicity of this operation.
*/
remove_backups = false;
if (rename(file_bck_path, file_path) < 0) {
elog(LOG, "Failed to rename file %s: %m", file_path);
goto Cleanup;
}
ReplaceMap:
/* At this moment defragmented file version is stored. We can perfrom in-place update of map.
* If crash happens at this point, map can be recovered from backup file */
memcpy(map->inodes, newMap->inodes, n_pages * sizeof(inode_t));
pg_atomic_write_u32(&map->usedSize, newSize);
pg_atomic_write_u32(&map->physSize, newSize);
map->generation += 1; /* force all backends to reopen the file */
/* Before removing backup files and releasing locks we need to flush updated map file */
if (cfs_msync(map) < 0) {
elog(LOG, "Failed to sync map %s: %m", map_path);
goto Cleanup;
}
if (pg_fsync(md) < 0) {
elog(LOG, "Failed to sync file %s: %m", map_path);
Cleanup:
if (fd >= 0) close(fd);
if (fd2 >= 0) close(fd2);
if (md2 >= 0) close(md2);
if (remove_backups) {
unlink(file_bck_path);
unlink(map_bck_path);
remove_backups = false;
}
succeed = false;
} else {
remove_backups = true; /* now backups are not need any more */
}
pg_atomic_fetch_sub_u32(&map->lock, CFS_GC_LOCK); /* release lock */
/* remove map backup file */
if (remove_backups && unlink(map_bck_path)) {
elog(LOG, "Failed to unlink file %s: %m", map_bck_path);
succeed = false;
}
endTime = GetCurrentTimestamp();
TimestampDifference(startTime, endTime, &secs, &usecs);
elog(LOG, "%d: defragment file %s: old size %d, new size %d, logical size %d, used %d, compression ratio %f, time %ld usec",
MyProcPid, file_path, physSize, newSize, virtSize, usedSize, (double)virtSize/newSize,
secs*USECS_PER_SEC + usecs);
pfree(file_path);
pfree(file_bck_path);
pfree(map_bck_path);
pfree(inodes);
pfree(newMap);
if (cfs_gc_delay != 0) {
int rc = WaitLatch(MyLatch,
WL_TIMEOUT | WL_POSTMASTER_DEATH,
cfs_gc_delay /* ms */ );
if (rc & WL_POSTMASTER_DEATH) {
exit(1);
}
}
} else if (cfs_state->max_iterations == 1) {
elog(LOG, "%d: file %.*s: physical size %d, logical size %d, used %d, compression ratio %f",
MyProcPid, suf, map_path, physSize, virtSize, usedSize, (double)virtSize/physSize);
}
if (cfs_munmap(map) < 0) {
elog(LOG, "Failed to unmap file %s: %m", map_path);
succeed = false;
}
if (close(md) < 0) {
elog(LOG, "Failed to close file %s: %m", map_path);
succeed = false;
}
return succeed;
}
/*
* Update logical file size
*/
void cfs_extend(FileMap* map, uint32 newSize)
{
uint32 oldSize = pg_atomic_read_u32(&map->virtSize);
while (newSize > oldSize && !pg_atomic_compare_exchange_u32(&map->virtSize, &oldSize, newSize));
}
/*
* LocalBufferAlloc -
* Find or create a local buffer for the given page of the given relation.
*
* API is similar to bufmgr.c's BufferAlloc, except that we do not need
* to do any locking since this is all local. Also, IO_IN_PROGRESS
* does not get set. Lastly, we support only default access strategy
* (hence, usage_count is always advanced).
*/
BufferDesc *
LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum,
bool *foundPtr)
{
BufferTag newTag; /* identity of requested block */
LocalBufferLookupEnt *hresult;
BufferDesc *bufHdr;
int b;
int trycounter;
bool found;
uint32 buf_state;
INIT_BUFFERTAG(newTag, smgr->smgr_rnode.node, forkNum, blockNum);
/* Initialize local buffers if first request in this session */
if (LocalBufHash == NULL)
InitLocalBuffers();
/* See if the desired buffer already exists */
hresult = (LocalBufferLookupEnt *)
hash_search(LocalBufHash, (void *) &newTag, HASH_FIND, NULL);
if (hresult)
{
b = hresult->id;
bufHdr = GetLocalBufferDescriptor(b);
Assert(BUFFERTAGS_EQUAL(bufHdr->tag, newTag));
#ifdef LBDEBUG
fprintf(stderr, "LB ALLOC (%u,%d,%d) %d\n",
smgr->smgr_rnode.node.relNode, forkNum, blockNum, -b - 1);
#endif
buf_state = pg_atomic_read_u32(&bufHdr->state);
/* this part is equivalent to PinBuffer for a shared buffer */
if (LocalRefCount[b] == 0)
{
if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT)
{
buf_state += BUF_USAGECOUNT_ONE;
pg_atomic_write_u32(&bufHdr->state, buf_state);
}
}
LocalRefCount[b]++;
ResourceOwnerRememberBuffer(CurrentResourceOwner,
BufferDescriptorGetBuffer(bufHdr));
if (buf_state & BM_VALID)
*foundPtr = TRUE;
else
{
/* Previous read attempt must have failed; try again */
*foundPtr = FALSE;
}
return bufHdr;
}
#ifdef LBDEBUG
fprintf(stderr, "LB ALLOC (%u,%d,%d) %d\n",
smgr->smgr_rnode.node.relNode, forkNum, blockNum,
-nextFreeLocalBuf - 1);
#endif
/*
* Need to get a new buffer. We use a clock sweep algorithm (essentially
* the same as what freelist.c does now...)
*/
trycounter = NLocBuffer;
for (;;)
{
b = nextFreeLocalBuf;
if (++nextFreeLocalBuf >= NLocBuffer)
nextFreeLocalBuf = 0;
bufHdr = GetLocalBufferDescriptor(b);
if (LocalRefCount[b] == 0)
{
buf_state = pg_atomic_read_u32(&bufHdr->state);
if (BUF_STATE_GET_USAGECOUNT(buf_state) > 0)
{
buf_state -= BUF_USAGECOUNT_ONE;
pg_atomic_write_u32(&bufHdr->state, buf_state);
trycounter = NLocBuffer;
}
else
{
/* Found a usable buffer */
LocalRefCount[b]++;
ResourceOwnerRememberBuffer(CurrentResourceOwner,
BufferDescriptorGetBuffer(bufHdr));
break;
}
}
else if (--trycounter == 0)
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_RESOURCES),
errmsg("no empty local buffer available")));
}
/*
* this buffer is not referenced but it might still be dirty. if that's
* the case, write it out before reusing it!
*/
if (buf_state & BM_DIRTY)
{
SMgrRelation oreln;
Page localpage = (char *) LocalBufHdrGetBlock(bufHdr);
/* Find smgr relation for buffer */
oreln = smgropen(bufHdr->tag.rnode, MyBackendId);
PageSetChecksumInplace(localpage, bufHdr->tag.blockNum);
/* And write... */
smgrwrite(oreln,
bufHdr->tag.forkNum,
bufHdr->tag.blockNum,
localpage,
false);
/* Mark not-dirty now in case we error out below */
buf_state &= ~BM_DIRTY;
pg_atomic_write_u32(&bufHdr->state, buf_state);
pgBufferUsage.local_blks_written++;
}
/*
* lazy memory allocation: allocate space on first use of a buffer.
*/
if (LocalBufHdrGetBlock(bufHdr) == NULL)
{
/* Set pointer for use by BufferGetBlock() macro */
LocalBufHdrGetBlock(bufHdr) = GetLocalBufferStorage();
}
/*
* Update the hash table: remove old entry, if any, and make new one.
*/
if (buf_state & BM_TAG_VALID)
{
hresult = (LocalBufferLookupEnt *)
hash_search(LocalBufHash, (void *) &bufHdr->tag,
HASH_REMOVE, NULL);
if (!hresult) /* shouldn't happen */
elog(ERROR, "local buffer hash table corrupted");
/* mark buffer invalid just in case hash insert fails */
CLEAR_BUFFERTAG(bufHdr->tag);
buf_state &= ~(BM_VALID | BM_TAG_VALID);
pg_atomic_write_u32(&bufHdr->state, buf_state);
}
hresult = (LocalBufferLookupEnt *)
hash_search(LocalBufHash, (void *) &newTag, HASH_ENTER, &found);
if (found) /* shouldn't happen */
elog(ERROR, "local buffer hash table corrupted");
hresult->id = b;
/*
* it's all ours now.
*/
bufHdr->tag = newTag;
buf_state &= ~(BM_VALID | BM_DIRTY | BM_JUST_DIRTIED | BM_IO_ERROR);
buf_state |= BM_TAG_VALID;
buf_state &= ~BUF_USAGECOUNT_MASK;
buf_state += BUF_USAGECOUNT_ONE;
pg_atomic_write_u32(&bufHdr->state, buf_state);
*foundPtr = FALSE;
return bufHdr;
}
