/*
* Correct the size for a BFZ file when we are done appending to it.
*
* During writing to a BFZ, the amount of bytes writen to disk differs from
* the bytes passed in to write. We can have one of two situations:
* - BFZ is compressed. The size on disk is smaller than the bytes written
* - BFZ is uncompressed, and we're using checksumming. The size on disk is
* slightly larger than the bytes written
* Make the necessary correction here.
*/
static void
ExecWorkFile_AdjustBFZSize(ExecWorkFile *workfile, int64 file_size)
{
Assert(workfile != NULL);
#if USE_ASSERT_CHECKING
bfz_t *bfz_file = (bfz_t *) workfile->file;
#endif
if (file_size <= workfile->size)
{
/*
* Actual size on disk is smaller than expected. This can happen in two cases:
* - file on disk is compressed
* - we hit out of disk space
*/
Assert(bfz_file->compression_index > 0 || WorkfileDiskspace_IsFull());
WorkfileDiskspace_Commit(file_size, workfile->size, true /* update_query_size */);
workfile_update_in_progress_size(workfile, file_size - workfile->size);
workfile->size = file_size;
}
else
{
int64 extra_bytes = file_size - workfile->size;
/* Actual file on disk is bigger than expected. This can happen when:
* - added checksums to an uncompressed file
* - closing empty or very small compressed file (zlib header overhead larger than saved space)
*/
Assert( (bfz_file->has_checksum && bfz_file->compression_index == 0) || (bfz_file->compression_index > 0 && workfile->size < BFZ_BUFFER_SIZE));
/*
* If we're already under disk full, don't try to reserve, as it will
* fail anyway. We're in cleanup code in that case, and the file
* will be deleted soon.
*/
if (!WorkfileDiskspace_IsFull())
{
bool reserved = WorkfileDiskspace_Reserve(extra_bytes);
if (!reserved)
{
elog(gp_workfile_caching_loglevel, "Could not reserve " INT64_FORMAT " additional bytes while adjusting for BFZ addtional size",
extra_bytes);
workfile_mgr_report_error();
}
WorkfileDiskspace_Commit(extra_bytes, extra_bytes, true /* update_query_size */);
workfile_update_in_progress_size(workfile, extra_bytes);
workfile->size = file_size;
}
}
}
/*
* Close a BufFile
*
* Like fclose(), this also implicitly FileCloses the underlying File.
*/
void
BufFileClose(BufFile *file)
{
if (file->isWorkfile && WorkfileDiskspace_IsFull())
{
elog(gp_workfile_caching_loglevel, "closing workfile while workfile diskspace full, skipping flush");
}
else
{
/* flush any unwritten data */
if (!file->isTemp)
{
/* This can thrown an exception */
BufFileFlush(file);
}
}
FileClose(file->file);
/* release the buffer space */
if (file->buffer)
pfree(file->buffer);
pfree(file);
}
/*
* BufFileDumpBuffer
*
* Dump buffer contents starting at curOffset.
* At call, should have dirty = true, nbytes > 0.
* On exit, dirty is cleared if successful write, and curOffset is advanced.
*/
static void BufFileDumpBuffer(BufFile *file, const void* buffer, Size nbytes)
{
size_t wpos = 0;
size_t bytestowrite;
int wrote = 0;
/*
* Unlike BufFileLoadBuffer, we must dump the whole buffer.
*/
while (wpos < nbytes)
{
bytestowrite = nbytes - wpos;
if (FileSeek(file->file, file->offset, SEEK_SET) != file->offset)
{
elog(ERROR, "could not seek in temporary file: %m");
}
wrote = FileWrite(file->file, (char *)buffer + wpos, (int)bytestowrite);
if (wrote != bytestowrite)
{
if (file->isWorkfile)
{
elog(gp_workfile_caching_loglevel, "FileWrite failed while writing to a workfile. Marking IO Error flag."
" offset=" INT64_FORMAT " pos=" INT64_FORMAT " maxoffset=" INT64_FORMAT " wpos=%d",
file->offset, file->pos, file->maxoffset, (int) wpos);
Assert(!WorkfileDiskspace_IsFull());
WorkfileDiskspace_SetFull(true /* isFull */);
}
elog(ERROR, "could not write %d bytes to temporary file: %m", (int)bytestowrite);
}
file->offset += wrote;
wpos += wrote;
}
file->dirty = false;
/*
* Now we can set the buffer empty without changing the logical position
*/
file->pos = 0;
file->nbytes = 0;
}
/*
* ExecWorkFile_Write
* write the given data from the end of the last write position.
*
* This function returns true if the write succeeds. Otherwise, return false.
*/
bool
ExecWorkFile_Write(ExecWorkFile *workfile,
void *data,
uint64 size)
{
Assert(workfile != NULL);
uint64 bytes;
if (data == NULL || size == 0)
{
return false;
}
/* Test the per-query and per-segment limit */
if ((workfile->flags & EXEC_WORKFILE_LIMIT_SIZE) &&
!WorkfileDiskspace_Reserve(size))
{
/* Failed to reserve additional disk space, notify caller */
workfile_mgr_report_error();
}
switch(workfile->fileType)
{
case BUFFILE:
{}
BufFile *buffile = (BufFile *)workfile->file;
int64 current_size = BufFileGetSize(buffile);
int64 new_size = 0;
PG_TRY();
{
bytes = BufFileWrite(buffile, data, size);
}
PG_CATCH();
{
new_size = BufFileGetSize(buffile);
workfile->size = new_size;
WorkfileDiskspace_Commit( (new_size - current_size), size, true /* update_query_size */);
int64 size_evicted = workfile_mgr_evict(MIN_EVICT_SIZE);
elog(gp_workfile_caching_loglevel, "Hit out of disk space, evicted " INT64_FORMAT " bytes", size_evicted);
PG_RE_THROW();
}
PG_END_TRY();
new_size = BufFileGetSize(buffile);
workfile->size = new_size;
WorkfileDiskspace_Commit( (new_size - current_size), size, true /* update_query_size */);
workfile_update_in_progress_size(workfile, new_size - current_size);
if (bytes != size)
{
workfile_mgr_report_error();
}
break;
case BFZ:
PG_TRY();
{
bfz_append((bfz_t *)workfile->file, data, size);
}
PG_CATCH();
{
Assert(WorkfileDiskspace_IsFull());
WorkfileDiskspace_Commit(0, size, true /* update_query_size */);
int64 size_evicted = workfile_mgr_evict(MIN_EVICT_SIZE);
elog(gp_workfile_caching_loglevel, "Hit out of disk space, evicted " INT64_FORMAT " bytes", size_evicted);
PG_RE_THROW();
}
PG_END_TRY();
/* bfz_append always adds to the file size */
workfile->size += size;
if ((workfile->flags & EXEC_WORKFILE_LIMIT_SIZE))
{
WorkfileDiskspace_Commit(size, size, true /* update_query_size */);
}
workfile_update_in_progress_size(workfile, size);
break;
default:
insist_log(false, "invalid work file type: %d", workfile->fileType);
}
return true;
}
