/* * Correct the size for a BFZ file when we are done appending to it. * * During writing to a BFZ, the amount of bytes writen to disk differs from * the bytes passed in to write. We can have one of two situations: * - BFZ is compressed. The size on disk is smaller than the bytes written * - BFZ is uncompressed, and we're using checksumming. The size on disk is * slightly larger than the bytes written * Make the necessary correction here. */ static void ExecWorkFile_AdjustBFZSize(ExecWorkFile *workfile, int64 file_size) { Assert(workfile != NULL); #if USE_ASSERT_CHECKING bfz_t *bfz_file = (bfz_t *) workfile->file; #endif if (file_size <= workfile->size) { /* * Actual size on disk is smaller than expected. This can happen in two cases: * - file on disk is compressed * - we hit out of disk space */ Assert(bfz_file->compression_index > 0 || WorkfileDiskspace_IsFull()); WorkfileDiskspace_Commit(file_size, workfile->size, true /* update_query_size */); workfile_update_in_progress_size(workfile, file_size - workfile->size); workfile->size = file_size; } else { int64 extra_bytes = file_size - workfile->size; /* Actual file on disk is bigger than expected. This can happen when: * - added checksums to an uncompressed file * - closing empty or very small compressed file (zlib header overhead larger than saved space) */ Assert( (bfz_file->has_checksum && bfz_file->compression_index == 0) || (bfz_file->compression_index > 0 && workfile->size < BFZ_BUFFER_SIZE)); /* * If we're already under disk full, don't try to reserve, as it will * fail anyway. We're in cleanup code in that case, and the file * will be deleted soon. */ if (!WorkfileDiskspace_IsFull()) { bool reserved = WorkfileDiskspace_Reserve(extra_bytes); if (!reserved) { elog(gp_workfile_caching_loglevel, "Could not reserve " INT64_FORMAT " additional bytes while adjusting for BFZ addtional size", extra_bytes); workfile_mgr_report_error(); } WorkfileDiskspace_Commit(extra_bytes, extra_bytes, true /* update_query_size */); workfile_update_in_progress_size(workfile, extra_bytes); workfile->size = file_size; } } }
/* * Close a BufFile * * Like fclose(), this also implicitly FileCloses the underlying File. */ void BufFileClose(BufFile *file) { if (file->isWorkfile && WorkfileDiskspace_IsFull()) { elog(gp_workfile_caching_loglevel, "closing workfile while workfile diskspace full, skipping flush"); } else { /* flush any unwritten data */ if (!file->isTemp) { /* This can thrown an exception */ BufFileFlush(file); } } FileClose(file->file); /* release the buffer space */ if (file->buffer) pfree(file->buffer); pfree(file); }
/* * BufFileDumpBuffer * * Dump buffer contents starting at curOffset. * At call, should have dirty = true, nbytes > 0. * On exit, dirty is cleared if successful write, and curOffset is advanced. */ static void BufFileDumpBuffer(BufFile *file, const void* buffer, Size nbytes) { size_t wpos = 0; size_t bytestowrite; int wrote = 0; /* * Unlike BufFileLoadBuffer, we must dump the whole buffer. */ while (wpos < nbytes) { bytestowrite = nbytes - wpos; if (FileSeek(file->file, file->offset, SEEK_SET) != file->offset) { elog(ERROR, "could not seek in temporary file: %m"); } wrote = FileWrite(file->file, (char *)buffer + wpos, (int)bytestowrite); if (wrote != bytestowrite) { if (file->isWorkfile) { elog(gp_workfile_caching_loglevel, "FileWrite failed while writing to a workfile. Marking IO Error flag." " offset=" INT64_FORMAT " pos=" INT64_FORMAT " maxoffset=" INT64_FORMAT " wpos=%d", file->offset, file->pos, file->maxoffset, (int) wpos); Assert(!WorkfileDiskspace_IsFull()); WorkfileDiskspace_SetFull(true /* isFull */); } elog(ERROR, "could not write %d bytes to temporary file: %m", (int)bytestowrite); } file->offset += wrote; wpos += wrote; } file->dirty = false; /* * Now we can set the buffer empty without changing the logical position */ file->pos = 0; file->nbytes = 0; }
/* * ExecWorkFile_Write * write the given data from the end of the last write position. * * This function returns true if the write succeeds. Otherwise, return false. */ bool ExecWorkFile_Write(ExecWorkFile *workfile, void *data, uint64 size) { Assert(workfile != NULL); uint64 bytes; if (data == NULL || size == 0) { return false; } /* Test the per-query and per-segment limit */ if ((workfile->flags & EXEC_WORKFILE_LIMIT_SIZE) && !WorkfileDiskspace_Reserve(size)) { /* Failed to reserve additional disk space, notify caller */ workfile_mgr_report_error(); } switch(workfile->fileType) { case BUFFILE: {} BufFile *buffile = (BufFile *)workfile->file; int64 current_size = BufFileGetSize(buffile); int64 new_size = 0; PG_TRY(); { bytes = BufFileWrite(buffile, data, size); } PG_CATCH(); { new_size = BufFileGetSize(buffile); workfile->size = new_size; WorkfileDiskspace_Commit( (new_size - current_size), size, true /* update_query_size */); int64 size_evicted = workfile_mgr_evict(MIN_EVICT_SIZE); elog(gp_workfile_caching_loglevel, "Hit out of disk space, evicted " INT64_FORMAT " bytes", size_evicted); PG_RE_THROW(); } PG_END_TRY(); new_size = BufFileGetSize(buffile); workfile->size = new_size; WorkfileDiskspace_Commit( (new_size - current_size), size, true /* update_query_size */); workfile_update_in_progress_size(workfile, new_size - current_size); if (bytes != size) { workfile_mgr_report_error(); } break; case BFZ: PG_TRY(); { bfz_append((bfz_t *)workfile->file, data, size); } PG_CATCH(); { Assert(WorkfileDiskspace_IsFull()); WorkfileDiskspace_Commit(0, size, true /* update_query_size */); int64 size_evicted = workfile_mgr_evict(MIN_EVICT_SIZE); elog(gp_workfile_caching_loglevel, "Hit out of disk space, evicted " INT64_FORMAT " bytes", size_evicted); PG_RE_THROW(); } PG_END_TRY(); /* bfz_append always adds to the file size */ workfile->size += size; if ((workfile->flags & EXEC_WORKFILE_LIMIT_SIZE)) { WorkfileDiskspace_Commit(size, size, true /* update_query_size */); } workfile_update_in_progress_size(workfile, size); break; default: insist_log(false, "invalid work file type: %d", workfile->fileType); } return true; }