/*
* Internal routine to grow the BufferedRead buffer to be the whole current block and
* to get header and content pointers of current block.
*
* Since we are growing the BufferedRead buffer to the whole block, old pointers to
* the header must be abandoned.
*
* Header to current block was read and verified by AppendOnlyStorageRead_ReadNextBlock.
*/
static void AppendOnlyStorageRead_InternalGetBuffer(
AppendOnlyStorageRead *storageRead,
uint8 **header,
uint8 **content)
{
int32 availableLen;
pg_crc32 storedChecksum;
pg_crc32 computedChecksum;
/*
* Verify next block is type Block.
*/
Assert(storageRead->current.headerKind == AoHeaderKind_SmallContent ||
storageRead->current.headerKind == AoHeaderKind_NonBulkDenseContent ||
storageRead->current.headerKind == AoHeaderKind_BulkDenseContent);
/*
* Grow the buffer to the full block length to avoid any
* unnecessary copying by BufferedRead.
*
* Since the BufferedRead module may have to copy information around,
* we do not save any pointers to the prior buffer call. This why
* AppendOnlyStorageFormat_GetHeaderInfo passes back the offset to the data,
* not a pointer.
*/
*header = BufferedReadGrowBuffer(&storageRead->bufferedRead,
storageRead->current.overallBlockLen,
&availableLen);
if (storageRead->current.overallBlockLen != availableLen)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("Wrong buffer length. Expected %d byte length buffer and got %d ",
storageRead->current.overallBlockLen,
availableLen),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
if (storageRead->storageAttributes.checksum &&
gp_appendonly_verify_block_checksums)
{
/*
* Now that the header has been verified, verify the block checksum
* in the header with the checksum of the data portion.
*/
if (!AppendOnlyStorageFormat_VerifyBlockChecksum(
*header,
storageRead->current.overallBlockLen,
&storedChecksum,
&computedChecksum))
ereport(ERROR,
(errmsg("Block checksum does not match. Expected 0x%X and found 0x%X",
storedChecksum,
computedChecksum),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
}
*content = &((*header)[storageRead->current.contentOffset]);
}
static int
datumstreamread_context_callback(void *arg)
{
DatumStreamRead *acc = (DatumStreamRead *) arg;
if (Debug_appendonly_print_datumstream)
elog(LOG,
"datumstream_advance filePathName %s nth %u ndatum %u datump %p ",
acc->ao_read.bufferedRead.filePathName,
acc->blockRead.nth,
acc->blockRead.logical_row_count,
acc->blockRead.datump);
/*
* Append-Only Storage Read's context.
*/
if (acc->need_close_file)
{
errcontext_appendonly_read_storage_block(&acc->ao_read);
}
else
{
errcontext("%s", acc->title);
}
return 0;
}
/*
* Get information on the next Append-Only Storage Block.
*
* Return true if another block was found. Otherwise, we have reached the
* end of the current segment file.
*/
bool
AppendOnlyStorageRead_ReadNextBlock(AppendOnlyStorageRead *storageRead)
{
uint8 *header;
AOHeaderCheckError checkError;
int32 blockLimitLen = 0; /* Shutup compiler. */
pg_crc32 storedChecksum;
pg_crc32 computedChecksum;
/*
* Reset current* variables.
*/
/* For efficiency, zero out. Comment out lines that set fields to 0. */
memset(&storageRead->current, 0, sizeof(AppendOnlyStorageReadCurrent));
/* storageRead->current.headerOffsetInFile = 0; */
storageRead->current.headerKind = AoHeaderKind_None;
/* storageRead->current.actualHeaderLen = 0; */
/* storageRead->current.contentLen = 0; */
/* storageRead->current.overallBlockLen = 0; */
/* storageRead->current.contentOffset = 0; */
/* storageRead->current.executorBlockKind = 0; */
/* storageRead->current.hasFirstRowNum = false; */
storageRead->current.firstRowNum = INT64CONST(-1);
/* storageRead->current.rowCount = 0; */
/* storageRead->current.isLarge = false; */
/* storageRead->current.isCompressed = false; */
/* storageRead->current.compressedLen = 0; */
elogif(Debug_appendonly_print_datumstream, LOG,
"before AppendOnlyStorageRead_PositionToNextBlock, storageRead->current.headerOffsetInFile is" INT64_FORMAT "storageRead->current.overallBlockLen is %d",
storageRead->current.headerOffsetInFile, storageRead->current.overallBlockLen);
if (!AppendOnlyStorageRead_PositionToNextBlock(storageRead,
&storageRead->current.headerOffsetInFile,
&header,
&blockLimitLen))
{
/* Done reading the file */
return false;
}
elogif(Debug_appendonly_print_datumstream, LOG,
"after AppendOnlyStorageRead_PositionToNextBlock, storageRead->current.headerOffsetInFile is" INT64_FORMAT "storageRead->current.overallBlockLen is %d",
storageRead->current.headerOffsetInFile, storageRead->current.overallBlockLen);
/*----------
* Proceed very carefully:
* [ 1. Verify header checksum ]
* 2. Examine (basic) header.
* 3. Examine specific header.
* [ 4. Verify the block checksum ]
*----------
*/
if (storageRead->storageAttributes.checksum &&
gp_appendonly_verify_block_checksums)
{
if (!AppendOnlyStorageFormat_VerifyHeaderChecksum(header,
&storedChecksum,
&computedChecksum))
ereport(ERROR,
(errmsg("Header checksum does not match. Expected 0x%X and found 0x%X ",
storedChecksum,
computedChecksum),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
}
/*
* Check the (basic) header information.
*/
checkError = AppendOnlyStorageFormat_GetHeaderInfo(header,
storageRead->storageAttributes.checksum,
&storageRead->current.headerKind,
&storageRead->current.actualHeaderLen);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
/*
* Get more header since AppendOnlyStorageRead_PositionToNextBlock only
* gets minimum.
*/
if (storageRead->minimumHeaderLen < storageRead->current.actualHeaderLen)
{
int32 availableLen;
header = BufferedReadGrowBuffer(&storageRead->bufferedRead,
storageRead->current.actualHeaderLen,
&availableLen);
if (header == NULL ||
availableLen != storageRead->current.actualHeaderLen)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("Expected %d bytes and found %d bytes in table %s "
"(segment file '%s', header offset in file = " INT64_FORMAT ", bufferCount " INT64_FORMAT ")",
storageRead->current.actualHeaderLen,
availableLen,
storageRead->relationName,
storageRead->segmentFileName,
storageRead->current.headerOffsetInFile,
storageRead->bufferCount)));
}
/*
* Based on the kind of header, we either have small or large content.
*/
switch (storageRead->current.headerKind)
{
case AoHeaderKind_SmallContent:
/*
* Check the SmallContent header information.
*/
checkError = AppendOnlyStorageFormat_GetSmallContentHeaderInfo
(header,
storageRead->current.actualHeaderLen,
storageRead->storageAttributes.checksum,
blockLimitLen,
&storageRead->current.overallBlockLen,
&storageRead->current.contentOffset,
&storageRead->current.uncompressedLen,
&storageRead->current.executorBlockKind,
&storageRead->current.hasFirstRowNum,
storageRead->formatVersion,
&storageRead->current.firstRowNum,
&storageRead->current.rowCount,
&storageRead->current.isCompressed,
&storageRead->current.compressedLen
);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header of type small content. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
break;
case AoHeaderKind_LargeContent:
/*
* Check the LargeContent metadata header information.
*/
checkError = AppendOnlyStorageFormat_GetLargeContentHeaderInfo
(header,
storageRead->current.actualHeaderLen,
storageRead->storageAttributes.checksum,
&storageRead->current.uncompressedLen,
&storageRead->current.executorBlockKind,
&storageRead->current.hasFirstRowNum,
&storageRead->current.firstRowNum,
&storageRead->current.rowCount);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header of type large content. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
storageRead->current.isLarge = true;
break;
case AoHeaderKind_NonBulkDenseContent:
/*
* Check the NonBulkDense header information.
*/
checkError =
AppendOnlyStorageFormat_GetNonBulkDenseContentHeaderInfo
(header,
storageRead->current.actualHeaderLen,
storageRead->storageAttributes.checksum,
blockLimitLen,
&storageRead->current.overallBlockLen,
&storageRead->current.contentOffset,
&storageRead->current.uncompressedLen,
&storageRead->current.executorBlockKind,
&storageRead->current.hasFirstRowNum,
storageRead->formatVersion,
&storageRead->current.firstRowNum,
&storageRead->current.rowCount
);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header of type non-bulk dense content. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
break;
case AoHeaderKind_BulkDenseContent:
/*
* Check the BulkDenseContent header information.
*/
checkError =
AppendOnlyStorageFormat_GetBulkDenseContentHeaderInfo
(header,
storageRead->current.actualHeaderLen,
storageRead->storageAttributes.checksum,
blockLimitLen,
&storageRead->current.overallBlockLen,
&storageRead->current.contentOffset,
&storageRead->current.uncompressedLen,
&storageRead->current.executorBlockKind,
&storageRead->current.hasFirstRowNum,
storageRead->formatVersion,
&storageRead->current.firstRowNum,
&storageRead->current.rowCount,
&storageRead->current.isCompressed,
&storageRead->current.compressedLen
);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header of type bulk dense content. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
break;
default:
elog(ERROR, "Unexpected Append-Only header kind %d",
storageRead->current.headerKind);
break;
}
if (Debug_appendonly_print_storage_headers)
{
AppendOnlyStorageRead_LogBlockHeader(storageRead, header);
}
if (storageRead->current.hasFirstRowNum)
{
/* UNDONE: Grow buffer and read the value into firstRowNum. */
}
if (storageRead->current.headerKind == AoHeaderKind_LargeContent)
{
/* UNDONE: Finish the read for the information only header. */
}
return true;
}
/*
* Skip the current block found with ~_GetBlockInfo.
*
* Do not decompress the block contents.
*
* Call this routine instead of calling ~_GetBuffer or ~_Contents that look at
* contents. Useful when the desired row(s) are not within the row range of
* the current block.
*/
void
AppendOnlyStorageRead_SkipCurrentBlock(AppendOnlyStorageRead *storageRead)
{
Assert(storageRead != NULL);
Assert(storageRead->isActive);
if (storageRead->current.isLarge)
{
int64 largeContentPosition; /* Position of the large
* content metadata block. */
int32 largeContentLen; /* Total length of the large content. */
int32 remainingLargeContentLen; /* Remaining number of bytes
* to read for the large
* content. */
int32 regularBlockReadCount; /* Number of regular blocks
* read after the metadata
* block. */
int32 regularContentLen; /* Length of the current regular
* block's content. */
int32 availableLen;
/*
* Large content.
*
* We have the LargeContent "metadata" AO block with the total length
* (already read) followed by N SmallContent blocks with the fragments
* of the large content.
*/
/*
* Save any values needed from the current* members since they will be
* modifed as we read the regular blocks.
*/
largeContentPosition = storageRead->current.headerOffsetInFile;
largeContentLen = storageRead->current.uncompressedLen;
/*
* Loop to read regular blocks.
*/
remainingLargeContentLen = largeContentLen;
regularBlockReadCount = 0;
while (true)
{
/*
* Read next regular block.
*/
regularBlockReadCount++;
if (!AppendOnlyStorageRead_ReadNextBlock(storageRead))
{
/*
* Unexpected end of file.
*/
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("Unexpected end of file trying to read block %d of large content in segment file '%s' of table '%s'. "
"Large content metadata block is at position " INT64_FORMAT " "
"Large content length %d",
regularBlockReadCount,
storageRead->segmentFileName,
storageRead->relationName,
largeContentPosition,
largeContentLen)));
}
if (storageRead->current.headerKind != AoHeaderKind_SmallContent)
{
/*
* Unexpected headerKind.
*/
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("Expected header kind 'Block' for block %d of large content in segment file '%s' of table '%s'. "
"Large content metadata block is at position " INT64_FORMAT " "
"Large content length %d",
regularBlockReadCount,
storageRead->segmentFileName,
storageRead->relationName,
largeContentPosition,
largeContentLen)));
}
Assert(!storageRead->current.isLarge);
BufferedReadGrowBuffer(&storageRead->bufferedRead,
storageRead->current.overallBlockLen,
&availableLen);
if (storageRead->current.overallBlockLen != availableLen)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("Wrong buffer length. Expected %d byte length"
" buffer and got %d ",
storageRead->current.overallBlockLen,
availableLen),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
regularContentLen = storageRead->current.uncompressedLen;
remainingLargeContentLen -= regularContentLen;
if (remainingLargeContentLen < 0)
{
/*
* Too much data found???
*/
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("Too much data found after reading %d blocks for large content in segment file '%s' of table '%s'. "
"Large content metadata block is at position " INT64_FORMAT " "
"Large content length %d; extra data length %d",
regularBlockReadCount,
storageRead->segmentFileName,
storageRead->relationName,
largeContentPosition,
largeContentLen,
-remainingLargeContentLen)));
}
/*
* Since we are skipping, we do not use the compressed or
* uncompressed content.
*/
if (remainingLargeContentLen == 0)
break;
}
}
else
{
uint8 *header;
uint8 *content;
/*
* "Small" content in one regular block.
*/
/*
* Fetch pointers to content.
*
* Since we are skipping, we do not look at the content.
*/
AppendOnlyStorageRead_InternalGetBuffer(storageRead,
&header,
&content);
}
}
