/*
* Get information on the next Append-Only Storage Block.
*
* Return true if another block was found. Otherwise, we have reached the
* end of the current segment file.
*/
bool
AppendOnlyStorageRead_ReadNextBlock(AppendOnlyStorageRead *storageRead)
{
uint8 *header;
AOHeaderCheckError checkError;
int32 blockLimitLen = 0; /* Shutup compiler. */
pg_crc32 storedChecksum;
pg_crc32 computedChecksum;
/*
* Reset current* variables.
*/
/* For efficiency, zero out. Comment out lines that set fields to 0. */
memset(&storageRead->current, 0, sizeof(AppendOnlyStorageReadCurrent));
/* storageRead->current.headerOffsetInFile = 0; */
storageRead->current.headerKind = AoHeaderKind_None;
/* storageRead->current.actualHeaderLen = 0; */
/* storageRead->current.contentLen = 0; */
/* storageRead->current.overallBlockLen = 0; */
/* storageRead->current.contentOffset = 0; */
/* storageRead->current.executorBlockKind = 0; */
/* storageRead->current.hasFirstRowNum = false; */
storageRead->current.firstRowNum = INT64CONST(-1);
/* storageRead->current.rowCount = 0; */
/* storageRead->current.isLarge = false; */
/* storageRead->current.isCompressed = false; */
/* storageRead->current.compressedLen = 0; */
elogif(Debug_appendonly_print_datumstream, LOG,
"before AppendOnlyStorageRead_PositionToNextBlock, storageRead->current.headerOffsetInFile is" INT64_FORMAT "storageRead->current.overallBlockLen is %d",
storageRead->current.headerOffsetInFile, storageRead->current.overallBlockLen);
if (!AppendOnlyStorageRead_PositionToNextBlock(storageRead,
&storageRead->current.headerOffsetInFile,
&header,
&blockLimitLen))
{
/* Done reading the file */
return false;
}
elogif(Debug_appendonly_print_datumstream, LOG,
"after AppendOnlyStorageRead_PositionToNextBlock, storageRead->current.headerOffsetInFile is" INT64_FORMAT "storageRead->current.overallBlockLen is %d",
storageRead->current.headerOffsetInFile, storageRead->current.overallBlockLen);
/*----------
* Proceed very carefully:
* [ 1. Verify header checksum ]
* 2. Examine (basic) header.
* 3. Examine specific header.
* [ 4. Verify the block checksum ]
*----------
*/
if (storageRead->storageAttributes.checksum &&
gp_appendonly_verify_block_checksums)
{
if (!AppendOnlyStorageFormat_VerifyHeaderChecksum(header,
&storedChecksum,
&computedChecksum))
ereport(ERROR,
(errmsg("Header checksum does not match. Expected 0x%X and found 0x%X ",
storedChecksum,
computedChecksum),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
}
/*
* Check the (basic) header information.
*/
checkError = AppendOnlyStorageFormat_GetHeaderInfo(header,
storageRead->storageAttributes.checksum,
&storageRead->current.headerKind,
&storageRead->current.actualHeaderLen);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
/*
* Get more header since AppendOnlyStorageRead_PositionToNextBlock only
* gets minimum.
*/
if (storageRead->minimumHeaderLen < storageRead->current.actualHeaderLen)
{
int32 availableLen;
header = BufferedReadGrowBuffer(&storageRead->bufferedRead,
storageRead->current.actualHeaderLen,
&availableLen);
if (header == NULL ||
availableLen != storageRead->current.actualHeaderLen)
ereport(ERROR,
(errcode(ERRCODE_GP_INTERNAL_ERROR),
errmsg("Expected %d bytes and found %d bytes in table %s "
"(segment file '%s', header offset in file = " INT64_FORMAT ", bufferCount " INT64_FORMAT ")",
storageRead->current.actualHeaderLen,
availableLen,
storageRead->relationName,
storageRead->segmentFileName,
storageRead->current.headerOffsetInFile,
storageRead->bufferCount)));
}
/*
* Based on the kind of header, we either have small or large content.
*/
switch (storageRead->current.headerKind)
{
case AoHeaderKind_SmallContent:
/*
* Check the SmallContent header information.
*/
checkError = AppendOnlyStorageFormat_GetSmallContentHeaderInfo
(header,
storageRead->current.actualHeaderLen,
storageRead->storageAttributes.checksum,
blockLimitLen,
&storageRead->current.overallBlockLen,
&storageRead->current.contentOffset,
&storageRead->current.uncompressedLen,
&storageRead->current.executorBlockKind,
&storageRead->current.hasFirstRowNum,
storageRead->formatVersion,
&storageRead->current.firstRowNum,
&storageRead->current.rowCount,
&storageRead->current.isCompressed,
&storageRead->current.compressedLen
);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header of type small content. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
break;
case AoHeaderKind_LargeContent:
/*
* Check the LargeContent metadata header information.
*/
checkError = AppendOnlyStorageFormat_GetLargeContentHeaderInfo
(header,
storageRead->current.actualHeaderLen,
storageRead->storageAttributes.checksum,
&storageRead->current.uncompressedLen,
&storageRead->current.executorBlockKind,
&storageRead->current.hasFirstRowNum,
&storageRead->current.firstRowNum,
&storageRead->current.rowCount);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header of type large content. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
storageRead->current.isLarge = true;
break;
case AoHeaderKind_NonBulkDenseContent:
/*
* Check the NonBulkDense header information.
*/
checkError =
AppendOnlyStorageFormat_GetNonBulkDenseContentHeaderInfo
(header,
storageRead->current.actualHeaderLen,
storageRead->storageAttributes.checksum,
blockLimitLen,
&storageRead->current.overallBlockLen,
&storageRead->current.contentOffset,
&storageRead->current.uncompressedLen,
&storageRead->current.executorBlockKind,
&storageRead->current.hasFirstRowNum,
storageRead->formatVersion,
&storageRead->current.firstRowNum,
&storageRead->current.rowCount
);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header of type non-bulk dense content. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
break;
case AoHeaderKind_BulkDenseContent:
/*
* Check the BulkDenseContent header information.
*/
checkError =
AppendOnlyStorageFormat_GetBulkDenseContentHeaderInfo
(header,
storageRead->current.actualHeaderLen,
storageRead->storageAttributes.checksum,
blockLimitLen,
&storageRead->current.overallBlockLen,
&storageRead->current.contentOffset,
&storageRead->current.uncompressedLen,
&storageRead->current.executorBlockKind,
&storageRead->current.hasFirstRowNum,
storageRead->formatVersion,
&storageRead->current.firstRowNum,
&storageRead->current.rowCount,
&storageRead->current.isCompressed,
&storageRead->current.compressedLen
);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Bad append-only storage header of type bulk dense content. Header check error %d, detail '%s'",
(int) checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr()),
errdetail_appendonly_read_storage_content_header(storageRead),
errcontext_appendonly_read_storage_block(storageRead)));
break;
default:
elog(ERROR, "Unexpected Append-Only header kind %d",
storageRead->current.headerKind);
break;
}
if (Debug_appendonly_print_storage_headers)
{
AppendOnlyStorageRead_LogBlockHeader(storageRead, header);
}
if (storageRead->current.hasFirstRowNum)
{
/* UNDONE: Grow buffer and read the value into firstRowNum. */
}
if (storageRead->current.headerKind == AoHeaderKind_LargeContent)
{
/* UNDONE: Finish the read for the information only header. */
}
return true;
}
void
AppendOnlyStorageFormat_MakeBulkDenseContentHeader(
uint8 *headerPtr,
bool usingChecksums,
bool hasFirstRowNum,
int version,
int64 firstRowNum,
int executorKind,
int rowCount,
int32 dataLength,
int32 compressedLength)
{
AOBulkDenseContentHeader *blockHeader;
int32 firstHeaderAndChecksumsLen;
AOBulkDenseContentHeaderExt *extHeader;
bool isCompressed;
Assert(headerPtr != NULL);
blockHeader = (AOBulkDenseContentHeader*)headerPtr;
firstHeaderAndChecksumsLen =
AoHeader_RegularSize +
(usingChecksums ? 2 * sizeof(pg_crc32) : 0);
/*
* The extension header is in the data portion with first row number.
*/
extHeader = (AOBulkDenseContentHeaderExt*)(headerPtr + firstHeaderAndChecksumsLen);
elogif(Debug_appendonly_print_storage_headers, LOG,
"Append-Only storage make Bulk Dense Content header parameters: wantChecksum = %s, hasFirstRowNum %s, executorKind = %d, "
"rowCount = %d, dataLength %d, compressedLength %d",
(usingChecksums ? "true" : "false"),
(hasFirstRowNum ? "true" : "false"),
executorKind,
rowCount,
dataLength,
compressedLength);
/* Zero out whole header */
AOBulkDenseContentHeaderInit_Init(blockHeader);
AOBulkDenseContentHeaderInit_headerKind(blockHeader,AoHeaderKind_BulkDenseContent);
AOBulkDenseContentHeaderInit_executorBlockKind(blockHeader,executorKind);
AOBulkDenseContentHeaderInit_dataLength(blockHeader,dataLength);
AOBulkDenseContentHeaderInit_hasFirstRowNum(blockHeader,hasFirstRowNum);
isCompressed = (compressedLength > 0);
if (isCompressed)
AOBulkDenseContentHeaderInit_compressedLength(blockHeader,compressedLength);
/* Zero out whole extension */
AOBulkDenseContentHeaderExtInit_Init(extHeader);
AOBulkDenseContentHeaderExtInit_largeRowCount(extHeader,rowCount);
/*
* Add the optional firstRowNum.
*
* NOTE: This is not part of the 8-byte (64-bit) header because it is so big.
* NOTE: And, it is not covered by the header checksum because in order to
* NOTE: determine if we should checksum more data we would need to examine
* NOTE: the header data not verified by checksum yet...
*
* So, the firstRowNum is extra data between the header (and checksums) and
* the content. We must add it before computing the checksum.
*/
if (hasFirstRowNum)
{
AppendOnlyStorageFormat_AddFirstRowNum(
headerPtr,
usingChecksums,
firstRowNum);
}
if (usingChecksums)
{
AppendOnlyStorageFormat_AddBlockHeaderChecksums(
headerPtr,
isCompressed,
hasFirstRowNum,
version,
dataLength,
compressedLength);
}
else
{
elogif(Debug_appendonly_print_storage_headers, LOG,
"Append-Only storage make Bulk Dense Content header result: "
"bulkdensecontent_bytes_0_3 0x%X, bulkdensecontent_bytes_4_7 0x%X "
"bulkdensecontent_ext_bytes_0_3 0x%X, bulkdensecontent_ext_bytes_4_7 0x%X ",
blockHeader->bulkdensecontent_bytes_0_3,
blockHeader->bulkdensecontent_bytes_4_7,
extHeader->bulkdensecontent_ext_bytes_0_3,
extHeader->bulkdensecontent_ext_bytes_4_7);
}
#ifdef USE_ASSERT_CHECKING
{
int checkHeaderLen;
int32 checkLength;
int32 checkBlockLimitLen;
int32 checkOverallBlockLen;
int32 checkOffset;
int32 checkUncompressedLen;
int checkExecutorBlockKind;
bool checkHasFirstRowNum;
int64 checkFirstRowNum;
int checkRowCount;
bool checkIsCompressed;
int32 checkCompressedLen;
AOHeaderCheckError checkError;
checkHeaderLen = firstHeaderAndChecksumsLen +
AoHeader_RegularSize;
if (hasFirstRowNum)
checkHeaderLen += sizeof(int64);
if (compressedLength == 0)
{
checkLength = dataLength;
}
else
{
checkLength = compressedLength;
}
checkBlockLimitLen = checkHeaderLen +
AOStorage_RoundUp(checkLength, version);
checkError =
AppendOnlyStorageFormat_GetBulkDenseContentHeaderInfo(
headerPtr,
checkHeaderLen,
usingChecksums,
checkBlockLimitLen,
&checkOverallBlockLen,
&checkOffset,
&checkUncompressedLen,
&checkExecutorBlockKind,
&checkHasFirstRowNum,
version,
&checkFirstRowNum,
&checkRowCount,
&checkIsCompressed,
&checkCompressedLen);
if (checkError != AOHeaderCheckOk)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. Header check error %d, detail '%s'",
(int)checkError,
AppendOnlyStorageFormat_GetHeaderCheckErrorStr())));
if (checkOverallBlockLen != checkBlockLimitLen)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. Found block length %d, expected %d",
checkOverallBlockLen,
checkBlockLimitLen)));
if (checkOffset != checkHeaderLen)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. Found data offset %d, expected %d",
checkOffset,
checkHeaderLen)));
if (checkUncompressedLen != dataLength)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. Found uncompressed length %d, expected %d",
checkUncompressedLen,
dataLength)));
if (checkExecutorBlockKind != executorKind)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. Found executor kind %d, expected %d",
checkExecutorBlockKind,
executorKind)));
if (checkHasFirstRowNum != hasFirstRowNum)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. Found has first row number flag %s, expected %s",
(checkHasFirstRowNum ? "true" : "false"),
(hasFirstRowNum ? "true" : "false"))));
if (hasFirstRowNum)
{
if (checkFirstRowNum != firstRowNum)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. "
"Found first row number " INT64_FORMAT ", expected " INT64_FORMAT,
checkFirstRowNum,
firstRowNum)));
}
if (checkRowCount != rowCount)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. Found row count %d, expected %d",
checkRowCount,
rowCount)));
if (checkIsCompressed != isCompressed)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. Found is compressed flag %s, expected %s",
(checkIsCompressed ? "true" : "false"),
(isCompressed ? "true" : "false"))));
if (checkCompressedLen != compressedLength)
ereport(ERROR,
(errmsg("Problem making append-only storage header of type bulk dense content. Found data length %d, expected %d",
checkCompressedLen,
dataLength)));
}
#endif
}
