/* * Get information on the next Append-Only Storage Block. * * Return true if another block was found. Otherwise, when we have reached * the end of the current segment file. * * OUTPUTS: * * contentLen - total byte length of the content. * executorBlockKind - executor supplied value stored in the Append-Only * Storage Block header * firstRowNum - When the first row number for this block was explicitly set, * that value is returned here. Otherwise, INT64CONST(-1) is * returned. * rowCount - number of rows in the content * isLarge - When true, the content was longer than the maxBufferLen * (i.e. blocksize) minus Append-Only Storage Block header and * had to be stored in more than one storage block. * isCompressed - When true, the content is compressed and cannot be looked at * directly in the buffer. */ bool AppendOnlyStorageRead_GetBlockInfo(AppendOnlyStorageRead *storageRead, int32 *contentLen, int *executorBlockKind, int64 *firstRowNum, int *rowCount, bool *isLarge, bool *isCompressed) { bool isNext; Assert(storageRead != NULL); Assert(storageRead->isActive); isNext = AppendOnlyStorageRead_ReadNextBlock(storageRead); /* * The current* variables have good values even when there is no next * block. */ *contentLen = storageRead->current.uncompressedLen; *executorBlockKind = storageRead->current.executorBlockKind; *firstRowNum = storageRead->current.firstRowNum; *rowCount = storageRead->current.rowCount; *isLarge = storageRead->current.isLarge; *isCompressed = storageRead->current.isCompressed; return isNext; }
/* * Get information on the next Append-Only Storage Block. * * Return true if another block was found. Otherwise, * when we have reached the end of the current segment * file. */ bool AppendOnlyStorageRead_GetBlockInfo( AppendOnlyStorageRead *storageRead, int32 *contentLen, /* The total byte length of the content. */ int *executorBlockKind, /* * The executor supplied value stored in the * Append-Only Storage Block header. */ int64 *firstRowNum, /* * When the first row number for this block * was explicitly set, that value is * returned here. Otherwise, INT64CONST(-1) * is returned. */ int *rowCount, /* The number of rows in the content. */ bool *isLarge, /* * When true, the content was longer than the * maxBufferLen (i.e. blocksize) minus * Append-Only Storage Block header and had * to be stored in more than one storage block. */ bool *isCompressed) /* * When true, the content is compressed and * cannot be looked at directly in the buffer. */ { bool isNext; Assert(storageRead != NULL); Assert(storageRead->isActive); isNext = AppendOnlyStorageRead_ReadNextBlock(storageRead); /* * The current* variables have good values even when there is no next block. */ *contentLen = storageRead->current.uncompressedLen; *executorBlockKind = storageRead->current.executorBlockKind; *firstRowNum = storageRead->current.firstRowNum; *rowCount = storageRead->current.rowCount; *isLarge = storageRead->current.isLarge; *isCompressed = storageRead->current.isCompressed; return isNext; }
/* * Skip the current block found with ~_GetBlockInfo. * * Do not decompress the block contents. * * Call this routine instead of calling ~_GetBuffer or ~_Contents that look at * contents. Useful when the desired row(s) are not within the row range of * the current block. */ void AppendOnlyStorageRead_SkipCurrentBlock(AppendOnlyStorageRead *storageRead) { Assert(storageRead != NULL); Assert(storageRead->isActive); if (storageRead->current.isLarge) { int64 largeContentPosition; /* Position of the large * content metadata block. */ int32 largeContentLen; /* Total length of the large content. */ int32 remainingLargeContentLen; /* Remaining number of bytes * to read for the large * content. */ int32 regularBlockReadCount; /* Number of regular blocks * read after the metadata * block. */ int32 regularContentLen; /* Length of the current regular * block's content. */ int32 availableLen; /* * Large content. * * We have the LargeContent "metadata" AO block with the total length * (already read) followed by N SmallContent blocks with the fragments * of the large content. */ /* * Save any values needed from the current* members since they will be * modifed as we read the regular blocks. */ largeContentPosition = storageRead->current.headerOffsetInFile; largeContentLen = storageRead->current.uncompressedLen; /* * Loop to read regular blocks. */ remainingLargeContentLen = largeContentLen; regularBlockReadCount = 0; while (true) { /* * Read next regular block. */ regularBlockReadCount++; if (!AppendOnlyStorageRead_ReadNextBlock(storageRead)) { /* * Unexpected end of file. */ ereport(ERROR, (errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("Unexpected end of file trying to read block %d of large content in segment file '%s' of table '%s'. " "Large content metadata block is at position " INT64_FORMAT " " "Large content length %d", regularBlockReadCount, storageRead->segmentFileName, storageRead->relationName, largeContentPosition, largeContentLen))); } if (storageRead->current.headerKind != AoHeaderKind_SmallContent) { /* * Unexpected headerKind. */ ereport(ERROR, (errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("Expected header kind 'Block' for block %d of large content in segment file '%s' of table '%s'. " "Large content metadata block is at position " INT64_FORMAT " " "Large content length %d", regularBlockReadCount, storageRead->segmentFileName, storageRead->relationName, largeContentPosition, largeContentLen))); } Assert(!storageRead->current.isLarge); BufferedReadGrowBuffer(&storageRead->bufferedRead, storageRead->current.overallBlockLen, &availableLen); if (storageRead->current.overallBlockLen != availableLen) ereport(ERROR, (errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("Wrong buffer length. Expected %d byte length" " buffer and got %d ", storageRead->current.overallBlockLen, availableLen), errdetail_appendonly_read_storage_content_header(storageRead), errcontext_appendonly_read_storage_block(storageRead))); regularContentLen = storageRead->current.uncompressedLen; remainingLargeContentLen -= regularContentLen; if (remainingLargeContentLen < 0) { /* * Too much data found??? */ ereport(ERROR, (errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("Too much data found after reading %d blocks for large content in segment file '%s' of table '%s'. " "Large content metadata block is at position " INT64_FORMAT " " "Large content length %d; extra data length %d", regularBlockReadCount, storageRead->segmentFileName, storageRead->relationName, largeContentPosition, largeContentLen, -remainingLargeContentLen))); } /* * Since we are skipping, we do not use the compressed or * uncompressed content. */ if (remainingLargeContentLen == 0) break; } } else { uint8 *header; uint8 *content; /* * "Small" content in one regular block. */ /* * Fetch pointers to content. * * Since we are skipping, we do not look at the content. */ AppendOnlyStorageRead_InternalGetBuffer(storageRead, &header, &content); } }
/* * Copy the large and/or decompressed content out. * * The contentOutLen parameter value must match the contentLen from the * AppendOnlyStorageReadGetBlockInfo call. * * Note this routine will work for small non-compressed content, too. * * contentOut - memory to receive the contiguous content. * contentOutLen - byte length of the contentOut buffer. */ void AppendOnlyStorageRead_Content(AppendOnlyStorageRead *storageRead, uint8 *contentOut, int32 contentOutLen) { Assert(storageRead != NULL); Assert(storageRead->isActive); Assert(contentOutLen == storageRead->current.uncompressedLen); if (storageRead->current.isLarge) { int64 largeContentPosition; /* Position of the large * content metadata block. */ int32 largeContentLen; /* Total length of the large content. */ int32 remainingLargeContentLen; /* The remaining number of * bytes to read for the large * content. */ uint8 *contentNext;/* Pointer inside the contentOut buffer to put * the next byte. */ int32 regularBlockReadCount; /* Number of regular blocks * read after the metadata * block. */ int32 regularContentLen; /* Length of the current regular * block's content. */ /* * Large content. * * We have the LargeContent "metadata" AO block with the total length * (already read) followed by N SmallContent blocks with the fragments * of the large content. */ /* * Save any values needed from the current* members since they will be * modifed as we read the regular blocks. */ largeContentPosition = storageRead->current.headerOffsetInFile; largeContentLen = storageRead->current.uncompressedLen; /* * Loop to read regular blocks. */ contentNext = contentOut; remainingLargeContentLen = largeContentLen; regularBlockReadCount = 0; while (true) { /* * Read next regular block. */ regularBlockReadCount++; if (!AppendOnlyStorageRead_ReadNextBlock(storageRead)) { /* * Unexpected end of file. */ ereport(ERROR, (errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("Unexpected end of file trying to read block %d of large content in segment file '%s' of table '%s'. " "Large content metadata block is at position " INT64_FORMAT " " "Large content length %d", regularBlockReadCount, storageRead->segmentFileName, storageRead->relationName, largeContentPosition, largeContentLen))); } if (storageRead->current.headerKind != AoHeaderKind_SmallContent) { /* * Unexpected headerKind. */ ereport(ERROR, (errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("Expected header kind 'Block' for block %d of large content in segment file '%s' of table '%s'. " "Large content metadata block is at position " INT64_FORMAT " " "Large content length %d", regularBlockReadCount, storageRead->segmentFileName, storageRead->relationName, largeContentPosition, largeContentLen))); } Assert(!storageRead->current.isLarge); regularContentLen = storageRead->current.uncompressedLen; remainingLargeContentLen -= regularContentLen; if (remainingLargeContentLen < 0) { /* * Too much data found??? */ ereport(ERROR, (errcode(ERRCODE_GP_INTERNAL_ERROR), errmsg("Too much data found after reading %d blocks for large content in segment file '%s' of table '%s'. " "Large content metadata block is at position " INT64_FORMAT " " "Large content length %d; extra data length %d", regularBlockReadCount, storageRead->segmentFileName, storageRead->relationName, largeContentPosition, largeContentLen, -remainingLargeContentLen))); } /* * We can safely recurse one level here. */ AppendOnlyStorageRead_Content(storageRead, contentNext, regularContentLen); if (remainingLargeContentLen == 0) break; /* * Advance our pointer inside the contentOut buffer to put the * next bytes. */ contentNext += regularContentLen; } } else { uint8 *header; uint8 *content; /* * "Small" content in one regular block. */ /* * Fetch pointers to content. */ AppendOnlyStorageRead_InternalGetBuffer(storageRead, &header, &content); if (!storageRead->current.isCompressed) { /* * Not compressed. */ memcpy(contentOut, content, storageRead->current.uncompressedLen); if (Debug_appendonly_print_scan) elog(LOG, "Append-only Storage Read non-compressed block for table '%s' " "(length = %d, segment file '%s', header offset in file = " INT64_FORMAT ", block count " INT64_FORMAT ")", storageRead->relationName, storageRead->current.uncompressedLen, storageRead->segmentFileName, storageRead->current.headerOffsetInFile, storageRead->bufferCount); } else { /* * Compressed. */ PGFunction decompressor; PGFunction *cfns = storageRead->compression_functions; /* * How can it be valid that decompressor is NULL, * gp_decompress_new will always crash if decompresor is NULL */ if (cfns == NULL) decompressor = NULL; else decompressor = cfns[COMPRESSION_DECOMPRESS]; gp_decompress_new(content, /* Compressed data in block. */ storageRead->current.compressedLen, contentOut, storageRead->current.uncompressedLen, decompressor, storageRead->compressionState, storageRead->bufferCount); if (Debug_appendonly_print_scan) elog(LOG, "Append-only Storage Read decompressed block for table '%s' " "(compressed length %d, uncompressed length = %d, segment file '%s', " "header offset in file = " INT64_FORMAT ", block count " INT64_FORMAT ")", storageRead->relationName, AppendOnlyStorageFormat_GetCompressedLen(header), storageRead->current.uncompressedLen, storageRead->segmentFileName, storageRead->current.headerOffsetInFile, storageRead->bufferCount); } } }