/*
* CStoreEndWrite finishes a cstore data load operation. If we have an unflushed
* stripe, we flush it. Then, we sync and close the cstore data file. Last, we
* flush the footer to a temporary file, and atomically rename this temporary
* file to the original footer file.
*/
void
CStoreEndWrite(TableWriteState *writeState)
{
StringInfo tableFooterFilename = NULL;
StringInfo tempTableFooterFileName = NULL;
int renameResult = 0;
int columnCount = writeState->tupleDescriptor->natts;
StripeBuffers *stripeBuffers = writeState->stripeBuffers;
if (stripeBuffers != NULL)
{
MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext);
StripeMetadata stripeMetadata = FlushStripe(writeState);
MemoryContextReset(writeState->stripeWriteContext);
MemoryContextSwitchTo(oldContext);
AppendStripeMetadata(writeState->tableFooter, stripeMetadata);
}
SyncAndCloseFile(writeState->tableFile);
tableFooterFilename = writeState->tableFooterFilename;
tempTableFooterFileName = makeStringInfo();
appendStringInfo(tempTableFooterFileName, "%s%s", tableFooterFilename->data,
CSTORE_TEMP_FILE_SUFFIX);
CStoreWriteFooter(tempTableFooterFileName, writeState->tableFooter);
renameResult = rename(tempTableFooterFileName->data, tableFooterFilename->data);
if (renameResult != 0)
{
ereport(ERROR, (errcode_for_file_access(),
errmsg("could not rename file \"%s\" to \"%s\": %m",
tempTableFooterFileName->data,
tableFooterFilename->data)));
}
pfree(tempTableFooterFileName->data);
pfree(tempTableFooterFileName);
MemoryContextDelete(writeState->stripeWriteContext);
list_free_deep(writeState->tableFooter->stripeMetadataList);
pfree(writeState->tableFooter);
pfree(writeState->tableFooterFilename->data);
pfree(writeState->tableFooterFilename);
pfree(writeState->comparisonFunctionArray);
FreeColumnBlockDataArray(writeState->blockDataArray, columnCount);
pfree(writeState);
}
/*
* CStoreWriteRow adds a row to the cstore file. If the stripe is not initialized,
* we create structures to hold stripe data and skip list. Then, we serialize and
* append data to serialized value buffer for each of the columns and update
* corresponding skip nodes. Then, whole block data is compressed at every
* rowBlockCount insertion. Then, if row count exceeds stripeMaxRowCount, we flush
* the stripe, and add its metadata to the table footer.
*/
void
CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNulls)
{
uint32 columnIndex = 0;
uint32 blockIndex = 0;
uint32 blockRowIndex = 0;
StripeBuffers *stripeBuffers = writeState->stripeBuffers;
StripeSkipList *stripeSkipList = writeState->stripeSkipList;
uint32 columnCount = writeState->tupleDescriptor->natts;
TableFooter *tableFooter = writeState->tableFooter;
const uint32 blockRowCount = tableFooter->blockRowCount;
ColumnBlockData **blockDataArray = writeState->blockDataArray;
MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext);
if (stripeBuffers == NULL)
{
stripeBuffers = CreateEmptyStripeBuffers(writeState->stripeMaxRowCount,
blockRowCount, columnCount);
stripeSkipList = CreateEmptyStripeSkipList(writeState->stripeMaxRowCount,
blockRowCount, columnCount);
writeState->stripeBuffers = stripeBuffers;
writeState->stripeSkipList = stripeSkipList;
writeState->compressionBuffer = makeStringInfo();
/*
* serializedValueBuffer lives in stripe write memory context so it needs to be
* initialized when the stripe is created.
*/
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
ColumnBlockData *blockData = blockDataArray[columnIndex];
blockData->valueBuffer = makeStringInfo();
}
}
blockIndex = stripeBuffers->rowCount / blockRowCount;
blockRowIndex = stripeBuffers->rowCount % blockRowCount;
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
ColumnBlockData *blockData = blockDataArray[columnIndex];
ColumnBlockSkipNode **blockSkipNodeArray = stripeSkipList->blockSkipNodeArray;
ColumnBlockSkipNode *blockSkipNode =
&blockSkipNodeArray[columnIndex][blockIndex];
if (columnNulls[columnIndex])
{
blockData->existsArray[blockRowIndex] = false;
}
else
{
FmgrInfo *comparisonFunction =
writeState->comparisonFunctionArray[columnIndex];
Form_pg_attribute attributeForm =
writeState->tupleDescriptor->attrs[columnIndex];
bool columnTypeByValue = attributeForm->attbyval;
int columnTypeLength = attributeForm->attlen;
Oid columnCollation = attributeForm->attcollation;
char columnTypeAlign = attributeForm->attalign;
blockData->existsArray[blockRowIndex] = true;
SerializeSingleDatum(blockData->valueBuffer, columnValues[columnIndex],
columnTypeByValue, columnTypeLength, columnTypeAlign);
UpdateBlockSkipNodeMinMax(blockSkipNode, columnValues[columnIndex],
columnTypeByValue, columnTypeLength,
columnCollation, comparisonFunction);
}
blockSkipNode->rowCount++;
}
stripeSkipList->blockCount = blockIndex + 1;
/* last row of the block is inserted serialize the block */
if (blockRowIndex == blockRowCount - 1)
{
SerializeBlockData(writeState, blockIndex, blockRowCount);
}
stripeBuffers->rowCount++;
if (stripeBuffers->rowCount >= writeState->stripeMaxRowCount)
{
StripeMetadata stripeMetadata = FlushStripe(writeState);
MemoryContextReset(writeState->stripeWriteContext);
/* set stripe data and skip list to NULL so they are recreated next time */
writeState->stripeBuffers = NULL;
writeState->stripeSkipList = NULL;
/*
* Append stripeMetadata in old context so next MemoryContextReset
* doesn't free it.
*/
MemoryContextSwitchTo(oldContext);
AppendStripeMetadata(tableFooter, stripeMetadata);
}
else
{
MemoryContextSwitchTo(oldContext);
}
}
/*
* CStoreWriteRow adds a row to the cstore file. If the stripe is not initialized,
* we create structures to hold stripe data and skip list. Then, we add data for
* each of the columns and update corresponding skip nodes. Then, if row count
* exceeds stripeMaxRowCount, we flush the stripe, and add its metadata to the
* table footer.
*/
void
CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNulls)
{
uint32 columnIndex = 0;
uint32 blockIndex = 0;
uint32 blockRowIndex = 0;
StripeData *stripeData = writeState->stripeData;
StripeSkipList *stripeSkipList = writeState->stripeSkipList;
uint32 columnCount = writeState->tupleDescriptor->natts;
TableFooter *tableFooter = writeState->tableFooter;
const uint32 blockRowCount = tableFooter->blockRowCount;
MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext);
if (stripeData == NULL)
{
stripeData = CreateEmptyStripeData(writeState->stripeMaxRowCount,
blockRowCount, columnCount);
stripeSkipList = CreateEmptyStripeSkipList(writeState->stripeMaxRowCount,
blockRowCount, columnCount);
writeState->stripeData = stripeData;
writeState->stripeSkipList = stripeSkipList;
}
blockIndex = stripeData->rowCount / blockRowCount;
blockRowIndex = stripeData->rowCount % blockRowCount;
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
ColumnData *columnData = stripeData->columnDataArray[columnIndex];
ColumnBlockData *blockData = columnData->blockDataArray[blockIndex];
ColumnBlockSkipNode **blockSkipNodeArray = stripeSkipList->blockSkipNodeArray;
ColumnBlockSkipNode *blockSkipNode =
&blockSkipNodeArray[columnIndex][blockIndex];
if (columnNulls[columnIndex])
{
blockData->existsArray[blockRowIndex] = false;
}
else
{
FmgrInfo *comparisonFunction =
writeState->comparisonFunctionArray[columnIndex];
Form_pg_attribute attributeForm =
writeState->tupleDescriptor->attrs[columnIndex];
bool columnTypeByValue = attributeForm->attbyval;
int columnTypeLength = attributeForm->attlen;
Oid columnCollation = attributeForm->attcollation;
blockData->existsArray[blockRowIndex] = true;
blockData->valueArray[blockRowIndex] = DatumCopy(columnValues[columnIndex],
columnTypeByValue,
columnTypeLength);
UpdateBlockSkipNodeMinMax(blockSkipNode, columnValues[columnIndex],
columnTypeByValue, columnTypeLength,
columnCollation, comparisonFunction);
}
blockSkipNode->rowCount++;
}
stripeSkipList->blockCount = blockIndex + 1;
stripeData->rowCount++;
if (stripeData->rowCount >= writeState->stripeMaxRowCount)
{
StripeMetadata stripeMetadata = FlushStripe(writeState);
MemoryContextReset(writeState->stripeWriteContext);
/* set stripe data and skip list to NULL so they are recreated next time */
writeState->stripeData = NULL;
writeState->stripeSkipList = NULL;
/*
* Append stripeMetadata in old context so next MemoryContextReset
* doesn't free it.
*/
MemoryContextSwitchTo(oldContext);
AppendStripeMetadata(tableFooter, stripeMetadata);
}
else
{
MemoryContextSwitchTo(oldContext);
}
}