StreamBlock*
TupleStreamWrapper::getCommittedExportBytes()
{
StreamBlock* first_unpolled_block = NULL;
deque<StreamBlock*>::iterator pending_iter = m_pendingBlocks.begin();
while (pending_iter != m_pendingBlocks.end())
{
StreamBlock* block = *pending_iter;
// find the first block that has unpolled data
if (m_firstUnpolledUso < (block->uso() + block->offset()))
{
// check that the entire remainder is committed
if (m_committedUso >= (block->uso() + block->offset()))
{
first_unpolled_block = block;
// find the value to update m_firstUnpolledUso
m_firstUnpolledUso = block->uso() + block->offset();
}
else
{
// if the unpolled block is not committed,
// -- construct a fake StreamBlock that makes no progress
// --- unreleased USO of this block but offset of 0
// don't advance the first unpolled USO
delete m_fakeBlock;
m_fakeBlock = new StreamBlock(0, 0, block->unreleasedUso());
first_unpolled_block = m_fakeBlock;
}
break;
}
++pending_iter;
}
// The first unpolled block wasn't found in the pending.
// It had better be m_currBlock or we've got troubles
// Since we're here, m_currBlock is not fully committed, so
// we just want to create a fake block based on its metadata
if (first_unpolled_block == NULL)
{
delete m_fakeBlock;
m_fakeBlock = new StreamBlock(0, 0, m_currBlock->unreleasedUso());
first_unpolled_block = m_fakeBlock;
}
// return the appropriate pointah
return first_unpolled_block;
}
void TupleStreamBase::pushPendingBlocks() {
while (!m_pendingBlocks.empty())
{
StreamBlock* block = m_pendingBlocks.front();
//std::cout << "m_committedUso(" << m_committedUso << "), block->uso() + block->offset() == "
//<< (block->uso() + block->offset()) << std::endl;
// check that the entire remainder is committed
if (m_committedUso >= (block->uso() + block->offset()))
{
//The block is handed off to the topend which is responsible for releasing the
//memory associated with the block data. The metadata is deleted here.
pushExportBuffer(
block,
false,
false);
delete block;
m_pendingBlocks.pop_front();
}
else
{
break;
}
}
}
bool
TupleStreamWrapper::releaseExportBytes(int64_t releaseOffset)
{
// if released offset is in an already-released past, just return success
if ((m_pendingBlocks.empty() && releaseOffset < m_currBlock->uso()) ||
(!m_pendingBlocks.empty() && releaseOffset < m_pendingBlocks.front()->uso()))
{
return true;
}
// if released offset is in the uncommitted bytes, then set up
// to release everything that is committed
if (releaseOffset > m_committedUso)
{
releaseOffset = m_committedUso;
}
bool retval = false;
if (releaseOffset >= m_currBlock->uso())
{
while (m_pendingBlocks.empty() != true) {
StreamBlock* sb = m_pendingBlocks.back();
m_pendingBlocks.pop_back();
discardBlock(sb);
}
m_currBlock->releaseUso(releaseOffset);
retval = true;
}
else
{
StreamBlock* sb = m_pendingBlocks.front();
while (!m_pendingBlocks.empty() && !retval)
{
if (releaseOffset >= sb->uso() + sb->offset())
{
m_pendingBlocks.pop_front();
discardBlock(sb);
sb = m_pendingBlocks.front();
}
else
{
sb->releaseUso(releaseOffset);
retval = true;
}
}
}
if (retval)
{
if (m_firstUnpolledUso < releaseOffset)
{
m_firstUnpolledUso = releaseOffset;
}
}
return retval;
}
/**
* The goal of this test is simply to run through the mechanics.
* Fill a buffer repeatedly and make sure nothing breaks.
*/
TEST_F(StreamedTableTest, BaseCase) {
int64_t tokenOffset = 2000; // just so tokens != txnIds
// repeat for more tuples than fit in the default buffer
for (int i = 1; i < 1000; i++) {
// pretend to be a plan fragment execution
m_quantum->release();
m_quantum =
new (m_pool->allocate(sizeof(UndoQuantum)))
UndoQuantum(i + tokenOffset, m_pool);
// quant, currTxnId, committedTxnId
m_context->setupForPlanFragments(m_quantum, i, i - 1);
// fill a tuple
for (int col = 0; col < COLUMN_COUNT; col++) {
int value = rand();
m_tuple->setNValue(col, ValueFactory::getIntegerValue(value));
}
m_table->insertTuple(*m_tuple);
}
// a negative flush implies "now". this helps valgrind heap block test
m_table->flushOldTuples(-1);
// poll from the table and make sure we get "stuff", releasing as
// we go. This just makes sure we don't fail catastrophically and
// that things are basically as we expect.
StreamBlock* block = m_table->getCommittedExportBytes();
int64_t uso = block->uso();
EXPECT_EQ(uso, 0);
size_t offset = block->offset();
EXPECT_TRUE(offset != 0);
while (block->offset() > 0)
{
m_table->releaseExportBytes(uso);
block = m_table->getCommittedExportBytes();
uso = block->uso();
EXPECT_EQ(uso, offset);
offset += block->offset();
}
}
/*
* Handoff fully committed blocks to the top end.
*
* This is the only function that should modify m_openTransactionId,
* m_openTransactionUso.
*/
void TupleStreamWrapper::commit(int64_t lastCommittedTxnId, int64_t currentTxnId, bool sync)
{
if (currentTxnId < m_openTransactionId)
{
throwFatalException("Active transactions moving backwards");
}
// more data for an ongoing transaction with no new committed data
if ((currentTxnId == m_openTransactionId) &&
(lastCommittedTxnId == m_committedTransactionId))
{
//std::cout << "Current txnid(" << currentTxnId << ") == m_openTransactionId(" << m_openTransactionId <<
//") && lastCommittedTxnId(" << lastCommittedTxnId << ") m_committedTransactionId(" <<
//m_committedTransactionId << ")" << std::endl;
if (sync) {
ExecutorContext::getExecutorContext()->getTopend()->pushExportBuffer(
m_generation,
m_partitionId,
m_signature,
NULL,
true,
false);
}
return;
}
// If the current TXN ID has advanced, then we know that:
// - The old open transaction has been committed
// - The current transaction is now our open transaction
if (m_openTransactionId < currentTxnId)
{
//std::cout << "m_openTransactionId(" << m_openTransactionId << ") < currentTxnId("
//<< currentTxnId << ")" << std::endl;
m_committedUso = m_uso;
// Advance the tip to the new transaction.
m_committedTransactionId = m_openTransactionId;
m_openTransactionId = currentTxnId;
}
// now check to see if the lastCommittedTxn tells us that our open
// transaction should really be committed. If so, update the
// committed state.
if (m_openTransactionId <= lastCommittedTxnId)
{
//std::cout << "m_openTransactionId(" << m_openTransactionId << ") <= lastCommittedTxnId(" <<
//lastCommittedTxnId << ")" << std::endl;
m_committedUso = m_uso;
m_committedTransactionId = m_openTransactionId;
}
while (!m_pendingBlocks.empty())
{
StreamBlock* block = m_pendingBlocks.front();
//std::cout << "m_committedUso(" << m_committedUso << "), block->uso() + block->offset() == "
//<< (block->uso() + block->offset()) << std::endl;
// check that the entire remainder is committed
if (m_committedUso >= (block->uso() + block->offset()))
{
//The block is handed off to the topend which is responsible for releasing the
//memory associated with the block data. The metadata is deleted here.
ExecutorContext::getExecutorContext()->getTopend()->pushExportBuffer(
m_generation,
m_partitionId,
m_signature,
block,
false,
false);
delete block;
m_pendingBlocks.pop_front();
}
else
{
break;
}
}
if (sync) {
ExecutorContext::getExecutorContext()->getTopend()->pushExportBuffer(
m_generation,
m_partitionId,
m_signature,
NULL,
true,
false);
}
}