/**
* Iterate through the table blocks until all the active tuples have been found. Skip dirty tuples
* and mark them as clean so that they can be copied during the next snapshot.
*/
bool CopyOnWriteIterator::next(TableTuple &out) {
assert(m_currentBlock != NULL);
while (true) {
if (m_blockOffset >= m_currentBlock->unusedTupleBoundry()) {
if (m_blockIterator == m_end) {
m_table->snapshotFinishedScanningBlock(m_currentBlock, TBPtr());
break;
}
m_table->snapshotFinishedScanningBlock(m_currentBlock, m_blockIterator.data());
m_location = m_blockIterator.key();
m_currentBlock = m_blockIterator.data();
assert(m_currentBlock->address() == m_location);
m_blockIterator.data() = TBPtr();
m_blockOffset = 0;
m_blockIterator++;
}
assert(m_location < m_currentBlock.get()->address() + m_table->m_tableAllocationSize);
assert(m_location < m_currentBlock.get()->address() + (m_table->m_tupleLength * m_table->m_tuplesPerBlock));
assert (out.sizeInValues() == m_table->columnCount());
m_blockOffset++;
out.move(m_location);
const bool active = out.isActive();
const bool dirty = out.isDirty();
// Return this tuple only when this tuple is not marked as deleted and isn't dirty
if (active && !dirty) {
out.setDirtyFalse();
m_location += m_tupleLength;
return true;
} else {
out.setDirtyFalse();
m_location += m_tupleLength;
}
}
return false;
}
void AntiCacheEvictionManager::printLRUChain(PersistentTable* table, int max, bool forward)
{
VOLT_INFO("num tuples in chain: %d", table->getNumTuplesInEvictionChain());
VOLT_INFO("oldest tuple id: %u", table->getOldestTupleID());
VOLT_INFO("newest tuple id: %u", table->getNewestTupleID());
char chain[max * 4];
int tuple_id;
TableTuple tuple = table->tempTuple();
if(forward)
tuple_id = table->getOldestTupleID();
else
tuple_id = table->getNewestTupleID();
chain[0] = '\0';
int iterations = 0;
while(iterations < table->getNumTuplesInEvictionChain() && iterations < max)
{
strcat(chain, itoa(tuple_id));
strcat(chain, " ");
tuple.move(table->dataPtrForTuple(tuple_id));
if(forward)
tuple_id = tuple.getNextTupleInChain();
else
tuple_id = tuple.getPreviousTupleInChain();
iterations++;
}
VOLT_INFO("LRU CHAIN: %s", chain);
}
bool EvictionIterator::next(TableTuple &tuple)
{
PersistentTable* ptable = static_cast<PersistentTable*>(table);
if(current_tuple_id == ptable->getNewestTupleID()) // we've already returned the last tuple in the chain
{
VOLT_DEBUG("No more tuples in the chain.");
return false;
}
if(current_tuple_id == -1) // this is the first call to next
{
VOLT_DEBUG("This is the first tuple in the chain.");
if(ptable->getNumTuplesInEvictionChain() == 0) // there are no tuples in the chain
{
VOLT_DEBUG("There are no tuples in the eviction chain.");
return false;
}
current_tuple_id = ptable->getOldestTupleID();
}
else // advance the iterator to the next tuple in the chain
{
current_tuple_id = current_tuple->getTupleID();
}
current_tuple->move(ptable->dataPtrForTuple(current_tuple_id));
tuple.move(current_tuple->address());
VOLT_DEBUG("current_tuple_id = %d", current_tuple_id);
return true;
}
TableTuple *newTuple(TupleSchema *schema, int idx, long value) {
TableTuple *tuple = new TableTuple(schema);
char *data = new char[tuple->tupleLength()];
memset(data, 0, tuple->tupleLength());
tuple->move(data);
tuple->setNValue(idx, ValueFactory::getBigIntValue(value));
return tuple;
}
bool EvictionIterator::next(TableTuple &tuple)
{
#ifndef ANTICACHE_TIMESTAMPS
PersistentTable* ptable = static_cast<PersistentTable*>(table);
if(current_tuple_id == ptable->getNewestTupleID()) // we've already returned the last tuple in the chain
{
VOLT_DEBUG("No more tuples in the chain.");
return false;
}
if(is_first) // this is the first call to next
{
is_first = false;
VOLT_DEBUG("This is the first tuple in the chain.");
if(ptable->getNumTuplesInEvictionChain() == 0) // there are no tuples in the chain
{
VOLT_DEBUG("There are no tuples in the eviction chain.");
return false;
}
current_tuple_id = ptable->getOldestTupleID();
}
else // advance the iterator to the next tuple in the chain
{
current_tuple_id = current_tuple->getNextTupleInChain();
}
current_tuple->move(ptable->dataPtrForTuple(current_tuple_id));
tuple.move(current_tuple->address());
VOLT_DEBUG("current_tuple_id = %d", current_tuple_id);
#else
tuple.move(candidates[current_tuple_id].m_addr);
current_tuple_id++;
while (candidates[current_tuple_id].m_addr == candidates[current_tuple_id - 1].m_addr) {
current_tuple_id++;
if (current_tuple_id == m_size) break;
}
#endif
return true;
}
/**
* Iterate through the table blocks until all the active tuples have been found. Skip dirty tuples
* and mark them as clean so that they can be copied during the next snapshot.
*/
bool CopyOnWriteIterator::next(TableTuple &out) {
if (m_currentBlock == NULL) {
return false;
}
while (true) {
if (m_blockOffset >= m_currentBlock->unusedTupleBoundary()) {
if (m_blockIterator == m_end) {
m_surgeon->snapshotFinishedScanningBlock(m_currentBlock, TBPtr());
break;
}
m_surgeon->snapshotFinishedScanningBlock(m_currentBlock, m_blockIterator.data());
char *finishedBlock = m_currentBlock->address();
m_location = m_blockIterator.key();
m_currentBlock = m_blockIterator.data();
assert(m_currentBlock->address() == m_location);
m_blockOffset = 0;
// Remove the finished block from the map so that it can be released
// back to the OS if all tuples in the block is deleted.
//
// This invalidates the iterators, so we have to get new iterators
// using the current block's start address. m_blockIterator has to
// point to the next block, hence the upper_bound() call.
m_blocks.erase(finishedBlock);
m_blockIterator = m_blocks.upper_bound(m_currentBlock->address());
m_end = m_blocks.end();
}
assert(m_location < m_currentBlock.get()->address() + m_table->getTableAllocationSize());
assert(m_location < m_currentBlock.get()->address() + (m_table->getTupleLength() * m_table->getTuplesPerBlock()));
assert (out.columnCount() == m_table->columnCount());
m_blockOffset++;
out.move(m_location);
const bool active = out.isActive();
const bool dirty = out.isDirty();
if (dirty) m_skippedDirtyRows++;
if (!active) m_skippedInactiveRows++;
// Return this tuple only when this tuple is not marked as deleted and isn't dirty
if (active && !dirty) {
out.setDirtyFalse();
m_location += m_tupleLength;
return true;
} else {
out.setDirtyFalse();
m_location += m_tupleLength;
}
}
return false;
}
TEST_F(TableTupleFilterTest, tableTupleFilterTest)
{
static const int MARKER = 33;
TempTable* table = getTempTable();
TableTupleFilter tableFilter;
tableFilter.init(table);
int tuplePerBlock = table->getTuplesPerBlock();
// make sure table spans more than one block
ASSERT_TRUE(NUM_OF_TUPLES / tuplePerBlock > 1);
TableTuple tuple = table->tempTuple();
TableIterator iterator = table->iterator();
// iterator over and mark every 5th tuple
int counter = 0;
std::multiset<int64_t> control_values;
while(iterator.next(tuple)) {
if (++counter % 5 == 0) {
NValue nvalue = tuple.getNValue(1);
int64_t value = ValuePeeker::peekBigInt(nvalue);
control_values.insert(value);
tableFilter.updateTuple(tuple, MARKER);
}
}
TableTupleFilter_iter<MARKER> endItr = tableFilter.end<MARKER>();
for (TableTupleFilter_iter<MARKER> itr = tableFilter.begin<MARKER>(); itr != endItr; ++itr) {
uint64_t tupleAddr = tableFilter.getTupleAddress(*itr);
tuple.move((char *)tupleAddr);
ASSERT_TRUE(tuple.isActive());
NValue nvalue = tuple.getNValue(1);
int64_t value = ValuePeeker::peekBigInt(nvalue);
ASSERT_FALSE(control_values.empty());
auto it = control_values.find(value);
ASSERT_NE(it, control_values.end());
control_values.erase(it);
}
ASSERT_TRUE(control_values.empty());
}
/**
* Get the next tuple or return false if none is available.
*/
bool ElasticScanner::next(TableTuple &out)
{
bool found = false;
while (!found && continueScan()) {
assert(m_currentBlockPtr != NULL);
// Sanity checks.
assert(m_tuplePtr < m_currentBlockPtr.get()->address() + m_table.getTableAllocationSize());
assert(m_tuplePtr < m_currentBlockPtr.get()->address() + (m_tupleSize * m_table.getTuplesPerBlock()));
assert (out.sizeInValues() == m_table.columnCount());
// Grab the tuple pointer.
out.move(m_tuplePtr);
// Shift to the next tuple in block.
// continueScan() will check if it's the last one in the block.
m_tupleIndex++;
m_tuplePtr += m_tupleSize;
// The next active/non-dirty tuple is return-worthy.
found = out.isActive() && !out.isDirty();
}
return found;
}
int64_t TupleTrackerManager::getPrimaryKey(std::string tableName, uint32_t tupleId){
Table* table = voltDBEngine->getTable(tableName);
TableTuple tuple = TableTuple(table->schema());
tuple.move(table->dataPtrForTuple(tupleId));
TableIndex *m_index = table->primaryKeyIndex();
std::vector<int> column_indices_vector = m_index->getColumnIndices();
std::vector<int>::iterator it = column_indices_vector.begin();
NValue colValue;
int i = 0;
while (it != column_indices_vector.end()) // this is for non composite key
{
colValue = tuple.getNValue(*it);
it++;
i++;
}
return colValue.castAsBigIntAndGetValue();
//return i;
//return colValue.isNull();
/*
it = std::find(column_indices_vector.begin(), column_indices_vector.end(), tupleId);
return (int) std:: distance(column_indices_vector.begin(), it);
TableTuple outputTuple = ... // tuple for your output table
TableTuple inputTuple = ... // tuple from your tracking table
TableTuple origTuple = ... // tuple from the original PersistantTable
foreach (inputTuple in tracking table) {
// (1) Get offset from inputTuple and move the origTuple to that location
origTuple.move(table->dataPtrForTuple(tupleId));
// (2) Now iterate over the pkey column offsets and copy the values into the inputTuple
int col_idx = 0;
for (pkey_offset in m_index->getColumnIndices()) {
NValue colValue = origTuple.getNValue(pkey_offset);
outputTuple.setNValue(col_idx, colValue);
col_idx++;
}
// (3) Insert outputTuple into output table
}
int colCount = (int)column_indices_vector.size();
if (colCount < tupleId)
return -1;
return column_indices_vector[tupleId];
//*/
}
bool InsertExecutor::p_execute(const NValueArray ¶ms) {
assert(m_node == dynamic_cast<InsertPlanNode*>(m_abstractNode));
assert(m_node);
assert(m_inputTable == dynamic_cast<TempTable*>(m_node->getInputTable()));
assert(m_inputTable);
// Target table can be StreamedTable or PersistentTable and must not be NULL
// Update target table reference from table delegate
Table* targetTable = m_node->getTargetTable();
assert(targetTable);
assert((targetTable == dynamic_cast<PersistentTable*>(targetTable)) ||
(targetTable == dynamic_cast<StreamedTable*>(targetTable)));
PersistentTable* persistentTable = m_isStreamed ?
NULL : static_cast<PersistentTable*>(targetTable);
TableTuple upsertTuple = TableTuple(targetTable->schema());
VOLT_TRACE("INPUT TABLE: %s\n", m_inputTable->debug().c_str());
// count the number of successful inserts
int modifiedTuples = 0;
Table* outputTable = m_node->getOutputTable();
assert(outputTable);
TableTuple templateTuple = m_templateTuple.tuple();
std::vector<int>::iterator it;
for (it = m_nowFields.begin(); it != m_nowFields.end(); ++it) {
templateTuple.setNValue(*it, NValue::callConstant<FUNC_CURRENT_TIMESTAMP>());
}
VOLT_DEBUG("This is a %s-row insert on partition with id %d",
m_node->getChildren()[0]->getPlanNodeType() == PLAN_NODE_TYPE_MATERIALIZE ?
"single" : "multi", m_engine->getPartitionId());
VOLT_DEBUG("Offset of partition column is %d", m_partitionColumn);
//
// An insert is quite simple really. We just loop through our m_inputTable
// and insert any tuple that we find into our targetTable. It doesn't get any easier than that!
//
TableTuple inputTuple(m_inputTable->schema());
assert (inputTuple.sizeInValues() == m_inputTable->columnCount());
TableIterator iterator = m_inputTable->iterator();
while (iterator.next(inputTuple)) {
for (int i = 0; i < m_node->getFieldMap().size(); ++i) {
// Most executors will just call setNValue instead of
// setNValueAllocateForObjectCopies.
//
// However, We need to call
// setNValueAlocateForObjectCopies here. Sometimes the
// input table's schema has an inlined string field, and
// it's being assigned to the target table's outlined
// string field. In this case we need to tell the NValue
// where to allocate the string data.
templateTuple.setNValueAllocateForObjectCopies(m_node->getFieldMap()[i],
inputTuple.getNValue(i),
ExecutorContext::getTempStringPool());
}
VOLT_TRACE("Inserting tuple '%s' into target table '%s' with table schema: %s",
templateTuple.debug(targetTable->name()).c_str(), targetTable->name().c_str(),
targetTable->schema()->debug().c_str());
// if there is a partition column for the target table
if (m_partitionColumn != -1) {
// get the value for the partition column
NValue value = templateTuple.getNValue(m_partitionColumn);
bool isLocal = m_engine->isLocalSite(value);
// if it doesn't map to this site
if (!isLocal) {
if (!m_multiPartition) {
throw ConstraintFailureException(
dynamic_cast<PersistentTable*>(targetTable),
templateTuple,
"Mispartitioned tuple in single-partition insert statement.");
}
// don't insert
continue;
}
}
// for multi partition export tables, only insert into one
// place (the partition with hash(0)), if the data is from a
// replicated source. If the data is coming from a subquery
// with partitioned tables, we need to perform the insert on
// every partition.
if (m_isStreamed && m_multiPartition && !m_sourceIsPartitioned) {
bool isLocal = m_engine->isLocalSite(ValueFactory::getBigIntValue(0));
if (!isLocal) continue;
}
if (! m_isUpsert) {
// try to put the tuple into the target table
if (m_hasPurgeFragment) {
if (!executePurgeFragmentIfNeeded(&persistentTable))
return false;
// purge fragment might have truncated the table, and
// refreshed the persistent table pointer. Make sure to
// use it when doing the insert below.
targetTable = persistentTable;
}
if (!targetTable->insertTuple(templateTuple)) {
VOLT_ERROR("Failed to insert tuple from input table '%s' into"
" target table '%s'",
m_inputTable->name().c_str(),
targetTable->name().c_str());
return false;
}
} else {
// upsert execution logic
assert(persistentTable->primaryKeyIndex() != NULL);
TableTuple existsTuple = persistentTable->lookupTupleByValues(templateTuple);
if (existsTuple.isNullTuple()) {
// try to put the tuple into the target table
if (m_hasPurgeFragment) {
if (!executePurgeFragmentIfNeeded(&persistentTable))
return false;
}
if (!persistentTable->insertTuple(templateTuple)) {
VOLT_ERROR("Failed to insert tuple from input table '%s' into"
" target table '%s'",
m_inputTable->name().c_str(),
persistentTable->name().c_str());
return false;
}
} else {
// tuple exists already, try to update the tuple instead
upsertTuple.move(templateTuple.address());
TableTuple &tempTuple = persistentTable->getTempTupleInlined(upsertTuple);
if (!persistentTable->updateTupleWithSpecificIndexes(existsTuple, tempTuple,
persistentTable->allIndexes())) {
VOLT_INFO("Failed to update existsTuple from table '%s'",
persistentTable->name().c_str());
return false;
}
}
}
// successfully inserted or updated
modifiedTuples++;
}
TableTuple& count_tuple = outputTable->tempTuple();
count_tuple.setNValue(0, ValueFactory::getBigIntValue(modifiedTuples));
// try to put the tuple into the output table
if (!outputTable->insertTuple(count_tuple)) {
VOLT_ERROR("Failed to insert tuple count (%d) into"
" output table '%s'",
modifiedTuples,
outputTable->name().c_str());
return false;
}
// add to the planfragments count of modified tuples
m_engine->addToTuplesModified(modifiedTuples);
VOLT_DEBUG("Finished inserting %d tuples", modifiedTuples);
return true;
}
bool AntiCacheEvictionManager::removeTupleSingleLinkedList(PersistentTable* table, uint32_t removal_id) {
bool tuple_found = false;
int tuples_in_chain;
// ids for iterating through the list
uint32_t current_tuple_id;
uint32_t previous_tuple_id;
uint32_t next_tuple_id;
uint32_t newest_tuple_id;
// assert we have tuples in the eviction chain before we try to remove anything
tuples_in_chain = table->getNumTuplesInEvictionChain();
if (tuples_in_chain <= 0)
return false;
previous_tuple_id = 0;
current_tuple_id = table->getOldestTupleID();
newest_tuple_id = table->getNewestTupleID();
// set the tuple to the first tuple in the chain (i.e. oldest)
TableTuple tuple = table->tempTuple();
tuple.move(table->dataPtrForTuple(current_tuple_id));
// we're removing the head of the chain, i.e. the oldest tuple
if (table->getOldestTupleID() == removal_id) {
//VOLT_INFO("Removing the first tuple in the eviction chain.");
if (table->getNumTuplesInEvictionChain() == 1) { // this is the only tuple in the chain
table->setOldestTupleID(0);
table->setNewestTupleID(0);
} else {
next_tuple_id = tuple.getNextTupleInChain();
table->setOldestTupleID(next_tuple_id);
}
tuple_found = true;
}
int iterations = 0;
while(!tuple_found && iterations < table->getNumTuplesInEvictionChain()) {
// we've found the tuple we want to remove
if (current_tuple_id == removal_id) {
next_tuple_id = tuple.getNextTupleInChain();
// create a tuple from the previous tuple id in the chain
tuple.move(table->dataPtrForTuple(previous_tuple_id));
// set the previous tuple to point to the next tuple
tuple.setNextTupleInChain(next_tuple_id);
tuple_found = true;
break;
}
// advance pointers
previous_tuple_id = current_tuple_id;
current_tuple_id = tuple.getNextTupleInChain();
tuple.move(table->dataPtrForTuple(current_tuple_id));
iterations++;
}
if (current_tuple_id == newest_tuple_id && !tuple_found) { // we are at the back of the chain
if (current_tuple_id == removal_id) { // we're removing the back of the chain
// set the previous tuple pointer to 0 since it is now the back of the chain
tuple.move(table->dataPtrForTuple(previous_tuple_id));
tuple.setNextTupleInChain(0);
table->setNewestTupleID(previous_tuple_id);
tuple_found = true;
}
}
if (tuple_found) {
--tuples_in_chain;
table->setNumTuplesInEvictionChain(tuples_in_chain);
return true;
}
return false;
}
// for the double linked list we start from the tail of the chain and iterate backwards
bool AntiCacheEvictionManager::removeTupleDoubleLinkedList(PersistentTable* table, uint32_t removal_id) {
bool tuple_found = false;
int tuples_in_chain;
// ids for iterating through the list
uint32_t current_tuple_id;
uint32_t previous_tuple_id;
uint32_t next_tuple_id;
uint32_t oldest_tuple_id;
// assert we have tuples in the eviction chain before we try to remove anything
tuples_in_chain = table->getNumTuplesInEvictionChain();
if (tuples_in_chain <= 0)
return false;
previous_tuple_id = 0;
oldest_tuple_id = table->getOldestTupleID();
current_tuple_id = table->getNewestTupleID(); // start iteration at back of chain
// set the tuple to the back of the chain (i.e. the newest)
TableTuple tuple = table->tempTuple();
tuple.move(table->dataPtrForTuple(current_tuple_id));
// we're removing the tail of the chain, i.e. the newest tuple
if (table->getNewestTupleID() == removal_id) {
if (table->getNumTuplesInEvictionChain() == 1) { // this is the only tuple in the chain
table->setOldestTupleID(0);
table->setNewestTupleID(0);
} else if(table->getNumTuplesInEvictionChain() == 2) {
table->setNewestTupleID(oldest_tuple_id);
table->setOldestTupleID(oldest_tuple_id);
}
else {
tuple.move(table->dataPtrForTuple(table->getNewestTupleID()));
// we need the previous tuple in the chain, since we're iterating from back to front
previous_tuple_id = tuple.getPreviousTupleInChain();
table->setNewestTupleID(previous_tuple_id);
}
tuple_found = true;
}
// we're removing the head of the chain, i.e. the oldest tuple
if(table->getOldestTupleID() == removal_id && !tuple_found) {
if (table->getNumTuplesInEvictionChain() == 1) { // this is the only tuple in the chain
table->setOldestTupleID(0);
table->setNewestTupleID(0);
}
else if(table->getNumTuplesInEvictionChain() == 2) {
table->setNewestTupleID(table->getNewestTupleID());
table->setOldestTupleID(table->getNewestTupleID());
}
else {
tuple.move(table->dataPtrForTuple(table->getOldestTupleID()));
next_tuple_id = tuple.getNextTupleInChain();
table->setOldestTupleID(next_tuple_id);
}
tuple_found = true;
}
int iterations = 0;
while(!tuple_found && iterations < table->getNumTuplesInEvictionChain()) {
if(current_tuple_id == oldest_tuple_id)
break;
// we've found the tuple we want to remove
if (current_tuple_id == removal_id) {
next_tuple_id = tuple.getPreviousTupleInChain();
// point previous tuple in chain to next tuple
tuple.move(table->dataPtrForTuple(previous_tuple_id));
tuple.setPreviousTupleInChain(next_tuple_id);
// point next tuple in chain to previous tuple
tuple.move(table->dataPtrForTuple(next_tuple_id));
tuple.setNextTupleInChain(previous_tuple_id);
tuple_found = true;
break;
}
// advance pointers
previous_tuple_id = current_tuple_id;
current_tuple_id = tuple.getPreviousTupleInChain(); // iterate back to front
tuple.move(table->dataPtrForTuple(current_tuple_id));
iterations++;
}
if (tuple_found) {
tuples_in_chain = table->getNumTuplesInEvictionChain();
--tuples_in_chain;
table->setNumTuplesInEvictionChain(tuples_in_chain);
return true;
}
return false;
}