Example #1
0
TEST_F(LoggingTests, BasicDeleteTest) {
  auto recovery_table = ExecutorTestsUtil::CreateTable(1024);
  auto &manager = catalog::Manager::GetInstance();
  storage::Database db(DEFAULT_DB_ID);
  manager.AddDatabase(&db);
  db.AddTable(recovery_table);

  EXPECT_EQ(recovery_table->GetNumberOfTuples(), 0);
  EXPECT_EQ(recovery_table->GetTileGroupCount(), 1);
  logging::WriteAheadFrontendLogger fel(true);

  cid_t test_commit_id = 10;

  auto curr_rec = new logging::TupleRecord(
      LOGRECORD_TYPE_TUPLE_UPDATE, test_commit_id, recovery_table->GetOid(),
      INVALID_ITEMPOINTER, ItemPointer(100, 4), nullptr, DEFAULT_DB_ID);
  fel.DeleteTuple(curr_rec);

  delete curr_rec;

  auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader();
  EXPECT_EQ(tg_header->GetEndCommitId(4), test_commit_id);

  //  EXPECT_EQ(recovery_table->GetNumberOfTuples(), 1);
  EXPECT_EQ(recovery_table->GetTileGroupCount(), 2);
}
bool PessimisticTxnManager::PerformUpdate(const oid_t &tile_group_id,
                                          const oid_t &tuple_id,
                                          const ItemPointer &new_location) {
  LOG_INFO("Performing Write %lu %lu", tile_group_id, tuple_id);

  auto transaction_id = current_txn->GetTransactionId();

  auto tile_group_header =
      catalog::Manager::GetInstance().GetTileGroup(tile_group_id)->GetHeader();
  auto new_tile_group_header = catalog::Manager::GetInstance()
      .GetTileGroup(new_location.block)->GetHeader();

  // if we can perform update, then we must have already locked the older
  // version.
  assert(tile_group_header->GetTransactionId(tuple_id) == transaction_id);
  assert(new_tile_group_header->GetTransactionId(new_location.offset) == INVALID_TXN_ID);
  assert(new_tile_group_header->GetBeginCommitId(new_location.offset) == MAX_CID);
  assert(new_tile_group_header->GetEndCommitId(new_location.offset) == MAX_CID);
  tile_group_header->SetTransactionId(tuple_id, transaction_id);

  // The write lock must have been acquired
  // Notice: if the executor doesn't call PerformUpdate after AcquireOwnership,
  // no
  // one will possibly release the write lock acquired by this txn.
  // Set double linked list
  tile_group_header->SetNextItemPointer(tuple_id, new_location);
  new_tile_group_header->SetPrevItemPointer(
      new_location.offset, ItemPointer(tile_group_id, tuple_id));

  new_tile_group_header->SetTransactionId(new_location.offset, transaction_id);

  // Add the old tuple into the update set
  current_txn->RecordUpdate(tile_group_id, tuple_id);
  return true;
}
const std::string TileGroupHeader::GetInfo() const {
  std::ostringstream os;

  os << "\t-----------------------------------------------------------\n";
  os << "\tTILE GROUP HEADER \n";

  oid_t active_tuple_slots = GetCurrentNextTupleSlot();
  peloton::ItemPointer item;

  for (oid_t header_itr = 0; header_itr < active_tuple_slots; header_itr++) {
    txn_id_t txn_id = GetTransactionId(header_itr);
    cid_t beg_commit_id = GetBeginCommitId(header_itr);
    cid_t end_commit_id = GetEndCommitId(header_itr);
    bool insert_commit = GetInsertCommit(header_itr);
    bool delete_commit = GetDeleteCommit(header_itr);

    int width = 10;
    os << "\t txn id : ";
    if (txn_id == MAX_TXN_ID)
      os << std::setw(width) << "MAX_TXN_ID";
    else
      os << std::setw(width) << txn_id;

    os << " beg cid : ";
    if (beg_commit_id == MAX_CID)
      os << std::setw(width) << "MAX_CID";
    else
      os << std::setw(width) << beg_commit_id;

    os << " end cid : ";
    if (end_commit_id == MAX_CID)
      os << std::setw(width) << "MAX_CID";
    else
      os << std::setw(width) << end_commit_id;

    os << " insert commit : ";
    if (insert_commit == true)
      os << "O";
    else
      os << "X";

    os << " delete commit : ";
    if (delete_commit == true)
      os << "O";
    else
      os << "X";

    peloton::ItemPointer location = GetNextItemPointer(header_itr);
    peloton::ItemPointer location2 = GetPrevItemPointer(header_itr);
    os << " next : "
       << "[ " << location.block << " , " << location.offset << " ]\n"
       << " prev : "
       << "[ " << location2.block << " , " << location2.offset << " ]\n";
  }

  os << "\t-----------------------------------------------------------\n";

  return os.str();
}
void PessimisticTxnManager::SetOwnership(const oid_t &tile_group_id,
                                         const oid_t &tuple_id) {
  auto &manager = catalog::Manager::GetInstance();
  auto tile_group_header = manager.GetTileGroup(tile_group_id)->GetHeader();
  auto transaction_id = current_txn->GetTransactionId();

  // Set MVCC info
  assert(tile_group_header->GetTransactionId(tuple_id) == INVALID_TXN_ID);
  assert(tile_group_header->GetBeginCommitId(tuple_id) == MAX_CID);
  assert(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID);

  tile_group_header->SetTransactionId(tuple_id, transaction_id);
}
void PessimisticTxnManager::PerformUpdate(const oid_t &tile_group_id,
                                          const oid_t &tuple_id) {
  auto &manager = catalog::Manager::GetInstance();
  auto tile_group_header = manager.GetTileGroup(tile_group_id)->GetHeader();
  
  // Set MVCC info
  assert(tile_group_header->GetTransactionId(tuple_id) == current_txn->GetTransactionId());
  assert(tile_group_header->GetBeginCommitId(tuple_id) == MAX_CID);
  assert(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID);

  // Add the old tuple into the update set
  auto old_location = tile_group_header->GetPrevItemPointer(tuple_id);
  if (old_location.IsNull() == false) {
    // update an inserted version
    current_txn->RecordUpdate(old_location.block, old_location.offset);
  }
}
Example #6
0
TEST_F(LoggingTests, BasicUpdateTest) {
  auto recovery_table = ExecutorTestsUtil::CreateTable(1024);
  auto &manager = catalog::Manager::GetInstance();
  storage::Database db(DEFAULT_DB_ID);
  manager.AddDatabase(&db);
  db.AddTable(recovery_table);

  auto tuples = BuildLoggingTuples(recovery_table, 1, false, false);
  EXPECT_EQ(recovery_table->GetNumberOfTuples(), 0);
  EXPECT_EQ(recovery_table->GetTileGroupCount(), 1);
  EXPECT_EQ(tuples.size(), 1);
  logging::WriteAheadFrontendLogger fel(true);
  //  auto bel = logging::WriteAheadBackendLogger::GetInstance();
  cid_t test_commit_id = 10;

  Value val0 = tuples[0]->GetValue(0);
  Value val1 = tuples[0]->GetValue(1);
  Value val2 = tuples[0]->GetValue(2);
  Value val3 = tuples[0]->GetValue(3);

  auto curr_rec = new logging::TupleRecord(
      LOGRECORD_TYPE_TUPLE_UPDATE, test_commit_id, recovery_table->GetOid(),
      ItemPointer(100, 5), ItemPointer(100, 4), tuples[0], DEFAULT_DB_ID);
  curr_rec->SetTuple(tuples[0]);
  fel.UpdateTuple(curr_rec);
  delete curr_rec;

  auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader();
  EXPECT_TRUE(tg_header->GetBeginCommitId(5) <= test_commit_id);
  EXPECT_EQ(tg_header->GetEndCommitId(5), MAX_CID);
  EXPECT_EQ(tg_header->GetEndCommitId(4), test_commit_id);

  EXPECT_TRUE(
      val0.Compare(recovery_table->GetTileGroupById(100)->GetValue(5, 0)) == 0);
  EXPECT_TRUE(
      val1.Compare(recovery_table->GetTileGroupById(100)->GetValue(5, 1)) == 0);
  EXPECT_TRUE(
      val2.Compare(recovery_table->GetTileGroupById(100)->GetValue(5, 2)) == 0);
  EXPECT_TRUE(
      val3.Compare(recovery_table->GetTileGroupById(100)->GetValue(5, 3)) == 0);

  EXPECT_EQ(recovery_table->GetNumberOfTuples(), 0);
  EXPECT_EQ(recovery_table->GetTileGroupCount(), 2);
}
std::pair<cid_t, storage::TileGroupHeader *>
WriteBehindFrontendLogger::SetDeleteCommitMark(ItemPointer location) {
  auto &manager = catalog::Manager::GetInstance();
  auto tile_group = manager.GetTileGroup(location.block);
  assert(tile_group != nullptr);
  auto tile_group_header = tile_group->GetHeader();
  assert(tile_group_header != nullptr);

  // Set the commit mark
  tile_group_header->SetDeleteCommit(location.offset, true);
  LOG_TRACE("<%p, %lu> : slot is delete committed", tile_group.get(),
            location.offset);

  // Update max oid
  if (max_oid < location.block) {
    max_oid = location.block;
  }

  auto end_commit_id = tile_group_header->GetEndCommitId(location.offset);
  return std::make_pair(end_commit_id, tile_group_header);
}
void PessimisticTxnManager::PerformDelete(const oid_t &tile_group_id,
                                          const oid_t &tuple_id) {
  auto &manager = catalog::Manager::GetInstance();
  auto tile_group_header = manager.GetTileGroup(tile_group_id)->GetHeader();

  assert(tile_group_header->GetTransactionId(tuple_id) ==
         current_txn->GetTransactionId());
  assert(tile_group_header->GetBeginCommitId(tuple_id) == MAX_CID);
  assert(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID);

  tile_group_header->SetEndCommitId(tuple_id, INVALID_CID);

  // Add the old tuple into the delete set
  auto old_location = tile_group_header->GetPrevItemPointer(tuple_id);
  if (old_location.IsNull() == false) {
    // delete an inserted version
    current_txn->RecordDelete(old_location.block, old_location.offset);
  } else {
    // if this version is newly inserted.
    current_txn->RecordDelete(tile_group_id, tuple_id);
  }
}
Example #9
0
TEST_F(LoggingTests, OutOfOrderCommitTest) {
  auto recovery_table = ExecutorTestsUtil::CreateTable(1024);
  auto &manager = catalog::Manager::GetInstance();
  storage::Database db(DEFAULT_DB_ID);
  manager.AddDatabase(&db);
  db.AddTable(recovery_table);

  auto tuples = BuildLoggingTuples(recovery_table, 1, false, false);
  EXPECT_EQ(recovery_table->GetNumberOfTuples(), 0);
  EXPECT_EQ(recovery_table->GetTileGroupCount(), 1);
  EXPECT_EQ(tuples.size(), 1);
  logging::WriteAheadFrontendLogger fel(true);
  //  auto bel = logging::WriteAheadBackendLogger::GetInstance();
  cid_t test_commit_id = 10;

  auto curr_rec = new logging::TupleRecord(
      LOGRECORD_TYPE_TUPLE_UPDATE, test_commit_id + 1, recovery_table->GetOid(),
      INVALID_ITEMPOINTER, ItemPointer(100, 5), nullptr, DEFAULT_DB_ID);
  fel.DeleteTuple(curr_rec);
  delete curr_rec;

  EXPECT_EQ(recovery_table->GetTileGroupCount(), 2);

  curr_rec = new logging::TupleRecord(
      LOGRECORD_TYPE_TUPLE_INSERT, test_commit_id, recovery_table->GetOid(),
      ItemPointer(100, 5), INVALID_ITEMPOINTER, tuples[0], DEFAULT_DB_ID);

  curr_rec->SetTuple(tuples[0]);
  fel.InsertTuple(curr_rec);

  delete curr_rec;

  auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader();
  EXPECT_EQ(tg_header->GetEndCommitId(5), test_commit_id + 1);

  EXPECT_EQ(recovery_table->GetNumberOfTuples(), 0);
  EXPECT_EQ(recovery_table->GetTileGroupCount(), 2);
}
Example #10
0
// count number of expired versions.
int GarbageNum(storage::DataTable *table) {
  auto table_tile_group_count_ = table->GetTileGroupCount();
  auto current_tile_group_offset_ = START_OID;

  int old_num = 0;

  while (current_tile_group_offset_ < table_tile_group_count_) {
    auto tile_group = table->GetTileGroup(current_tile_group_offset_++);
    auto tile_group_header = tile_group->GetHeader();
    oid_t active_tuple_count = tile_group->GetNextTupleSlot();

    for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) {
      auto tuple_txn_id = tile_group_header->GetTransactionId(tuple_id);
      auto tuple_end_cid = tile_group_header->GetEndCommitId(tuple_id);
      if (tuple_txn_id == INITIAL_TXN_ID && tuple_end_cid != MAX_CID) {
        old_num++;
      }
    }
  }

  LOG_INFO("old version num = %d", old_num);
  return old_num;
}
void TimestampOrderingTransactionManager::PerformInsert(
    TransactionContext *const current_txn, const ItemPointer &location,
    ItemPointer *index_entry_ptr) {
  PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY);

  oid_t tile_group_id = location.block;
  oid_t tuple_id = location.offset;

  auto &manager = catalog::Manager::GetInstance();
  auto tile_group_header = manager.GetTileGroup(tile_group_id)->GetHeader();
  auto transaction_id = current_txn->GetTransactionId();

  // check MVCC info
  // the tuple slot must be empty.
  PL_ASSERT(tile_group_header->GetTransactionId(tuple_id) == INVALID_TXN_ID);
  PL_ASSERT(tile_group_header->GetBeginCommitId(tuple_id) == MAX_CID);
  PL_ASSERT(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID);

  tile_group_header->SetTransactionId(tuple_id, transaction_id);

  // no need to set next item pointer.

  // Add the new tuple into the insert set
  current_txn->RecordInsert(location);

  InitTupleReserved(tile_group_header, tuple_id);

  // Write down the head pointer's address in tile group header
  tile_group_header->SetIndirection(tuple_id, index_entry_ptr);

  // Increment table insert op stats
  if (static_cast<StatsType>(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) !=
      StatsType::INVALID) {
    stats::BackendStatsContext::GetInstance()->IncrementTableInserts(
        location.block);
  }
}
Example #12
0
// Validate that MVCC storage is correct, it assumes an old-to-new chain
// Invariants
// 1. Transaction id should either be INVALID_TXNID or INITIAL_TXNID
// 2. Begin commit id should <= end commit id
// 3. Timestamp consistence
// 4. Version doubly linked list consistency
static void ValidateMVCC_OldToNew(storage::DataTable *table) {
  auto &catalog_manager = catalog::Manager::GetInstance();
  LOG_INFO("Validating MVCC storage");
  int tile_group_count = table->GetTileGroupCount();
  LOG_INFO("The table has %d tile groups in the table", tile_group_count);

  for (int tile_group_offset = 0; tile_group_offset < tile_group_count;
       tile_group_offset++) {
    LOG_INFO("Validate tile group #%d", tile_group_offset);
    auto tile_group = table->GetTileGroup(tile_group_offset);
    auto tile_group_header = tile_group->GetHeader();
    size_t tuple_count = tile_group->GetAllocatedTupleCount();
    LOG_INFO("Tile group #%d has allocated %lu tuples", tile_group_offset,
             tuple_count);

    // 1. Transaction id should either be INVALID_TXNID or INITIAL_TXNID
    for (oid_t tuple_slot = 0; tuple_slot < tuple_count; tuple_slot++) {
      txn_id_t txn_id = tile_group_header->GetTransactionId(tuple_slot);
      EXPECT_TRUE(txn_id == INVALID_TXN_ID || txn_id == INITIAL_TXN_ID)
          << "Transaction id is not INVALID_TXNID or INITIAL_TXNID";
    }

    LOG_INFO("[OK] All tuples have valid txn id");

    // double avg_version_chain_length = 0.0;
    for (oid_t tuple_slot = 0; tuple_slot < tuple_count; tuple_slot++) {
      txn_id_t txn_id = tile_group_header->GetTransactionId(tuple_slot);
      cid_t begin_cid = tile_group_header->GetBeginCommitId(tuple_slot);
      cid_t end_cid = tile_group_header->GetEndCommitId(tuple_slot);
      ItemPointer next_location =
          tile_group_header->GetNextItemPointer(tuple_slot);
      ItemPointer prev_location =
          tile_group_header->GetPrevItemPointer(tuple_slot);

      // 2. Begin commit id should <= end commit id
      EXPECT_TRUE(begin_cid <= end_cid)
          << "Tuple begin commit id is less than or equal to end commit id";

      // This test assumes a oldest-to-newest version chain
      if (txn_id != INVALID_TXN_ID) {
        EXPECT_TRUE(begin_cid != MAX_CID)
            << "Non invalid txn shouldn't have a MAX_CID begin commit id";

        // The version is an oldest version
        if (prev_location.IsNull()) {
          if (next_location.IsNull()) {
            EXPECT_EQ(end_cid, MAX_CID)
                << "Single version has a non MAX_CID end commit time";
          } else {
            cid_t prev_end_cid = end_cid;
            ItemPointer prev_location(tile_group->GetTileGroupId(), tuple_slot);
            while (!next_location.IsNull()) {
              auto next_tile_group =
                  catalog_manager.GetTileGroup(next_location.block);
              auto next_tile_group_header = next_tile_group->GetHeader();

              txn_id_t next_txn_id = next_tile_group_header->GetTransactionId(
                  next_location.offset);

              if (next_txn_id == INVALID_TXN_ID) {
                // If a version in the version chain has a INVALID_TXN_ID, it
                // must be at the tail
                // of the chain. It is either because we have deleted a tuple
                // (so append a invalid tuple),
                // or because this new version is aborted.
                EXPECT_TRUE(
                    next_tile_group_header->GetNextItemPointer(
                                                next_location.offset).IsNull())
                    << "Invalid version in a version chain and is not delete";
              }

              cid_t next_begin_cid = next_tile_group_header->GetBeginCommitId(
                  next_location.offset);
              cid_t next_end_cid =
                  next_tile_group_header->GetEndCommitId(next_location.offset);

              // 3. Timestamp consistence
              if (next_begin_cid == MAX_CID) {
                // It must be an aborted version, it should be at the end of the
                // chain
                EXPECT_TRUE(
                    next_tile_group_header->GetNextItemPointer(
                                                next_location.offset).IsNull())
                    << "Version with MAX_CID begin cid is not version tail";
              } else {
                EXPECT_EQ(prev_end_cid, next_begin_cid)
                    << "Prev end commit id should equal net begin commit id";
                ItemPointer next_prev_location =
                    next_tile_group_header->GetPrevItemPointer(
                        next_location.offset);

                // 4. Version doubly linked list consistency
                EXPECT_TRUE(next_prev_location.offset == prev_location.offset &&
                            next_prev_location.block == prev_location.block)
                    << "Next version's prev version does not match";
              }

              prev_location = next_location;
              prev_end_cid = next_end_cid;
              next_location = next_tile_group_header->GetNextItemPointer(
                  next_location.offset);
            }

            // Now prev_location is at the tail of the version chain
            ItemPointer last_location = prev_location;
            auto last_tile_group =
                catalog_manager.GetTileGroup(last_location.block);
            auto last_tile_group_header = last_tile_group->GetHeader();
            // txn_id_t last_txn_id =
            // last_tile_group_header->GetTransactionId(last_location.offset);
            cid_t last_end_cid =
                last_tile_group_header->GetEndCommitId(last_location.offset);
            EXPECT_TRUE(
                last_tile_group_header->GetNextItemPointer(last_location.offset)
                    .IsNull())
                << "Last version has a next pointer";

            EXPECT_EQ(last_end_cid, MAX_CID)
                << "Last version doesn't end with MAX_CID";
          }
        }
      } else {
        EXPECT_TRUE(tile_group_header->GetNextItemPointer(tuple_slot).IsNull())
            << "Invalid tuple must not have next item pointer";
      }
    }
    LOG_INFO("[OK] oldest-to-newest version chain validated");
  }
}
Example #13
0
bool IndexScanExecutor::ExecPrimaryIndexLookup() {
  PL_ASSERT(!done_);

  std::vector<ItemPointer *> tuple_location_ptrs;

  PL_ASSERT(index_->GetIndexType() == INDEX_CONSTRAINT_TYPE_PRIMARY_KEY);

  if (0 == key_column_ids_.size()) {
    index_->ScanAllKeys(tuple_location_ptrs);
  } else {
    index_->Scan(values_, key_column_ids_, expr_types_,
                 SCAN_DIRECTION_TYPE_FORWARD, tuple_location_ptrs);
  }


  if (tuple_location_ptrs.size() == 0) return false;

  auto &transaction_manager =
      concurrency::TransactionManagerFactory::GetInstance();

  std::map<oid_t, std::vector<oid_t>> visible_tuples;
  std::vector<ItemPointer> garbage_tuples;
  // for every tuple that is found in the index.
  for (auto tuple_location_ptr : tuple_location_ptrs) {
    
    ItemPointer tuple_location = *tuple_location_ptr;
    
    auto &manager = catalog::Manager::GetInstance();
    auto tile_group = manager.GetTileGroup(tuple_location.block);
    auto tile_group_header = tile_group.get()->GetHeader();

    size_t chain_length = 0;
    while (true) {

      ++chain_length;

      // if the tuple is visible.
      if (transaction_manager.IsVisible(tile_group_header,
                                        tuple_location.offset)) {

        LOG_TRACE("traverse chain length : %lu", chain_length);
        LOG_TRACE("perform read: %u, %u", tuple_location.block,
                 tuple_location.offset);

        // perform predicate evaluation.
        if (predicate_ == nullptr) {
          visible_tuples[tuple_location.block].push_back(tuple_location.offset);

          auto res = transaction_manager.PerformRead(tuple_location);
          if (!res) {
            transaction_manager.SetTransactionResult(RESULT_FAILURE);
            return res;
          }
        } else {
          expression::ContainerTuple<storage::TileGroup> tuple(
              tile_group.get(), tuple_location.offset);
          auto eval =
              predicate_->Evaluate(&tuple, nullptr, executor_context_).IsTrue();
          if (eval == true) {
            visible_tuples[tuple_location.block]
                .push_back(tuple_location.offset);

            auto res = transaction_manager.PerformRead(tuple_location);
            if (!res) {
              transaction_manager.SetTransactionResult(RESULT_FAILURE);
              return res;
            }
          }
        }
        break;
      }
      // if the tuple is not visible.
      else {
        ItemPointer old_item = tuple_location;
        cid_t old_end_cid = tile_group_header->GetEndCommitId(old_item.offset);

        tuple_location = tile_group_header->GetNextItemPointer(old_item.offset);
        // there must exist a visible version.

        // FIXME: currently, only speculative read transaction manager **may** see a null version
        // it's a potential bug
        if(tuple_location.IsNull()) {
          transaction_manager.SetTransactionResult(RESULT_FAILURE);
          // FIXME: this cause unnecessary abort when we have delete operations
          return false;
        }

        // FIXME: Is this always true? what if we have a deleted tuple? --jiexi
        PL_ASSERT(tuple_location.IsNull() == false);

        cid_t max_committed_cid = transaction_manager.GetMaxCommittedCid();

        // check whether older version is garbage.

        if (old_end_cid <= max_committed_cid) {
          PL_ASSERT(tile_group_header->GetTransactionId(old_item.offset) == INITIAL_TXN_ID ||
                      tile_group_header->GetTransactionId(old_item.offset) == INVALID_TXN_ID);

          if (tile_group_header->SetAtomicTransactionId(old_item.offset, INVALID_TXN_ID) == true) {

            // atomically swap item pointer held in the index bucket.
            AtomicUpdateItemPointer(tuple_location_ptr, tuple_location);

            // currently, let's assume only primary index exists.
            // gc::GCManagerFactory::GetInstance().RecycleTupleSlot(
            //     table_->GetOid(), old_item.block, old_item.offset,
            //     transaction_manager.GetNextCommitId());
            garbage_tuples.push_back(old_item);

            tile_group = manager.GetTileGroup(tuple_location.block);
            tile_group_header = tile_group.get()->GetHeader();
            tile_group_header->SetPrevItemPointer(tuple_location.offset, INVALID_ITEMPOINTER);

          } else {

            tile_group = manager.GetTileGroup(tuple_location.block);
            tile_group_header = tile_group.get()->GetHeader();
          }

        } else {
        tile_group = manager.GetTileGroup(tuple_location.block);
        tile_group_header = tile_group.get()->GetHeader();

        }


      }
    }
  }

  // Add all garbage tuples to GC manager
  if(garbage_tuples.size() != 0) {
    cid_t garbage_timestamp = transaction_manager.GetNextCommitId();
    for (auto garbage : garbage_tuples) {
      gc::GCManagerFactory::GetInstance().RecycleTupleSlot(
        table_->GetOid(), garbage.block, garbage.offset, garbage_timestamp);
    }
  }

  // Construct a logical tile for each block
  for (auto tuples : visible_tuples) {
    auto &manager = catalog::Manager::GetInstance();
    auto tile_group = manager.GetTileGroup(tuples.first);

    std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile());
    // Add relevant columns to logical tile
    logical_tile->AddColumns(tile_group, full_column_ids_);
    logical_tile->AddPositionList(std::move(tuples.second));
    if (column_ids_.size() != 0) {
      logical_tile->ProjectColumns(full_column_ids_, column_ids_);
    }

    result_.push_back(logical_tile.release());
  }

  done_ = true;

  LOG_TRACE("Result tiles : %lu", result_.size());

  return true;
}
void TileGroupHeader::PrintVisibility(txn_id_t txn_id, cid_t at_cid) {
  oid_t active_tuple_slots = GetNextTupleSlot();
  std::stringstream os;

  os << "\t-----------------------------------------------------------\n";

  for (oid_t header_itr = 0; header_itr < active_tuple_slots; header_itr++) {
    bool own = (txn_id == GetTransactionId(header_itr));
    bool activated = (at_cid >= GetBeginCommitId(header_itr));
    bool invalidated = (at_cid >= GetEndCommitId(header_itr));

    txn_id_t txn_id = GetTransactionId(header_itr);
    cid_t beg_commit_id = GetBeginCommitId(header_itr);
    cid_t end_commit_id = GetEndCommitId(header_itr);
    bool insert_commit = GetInsertCommit(header_itr);
    bool delete_commit = GetDeleteCommit(header_itr);

    int width = 10;

    os << "\tslot :: " << std::setw(width) << header_itr;

    os << " txn id : ";
    if (txn_id == MAX_TXN_ID)
      os << std::setw(width) << "MAX_TXN_ID";
    else
      os << std::setw(width) << txn_id;

    os << " beg cid : ";
    if (beg_commit_id == MAX_CID)
      os << std::setw(width) << "MAX_CID";
    else
      os << std::setw(width) << beg_commit_id;

    os << " end cid : ";
    if (end_commit_id == MAX_CID)
      os << std::setw(width) << "MAX_CID";
    else
      os << std::setw(width) << end_commit_id;

    os << " insert commit : ";
    if (insert_commit == true)
      os << "O";
    else
      os << "X";

    os << " delete commit : ";
    if (delete_commit == true)
      os << "O";
    else
      os << "X";

    peloton::ItemPointer location = GetPrevItemPointer(header_itr);
    os << " prev : "
       << "[ " << location.block << " , " << location.offset << " ]";  //<<

    os << " own : " << own;
    os << " activated : " << activated;
    os << " invalidated : " << invalidated << " ";

    // Visible iff past Insert || Own Insert
    if ((!own && activated && !invalidated) ||
        (own && !activated && !invalidated))
      os << "\t\t[ true  ]\n";
    else
      os << "\t\t[ false ]\n";
  }

  os << "\t-----------------------------------------------------------\n";

  std::cout << os.str().c_str();
}
void TimestampOrderingTransactionManager::PerformUpdate(
    TransactionContext *const current_txn, const ItemPointer &location,
    const ItemPointer &new_location) {
  PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY);

  ItemPointer old_location = location;

  LOG_TRACE("Performing Update old tuple %u %u", old_location.block,
            old_location.offset);
  LOG_TRACE("Performing Update new tuple %u %u", new_location.block,
            new_location.offset);

  auto &manager = catalog::Manager::GetInstance();

  auto tile_group_header =
      manager.GetTileGroup(old_location.block)->GetHeader();
  auto new_tile_group_header =
      manager.GetTileGroup(new_location.block)->GetHeader();

  auto transaction_id = current_txn->GetTransactionId();
  // if we can perform update, then we must have already locked the older
  // version.
  PL_ASSERT(tile_group_header->GetTransactionId(old_location.offset) ==
            transaction_id);
  PL_ASSERT(tile_group_header->GetPrevItemPointer(old_location.offset)
                .IsNull() == true);

  // check whether the new version is empty.
  PL_ASSERT(new_tile_group_header->GetTransactionId(new_location.offset) ==
            INVALID_TXN_ID);
  PL_ASSERT(new_tile_group_header->GetBeginCommitId(new_location.offset) ==
            MAX_CID);
  PL_ASSERT(new_tile_group_header->GetEndCommitId(new_location.offset) ==
            MAX_CID);

  // if the executor doesn't call PerformUpdate after AcquireOwnership,
  // no one will possibly release the write lock acquired by this txn.

  // Set double linked list
  tile_group_header->SetPrevItemPointer(old_location.offset, new_location);

  new_tile_group_header->SetNextItemPointer(new_location.offset, old_location);

  new_tile_group_header->SetTransactionId(new_location.offset, transaction_id);

  // we should guarantee that the newer version is all set before linking the
  // newer version to older version.
  COMPILER_MEMORY_FENCE;

  InitTupleReserved(new_tile_group_header, new_location.offset);

  // we must be updating the latest version.
  // Set the header information for the new version
  ItemPointer *index_entry_ptr =
      tile_group_header->GetIndirection(old_location.offset);

  // if there's no primary index on a table, then index_entry_ptr == nullptr.
  if (index_entry_ptr != nullptr) {
    new_tile_group_header->SetIndirection(new_location.offset, index_entry_ptr);

    // Set the index header in an atomic way.
    // We do it atomically because we don't want any one to see a half-done
    // pointer.
    // In case of contention, no one can update this pointer when we are
    // updating it
    // because we are holding the write lock. This update should success in
    // its first trial.
    UNUSED_ATTRIBUTE auto res =
        AtomicUpdateItemPointer(index_entry_ptr, new_location);
    PL_ASSERT(res == true);
  }

  // Add the old tuple into the update set
  current_txn->RecordUpdate(old_location);

  // Increment table update op stats
  if (static_cast<StatsType>(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) !=
      StatsType::INVALID) {
    stats::BackendStatsContext::GetInstance()->IncrementTableUpdates(
        new_location.block);
  }
}
Example #16
0
bool HybridScanExecutor::ExecPrimaryIndexLookup() {
  assert(!index_done_);

  std::vector<ItemPointer *> tuple_location_ptrs;

  assert(index_->GetIndexType() == INDEX_CONSTRAINT_TYPE_PRIMARY_KEY);

  if (0 == key_column_ids_.size()) {
    index_->ScanAllKeys(tuple_location_ptrs);
  } else {
    index_->Scan(values_, key_column_ids_, expr_types_,
                 SCAN_DIRECTION_TYPE_FORWARD, tuple_location_ptrs);
  }

  LOG_INFO("Tuple_locations.size(): %lu", tuple_location_ptrs.size());

  auto &transaction_manager =
    concurrency::TransactionManagerFactory::GetInstance();

   if (tuple_location_ptrs.size() == 0) {
    index_done_ = true;
    return false;
  }

  //std::set<oid_t> oid_ts;

  std::map<oid_t, std::vector<oid_t>> visible_tuples;
  // for every tuple that is found in the index.
  for (auto tuple_location_ptr : tuple_location_ptrs) {
    ItemPointer tuple_location = *tuple_location_ptr;

    if (type_ == planner::HYBRID &&
      tuple_location.block >= (block_threshold)) {
        item_pointers_.insert(tuple_location);
    //  oid_ts.insert(tuple_location.block);
    }

    auto &manager = catalog::Manager::GetInstance();
    auto tile_group = manager.GetTileGroup(tuple_location.block);
    auto tile_group_header = tile_group.get()->GetHeader();

    // perform transaction read
    size_t chain_length = 0;
    while (true) {

      ++chain_length;

      if (transaction_manager.IsVisible(tile_group_header,
                                        tuple_location.offset)) {
        visible_tuples[tuple_location.block].push_back(tuple_location.offset);
        auto res = transaction_manager.PerformRead(tuple_location);
        if (!res) {
          transaction_manager.SetTransactionResult(RESULT_FAILURE);
          return res;
        }
        break;
      } else {
        ItemPointer old_item = tuple_location;
        cid_t old_end_cid = tile_group_header->GetEndCommitId(old_item.offset);

        tuple_location = tile_group_header->GetNextItemPointer(old_item.offset);
        // there must exist a visible version.
        assert(tuple_location.IsNull() == false);

        cid_t max_committed_cid = transaction_manager.GetMaxCommittedCid();

        // check whether older version is garbage.
        if (old_end_cid < max_committed_cid) {
          assert(tile_group_header->GetTransactionId(old_item.offset) == INITIAL_TXN_ID ||
                 tile_group_header->GetTransactionId(old_item.offset) == INVALID_TXN_ID);

          if (tile_group_header->SetAtomicTransactionId(old_item.offset, INVALID_TXN_ID) == true) {


            // atomically swap item pointer held in the index bucket.
            AtomicUpdateItemPointer(tuple_location_ptr, tuple_location);

            // currently, let's assume only primary index exists.
            //gc::GCManagerFactory::GetInstance().RecycleTupleSlot(
            //  table_->GetOid(), old_item.block, old_item.offset,
            //  max_committed_cid);
          }
        }

        tile_group = manager.GetTileGroup(tuple_location.block);
        tile_group_header = tile_group.get()->GetHeader();
      }
    }
  }

  // Construct a logical tile for each block
  for (auto tuples : visible_tuples) {
    auto &manager = catalog::Manager::GetInstance();
    auto tile_group = manager.GetTileGroup(tuples.first);

    std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile());
    // Add relevant columns to logical tile
    logical_tile->AddColumns(tile_group, full_column_ids_);
    logical_tile->AddPositionList(std::move(tuples.second));
    if (column_ids_.size() != 0) {
      logical_tile->ProjectColumns(full_column_ids_, column_ids_);
    }

    result_.push_back(logical_tile.release());
  }

  index_done_ = true;

  LOG_TRACE("Result tiles : %lu", result_.size());

  return true;
}
bool HybridScanExecutor::ExecPrimaryIndexLookup() {
  PL_ASSERT(index_done_ == false);

  const planner::HybridScanPlan &node = GetPlanNode<planner::HybridScanPlan>();
  bool acquire_owner = GetPlanNode<planner::AbstractScan>().IsForUpdate();

  auto key_column_ids_ = node.GetKeyColumnIds();
  auto expr_type_ = node.GetExprTypes();

  std::vector<ItemPointer *> tuple_location_ptrs;

  PL_ASSERT(index_->GetIndexType() == INDEX_CONSTRAINT_TYPE_PRIMARY_KEY);

  if (0 == key_column_ids_.size()) {
    LOG_TRACE("Scan all keys");
    index_->ScanAllKeys(tuple_location_ptrs);
  } else {
    LOG_TRACE("Scan");
    index_->Scan(values_,
                 key_column_ids_,
                 expr_type_,
                 SCAN_DIRECTION_TYPE_FORWARD,
                 tuple_location_ptrs,
                 &node.GetIndexPredicate().GetConjunctionList()[0]);
  }

  LOG_TRACE("Result tuple count: %lu", tuple_location_ptrs.size());

  auto &transaction_manager =
      concurrency::TransactionManagerFactory::GetInstance();

  auto current_txn = executor_context_->GetTransaction();

  if (tuple_location_ptrs.size() == 0) {
    index_done_ = true;
    return false;
  }

  std::map<oid_t, std::vector<oid_t>> visible_tuples;

  // for every tuple that is found in the index.
  for (auto tuple_location_ptr : tuple_location_ptrs) {
    ItemPointer tuple_location = *tuple_location_ptr;

    if (type_ == HYBRID_SCAN_TYPE_HYBRID &&
        tuple_location.block >= (block_threshold)) {
      item_pointers_.insert(tuple_location);
    }

    auto &manager = catalog::Manager::GetInstance();
    auto tile_group = manager.GetTileGroup(tuple_location.block);
    auto tile_group_header = tile_group.get()->GetHeader();

    // perform transaction read
    size_t chain_length = 0;
    while (true) {
      ++chain_length;

      auto visibility = transaction_manager.IsVisible(current_txn, tile_group_header, tuple_location.offset);

      if (visibility == VISIBILITY_OK) {

        visible_tuples[tuple_location.block].push_back(tuple_location.offset);
        auto res = transaction_manager.PerformRead(current_txn, tuple_location, acquire_owner);
        if (!res) {
          transaction_manager.SetTransactionResult(current_txn, RESULT_FAILURE);
          return res;
        }
        break;
      } else {
        ItemPointer old_item = tuple_location;
        cid_t old_end_cid = tile_group_header->GetEndCommitId(old_item.offset);

        tuple_location = tile_group_header->GetNextItemPointer(old_item.offset);
        // there must exist a visible version.
        assert(tuple_location.IsNull() == false);

        cid_t max_committed_cid = transaction_manager.GetMaxCommittedCid();

        // check whether older version is garbage.
        if (old_end_cid < max_committed_cid) {
          assert(tile_group_header->GetTransactionId(old_item.offset) ==
              INITIAL_TXN_ID ||
              tile_group_header->GetTransactionId(old_item.offset) ==
                  INVALID_TXN_ID);

          if (tile_group_header->SetAtomicTransactionId(
              old_item.offset, INVALID_TXN_ID) == true) {
            // atomically swap item pointer held in the index bucket.
            AtomicUpdateItemPointer(tuple_location_ptr, tuple_location);
          }
        }

        tile_group = manager.GetTileGroup(tuple_location.block);
        tile_group_header = tile_group.get()->GetHeader();
      }
    }
  }

  // Construct a logical tile for each block
  for (auto tuples : visible_tuples) {
    auto &manager = catalog::Manager::GetInstance();
    auto tile_group = manager.GetTileGroup(tuples.first);

    std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile());

    // Add relevant columns to logical tile
    logical_tile->AddColumns(tile_group, full_column_ids_);
    logical_tile->AddPositionList(std::move(tuples.second));

    if (column_ids_.size() != 0) {
      logical_tile->ProjectColumns(full_column_ids_, column_ids_);
    }

    result_.push_back(logical_tile.release());
  }

  index_done_ = true;

  LOG_TRACE("Result tiles : %lu", result_.size());

  return true;
}