/**
* @brief recover the database and check the tuples
*/
void DoRecovery() {
//===--------------------------------------------------------------------===//
// RECOVERY
//===--------------------------------------------------------------------===//
// Reset log manager state
auto& log_manager = peloton::logging::LogManager::GetInstance();
log_manager.ResetLogStatus();
log_manager.ResetFrontendLoggers();
Timer<std::milli> timer;
std::thread thread;
timer.Start();
// Do recovery
StartLogging(thread);
// Synchronize and finish recovery
if (log_manager.EndLogging()) {
thread.join();
} else {
LOG_ERROR("Failed to terminate logging thread");
}
timer.Stop();
// Recovery time (in ms)
if (state.experiment_type == EXPERIMENT_TYPE_RECOVERY) {
WriteOutput(timer.GetDuration());
}
}
void RunBackend(oid_t thread_id) {
auto txn_count = state.transaction_count;
UniformGenerator generator;
Timer<> timer;
// Start timer
timer.Reset();
timer.Start();
// Run these many transactions
for (oid_t txn_itr = 0; txn_itr < txn_count; txn_itr++) {
auto rng_val = generator.GetSample();
if (rng_val <= 0.04) {
RunStockLevel();
} else if (rng_val <= 0.08) {
RunDelivery();
} else if (rng_val <= 0.12) {
RunOrderStatus();
} else if (rng_val <= 0.55) {
RunPayment();
} else {
RunNewOrder();
}
}
// Stop timer
timer.Stop();
// Set duration
durations[thread_id] = timer.GetDuration();
}
void RunBackend(oid_t thread_id) {
auto txn_count = state.transaction_count;
auto update_ratio = state.update_ratio;
UniformGenerator generator;
Timer<> timer;
// Start timer
timer.Reset();
timer.Start();
// Run these many transactions
for (oid_t txn_itr = 0; txn_itr < txn_count; txn_itr++) {
auto rng_val = generator.GetSample();
if (rng_val < update_ratio) {
RunUpdate();
}
else {
RunRead();
}
}
// Stop timer
timer.Stop();
// Set duration
durations[thread_id] = timer.GetDuration();
}
void ExecuteTest(executor::AbstractExecutor *executor) {
Timer<> timer;
bool status = false;
status = executor->Init();
if (status == false) {
throw Exception("Init failed");
}
timer.Start();
size_t result_tuple_count = 0;
while (executor->Execute() == true) {
std::unique_ptr<executor::LogicalTile> result_tile(executor->GetOutput());
result_tuple_count += result_tile->GetTupleCount();
}
timer.Stop();
UNUSED_ATTRIBUTE double time_per_transaction = timer.GetDuration();
LOG_TRACE("%f", time_per_transaction);
LOG_TRACE("Lower bound : %.0lf", tuple_start_offset);
LOG_TRACE("Upper bound : %.0lf", tuple_end_offset);
LOG_TRACE("Result tuple count : %lu", result_tuple_count);
// EXPECT_EQ(result_tuple_count, selectivity * tuple_count);
}
// Generate the cave.
// Here we will also time it and calculate the CPU usage.
// TODO(Kim): Calculate CPU usage.
void CellularAutomata::GenerateCave()
{
Timer t;
t.StartTimer();
for (int i = 0; i < GetGenerations(); i++)
StepInGeneration();
t.StopTimer();
SetTimeToGenerate(t.GetDuration());
}
TEST_F(InsertTests, LoadingTest) {
// We are going to simply load tile groups concurrently in this test
// WARNING: This test may potentially run for a long time if
// TEST_TUPLES_PER_TILEGROUP is large, consider rewrite the test or hard
// code the number of tuples per tile group in this test
oid_t tuples_per_tilegroup = TEST_TUPLES_PER_TILEGROUP;
bool build_indexes = false;
// Control the scale
oid_t loader_threads_count = 1;
oid_t tilegroup_count_per_loader = 1;
// Each tuple size ~40 B.
oid_t tuple_size = 41;
std::unique_ptr<storage::DataTable> data_table(
ExecutorTestsUtil::CreateTable(tuples_per_tilegroup, build_indexes));
auto testing_pool = TestingHarness::GetInstance().GetTestingPool();
Timer<> timer;
timer.Start();
LaunchParallelTest(loader_threads_count, InsertTuple, data_table.get(),
testing_pool, tilegroup_count_per_loader);
timer.Stop();
auto duration = timer.GetDuration();
LOG_INFO("Duration: %.2lf", duration);
//EXPECT_LE(duration, 0.2);
auto expected_tile_group_count =
loader_threads_count * tilegroup_count_per_loader + 1;
auto bytes_to_megabytes_converter = (1024 * 1024);
EXPECT_EQ(data_table->GetTileGroupCount(), expected_tile_group_count);
LOG_INFO("Dataset size : %u MB \n",
(expected_tile_group_count * tuples_per_tilegroup * tuple_size) /
bytes_to_megabytes_converter);
}
/**
* @brief recover the database and check the tuples
*/
void DoRecovery(std::string file_name) {
auto file_path = GetFilePath(state.log_file_dir, file_name);
std::ifstream log_file(file_path);
// Reset the log file if exists
log_file.close();
ycsb::CreateYCSBDatabase();
//===--------------------------------------------------------------------===//
// RECOVERY
//===--------------------------------------------------------------------===//
Timer<std::milli> timer;
std::thread thread;
timer.Start();
auto& log_manager = peloton::logging::LogManager::GetInstance();
log_manager.ResetLogStatus();
log_manager.ResetFrontendLoggers();
if (peloton_logging_mode != LOGGING_TYPE_INVALID) {
// Launching a thread for logging
if (!log_manager.IsInLoggingMode()) {
// Set sync commit mode
log_manager.SetSyncCommit(false);
// Wait for standby mode
auto local_thread = std::thread(
&peloton::logging::LogManager::StartStandbyMode, &log_manager);
thread.swap(local_thread);
log_manager.WaitForModeTransition(peloton::LOGGING_STATUS_TYPE_STANDBY,
true);
// Clean up database tile state before recovery from checkpoint
log_manager.PrepareRecovery();
// Do any recovery
log_manager.StartRecoveryMode();
// Wait for logging mode
log_manager.WaitForModeTransition(peloton::LOGGING_STATUS_TYPE_LOGGING,
true);
// Done recovery
log_manager.DoneRecovery();
}
}
if (log_manager.EndLogging()) {
thread.join();
} else {
LOG_ERROR("Failed to terminate logging thread");
}
timer.Stop();
// Recovery time (in ms)
WriteOutput(timer.GetDuration());
}
/**
* @brief writing a simple log file
*/
bool PrepareLogFile() {
if (chdir(state.log_file_dir.c_str())) {
LOG_ERROR("change directory failed");
}
// start a thread for logging
auto& log_manager = logging::LogManager::GetInstance();
if (log_manager.ContainsFrontendLogger() == true) {
LOG_ERROR("another logging thread is running now");
return false;
}
Timer<> timer;
std::thread thread;
timer.Start();
if (peloton_logging_mode != LOGGING_TYPE_INVALID) {
// Launching a thread for logging
if (!log_manager.IsInLoggingMode()) {
// Set sync commit mode
log_manager.SetSyncCommit(false);
// Wait for standby mode
auto local_thread = std::thread(
&peloton::logging::LogManager::StartStandbyMode, &log_manager);
thread.swap(local_thread);
log_manager.WaitForModeTransition(peloton::LOGGING_STATUS_TYPE_STANDBY,
true);
// Clean up database tile state before recovery from checkpoint
log_manager.PrepareRecovery();
// Do any recovery
log_manager.StartRecoveryMode();
// Wait for logging mode
log_manager.WaitForModeTransition(peloton::LOGGING_STATUS_TYPE_LOGGING,
true);
// Done recovery
log_manager.DoneRecovery();
}
}
// Build the log
BuildLog();
// Stop frontend logger if in a valid logging mode
if (peloton_logging_mode != LOGGING_TYPE_INVALID) {
// Wait for the mode transition :: LOGGING -> TERMINATE -> SLEEP
if (log_manager.EndLogging()) {
thread.join();
}
}
timer.Stop();
auto duration = timer.GetDuration();
auto throughput =
(ycsb::state.transaction_count * ycsb::state.backend_count) / duration;
// Log the build log time
if (state.experiment_type == EXPERIMENT_TYPE_INVALID ||
state.experiment_type == EXPERIMENT_TYPE_ACTIVE ||
state.experiment_type == EXPERIMENT_TYPE_WAIT) {
WriteOutput(throughput);
} else if (state.experiment_type == EXPERIMENT_TYPE_STORAGE) {
auto log_file_size = GetLogFileSize();
WriteOutput(log_file_size);
}
return true;
}
int main(const int ac, const char* av[])
{
//////////////////////////////////////////////////////////////////////////
// Command Line Options
std::vector<std::string> inputFiles;
for (int i = 1; i < ac; ++i) {
std::string str(av[i]);
if (str == "-w") {
Parse<1>(ac, av, i, width);
}
else if (str == "-h") {
Parse<1>(ac, av, i, height);
}
else if (str == "-random") {
Parse<1>(ac, av, i, random_size);
}
else if (str == "-numPatches") {
Parse<1>(ac, av, i, numPatches);
}
}
//////////////////////////////////////////////////////////////////////////
// MPI stuff
int mpiRank, mpiSize;
MPI_Init(NULL, NULL);
MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
int hostnamelen;
char hostname[512];
MPI_Get_processor_name(hostname, &hostnamelen);
//std::cout << "MPISize = " << mpiSize << std::endl;
//////////////////////////////////////////////////////////////////////////
// Initialization
srand((mpiRank+1)*25*time(NULL)); // random seed
Timer clock; // Timer
//////////////////////////////////////////////////////////////////////////
// Randomly create images
const size_t minX = 0, minY = 0, maxX = width, maxY = height;
std::vector<Image> imgList(numPatches);
for (int i = 0; i < numPatches; ++i) {
InitImage(minX, maxX, minY, maxY, mpiRank, imgList[i], random_size);
}
//////////////////////////////////////////////////////////////////////////
// Composition
QCT::Init(ac, av);
if (mpiSize == 1) {
////////////////////////////////////////////////////////////////////////
// Using VisIt Method
auto visit = QCT::Create(QCT::ALGO_VISIT_ONE_NODE, width, height);
clock.Start();
QCT::BeginFrame(visit);
for (int i=0; i<numPatches; i++) {
/* porting the code into our API */
float depth = imgList[i].GetDepth();
QCT::Tile tile(imgList[i].GetExtents(0),
imgList[i].GetExtents(2),
imgList[i].GetExtents(1),
imgList[i].GetExtents(3),
width,
height,
imgList[i].GetData(),
&depth,
QCT_TILE_REDUCED_DEPTH);
QCT::SetTile(visit, tile);
}
QCT::EndFrame(visit);
clock.Stop();
for (int i=0; i<numPatches; i++) {
imgList[i].DeleteImage();
}
////////////////////////////////////////////////////////////////////////
// Timing
CreatePPM((float*)QCT::MapColorBuffer(visit),
width, height, outputdir);
std::cout << "[Single Node (VisIt method)] " << clock.GetDuration()
<< " seconds to finish" << std::endl;
}
else {
#if 1
////////////////////////////////////////////////////////////////////////
// Test Tree Method
auto tree = QCT::Create(QCT::ALGO_TREE, width, height);
clock.Start();
float depth = imgList[0].GetDepth();
QCT::Tile tile(imgList[0].GetExtents(0),
imgList[0].GetExtents(2),
imgList[0].GetExtents(1),
imgList[0].GetExtents(3),
width,
height,
imgList[0].GetData(),
&depth,
QCT_TILE_REDUCED_DEPTH);
QCT::BeginFrame(tree);
QCT::SetTile(tree, tile);
QCT::EndFrame(tree);
clock.Stop();
if (mpiRank == 0) {
CreatePPM((float*)QCT::MapColorBuffer(tree), width, height, outputdir);
std::cout << "[Multiple Node (Tree method)] " << clock.GetDuration()
<< " seconds to finish" << std::endl;
}
#else
////////////////////////////////////////////////////////////////////////
// Using VisIt Method
auto visit = QCT::Create(QCT::ALGO_VISIT_ICET, width, height);
std::cout << "visit is " << QCT::IsValid(visit) << std::endl;
if (QCT::IsValid(visit) && numPatches == 1) {
clock.Start();
/* porting the code into our API */
float depth = imgList[0].GetDepth();
QCT::Tile tile(imgList[0].GetExtents(0),
imgList[0].GetExtents(2),
imgList[0].GetExtents(1),
imgList[0].GetExtents(3),
width,
height,
imgList[0].GetData(),
&depth,
QCT_TILE_REDUCED_DEPTH);
QCT::BeginFrame(visit);
QCT::SetTile(visit, tile);
QCT::EndFrame(visit);
clock.Stop();
////////////////////////////////////////////////////////////////////////
// Timing
if (MPIRank == 0) {
CreatePPM((float*)QCT::MapColorBuffer(visit),
width, height, outputdir);
std::cout << "[Multiple Node (VisIt method)] " << clock.GetDuration()
<< " seconds to finish" << std::endl;
}
}
#endif
}
MPI_Finalize();
return 0;
}
/*
* TestIndexPerformance() - Test driver for indices of a given type
*
* This function tests Insert and Delete performance together with
* key scan
*/
static void TestIndexPerformance(const IndexType &index_type) {
// This is where we read all values in and verify them
std::vector<ItemPointer *> location_ptrs;
// INDEX
std::unique_ptr<index::Index> index(BuildIndex(false, index_type));
// Parallel Test by default 1 Million key
// Number of threads doing insert or delete
size_t num_thread = 4;
// Number of keys inserted by each thread
size_t num_key = 1024 * 256;
Timer<> timer;
///////////////////////////////////////////////////////////////////
// Start InsertTest1
///////////////////////////////////////////////////////////////////
timer.Start();
// First two arguments are used for launching tasks
// All remaining arguments are passed to the thread body
LaunchParallelTest(num_thread, InsertTest1, index.get(), num_thread, num_key);
// Perform garbage collection
if (index->NeedGC() == true) {
index->PerformGC();
}
index->ScanAllKeys(location_ptrs);
EXPECT_EQ(num_thread * num_key, location_ptrs.size());
location_ptrs.clear();
timer.Stop();
LOG_INFO("InsertTest1 :: Type=%s; Duration=%.2lf",
IndexTypeToString(index_type).c_str(), timer.GetDuration());
///////////////////////////////////////////////////////////////////
// Start DeleteTest1
///////////////////////////////////////////////////////////////////
timer.Start();
LaunchParallelTest(num_thread, DeleteTest1, index.get(), num_thread, num_key);
// Perform garbage collection
if (index->NeedGC() == true) {
index->PerformGC();
}
index->ScanAllKeys(location_ptrs);
EXPECT_EQ(0, location_ptrs.size());
location_ptrs.clear();
timer.Stop();
LOG_INFO("DeleteTest1 :: Type=%s; Duration=%.2lf",
IndexTypeToString(index_type).c_str(), timer.GetDuration());
///////////////////////////////////////////////////////////////////
// Start InsertTest2
///////////////////////////////////////////////////////////////////
timer.Start();
LaunchParallelTest(num_thread, InsertTest2, index.get(), num_thread, num_key);
// Perform garbage collection
if (index->NeedGC() == true) {
index->PerformGC();
}
index->ScanAllKeys(location_ptrs);
EXPECT_EQ(num_thread * num_key, location_ptrs.size());
location_ptrs.clear();
timer.Stop();
LOG_INFO("InsertTest2 :: Type=%s; Duration=%.2lf",
IndexTypeToString(index_type).c_str(), timer.GetDuration());
///////////////////////////////////////////////////////////////////
// Start DeleteTest2
///////////////////////////////////////////////////////////////////
timer.Start();
LaunchParallelTest(num_thread, DeleteTest2, index.get(), num_thread, num_key);
// Perform garbage collection
if (index->NeedGC() == true) {
index->PerformGC();
}
index->ScanAllKeys(location_ptrs);
EXPECT_EQ(0, location_ptrs.size());
location_ptrs.clear();
timer.Stop();
LOG_INFO("DeleteTest :: Type=%s; Duration=%.2lf",
IndexTypeToString(index_type).c_str(), timer.GetDuration());
///////////////////////////////////////////////////////////////////
// End of all tests
///////////////////////////////////////////////////////////////////
delete tuple_schema;
return;
}