int main(int argc, char* argv[]) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
// Detect the hardware concurrency level.
const std::size_t num_hw_threads =
DefaultsConfigurator::GetNumHardwareThreads();
// Use the command-line value if that was supplied, else use the value
// that we computed above, provided it did return a valid value.
// TODO(jmp): May need to change this at some point to keep one thread
// available for the OS if the hardware concurrency level is high.
const unsigned int real_num_workers = quickstep::FLAGS_num_workers != 0
? quickstep::FLAGS_num_workers
: (num_hw_threads != 0 ?
num_hw_threads
: 1);
if (real_num_workers > 0) {
printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool\n",
real_num_workers,
(static_cast<double>(quickstep::FLAGS_buffer_pool_slots) * quickstep::kSlotSizeBytes)/quickstep::kAGigaByte);
} else {
LOG(FATAL) << "Quickstep needs at least one worker thread to run";
}
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
if (quickstep::FLAGS_use_hdfs) {
LOG(INFO) << "Using HDFS as the default persistent storage, with namenode at "
<< quickstep::FLAGS_hdfs_namenode_host << ":"
<< quickstep::FLAGS_hdfs_namenode_port << " and block replication factor "
<< quickstep::FLAGS_hdfs_num_replications << "\n";
}
#endif
// Initialize the thread ID based map here before the Foreman and workers are
// constructed because the initialization isn't thread safe.
quickstep::ClientIDMap::Instance();
MessageBusImpl bus;
bus.Initialize();
// The TMB client id for the main thread, used to kill workers at the end.
const client_id main_thread_client_id = bus.Connect();
bus.RegisterClientAsSender(main_thread_client_id, kPoisonMessage);
// Setup the paths used by StorageManager.
string fixed_storage_path(quickstep::FLAGS_storage_path);
if (!fixed_storage_path.empty()
&& (fixed_storage_path.back() != quickstep::kPathSeparator)) {
fixed_storage_path.push_back(quickstep::kPathSeparator);
}
string catalog_path(fixed_storage_path);
catalog_path.append("catalog.pb.bin");
if (quickstep::FLAGS_initialize_db) { // Initialize the database
// TODO(jmp): Refactor the code in this file!
LOG(INFO) << "Initializing the database, creating a new catalog file and storage directory\n";
// Create the directory
// TODO(jmp): At some point, likely in C++-17, we will just have the
// filesystem path, and we can clean this up
#ifdef QUICKSTEP_OS_WINDOWS
std::filesystem::create_directories(fixed_storage_path);
LOG(FATAL) << "Failed when attempting to create the directory: " << fixed_storage_path << "\n";
LOG(FATAL) << "Check if the directory already exists. If so, delete it or move it before initializing \n";
#else
{
string path_name = "mkdir " + fixed_storage_path;
if (std::system(path_name.c_str())) {
LOG(FATAL) << "Failed when attempting to create the directory: " << fixed_storage_path << "\n";
}
}
#endif
// Create the default catalog file.
std::ofstream catalog_file(catalog_path);
if (!catalog_file.good()) {
LOG(FATAL) << "ERROR: Unable to open catalog.pb.bin for writing.\n";
}
quickstep::Catalog catalog;
catalog.addDatabase(new quickstep::CatalogDatabase(nullptr, "default"));
if (!catalog.getProto().SerializeToOstream(&catalog_file)) {
LOG(FATAL) << "ERROR: Unable to serialize catalog proto to file catalog.pb.bin\n";
return 1;
}
// Close the catalog file - it will be reopened below by the QueryProcessor.
catalog_file.close();
}
// Setup QueryProcessor, including CatalogDatabase and StorageManager.
std::unique_ptr<QueryProcessor> query_processor;
try {
query_processor.reset(new QueryProcessor(catalog_path, fixed_storage_path));
} catch (const std::exception &e) {
LOG(FATAL) << "FATAL ERROR DURING STARTUP: "
<< e.what()
<< "\nIf you intended to create a new database, "
<< "please use the \"-initialize_db=true\" command line option.";
} catch (...) {
LOG(FATAL) << "NON-STANDARD EXCEPTION DURING STARTUP";
}
// Parse the CPU affinities for workers and the preloader thread, if enabled
// to warm up the buffer pool.
const vector<int> worker_cpu_affinities =
InputParserUtil::ParseWorkerAffinities(real_num_workers,
quickstep::FLAGS_worker_affinities);
const std::size_t num_numa_nodes_covered =
DefaultsConfigurator::GetNumNUMANodesCoveredByWorkers(worker_cpu_affinities);
if (quickstep::FLAGS_preload_buffer_pool) {
std::chrono::time_point<std::chrono::steady_clock> preload_start, preload_end;
preload_start = std::chrono::steady_clock::now();
printf("Preloading the buffer pool ... ");
fflush(stdout);
quickstep::PreloaderThread preloader(*query_processor->getDefaultDatabase(),
query_processor->getStorageManager(),
worker_cpu_affinities.front());
preloader.start();
preloader.join();
preload_end = std::chrono::steady_clock::now();
printf("in %g seconds\n",
std::chrono::duration<double>(preload_end - preload_start).count());
}
Foreman foreman(&bus,
query_processor->getDefaultDatabase(),
query_processor->getStorageManager(),
num_numa_nodes_covered);
// Get the NUMA affinities for workers.
vector<int> cpu_numa_nodes = InputParserUtil::GetNUMANodesForCPUs();
if (cpu_numa_nodes.empty()) {
// libnuma is not present. Assign -1 as the NUMA node for every worker.
cpu_numa_nodes.assign(worker_cpu_affinities.size(), -1);
}
vector<int> worker_numa_nodes;
PtrVector<Worker> workers;
vector<client_id> worker_client_ids;
// Initialize the worker threads.
DCHECK_EQ(static_cast<std::size_t>(real_num_workers),
worker_cpu_affinities.size());
for (std::size_t worker_thread_index = 0;
worker_thread_index < worker_cpu_affinities.size();
++worker_thread_index) {
int numa_node_id = -1;
if (worker_cpu_affinities[worker_thread_index] >= 0) {
// This worker can be NUMA affinitized.
numa_node_id = cpu_numa_nodes[worker_cpu_affinities[worker_thread_index]];
}
worker_numa_nodes.push_back(numa_node_id);
workers.push_back(
new Worker(worker_thread_index, &bus, worker_cpu_affinities[worker_thread_index]));
worker_client_ids.push_back(workers.back().getBusClientID());
}
// TODO(zuyu): Move WorkerDirectory within Shiftboss once the latter is added.
WorkerDirectory worker_directory(worker_cpu_affinities.size(),
worker_client_ids,
worker_numa_nodes);
foreman.setWorkerDirectory(&worker_directory);
// Start the worker threads.
for (Worker &worker : workers) {
worker.start();
}
LineReaderImpl line_reader("quickstep> ",
" ...> ");
std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper());
std::chrono::time_point<std::chrono::steady_clock> start, end;
for (;;) {
string *command_string = new string();
*command_string = line_reader.getNextCommand();
if (command_string->size() == 0) {
delete command_string;
break;
}
parser_wrapper->feedNextBuffer(command_string);
bool quitting = false;
// A parse error should reset the parser. This is because the thrown quickstep
// SqlError does not do the proper reset work of the YYABORT macro.
bool reset_parser = false;
for (;;) {
ParseResult result = parser_wrapper->getNextStatement();
if (result.condition == ParseResult::kSuccess) {
if (result.parsed_statement->getStatementType() == ParseStatement::kQuit) {
quitting = true;
break;
}
if (result.parsed_statement->getStatementType() == ParseStatement::kCommand) {
try {
quickstep::cli::executeCommand(
*result.parsed_statement,
*(query_processor->getDefaultDatabase()), stdout);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s",
sql_error.formatMessage(*command_string).c_str());
reset_parser = true;
break;
}
continue;
}
std::unique_ptr<QueryHandle> query_handle;
try {
query_handle.reset(query_processor->generateQueryHandle(*result.parsed_statement));
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s", sql_error.formatMessage(*command_string).c_str());
reset_parser = true;
break;
}
DCHECK(query_handle->getQueryPlanMutable() != nullptr);
foreman.setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());
foreman.reconstructQueryContextFromProto(query_handle->getQueryContextProto());
try {
start = std::chrono::steady_clock::now();
foreman.start();
foreman.join();
end = std::chrono::steady_clock::now();
const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
if (query_result_relation) {
PrintToScreen::PrintRelation(*query_result_relation,
query_processor->getStorageManager(),
stdout);
DropRelation::Drop(*query_result_relation,
query_processor->getDefaultDatabase(),
query_processor->getStorageManager());
}
query_processor->saveCatalog();
printf("Execution time: %g seconds\n",
std::chrono::duration<double>(end - start).count());
} catch (const std::exception &e) {
fprintf(stderr, "QUERY EXECUTION ERROR: %s\n", e.what());
break;
}
printf("Query Complete\n");
} else {
if (result.condition == ParseResult::kError) {
fprintf(stderr, "%s", result.error_message.c_str());
}
reset_parser = true;
break;
}
}
if (quitting) {
break;
} else if (reset_parser) {
parser_wrapper.reset(new SqlParserWrapper());
reset_parser = false;
}
}
// Terminate all workers before exiting.
// The main thread broadcasts poison message to the workers. Each worker dies
// after receiving poison message. The order of workers' death is irrelavant.
MessageStyle style;
style.Broadcast(true);
Address address;
address.All(true);
std::unique_ptr<WorkerMessage> poison_message(WorkerMessage::PoisonMessage());
TaggedMessage poison_tagged_message(poison_message.get(),
sizeof(*poison_message),
kPoisonMessage);
const tmb::MessageBus::SendStatus send_status =
bus.Send(main_thread_client_id,
address,
style,
std::move(poison_tagged_message));
CHECK(send_status == tmb::MessageBus::SendStatus::kOK) <<
"Broadcast message from Foreman to workers failed";
for (Worker &worker : workers) {
worker.join();
}
return 0;
}
int main(int argc, char *argv[])
{
int test = argc > 1 ? atoi(argv[1]) : 0;
int verbose = argc > 2;
int veryVerbose = argc > 3;
printf("TEST " __FILE__ " CASE %d\n", test);
switch (test) { case 0:
case 2: {
// --------------------------------------------------------------------
// USAGE EXAMPLE
//
// Concerns:
//: 1 The usage example provided in the component header file compiles,
//: links, and runs as shown.
//
// Plan:
//: 1 Incorporate usage example from header into test driver, remove
//: leading comment characters, and replace 'assert' with 'ASSERT'.
//: (C-1)
//
// Testing:
// USAGE EXAMPLE
// --------------------------------------------------------------------
if (verbose) printf("\nUSAGE EXAMPLE\n"
"\n=============\n");
///Usage
///-----
// In this section we show intended use of this component.
//
///Example 1: Removing The 'const'-qualifier of A Type
///- - - - - - - - - - - - - - - - - - - - - - - - - -
// Suppose that we want to remove any top-level 'const'-qualifier from a
// particular type.
//
// First, we create two 'typedef's -- a 'const'-qualified type ('MyConstType')
// and the same type without the 'const'-qualifier ('MyType'):
//..
typedef int MyType;
typedef const int MyConstType;
//..
// Now, we remove the 'const'-qualifier from 'MyConstType' using
// 'bsl::remove_const' and verify that the resulting type is the same as
// 'MyType':
//..
ASSERT(true == (bsl::is_same<bsl::remove_const<MyConstType>::type,
MyType>::value));
//..
} break;
case 1: {
// --------------------------------------------------------------------
// 'bsl::remove_const::type'
// Ensure that the 'typedef' 'type' of 'bsl::remove_const' has the
// correct type for a variety of template parameter types.
//
// Concerns:
//: 1 'bsl::remove_const' leaves types that are not 'const'-qualified
//: at the top-level as-is.
//:
//: 2 'bsl::remove_const' remove any top-level 'const'-qualifier.
//
// Plan:
// Verify that 'bsl::remove_const::type' has the correct type for
// each concern.
//
// Testing:
// bsl::remove_const::type
// --------------------------------------------------------------------
if (verbose) printf("\n'bsl::remove_const::type'\n"
"\n=========================\n");
// C-1
ASSERT((is_same<remove_const<int>::type, int>::value));
ASSERT((is_same<remove_const<int *>::type, int *>::value));
ASSERT((is_same<remove_const<TestType>::type, TestType>::value));
ASSERT((is_same<remove_const<int const *>::type,
int const *>::value));
// C-2
ASSERT((is_same<remove_const<int const>::type, int>::value));
ASSERT((is_same<remove_const<int * const>::type, int *>::value));
ASSERT((is_same<remove_const<TestType const>::type, TestType>::value));
ASSERT((is_same<remove_const<int const volatile>::type,
int volatile>::value));
ASSERT((is_same<remove_const<int * const volatile>::type,
int * volatile>::value));
ASSERT((is_same<remove_const<TestType const volatile>::type,
TestType volatile>::value));
} break;
default: {
fprintf(stderr, "WARNING: CASE `%d' NOT FOUND.\n", test);
testStatus = -1;
}
}
if (testStatus > 0) {
fprintf(stderr, "Error, non-zero test status = %d.\n", testStatus);
}
return testStatus;
}
int main(int argc, char *argv[])
{
int test = argc > 1 ? atoi(argv[1]) : 0;
int verbose = argc > 2;
int veryVerbose = argc > 3;
(void) verbose;
(void) veryVerbose;
printf("TEST " __FILE__ " CASE %d\n", test);
switch (test) { case 0:
case 2: {
// --------------------------------------------------------------------
// USAGE EXAMPLE
//
// Concerns:
//: 1 The usage example provided in the component header file compiles,
//: links, and runs as shown.
//
// Plan:
//: 1 Incorporate usage example from header into test driver, remove
//: leading comment characters, and replace 'assert' with 'ASSERT'.
//: (C-1)
//
// Testing:
// USAGE EXAMPLE
// --------------------------------------------------------------------
if (verbose) printf("\nUSAGE EXAMPLE\n"
"\n=============\n");
///Usage
///-----
// In this section we show intended use of this component.
//
///Example 1: Transform Type to Pointer Type to that Type
/// - - - - - - - - - - - - - - - - - - - - - - - - - - -
// Suppose that we want to transform a type to a pointer type to that type.
//
// First, we create two 'typedef's -- a pointer type ('MyPtrType')
// and the type pointed to by the pointer type ('MyType'):
//..
typedef int MyType;
typedef int * MyPtrType;
//..
// Now, we transform 'MyType' to a pointer type to 'MyType' using
// 'bsl::add_pointer' and verify that the resulting type is the same as
// 'MyPtrType':
//..
ASSERT((bsl::is_same<bsl::add_pointer<MyType>::type,
MyPtrType>::value));
//..
} break;
case 1: {
// --------------------------------------------------------------------
// 'bsl::add_pointer::type'
// Ensure that the 'typedef' 'type' of 'bsl::add_pointer' has the
// correct type for a variety of template parameter types.
//
// Concerns:
//: 1 'bsl::add_pointer' transforms a non-reference type to a pointer
//: type pointing to the original type.
//:
//: 2 'bsl::add_pointer' transforms a reference type to a pointer
//: type pointing to the type referred to by the reference type.
//
// Plan:
// Verify that 'bsl::add_pointer::type' has the correct type for
// each concern.
//
// Testing:
// bsl::remove_pointer::type
// --------------------------------------------------------------------
if (verbose) printf("\nbsl::add_pointer::type\n"
"\n======================\n");
// C-1
ASSERT((is_same<add_pointer<int>::type, int *>::value));
ASSERT((is_same<add_pointer<int const>::type, int const *>::value));
ASSERT((is_same<add_pointer<int *>::type, int **>::value));
ASSERT((is_same<add_pointer<TestType>::type, TestType *>::value));
// C-2
ASSERT((is_same<add_pointer<int &>::type, int *>::value));
ASSERT((is_same<add_pointer<int const &>::type, int const *>::value));
ASSERT((is_same<add_pointer<TestType &>::type, TestType *>::value));
#if defined(BSLS_COMPILERFEATURES_SUPPORT_RVALUE_REFERENCES)
ASSERT((is_same<add_pointer<int &&>::type, int *>::value));
ASSERT((is_same<add_pointer<int const &&>::type, int const *>::value));
#endif
} break;
default: {
fprintf(stderr, "WARNING: CASE `%d' NOT FOUND.\n", test);
testStatus = -1;
}
}
if (testStatus > 0) {
fprintf(stderr, "Error, non-zero test status = %d.\n", testStatus);
}
return testStatus;
}
int main(int argc, char *argv[])
{
int test = argc > 1 ? atoi(argv[1]) : 0;
int verbose = argc > 2;
int veryVerbose = argc > 3;
printf("TEST " __FILE__ " CASE %d\n", test);
switch (test) { case 0:
case 2: {
// --------------------------------------------------------------------
// USAGE EXAMPLE
//
// Concerns:
//: 1 The usage example provided in the component header file compiles,
//: links, and runs as shown.
//
// Plan:
//: 1 Incorporate usage example from header into test driver, remove
//: leading comment characters, and replace 'assert' with 'ASSERT'.
//: (C-1)
//
// Testing:
// USAGE EXAMPLE
// --------------------------------------------------------------------
if (verbose) printf("\nUSAGE EXAMPLE\n"
"\n=============\n");
///Usage
///-----
// In this section we show intended use of this component.
//
///Example 1: Verify 'Const' Types
///- - - - - - - - - - - - - - - -
// Suppose that we want to assert whether a particular type is a
// 'const'-qualified.
//
// First, we create two 'typedef's -- a 'const'-qualified type and a
// unqualified type:
//..
typedef int MyType;
typedef const int MyConstType;
//..
// Now, we instantiate the 'bsl::is_const' template for each of the
// 'typedef's and assert the 'value' static data member of each instantiation:
//..
ASSERT(false == bsl::is_const<MyType>::value);
ASSERT(true == bsl::is_const<MyConstType>::value);
//..
} break;
case 1: {
// --------------------------------------------------------------------
// 'bsl::is_const::value'
// Ensure that the static data member 'value' of 'bsl::is_const'
// instantiations having various (template parameter) 'TYPES' has the
// correct value.
//
// Concerns:
//: 1 'is_const::value' is 'false' when 'TYPE' is a (possibly
//: 'volatile'-qualified) type.
//:
//: 2 'is_const::value' is 'true' when 'TYPE' is a 'const'-qualified or
//: cv-qualified type.
//
// Plan:
// Verify that 'bsl::is_const::value' has the correct value for
// each (template parameter) 'TYPE' in the concerns.
//
// Testing:
// bsl::is_const::value
// --------------------------------------------------------------------
if (verbose) printf("\nbsl::is_const::value\n"
"\n====================\n");
// C-1
ASSERT(false == is_const<int>::value);
ASSERT(false == is_const<int volatile>::value);
ASSERT(false == is_const<TestType>::value);
ASSERT(false == is_const<TestType volatile>::value);
// C-2
ASSERT(true == is_const<int const>::value);
ASSERT(true == is_const<int const volatile>::value);
ASSERT(true == is_const<TestType const>::value);
ASSERT(true == is_const<TestType const volatile>::value);
} break;
default: {
fprintf(stderr, "WARNING: CASE `%d' NOT FOUND.\n", test);
testStatus = -1;
}
}
if (testStatus > 0) {
fprintf(stderr, "Error, non-zero test status = %d.\n", testStatus);
}
return testStatus;
}
void myMessageOutput(QtMsgType type, const char* msg){
fprintf(stdout, "%s\n", msg);
fflush(stdout);
}