Esempio n. 1
0
int main(int argc, char* argv[]) {
  google::InitGoogleLogging(argv[0]);
  gflags::ParseCommandLineFlags(&argc, &argv, true);

  // Detect the hardware concurrency level.
  const std::size_t num_hw_threads =
      DefaultsConfigurator::GetNumHardwareThreads();

  // Use the command-line value if that was supplied, else use the value
  // that we computed above, provided it did return a valid value.
  // TODO(jmp): May need to change this at some point to keep one thread
  //            available for the OS if the hardware concurrency level is high.
  const unsigned int real_num_workers = quickstep::FLAGS_num_workers != 0
                                      ? quickstep::FLAGS_num_workers
                                      : (num_hw_threads != 0 ?
                                         num_hw_threads
                                         : 1);

  if (real_num_workers > 0) {
    printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool\n",
           real_num_workers,
           (static_cast<double>(quickstep::FLAGS_buffer_pool_slots) * quickstep::kSlotSizeBytes)/quickstep::kAGigaByte);
  } else {
    LOG(FATAL) << "Quickstep needs at least one worker thread to run";
  }

#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
  if (quickstep::FLAGS_use_hdfs) {
    LOG(INFO) << "Using HDFS as the default persistent storage, with namenode at "
              << quickstep::FLAGS_hdfs_namenode_host << ":"
              << quickstep::FLAGS_hdfs_namenode_port << " and block replication factor "
              << quickstep::FLAGS_hdfs_num_replications << "\n";
  }
#endif

  // Initialize the thread ID based map here before the Foreman and workers are
  // constructed because the initialization isn't thread safe.
  quickstep::ClientIDMap::Instance();

  MessageBusImpl bus;
  bus.Initialize();

  // The TMB client id for the main thread, used to kill workers at the end.
  const client_id main_thread_client_id = bus.Connect();
  bus.RegisterClientAsSender(main_thread_client_id, kPoisonMessage);

  // Setup the paths used by StorageManager.
  string fixed_storage_path(quickstep::FLAGS_storage_path);
  if (!fixed_storage_path.empty()
      && (fixed_storage_path.back() != quickstep::kPathSeparator)) {
    fixed_storage_path.push_back(quickstep::kPathSeparator);
  }

  string catalog_path(fixed_storage_path);
  catalog_path.append("catalog.pb.bin");
  if (quickstep::FLAGS_initialize_db) {  // Initialize the database
    // TODO(jmp): Refactor the code in this file!
    LOG(INFO) << "Initializing the database, creating a new catalog file and storage directory\n";

    // Create the directory
    // TODO(jmp): At some point, likely in C++-17, we will just have the
    //            filesystem path, and we can clean this up
#ifdef QUICKSTEP_OS_WINDOWS
    std::filesystem::create_directories(fixed_storage_path);
    LOG(FATAL) << "Failed when attempting to create the directory: " << fixed_storage_path << "\n";
    LOG(FATAL) << "Check if the directory already exists. If so, delete it or move it before initializing \n";
#else
    {
      string path_name = "mkdir " + fixed_storage_path;
      if (std::system(path_name.c_str())) {
        LOG(FATAL) << "Failed when attempting to create the directory: " << fixed_storage_path << "\n";
      }
    }
#endif

    // Create the default catalog file.
    std::ofstream catalog_file(catalog_path);
    if (!catalog_file.good()) {
      LOG(FATAL) << "ERROR: Unable to open catalog.pb.bin for writing.\n";
    }

    quickstep::Catalog catalog;
    catalog.addDatabase(new quickstep::CatalogDatabase(nullptr, "default"));

    if (!catalog.getProto().SerializeToOstream(&catalog_file)) {
      LOG(FATAL) << "ERROR: Unable to serialize catalog proto to file catalog.pb.bin\n";
      return 1;
    }

    // Close the catalog file - it will be reopened below by the QueryProcessor.
    catalog_file.close();
  }

  // Setup QueryProcessor, including CatalogDatabase and StorageManager.
  std::unique_ptr<QueryProcessor> query_processor;
  try {
    query_processor.reset(new QueryProcessor(catalog_path, fixed_storage_path));
  } catch (const std::exception &e) {
    LOG(FATAL) << "FATAL ERROR DURING STARTUP: "
               << e.what()
               << "\nIf you intended to create a new database, "
               << "please use the \"-initialize_db=true\" command line option.";
  } catch (...) {
    LOG(FATAL) << "NON-STANDARD EXCEPTION DURING STARTUP";
  }

  // Parse the CPU affinities for workers and the preloader thread, if enabled
  // to warm up the buffer pool.
  const vector<int> worker_cpu_affinities =
      InputParserUtil::ParseWorkerAffinities(real_num_workers,
                                             quickstep::FLAGS_worker_affinities);

  const std::size_t num_numa_nodes_covered =
      DefaultsConfigurator::GetNumNUMANodesCoveredByWorkers(worker_cpu_affinities);

  if (quickstep::FLAGS_preload_buffer_pool) {
    std::chrono::time_point<std::chrono::steady_clock> preload_start, preload_end;
    preload_start = std::chrono::steady_clock::now();
    printf("Preloading the buffer pool ... ");
    fflush(stdout);
    quickstep::PreloaderThread preloader(*query_processor->getDefaultDatabase(),
                                         query_processor->getStorageManager(),
                                         worker_cpu_affinities.front());

    preloader.start();
    preloader.join();
    preload_end = std::chrono::steady_clock::now();
    printf("in %g seconds\n",
           std::chrono::duration<double>(preload_end - preload_start).count());
  }

  Foreman foreman(&bus,
                  query_processor->getDefaultDatabase(),
                  query_processor->getStorageManager(),
                  num_numa_nodes_covered);

  // Get the NUMA affinities for workers.
  vector<int> cpu_numa_nodes = InputParserUtil::GetNUMANodesForCPUs();
  if (cpu_numa_nodes.empty()) {
    // libnuma is not present. Assign -1 as the NUMA node for every worker.
    cpu_numa_nodes.assign(worker_cpu_affinities.size(), -1);
  }

  vector<int> worker_numa_nodes;
  PtrVector<Worker> workers;
  vector<client_id> worker_client_ids;

  // Initialize the worker threads.
  DCHECK_EQ(static_cast<std::size_t>(real_num_workers),
            worker_cpu_affinities.size());
  for (std::size_t worker_thread_index = 0;
       worker_thread_index < worker_cpu_affinities.size();
       ++worker_thread_index) {
    int numa_node_id = -1;
    if (worker_cpu_affinities[worker_thread_index] >= 0) {
      // This worker can be NUMA affinitized.
      numa_node_id = cpu_numa_nodes[worker_cpu_affinities[worker_thread_index]];
    }
    worker_numa_nodes.push_back(numa_node_id);

    workers.push_back(
        new Worker(worker_thread_index, &bus, worker_cpu_affinities[worker_thread_index]));
    worker_client_ids.push_back(workers.back().getBusClientID());
  }

  // TODO(zuyu): Move WorkerDirectory within Shiftboss once the latter is added.
  WorkerDirectory worker_directory(worker_cpu_affinities.size(),
                                   worker_client_ids,
                                   worker_numa_nodes);

  foreman.setWorkerDirectory(&worker_directory);

  // Start the worker threads.
  for (Worker &worker : workers) {
    worker.start();
  }

  LineReaderImpl line_reader("quickstep> ",
                             "      ...> ");
  std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper());
  std::chrono::time_point<std::chrono::steady_clock> start, end;

  for (;;) {
    string *command_string = new string();
    *command_string = line_reader.getNextCommand();
    if (command_string->size() == 0) {
      delete command_string;
      break;
    }

    parser_wrapper->feedNextBuffer(command_string);

    bool quitting = false;
    // A parse error should reset the parser. This is because the thrown quickstep
    // SqlError does not do the proper reset work of the YYABORT macro.
    bool reset_parser = false;
    for (;;) {
      ParseResult result = parser_wrapper->getNextStatement();
      if (result.condition == ParseResult::kSuccess) {
        if (result.parsed_statement->getStatementType() == ParseStatement::kQuit) {
          quitting = true;
          break;
        }

        if (result.parsed_statement->getStatementType() == ParseStatement::kCommand) {
          try {
            quickstep::cli::executeCommand(
                *result.parsed_statement,
                *(query_processor->getDefaultDatabase()), stdout);
          } catch (const quickstep::SqlError &sql_error) {
            fprintf(stderr, "%s",
                    sql_error.formatMessage(*command_string).c_str());
            reset_parser = true;
            break;
          }
        continue;
        }

        std::unique_ptr<QueryHandle> query_handle;
        try {
          query_handle.reset(query_processor->generateQueryHandle(*result.parsed_statement));
        } catch (const quickstep::SqlError &sql_error) {
          fprintf(stderr, "%s", sql_error.formatMessage(*command_string).c_str());
          reset_parser = true;
          break;
        }

        DCHECK(query_handle->getQueryPlanMutable() != nullptr);
        foreman.setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());

        foreman.reconstructQueryContextFromProto(query_handle->getQueryContextProto());

        try {
          start = std::chrono::steady_clock::now();
          foreman.start();
          foreman.join();
          end = std::chrono::steady_clock::now();

          const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
          if (query_result_relation) {
            PrintToScreen::PrintRelation(*query_result_relation,
                                         query_processor->getStorageManager(),
                                         stdout);

            DropRelation::Drop(*query_result_relation,
                               query_processor->getDefaultDatabase(),
                               query_processor->getStorageManager());
          }

          query_processor->saveCatalog();
          printf("Execution time: %g seconds\n",
                 std::chrono::duration<double>(end - start).count());
        } catch (const std::exception &e) {
          fprintf(stderr, "QUERY EXECUTION ERROR: %s\n", e.what());
          break;
        }
        printf("Query Complete\n");
      } else {
        if (result.condition == ParseResult::kError) {
          fprintf(stderr, "%s", result.error_message.c_str());
        }
        reset_parser = true;
        break;
      }
    }

    if (quitting) {
      break;
    } else if (reset_parser) {
      parser_wrapper.reset(new SqlParserWrapper());
      reset_parser = false;
    }
  }

  // Terminate all workers before exiting.
  // The main thread broadcasts poison message to the workers. Each worker dies
  // after receiving poison message. The order of workers' death is irrelavant.
  MessageStyle style;
  style.Broadcast(true);
  Address address;
  address.All(true);
  std::unique_ptr<WorkerMessage> poison_message(WorkerMessage::PoisonMessage());
  TaggedMessage poison_tagged_message(poison_message.get(),
                                      sizeof(*poison_message),
                                      kPoisonMessage);

  const tmb::MessageBus::SendStatus send_status =
      bus.Send(main_thread_client_id,
               address,
               style,
               std::move(poison_tagged_message));
  CHECK(send_status == tmb::MessageBus::SendStatus::kOK) <<
     "Broadcast message from Foreman to workers failed";

  for (Worker &worker : workers) {
    worker.join();
  }

  return 0;
}
Esempio n. 2
0
int main(int argc, char *argv[])
{
    int test = argc > 1 ? atoi(argv[1]) : 0;
    int verbose = argc > 2;
    int veryVerbose = argc > 3;

    printf("TEST " __FILE__ " CASE %d\n", test);

    switch (test) { case 0:
      case 2: {
        // --------------------------------------------------------------------
        // USAGE EXAMPLE
        //
        // Concerns:
        //: 1 The usage example provided in the component header file compiles,
        //:   links, and runs as shown.
        //
        // Plan:
        //: 1 Incorporate usage example from header into test driver, remove
        //:   leading comment characters, and replace 'assert' with 'ASSERT'.
        //:   (C-1)
        //
        // Testing:
        //   USAGE EXAMPLE
        // --------------------------------------------------------------------

        if (verbose) printf("\nUSAGE EXAMPLE\n"
                            "\n=============\n");

///Usage
///-----
// In this section we show intended use of this component.
//
///Example 1: Removing The 'const'-qualifier of A Type
///- - - - - - - - - - - - - - - - - - - - - - - - - -
// Suppose that we want to remove any top-level 'const'-qualifier from a
// particular type.
//
// First, we create two 'typedef's -- a 'const'-qualified type ('MyConstType')
// and the same type without the 'const'-qualifier ('MyType'):
//..
        typedef int       MyType;
        typedef const int MyConstType;
//..
// Now, we remove the 'const'-qualifier from 'MyConstType' using
// 'bsl::remove_const' and verify that the resulting type is the same as
// 'MyType':
//..
        ASSERT(true == (bsl::is_same<bsl::remove_const<MyConstType>::type,
                                                              MyType>::value));
//..

      } break;
      case 1: {
        // --------------------------------------------------------------------
        // 'bsl::remove_const::type'
        //   Ensure that the 'typedef' 'type' of 'bsl::remove_const' has the
        //   correct type for a variety of template parameter types.
        //
        // Concerns:
        //: 1 'bsl::remove_const' leaves types that are not 'const'-qualified
        //:   at the top-level as-is.
        //:
        //: 2 'bsl::remove_const' remove any top-level 'const'-qualifier.
        //
        // Plan:
        //   Verify that 'bsl::remove_const::type' has the correct type for
        //   each concern.
        //
        // Testing:
        //   bsl::remove_const::type
        // --------------------------------------------------------------------

        if (verbose) printf("\n'bsl::remove_const::type'\n"
                            "\n=========================\n");

        // C-1
        ASSERT((is_same<remove_const<int>::type, int>::value));
        ASSERT((is_same<remove_const<int *>::type, int *>::value));
        ASSERT((is_same<remove_const<TestType>::type, TestType>::value));
        ASSERT((is_same<remove_const<int const *>::type,
                                                         int const *>::value));

        // C-2
        ASSERT((is_same<remove_const<int const>::type, int>::value));
        ASSERT((is_same<remove_const<int * const>::type, int *>::value));
        ASSERT((is_same<remove_const<TestType const>::type, TestType>::value));

        ASSERT((is_same<remove_const<int const volatile>::type,
                                                        int volatile>::value));
        ASSERT((is_same<remove_const<int * const volatile>::type,
                                                      int * volatile>::value));
        ASSERT((is_same<remove_const<TestType const volatile>::type,
                                                   TestType volatile>::value));

      } break;
      default: {

        fprintf(stderr, "WARNING: CASE `%d' NOT FOUND.\n", test);
        testStatus = -1;
      }
    }

    if (testStatus > 0) {
        fprintf(stderr, "Error, non-zero test status = %d.\n", testStatus);
    }

    return testStatus;
}
Esempio n. 3
0
int main(int argc, char *argv[])
{
    int test = argc > 1 ? atoi(argv[1]) : 0;
    int verbose = argc > 2;
    int veryVerbose = argc > 3;

    (void) verbose;
    (void) veryVerbose;

    printf("TEST " __FILE__ " CASE %d\n", test);

    switch (test) { case 0:
      case 2: {
        // --------------------------------------------------------------------
        // USAGE EXAMPLE
        //
        // Concerns:
        //: 1 The usage example provided in the component header file compiles,
        //:   links, and runs as shown.
        //
        // Plan:
        //: 1 Incorporate usage example from header into test driver, remove
        //:   leading comment characters, and replace 'assert' with 'ASSERT'.
        //:   (C-1)
        //
        // Testing:
        //   USAGE EXAMPLE
        // --------------------------------------------------------------------

        if (verbose) printf("\nUSAGE EXAMPLE\n"
                            "\n=============\n");

///Usage
///-----
// In this section we show intended use of this component.
//
///Example 1: Transform Type to Pointer Type to that Type
/// - - - - - - - - - - - - - - - - - - - - - - - - - - -
// Suppose that we want to transform a type to a pointer type to that type.
//
// First, we create two 'typedef's -- a pointer type ('MyPtrType')
// and the type pointed to by the pointer type ('MyType'):
//..
        typedef int   MyType;
        typedef int * MyPtrType;
//..
// Now, we transform 'MyType' to a pointer type to 'MyType' using
// 'bsl::add_pointer' and verify that the resulting type is the same as
// 'MyPtrType':
//..
        ASSERT((bsl::is_same<bsl::add_pointer<MyType>::type,
                             MyPtrType>::value));
//..

      } break;
      case 1: {
        // --------------------------------------------------------------------
        // 'bsl::add_pointer::type'
        //   Ensure that the 'typedef' 'type' of 'bsl::add_pointer' has the
        //   correct type for a variety of template parameter types.
        //
        // Concerns:
        //: 1 'bsl::add_pointer' transforms a non-reference type to a pointer
        //:   type pointing to the original type.
        //:
        //: 2 'bsl::add_pointer' transforms a reference type to a pointer
        //:   type pointing to the type referred to by the reference type.
        //
        // Plan:
        //   Verify that 'bsl::add_pointer::type' has the correct type for
        //   each concern.
        //
        // Testing:
        //   bsl::remove_pointer::type
        // --------------------------------------------------------------------

        if (verbose) printf("\nbsl::add_pointer::type\n"
                            "\n======================\n");

        // C-1
        ASSERT((is_same<add_pointer<int>::type, int *>::value));
        ASSERT((is_same<add_pointer<int const>::type, int const *>::value));
        ASSERT((is_same<add_pointer<int *>::type, int **>::value));
        ASSERT((is_same<add_pointer<TestType>::type, TestType *>::value));

        // C-2
        ASSERT((is_same<add_pointer<int &>::type, int *>::value));
        ASSERT((is_same<add_pointer<int const &>::type, int const *>::value));
        ASSERT((is_same<add_pointer<TestType &>::type, TestType *>::value));

#if defined(BSLS_COMPILERFEATURES_SUPPORT_RVALUE_REFERENCES)
        ASSERT((is_same<add_pointer<int &&>::type, int *>::value));
        ASSERT((is_same<add_pointer<int const &&>::type, int const *>::value));
#endif
      } break;
      default: {
        fprintf(stderr, "WARNING: CASE `%d' NOT FOUND.\n", test);
        testStatus = -1;
      }
    }

    if (testStatus > 0) {
        fprintf(stderr, "Error, non-zero test status = %d.\n", testStatus);
    }

    return testStatus;
}
Esempio n. 4
0
int main(int argc, char *argv[])
{
    int test = argc > 1 ? atoi(argv[1]) : 0;
    int verbose = argc > 2;
    int veryVerbose = argc > 3;

    printf("TEST " __FILE__ " CASE %d\n", test);

    switch (test) { case 0:
      case 2: {
        // --------------------------------------------------------------------
        // USAGE EXAMPLE
        //
        // Concerns:
        //: 1 The usage example provided in the component header file compiles,
        //:   links, and runs as shown.
        //
        // Plan:
        //: 1 Incorporate usage example from header into test driver, remove
        //:   leading comment characters, and replace 'assert' with 'ASSERT'.
        //:   (C-1)
        //
        // Testing:
        //   USAGE EXAMPLE
        // --------------------------------------------------------------------

        if (verbose) printf("\nUSAGE EXAMPLE\n"
                            "\n=============\n");

///Usage
///-----
// In this section we show intended use of this component.
//
///Example 1: Verify 'Const' Types
///- - - - - - - - - - - - - - - -
// Suppose that we want to assert whether a particular type is a
// 'const'-qualified.
//
// First, we create two 'typedef's -- a 'const'-qualified type and a
// unqualified type:
//..
        typedef int        MyType;
        typedef const int  MyConstType;
//..
// Now, we instantiate the 'bsl::is_const' template for each of the
// 'typedef's and assert the 'value' static data member of each instantiation:
//..
        ASSERT(false == bsl::is_const<MyType>::value);
        ASSERT(true == bsl::is_const<MyConstType>::value);
//..

      } break;
      case 1: {
        // --------------------------------------------------------------------
        // 'bsl::is_const::value'
        //   Ensure that the static data member 'value' of 'bsl::is_const'
        //   instantiations having various (template parameter) 'TYPES' has the
        //   correct value.
        //
        // Concerns:
        //: 1 'is_const::value' is 'false' when 'TYPE' is a (possibly
        //:   'volatile'-qualified) type.
        //:
        //: 2 'is_const::value' is 'true' when 'TYPE' is a 'const'-qualified or
        //:    cv-qualified type.
        //
        // Plan:
        //   Verify that 'bsl::is_const::value' has the correct value for
        //   each (template parameter) 'TYPE' in the concerns.
        //
        // Testing:
        //   bsl::is_const::value
        // --------------------------------------------------------------------

        if (verbose) printf("\nbsl::is_const::value\n"
                            "\n====================\n");

        // C-1
        ASSERT(false == is_const<int>::value);
        ASSERT(false == is_const<int volatile>::value);

        ASSERT(false == is_const<TestType>::value);
        ASSERT(false == is_const<TestType volatile>::value);

        // C-2
        ASSERT(true == is_const<int const>::value);
        ASSERT(true == is_const<int const volatile>::value);

        ASSERT(true == is_const<TestType const>::value);
        ASSERT(true == is_const<TestType const volatile>::value);
      } break;
      default: {
        fprintf(stderr, "WARNING: CASE `%d' NOT FOUND.\n", test);
        testStatus = -1;
      }
    }

    if (testStatus > 0) {
        fprintf(stderr, "Error, non-zero test status = %d.\n", testStatus);
    }

    return testStatus;
}
void myMessageOutput(QtMsgType type, const char* msg){
               fprintf(stdout, "%s\n", msg);
               fflush(stdout);
}