Пример #1
0
static void ParseProgramOptions (int argc, char* argv[]) {
    ProgramOptionsDescription description("STANDARD options");

    description
    ("async", &Async, "send asynchronous requests")
    ("concurrency", &ThreadConcurrency, "number of parallel connections")
    ("requests", &Operations, "total number of operations")
    ("batch-size", &BatchSize, "number of operations in one batch (0 disables batching)")
    ("keep-alive", &KeepAlive, "use HTTP keep-alive")
    ("collection", &Collection, "collection name to use in tests")
    ("test-case", &TestCase, "test case to use (possible values: version, document, collection, import-document, hash, skiplist, edge, shapes, shapes-append, random-shapes, crud, crud-append, crud-write-read, aqltrx, counttrx, multitrx, multi-collection, aqlinsert, aqlv8)")
    ("complexity", &Complexity, "complexity parameter for the test")
    ("delay", &Delay, "use a startup delay (necessary only when run in series)")
    ("progress", &Progress, "show progress")
    ("verbose", &verbose, "print out replies if the http-header indicates db-errors")
    ;

    BaseClient.setupGeneral(description);
    BaseClient.setupServer(description);

    vector<string> arguments;
    description.arguments(&arguments);

    ProgramOptions options;
    BaseClient.parse(options, description, "--concurrency <concurrency> --requests <request> --test-case <case> ...", argc, argv, "arangob.conf");
}
Пример #2
0
static void ParseProgramOptions (int argc, char* argv[]) {
  ProgramOptionsDescription description("STANDARD options");

  description
    ("async", &Async, "send asychronous requests")
    ("concurrency", &Concurrency, "number of parallel connections")
    ("requests", &Operations, "total number of operations")
    ("batch-size", &BatchSize, "number of operations in one batch (0 disables batching)")
    ("keep-alive", &KeepAlive, "use HTTP keep-alive")
    ("collection", &Collection, "collection name to use in tests")
    ("test-case", &TestCase, "test case to use")
    ("complexity", &Complexity, "complexity parameter for the test")
    ("delay", &Delay, "use a startup delay (necessary only when run in series)")
    ("progress", &Progress, "show progress")
  ;

  BaseClient.setupGeneral(description);
  BaseClient.setupServer(description);

  vector<string> arguments;
  description.arguments(&arguments);

  ProgramOptions options;
  BaseClient.parse(options, description, "--concurrency <concurrency> --requests <request> --test-case <case> ...", argc, argv, "arangob.conf");
}
Пример #3
0
static void ParseProgramOptions (int argc, char* argv[]) {
  ProgramOptionsDescription description("STANDARD options");

  description
    ("collection", &Collections, "restrict to collection name (can be specified multiple times)")
    ("create-database", &CreateDatabase, "create the target database if it does not exist")
    ("batch-size", &ChunkSize, "maximum size for individual data batches (in bytes)")
    ("import-data", &ImportData, "import data into collection")
    ("recycle-ids", &RecycleIds, "recycle collection and revision ids from dump")
    ("force", &Force, "continue restore even in the face of some server-side errors")
    ("create-collection", &ImportStructure, "create collection structure")
    ("include-system-collections", &IncludeSystemCollections, "include system collections")
    ("input-directory", &InputDirectory, "input directory")
    ("overwrite", &Overwrite, "overwrite collections if they exist")
    ("progress", &Progress, "show progress")
  ;

  BaseClient.setupGeneral(description);
  BaseClient.setupServer(description);

  vector<string> arguments;
  description.arguments(&arguments);

  ProgramOptions options;
  BaseClient.parse(options, description, "", argc, argv, "arangorestore.conf");

  if (1 == arguments.size()) {
    InputDirectory = arguments[0];
  }
}
Пример #4
0
static void ParseProgramOptions (int argc, char* argv[]) {
  ProgramOptionsDescription description("STANDARD options");

  description
    ("collection", &Collections, "restrict to collection name (can be specified multiple times)")
    ("batch-size", &ChunkSize, "maximum size for individual data batches (in bytes)")
    ("dump-data", &DumpData, "dump collection data")
    ("include-system-collections", &IncludeSystemCollections, "include system collections")
    ("output-directory", &OutputDirectory, "output directory")
    ("overwrite", &Overwrite, "overwrite data in output directory")
    ("progress", &Progress, "show progress")
    ("tick-start", &TickStart, "only include data after this tick")
    ("tick-end", &TickEnd, "last tick to be included in data dump")
  ;

  BaseClient.setupGeneral(description);
  BaseClient.setupServer(description);

  vector<string> arguments;
  description.arguments(&arguments);

  ProgramOptions options;
  BaseClient.parse(options, description, argc, argv, "arangodump.conf");
  
  if (1 == arguments.size()) {
    OutputDirectory = arguments[0];
  }
}
Пример #5
0
static void ParseProgramOptions (int argc, char* argv[]) {
  ProgramOptionsDescription description("STANDARD options");
  ProgramOptionsDescription ruby("RUBY options");

  ruby
    ("ruby.directory", &StartupPath, "startup paths containing the Ruby files; multiple directories can be separated by cola")
    ("ruby.modules-path", &StartupModules, "one or more directories separated by cola")
  ;

  description
    (ruby, false)
  ;

  // fill in used options
  BaseClient.setupGeneral(description);
  BaseClient.setupServer(description);

  // and parse the command line and config file
  ProgramOptions options;
  BaseClient.parse(options, description, argc, argv, "arangoirb.conf");

  // check module path
  if (StartupModules.empty()) {
    LOGGER_FATAL_AND_EXIT("module path not known, please use '--ruby.modules-path'");
  }
}
Пример #6
0
static void ParseProgramOptions (int argc, char* argv[]) {
  ProgramOptionsDescription deprecatedOptions("DEPRECATED options");

  deprecatedOptions
    ("max-upload-size", &ChunkSize, "size for individual data batches (in bytes)")
  ;

  ProgramOptionsDescription description("STANDARD options");

  description
    ("file", &FileName, "file name (\"-\" for STDIN)")
    ("batch-size", &ChunkSize, "size for individual data batches (in bytes)")
    ("collection", &CollectionName, "collection name")
    ("create-collection", &CreateCollection, "create collection if it does not yet exist")
    ("type", &TypeImport, "type of file (\"csv\", \"tsv\", or \"json\")")
    ("quote", &Quote, "quote character(s)")
    ("separator", &Separator, "separator")
    ("progress", &Progress, "show progress")
    (deprecatedOptions, true)
  ;

  BaseClient.setupGeneral(description);
  BaseClient.setupServer(description);

  vector<string> arguments;
  description.arguments(&arguments);

  ProgramOptions options;
  BaseClient.parse(options, description, "--file <file> --type <type> --collection <collection>", argc, argv, "arangoimp.conf");

  if (FileName == "" && arguments.size() > 0) {
    FileName = arguments[0];
  }
}
Пример #7
0
static MRubyClientConnection* createConnection (mrb_state* mrb) {
  return new MRubyClientConnection(mrb,
                                   BaseClient.endpointServer(),
                                   BaseClient.username(),
                                   BaseClient.password(),
                                   BaseClient.requestTimeout(),
                                   BaseClient.connectTimeout(),
                                   ArangoClient::DEFAULT_RETRIES,
                                   false);
}
Пример #8
0
static string rewriteLocation (void* data, const string& location) {
  if (location.substr(0, 5) == "/_db/") {
    // location already contains /_db/
    return location;
  }

  if (location[0] == '/') {
    return "/_db/" + BaseClient.databaseName() + location;
  }
  else {
    return "/_db/" + BaseClient.databaseName() + "/" + location;
  }
}
Пример #9
0
static int TryCreateDatabase (std::string const& name) {

  triagens::basics::Json json(triagens::basics::Json::Object);
  json("name", triagens::basics::Json(name));

  triagens::basics::Json user(triagens::basics::Json::Object);
  user("username", triagens::basics::Json(BaseClient.username()));
  user("passwd", triagens::basics::Json(BaseClient.password()));

  triagens::basics::Json users(triagens::basics::Json::Array);
  users.add(user);
  json("users", users);
                                               
  std::string const body(triagens::basics::JsonHelper::toString(json.json()));

  std::unique_ptr<SimpleHttpResult> response(Client->request(HttpRequest::HTTP_REQUEST_POST,
                                               "/_api/database",
                                               body.c_str(),
                                               body.size()));

  if (response == nullptr || ! response->isComplete()) {
    return TRI_ERROR_INTERNAL;
  }

  auto returnCode = response->getHttpReturnCode();

  if (returnCode == HttpResponse::OK ||
      returnCode == HttpResponse::CREATED) {
    // all ok
    return TRI_ERROR_NO_ERROR;
  }
  else if (returnCode == HttpResponse::UNAUTHORIZED ||
           returnCode == HttpResponse::FORBIDDEN) {
    // invalid authorization
    Client->setErrorMessage(GetHttpErrorMessage(response.get()), false);
    return TRI_ERROR_FORBIDDEN;
  }

  // any other error
  Client->setErrorMessage(GetHttpErrorMessage(response.get()), false);
  return TRI_ERROR_INTERNAL;
}
Пример #10
0
static void RunShell (mrb_state* mrb) {
  MRLineEditor console(mrb, ".arangoirb.history");

  console.open(false /*! NoAutoComplete*/);

  while (true) {
    char* input = console.prompt("arangoirb> ");

    if (input == 0) {
      break;
    }

    if (*input == '\0') {
      TRI_FreeString(TRI_CORE_MEM_ZONE, input);
      continue;
    }

    console.addHistory(input);

    struct mrb_parser_state* p = mrb_parse_nstring(mrb, input, strlen(input), NULL);
    TRI_FreeString(TRI_CORE_MEM_ZONE, input);

    if (p == 0 || p->tree == 0 || 0 < p->nerr) {
      cout << "UPPS!\n";
      continue;
    }

    int n = mrb_generate_code(mrb, p);

    if (n < 0) {
      cout << "UPPS: " << n << " returned by mrb_generate_code\n";
      continue;
    }

    mrb_value result = mrb_run(mrb,
                               mrb_proc_new(mrb, mrb->irep[n]),
                               mrb_top_self(mrb));

    if (mrb->exc) {
      cout << "Caught exception:\n";
      mrb_p(mrb, mrb_obj_value(mrb->exc));
      mrb->exc = 0;
    }
    else if (! mrb_nil_p(result)) {
      mrb_p(mrb, result);
    }
  }

  console.close();

  cout << endl;

  BaseClient.printByeBye();
}
Пример #11
0
int main (int argc, char* argv[]) {
  int ret = EXIT_SUCCESS;

  arangodumpEntryFunction();

  TRIAGENS_C_INITIALISE(argc, argv);
  TRIAGENS_REST_INITIALISE(argc, argv);

  TRI_InitialiseLogging(false);
  
  // .............................................................................
  // set defaults
  // .............................................................................

  int err = 0;
  OutputDirectory = FileUtils::currentDirectory(&err).append(TRI_DIR_SEPARATOR_STR).append("dump");
  BaseClient.setEndpointString(Endpoint::getDefaultEndpoint());

  // .............................................................................
  // parse the program options
  // .............................................................................

  ParseProgramOptions(argc, argv);

  // use a minimum value for batches
  if (ChunkSize < 1024 * 128) {
    ChunkSize = 1024 * 128;
  }

  if (TickStart < TickEnd) {
    cerr << "invalid values for --tick-start or --tick-end" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  // .............................................................................
  // create output directory
  // .............................................................................

  bool isDirectory = false;
  if (OutputDirectory != "") {
    isDirectory = TRI_IsDirectory(OutputDirectory.c_str());
  }

  if (OutputDirectory == "" ||
      (TRI_ExistsFile(OutputDirectory.c_str()) && ! isDirectory)) {
    cerr << "cannot write to output directory '" << OutputDirectory << "'" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  if (isDirectory && ! Overwrite) {
    cerr << "output directory '" << OutputDirectory << "' already exists. use --overwrite to overwrite data in in it" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  // .............................................................................
  // set-up client connection
  // .............................................................................

  BaseClient.createEndpoint();

  if (BaseClient.endpointServer() == 0) {
    cerr << "invalid value for --server.endpoint ('" << BaseClient.endpointString() << "')" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  Connection = GeneralClientConnection::factory(BaseClient.endpointServer(),
                                                BaseClient.requestTimeout(),
                                                BaseClient.connectTimeout(),
                                                ArangoClient::DEFAULT_RETRIES,
                                                BaseClient.sslProtocol());
  
  if (Connection == 0) {
    cerr << "out of memory" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }
  
  Client = new SimpleHttpClient(Connection, BaseClient.requestTimeout(), false);

  if (Client == 0) {
    cerr << "out of memory" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  Client->setLocationRewriter(0, &rewriteLocation);
  Client->setUserNamePassword("/", BaseClient.username(), BaseClient.password());

  const string versionString = GetArangoVersion();

  if (! Connection->isConnected()) {
    cerr << "Could not connect to endpoint '" << BaseClient.endpointString() 
         << "', database: '" << BaseClient.databaseName() << "', username: '******'" << endl;
    cerr << "Error message: '" << Client->getErrorMessage() << "'" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }
    
  // successfully connected

  // validate server version 
  int major = 0;
  int minor = 0;

  if (sscanf(versionString.c_str(), "%d.%d", &major, &minor) != 2) {
    cerr << "Invalid server version '" << versionString << "'" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  if (major != 1 ||
      (major == 1 && minor < 4)) {
    // we can connect to 1.4 and higher only
    cerr << "Got an incompatible server version '" << versionString << "'" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }
  
  
  if (! isDirectory) {
    int res = TRI_CreateDirectory(OutputDirectory.c_str());

    if (res != TRI_ERROR_NO_ERROR) {
      cerr << "unable to create output directory '" << OutputDirectory << "': " << string(TRI_errno_string(res)) << endl;
      TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
    }
  }


  if (Progress) {
    cout << "Connected to ArangoDB '" << BaseClient.endpointString() 
          << "', database: '" << BaseClient.databaseName() << "', username: '******'" << endl;
    
    cout << "Writing dump to output directory '" << OutputDirectory << "'" << endl;
  }
  
  memset(&Stats, 0, sizeof(Stats));

  string errorMsg = "";

  int res = StartBatch(errorMsg);

  if (res == TRI_ERROR_NO_ERROR) {
    res = RunDump(errorMsg);
  }
  
  if (BatchId > 0) {
    EndBatch();
  }
  
  if (Progress) {
    if (DumpData) {
      cout << "Processed " << Stats._totalCollections << " collection(s), " << 
              "wrote " << Stats._totalWritten << " byte(s) into datafiles, " << 
              "sent " << Stats._totalBatches << " batch(es)" << endl;
    }
    else {
      cout << "Processed " << Stats._totalCollections << " collection(s)" << endl;
    }
  }

  if (res != TRI_ERROR_NO_ERROR) {
    cerr << errorMsg << endl;
    ret = EXIT_FAILURE;
  }

  TRIAGENS_REST_SHUTDOWN;

  arangodumpExitFunction(ret, NULL);

  return ret;
}
Пример #12
0
int main (int argc, char* argv[]) {
  int ret = EXIT_SUCCESS;

  LocalEntryFunction();

  TRIAGENS_C_INITIALIZE(argc, argv);
  TRIAGENS_REST_INITIALIZE(argc, argv);

  TRI_InitializeLogging(false);

  // .............................................................................
  // set defaults
  // .............................................................................

  BaseClient.setEndpointString(Endpoint::getDefaultEndpoint());

  // .............................................................................
  // parse the program options
  // .............................................................................

  ParseProgramOptions(argc, argv);

  // use a minimum value for batches
  if (ChunkSize < 1024 * 128) {
    ChunkSize = 1024 * 128;
  }

  if (! InputDirectory.empty() &&
      InputDirectory.back() == TRI_DIR_SEPARATOR_CHAR) {
    // trim trailing slash from path because it may cause problems on ... Windows
    TRI_ASSERT(InputDirectory.size() > 0);
    InputDirectory.pop_back();
  }

  // .............................................................................
  // check input directory
  // .............................................................................

  if (InputDirectory == "" || ! TRI_IsDirectory(InputDirectory.c_str())) {
    cerr << "Error: input directory '" << InputDirectory << "' does not exist" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, nullptr);
  }

  if (! ImportStructure && ! ImportData) {
    cerr << "Error: must specify either --create-collection or --import-data" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, nullptr);
  }

  // .............................................................................
  // set-up client connection
  // .............................................................................

  BaseClient.createEndpoint();

  if (BaseClient.endpointServer() == nullptr) {
    cerr << "Error: invalid value for --server.endpoint ('" << BaseClient.endpointString() << "')" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, nullptr);
  }

  Connection = GeneralClientConnection::factory(BaseClient.endpointServer(),
                                                BaseClient.requestTimeout(),
                                                BaseClient.connectTimeout(),
                                                ArangoClient::DEFAULT_RETRIES,
                                                BaseClient.sslProtocol());

  Client = new SimpleHttpClient(Connection, BaseClient.requestTimeout(), false);

  Client->setLocationRewriter(nullptr, &rewriteLocation);
  Client->setUserNamePassword("/", BaseClient.username(), BaseClient.password());

  string versionString = GetArangoVersion();
    
  if (CreateDatabase && LastErrorCode == TRI_ERROR_ARANGO_DATABASE_NOT_FOUND) {
    // database not found, but database creation requested
   
    std::string old = BaseClient.databaseName();
    cout << "Creating database '" << old << "'" << endl;

    BaseClient.setDatabaseName("_system");

    int res = TryCreateDatabase(old);

    if (res != TRI_ERROR_NO_ERROR) {
      cerr << "Could not create database '" << old << "'" << endl;
      cerr << "Error message: '" << Client->getErrorMessage() << "'" << endl;
      TRI_EXIT_FUNCTION(EXIT_FAILURE, nullptr);
    }

    // restore old database name
    BaseClient.setDatabaseName(old);

    // re-fetch version
    versionString = GetArangoVersion();
  }

  if (! Connection->isConnected()) {
    cerr << "Could not connect to endpoint " << BaseClient.endpointServer()->getSpecification() << endl;
    cerr << "Error message: '" << Client->getErrorMessage() << "'" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, nullptr);
  }

  // successfully connected
  cout << "Server version: " << versionString << endl;

  // validate server version
  int major = 0;
  int minor = 0;

  if (sscanf(versionString.c_str(), "%d.%d", &major, &minor) != 2) {
    cerr << "Error: invalid server version '" << versionString << "'" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, nullptr);
  }

  if (major < 1 ||
      major > 2 ||
      (major == 1 && minor < 4)) {
    // we can connect to 1.4, 2.0 and higher only
    cerr << "Error: got incompatible server version '" << versionString << "'" << endl;
    if (! Force) {
      TRI_EXIT_FUNCTION(EXIT_FAILURE, nullptr);
    }
  }

  if (major >= 2) {
    // Version 1.4 did not yet have a cluster mode
    ClusterMode = GetArangoIsCluster();
  }

  if (Progress) {
    cout << "Connected to ArangoDB '" << BaseClient.endpointServer()->getSpecification() << endl;
  }

  memset(&Stats, 0, sizeof(Stats));

  string errorMsg = "";

  int res;
  try {
    res = ProcessInputDirectory(errorMsg);
  }
  catch (std::exception const& ex) {
    cerr << "Error: caught exception " << ex.what() << endl;
    res = TRI_ERROR_INTERNAL;
  }
  catch (...) {
    cerr << "Error: caught unknown exception" << endl;
    res = TRI_ERROR_INTERNAL;
  }


  if (Progress) {
    if (ImportData) {
      cout << "Processed " << Stats._totalCollections << " collection(s), " <<
              "read " << Stats._totalRead << " byte(s) from datafiles, " <<
              "sent " << Stats._totalBatches << " batch(es)" << endl;
    }
    else if (ImportStructure) {
      cout << "Processed " << Stats._totalCollections << " collection(s)" << endl;
    }
  }


  if (res != TRI_ERROR_NO_ERROR) {
    if (! errorMsg.empty()) {
      cerr << "Error: " << errorMsg << endl;
    }
    ret = EXIT_FAILURE;
  }

  if (Client != nullptr) {
    delete Client;
  }

  TRIAGENS_REST_SHUTDOWN;

  LocalExitFunction(ret, nullptr);

  return ret;
}
Пример #13
0
int main (int argc, char* argv[]) {
  int ret = EXIT_SUCCESS;

  arangorestoreEntryFunction();

  TRIAGENS_C_INITIALISE(argc, argv);
  TRIAGENS_REST_INITIALISE(argc, argv);

  TRI_InitialiseLogging(false);

  // .............................................................................
  // set defaults
  // .............................................................................

  BaseClient.setEndpointString(Endpoint::getDefaultEndpoint());

  // .............................................................................
  // parse the program options
  // .............................................................................

  ParseProgramOptions(argc, argv);

  // use a minimum value for batches
  if (ChunkSize < 1024 * 128) {
    ChunkSize = 1024 * 128;
  }

  // .............................................................................
  // check input directory
  // .............................................................................

  if (InputDirectory == "" || ! TRI_IsDirectory(InputDirectory.c_str())) {
    cerr << "input directory '" << InputDirectory << "' does not exist" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  if (! ImportStructure && ! ImportData) {
    cerr << "must specify either --create-collection or --import-data" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  // .............................................................................
  // set-up client connection
  // .............................................................................

  BaseClient.createEndpoint();

  if (BaseClient.endpointServer() == 0) {
    cerr << "invalid value for --server.endpoint ('" << BaseClient.endpointString() << "')" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  Connection = GeneralClientConnection::factory(BaseClient.endpointServer(),
                                                BaseClient.requestTimeout(),
                                                BaseClient.connectTimeout(),
                                                ArangoClient::DEFAULT_RETRIES,
                                                BaseClient.sslProtocol());

  if (Connection == 0) {
    cerr << "out of memory" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  Client = new SimpleHttpClient(Connection, BaseClient.requestTimeout(), false);

  if (Client == 0) {
    cerr << "out of memory" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  Client->setLocationRewriter(0, &rewriteLocation);
  Client->setUserNamePassword("/", BaseClient.username(), BaseClient.password());

  const string versionString = GetArangoVersion();

  if (! Connection->isConnected()) {
    cerr << "Could not connect to endpoint " << BaseClient.endpointServer()->getSpecification() << endl;
    cerr << "Error message: '" << Client->getErrorMessage() << "'" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  // successfully connected
  cout << "Server version: " << versionString << endl;

  // validate server version
  int major = 0;
  int minor = 0;

  if (sscanf(versionString.c_str(), "%d.%d", &major, &minor) != 2) {
    cerr << "invalid server version '" << versionString << "'" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  if (major < 1 ||
      major > 2 ||
      (major == 1 && minor < 4)) {
    // we can connect to 1.4, 2.0 and higher only
    cerr << "got incompatible server version '" << versionString << "'" << endl;
    if (! Force) {
      TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
    }
  }

  if (major >= 2) {
    // Version 1.4 did not yet have a cluster mode
    clusterMode = GetArangoIsCluster();
  }

  if (Progress) {
    cout << "Connected to ArangoDB '" << BaseClient.endpointServer()->getSpecification() << endl;
  }

  memset(&Stats, 0, sizeof(Stats));

  string errorMsg = "";
  int res = ProcessInputDirectory(errorMsg);

  if (Progress) {
    if (ImportData) {
      cout << "Processed " << Stats._totalCollections << " collection(s), " <<
              "read " << Stats._totalRead << " byte(s) from datafiles, " <<
              "sent " << Stats._totalBatches << " batch(es)" << endl;
    }
    else if (ImportStructure) {
      cout << "Processed " << Stats._totalCollections << " collection(s)" << endl;
    }
  }


  if (res != TRI_ERROR_NO_ERROR) {
    cerr << errorMsg << endl;
    ret = EXIT_FAILURE;
  }

  if (Client != 0) {
    delete Client;
  }

  TRIAGENS_REST_SHUTDOWN;

  arangorestoreExitFunction(ret, NULL);

  return ret;
}
Пример #14
0
int main (int argc, char* argv[]) {
    int ret = EXIT_SUCCESS;

    arangobEntryFunction();

    TRIAGENS_C_INITIALIZE(argc, argv);
    TRIAGENS_REST_INITIALIZE(argc, argv);

    TRI_InitializeLogging(false);

    BaseClient.setEndpointString(Endpoint::getDefaultEndpoint());

    // .............................................................................
    // parse the program options
    // .............................................................................

    ParseProgramOptions(argc, argv);

    // .............................................................................
    // set-up client connection
    // .............................................................................

    BaseClient.createEndpoint();

    if (BaseClient.endpointServer() == nullptr) {
        LOG_FATAL_AND_EXIT("invalid value for --server.endpoint ('%s')", BaseClient.endpointString().c_str());
    }

    BenchmarkOperation* testCase = GetTestCase(TestCase);

    if (testCase == nullptr) {
        LOG_FATAL_AND_EXIT("invalid test case name '%s'", TestCase.c_str());
        return EXIT_FAILURE; // will not be reached
    }

    Status("starting threads...");

    BenchmarkCounter<unsigned long> operationsCounter(0, (unsigned long) Operations);
    ConditionVariable startCondition;


    vector<Endpoint*> endpoints;
    vector<BenchmarkThread*> threads;

    const double stepSize = (double) Operations / (double) ThreadConcurrency;
    int64_t realStep = (int64_t) stepSize;
    if (stepSize - (double) ((int64_t) stepSize) > 0.0) {
        realStep++;
    }
    if (realStep % 1000 != 0) {
        realStep += 1000 - (realStep % 1000);
    }
    // add some more offset so we don't get into trouble with threads of different speed
    realStep += 10000;

    // start client threads
    for (int i = 0; i < ThreadConcurrency; ++i) {
        Endpoint* endpoint = Endpoint::clientFactory(BaseClient.endpointString());
        endpoints.push_back(endpoint);

        BenchmarkThread* thread = new BenchmarkThread(testCase,
                &startCondition,
                &UpdateStartCounter,
                i,
                (unsigned long) BatchSize,
                &operationsCounter,
                endpoint,
                BaseClient.databaseName(),
                BaseClient.username(),
                BaseClient.password(),
                BaseClient.requestTimeout(),
                BaseClient.connectTimeout(),
                BaseClient.sslProtocol(),
                KeepAlive,
                Async,
                verbose);

        threads.push_back(thread);
        thread->setOffset((size_t) (i * realStep));
        thread->start();
    }

    // give all threads a chance to start so they will not miss the broadcast
    while (GetStartCounter() < ThreadConcurrency) {
        usleep(5000);
    }


    if (Delay) {
        Status("sleeping (startup delay)...");
        sleep(10);
    }
    Status("executing tests...");

    double start = TRI_microtime();

    // broadcast the start signal to all threads
    {
        CONDITION_LOCKER(guard, startCondition);
        guard.broadcast();
    }

    const size_t stepValue = (Operations / 20);
    size_t nextReportValue = stepValue;

    if (nextReportValue < 100) {
        nextReportValue = 100;
    }

    while (1) {
        const size_t numOperations = operationsCounter.getDone();

        if (numOperations >= (size_t) Operations) {
            break;
        }

        if (Progress && numOperations >= nextReportValue) {
            LOG_INFO("number of operations: %d", (int) nextReportValue);
            nextReportValue += stepValue;
        }

        usleep(20000);
    }

    double time = TRI_microtime() - start;
    double requestTime = 0.0;

    for (int i = 0; i < ThreadConcurrency; ++i) {
        requestTime += threads[i]->getTime();
    }

    size_t failures = operationsCounter.failures();
    size_t incomplete = operationsCounter.incompleteFailures();

    cout << endl;
    cout << "Total number of operations: " << Operations <<
         ", keep alive: " << (KeepAlive ? "yes" : "no") <<
         ", async: " << (Async ? "yes" : "no")  <<
         ", batch size: " << BatchSize <<
         ", concurrency level (threads): " << ThreadConcurrency <<
         endl;

    cout << "Test case: " << TestCase <<
         ", complexity: " << Complexity <<
         ", database: '" << BaseClient.databaseName() <<
         "', collection: '" << Collection << "'" <<
         endl;

    cout << "Total request/response duration (sum of all threads): " << fixed << requestTime << " s" << endl;
    cout << "Request/response duration (per thread): " << fixed << (requestTime / (double) ThreadConcurrency) << " s" << endl;
    cout << "Time needed per operation: " << fixed << (time / Operations) << " s" << endl;
    cout << "Time needed per operation per thread: " << fixed << (time / (double) Operations * (double) ThreadConcurrency) << " s" << endl;
    cout << "Operations per second rate: " << fixed << ((double) Operations / time) << endl;
    cout << "Elapsed time since start: " << fixed << time << " s" << endl << endl;

    if (failures > 0) {
        cerr << "WARNING: " << failures << " arangob request(s) failed!!" << endl;
    }
    if (incomplete > 0) {
        cerr << "WARNING: " << incomplete << " arangob requests with incomplete results!!" << endl;
    }

    testCase->tearDown();

    for (int i = 0; i < ThreadConcurrency; ++i) {
        threads[i]->join();
        delete threads[i];
        delete endpoints[i];
    }

    delete testCase;

    TRIAGENS_REST_SHUTDOWN;

    if (failures > 0) {
        ret = EXIT_FAILURE;
    }

    arangobExitFunction(ret, nullptr);

    return ret;
}
Пример #15
0
static void Status (const string& value) {
    if (! BaseClient.quiet()) {
        cout << value << endl;
    }
}
Пример #16
0
int main (int argc, char* argv[]) {
  TRIAGENS_C_INITIALISE(argc, argv);
  TRIAGENS_REST_INITIALISE(argc, argv);

  TRI_InitialiseLogging(false);
  int ret = EXIT_SUCCESS;

  BaseClient.setEndpointString(Endpoint::getDefaultEndpoint());

  // .............................................................................
  // parse the program options
  // .............................................................................

  ParseProgramOptions(argc, argv);

  // .............................................................................
  // set-up MRuby objects
  // .............................................................................

  // create a new ruby shell
  mrb_state* mrb = MR_OpenShell();

  TRI_InitMRUtils(mrb);

  // .............................................................................
  // set-up client connection
  // .............................................................................

  // check if we want to connect to a server
  bool useServer = (BaseClient.endpointString() != "none");

  if (useServer) {
    BaseClient.createEndpoint();

    if (BaseClient.endpointServer() == 0) {
      LOGGER_FATAL_AND_EXIT("invalid value for --server.endpoint ('" << BaseClient.endpointString() << "')");
    }

    ClientConnection = createConnection(mrb);
    InitMRClientConnection(mrb, ClientConnection);
  }

  // .............................................................................
  // banner
  // .............................................................................

  // http://www.network-science.de/ascii/   Font: ogre
  if (! BaseClient.quiet()) {
    char const* g = TRI_SHELL_COLOR_GREEN;
    char const* r = TRI_SHELL_COLOR_RED;
    char const* z = TRI_SHELL_COLOR_RESET;

    if (! BaseClient.colors()) {
      g = "";
      r = "";
      z = "";
    }

    printf("%s                                  %s _      _     %s\n", g, r, z);
    printf("%s  __ _ _ __ __ _ _ __   __ _  ___ %s(_)_ __| |__  %s\n", g, r, z);
    printf("%s / _` | '__/ _` | '_ \\ / _` |/ _ \\%s| | '__| '_ \\ %s\n", g, r, z);
    printf("%s| (_| | | | (_| | | | | (_| | (_) %s| | |  | |_) |%s\n", g, r, z);
    printf("%s \\__,_|_|  \\__,_|_| |_|\\__, |\\___/%s|_|_|  |_.__/ %s\n", g, r, z);
    printf("%s                       |___/      %s              %s\n", g, r, z);

    cout << endl << "Welcome to arangosh " << TRI_VERSION_FULL << ". Copyright (c) 2012 triAGENS GmbH" << endl;

#ifdef TRI_MRUBY_VERSION
    cout << "Using MRUBY " << TRI_MRUBY_VERSION << " engine. Copyright (c) 2012 mruby developers." << endl;
#endif

#ifdef TRI_READLINE_VERSION
    cout << "Using READLINE " << TRI_READLINE_VERSION << endl;
#endif

    cout << endl;

    BaseClient.printWelcomeInfo();

    if (useServer) {
      if (ClientConnection->isConnected()) {
        if (! BaseClient.quiet()) {
          cout << "Connected to ArangoDB '" << BaseClient.endpointServer()->getSpecification()
               << "' Version " << ClientConnection->getVersion() << endl;
        }
      }
      else {
        cerr << "Could not connect to endpoint '" << BaseClient.endpointString() << "'" << endl;
        cerr << "Error message '" << ClientConnection->getErrorMessage() << "'" << endl;
      }
    }
  }

  // .............................................................................
  // read files
  // .............................................................................

  // load java script from js/bootstrap/*.h files
  if (StartupPath.empty()) {
    StartupLoader.defineScript("common/bootstrap/error.rb", MR_common_bootstrap_error);
  }
  else {
    LOGGER_DEBUG("using Ruby startup files at '" << StartupPath << "'");
    StartupLoader.setDirectory(StartupPath);
  }

  // load all init files
  char const* files[] = {
    "common/bootstrap/error.rb"
  };

  for (size_t i = 0;  i < sizeof(files) / sizeof(files[0]);  ++i) {
    bool ok = StartupLoader.loadScript(mrb, files[i]);

    if (ok) {
      LOGGER_TRACE("loaded ruby file '" << files[i] << "'");
    }
    else {
      LOGGER_FATAL_AND_EXIT("cannot load ruby file '" << files[i] << "'");
    }
  }

  // .............................................................................
  // run normal shell
  // .............................................................................

  RunShell(mrb);

  TRIAGENS_REST_SHUTDOWN;

  return ret;
}
Пример #17
0
int main (int argc, char* argv[]) {

  int ret = EXIT_SUCCESS;

  arangoimpEntryFunction();

  TRIAGENS_C_INITIALISE(argc, argv);
  TRIAGENS_REST_INITIALISE(argc, argv);

  TRI_InitialiseLogging(false);

  BaseClient.setEndpointString(Endpoint::getDefaultEndpoint());

  // .............................................................................
  // parse the program options
  // .............................................................................

  ParseProgramOptions(argc, argv);

  // .............................................................................
  // set-up client connection
  // .............................................................................

  BaseClient.createEndpoint();

  if (BaseClient.endpointServer() == 0) {
    cerr << "invalid value for --server.endpoint ('" << BaseClient.endpointString() << "')" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  ClientConnection = new V8ClientConnection(BaseClient.endpointServer(),
                                            BaseClient.databaseName(),
                                            BaseClient.username(),
                                            BaseClient.password(),
                                            BaseClient.requestTimeout(),
                                            BaseClient.connectTimeout(),
                                            ArangoClient::DEFAULT_RETRIES,
                                            BaseClient.sslProtocol(),
                                            false);

  if (! ClientConnection->isConnected() || 
      ClientConnection->getLastHttpReturnCode() != HttpResponse::OK) {
    cerr << "Could not connect to endpoint '" << BaseClient.endpointServer()->getSpecification() 
         << "', database: '" << BaseClient.databaseName() << "'" << endl;
    cerr << "Error message: '" << ClientConnection->getErrorMessage() << "'" << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  // successfully connected
  cout << "Connected to ArangoDB '" << BaseClient.endpointServer()->getSpecification()
       << "', version " << ClientConnection->getVersion() << ", database: '" 
       << BaseClient.databaseName() << "', username: '******'" << endl;

  cout << "----------------------------------------" << endl;
  cout << "database:         " << BaseClient.databaseName() << endl;
  cout << "collection:       " << CollectionName << endl;
  cout << "create:           " << (CreateCollection ? "yes" : "no") << endl;
  cout << "file:             " << FileName << endl;
  cout << "type:             " << TypeImport << endl;

  if (TypeImport == "csv") {
    cout << "quote:            " << Quote << endl;
    cout << "separator:        " << Separator << endl;
  }

  cout << "connect timeout:  " << BaseClient.connectTimeout() << endl;
  cout << "request timeout:  " << BaseClient.requestTimeout() << endl;
  cout << "----------------------------------------" << endl;

  ImportHelper ih(ClientConnection->getHttpClient(), ChunkSize);

  // create colletion
  if (CreateCollection) {
    ih.setCreateCollection(true);
  }

  // quote
  if (Quote.length() <= 1) {
    ih.setQuote(Quote);
  }
  else {
    cerr << "Wrong length of quote character." << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  // separator
  if (Separator.length() == 1) {
    ih.setSeparator(Separator);
  }
  else {
    cerr << "Separator must be exactly one character." << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  // collection name
  if (CollectionName == "") {
    cerr << "Collection name is missing." << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  // filename
  if (FileName == "") {
    cerr << "File name is missing." << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  if (FileName != "-" && ! FileUtils::isRegularFile(FileName)) {
    if (! FileUtils::exists(FileName)) {
      cerr << "Cannot open file '" << FileName << "'. File not found." << endl;
    }
    else if (FileUtils::isDirectory(FileName)) {
      cerr << "Specified file '" << FileName << "' is a directory. Please use a regular file." << endl;
    }
    else {
      cerr << "Cannot open '" << FileName << "'. Invalid file type." << endl;
    }

    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  // progress
  if (Progress) {
    ih.setProgress(true);
  }

  // import type
  bool ok = false;

  if (TypeImport == "csv") {
    cout << "Starting CSV import..." << endl;
    ok = ih.importDelimited(CollectionName, FileName, ImportHelper::CSV);
  }

  else if (TypeImport == "tsv") {
    cout << "Starting TSV import..." << endl;
    ih.setQuote("");
    ih.setSeparator("\\t");
    ok = ih.importDelimited(CollectionName, FileName, ImportHelper::TSV);
  }

  else if (TypeImport == "json") {
    cout << "Starting JSON import..." << endl;
    ok = ih.importJson(CollectionName, FileName);
  }

  else {
    cerr << "Wrong type '" << TypeImport << "'." << endl;
    TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
  }

  cout << endl;

  // give information about import
  if (ok) {
    cout << "created:          " << ih.getImportedLines() << endl;
    cout << "errors:           " << ih.getErrorLines() << endl;
    cout << "total:            " << ih.getReadLines() << endl;
  }
  else {
    cerr << "error message:    " << ih.getErrorMessage() << endl;
  }

  TRIAGENS_REST_SHUTDOWN;

  arangoimpExitFunction(ret, NULL);

  return ret;
}