Beispiel #1
0
void RestoreFeature::start() {
  ClientFeature* client = application_features::ApplicationServer::getFeature<ClientFeature>("Client");

  int ret = EXIT_SUCCESS;
  *_result = ret;

  try {
    _httpClient = client->createHttpClient();
  } catch (...) {
    LOG(FATAL) << "cannot create server connection, giving up!";
    FATAL_ERROR_EXIT();
  }

  std::string dbName = client->databaseName();

  _httpClient->setLocationRewriter(static_cast<void*>(client), &rewriteLocation);
  _httpClient->setUserNamePassword("/", client->username(), client->password());

  int err = TRI_ERROR_NO_ERROR;
  std::string versionString = _httpClient->getServerVersion(&err);

  if (_createDatabase && err == TRI_ERROR_ARANGO_DATABASE_NOT_FOUND) {
    // database not found, but database creation requested
    std::cout << "Creating database '" << dbName << "'" << std::endl;

    client->setDatabaseName("_system");

    int res = tryCreateDatabase(client, dbName);

    if (res != TRI_ERROR_NO_ERROR) {
      LOG(ERR) << "Could not create database '" << dbName << "'";
      LOG(FATAL) << _httpClient->getErrorMessage() << "'";
      FATAL_ERROR_EXIT();
    }

    // restore old database name
    client->setDatabaseName(dbName);

    // re-fetch version
    versionString = _httpClient->getServerVersion(nullptr);
  }

  if (!_httpClient->isConnected()) {
    LOG(ERR) << "Could not connect to endpoint "
             << _httpClient->getEndpointSpecification();
    LOG(FATAL) << _httpClient->getErrorMessage() << "'";
    FATAL_ERROR_EXIT();
  }

  // successfully connected
  std::cout << "Server version: " << versionString << std::endl;

  // validate server version
  std::pair<int, int> version = Version::parseVersionString(versionString);

  if (version.first < 3) {
    // we can connect to 3.x
    LOG(ERR) << "got incompatible server version '" << versionString << "'";

    if (!_force) {
      LOG(FATAL) << "giving up!";
      FATAL_ERROR_EXIT();
    }
  }

  // Version 1.4 did not yet have a cluster mode
  _clusterMode = getArangoIsCluster(nullptr);

  if (_progress) {
    std::cout << "# Connected to ArangoDB '"
              << _httpClient->getEndpointSpecification() << "'" << std::endl;
  }

  std::string errorMsg = "";

  int res;
  try {
    res = processInputDirectory(errorMsg);
  } catch (std::exception const& ex) {
    LOG(ERR) << "caught exception " << ex.what();
    res = TRI_ERROR_INTERNAL;
  } catch (...) {
    LOG(ERR) << "Error: caught unknown exception";
    res = TRI_ERROR_INTERNAL;
  }

  if (res != TRI_ERROR_NO_ERROR) {
    if (!errorMsg.empty()) {
      LOG(ERR) << errorMsg;
    } else {
      LOG(ERR) << "An error occurred";
    }
    ret = EXIT_FAILURE;
  }

  if (_progress) {
    if (_importData) {
      std::cout << "Processed " << _stats._totalCollections
                << " collection(s), "
                << "read " << _stats._totalRead << " byte(s) from datafiles, "
                << "sent " << _stats._totalBatches << " batch(es)" << std::endl;
    } else if (_importStructure) {
      std::cout << "Processed " << _stats._totalCollections << " collection(s)"
                << std::endl;
    }
  }
  
  *_result = ret;
}
Beispiel #2
0
void ImportFeature::start() {
  ClientFeature* client = application_features::ApplicationServer::getFeature<ClientFeature>("Client");

  int ret = EXIT_SUCCESS;
  *_result = ret;

  std::unique_ptr<SimpleHttpClient> httpClient;

  try {
    httpClient = client->createHttpClient();
  } catch (...) {
    LOG(FATAL) << "cannot create server connection, giving up!";
    FATAL_ERROR_EXIT();
  }

  httpClient->setLocationRewriter(static_cast<void*>(client), &rewriteLocation);
  httpClient->setUserNamePassword("/", client->username(), client->password());

  // must stay here in order to establish the connection
  httpClient->getServerVersion();

  if (!httpClient->isConnected()) {
    LOG(ERR) << "Could not connect to endpoint '" << client->endpoint()
             << "', database: '" << client->databaseName() << "', username: '******'";
    LOG(FATAL) << httpClient->getErrorMessage() << "'";
    FATAL_ERROR_EXIT();
  }

  // successfully connected
  std::cout << "Connected to ArangoDB '"
            << httpClient->getEndpointSpecification() << "', version "
            << httpClient->getServerVersion() << ", database: '"
            << client->databaseName() << "', username: '******'" << std::endl;

  std::cout << "----------------------------------------" << std::endl;
  std::cout << "database:               " << client->databaseName() << std::endl;
  std::cout << "collection:             " << _collectionName << std::endl;
  if (!_fromCollectionPrefix.empty()) {
    std::cout << "from collection prefix: " << _fromCollectionPrefix << std::endl;
  }
  if (!_toCollectionPrefix.empty()) {
    std::cout << "to collection prefix:   " << _toCollectionPrefix << std::endl;
  }
  std::cout << "create:                 " << (_createCollection ? "yes" : "no")
            << std::endl;
  std::cout << "source filename:        " << _filename << std::endl;
  std::cout << "file type:              " << _typeImport << std::endl;

  if (_typeImport == "csv") {
    std::cout << "quote:                  " << _quote << std::endl;
  } 
  if (_typeImport == "csv" || _typeImport == "tsv") {
    std::cout << "separator:              " << _separator << std::endl;
  }

  std::cout << "connect timeout:        " << client->connectionTimeout() << std::endl;
  std::cout << "request timeout:        " << client->requestTimeout() << std::endl;
  std::cout << "----------------------------------------" << std::endl;

  arangodb::import::ImportHelper ih(httpClient.get(), _chunkSize);

  // create colletion
  if (_createCollection) {
    ih.setCreateCollection(true);
  }

  if (_createCollectionType == "document" || _createCollectionType == "edge") {
    ih.setCreateCollectionType(_createCollectionType);
  }

  ih.setConversion(_convert);
  ih.setRowsToSkip(static_cast<size_t>(_rowsToSkip));
  ih.setOverwrite(_overwrite);
  ih.useBackslash(_useBackslash);

  // quote
  if (_quote.length() <= 1) {
    ih.setQuote(_quote);
  } else {
    LOG(FATAL) << "Wrong length of quote character.";
    FATAL_ERROR_EXIT();
  }

  if (_separator.empty()) {
    _separator = ",";
    if (_typeImport == "tsv") {
      _separator = "\\t";
    }
  }

  // separator
  if (_separator.length() == 1 || _separator == "\\r" || _separator == "\\n" || _separator == "\\t") {
    ih.setSeparator(_separator);
  } else {
    LOG(FATAL) << "_separator must be exactly one character.";
    FATAL_ERROR_EXIT();
  }

  // collection name
  if (_collectionName == "") {
    LOG(FATAL) << "Collection name is missing.";
    FATAL_ERROR_EXIT();
  }

  // filename
  if (_filename == "") {
    LOG(FATAL) << "File name is missing.";
    FATAL_ERROR_EXIT();
  }

  if (_filename != "-" && !FileUtils::isRegularFile(_filename)) {
    if (!FileUtils::exists(_filename)) {
      LOG(FATAL) << "Cannot open file '" << _filename << "'. File not found.";
    } else if (FileUtils::isDirectory(_filename)) {
      LOG(FATAL) << "Specified file '" << _filename
                 << "' is a directory. Please use a regular file.";
    } else {
      LOG(FATAL) << "Cannot open '" << _filename << "'. Invalid file type.";
    }

    FATAL_ERROR_EXIT();
  }

  // progress
  if (_progress) {
    ih.setProgress(true);
  }

  if (_onDuplicateAction != "error" && _onDuplicateAction != "update" &&
      _onDuplicateAction != "replace" && _onDuplicateAction != "ignore") {
    LOG(FATAL)
        << "Invalid value for '--on-duplicate'. Possible values: 'error', "
           "'update', 'replace', 'ignore'.";
    FATAL_ERROR_EXIT();
  }

  ih.setOnDuplicateAction(_onDuplicateAction);

  try {
    bool ok = false;
    // set prefixes
    ih.setFrom(_fromCollectionPrefix);
    ih.setTo(_toCollectionPrefix);

    // import type
    if (_typeImport == "csv") {
      std::cout << "Starting CSV import..." << std::endl;
      ok = ih.importDelimited(_collectionName, _filename,
                              arangodb::import::ImportHelper::CSV);
    }

    else if (_typeImport == "tsv") {
      std::cout << "Starting TSV import..." << std::endl;
      ih.setQuote("");
      ok = ih.importDelimited(_collectionName, _filename,
                              arangodb::import::ImportHelper::TSV);
    }

    else if (_typeImport == "json") {
      std::cout << "Starting JSON import..." << std::endl;
      ok = ih.importJson(_collectionName, _filename);
    }

    else {
      LOG(FATAL) << "Wrong type '" << _typeImport << "'.";
      FATAL_ERROR_EXIT();
    }

    std::cout << std::endl;

    // give information about import
    if (ok) {
      std::cout << "created:          " << ih.getNumberCreated() << std::endl;
      std::cout << "warnings/errors:  " << ih.getNumberErrors() << std::endl;
      std::cout << "updated/replaced: " << ih.getNumberUpdated() << std::endl;
      std::cout << "ignored:          " << ih.getNumberIgnored() << std::endl;

      if (_typeImport == "csv" || _typeImport == "tsv") {
        std::cout << "lines read:       " << ih.getReadLines() << std::endl;
      }

    } else {
      LOG(ERR) << "error message:    " << ih.getErrorMessage();
    }
  } catch (std::exception const& ex) {
    LOG(ERR) << "Caught exception " << ex.what() << " during import";
  } catch (...) {
    LOG(ERR) << "Got an unknown exception during import";
  }

  *_result = ret;
}