bool RestEdgesHandler::getEdgesForVertexList( VPackSlice const ids, std::vector<traverser::TraverserExpression*> const& expressions, TRI_edge_direction_e direction, SingleCollectionTransaction& trx, VPackBuilder& result, size_t& scannedIndex, size_t& filtered) { TRI_ASSERT(ids.isArray()); trx.orderDitch(trx.cid()); // will throw when it fails std::string const collectionName = trx.resolver()->getCollectionName(trx.cid()); Transaction::IndexHandle indexId = trx.edgeIndexHandle(collectionName); VPackBuilder searchValueBuilder; EdgeIndex::buildSearchValueFromArray(direction, ids, searchValueBuilder); VPackSlice search = searchValueBuilder.slice(); std::unique_ptr<OperationCursor> cursor = trx.indexScan(collectionName, arangodb::Transaction::CursorType::INDEX, indexId, search, nullptr, 0, UINT64_MAX, 1000, false); if (cursor->failed()) { THROW_ARANGO_EXCEPTION(cursor->code); } auto opRes = std::make_shared<OperationResult>(TRI_ERROR_NO_ERROR); while (cursor->hasMore()) { cursor->getMore(opRes); if (opRes->failed()) { THROW_ARANGO_EXCEPTION(opRes->code); } VPackSlice edges = opRes->slice(); TRI_ASSERT(edges.isArray()); // generate result scannedIndex += static_cast<size_t>(edges.length()); for (auto const& edge : VPackArrayIterator(edges)) { bool add = true; if (!expressions.empty()) { for (auto& exp : expressions) { if (exp->isEdgeAccess && !exp->matchesCheck(&trx, edge)) { ++filtered; add = false; break; } } } if (add) { result.add(edge); } } } return true; }
void RestVocbaseBaseHandler::generate20x( arangodb::OperationResult const& result, std::string const& collectionName, TRI_col_type_e type, VPackOptions const* options) { VPackSlice slice = result.slice(); if (slice.isNone()) { // will happen if silent == true slice = VelocyPackHelper::EmptyObjectValue(); } else { TRI_ASSERT(slice.isObject() || slice.isArray()); if (slice.isObject()) { _response->setHeaderNC( StaticStrings::Etag, "\"" + slice.get(StaticStrings::RevString).copyString() + "\""); // pre 1.4 location headers withdrawn for >= 3.0 std::string escapedHandle(assembleDocumentId( collectionName, slice.get(StaticStrings::KeyString).copyString(), true)); _response->setHeaderNC(StaticStrings::Location, std::string("/_db/" + _request->databaseName() + DOCUMENT_PATH + "/" + escapedHandle)); } } writeResult(slice, *options); }
static v8::Handle<v8::Value> ObjectVPackArray(v8::Isolate* isolate, VPackSlice const& slice, VPackOptions const* options, VPackSlice const* base) { TRI_ASSERT(slice.isArray()); v8::Handle<v8::Array> object = v8::Array::New(isolate, static_cast<int>(slice.length())); if (object.IsEmpty()) { return v8::Undefined(isolate); } uint32_t j = 0; VPackArrayIterator it(slice, true); while (it.valid()) { v8::Handle<v8::Value> val = TRI_VPackToV8(isolate, it.value(), options, &slice); if (!val.IsEmpty()) { object->Set(j++, val); } it.next(); } return object; }
/// Log transactions (follower) arangodb::consensus::index_t State::log(query_t const& transactions, size_t ndups) { VPackSlice slices = transactions->slice(); TRI_ASSERT(slices.isArray()); size_t nqs = slices.length(); TRI_ASSERT(nqs > ndups); MUTEX_LOCKER(mutexLocker, _logLock); // log entries must stay in order for (size_t i = ndups; i < nqs; ++i) { VPackSlice slice = slices[i]; try { auto idx = slice.get("index").getUInt(); auto trm = slice.get("term").getUInt(); auto buf = std::make_shared<Buffer<uint8_t>>(); buf->append((char const*)slice.get("query").begin(), slice.get("query").byteSize()); // to RAM _log.push_back(log_t(idx, trm, buf)); // to disk persist(idx, trm, slice.get("query")); } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCY) << e.what() << " " << __FILE__ << __LINE__; } } TRI_ASSERT(!_log.empty()); return _log.back().index; }
void AuthInfo::reload() { insertInitial(); TRI_vocbase_t* vocbase = DatabaseFeature::DATABASE->systemDatabase(); if (vocbase == nullptr) { LOG(DEBUG) << "system database is unknown, cannot load authentication " << "and authorization information"; return; } MUTEX_LOCKER(locker, _queryLock); if (!_outdated) { return; } std::string queryStr("FOR user IN _users RETURN user"); auto nullBuilder = std::make_shared<VPackBuilder>(); VPackBuilder options; { VPackObjectBuilder b(&options); } auto objectBuilder = std::make_shared<VPackBuilder>(options); arangodb::aql::Query query(false, vocbase, queryStr.c_str(), queryStr.length(), nullBuilder, objectBuilder, arangodb::aql::PART_MAIN); LOG(DEBUG) << "starting to load authentication and authorization information"; TRI_ASSERT(_queryRegistry != nullptr); auto queryResult = query.execute(_queryRegistry); if (queryResult.code != TRI_ERROR_NO_ERROR) { if (queryResult.code == TRI_ERROR_REQUEST_CANCELED || (queryResult.code == TRI_ERROR_QUERY_KILLED)) { THROW_ARANGO_EXCEPTION(TRI_ERROR_REQUEST_CANCELED); } _outdated = false; return; } VPackSlice usersSlice = queryResult.result->slice(); if (usersSlice.isNone()) { THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); } if (!usersSlice.isArray()) { LOG(ERR) << "cannot read users from _users collection"; return; } if (usersSlice.length() == 0) { insertInitial(); } else { populate(usersSlice); } _outdated = false; }
arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo( arangodb::aql::Query* query, VPackSlice const& info, VPackSlice const& shards) { TRI_ASSERT(shards.isArray()); idxHandles.reserve(shards.length()); conditionNeedUpdate = arangodb::basics::VelocyPackHelper::getBooleanValue( info, "condNeedUpdate", false); conditionMemberToUpdate = arangodb::basics::VelocyPackHelper::getNumericValue<size_t>( info, "condMemberToUpdate", 0); VPackSlice read = info.get("handle"); if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Each lookup requires handle to be an object"); } read = read.get("id"); if (!read.isString()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Each handle requires id to be a string"); } std::string idxId = read.copyString(); auto trx = query->trx(); for (auto const& it : VPackArrayIterator(shards)) { if (!it.isString()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Shards have to be a list of strings"); } idxHandles.emplace_back(trx->getIndexByIdentifier(it.copyString(), idxId)); } read = info.get("expression"); if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Each lookup requires expression to be an object"); } expression = new aql::Expression(query->ast(), read); read = info.get("condition"); if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Each lookup requires condition to be an object"); } indexCondition = new aql::AstNode(query->ast(), read); }
/// @brief shutdown, will be called exactly once for the whole query int RemoteBlock::shutdown(int errorCode) { DEBUG_BEGIN_BLOCK(); if (!_isResponsibleForInitializeCursor) { // do nothing... return TRI_ERROR_NO_ERROR; } // For every call we simply forward via HTTP std::unique_ptr<ClusterCommResult> res = sendRequest(rest::RequestType::PUT, "/_api/aql/shutdown/", std::string("{\"code\":" + std::to_string(errorCode) + "}")); if (throwExceptionAfterBadSyncRequest(res.get(), true)) { // artificially ignore error in case query was not found during shutdown return TRI_ERROR_NO_ERROR; } StringBuffer const& responseBodyBuf(res->result->getBody()); std::shared_ptr<VPackBuilder> builder = VPackParser::fromJson(responseBodyBuf.c_str(), responseBodyBuf.length()); VPackSlice slice = builder->slice(); // read "warnings" attribute if present and add it to our query if (slice.isObject()) { VPackSlice warnings = slice.get("warnings"); if (warnings.isArray()) { auto query = _engine->getQuery(); for (auto const& it : VPackArrayIterator(warnings)) { if (it.isObject()) { VPackSlice code = it.get("code"); VPackSlice message = it.get("message"); if (code.isNumber() && message.isString()) { query->registerWarning(code.getNumericValue<int>(), message.copyString().c_str()); } } } } } if (slice.hasKey("code")) { return slice.get("code").getNumericValue<int>(); } return TRI_ERROR_INTERNAL; // cppcheck-suppress style DEBUG_END_BLOCK(); }
bool AuthInfo::populate(VPackSlice const& slice) { TRI_ASSERT(slice.isArray()); WRITE_LOCKER(writeLocker, _authInfoLock); clear(); for (VPackSlice const& authSlice : VPackArrayIterator(slice)) { AuthEntry auth = CreateAuthEntry(authSlice.resolveExternal()); if (auth.isActive()) { _authInfo.emplace(auth.username(), auth); } } return true; }
void VelocyPackHelper::SanitizeExternals(VPackSlice const input, VPackBuilder& output) { if (input.isExternal()) { output.add(input.resolveExternal()); } else if (input.isObject()) { output.openObject(); for (auto const& it : VPackObjectIterator(input)) { output.add(VPackValue(it.key.copyString())); SanitizeExternals(it.value, output); } output.close(); } else if (input.isArray()) { output.openArray(); for (auto const& it : VPackArrayIterator(input)) { SanitizeExternals(it, output); } output.close(); } else { output.add(input); } }
/// Load compaction collection bool State::loadCompacted() { auto bindVars = std::make_shared<VPackBuilder>(); bindVars->openObject(); bindVars->close(); std::string const aql( std::string("FOR c IN compact SORT c._key DESC LIMIT 1 RETURN c")); arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars, nullptr, arangodb::aql::PART_MAIN); auto queryResult = query.execute(QueryRegistryFeature::QUERY_REGISTRY); if (queryResult.code != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details); } VPackSlice result = queryResult.result->slice(); if (result.isArray() && result.length()) { MUTEX_LOCKER(logLock, _logLock); for (auto const& i : VPackArrayIterator(result)) { auto ii = i.resolveExternals(); buffer_t tmp = std::make_shared<arangodb::velocypack::Buffer<uint8_t>>(); (*_agent) = ii; try { _cur = std::stoul(ii.get("_key").copyString()); } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCY) << e.what() << " " << __FILE__ << __LINE__; } } } return true; }
/// Internal function for optimized edge retrieval. /// Allows to send an TraverserExpression for filtering in the body /// Not publicly documented on purpose. bool RestEdgesHandler::readFilteredEdges() { std::vector<traverser::TraverserExpression*> expressions; bool parseSuccess = true; std::shared_ptr<VPackBuilder> parsedBody = parseVelocyPackBody(&VPackOptions::Defaults, parseSuccess); if (!parseSuccess) { // We continue unfiltered // Filter could be done by caller return readEdges(expressions); } VPackSlice body = parsedBody->slice(); arangodb::basics::ScopeGuard guard{[]() -> void {}, [&expressions]() -> void { for (auto& e : expressions) { delete e; } }}; if (!body.isArray()) { generateError( rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, "Expected an array of traverser expressions as body parameter"); return false; } expressions.reserve(static_cast<size_t>(body.length())); for (auto const& exp : VPackArrayIterator(body)) { if (exp.isObject()) { auto expression = std::make_unique<traverser::TraverserExpression>(exp); expressions.emplace_back(expression.get()); expression.release(); } } return readEdges(expressions); }
// Internal function to receive all edges for a list of vertices // Not publicly documented on purpose. // NOTE: It ONLY except _id strings. Nothing else bool RestEdgesHandler::readEdgesForMultipleVertices() { std::vector<std::string> const& suffix = _request->suffix(); if (suffix.size() != 1) { generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, "expected POST " + EDGES_PATH + "/<collection-identifier>?direction=<direction>"); return false; } bool parseSuccess = true; std::shared_ptr<VPackBuilder> parsedBody = parseVelocyPackBody(&VPackOptions::Defaults, parseSuccess); if (!parseSuccess) { generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, "expected POST " + EDGES_PATH + "/<collection-identifier>?direction=<direction>"); // A body is required return false; } VPackSlice body = parsedBody->slice(); if (!body.isArray()) { generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, "Expected an array of vertex _id's in body parameter"); return false; } std::string collectionName = suffix[0]; CollectionNameResolver resolver(_vocbase); TRI_col_type_e colType = resolver.getCollectionTypeCluster(collectionName); if (colType == TRI_COL_TYPE_UNKNOWN) { generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND); return false; } else if (colType != TRI_COL_TYPE_EDGE) { generateError(rest::ResponseCode::BAD, TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID); return false; } bool found; std::string dirString = _request->value("direction", found); if (!found || dirString.empty()) { dirString = "any"; } TRI_edge_direction_e direction; if (dirString == "any") { direction = TRI_EDGE_ANY; } else if (dirString == "out" || dirString == "outbound") { direction = TRI_EDGE_OUT; } else if (dirString == "in" || dirString == "inbound") { direction = TRI_EDGE_IN; } else { generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, "<direction> must by any, in, or out, not: " + dirString); return false; } std::vector<traverser::TraverserExpression*> const expressions; if (ServerState::instance()->isCoordinator()) { rest::ResponseCode responseCode; VPackBuilder resultDocument; resultDocument.openObject(); for (auto const& it : VPackArrayIterator(body)) { if (it.isString()) { std::string vertexString(it.copyString()); int res = getFilteredEdgesOnCoordinator( _vocbase->name(), collectionName, vertexString, direction, expressions, responseCode, resultDocument); if (res != TRI_ERROR_NO_ERROR) { generateError(responseCode, res); return false; } } } resultDocument.add("error", VPackValue(false)); resultDocument.add("code", VPackValue(200)); resultDocument.close(); generateResult(rest::ResponseCode::OK, resultDocument.slice()); return true; } // find and load collection given by name or identifier SingleCollectionTransaction trx( StandaloneTransactionContext::Create(_vocbase), collectionName, TRI_TRANSACTION_READ); // ............................................................................. // inside read transaction // ............................................................................. int res = trx.begin(); if (res != TRI_ERROR_NO_ERROR) { generateTransactionError(collectionName, res, ""); return false; } // If we are a DBserver, we want to use the cluster-wide collection // name for error reporting: if (ServerState::instance()->isDBServer()) { collectionName = trx.resolver()->getCollectionName(trx.cid()); } size_t filtered = 0; size_t scannedIndex = 0; VPackBuilder resultBuilder; resultBuilder.openObject(); // build edges resultBuilder.add(VPackValue("edges")); // only key resultBuilder.openArray(); bool ok = getEdgesForVertexList(body, expressions, direction, trx, resultBuilder, scannedIndex, filtered); if (!ok) { // Ignore the error } resultBuilder.close(); res = trx.finish(res); if (res != TRI_ERROR_NO_ERROR) { generateTransactionError(collectionName, res, ""); return false; } resultBuilder.add("error", VPackValue(false)); resultBuilder.add("code", VPackValue(200)); resultBuilder.add("stats", VPackValue(VPackValueType::Object)); resultBuilder.add("scannedIndex", VPackValue(scannedIndex)); resultBuilder.add("filtered", VPackValue(filtered)); resultBuilder.close(); // inner object resultBuilder.close(); // and generate a response generateResult(rest::ResponseCode::OK, resultBuilder.slice(), trx.transactionContext()); return true; }
size_t State::removeConflicts(query_t const& transactions) { // Under MUTEX in Agent VPackSlice slices = transactions->slice(); TRI_ASSERT(slices.isArray()); size_t ndups = 0; if (slices.length() > 0) { auto bindVars = std::make_shared<VPackBuilder>(); bindVars->openObject(); bindVars->close(); try { MUTEX_LOCKER(logLock, _logLock); auto idx = slices[0].get("index").getUInt(); auto pos = idx - _cur; if (pos < _log.size()) { for (auto const& slice : VPackArrayIterator(slices)) { auto trm = slice.get("term").getUInt(); idx = slice.get("index").getUInt(); pos = idx - _cur; if (pos < _log.size()) { if (idx == _log.at(pos).index && trm != _log.at(pos).term) { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Removing " << _log.size() - pos << " entries from log starting with " << idx << "==" << _log.at(pos).index << " and " << trm << "=" << _log.at(pos).term; // persisted logs std::string const aql(std::string("FOR l IN log FILTER l._key >= '") + stringify(idx) + "' REMOVE l IN log"); arangodb::aql::Query query( false, _vocbase, aql.c_str(), aql.size(), bindVars, nullptr, arangodb::aql::PART_MAIN); auto queryResult = query.execute(_queryRegistry); if (queryResult.code != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details); } // volatile logs _log.erase(_log.begin() + pos, _log.end()); break; } else { ++ndups; } } } } } catch (std::exception const& e) { LOG_TOPIC(DEBUG, Logger::AGENCY) << e.what() << " " << __FILE__ << __LINE__; } } return ndups; }
int RestoreFeature::processInputDirectory(std::string& errorMsg) { // create a lookup table for collections std::map<std::string, bool> restrictList; for (size_t i = 0; i < _collections.size(); ++i) { restrictList.insert(std::pair<std::string, bool>(_collections[i], true)); } try { std::vector<std::string> const files = FileUtils::listFiles(_inputDirectory); std::string const suffix = std::string(".structure.json"); std::vector<std::shared_ptr<VPackBuilder>> collectionBuilders; std::vector<VPackSlice> collections; // Step 1 determine all collections to process { // loop over all files in InputDirectory, and look for all structure.json // files for (std::string const& file : files) { size_t const nameLength = file.size(); if (nameLength <= suffix.size() || file.substr(file.size() - suffix.size()) != suffix) { // some other file continue; } // found a structure.json file std::string name = file.substr(0, file.size() - suffix.size()); if (!_includeSystemCollections && name[0] == '_') { continue; } std::string const fqn = _inputDirectory + TRI_DIR_SEPARATOR_STR + file; std::shared_ptr<VPackBuilder> fileContentBuilder = arangodb::basics::VelocyPackHelper::velocyPackFromFile(fqn); VPackSlice const fileContent = fileContentBuilder->slice(); if (!fileContent.isObject()) { errorMsg = "could not read collection structure file '" + fqn + "'"; return TRI_ERROR_INTERNAL; } VPackSlice const parameters = fileContent.get("parameters"); VPackSlice const indexes = fileContent.get("indexes"); if (!parameters.isObject() || !indexes.isArray()) { errorMsg = "could not read collection structure file '" + fqn + "'"; return TRI_ERROR_INTERNAL; } std::string const cname = arangodb::basics::VelocyPackHelper::getStringValue(parameters, "name", ""); bool overwriteName = false; if (cname != name && name != (cname + "_" + arangodb::rest::SslInterface::sslMD5(cname))) { // file has a different name than found in structure file if (_importStructure) { // we cannot go on if there is a mismatch errorMsg = "collection name mismatch in collection structure file '" + fqn + "' (offending value: '" + cname + "')"; return TRI_ERROR_INTERNAL; } else { // we can patch the name in our array and go on std::cout << "ignoring collection name mismatch in collection " "structure file '" + fqn + "' (offending value: '" + cname + "')" << std::endl; overwriteName = true; } } if (!restrictList.empty() && restrictList.find(cname) == restrictList.end()) { // collection name not in list continue; } if (overwriteName) { // TODO MAX // Situation: // Ich habe ein Json-Object von Datei (teile des Inhalts im Zweifel // unbekannt) // Es gibt ein Sub-Json-Object "parameters" mit einem Attribute "name" // der gesetzt ist. // Ich muss nur diesen namen überschreiben, der Rest soll identisch // bleiben. } else { collectionBuilders.emplace_back(fileContentBuilder); collections.emplace_back(fileContent); } } } std::sort(collections.begin(), collections.end(), SortCollections); StringBuffer buffer(TRI_UNKNOWN_MEM_ZONE); // step2: run the actual import for (VPackSlice const& collection : collections) { VPackSlice const parameters = collection.get("parameters"); VPackSlice const indexes = collection.get("indexes"); std::string const cname = arangodb::basics::VelocyPackHelper::getStringValue(parameters, "name", ""); int type = arangodb::basics::VelocyPackHelper::getNumericValue<int>( parameters, "type", 2); std::string const collectionType(type == 2 ? "document" : "edge"); if (_importStructure) { // re-create collection if (_progress) { if (_overwrite) { std::cout << "# Re-creating " << collectionType << " collection '" << cname << "'..." << std::endl; } else { std::cout << "# Creating " << collectionType << " collection '" << cname << "'..." << std::endl; } } int res = sendRestoreCollection(collection, cname, errorMsg); if (res != TRI_ERROR_NO_ERROR) { if (_force) { std::cerr << errorMsg << std::endl; continue; } return TRI_ERROR_INTERNAL; } } _stats._totalCollections++; if (_importData) { // import data. check if we have a datafile std::string datafile = _inputDirectory + TRI_DIR_SEPARATOR_STR + cname + "_" + arangodb::rest::SslInterface::sslMD5(cname) + ".data.json"; if (!TRI_ExistsFile(datafile.c_str())) { datafile = _inputDirectory + TRI_DIR_SEPARATOR_STR + cname + ".data.json"; } if (TRI_ExistsFile(datafile.c_str())) { // found a datafile if (_progress) { std::cout << "# Loading data into " << collectionType << " collection '" << cname << "'..." << std::endl; } int fd = TRI_OPEN(datafile.c_str(), O_RDONLY | TRI_O_CLOEXEC); if (fd < 0) { errorMsg = "cannot open collection data file '" + datafile + "'"; return TRI_ERROR_INTERNAL; } buffer.clear(); while (true) { if (buffer.reserve(16384) != TRI_ERROR_NO_ERROR) { TRI_CLOSE(fd); errorMsg = "out of memory"; return TRI_ERROR_OUT_OF_MEMORY; } ssize_t numRead = TRI_READ(fd, buffer.end(), 16384); if (numRead < 0) { // error while reading int res = TRI_errno(); TRI_CLOSE(fd); errorMsg = std::string(TRI_errno_string(res)); return res; } // read something buffer.increaseLength(numRead); _stats._totalRead += (uint64_t)numRead; if (buffer.length() < _chunkSize && numRead > 0) { // still continue reading continue; } // do we have a buffer? if (buffer.length() > 0) { // look for the last \n in the buffer char* found = (char*)memrchr((const void*)buffer.begin(), '\n', buffer.length()); size_t length; if (found == nullptr) { // no \n found... if (numRead == 0) { // we're at the end. send the complete buffer anyway length = buffer.length(); } else { // read more continue; } } else { // found a \n somewhere length = found - buffer.begin(); } TRI_ASSERT(length > 0); _stats._totalBatches++; int res = sendRestoreData(cname, buffer.begin(), length, errorMsg); if (res != TRI_ERROR_NO_ERROR) { TRI_CLOSE(fd); if (errorMsg.empty()) { errorMsg = std::string(TRI_errno_string(res)); } else { errorMsg = std::string(TRI_errno_string(res)) + ": " + errorMsg; } if (_force) { std::cerr << errorMsg << std::endl; continue; } return res; } buffer.erase_front(length); } if (numRead == 0) { // EOF break; } } TRI_CLOSE(fd); } } if (_importStructure) { // re-create indexes if (indexes.length() > 0) { // we actually have indexes if (_progress) { std::cout << "# Creating indexes for collection '" << cname << "'..." << std::endl; } int res = sendRestoreIndexes(collection, errorMsg); if (res != TRI_ERROR_NO_ERROR) { if (_force) { std::cerr << errorMsg << std::endl; continue; } return TRI_ERROR_INTERNAL; } } } } } catch (...) { errorMsg = "out of memory"; return TRI_ERROR_OUT_OF_MEMORY; } return TRI_ERROR_NO_ERROR; }
/// Get persisted information and run election process void Constituent::run() { TRI_ASSERT(_vocbase != nullptr); auto bindVars = std::make_shared<VPackBuilder>(); bindVars->openObject(); bindVars->close(); // Most recent vote std::string const aql("FOR l IN election SORT l._key DESC LIMIT 1 RETURN l"); arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars, nullptr, arangodb::aql::PART_MAIN); auto queryResult = query.execute(_queryRegistry); if (queryResult.code != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details); } VPackSlice result = queryResult.result->slice(); if (result.isArray()) { for (auto const& i : VPackArrayIterator(result)) { try { _term = i.get("term").getUInt(); _votedFor = static_cast<decltype(_votedFor)>(i.get("voted_for").getUInt()); } catch (std::exception const&) { LOG_TOPIC(ERR, Logger::AGENCY) << "Persisted election entries corrupt! Defaulting term,vote (0,0)"; } } } // Always start off as follower while (!this->isStopping() && size() > 1) { if (_role == FOLLOWER) { bool cast = false; { MUTEX_LOCKER(guard, _castLock); _cast = false; // New round set not cast vote } int32_t left = 1000000*config().minPing, right = 1000000*config().maxPing; long rand_wait = static_cast<long>(RandomGenerator::interval(left, right)); { CONDITION_LOCKER(guardv, _cv); _cv.wait(rand_wait); } { MUTEX_LOCKER(guard, _castLock); cast = _cast; } if (!cast) { candidate(); // Next round, we are running } } else { callElection(); // Run for office } } }
/// Get persisted information and run election process void Constituent::run() { // single instance _id = _agent->config().id(); TRI_ASSERT(_vocbase != nullptr); auto bindVars = std::make_shared<VPackBuilder>(); bindVars->openObject(); bindVars->close(); // Most recent vote std::string const aql("FOR l IN election SORT l._key DESC LIMIT 1 RETURN l"); arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars, nullptr, arangodb::aql::PART_MAIN); auto queryResult = query.execute(_queryRegistry); if (queryResult.code != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details); } VPackSlice result = queryResult.result->slice(); if (result.isArray()) { for (auto const& i : VPackArrayIterator(result)) { auto ii = i.resolveExternals(); try { MUTEX_LOCKER(locker, _castLock); _term = ii.get("term").getUInt(); _votedFor = ii.get("voted_for").copyString(); } catch (std::exception const&) { LOG_TOPIC(ERR, Logger::AGENCY) << "Persisted election entries corrupt! Defaulting term,vote (0,0)"; } } } std::vector<std::string> act = _agent->config().active(); while ( !this->isStopping() // Obvious && (!_agent->ready() || find(act.begin(), act.end(), _id) == act.end())) { // Active agent CONDITION_LOCKER(guardv, _cv); _cv.wait(50000); act = _agent->config().active(); } if (size() == 1) { _leaderID = _agent->config().id(); LOG_TOPIC(DEBUG, Logger::AGENCY) << "Set _leaderID to " << _leaderID << " in term " << _term; } else { while (!this->isStopping()) { if (_role == FOLLOWER) { static double const M = 1.0e6; int64_t a = static_cast<int64_t>(M * _agent->config().minPing()); int64_t b = static_cast<int64_t>(M * _agent->config().maxPing()); int64_t randTimeout = RandomGenerator::interval(a, b); int64_t randWait = randTimeout; { MUTEX_LOCKER(guard, _castLock); // in the beginning, pure random if (_lastHeartbeatSeen > 0.0) { double now = TRI_microtime(); randWait -= static_cast<int64_t>(M * (now - _lastHeartbeatSeen)); } } LOG_TOPIC(DEBUG, Logger::AGENCY) << "Random timeout: " << randTimeout << ", wait: " << randWait; if (randWait > 0.0) { CONDITION_LOCKER(guardv, _cv); _cv.wait(randWait); } bool isTimeout = false; { MUTEX_LOCKER(guard, _castLock); if (_lastHeartbeatSeen <= 0.0) { LOG_TOPIC(TRACE, Logger::AGENCY) << "no heartbeat seen"; isTimeout = true; } else { double diff = TRI_microtime() - _lastHeartbeatSeen; LOG_TOPIC(TRACE, Logger::AGENCY) << "last heartbeat: " << diff << "sec ago"; isTimeout = (static_cast<int32_t>(M * diff) > randTimeout); } } if (isTimeout) { LOG_TOPIC(TRACE, Logger::AGENCY) << "timeout, calling an election"; candidate(); } } else if (_role == CANDIDATE) { callElection(); // Run for office } else { int32_t left = static_cast<int32_t>(100000.0 * _agent->config().minPing()); long randTimeout = static_cast<long>(left); { CONDITION_LOCKER(guardv, _cv); _cv.wait(randTimeout); } } } } }
/// Load persisted configuration bool State::loadOrPersistConfiguration() { auto bindVars = std::make_shared<VPackBuilder>(); bindVars->openObject(); bindVars->close(); std::string const aql( std::string("FOR c in configuration FILTER c._key==\"0\" RETURN c.cfg")); arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars, nullptr, arangodb::aql::PART_MAIN); auto queryResult = query.execute(QueryRegistryFeature::QUERY_REGISTRY); if (queryResult.code != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details); } VPackSlice result = queryResult.result->slice(); if (result.isArray() && result.length()) { // We already have a persisted conf try { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Merging configuration " << result[0].resolveExternals().toJson(); _agent->mergeConfiguration(result[0].resolveExternals()); } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to merge persisted configuration into runtime " "configuration: " << e.what(); FATAL_ERROR_EXIT(); } } else { // Fresh start LOG_TOPIC(DEBUG, Logger::AGENCY) << "New agency!"; TRI_ASSERT(_agent != nullptr); _agent->id(to_string(boost::uuids::random_generator()())); auto transactionContext = std::make_shared<StandaloneTransactionContext>(_vocbase); SingleCollectionTransaction trx(transactionContext, "configuration", TRI_TRANSACTION_WRITE); int res = trx.begin(); OperationResult result; if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(res); } Builder doc; doc.openObject(); doc.add("_key", VPackValue("0")); doc.add("cfg", _agent->config().toBuilder()->slice()); doc.close(); try { result = trx.insert("configuration", doc.slice(), _options); } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to persist configuration entry:" << e.what(); FATAL_ERROR_EXIT(); } res = trx.finish(result.code); return (res == TRI_ERROR_NO_ERROR); } return true; }
arangodb::traverser::TraverserOptions::TraverserOptions( arangodb::aql::Query* query, VPackSlice info, VPackSlice collections) : _trx(query->trx()), _baseVertexExpression(nullptr), _tmpVar(nullptr), _ctx(new aql::FixedVarExpressionContext()), _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), minDepth(1), maxDepth(1), useBreadthFirst(false), uniqueVertices(UniquenessLevel::NONE), uniqueEdges(UniquenessLevel::PATH) { // NOTE collections is an array of arrays of strings VPackSlice read = info.get("minDepth"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a minDepth"); } minDepth = read.getNumber<uint64_t>(); read = info.get("maxDepth"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a maxDepth"); } maxDepth = read.getNumber<uint64_t>(); read = info.get("bfs"); if (!read.isBoolean()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a bfs"); } useBreadthFirst = read.getBool(); read = info.get("tmpVar"); if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a tmpVar"); } _tmpVar = query->ast()->variables()->createVariable(read); read = info.get("uniqueVertices"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a uniqueVertices"); } size_t i = read.getNumber<size_t>(); switch (i) { case 0: uniqueVertices = UniquenessLevel::NONE; break; case 1: uniqueVertices = UniquenessLevel::PATH; break; case 2: uniqueVertices = UniquenessLevel::GLOBAL; break; default: THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a uniqueVertices"); } read = info.get("uniqueEdges"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a uniqueEdges"); } i = read.getNumber<size_t>(); switch (i) { case 0: uniqueEdges = UniquenessLevel::NONE; break; case 1: uniqueEdges = UniquenessLevel::PATH; break; case 2: uniqueEdges = UniquenessLevel::GLOBAL; break; default: THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a uniqueEdges"); } read = info.get("baseLookupInfos"); if (!read.isArray()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a baseLookupInfos"); } size_t length = read.length(); TRI_ASSERT(read.length() == collections.length()); _baseLookupInfos.reserve(length); for (size_t j = 0; j < length; ++j) { _baseLookupInfos.emplace_back(query, read.at(j), collections.at(j)); } read = info.get("depthLookupInfo"); if (!read.isNone()) { if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "The options require depthLookupInfo to be an object"); } _depthLookupInfo.reserve(read.length()); for (auto const& depth : VPackObjectIterator(read)) { size_t d = basics::StringUtils::uint64(depth.key.copyString()); auto it = _depthLookupInfo.emplace(d, std::vector<LookupInfo>()); TRI_ASSERT(it.second); VPackSlice list = depth.value; TRI_ASSERT(length == list.length()); it.first->second.reserve(length); for (size_t j = 0; j < length; ++j) { it.first->second.emplace_back(query, list.at(j), collections.at(j)); } } } read = info.get("vertexExpressions"); if (!read.isNone()) { if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "The options require vertexExpressions to be an object"); } _vertexExpressions.reserve(read.length()); for (auto const& info : VPackObjectIterator(read)) { size_t d = basics::StringUtils::uint64(info.key.copyString()); #ifdef ARANGODB_ENABLE_MAINAINER_MODE auto it = _vertexExpressions.emplace( d, new aql::Expression(query->ast(), info.value)); TRI_ASSERT(it.second); #else _vertexExpressions.emplace( d, new aql::Expression(query->ast(), info.value)); #endif } } read = info.get("baseVertexExpression"); if (!read.isNone()) { if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "The options require vertexExpressions to be an object"); } _baseVertexExpression = new aql::Expression(query->ast(), read); } // Check for illegal option combination: TRI_ASSERT(uniqueEdges != arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL); TRI_ASSERT( uniqueVertices != arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL || useBreadthFirst); }