/// Log transactions (follower) arangodb::consensus::index_t State::log(query_t const& transactions, size_t ndups) { VPackSlice slices = transactions->slice(); TRI_ASSERT(slices.isArray()); size_t nqs = slices.length(); TRI_ASSERT(nqs > ndups); MUTEX_LOCKER(mutexLocker, _logLock); // log entries must stay in order for (size_t i = ndups; i < nqs; ++i) { VPackSlice slice = slices[i]; try { auto idx = slice.get("index").getUInt(); auto trm = slice.get("term").getUInt(); auto buf = std::make_shared<Buffer<uint8_t>>(); buf->append((char const*)slice.get("query").begin(), slice.get("query").byteSize()); // to RAM _log.push_back(log_t(idx, trm, buf)); // to disk persist(idx, trm, slice.get("query")); } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCY) << e.what() << " " << __FILE__ << __LINE__; } } TRI_ASSERT(!_log.empty()); return _log.back().index; }
static v8::Handle<v8::Value> ObjectVPackArray(v8::Isolate* isolate, VPackSlice const& slice, VPackOptions const* options, VPackSlice const* base) { TRI_ASSERT(slice.isArray()); v8::Handle<v8::Array> object = v8::Array::New(isolate, static_cast<int>(slice.length())); if (object.IsEmpty()) { return v8::Undefined(isolate); } uint32_t j = 0; VPackArrayIterator it(slice, true); while (it.valid()) { v8::Handle<v8::Value> val = TRI_VPackToV8(isolate, it.value(), options, &slice); if (!val.IsEmpty()) { object->Set(j++, val); } it.next(); } return object; }
void AuthInfo::reload() { insertInitial(); TRI_vocbase_t* vocbase = DatabaseFeature::DATABASE->systemDatabase(); if (vocbase == nullptr) { LOG(DEBUG) << "system database is unknown, cannot load authentication " << "and authorization information"; return; } MUTEX_LOCKER(locker, _queryLock); if (!_outdated) { return; } std::string queryStr("FOR user IN _users RETURN user"); auto nullBuilder = std::make_shared<VPackBuilder>(); VPackBuilder options; { VPackObjectBuilder b(&options); } auto objectBuilder = std::make_shared<VPackBuilder>(options); arangodb::aql::Query query(false, vocbase, queryStr.c_str(), queryStr.length(), nullBuilder, objectBuilder, arangodb::aql::PART_MAIN); LOG(DEBUG) << "starting to load authentication and authorization information"; TRI_ASSERT(_queryRegistry != nullptr); auto queryResult = query.execute(_queryRegistry); if (queryResult.code != TRI_ERROR_NO_ERROR) { if (queryResult.code == TRI_ERROR_REQUEST_CANCELED || (queryResult.code == TRI_ERROR_QUERY_KILLED)) { THROW_ARANGO_EXCEPTION(TRI_ERROR_REQUEST_CANCELED); } _outdated = false; return; } VPackSlice usersSlice = queryResult.result->slice(); if (usersSlice.isNone()) { THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); } if (!usersSlice.isArray()) { LOG(ERR) << "cannot read users from _users collection"; return; } if (usersSlice.length() == 0) { insertInitial(); } else { populate(usersSlice); } _outdated = false; }
double VelocyPackHelper::toDouble(VPackSlice const& slice, bool& failed) { TRI_ASSERT(!slice.isNone()); failed = false; switch (slice.type()) { case VPackValueType::None: case VPackValueType::Null: return 0.0; case VPackValueType::Bool: return (slice.getBoolean() ? 1.0 : 0.0); case VPackValueType::Double: case VPackValueType::Int: case VPackValueType::UInt: case VPackValueType::SmallInt: return slice.getNumericValue<double>(); case VPackValueType::String: { std::string tmp(slice.copyString()); try { // try converting string to number return std::stod(tmp); } catch (...) { if (tmp.empty()) { return 0.0; } // conversion failed } break; } case VPackValueType::Array: { VPackValueLength const n = slice.length(); if (n == 0) { return 0.0; } else if (n == 1) { return VelocyPackHelper::toDouble(slice.at(0).resolveExternal(), failed); } break; } case VPackValueType::External: { return VelocyPackHelper::toDouble(slice.resolveExternal(), failed); } case VPackValueType::Illegal: case VPackValueType::Object: case VPackValueType::UTCDate: case VPackValueType::MinKey: case VPackValueType::MaxKey: case VPackValueType::Binary: case VPackValueType::BCD: case VPackValueType::Custom: break; } failed = true; return 0.0; }
bool RestEdgesHandler::getEdgesForVertexList( VPackSlice const ids, std::vector<traverser::TraverserExpression*> const& expressions, TRI_edge_direction_e direction, SingleCollectionTransaction& trx, VPackBuilder& result, size_t& scannedIndex, size_t& filtered) { TRI_ASSERT(ids.isArray()); trx.orderDitch(trx.cid()); // will throw when it fails std::string const collectionName = trx.resolver()->getCollectionName(trx.cid()); Transaction::IndexHandle indexId = trx.edgeIndexHandle(collectionName); VPackBuilder searchValueBuilder; EdgeIndex::buildSearchValueFromArray(direction, ids, searchValueBuilder); VPackSlice search = searchValueBuilder.slice(); std::unique_ptr<OperationCursor> cursor = trx.indexScan(collectionName, arangodb::Transaction::CursorType::INDEX, indexId, search, nullptr, 0, UINT64_MAX, 1000, false); if (cursor->failed()) { THROW_ARANGO_EXCEPTION(cursor->code); } auto opRes = std::make_shared<OperationResult>(TRI_ERROR_NO_ERROR); while (cursor->hasMore()) { cursor->getMore(opRes); if (opRes->failed()) { THROW_ARANGO_EXCEPTION(opRes->code); } VPackSlice edges = opRes->slice(); TRI_ASSERT(edges.isArray()); // generate result scannedIndex += static_cast<size_t>(edges.length()); for (auto const& edge : VPackArrayIterator(edges)) { bool add = true; if (!expressions.empty()) { for (auto& exp : expressions) { if (exp->isEdgeAccess && !exp->matchesCheck(&trx, edge)) { ++filtered; add = false; break; } } } if (add) { result.add(edge); } } } return true; }
arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo( arangodb::aql::Query* query, VPackSlice const& info, VPackSlice const& shards) { TRI_ASSERT(shards.isArray()); idxHandles.reserve(shards.length()); conditionNeedUpdate = arangodb::basics::VelocyPackHelper::getBooleanValue( info, "condNeedUpdate", false); conditionMemberToUpdate = arangodb::basics::VelocyPackHelper::getNumericValue<size_t>( info, "condMemberToUpdate", 0); VPackSlice read = info.get("handle"); if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Each lookup requires handle to be an object"); } read = read.get("id"); if (!read.isString()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Each handle requires id to be a string"); } std::string idxId = read.copyString(); auto trx = query->trx(); for (auto const& it : VPackArrayIterator(shards)) { if (!it.isString()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Shards have to be a list of strings"); } idxHandles.emplace_back(trx->getIndexByIdentifier(it.copyString(), idxId)); } read = info.get("expression"); if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Each lookup requires expression to be an object"); } expression = new aql::Expression(query->ast(), read); read = info.get("condition"); if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "Each lookup requires condition to be an object"); } indexCondition = new aql::AstNode(query->ast(), read); }
/// Load compaction collection bool State::loadCompacted() { auto bindVars = std::make_shared<VPackBuilder>(); bindVars->openObject(); bindVars->close(); std::string const aql( std::string("FOR c IN compact SORT c._key DESC LIMIT 1 RETURN c")); arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars, nullptr, arangodb::aql::PART_MAIN); auto queryResult = query.execute(QueryRegistryFeature::QUERY_REGISTRY); if (queryResult.code != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details); } VPackSlice result = queryResult.result->slice(); if (result.isArray() && result.length()) { MUTEX_LOCKER(logLock, _logLock); for (auto const& i : VPackArrayIterator(result)) { auto ii = i.resolveExternals(); buffer_t tmp = std::make_shared<arangodb::velocypack::Buffer<uint8_t>>(); (*_agent) = ii; try { _cur = std::stoul(ii.get("_key").copyString()); } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCY) << e.what() << " " << __FILE__ << __LINE__; } } } return true; }
/// Internal function for optimized edge retrieval. /// Allows to send an TraverserExpression for filtering in the body /// Not publicly documented on purpose. bool RestEdgesHandler::readFilteredEdges() { std::vector<traverser::TraverserExpression*> expressions; bool parseSuccess = true; std::shared_ptr<VPackBuilder> parsedBody = parseVelocyPackBody(&VPackOptions::Defaults, parseSuccess); if (!parseSuccess) { // We continue unfiltered // Filter could be done by caller return readEdges(expressions); } VPackSlice body = parsedBody->slice(); arangodb::basics::ScopeGuard guard{[]() -> void {}, [&expressions]() -> void { for (auto& e : expressions) { delete e; } }}; if (!body.isArray()) { generateError( rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, "Expected an array of traverser expressions as body parameter"); return false; } expressions.reserve(static_cast<size_t>(body.length())); for (auto const& exp : VPackArrayIterator(body)) { if (exp.isObject()) { auto expression = std::make_unique<traverser::TraverserExpression>(exp); expressions.emplace_back(expression.get()); expression.release(); } } return readEdges(expressions); }
int RestoreFeature::processInputDirectory(std::string& errorMsg) { // create a lookup table for collections std::map<std::string, bool> restrictList; for (size_t i = 0; i < _collections.size(); ++i) { restrictList.insert(std::pair<std::string, bool>(_collections[i], true)); } try { std::vector<std::string> const files = FileUtils::listFiles(_inputDirectory); std::string const suffix = std::string(".structure.json"); std::vector<std::shared_ptr<VPackBuilder>> collectionBuilders; std::vector<VPackSlice> collections; // Step 1 determine all collections to process { // loop over all files in InputDirectory, and look for all structure.json // files for (std::string const& file : files) { size_t const nameLength = file.size(); if (nameLength <= suffix.size() || file.substr(file.size() - suffix.size()) != suffix) { // some other file continue; } // found a structure.json file std::string name = file.substr(0, file.size() - suffix.size()); if (!_includeSystemCollections && name[0] == '_') { continue; } std::string const fqn = _inputDirectory + TRI_DIR_SEPARATOR_STR + file; std::shared_ptr<VPackBuilder> fileContentBuilder = arangodb::basics::VelocyPackHelper::velocyPackFromFile(fqn); VPackSlice const fileContent = fileContentBuilder->slice(); if (!fileContent.isObject()) { errorMsg = "could not read collection structure file '" + fqn + "'"; return TRI_ERROR_INTERNAL; } VPackSlice const parameters = fileContent.get("parameters"); VPackSlice const indexes = fileContent.get("indexes"); if (!parameters.isObject() || !indexes.isArray()) { errorMsg = "could not read collection structure file '" + fqn + "'"; return TRI_ERROR_INTERNAL; } std::string const cname = arangodb::basics::VelocyPackHelper::getStringValue(parameters, "name", ""); bool overwriteName = false; if (cname != name && name != (cname + "_" + arangodb::rest::SslInterface::sslMD5(cname))) { // file has a different name than found in structure file if (_importStructure) { // we cannot go on if there is a mismatch errorMsg = "collection name mismatch in collection structure file '" + fqn + "' (offending value: '" + cname + "')"; return TRI_ERROR_INTERNAL; } else { // we can patch the name in our array and go on std::cout << "ignoring collection name mismatch in collection " "structure file '" + fqn + "' (offending value: '" + cname + "')" << std::endl; overwriteName = true; } } if (!restrictList.empty() && restrictList.find(cname) == restrictList.end()) { // collection name not in list continue; } if (overwriteName) { // TODO MAX // Situation: // Ich habe ein Json-Object von Datei (teile des Inhalts im Zweifel // unbekannt) // Es gibt ein Sub-Json-Object "parameters" mit einem Attribute "name" // der gesetzt ist. // Ich muss nur diesen namen überschreiben, der Rest soll identisch // bleiben. } else { collectionBuilders.emplace_back(fileContentBuilder); collections.emplace_back(fileContent); } } } std::sort(collections.begin(), collections.end(), SortCollections); StringBuffer buffer(TRI_UNKNOWN_MEM_ZONE); // step2: run the actual import for (VPackSlice const& collection : collections) { VPackSlice const parameters = collection.get("parameters"); VPackSlice const indexes = collection.get("indexes"); std::string const cname = arangodb::basics::VelocyPackHelper::getStringValue(parameters, "name", ""); int type = arangodb::basics::VelocyPackHelper::getNumericValue<int>( parameters, "type", 2); std::string const collectionType(type == 2 ? "document" : "edge"); if (_importStructure) { // re-create collection if (_progress) { if (_overwrite) { std::cout << "# Re-creating " << collectionType << " collection '" << cname << "'..." << std::endl; } else { std::cout << "# Creating " << collectionType << " collection '" << cname << "'..." << std::endl; } } int res = sendRestoreCollection(collection, cname, errorMsg); if (res != TRI_ERROR_NO_ERROR) { if (_force) { std::cerr << errorMsg << std::endl; continue; } return TRI_ERROR_INTERNAL; } } _stats._totalCollections++; if (_importData) { // import data. check if we have a datafile std::string datafile = _inputDirectory + TRI_DIR_SEPARATOR_STR + cname + "_" + arangodb::rest::SslInterface::sslMD5(cname) + ".data.json"; if (!TRI_ExistsFile(datafile.c_str())) { datafile = _inputDirectory + TRI_DIR_SEPARATOR_STR + cname + ".data.json"; } if (TRI_ExistsFile(datafile.c_str())) { // found a datafile if (_progress) { std::cout << "# Loading data into " << collectionType << " collection '" << cname << "'..." << std::endl; } int fd = TRI_OPEN(datafile.c_str(), O_RDONLY | TRI_O_CLOEXEC); if (fd < 0) { errorMsg = "cannot open collection data file '" + datafile + "'"; return TRI_ERROR_INTERNAL; } buffer.clear(); while (true) { if (buffer.reserve(16384) != TRI_ERROR_NO_ERROR) { TRI_CLOSE(fd); errorMsg = "out of memory"; return TRI_ERROR_OUT_OF_MEMORY; } ssize_t numRead = TRI_READ(fd, buffer.end(), 16384); if (numRead < 0) { // error while reading int res = TRI_errno(); TRI_CLOSE(fd); errorMsg = std::string(TRI_errno_string(res)); return res; } // read something buffer.increaseLength(numRead); _stats._totalRead += (uint64_t)numRead; if (buffer.length() < _chunkSize && numRead > 0) { // still continue reading continue; } // do we have a buffer? if (buffer.length() > 0) { // look for the last \n in the buffer char* found = (char*)memrchr((const void*)buffer.begin(), '\n', buffer.length()); size_t length; if (found == nullptr) { // no \n found... if (numRead == 0) { // we're at the end. send the complete buffer anyway length = buffer.length(); } else { // read more continue; } } else { // found a \n somewhere length = found - buffer.begin(); } TRI_ASSERT(length > 0); _stats._totalBatches++; int res = sendRestoreData(cname, buffer.begin(), length, errorMsg); if (res != TRI_ERROR_NO_ERROR) { TRI_CLOSE(fd); if (errorMsg.empty()) { errorMsg = std::string(TRI_errno_string(res)); } else { errorMsg = std::string(TRI_errno_string(res)) + ": " + errorMsg; } if (_force) { std::cerr << errorMsg << std::endl; continue; } return res; } buffer.erase_front(length); } if (numRead == 0) { // EOF break; } } TRI_CLOSE(fd); } } if (_importStructure) { // re-create indexes if (indexes.length() > 0) { // we actually have indexes if (_progress) { std::cout << "# Creating indexes for collection '" << cname << "'..." << std::endl; } int res = sendRestoreIndexes(collection, errorMsg); if (res != TRI_ERROR_NO_ERROR) { if (_force) { std::cerr << errorMsg << std::endl; continue; } return TRI_ERROR_INTERNAL; } } } } } catch (...) { errorMsg = "out of memory"; return TRI_ERROR_OUT_OF_MEMORY; } return TRI_ERROR_NO_ERROR; }
/// @brief create the block from VelocyPack, note that this can throw AqlItemBlock::AqlItemBlock(VPackSlice const slice) { bool exhausted = VelocyPackHelper::getBooleanValue(slice, "exhausted", false); if (exhausted) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "exhausted must be false"); } _nrItems = VelocyPackHelper::getNumericValue<size_t>(slice, "nrItems", 0); if (_nrItems == 0) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "nrItems must be > 0"); } _nrRegs = VelocyPackHelper::getNumericValue<RegisterId>(slice, "nrRegs", 0); // Initialize the data vector: if (_nrRegs > 0) { _data.resize(_nrItems * _nrRegs); } // Now put in the data: VPackSlice data = slice.get("data"); VPackSlice raw = slice.get("raw"); std::vector<AqlValue> madeHere; madeHere.reserve(static_cast<size_t>(raw.length())); madeHere.emplace_back(); // an empty AqlValue madeHere.emplace_back(); // another empty AqlValue, indices start w. 2 try { size_t posInRaw = 2; size_t posInData = 0; int64_t emptyRun = 0; for (RegisterId column = 0; column < _nrRegs; column++) { for (size_t i = 0; i < _nrItems; i++) { if (emptyRun > 0) { emptyRun--; } else { VPackSlice dataEntry = data.at(posInData++); if (!dataEntry.isNumber()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "data must contain only numbers"); } int64_t n = dataEntry.getNumericValue<int64_t>(); if (n == 0) { // empty, do nothing here } else if (n == -1) { // empty run: VPackSlice runLength = data.at(posInData++); TRI_ASSERT(runLength.isNumber()); emptyRun = runLength.getNumericValue<int64_t>(); emptyRun--; } else if (n == -2) { // a range VPackSlice lowBound = data.at(posInData++); VPackSlice highBound = data.at(posInData++); int64_t low = VelocyPackHelper::getNumericValue<int64_t>(lowBound, 0); int64_t high = VelocyPackHelper::getNumericValue<int64_t>(highBound, 0); AqlValue a(low, high); try { setValue(i, column, a); } catch (...) { a.destroy(); throw; } } else if (n == 1) { // a VelocyPack value AqlValue a(raw.at(posInRaw++)); try { setValue(i, column, a); // if this throws, a is destroyed again } catch (...) { a.destroy(); throw; } madeHere.emplace_back(a); } else if (n >= 2) { setValue(i, column, madeHere[static_cast<size_t>(n)]); // If this throws, all is OK, because it was already put into // the block elsewhere. } else { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "found undefined data value"); } } } } } catch (...) { destroy(); } }
int VelocyPackHelper::compare(VPackSlice lhs, VPackSlice rhs, bool useUTF8, VPackOptions const* options, VPackSlice const* lhsBase, VPackSlice const* rhsBase) { { // will resolve externals... int lWeight = TypeWeight(lhs); int rWeight = TypeWeight(rhs); if (lWeight < rWeight) { return -1; } if (lWeight > rWeight) { return 1; } TRI_ASSERT(lWeight == rWeight); } lhs = lhs.resolveExternal(); // follow externals rhs = rhs.resolveExternal(); // follow externals // lhs and rhs have equal weights if (lhs.isNone() || rhs.isNone()) { // either lhs or rhs is none. we cannot be sure here that both are // nones. // there can also exist the situation that lhs is a none and rhs is a // null value // (or vice versa). Anyway, the compare value is the same for both, return 0; } auto lhsType = lhs.type(); switch (lhsType) { case VPackValueType::Illegal: case VPackValueType::MinKey: case VPackValueType::MaxKey: case VPackValueType::None: case VPackValueType::Null: return 0; case VPackValueType::Bool: { bool left = lhs.getBoolean(); bool right = rhs.getBoolean(); if (left == right) { return 0; } if (!left && right) { return -1; } return 1; } case VPackValueType::Double: case VPackValueType::Int: case VPackValueType::UInt: case VPackValueType::SmallInt: { return compareNumberValues(lhsType, lhs, rhs); } case VPackValueType::Custom: case VPackValueType::String: { std::string lhsString; VPackValueLength nl; char const* left; if (lhs.isCustom()) { if (lhsBase == nullptr || options == nullptr || options->customTypeHandler == nullptr) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "Could not extract custom attribute."); } lhsString.assign(options->customTypeHandler->toString(lhs, options, *lhsBase)); left = lhsString.c_str(); nl = lhsString.size(); } else { left = lhs.getString(nl); } TRI_ASSERT(left != nullptr); std::string rhsString; VPackValueLength nr; char const* right; if (rhs.isCustom()) { if (rhsBase == nullptr || options == nullptr || options->customTypeHandler == nullptr) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "Could not extract custom attribute."); } rhsString.assign(options->customTypeHandler->toString(rhs, options, *rhsBase)); right = rhsString.c_str(); nr = rhsString.size(); } else { right = rhs.getString(nr); } TRI_ASSERT(right != nullptr); int res; if (useUTF8) { res = TRI_compare_utf8(left, static_cast<size_t>(nl), right, static_cast<size_t>(nr)); } else { size_t len = static_cast<size_t>(nl < nr ? nl : nr); res = memcmp(left, right, len); } if (res < 0) { return -1; } if (res > 0) { return 1; } // res == 0 if (nl == nr) { return 0; } // res == 0, but different string lengths return nl < nr ? -1 : 1; } case VPackValueType::Array: { VPackValueLength const nl = lhs.length(); VPackValueLength const nr = rhs.length(); VPackValueLength const n = (std::max)(nr, nl); for (VPackValueLength i = 0; i < n; ++i) { VPackSlice lhsValue; if (i < nl) { lhsValue = lhs.at(i).resolveExternal(); } VPackSlice rhsValue; if (i < nr) { rhsValue = rhs.at(i).resolveExternal(); } int result = compare(lhsValue, rhsValue, useUTF8, options, &lhs, &rhs); if (result != 0) { return result; } } return 0; } case VPackValueType::Object: { std::set<std::string, AttributeSorterUTF8> keys; VPackCollection::keys(lhs, keys); VPackCollection::keys(rhs, keys); for (auto const& key : keys) { VPackSlice lhsValue = lhs.get(key).resolveExternal(); if (lhsValue.isNone()) { // not present => null lhsValue = VPackSlice::nullSlice(); } VPackSlice rhsValue = rhs.get(key).resolveExternal(); if (rhsValue.isNone()) { // not present => null rhsValue = VPackSlice::nullSlice(); } int result = compare(lhsValue, rhsValue, useUTF8, options, &lhs, &rhs); if (result != 0) { return result; } } return 0; } default: // Contains all other ValueTypes of VelocyPack. // They are not used in ArangoDB so this cannot occur TRI_ASSERT(false); return 0; } }
arangodb::traverser::TraverserOptions::TraverserOptions( arangodb::aql::Query* query, VPackSlice info, VPackSlice collections) : _trx(query->trx()), _baseVertexExpression(nullptr), _tmpVar(nullptr), _ctx(new aql::FixedVarExpressionContext()), _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), minDepth(1), maxDepth(1), useBreadthFirst(false), uniqueVertices(UniquenessLevel::NONE), uniqueEdges(UniquenessLevel::PATH) { // NOTE collections is an array of arrays of strings VPackSlice read = info.get("minDepth"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a minDepth"); } minDepth = read.getNumber<uint64_t>(); read = info.get("maxDepth"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a maxDepth"); } maxDepth = read.getNumber<uint64_t>(); read = info.get("bfs"); if (!read.isBoolean()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a bfs"); } useBreadthFirst = read.getBool(); read = info.get("tmpVar"); if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a tmpVar"); } _tmpVar = query->ast()->variables()->createVariable(read); read = info.get("uniqueVertices"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a uniqueVertices"); } size_t i = read.getNumber<size_t>(); switch (i) { case 0: uniqueVertices = UniquenessLevel::NONE; break; case 1: uniqueVertices = UniquenessLevel::PATH; break; case 2: uniqueVertices = UniquenessLevel::GLOBAL; break; default: THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a uniqueVertices"); } read = info.get("uniqueEdges"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a uniqueEdges"); } i = read.getNumber<size_t>(); switch (i) { case 0: uniqueEdges = UniquenessLevel::NONE; break; case 1: uniqueEdges = UniquenessLevel::PATH; break; case 2: uniqueEdges = UniquenessLevel::GLOBAL; break; default: THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a uniqueEdges"); } read = info.get("baseLookupInfos"); if (!read.isArray()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "The options require a baseLookupInfos"); } size_t length = read.length(); TRI_ASSERT(read.length() == collections.length()); _baseLookupInfos.reserve(length); for (size_t j = 0; j < length; ++j) { _baseLookupInfos.emplace_back(query, read.at(j), collections.at(j)); } read = info.get("depthLookupInfo"); if (!read.isNone()) { if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "The options require depthLookupInfo to be an object"); } _depthLookupInfo.reserve(read.length()); for (auto const& depth : VPackObjectIterator(read)) { size_t d = basics::StringUtils::uint64(depth.key.copyString()); auto it = _depthLookupInfo.emplace(d, std::vector<LookupInfo>()); TRI_ASSERT(it.second); VPackSlice list = depth.value; TRI_ASSERT(length == list.length()); it.first->second.reserve(length); for (size_t j = 0; j < length; ++j) { it.first->second.emplace_back(query, list.at(j), collections.at(j)); } } } read = info.get("vertexExpressions"); if (!read.isNone()) { if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "The options require vertexExpressions to be an object"); } _vertexExpressions.reserve(read.length()); for (auto const& info : VPackObjectIterator(read)) { size_t d = basics::StringUtils::uint64(info.key.copyString()); #ifdef ARANGODB_ENABLE_MAINAINER_MODE auto it = _vertexExpressions.emplace( d, new aql::Expression(query->ast(), info.value)); TRI_ASSERT(it.second); #else _vertexExpressions.emplace( d, new aql::Expression(query->ast(), info.value)); #endif } } read = info.get("baseVertexExpression"); if (!read.isNone()) { if (!read.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "The options require vertexExpressions to be an object"); } _baseVertexExpression = new aql::Expression(query->ast(), read); } // Check for illegal option combination: TRI_ASSERT(uniqueEdges != arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL); TRI_ASSERT( uniqueVertices != arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL || useBreadthFirst); }
/// Load persisted configuration bool State::loadOrPersistConfiguration() { auto bindVars = std::make_shared<VPackBuilder>(); bindVars->openObject(); bindVars->close(); std::string const aql( std::string("FOR c in configuration FILTER c._key==\"0\" RETURN c.cfg")); arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars, nullptr, arangodb::aql::PART_MAIN); auto queryResult = query.execute(QueryRegistryFeature::QUERY_REGISTRY); if (queryResult.code != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details); } VPackSlice result = queryResult.result->slice(); if (result.isArray() && result.length()) { // We already have a persisted conf try { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Merging configuration " << result[0].resolveExternals().toJson(); _agent->mergeConfiguration(result[0].resolveExternals()); } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to merge persisted configuration into runtime " "configuration: " << e.what(); FATAL_ERROR_EXIT(); } } else { // Fresh start LOG_TOPIC(DEBUG, Logger::AGENCY) << "New agency!"; TRI_ASSERT(_agent != nullptr); _agent->id(to_string(boost::uuids::random_generator()())); auto transactionContext = std::make_shared<StandaloneTransactionContext>(_vocbase); SingleCollectionTransaction trx(transactionContext, "configuration", TRI_TRANSACTION_WRITE); int res = trx.begin(); OperationResult result; if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(res); } Builder doc; doc.openObject(); doc.add("_key", VPackValue("0")); doc.add("cfg", _agent->config().toBuilder()->slice()); doc.close(); try { result = trx.insert("configuration", doc.slice(), _options); } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to persist configuration entry:" << e.what(); FATAL_ERROR_EXIT(); } res = trx.finish(result.code); return (res == TRI_ERROR_NO_ERROR); } return true; }
size_t State::removeConflicts(query_t const& transactions) { // Under MUTEX in Agent VPackSlice slices = transactions->slice(); TRI_ASSERT(slices.isArray()); size_t ndups = 0; if (slices.length() > 0) { auto bindVars = std::make_shared<VPackBuilder>(); bindVars->openObject(); bindVars->close(); try { MUTEX_LOCKER(logLock, _logLock); auto idx = slices[0].get("index").getUInt(); auto pos = idx - _cur; if (pos < _log.size()) { for (auto const& slice : VPackArrayIterator(slices)) { auto trm = slice.get("term").getUInt(); idx = slice.get("index").getUInt(); pos = idx - _cur; if (pos < _log.size()) { if (idx == _log.at(pos).index && trm != _log.at(pos).term) { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Removing " << _log.size() - pos << " entries from log starting with " << idx << "==" << _log.at(pos).index << " and " << trm << "=" << _log.at(pos).term; // persisted logs std::string const aql(std::string("FOR l IN log FILTER l._key >= '") + stringify(idx) + "' REMOVE l IN log"); arangodb::aql::Query query( false, _vocbase, aql.c_str(), aql.size(), bindVars, nullptr, arangodb::aql::PART_MAIN); auto queryResult = query.execute(_queryRegistry); if (queryResult.code != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details); } // volatile logs _log.erase(_log.begin() + pos, _log.end()); break; } else { ++ndups; } } } } } catch (std::exception const& e) { LOG_TOPIC(DEBUG, Logger::AGENCY) << e.what() << " " << __FILE__ << __LINE__; } } return ndups; }