int Syncer::createCollection(VPackSlice const& slice, arangodb::LogicalCollection** dst) {
if (dst != nullptr) {
*dst = nullptr;
}
if (!slice.isObject()) {
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
std::string const name = VelocyPackHelper::getStringValue(slice, "name", "");
if (name.empty()) {
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
TRI_voc_cid_t const cid = getCid(slice);
if (cid == 0) {
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
TRI_col_type_e const type = static_cast<TRI_col_type_e>(VelocyPackHelper::getNumericValue<int>(
slice, "type", TRI_COL_TYPE_DOCUMENT));
arangodb::LogicalCollection* col = getCollectionByIdOrName(cid, name);
if (col != nullptr && col->type() == type) {
// collection already exists. TODO: compare attributes
return TRI_ERROR_NO_ERROR;
}
// merge in "isSystem" attribute
VPackBuilder s;
s.openObject();
s.add("isSystem", VPackValue(true));
s.close();
VPackBuilder merged = VPackCollection::merge(s.slice(), slice, true);
int res = TRI_ERROR_NO_ERROR;
try {
col = _vocbase->createCollection(merged.slice(), cid, true);
} catch (basics::Exception const& ex) {
res = ex.code();
} catch (...) {
res = TRI_ERROR_INTERNAL;
}
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
TRI_ASSERT(col != nullptr);
if (dst != nullptr) {
*dst = col;
}
return TRI_ERROR_NO_ERROR;
}
std::string AuthInfo::generateJwt(VPackBuilder const& payload) {
if (!payload.slice().isObject()) {
std::string error = "Need an object to generate a JWT. Got: ";
error += payload.slice().typeName();
throw std::runtime_error(error);
}
bool hasIss = payload.slice().hasKey("iss");
bool hasIat = payload.slice().hasKey("iat");
VPackBuilder bodyBuilder;
if (hasIss && hasIat) {
bodyBuilder = payload;
} else {
VPackObjectBuilder p(&bodyBuilder);
if (!hasIss) {
bodyBuilder.add("iss", VPackValue("arangodb"));
}
if (!hasIat) {
bodyBuilder.add("iat", VPackValue(TRI_microtime() / 1000));
}
for (auto const& obj : VPackObjectIterator(payload.slice())) {
bodyBuilder.add(obj.key.copyString(), obj.value);
}
}
return generateRawJwt(bodyBuilder);
}
int Syncer::createCollection(VPackSlice const& slice, TRI_vocbase_col_t** dst) {
if (dst != nullptr) {
*dst = nullptr;
}
if (!slice.isObject()) {
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
std::string const name = VelocyPackHelper::getStringValue(slice, "name", "");
if (name.empty()) {
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
TRI_voc_cid_t const cid = getCid(slice);
if (cid == 0) {
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
TRI_col_type_e const type = static_cast<TRI_col_type_e>(VelocyPackHelper::getNumericValue<int>(
slice, "type", static_cast<int>(TRI_COL_TYPE_DOCUMENT)));
TRI_vocbase_col_t* col = getCollectionByIdOrName(cid, name);
if (col != nullptr && static_cast<TRI_col_type_t>(col->_type) == static_cast<TRI_col_type_t>(type)) {
// collection already exists. TODO: compare attributes
return TRI_ERROR_NO_ERROR;
}
// merge in "isSystem" attribute
VPackBuilder s;
s.openObject();
s.add("isSystem", VPackValue(true));
s.close();
VPackBuilder merged = VPackCollection::merge(s.slice(), slice, true);
VocbaseCollectionInfo params(_vocbase, name.c_str(), merged.slice());
col = TRI_CreateCollectionVocBase(_vocbase, params, cid, true);
if (col == nullptr) {
return TRI_errno();
}
if (dst != nullptr) {
*dst = col;
}
return TRI_ERROR_NO_ERROR;
}
/// @brief initializeCursor, could be called multiple times
int RemoteBlock::initializeCursor(AqlItemBlock* items, size_t pos) {
DEBUG_BEGIN_BLOCK();
// For every call we simply forward via HTTP
if (!_isResponsibleForInitializeCursor) {
// do nothing...
return TRI_ERROR_NO_ERROR;
}
VPackBuilder builder;
builder.openObject();
if (items == nullptr) {
// first call, items is still a nullptr
builder.add("exhausted", VPackValue(true));
builder.add("error", VPackValue(false));
} else {
builder.add("exhausted", VPackValue(false));
builder.add("error", VPackValue(false));
builder.add("pos", VPackValue(pos));
builder.add(VPackValue("items"));
builder.openObject();
items->toVelocyPack(_engine->getQuery()->trx(), builder);
builder.close();
}
builder.close();
std::string bodyString(builder.slice().toJson());
std::unique_ptr<ClusterCommResult> res = sendRequest(
rest::RequestType::PUT, "/_api/aql/initializeCursor/", bodyString);
throwExceptionAfterBadSyncRequest(res.get(), false);
// If we get here, then res->result is the response which will be
// a serialized AqlItemBlock:
StringBuffer const& responseBodyBuf(res->result->getBody());
{
std::shared_ptr<VPackBuilder> builder = VPackParser::fromJson(
responseBodyBuf.c_str(), responseBodyBuf.length());
VPackSlice slice = builder->slice();
if (slice.hasKey("code")) {
return slice.get("code").getNumericValue<int>();
}
return TRI_ERROR_INTERNAL;
}
// cppcheck-suppress style
DEBUG_END_BLOCK();
}
uint64_t VelocyPackHelper::hashByAttributes(
VPackSlice slice, std::vector<std::string> const& attributes,
bool docComplete, int& error, std::string const& key) {
uint64_t hash = TRI_FnvHashBlockInitial();
error = TRI_ERROR_NO_ERROR;
slice = slice.resolveExternal();
if (slice.isObject()) {
for (auto const& attr : attributes) {
VPackSlice sub = slice.get(attr).resolveExternal();
if (sub.isNone()) {
if (attr == StaticStrings::KeyString && !key.empty()) {
VPackBuilder temporaryBuilder;
temporaryBuilder.add(VPackValue(key));
hash = temporaryBuilder.slice().normalizedHash(hash);
continue;
}
if (!docComplete) {
error = TRI_ERROR_CLUSTER_NOT_ALL_SHARDING_ATTRIBUTES_GIVEN;
}
// Null is equal to None/not present
sub = VPackSlice::nullSlice();
}
hash = sub.normalizedHash(hash);
}
}
return hash;
}
HttpHandler::status_t RestVersionHandler::execute() {
try {
VPackBuilder result;
result.add(VPackValue(VPackValueType::Object));
result.add("server", VPackValue("arango"));
result.add("version", VPackValue(ARANGODB_VERSION));
bool found;
std::string const& detailsStr = _request->value("details", found);
if (found && StringUtils::boolean(detailsStr)) {
result.add("details", VPackValue(VPackValueType::Object));
Version::getVPack(result);
if (application_features::ApplicationServer::server != nullptr) {
auto server = application_features::ApplicationServer::server
->getFeature<ServerFeature>("Server");
result.add("mode", VPackValue(server->operationModeString()));
}
result.close();
}
result.close();
generateResult(GeneralResponse::ResponseCode::OK, result.slice());
} catch (...) {
// Ignore this error
}
return status_t(HANDLER_DONE);
}
void operator()(VPackSlice const& v, std::vector<VPackSlice>& resEdges,
std::vector<VPackSlice>& neighbors) {
int res = TRI_ERROR_NO_ERROR;
for (auto const& edgeCollection : _block->_collectionInfos) {
VPackBuilder result;
TRI_ASSERT(edgeCollection != nullptr);
if (_isReverse) {
res = edgeCollection->getReverseEdgesCoordinator(v, result);
} else {
res = edgeCollection->getEdgesCoordinator(v, result);
}
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
VPackSlice edges = result.slice().get("edges");
for (auto const& edge : VPackArrayIterator(edges)) {
VPackSlice from = arangodb::Transaction::extractFromFromDocument(edge);
if (from == v) {
VPackSlice to = arangodb::Transaction::extractToFromDocument(edge);
if (to != v) {
resEdges.emplace_back(edge);
neighbors.emplace_back(to);
}
} else {
resEdges.emplace_back(edge);
neighbors.emplace_back(from);
}
}
// Make sure the data Slices are pointing to is not running out of scope.
// This is not thread-safe!
_block->_coordinatorCache.emplace_back(result.steal());
}
}
void RestWalHandler::transactions() {
auto const& info =
arangodb::wal::LogfileManager::instance()->runningTransactions();
VPackBuilder builder;
builder.openObject();
builder.add("runningTransactions", VPackValue(static_cast<double>(std::get<0>(info))));
// lastCollectedId
{
auto value = std::get<1>(info);
if (value == UINT64_MAX) {
builder.add("minLastCollected", VPackValue(VPackValueType::Null));
} else {
builder.add("minLastCollected", VPackValue(value));
}
}
// lastSealedId
{
auto value = std::get<2>(info);
if (value == UINT64_MAX) {
builder.add("minLastSealed", VPackValue(VPackValueType::Null));
} else {
builder.add("minLastSealed", VPackValue(value));
}
}
builder.close();
generateResult(rest::ResponseCode::OK, builder.slice());
}
void RestVocbaseBaseHandler::generatePreconditionFailed(
VPackSlice const& slice) {
createResponse(GeneralResponse::ResponseCode::PRECONDITION_FAILED);
if (slice.isObject()) { // single document case
std::string const rev =
VelocyPackHelper::getStringValue(slice, StaticStrings::RevString, "");
_response->setHeaderNC(StaticStrings::Etag, "\"" + rev + "\"");
}
VPackBuilder builder;
{
VPackObjectBuilder guard(&builder);
builder.add("error", VPackValue(true));
builder.add("code",
VPackValue(static_cast<int32_t>(
GeneralResponse::ResponseCode::PRECONDITION_FAILED)));
builder.add("errorNum", VPackValue(TRI_ERROR_ARANGO_CONFLICT));
builder.add("errorMessage", VPackValue("precondition failed"));
if (slice.isObject()) {
builder.add(StaticStrings::IdString, slice.get(StaticStrings::IdString));
builder.add(StaticStrings::KeyString,
slice.get(StaticStrings::KeyString));
builder.add(StaticStrings::RevString,
slice.get(StaticStrings::RevString));
} else {
builder.add("result", slice);
}
}
auto transactionContext(StandaloneTransactionContext::Create(_vocbase));
writeResult(builder.slice(), *(transactionContext->getVPackOptionsForDump()));
}
bool RestEdgesHandler::getEdgesForVertexList(
VPackSlice const ids,
std::vector<traverser::TraverserExpression*> const& expressions,
TRI_edge_direction_e direction, SingleCollectionTransaction& trx,
VPackBuilder& result, size_t& scannedIndex, size_t& filtered) {
TRI_ASSERT(ids.isArray());
trx.orderDitch(trx.cid()); // will throw when it fails
std::string const collectionName =
trx.resolver()->getCollectionName(trx.cid());
Transaction::IndexHandle indexId = trx.edgeIndexHandle(collectionName);
VPackBuilder searchValueBuilder;
EdgeIndex::buildSearchValueFromArray(direction, ids, searchValueBuilder);
VPackSlice search = searchValueBuilder.slice();
std::unique_ptr<OperationCursor> cursor =
trx.indexScan(collectionName, arangodb::Transaction::CursorType::INDEX,
indexId, search, nullptr, 0, UINT64_MAX, 1000, false);
if (cursor->failed()) {
THROW_ARANGO_EXCEPTION(cursor->code);
}
auto opRes = std::make_shared<OperationResult>(TRI_ERROR_NO_ERROR);
while (cursor->hasMore()) {
cursor->getMore(opRes);
if (opRes->failed()) {
THROW_ARANGO_EXCEPTION(opRes->code);
}
VPackSlice edges = opRes->slice();
TRI_ASSERT(edges.isArray());
// generate result
scannedIndex += static_cast<size_t>(edges.length());
for (auto const& edge : VPackArrayIterator(edges)) {
bool add = true;
if (!expressions.empty()) {
for (auto& exp : expressions) {
if (exp->isEdgeAccess && !exp->matchesCheck(&trx, edge)) {
++filtered;
add = false;
break;
}
}
}
if (add) {
result.add(edge);
}
}
}
return true;
}
void operator()(VPackSlice const& source,
std::vector<ArangoDBPathFinder::Step*>& result) {
int res = TRI_ERROR_NO_ERROR;
for (auto const& edgeCollection : _block->_collectionInfos) {
TRI_ASSERT(edgeCollection != nullptr);
VPackBuilder edgesBuilder;
if (_reverse) {
res = edgeCollection->getReverseEdgesCoordinator(source, edgesBuilder);
} else {
res = edgeCollection->getEdgesCoordinator(source, edgesBuilder);
}
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
std::unordered_map<VPackSlice, size_t> candidates;
auto inserter = [&](VPackSlice const& s, VPackSlice const& t,
double currentWeight, VPackSlice const& edge) {
auto cand = candidates.find(t);
if (cand == candidates.end()) {
// Add weight
auto step = std::make_unique<ArangoDBPathFinder::Step>(
t, s, currentWeight, std::move(edge));
result.emplace_back(step.release());
candidates.emplace(t, result.size() - 1);
} else {
// Compare weight
auto old = result[cand->second];
auto oldWeight = old->weight();
if (currentWeight < oldWeight) {
old->setWeight(currentWeight);
old->_predecessor = s;
old->_edge = edge;
}
}
};
VPackSlice edges = edgesBuilder.slice().get("edges");
for (auto const& edge : VPackArrayIterator(edges)) {
VPackSlice from = arangodb::Transaction::extractFromFromDocument(edge);
VPackSlice to = arangodb::Transaction::extractToFromDocument(edge);
double currentWeight = edgeCollection->weightEdge(edge);
if (from == source) {
inserter(from, to, currentWeight, edge);
} else {
inserter(to, from, currentWeight, edge);
}
}
_block->_coordinatorCache.emplace_back(edgesBuilder.steal());
}
}
HttpHandler::status_t RestAuthHandler::execute() {
auto const type = _request->requestType();
if (type != GeneralRequest::RequestType::POST) {
generateError(GeneralResponse::ResponseCode::METHOD_NOT_ALLOWED,
TRI_ERROR_HTTP_METHOD_NOT_ALLOWED);
return status_t(HANDLER_DONE);
}
VPackOptions options = VPackOptions::Defaults;
options.checkAttributeUniqueness = true;
bool parseSuccess;
std::shared_ptr<VPackBuilder> parsedBody =
parseVelocyPackBody(&options, parseSuccess);
if (!parseSuccess) {
return badRequest();
}
VPackSlice slice = parsedBody->slice();
if (!slice.isObject()) {
return badRequest();
}
VPackSlice usernameSlice = slice.get("username");
VPackSlice passwordSlice = slice.get("password");
if (!usernameSlice.isString() || !passwordSlice.isString()) {
return badRequest();
}
std::string const username = usernameSlice.copyString();
std::string const password = passwordSlice.copyString();
AuthResult auth = RestServerFeature::AUTH_INFO.checkPassword(username, password);
if (auth._authorized) {
VPackBuilder resultBuilder;
{
VPackObjectBuilder b(&resultBuilder);
std::string jwt = generateJwt(username, password);
resultBuilder.add("jwt", VPackValue(jwt));
resultBuilder.add("must_change_password", VPackValue(auth._mustChange));
}
generateDocument(resultBuilder.slice(), true, &VPackOptions::Defaults);
return status_t(HANDLER_DONE);
} else {
// mop: rfc 2616 10.4.2 (if credentials wrong 401)
generateError(GeneralResponse::ResponseCode::UNAUTHORIZED, TRI_ERROR_HTTP_UNAUTHORIZED,
"Wrong credentials");
return status_t(HANDLER_DONE);
}
}
RestStatus RestShutdownHandler::execute() {
if (_request->requestType() != rest::RequestType::DELETE_REQ) {
generateError(rest::ResponseCode::METHOD_NOT_ALLOWED, 405);
return RestStatus::DONE;
}
bool removeFromCluster;
std::string const& remove = _request->value("remove_from_cluster", removeFromCluster);
removeFromCluster = removeFromCluster && remove == "1";
bool shutdownClusterFound;
std::string const& shutdownCluster = _request->value("shutdown_cluster", shutdownClusterFound);
if (shutdownClusterFound && shutdownCluster == "1") {
AgencyComm agency;
VPackBuilder builder;
builder.add(VPackValue(true));
AgencyCommResult result = agency.setValue("Shutdown", builder.slice(), 0.0);
if (!result.successful()) {
generateError(rest::ResponseCode::SERVER_ERROR, 500);
return RestStatus::DONE;
}
removeFromCluster = true;
}
if (removeFromCluster) {
ClusterFeature* clusterFeature = ApplicationServer::getFeature<ClusterFeature>("Cluster");
clusterFeature->setUnregisterOnShutdown(true);
}
ApplicationServer::server->beginShutdown();
try {
VPackBuilder result;
result.add(VPackValue("OK"));
generateResult(rest::ResponseCode::OK, result.slice());
} catch (...) {
// Ignore the error
}
return RestStatus::DONE;
}
void RestVocbaseBaseHandler::generatePreconditionFailed(
std::string const& collectionName, std::string const& key,
TRI_voc_rid_t rev) {
VPackBuilder builder;
builder.openObject();
builder.add(StaticStrings::IdString,
VPackValue(assembleDocumentId(collectionName, key, false)));
builder.add(StaticStrings::KeyString, VPackValue(key));
builder.add(StaticStrings::RevString, VPackValue(std::to_string(rev)));
builder.close();
generatePreconditionFailed(builder.slice());
}
/// @brief skipSome
size_t RemoteBlock::skipSome(size_t atLeast, size_t atMost) {
DEBUG_BEGIN_BLOCK();
// For every call we simply forward via HTTP
VPackBuilder builder;
builder.openObject();
builder.add("atLeast", VPackValue(atLeast));
builder.add("atMost", VPackValue(atMost));
builder.close();
std::string bodyString(builder.slice().toJson());
std::unique_ptr<ClusterCommResult> res =
sendRequest(rest::RequestType::PUT, "/_api/aql/skipSome/", bodyString);
throwExceptionAfterBadSyncRequest(res.get(), false);
// If we get here, then res->result is the response which will be
// a serialized AqlItemBlock:
StringBuffer const& responseBodyBuf(res->result->getBody());
{
std::shared_ptr<VPackBuilder> builder = VPackParser::fromJson(
responseBodyBuf.c_str(), responseBodyBuf.length());
VPackSlice slice = builder->slice();
if (!slice.hasKey("error") || slice.get("error").getBoolean()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_AQL_COMMUNICATION);
}
size_t skipped = 0;
if (slice.hasKey("skipped")) {
skipped = slice.get("skipped").getNumericValue<size_t>();
}
return skipped;
}
// cppcheck-suppress style
DEBUG_END_BLOCK();
}
void AuthInfo::insertInitial() {
if (!_authInfo.empty()) {
return;
}
try {
VPackBuilder builder;
builder.openArray();
// The only users object
builder.add(VPackValue(VPackValueType::Object));
// username
builder.add("user", VPackValue("root"));
builder.add("authData", VPackValue(VPackValueType::Object));
// simple auth
builder.add("simple", VPackValue(VPackValueType::Object));
builder.add("method", VPackValue("sha256"));
char const* salt = "c776f5f4";
builder.add("salt", VPackValue(salt));
char const* hash =
"ef74bc6fd59ac713bf5929c5ac2f42233e50d4d58748178132ea46dec433bd5b";
builder.add("hash", VPackValue(hash));
builder.close(); // simple
builder.add("active", VPackValue(true));
builder.close(); // authData
builder.add("databases", VPackValue(VPackValueType::Object));
builder.add("*", VPackValue("rw"));
builder.close();
builder.close(); // The user object
builder.close(); // The Array
populate(builder.slice());
} catch (...) {
// No action
}
}
int RestoreFeature::tryCreateDatabase(ClientFeature* client,
std::string const& name) {
VPackBuilder builder;
builder.openObject();
builder.add("name", VPackValue(name));
builder.add("users", VPackValue(VPackValueType::Array));
builder.openObject();
builder.add("username", VPackValue(client->username()));
builder.add("passwd", VPackValue(client->password()));
builder.close();
builder.close();
builder.close();
std::string const body = builder.slice().toJson();
std::unique_ptr<SimpleHttpResult> response(
_httpClient->request(GeneralRequest::RequestType::POST, "/_api/database",
body.c_str(), body.size()));
if (response == nullptr || !response->isComplete()) {
return TRI_ERROR_INTERNAL;
}
auto returnCode = response->getHttpReturnCode();
if (returnCode == static_cast<int>(GeneralResponse::ResponseCode::OK) ||
returnCode == static_cast<int>(GeneralResponse::ResponseCode::CREATED)) {
// all ok
return TRI_ERROR_NO_ERROR;
}
if (returnCode == static_cast<int>(GeneralResponse::ResponseCode::UNAUTHORIZED) ||
returnCode == static_cast<int>(GeneralResponse::ResponseCode::FORBIDDEN)) {
// invalid authorization
_httpClient->setErrorMessage(getHttpErrorMessage(response.get(), nullptr),
false);
return TRI_ERROR_FORBIDDEN;
}
// any other error
_httpClient->setErrorMessage(getHttpErrorMessage(response.get(), nullptr),
false);
return TRI_ERROR_INTERNAL;
}
RestStatus RestAqlFunctionsHandler::execute() {
// extract the sub-request type
auto const type = _request->requestType();
if (type == rest::RequestType::GET) {
VPackBuilder builder;
builder.openObject();
builder.add(VPackValue("functions"));
aql::FunctionDefinitions::toVelocyPack(builder);
builder.close();
generateResult(rest::ResponseCode::OK, builder.slice());
return RestStatus::DONE;
}
generateError(rest::ResponseCode::METHOD_NOT_ALLOWED,
TRI_ERROR_HTTP_METHOD_NOT_ALLOWED);
return RestStatus::DONE;
}
/// @brief getSome
AqlItemBlock* RemoteBlock::getSome(size_t atLeast, size_t atMost) {
DEBUG_BEGIN_BLOCK();
// For every call we simply forward via HTTP
traceGetSomeBegin();
VPackBuilder builder;
builder.openObject();
builder.add("atLeast", VPackValue(atLeast));
builder.add("atMost", VPackValue(atMost));
builder.close();
std::string bodyString(builder.slice().toJson());
std::unique_ptr<ClusterCommResult> res =
sendRequest(rest::RequestType::PUT, "/_api/aql/getSome/", bodyString);
throwExceptionAfterBadSyncRequest(res.get(), false);
// If we get here, then res->result is the response which will be
// a serialized AqlItemBlock:
std::shared_ptr<VPackBuilder> responseBodyBuilder =
res->result->getBodyVelocyPack();
VPackSlice responseBody = responseBodyBuilder->slice();
ExecutionStats newStats(responseBody.get("stats"));
_engine->_stats.addDelta(_deltaStats, newStats);
_deltaStats = newStats;
if (VelocyPackHelper::getBooleanValue(responseBody, "exhausted", true)) {
traceGetSomeEnd(nullptr);
return nullptr;
}
auto r = new arangodb::aql::AqlItemBlock(responseBody);
traceGetSomeEnd(r);
return r;
// cppcheck-suppress style
DEBUG_END_BLOCK();
}
/// @brief hashes the value
uint64_t AqlValue::hash(arangodb::AqlTransaction* trx, uint64_t seed) const {
switch (type()) {
case VPACK_SLICE_POINTER:
case VPACK_INLINE:
case VPACK_MANAGED: {
// we must use the slow hash function here, because a value may have
// different representations in case its an array/object/number
return slice().normalizedHash(seed);
}
case DOCVEC:
case RANGE: {
VPackBuilder builder;
toVelocyPack(trx, builder, false);
// we must use the slow hash function here, because a value may have
// different representations in case its an array/object/number
return builder.slice().normalizedHash(seed);
}
}
return 0;
}
RestStatus RestDebugHandler::execute() {
// extract the sub-request type
auto const type = _request->requestType();
size_t const len = _request->suffix().size();
if (len == 0 || len > 2 || !(_request->suffix()[0] == "failat")) {
generateNotImplemented("ILLEGAL /_admin/debug/failat");
return RestStatus::DONE;
}
std::vector<std::string> const& suffix = _request->suffix();
// execute one of the CRUD methods
switch (type) {
case rest::RequestType::DELETE_REQ:
if (len == 1) {
TRI_ClearFailurePointsDebugging();
} else {
TRI_RemoveFailurePointDebugging(suffix[1].c_str());
}
break;
case rest::RequestType::PUT:
if (len == 2) {
TRI_AddFailurePointDebugging(suffix[1].c_str());
} else {
generateNotImplemented("ILLEGAL /_admin/debug/failat");
}
break;
default:
generateNotImplemented("ILLEGAL /_admin/debug/failat");
return RestStatus::DONE;
}
try {
VPackBuilder result;
result.add(VPackValue(true));
generateResult(rest::ResponseCode::OK, result.slice());
} catch (...) {
// Ignore this error
}
return RestStatus::DONE;
}
static void JS_VersionClient(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_V8_TRY_CATCH_BEGIN(isolate);
v8::HandleScope scope(isolate);
bool details = false;
if (args.Length() > 0) {
details = TRI_ObjectToBoolean(args[0]);
}
if (!details) {
// return version string
TRI_V8_RETURN(TRI_V8_ASCII_STRING(ARANGODB_VERSION));
}
// return version details
VPackBuilder builder;
builder.openObject();
rest::Version::getVPack(builder);
builder.close();
TRI_V8_RETURN(TRI_VPackToV8(isolate, builder.slice()));
TRI_V8_TRY_CATCH_END
}
arangodb::aql::Graph* arangodb::lookupGraphByName(TRI_vocbase_t* vocbase,
std::string const& name) {
SingleCollectionTransaction trx(StandaloneTransactionContext::Create(vocbase),
GRAPHS, TRI_TRANSACTION_READ);
int res = trx.begin();
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_FORMAT(res, "while looking up graph '%s'",
name.c_str());
}
VPackBuilder b;
{
VPackObjectBuilder guard(&b);
b.add(StaticStrings::KeyString, VPackValue(name));
}
// Default options are enough here
OperationOptions options;
OperationResult result = trx.document(GRAPHS, b.slice(), options);
// Commit or abort.
res = trx.finish(result.code);
if (!result.successful()) {
THROW_ARANGO_EXCEPTION_FORMAT(result.code, "while looking up graph '%s'",
name.c_str());
}
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_FORMAT(res, "while looking up graph '%s'",
name.c_str());
}
return new arangodb::aql::Graph(result.slice());
}
/// @brief sendToClient: for each row of the incoming AqlItemBlock use the
/// attributes <shardKeys> of the Aql value <val> to determine to which shard
/// the row should be sent and return its clientId
size_t DistributeBlock::sendToClient(AqlItemBlock* cur) {
DEBUG_BEGIN_BLOCK();
// inspect cur in row _pos and check to which shard it should be sent . .
AqlValue val = cur->getValueReference(_pos, _regId);
VPackSlice input = val.slice(); // will throw when wrong type
bool usedAlternativeRegId = false;
if (input.isNull() && _alternativeRegId != ExecutionNode::MaxRegisterId) {
// value is set, but null
// check if there is a second input register available (UPSERT makes use of
// two input registers,
// one for the search document, the other for the insert document)
val = cur->getValueReference(_pos, _alternativeRegId);
input = val.slice(); // will throw when wrong type
usedAlternativeRegId = true;
}
VPackSlice value = input;
VPackBuilder builder;
VPackBuilder builder2;
bool hasCreatedKeyAttribute = false;
if (input.isString() &&
static_cast<DistributeNode const*>(_exeNode)
->_allowKeyConversionToObject) {
builder.openObject();
builder.add(StaticStrings::KeyString, input);
builder.close();
// clear the previous value
cur->destroyValue(_pos, _regId);
// overwrite with new value
cur->setValue(_pos, _regId, AqlValue(builder));
value = builder.slice();
hasCreatedKeyAttribute = true;
} else if (!input.isObject()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_DOCUMENT_TYPE_INVALID);
}
TRI_ASSERT(value.isObject());
if (static_cast<DistributeNode const*>(_exeNode)->_createKeys) {
// we are responsible for creating keys if none present
if (_usesDefaultSharding) {
// the collection is sharded by _key...
if (!hasCreatedKeyAttribute && !value.hasKey(StaticStrings::KeyString)) {
// there is no _key attribute present, so we are responsible for
// creating one
VPackBuilder temp;
temp.openObject();
temp.add(StaticStrings::KeyString, VPackValue(createKey(value)));
temp.close();
builder2 = VPackCollection::merge(input, temp.slice(), true);
// clear the previous value and overwrite with new value:
if (usedAlternativeRegId) {
cur->destroyValue(_pos, _alternativeRegId);
cur->setValue(_pos, _alternativeRegId, AqlValue(builder2));
} else {
cur->destroyValue(_pos, _regId);
cur->setValue(_pos, _regId, AqlValue(builder2));
}
value = builder2.slice();
}
} else {
// the collection is not sharded by _key
if (hasCreatedKeyAttribute || value.hasKey(StaticStrings::KeyString)) {
// a _key was given, but user is not allowed to specify _key
if (usedAlternativeRegId || !_allowSpecifiedKeys) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_MUST_NOT_SPECIFY_KEY);
}
} else {
VPackBuilder temp;
temp.openObject();
temp.add(StaticStrings::KeyString, VPackValue(createKey(value)));
temp.close();
builder2 = VPackCollection::merge(input, temp.slice(), true);
// clear the previous value and overwrite with new value:
if (usedAlternativeRegId) {
cur->destroyValue(_pos, _alternativeRegId);
cur->setValue(_pos, _alternativeRegId, AqlValue(builder2.slice()));
} else {
cur->destroyValue(_pos, _regId);
cur->setValue(_pos, _regId, AqlValue(builder2.slice()));
}
value = builder2.slice();
}
}
}
std::string shardId;
bool usesDefaultShardingAttributes;
auto clusterInfo = arangodb::ClusterInfo::instance();
auto collInfo = _collection->getCollection();
int res = clusterInfo->getResponsibleShard(collInfo.get(), value, true,
shardId, usesDefaultShardingAttributes);
// std::cout << "SHARDID: " << shardId << "\n";
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
TRI_ASSERT(!shardId.empty());
return getClientId(shardId);
// cppcheck-suppress style
DEBUG_END_BLOCK();
}
AqlItemBlock* ShortestPathBlock::getSome(size_t, size_t atMost) {
DEBUG_BEGIN_BLOCK();
if (_done) {
return nullptr;
}
if (_buffer.empty()) {
size_t toFetch = (std::min)(DefaultBatchSize(), atMost);
if (!ExecutionBlock::getBlock(toFetch, toFetch)) {
_done = true;
return nullptr;
}
_pos = 0; // this is in the first block
}
// If we get here, we do have _buffer.front()
AqlItemBlock* cur = _buffer.front();
size_t const curRegs = cur->getNrRegs();
// Collect the next path:
if (_posInPath >= _pathLength) {
if (!nextPath(cur)) {
// This input does not have any path. maybe the next one has.
// we can only return nullptr iff the buffer is empty.
if (++_pos >= cur->size()) {
_buffer.pop_front(); // does not throw
delete cur;
_pos = 0;
}
return getSome(atMost, atMost);
}
}
size_t available = _pathLength - _posInPath;
size_t toSend = (std::min)(atMost, available);
RegisterId nrRegs =
getPlanNode()->getRegisterPlan()->nrRegs[getPlanNode()->getDepth()];
std::unique_ptr<AqlItemBlock> res(requestBlock(toSend, nrRegs));
// automatically freed if we throw
TRI_ASSERT(curRegs <= res->getNrRegs());
// only copy 1st row of registers inherited from previous frame(s)
inheritRegisters(cur, res.get(), _pos);
// TODO this might be optimized in favor of direct mptr.
VPackBuilder resultBuilder;
for (size_t j = 0; j < toSend; j++) {
if (usesVertexOutput()) {
// TODO this might be optimized in favor of direct mptr.
resultBuilder.clear();
_path->vertexToVelocyPack(_trx, _posInPath, resultBuilder);
res->setValue(j, _vertexReg, AqlValue(resultBuilder.slice()));
}
if (usesEdgeOutput()) {
// TODO this might be optimized in favor of direct mptr.
resultBuilder.clear();
_path->edgeToVelocyPack(_trx, _posInPath, resultBuilder);
res->setValue(j, _edgeReg, AqlValue(resultBuilder.slice()));
}
if (j > 0) {
// re-use already copied aqlvalues
res->copyValuesFromFirstRow(j, static_cast<RegisterId>(curRegs));
}
++_posInPath;
}
if (_posInPath >= _pathLength) {
// Advance read position for next call
if (++_pos >= cur->size()) {
_buffer.pop_front(); // does not throw
delete cur;
_pos = 0;
}
}
// Clear out registers no longer needed later:
clearRegisters(res.get());
return res.release();
DEBUG_END_BLOCK();
}
int RocksDBFeature::dropPrefix(std::string const& prefix) {
if (!isEnabled()) {
return TRI_ERROR_NO_ERROR;
}
TRI_ASSERT(Instance != nullptr);
try {
VPackBuilder builder;
// create lower and upper bound for deletion
builder.openArray();
builder.add(VPackSlice::minKeySlice());
builder.close();
std::string l;
l.reserve(prefix.size() + builder.slice().byteSize());
l.append(prefix);
// extend the prefix to at least 24 bytes
while (l.size() < RocksDBIndex::keyPrefixSize()) {
uint64_t value = 0;
l.append(reinterpret_cast<char const*>(&value), sizeof(uint64_t));
}
l.append(builder.slice().startAs<char const>(), builder.slice().byteSize());
builder.clear();
builder.openArray();
builder.add(VPackSlice::maxKeySlice());
builder.close();
std::string u;
u.reserve(prefix.size() + builder.slice().byteSize());
u.append(prefix);
// extend the prefix to at least 24 bytes
while (u.size() < RocksDBIndex::keyPrefixSize()) {
uint64_t value = UINT64_MAX;
u.append(reinterpret_cast<char const*>(&value), sizeof(uint64_t));
}
u.append(builder.slice().startAs<char const>(), builder.slice().byteSize());
#if 0
for (size_t i = 0; i < prefix.size(); i += sizeof(TRI_idx_iid_t)) {
char const* x = prefix.c_str() + i;
size_t o;
char* q = TRI_EncodeHexString(x, 8, &o);
if (q != nullptr) {
LOG(TRACE) << "RocksDB prefix part: " << q;
TRI_FreeString(TRI_CORE_MEM_ZONE, q);
}
}
LOG(TRACE) << "dropping RocksDB range: " << VPackSlice(l.c_str() + RocksDBIndex::keyPrefixSize()).toJson() << " - " << VPackSlice(u.c_str() + RocksDBIndex::keyPrefixSize()).toJson();
#endif
// delete files in range lower..upper
rocksdb::Slice lower(l.c_str(), l.size());
rocksdb::Slice upper(u.c_str(), u.size());
{
rocksdb::Status status = rocksdb::DeleteFilesInRange(_db->GetBaseDB(), _db->GetBaseDB()->DefaultColumnFamily(), &lower, &upper);
if (!status.ok()) {
// if file deletion failed, we will still iterate over the remaining keys, so we
// don't need to abort and raise an error here
LOG(WARN) << "RocksDB file deletion failed";
}
}
// go on and delete the remaining keys (delete files in range does not necessarily
// find them all, just complete files)
auto comparator = RocksDBFeature::instance()->comparator();
rocksdb::DB* db = _db->GetBaseDB();
rocksdb::WriteBatch batch;
std::unique_ptr<rocksdb::Iterator> it(db->NewIterator(rocksdb::ReadOptions()));
it->Seek(lower);
while (it->Valid()) {
int res = comparator->Compare(it->key(), upper);
if (res >= 0) {
break;
}
batch.Delete(it->key());
it->Next();
}
// now apply deletion batch
rocksdb::Status status = db->Write(rocksdb::WriteOptions(), &batch);
if (!status.ok()) {
LOG(WARN) << "RocksDB key deletion failed: " << status.ToString();
return TRI_ERROR_INTERNAL;
}
return TRI_ERROR_NO_ERROR;
} catch (arangodb::basics::Exception const& ex) {
LOG(ERR) << "caught exception during RocksDB key prefix deletion: " << ex.what();
return ex.code();
} catch (std::exception const& ex) {
LOG(ERR) << "caught exception during RocksDB key prefix deletion: " << ex.what();
return TRI_ERROR_INTERNAL;
} catch (...) {
LOG(ERR) << "caught unknown exception during RocksDB key prefix deletion";
return TRI_ERROR_INTERNAL;
}
}
static void raceForClusterBootstrap() {
AgencyComm agency;
auto ci = ClusterInfo::instance();
while (true) {
AgencyCommResult result = agency.getValues("Bootstrap");
if (!result.successful()) {
// Error in communication, note that value not found is not an error
LOG_TOPIC(TRACE, Logger::STARTUP)
<< "raceForClusterBootstrap: no agency communication";
sleep(1);
continue;
}
VPackSlice value = result.slice()[0].get(
std::vector<std::string>({agency.prefix(), "Bootstrap"}));
if (value.isString()) {
// key was found and is a string
if (value.copyString().find("done") != std::string::npos) {
// all done, let's get out of here:
LOG_TOPIC(TRACE, Logger::STARTUP)
<< "raceForClusterBootstrap: bootstrap already done";
return;
}
LOG_TOPIC(DEBUG, Logger::STARTUP)
<< "raceForClusterBootstrap: somebody else does the bootstrap";
sleep(1);
continue;
}
// No value set, we try to do the bootstrap ourselves:
VPackBuilder b;
b.add(VPackValue(arangodb::ServerState::instance()->getId()));
result = agency.casValue("Bootstrap", b.slice(), false, 300, 15);
if (!result.successful()) {
LOG_TOPIC(DEBUG, Logger::STARTUP)
<< "raceForClusterBootstrap: lost race, somebody else will bootstrap";
// Cannot get foot into the door, try again later:
sleep(1);
continue;
}
// OK, we handle things now, let's see whether a DBserver is there:
auto dbservers = ci->getCurrentDBServers();
if (dbservers.size() == 0) {
LOG_TOPIC(TRACE, Logger::STARTUP)
<< "raceForClusterBootstrap: no DBservers, waiting";
agency.removeValues("Bootstrap", false);
sleep(1);
continue;
}
LOG_TOPIC(DEBUG, Logger::STARTUP)
<< "raceForClusterBootstrap: race won, we do the bootstrap";
auto vocbase = DatabaseFeature::DATABASE->systemDatabase();
V8DealerFeature::DEALER->loadJavascriptFiles(vocbase, "server/bootstrap/cluster-bootstrap.js", 0);
LOG_TOPIC(DEBUG, Logger::STARTUP)
<< "raceForClusterBootstrap: bootstrap done";
b.clear();
b.add(VPackValue(arangodb::ServerState::instance()->getId() + ": done"));
result = agency.setValue("Bootstrap", b.slice(), 0);
if (result.successful()) {
return;
}
LOG_TOPIC(TRACE, Logger::STARTUP)
<< "raceForClusterBootstrap: could not indicate success";
sleep(1);
}
}
void ExportCursor::dump(arangodb::basics::StringBuffer& buffer) {
auto transactionContext = std::make_shared<StandaloneTransactionContext>(_vocbase);
VPackOptions* options = transactionContext->getVPackOptions();
TRI_ASSERT(_ex != nullptr);
auto const restrictionType = _ex->_restrictions.type;
buffer.appendText("\"result\":[");
VPackBuilder result;
size_t const n = batchSize();
for (size_t i = 0; i < n; ++i) {
if (!hasNext()) {
break;
}
if (i > 0) {
buffer.appendChar(',');
}
VPackSlice const slice(reinterpret_cast<char const*>(_ex->_documents->at(_position++)));
{
result.clear();
VPackObjectBuilder b(&result);
// Copy over shaped values
for (auto const& entry : VPackObjectIterator(slice)) {
std::string key(entry.key.copyString());
if (!IncludeAttribute(restrictionType, _ex->_restrictions.fields, key)) {
// Ignore everything that should be excluded or not included
continue;
}
// If we get here we need this entry in the final result
if (entry.value.isCustom()) {
result.add(key, VPackValue(options->customTypeHandler->toString(entry.value, options, slice)));
} else {
result.add(key, entry.value);
}
}
}
arangodb::basics::VPackStringBufferAdapter bufferAdapter(buffer.stringBuffer());
try {
VPackDumper dumper(&bufferAdapter, options);
dumper.dump(result.slice());
} catch (arangodb::basics::Exception const& ex) {
THROW_ARANGO_EXCEPTION_MESSAGE(ex.code(), ex.what());
} catch (std::exception const& ex) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, ex.what());
} catch (...) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
}
}
buffer.appendText("],\"hasMore\":");
buffer.appendText(hasNext() ? "true" : "false");
if (hasNext()) {
// only return cursor id if there are more documents
buffer.appendText(",\"id\":\"");
buffer.appendInteger(id());
buffer.appendText("\"");
}
if (hasCount()) {
buffer.appendText(",\"count\":");
buffer.appendInteger(static_cast<uint64_t>(count()));
}
if (!hasNext()) {
delete _ex;
_ex = nullptr;
// mark the cursor as deleted
this->deleted();
}
}
/// @brief toJson, transfer a whole AqlItemBlock to Json, the result can
/// be used to recreate the AqlItemBlock via the Json constructor
/// Here is a description of the data format: The resulting Json has
/// the following attributes:
/// "nrItems": the number of rows of the AqlItemBlock
/// "nrRegs": the number of registers of the AqlItemBlock
/// "error": always set to false
/// "data": this contains the actual data in the form of a list of
/// numbers. The AqlItemBlock is stored columnwise, starting
/// from the first column (top to bottom) and going right.
/// Each entry found is encoded in the following way:
/// 0.0 means a single empty entry
/// -1.0 followed by a positive integer N (encoded as number)
/// means a run of that many empty entries
/// -2.0 followed by two numbers LOW and HIGH means a range
/// and LOW and HIGH are the boundaries (inclusive)
/// 1.0 means a JSON entry at the "next" position in "raw"
/// the "next" position starts with 2 and is increased
/// by one for every 1.0 found in data
/// integer values >= 2.0 mean a JSON entry, in this
/// case the "raw" list contains an entry in the
/// corresponding position
/// "raw": List of actual values, positions 0 and 1 are always null
/// such that actual indices start at 2
void AqlItemBlock::toVelocyPack(arangodb::AqlTransaction* trx,
VPackBuilder& result) const {
VPackBuilder data;
data.openArray();
VPackBuilder raw;
raw.openArray();
// Two nulls in the beginning such that indices start with 2
raw.add(VPackValue(VPackValueType::Null));
raw.add(VPackValue(VPackValueType::Null));
std::unordered_map<AqlValue, size_t> table; // remember duplicates
size_t emptyCount = 0; // here we count runs of empty AqlValues
auto commitEmpties = [&]() { // this commits an empty run to the data
if (emptyCount > 0) {
if (emptyCount == 1) {
data.add(VPackValue(0));
} else {
data.add(VPackValue(-1));
data.add(VPackValue(emptyCount));
}
emptyCount = 0;
}
};
size_t pos = 2; // write position in raw
for (RegisterId column = 0; column < _nrRegs; column++) {
for (size_t i = 0; i < _nrItems; i++) {
AqlValue const& a(_data[i * _nrRegs + column]);
if (a.isEmpty()) {
emptyCount++;
} else {
commitEmpties();
if (a.isRange()) {
data.add(VPackValue(-2));
data.add(VPackValue(a.range()->_low));
data.add(VPackValue(a.range()->_high));
} else {
auto it = table.find(a);
if (it == table.end()) {
a.toVelocyPack(trx, raw, false);
data.add(VPackValue(1));
table.emplace(a, pos++);
} else {
data.add(VPackValue(it->second));
}
}
}
}
}
commitEmpties();
raw.close();
data.close();
result.add("nrItems", VPackValue(_nrItems));
result.add("nrRegs", VPackValue(_nrRegs));
result.add("data", data.slice());
result.add("raw", raw.slice());
result.add("error", VPackValue(false));
result.add("exhausted", VPackValue(false));
}
void ClusterFeature::start() {
// return if cluster is disabled
if (!_enableCluster) {
return;
}
ServerState::instance()->setState(ServerState::STATE_STARTUP);
// the agency about our state
AgencyComm comm;
comm.sendServerState(0.0);
std::string const version = comm.getVersion();
ServerState::instance()->setInitialized();
std::string const endpoints = AgencyComm::getEndpointsString();
ServerState::RoleEnum role = ServerState::instance()->getRole();
LOG(INFO) << "Cluster feature is turned on. Agency version: " << version
<< ", Agency endpoints: " << endpoints << ", server id: '" << _myId
<< "', internal address: " << _myAddress
<< ", role: " << ServerState::roleToString(role);
if (!_disableHeartbeat) {
AgencyCommResult result = comm.getValues("Sync/HeartbeatIntervalMs");
if (result.successful()) {
velocypack::Slice HeartbeatIntervalMs =
result.slice()[0].get(std::vector<std::string>(
{AgencyComm::prefix(), "Sync", "HeartbeatIntervalMs"}));
if (HeartbeatIntervalMs.isInteger()) {
try {
_heartbeatInterval = HeartbeatIntervalMs.getUInt();
LOG(INFO) << "using heartbeat interval value '" << _heartbeatInterval
<< " ms' from agency";
} catch (...) {
// Ignore if it is not a small int or uint
}
}
}
// no value set in agency. use default
if (_heartbeatInterval == 0) {
_heartbeatInterval = 5000; // 1/s
LOG(WARN) << "unable to read heartbeat interval from agency. Using "
<< "default value '" << _heartbeatInterval << " ms'";
}
// start heartbeat thread
_heartbeatThread = std::make_shared<HeartbeatThread>(
_agencyCallbackRegistry.get(), _heartbeatInterval * 1000, 5,
SchedulerFeature::SCHEDULER->ioService());
if (!_heartbeatThread->init() || !_heartbeatThread->start()) {
LOG(FATAL) << "heartbeat could not connect to agency endpoints ("
<< endpoints << ")";
FATAL_ERROR_EXIT();
}
while (!_heartbeatThread->isReady()) {
// wait until heartbeat is ready
usleep(10000);
}
}
AgencyCommResult result;
while (true) {
VPackBuilder builder;
try {
VPackObjectBuilder b(&builder);
builder.add("endpoint", VPackValue(_myAddress));
} catch (...) {
LOG(FATAL) << "out of memory";
FATAL_ERROR_EXIT();
}
result = comm.setValue("Current/ServersRegistered/" + _myId,
builder.slice(), 0.0);
if (!result.successful()) {
LOG(FATAL) << "unable to register server in agency: http code: "
<< result.httpCode() << ", body: " << result.body();
FATAL_ERROR_EXIT();
} else {
break;
}
sleep(1);
}
if (role == ServerState::ROLE_COORDINATOR) {
ServerState::instance()->setState(ServerState::STATE_SERVING);
} else if (role == ServerState::ROLE_PRIMARY) {
ServerState::instance()->setState(ServerState::STATE_SERVINGASYNC);
} else if (role == ServerState::ROLE_SECONDARY) {
ServerState::instance()->setState(ServerState::STATE_SYNCING);
}
}
