bool AuthInfo::validateJwtHeader(std::string const& header) {
std::shared_ptr<VPackBuilder> headerBuilder =
parseJson(StringUtils::decodeBase64(header), "jwt header");
if (headerBuilder.get() == nullptr) {
return false;
}
VPackSlice const headerSlice = headerBuilder->slice();
if (!headerSlice.isObject()) {
return false;
}
VPackSlice const algSlice = headerSlice.get("alg");
VPackSlice const typSlice = headerSlice.get("typ");
if (!algSlice.isString()) {
return false;
}
if (!typSlice.isString()) {
return false;
}
if (algSlice.copyString() != "HS256") {
return false;
}
std::string typ = typSlice.copyString();
if (typ != "JWT") {
return false;
}
return true;
}
AuthJwtResult AuthInfo::validateJwtBody(std::string const& body) {
std::shared_ptr<VPackBuilder> bodyBuilder =
parseJson(StringUtils::decodeBase64(body), "jwt body");
AuthJwtResult authResult;
if (bodyBuilder.get() == nullptr) {
return authResult;
}
VPackSlice const bodySlice = bodyBuilder->slice();
if (!bodySlice.isObject()) {
return authResult;
}
VPackSlice const issSlice = bodySlice.get("iss");
if (!issSlice.isString()) {
return authResult;
}
if (issSlice.copyString() != "arangodb") {
return authResult;
}
if (bodySlice.hasKey("preferred_username")) {
VPackSlice const usernameSlice = bodySlice.get("preferred_username");
if (!usernameSlice.isString()) {
return authResult;
}
authResult._username = usernameSlice.copyString();
} else if (bodySlice.hasKey("server_id")) {
// mop: hmm...nothing to do here :D
// authResult._username = "******";
} else {
return authResult;
}
// mop: optional exp (cluster currently uses non expiring jwts)
if (bodySlice.hasKey("exp")) {
VPackSlice const expSlice = bodySlice.get("exp");
if (!expSlice.isNumber()) {
return authResult;
}
std::chrono::system_clock::time_point expires(
std::chrono::seconds(expSlice.getNumber<uint64_t>()));
std::chrono::system_clock::time_point now =
std::chrono::system_clock::now();
if (now >= expires) {
return authResult;
}
authResult._expires = true;
authResult._expireTime = expires;
}
authResult._authorized = true;
return authResult;
}
HttpHandler::status_t RestAuthHandler::execute() {
auto const type = _request->requestType();
if (type != GeneralRequest::RequestType::POST) {
generateError(GeneralResponse::ResponseCode::METHOD_NOT_ALLOWED,
TRI_ERROR_HTTP_METHOD_NOT_ALLOWED);
return status_t(HANDLER_DONE);
}
VPackOptions options = VPackOptions::Defaults;
options.checkAttributeUniqueness = true;
bool parseSuccess;
std::shared_ptr<VPackBuilder> parsedBody =
parseVelocyPackBody(&options, parseSuccess);
if (!parseSuccess) {
return badRequest();
}
VPackSlice slice = parsedBody->slice();
if (!slice.isObject()) {
return badRequest();
}
VPackSlice usernameSlice = slice.get("username");
VPackSlice passwordSlice = slice.get("password");
if (!usernameSlice.isString() || !passwordSlice.isString()) {
return badRequest();
}
std::string const username = usernameSlice.copyString();
std::string const password = passwordSlice.copyString();
AuthResult auth = RestServerFeature::AUTH_INFO.checkPassword(username, password);
if (auth._authorized) {
VPackBuilder resultBuilder;
{
VPackObjectBuilder b(&resultBuilder);
std::string jwt = generateJwt(username, password);
resultBuilder.add("jwt", VPackValue(jwt));
resultBuilder.add("must_change_password", VPackValue(auth._mustChange));
}
generateDocument(resultBuilder.slice(), true, &VPackOptions::Defaults);
return status_t(HANDLER_DONE);
} else {
// mop: rfc 2616 10.4.2 (if credentials wrong 401)
generateError(GeneralResponse::ResponseCode::UNAUTHORIZED, TRI_ERROR_HTTP_UNAUTHORIZED,
"Wrong credentials");
return status_t(HANDLER_DONE);
}
}
std::string VelocyPackHelper::getStringValue(VPackSlice const& slice,
std::string const& defaultValue) {
if (!slice.isString()) {
return defaultValue;
}
return slice.copyString();
}
uint64_t VelocyPackHelper::stringUInt64(VPackSlice const& slice) {
if (slice.isString()) {
return arangodb::basics::StringUtils::uint64(slice.copyString());
}
if (slice.isNumber()) {
return slice.getNumericValue<uint64_t>();
}
return 0;
}
arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo(
arangodb::aql::Query* query, VPackSlice const& info, VPackSlice const& shards) {
TRI_ASSERT(shards.isArray());
idxHandles.reserve(shards.length());
conditionNeedUpdate = arangodb::basics::VelocyPackHelper::getBooleanValue(
info, "condNeedUpdate", false);
conditionMemberToUpdate =
arangodb::basics::VelocyPackHelper::getNumericValue<size_t>(
info, "condMemberToUpdate", 0);
VPackSlice read = info.get("handle");
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Each lookup requires handle to be an object");
}
read = read.get("id");
if (!read.isString()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Each handle requires id to be a string");
}
std::string idxId = read.copyString();
auto trx = query->trx();
for (auto const& it : VPackArrayIterator(shards)) {
if (!it.isString()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Shards have to be a list of strings");
}
idxHandles.emplace_back(trx->getIndexByIdentifier(it.copyString(), idxId));
}
read = info.get("expression");
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Each lookup requires expression to be an object");
}
expression = new aql::Expression(query->ast(), read);
read = info.get("condition");
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Each lookup requires condition to be an object");
}
indexCondition = new aql::AstNode(query->ast(), read);
}
bool arangodb::traverser::Traverser::vertexMatchesConditions(VPackSlice v, size_t depth) {
TRI_ASSERT(v.isString());
if (_opts->vertexHasFilter(depth)) {
aql::AqlValue vertex = fetchVertexData(v);
if (!_opts->evaluateVertexExpression(vertex.slice(), depth)) {
++_filteredPaths;
return false;
}
}
return true;
}
Aggregator* Aggregator::fromVPack(arangodb::Transaction* trx,
arangodb::velocypack::Slice const& slice,
char const* variableName) {
VPackSlice variable = slice.get(variableName);
if (variable.isString()) {
return fromTypeString(trx, variable.copyString());
}
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"invalid aggregate function");
}
/// @brief shutdown, will be called exactly once for the whole query
int RemoteBlock::shutdown(int errorCode) {
DEBUG_BEGIN_BLOCK();
if (!_isResponsibleForInitializeCursor) {
// do nothing...
return TRI_ERROR_NO_ERROR;
}
// For every call we simply forward via HTTP
std::unique_ptr<ClusterCommResult> res =
sendRequest(rest::RequestType::PUT, "/_api/aql/shutdown/",
std::string("{\"code\":" + std::to_string(errorCode) + "}"));
if (throwExceptionAfterBadSyncRequest(res.get(), true)) {
// artificially ignore error in case query was not found during shutdown
return TRI_ERROR_NO_ERROR;
}
StringBuffer const& responseBodyBuf(res->result->getBody());
std::shared_ptr<VPackBuilder> builder =
VPackParser::fromJson(responseBodyBuf.c_str(), responseBodyBuf.length());
VPackSlice slice = builder->slice();
// read "warnings" attribute if present and add it to our query
if (slice.isObject()) {
VPackSlice warnings = slice.get("warnings");
if (warnings.isArray()) {
auto query = _engine->getQuery();
for (auto const& it : VPackArrayIterator(warnings)) {
if (it.isObject()) {
VPackSlice code = it.get("code");
VPackSlice message = it.get("message");
if (code.isNumber() && message.isString()) {
query->registerWarning(code.getNumericValue<int>(),
message.copyString().c_str());
}
}
}
}
}
if (slice.hasKey("code")) {
return slice.get("code").getNumericValue<int>();
}
return TRI_ERROR_INTERNAL;
// cppcheck-suppress style
DEBUG_END_BLOCK();
}
TRI_voc_cid_t Syncer::getCid(VPackSlice const& slice) const {
if (!slice.isObject()) {
return 0;
}
VPackSlice id = slice.get("cid");
if (id.isString()) {
// string cid, e.g. "9988488"
return StringUtils::uint64(id.copyString());
} else if (id.isNumber()) {
// numeric cid, e.g. 9988488
return id.getNumericValue<TRI_voc_cid_t>();
}
return 0;
}
std::string VelocyPackHelper::getStringValue(VPackSlice slice,
std::string const& name,
std::string const& defaultValue) {
if (slice.isExternal()) {
slice = VPackSlice(slice.getExternal());
}
TRI_ASSERT(slice.isObject());
if (!slice.hasKey(name)) {
return defaultValue;
}
VPackSlice const sub = slice.get(name);
if (!sub.isString()) {
return defaultValue;
}
return sub.copyString();
}
std::string VelocyPackHelper::checkAndGetStringValue(VPackSlice const& slice,
std::string const& name) {
TRI_ASSERT(slice.isObject());
if (!slice.hasKey(name)) {
std::string msg =
"The attribute '" + name + "' was not found.";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg);
}
VPackSlice const sub = slice.get(name);
if (!sub.isString()) {
std::string msg =
"The attribute '" + name + "' is not a string.";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg);
}
return sub.copyString();
}
aql::AqlValue SingleServerTraverser::fetchVertexData(VPackSlice id) {
TRI_ASSERT(id.isString());
auto it = _vertices.find(id);
if (it == _vertices.end()) {
StringRef ref(id);
int res = FetchDocumentById(_trx, ref, *_mmdr);
++_readDocuments;
if (res != TRI_ERROR_NO_ERROR) {
return aql::AqlValue(basics::VelocyPackHelper::NullValue());
}
uint8_t const* p = _mmdr->vpack();
_vertices.emplace(id, p);
return aql::AqlValue(p, aql::AqlValueFromManagedDocument());
}
return aql::AqlValue((*it).second, aql::AqlValueFromManagedDocument());
}
ShortestPathOptions::ShortestPathOptions(VPackSlice const& slice) {
VPackSlice obj = slice.get("shortestpathFlags");
weightAttribute = "";
if (obj.hasKey("weightAttribute")) {
VPackSlice v = obj.get("weightAttribute");
if (v.isString()) {
weightAttribute = v.copyString();
}
}
defaultWeight = 1;
if (obj.hasKey("defaultWeight")) {
VPackSlice v = obj.get("defaultWeight");
if (v.isNumber()) {
defaultWeight = v.getNumericValue<double>();
}
}
}
void SingleServerTraverser::addVertexToVelocyPack(VPackSlice id,
VPackBuilder& result) {
TRI_ASSERT(id.isString());
auto it = _vertices.find(id);
if (it == _vertices.end()) {
StringRef ref(id);
int res = FetchDocumentById(_trx, ref, *_mmdr);
++_readDocuments;
if (res != TRI_ERROR_NO_ERROR) {
result.add(basics::VelocyPackHelper::NullValue());
} else {
uint8_t const* p = _mmdr->vpack();
_vertices.emplace(id, p);
result.addExternal(p);
}
} else {
result.addExternal((*it).second);
}
}
void arangodb::traverser::ShortestPath::vertexToVelocyPack(Transaction* trx, ManagedDocumentResult* mmdr,
size_t position, VPackBuilder& builder) {
TRI_ASSERT(position < length());
VPackSlice v = _vertices[position];
TRI_ASSERT(v.isString());
std::string collection = v.copyString();
size_t p = collection.find("/");
TRI_ASSERT(p != std::string::npos);
TransactionBuilderLeaser searchBuilder(trx);
searchBuilder->add(VPackValue(collection.substr(p + 1)));
collection = collection.substr(0, p);
int res =
trx->documentFastPath(collection, mmdr, searchBuilder->slice(), builder, true);
if (res != TRI_ERROR_NO_ERROR) {
builder.clear(); // Just in case...
builder.add(basics::VelocyPackHelper::NullValue());
}
}
ShortestPathNode::ShortestPathNode(ExecutionPlan* plan,
arangodb::velocypack::Slice const& base)
: ExecutionNode(plan, base),
_vocbase(plan->getAst()->query()->vocbase()),
_vertexOutVariable(nullptr),
_edgeOutVariable(nullptr),
_inStartVariable(nullptr),
_inTargetVariable(nullptr),
_graphObj(nullptr) {
// Directions
VPackSlice dirList = base.get("directions");
for (auto const& it : VPackArrayIterator(dirList)) {
uint64_t dir = arangodb::basics::VelocyPackHelper::stringUInt64(it);
TRI_edge_direction_e d;
switch (dir) {
case 0:
d = TRI_EDGE_ANY;
break;
case 1:
d = TRI_EDGE_IN;
break;
case 2:
d = TRI_EDGE_OUT;
break;
default:
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"Invalid direction value");
break;
}
_directions.emplace_back(d);
}
// Start Vertex
if (base.hasKey("startInVariable")) {
_inStartVariable = varFromVPack(plan->getAst(), base, "startInVariable");
} else {
VPackSlice v = base.get("startVertexId");
if (!v.isString()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN,
"start vertex must be a string");
}
_startVertexId = v.copyString();
if (_startVertexId.empty()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN,
"start vertex mustn't be empty");
}
}
// Target Vertex
if (base.hasKey("targetInVariable")) {
_inTargetVariable = varFromVPack(plan->getAst(), base, "targetInVariable");
} else {
VPackSlice v = base.get("targetVertexId");
if (!v.isString()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN,
"target vertex must be a string");
}
_targetVertexId = v.copyString();
if (_targetVertexId.empty()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN,
"target vertex mustn't be empty");
}
}
std::string graphName;
if (base.hasKey("graph") && (base.get("graph").isString())) {
graphName = base.get("graph").copyString();
if (base.hasKey("graphDefinition")) {
_graphObj = plan->getAst()->query()->lookupGraphByName(graphName);
if (_graphObj == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND);
}
auto eColls = _graphObj->edgeCollections();
for (auto const& n : eColls) {
_edgeColls.push_back(n);
}
} else {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN,
"missing graphDefinition.");
}
} else {
_graphInfo.add(base.get("graph"));
if (!_graphInfo.slice().isArray()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN,
"graph has to be an array.");
}
// List of edge collection names
for (auto const& it : VPackArrayIterator(_graphInfo.slice())) {
if (!it.isString()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN,
"graph has to be an array of strings.");
}
_edgeColls.emplace_back(it.copyString());
}
if (_edgeColls.empty()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_QUERY_BAD_JSON_PLAN,
"graph has to be a non empty array of strings.");
}
}
// Out variables
if (base.hasKey("vertexOutVariable")) {
_vertexOutVariable = varFromVPack(plan->getAst(), base, "vertexOutVariable");
}
if (base.hasKey("edgeOutVariable")) {
_edgeOutVariable = varFromVPack(plan->getAst(), base, "edgeOutVariable");
}
// Flags
if (base.hasKey("shortestPathFlags")) {
_options = ShortestPathOptions(base);
}
}
int Syncer::handleStateResponse(VPackSlice const& slice, std::string& errorMsg) {
std::string const endpointString =
" from endpoint '" + _masterInfo._endpoint + "'";
// process "state" section
VPackSlice const state = slice.get("state");
if (!state.isObject()) {
errorMsg = "state section is missing in response" + endpointString;
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
// state."lastLogTick"
VPackSlice const tick = state.get("lastLogTick");
if (!tick.isString()) {
errorMsg = "lastLogTick is missing in response" + endpointString;
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
TRI_voc_tick_t const lastLogTick = VelocyPackHelper::stringUInt64(tick);
// state."running"
bool running = VelocyPackHelper::getBooleanValue(state, "running", false);
// process "server" section
VPackSlice const server = slice.get("server");
if (!server.isObject()) {
errorMsg = "server section is missing in response" + endpointString;
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
// server."version"
VPackSlice const version = server.get("version");
if (!version.isString()) {
errorMsg = "server version is missing in response" + endpointString;
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
// server."serverId"
VPackSlice const serverId = server.get("serverId");
if (!serverId.isString()) {
errorMsg = "server id is missing in response" + endpointString;
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
// validate all values we got
std::string const masterIdString(serverId.copyString());
TRI_server_id_t const masterId = StringUtils::uint64(masterIdString);
if (masterId == 0) {
// invalid master id
errorMsg = "invalid server id in response" + endpointString;
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
if (masterIdString == _localServerIdString) {
// master and replica are the same instance. this is not supported.
errorMsg = "got same server id (" + _localServerIdString + ")" +
endpointString + " as the local applier server's id";
return TRI_ERROR_REPLICATION_LOOP;
}
int major = 0;
int minor = 0;
std::string const versionString(version.copyString());
if (sscanf(versionString.c_str(), "%d.%d", &major, &minor) != 2) {
errorMsg = "invalid master version info" + endpointString + ": '" +
versionString + "'";
return TRI_ERROR_REPLICATION_MASTER_INCOMPATIBLE;
}
if (major != 3) {
// we can connect to 3.x only
errorMsg = "got incompatible master version" + endpointString + ": '" +
versionString + "'";
return TRI_ERROR_REPLICATION_MASTER_INCOMPATIBLE;
}
_masterInfo._majorVersion = major;
_masterInfo._minorVersion = minor;
_masterInfo._serverId = masterId;
_masterInfo._lastLogTick = lastLogTick;
_masterInfo._active = running;
LOG_TOPIC(INFO, Logger::REPLICATION)
<< "connected to master at " << _masterInfo._endpoint << ", id "
<< _masterInfo._serverId << ", version " << _masterInfo._majorVersion
<< "." << _masterInfo._minorVersion << ", last log tick "
<< _masterInfo._lastLogTick;
return TRI_ERROR_NO_ERROR;
}
static void raceForClusterBootstrap() {
AgencyComm agency;
auto ci = ClusterInfo::instance();
while (true) {
AgencyCommResult result = agency.getValues("Bootstrap");
if (!result.successful()) {
// Error in communication, note that value not found is not an error
LOG_TOPIC(TRACE, Logger::STARTUP)
<< "raceForClusterBootstrap: no agency communication";
sleep(1);
continue;
}
VPackSlice value = result.slice()[0].get(
std::vector<std::string>({agency.prefix(), "Bootstrap"}));
if (value.isString()) {
// key was found and is a string
if (value.copyString().find("done") != std::string::npos) {
// all done, let's get out of here:
LOG_TOPIC(TRACE, Logger::STARTUP)
<< "raceForClusterBootstrap: bootstrap already done";
return;
}
LOG_TOPIC(DEBUG, Logger::STARTUP)
<< "raceForClusterBootstrap: somebody else does the bootstrap";
sleep(1);
continue;
}
// No value set, we try to do the bootstrap ourselves:
VPackBuilder b;
b.add(VPackValue(arangodb::ServerState::instance()->getId()));
result = agency.casValue("Bootstrap", b.slice(), false, 300, 15);
if (!result.successful()) {
LOG_TOPIC(DEBUG, Logger::STARTUP)
<< "raceForClusterBootstrap: lost race, somebody else will bootstrap";
// Cannot get foot into the door, try again later:
sleep(1);
continue;
}
// OK, we handle things now, let's see whether a DBserver is there:
auto dbservers = ci->getCurrentDBServers();
if (dbservers.size() == 0) {
LOG_TOPIC(TRACE, Logger::STARTUP)
<< "raceForClusterBootstrap: no DBservers, waiting";
agency.removeValues("Bootstrap", false);
sleep(1);
continue;
}
LOG_TOPIC(DEBUG, Logger::STARTUP)
<< "raceForClusterBootstrap: race won, we do the bootstrap";
auto vocbase = DatabaseFeature::DATABASE->systemDatabase();
V8DealerFeature::DEALER->loadJavascriptFiles(vocbase, "server/bootstrap/cluster-bootstrap.js", 0);
LOG_TOPIC(DEBUG, Logger::STARTUP)
<< "raceForClusterBootstrap: bootstrap done";
b.clear();
b.add(VPackValue(arangodb::ServerState::instance()->getId() + ": done"));
result = agency.setValue("Bootstrap", b.slice(), 0);
if (result.successful()) {
return;
}
LOG_TOPIC(TRACE, Logger::STARTUP)
<< "raceForClusterBootstrap: could not indicate success";
sleep(1);
}
}
/// @brief local helper to throw an exception if a HTTP request went wrong
static bool throwExceptionAfterBadSyncRequest(ClusterCommResult* res,
bool isShutdown) {
DEBUG_BEGIN_BLOCK();
if (res->status == CL_COMM_TIMEOUT ||
res->status == CL_COMM_BACKEND_UNAVAILABLE) {
THROW_ARANGO_EXCEPTION_MESSAGE(res->getErrorCode(),
res->stringifyErrorMessage());
}
if (res->status == CL_COMM_ERROR) {
std::string errorMessage;
TRI_ASSERT(nullptr != res->result);
arangodb::basics::StringBuffer const& responseBodyBuf(res->result->getBody());
// extract error number and message from response
int errorNum = TRI_ERROR_NO_ERROR;
std::shared_ptr<VPackBuilder> builder = VPackParser::fromJson(
responseBodyBuf.c_str(), responseBodyBuf.length());
VPackSlice slice = builder->slice();
if (!slice.hasKey("error") || slice.get("error").getBoolean()) {
errorNum = TRI_ERROR_INTERNAL;
errorMessage = std::string("Error message received from shard '") +
std::string(res->shardID) +
std::string("' on cluster node '") +
std::string(res->serverID) + std::string("': ");
}
if (slice.isObject()) {
VPackSlice v = slice.get("errorNum");
if (v.isNumber()) {
if (v.getNumericValue<int>() != TRI_ERROR_NO_ERROR) {
/* if we've got an error num, error has to be true. */
TRI_ASSERT(errorNum == TRI_ERROR_INTERNAL);
errorNum = v.getNumericValue<int>();
}
}
v = slice.get("errorMessage");
if (v.isString()) {
errorMessage += v.copyString();
} else {
errorMessage += std::string("(no valid error in response)");
}
} else {
errorMessage += std::string("(no valid response)");
}
if (isShutdown && errorNum == TRI_ERROR_QUERY_NOT_FOUND) {
// this error may happen on shutdown and is thus tolerated
// pass the info to the caller who can opt to ignore this error
return true;
}
// In this case a proper HTTP error was reported by the DBserver,
if (errorNum > 0 && !errorMessage.empty()) {
THROW_ARANGO_EXCEPTION_MESSAGE(errorNum, errorMessage);
}
// default error
THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_AQL_COMMUNICATION);
}
return false;
// cppcheck-suppress style
DEBUG_END_BLOCK();
}
/// @brief sendToClient: for each row of the incoming AqlItemBlock use the
/// attributes <shardKeys> of the Aql value <val> to determine to which shard
/// the row should be sent and return its clientId
size_t DistributeBlock::sendToClient(AqlItemBlock* cur) {
DEBUG_BEGIN_BLOCK();
// inspect cur in row _pos and check to which shard it should be sent . .
AqlValue val = cur->getValueReference(_pos, _regId);
VPackSlice input = val.slice(); // will throw when wrong type
bool usedAlternativeRegId = false;
if (input.isNull() && _alternativeRegId != ExecutionNode::MaxRegisterId) {
// value is set, but null
// check if there is a second input register available (UPSERT makes use of
// two input registers,
// one for the search document, the other for the insert document)
val = cur->getValueReference(_pos, _alternativeRegId);
input = val.slice(); // will throw when wrong type
usedAlternativeRegId = true;
}
VPackSlice value = input;
VPackBuilder builder;
VPackBuilder builder2;
bool hasCreatedKeyAttribute = false;
if (input.isString() &&
static_cast<DistributeNode const*>(_exeNode)
->_allowKeyConversionToObject) {
builder.openObject();
builder.add(StaticStrings::KeyString, input);
builder.close();
// clear the previous value
cur->destroyValue(_pos, _regId);
// overwrite with new value
cur->setValue(_pos, _regId, AqlValue(builder));
value = builder.slice();
hasCreatedKeyAttribute = true;
} else if (!input.isObject()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_DOCUMENT_TYPE_INVALID);
}
TRI_ASSERT(value.isObject());
if (static_cast<DistributeNode const*>(_exeNode)->_createKeys) {
// we are responsible for creating keys if none present
if (_usesDefaultSharding) {
// the collection is sharded by _key...
if (!hasCreatedKeyAttribute && !value.hasKey(StaticStrings::KeyString)) {
// there is no _key attribute present, so we are responsible for
// creating one
VPackBuilder temp;
temp.openObject();
temp.add(StaticStrings::KeyString, VPackValue(createKey(value)));
temp.close();
builder2 = VPackCollection::merge(input, temp.slice(), true);
// clear the previous value and overwrite with new value:
if (usedAlternativeRegId) {
cur->destroyValue(_pos, _alternativeRegId);
cur->setValue(_pos, _alternativeRegId, AqlValue(builder2));
} else {
cur->destroyValue(_pos, _regId);
cur->setValue(_pos, _regId, AqlValue(builder2));
}
value = builder2.slice();
}
} else {
// the collection is not sharded by _key
if (hasCreatedKeyAttribute || value.hasKey(StaticStrings::KeyString)) {
// a _key was given, but user is not allowed to specify _key
if (usedAlternativeRegId || !_allowSpecifiedKeys) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_MUST_NOT_SPECIFY_KEY);
}
} else {
VPackBuilder temp;
temp.openObject();
temp.add(StaticStrings::KeyString, VPackValue(createKey(value)));
temp.close();
builder2 = VPackCollection::merge(input, temp.slice(), true);
// clear the previous value and overwrite with new value:
if (usedAlternativeRegId) {
cur->destroyValue(_pos, _alternativeRegId);
cur->setValue(_pos, _alternativeRegId, AqlValue(builder2.slice()));
} else {
cur->destroyValue(_pos, _regId);
cur->setValue(_pos, _regId, AqlValue(builder2.slice()));
}
value = builder2.slice();
}
}
}
std::string shardId;
bool usesDefaultShardingAttributes;
auto clusterInfo = arangodb::ClusterInfo::instance();
auto collInfo = _collection->getCollection();
int res = clusterInfo->getResponsibleShard(collInfo.get(), value, true,
shardId, usesDefaultShardingAttributes);
// std::cout << "SHARDID: " << shardId << "\n";
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
TRI_ASSERT(!shardId.empty());
return getClientId(shardId);
// cppcheck-suppress style
DEBUG_END_BLOCK();
}
static AuthEntry CreateAuthEntry(VPackSlice const& slice) {
if (slice.isNone() || !slice.isObject()) {
return AuthEntry();
}
// extract "user" attribute
VPackSlice const userSlice = slice.get("user");
if (!userSlice.isString()) {
LOG(DEBUG) << "cannot extract username";
return AuthEntry();
}
VPackSlice const authDataSlice = slice.get("authData");
if (!authDataSlice.isObject()) {
LOG(DEBUG) << "cannot extract authData";
return AuthEntry();
}
VPackSlice const simpleSlice = authDataSlice.get("simple");
if (!simpleSlice.isObject()) {
LOG(DEBUG) << "cannot extract simple";
return AuthEntry();
}
VPackSlice const methodSlice = simpleSlice.get("method");
VPackSlice const saltSlice = simpleSlice.get("salt");
VPackSlice const hashSlice = simpleSlice.get("hash");
if (!methodSlice.isString() || !saltSlice.isString() ||
!hashSlice.isString()) {
LOG(DEBUG) << "cannot extract password internals";
return AuthEntry();
}
// extract "active" attribute
bool active;
VPackSlice const activeSlice = authDataSlice.get("active");
if (!activeSlice.isBoolean()) {
LOG(DEBUG) << "cannot extract active flag";
return AuthEntry();
}
active = activeSlice.getBool();
// extract "changePassword" attribute
bool mustChange =
VelocyPackHelper::getBooleanValue(authDataSlice, "changePassword", false);
// extract "databases" attribute
VPackSlice const databasesSlice = slice.get("databases");
std::unordered_map<std::string, AuthLevel> databases;
AuthLevel allDatabases = AuthLevel::NONE;
if (databasesSlice.isObject()) {
for (auto const& obj : VPackObjectIterator(databasesSlice)) {
std::string const key = obj.key.copyString();
ValueLength length;
char const* value = obj.value.getString(length);
if (TRI_CaseEqualString(value, "rw", 2)) {
if (key == "*") {
allDatabases = AuthLevel::RW;
} else {
databases.emplace(key, AuthLevel::RW);
}
} else if (TRI_CaseEqualString(value, "ro", 2)) {
if (key == "*") {
allDatabases = AuthLevel::RO;
} else {
databases.emplace(key, AuthLevel::RO);
}
}
}
}
// build authentication entry
return AuthEntry(userSlice.copyString(), methodSlice.copyString(),
saltSlice.copyString(), hashSlice.copyString(), databases,
allDatabases, active, mustChange);
}
// Check all DB servers, guarded above doChecks
std::vector<check_t> Supervision::checkDBServers() {
std::vector<check_t> ret;
Node::Children const& machinesPlanned =
_snapshot(planDBServersPrefix).children();
Node::Children const serversRegistered =
_snapshot(currentServersRegisteredPrefix).children();
std::vector<std::string> todelete;
for (auto const& machine : _snapshot(healthPrefix).children()) {
if (machine.first.substr(0, 2) == "DB") {
todelete.push_back(machine.first);
}
}
for (auto const& machine : machinesPlanned) {
bool good = false;
std::string lastHeartbeatTime, lastHeartbeatAcked, lastStatus,
heartbeatTime, heartbeatStatus, serverID;
serverID = machine.first;
heartbeatTime = _snapshot(syncPrefix + serverID + "/time").toJson();
heartbeatStatus = _snapshot(syncPrefix + serverID + "/status").toJson();
todelete.erase(std::remove(todelete.begin(), todelete.end(), serverID),
todelete.end());
try { // Existing
lastHeartbeatTime =
_snapshot(healthPrefix + serverID + "/LastHeartbeatSent").toJson();
lastHeartbeatAcked =
_snapshot(healthPrefix + serverID + "/LastHeartbeatAcked").toJson();
lastStatus = _snapshot(healthPrefix + serverID + "/Status").toJson();
if (lastHeartbeatTime != heartbeatTime) { // Update
good = true;
}
} catch (...) { // New server
good = true;
}
query_t report = std::make_shared<Builder>();
report->openArray();
report->openArray();
report->openObject();
report->add(_agencyPrefix + healthPrefix + serverID,
VPackValue(VPackValueType::Object));
report->add("LastHeartbeatSent", VPackValue(heartbeatTime));
report->add("LastHeartbeatStatus", VPackValue(heartbeatStatus));
report->add("Role", VPackValue("DBServer"));
auto endpoint = serversRegistered.find(serverID);
if (endpoint != serversRegistered.end()) {
endpoint = endpoint->second->children().find("endpoint");
if (endpoint != endpoint->second->children().end()) {
if (endpoint->second->children().size() == 0) {
VPackSlice epString = endpoint->second->slice();
if (epString.isString()) {
report->add("Endpoint", epString);
}
}
}
}
if (good) {
report->add(
"LastHeartbeatAcked",
VPackValue(timepointToString(std::chrono::system_clock::now())));
report->add("Status", VPackValue(Supervision::HEALTH_STATUS_GOOD));
std::string failedServerPath = failedServersPrefix + "/" + serverID;
if (_snapshot.exists(failedServerPath).size() == 3) {
Builder del;
del.openArray();
del.openObject();
del.add(_agencyPrefix + failedServerPath,
VPackValue(VPackValueType::Object));
del.add("op", VPackValue("delete"));
del.close();
del.close();
del.close();
transact(_agent, del);
}
} else {
std::chrono::seconds t{0};
t = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now() -
stringToTimepoint(lastHeartbeatAcked));
if (t.count() > _gracePeriod) { // Failure
if (lastStatus == "BAD") {
report->add("Status", VPackValue("FAILED"));
FailedServer fsj(_snapshot, _agent, std::to_string(_jobId++),
"supervision", _agencyPrefix, serverID);
}
} else {
report->add("Status", VPackValue("BAD"));
}
}
report->close();
report->close();
report->close();
report->close();
if (!this->isStopping()) {
_agent->write(report);
}
}
if (!todelete.empty()) {
query_t del = std::make_shared<Builder>();
del->openArray();
del->openArray();
del->openObject();
for (auto const& srv : todelete) {
del->add(_agencyPrefix + healthPrefix + srv,
VPackValue(VPackValueType::Object));
del->add("op", VPackValue("delete"));
del->close();
}
del->close();
del->close();
del->close();
_agent->write(del);
}
return ret;
}
// Check all coordinators, guarded above doChecks
std::vector<check_t> Supervision::checkCoordinators() {
std::vector<check_t> ret;
Node::Children const& machinesPlanned =
_snapshot(planCoordinatorsPrefix).children();
Node::Children const serversRegistered =
_snapshot(currentServersRegisteredPrefix).children();
std::string currentFoxxmaster;
try {
currentFoxxmaster = _snapshot(foxxmaster).getString();
} catch (...) {
}
std::string goodServerId;
bool foxxmasterOk = false;
std::vector<std::string> todelete;
for (auto const& machine : _snapshot(healthPrefix).children()) {
if (machine.first.substr(0, 2) == "Co") {
todelete.push_back(machine.first);
}
}
for (auto const& machine : machinesPlanned) {
bool good = false;
std::string lastHeartbeatTime, lastHeartbeatAcked, lastStatus,
heartbeatTime, heartbeatStatus, serverID;
serverID = machine.first;
heartbeatTime = _snapshot(syncPrefix + serverID + "/time").toJson();
heartbeatStatus = _snapshot(syncPrefix + serverID + "/status").toJson();
todelete.erase(std::remove(todelete.begin(), todelete.end(), serverID),
todelete.end());
try { // Existing
lastHeartbeatTime =
_snapshot(healthPrefix + serverID + "/LastHeartbeatSent").toJson();
lastStatus = _snapshot(healthPrefix + serverID + "/Status").toJson();
if (lastHeartbeatTime != heartbeatTime) { // Update
good = true;
}
} catch (...) { // New server
good = true;
}
query_t report = std::make_shared<Builder>();
report->openArray();
report->openArray();
report->openObject();
report->add(_agencyPrefix + healthPrefix + serverID,
VPackValue(VPackValueType::Object));
report->add("LastHeartbeatSent", VPackValue(heartbeatTime));
report->add("LastHeartbeatStatus", VPackValue(heartbeatStatus));
report->add("Role", VPackValue("Coordinator"));
auto endpoint = serversRegistered.find(serverID);
if (endpoint != serversRegistered.end()) {
endpoint = endpoint->second->children().find("endpoint");
if (endpoint != endpoint->second->children().end()) {
if (endpoint->second->children().size() == 0) {
VPackSlice epString = endpoint->second->slice();
if (epString.isString()) {
report->add("Endpoint", epString);
}
}
}
}
if (good) {
if (goodServerId.empty()) {
goodServerId = serverID;
}
if (serverID == currentFoxxmaster) {
foxxmasterOk = true;
}
report->add(
"LastHeartbeatAcked",
VPackValue(timepointToString(std::chrono::system_clock::now())));
report->add("Status", VPackValue(Supervision::HEALTH_STATUS_GOOD));
} else {
std::chrono::seconds t{0};
t = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now() -
stringToTimepoint(lastHeartbeatAcked));
if (t.count() > _gracePeriod) { // Failure
if (lastStatus == Supervision::HEALTH_STATUS_BAD) {
report->add("Status", VPackValue(Supervision::HEALTH_STATUS_FAILED));
}
} else {
report->add("Status", VPackValue(Supervision::HEALTH_STATUS_BAD));
}
}
report->close();
report->close();
report->close();
report->close();
if (!this->isStopping()) {
_agent->write(report);
}
}
if (!todelete.empty()) {
query_t del = std::make_shared<Builder>();
del->openArray();
del->openArray();
del->openObject();
for (auto const& srv : todelete) {
del->add(_agencyPrefix + healthPrefix + srv,
VPackValue(VPackValueType::Object));
del->add("op", VPackValue("delete"));
del->close();
}
del->close();
del->close();
del->close();
_agent->write(del);
}
if (!foxxmasterOk && !goodServerId.empty()) {
query_t create = std::make_shared<Builder>();
create->openArray();
create->openArray();
create->openObject();
create->add(_agencyPrefix + foxxmaster, VPackValue(goodServerId));
create->close();
create->close();
create->close();
_agent->write(create);
}
return ret;
}
void RestWalHandler::flush() {
std::shared_ptr<VPackBuilder> parsedRequest;
VPackSlice slice;
try {
slice = _request->payload();
} catch (...) {
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid body value. expecting object");
return;
}
if (!slice.isObject() && !slice.isNone()) {
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid body value. expecting object");
}
bool waitForSync = false;
bool waitForCollector = false;
if (slice.isObject()) {
// got a request body
VPackSlice value = slice.get("waitForSync");
if (value.isString()) {
waitForSync = (value.copyString() == "true");
} else if (value.isBoolean()) {
waitForSync = value.getBoolean();
}
value = slice.get("waitForCollector");
if (value.isString()) {
waitForCollector = (value.copyString() == "true");
} else if (value.isBoolean()) {
waitForCollector = value.getBoolean();
}
} else {
// no request body
bool found;
{
std::string const& v = _request->value("waitForSync", found);
if (found) {
waitForSync = (v == "1" || v == "true");
}
}
{
std::string const& v = _request->value("waitForCollector", found);
if (found) {
waitForCollector = (v == "1" || v == "true");
}
}
}
int res;
if (ServerState::instance()->isCoordinator()) {
res = flushWalOnAllDBServers(waitForSync, waitForCollector);
} else {
res = arangodb::wal::LogfileManager::instance()->flush(
waitForSync, waitForCollector, false);
}
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
generateResult(rest::ResponseCode::OK, basics::VelocyPackHelper::EmptyObjectValue());
}
