bool RestEdgesHandler::getEdgesForVertexList(
VPackSlice const ids,
std::vector<traverser::TraverserExpression*> const& expressions,
TRI_edge_direction_e direction, SingleCollectionTransaction& trx,
VPackBuilder& result, size_t& scannedIndex, size_t& filtered) {
TRI_ASSERT(ids.isArray());
trx.orderDitch(trx.cid()); // will throw when it fails
std::string const collectionName =
trx.resolver()->getCollectionName(trx.cid());
Transaction::IndexHandle indexId = trx.edgeIndexHandle(collectionName);
VPackBuilder searchValueBuilder;
EdgeIndex::buildSearchValueFromArray(direction, ids, searchValueBuilder);
VPackSlice search = searchValueBuilder.slice();
std::unique_ptr<OperationCursor> cursor =
trx.indexScan(collectionName, arangodb::Transaction::CursorType::INDEX,
indexId, search, nullptr, 0, UINT64_MAX, 1000, false);
if (cursor->failed()) {
THROW_ARANGO_EXCEPTION(cursor->code);
}
auto opRes = std::make_shared<OperationResult>(TRI_ERROR_NO_ERROR);
while (cursor->hasMore()) {
cursor->getMore(opRes);
if (opRes->failed()) {
THROW_ARANGO_EXCEPTION(opRes->code);
}
VPackSlice edges = opRes->slice();
TRI_ASSERT(edges.isArray());
// generate result
scannedIndex += static_cast<size_t>(edges.length());
for (auto const& edge : VPackArrayIterator(edges)) {
bool add = true;
if (!expressions.empty()) {
for (auto& exp : expressions) {
if (exp->isEdgeAccess && !exp->matchesCheck(&trx, edge)) {
++filtered;
add = false;
break;
}
}
}
if (add) {
result.add(edge);
}
}
}
return true;
}
void RestVocbaseBaseHandler::generate20x(
arangodb::OperationResult const& result, std::string const& collectionName,
TRI_col_type_e type, VPackOptions const* options) {
VPackSlice slice = result.slice();
if (slice.isNone()) {
// will happen if silent == true
slice = VelocyPackHelper::EmptyObjectValue();
} else {
TRI_ASSERT(slice.isObject() || slice.isArray());
if (slice.isObject()) {
_response->setHeaderNC(
StaticStrings::Etag,
"\"" + slice.get(StaticStrings::RevString).copyString() + "\"");
// pre 1.4 location headers withdrawn for >= 3.0
std::string escapedHandle(assembleDocumentId(
collectionName, slice.get(StaticStrings::KeyString).copyString(),
true));
_response->setHeaderNC(StaticStrings::Location,
std::string("/_db/" + _request->databaseName() +
DOCUMENT_PATH + "/" + escapedHandle));
}
}
writeResult(slice, *options);
}
static v8::Handle<v8::Value> ObjectVPackArray(v8::Isolate* isolate,
VPackSlice const& slice,
VPackOptions const* options,
VPackSlice const* base) {
TRI_ASSERT(slice.isArray());
v8::Handle<v8::Array> object =
v8::Array::New(isolate, static_cast<int>(slice.length()));
if (object.IsEmpty()) {
return v8::Undefined(isolate);
}
uint32_t j = 0;
VPackArrayIterator it(slice, true);
while (it.valid()) {
v8::Handle<v8::Value> val =
TRI_VPackToV8(isolate, it.value(), options, &slice);
if (!val.IsEmpty()) {
object->Set(j++, val);
}
it.next();
}
return object;
}
/// Log transactions (follower)
arangodb::consensus::index_t State::log(query_t const& transactions,
size_t ndups) {
VPackSlice slices = transactions->slice();
TRI_ASSERT(slices.isArray());
size_t nqs = slices.length();
TRI_ASSERT(nqs > ndups);
MUTEX_LOCKER(mutexLocker, _logLock); // log entries must stay in order
for (size_t i = ndups; i < nqs; ++i) {
VPackSlice slice = slices[i];
try {
auto idx = slice.get("index").getUInt();
auto trm = slice.get("term").getUInt();
auto buf = std::make_shared<Buffer<uint8_t>>();
buf->append((char const*)slice.get("query").begin(),
slice.get("query").byteSize());
// to RAM
_log.push_back(log_t(idx, trm, buf));
// to disk
persist(idx, trm, slice.get("query"));
} catch (std::exception const& e) {
LOG_TOPIC(ERR, Logger::AGENCY) << e.what() << " " << __FILE__ << __LINE__;
}
}
TRI_ASSERT(!_log.empty());
return _log.back().index;
}
void AuthInfo::reload() {
insertInitial();
TRI_vocbase_t* vocbase = DatabaseFeature::DATABASE->systemDatabase();
if (vocbase == nullptr) {
LOG(DEBUG) << "system database is unknown, cannot load authentication "
<< "and authorization information";
return;
}
MUTEX_LOCKER(locker, _queryLock);
if (!_outdated) {
return;
}
std::string queryStr("FOR user IN _users RETURN user");
auto nullBuilder = std::make_shared<VPackBuilder>();
VPackBuilder options;
{
VPackObjectBuilder b(&options);
}
auto objectBuilder = std::make_shared<VPackBuilder>(options);
arangodb::aql::Query query(false, vocbase, queryStr.c_str(),
queryStr.length(), nullBuilder, objectBuilder,
arangodb::aql::PART_MAIN);
LOG(DEBUG) << "starting to load authentication and authorization information";
TRI_ASSERT(_queryRegistry != nullptr);
auto queryResult = query.execute(_queryRegistry);
if (queryResult.code != TRI_ERROR_NO_ERROR) {
if (queryResult.code == TRI_ERROR_REQUEST_CANCELED ||
(queryResult.code == TRI_ERROR_QUERY_KILLED)) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_REQUEST_CANCELED);
}
_outdated = false;
return;
}
VPackSlice usersSlice = queryResult.result->slice();
if (usersSlice.isNone()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
}
if (!usersSlice.isArray()) {
LOG(ERR) << "cannot read users from _users collection";
return;
}
if (usersSlice.length() == 0) {
insertInitial();
} else {
populate(usersSlice);
}
_outdated = false;
}
arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo(
arangodb::aql::Query* query, VPackSlice const& info, VPackSlice const& shards) {
TRI_ASSERT(shards.isArray());
idxHandles.reserve(shards.length());
conditionNeedUpdate = arangodb::basics::VelocyPackHelper::getBooleanValue(
info, "condNeedUpdate", false);
conditionMemberToUpdate =
arangodb::basics::VelocyPackHelper::getNumericValue<size_t>(
info, "condMemberToUpdate", 0);
VPackSlice read = info.get("handle");
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Each lookup requires handle to be an object");
}
read = read.get("id");
if (!read.isString()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Each handle requires id to be a string");
}
std::string idxId = read.copyString();
auto trx = query->trx();
for (auto const& it : VPackArrayIterator(shards)) {
if (!it.isString()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Shards have to be a list of strings");
}
idxHandles.emplace_back(trx->getIndexByIdentifier(it.copyString(), idxId));
}
read = info.get("expression");
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Each lookup requires expression to be an object");
}
expression = new aql::Expression(query->ast(), read);
read = info.get("condition");
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"Each lookup requires condition to be an object");
}
indexCondition = new aql::AstNode(query->ast(), read);
}
/// @brief shutdown, will be called exactly once for the whole query
int RemoteBlock::shutdown(int errorCode) {
DEBUG_BEGIN_BLOCK();
if (!_isResponsibleForInitializeCursor) {
// do nothing...
return TRI_ERROR_NO_ERROR;
}
// For every call we simply forward via HTTP
std::unique_ptr<ClusterCommResult> res =
sendRequest(rest::RequestType::PUT, "/_api/aql/shutdown/",
std::string("{\"code\":" + std::to_string(errorCode) + "}"));
if (throwExceptionAfterBadSyncRequest(res.get(), true)) {
// artificially ignore error in case query was not found during shutdown
return TRI_ERROR_NO_ERROR;
}
StringBuffer const& responseBodyBuf(res->result->getBody());
std::shared_ptr<VPackBuilder> builder =
VPackParser::fromJson(responseBodyBuf.c_str(), responseBodyBuf.length());
VPackSlice slice = builder->slice();
// read "warnings" attribute if present and add it to our query
if (slice.isObject()) {
VPackSlice warnings = slice.get("warnings");
if (warnings.isArray()) {
auto query = _engine->getQuery();
for (auto const& it : VPackArrayIterator(warnings)) {
if (it.isObject()) {
VPackSlice code = it.get("code");
VPackSlice message = it.get("message");
if (code.isNumber() && message.isString()) {
query->registerWarning(code.getNumericValue<int>(),
message.copyString().c_str());
}
}
}
}
}
if (slice.hasKey("code")) {
return slice.get("code").getNumericValue<int>();
}
return TRI_ERROR_INTERNAL;
// cppcheck-suppress style
DEBUG_END_BLOCK();
}
bool AuthInfo::populate(VPackSlice const& slice) {
TRI_ASSERT(slice.isArray());
WRITE_LOCKER(writeLocker, _authInfoLock);
clear();
for (VPackSlice const& authSlice : VPackArrayIterator(slice)) {
AuthEntry auth = CreateAuthEntry(authSlice.resolveExternal());
if (auth.isActive()) {
_authInfo.emplace(auth.username(), auth);
}
}
return true;
}
void VelocyPackHelper::SanitizeExternals(VPackSlice const input, VPackBuilder& output) {
if (input.isExternal()) {
output.add(input.resolveExternal());
} else if (input.isObject()) {
output.openObject();
for (auto const& it : VPackObjectIterator(input)) {
output.add(VPackValue(it.key.copyString()));
SanitizeExternals(it.value, output);
}
output.close();
} else if (input.isArray()) {
output.openArray();
for (auto const& it : VPackArrayIterator(input)) {
SanitizeExternals(it, output);
}
output.close();
} else {
output.add(input);
}
}
/// Load compaction collection
bool State::loadCompacted() {
auto bindVars = std::make_shared<VPackBuilder>();
bindVars->openObject();
bindVars->close();
std::string const aql(
std::string("FOR c IN compact SORT c._key DESC LIMIT 1 RETURN c"));
arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars,
nullptr, arangodb::aql::PART_MAIN);
auto queryResult = query.execute(QueryRegistryFeature::QUERY_REGISTRY);
if (queryResult.code != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details);
}
VPackSlice result = queryResult.result->slice();
if (result.isArray() && result.length()) {
MUTEX_LOCKER(logLock, _logLock);
for (auto const& i : VPackArrayIterator(result)) {
auto ii = i.resolveExternals();
buffer_t tmp = std::make_shared<arangodb::velocypack::Buffer<uint8_t>>();
(*_agent) = ii;
try {
_cur = std::stoul(ii.get("_key").copyString());
} catch (std::exception const& e) {
LOG_TOPIC(ERR, Logger::AGENCY) << e.what() << " " << __FILE__
<< __LINE__;
}
}
}
return true;
}
/// Internal function for optimized edge retrieval.
/// Allows to send an TraverserExpression for filtering in the body
/// Not publicly documented on purpose.
bool RestEdgesHandler::readFilteredEdges() {
std::vector<traverser::TraverserExpression*> expressions;
bool parseSuccess = true;
std::shared_ptr<VPackBuilder> parsedBody =
parseVelocyPackBody(&VPackOptions::Defaults, parseSuccess);
if (!parseSuccess) {
// We continue unfiltered
// Filter could be done by caller
return readEdges(expressions);
}
VPackSlice body = parsedBody->slice();
arangodb::basics::ScopeGuard guard{[]() -> void {},
[&expressions]() -> void {
for (auto& e : expressions) {
delete e;
}
}};
if (!body.isArray()) {
generateError(
rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
"Expected an array of traverser expressions as body parameter");
return false;
}
expressions.reserve(static_cast<size_t>(body.length()));
for (auto const& exp : VPackArrayIterator(body)) {
if (exp.isObject()) {
auto expression = std::make_unique<traverser::TraverserExpression>(exp);
expressions.emplace_back(expression.get());
expression.release();
}
}
return readEdges(expressions);
}
// Internal function to receive all edges for a list of vertices
// Not publicly documented on purpose.
// NOTE: It ONLY except _id strings. Nothing else
bool RestEdgesHandler::readEdgesForMultipleVertices() {
std::vector<std::string> const& suffix = _request->suffix();
if (suffix.size() != 1) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"expected POST " + EDGES_PATH +
"/<collection-identifier>?direction=<direction>");
return false;
}
bool parseSuccess = true;
std::shared_ptr<VPackBuilder> parsedBody =
parseVelocyPackBody(&VPackOptions::Defaults, parseSuccess);
if (!parseSuccess) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"expected POST " + EDGES_PATH +
"/<collection-identifier>?direction=<direction>");
// A body is required
return false;
}
VPackSlice body = parsedBody->slice();
if (!body.isArray()) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"Expected an array of vertex _id's in body parameter");
return false;
}
std::string collectionName = suffix[0];
CollectionNameResolver resolver(_vocbase);
TRI_col_type_e colType = resolver.getCollectionTypeCluster(collectionName);
if (colType == TRI_COL_TYPE_UNKNOWN) {
generateError(rest::ResponseCode::NOT_FOUND,
TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND);
return false;
} else if (colType != TRI_COL_TYPE_EDGE) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID);
return false;
}
bool found;
std::string dirString = _request->value("direction", found);
if (!found || dirString.empty()) {
dirString = "any";
}
TRI_edge_direction_e direction;
if (dirString == "any") {
direction = TRI_EDGE_ANY;
} else if (dirString == "out" || dirString == "outbound") {
direction = TRI_EDGE_OUT;
} else if (dirString == "in" || dirString == "inbound") {
direction = TRI_EDGE_IN;
} else {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"<direction> must by any, in, or out, not: " + dirString);
return false;
}
std::vector<traverser::TraverserExpression*> const expressions;
if (ServerState::instance()->isCoordinator()) {
rest::ResponseCode responseCode;
VPackBuilder resultDocument;
resultDocument.openObject();
for (auto const& it : VPackArrayIterator(body)) {
if (it.isString()) {
std::string vertexString(it.copyString());
int res = getFilteredEdgesOnCoordinator(
_vocbase->name(), collectionName, vertexString, direction,
expressions, responseCode, resultDocument);
if (res != TRI_ERROR_NO_ERROR) {
generateError(responseCode, res);
return false;
}
}
}
resultDocument.add("error", VPackValue(false));
resultDocument.add("code", VPackValue(200));
resultDocument.close();
generateResult(rest::ResponseCode::OK, resultDocument.slice());
return true;
}
// find and load collection given by name or identifier
SingleCollectionTransaction trx(
StandaloneTransactionContext::Create(_vocbase), collectionName,
TRI_TRANSACTION_READ);
// .............................................................................
// inside read transaction
// .............................................................................
int res = trx.begin();
if (res != TRI_ERROR_NO_ERROR) {
generateTransactionError(collectionName, res, "");
return false;
}
// If we are a DBserver, we want to use the cluster-wide collection
// name for error reporting:
if (ServerState::instance()->isDBServer()) {
collectionName = trx.resolver()->getCollectionName(trx.cid());
}
size_t filtered = 0;
size_t scannedIndex = 0;
VPackBuilder resultBuilder;
resultBuilder.openObject();
// build edges
resultBuilder.add(VPackValue("edges")); // only key
resultBuilder.openArray();
bool ok = getEdgesForVertexList(body, expressions, direction, trx,
resultBuilder, scannedIndex, filtered);
if (!ok) {
// Ignore the error
}
resultBuilder.close();
res = trx.finish(res);
if (res != TRI_ERROR_NO_ERROR) {
generateTransactionError(collectionName, res, "");
return false;
}
resultBuilder.add("error", VPackValue(false));
resultBuilder.add("code", VPackValue(200));
resultBuilder.add("stats", VPackValue(VPackValueType::Object));
resultBuilder.add("scannedIndex", VPackValue(scannedIndex));
resultBuilder.add("filtered", VPackValue(filtered));
resultBuilder.close(); // inner object
resultBuilder.close();
// and generate a response
generateResult(rest::ResponseCode::OK, resultBuilder.slice(),
trx.transactionContext());
return true;
}
size_t State::removeConflicts(query_t const& transactions) { // Under MUTEX in Agent
VPackSlice slices = transactions->slice();
TRI_ASSERT(slices.isArray());
size_t ndups = 0;
if (slices.length() > 0) {
auto bindVars = std::make_shared<VPackBuilder>();
bindVars->openObject();
bindVars->close();
try {
MUTEX_LOCKER(logLock, _logLock);
auto idx = slices[0].get("index").getUInt();
auto pos = idx - _cur;
if (pos < _log.size()) {
for (auto const& slice : VPackArrayIterator(slices)) {
auto trm = slice.get("term").getUInt();
idx = slice.get("index").getUInt();
pos = idx - _cur;
if (pos < _log.size()) {
if (idx == _log.at(pos).index && trm != _log.at(pos).term) {
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Removing " << _log.size() - pos
<< " entries from log starting with " << idx
<< "==" << _log.at(pos).index << " and " << trm << "="
<< _log.at(pos).term;
// persisted logs
std::string const aql(std::string("FOR l IN log FILTER l._key >= '") +
stringify(idx) + "' REMOVE l IN log");
arangodb::aql::Query query(
false, _vocbase, aql.c_str(), aql.size(), bindVars, nullptr,
arangodb::aql::PART_MAIN);
auto queryResult = query.execute(_queryRegistry);
if (queryResult.code != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code,
queryResult.details);
}
// volatile logs
_log.erase(_log.begin() + pos, _log.end());
break;
} else {
++ndups;
}
}
}
}
} catch (std::exception const& e) {
LOG_TOPIC(DEBUG, Logger::AGENCY) << e.what() << " " << __FILE__
<< __LINE__;
}
}
return ndups;
}
int RestoreFeature::processInputDirectory(std::string& errorMsg) {
// create a lookup table for collections
std::map<std::string, bool> restrictList;
for (size_t i = 0; i < _collections.size(); ++i) {
restrictList.insert(std::pair<std::string, bool>(_collections[i], true));
}
try {
std::vector<std::string> const files =
FileUtils::listFiles(_inputDirectory);
std::string const suffix = std::string(".structure.json");
std::vector<std::shared_ptr<VPackBuilder>> collectionBuilders;
std::vector<VPackSlice> collections;
// Step 1 determine all collections to process
{
// loop over all files in InputDirectory, and look for all structure.json
// files
for (std::string const& file : files) {
size_t const nameLength = file.size();
if (nameLength <= suffix.size() ||
file.substr(file.size() - suffix.size()) != suffix) {
// some other file
continue;
}
// found a structure.json file
std::string name = file.substr(0, file.size() - suffix.size());
if (!_includeSystemCollections && name[0] == '_') {
continue;
}
std::string const fqn = _inputDirectory + TRI_DIR_SEPARATOR_STR + file;
std::shared_ptr<VPackBuilder> fileContentBuilder =
arangodb::basics::VelocyPackHelper::velocyPackFromFile(fqn);
VPackSlice const fileContent = fileContentBuilder->slice();
if (!fileContent.isObject()) {
errorMsg = "could not read collection structure file '" + fqn + "'";
return TRI_ERROR_INTERNAL;
}
VPackSlice const parameters = fileContent.get("parameters");
VPackSlice const indexes = fileContent.get("indexes");
if (!parameters.isObject() || !indexes.isArray()) {
errorMsg = "could not read collection structure file '" + fqn + "'";
return TRI_ERROR_INTERNAL;
}
std::string const cname =
arangodb::basics::VelocyPackHelper::getStringValue(parameters,
"name", "");
bool overwriteName = false;
if (cname != name &&
name !=
(cname + "_" + arangodb::rest::SslInterface::sslMD5(cname))) {
// file has a different name than found in structure file
if (_importStructure) {
// we cannot go on if there is a mismatch
errorMsg =
"collection name mismatch in collection structure file '" +
fqn + "' (offending value: '" + cname + "')";
return TRI_ERROR_INTERNAL;
} else {
// we can patch the name in our array and go on
std::cout << "ignoring collection name mismatch in collection "
"structure file '" +
fqn + "' (offending value: '" + cname + "')"
<< std::endl;
overwriteName = true;
}
}
if (!restrictList.empty() &&
restrictList.find(cname) == restrictList.end()) {
// collection name not in list
continue;
}
if (overwriteName) {
// TODO MAX
// Situation:
// Ich habe ein Json-Object von Datei (teile des Inhalts im Zweifel
// unbekannt)
// Es gibt ein Sub-Json-Object "parameters" mit einem Attribute "name"
// der gesetzt ist.
// Ich muss nur diesen namen überschreiben, der Rest soll identisch
// bleiben.
} else {
collectionBuilders.emplace_back(fileContentBuilder);
collections.emplace_back(fileContent);
}
}
}
std::sort(collections.begin(), collections.end(), SortCollections);
StringBuffer buffer(TRI_UNKNOWN_MEM_ZONE);
// step2: run the actual import
for (VPackSlice const& collection : collections) {
VPackSlice const parameters = collection.get("parameters");
VPackSlice const indexes = collection.get("indexes");
std::string const cname =
arangodb::basics::VelocyPackHelper::getStringValue(parameters, "name",
"");
int type = arangodb::basics::VelocyPackHelper::getNumericValue<int>(
parameters, "type", 2);
std::string const collectionType(type == 2 ? "document" : "edge");
if (_importStructure) {
// re-create collection
if (_progress) {
if (_overwrite) {
std::cout << "# Re-creating " << collectionType << " collection '"
<< cname << "'..." << std::endl;
} else {
std::cout << "# Creating " << collectionType << " collection '"
<< cname << "'..." << std::endl;
}
}
int res = sendRestoreCollection(collection, cname, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
if (_force) {
std::cerr << errorMsg << std::endl;
continue;
}
return TRI_ERROR_INTERNAL;
}
}
_stats._totalCollections++;
if (_importData) {
// import data. check if we have a datafile
std::string datafile =
_inputDirectory + TRI_DIR_SEPARATOR_STR + cname + "_" +
arangodb::rest::SslInterface::sslMD5(cname) + ".data.json";
if (!TRI_ExistsFile(datafile.c_str())) {
datafile =
_inputDirectory + TRI_DIR_SEPARATOR_STR + cname + ".data.json";
}
if (TRI_ExistsFile(datafile.c_str())) {
// found a datafile
if (_progress) {
std::cout << "# Loading data into " << collectionType
<< " collection '" << cname << "'..." << std::endl;
}
int fd = TRI_OPEN(datafile.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) {
errorMsg = "cannot open collection data file '" + datafile + "'";
return TRI_ERROR_INTERNAL;
}
buffer.clear();
while (true) {
if (buffer.reserve(16384) != TRI_ERROR_NO_ERROR) {
TRI_CLOSE(fd);
errorMsg = "out of memory";
return TRI_ERROR_OUT_OF_MEMORY;
}
ssize_t numRead = TRI_READ(fd, buffer.end(), 16384);
if (numRead < 0) {
// error while reading
int res = TRI_errno();
TRI_CLOSE(fd);
errorMsg = std::string(TRI_errno_string(res));
return res;
}
// read something
buffer.increaseLength(numRead);
_stats._totalRead += (uint64_t)numRead;
if (buffer.length() < _chunkSize && numRead > 0) {
// still continue reading
continue;
}
// do we have a buffer?
if (buffer.length() > 0) {
// look for the last \n in the buffer
char* found = (char*)memrchr((const void*)buffer.begin(), '\n',
buffer.length());
size_t length;
if (found == nullptr) {
// no \n found...
if (numRead == 0) {
// we're at the end. send the complete buffer anyway
length = buffer.length();
} else {
// read more
continue;
}
} else {
// found a \n somewhere
length = found - buffer.begin();
}
TRI_ASSERT(length > 0);
_stats._totalBatches++;
int res =
sendRestoreData(cname, buffer.begin(), length, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
TRI_CLOSE(fd);
if (errorMsg.empty()) {
errorMsg = std::string(TRI_errno_string(res));
} else {
errorMsg =
std::string(TRI_errno_string(res)) + ": " + errorMsg;
}
if (_force) {
std::cerr << errorMsg << std::endl;
continue;
}
return res;
}
buffer.erase_front(length);
}
if (numRead == 0) {
// EOF
break;
}
}
TRI_CLOSE(fd);
}
}
if (_importStructure) {
// re-create indexes
if (indexes.length() > 0) {
// we actually have indexes
if (_progress) {
std::cout << "# Creating indexes for collection '" << cname
<< "'..." << std::endl;
}
int res = sendRestoreIndexes(collection, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
if (_force) {
std::cerr << errorMsg << std::endl;
continue;
}
return TRI_ERROR_INTERNAL;
}
}
}
}
} catch (...) {
errorMsg = "out of memory";
return TRI_ERROR_OUT_OF_MEMORY;
}
return TRI_ERROR_NO_ERROR;
}
/// Get persisted information and run election process
void Constituent::run() {
TRI_ASSERT(_vocbase != nullptr);
auto bindVars = std::make_shared<VPackBuilder>();
bindVars->openObject();
bindVars->close();
// Most recent vote
std::string const aql("FOR l IN election SORT l._key DESC LIMIT 1 RETURN l");
arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars,
nullptr, arangodb::aql::PART_MAIN);
auto queryResult = query.execute(_queryRegistry);
if (queryResult.code != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details);
}
VPackSlice result = queryResult.result->slice();
if (result.isArray()) {
for (auto const& i : VPackArrayIterator(result)) {
try {
_term = i.get("term").getUInt();
_votedFor =
static_cast<decltype(_votedFor)>(i.get("voted_for").getUInt());
} catch (std::exception const&) {
LOG_TOPIC(ERR, Logger::AGENCY)
<< "Persisted election entries corrupt! Defaulting term,vote (0,0)";
}
}
}
// Always start off as follower
while (!this->isStopping() && size() > 1) {
if (_role == FOLLOWER) {
bool cast = false;
{
MUTEX_LOCKER(guard, _castLock);
_cast = false; // New round set not cast vote
}
int32_t left = 1000000*config().minPing, right = 1000000*config().maxPing;
long rand_wait = static_cast<long>(RandomGenerator::interval(left, right));
{
CONDITION_LOCKER(guardv, _cv);
_cv.wait(rand_wait);
}
{
MUTEX_LOCKER(guard, _castLock);
cast = _cast;
}
if (!cast) {
candidate(); // Next round, we are running
}
} else {
callElection(); // Run for office
}
}
}
/// Get persisted information and run election process
void Constituent::run() {
// single instance
_id = _agent->config().id();
TRI_ASSERT(_vocbase != nullptr);
auto bindVars = std::make_shared<VPackBuilder>();
bindVars->openObject();
bindVars->close();
// Most recent vote
std::string const aql("FOR l IN election SORT l._key DESC LIMIT 1 RETURN l");
arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars,
nullptr, arangodb::aql::PART_MAIN);
auto queryResult = query.execute(_queryRegistry);
if (queryResult.code != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details);
}
VPackSlice result = queryResult.result->slice();
if (result.isArray()) {
for (auto const& i : VPackArrayIterator(result)) {
auto ii = i.resolveExternals();
try {
MUTEX_LOCKER(locker, _castLock);
_term = ii.get("term").getUInt();
_votedFor = ii.get("voted_for").copyString();
} catch (std::exception const&) {
LOG_TOPIC(ERR, Logger::AGENCY)
<< "Persisted election entries corrupt! Defaulting term,vote (0,0)";
}
}
}
std::vector<std::string> act = _agent->config().active();
while (
!this->isStopping() // Obvious
&& (!_agent->ready()
|| find(act.begin(), act.end(), _id) == act.end())) { // Active agent
CONDITION_LOCKER(guardv, _cv);
_cv.wait(50000);
act = _agent->config().active();
}
if (size() == 1) {
_leaderID = _agent->config().id();
LOG_TOPIC(DEBUG, Logger::AGENCY) << "Set _leaderID to " << _leaderID
<< " in term " << _term;
} else {
while (!this->isStopping()) {
if (_role == FOLLOWER) {
static double const M = 1.0e6;
int64_t a = static_cast<int64_t>(M * _agent->config().minPing());
int64_t b = static_cast<int64_t>(M * _agent->config().maxPing());
int64_t randTimeout = RandomGenerator::interval(a, b);
int64_t randWait = randTimeout;
{
MUTEX_LOCKER(guard, _castLock);
// in the beginning, pure random
if (_lastHeartbeatSeen > 0.0) {
double now = TRI_microtime();
randWait -= static_cast<int64_t>(M * (now - _lastHeartbeatSeen));
}
}
LOG_TOPIC(DEBUG, Logger::AGENCY) << "Random timeout: " << randTimeout
<< ", wait: " << randWait;
if (randWait > 0.0) {
CONDITION_LOCKER(guardv, _cv);
_cv.wait(randWait);
}
bool isTimeout = false;
{
MUTEX_LOCKER(guard, _castLock);
if (_lastHeartbeatSeen <= 0.0) {
LOG_TOPIC(TRACE, Logger::AGENCY) << "no heartbeat seen";
isTimeout = true;
} else {
double diff = TRI_microtime() - _lastHeartbeatSeen;
LOG_TOPIC(TRACE, Logger::AGENCY) << "last heartbeat: " << diff << "sec ago";
isTimeout = (static_cast<int32_t>(M * diff) > randTimeout);
}
}
if (isTimeout) {
LOG_TOPIC(TRACE, Logger::AGENCY) << "timeout, calling an election";
candidate();
}
} else if (_role == CANDIDATE) {
callElection(); // Run for office
} else {
int32_t left =
static_cast<int32_t>(100000.0 * _agent->config().minPing());
long randTimeout = static_cast<long>(left);
{
CONDITION_LOCKER(guardv, _cv);
_cv.wait(randTimeout);
}
}
}
}
}
/// Load persisted configuration
bool State::loadOrPersistConfiguration() {
auto bindVars = std::make_shared<VPackBuilder>();
bindVars->openObject();
bindVars->close();
std::string const aql(
std::string("FOR c in configuration FILTER c._key==\"0\" RETURN c.cfg"));
arangodb::aql::Query query(false, _vocbase, aql.c_str(), aql.size(), bindVars,
nullptr, arangodb::aql::PART_MAIN);
auto queryResult = query.execute(QueryRegistryFeature::QUERY_REGISTRY);
if (queryResult.code != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_MESSAGE(queryResult.code, queryResult.details);
}
VPackSlice result = queryResult.result->slice();
if (result.isArray() &&
result.length()) { // We already have a persisted conf
try {
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Merging configuration " << result[0].resolveExternals().toJson();
_agent->mergeConfiguration(result[0].resolveExternals());
} catch (std::exception const& e) {
LOG_TOPIC(ERR, Logger::AGENCY)
<< "Failed to merge persisted configuration into runtime "
"configuration: "
<< e.what();
FATAL_ERROR_EXIT();
}
} else { // Fresh start
LOG_TOPIC(DEBUG, Logger::AGENCY) << "New agency!";
TRI_ASSERT(_agent != nullptr);
_agent->id(to_string(boost::uuids::random_generator()()));
auto transactionContext =
std::make_shared<StandaloneTransactionContext>(_vocbase);
SingleCollectionTransaction trx(transactionContext, "configuration",
TRI_TRANSACTION_WRITE);
int res = trx.begin();
OperationResult result;
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
Builder doc;
doc.openObject();
doc.add("_key", VPackValue("0"));
doc.add("cfg", _agent->config().toBuilder()->slice());
doc.close();
try {
result = trx.insert("configuration", doc.slice(), _options);
} catch (std::exception const& e) {
LOG_TOPIC(ERR, Logger::AGENCY) << "Failed to persist configuration entry:"
<< e.what();
FATAL_ERROR_EXIT();
}
res = trx.finish(result.code);
return (res == TRI_ERROR_NO_ERROR);
}
return true;
}
arangodb::traverser::TraverserOptions::TraverserOptions(
arangodb::aql::Query* query, VPackSlice info, VPackSlice collections)
: _trx(query->trx()),
_baseVertexExpression(nullptr),
_tmpVar(nullptr),
_ctx(new aql::FixedVarExpressionContext()),
_isCoordinator(arangodb::ServerState::instance()->isCoordinator()),
minDepth(1),
maxDepth(1),
useBreadthFirst(false),
uniqueVertices(UniquenessLevel::NONE),
uniqueEdges(UniquenessLevel::PATH) {
// NOTE collections is an array of arrays of strings
VPackSlice read = info.get("minDepth");
if (!read.isInteger()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"The options require a minDepth");
}
minDepth = read.getNumber<uint64_t>();
read = info.get("maxDepth");
if (!read.isInteger()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"The options require a maxDepth");
}
maxDepth = read.getNumber<uint64_t>();
read = info.get("bfs");
if (!read.isBoolean()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"The options require a bfs");
}
useBreadthFirst = read.getBool();
read = info.get("tmpVar");
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"The options require a tmpVar");
}
_tmpVar = query->ast()->variables()->createVariable(read);
read = info.get("uniqueVertices");
if (!read.isInteger()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"The options require a uniqueVertices");
}
size_t i = read.getNumber<size_t>();
switch (i) {
case 0:
uniqueVertices = UniquenessLevel::NONE;
break;
case 1:
uniqueVertices = UniquenessLevel::PATH;
break;
case 2:
uniqueVertices = UniquenessLevel::GLOBAL;
break;
default:
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"The options require a uniqueVertices");
}
read = info.get("uniqueEdges");
if (!read.isInteger()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"The options require a uniqueEdges");
}
i = read.getNumber<size_t>();
switch (i) {
case 0:
uniqueEdges = UniquenessLevel::NONE;
break;
case 1:
uniqueEdges = UniquenessLevel::PATH;
break;
case 2:
uniqueEdges = UniquenessLevel::GLOBAL;
break;
default:
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"The options require a uniqueEdges");
}
read = info.get("baseLookupInfos");
if (!read.isArray()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"The options require a baseLookupInfos");
}
size_t length = read.length();
TRI_ASSERT(read.length() == collections.length());
_baseLookupInfos.reserve(length);
for (size_t j = 0; j < length; ++j) {
_baseLookupInfos.emplace_back(query, read.at(j), collections.at(j));
}
read = info.get("depthLookupInfo");
if (!read.isNone()) {
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"The options require depthLookupInfo to be an object");
}
_depthLookupInfo.reserve(read.length());
for (auto const& depth : VPackObjectIterator(read)) {
size_t d = basics::StringUtils::uint64(depth.key.copyString());
auto it = _depthLookupInfo.emplace(d, std::vector<LookupInfo>());
TRI_ASSERT(it.second);
VPackSlice list = depth.value;
TRI_ASSERT(length == list.length());
it.first->second.reserve(length);
for (size_t j = 0; j < length; ++j) {
it.first->second.emplace_back(query, list.at(j), collections.at(j));
}
}
}
read = info.get("vertexExpressions");
if (!read.isNone()) {
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"The options require vertexExpressions to be an object");
}
_vertexExpressions.reserve(read.length());
for (auto const& info : VPackObjectIterator(read)) {
size_t d = basics::StringUtils::uint64(info.key.copyString());
#ifdef ARANGODB_ENABLE_MAINAINER_MODE
auto it = _vertexExpressions.emplace(
d, new aql::Expression(query->ast(), info.value));
TRI_ASSERT(it.second);
#else
_vertexExpressions.emplace(
d, new aql::Expression(query->ast(), info.value));
#endif
}
}
read = info.get("baseVertexExpression");
if (!read.isNone()) {
if (!read.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"The options require vertexExpressions to be an object");
}
_baseVertexExpression = new aql::Expression(query->ast(), read);
}
// Check for illegal option combination:
TRI_ASSERT(uniqueEdges !=
arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL);
TRI_ASSERT(
uniqueVertices !=
arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL ||
useBreadthFirst);
}
