int RocksDBFeature::dropPrefix(std::string const& prefix) {
if (!isEnabled()) {
return TRI_ERROR_NO_ERROR;
}
TRI_ASSERT(Instance != nullptr);
try {
VPackBuilder builder;
// create lower and upper bound for deletion
builder.openArray();
builder.add(VPackSlice::minKeySlice());
builder.close();
std::string l;
l.reserve(prefix.size() + builder.slice().byteSize());
l.append(prefix);
// extend the prefix to at least 24 bytes
while (l.size() < RocksDBIndex::keyPrefixSize()) {
uint64_t value = 0;
l.append(reinterpret_cast<char const*>(&value), sizeof(uint64_t));
}
l.append(builder.slice().startAs<char const>(), builder.slice().byteSize());
builder.clear();
builder.openArray();
builder.add(VPackSlice::maxKeySlice());
builder.close();
std::string u;
u.reserve(prefix.size() + builder.slice().byteSize());
u.append(prefix);
// extend the prefix to at least 24 bytes
while (u.size() < RocksDBIndex::keyPrefixSize()) {
uint64_t value = UINT64_MAX;
u.append(reinterpret_cast<char const*>(&value), sizeof(uint64_t));
}
u.append(builder.slice().startAs<char const>(), builder.slice().byteSize());
#if 0
for (size_t i = 0; i < prefix.size(); i += sizeof(TRI_idx_iid_t)) {
char const* x = prefix.c_str() + i;
size_t o;
char* q = TRI_EncodeHexString(x, 8, &o);
if (q != nullptr) {
LOG(TRACE) << "RocksDB prefix part: " << q;
TRI_FreeString(TRI_CORE_MEM_ZONE, q);
}
}
LOG(TRACE) << "dropping RocksDB range: " << VPackSlice(l.c_str() + RocksDBIndex::keyPrefixSize()).toJson() << " - " << VPackSlice(u.c_str() + RocksDBIndex::keyPrefixSize()).toJson();
#endif
// delete files in range lower..upper
rocksdb::Slice lower(l.c_str(), l.size());
rocksdb::Slice upper(u.c_str(), u.size());
{
rocksdb::Status status = rocksdb::DeleteFilesInRange(_db->GetBaseDB(), _db->GetBaseDB()->DefaultColumnFamily(), &lower, &upper);
if (!status.ok()) {
// if file deletion failed, we will still iterate over the remaining keys, so we
// don't need to abort and raise an error here
LOG(WARN) << "RocksDB file deletion failed";
}
}
// go on and delete the remaining keys (delete files in range does not necessarily
// find them all, just complete files)
auto comparator = RocksDBFeature::instance()->comparator();
rocksdb::DB* db = _db->GetBaseDB();
rocksdb::WriteBatch batch;
std::unique_ptr<rocksdb::Iterator> it(db->NewIterator(rocksdb::ReadOptions()));
it->Seek(lower);
while (it->Valid()) {
int res = comparator->Compare(it->key(), upper);
if (res >= 0) {
break;
}
batch.Delete(it->key());
it->Next();
}
// now apply deletion batch
rocksdb::Status status = db->Write(rocksdb::WriteOptions(), &batch);
if (!status.ok()) {
LOG(WARN) << "RocksDB key deletion failed: " << status.ToString();
return TRI_ERROR_INTERNAL;
}
return TRI_ERROR_NO_ERROR;
} catch (arangodb::basics::Exception const& ex) {
LOG(ERR) << "caught exception during RocksDB key prefix deletion: " << ex.what();
return ex.code();
} catch (std::exception const& ex) {
LOG(ERR) << "caught exception during RocksDB key prefix deletion: " << ex.what();
return TRI_ERROR_INTERNAL;
} catch (...) {
LOG(ERR) << "caught unknown exception during RocksDB key prefix deletion";
return TRI_ERROR_INTERNAL;
}
}
/// @brief sendToClient: for each row of the incoming AqlItemBlock use the
/// attributes <shardKeys> of the Aql value <val> to determine to which shard
/// the row should be sent and return its clientId
size_t DistributeBlock::sendToClient(AqlItemBlock* cur) {
DEBUG_BEGIN_BLOCK();
// inspect cur in row _pos and check to which shard it should be sent . .
AqlValue val = cur->getValueReference(_pos, _regId);
VPackSlice input = val.slice(); // will throw when wrong type
bool usedAlternativeRegId = false;
if (input.isNull() && _alternativeRegId != ExecutionNode::MaxRegisterId) {
// value is set, but null
// check if there is a second input register available (UPSERT makes use of
// two input registers,
// one for the search document, the other for the insert document)
val = cur->getValueReference(_pos, _alternativeRegId);
input = val.slice(); // will throw when wrong type
usedAlternativeRegId = true;
}
VPackSlice value = input;
VPackBuilder builder;
VPackBuilder builder2;
bool hasCreatedKeyAttribute = false;
if (input.isString() &&
static_cast<DistributeNode const*>(_exeNode)
->_allowKeyConversionToObject) {
builder.openObject();
builder.add(StaticStrings::KeyString, input);
builder.close();
// clear the previous value
cur->destroyValue(_pos, _regId);
// overwrite with new value
cur->setValue(_pos, _regId, AqlValue(builder));
value = builder.slice();
hasCreatedKeyAttribute = true;
} else if (!input.isObject()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_DOCUMENT_TYPE_INVALID);
}
TRI_ASSERT(value.isObject());
if (static_cast<DistributeNode const*>(_exeNode)->_createKeys) {
// we are responsible for creating keys if none present
if (_usesDefaultSharding) {
// the collection is sharded by _key...
if (!hasCreatedKeyAttribute && !value.hasKey(StaticStrings::KeyString)) {
// there is no _key attribute present, so we are responsible for
// creating one
VPackBuilder temp;
temp.openObject();
temp.add(StaticStrings::KeyString, VPackValue(createKey(value)));
temp.close();
builder2 = VPackCollection::merge(input, temp.slice(), true);
// clear the previous value and overwrite with new value:
if (usedAlternativeRegId) {
cur->destroyValue(_pos, _alternativeRegId);
cur->setValue(_pos, _alternativeRegId, AqlValue(builder2));
} else {
cur->destroyValue(_pos, _regId);
cur->setValue(_pos, _regId, AqlValue(builder2));
}
value = builder2.slice();
}
} else {
// the collection is not sharded by _key
if (hasCreatedKeyAttribute || value.hasKey(StaticStrings::KeyString)) {
// a _key was given, but user is not allowed to specify _key
if (usedAlternativeRegId || !_allowSpecifiedKeys) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_MUST_NOT_SPECIFY_KEY);
}
} else {
VPackBuilder temp;
temp.openObject();
temp.add(StaticStrings::KeyString, VPackValue(createKey(value)));
temp.close();
builder2 = VPackCollection::merge(input, temp.slice(), true);
// clear the previous value and overwrite with new value:
if (usedAlternativeRegId) {
cur->destroyValue(_pos, _alternativeRegId);
cur->setValue(_pos, _alternativeRegId, AqlValue(builder2.slice()));
} else {
cur->destroyValue(_pos, _regId);
cur->setValue(_pos, _regId, AqlValue(builder2.slice()));
}
value = builder2.slice();
}
}
}
std::string shardId;
bool usesDefaultShardingAttributes;
auto clusterInfo = arangodb::ClusterInfo::instance();
auto collInfo = _collection->getCollection();
int res = clusterInfo->getResponsibleShard(collInfo.get(), value, true,
shardId, usesDefaultShardingAttributes);
// std::cout << "SHARDID: " << shardId << "\n";
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
TRI_ASSERT(!shardId.empty());
return getClientId(shardId);
// cppcheck-suppress style
DEBUG_END_BLOCK();
}
void arangodb::traverser::TraverserOptions::buildEngineInfo(VPackBuilder& result) const {
result.openObject();
result.add("minDepth", VPackValue(minDepth));
result.add("maxDepth", VPackValue(maxDepth));
result.add("bfs", VPackValue(useBreadthFirst));
result.add(VPackValue("uniqueVertices"));
switch (uniqueVertices) {
case UniquenessLevel::NONE:
result.add(VPackValue(0));
break;
case UniquenessLevel::PATH:
result.add(VPackValue(1));
break;
case UniquenessLevel::GLOBAL:
result.add(VPackValue(2));
break;
}
result.add(VPackValue("uniqueEdges"));
switch (uniqueEdges) {
case UniquenessLevel::NONE:
result.add(VPackValue(0));
break;
case UniquenessLevel::PATH:
result.add(VPackValue(1));
break;
case UniquenessLevel::GLOBAL:
result.add(VPackValue(2));
break;
}
result.add(VPackValue("baseLookupInfos"));
result.openArray();
for (auto const& it: _baseLookupInfos) {
it.buildEngineInfo(result);
}
result.close();
if (!_depthLookupInfo.empty()) {
result.add(VPackValue("depthLookupInfo"));
result.openObject();
for (auto const& pair : _depthLookupInfo) {
result.add(VPackValue(basics::StringUtils::itoa(pair.first)));
result.openArray();
for (auto const& it : pair.second) {
it.buildEngineInfo(result);
}
result.close();
}
result.close();
}
if (!_vertexExpressions.empty()) {
result.add(VPackValue("vertexExpressions"));
result.openObject();
for (auto const& pair : _vertexExpressions) {
result.add(VPackValue(basics::StringUtils::itoa(pair.first)));
result.openObject();
result.add(VPackValue("expression"));
pair.second->toVelocyPack(result, true);
result.close();
}
result.close();
}
if (_baseVertexExpression != nullptr) {
result.add(VPackValue("baseVertexExpression"));
result.openObject();
result.add(VPackValue("expression"));
_baseVertexExpression->toVelocyPack(result, true);
result.close();
}
result.add(VPackValue("tmpVar"));
_tmpVar->toVelocyPack(result);
result.close();
}
void RestWalHandler::properties() {
auto l = arangodb::wal::LogfileManager::instance();
if (_request->requestType() == rest::RequestType::PUT) {
std::shared_ptr<VPackBuilder> parsedRequest;
VPackSlice slice;
try {
slice = _request->payload();
} catch (...) {
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid body value. expecting object");
return;
}
if (!slice.isObject()) {
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid body value. expecting object");
}
if (slice.hasKey("allowOversizeEntries")) {
bool value = slice.get("allowOversizeEntries").getBoolean();
l->allowOversizeEntries(value);
}
if (slice.hasKey("logfileSize")) {
uint32_t value = slice.get("logfileSize").getNumericValue<uint32_t>();
l->filesize(value);
}
if (slice.hasKey("historicLogfiles")) {
uint32_t value = slice.get("historicLogfiles").getNumericValue<uint32_t>();
l->historicLogfiles(value);
}
if (slice.hasKey("reserveLogfiles")) {
uint32_t value = slice.get("reserveLogfiles").getNumericValue<uint32_t>();
l->reserveLogfiles(value);
}
if (slice.hasKey("throttleWait")) {
uint64_t value = slice.get("throttleWait").getNumericValue<uint64_t>();
l->maxThrottleWait(value);
}
if (slice.hasKey("throttleWhenPending")) {
uint64_t value = slice.get("throttleWhenPending").getNumericValue<uint64_t>();
l->throttleWhenPending(value);
}
}
VPackBuilder builder;
builder.openObject();
builder.add("allowOversizeEntries", VPackValue(l->allowOversizeEntries()));
builder.add("logfileSize", VPackValue(l->filesize()));
builder.add("historicLogfiles", VPackValue(l->historicLogfiles()));
builder.add("reserveLogfiles", VPackValue(l->reserveLogfiles()));
builder.add("syncInterval", VPackValue(l->syncInterval()));
builder.add("throttleWait", VPackValue(l->maxThrottleWait()));
builder.add("throttleWhenPending", VPackValue(l->throttleWhenPending()));
builder.close();
generateResult(rest::ResponseCode::OK, builder.slice());
}
// Internal function to receive all edges for a list of vertices
// Not publicly documented on purpose.
// NOTE: It ONLY except _id strings. Nothing else
bool RestEdgesHandler::readEdgesForMultipleVertices() {
std::vector<std::string> const& suffix = _request->suffix();
if (suffix.size() != 1) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"expected POST " + EDGES_PATH +
"/<collection-identifier>?direction=<direction>");
return false;
}
bool parseSuccess = true;
std::shared_ptr<VPackBuilder> parsedBody =
parseVelocyPackBody(&VPackOptions::Defaults, parseSuccess);
if (!parseSuccess) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"expected POST " + EDGES_PATH +
"/<collection-identifier>?direction=<direction>");
// A body is required
return false;
}
VPackSlice body = parsedBody->slice();
if (!body.isArray()) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"Expected an array of vertex _id's in body parameter");
return false;
}
std::string collectionName = suffix[0];
CollectionNameResolver resolver(_vocbase);
TRI_col_type_e colType = resolver.getCollectionTypeCluster(collectionName);
if (colType == TRI_COL_TYPE_UNKNOWN) {
generateError(rest::ResponseCode::NOT_FOUND,
TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND);
return false;
} else if (colType != TRI_COL_TYPE_EDGE) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID);
return false;
}
bool found;
std::string dirString = _request->value("direction", found);
if (!found || dirString.empty()) {
dirString = "any";
}
TRI_edge_direction_e direction;
if (dirString == "any") {
direction = TRI_EDGE_ANY;
} else if (dirString == "out" || dirString == "outbound") {
direction = TRI_EDGE_OUT;
} else if (dirString == "in" || dirString == "inbound") {
direction = TRI_EDGE_IN;
} else {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"<direction> must by any, in, or out, not: " + dirString);
return false;
}
std::vector<traverser::TraverserExpression*> const expressions;
if (ServerState::instance()->isCoordinator()) {
rest::ResponseCode responseCode;
VPackBuilder resultDocument;
resultDocument.openObject();
for (auto const& it : VPackArrayIterator(body)) {
if (it.isString()) {
std::string vertexString(it.copyString());
int res = getFilteredEdgesOnCoordinator(
_vocbase->name(), collectionName, vertexString, direction,
expressions, responseCode, resultDocument);
if (res != TRI_ERROR_NO_ERROR) {
generateError(responseCode, res);
return false;
}
}
}
resultDocument.add("error", VPackValue(false));
resultDocument.add("code", VPackValue(200));
resultDocument.close();
generateResult(rest::ResponseCode::OK, resultDocument.slice());
return true;
}
// find and load collection given by name or identifier
SingleCollectionTransaction trx(
StandaloneTransactionContext::Create(_vocbase), collectionName,
TRI_TRANSACTION_READ);
// .............................................................................
// inside read transaction
// .............................................................................
int res = trx.begin();
if (res != TRI_ERROR_NO_ERROR) {
generateTransactionError(collectionName, res, "");
return false;
}
// If we are a DBserver, we want to use the cluster-wide collection
// name for error reporting:
if (ServerState::instance()->isDBServer()) {
collectionName = trx.resolver()->getCollectionName(trx.cid());
}
size_t filtered = 0;
size_t scannedIndex = 0;
VPackBuilder resultBuilder;
resultBuilder.openObject();
// build edges
resultBuilder.add(VPackValue("edges")); // only key
resultBuilder.openArray();
bool ok = getEdgesForVertexList(body, expressions, direction, trx,
resultBuilder, scannedIndex, filtered);
if (!ok) {
// Ignore the error
}
resultBuilder.close();
res = trx.finish(res);
if (res != TRI_ERROR_NO_ERROR) {
generateTransactionError(collectionName, res, "");
return false;
}
resultBuilder.add("error", VPackValue(false));
resultBuilder.add("code", VPackValue(200));
resultBuilder.add("stats", VPackValue(VPackValueType::Object));
resultBuilder.add("scannedIndex", VPackValue(scannedIndex));
resultBuilder.add("filtered", VPackValue(filtered));
resultBuilder.close(); // inner object
resultBuilder.close();
// and generate a response
generateResult(rest::ResponseCode::OK, resultBuilder.slice(),
trx.transactionContext());
return true;
}
// read in- or outbound edges
bool RestEdgesHandler::readEdges(
std::vector<traverser::TraverserExpression*> const& expressions) {
std::vector<std::string> const& suffix = _request->suffix();
if (suffix.size() != 1) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"expected GET " + EDGES_PATH +
"/<collection-identifier>?vertex=<vertex-handle>&"
"direction=<direction>");
return false;
}
std::string collectionName = suffix[0];
CollectionNameResolver resolver(_vocbase);
TRI_col_type_e colType = resolver.getCollectionTypeCluster(collectionName);
if (colType == TRI_COL_TYPE_UNKNOWN) {
generateError(rest::ResponseCode::NOT_FOUND,
TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND);
return false;
}
if (colType != TRI_COL_TYPE_EDGE) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID);
return false;
}
bool found;
std::string dir = _request->value("direction", found);
if (!found || dir.empty()) {
dir = "any";
}
std::string dirString(dir);
TRI_edge_direction_e direction;
if (dirString == "any") {
direction = TRI_EDGE_ANY;
} else if (dirString == "out" || dirString == "outbound") {
direction = TRI_EDGE_OUT;
} else if (dirString == "in" || dirString == "inbound") {
direction = TRI_EDGE_IN;
} else {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"<direction> must by any, in, or out, not: " + dirString);
return false;
}
std::string const& startVertex = _request->value("vertex", found);
if (!found || startVertex.empty()) {
generateError(rest::ResponseCode::BAD,
TRI_ERROR_ARANGO_DOCUMENT_HANDLE_BAD,
"illegal document handle");
return false;
}
if (ServerState::instance()->isCoordinator()) {
std::string vertexString(startVertex);
rest::ResponseCode responseCode;
VPackBuilder resultDocument;
resultDocument.openObject();
int res = getFilteredEdgesOnCoordinator(
_vocbase->name(), collectionName, vertexString, direction, expressions,
responseCode, resultDocument);
if (res != TRI_ERROR_NO_ERROR) {
generateError(responseCode, res);
return false;
}
resultDocument.add("error", VPackValue(false));
resultDocument.add("code", VPackValue(200));
resultDocument.close();
generateResult(rest::ResponseCode::OK, resultDocument.slice());
return true;
}
// find and load collection given by name or identifier
SingleCollectionTransaction trx(
StandaloneTransactionContext::Create(_vocbase), collectionName,
TRI_TRANSACTION_READ);
// .............................................................................
// inside read transaction
// .............................................................................
int res = trx.begin();
if (res != TRI_ERROR_NO_ERROR) {
generateTransactionError(collectionName, res, "");
return false;
}
size_t filtered = 0;
size_t scannedIndex = 0;
VPackBuilder resultBuilder;
resultBuilder.openObject();
// build edges
resultBuilder.add(VPackValue("edges")); // only key
resultBuilder.openArray();
// NOTE: collecitonName is the shard-name in DBServer case
bool ok =
getEdgesForVertex(startVertex, collectionName, expressions, direction,
trx, resultBuilder, scannedIndex, filtered);
resultBuilder.close();
res = trx.finish(res);
if (!ok) {
// Error has been built internally
return false;
}
if (res != TRI_ERROR_NO_ERROR) {
if (ServerState::instance()->isDBServer()) {
// If we are a DBserver, we want to use the cluster-wide collection
// name for error reporting:
collectionName = trx.resolver()->getCollectionName(trx.cid());
}
generateTransactionError(collectionName, res, "");
return false;
}
resultBuilder.add("error", VPackValue(false));
resultBuilder.add("code", VPackValue(200));
resultBuilder.add("stats", VPackValue(VPackValueType::Object));
resultBuilder.add("scannedIndex", VPackValue(scannedIndex));
resultBuilder.add("filtered", VPackValue(filtered));
resultBuilder.close(); // inner object
resultBuilder.close();
// and generate a response
generateResult(rest::ResponseCode::OK, resultBuilder.slice(),
trx.transactionContext());
return true;
}
