static int SendRestoreIndexes (TRI_json_t const* json,
string& errorMsg) {
map<string, string> headers;
const string url = "/_api/replication/restore-indexes?force=" + string(Force ? "true" : "false");
const string body = JsonHelper::toString(json);
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_PUT,
url,
body.c_str(),
body.size(),
headers);
if (response == nullptr || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != nullptr) {
delete response;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = GetHttpErrorMessage(response);
delete response;
return TRI_ERROR_INTERNAL;
}
delete response;
return TRI_ERROR_NO_ERROR;
}
SimpleHttpResult* SimpleHttpClient::request (
int method,
const std::string& location,
const char* body,
size_t bodyLength,
const map<string, string>& headerFields) {
SimpleHttpResult* result = new SimpleHttpResult();
SimpleHttpResult::resultTypes type = SimpleHttpResult::UNKNOWN;
size_t retries = 0;
// build request
stringstream requestBuffer;
fillRequestBuffer(requestBuffer, method, location, body, bodyLength, headerFields);
LOGGER_TRACE << "Request: " << requestBuffer.str();
double start = now();
double runtime = 0.0;
// TODO requestTimeout
while (++retries < _retries && runtime < _requestTimeout) {
// check connection
if (!checkConnection()) {
type = SimpleHttpResult::COULD_NOT_CONNECT;
// LOGGER_WARNING << "could not connect to '" << _connection->getHostname() << ":" << _connection->getPort() << "'";
}
// write
else if (!write(requestBuffer, _requestTimeout - runtime)) {
closeConnection();
type = SimpleHttpResult::WRITE_ERROR;
// LOGGER_WARNING << "write error";
}
// read
else if (!read(result, _requestTimeout - runtime)) {
closeConnection();
type = SimpleHttpResult::READ_ERROR;
// LOGGER_WARNING << "read error";
}
else {
type = SimpleHttpResult::COMPLETE;
break;
}
runtime = now() - start;
}
result->setResultType(type);
return result;
}
static string GetArangoVersion () {
map<string, string> headers;
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET,
"/_api/version",
nullptr,
0,
headers);
if (response == nullptr || ! response->isComplete()) {
if (response != nullptr) {
delete response;
}
return "";
}
string version;
if (response->getHttpReturnCode() == HttpResponse::OK) {
// default value
version = "arango";
// convert response body to json
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE,
response->getBody().c_str());
if (json) {
// look up "server" value
const string server = JsonHelper::getStringValue(json, "server", "");
// "server" value is a string and content is "arango"
if (server == "arango") {
// look up "version" value
version = JsonHelper::getStringValue(json, "version", "");
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
}
else {
if (response->wasHttpError()) {
Client->setErrorMessage(GetHttpErrorMessage(response), false);
}
Connection->disconnect();
}
delete response;
return version;
}
int InitialSyncer::sendExtendBatch () {
if (_batchId == 0) {
return TRI_ERROR_NO_ERROR;
}
double now = TRI_microtime();
if (now <= _batchUpdateTime + _batchTtl - 60) {
// no need to extend the batch yet
return TRI_ERROR_NO_ERROR;
}
map<string, string> const headers;
string const url = BaseUrl + "/batch/" + StringUtils::itoa(_batchId);
string const body = "{\"ttl\":" + StringUtils::itoa(_batchTtl) + "}";
// send request
string const progress = "send batch start command to url " + url;
setProgress(progress);
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_PUT,
url,
body.c_str(),
body.size(),
headers);
if (response == nullptr || ! response->isComplete()) {
if (response != nullptr) {
delete response;
}
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
int res = TRI_ERROR_NO_ERROR;
if (response->wasHttpError()) {
res = TRI_ERROR_REPLICATION_MASTER_ERROR;
}
else {
_batchUpdateTime = TRI_microtime();
}
delete response;
return res;
}
static void FlushWal () {
map<string, string> headers;
const string url = "/_admin/wal/flush?waitForSync=true&waitForCollector=true";
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_PUT,
url,
nullptr,
0,
headers);
if (response == nullptr || ! response->isComplete() || response->wasHttpError()) {
cerr << "got invalid response from server: " + Client->getErrorMessage() << endl;
}
if (response != nullptr) {
delete response;
}
}
static bool GetArangoIsCluster () {
map<string, string> headers;
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET,
"/_admin/server/role",
"",
0,
headers);
if (response == nullptr || ! response->isComplete()) {
if (response != nullptr) {
delete response;
}
return false;
}
string role = "UNDEFINED";
if (response->getHttpReturnCode() == HttpResponse::OK) {
// convert response body to json
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE,
response->getBody().c_str());
if (json != nullptr) {
// look up "server" value
role = JsonHelper::getStringValue(json, "role", "UNDEFINED");
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
}
else {
if (response->wasHttpError()) {
Client->setErrorMessage(GetHttpErrorMessage(response), false);
}
Connection->disconnect();
}
delete response;
return role == "COORDINATOR";
}
static int SendRestoreData (string const& cid,
string const& cname,
char const* buffer,
size_t bufferSize,
string& errorMsg) {
map<string, string> headers;
const string url = "/_api/replication/restore-data?collection=" +
StringUtils::urlEncode(cname) +
"&recycleIds=" + (RecycleIds ? "true" : "false") +
"&force=" + (Force ? "true" : "false");
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_PUT,
url,
buffer,
bufferSize,
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != 0) {
delete response;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = GetHttpErrorMessage(response);
delete response;
return TRI_ERROR_INTERNAL;
}
delete response;
return TRI_ERROR_NO_ERROR;
}
int InitialSyncer::sendFinishBatch () {
if (_batchId == 0) {
return TRI_ERROR_NO_ERROR;
}
map<string, string> const headers;
string const url = BaseUrl + "/batch/" + StringUtils::itoa(_batchId);
// send request
string const progress = "send batch finish command to url " + url;
setProgress(progress);
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_DELETE,
url,
nullptr,
0,
headers);
if (response == nullptr || ! response->isComplete()) {
if (response != nullptr) {
delete response;
}
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
int res = TRI_ERROR_NO_ERROR;
if (response->wasHttpError()) {
res = TRI_ERROR_REPLICATION_MASTER_ERROR;
}
else {
_batchId = 0;
_batchUpdateTime = 0;
}
delete response;
return res;
}
static int StartBatch (string DBserver, string& errorMsg) {
map<string, string> headers;
const string url = "/_api/replication/batch";
const string body = "{\"ttl\":300}";
string urlExt;
if (! DBserver.empty()) {
urlExt = "?DBserver="+DBserver;
}
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_POST,
url + urlExt,
body.c_str(),
body.size(),
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != 0) {
delete response;
}
if (Force) {
return TRI_ERROR_NO_ERROR;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = "got invalid response from server: HTTP " +
StringUtils::itoa(response->getHttpReturnCode()) + ": " +
response->getHttpReturnMessage();
delete response;
return TRI_ERROR_INTERNAL;
}
// convert response body to json
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, response->getBody().c_str());
delete response;
if (json == 0) {
errorMsg = "got malformed JSON";
return TRI_ERROR_INTERNAL;
}
// look up "id" value
const string id = JsonHelper::getStringValue(json, "id", "");
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
BatchId = StringUtils::uint64(id);
return TRI_ERROR_NO_ERROR;
}
int Syncer::getMasterState (string& errorMsg) {
map<string, string> headers;
static const string url = BaseUrl +
"/logger-state" +
"?serverId=" + _localServerIdString;
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET,
url,
0,
0,
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) +
": " + _client->getErrorMessage();
if (response != 0) {
delete response;
}
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
int res = TRI_ERROR_NO_ERROR;
if (response->wasHttpError()) {
res = TRI_ERROR_REPLICATION_MASTER_ERROR;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) +
": " + response->getHttpReturnMessage();
}
else {
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, response->getBody().str().c_str());
if (JsonHelper::isArray(json)) {
res = handleStateResponse(json, errorMsg);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
else {
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": invalid JSON";
}
}
delete response;
return res;
}
static int RunDump (string& errorMsg) {
map<string, string> headers;
const string url = "/_api/replication/inventory?includeSystem=" +
string(IncludeSystemCollections ? "true" : "false");
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET,
url,
0,
0,
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != 0) {
delete response;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = "got invalid response from server: HTTP " +
StringUtils::itoa(response->getHttpReturnCode()) + ": " +
response->getHttpReturnMessage();
delete response;
return TRI_ERROR_INTERNAL;
}
const string& data = response->getBody().str();
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, data.c_str());
delete response;
if (! JsonHelper::isArray(json)) {
if (json != 0) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
TRI_json_t const* collections = JsonHelper::getArrayElement(json, "collections");
if (! JsonHelper::isList(collections)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
const string tickString = JsonHelper::getStringValue(json, "tick", "");
if (tickString == "") {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
cout << "Last tick provided by server is: " << tickString << endl;
// read the server's max tick value
uint64_t maxTick = StringUtils::uint64(tickString);
// check if the user specific a max tick value
if (TickEnd > 0 && maxTick > TickEnd) {
maxTick = TickEnd;
}
// create a lookup table for collections
map<string, bool> restrictList;
for (size_t i = 0; i < Collections.size(); ++i) {
restrictList.insert(pair<string, bool>(Collections[i], true));
}
// iterate over collections
const size_t n = collections->_value._objects._length;
for (size_t i = 0; i < n; ++i) {
TRI_json_t const* collection = (TRI_json_t const*) TRI_AtVector(&collections->_value._objects, i);
if (! JsonHelper::isArray(collection)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
TRI_json_t const* parameters = JsonHelper::getArrayElement(collection, "parameters");
if (! JsonHelper::isArray(parameters)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
const string cid = JsonHelper::getStringValue(parameters, "cid", "");
const string name = JsonHelper::getStringValue(parameters, "name", "");
const bool deleted = JsonHelper::getBooleanValue(parameters, "deleted", false);
if (cid == "" || name == "") {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
if (deleted) {
continue;
}
if (name[0] == '_' && ! IncludeSystemCollections) {
continue;
}
if (restrictList.size() > 0 &&
restrictList.find(name) == restrictList.end()) {
// collection name not in list
continue;
}
// found a collection!
if (Progress) {
cout << "dumping collection '" << name << "'..." << endl;
}
// now save the collection meta data and/or the actual data
Stats._totalCollections++;
{
// save meta data
string fileName;
fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".structure.json";
int fd;
// remove an existing file first
if (TRI_ExistsFile(fileName.c_str())) {
TRI_UnlinkFile(fileName.c_str());
}
fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
const string collectionInfo = JsonHelper::toString(collection);
if (! TRI_WritePointer(fd, collectionInfo.c_str(), collectionInfo.size())) {
TRI_CLOSE(fd);
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
TRI_CLOSE(fd);
}
if (DumpData) {
// save the actual data
string fileName;
fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".data.json";
int fd;
// remove an existing file first
if (TRI_ExistsFile(fileName.c_str())) {
TRI_UnlinkFile(fileName.c_str());
}
fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
ExtendBatch();
int res = DumpCollection(fd, cid, name, parameters, maxTick, errorMsg);
TRI_CLOSE(fd);
if (res != TRI_ERROR_NO_ERROR) {
if (errorMsg.empty()) {
errorMsg = "cannot write to file '" + fileName + "'";
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return res;
}
}
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_NO_ERROR;
}
static int DumpCollection (int fd,
const string& cid,
const string& name,
TRI_json_t const* parameters,
const uint64_t maxTick,
string& errorMsg) {
const string baseUrl = "/_api/replication/dump?collection=" + cid +
"&chunkSize=" + StringUtils::itoa(ChunkSize) +
"&ticks=false&translateIds=true";
map<string, string> headers;
uint64_t fromTick = TickStart;
while (1) {
string url = baseUrl + "&from=" + StringUtils::itoa(fromTick);
if (maxTick > 0) {
url += "&to=" + StringUtils::itoa(maxTick);
}
Stats._totalBatches++;
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET,
url,
0,
0,
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != 0) {
delete response;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = GetHttpErrorMessage(response);
delete response;
return TRI_ERROR_INTERNAL;
}
int res;
bool checkMore = false;
bool found;
uint64_t tick;
// TODO: fix hard-coded headers
string header = response->getHeaderField("x-arango-replication-checkmore", found);
if (found) {
checkMore = StringUtils::boolean(header);
res = TRI_ERROR_NO_ERROR;
if (checkMore) {
// TODO: fix hard-coded headers
header = response->getHeaderField("x-arango-replication-lastincluded", found);
if (found) {
tick = StringUtils::uint64(header);
if (tick > fromTick) {
fromTick = tick;
}
else {
// we got the same tick again, this indicates we're at the end
checkMore = false;
}
}
}
}
if (! found) {
errorMsg = "got invalid response server: required header is missing";
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
if (res == TRI_ERROR_NO_ERROR) {
stringstream& responseBody = response->getBody();
const string body = responseBody.str();
const size_t len = body.size();
if (! TRI_WritePointer(fd, body.c_str(), len)) {
res = TRI_ERROR_CANNOT_WRITE_FILE;
}
else {
Stats._totalWritten += (uint64_t) len;
}
}
delete response;
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
if (! checkMore || fromTick == 0) {
// done
return res;
}
}
assert(false);
return TRI_ERROR_INTERNAL;
}
int InitialSyncer::handleCollectionDump (string const& cid,
TRI_transaction_collection_t* trxCollection,
string const& collectionName,
TRI_voc_tick_t maxTick,
string& errorMsg) {
std::string appendix;
if (_hasFlushed) {
appendix = "&flush=false";
}
else {
// only flush WAL once
appendix = "&flush=true";
_hasFlushed = true;
}
string const baseUrl = BaseUrl +
"/dump?collection=" + cid +
"&chunkSize=" + _chunkSize +
appendix;
map<string, string> headers;
TRI_voc_tick_t fromTick = 0;
int batch = 1;
while (1) {
sendExtendBatch();
string url = baseUrl + "&from=" + StringUtils::itoa(fromTick);
if (maxTick > 0) {
url += "&to=" + StringUtils::itoa(maxTick);
}
url += "&serverId=" + _localServerIdString;
// send request
string const progress = "fetching master collection dump for collection '" + collectionName +
"', id " + cid + ", batch " + StringUtils::itoa(batch);
setProgress(progress.c_str());
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET,
url,
nullptr,
0,
headers);
if (response == nullptr || ! response->isComplete()) {
errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) +
": " + _client->getErrorMessage();
if (response != nullptr) {
delete response;
}
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
if (response->wasHttpError()) {
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) +
": " + response->getHttpReturnMessage();
delete response;
return TRI_ERROR_REPLICATION_MASTER_ERROR;
}
int res = TRI_ERROR_NO_ERROR; // Just to please the compiler
bool checkMore = false;
bool found;
TRI_voc_tick_t tick;
string header = response->getHeaderField(TRI_REPLICATION_HEADER_CHECKMORE, found);
if (found) {
checkMore = StringUtils::boolean(header);
res = TRI_ERROR_NO_ERROR;
if (checkMore) {
header = response->getHeaderField(TRI_REPLICATION_HEADER_LASTINCLUDED, found);
if (found) {
tick = StringUtils::uint64(header);
if (tick > fromTick) {
fromTick = tick;
}
else {
// we got the same tick again, this indicates we're at the end
checkMore = false;
}
}
}
}
if (! found) {
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": required header is missing";
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
if (res == TRI_ERROR_NO_ERROR) {
res = applyCollectionDump(trxCollection, response, errorMsg);
}
delete response;
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
if (! checkMore || fromTick == 0) {
// done
return res;
}
batch++;
}
TRI_ASSERT(false);
return TRI_ERROR_INTERNAL;
}
static int RunClusterDump (string& errorMsg) {
int res;
map<string, string> headers;
const string url = "/_api/replication/clusterInventory?includeSystem=" +
string(IncludeSystemCollections ? "true" : "false");
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET,
url,
0,
0,
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != 0) {
delete response;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = "got invalid response from server: HTTP " +
StringUtils::itoa(response->getHttpReturnCode()) + ": " +
response->getHttpReturnMessage();
delete response;
return TRI_ERROR_INTERNAL;
}
const StringBuffer& data = response->getBody();
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, data.c_str());
delete response;
if (! JsonHelper::isArray(json)) {
if (json != 0) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
TRI_json_t const* collections = JsonHelper::getArrayElement(json, "collections");
if (! JsonHelper::isList(collections)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
// create a lookup table for collections
map<string, bool> restrictList;
for (size_t i = 0; i < Collections.size(); ++i) {
restrictList.insert(pair<string, bool>(Collections[i], true));
}
// iterate over collections
const size_t n = collections->_value._objects._length;
for (size_t i = 0; i < n; ++i) {
TRI_json_t const* collection = (TRI_json_t const*) TRI_AtVector(&collections->_value._objects, i);
if (! JsonHelper::isArray(collection)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
TRI_json_t const* parameters = JsonHelper::getArrayElement(collection, "parameters");
if (! JsonHelper::isArray(parameters)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
const string id = JsonHelper::getStringValue(parameters, "id", "");
const string name = JsonHelper::getStringValue(parameters, "name", "");
const bool deleted = JsonHelper::getBooleanValue(parameters, "deleted", false);
if (id == "" || name == "") {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
if (deleted) {
continue;
}
if (name[0] == '_' && ! IncludeSystemCollections) {
continue;
}
if (restrictList.size() > 0 &&
restrictList.find(name) == restrictList.end()) {
// collection name not in list
continue;
}
// found a collection!
if (Progress) {
cout << "dumping collection '" << name << "'..." << endl;
}
// now save the collection meta data and/or the actual data
Stats._totalCollections++;
{
// save meta data
string fileName;
fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".structure.json";
int fd;
// remove an existing file first
if (TRI_ExistsFile(fileName.c_str())) {
TRI_UnlinkFile(fileName.c_str());
}
fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
const string collectionInfo = JsonHelper::toString(collection);
if (! TRI_WritePointer(fd, collectionInfo.c_str(), collectionInfo.size())) {
TRI_CLOSE(fd);
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
TRI_CLOSE(fd);
}
if (DumpData) {
// save the actual data
// First we have to go through all the shards, what are they?
TRI_json_t const* shards = JsonHelper::getArrayElement(parameters,
"shards");
map<string, string> shardTab = JsonHelper::stringObject(shards);
// This is now a map from shardIDs to DBservers
// Now set up the output file:
string fileName;
fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".data.json";
int fd;
// remove an existing file first
if (TRI_ExistsFile(fileName.c_str())) {
TRI_UnlinkFile(fileName.c_str());
}
fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
map<string, string>::iterator it;
for (it = shardTab.begin(); it != shardTab.end(); it++) {
string shardName = it->first;
string DBserver = it->second;
if (Progress) {
cout << "dumping shard '" << shardName << "' from DBserver '"
<< DBserver << "' ..." << endl;
}
res = StartBatch(DBserver, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
TRI_CLOSE(fd);
return res;
}
res = DumpShard(fd, DBserver, shardName, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
TRI_CLOSE(fd);
return res;
}
EndBatch(DBserver);
}
res = TRI_CLOSE(fd);
if (res != 0) {
if (errorMsg.empty()) {
errorMsg = "cannot write to file '" + fileName + "'";
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return res;
}
}
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_NO_ERROR;
}
MRubyClientConnection::MRubyClientConnection (mrb_state* mrb,
Endpoint* endpoint,
const string& username,
const string& password,
double requestTimeout,
double connectionTimeout,
size_t numRetries,
bool warn)
: _mrb(mrb),
_connection(0),
_lastHttpReturnCode(0),
_lastErrorMessage(""),
_client(0),
_httpResult(0) {
_connection = GeneralClientConnection::factory(endpoint, connectionTimeout, requestTimeout, numRetries, 0);
if (_connection == 0) {
throw "out of memory";
}
_client = new SimpleHttpClient(_connection, requestTimeout, warn);
_client->setUserNamePassword("/", username, password);
// connect to server and get version number
map<string, string> headerFields;
SimpleHttpResult* result = _client->request(HttpRequest::HTTP_REQUEST_GET, "/_api/version", 0, 0, headerFields);
if (!result->isComplete()) {
// save error message
_lastErrorMessage = _client->getErrorMessage();
_lastHttpReturnCode = 500;
}
else {
_lastHttpReturnCode = result->getHttpReturnCode();
if (result->getHttpReturnCode() == HttpResponse::OK) {
// default value
_version = "arango";
// convert response body to json
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, result->getBody().c_str());
if (json) {
// look up "server" value (this returns a pointer, not a copy)
TRI_json_t* server = TRI_LookupArrayJson(json, "server");
if (TRI_IsStringJson(server)) {
// "server" value is a string and content is "arango"
if (TRI_EqualString(server->_value._string.data, "arango")) {
// look up "version" value (this returns a pointer, not a copy)
TRI_json_t* vs = TRI_LookupArrayJson(json, "version");
if (TRI_IsStringJson(vs)) {
// "version" value is a string
_version = string(vs->_value._string.data, vs->_value._string.length);
}
}
// must not free server and vs, they are contained in the "json" variable and freed below
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
}
}
delete result;
}
int InitialSyncer::sendStartBatch (string& errorMsg) {
_batchId = 0;
map<string, string> const headers;
string const url = BaseUrl + "/batch";
string const body = "{\"ttl\":" + StringUtils::itoa(_batchTtl) + "}";
// send request
string const progress = "send batch start command to url " + url;
setProgress(progress);
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_POST,
url,
body.c_str(),
body.size(),
headers);
if (response == nullptr || ! response->isComplete()) {
errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) +
": " + _client->getErrorMessage();
if (response != nullptr) {
delete response;
}
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
int res = TRI_ERROR_NO_ERROR;
if (response->wasHttpError()) {
res = TRI_ERROR_REPLICATION_MASTER_ERROR;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) +
": " + response->getHttpReturnMessage();
}
if (res == TRI_ERROR_NO_ERROR) {
TRI_json_t* json = TRI_JsonString(TRI_CORE_MEM_ZONE,
response->getBody().c_str());
if (json == nullptr) {
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
else {
string const id = JsonHelper::getStringValue(json, "id", "");
if (id.empty()) {
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
else {
_batchId = StringUtils::uint64(id);
_batchUpdateTime = TRI_microtime();
}
TRI_FreeJson(TRI_CORE_MEM_ZONE, json);
}
}
delete response;
return res;
}
int InitialSyncer::run (string& errorMsg) {
if (_client == nullptr ||
_connection == nullptr ||
_endpoint == nullptr) {
errorMsg = "invalid endpoint";
return TRI_ERROR_INTERNAL;
}
setProgress("fetching master state");
int res = getMasterState(errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
res = sendStartBatch(errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
map<string, string> headers;
string url = BaseUrl + "/inventory?serverId=" + _localServerIdString;
if (_includeSystem) {
url += "&includeSystem=true";
}
// send request
string const progress = "fetching master inventory from " + url;
setProgress(progress);
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET,
url,
nullptr,
0,
headers);
if (response == nullptr || ! response->isComplete()) {
errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) +
": " + _client->getErrorMessage();
if (response != nullptr) {
delete response;
}
sendFinishBatch();
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
if (response->wasHttpError()) {
res = TRI_ERROR_REPLICATION_MASTER_ERROR;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) +
": " + response->getHttpReturnMessage();
}
else {
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE,
response->getBody().c_str());
if (JsonHelper::isObject(json)) {
res = handleInventoryResponse(json, errorMsg);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
else {
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": invalid JSON";
}
}
delete response;
sendFinishBatch();
return res;
}
