static int StartBatch (string DBserver, string& errorMsg) { map<string, string> headers; const string url = "/_api/replication/batch"; const string body = "{\"ttl\":300}"; string urlExt; if (! DBserver.empty()) { urlExt = "?DBserver="+DBserver; } SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_POST, url + urlExt, body.c_str(), body.size(), headers); if (response == 0 || ! response->isComplete()) { errorMsg = "got invalid response from server: " + Client->getErrorMessage(); if (response != 0) { delete response; } if (Force) { return TRI_ERROR_NO_ERROR; } return TRI_ERROR_INTERNAL; } if (response->wasHttpError()) { errorMsg = "got invalid response from server: HTTP " + StringUtils::itoa(response->getHttpReturnCode()) + ": " + response->getHttpReturnMessage(); delete response; return TRI_ERROR_INTERNAL; } // convert response body to json TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, response->getBody().c_str()); delete response; if (json == 0) { errorMsg = "got malformed JSON"; return TRI_ERROR_INTERNAL; } // look up "id" value const string id = JsonHelper::getStringValue(json, "id", ""); TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); BatchId = StringUtils::uint64(id); return TRI_ERROR_NO_ERROR; }
int Syncer::getMasterState (string& errorMsg) { map<string, string> headers; static const string url = BaseUrl + "/logger-state" + "?serverId=" + _localServerIdString; SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET, url, 0, 0, headers); if (response == 0 || ! response->isComplete()) { errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) + ": " + _client->getErrorMessage(); if (response != 0) { delete response; } return TRI_ERROR_REPLICATION_NO_RESPONSE; } int res = TRI_ERROR_NO_ERROR; if (response->wasHttpError()) { res = TRI_ERROR_REPLICATION_MASTER_ERROR; errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) + ": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) + ": " + response->getHttpReturnMessage(); } else { TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, response->getBody().str().c_str()); if (JsonHelper::isArray(json)) { res = handleStateResponse(json, errorMsg); TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); } else { res = TRI_ERROR_REPLICATION_INVALID_RESPONSE; errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) + ": invalid JSON"; } } delete response; return res; }
static string GetArangoVersion () { map<string, string> headers; SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET, "/_api/version", nullptr, 0, headers); if (response == nullptr || ! response->isComplete()) { if (response != nullptr) { delete response; } return ""; } string version; if (response->getHttpReturnCode() == HttpResponse::OK) { // default value version = "arango"; // convert response body to json TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, response->getBody().c_str()); if (json) { // look up "server" value const string server = JsonHelper::getStringValue(json, "server", ""); // "server" value is a string and content is "arango" if (server == "arango") { // look up "version" value version = JsonHelper::getStringValue(json, "version", ""); } TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); } } else { if (response->wasHttpError()) { Client->setErrorMessage(GetHttpErrorMessage(response), false); } Connection->disconnect(); } delete response; return version; }
static bool GetArangoIsCluster () { map<string, string> headers; SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET, "/_admin/server/role", "", 0, headers); if (response == nullptr || ! response->isComplete()) { if (response != nullptr) { delete response; } return false; } string role = "UNDEFINED"; if (response->getHttpReturnCode() == HttpResponse::OK) { // convert response body to json TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, response->getBody().c_str()); if (json != nullptr) { // look up "server" value role = JsonHelper::getStringValue(json, "role", "UNDEFINED"); TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); } } else { if (response->wasHttpError()) { Client->setErrorMessage(GetHttpErrorMessage(response), false); } Connection->disconnect(); } delete response; return role == "COORDINATOR"; }
static int RunDump (string& errorMsg) { map<string, string> headers; const string url = "/_api/replication/inventory?includeSystem=" + string(IncludeSystemCollections ? "true" : "false"); SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET, url, 0, 0, headers); if (response == 0 || ! response->isComplete()) { errorMsg = "got invalid response from server: " + Client->getErrorMessage(); if (response != 0) { delete response; } return TRI_ERROR_INTERNAL; } if (response->wasHttpError()) { errorMsg = "got invalid response from server: HTTP " + StringUtils::itoa(response->getHttpReturnCode()) + ": " + response->getHttpReturnMessage(); delete response; return TRI_ERROR_INTERNAL; } const string& data = response->getBody().str(); TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, data.c_str()); delete response; if (! JsonHelper::isArray(json)) { if (json != 0) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); } errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } TRI_json_t const* collections = JsonHelper::getArrayElement(json, "collections"); if (! JsonHelper::isList(collections)) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } const string tickString = JsonHelper::getStringValue(json, "tick", ""); if (tickString == "") { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } cout << "Last tick provided by server is: " << tickString << endl; // read the server's max tick value uint64_t maxTick = StringUtils::uint64(tickString); // check if the user specific a max tick value if (TickEnd > 0 && maxTick > TickEnd) { maxTick = TickEnd; } // create a lookup table for collections map<string, bool> restrictList; for (size_t i = 0; i < Collections.size(); ++i) { restrictList.insert(pair<string, bool>(Collections[i], true)); } // iterate over collections const size_t n = collections->_value._objects._length; for (size_t i = 0; i < n; ++i) { TRI_json_t const* collection = (TRI_json_t const*) TRI_AtVector(&collections->_value._objects, i); if (! JsonHelper::isArray(collection)) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } TRI_json_t const* parameters = JsonHelper::getArrayElement(collection, "parameters"); if (! JsonHelper::isArray(parameters)) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } const string cid = JsonHelper::getStringValue(parameters, "cid", ""); const string name = JsonHelper::getStringValue(parameters, "name", ""); const bool deleted = JsonHelper::getBooleanValue(parameters, "deleted", false); if (cid == "" || name == "") { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } if (deleted) { continue; } if (name[0] == '_' && ! IncludeSystemCollections) { continue; } if (restrictList.size() > 0 && restrictList.find(name) == restrictList.end()) { // collection name not in list continue; } // found a collection! if (Progress) { cout << "dumping collection '" << name << "'..." << endl; } // now save the collection meta data and/or the actual data Stats._totalCollections++; { // save meta data string fileName; fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".structure.json"; int fd; // remove an existing file first if (TRI_ExistsFile(fileName.c_str())) { TRI_UnlinkFile(fileName.c_str()); } fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR); if (fd < 0) { errorMsg = "cannot write to file '" + fileName + "'"; TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return TRI_ERROR_CANNOT_WRITE_FILE; } const string collectionInfo = JsonHelper::toString(collection); if (! TRI_WritePointer(fd, collectionInfo.c_str(), collectionInfo.size())) { TRI_CLOSE(fd); errorMsg = "cannot write to file '" + fileName + "'"; TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return TRI_ERROR_CANNOT_WRITE_FILE; } TRI_CLOSE(fd); } if (DumpData) { // save the actual data string fileName; fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".data.json"; int fd; // remove an existing file first if (TRI_ExistsFile(fileName.c_str())) { TRI_UnlinkFile(fileName.c_str()); } fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR); if (fd < 0) { errorMsg = "cannot write to file '" + fileName + "'"; TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return TRI_ERROR_CANNOT_WRITE_FILE; } ExtendBatch(); int res = DumpCollection(fd, cid, name, parameters, maxTick, errorMsg); TRI_CLOSE(fd); if (res != TRI_ERROR_NO_ERROR) { if (errorMsg.empty()) { errorMsg = "cannot write to file '" + fileName + "'"; } TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return res; } } } TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return TRI_ERROR_NO_ERROR; }
static int DumpCollection (int fd, const string& cid, const string& name, TRI_json_t const* parameters, const uint64_t maxTick, string& errorMsg) { const string baseUrl = "/_api/replication/dump?collection=" + cid + "&chunkSize=" + StringUtils::itoa(ChunkSize) + "&ticks=false&translateIds=true"; map<string, string> headers; uint64_t fromTick = TickStart; while (1) { string url = baseUrl + "&from=" + StringUtils::itoa(fromTick); if (maxTick > 0) { url += "&to=" + StringUtils::itoa(maxTick); } Stats._totalBatches++; SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET, url, 0, 0, headers); if (response == 0 || ! response->isComplete()) { errorMsg = "got invalid response from server: " + Client->getErrorMessage(); if (response != 0) { delete response; } return TRI_ERROR_INTERNAL; } if (response->wasHttpError()) { errorMsg = GetHttpErrorMessage(response); delete response; return TRI_ERROR_INTERNAL; } int res; bool checkMore = false; bool found; uint64_t tick; // TODO: fix hard-coded headers string header = response->getHeaderField("x-arango-replication-checkmore", found); if (found) { checkMore = StringUtils::boolean(header); res = TRI_ERROR_NO_ERROR; if (checkMore) { // TODO: fix hard-coded headers header = response->getHeaderField("x-arango-replication-lastincluded", found); if (found) { tick = StringUtils::uint64(header); if (tick > fromTick) { fromTick = tick; } else { // we got the same tick again, this indicates we're at the end checkMore = false; } } } } if (! found) { errorMsg = "got invalid response server: required header is missing"; res = TRI_ERROR_REPLICATION_INVALID_RESPONSE; } if (res == TRI_ERROR_NO_ERROR) { stringstream& responseBody = response->getBody(); const string body = responseBody.str(); const size_t len = body.size(); if (! TRI_WritePointer(fd, body.c_str(), len)) { res = TRI_ERROR_CANNOT_WRITE_FILE; } else { Stats._totalWritten += (uint64_t) len; } } delete response; if (res != TRI_ERROR_NO_ERROR) { return res; } if (! checkMore || fromTick == 0) { // done return res; } } assert(false); return TRI_ERROR_INTERNAL; }
int InitialSyncer::sendStartBatch (string& errorMsg) { _batchId = 0; map<string, string> const headers; string const url = BaseUrl + "/batch"; string const body = "{\"ttl\":" + StringUtils::itoa(_batchTtl) + "}"; // send request string const progress = "send batch start command to url " + url; setProgress(progress); SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_POST, url, body.c_str(), body.size(), headers); if (response == nullptr || ! response->isComplete()) { errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) + ": " + _client->getErrorMessage(); if (response != nullptr) { delete response; } return TRI_ERROR_REPLICATION_NO_RESPONSE; } int res = TRI_ERROR_NO_ERROR; if (response->wasHttpError()) { res = TRI_ERROR_REPLICATION_MASTER_ERROR; errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) + ": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) + ": " + response->getHttpReturnMessage(); } if (res == TRI_ERROR_NO_ERROR) { TRI_json_t* json = TRI_JsonString(TRI_CORE_MEM_ZONE, response->getBody().c_str()); if (json == nullptr) { res = TRI_ERROR_REPLICATION_INVALID_RESPONSE; } else { string const id = JsonHelper::getStringValue(json, "id", ""); if (id.empty()) { res = TRI_ERROR_REPLICATION_INVALID_RESPONSE; } else { _batchId = StringUtils::uint64(id); _batchUpdateTime = TRI_microtime(); } TRI_FreeJson(TRI_CORE_MEM_ZONE, json); } } delete response; return res; }
int InitialSyncer::run (string& errorMsg) { if (_client == nullptr || _connection == nullptr || _endpoint == nullptr) { errorMsg = "invalid endpoint"; return TRI_ERROR_INTERNAL; } setProgress("fetching master state"); int res = getMasterState(errorMsg); if (res != TRI_ERROR_NO_ERROR) { return res; } res = sendStartBatch(errorMsg); if (res != TRI_ERROR_NO_ERROR) { return res; } map<string, string> headers; string url = BaseUrl + "/inventory?serverId=" + _localServerIdString; if (_includeSystem) { url += "&includeSystem=true"; } // send request string const progress = "fetching master inventory from " + url; setProgress(progress); SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET, url, nullptr, 0, headers); if (response == nullptr || ! response->isComplete()) { errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) + ": " + _client->getErrorMessage(); if (response != nullptr) { delete response; } sendFinishBatch(); return TRI_ERROR_REPLICATION_NO_RESPONSE; } if (response->wasHttpError()) { res = TRI_ERROR_REPLICATION_MASTER_ERROR; errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) + ": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) + ": " + response->getHttpReturnMessage(); } else { TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, response->getBody().c_str()); if (JsonHelper::isObject(json)) { res = handleInventoryResponse(json, errorMsg); TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); } else { res = TRI_ERROR_REPLICATION_INVALID_RESPONSE; errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) + ": invalid JSON"; } } delete response; sendFinishBatch(); return res; }
MRubyClientConnection::MRubyClientConnection (mrb_state* mrb, Endpoint* endpoint, const string& username, const string& password, double requestTimeout, double connectionTimeout, size_t numRetries, bool warn) : _mrb(mrb), _connection(0), _lastHttpReturnCode(0), _lastErrorMessage(""), _client(0), _httpResult(0) { _connection = GeneralClientConnection::factory(endpoint, connectionTimeout, requestTimeout, numRetries, 0); if (_connection == 0) { throw "out of memory"; } _client = new SimpleHttpClient(_connection, requestTimeout, warn); _client->setUserNamePassword("/", username, password); // connect to server and get version number map<string, string> headerFields; SimpleHttpResult* result = _client->request(HttpRequest::HTTP_REQUEST_GET, "/_api/version", 0, 0, headerFields); if (!result->isComplete()) { // save error message _lastErrorMessage = _client->getErrorMessage(); _lastHttpReturnCode = 500; } else { _lastHttpReturnCode = result->getHttpReturnCode(); if (result->getHttpReturnCode() == HttpResponse::OK) { // default value _version = "arango"; // convert response body to json TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, result->getBody().c_str()); if (json) { // look up "server" value (this returns a pointer, not a copy) TRI_json_t* server = TRI_LookupArrayJson(json, "server"); if (TRI_IsStringJson(server)) { // "server" value is a string and content is "arango" if (TRI_EqualString(server->_value._string.data, "arango")) { // look up "version" value (this returns a pointer, not a copy) TRI_json_t* vs = TRI_LookupArrayJson(json, "version"); if (TRI_IsStringJson(vs)) { // "version" value is a string _version = string(vs->_value._string.data, vs->_value._string.length); } } // must not free server and vs, they are contained in the "json" variable and freed below } TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); } } } delete result; }
static int RunClusterDump (string& errorMsg) { int res; map<string, string> headers; const string url = "/_api/replication/clusterInventory?includeSystem=" + string(IncludeSystemCollections ? "true" : "false"); SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET, url, 0, 0, headers); if (response == 0 || ! response->isComplete()) { errorMsg = "got invalid response from server: " + Client->getErrorMessage(); if (response != 0) { delete response; } return TRI_ERROR_INTERNAL; } if (response->wasHttpError()) { errorMsg = "got invalid response from server: HTTP " + StringUtils::itoa(response->getHttpReturnCode()) + ": " + response->getHttpReturnMessage(); delete response; return TRI_ERROR_INTERNAL; } const StringBuffer& data = response->getBody(); TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, data.c_str()); delete response; if (! JsonHelper::isArray(json)) { if (json != 0) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); } errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } TRI_json_t const* collections = JsonHelper::getArrayElement(json, "collections"); if (! JsonHelper::isList(collections)) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } // create a lookup table for collections map<string, bool> restrictList; for (size_t i = 0; i < Collections.size(); ++i) { restrictList.insert(pair<string, bool>(Collections[i], true)); } // iterate over collections const size_t n = collections->_value._objects._length; for (size_t i = 0; i < n; ++i) { TRI_json_t const* collection = (TRI_json_t const*) TRI_AtVector(&collections->_value._objects, i); if (! JsonHelper::isArray(collection)) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } TRI_json_t const* parameters = JsonHelper::getArrayElement(collection, "parameters"); if (! JsonHelper::isArray(parameters)) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } const string id = JsonHelper::getStringValue(parameters, "id", ""); const string name = JsonHelper::getStringValue(parameters, "name", ""); const bool deleted = JsonHelper::getBooleanValue(parameters, "deleted", false); if (id == "" || name == "") { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); errorMsg = "got malformed JSON response from server"; return TRI_ERROR_INTERNAL; } if (deleted) { continue; } if (name[0] == '_' && ! IncludeSystemCollections) { continue; } if (restrictList.size() > 0 && restrictList.find(name) == restrictList.end()) { // collection name not in list continue; } // found a collection! if (Progress) { cout << "dumping collection '" << name << "'..." << endl; } // now save the collection meta data and/or the actual data Stats._totalCollections++; { // save meta data string fileName; fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".structure.json"; int fd; // remove an existing file first if (TRI_ExistsFile(fileName.c_str())) { TRI_UnlinkFile(fileName.c_str()); } fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR); if (fd < 0) { errorMsg = "cannot write to file '" + fileName + "'"; TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return TRI_ERROR_CANNOT_WRITE_FILE; } const string collectionInfo = JsonHelper::toString(collection); if (! TRI_WritePointer(fd, collectionInfo.c_str(), collectionInfo.size())) { TRI_CLOSE(fd); errorMsg = "cannot write to file '" + fileName + "'"; TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return TRI_ERROR_CANNOT_WRITE_FILE; } TRI_CLOSE(fd); } if (DumpData) { // save the actual data // First we have to go through all the shards, what are they? TRI_json_t const* shards = JsonHelper::getArrayElement(parameters, "shards"); map<string, string> shardTab = JsonHelper::stringObject(shards); // This is now a map from shardIDs to DBservers // Now set up the output file: string fileName; fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".data.json"; int fd; // remove an existing file first if (TRI_ExistsFile(fileName.c_str())) { TRI_UnlinkFile(fileName.c_str()); } fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR); if (fd < 0) { errorMsg = "cannot write to file '" + fileName + "'"; TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return TRI_ERROR_CANNOT_WRITE_FILE; } map<string, string>::iterator it; for (it = shardTab.begin(); it != shardTab.end(); it++) { string shardName = it->first; string DBserver = it->second; if (Progress) { cout << "dumping shard '" << shardName << "' from DBserver '" << DBserver << "' ..." << endl; } res = StartBatch(DBserver, errorMsg); if (res != TRI_ERROR_NO_ERROR) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); TRI_CLOSE(fd); return res; } res = DumpShard(fd, DBserver, shardName, errorMsg); if (res != TRI_ERROR_NO_ERROR) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); TRI_CLOSE(fd); return res; } EndBatch(DBserver); } res = TRI_CLOSE(fd); if (res != 0) { if (errorMsg.empty()) { errorMsg = "cannot write to file '" + fileName + "'"; } TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return res; } } } TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); return TRI_ERROR_NO_ERROR; }