static int StartBatch (string DBserver, string& errorMsg) {
map<string, string> headers;
const string url = "/_api/replication/batch";
const string body = "{\"ttl\":300}";
string urlExt;
if (! DBserver.empty()) {
urlExt = "?DBserver="+DBserver;
}
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_POST,
url + urlExt,
body.c_str(),
body.size(),
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != 0) {
delete response;
}
if (Force) {
return TRI_ERROR_NO_ERROR;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = "got invalid response from server: HTTP " +
StringUtils::itoa(response->getHttpReturnCode()) + ": " +
response->getHttpReturnMessage();
delete response;
return TRI_ERROR_INTERNAL;
}
// convert response body to json
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, response->getBody().c_str());
delete response;
if (json == 0) {
errorMsg = "got malformed JSON";
return TRI_ERROR_INTERNAL;
}
// look up "id" value
const string id = JsonHelper::getStringValue(json, "id", "");
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
BatchId = StringUtils::uint64(id);
return TRI_ERROR_NO_ERROR;
}
int Syncer::getMasterState (string& errorMsg) {
map<string, string> headers;
static const string url = BaseUrl +
"/logger-state" +
"?serverId=" + _localServerIdString;
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET,
url,
0,
0,
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) +
": " + _client->getErrorMessage();
if (response != 0) {
delete response;
}
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
int res = TRI_ERROR_NO_ERROR;
if (response->wasHttpError()) {
res = TRI_ERROR_REPLICATION_MASTER_ERROR;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) +
": " + response->getHttpReturnMessage();
}
else {
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, response->getBody().str().c_str());
if (JsonHelper::isArray(json)) {
res = handleStateResponse(json, errorMsg);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
else {
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": invalid JSON";
}
}
delete response;
return res;
}
static int RunDump (string& errorMsg) {
map<string, string> headers;
const string url = "/_api/replication/inventory?includeSystem=" +
string(IncludeSystemCollections ? "true" : "false");
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET,
url,
0,
0,
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != 0) {
delete response;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = "got invalid response from server: HTTP " +
StringUtils::itoa(response->getHttpReturnCode()) + ": " +
response->getHttpReturnMessage();
delete response;
return TRI_ERROR_INTERNAL;
}
const string& data = response->getBody().str();
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, data.c_str());
delete response;
if (! JsonHelper::isArray(json)) {
if (json != 0) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
TRI_json_t const* collections = JsonHelper::getArrayElement(json, "collections");
if (! JsonHelper::isList(collections)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
const string tickString = JsonHelper::getStringValue(json, "tick", "");
if (tickString == "") {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
cout << "Last tick provided by server is: " << tickString << endl;
// read the server's max tick value
uint64_t maxTick = StringUtils::uint64(tickString);
// check if the user specific a max tick value
if (TickEnd > 0 && maxTick > TickEnd) {
maxTick = TickEnd;
}
// create a lookup table for collections
map<string, bool> restrictList;
for (size_t i = 0; i < Collections.size(); ++i) {
restrictList.insert(pair<string, bool>(Collections[i], true));
}
// iterate over collections
const size_t n = collections->_value._objects._length;
for (size_t i = 0; i < n; ++i) {
TRI_json_t const* collection = (TRI_json_t const*) TRI_AtVector(&collections->_value._objects, i);
if (! JsonHelper::isArray(collection)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
TRI_json_t const* parameters = JsonHelper::getArrayElement(collection, "parameters");
if (! JsonHelper::isArray(parameters)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
const string cid = JsonHelper::getStringValue(parameters, "cid", "");
const string name = JsonHelper::getStringValue(parameters, "name", "");
const bool deleted = JsonHelper::getBooleanValue(parameters, "deleted", false);
if (cid == "" || name == "") {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
if (deleted) {
continue;
}
if (name[0] == '_' && ! IncludeSystemCollections) {
continue;
}
if (restrictList.size() > 0 &&
restrictList.find(name) == restrictList.end()) {
// collection name not in list
continue;
}
// found a collection!
if (Progress) {
cout << "dumping collection '" << name << "'..." << endl;
}
// now save the collection meta data and/or the actual data
Stats._totalCollections++;
{
// save meta data
string fileName;
fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".structure.json";
int fd;
// remove an existing file first
if (TRI_ExistsFile(fileName.c_str())) {
TRI_UnlinkFile(fileName.c_str());
}
fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
const string collectionInfo = JsonHelper::toString(collection);
if (! TRI_WritePointer(fd, collectionInfo.c_str(), collectionInfo.size())) {
TRI_CLOSE(fd);
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
TRI_CLOSE(fd);
}
if (DumpData) {
// save the actual data
string fileName;
fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".data.json";
int fd;
// remove an existing file first
if (TRI_ExistsFile(fileName.c_str())) {
TRI_UnlinkFile(fileName.c_str());
}
fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
ExtendBatch();
int res = DumpCollection(fd, cid, name, parameters, maxTick, errorMsg);
TRI_CLOSE(fd);
if (res != TRI_ERROR_NO_ERROR) {
if (errorMsg.empty()) {
errorMsg = "cannot write to file '" + fileName + "'";
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return res;
}
}
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_NO_ERROR;
}
int InitialSyncer::handleCollectionDump (string const& cid,
TRI_transaction_collection_t* trxCollection,
string const& collectionName,
TRI_voc_tick_t maxTick,
string& errorMsg) {
std::string appendix;
if (_hasFlushed) {
appendix = "&flush=false";
}
else {
// only flush WAL once
appendix = "&flush=true";
_hasFlushed = true;
}
string const baseUrl = BaseUrl +
"/dump?collection=" + cid +
"&chunkSize=" + _chunkSize +
appendix;
map<string, string> headers;
TRI_voc_tick_t fromTick = 0;
int batch = 1;
while (1) {
sendExtendBatch();
string url = baseUrl + "&from=" + StringUtils::itoa(fromTick);
if (maxTick > 0) {
url += "&to=" + StringUtils::itoa(maxTick);
}
url += "&serverId=" + _localServerIdString;
// send request
string const progress = "fetching master collection dump for collection '" + collectionName +
"', id " + cid + ", batch " + StringUtils::itoa(batch);
setProgress(progress.c_str());
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET,
url,
nullptr,
0,
headers);
if (response == nullptr || ! response->isComplete()) {
errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) +
": " + _client->getErrorMessage();
if (response != nullptr) {
delete response;
}
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
if (response->wasHttpError()) {
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) +
": " + response->getHttpReturnMessage();
delete response;
return TRI_ERROR_REPLICATION_MASTER_ERROR;
}
int res = TRI_ERROR_NO_ERROR; // Just to please the compiler
bool checkMore = false;
bool found;
TRI_voc_tick_t tick;
string header = response->getHeaderField(TRI_REPLICATION_HEADER_CHECKMORE, found);
if (found) {
checkMore = StringUtils::boolean(header);
res = TRI_ERROR_NO_ERROR;
if (checkMore) {
header = response->getHeaderField(TRI_REPLICATION_HEADER_LASTINCLUDED, found);
if (found) {
tick = StringUtils::uint64(header);
if (tick > fromTick) {
fromTick = tick;
}
else {
// we got the same tick again, this indicates we're at the end
checkMore = false;
}
}
}
}
if (! found) {
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": required header is missing";
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
if (res == TRI_ERROR_NO_ERROR) {
res = applyCollectionDump(trxCollection, response, errorMsg);
}
delete response;
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
if (! checkMore || fromTick == 0) {
// done
return res;
}
batch++;
}
TRI_ASSERT(false);
return TRI_ERROR_INTERNAL;
}
int InitialSyncer::sendStartBatch (string& errorMsg) {
_batchId = 0;
map<string, string> const headers;
string const url = BaseUrl + "/batch";
string const body = "{\"ttl\":" + StringUtils::itoa(_batchTtl) + "}";
// send request
string const progress = "send batch start command to url " + url;
setProgress(progress);
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_POST,
url,
body.c_str(),
body.size(),
headers);
if (response == nullptr || ! response->isComplete()) {
errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) +
": " + _client->getErrorMessage();
if (response != nullptr) {
delete response;
}
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
int res = TRI_ERROR_NO_ERROR;
if (response->wasHttpError()) {
res = TRI_ERROR_REPLICATION_MASTER_ERROR;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) +
": " + response->getHttpReturnMessage();
}
if (res == TRI_ERROR_NO_ERROR) {
TRI_json_t* json = TRI_JsonString(TRI_CORE_MEM_ZONE,
response->getBody().c_str());
if (json == nullptr) {
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
else {
string const id = JsonHelper::getStringValue(json, "id", "");
if (id.empty()) {
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
else {
_batchId = StringUtils::uint64(id);
_batchUpdateTime = TRI_microtime();
}
TRI_FreeJson(TRI_CORE_MEM_ZONE, json);
}
}
delete response;
return res;
}
int InitialSyncer::run (string& errorMsg) {
if (_client == nullptr ||
_connection == nullptr ||
_endpoint == nullptr) {
errorMsg = "invalid endpoint";
return TRI_ERROR_INTERNAL;
}
setProgress("fetching master state");
int res = getMasterState(errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
res = sendStartBatch(errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
map<string, string> headers;
string url = BaseUrl + "/inventory?serverId=" + _localServerIdString;
if (_includeSystem) {
url += "&includeSystem=true";
}
// send request
string const progress = "fetching master inventory from " + url;
setProgress(progress);
SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET,
url,
nullptr,
0,
headers);
if (response == nullptr || ! response->isComplete()) {
errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) +
": " + _client->getErrorMessage();
if (response != nullptr) {
delete response;
}
sendFinishBatch();
return TRI_ERROR_REPLICATION_NO_RESPONSE;
}
if (response->wasHttpError()) {
res = TRI_ERROR_REPLICATION_MASTER_ERROR;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) +
": " + response->getHttpReturnMessage();
}
else {
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE,
response->getBody().c_str());
if (JsonHelper::isObject(json)) {
res = handleInventoryResponse(json, errorMsg);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
else {
res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
": invalid JSON";
}
}
delete response;
sendFinishBatch();
return res;
}
static int RunClusterDump (string& errorMsg) {
int res;
map<string, string> headers;
const string url = "/_api/replication/clusterInventory?includeSystem=" +
string(IncludeSystemCollections ? "true" : "false");
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET,
url,
0,
0,
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != 0) {
delete response;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = "got invalid response from server: HTTP " +
StringUtils::itoa(response->getHttpReturnCode()) + ": " +
response->getHttpReturnMessage();
delete response;
return TRI_ERROR_INTERNAL;
}
const StringBuffer& data = response->getBody();
TRI_json_t* json = TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, data.c_str());
delete response;
if (! JsonHelper::isArray(json)) {
if (json != 0) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
TRI_json_t const* collections = JsonHelper::getArrayElement(json, "collections");
if (! JsonHelper::isList(collections)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
// create a lookup table for collections
map<string, bool> restrictList;
for (size_t i = 0; i < Collections.size(); ++i) {
restrictList.insert(pair<string, bool>(Collections[i], true));
}
// iterate over collections
const size_t n = collections->_value._objects._length;
for (size_t i = 0; i < n; ++i) {
TRI_json_t const* collection = (TRI_json_t const*) TRI_AtVector(&collections->_value._objects, i);
if (! JsonHelper::isArray(collection)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
TRI_json_t const* parameters = JsonHelper::getArrayElement(collection, "parameters");
if (! JsonHelper::isArray(parameters)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
const string id = JsonHelper::getStringValue(parameters, "id", "");
const string name = JsonHelper::getStringValue(parameters, "name", "");
const bool deleted = JsonHelper::getBooleanValue(parameters, "deleted", false);
if (id == "" || name == "") {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
errorMsg = "got malformed JSON response from server";
return TRI_ERROR_INTERNAL;
}
if (deleted) {
continue;
}
if (name[0] == '_' && ! IncludeSystemCollections) {
continue;
}
if (restrictList.size() > 0 &&
restrictList.find(name) == restrictList.end()) {
// collection name not in list
continue;
}
// found a collection!
if (Progress) {
cout << "dumping collection '" << name << "'..." << endl;
}
// now save the collection meta data and/or the actual data
Stats._totalCollections++;
{
// save meta data
string fileName;
fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".structure.json";
int fd;
// remove an existing file first
if (TRI_ExistsFile(fileName.c_str())) {
TRI_UnlinkFile(fileName.c_str());
}
fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
const string collectionInfo = JsonHelper::toString(collection);
if (! TRI_WritePointer(fd, collectionInfo.c_str(), collectionInfo.size())) {
TRI_CLOSE(fd);
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
TRI_CLOSE(fd);
}
if (DumpData) {
// save the actual data
// First we have to go through all the shards, what are they?
TRI_json_t const* shards = JsonHelper::getArrayElement(parameters,
"shards");
map<string, string> shardTab = JsonHelper::stringObject(shards);
// This is now a map from shardIDs to DBservers
// Now set up the output file:
string fileName;
fileName = OutputDirectory + TRI_DIR_SEPARATOR_STR + name + ".data.json";
int fd;
// remove an existing file first
if (TRI_ExistsFile(fileName.c_str())) {
TRI_UnlinkFile(fileName.c_str());
}
fd = TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_CANNOT_WRITE_FILE;
}
map<string, string>::iterator it;
for (it = shardTab.begin(); it != shardTab.end(); it++) {
string shardName = it->first;
string DBserver = it->second;
if (Progress) {
cout << "dumping shard '" << shardName << "' from DBserver '"
<< DBserver << "' ..." << endl;
}
res = StartBatch(DBserver, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
TRI_CLOSE(fd);
return res;
}
res = DumpShard(fd, DBserver, shardName, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
TRI_CLOSE(fd);
return res;
}
EndBatch(DBserver);
}
res = TRI_CLOSE(fd);
if (res != 0) {
if (errorMsg.empty()) {
errorMsg = "cannot write to file '" + fileName + "'";
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return res;
}
}
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_NO_ERROR;
}