static int DumpCollection (int fd, const string& cid, const string& name, TRI_json_t const* parameters, const uint64_t maxTick, string& errorMsg) { const string baseUrl = "/_api/replication/dump?collection=" + cid + "&chunkSize=" + StringUtils::itoa(ChunkSize) + "&ticks=false&translateIds=true"; map<string, string> headers; uint64_t fromTick = TickStart; while (1) { string url = baseUrl + "&from=" + StringUtils::itoa(fromTick); if (maxTick > 0) { url += "&to=" + StringUtils::itoa(maxTick); } Stats._totalBatches++; SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET, url, 0, 0, headers); if (response == 0 || ! response->isComplete()) { errorMsg = "got invalid response from server: " + Client->getErrorMessage(); if (response != 0) { delete response; } return TRI_ERROR_INTERNAL; } if (response->wasHttpError()) { errorMsg = GetHttpErrorMessage(response); delete response; return TRI_ERROR_INTERNAL; } int res; bool checkMore = false; bool found; uint64_t tick; // TODO: fix hard-coded headers string header = response->getHeaderField("x-arango-replication-checkmore", found); if (found) { checkMore = StringUtils::boolean(header); res = TRI_ERROR_NO_ERROR; if (checkMore) { // TODO: fix hard-coded headers header = response->getHeaderField("x-arango-replication-lastincluded", found); if (found) { tick = StringUtils::uint64(header); if (tick > fromTick) { fromTick = tick; } else { // we got the same tick again, this indicates we're at the end checkMore = false; } } } } if (! found) { errorMsg = "got invalid response server: required header is missing"; res = TRI_ERROR_REPLICATION_INVALID_RESPONSE; } if (res == TRI_ERROR_NO_ERROR) { stringstream& responseBody = response->getBody(); const string body = responseBody.str(); const size_t len = body.size(); if (! TRI_WritePointer(fd, body.c_str(), len)) { res = TRI_ERROR_CANNOT_WRITE_FILE; } else { Stats._totalWritten += (uint64_t) len; } } delete response; if (res != TRI_ERROR_NO_ERROR) { return res; } if (! checkMore || fromTick == 0) { // done return res; } } assert(false); return TRI_ERROR_INTERNAL; }
int InitialSyncer::handleCollectionDump (string const& cid, TRI_transaction_collection_t* trxCollection, string const& collectionName, TRI_voc_tick_t maxTick, string& errorMsg) { std::string appendix; if (_hasFlushed) { appendix = "&flush=false"; } else { // only flush WAL once appendix = "&flush=true"; _hasFlushed = true; } string const baseUrl = BaseUrl + "/dump?collection=" + cid + "&chunkSize=" + _chunkSize + appendix; map<string, string> headers; TRI_voc_tick_t fromTick = 0; int batch = 1; while (1) { sendExtendBatch(); string url = baseUrl + "&from=" + StringUtils::itoa(fromTick); if (maxTick > 0) { url += "&to=" + StringUtils::itoa(maxTick); } url += "&serverId=" + _localServerIdString; // send request string const progress = "fetching master collection dump for collection '" + collectionName + "', id " + cid + ", batch " + StringUtils::itoa(batch); setProgress(progress.c_str()); SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET, url, nullptr, 0, headers); if (response == nullptr || ! response->isComplete()) { errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) + ": " + _client->getErrorMessage(); if (response != nullptr) { delete response; } return TRI_ERROR_REPLICATION_NO_RESPONSE; } if (response->wasHttpError()) { errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) + ": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) + ": " + response->getHttpReturnMessage(); delete response; return TRI_ERROR_REPLICATION_MASTER_ERROR; } int res = TRI_ERROR_NO_ERROR; // Just to please the compiler bool checkMore = false; bool found; TRI_voc_tick_t tick; string header = response->getHeaderField(TRI_REPLICATION_HEADER_CHECKMORE, found); if (found) { checkMore = StringUtils::boolean(header); res = TRI_ERROR_NO_ERROR; if (checkMore) { header = response->getHeaderField(TRI_REPLICATION_HEADER_LASTINCLUDED, found); if (found) { tick = StringUtils::uint64(header); if (tick > fromTick) { fromTick = tick; } else { // we got the same tick again, this indicates we're at the end checkMore = false; } } } } if (! found) { errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) + ": required header is missing"; res = TRI_ERROR_REPLICATION_INVALID_RESPONSE; } if (res == TRI_ERROR_NO_ERROR) { res = applyCollectionDump(trxCollection, response, errorMsg); } delete response; if (res != TRI_ERROR_NO_ERROR) { return res; } if (! checkMore || fromTick == 0) { // done return res; } batch++; } TRI_ASSERT(false); return TRI_ERROR_INTERNAL; }