Пример #1
0
static int DumpCollection (int fd,
                           const string& cid,
                           const string& name,
                           TRI_json_t const* parameters,
                           const uint64_t maxTick,
                           string& errorMsg) {

  const string baseUrl = "/_api/replication/dump?collection=" + cid + 
                         "&chunkSize=" + StringUtils::itoa(ChunkSize) +
                         "&ticks=false&translateIds=true";
    
  map<string, string> headers;

  uint64_t fromTick = TickStart;

  while (1) {
    string url = baseUrl + "&from=" + StringUtils::itoa(fromTick);
  
    if (maxTick > 0) {
      url += "&to=" + StringUtils::itoa(maxTick);
    }

    Stats._totalBatches++;

    SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_GET, 
                                                 url,
                                                 0, 
                                                 0,  
                                                 headers); 

    if (response == 0 || ! response->isComplete()) {
      errorMsg = "got invalid response from server: " + Client->getErrorMessage();

      if (response != 0) {
        delete response;
      }

      return TRI_ERROR_INTERNAL;
    }

    if (response->wasHttpError()) {
      errorMsg = GetHttpErrorMessage(response);
      delete response;

      return TRI_ERROR_INTERNAL;
    }

    int res;
    bool checkMore = false;
    bool found;
    uint64_t tick;

    // TODO: fix hard-coded headers
    string header = response->getHeaderField("x-arango-replication-checkmore", found);

    if (found) {
      checkMore = StringUtils::boolean(header);
      res = TRI_ERROR_NO_ERROR;
   
      if (checkMore) { 
        // TODO: fix hard-coded headers
        header = response->getHeaderField("x-arango-replication-lastincluded", found);

        if (found) {
          tick = StringUtils::uint64(header);

          if (tick > fromTick) {
            fromTick = tick;
          }
          else {
            // we got the same tick again, this indicates we're at the end
            checkMore = false;
          }
        }
      }
    }

    if (! found) {
      errorMsg = "got invalid response server: required header is missing";
      res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
    }
      
    if (res == TRI_ERROR_NO_ERROR) {
      stringstream& responseBody = response->getBody();
      const string body = responseBody.str();
      const size_t len = body.size();
     
      if (! TRI_WritePointer(fd, body.c_str(), len)) {
        res = TRI_ERROR_CANNOT_WRITE_FILE;
      }
      else {
        Stats._totalWritten += (uint64_t) len;
      }
    }

    delete response;

    if (res != TRI_ERROR_NO_ERROR) {
      return res;
    }

    if (! checkMore || fromTick == 0) {
      // done
      return res;
    }
  }

  assert(false);
  return TRI_ERROR_INTERNAL;
}
Пример #2
0
int InitialSyncer::handleCollectionDump (string const& cid,
                                         TRI_transaction_collection_t* trxCollection,
                                         string const& collectionName,
                                         TRI_voc_tick_t maxTick,
                                         string& errorMsg) {

  std::string appendix;

  if (_hasFlushed) {
    appendix = "&flush=false";
  }
  else {
    // only flush WAL once
    appendix = "&flush=true";
    _hasFlushed = true;
  }

  string const baseUrl = BaseUrl +
                         "/dump?collection=" + cid +
                         "&chunkSize=" + _chunkSize + 
                         appendix;

  map<string, string> headers;

  TRI_voc_tick_t fromTick = 0;
  int batch = 1;

  while (1) {
    sendExtendBatch();

    string url = baseUrl + "&from=" + StringUtils::itoa(fromTick);

    if (maxTick > 0) {
      url += "&to=" + StringUtils::itoa(maxTick);
    }

    url += "&serverId=" + _localServerIdString;

    // send request
    string const progress = "fetching master collection dump for collection '" + collectionName +
                            "', id " + cid + ", batch " + StringUtils::itoa(batch);

    setProgress(progress.c_str());

    SimpleHttpResult* response = _client->request(HttpRequest::HTTP_REQUEST_GET,
                                                  url,
                                                  nullptr,
                                                  0,
                                                  headers);

    if (response == nullptr || ! response->isComplete()) {
      errorMsg = "could not connect to master at " + string(_masterInfo._endpoint) +
                 ": " + _client->getErrorMessage();

      if (response != nullptr) {
        delete response;
      }

      return TRI_ERROR_REPLICATION_NO_RESPONSE;
    }

    if (response->wasHttpError()) {
      errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
                 ": HTTP " + StringUtils::itoa(response->getHttpReturnCode()) +
                 ": " + response->getHttpReturnMessage();

      delete response;

      return TRI_ERROR_REPLICATION_MASTER_ERROR;
    }

    int res = TRI_ERROR_NO_ERROR;  // Just to please the compiler
    bool checkMore = false;
    bool found;
    TRI_voc_tick_t tick;

    string header = response->getHeaderField(TRI_REPLICATION_HEADER_CHECKMORE, found);
    if (found) {
      checkMore = StringUtils::boolean(header);
      res = TRI_ERROR_NO_ERROR;

      if (checkMore) {
        header = response->getHeaderField(TRI_REPLICATION_HEADER_LASTINCLUDED, found);
        if (found) {
          tick = StringUtils::uint64(header);

          if (tick > fromTick) {
            fromTick = tick;
          }
          else {
            // we got the same tick again, this indicates we're at the end
            checkMore = false;
          }
        }
      }
    }

    if (! found) {
      errorMsg = "got invalid response from master at " + string(_masterInfo._endpoint) +
                 ": required header is missing";
      res = TRI_ERROR_REPLICATION_INVALID_RESPONSE;
    }

    if (res == TRI_ERROR_NO_ERROR) {
      res = applyCollectionDump(trxCollection, response, errorMsg);
    }

    delete response;

    if (res != TRI_ERROR_NO_ERROR) {
      return res;
    }

    if (! checkMore || fromTick == 0) {
      // done
      return res;
    }

    batch++;
  }

  TRI_ASSERT(false);
  return TRI_ERROR_INTERNAL;
}