HttpRequest::HttpRequest(Context & context, const Uri & uri, DavixError** err) :
d_ptr(new NEONRequest(ContextExplorer::SessionFactoryFromContext(context),uri)){
if(uri.getStatus() != StatusCode::OK){
DavixError::setupError(err, davix_scope_http_request(), StatusCode::UriParsingError, "impossible to parse " + uri.getString() + " ,not a valid HTTP or Webdav URL");
}
}
ssize_t HttpVecOps::readPartialBufferVec(const DavIOVecInput * input_vec,
DavIOVecOuput * output_vec,
const dav_size_t count_vec, DavixError** err){
ssize_t ret;
DavixError* tmp_err=NULL;
DAVIX_TRACE(" -> Davix Vector operation");
configure_iovec_range_header(_req, input_vec, count_vec);
if( _req.beginRequest(&tmp_err) == 0){
const int retcode = _req.getRequestCode();
switch(retcode){
case 206: // multipart req
ret = parseMultipartRequest( input_vec,
output_vec, count_vec, &tmp_err);
break;
case 200: // classical req, simulate vector ops
// TODO
ret = simulateMultiPartRequest(input_vec, output_vec, count_vec, &tmp_err);
break;
default:
httpcodeToDavixCode(_req.getRequestCode(),davix_scope_http_request(),", ", &tmp_err);
ret = -1;
}
}
if(tmp_err)
DavixError::propagateError(err, tmp_err);
DAVIX_TRACE(" <- Davix Vector operation");
return ret;
}
int DeleteOp::executeOp() {
DavixError* tmp_err=NULL;
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CORE, "{} executing op on ", _scope, _destination_url);
if(_opts.params.getProtocol() == RequestProtocol::AwsS3) {
_destination_url += "/?delete";
PostRequest req(_c, _destination_url, &tmp_err);
if(tmp_err) {
Tool::errorPrint(&tmp_err);
return -1;
}
req.setParameters(_opts.params);
std::ostringstream ss;
ss << _buf.size();
// calculate md5 of body and set header fields, these are required for S3 multi-objects delete
std::string md5;
S3::calculateMD5(_buf, md5);
req.addHeaderField("Content-MD5", md5);
req.addHeaderField("Content-Length", ss.str());
req.setRequestBody(_buf);
req.executeRequest(&tmp_err);
if(tmp_err) {
Tool::errorPrint(&tmp_err);
return -1;
}
// check response code
int code = req.getRequestCode();
if(!httpcodeIsValid(code)) {
httpcodeToDavixError(req.getRequestCode(), davix_scope_http_request(), "during S3 multi-objects delete operation", &tmp_err);
if(tmp_err) {
Tool::errorPrint(&tmp_err);
return -1;
}
}
std::vector<char> body = req.getAnswerContentVec();
TRY_DAVIX{
parse_deletion_result(code, Uri(_destination_url), _scope, body);
} CATCH_DAVIX(&tmp_err);
if(tmp_err) {
Tool::errorPrint(&tmp_err);
return -1;
}
}
// do a multi-range on selected ranges
MultirangeResult HttpIOVecOps::performMultirange(IOChainContext & iocontext,
const IntervalTree<ElemChunk> &tree,
const SortedRanges & ranges) {
DavixError * tmp_err=NULL;
dav_ssize_t tmp_ret=-1, ret = 0;
ptrdiff_t p_diff=0;
dav_size_t counter = 0;
MultirangeResult::OperationResult opresult = MultirangeResult::SUCCESS;
// calculate total bytes to be read (approximate, since ranges could overlap)
dav_ssize_t bytes_to_read = 0;
for(dav_size_t i = 0; i < ranges.size(); i++) {
bytes_to_read += (ranges[i].second - ranges[i].first + 1);
}
std::function<int (dav_off_t &, dav_off_t &)> offsetProvider = std::bind(&davIOVecProvider, ranges, std::ref(counter), std::placeholders::_1, std::placeholders::_2);
// header line need to be inferior to 8K on Apache2 / ngnix
// in Addition, some S3 implementation limit the total header size to 4k....
// 3900 bytes maximum for the range seems to be a ood compromise
std::vector< std::pair<dav_size_t, std::string> > vecRanges = generateRangeHeaders(3900, offsetProvider);
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> getPartialVec operation for {} vectors", ranges.size());
for(std::vector< std::pair<dav_size_t, std::string> >::iterator it = vecRanges.begin(); it < vecRanges.end(); ++it){
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> getPartialVec request for {} chunks", it->first);
if(it->first == 1){ // one chunk only : no need of multi part
ret += singleRangeRequest(iocontext, tree, ranges[p_diff].first, ranges[p_diff].second - ranges[p_diff].first + 1);
p_diff += 1;
}else{
GetRequest req (iocontext._context, iocontext._uri, &tmp_err);
if(tmp_err == NULL){
RequestParams request_params(iocontext._reqparams);
req.setParameters(request_params);
req.addHeaderField(req_header_byte_range, it->second);
if( req.beginRequest(&tmp_err) == 0){
const int retcode = req.getRequestCode();
// looks like the server supports multi-range requests.. yay
if(retcode == 206) {
ret = parseMultipartRequest(req, tree, &tmp_err);
// could not parse multipart response - server's broken?
// known to happen with ceph - return code is 206, but only
// returns the first range
if(ret == -1) {
opresult = MultirangeResult::NOMULTIRANGE;
req.endRequest(&tmp_err);
break;
}
}
// no multi-range.. bad server, bad
else if(retcode == 200) {
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Multi-range request resulted in getting the whole file.");
// we have two options: read the entire file or abort current
// request and start a multi-range simulation
// if this is a huge file, reading the entire contents is
// definitely not an option
if(req.getAnswerSize() > 1000000 && req.getAnswerSize() > 2*bytes_to_read) {
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "File is too large; will not waste bandwidth, bailing out");
opresult = MultirangeResult::NOMULTIRANGE;
req.endRequest(&tmp_err);
}
else {
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, "Simulating multi-part response from the contents of the entire file");
opresult = MultirangeResult::SUCCESS_BUT_NO_MULTIRANGE;
ret = simulateMultiPartRequest(req, tree, &tmp_err);
}
break;
}
else if(retcode == 416) {
ret = 0;
DavixError::clearError(&tmp_err);
}
else {
httpcodeToDavixError(req.getRequestCode(),davix_scope_http_request(),", ", &tmp_err);
ret = -1;
break;
}
p_diff += it->first;
ret += tmp_ret;
} else {
ret = -1;
break;
}
}
}
}
DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " <- getPartialVec operation for {} vectors", ranges.size());
checkDavixError(&tmp_err);
return MultirangeResult(opresult, ret);
}