KLUPD::CoreError KLUPD::HttpProtocol::receiveResponseAndData(HttpHeader &httpHeader)
{
bool httpHeaderFullyReceived = false;
const char *entity = 0;
size_t total_entity_bytes = 0;
const size_t httpBufferSize = 65536;
char httpBuffer[httpBufferSize + 1];
memset(httpBuffer, 0, httpBufferSize + 1);
size_t currentOffsetInHttpBuffer = 0;
while(true)
{
CoreError receiveResult = CORE_NO_ERROR;
const int bytesReceived = m_socket.recv(httpBuffer + currentOffsetInHttpBuffer,
httpBufferSize - currentOffsetInHttpBuffer, receiveResult);
currentOffsetInHttpBuffer += bytesReceived;
// according to RFC 2616, 4.4 point 5: connection close may indicate
// file transfer completion if Range or Content-Length is not specified
if(httpHeaderFullyReceived
&& (receiveResult == CORE_REMOTE_HOST_CLOSED_CONNECTION)
&& (httpHeader.m_contentLength == -1))
{
TRACE_MESSAGE2("HTTP connection closed by server, no Content-Length field, consider file is obtained, size %d bytes", currentOffsetInHttpBuffer);
return CORE_NO_ERROR;
}
if(receiveResult != CORE_NO_ERROR)
return receiveResult;
// size of entity part that is currently in buffer
size_t entityBytes = 0;
if(httpHeaderFullyReceived)
{
// http header was obtained on previous iteration
entityBytes = currentOffsetInHttpBuffer;
entity = httpBuffer;
}
else
{
// HTTP header has not been obtained yet, check if it is in httpBuffer
const char *const headerEndMarker = "\r\n\r\n";
char *const headerLastPosition = strstr(httpBuffer, headerEndMarker);
if(!headerLastPosition)
{
// no end-of-header marker found
if(httpBufferSize < currentOffsetInHttpBuffer)
{
TRACE_MESSAGE3("Error: response HTTP header can not fit into %d-bytes buffer (already received %d bytes)",
httpBufferSize, currentOffsetInHttpBuffer);
return CORE_DOWNLOAD_ERROR;
}
// continue reading from socket
continue;
}
httpHeaderFullyReceived = true;
// pass to applicaiton data included into HTTP header
fullHttpHeaderReceived(std::string(httpBuffer, headerLastPosition + strlen(headerEndMarker)));
*headerLastPosition = 0;
TRACE_MESSAGE2("HTTP response received:\n%s", HTTPRequestBuilder::toString(httpBuffer).c_str());
// Parse HTTP header, data loaded into httpHeader structure
if(!httpHeader.load(httpBuffer, m_authorizationDriver))
{
TRACE_MESSAGE("Failed to parse HTTP response header");
return CORE_DOWNLOAD_ERROR;
}
// it is expected 206 code in case regetting is used
if(m_regettingPosition && (httpHeader.httpCode() == 200))
{
TRACE_MESSAGE("It is expected 206 code in case regetting is used, but 200 code is received");
return CORE_DOWNLOAD_ERROR;
}
// data from socket after the HTTP header
entityBytes = (httpBuffer + currentOffsetInHttpBuffer) // pointer to the end of received bytes
- (headerLastPosition + strlen(headerEndMarker)); // pointer to the data (begin of the file)
entity = entityBytes ? headerLastPosition + strlen(headerEndMarker) : 0;
}
total_entity_bytes += entityBytes;
if(httpHeader.isFile()
// only header has been received, but data is not received yet
&& entityBytes)
{
const CoreError saveDataToFileResult = dataReceived(reinterpret_cast<const unsigned char *>(entity), entityBytes);
if(!isSuccess(saveDataToFileResult))
{
TRACE_MESSAGE2("Failed to write data obtained from HTTP Server, result %S",
toString(saveDataToFileResult).toWideChar());
return saveDataToFileResult;
}
m_regettingPosition += entityBytes;
}
//////////////////////////////////////////////////////////////////////////
/// check if complete file is downloaded
// Content-Length header presents
if(httpHeader.m_contentLength == -1)
{
TRACE_MESSAGE("HTTP chunk received (header does not contain the content-length field), continue receiving message until connection is closed by remote peer");
}
else if(static_cast<size_t>(httpHeader.m_contentLength) <= total_entity_bytes)
{
TRACE_MESSAGE2("HTTP message successfully received, content length = %d", httpHeader.m_contentLength);
return CORE_NO_ERROR;
}
currentOffsetInHttpBuffer = 0;
entityBytes = 0;
}
}
KLUPD::CoreError KLUPD::HttpProtocol::httpRequestAttempt(
const Path &fileNameIn,
const Path &relativeUrlPathIn,
const Address &serverAddressIn, const std::string &userAgent,
const bool useProxy, const Address &proxyAddressIn,
const AuthorizationTypeList &proxyAuthorizationMethods,
const size_t regettingPosition)
{
size_t infiniteRedirectLoop = 0;
Path fileName = fileNameIn;
Path relativeUrlPath = relativeUrlPathIn;
Address serverAddress = serverAddressIn;
Address proxyAddress = proxyAddressIn;
m_regettingPosition = regettingPosition;
// cache for next iteration generate request
// at first attempt the target is not known, but proxy server target is used,
// because of possible protection from client for authorization on server
DownloadProgress::AuthorizationTarget authorizationTarget = DownloadProgress::proxyServer;
////////////////////
// the State flags agains proxies that close connection after authorization switch.
// There are proxies (e.g. Squid/2.4.STABLE7) which deals in next way:
// *** Client -> Server: GET file
// *** Server -> Client: HTTP 407 / Proxy-Connection: keep-alive
// *** Server closes connection
// *** Client -> Server GET 407
//
// Here are 3 flags to track such situations, that previous response is received,
// BUT then server closes connection
bool previousResponseReceived = false;
bool authorizationTypeSwitched = false;
bool requestHasAlreadyBeenRepeated = false;
////////////////////
////////////////////
// in case proxy server requires authorization, but sends incorrect (from HTTP)
// code for authorization failure in next way
// *** Client -> Server: GET file
// *** Server -> Client: HTTP 407 / Proxy-Authorization: NTLM
// *** Client -> Server GET / Proxy-Authorization: NTLM
// *** Server -> Client: HTTP 502 (or 403 or other code)
// *** here is authorization state is forgotten, because 502 request does NOT contain "Proxy-Authorization: NTLM"
// *** Client ask from product new credentials
// *** Client -> Server GET / Proxy-Authorization: NTLM (with new credentials)
bool authorizationWasNeededOnProxy = false;
// it makes no sence to try Ntlm without credentials authorization type,
// because this authorization type does not depend on provided credentials
bool ntlmAuthorizationTriedAlready = false;
int lastHttpCode = 0;
for(size_t protectionAgainstCyclingCounter = 100; protectionAgainstCyclingCounter; --protectionAgainstCyclingCounter)
{
const CoreError connectionResult = setupLowLevelConnectionIfNeed(useProxy, useProxy ? proxyAddress : serverAddress, proxyAuthorizationMethods);
if(connectionResult != CORE_NO_ERROR)
{
TRACE_MESSAGE2("Failed to setup connection to HTTP Server, result '%S'",
toString(connectionResult).toWideChar());
return connectionResult;
}
// make proxy authorization header
std::string proxyAuthorizationHeader;
if(!m_authorizationDriver.makeProxyAuthorizationHeader(proxyAuthorizationHeader)
// we know authorization methods supported by proxy server
|| ((m_authorizationDriver.currentAuthorizationType() == noAuthorization) && !m_authorizationDriver.supportedAuthorizationTypesByServer().empty()))
{
if(!switchAuthorization(fileName, relativeUrlPath, authorizationTarget, proxyAddress,
authorizationTypeSwitched, ntlmAuthorizationTriedAlready))
{
if(lastHttpCode)
return HttpHeader::convertHttpCodeToUpdaterCode(lastHttpCode);
return useProxy ? CORE_PROXY_AUTH_ERROR : CORE_SERVER_AUTH_ERROR;
}
// try other authorization type
continue;
}
HTTPRequestBuilder requestBuilder(m_method);
const std::vector<unsigned char> requestBuffer = requestBuilder.generateRequest(
fileName,
relativeUrlPath,
useProxy,
serverAddress,
userAgent,
proxyAuthorizationHeader,
m_regettingPosition,
m_postData);
TRACE_MESSAGE2("Sending HTTP request\n%s", requestBuilder.toString().c_str());
if(requestBuffer.empty())
{
TRACE_MESSAGE("Failed to send empty HTTP request");
return CORE_DOWNLOAD_ERROR;
}
const CoreError sendRequestResult = m_socket.send(reinterpret_cast<const char *>(&requestBuffer[0]), requestBuffer.size());
if(sendRequestResult != CORE_NO_ERROR)
{
TRACE_MESSAGE2("Failed to send HTTP request, error %S", toString(sendRequestResult).toWideChar());
if((sendRequestResult == CORE_REMOTE_HOST_CLOSED_CONNECTION)
&& previousResponseReceived && authorizationTypeSwitched && !requestHasAlreadyBeenRepeated)
{
TRACE_MESSAGE("Repeating the same request (without authorization switch), because server was reachable, but unexpectedly closed connection");
requestHasAlreadyBeenRepeated = true;
continue;
}
return sendRequestResult;
}
HttpHeader httpHeader;
const CoreError receiveResponseAndDataResult = receiveResponseAndData(httpHeader);
const bool needCloseConnection = httpHeader.needCloseConnection(useProxy)
|| (receiveResponseAndDataResult != CORE_NO_ERROR);
if(needCloseConnection)
{
TRACE_MESSAGE2("Closing connection to HTTP server, get file result %S",
toString(receiveResponseAndDataResult).toWideChar());
closeSession();
}
if(receiveResponseAndDataResult != CORE_NO_ERROR)
{
TRACE_MESSAGE2("Failed to receive HTTP response, error %S",
toString(receiveResponseAndDataResult).toWideChar());
if((receiveResponseAndDataResult == CORE_REMOTE_HOST_CLOSED_CONNECTION)
&& previousResponseReceived && authorizationTypeSwitched && !requestHasAlreadyBeenRepeated)
{
TRACE_MESSAGE("Repeating the same request (without authorization switch), because server was reachable, but unexpectedly closed connection");
requestHasAlreadyBeenRepeated = true;
continue;
}
return receiveResponseAndDataResult;
}
previousResponseReceived = true;
requestHasAlreadyBeenRepeated = false;
m_authorizationDriver.authorized(
// authorization information is reset in case connection is to be closed
!needCloseConnection
// authorization success in case file received or successful redirect
&& (httpHeader.isFile()
|| httpHeader.redirectRequired()
|| httpHeader.fileNotFound()));
if(httpHeader.isFile())
{
// in case proxy server by some ocassion desided to close
// connection after successful file receive event
if(needCloseConnection)
m_authorizationDriver.resetNtlmState();
return CORE_NO_ERROR;
}
authorizationTarget = httpHeader.authorizationTarget();
authorizationWasNeededOnProxy = !proxyAuthorizationHeader.empty();
lastHttpCode = httpHeader.httpCode();
// authorization error
if(httpHeader.authorizationNeeded())
{
authorizationWasNeededOnProxy = (authorizationTarget == DownloadProgress::proxyServer);
if(!switchAuthorization(fileName, relativeUrlPath, httpHeader.authorizationTarget(), proxyAddress,
authorizationTypeSwitched, ntlmAuthorizationTriedAlready))
{
return httpHeader.convertHttpCodeToUpdaterCode();
}
}
// redirect needed
else if(httpHeader.redirectRequired())
{
// protection against infinite loop (according to RFC 2616)
if(++infiniteRedirectLoop > 2)
{
TRACE_MESSAGE2("Infinite redirection loop detected for location '%S'",
httpHeader.m_location.toWideChar());
return CORE_NO_SOURCE_FILE;
}
if(httpHeader.m_location.isAbsoluteUri())
serverAddress.parse(httpHeader.m_location);
else
{
serverAddress.parse(toProtocolPrefix(serverAddress.m_protocol) + serverAddress.m_hostname
+ serverAddress.m_path + httpHeader.m_location);
serverAddress.m_path.correctPathDelimiters();
}
fileName = serverAddress.m_fileName;
relativeUrlPath.erase();
TRACE_MESSAGE3("HTTP Redirect to file '%S' on server %S",
fileName.toWideChar(), serverAddress.toString().toWideChar());
}
// known HTTP results
else if(httpHeader.resourceUnavailable()
|| httpHeader.fileNotFound())
{
return httpHeader.convertHttpCodeToUpdaterCode();
}
// retry authorization with other credentials in case Forbidden
// code received after successful authorization has happened
else if(httpHeader.retryAuthorization(authorizationWasNeededOnProxy)
&& treat_403_502_httpCodesAs407())
{
if(!switchAuthorization(fileName, relativeUrlPath, DownloadProgress::proxyServer,
proxyAddress, authorizationTypeSwitched, ntlmAuthorizationTriedAlready))
{
TRACE_MESSAGE2("Authorization was needed, but error HTTP code '%d' received and switch to next authorization type failed",
httpHeader.httpCode());
return httpHeader.convertHttpCodeToUpdaterCode();
}
TRACE_MESSAGE3("Authorization was needed, but error HTTP code '%d' received, try next authorization type '%S'",
httpHeader.httpCode(),
toString(m_authorizationDriver.currentAuthorizationType()).toWideChar());
}
else
{
// processing HTTP code is not implemented
m_authorizationDriver.resetAuthorizatoinState(proxyAuthorizationMethods);
return httpHeader.convertHttpCodeToUpdaterCode();
}
}
// Authorization state machine is complex and may contain bug,
// that is why protection against infinite loop is implemented
TRACE_MESSAGE3("Error in HTTP authorization state implementation: credentials '%S', current authorization type '%S'",
m_authorizationDriver.credentials().toString().toWideChar(),
toString(m_authorizationDriver.currentAuthorizationType()).toWideChar());
return CORE_DOWNLOAD_ERROR;
}
