KLUPD::CoreError KLUPD::HttpProtocol::receiveResponseAndData(HttpHeader &httpHeader) { bool httpHeaderFullyReceived = false; const char *entity = 0; size_t total_entity_bytes = 0; const size_t httpBufferSize = 65536; char httpBuffer[httpBufferSize + 1]; memset(httpBuffer, 0, httpBufferSize + 1); size_t currentOffsetInHttpBuffer = 0; while(true) { CoreError receiveResult = CORE_NO_ERROR; const int bytesReceived = m_socket.recv(httpBuffer + currentOffsetInHttpBuffer, httpBufferSize - currentOffsetInHttpBuffer, receiveResult); currentOffsetInHttpBuffer += bytesReceived; // according to RFC 2616, 4.4 point 5: connection close may indicate // file transfer completion if Range or Content-Length is not specified if(httpHeaderFullyReceived && (receiveResult == CORE_REMOTE_HOST_CLOSED_CONNECTION) && (httpHeader.m_contentLength == -1)) { TRACE_MESSAGE2("HTTP connection closed by server, no Content-Length field, consider file is obtained, size %d bytes", currentOffsetInHttpBuffer); return CORE_NO_ERROR; } if(receiveResult != CORE_NO_ERROR) return receiveResult; // size of entity part that is currently in buffer size_t entityBytes = 0; if(httpHeaderFullyReceived) { // http header was obtained on previous iteration entityBytes = currentOffsetInHttpBuffer; entity = httpBuffer; } else { // HTTP header has not been obtained yet, check if it is in httpBuffer const char *const headerEndMarker = "\r\n\r\n"; char *const headerLastPosition = strstr(httpBuffer, headerEndMarker); if(!headerLastPosition) { // no end-of-header marker found if(httpBufferSize < currentOffsetInHttpBuffer) { TRACE_MESSAGE3("Error: response HTTP header can not fit into %d-bytes buffer (already received %d bytes)", httpBufferSize, currentOffsetInHttpBuffer); return CORE_DOWNLOAD_ERROR; } // continue reading from socket continue; } httpHeaderFullyReceived = true; // pass to applicaiton data included into HTTP header fullHttpHeaderReceived(std::string(httpBuffer, headerLastPosition + strlen(headerEndMarker))); *headerLastPosition = 0; TRACE_MESSAGE2("HTTP response received:\n%s", HTTPRequestBuilder::toString(httpBuffer).c_str()); // Parse HTTP header, data loaded into httpHeader structure if(!httpHeader.load(httpBuffer, m_authorizationDriver)) { TRACE_MESSAGE("Failed to parse HTTP response header"); return CORE_DOWNLOAD_ERROR; } // it is expected 206 code in case regetting is used if(m_regettingPosition && (httpHeader.httpCode() == 200)) { TRACE_MESSAGE("It is expected 206 code in case regetting is used, but 200 code is received"); return CORE_DOWNLOAD_ERROR; } // data from socket after the HTTP header entityBytes = (httpBuffer + currentOffsetInHttpBuffer) // pointer to the end of received bytes - (headerLastPosition + strlen(headerEndMarker)); // pointer to the data (begin of the file) entity = entityBytes ? headerLastPosition + strlen(headerEndMarker) : 0; } total_entity_bytes += entityBytes; if(httpHeader.isFile() // only header has been received, but data is not received yet && entityBytes) { const CoreError saveDataToFileResult = dataReceived(reinterpret_cast<const unsigned char *>(entity), entityBytes); if(!isSuccess(saveDataToFileResult)) { TRACE_MESSAGE2("Failed to write data obtained from HTTP Server, result %S", toString(saveDataToFileResult).toWideChar()); return saveDataToFileResult; } m_regettingPosition += entityBytes; } ////////////////////////////////////////////////////////////////////////// /// check if complete file is downloaded // Content-Length header presents if(httpHeader.m_contentLength == -1) { TRACE_MESSAGE("HTTP chunk received (header does not contain the content-length field), continue receiving message until connection is closed by remote peer"); } else if(static_cast<size_t>(httpHeader.m_contentLength) <= total_entity_bytes) { TRACE_MESSAGE2("HTTP message successfully received, content length = %d", httpHeader.m_contentLength); return CORE_NO_ERROR; } currentOffsetInHttpBuffer = 0; entityBytes = 0; } }
KLUPD::CoreError KLUPD::HttpProtocol::httpRequestAttempt( const Path &fileNameIn, const Path &relativeUrlPathIn, const Address &serverAddressIn, const std::string &userAgent, const bool useProxy, const Address &proxyAddressIn, const AuthorizationTypeList &proxyAuthorizationMethods, const size_t regettingPosition) { size_t infiniteRedirectLoop = 0; Path fileName = fileNameIn; Path relativeUrlPath = relativeUrlPathIn; Address serverAddress = serverAddressIn; Address proxyAddress = proxyAddressIn; m_regettingPosition = regettingPosition; // cache for next iteration generate request // at first attempt the target is not known, but proxy server target is used, // because of possible protection from client for authorization on server DownloadProgress::AuthorizationTarget authorizationTarget = DownloadProgress::proxyServer; //////////////////// // the State flags agains proxies that close connection after authorization switch. // There are proxies (e.g. Squid/2.4.STABLE7) which deals in next way: // *** Client -> Server: GET file // *** Server -> Client: HTTP 407 / Proxy-Connection: keep-alive // *** Server closes connection // *** Client -> Server GET 407 // // Here are 3 flags to track such situations, that previous response is received, // BUT then server closes connection bool previousResponseReceived = false; bool authorizationTypeSwitched = false; bool requestHasAlreadyBeenRepeated = false; //////////////////// //////////////////// // in case proxy server requires authorization, but sends incorrect (from HTTP) // code for authorization failure in next way // *** Client -> Server: GET file // *** Server -> Client: HTTP 407 / Proxy-Authorization: NTLM // *** Client -> Server GET / Proxy-Authorization: NTLM // *** Server -> Client: HTTP 502 (or 403 or other code) // *** here is authorization state is forgotten, because 502 request does NOT contain "Proxy-Authorization: NTLM" // *** Client ask from product new credentials // *** Client -> Server GET / Proxy-Authorization: NTLM (with new credentials) bool authorizationWasNeededOnProxy = false; // it makes no sence to try Ntlm without credentials authorization type, // because this authorization type does not depend on provided credentials bool ntlmAuthorizationTriedAlready = false; int lastHttpCode = 0; for(size_t protectionAgainstCyclingCounter = 100; protectionAgainstCyclingCounter; --protectionAgainstCyclingCounter) { const CoreError connectionResult = setupLowLevelConnectionIfNeed(useProxy, useProxy ? proxyAddress : serverAddress, proxyAuthorizationMethods); if(connectionResult != CORE_NO_ERROR) { TRACE_MESSAGE2("Failed to setup connection to HTTP Server, result '%S'", toString(connectionResult).toWideChar()); return connectionResult; } // make proxy authorization header std::string proxyAuthorizationHeader; if(!m_authorizationDriver.makeProxyAuthorizationHeader(proxyAuthorizationHeader) // we know authorization methods supported by proxy server || ((m_authorizationDriver.currentAuthorizationType() == noAuthorization) && !m_authorizationDriver.supportedAuthorizationTypesByServer().empty())) { if(!switchAuthorization(fileName, relativeUrlPath, authorizationTarget, proxyAddress, authorizationTypeSwitched, ntlmAuthorizationTriedAlready)) { if(lastHttpCode) return HttpHeader::convertHttpCodeToUpdaterCode(lastHttpCode); return useProxy ? CORE_PROXY_AUTH_ERROR : CORE_SERVER_AUTH_ERROR; } // try other authorization type continue; } HTTPRequestBuilder requestBuilder(m_method); const std::vector<unsigned char> requestBuffer = requestBuilder.generateRequest( fileName, relativeUrlPath, useProxy, serverAddress, userAgent, proxyAuthorizationHeader, m_regettingPosition, m_postData); TRACE_MESSAGE2("Sending HTTP request\n%s", requestBuilder.toString().c_str()); if(requestBuffer.empty()) { TRACE_MESSAGE("Failed to send empty HTTP request"); return CORE_DOWNLOAD_ERROR; } const CoreError sendRequestResult = m_socket.send(reinterpret_cast<const char *>(&requestBuffer[0]), requestBuffer.size()); if(sendRequestResult != CORE_NO_ERROR) { TRACE_MESSAGE2("Failed to send HTTP request, error %S", toString(sendRequestResult).toWideChar()); if((sendRequestResult == CORE_REMOTE_HOST_CLOSED_CONNECTION) && previousResponseReceived && authorizationTypeSwitched && !requestHasAlreadyBeenRepeated) { TRACE_MESSAGE("Repeating the same request (without authorization switch), because server was reachable, but unexpectedly closed connection"); requestHasAlreadyBeenRepeated = true; continue; } return sendRequestResult; } HttpHeader httpHeader; const CoreError receiveResponseAndDataResult = receiveResponseAndData(httpHeader); const bool needCloseConnection = httpHeader.needCloseConnection(useProxy) || (receiveResponseAndDataResult != CORE_NO_ERROR); if(needCloseConnection) { TRACE_MESSAGE2("Closing connection to HTTP server, get file result %S", toString(receiveResponseAndDataResult).toWideChar()); closeSession(); } if(receiveResponseAndDataResult != CORE_NO_ERROR) { TRACE_MESSAGE2("Failed to receive HTTP response, error %S", toString(receiveResponseAndDataResult).toWideChar()); if((receiveResponseAndDataResult == CORE_REMOTE_HOST_CLOSED_CONNECTION) && previousResponseReceived && authorizationTypeSwitched && !requestHasAlreadyBeenRepeated) { TRACE_MESSAGE("Repeating the same request (without authorization switch), because server was reachable, but unexpectedly closed connection"); requestHasAlreadyBeenRepeated = true; continue; } return receiveResponseAndDataResult; } previousResponseReceived = true; requestHasAlreadyBeenRepeated = false; m_authorizationDriver.authorized( // authorization information is reset in case connection is to be closed !needCloseConnection // authorization success in case file received or successful redirect && (httpHeader.isFile() || httpHeader.redirectRequired() || httpHeader.fileNotFound())); if(httpHeader.isFile()) { // in case proxy server by some ocassion desided to close // connection after successful file receive event if(needCloseConnection) m_authorizationDriver.resetNtlmState(); return CORE_NO_ERROR; } authorizationTarget = httpHeader.authorizationTarget(); authorizationWasNeededOnProxy = !proxyAuthorizationHeader.empty(); lastHttpCode = httpHeader.httpCode(); // authorization error if(httpHeader.authorizationNeeded()) { authorizationWasNeededOnProxy = (authorizationTarget == DownloadProgress::proxyServer); if(!switchAuthorization(fileName, relativeUrlPath, httpHeader.authorizationTarget(), proxyAddress, authorizationTypeSwitched, ntlmAuthorizationTriedAlready)) { return httpHeader.convertHttpCodeToUpdaterCode(); } } // redirect needed else if(httpHeader.redirectRequired()) { // protection against infinite loop (according to RFC 2616) if(++infiniteRedirectLoop > 2) { TRACE_MESSAGE2("Infinite redirection loop detected for location '%S'", httpHeader.m_location.toWideChar()); return CORE_NO_SOURCE_FILE; } if(httpHeader.m_location.isAbsoluteUri()) serverAddress.parse(httpHeader.m_location); else { serverAddress.parse(toProtocolPrefix(serverAddress.m_protocol) + serverAddress.m_hostname + serverAddress.m_path + httpHeader.m_location); serverAddress.m_path.correctPathDelimiters(); } fileName = serverAddress.m_fileName; relativeUrlPath.erase(); TRACE_MESSAGE3("HTTP Redirect to file '%S' on server %S", fileName.toWideChar(), serverAddress.toString().toWideChar()); } // known HTTP results else if(httpHeader.resourceUnavailable() || httpHeader.fileNotFound()) { return httpHeader.convertHttpCodeToUpdaterCode(); } // retry authorization with other credentials in case Forbidden // code received after successful authorization has happened else if(httpHeader.retryAuthorization(authorizationWasNeededOnProxy) && treat_403_502_httpCodesAs407()) { if(!switchAuthorization(fileName, relativeUrlPath, DownloadProgress::proxyServer, proxyAddress, authorizationTypeSwitched, ntlmAuthorizationTriedAlready)) { TRACE_MESSAGE2("Authorization was needed, but error HTTP code '%d' received and switch to next authorization type failed", httpHeader.httpCode()); return httpHeader.convertHttpCodeToUpdaterCode(); } TRACE_MESSAGE3("Authorization was needed, but error HTTP code '%d' received, try next authorization type '%S'", httpHeader.httpCode(), toString(m_authorizationDriver.currentAuthorizationType()).toWideChar()); } else { // processing HTTP code is not implemented m_authorizationDriver.resetAuthorizatoinState(proxyAuthorizationMethods); return httpHeader.convertHttpCodeToUpdaterCode(); } } // Authorization state machine is complex and may contain bug, // that is why protection against infinite loop is implemented TRACE_MESSAGE3("Error in HTTP authorization state implementation: credentials '%S', current authorization type '%S'", m_authorizationDriver.credentials().toString().toWideChar(), toString(m_authorizationDriver.currentAuthorizationType()).toWideChar()); return CORE_DOWNLOAD_ERROR; }