void tst_QHttpNetworkReply::parseEndOfHeader() { QFETCH(QByteArray, headers); QFETCH(qint64, lengths); TestHeaderSocket socket(headers); TestHeaderReply reply; QHttpNetworkReplyPrivate *replyPrivate = reply.replyPrivate(); qint64 headerBytes = replyPrivate->readHeader(&socket); QCOMPARE(headerBytes, lengths); }
void QHttpNetworkConnectionChannel::_q_receiveReply() { Q_ASSERT(socket); if (!reply) { // heh, how should that happen! qWarning() << "QHttpNetworkConnectionChannel::_q_receiveReply() called without QHttpNetworkReply," << socket->bytesAvailable() << "bytes on socket."; close(); return; } // only run when the QHttpNetworkConnection is not currently being destructed, e.g. // this function is called from _q_disconnected which is called because // of ~QHttpNetworkConnectionPrivate if (!qobject_cast<QHttpNetworkConnection*>(connection)) { return; } QAbstractSocket::SocketState socketState = socket->state(); // connection might be closed to signal the end of data if (socketState == QAbstractSocket::UnconnectedState) { if (socket->bytesAvailable() <= 0) { if (reply->d_func()->state == QHttpNetworkReplyPrivate::ReadingDataState) { // finish this reply. this case happens when the server did not send a content length reply->d_func()->state = QHttpNetworkReplyPrivate::AllDoneState; allDone(); return; } else { handleUnexpectedEOF(); return; } } else { // socket not connected but still bytes for reading.. just continue in this function } } // read loop for the response qint64 bytes = 0; qint64 lastBytes = bytes; do { lastBytes = bytes; QHttpNetworkReplyPrivate::ReplyState state = reply->d_func()->state; switch (state) { case QHttpNetworkReplyPrivate::NothingDoneState: { state = reply->d_func()->state = QHttpNetworkReplyPrivate::ReadingStatusState; // fallthrough } case QHttpNetworkReplyPrivate::ReadingStatusState: { qint64 statusBytes = reply->d_func()->readStatus(socket); if (statusBytes == -1) { // connection broke while reading status. also handled if later _q_disconnected is called handleUnexpectedEOF(); return; } bytes += statusBytes; lastStatus = reply->d_func()->statusCode; break; } case QHttpNetworkReplyPrivate::ReadingHeaderState: { QHttpNetworkReplyPrivate *replyPrivate = reply->d_func(); qint64 headerBytes = replyPrivate->readHeader(socket); if (headerBytes == -1) { // connection broke while reading headers. also handled if later _q_disconnected is called handleUnexpectedEOF(); return; } bytes += headerBytes; // If headers were parsed successfully now it is the ReadingDataState if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) { if (replyPrivate->isCompressed() && replyPrivate->autoDecompress) { // remove the Content-Length from header replyPrivate->removeAutoDecompressHeader(); } else { replyPrivate->autoDecompress = false; } if (replyPrivate->statusCode == 100) { replyPrivate->clearHttpLayerInformation(); replyPrivate->state = QHttpNetworkReplyPrivate::ReadingStatusState; break; // ignore } if (replyPrivate->shouldEmitSignals()) emit reply->headerChanged(); // After headerChanged had been emitted // we can suddenly have a replyPrivate->userProvidedDownloadBuffer // this is handled in the ReadingDataState however if (!replyPrivate->expectContent()) { replyPrivate->state = QHttpNetworkReplyPrivate::AllDoneState; allDone(); break; } } break; } case QHttpNetworkReplyPrivate::ReadingDataState: { QHttpNetworkReplyPrivate *replyPrivate = reply->d_func(); if (socket->state() == QAbstractSocket::ConnectedState && replyPrivate->downstreamLimited && !replyPrivate->responseData.isEmpty() && replyPrivate->shouldEmitSignals()) { // (only do the following when still connected, not when we have already been disconnected and there is still data) // We already have some HTTP body data. We don't read more from the socket until // this is fetched by QHttpNetworkAccessHttpBackend. If we would read more, // we could not limit our read buffer usage. // We only do this when shouldEmitSignals==true because our HTTP parsing // always needs to parse the 401/407 replies. Therefore they don't really obey // to the read buffer maximum size, but we don't care since they should be small. return; } if (replyPrivate->userProvidedDownloadBuffer) { // the user provided a direct buffer where we should put all our data in. // this only works when we can tell the user the content length and he/she can allocate // the buffer in that size. // note that this call will read only from the still buffered data qint64 haveRead = replyPrivate->readBodyVeryFast(socket, replyPrivate->userProvidedDownloadBuffer + replyPrivate->totalProgress); if (haveRead > 0) { bytes += haveRead; replyPrivate->totalProgress += haveRead; // the user will get notified of it via progress signal emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength); } else if (haveRead == 0) { // Happens since this called in a loop. Currently no bytes available. } else if (haveRead < 0) { connection->d_func()->emitReplyError(socket, reply, QNetworkReply::RemoteHostClosedError); break; } } else if (!replyPrivate->isChunked() && !replyPrivate->autoDecompress && replyPrivate->bodyLength > 0) { // bulk files like images should fulfill these properties and // we can therefore save on memory copying qint64 haveRead = replyPrivate->readBodyFast(socket, &replyPrivate->responseData); bytes += haveRead; replyPrivate->totalProgress += haveRead; if (replyPrivate->shouldEmitSignals()) { emit reply->readyRead(); emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength); } } else { // use the traditional slower reading (for compressed encoding, chunked encoding, // no content-length etc) qint64 haveRead = replyPrivate->readBody(socket, &replyPrivate->responseData); if (haveRead > 0) { bytes += haveRead; replyPrivate->totalProgress += haveRead; if (replyPrivate->shouldEmitSignals()) { emit reply->readyRead(); emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength); } } else if (haveRead == -1) { // Some error occured connection->d_func()->emitReplyError(socket, reply, QNetworkReply::ProtocolFailure); break; } } // still in ReadingDataState? This function will be called again by the socket's readyRead if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) break; // everything done, fall through } case QHttpNetworkReplyPrivate::AllDoneState: allDone(); break; default: break; } } while (bytes != lastBytes && reply); }
bool QHttpNetworkConnectionChannel::sendRequest() { if (!reply) { // heh, how should that happen! qWarning() << "QHttpNetworkConnectionChannel::sendRequest() called without QHttpNetworkReply"; state = QHttpNetworkConnectionChannel::IdleState; return false; } switch (state) { case QHttpNetworkConnectionChannel::IdleState: { // write the header if (!ensureConnection()) { // wait for the connection (and encryption) to be done // sendRequest will be called again from either // _q_connected or _q_encrypted return false; } written = 0; // excluding the header bytesTotal = 0; QHttpNetworkReplyPrivate *replyPrivate = reply->d_func(); replyPrivate->clear(); replyPrivate->connection = connection; replyPrivate->connectionChannel = this; replyPrivate->autoDecompress = request.d->autoDecompress; replyPrivate->pipeliningUsed = false; // if the url contains authentication parameters, use the new ones // both channels will use the new authentication parameters if (!request.url().userInfo().isEmpty() && request.withCredentials()) { QUrl url = request.url(); QAuthenticator &auth = authenticator; if (url.userName() != auth.user() || (!url.password().isEmpty() && url.password() != auth.password())) { auth.setUser(url.userName()); auth.setPassword(url.password()); connection->d_func()->copyCredentials(connection->d_func()->indexOf(socket), &auth, false); } // clear the userinfo, since we use the same request for resending // userinfo in url can conflict with the one in the authenticator url.setUserInfo(QString()); request.setUrl(url); } // Will only be false if QtWebKit is performing a cross-origin XMLHttpRequest // and withCredentials has not been set to true. if (request.withCredentials()) connection->d_func()->createAuthorization(socket, request); #ifndef QT_NO_NETWORKPROXY QByteArray header = QHttpNetworkRequestPrivate::header(request, (connection->d_func()->networkProxy.type() != QNetworkProxy::NoProxy)); #else QByteArray header = QHttpNetworkRequestPrivate::header(request, false); #endif socket->write(header); // flushing is dangerous (QSslSocket calls transmit which might read or error) // socket->flush(); QNonContiguousByteDevice* uploadByteDevice = request.uploadByteDevice(); if (uploadByteDevice) { // connect the signals so this function gets called again QObject::connect(uploadByteDevice, SIGNAL(readyRead()),this, SLOT(_q_uploadDataReadyRead())); bytesTotal = request.contentLength(); state = QHttpNetworkConnectionChannel::WritingState; // start writing data sendRequest(); //recurse } else { state = QHttpNetworkConnectionChannel::WaitingState; // now wait for response sendRequest(); //recurse } break; } case QHttpNetworkConnectionChannel::WritingState: { // write the data QNonContiguousByteDevice* uploadByteDevice = request.uploadByteDevice(); if (!uploadByteDevice || bytesTotal == written) { if (uploadByteDevice) emit reply->dataSendProgress(written, bytesTotal); state = QHttpNetworkConnectionChannel::WaitingState; // now wait for response sendRequest(); // recurse break; } // only feed the QTcpSocket buffer when there is less than 32 kB in it const qint64 socketBufferFill = 32*1024; const qint64 socketWriteMaxSize = 16*1024; #ifndef QT_NO_OPENSSL QSslSocket *sslSocket = qobject_cast<QSslSocket*>(socket); // if it is really an ssl socket, check more than just bytesToWrite() while ((socket->bytesToWrite() + (sslSocket ? sslSocket->encryptedBytesToWrite() : 0)) <= socketBufferFill && bytesTotal != written) #else while (socket->bytesToWrite() <= socketBufferFill && bytesTotal != written) #endif { // get pointer to upload data qint64 currentReadSize = 0; qint64 desiredReadSize = qMin(socketWriteMaxSize, bytesTotal - written); const char *readPointer = uploadByteDevice->readPointer(desiredReadSize, currentReadSize); if (currentReadSize == -1) { // premature eof happened connection->d_func()->emitReplyError(socket, reply, QNetworkReply::UnknownNetworkError); return false; break; } else if (readPointer == 0 || currentReadSize == 0) { // nothing to read currently, break the loop break; } else { qint64 currentWriteSize = socket->write(readPointer, currentReadSize); if (currentWriteSize == -1 || currentWriteSize != currentReadSize) { // socket broke down connection->d_func()->emitReplyError(socket, reply, QNetworkReply::UnknownNetworkError); return false; } else { written += currentWriteSize; uploadByteDevice->advanceReadPointer(currentWriteSize); emit reply->dataSendProgress(written, bytesTotal); if (written == bytesTotal) { // make sure this function is called once again state = QHttpNetworkConnectionChannel::WaitingState; sendRequest(); break; } } } } break; } case QHttpNetworkConnectionChannel::WaitingState: { QNonContiguousByteDevice* uploadByteDevice = request.uploadByteDevice(); if (uploadByteDevice) { QObject::disconnect(uploadByteDevice, SIGNAL(readyRead()), this, SLOT(_q_uploadDataReadyRead())); } // HTTP pipelining //connection->d_func()->fillPipeline(socket); //socket->flush(); // ensure we try to receive a reply in all cases, even if _q_readyRead_ hat not been called // this is needed if the sends an reply before we have finished sending the request. In that // case receiveReply had been called before but ignored the server reply if (socket->bytesAvailable()) QMetaObject::invokeMethod(this, "_q_receiveReply", Qt::QueuedConnection); break; } case QHttpNetworkConnectionChannel::ReadingState: // ignore _q_bytesWritten in these states // fall through default: break; } return true; }
void QHttpNetworkConnectionChannel::_q_receiveReply() { Q_ASSERT(socket); if (!reply) { // heh, how should that happen! qWarning() << "QHttpNetworkConnectionChannel::_q_receiveReply() called without QHttpNetworkReply," << socket->bytesAvailable() << "bytes on socket."; close(); return; } // only run when the QHttpNetworkConnection is not currently being destructed, e.g. // this function is called from _q_disconnected which is called because // of ~QHttpNetworkConnectionPrivate if (!qobject_cast<QHttpNetworkConnection*>(connection)) { return; } qint64 bytes = 0; QAbstractSocket::SocketState socketState = socket->state(); // connection might be closed to signal the end of data if (socketState == QAbstractSocket::UnconnectedState) { if (socket->bytesAvailable() <= 0) { if (reply->d_func()->state == QHttpNetworkReplyPrivate::ReadingDataState) { // finish this reply. this case happens when the server did not send a content length reply->d_func()->state = QHttpNetworkReplyPrivate::AllDoneState; allDone(); return; } else { handleUnexpectedEOF(); return; } } else { // socket not connected but still bytes for reading.. just continue in this function } } // read loop for the response while (socket->bytesAvailable()) { QHttpNetworkReplyPrivate::ReplyState state = reply->d_func()->state; switch (state) { case QHttpNetworkReplyPrivate::NothingDoneState: { // only eat whitespace on the first call eatWhitespace(); state = reply->d_func()->state = QHttpNetworkReplyPrivate::ReadingStatusState; // fallthrough } case QHttpNetworkReplyPrivate::ReadingStatusState: { qint64 statusBytes = reply->d_func()->readStatus(socket); if (statusBytes == -1) { // connection broke while reading status. also handled if later _q_disconnected is called handleUnexpectedEOF(); return; } bytes += statusBytes; lastStatus = reply->d_func()->statusCode; break; } case QHttpNetworkReplyPrivate::ReadingHeaderState: { QHttpNetworkReplyPrivate *replyPrivate = reply->d_func(); qint64 headerBytes = replyPrivate->readHeader(socket); if (headerBytes == -1) { // connection broke while reading headers. also handled if later _q_disconnected is called handleUnexpectedEOF(); return; } bytes += headerBytes; if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) { if (replyPrivate->isGzipped() && replyPrivate->autoDecompress) { // remove the Content-Length from header replyPrivate->removeAutoDecompressHeader(); } else { replyPrivate->autoDecompress = false; } if (replyPrivate->statusCode == 100) { replyPrivate->clearHttpLayerInformation(); replyPrivate->state = QHttpNetworkReplyPrivate::ReadingStatusState; break; // ignore } if (replyPrivate->shouldEmitSignals()) emit reply->headerChanged(); if (!replyPrivate->expectContent()) { replyPrivate->state = QHttpNetworkReplyPrivate::AllDoneState; allDone(); break; } } break; } case QHttpNetworkReplyPrivate::ReadingDataState: { QHttpNetworkReplyPrivate *replyPrivate = reply->d_func(); if (replyPrivate->downstreamLimited && !replyPrivate->responseData.isEmpty() && replyPrivate->shouldEmitSignals()) { // We already have some HTTP body data. We don't read more from the socket until // this is fetched by QHttpNetworkAccessHttpBackend. If we would read more, // we could not limit our read buffer usage. // We only do this when shouldEmitSignals==true because our HTTP parsing // always needs to parse the 401/407 replies. Therefore they don't really obey // to the read buffer maximum size, but we don't care since they should be small. return; } if (!replyPrivate->isChunked() && !replyPrivate->autoDecompress && replyPrivate->bodyLength > 0) { // bulk files like images should fulfill these properties and // we can therefore save on memory copying bytes = replyPrivate->readBodyFast(socket, &replyPrivate->responseData); replyPrivate->totalProgress += bytes; if (replyPrivate->shouldEmitSignals()) { QPointer<QHttpNetworkReply> replyPointer = reply; emit reply->readyRead(); // make sure that the reply is valid if (replyPointer.isNull()) return; emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength); // make sure that the reply is valid if (replyPointer.isNull()) return; } } else { // use the traditional slower reading (for compressed encoding, chunked encoding, // no content-length etc) QByteDataBuffer byteDatas; bytes = replyPrivate->readBody(socket, &byteDatas); if (bytes) { if (replyPrivate->autoDecompress) replyPrivate->appendCompressedReplyData(byteDatas); else replyPrivate->appendUncompressedReplyData(byteDatas); if (!replyPrivate->autoDecompress) { replyPrivate->totalProgress += bytes; if (replyPrivate->shouldEmitSignals()) { QPointer<QHttpNetworkReply> replyPointer = reply; // important: At the point of this readyRead(), the byteDatas list must be empty, // else implicit sharing will trigger memcpy when the user is reading data! emit reply->readyRead(); // make sure that the reply is valid if (replyPointer.isNull()) return; emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength); // make sure that the reply is valid if (replyPointer.isNull()) return; } } #ifndef QT_NO_COMPRESS else if (!expand(false)) { // expand a chunk if possible return; // ### expand failed } #endif } } // still in ReadingDataState? This function will be called again by the socket's readyRead if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) break; // everything done, fall through } case QHttpNetworkReplyPrivate::AllDoneState: allDone(); break; default: break; } } }
void QHttpNetworkConnectionChannel::_q_receiveReply() { Q_ASSERT(socket); qint64 bytes = 0; QAbstractSocket::SocketState socketState = socket->state(); // connection might be closed to signal the end of data if (socketState == QAbstractSocket::UnconnectedState) { if (!socket->bytesAvailable()) { if (reply && reply->d_func()->state == QHttpNetworkReplyPrivate::ReadingDataState) { reply->d_func()->state = QHttpNetworkReplyPrivate::AllDoneState; this->state = QHttpNetworkConnectionChannel::IdleState; allDone(); } else { // try to reconnect/resend before sending an error. if (reconnectAttempts-- > 0) { closeAndResendCurrentRequest(); } else if (reply) { reply->d_func()->errorString = connection->d_func()->errorDetail(QNetworkReply::RemoteHostClosedError, socket); emit reply->finishedWithError(QNetworkReply::RemoteHostClosedError, reply->d_func()->errorString); QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection); } } } } // read loop for the response while (socket->bytesAvailable()) { QHttpNetworkReplyPrivate::ReplyState state = reply ? reply->d_func()->state : QHttpNetworkReplyPrivate::AllDoneState; switch (state) { case QHttpNetworkReplyPrivate::NothingDoneState: case QHttpNetworkReplyPrivate::ReadingStatusState: { eatWhitespace(); qint64 statusBytes = reply->d_func()->readStatus(socket); if (statusBytes == -1 && reconnectAttempts <= 0) { // too many errors reading/receiving/parsing the status, close the socket and emit error close(); reply->d_func()->errorString = connection->d_func()->errorDetail(QNetworkReply::ProtocolFailure, socket); emit reply->finishedWithError(QNetworkReply::ProtocolFailure, reply->d_func()->errorString); QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection); break; } else if (statusBytes == -1) { reconnectAttempts--; reply->d_func()->clear(); closeAndResendCurrentRequest(); break; } bytes += statusBytes; lastStatus = reply->d_func()->statusCode; break; } case QHttpNetworkReplyPrivate::ReadingHeaderState: { QHttpNetworkReplyPrivate *replyPrivate = reply->d_func(); bytes += replyPrivate->readHeader(socket); if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) { if (replyPrivate->isGzipped() && replyPrivate->autoDecompress) { // remove the Content-Length from header replyPrivate->removeAutoDecompressHeader(); } else { replyPrivate->autoDecompress = false; } if (replyPrivate->statusCode == 100) { replyPrivate->state = QHttpNetworkReplyPrivate::ReadingStatusState; break; // ignore } if (replyPrivate->shouldEmitSignals()) emit reply->headerChanged(); if (!replyPrivate->expectContent()) { replyPrivate->state = QHttpNetworkReplyPrivate::AllDoneState; this->state = QHttpNetworkConnectionChannel::IdleState; allDone(); return; } } break; } case QHttpNetworkReplyPrivate::ReadingDataState: { QHttpNetworkReplyPrivate *replyPrivate = reply->d_func(); if (replyPrivate->downstreamLimited && !replyPrivate->responseData.isEmpty() && replyPrivate->shouldEmitSignals()) { // We already have some HTTP body data. We don't read more from the socket until // this is fetched by QHttpNetworkAccessHttpBackend. If we would read more, // we could not limit our read buffer usage. // We only do this when shouldEmitSignals==true because our HTTP parsing // always needs to parse the 401/407 replies. Therefore they don't really obey // to the read buffer maximum size, but we don't care since they should be small. return; } if (!replyPrivate->isChunked() && !replyPrivate->autoDecompress && replyPrivate->bodyLength > 0) { // bulk files like images should fulfill these properties and // we can therefore save on memory copying bytes = replyPrivate->readBodyFast(socket, &replyPrivate->responseData); replyPrivate->totalProgress += bytes; if (replyPrivate->shouldEmitSignals()) { QPointer<QHttpNetworkReply> replyPointer = reply; emit reply->readyRead(); // make sure that the reply is valid if (replyPointer.isNull()) return; emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength); // make sure that the reply is valid if (replyPointer.isNull()) return; } } else { // use the traditional slower reading (for compressed encoding, chunked encoding, // no content-length etc) QByteDataBuffer byteDatas; bytes = replyPrivate->readBody(socket, &byteDatas); if (bytes) { if (replyPrivate->autoDecompress) replyPrivate->appendCompressedReplyData(byteDatas); else replyPrivate->appendUncompressedReplyData(byteDatas); if (!replyPrivate->autoDecompress) { replyPrivate->totalProgress += bytes; if (replyPrivate->shouldEmitSignals()) { QPointer<QHttpNetworkReply> replyPointer = reply; // important: At the point of this readyRead(), the byteDatas list must be empty, // else implicit sharing will trigger memcpy when the user is reading data! emit reply->readyRead(); // make sure that the reply is valid if (replyPointer.isNull()) return; emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength); // make sure that the reply is valid if (replyPointer.isNull()) return; } } #ifndef QT_NO_COMPRESS else if (!expand(false)) { // expand a chunk if possible return; // ### expand failed } #endif } } if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) break; // everything done, fall through } case QHttpNetworkReplyPrivate::AllDoneState: this->state = QHttpNetworkConnectionChannel::IdleState; allDone(); break; default: break; } } }