static void
http_client_request_finish_payload_out(struct http_client_request *req)
{
i_assert(req->conn != NULL);
/* drop payload output stream */
if (req->payload_output != NULL) {
o_stream_unref(&req->payload_output);
req->payload_output = NULL;
}
/* advance state only when request didn't get aborted in the mean time */
if (req->state != HTTP_REQUEST_STATE_ABORTED) {
i_assert(req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT);
/* we're now waiting for a response from the server */
req->state = HTTP_REQUEST_STATE_WAITING;
http_client_connection_start_request_timeout(req->conn);
}
/* release connection */
req->conn->output_locked = FALSE;
http_client_request_debug(req, "Finished sending%s payload",
(req->state == HTTP_REQUEST_STATE_ABORTED ? " aborted" : ""));
}
void http_client_request_destroy(struct http_client_request **_req)
{
struct http_client_request *req = *_req;
struct http_client *client = req->client;
*_req = NULL;
http_client_request_debug(req, "Destroy (requests left=%d)",
client->requests_count);
if (req->state < HTTP_REQUEST_STATE_FINISHED)
req->state = HTTP_REQUEST_STATE_ABORTED;
req->callback = NULL;
if (req->queue != NULL)
http_client_queue_drop_request(req->queue, req);
if (req->destroy_callback != NULL) {
void (*callback)(void *) = req->destroy_callback;
req->destroy_callback = NULL;
callback(req->destroy_context);
}
http_client_request_remove(req);
http_client_request_unref(&req);
}
void http_client_request_resubmit(struct http_client_request *req)
{
i_assert(!req->payload_wait);
http_client_request_debug(req, "Resubmitting request");
/* rewind payload stream */
if (req->payload_input != NULL && req->payload_size > 0) {
if (req->payload_input->v_offset != req->payload_offset &&
!req->payload_input->seekable) {
http_client_request_error(&req,
HTTP_CLIENT_REQUEST_ERROR_ABORTED,
"Resubmission failed: Cannot resend payload; stream is not seekable");
return;
} else {
i_stream_seek(req->payload_input, req->payload_offset);
}
}
/* drop payload output stream from previous attempt */
if (req->payload_output != NULL)
o_stream_unref(&req->payload_output);
req->conn = NULL;
req->peer = NULL;
req->state = HTTP_REQUEST_STATE_QUEUED;
http_client_host_submit_request(req->host, req);
}
void http_client_request_submit(struct http_client_request *req)
{
req->client->pending_requests++;
http_client_request_do_submit(req);
http_client_request_debug(req, "Submitted");
req->submitted = TRUE;
}
void http_client_request_submit(struct http_client_request *req)
{
req->submit_time = ioloop_timeval;
http_client_request_do_submit(req);
http_client_request_debug(req, "Submitted");
req->submitted = TRUE;
http_client_request_add(req);
}
bool http_client_request_try_retry(struct http_client_request *req)
{
/* limit the number of attempts for each request */
if (req->attempts+1 >= req->client->set.max_attempts)
return FALSE;
req->attempts++;
http_client_request_debug(req, "Retrying (attempts=%d)", req->attempts);
if (req->callback != NULL)
http_client_request_resubmit(req);
return TRUE;
}
static void
http_client_request_finish_payload_out(struct http_client_request *req)
{
i_assert(req->conn != NULL);
if (req->payload_output != NULL) {
o_stream_unref(&req->payload_output);
req->payload_output = NULL;
}
req->state = HTTP_REQUEST_STATE_WAITING;
req->conn->output_locked = FALSE;
http_client_request_debug(req, "Finished sending payload");
}
void http_client_request_finish(struct http_client_request **_req)
{
struct http_client_request *req = *_req;
if (req->state >= HTTP_REQUEST_STATE_FINISHED)
return;
http_client_request_debug(req, "Finished");
req->callback = NULL;
req->state = HTTP_REQUEST_STATE_FINISHED;
if (req->payload_wait && req->client->ioloop != NULL)
io_loop_stop(req->client->ioloop);
http_client_request_unref(_req);
}
void http_client_request_finish(struct http_client_request *req)
{
if (req->state >= HTTP_REQUEST_STATE_FINISHED)
return;
i_assert(req->refcount > 0);
http_client_request_debug(req, "Finished");
req->callback = NULL;
req->state = HTTP_REQUEST_STATE_FINISHED;
if (req->queue != NULL)
http_client_queue_drop_request(req->queue, req);
if (req->payload_wait && req->client->ioloop != NULL)
io_loop_stop(req->client->ioloop);
http_client_request_unref(&req);
}
bool http_client_request_try_retry(struct http_client_request *req)
{
/* don't ever retry if we're sending data in small blocks via
http_client_request_send_payload() and we're not waiting for a
100 continue (there's no way to rewind the payload for a retry)
*/
if (req->payload_wait &&
(!req->payload_sync || req->payload_sync_continue))
return FALSE;
/* limit the number of attempts for each request */
if (req->attempts+1 >= req->client->set.max_attempts)
return FALSE;
req->attempts++;
http_client_request_debug(req, "Retrying (attempts=%d)", req->attempts);
if (req->callback != NULL)
http_client_request_resubmit(req);
return TRUE;
}
bool http_client_request_unref(struct http_client_request **_req)
{
struct http_client_request *req = *_req;
struct http_client *client = req->client;
i_assert(req->refcount > 0);
*_req = NULL;
if (--req->refcount > 0)
return TRUE;
http_client_request_debug(req, "Free (requests left=%d)",
client->requests_count);
/* cannot be destroyed while it is still pending */
i_assert(req->conn == NULL);
if (req->queue != NULL)
http_client_queue_drop_request(req->queue, req);
if (req->destroy_callback != NULL) {
req->destroy_callback(req->destroy_context);
req->destroy_callback = NULL;
}
http_client_request_remove(req);
if (client->requests_count == 0 && client->ioloop != NULL)
io_loop_stop(client->ioloop);
if (req->delayed_error != NULL)
http_client_remove_request_error(req->client, req);
if (req->payload_input != NULL)
i_stream_unref(&req->payload_input);
if (req->payload_output != NULL)
o_stream_unref(&req->payload_output);
if (req->headers != NULL)
str_free(&req->headers);
pool_unref(&req->pool);
return FALSE;
}
void http_client_request_unref(struct http_client_request **_req)
{
struct http_client_request *req = *_req;
struct http_client *client = req->client;
i_assert(req->refcount > 0);
if (--req->refcount > 0)
return;
if (req->destroy_callback != NULL) {
req->destroy_callback(req->destroy_context);
req->destroy_callback = NULL;
}
/* only decrease pending request counter if this request was submitted */
if (req->state > HTTP_REQUEST_STATE_NEW)
req->client->pending_requests--;
http_client_request_debug(req, "Destroy (requests left=%d)",
client->pending_requests);
if (client->pending_requests == 0 && client->ioloop != NULL)
io_loop_stop(client->ioloop);
if (req->delayed_error != NULL)
http_client_host_remove_request_error(req->host, req);
if (req->payload_input != NULL)
i_stream_unref(&req->payload_input);
if (req->payload_output != NULL)
o_stream_unref(&req->payload_output);
if (req->headers != NULL)
str_free(&req->headers);
pool_unref(&req->pool);
*_req = NULL;
}
static int http_client_request_send_real(struct http_client_request *req,
bool pipelined, const char **error_r)
{
const struct http_client_settings *set = &req->client->set;
struct http_client_connection *conn = req->conn;
struct ostream *output = conn->conn.output;
string_t *rtext = t_str_new(256);
struct const_iovec iov[3];
int ret = 0;
i_assert(!req->conn->output_locked);
i_assert(req->payload_output == NULL);
/* create request line */
str_append(rtext, req->method);
str_append(rtext, " ");
str_append(rtext, req->target);
str_append(rtext, " HTTP/1.1\r\n");
/* create special headers implicitly if not set explicitly using
http_client_request_add_header() */
if (!req->have_hdr_host) {
str_append(rtext, "Host: ");
str_append(rtext, req->authority);
str_append(rtext, "\r\n");
}
if (!req->have_hdr_date) {
str_append(rtext, "Date: ");
str_append(rtext, http_date_create(req->date));
str_append(rtext, "\r\n");
}
if (!req->have_hdr_authorization &&
req->username != NULL && req->password != NULL) {
struct http_auth_credentials auth_creds;
http_auth_basic_credentials_init(&auth_creds,
req->username, req->password);
str_append(rtext, "Authorization: ");
http_auth_create_credentials(rtext, &auth_creds);
str_append(rtext, "\r\n");
}
if (http_client_request_to_proxy(req) &&
set->proxy_username != NULL && set->proxy_password != NULL) {
struct http_auth_credentials auth_creds;
http_auth_basic_credentials_init(&auth_creds,
set->proxy_username, set->proxy_password);
str_append(rtext, "Proxy-Authorization: ");
http_auth_create_credentials(rtext, &auth_creds);
str_append(rtext, "\r\n");
}
if (!req->have_hdr_user_agent && req->client->set.user_agent != NULL) {
str_printfa(rtext, "User-Agent: %s\r\n",
req->client->set.user_agent);
}
if (!req->have_hdr_expect && req->payload_sync) {
str_append(rtext, "Expect: 100-continue\r\n");
}
if (req->payload_input != NULL) {
if (req->payload_chunked) {
// FIXME: can't do this for a HTTP/1.0 server
if (!req->have_hdr_body_spec)
str_append(rtext, "Transfer-Encoding: chunked\r\n");
req->payload_output =
http_transfer_chunked_ostream_create(output);
} else {
/* send Content-Length if we have specified a payload,
even if it's 0 bytes. */
if (!req->have_hdr_body_spec) {
str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n",
req->payload_size);
}
req->payload_output = output;
o_stream_ref(output);
}
}
if (!req->have_hdr_connection &&
!http_client_request_to_proxy(req)) {
/* https://tools.ietf.org/html/rfc2068
Section 19.7.1:
A client MUST NOT send the Keep-Alive connection token to a proxy
server as HTTP/1.0 proxy servers do not obey the rules of HTTP/1.1
for parsing the Connection header field.
*/
str_append(rtext, "Connection: Keep-Alive\r\n");
}
/* request line + implicit headers */
iov[0].iov_base = str_data(rtext);
iov[0].iov_len = str_len(rtext);
/* explicit headers */
if (req->headers != NULL) {
iov[1].iov_base = str_data(req->headers);
iov[1].iov_len = str_len(req->headers);
} else {
iov[1].iov_base = "";
iov[1].iov_len = 0;
}
/* end of header */
iov[2].iov_base = "\r\n";
iov[2].iov_len = 2;
req->state = HTTP_REQUEST_STATE_PAYLOAD_OUT;
req->sent_time = ioloop_timeval;
o_stream_cork(output);
if (o_stream_sendv(output, iov, N_ELEMENTS(iov)) < 0) {
*error_r = t_strdup_printf("write(%s) failed: %s",
o_stream_get_name(output),
o_stream_get_error(output));
ret = -1;
} else {
http_client_request_debug(req, "Sent header");
if (req->payload_output != NULL) {
if (!req->payload_sync) {
if (http_client_request_send_more
(req, pipelined, error_r) < 0)
ret = -1;
} else {
http_client_request_debug(req, "Waiting for 100-continue");
conn->output_locked = TRUE;
}
} else {
req->state = HTTP_REQUEST_STATE_WAITING;
if (!pipelined)
http_client_connection_start_request_timeout(req->conn);
conn->output_locked = FALSE;
}
if (ret >= 0 && o_stream_flush(output) < 0) {
*error_r = t_strdup_printf("flush(%s) failed: %s",
o_stream_get_name(output),
o_stream_get_error(output));
ret = -1;
}
}
o_stream_uncork(output);
return ret;
}
int http_client_request_send_more(struct http_client_request *req,
bool pipelined, const char **error_r)
{
struct http_client_connection *conn = req->conn;
struct ostream *output = req->payload_output;
enum ostream_send_istream_result res;
i_assert(req->payload_input != NULL);
i_assert(req->payload_output != NULL);
if (conn->io_req_payload != NULL)
io_remove(&conn->io_req_payload);
/* chunked ostream needs to write to the parent stream's buffer */
o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE);
res = o_stream_send_istream(output, req->payload_input);
o_stream_set_max_buffer_size(output, (size_t)-1);
switch (res) {
case OSTREAM_SEND_ISTREAM_RESULT_FINISHED:
/* finished sending */
if (!req->payload_chunked &&
req->payload_input->v_offset - req->payload_offset != req->payload_size) {
*error_r = t_strdup_printf("BUG: stream '%s' input size changed: "
"%"PRIuUOFF_T"-%"PRIuUOFF_T" != %"PRIuUOFF_T,
i_stream_get_name(req->payload_input),
req->payload_input->v_offset, req->payload_offset, req->payload_size);
i_error("%s", *error_r); //FIXME: remove?
return -1;
}
if (req->payload_wait) {
/* this chunk of input is finished
(client needs to act; disable timeout) */
i_assert(!pipelined);
conn->output_locked = TRUE;
http_client_connection_stop_request_timeout(conn);
if (req->client->ioloop != NULL)
io_loop_stop(req->client->ioloop);
} else {
/* finished sending payload */
http_client_request_finish_payload_out(req);
}
return 0;
case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT:
/* input is blocking (client needs to act; disable timeout) */
conn->output_locked = TRUE;
if (!pipelined)
http_client_connection_stop_request_timeout(conn);
conn->io_req_payload = io_add_istream(req->payload_input,
http_client_request_payload_input, req);
return 0;
case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT:
/* output is blocking (server needs to act; enable timeout) */
conn->output_locked = TRUE;
if (!pipelined)
http_client_connection_start_request_timeout(conn);
http_client_request_debug(req, "Partially sent payload");
return 0;
case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT:
/* we're in the middle of sending a request, so the connection
will also have to be aborted */
*error_r = t_strdup_printf("read(%s) failed: %s",
i_stream_get_name(req->payload_input),
i_stream_get_error(req->payload_input));
/* the payload stream assigned to this request is broken,
fail this the request immediately */
http_client_request_error(&req,
HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD,
"Broken payload stream");
return -1;
case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT:
/* failed to send request */
*error_r = t_strdup_printf("write(%s) failed: %s",
o_stream_get_name(output),
o_stream_get_error(output));
return -1;
}
i_unreached();
}
int http_client_request_send_more(struct http_client_request *req,
const char **error_r)
{
struct http_client_connection *conn = req->conn;
struct ostream *output = req->payload_output;
off_t ret;
int fd;
i_assert(req->payload_input != NULL);
i_assert(req->payload_output != NULL);
if (conn->io_req_payload != NULL)
io_remove(&conn->io_req_payload);
/* chunked ostream needs to write to the parent stream's buffer */
o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE);
ret = o_stream_send_istream(output, req->payload_input);
o_stream_set_max_buffer_size(output, (size_t)-1);
if (req->payload_input->stream_errno != 0) {
/* the payload stream assigned to this request is broken,
fail this the request immediately */
http_client_request_send_error(req,
HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD,
"Broken payload stream");
/* we're in the middle of sending a request, so the connection
will also have to be aborted */
errno = req->payload_input->stream_errno;
*error_r = t_strdup_printf("read(%s) failed: %s",
i_stream_get_name(req->payload_input),
i_stream_get_error(req->payload_input));
ret = -1;
} else if (output->stream_errno != 0) {
/* failed to send request */
errno = output->stream_errno;
*error_r = t_strdup_printf("write(%s) failed: %s",
o_stream_get_name(output),
o_stream_get_error(output));
ret = -1;
} else {
i_assert(ret >= 0);
}
if (ret < 0 || i_stream_is_eof(req->payload_input)) {
if (!req->payload_chunked &&
req->payload_input->v_offset - req->payload_offset != req->payload_size) {
*error_r = "stream input size changed [BUG]";
i_error("stream input size changed"); //FIXME
return -1;
}
if (req->payload_wait) {
conn->output_locked = TRUE;
if (req->client->ioloop != NULL)
io_loop_stop(req->client->ioloop);
} else {
http_client_request_finish_payload_out(req);
}
} else if (i_stream_get_data_size(req->payload_input) > 0) {
/* output is blocking */
conn->output_locked = TRUE;
o_stream_set_flush_pending(output, TRUE);
http_client_request_debug(req, "Partially sent payload");
} else {
/* input is blocking */
fd = i_stream_get_fd(req->payload_input);
conn->output_locked = TRUE;
i_assert(fd >= 0);
conn->io_req_payload = io_add
(fd, IO_READ, http_client_request_payload_input, req);
}
return ret < 0 ? -1 : 0;
}
static int
http_client_request_continue_payload(struct http_client_request **_req,
const unsigned char *data, size_t size)
{
struct ioloop *prev_ioloop = current_ioloop;
struct http_client_request *req = *_req;
struct http_client_connection *conn = req->conn;
struct http_client *client = req->client;
int ret;
i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT);
i_assert(req->payload_input == NULL);
if (conn != NULL)
http_client_connection_ref(conn);
http_client_request_ref(req);
req->payload_wait = TRUE;
if (data == NULL) {
req->payload_input = NULL;
if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT)
http_client_request_finish_payload_out(req);
} else {
req->payload_input = i_stream_create_from_data(data, size);
i_stream_set_name(req->payload_input, "<HTTP request payload>");
}
req->payload_size = 0;
req->payload_chunked = TRUE;
if (req->state == HTTP_REQUEST_STATE_NEW)
http_client_request_submit(req);
/* Wait for payload data to be written */
i_assert(client->ioloop == NULL);
client->ioloop = io_loop_create();
http_client_switch_ioloop(client);
if (client->set.dns_client != NULL)
dns_client_switch_ioloop(client->set.dns_client);
while (req->state < HTTP_REQUEST_STATE_PAYLOAD_IN) {
http_client_request_debug(req, "Waiting for request to finish");
if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT)
o_stream_set_flush_pending(req->payload_output, TRUE);
io_loop_run(client->ioloop);
if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT &&
req->payload_input->eof) {
i_stream_unref(&req->payload_input);
req->payload_input = NULL;
break;
}
}
io_loop_set_current(prev_ioloop);
http_client_switch_ioloop(client);
if (client->set.dns_client != NULL)
dns_client_switch_ioloop(client->set.dns_client);
io_loop_set_current(client->ioloop);
io_loop_destroy(&client->ioloop);
switch (req->state) {
case HTTP_REQUEST_STATE_PAYLOAD_IN:
case HTTP_REQUEST_STATE_FINISHED:
ret = 1;
break;
case HTTP_REQUEST_STATE_ABORTED:
ret = -1;
break;
default:
ret = 0;
break;
}
req->payload_wait = FALSE;
/* callback may have messed with our pointer,
so unref using local variable */
if (!http_client_request_unref(&req))
*_req = NULL;
if (conn != NULL)
http_client_connection_unref(&conn);
/* Return status */
return ret;
}
void http_client_request_redirect(struct http_client_request *req,
unsigned int status, const char *location)
{
struct http_url *url;
const char *error, *target, *origin_url;
i_assert(!req->payload_wait);
/* parse URL */
if (http_url_parse(location, NULL, 0,
pool_datastack_create(), &url, &error) < 0) {
http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
t_strdup_printf("Invalid redirect location: %s", error));
return;
}
if (++req->redirects > req->client->set.max_redirects) {
if (req->client->set.max_redirects > 0) {
http_client_request_error(&req,
HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
t_strdup_printf("Redirected more than %d times",
req->client->set.max_redirects));
} else {
http_client_request_error(&req,
HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
"Redirect refused");
}
return;
}
/* rewind payload stream */
if (req->payload_input != NULL && req->payload_size > 0 && status != 303) {
if (req->payload_input->v_offset != req->payload_offset &&
!req->payload_input->seekable) {
http_client_request_error(&req,
HTTP_CLIENT_REQUEST_ERROR_ABORTED,
"Redirect failed: Cannot resend payload; stream is not seekable");
return;
} else {
i_stream_seek(req->payload_input, req->payload_offset);
}
}
/* drop payload output stream from previous attempt */
if (req->payload_output != NULL)
o_stream_unref(&req->payload_output);
target = http_url_create_target(url);
http_url_copy(req->pool, &req->origin_url, url);
req->target = p_strdup(req->pool, target);
req->host = NULL;
req->conn = NULL;
origin_url = http_url_create(&req->origin_url);
http_client_request_debug(req, "Redirecting to %s%s",
origin_url, target);
req->label = p_strdup_printf(req->pool, "[%s %s%s]",
req->method, origin_url, req->target);
/* RFC 7231, Section 6.4.4:
-> A 303 `See Other' redirect status response is handled a bit differently.
Basically, the response content is located elsewhere, but the original
(POST) request is handled already.
*/
if (status == 303 && strcasecmp(req->method, "HEAD") != 0 &&
strcasecmp(req->method, "GET") != 0) {
// FIXME: should we provide the means to skip this step? The original
// request was already handled at this point.
req->method = p_strdup(req->pool, "GET");
/* drop payload */
if (req->payload_input != NULL)
i_stream_unref(&req->payload_input);
req->payload_size = 0;
req->payload_offset = 0;
}
/* resubmit */
req->state = HTTP_REQUEST_STATE_NEW;
http_client_request_do_submit(req);
}
