static void http_client_request_finish_payload_out(struct http_client_request *req) { i_assert(req->conn != NULL); /* drop payload output stream */ if (req->payload_output != NULL) { o_stream_unref(&req->payload_output); req->payload_output = NULL; } /* advance state only when request didn't get aborted in the mean time */ if (req->state != HTTP_REQUEST_STATE_ABORTED) { i_assert(req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT); /* we're now waiting for a response from the server */ req->state = HTTP_REQUEST_STATE_WAITING; http_client_connection_start_request_timeout(req->conn); } /* release connection */ req->conn->output_locked = FALSE; http_client_request_debug(req, "Finished sending%s payload", (req->state == HTTP_REQUEST_STATE_ABORTED ? " aborted" : "")); }
void http_client_request_destroy(struct http_client_request **_req) { struct http_client_request *req = *_req; struct http_client *client = req->client; *_req = NULL; http_client_request_debug(req, "Destroy (requests left=%d)", client->requests_count); if (req->state < HTTP_REQUEST_STATE_FINISHED) req->state = HTTP_REQUEST_STATE_ABORTED; req->callback = NULL; if (req->queue != NULL) http_client_queue_drop_request(req->queue, req); if (req->destroy_callback != NULL) { void (*callback)(void *) = req->destroy_callback; req->destroy_callback = NULL; callback(req->destroy_context); } http_client_request_remove(req); http_client_request_unref(&req); }
void http_client_request_resubmit(struct http_client_request *req) { i_assert(!req->payload_wait); http_client_request_debug(req, "Resubmitting request"); /* rewind payload stream */ if (req->payload_input != NULL && req->payload_size > 0) { if (req->payload_input->v_offset != req->payload_offset && !req->payload_input->seekable) { http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Resubmission failed: Cannot resend payload; stream is not seekable"); return; } else { i_stream_seek(req->payload_input, req->payload_offset); } } /* drop payload output stream from previous attempt */ if (req->payload_output != NULL) o_stream_unref(&req->payload_output); req->conn = NULL; req->peer = NULL; req->state = HTTP_REQUEST_STATE_QUEUED; http_client_host_submit_request(req->host, req); }
void http_client_request_submit(struct http_client_request *req) { req->client->pending_requests++; http_client_request_do_submit(req); http_client_request_debug(req, "Submitted"); req->submitted = TRUE; }
void http_client_request_submit(struct http_client_request *req) { req->submit_time = ioloop_timeval; http_client_request_do_submit(req); http_client_request_debug(req, "Submitted"); req->submitted = TRUE; http_client_request_add(req); }
bool http_client_request_try_retry(struct http_client_request *req) { /* limit the number of attempts for each request */ if (req->attempts+1 >= req->client->set.max_attempts) return FALSE; req->attempts++; http_client_request_debug(req, "Retrying (attempts=%d)", req->attempts); if (req->callback != NULL) http_client_request_resubmit(req); return TRUE; }
static void http_client_request_finish_payload_out(struct http_client_request *req) { i_assert(req->conn != NULL); if (req->payload_output != NULL) { o_stream_unref(&req->payload_output); req->payload_output = NULL; } req->state = HTTP_REQUEST_STATE_WAITING; req->conn->output_locked = FALSE; http_client_request_debug(req, "Finished sending payload"); }
void http_client_request_finish(struct http_client_request **_req) { struct http_client_request *req = *_req; if (req->state >= HTTP_REQUEST_STATE_FINISHED) return; http_client_request_debug(req, "Finished"); req->callback = NULL; req->state = HTTP_REQUEST_STATE_FINISHED; if (req->payload_wait && req->client->ioloop != NULL) io_loop_stop(req->client->ioloop); http_client_request_unref(_req); }
void http_client_request_finish(struct http_client_request *req) { if (req->state >= HTTP_REQUEST_STATE_FINISHED) return; i_assert(req->refcount > 0); http_client_request_debug(req, "Finished"); req->callback = NULL; req->state = HTTP_REQUEST_STATE_FINISHED; if (req->queue != NULL) http_client_queue_drop_request(req->queue, req); if (req->payload_wait && req->client->ioloop != NULL) io_loop_stop(req->client->ioloop); http_client_request_unref(&req); }
bool http_client_request_try_retry(struct http_client_request *req) { /* don't ever retry if we're sending data in small blocks via http_client_request_send_payload() and we're not waiting for a 100 continue (there's no way to rewind the payload for a retry) */ if (req->payload_wait && (!req->payload_sync || req->payload_sync_continue)) return FALSE; /* limit the number of attempts for each request */ if (req->attempts+1 >= req->client->set.max_attempts) return FALSE; req->attempts++; http_client_request_debug(req, "Retrying (attempts=%d)", req->attempts); if (req->callback != NULL) http_client_request_resubmit(req); return TRUE; }
bool http_client_request_unref(struct http_client_request **_req) { struct http_client_request *req = *_req; struct http_client *client = req->client; i_assert(req->refcount > 0); *_req = NULL; if (--req->refcount > 0) return TRUE; http_client_request_debug(req, "Free (requests left=%d)", client->requests_count); /* cannot be destroyed while it is still pending */ i_assert(req->conn == NULL); if (req->queue != NULL) http_client_queue_drop_request(req->queue, req); if (req->destroy_callback != NULL) { req->destroy_callback(req->destroy_context); req->destroy_callback = NULL; } http_client_request_remove(req); if (client->requests_count == 0 && client->ioloop != NULL) io_loop_stop(client->ioloop); if (req->delayed_error != NULL) http_client_remove_request_error(req->client, req); if (req->payload_input != NULL) i_stream_unref(&req->payload_input); if (req->payload_output != NULL) o_stream_unref(&req->payload_output); if (req->headers != NULL) str_free(&req->headers); pool_unref(&req->pool); return FALSE; }
void http_client_request_unref(struct http_client_request **_req) { struct http_client_request *req = *_req; struct http_client *client = req->client; i_assert(req->refcount > 0); if (--req->refcount > 0) return; if (req->destroy_callback != NULL) { req->destroy_callback(req->destroy_context); req->destroy_callback = NULL; } /* only decrease pending request counter if this request was submitted */ if (req->state > HTTP_REQUEST_STATE_NEW) req->client->pending_requests--; http_client_request_debug(req, "Destroy (requests left=%d)", client->pending_requests); if (client->pending_requests == 0 && client->ioloop != NULL) io_loop_stop(client->ioloop); if (req->delayed_error != NULL) http_client_host_remove_request_error(req->host, req); if (req->payload_input != NULL) i_stream_unref(&req->payload_input); if (req->payload_output != NULL) o_stream_unref(&req->payload_output); if (req->headers != NULL) str_free(&req->headers); pool_unref(&req->pool); *_req = NULL; }
static int http_client_request_send_real(struct http_client_request *req, bool pipelined, const char **error_r) { const struct http_client_settings *set = &req->client->set; struct http_client_connection *conn = req->conn; struct ostream *output = conn->conn.output; string_t *rtext = t_str_new(256); struct const_iovec iov[3]; int ret = 0; i_assert(!req->conn->output_locked); i_assert(req->payload_output == NULL); /* create request line */ str_append(rtext, req->method); str_append(rtext, " "); str_append(rtext, req->target); str_append(rtext, " HTTP/1.1\r\n"); /* create special headers implicitly if not set explicitly using http_client_request_add_header() */ if (!req->have_hdr_host) { str_append(rtext, "Host: "); str_append(rtext, req->authority); str_append(rtext, "\r\n"); } if (!req->have_hdr_date) { str_append(rtext, "Date: "); str_append(rtext, http_date_create(req->date)); str_append(rtext, "\r\n"); } if (!req->have_hdr_authorization && req->username != NULL && req->password != NULL) { struct http_auth_credentials auth_creds; http_auth_basic_credentials_init(&auth_creds, req->username, req->password); str_append(rtext, "Authorization: "); http_auth_create_credentials(rtext, &auth_creds); str_append(rtext, "\r\n"); } if (http_client_request_to_proxy(req) && set->proxy_username != NULL && set->proxy_password != NULL) { struct http_auth_credentials auth_creds; http_auth_basic_credentials_init(&auth_creds, set->proxy_username, set->proxy_password); str_append(rtext, "Proxy-Authorization: "); http_auth_create_credentials(rtext, &auth_creds); str_append(rtext, "\r\n"); } if (!req->have_hdr_user_agent && req->client->set.user_agent != NULL) { str_printfa(rtext, "User-Agent: %s\r\n", req->client->set.user_agent); } if (!req->have_hdr_expect && req->payload_sync) { str_append(rtext, "Expect: 100-continue\r\n"); } if (req->payload_input != NULL) { if (req->payload_chunked) { // FIXME: can't do this for a HTTP/1.0 server if (!req->have_hdr_body_spec) str_append(rtext, "Transfer-Encoding: chunked\r\n"); req->payload_output = http_transfer_chunked_ostream_create(output); } else { /* send Content-Length if we have specified a payload, even if it's 0 bytes. */ if (!req->have_hdr_body_spec) { str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n", req->payload_size); } req->payload_output = output; o_stream_ref(output); } } if (!req->have_hdr_connection && !http_client_request_to_proxy(req)) { /* https://tools.ietf.org/html/rfc2068 Section 19.7.1: A client MUST NOT send the Keep-Alive connection token to a proxy server as HTTP/1.0 proxy servers do not obey the rules of HTTP/1.1 for parsing the Connection header field. */ str_append(rtext, "Connection: Keep-Alive\r\n"); } /* request line + implicit headers */ iov[0].iov_base = str_data(rtext); iov[0].iov_len = str_len(rtext); /* explicit headers */ if (req->headers != NULL) { iov[1].iov_base = str_data(req->headers); iov[1].iov_len = str_len(req->headers); } else { iov[1].iov_base = ""; iov[1].iov_len = 0; } /* end of header */ iov[2].iov_base = "\r\n"; iov[2].iov_len = 2; req->state = HTTP_REQUEST_STATE_PAYLOAD_OUT; req->sent_time = ioloop_timeval; o_stream_cork(output); if (o_stream_sendv(output, iov, N_ELEMENTS(iov)) < 0) { *error_r = t_strdup_printf("write(%s) failed: %s", o_stream_get_name(output), o_stream_get_error(output)); ret = -1; } else { http_client_request_debug(req, "Sent header"); if (req->payload_output != NULL) { if (!req->payload_sync) { if (http_client_request_send_more (req, pipelined, error_r) < 0) ret = -1; } else { http_client_request_debug(req, "Waiting for 100-continue"); conn->output_locked = TRUE; } } else { req->state = HTTP_REQUEST_STATE_WAITING; if (!pipelined) http_client_connection_start_request_timeout(req->conn); conn->output_locked = FALSE; } if (ret >= 0 && o_stream_flush(output) < 0) { *error_r = t_strdup_printf("flush(%s) failed: %s", o_stream_get_name(output), o_stream_get_error(output)); ret = -1; } } o_stream_uncork(output); return ret; }
int http_client_request_send_more(struct http_client_request *req, bool pipelined, const char **error_r) { struct http_client_connection *conn = req->conn; struct ostream *output = req->payload_output; enum ostream_send_istream_result res; i_assert(req->payload_input != NULL); i_assert(req->payload_output != NULL); if (conn->io_req_payload != NULL) io_remove(&conn->io_req_payload); /* chunked ostream needs to write to the parent stream's buffer */ o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE); res = o_stream_send_istream(output, req->payload_input); o_stream_set_max_buffer_size(output, (size_t)-1); switch (res) { case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: /* finished sending */ if (!req->payload_chunked && req->payload_input->v_offset - req->payload_offset != req->payload_size) { *error_r = t_strdup_printf("BUG: stream '%s' input size changed: " "%"PRIuUOFF_T"-%"PRIuUOFF_T" != %"PRIuUOFF_T, i_stream_get_name(req->payload_input), req->payload_input->v_offset, req->payload_offset, req->payload_size); i_error("%s", *error_r); //FIXME: remove? return -1; } if (req->payload_wait) { /* this chunk of input is finished (client needs to act; disable timeout) */ i_assert(!pipelined); conn->output_locked = TRUE; http_client_connection_stop_request_timeout(conn); if (req->client->ioloop != NULL) io_loop_stop(req->client->ioloop); } else { /* finished sending payload */ http_client_request_finish_payload_out(req); } return 0; case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: /* input is blocking (client needs to act; disable timeout) */ conn->output_locked = TRUE; if (!pipelined) http_client_connection_stop_request_timeout(conn); conn->io_req_payload = io_add_istream(req->payload_input, http_client_request_payload_input, req); return 0; case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: /* output is blocking (server needs to act; enable timeout) */ conn->output_locked = TRUE; if (!pipelined) http_client_connection_start_request_timeout(conn); http_client_request_debug(req, "Partially sent payload"); return 0; case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: /* we're in the middle of sending a request, so the connection will also have to be aborted */ *error_r = t_strdup_printf("read(%s) failed: %s", i_stream_get_name(req->payload_input), i_stream_get_error(req->payload_input)); /* the payload stream assigned to this request is broken, fail this the request immediately */ http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD, "Broken payload stream"); return -1; case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: /* failed to send request */ *error_r = t_strdup_printf("write(%s) failed: %s", o_stream_get_name(output), o_stream_get_error(output)); return -1; } i_unreached(); }
int http_client_request_send_more(struct http_client_request *req, const char **error_r) { struct http_client_connection *conn = req->conn; struct ostream *output = req->payload_output; off_t ret; int fd; i_assert(req->payload_input != NULL); i_assert(req->payload_output != NULL); if (conn->io_req_payload != NULL) io_remove(&conn->io_req_payload); /* chunked ostream needs to write to the parent stream's buffer */ o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE); ret = o_stream_send_istream(output, req->payload_input); o_stream_set_max_buffer_size(output, (size_t)-1); if (req->payload_input->stream_errno != 0) { /* the payload stream assigned to this request is broken, fail this the request immediately */ http_client_request_send_error(req, HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD, "Broken payload stream"); /* we're in the middle of sending a request, so the connection will also have to be aborted */ errno = req->payload_input->stream_errno; *error_r = t_strdup_printf("read(%s) failed: %s", i_stream_get_name(req->payload_input), i_stream_get_error(req->payload_input)); ret = -1; } else if (output->stream_errno != 0) { /* failed to send request */ errno = output->stream_errno; *error_r = t_strdup_printf("write(%s) failed: %s", o_stream_get_name(output), o_stream_get_error(output)); ret = -1; } else { i_assert(ret >= 0); } if (ret < 0 || i_stream_is_eof(req->payload_input)) { if (!req->payload_chunked && req->payload_input->v_offset - req->payload_offset != req->payload_size) { *error_r = "stream input size changed [BUG]"; i_error("stream input size changed"); //FIXME return -1; } if (req->payload_wait) { conn->output_locked = TRUE; if (req->client->ioloop != NULL) io_loop_stop(req->client->ioloop); } else { http_client_request_finish_payload_out(req); } } else if (i_stream_get_data_size(req->payload_input) > 0) { /* output is blocking */ conn->output_locked = TRUE; o_stream_set_flush_pending(output, TRUE); http_client_request_debug(req, "Partially sent payload"); } else { /* input is blocking */ fd = i_stream_get_fd(req->payload_input); conn->output_locked = TRUE; i_assert(fd >= 0); conn->io_req_payload = io_add (fd, IO_READ, http_client_request_payload_input, req); } return ret < 0 ? -1 : 0; }
static int http_client_request_continue_payload(struct http_client_request **_req, const unsigned char *data, size_t size) { struct ioloop *prev_ioloop = current_ioloop; struct http_client_request *req = *_req; struct http_client_connection *conn = req->conn; struct http_client *client = req->client; int ret; i_assert(req->state == HTTP_REQUEST_STATE_NEW || req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT); i_assert(req->payload_input == NULL); if (conn != NULL) http_client_connection_ref(conn); http_client_request_ref(req); req->payload_wait = TRUE; if (data == NULL) { req->payload_input = NULL; if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) http_client_request_finish_payload_out(req); } else { req->payload_input = i_stream_create_from_data(data, size); i_stream_set_name(req->payload_input, "<HTTP request payload>"); } req->payload_size = 0; req->payload_chunked = TRUE; if (req->state == HTTP_REQUEST_STATE_NEW) http_client_request_submit(req); /* Wait for payload data to be written */ i_assert(client->ioloop == NULL); client->ioloop = io_loop_create(); http_client_switch_ioloop(client); if (client->set.dns_client != NULL) dns_client_switch_ioloop(client->set.dns_client); while (req->state < HTTP_REQUEST_STATE_PAYLOAD_IN) { http_client_request_debug(req, "Waiting for request to finish"); if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) o_stream_set_flush_pending(req->payload_output, TRUE); io_loop_run(client->ioloop); if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT && req->payload_input->eof) { i_stream_unref(&req->payload_input); req->payload_input = NULL; break; } } io_loop_set_current(prev_ioloop); http_client_switch_ioloop(client); if (client->set.dns_client != NULL) dns_client_switch_ioloop(client->set.dns_client); io_loop_set_current(client->ioloop); io_loop_destroy(&client->ioloop); switch (req->state) { case HTTP_REQUEST_STATE_PAYLOAD_IN: case HTTP_REQUEST_STATE_FINISHED: ret = 1; break; case HTTP_REQUEST_STATE_ABORTED: ret = -1; break; default: ret = 0; break; } req->payload_wait = FALSE; /* callback may have messed with our pointer, so unref using local variable */ if (!http_client_request_unref(&req)) *_req = NULL; if (conn != NULL) http_client_connection_unref(&conn); /* Return status */ return ret; }
void http_client_request_redirect(struct http_client_request *req, unsigned int status, const char *location) { struct http_url *url; const char *error, *target, *origin_url; i_assert(!req->payload_wait); /* parse URL */ if (http_url_parse(location, NULL, 0, pool_datastack_create(), &url, &error) < 0) { http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, t_strdup_printf("Invalid redirect location: %s", error)); return; } if (++req->redirects > req->client->set.max_redirects) { if (req->client->set.max_redirects > 0) { http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, t_strdup_printf("Redirected more than %d times", req->client->set.max_redirects)); } else { http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, "Redirect refused"); } return; } /* rewind payload stream */ if (req->payload_input != NULL && req->payload_size > 0 && status != 303) { if (req->payload_input->v_offset != req->payload_offset && !req->payload_input->seekable) { http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Redirect failed: Cannot resend payload; stream is not seekable"); return; } else { i_stream_seek(req->payload_input, req->payload_offset); } } /* drop payload output stream from previous attempt */ if (req->payload_output != NULL) o_stream_unref(&req->payload_output); target = http_url_create_target(url); http_url_copy(req->pool, &req->origin_url, url); req->target = p_strdup(req->pool, target); req->host = NULL; req->conn = NULL; origin_url = http_url_create(&req->origin_url); http_client_request_debug(req, "Redirecting to %s%s", origin_url, target); req->label = p_strdup_printf(req->pool, "[%s %s%s]", req->method, origin_url, req->target); /* RFC 7231, Section 6.4.4: -> A 303 `See Other' redirect status response is handled a bit differently. Basically, the response content is located elsewhere, but the original (POST) request is handled already. */ if (status == 303 && strcasecmp(req->method, "HEAD") != 0 && strcasecmp(req->method, "GET") != 0) { // FIXME: should we provide the means to skip this step? The original // request was already handled at this point. req->method = p_strdup(req->pool, "GET"); /* drop payload */ if (req->payload_input != NULL) i_stream_unref(&req->payload_input); req->payload_size = 0; req->payload_offset = 0; } /* resubmit */ req->state = HTTP_REQUEST_STATE_NEW; http_client_request_do_submit(req); }