static ngx_int_t ngx_http_memcached_filter(void *data, ssize_t bytes) { ngx_http_memcached_ctx_t *ctx; ngx_http_upstream_t *u; u_char *last; ngx_buf_t *b; ngx_chain_t *cl, **ll; ctx = data; u = ctx->request->upstream; b = &u->buffer; last = b->last; b->last += bytes; /* set valid data for client */ if ((ssize_t)NGX_HTTP_MEMCACHED_END < u->length) { for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) ll = &cl->next; if (!(cl = ngx_chain_get_free_buf(ctx->request->pool, &u->free_bufs))) return NGX_ERROR; cl->buf->flush = 1; cl->buf->memory = 1; *ll = cl; cl->buf->pos = last; last += bytes; /* has \r\nEND\r\n ? */ if ((u->length -= bytes) < (ssize_t)NGX_HTTP_MEMCACHED_END) { last -= NGX_HTTP_MEMCACHED_END - u->length; u->length = NGX_HTTP_MEMCACHED_END; } cl->buf->last = last; cl->buf->tag = u->output.tag; ngx_log_debug3(NGX_LOG_DEBUG_HTTP, ctx->request->connection->log, 0, "memcached filter bytes:%z size:%z length:%z", bytes, cl->buf->last - cl->buf->pos, u->length); /* eat */ bytes -= cl->buf->last - cl->buf->pos; } /* eat \r\nEND\r\n */ if (0 < bytes) { if (ngx_strncmp(last, ngx_http_memcached_end + NGX_HTTP_MEMCACHED_END - u->length, bytes)) { ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0, "memcached sent invalid trailer"); u->length = 0; return NGX_OK; } if (!(u->length -= bytes)) u->keepalive = 1; } return NGX_OK; }
static ngx_int_t ngx_http_cwinux_empty_filter(void *data, ssize_t bytes) { ngx_http_cwinux_ctx_t *ctx = data; ngx_chain_t *cl, **ll; ngx_http_upstream_t *u; u = ctx->request->upstream; if (ctx->finish){ u->length = 0; return NGX_OK; } for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { ll = &cl->next; } cl = ngx_chain_get_free_buf(ctx->request->pool, &u->free_bufs); if (cl == NULL) { return NGX_ERROR; } cl->buf->flush = 1; cl->buf->memory = 1; *ll = cl; cl->buf->pos = (u_char*)ctx->out_buf; cl->buf->last = cl->buf->pos + ctx->out_buf_len; cl->buf->tag = u->output.tag; u->length = 0; ctx->finish = true; return NGX_OK; }
static ngx_chain_t * ngx_http_v2_filter_get_shadow(ngx_http_v2_stream_t *stream, ngx_buf_t *buf, off_t offset, off_t size) { ngx_buf_t *chunk; ngx_chain_t *cl; cl = ngx_chain_get_free_buf(stream->request->pool, &stream->free_bufs); if (cl == NULL) { return NULL; } chunk = cl->buf; ngx_memcpy(chunk, buf, sizeof(ngx_buf_t)); chunk->tag = (ngx_buf_tag_t) &ngx_http_v2_filter_get_shadow; chunk->shadow = buf; if (ngx_buf_in_memory(chunk)) { chunk->pos += offset; chunk->last = chunk->pos + size; } if (chunk->in_file) { chunk->file_pos += offset; chunk->file_last = chunk->file_pos + size; } return cl; }
static inline ngx_int_t ngx_http_tnt_output(ngx_http_request_t *r, ngx_http_upstream_t *u, ngx_buf_t *b) { ngx_chain_t *cl, **ll; for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { ll = &cl->next; } cl = ngx_chain_get_free_buf(r->pool, &u->free_bufs); if (cl == NULL) { return NGX_ERROR; } b->pos = b->start; b->flush = 1; b->last_in_chain = 1; b->tag = u->output.tag; cl->buf = b; cl->next = NULL; *ll = cl; return NGX_OK; }
static inline ngx_int_t ngx_http_tnt_output_err(ngx_http_request_t *r, ngx_http_tnt_ctx_t *ctx, ngx_int_t code) { ngx_http_upstream_t *u; ngx_chain_t *cl, **ll; u = r->upstream; if (ctx->in_err == NULL) { u->headers_in.status_n = 500; u->state->status = 500; u->headers_in.content_length_n = 0; return NGX_OK; } u->headers_in.status_n = code; u->state->status = code; u->headers_in.content_length_n = ctx->in_err->last - ctx->in_err->pos; u->length = 0; for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { ll = &cl->next; } *ll = cl = ngx_chain_get_free_buf(r->pool, &u->free_bufs); if (cl == NULL) { return NGX_ERROR; } cl->buf = ctx->in_err; cl->next = NULL; cl->buf->pos = cl->buf->start; cl->buf->end = cl->buf->last; cl->buf->flush = 1; cl->buf->memory = 1; cl->buf->tag = u->output.tag; cl->buf->last_in_chain = 1; return NGX_OK; }
ngx_int_t ngx_event_pipe_copy_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) { ngx_buf_t *b; ngx_chain_t *cl; if (buf->pos == buf->last) { return NGX_OK; } cl = ngx_chain_get_free_buf(p->pool, &p->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; ngx_memcpy(b, buf, sizeof(ngx_buf_t)); b->shadow = buf; b->tag = p->tag; b->last_shadow = 1; b->recycled = 1; buf->shadow = b; ngx_log_debug1(NGX_LOG_DEBUG_EVENT, p->log, 0, "input buf #%d", b->num); if (p->in) { *p->last_in = cl; } else { p->in = cl; } p->last_in = &cl->next; if (p->length == -1) { return NGX_OK; } p->length -= b->last - b->pos; return NGX_OK; }
ngx_int_t ngx_http_beanstalkd_write_simple_response(ngx_http_request_t *r, ngx_http_upstream_t *u, ngx_http_beanstalkd_ctx_t *ctx, ngx_uint_t status, ngx_str_t *resp) { dd("ngx_http_beanstalkd_write_simple_response"); ngx_chain_t *cl, **ll; r->headers_out.content_length_n = resp->len; u->headers_in.status_n = status; u->state->status = status; for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { ll = &cl->next; } cl = ngx_chain_get_free_buf(r->pool, &u->free_bufs); if (cl == NULL) { return NGX_ERROR; } cl->buf->flush = 1; cl->buf->memory = 1; cl->buf->pos = resp->data; cl->buf->last = cl->buf->pos + resp->len; *ll = cl; /* for subrequests in memory */ u->buffer.pos = resp->data; u->buffer.last = resp->data + resp->len; /* ctx->body_length = resp->len; */ return NGX_OK; }
ngx_chain_t * ngx_tcp_chain_get_free_buf(ngx_output_chain_ctx_t *ctx, size_t total_size) { ngx_chain_t *cl, **ll; size_t size = 0; cl = NULL; ll = &cl; while (size < total_size) { *ll = ngx_chain_get_free_buf(ctx->pool, &ctx->free); if (*ll == NULL) { goto failed; } if (NULL == (*ll)->buf->start) { size_t buf_size = total_size - size; (*ll)->buf->start = ngx_palloc(ctx->pool, buf_size); if (NULL == (*ll)->buf->start) { goto failed; } (*ll)->buf->pos = (*ll)->buf->start; (*ll)->buf->last = (*ll)->buf->start; (*ll)->buf->end = (*ll)->buf->start + buf_size; (*ll)->buf->temporary = 1; } size += (*ll)->buf->end - (*ll)->buf->last; (*ll)->next = NULL; ll = &(*ll)->next; } return cl; failed: if (cl != NULL) ngx_chain_update_chains(ctx->pool, &ctx->free, &ctx->busy, &cl, ctx->tag); return NULL; }
static ngx_int_t ngx_http_reqstat_show_handler(ngx_http_request_t *r) { ngx_int_t rc; ngx_buf_t *b; ngx_uint_t i, j; ngx_array_t *display; ngx_chain_t *tl, *free, *busy; ngx_queue_t *q; ngx_shm_zone_t **shm_zone; ngx_http_reqstat_ctx_t *ctx; ngx_http_reqstat_conf_t *slcf; ngx_http_reqstat_conf_t *smcf; ngx_http_reqstat_rbnode_t *node; slcf = ngx_http_get_module_loc_conf(r, ngx_http_reqstat_module); smcf = ngx_http_get_module_main_conf(r, ngx_http_reqstat_module); display = slcf->display == NULL ? smcf->monitor : slcf->display; if (display == NULL) { r->headers_out.status = NGX_HTTP_NO_CONTENT; return ngx_http_send_header(r); } r->headers_out.status = NGX_HTTP_OK; ngx_http_clear_content_length(r); rc = ngx_http_send_header(r); if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { return rc; } shm_zone = display->elts; for (free = busy = NULL, i = 0; i < display->nelts; i++) { ctx = shm_zone[i]->data; for (q = ngx_queue_head(&ctx->sh->queue); q != ngx_queue_sentinel(&ctx->sh->queue); q = ngx_queue_next(q)) { node = ngx_queue_data(q, ngx_http_reqstat_rbnode_t, queue); tl = ngx_chain_get_free_buf(r->pool, &free); if (tl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b = tl->buf; if (b->start == NULL) { b->start = ngx_pcalloc(r->pool, 512); if (b->start == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b->end = b->start + 512; } b->last = b->pos = b->start; b->memory = 1; b->temporary = 1; b->last = ngx_slprintf(b->last, b->end, "%*s,", (size_t) node->len, node->data); for (j = 0; j < sizeof(ngx_http_reqstat_fields) / sizeof(off_t); j++) { b->last = ngx_slprintf(b->last, b->end, "%uA,", *REQ_FIELD(node, ngx_http_reqstat_fields[j])); } *(b->last - 1) = '\n'; if (ngx_http_output_filter(r, tl) == NGX_ERROR) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } #if nginx_version >= 1002000 ngx_chain_update_chains(r->pool, &free, &busy, &tl, (ngx_buf_tag_t) &ngx_http_reqstat_module); #else ngx_chain_update_chains(&free, &busy, &tl, (ngx_buf_tag_t) &ngx_http_reqstat_module); #endif } } tl = ngx_chain_get_free_buf(r->pool, &free); if (tl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b = tl->buf; b->last_buf = 1; return ngx_http_output_filter(r, tl); }
static void ngx_http_groonga_context_receive_handler_raw(grn_ctx *context, int flags, ngx_http_groonga_handler_data_t *data) { char *chunk = NULL; unsigned int chunk_size = 0; int recv_flags; ngx_http_request_t *r; ngx_log_t *log; grn_bool is_last_chunk; grn_ctx_recv(context, &chunk, &chunk_size, &recv_flags); data->raw.processed = GRN_TRUE; if (data->raw.rc != NGX_OK) { return; } r = data->raw.r; log = r->connection->log; is_last_chunk = (flags & GRN_CTX_TAIL); if (!data->raw.header_sent) { ngx_http_groonga_handler_set_content_type(r, grn_ctx_get_mime_type(context)); r->headers_out.status = NGX_HTTP_OK; if (is_last_chunk) { r->headers_out.content_length_n = chunk_size; if (chunk_size == 0) { r->header_only = 1; } } else { r->headers_out.content_length_n = -1; } data->raw.rc = ngx_http_send_header(r); data->raw.header_sent = GRN_TRUE; if (data->raw.rc != NGX_OK) { return; } } if (chunk_size > 0 || is_last_chunk) { ngx_chain_t *chain; chain = ngx_chain_get_free_buf(r->pool, &(data->raw.free_chain)); if (!chain) { ngx_log_error(NGX_LOG_ERR, log, 0, "http_groonga: failed to allocate memory for chunked body"); data->raw.rc = NGX_ERROR; return; } if (chunk_size == 0) { chain->buf->pos = NULL; chain->buf->last = NULL; chain->buf->memory = 0; } else { chain->buf->pos = (u_char *)chunk; chain->buf->last = (u_char *)chunk + chunk_size; chain->buf->memory = 1; } chain->buf->tag = (ngx_buf_tag_t)&ngx_http_groonga_module; chain->buf->flush = 1; chain->buf->temporary = 0; chain->buf->in_file = 0; if (is_last_chunk) { chain->buf->last_buf = 1; } else { chain->buf->last_buf = 0; } chain->next = NULL; data->raw.rc = ngx_http_output_filter(r, chain); ngx_chain_update_chains(r->pool, &(data->raw.free_chain), &(data->raw.busy_chain), &chain, (ngx_buf_tag_t)&ngx_http_groonga_module); } }
static ngx_int_t ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in) { u_char *chunk; off_t size; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t *out, *cl, *tl, **ll; ngx_http_chunked_filter_ctx_t *ctx; if (in == NULL || !r->chunked || r->header_only) { return ngx_http_next_body_filter(r, in); } ctx = ngx_http_get_module_ctx(r, ngx_http_chunked_filter_module); out = NULL; ll = &out; size = 0; cl = in; for ( ;; ) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http chunk: %d", ngx_buf_size(cl->buf)); size += ngx_buf_size(cl->buf); if (cl->buf->flush || cl->buf->sync || ngx_buf_in_memory(cl->buf) || cl->buf->in_file) { tl = ngx_alloc_chain_link(r->pool); if (tl == NULL) { return NGX_ERROR; } tl->buf = cl->buf; *ll = tl; ll = &tl->next; } if (cl->next == NULL) { break; } cl = cl->next; } if (size) { tl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (tl == NULL) { return NGX_ERROR; } b = tl->buf; chunk = b->start; if (chunk == NULL) { /* the "0000000000000000" is 64-bit hexadecimal string */ chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1); if (chunk == NULL) { return NGX_ERROR; } b->start = chunk; b->end = chunk + sizeof("0000000000000000" CRLF) - 1; } b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module; b->memory = 0; b->temporary = 1; b->pos = chunk; b->last = ngx_sprintf(chunk, "%xO" CRLF, size); tl->next = out; out = tl; } if (cl->buf->last_buf) { tl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (tl == NULL) { return NGX_ERROR; } b = tl->buf; b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module; b->temporary = 0; b->memory = 1; b->last_buf = 1; b->pos = (u_char *) CRLF "0" CRLF CRLF; b->last = b->pos + 7; cl->buf->last_buf = 0; *ll = tl; if (size == 0) { b->pos += 2; } } else if (size > 0) { tl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (tl == NULL) { return NGX_ERROR; } b = tl->buf; b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module; b->temporary = 0; b->memory = 1; b->pos = (u_char *) CRLF; b->last = b->pos + 2; *ll = tl; } else { *ll = NULL; } ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "send chunked data: %d", size); rc = ngx_http_next_body_filter(r, out); ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &out, (ngx_buf_tag_t) &ngx_http_chunked_filter_module); return rc; }
/* 调用ngx_http_do_read_client_request_body方法接收包体。该方法的意义在于把客户端与Nginx之间TCP连接上套接字缓冲区中的当前字符流全 部读出来,并判断是否需要写入文件,以及是否接收到全部的包体,同时在接收到完整的包体后激活post_handler回调方法 负责具体的读取包体工作,该函数会在for循环中会反复读直到包体读取完毕,如果内核已经没有数据并且包体还没有读完,则添加读事件,并退出循环,这样HTTP模块还能继续作用其他功能,避免阻塞 读取的时候一个buf装满后,会把buf中存储的数据写到临时文件中(不管有没有配置request_body_in_file_only),然后继续使用该buf读取数据, 存储数据的内存分配地方有两个:1.在读取报文头部的时候ngx_http_wait_request_handler 2.如果在1中读到的内容里面不包括完整包体,则需要在 ngx_http_read_client_request_body中会重新分配内存读取,触发再次读取的地方未ngx_http_read_client_request_body中为读取到完整包体的时候添加的ngx_handle_read_event */ static ngx_int_t ngx_http_do_read_client_request_body(ngx_http_request_t *r) { off_t rest; size_t size; ssize_t n; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t *cl, out; ngx_connection_t *c; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; c = r->connection; rb = r->request_body; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http read client request body"); for ( ;; ) { for ( ;; ) { /* 首先检查请求的request_body成员中的buf缓冲区,如果缓冲区还有空闲的空间,则跳过该if{}去读取内核中套接字缓冲区里的TCP字符流; 如果缓冲区已经写满,则调用ngx_http_write_request_body方法把缓冲区中的字符流写入文件。不管有没有配置request_body_in_file_only置1 */ if (rb->buf->last == rb->buf->end) { if (rb->buf->pos != rb->buf->last) { /* pass buffer to request body filter chain */ out.buf = rb->buf; out.next = NULL; //这里肯定会调用ngx_http_request_body_save_filter->ngx_http_write_request_body写该buf中的内容到临时文件,因为该buf指向的空间 //会重复利用来读取包体内容 rc = ngx_http_request_body_filter(r, &out); if (rc != NGX_OK) { return rc; } } else { /* update chains */ rc = ngx_http_request_body_filter(r, NULL); if (rc != NGX_OK) { return rc; } } if (rb->busy != NULL) { //如果头部行中的content-length:LEN中的len长度表示后面的包体大小,如果后面的包体数据长度实际比头部中的LEN大,则会走这里, if (r->request_body_no_buffering) { if (c->read->timer_set) { ngx_del_timer(c->read, NGX_FUNC_LINE); } if (ngx_handle_read_event(c->read, 0, NGX_FUNC_LINE) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } return NGX_AGAIN; } return NGX_HTTP_INTERNAL_SERVER_ERROR; } /* 为什么能下次还可以直接利用rb->buf空间来读取数据呢? 当一个rb->buf填满后就会通过ngx_http_write_request_body把bufs链表中的所有ngx_chain_t->ngx_buf_t中指向的数据写入到临时文件,因此rb->buf中的内存就可以再次使用了 */ //只需要把缓冲区ngx_buf_t结构体的last指针指向start指针,缓冲区即可复用。 rb->buf->pos = rb->buf->start; rb->buf->last = rb->buf->start; } size = rb->buf->end - rb->buf->last; //buf中还剩余这么多空间 rest = rb->rest - (rb->buf->last - rb->buf->pos); //还有多少字节包体没有读取 if ((off_t) size > rest) { //说明空间够用来存储剩余的没有读取的字节数 size = (size_t) rest; } //负责具体的读取包体工作,该函数会在for循环中会反复读直到包体读取完毕,如果内核已经没有数据并且包体还没有读完,则添加读事件,并退出循环,这样HTTP模块还能继续作用其他功能,避免阻塞 //调用封装了recv的方法从套接字缓冲区中读取包体到缓冲区中。 n = c->recv(c, rb->buf->last, size); ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "http client request body recv %z", n); if (n == NGX_AGAIN) { break; } if (n == 0) {//如果recv方法返回错误,或者客户端主动关闭了连接 ngx_log_error(NGX_LOG_INFO, c->log, 0, "client prematurely closed connection"); } if (n == 0 || n == NGX_ERROR) {//如果recv方法返回错误,或者客户端主动关闭了连接 c->error = 1; return NGX_HTTP_BAD_REQUEST; } /* 根据接收到的TCP流长度,修改缓冲区参数。例如,把缓冲区ngx_buf_t结构体的last揩针加上接收到的长度,同时更新request_body结 构体中表示待接收的剩余包体长度的rest成员、更新ngx_http_request_t结构体中表示已接收请求长度的request_length成员。 */ //从这里可以看出在多次读取包体的时候,需要先把前面开辟空间buf中没有填充的部分填满,如果buf填满了,则重新利用该buf读取数据 //之前读取到填满buf中的数据取出来存放到临时文件中,参考前面的if (rb->buf->last == rb->buf->end) rb->buf->last += n; r->request_length += n; if (n == rest) {//根据rest成员检查是否接收到完整的包体 /* pass buffer to request body filter chain */ out.buf = rb->buf; out.next = NULL; rc = ngx_http_request_body_filter(r, &out); if (rc != NGX_OK) { return rc; } } if (rb->rest == 0) { break; //所有包体读取处理完毕,则退出for } if (rb->buf->last < rb->buf->end) { break; } } ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "http client request body rest %O", rb->rest); if (rb->rest == 0) { break; } /* 如果当前已经没有可读的字符流,同时还没有接收到完整的包体,则说明需要把读事件添加到事件模块,等待可读事件发生时,事件框架可以再次 调度到这个方法接收包体。这一步是调用ngx_add_timer方法将读事件添加到定时器中,超时时间以nginx.conf文件中的client_body_timeout配置项参数为准。 */ //说明前面的 n = c->recv(c, rb->buf->last, size);返回的是NGX_AGAIN,所以在recv中会把ready置0 if (!c->read->ready) { if (r->request_body_no_buffering && rb->buf->pos != rb->buf->last) { /* pass buffer to request body filter chain */ out.buf = rb->buf; out.next = NULL; rc = ngx_http_request_body_filter(r, &out); if (rc != NGX_OK) { return rc; } } clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); //当读取到完整的包体后,会删除该定时器,见后面的ngx_del_timer(c->read); ngx_add_timer(c->read, clcf->client_body_timeout, NGX_FUNC_LINE);//handle应该是ngx_http_request_handler /* 这个请求连接上的读事件触发时的回调方法ngx_http_request_handler,从而会调用read_event_handler方法(ngx_http_read_client_request_body_handler) */ if (ngx_handle_read_event(c->read, 0, NGX_FUNC_LINE) != NGX_OK) { //handle应该是ngx_http_request_handler,通过这里触发再次读取包体 return NGX_HTTP_INTERNAL_SERVER_ERROR; } return NGX_AGAIN; //把控制器交给HTTP框架,由框架感知读事件,当读事件发生,也就是数据到来,继续读取包体 } } //只有包体读取完毕,才会从上面的for()循环中退出 /* 表明已经接收到完整的包体,需要做一些收尾工作了。首先不需要检查是否接收HTTP包体超时了,要把读事件从定时器中取出,防止不必要的定时器触发。 这一步会检查读事件的timer set标志位,如果为1,则调用ngx_del_timer方法把读事件从定时器中移除。 */ if (c->read->timer_set) { ngx_del_timer(c->read, NGX_FUNC_LINE); } //如果缓冲区中还有未写入文件的内容,调用ngx_http_write_request_body方法把最后的包体内容也写入文件。 if (rb->temp_file || r->request_body_in_file_only) { //只要之前的内存有写入文件,那么剩余的部分也要写入文件 /* save the last part */ if (ngx_http_write_request_body(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } if (rb->temp_file->file.offset != 0) { cl = ngx_chain_get_free_buf(r->pool, &rb->free); if (cl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->in_file = 1; b->file_last = rb->temp_file->file.offset; b->file = &rb->temp_file->file; rb->bufs = cl; //读取客户包体即使是存入临时文件中,当所有包体读取完毕后(ngx_http_do_read_client_request_body),还是会让r->request_body->bufs指向文件中的相关偏移内存地址 } else { rb->bufs = NULL; } } /* 在之前read_event_handler成员设置为ngx_http_read_client_request_body_handler方法,现在既然已经接收到完整的包体了,就会把 read_event_handler设为ngx_http_block_reading方法,表示连接上再有读事件将不做任何处理。 */ if (!r->request_body_no_buffering) { r->read_event_handler = ngx_http_block_reading; rb->post_handler(r); //执行ngx_http_read_client_request_body的第二个参数 } return NGX_OK; }
static ngx_int_t ngx_http_trim_body_filter(ngx_http_request_t *r, ngx_chain_t *in) { ngx_int_t rc; ngx_chain_t *cl, *ln, *out, **ll; ngx_http_trim_ctx_t *ctx; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http trim filter"); ctx = ngx_http_get_module_ctx(r, ngx_http_trim_filter_module); if (ctx == NULL) { return ngx_http_next_body_filter(r, in); } if (in == NULL) { return ngx_http_next_body_filter(r, in); } ctx->in = NULL; if (ngx_chain_add_copy(r->pool, &ctx->in, in) != NGX_OK) { return NGX_ERROR; } out = NULL; ll = &out; for (ln = ctx->in; ln; ln = ln->next) { ngx_http_trim_parse(r, ln->buf, ctx); if (ctx->saved) { cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } cl->buf->tag = (ngx_buf_tag_t) &ngx_http_trim_filter_module; cl->buf->memory = 1; if (ctx->saved > 0) { cl->buf->pos = ngx_http_trim_saved_html.data; cl->buf->last = cl->buf->pos + ctx->saved; } else if (ctx->saved == NGX_HTTP_TRIM_SAVE_SLASH) { cl->buf->pos = ngx_http_trim_saved_jscss.data; cl->buf->last = cl->buf->pos + 1; } else if (ctx->saved == NGX_HTTP_TRIM_SAVE_JSCSS) { cl->buf->pos = ngx_http_trim_saved_jscss.data; cl->buf->last = cl->buf->pos + ngx_http_trim_saved_jscss.len; } else if (ctx->saved == NGX_HTTP_TRIM_SAVE_HACKCSS) { cl->buf->pos = ngx_http_trim_saved_css_hack.data; cl->buf->last = cl->buf->pos + ngx_http_trim_saved_css_hack.len; } *ll = cl; ll = &cl->next; ctx->saved = 0; } if(ln->buf->in_file && (ln->buf->file_last - ln->buf->file_pos) != (off_t) (ln->buf->last - ln->buf->pos)) { ln->buf->in_file = 0; } if (ngx_buf_size(ln->buf) == 0) { if (ln->buf->last_buf) { cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } ngx_memzero(cl->buf, sizeof(ngx_buf_t)); cl->buf->tag = (ngx_buf_tag_t) &ngx_http_trim_filter_module; cl->buf->last_buf = 1; *ll = cl; ll = &cl->next; } else { if (ln->next == NULL) { *ll = NULL; } } } else { *ll = ln; ll = &ln->next; } } if (out == NULL) { return NGX_OK; } rc = ngx_http_next_body_filter(r, out); ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &out, (ngx_buf_tag_t) &ngx_http_trim_filter_module); return rc; }
static ngx_int_t ngx_http_request_body_length_filter(ngx_http_request_t *r, ngx_chain_t *in) { size_t size; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t *cl, *tl, *out, **ll; ngx_http_request_body_t *rb; rb = r->request_body; if (rb->rest == -1) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http request body content length filter"); rb->rest = r->headers_in.content_length_n; } out = NULL; ll = &out; for (cl = in; cl; cl = cl->next) { if (rb->rest == 0) { break; } tl = ngx_chain_get_free_buf(r->pool, &rb->free); if (tl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b = tl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->temporary = 1; b->tag = (ngx_buf_tag_t) &ngx_http_read_client_request_body; b->start = cl->buf->pos; b->pos = cl->buf->pos; b->last = cl->buf->last; b->end = cl->buf->end; b->flush = r->request_body_no_buffering; size = cl->buf->last - cl->buf->pos; if ((off_t) size < rb->rest) { cl->buf->pos = cl->buf->last; rb->rest -= size; } else { cl->buf->pos += (size_t) rb->rest; rb->rest = 0; b->last = cl->buf->pos; b->last_buf = 1; } *ll = tl; ll = &tl->next; } rc = ngx_http_top_request_body_filter(r, out); ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, (ngx_buf_tag_t) &ngx_http_read_client_request_body); return rc; }
//只有在连接后端服务器的时候才会读取客户端请求包体,见ngx_http_xxx_handler(proxy fastcgi等) //post_handler在ngx_http_do_read_client_request_body接收完所有包体后执行,或者在本函数能读取完包体后也会执行 //post_handler方法被回调时,务必调用类似ngx_http_finalize_request的方法去结束请求,否则引用计数会始终无法清零,从而导致请求无法释放。 ngx_int_t ngx_http_read_client_request_body(ngx_http_request_t *r, ngx_http_client_body_handler_pt post_handler) { size_t preread; ssize_t size; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t out, *cl; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; /* 首先把该请求对应的原始请求的引用计数加1。这同时是在要求每一个HTTP模块在传入的post_handler方法被回调时,务必调用类似 ngx_http_finalize_request的方法去结束请求,否则引用计数会始终无法清零,从而导致请求无法释放。 */ r->main->count++; #if (NGX_HTTP_SPDY) if (r->spdy_stream && r == r->main) { r->request_body_no_buffering = 0; rc = ngx_http_spdy_read_request_body(r, post_handler); goto done; } #endif /* 检查请求ngx_http_request_t结构体中的request_body成员,如果它已经被分配过了,证明已经读取过HTTP包体了,不需要再次读取一遍; 再检查请求ngx_http_request_t结构体中的discard_body标志位,如果discard_body为1,则证明曾经执行过丢弃包体的方法,现在包体正在被丢弃中。 只有这两个条件都不满足,才说明真正需要接收HTTP包体。 */ if (r != r->main || r->request_body || r->discard_body) { r->request_body_no_buffering = 0; post_handler(r); //直接执行各HTTP模块提供的post_handler回调方法 return NGX_OK; } if (ngx_http_test_expect(r) != NGX_OK) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } if (r->request_body_no_buffering) { //如果不缓存包体,request_body_no_buffering和request_body_in_file_only是互斥的 r->request_body_in_file_only = 0; //设置为不缓存包体,则就不能把包体写道文件中 } /* 分配请求的ngx_http_request_t结构体中的request_body成员(之前request_body是NULL空指针),准备接收包体。 */ rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); if (rb == NULL) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } /* * set by ngx_pcalloc(): * * rb->bufs = NULL; * rb->buf = NULL; * rb->free = NULL; * rb->busy = NULL; * rb->chunked = NULL; */ rb->rest = -1; rb->post_handler = post_handler; r->request_body = rb; //把创建的ngx_http_request_body_t空间赋值给request_body /* 检查请求的content-length头部,如果指定了包体长度的content-length字段小于或等于0,当然不用继续接收包体: 如果content-length大于0,则意味着继续执行,但HTTP模块定义的post_handler方法不会知道在哪一次事件的触发中会被回调, 所以先把它设置到request_body结构体的post_handler成员中。 */ if (r->headers_in.content_length_n < 0 && !r->headers_in.chunked) { r->request_body_no_buffering = 0; post_handler(r); return NGX_OK; } /* * 接收HTTP头部的流程中,是有可能接收到HTTP包体的。 * 首先我们需要检查在header_in缓冲区中已经接收到的包体长度,确定其是否大于或者等于content-length头部指定的长度, * 如果大干或等于则说明已经接收到完整的包体 */ preread = r->header_in->last - r->header_in->pos; if (preread) { //注意在ngx_http_wait_request_handler中第一次读的时候默认是读1024字节,有可能ngx_http_wait_request_handler已经把包体读了 /* there is the pre-read part of the request body */ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http client request body preread %uz", preread); out.buf = r->header_in;// out.next = NULL; //把最新读取到的buf数据添加到r->request_body->bufs中,并且让free指向该bufs中所有数据中已经解析了的数据节点信息(重复利用ngx_buf_t) //busy链表中的ngx_buf_t节点指向bufs中所有数据中还没有解析完毕的数据 rc = ngx_http_request_body_filter(r, &out); if (rc != NGX_OK) { goto done; } r->request_length += preread - (r->header_in->last - r->header_in->pos); /* 当上述条件不满足时,再检查header_in缓冲区里的剩余空闲空间是否可以存放下全部的包体(content-length头部指定),如果可以,就不用分配新的包体缓冲区浪费内存了 */ if (!r->headers_in.chunked && rb->rest > 0 //还需要读取rb->rest才能保证包体读完 && rb->rest <= (off_t) (r->header_in->end - r->header_in->last)) //判断header_in指向的剩余未用空间是否足够存取剩余的rest字节数据 { /* the whole request body may be placed in r->header_in */ //header_in中剩余的未用空间足够,例如还差rest = 1000字节才能读取完包体,但是header_in中剩余空间end - last超过1000,则不需要从新开辟空间 //直接使用header_in剩余空间,开辟新的ngx_buf_t空间,使用新的ngx_buf_t中的各个指针指向header_in中剩余未用空间,用来继续读取 b = ngx_calloc_buf(r->pool); if (b == NULL) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } b->temporary = 1; b->start = r->header_in->pos; b->pos = r->header_in->pos; b->last = r->header_in->last; b->end = r->header_in->end; rb->buf = b; r->read_event_handler = ngx_http_read_client_request_body_handler; r->write_event_handler = ngx_http_request_empty_handler; rc = ngx_http_do_read_client_request_body(r); goto done; } } else { /* set rb->rest */ if (ngx_http_request_body_filter(r, NULL) != NGX_OK) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } } //包体一次读取完毕 if (rb->rest == 0) { /* the whole request body was pre-read */ //如果配置"client_body_in_file_only" on | clean 表示包体存储在磁盘文件中 if (r->request_body_in_file_only) { if (ngx_http_write_request_body(r) != NGX_OK) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } if (rb->temp_file->file.offset != 0) { cl = ngx_chain_get_free_buf(r->pool, &rb->free); if (cl == NULL) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->in_file = 1; b->file_last = rb->temp_file->file.offset; b->file = &rb->temp_file->file; rb->bufs = cl; //如果包体存入临时文件中,则读取包体完成后,bufs指向的ngx_chain_t中的各个指针指向文件中的相关偏移 } else { rb->bufs = NULL; } } r->request_body_no_buffering = 0; post_handler(r); return NGX_OK; } //包体一次没有读取完毕 if (rb->rest < 0) { ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, "negative request body rest"); rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); size = clcf->client_body_buffer_size; size += size >> 2; //实际上就是四分之五5/4个client_body_buffer_size /* TODO: honor r->request_body_in_single_buf */ //走到这里之前至少在ngx_http_wait_request_handler函数中读取过一次,也就是读取头部的时候,可能会读取一部分包体,在读取头部的时候 //读取的最大报文长度为client_header_buffer_size,所以包体有可能在那里读取后处理了头部行后,会走到本函数处理包体,这时候可能包体没有读完 if (!r->headers_in.chunked && rb->rest < size) { size = (ssize_t) rb->rest; if (r->request_body_in_single_buf) { //需要缓存到同一个buf中,那么开辟的空间就必须一次分配完,这样可以存储后面所有的。 size += preread; //如果是把读取的网络数据存到同一个single buffer中,则本次读到preread字节,但是还有size字节没读,所以需要相加,表示一共需要这么多空间, } } else { size = clcf->client_body_buffer_size; //如果不是缓存到同一个buf,则一次最多开辟这么多空间,这样可能需要多个buf才能读取完 } /* 说明确实需要分配用于接收包体的缓冲区了。缓冲区长度由nginx.conf丈件中的client_body_buffer_size配置项指定,缓冲区就在ngx_http_request_body_t 结构体的buf成员中存放着,同时,bufs和to_ write这两个缓冲区链表首部也指向该buf。 */ rb->buf = ngx_create_temp_buf(r->pool, size); //这个是为下次读取准备的 if (rb->buf == NULL) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } /* 设置请求ngx_http_request_t结构体的read_ event_ handler成员为上面介绍过的ngx_http_read_client_request_body_handler方法, 它意味着如果epoll再次检测到可读事件或者读事件的定时器超时,HTTP框架将调用ngx_http_read_client_request_body_handler方法处理 */ r->read_event_handler = ngx_http_read_client_request_body_handler; r->write_event_handler = ngx_http_request_empty_handler; /* 调用ngx_http_do_read_client_request_body方法接收包体。该方法的意义在于把客户端与Nginx之间TCP连接上套接字缓冲区中的当前字符流全 部读出来,并判断是否需要写入文件,以及是否接收到全部的包体,同时在接收到完整的包体后激活post_handler回调方法 */ rc = ngx_http_do_read_client_request_body(r);//这里面添加ngx_handle_read_event的时候,对应的handler为ngx_http_read_client_request_body_handler done: if (r->request_body_no_buffering && (rc == NGX_OK || rc == NGX_AGAIN)) { if (rc == NGX_OK) { r->request_body_no_buffering = 0; } else { /* rc == NGX_AGAIN */ r->reading_body = 1; } r->read_event_handler = ngx_http_block_reading; post_handler(r); } if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {//如果返回出错 r->main->count--; //该函数处理结束后-1,因为该函数开始处理的时候有+1 } return rc; }
ngx_int_t ngx_http_read_client_request_body(ngx_http_request_t *r, ngx_http_client_body_handler_pt post_handler) { size_t preread; ssize_t size; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t out, *cl; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; r->main->count++; if (r != r->main || r->request_body || r->discard_body) { r->request_body_no_buffering = 0; post_handler(r); return NGX_OK; } if (ngx_http_test_expect(r) != NGX_OK) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } if (r->request_body_no_buffering) { r->request_body_in_file_only = 0; } rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); if (rb == NULL) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } /* * set by ngx_pcalloc(): * * rb->bufs = NULL; * rb->buf = NULL; * rb->free = NULL; * rb->busy = NULL; * rb->chunked = NULL; */ rb->rest = -1; rb->post_handler = post_handler; r->request_body = rb; if (r->headers_in.content_length_n < 0 && !r->headers_in.chunked) { r->request_body_no_buffering = 0; post_handler(r); return NGX_OK; } preread = r->header_in->last - r->header_in->pos; if (preread) { /* there is the pre-read part of the request body */ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http client request body preread %uz", preread); out.buf = r->header_in; out.next = NULL; rc = ngx_http_request_body_filter(r, &out); if (rc != NGX_OK) { goto done; } r->request_length += preread - (r->header_in->last - r->header_in->pos); if (!r->headers_in.chunked && rb->rest > 0 && rb->rest <= (off_t) (r->header_in->end - r->header_in->last)) { /* the whole request body may be placed in r->header_in */ b = ngx_calloc_buf(r->pool); if (b == NULL) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } b->temporary = 1; b->start = r->header_in->pos; b->pos = r->header_in->pos; b->last = r->header_in->last; b->end = r->header_in->end; rb->buf = b; r->read_event_handler = ngx_http_read_client_request_body_handler; r->write_event_handler = ngx_http_request_empty_handler; rc = ngx_http_do_read_client_request_body(r); goto done; } } else { /* set rb->rest */ if (ngx_http_request_body_filter(r, NULL) != NGX_OK) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } } if (rb->rest == 0) { /* the whole request body was pre-read */ if (r->request_body_in_file_only) { if (ngx_http_write_request_body(r) != NGX_OK) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } if (rb->temp_file->file.offset != 0) { cl = ngx_chain_get_free_buf(r->pool, &rb->free); if (cl == NULL) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->in_file = 1; b->file_last = rb->temp_file->file.offset; b->file = &rb->temp_file->file; rb->bufs = cl; } else { rb->bufs = NULL; } } r->request_body_no_buffering = 0; post_handler(r); return NGX_OK; } if (rb->rest < 0) { ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, "negative request body rest"); rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); size = clcf->client_body_buffer_size; size += size >> 2; /* TODO: honor r->request_body_in_single_buf */ if (!r->headers_in.chunked && rb->rest < size) { size = (ssize_t) rb->rest; if (r->request_body_in_single_buf) { size += preread; } } else { size = clcf->client_body_buffer_size; } rb->buf = ngx_create_temp_buf(r->pool, size); if (rb->buf == NULL) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } r->read_event_handler = ngx_http_read_client_request_body_handler; r->write_event_handler = ngx_http_request_empty_handler; rc = ngx_http_do_read_client_request_body(r); done: if (r->request_body_no_buffering && (rc == NGX_OK || rc == NGX_AGAIN)) { if (rc == NGX_OK) { r->request_body_no_buffering = 0; } else { /* rc == NGX_AGAIN */ r->reading_body = 1; } r->read_event_handler = ngx_http_block_reading; post_handler(r); } if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { r->main->count--; } return rc; }
// 处理确定长度的请求体数据,参数in是已经读取的数据链表 // 先看free里是否有空闲节点,有则直接使用 // 如果没有,就从内存池的空闲链表里获取 // 创建新的链表节点,加入到out链表里 // 这里只是指针操作,没有内存拷贝 // 调用请求体过滤链表,对数据进行过滤处理 // 实际上是ngx_http_request_body_save_filter // 拷贝in链表里的buf到rb->bufs里,不是直接连接 // 最后把用完的ngx_chaint_t挂到free里供复用,提高效率 static ngx_int_t ngx_http_request_body_length_filter(ngx_http_request_t *r, ngx_chain_t *in) { size_t size; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t *cl, *tl, *out, **ll; ngx_http_request_body_t *rb; // 请求体数据的结构体 rb = r->request_body; // -1表示无效,即还没有开始读取 // 那么剩余字节数就是content_length_n if (rb->rest == -1) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http request body content length filter"); rb->rest = r->headers_in.content_length_n; } out = NULL; ll = &out; // 遍历已经读取的数据链表 // 创建新的链表节点,加入到out链表里 // 这里只是指针操作,没有内存拷贝 for (cl = in; cl; cl = cl->next) { // 已经读完,无剩余字节,那么就结束循环 if (rb->rest == 0) { break; } // 先看free里是否有空闲节点,有则直接使用 // 如果没有,就从内存池的空闲链表里获取 // 最开始rb->free是空的,所以要从内存池里获取 tl = ngx_chain_get_free_buf(r->pool, &rb->free); if (tl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } // 这个缓冲区对象并不持有实际的内存块 b = tl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); // 调整缓冲区的指针,指向链表节点里的地址 b->temporary = 1; b->tag = (ngx_buf_tag_t) &ngx_http_read_client_request_body; b->start = cl->buf->pos; b->pos = cl->buf->pos; b->last = cl->buf->last; b->end = cl->buf->end; b->flush = r->request_body_no_buffering; // 计算此缓冲区里的数据长度 size = cl->buf->last - cl->buf->pos; // 剩余的数据还很多 // 这里处理的是in链表,不是out // 消费的是in链表里的缓冲区 if ((off_t) size < rb->rest) { // 消费缓冲区里的数据 cl->buf->pos = cl->buf->last; // 剩余字节数减少 rb->rest -= size; } else { // 这里rest字节已经读取足够了 // 消费缓冲区里的数据 cl->buf->pos += (size_t) rb->rest; // 剩余数据全部读取完毕,rest=0 rb->rest = 0; // 如果还有多余的数据也不再考虑,调整last b->last = cl->buf->pos; // 标记为最后一块数据,重要 b->last_buf = 1; } // 加入链表 *ll = tl; ll = &tl->next; } // 这里调用请求体过滤链表,对数据进行过滤处理 // 实际上是ngx_http_request_body_save_filter // 从内存池里分配节点 // 拷贝in链表里的buf到rb->bufs里,不是直接连接 // 同样是指针操作,没有内存拷贝 // 如果要求写磁盘文件,那么调用ngx_http_write_request_body rc = ngx_http_top_request_body_filter(r, out); // 用于处理请求体数据,更新free/busy几个链表指针 // 先把out链表挂到busy指针上 // 遍历busy链表 // 缓冲区为空,说明可以复用,应该挂到free链表里 // 把缓冲区复位,都指向start,即完全可用 // 此节点不应该在busy里,从busy链表摘除 // 加入到free链表里,供以后复用 ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, (ngx_buf_tag_t) &ngx_http_read_client_request_body); return rc; }
static ngx_int_t ngx_http_sub_body_filter(ngx_http_request_t *r, ngx_chain_t *in) { ngx_int_t rc; ngx_buf_t *b; ngx_chain_t *cl; ngx_http_sub_ctx_t *ctx; ngx_http_sub_loc_conf_t *slcf; ctx = ngx_http_get_module_ctx(r, ngx_http_sub_filter_module); if (ctx == NULL) { return ngx_http_next_body_filter(r, in); } if ((in == NULL && ctx->buf == NULL && ctx->in == NULL && ctx->busy == NULL)) { return ngx_http_next_body_filter(r, in); } if (ctx->once && (ctx->buf == NULL || ctx->in == NULL)) { if (ctx->busy) { if (ngx_http_sub_output(r, ctx) == NGX_ERROR) { return NGX_ERROR; } } return ngx_http_next_body_filter(r, in); } /* add the incoming chain to the chain ctx->in */ if (in) { if (ngx_chain_add_copy(r->pool, &ctx->in, in) != NGX_OK) { return NGX_ERROR; } } ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http sub filter \"%V\"", &r->uri); while (ctx->in || ctx->buf) { if (ctx->buf == NULL) { ctx->buf = ctx->in->buf; ctx->in = ctx->in->next; ctx->pos = ctx->buf->pos; } if (ctx->state == sub_start_state) { ctx->copy_start = ctx->pos; ctx->copy_end = ctx->pos; } b = NULL; while (ctx->pos < ctx->buf->last) { ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "saved: \"%V\" state: %d", &ctx->saved, ctx->state); rc = ngx_http_sub_parse(r, ctx); ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "parse: %d, looked: \"%V\" %p-%p", rc, &ctx->looked, ctx->copy_start, ctx->copy_end); if (rc == NGX_ERROR) { return rc; } if (ctx->saved.len) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "saved: \"%V\"", &ctx->saved); cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->pos = ngx_pnalloc(r->pool, ctx->saved.len); if (b->pos == NULL) { return NGX_ERROR; } ngx_memcpy(b->pos, ctx->saved.data, ctx->saved.len); b->last = b->pos + ctx->saved.len; b->memory = 1; *ctx->last_out = cl; ctx->last_out = &cl->next; ctx->saved.len = 0; } if (ctx->copy_start != ctx->copy_end) { cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; ngx_memcpy(b, ctx->buf, sizeof(ngx_buf_t)); b->pos = ctx->copy_start; b->last = ctx->copy_end; b->shadow = NULL; b->last_buf = 0; b->last_in_chain = 0; b->recycled = 0; if (b->in_file) { b->file_last = b->file_pos + (b->last - ctx->buf->pos); b->file_pos += b->pos - ctx->buf->pos; } *ctx->last_out = cl; ctx->last_out = &cl->next; } if (ctx->state == sub_start_state) { ctx->copy_start = ctx->pos; ctx->copy_end = ctx->pos; } else { ctx->copy_start = NULL; ctx->copy_end = NULL; } if (ctx->looked.len > (size_t) (ctx->pos - ctx->buf->pos)) { ctx->saved.len = ctx->looked.len - (ctx->pos - ctx->buf->pos); ngx_memcpy(ctx->saved.data, ctx->looked.data, ctx->saved.len); } if (rc == NGX_AGAIN) { continue; } /* rc == NGX_OK */ cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); slcf = ngx_http_get_module_loc_conf(r, ngx_http_sub_filter_module); if (ctx->sub.data == NULL) { if (ngx_http_complex_value(r, &slcf->value, &ctx->sub) != NGX_OK) { return NGX_ERROR; } } if (ctx->sub.len) { b->memory = 1; b->pos = ctx->sub.data; b->last = ctx->sub.data + ctx->sub.len; } else { b->sync = 1; } *ctx->last_out = cl; ctx->last_out = &cl->next; ctx->once = slcf->once; continue; } if (ctx->looked.len && (ctx->buf->last_buf || ctx->buf->last_in_chain)) { cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->pos = ctx->looked.data; b->last = b->pos + ctx->looked.len; b->memory = 1; *ctx->last_out = cl; ctx->last_out = &cl->next; ctx->looked.len = 0; } if (ctx->buf->last_buf || ctx->buf->flush || ctx->buf->sync || ngx_buf_in_memory(ctx->buf)) { if (b == NULL) { cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->sync = 1; *ctx->last_out = cl; ctx->last_out = &cl->next; } b->last_buf = ctx->buf->last_buf; b->last_in_chain = ctx->buf->last_in_chain; b->flush = ctx->buf->flush; b->shadow = ctx->buf; b->recycled = ctx->buf->recycled; } ctx->buf = NULL; ctx->saved.len = ctx->looked.len; ngx_memcpy(ctx->saved.data, ctx->looked.data, ctx->looked.len); } if (ctx->out == NULL && ctx->busy == NULL) { return NGX_OK; } return ngx_http_sub_output(r, ctx); }
// 参数in实际上是ngx_http_request_body_length_filter里的out,即读取到的数据 // 从内存池里分配节点 // 拷贝in链表里的buf到rb->bufs里,不是直接连接 // 同样是指针操作,没有内存拷贝 // 如果要求写磁盘文件,那么调用ngx_http_write_request_body ngx_int_t ngx_http_request_body_save_filter(ngx_http_request_t *r, ngx_chain_t *in) { ngx_buf_t *b; ngx_chain_t *cl; ngx_http_request_body_t *rb; // 请求体数据的结构体 rb = r->request_body; #if (NGX_DEBUG) for (cl = rb->bufs; cl; cl = cl->next) { ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, "http body old buf t:%d f:%d %p, pos %p, size: %z " "file: %O, size: %O", cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, cl->buf->last - cl->buf->pos, cl->buf->file_pos, cl->buf->file_last - cl->buf->file_pos); } for (cl = in; cl; cl = cl->next) { ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, "http body new buf t:%d f:%d %p, pos %p, size: %z " "file: %O, size: %O", cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, cl->buf->last - cl->buf->pos, cl->buf->file_pos, cl->buf->file_last - cl->buf->file_pos); } #endif /* TODO: coalesce neighbouring buffers */ // 从内存池里分配节点 // 拷贝in链表里的buf到rb->bufs里,不是直接连接 // 同样是指针操作,没有内存拷贝 if (ngx_chain_add_copy(r->pool, &rb->bufs, in) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } if (r->request_body_no_buffering) { return NGX_OK; } if (rb->rest > 0) { if (rb->buf && rb->buf->last == rb->buf->end && ngx_http_write_request_body(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } return NGX_OK; } /* rb->rest == 0 */ // 如果要求写磁盘文件,那么调用ngx_http_write_request_body if (rb->temp_file || r->request_body_in_file_only) { if (ngx_http_write_request_body(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } if (rb->temp_file->file.offset != 0) { cl = ngx_chain_get_free_buf(r->pool, &rb->free); if (cl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->in_file = 1; b->file_last = rb->temp_file->file.offset; b->file = &rb->temp_file->file; rb->bufs = cl; } } return NGX_OK; }
static ngx_int_t ngx_http_parallel_start_fiber( ngx_http_request_t *r, ngx_http_parallel_fiber_ctx_t* fiber, uint64_t chunk_index) { ngx_http_parallel_loc_conf_t *conf; ngx_http_parallel_ctx_t *ctx; ngx_http_request_t* sr; ngx_str_t args = ngx_null_string; ngx_table_elt_t *h; ngx_int_t rc; size_t alloc_size; off_t start_offset; off_t end_offset; ngx_chain_t* cl; ngx_buf_t* b; ctx = ngx_http_get_module_ctx(r, ngx_http_parallel_module); conf = ngx_http_get_module_loc_conf(r, ngx_http_parallel_module); // get a buffer for the response cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "ngx_http_parallel_start_fiber: ngx_chain_get_free_buf failed"); return NGX_ERROR; } b = cl->buf; if (cl->buf->start == NULL) { alloc_size = conf->max_headers_size + conf->max_chunk_size; b->start = ngx_palloc(r->pool, alloc_size); if (b->start == NULL) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "ngx_http_parallel_start_fiber: ngx_palloc failed"); return NGX_ERROR; } b->pos = b->start; b->last = b->start; b->end = b->last + alloc_size; b->temporary = 1; b->tag = (ngx_buf_tag_t)&ngx_http_parallel_module; ctx->allocated_count++; } // allocate a list for the input headers if (ngx_list_init(&fiber->upstream_headers, r->pool, 8, sizeof(ngx_table_elt_t)) != NGX_OK) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "ngx_http_parallel_start_fiber: ngx_list_init failed"); return NGX_ERROR; } // set the range if (ctx->fiber_count != 1) { start_offset = ctx->range.start; if (chunk_index < ctx->fiber_count) { start_offset += chunk_index * ctx->chunk_size; } else { start_offset += ctx->initial_requested_size + (chunk_index - ctx->fiber_count) * ctx->chunk_size; } end_offset = start_offset + ctx->chunk_size; if (ctx->range.end != 0 && ctx->range.end < end_offset) { end_offset = ctx->range.end; } h = fiber->headers_in.range; h->value.len = ngx_sprintf( h->value.data, RANGE_FORMAT, start_offset, end_offset - 1) - h->value.data; h->value.data[h->value.len] = '\0'; } // start the request r->headers_in = fiber->headers_in; rc = ngx_http_subrequest(r, &ctx->sr_uri, &args, &sr, fiber->psr, NGX_HTTP_SUBREQUEST_WAITED | NGX_HTTP_SUBREQUEST_IN_MEMORY); r->headers_in = ctx->original_headers_in; if (rc != NGX_OK) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "ngx_http_parallel_start_fiber: ngx_http_subrequest failed %i", rc); return rc; } ngx_http_set_ctx(sr, fiber, ngx_http_parallel_module); ctx->active_fibers++; sr->write_event_handler = ngx_http_parallel_fiber_initial_wev_handler; sr->method = r->method; // copy the method to the subrequest sr->method_name = r->method_name; // (ngx_http_subrequest always uses GET) sr->header_in = r->header_in; // fix the last pointer in headers_in (from echo-nginx-module) if (fiber->headers_in.headers.last == &fiber->headers_in.headers.part) { sr->headers_in.headers.last = &sr->headers_in.headers.part; } fiber->chunk_index = chunk_index; fiber->cl = cl; fiber->sr = sr; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "ngx_http_parallel_start_fiber: started fiber %uL", chunk_index); return NGX_OK; }
// the implementation is based on ngx_http_sub_body_filter static ngx_int_t ngx_http_secure_token_body_filter(ngx_http_request_t *r, ngx_chain_t *in) { ngx_http_secure_token_processor_output_t output; ngx_http_secure_token_loc_conf_t *conf; ngx_http_secure_token_ctx_t* ctx; ngx_chain_t* in_copy = NULL; ngx_chain_t* cl; ngx_buf_t* buf; ngx_buf_t* b; ngx_int_t rc; u_char* copy_start; u_char* pos = NULL; ctx = ngx_http_get_module_ctx(r, ngx_http_secure_token_filter_module); if (ctx == NULL) { return ngx_http_next_body_filter(r, in); } if ((in == NULL && ctx->busy == NULL)) { return ngx_http_next_body_filter(r, in); } conf = ngx_http_get_module_loc_conf(r, ngx_http_secure_token_filter_module); /* add the incoming chain to the chain in_copy */ if (in) { if (ngx_chain_add_copy(r->pool, &in_copy, in) != NGX_OK) { return NGX_ERROR; } } ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http secure token filter \"%V\"", &r->uri); buf = NULL; while (in_copy || buf) { if (buf == NULL) { buf = in_copy->buf; in_copy = in_copy->next; pos = buf->pos; } b = NULL; while (pos < buf->last) { copy_start = pos; output.copy_input = 1; output.output_buffer.len = 0; output.token_index = -1; rc = ctx->process( r, &conf->processor_conf, ctx->processor_params, &pos, buf->last, (u_char*)ctx + ctx->processor_context_offset, &output); ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "process: %d, %p-%p", rc, copy_start, pos); if (rc == NGX_ERROR) { return rc; } if (output.copy_input && copy_start != pos) { cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; ngx_memcpy(b, buf, sizeof(ngx_buf_t)); b->pos = copy_start; b->last = pos; b->shadow = NULL; b->last_buf = 0; b->last_in_chain = 0; b->recycled = 0; if (b->in_file) { b->file_last = b->file_pos + (b->last - buf->pos); b->file_pos += b->pos - buf->pos; } *ctx->last_out = cl; ctx->last_out = &cl->next; } if (output.output_buffer.len != 0) { cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->memory = 1; b->pos = output.output_buffer.data; b->last = output.output_buffer.data + output.output_buffer.len; *ctx->last_out = cl; ctx->last_out = &cl->next; } if (output.token_index >= 0 && ctx->token.len != 0) { cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; rc = ngx_http_secure_token_get_token_buffer(r, ctx, b, output.token_index); if (rc != NGX_OK) { return rc; } *ctx->last_out = cl; ctx->last_out = &cl->next; } continue; } if (buf->last_buf || buf->flush || buf->sync || ngx_buf_in_memory(buf)) { if (b == NULL) { cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->sync = 1; *ctx->last_out = cl; ctx->last_out = &cl->next; } b->last_buf = buf->last_buf; b->last_in_chain = buf->last_in_chain; b->flush = buf->flush; b->shadow = buf; b->recycled = buf->recycled; } buf = NULL; } if (ctx->out == NULL && ctx->busy == NULL) { return NGX_OK; } return ngx_http_secure_token_output(r, ctx); }
static ngx_http_spdy_out_frame_t * ngx_http_spdy_filter_get_data_frame(ngx_http_spdy_stream_t *stream, size_t len, ngx_chain_t *first, ngx_chain_t *last) { u_char *p; ngx_buf_t *buf; ngx_uint_t flags; ngx_chain_t *cl; ngx_http_spdy_out_frame_t *frame; frame = stream->free_frames; if (frame) { stream->free_frames = frame->next; } else { frame = ngx_palloc(stream->request->pool, sizeof(ngx_http_spdy_out_frame_t)); if (frame == NULL) { return NULL; } } flags = last->buf->last_buf ? NGX_SPDY_FLAG_FIN : 0; ngx_log_debug4(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, "spdy:%ui create DATA frame %p: len:%uz flags:%ui", stream->id, frame, len, flags); cl = ngx_chain_get_free_buf(stream->request->pool, &stream->free_data_headers); if (cl == NULL) { return NULL; } buf = cl->buf; if (buf->start) { p = buf->start; buf->pos = p; p += NGX_SPDY_SID_SIZE; (void) ngx_spdy_frame_write_flags_and_len(p, flags, len); } else { p = ngx_palloc(stream->request->pool, NGX_SPDY_FRAME_HEADER_SIZE); if (p == NULL) { return NULL; } buf->pos = p; buf->start = p; p = ngx_spdy_frame_write_sid(p, stream->id); p = ngx_spdy_frame_write_flags_and_len(p, flags, len); buf->last = p; buf->end = p; buf->tag = (ngx_buf_tag_t) &ngx_http_spdy_filter_get_data_frame; buf->memory = 1; } cl->next = first; first = cl; last->buf->flush = 1; frame->first = first; frame->last = last; frame->handler = ngx_http_spdy_data_frame_handler; frame->stream = stream; frame->length = len; frame->priority = stream->priority; frame->blocked = 0; frame->fin = last->buf->last_buf; return frame; }
static ngx_http_v2_out_frame_t * ngx_http_v2_filter_get_data_frame(ngx_http_v2_stream_t *stream, size_t len, ngx_chain_t *first, ngx_chain_t *last) { u_char flags; ngx_buf_t *buf; ngx_chain_t *cl; ngx_http_v2_out_frame_t *frame; frame = stream->free_frames; if (frame) { stream->free_frames = frame->next; } else { frame = ngx_palloc(stream->request->pool, sizeof(ngx_http_v2_out_frame_t)); if (frame == NULL) { return NULL; } } flags = last->buf->last_buf ? NGX_HTTP_V2_END_STREAM_FLAG : 0; ngx_log_debug4(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, "http2:%ui create DATA frame %p: len:%uz flags:%ui", stream->node->id, frame, len, (ngx_uint_t) flags); cl = ngx_chain_get_free_buf(stream->request->pool, &stream->free_frame_headers); if (cl == NULL) { return NULL; } buf = cl->buf; if (!buf->start) { buf->start = ngx_palloc(stream->request->pool, NGX_HTTP_V2_FRAME_HEADER_SIZE); if (buf->start == NULL) { return NULL; } buf->end = buf->start + NGX_HTTP_V2_FRAME_HEADER_SIZE; buf->last = buf->end; buf->tag = (ngx_buf_tag_t) &ngx_http_v2_module; buf->memory = 1; } buf->pos = buf->start; buf->last = buf->pos; buf->last = ngx_http_v2_write_len_and_type(buf->last, len, NGX_HTTP_V2_DATA_FRAME); *buf->last++ = flags; buf->last = ngx_http_v2_write_sid(buf->last, stream->node->id); cl->next = first; first = cl; last->buf->flush = 1; frame->first = first; frame->last = last; frame->handler = ngx_http_v2_data_frame_handler; frame->stream = stream; frame->length = len; frame->blocked = 0; frame->fin = last->buf->last_buf; return frame; }
static ngx_int_t ngx_event_pipe_write_chain_to_temp_file(ngx_event_pipe_t *p) { ssize_t size, bsize, n; ngx_buf_t *b; ngx_uint_t prev_last_shadow; ngx_chain_t *cl, *tl, *next, *out, **ll, **last_out, **last_free, fl; if (p->buf_to_file) { fl.buf = p->buf_to_file; fl.next = p->in; out = &fl; } else { out = p->in; } if (!p->cacheable) { size = 0; cl = out; ll = NULL; prev_last_shadow = 1; ngx_log_debug1(NGX_LOG_DEBUG_EVENT, p->log, 0, "pipe offset: %O", p->temp_file->offset); do { bsize = cl->buf->last - cl->buf->pos; ngx_log_debug4(NGX_LOG_DEBUG_EVENT, p->log, 0, "pipe buf ls:%d %p, pos %p, size: %z", cl->buf->last_shadow, cl->buf->start, cl->buf->pos, bsize); if (prev_last_shadow && ((size + bsize > p->temp_file_write_size) || (p->temp_file->offset + size + bsize > p->max_temp_file_size))) { break; } prev_last_shadow = cl->buf->last_shadow; size += bsize; ll = &cl->next; cl = cl->next; } while (cl); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, p->log, 0, "size: %z", size); if (ll == NULL) { return NGX_BUSY; } if (cl) { p->in = cl; *ll = NULL; } else { p->in = NULL; p->last_in = &p->in; } } else { p->in = NULL; p->last_in = &p->in; } n = ngx_write_chain_to_temp_file(p->temp_file, out); if (n == NGX_ERROR) { return NGX_ABORT; } if (p->buf_to_file) { p->temp_file->offset = p->buf_to_file->last - p->buf_to_file->pos; n -= p->buf_to_file->last - p->buf_to_file->pos; p->buf_to_file = NULL; out = out->next; } if (n > 0) { /* update previous buffer or add new buffer */ if (p->out) { for (cl = p->out; cl->next; cl = cl->next) { /* void */ } b = cl->buf; if (b->file_last == p->temp_file->offset) { p->temp_file->offset += n; b->file_last = p->temp_file->offset; goto free; } last_out = &cl->next; } else { last_out = &p->out; } cl = ngx_chain_get_free_buf(p->pool, &p->free); if (cl == NULL) { return NGX_ABORT; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->tag = p->tag; b->file = &p->temp_file->file; b->file_pos = p->temp_file->offset; p->temp_file->offset += n; b->file_last = p->temp_file->offset; b->in_file = 1; b->temp_file = 1; *last_out = cl; } free: for (last_free = &p->free_raw_bufs; *last_free != NULL; last_free = &(*last_free)->next) { /* void */ } for (cl = out; cl; cl = next) { next = cl->next; cl->next = p->free; p->free = cl; b = cl->buf; if (b->last_shadow) { tl = ngx_alloc_chain_link(p->pool); if (tl == NULL) { return NGX_ABORT; } tl->buf = b->shadow; tl->next = NULL; *last_free = tl; last_free = &tl->next; b->shadow->pos = b->shadow->start; b->shadow->last = b->shadow->start; ngx_event_pipe_remove_shadow_links(b->shadow); } } return NGX_OK; }
static ngx_int_t ngx_http_do_read_client_request_body(ngx_http_request_t *r) { off_t rest; size_t size; ssize_t n; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t *cl, out; ngx_connection_t *c; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; c = r->connection; rb = r->request_body; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http read client request body"); for ( ;; ) { for ( ;; ) { if (rb->buf->last == rb->buf->end) { if (rb->buf->pos != rb->buf->last) { /* pass buffer to request body filter chain */ out.buf = rb->buf; out.next = NULL; rc = ngx_http_request_body_filter(r, &out); if (rc != NGX_OK) { return rc; } } else { /* update chains */ rc = ngx_http_request_body_filter(r, NULL); if (rc != NGX_OK) { return rc; } } if (rb->busy != NULL) { if (r->request_body_no_buffering) { if (c->read->timer_set) { ngx_del_timer(c->read); } if (ngx_handle_read_event(c->read, 0) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } return NGX_AGAIN; } return NGX_HTTP_INTERNAL_SERVER_ERROR; } rb->buf->pos = rb->buf->start; rb->buf->last = rb->buf->start; } size = rb->buf->end - rb->buf->last; rest = rb->rest - (rb->buf->last - rb->buf->pos); if ((off_t) size > rest) { size = (size_t) rest; } n = c->recv(c, rb->buf->last, size); ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "http client request body recv %z", n); if (n == NGX_AGAIN) { break; } if (n == 0) { ngx_log_error(NGX_LOG_INFO, c->log, 0, "client prematurely closed connection"); } if (n == 0 || n == NGX_ERROR) { c->error = 1; return NGX_HTTP_BAD_REQUEST; } rb->buf->last += n; r->request_length += n; if (n == rest) { /* pass buffer to request body filter chain */ out.buf = rb->buf; out.next = NULL; rc = ngx_http_request_body_filter(r, &out); if (rc != NGX_OK) { return rc; } } if (rb->rest == 0) { break; } if (rb->buf->last < rb->buf->end) { break; } } ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "http client request body rest %O", rb->rest); if (rb->rest == 0) { break; } if (!c->read->ready) { if (r->request_body_no_buffering && rb->buf->pos != rb->buf->last) { /* pass buffer to request body filter chain */ out.buf = rb->buf; out.next = NULL; rc = ngx_http_request_body_filter(r, &out); if (rc != NGX_OK) { return rc; } } clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ngx_add_timer(c->read, clcf->client_body_timeout); if (ngx_handle_read_event(c->read, 0) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } return NGX_AGAIN; } } if (c->read->timer_set) { ngx_del_timer(c->read); } if (rb->temp_file || r->request_body_in_file_only) { /* save the last part */ if (ngx_http_write_request_body(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } if (rb->temp_file->file.offset != 0) { cl = ngx_chain_get_free_buf(r->pool, &rb->free); if (cl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b = cl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->in_file = 1; b->file_last = rb->temp_file->file.offset; b->file = &rb->temp_file->file; rb->bufs = cl; } else { rb->bufs = NULL; } } if (!r->request_body_no_buffering) { r->read_event_handler = ngx_http_block_reading; rb->post_handler(r); } return NGX_OK; }
static ngx_http_spdy_out_frame_t * ngx_http_spdy_filter_get_data_frame(ngx_http_spdy_stream_t *stream, size_t len, ngx_uint_t fin, ngx_chain_t *first, ngx_chain_t *last) { u_char *p; ngx_buf_t *buf; ngx_uint_t flags; ngx_chain_t *cl; ngx_http_spdy_out_frame_t *frame; frame = stream->free_frames; if (frame) { stream->free_frames = frame->free; } else { frame = ngx_palloc(stream->request->pool, sizeof(ngx_http_spdy_out_frame_t)); if (frame == NULL) { return NULL; } } ngx_log_debug4(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, "Blaha spdy:%ui create DATA frame %p: len:%uz fin:%ui", stream->id, frame, len, fin); if (len || fin) { flags = fin ? NGX_SPDY_FLAG_FIN : 0; cl = ngx_chain_get_free_buf(stream->request->pool, &stream->free_data_headers); if (cl == NULL) { return NULL; } buf = cl->buf; if (buf->start) { p = buf->start; buf->pos = p; p += sizeof(uint32_t); ngx_log_debug1(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, "sai ram IFF:%ui DATA frame", stream->id); (void) ngx_spdy_frame_write_flags_and_len(p, flags, len); } else { p = ngx_palloc(stream->request->pool, NGX_SPDY_FRAME_HEADER_SIZE); if (p == NULL) { return NULL; } buf->pos = p; buf->start = p; p = ngx_spdy_frame_write_sid(p, stream->id); p = ngx_spdy_frame_write_flags_and_len(p, flags, len); buf->last = p; buf->end = p; buf->tag = (ngx_buf_tag_t) &ngx_http_spdy_serverpush_filter_module; buf->memory = 1; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, "sai ram ELSEE:%ui DATA frame", stream->id); if(buf->pos == buf->last) ngx_log_debug0(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, "EQUAL !!!!!!"); } cl->next = first; first = cl; } frame->first = first; frame->last = last; frame->handler = ngx_http_spdy_data_frame_handler; frame->free = NULL; frame->stream = stream; frame->size = NGX_SPDY_FRAME_HEADER_SIZE + len; frame->priority = stream->priority; frame->blocked = 0; frame->fin = fin; //ngx_chain_t *cl1; //cl = frame->first; //frame->first->buf->pos = frame->first->buf->last; if (frame->first->buf->pos != frame->first->buf->last) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, stream->request->connection->log, 0, "sai ram spdy:%ui DATA frame", stream->id); } return frame; }
static ngx_int_t ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in) { size_t size; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t *cl, *out, *tl, **ll; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; rb = r->request_body; if (rb->rest == -1) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http request body chunked filter"); rb->chunked = ngx_pcalloc(r->pool, sizeof(ngx_http_chunked_t)); if (rb->chunked == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } r->headers_in.content_length_n = 0; rb->rest = 3; } out = NULL; ll = &out; for (cl = in; cl; cl = cl->next) { for ( ;; ) { ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, "http body chunked buf " "t:%d f:%d %p, pos %p, size: %z file: %O, size: %O", cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, cl->buf->last - cl->buf->pos, cl->buf->file_pos, cl->buf->file_last - cl->buf->file_pos); rc = ngx_http_parse_chunked(r, cl->buf, rb->chunked); if (rc == NGX_OK) { /* a chunk has been parsed successfully */ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); if (clcf->client_max_body_size && clcf->client_max_body_size - r->headers_in.content_length_n < rb->chunked->size) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "client intended to send too large chunked " "body: %O+%O bytes", r->headers_in.content_length_n, rb->chunked->size); r->lingering_close = 1; return NGX_HTTP_REQUEST_ENTITY_TOO_LARGE; } tl = ngx_chain_get_free_buf(r->pool, &rb->free); if (tl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b = tl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->temporary = 1; b->tag = (ngx_buf_tag_t) &ngx_http_read_client_request_body; b->start = cl->buf->pos; b->pos = cl->buf->pos; b->last = cl->buf->last; b->end = cl->buf->end; b->flush = r->request_body_no_buffering; *ll = tl; ll = &tl->next; size = cl->buf->last - cl->buf->pos; if ((off_t) size > rb->chunked->size) { cl->buf->pos += (size_t) rb->chunked->size; r->headers_in.content_length_n += rb->chunked->size; rb->chunked->size = 0; } else { rb->chunked->size -= size; r->headers_in.content_length_n += size; cl->buf->pos = cl->buf->last; } b->last = cl->buf->pos; continue; } if (rc == NGX_DONE) { /* a whole response has been parsed successfully */ rb->rest = 0; tl = ngx_chain_get_free_buf(r->pool, &rb->free); if (tl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b = tl->buf; ngx_memzero(b, sizeof(ngx_buf_t)); b->last_buf = 1; *ll = tl; ll = &tl->next; break; } if (rc == NGX_AGAIN) { /* set rb->rest, amount of data we want to see next time */ rb->rest = rb->chunked->length; break; } /* invalid */ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "client sent invalid chunked body"); return NGX_HTTP_BAD_REQUEST; } } rc = ngx_http_top_request_body_filter(r, out); ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, (ngx_buf_tag_t) &ngx_http_read_client_request_body); return rc; }
ngx_int_t ngx_http_lua_add_copy_chain(ngx_http_request_t *r, ngx_http_lua_ctx_t *ctx, ngx_chain_t **chain, ngx_chain_t *in) { ngx_chain_t *cl, **ll; size_t len; ngx_buf_t *b; ll = chain; for (cl = *chain; cl; cl = cl->next) { ll = &cl->next; } len = 0; for (cl = in; cl; cl = cl->next) { if (ngx_buf_in_memory(cl->buf)) { len += cl->buf->last - cl->buf->pos; } } if (len == 0) { return NGX_OK; } cl = ngx_chain_get_free_buf(r->pool, &ctx->free); if (cl == NULL) { return NGX_ERROR; } b = cl->buf; b->start = ngx_palloc(r->pool, len); if (b->start == NULL) { return NGX_ERROR; } b->end = b->start + len; b->pos = b->start; b->last = b->pos; b->memory = 1; #if 0 b->tag = (ngx_buf_tag_t) &ngx_http_lua_module; #endif while (in) { if (ngx_buf_in_memory(in->buf)) { b->last = ngx_copy(b->last, in->buf->pos, in->buf->last - in->buf->pos); } in = in->next; } *ll = cl; cl->next = NULL; return NGX_OK; }
static ngx_int_t ngx_http_redis_filter(void *data, ssize_t bytes) { ngx_http_redis_ctx_t *ctx = data; u_char *last; ngx_buf_t *b; ngx_chain_t *cl, **ll; ngx_http_upstream_t *u; u = ctx->request->upstream; b = &u->buffer; if (u->length == ctx->rest) { if (ngx_strncmp(b->last, ngx_http_redis_end + NGX_HTTP_REDIS_END - ctx->rest, ctx->rest) != 0) { ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0, "redis sent invalid trailer"); } u->length = 0; ctx->rest = 0; return NGX_OK; } for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { ll = &cl->next; } cl = ngx_chain_get_free_buf(ctx->request->pool, &u->free_bufs); if (cl == NULL) { return NGX_ERROR; } cl->buf->flush = 1; cl->buf->memory = 1; *ll = cl; last = b->last; cl->buf->pos = last; b->last += bytes; cl->buf->last = b->last; cl->buf->tag = u->output.tag; ngx_log_debug4(NGX_LOG_DEBUG_HTTP, ctx->request->connection->log, 0, "redis filter bytes:%z size:%z length:%z rest:%z", bytes, b->last - b->pos, u->length, ctx->rest); if (bytes <= (ssize_t) (u->length - NGX_HTTP_REDIS_END)) { u->length -= bytes; return NGX_OK; } last += u->length - NGX_HTTP_REDIS_END; if (ngx_strncmp(last, ngx_http_redis_end, b->last - last) != 0) { ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0, "redis sent invalid trailer"); } ctx->rest -= b->last - last; b->last = last; cl->buf->last = last; u->length = ctx->rest; return NGX_OK; }
static ngx_int_t ngx_http_request_body_length_filter(ngx_http_request_t *r, ngx_chain_t *in) { //in其实也是从r->request_body->buf中来的 size_t size; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t *cl, *tl, *out, **ll; ngx_http_request_body_t *rb; rb = r->request_body; if (rb->rest == -1) {//第一次执行该函数 rest 设置为请求头的 content-length ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http request body content length filter"); rb->rest = r->headers_in.content_length_n; } out = NULL; ll = &out; //把in中的所有数据节点数据连接在一起添加到out头中 for (cl = in; cl; cl = cl->next) {//遍历r->request_body中的所有buf if (rb->rest == 0) {//表示包体数据已经处理完毕 break; } tl = ngx_chain_get_free_buf(r->pool, &rb->free); //从free链表中poll中获取ngx_chain_t空间,如果free中为空,则直接创建 if (tl == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b = tl->buf; //获取tl中的buf ngx_memzero(b, sizeof(ngx_buf_t)); //b的相关成员指针指向in中的各个节点里面的对于成员,即新的b指向获取到的包体数据中的各个ngx_buf_t //b指向的数据和cl->buf指向的数据时一致的,最后实际读取到的数据的头尾用b指向 b->temporary = 1; b->tag = (ngx_buf_tag_t) &ngx_http_read_client_request_body; b->start = cl->buf->pos; b->pos = cl->buf->pos; b->last = cl->buf->last; b->end = cl->buf->end; b->flush = r->request_body_no_buffering; size = cl->buf->last - cl->buf->pos; if ((off_t) size < rb->rest) { //说明数据还不够r->headers_in.content_length_n; cl->buf->pos = cl->buf->last; rb->rest -= size; //已经获取到size了,还差rb->rest才到content_length_n } else { //说明已经获取到了content_length_n这么多包体,也就是包体已经够了 cl->buf->pos += (size_t) rb->rest; //注意这时候的last没有移动,如果头部行content-length:len中的len小于实际携带的包体数据,就会造成pos小于last rb->rest = 0;//表示包体有这么多了 b->last = cl->buf->pos; //实际读到的数据比我们期望的rest数据多,因此我们截取实际需要的数据即可 b->last_buf = 1; //标记该buf是组成包体数据的buf数组中的最后一个ngx_buf_t } *ll = tl; ll = &tl->next; //前面创建的所有tl(ngx_chain_t)通过next连接在一起,所有这些节点的头部是前面的out } //把in表中的数据(通过从新创建ngx_buf_t指向in表中各个数据的成员)连接到r->request_body->bufs中,这样所有的out数据都会添加到r->request_body->bufs数组中 //通过新创建的ngx_chain_t(之前out中的ngx_chain_t就通过下面的ngx_chain_update_chains进行回收)中的各个指针来执行新读取到的out数据,ngx_http_request_body_save_filter,通过该函数后所有的out数据都连接到rb->bufs中缓存了 rc = ngx_http_top_request_body_filter(r, out); //ngx_http_request_body_save_filter //rb中的bufs链表中的成员中的各个指针指向读取到的原始数据位置(如pos指向读取到数据的头,last指向读取到数据的尾部) //rb->busy最初是直接从out中拷贝的,指向的数据空间最初与bufs是一样的,但是一旦http模块从网络读取到的数据中解析出一部分数据,那么free中成员的各个指针就会移动,例如pos会 //向last方向移动,直到pos=list,这时候busy中的这个ngx_buf_t节点成员就可以从busy链表中取出,然后添加到free链表中 ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, (ngx_buf_tag_t) &ngx_http_read_client_request_body); return rc; }