bool proxy_connection::read_chunked_body(unsigned fd, size_t& total) { io_result res; do { if ((res = read(fd, _M_client->_M_body, total)) == IO_ERROR) { _M_client->_M_error = http_error::GATEWAY_TIMEOUT; return false; } else if (res == IO_NO_DATA_READ) { return true; } _M_client->_M_timestamp = now::_M_time; switch (parse_chunk(_M_client->_M_body.data(), _M_client->_M_body.count())) { case chunked_parser::INVALID_CHUNKED_RESPONSE: _M_client->_M_error = http_error::BAD_GATEWAY; return false; case chunked_parser::CALLBACK_FAILED: _M_client->_M_error = http_error::INTERNAL_SERVER_ERROR; return false; case chunked_parser::NOT_END_OF_RESPONSE: _M_client->_M_body.reset(); break; default: _M_state = RESPONSE_COMPLETED_STATE; return true; } } while (res == IO_SUCCESS); return true; }
int hrepack_addchunk(const char* str, options_t *options) { obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */ int n_objs; /*number of objects in the current -t or -c option entry */ int32 chunk_lengths[H4_MAX_VAR_DIMS]; /* chunk lengths along each dimension */ int chunk_rank; /*global rank for chunks */ int i, j; if (options->all_chunk==1){ printf("Error: Invalid chunking input: '*' is present with other objects <%s>\n",str); return FAIL; } /* parse the -c option */ if ((obj_list = parse_chunk(str,&n_objs,chunk_lengths,&chunk_rank)) == NULL) return FAIL; /* searh for the "*" all objects character */ for (i = 0; i < n_objs; i++) { if (HDstrcmp("*",obj_list[i].obj)==0) { /* if we are chunking all set the global chunking type */ options->all_chunk=1; options->chunk_g.rank=chunk_rank; for (j = 0; j < chunk_rank; j++) options->chunk_g.chunk_lengths[j] = chunk_lengths[j]; } } if (i>1) { printf("\nError: '*' cannot be with other objects, <%s>. Exiting...\n",str); goto out; } if (options->all_chunk==0) { if (options_add_chunk(obj_list,n_objs,chunk_lengths,chunk_rank,options->op_tbl)<0) goto out; } HDfree(obj_list); return SUCCEED; out: HDfree(obj_list); return FAIL; }
static lcb_error_t parse_body(struct htvb_st *vbs, int *done) { lcb_error_t err = LCB_BUSY; char *term; if ((err = parse_chunk(vbs)) != LCB_SUCCESS) { *done = 1; /* no data */ lcb_assert(err == LCB_BUSY); return err; } if (lcb_string_append(&vbs->input, vbs->chunk.base, vbs->chunk_size)) { return LCB_CLIENT_ENOMEM; } lcb_string_erase_end(&vbs->input, 2); lcb_string_erase_beginning(&vbs->chunk, vbs->chunk_size); vbs->chunk_size = (lcb_size_t) - 1; if (vbs->chunk.nused > 0) { *done = 0; } term = strstr(vbs->input.base, "\n\n\n\n"); if (term != NULL) { lcb_string tmp; lcb_error_t ret; /** Next input */ lcb_string_init(&tmp); lcb_string_appendz(&tmp, term + 4); *term = '\0'; ret = set_next_config(vbs); /** Now, erase everything until the end of the 'term' */ if (vbs->input.base) { lcb_string_release(&vbs->input); } lcb_string_transfer(&tmp, &vbs->input); return ret; } return err; }
bool http_connection::read_chunked_body(unsigned fd, size_t& total) { io_result res; do { if ((res = read(fd, total)) == IO_ERROR) { return false; } else if (res == IO_NO_DATA_READ) { return true; } if (_M_rule->handler == rulelist::HTTP_HANDLER) { size_t size; switch (parse_chunk(_M_in.data(), _M_in.count(), size)) { case chunked_parser::INVALID_CHUNKED_RESPONSE: _M_error = http_error::BAD_REQUEST; _M_state = PREPARING_ERROR_PAGE_STATE; return true; case chunked_parser::CALLBACK_FAILED: _M_error = http_error::INTERNAL_SERVER_ERROR; _M_state = PREPARING_ERROR_PAGE_STATE; return true; case chunked_parser::END_OF_RESPONSE: _M_inp += size; _M_state = PREPARING_HTTP_REQUEST_STATE; return true; default: ; } } _M_in.reset(); } while (res == IO_SUCCESS); return true; }
bool http_connection::process_non_local_handler(unsigned fd) { if ((_M_method == http_method::GET) || (_M_method == http_method::HEAD)) { _M_request_body_size = 0; _M_payload_in_memory = 1; _M_state = PREPARING_HTTP_REQUEST_STATE; _M_error = http_error::OK; return true; } else if ((_M_method != http_method::POST) && (_M_method != http_method::PUT)) { _M_error = http_error::NOT_IMPLEMENTED; return true; } size_t count = _M_in.count() - _M_request_header_size; // Transfer-Encoding: chunked? const char* value; unsigned short valuelen; bool chunked; if ((_M_headers.get_value_known_header(http_headers::TRANSFER_ENCODING_HEADER, value, &valuelen)) && (memcasemem(value, valuelen, "chunked", 7))) { chunked = true; _M_request_body_size = 0; _M_payload_in_memory = 0; } else { chunked = false; // Get payload size (Content-Length). if (!_M_headers.get_value_known_header(http_headers::CONTENT_LENGTH_HEADER, value, &valuelen)) { _M_error = http_error::LENGTH_REQUIRED; return true; } if (number::parse_size_t(value, valuelen, _M_request_body_size) != number::PARSE_SUCCEEDED) { _M_error = http_error::BAD_REQUEST; return true; } // If the payload is already in memory... if (_M_request_header_size + _M_request_body_size <= _M_in.count()) { _M_payload_in_memory = 1; _M_inp += _M_request_body_size; _M_state = PREPARING_HTTP_REQUEST_STATE; } else { _M_payload_in_memory = (_M_request_body_size <= static_cast<http_server*>(_M_server)->_M_max_payload_in_memory); _M_inp += count; _M_filesize = _M_request_body_size - count; _M_state = READING_BODY_STATE; } } logger::instance().log(logger::LOG_DEBUG, "[http_connection::process_non_local_handler] (fd %d) %s payload will be saved in %s.", fd, chunked ? "Chunked" : "Not chunked", _M_payload_in_memory ? "memory" : "disk"); // If the payload should be written to disk... if (!_M_payload_in_memory) { // Open a temporary file. if ((_M_tmpfile = static_cast<http_server*>(_M_server)->_M_tmpfiles.open()) < 0) { _M_error = http_error::INTERNAL_SERVER_ERROR; return true; } _M_tmpfilesize = 0; // If there is some payload already in the input buffer... if (count > 0) { if (chunked) { size_t size; switch (parse_chunk(_M_in.data() + _M_request_header_size, count, size)) { case chunked_parser::INVALID_CHUNKED_RESPONSE: _M_error = http_error::BAD_REQUEST; return true; case chunked_parser::CALLBACK_FAILED: _M_error = http_error::INTERNAL_SERVER_ERROR; return true; case chunked_parser::NOT_END_OF_RESPONSE: _M_state = READING_CHUNKED_BODY_STATE; break; default: _M_inp += size; _M_state = PREPARING_HTTP_REQUEST_STATE; _M_error = http_error::OK; return true; } } else { if (_M_rule->handler == rulelist::HTTP_HANDLER) { if (!file_wrapper::write(_M_tmpfile, _M_in.data() + _M_request_header_size, count)) { _M_error = http_error::INTERNAL_SERVER_ERROR; return true; } } else { if (!fastcgi::stdin_stream(REQUEST_ID, _M_in.data() + _M_request_header_size, count, _M_tmpfile, _M_tmpfilesize)) { _M_error = http_error::INTERNAL_SERVER_ERROR; return true; } } } } _M_in.reset(); _M_inp = 0; } _M_error = http_error::OK; return true; }
bool proxy_connection::process_response(unsigned fd) { const char* value; unsigned short valuelen; if (_M_client->_M_method == http_method::HEAD) { _M_client->_M_payload_in_memory = 1; if (!_M_client->_M_headers.get_value_known_header(http_headers::CONTENT_LENGTH_HEADER, value, &valuelen)) { _M_client->_M_filesize = 0; } else { if (number::parse_off_t(value, valuelen, _M_client->_M_filesize, 0) != number::PARSE_SUCCEEDED) { logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Invalid body size.", fd); _M_client->_M_error = http_error::BAD_GATEWAY; return false; } } _M_state = RESPONSE_COMPLETED_STATE; } else { size_t count = _M_client->_M_body.count() - _M_body_offset; // Transfer-Encoding: chunked? bool chunked; if ((_M_client->_M_headers.get_value_known_header(http_headers::TRANSFER_ENCODING_HEADER, value, &valuelen)) && (memcasemem(value, valuelen, "chunked", 7))) { logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Chunked body.", fd); chunked = true; _M_client->_M_filesize = 0; _M_client->_M_payload_in_memory = 0; _M_state = READING_CHUNKED_BODY_STATE; } else { chunked = false; // Get Content-Length. if (!_M_client->_M_headers.get_value_known_header(http_headers::CONTENT_LENGTH_HEADER, value, &valuelen)) { if ((_M_status_code < 200) || (_M_status_code == 204) || (_M_status_code == 302) || (_M_status_code == 304)) { _M_client->_M_filesize = 0; _M_client->_M_payload_in_memory = 1; _M_state = RESPONSE_COMPLETED_STATE; } else { if (_M_client->_M_headers.get_value_known_header(http_headers::CONNECTION_HEADER, value, &valuelen)) { if (memcasemem(value, valuelen, "Keep-Alive", 10)) { // We cannot know the Content-Length. logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) !Content-Length && Keep-Alive.", fd); _M_client->_M_error = http_error::BAD_GATEWAY; return false; } } else { if ((_M_major_number == 1) && (_M_minor_number == 1)) { // We cannot know the Content-Length. logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) !Content-Length && !Connection && HTTP/1.1.", fd); _M_client->_M_error = http_error::BAD_GATEWAY; return false; } } logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Unknown body size.", fd); _M_client->_M_filesize = count; _M_client->_M_payload_in_memory = 0; _M_state = READING_UNKNOWN_SIZE_BODY_STATE; } } else { if (number::parse_off_t(value, valuelen, _M_client->_M_filesize, 0) != number::PARSE_SUCCEEDED) { logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Invalid body size.", fd); _M_client->_M_error = http_error::BAD_GATEWAY; return false; } logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Body size %lld.", fd, _M_client->_M_filesize); // If we have received the whole body already... if ((off_t) count >= _M_client->_M_filesize) { _M_client->_M_payload_in_memory = 1; _M_state = RESPONSE_COMPLETED_STATE; } else { _M_client->_M_payload_in_memory = (_M_client->_M_filesize <= (off_t) static_cast<http_server*>(_M_server)->_M_max_payload_in_memory); _M_left = _M_client->_M_filesize - count; _M_state = READING_BODY_STATE; } } } logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Payload will be saved %s.", fd, _M_client->_M_payload_in_memory ? "in memory" : "to disk"); if (!_M_client->_M_payload_in_memory) { // Open a temporary file. if ((_M_tmpfile = static_cast<http_server*>(_M_server)->_M_tmpfiles.open()) < 0) { _M_client->_M_error = http_error::INTERNAL_SERVER_ERROR; return false; } if (count > 0) { if (!chunked) { if (!file_wrapper::write(_M_tmpfile, _M_client->_M_body.data() + _M_body_offset, count)) { _M_client->_M_error = http_error::INTERNAL_SERVER_ERROR; return false; } } else { switch (parse_chunk(_M_client->_M_body.data() + _M_body_offset, count)) { case chunked_parser::INVALID_CHUNKED_RESPONSE: _M_client->_M_error = http_error::BAD_GATEWAY; return false; case chunked_parser::CALLBACK_FAILED: _M_client->_M_error = http_error::INTERNAL_SERVER_ERROR; return false; case chunked_parser::END_OF_RESPONSE: _M_state = RESPONSE_COMPLETED_STATE; break; default: // NOT_END_OF_RESPONSE. ; } } } // If there is reason phrase... if (_M_reason_phrase_len > 0) { _M_out.reset(); if (!_M_out.append(_M_client->_M_body.data() + _M_reason_phrase, _M_reason_phrase_len)) { _M_client->_M_error = http_error::INTERNAL_SERVER_ERROR; return false; } } _M_client->_M_body.reset(); _M_inp = 0; } } return true; }
int era_create(int argc, char **argv) { char table[64], uuid[64]; char *name, *meta, *data, *orig; unsigned orig_major, orig_minor; unsigned meta_major, meta_minor; unsigned data_major, data_minor; struct era_dm_info orig_info; uint64_t sectors; struct md *md; int chunk; int fd; /* * check and save arguments */ switch (argc) { case 0: error(0, "device name argument expected"); usage(stderr, 1); case 1: error(0, "metadata device argument expected"); usage(stderr, 1); case 2: error(0, "data device argument expected"); usage(stderr, 1); case 3: chunk = DEF_CHUNK_SIZE; break; case 4: chunk = parse_chunk(argv[3]); if (chunk == -1) { error(0, "can't parse chunk size: %s", argv[3]); return -1; } break; default: error(0, "unknown argument: %s", argv[4]); usage(stderr, 1); } name = argv[0]; meta = argv[1]; data = argv[2]; orig = malloc(16 + strlen(data)); if (!orig) { error(ENOMEM, NULL); return -1; } sprintf(orig, "%s-orig", name); /* * stat data device */ fd = blkopen(data, 0, &data_major, &data_minor, §ors); if (fd == -1) goto out; close(fd); /* * open metadata device */ md = md_open(meta, 1); if (!md) goto out; meta_major = md->major; meta_minor = md->minor; /* * create empty era target */ snprintf(uuid, sizeof(uuid), "%s%u-%u", UUID_PREFIX, meta_major, meta_minor); if (era_dm_create_empty(name, uuid, NULL)) { md_close(md); goto out; } /* * clear and close metadata */ if (clear_metadata(md, meta)) { (void)era_dm_remove(name); md_close(md); goto out; } md_close(md); /* * create origin target */ snprintf(uuid, sizeof(uuid), "%s%u-%u-orig", UUID_PREFIX, meta_major, meta_minor); snprintf(table, sizeof(table), "%u:%u 0", data_major, data_minor); if (era_dm_create(orig, uuid, 0, sectors, TARGET_LINEAR, table, &orig_info) == -1) { (void)era_dm_remove(name); goto out; } orig_major = (unsigned)orig_info.major; orig_minor = (unsigned)orig_info.minor; /* * load and resume era target */ snprintf(table, sizeof(table), "%u:%u %u:%u %d", meta_major, meta_minor, orig_major, orig_minor, chunk); if (era_dm_load(name, 0, sectors, TARGET_ERA, table, NULL)) { (void)era_dm_remove(orig); (void)era_dm_remove(name); goto out; } if (era_dm_resume(name)) { (void)era_dm_remove(orig); (void)era_dm_remove(name); goto out; } free(orig); return 0; out: free(orig); return -1; }
/** * Dechunk more data from the input buffer `mb'. * @returns dechunked data in a new buffer, or NULL if no more data. */ static pmsg_t * dechunk_data(rxdrv_t *rx, pmsg_t *mb) { struct attr *attr = rx->opaque; const char *error_str, *src; size_t size; /* * Prepare call to parse_chunk(). */ size = pmsg_size(mb); src = pmsg_read_base(mb); while (size > 0) { size_t ret; g_assert(CHUNK_STATE_ERROR != attr->state); /* * Copy avoidance: if the data we got fits into the current chunk size, * then we don't have to parse anything: all the data belong to the * current chunk, so we can simply pass them to the upper layer. */ if (CHUNK_STATE_DATA == attr->state) { pmsg_t *nmb; nmb = pmsg_clone(mb); if (size < attr->data_remain) { /* The complete chunk data is forwarded to the upper layer */ mb->m_rptr += size; attr->data_remain -= size; } else { /* Only the first ``data_remain'' bytes are forwarded */ mb->m_rptr += attr->data_remain; nmb->m_wptr = deconstify_pointer(&nmb->m_rptr[attr->data_remain]); attr->data_remain = 0; attr->state = CHUNK_STATE_DATA_CRLF; } if (GNET_PROPERTY(rx_debug) > 9) g_debug("dechunk_data: returning chunk of %u bytes", pmsg_size(nmb)); return nmb; } g_assert(size > 0); g_assert(CHUNK_STATE_DATA != attr->state); /* * Parse chunk headers */ ret = parse_chunk(rx, src, size, &error_str); if (0 == ret) { /* * We can't continue if we meet a dechunking error. Signal * our user so that the connection is terminated. */ errno = EIO; attr->cb->chunk_error(rx->owner, "dechunk() failed: %s", error_str); g_warning("dechunk_data(): %s", error_str); break; } g_assert(ret <= size); size -= ret; mb->m_rptr += ret; /* Read that far */ } return NULL; /* No more data */ }