bool proxy_connection::read_chunked_body(unsigned fd, size_t& total)
{
io_result res;
do {
if ((res = read(fd, _M_client->_M_body, total)) == IO_ERROR) {
_M_client->_M_error = http_error::GATEWAY_TIMEOUT;
return false;
} else if (res == IO_NO_DATA_READ) {
return true;
}
_M_client->_M_timestamp = now::_M_time;
switch (parse_chunk(_M_client->_M_body.data(), _M_client->_M_body.count())) {
case chunked_parser::INVALID_CHUNKED_RESPONSE:
_M_client->_M_error = http_error::BAD_GATEWAY;
return false;
case chunked_parser::CALLBACK_FAILED:
_M_client->_M_error = http_error::INTERNAL_SERVER_ERROR;
return false;
case chunked_parser::NOT_END_OF_RESPONSE:
_M_client->_M_body.reset();
break;
default:
_M_state = RESPONSE_COMPLETED_STATE;
return true;
}
} while (res == IO_SUCCESS);
return true;
}
int hrepack_addchunk(const char* str,
options_t *options)
{
obj_list_t *obj_list=NULL; /*one object list for the -t and -c option entry */
int n_objs; /*number of objects in the current -t or -c option entry */
int32 chunk_lengths[H4_MAX_VAR_DIMS]; /* chunk lengths along each dimension */
int chunk_rank; /*global rank for chunks */
int i, j;
if (options->all_chunk==1){
printf("Error: Invalid chunking input: '*' is present with other objects <%s>\n",str);
return FAIL;
}
/* parse the -c option */
if ((obj_list = parse_chunk(str,&n_objs,chunk_lengths,&chunk_rank)) == NULL)
return FAIL;
/* searh for the "*" all objects character */
for (i = 0; i < n_objs; i++)
{
if (HDstrcmp("*",obj_list[i].obj)==0)
{
/* if we are chunking all set the global chunking type */
options->all_chunk=1;
options->chunk_g.rank=chunk_rank;
for (j = 0; j < chunk_rank; j++)
options->chunk_g.chunk_lengths[j] = chunk_lengths[j];
}
}
if (i>1)
{
printf("\nError: '*' cannot be with other objects, <%s>. Exiting...\n",str);
goto out;
}
if (options->all_chunk==0)
{
if (options_add_chunk(obj_list,n_objs,chunk_lengths,chunk_rank,options->op_tbl)<0)
goto out;
}
HDfree(obj_list);
return SUCCEED;
out:
HDfree(obj_list);
return FAIL;
}
static lcb_error_t parse_body(struct htvb_st *vbs, int *done)
{
lcb_error_t err = LCB_BUSY;
char *term;
if ((err = parse_chunk(vbs)) != LCB_SUCCESS) {
*done = 1; /* no data */
lcb_assert(err == LCB_BUSY);
return err;
}
if (lcb_string_append(&vbs->input, vbs->chunk.base, vbs->chunk_size)) {
return LCB_CLIENT_ENOMEM;
}
lcb_string_erase_end(&vbs->input, 2);
lcb_string_erase_beginning(&vbs->chunk, vbs->chunk_size);
vbs->chunk_size = (lcb_size_t) - 1;
if (vbs->chunk.nused > 0) {
*done = 0;
}
term = strstr(vbs->input.base, "\n\n\n\n");
if (term != NULL) {
lcb_string tmp;
lcb_error_t ret;
/** Next input */
lcb_string_init(&tmp);
lcb_string_appendz(&tmp, term + 4);
*term = '\0';
ret = set_next_config(vbs);
/** Now, erase everything until the end of the 'term' */
if (vbs->input.base) {
lcb_string_release(&vbs->input);
}
lcb_string_transfer(&tmp, &vbs->input);
return ret;
}
return err;
}
bool http_connection::read_chunked_body(unsigned fd, size_t& total)
{
io_result res;
do {
if ((res = read(fd, total)) == IO_ERROR) {
return false;
} else if (res == IO_NO_DATA_READ) {
return true;
}
if (_M_rule->handler == rulelist::HTTP_HANDLER) {
size_t size;
switch (parse_chunk(_M_in.data(), _M_in.count(), size)) {
case chunked_parser::INVALID_CHUNKED_RESPONSE:
_M_error = http_error::BAD_REQUEST;
_M_state = PREPARING_ERROR_PAGE_STATE;
return true;
case chunked_parser::CALLBACK_FAILED:
_M_error = http_error::INTERNAL_SERVER_ERROR;
_M_state = PREPARING_ERROR_PAGE_STATE;
return true;
case chunked_parser::END_OF_RESPONSE:
_M_inp += size;
_M_state = PREPARING_HTTP_REQUEST_STATE;
return true;
default:
;
}
}
_M_in.reset();
} while (res == IO_SUCCESS);
return true;
}
bool http_connection::process_non_local_handler(unsigned fd)
{
if ((_M_method == http_method::GET) || (_M_method == http_method::HEAD)) {
_M_request_body_size = 0;
_M_payload_in_memory = 1;
_M_state = PREPARING_HTTP_REQUEST_STATE;
_M_error = http_error::OK;
return true;
} else if ((_M_method != http_method::POST) && (_M_method != http_method::PUT)) {
_M_error = http_error::NOT_IMPLEMENTED;
return true;
}
size_t count = _M_in.count() - _M_request_header_size;
// Transfer-Encoding: chunked?
const char* value;
unsigned short valuelen;
bool chunked;
if ((_M_headers.get_value_known_header(http_headers::TRANSFER_ENCODING_HEADER, value, &valuelen)) && (memcasemem(value, valuelen, "chunked", 7))) {
chunked = true;
_M_request_body_size = 0;
_M_payload_in_memory = 0;
} else {
chunked = false;
// Get payload size (Content-Length).
if (!_M_headers.get_value_known_header(http_headers::CONTENT_LENGTH_HEADER, value, &valuelen)) {
_M_error = http_error::LENGTH_REQUIRED;
return true;
}
if (number::parse_size_t(value, valuelen, _M_request_body_size) != number::PARSE_SUCCEEDED) {
_M_error = http_error::BAD_REQUEST;
return true;
}
// If the payload is already in memory...
if (_M_request_header_size + _M_request_body_size <= _M_in.count()) {
_M_payload_in_memory = 1;
_M_inp += _M_request_body_size;
_M_state = PREPARING_HTTP_REQUEST_STATE;
} else {
_M_payload_in_memory = (_M_request_body_size <= static_cast<http_server*>(_M_server)->_M_max_payload_in_memory);
_M_inp += count;
_M_filesize = _M_request_body_size - count;
_M_state = READING_BODY_STATE;
}
}
logger::instance().log(logger::LOG_DEBUG, "[http_connection::process_non_local_handler] (fd %d) %s payload will be saved in %s.", fd, chunked ? "Chunked" : "Not chunked", _M_payload_in_memory ? "memory" : "disk");
// If the payload should be written to disk...
if (!_M_payload_in_memory) {
// Open a temporary file.
if ((_M_tmpfile = static_cast<http_server*>(_M_server)->_M_tmpfiles.open()) < 0) {
_M_error = http_error::INTERNAL_SERVER_ERROR;
return true;
}
_M_tmpfilesize = 0;
// If there is some payload already in the input buffer...
if (count > 0) {
if (chunked) {
size_t size;
switch (parse_chunk(_M_in.data() + _M_request_header_size, count, size)) {
case chunked_parser::INVALID_CHUNKED_RESPONSE:
_M_error = http_error::BAD_REQUEST;
return true;
case chunked_parser::CALLBACK_FAILED:
_M_error = http_error::INTERNAL_SERVER_ERROR;
return true;
case chunked_parser::NOT_END_OF_RESPONSE:
_M_state = READING_CHUNKED_BODY_STATE;
break;
default:
_M_inp += size;
_M_state = PREPARING_HTTP_REQUEST_STATE;
_M_error = http_error::OK;
return true;
}
} else {
if (_M_rule->handler == rulelist::HTTP_HANDLER) {
if (!file_wrapper::write(_M_tmpfile, _M_in.data() + _M_request_header_size, count)) {
_M_error = http_error::INTERNAL_SERVER_ERROR;
return true;
}
} else {
if (!fastcgi::stdin_stream(REQUEST_ID, _M_in.data() + _M_request_header_size, count, _M_tmpfile, _M_tmpfilesize)) {
_M_error = http_error::INTERNAL_SERVER_ERROR;
return true;
}
}
}
}
_M_in.reset();
_M_inp = 0;
}
_M_error = http_error::OK;
return true;
}
bool proxy_connection::process_response(unsigned fd)
{
const char* value;
unsigned short valuelen;
if (_M_client->_M_method == http_method::HEAD) {
_M_client->_M_payload_in_memory = 1;
if (!_M_client->_M_headers.get_value_known_header(http_headers::CONTENT_LENGTH_HEADER, value, &valuelen)) {
_M_client->_M_filesize = 0;
} else {
if (number::parse_off_t(value, valuelen, _M_client->_M_filesize, 0) != number::PARSE_SUCCEEDED) {
logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Invalid body size.", fd);
_M_client->_M_error = http_error::BAD_GATEWAY;
return false;
}
}
_M_state = RESPONSE_COMPLETED_STATE;
} else {
size_t count = _M_client->_M_body.count() - _M_body_offset;
// Transfer-Encoding: chunked?
bool chunked;
if ((_M_client->_M_headers.get_value_known_header(http_headers::TRANSFER_ENCODING_HEADER, value, &valuelen)) && (memcasemem(value, valuelen, "chunked", 7))) {
logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Chunked body.", fd);
chunked = true;
_M_client->_M_filesize = 0;
_M_client->_M_payload_in_memory = 0;
_M_state = READING_CHUNKED_BODY_STATE;
} else {
chunked = false;
// Get Content-Length.
if (!_M_client->_M_headers.get_value_known_header(http_headers::CONTENT_LENGTH_HEADER, value, &valuelen)) {
if ((_M_status_code < 200) || (_M_status_code == 204) || (_M_status_code == 302) || (_M_status_code == 304)) {
_M_client->_M_filesize = 0;
_M_client->_M_payload_in_memory = 1;
_M_state = RESPONSE_COMPLETED_STATE;
} else {
if (_M_client->_M_headers.get_value_known_header(http_headers::CONNECTION_HEADER, value, &valuelen)) {
if (memcasemem(value, valuelen, "Keep-Alive", 10)) {
// We cannot know the Content-Length.
logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) !Content-Length && Keep-Alive.", fd);
_M_client->_M_error = http_error::BAD_GATEWAY;
return false;
}
} else {
if ((_M_major_number == 1) && (_M_minor_number == 1)) {
// We cannot know the Content-Length.
logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) !Content-Length && !Connection && HTTP/1.1.", fd);
_M_client->_M_error = http_error::BAD_GATEWAY;
return false;
}
}
logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Unknown body size.", fd);
_M_client->_M_filesize = count;
_M_client->_M_payload_in_memory = 0;
_M_state = READING_UNKNOWN_SIZE_BODY_STATE;
}
} else {
if (number::parse_off_t(value, valuelen, _M_client->_M_filesize, 0) != number::PARSE_SUCCEEDED) {
logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Invalid body size.", fd);
_M_client->_M_error = http_error::BAD_GATEWAY;
return false;
}
logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Body size %lld.", fd, _M_client->_M_filesize);
// If we have received the whole body already...
if ((off_t) count >= _M_client->_M_filesize) {
_M_client->_M_payload_in_memory = 1;
_M_state = RESPONSE_COMPLETED_STATE;
} else {
_M_client->_M_payload_in_memory = (_M_client->_M_filesize <= (off_t) static_cast<http_server*>(_M_server)->_M_max_payload_in_memory);
_M_left = _M_client->_M_filesize - count;
_M_state = READING_BODY_STATE;
}
}
}
logger::instance().log(logger::LOG_DEBUG, "[proxy_connection::process_response] (fd %d) Payload will be saved %s.", fd, _M_client->_M_payload_in_memory ? "in memory" : "to disk");
if (!_M_client->_M_payload_in_memory) {
// Open a temporary file.
if ((_M_tmpfile = static_cast<http_server*>(_M_server)->_M_tmpfiles.open()) < 0) {
_M_client->_M_error = http_error::INTERNAL_SERVER_ERROR;
return false;
}
if (count > 0) {
if (!chunked) {
if (!file_wrapper::write(_M_tmpfile, _M_client->_M_body.data() + _M_body_offset, count)) {
_M_client->_M_error = http_error::INTERNAL_SERVER_ERROR;
return false;
}
} else {
switch (parse_chunk(_M_client->_M_body.data() + _M_body_offset, count)) {
case chunked_parser::INVALID_CHUNKED_RESPONSE:
_M_client->_M_error = http_error::BAD_GATEWAY;
return false;
case chunked_parser::CALLBACK_FAILED:
_M_client->_M_error = http_error::INTERNAL_SERVER_ERROR;
return false;
case chunked_parser::END_OF_RESPONSE:
_M_state = RESPONSE_COMPLETED_STATE;
break;
default: // NOT_END_OF_RESPONSE.
;
}
}
}
// If there is reason phrase...
if (_M_reason_phrase_len > 0) {
_M_out.reset();
if (!_M_out.append(_M_client->_M_body.data() + _M_reason_phrase, _M_reason_phrase_len)) {
_M_client->_M_error = http_error::INTERNAL_SERVER_ERROR;
return false;
}
}
_M_client->_M_body.reset();
_M_inp = 0;
}
}
return true;
}
int era_create(int argc, char **argv)
{
char table[64], uuid[64];
char *name, *meta, *data, *orig;
unsigned orig_major, orig_minor;
unsigned meta_major, meta_minor;
unsigned data_major, data_minor;
struct era_dm_info orig_info;
uint64_t sectors;
struct md *md;
int chunk;
int fd;
/*
* check and save arguments
*/
switch (argc)
{
case 0:
error(0, "device name argument expected");
usage(stderr, 1);
case 1:
error(0, "metadata device argument expected");
usage(stderr, 1);
case 2:
error(0, "data device argument expected");
usage(stderr, 1);
case 3:
chunk = DEF_CHUNK_SIZE;
break;
case 4:
chunk = parse_chunk(argv[3]);
if (chunk == -1)
{
error(0, "can't parse chunk size: %s", argv[3]);
return -1;
}
break;
default:
error(0, "unknown argument: %s", argv[4]);
usage(stderr, 1);
}
name = argv[0];
meta = argv[1];
data = argv[2];
orig = malloc(16 + strlen(data));
if (!orig)
{
error(ENOMEM, NULL);
return -1;
}
sprintf(orig, "%s-orig", name);
/*
* stat data device
*/
fd = blkopen(data, 0, &data_major, &data_minor, §ors);
if (fd == -1)
goto out;
close(fd);
/*
* open metadata device
*/
md = md_open(meta, 1);
if (!md)
goto out;
meta_major = md->major;
meta_minor = md->minor;
/*
* create empty era target
*/
snprintf(uuid, sizeof(uuid), "%s%u-%u",
UUID_PREFIX, meta_major, meta_minor);
if (era_dm_create_empty(name, uuid, NULL))
{
md_close(md);
goto out;
}
/*
* clear and close metadata
*/
if (clear_metadata(md, meta))
{
(void)era_dm_remove(name);
md_close(md);
goto out;
}
md_close(md);
/*
* create origin target
*/
snprintf(uuid, sizeof(uuid), "%s%u-%u-orig",
UUID_PREFIX, meta_major, meta_minor);
snprintf(table, sizeof(table), "%u:%u 0",
data_major, data_minor);
if (era_dm_create(orig, uuid, 0, sectors,
TARGET_LINEAR, table, &orig_info) == -1)
{
(void)era_dm_remove(name);
goto out;
}
orig_major = (unsigned)orig_info.major;
orig_minor = (unsigned)orig_info.minor;
/*
* load and resume era target
*/
snprintf(table, sizeof(table), "%u:%u %u:%u %d",
meta_major, meta_minor, orig_major, orig_minor, chunk);
if (era_dm_load(name, 0, sectors, TARGET_ERA, table, NULL))
{
(void)era_dm_remove(orig);
(void)era_dm_remove(name);
goto out;
}
if (era_dm_resume(name))
{
(void)era_dm_remove(orig);
(void)era_dm_remove(name);
goto out;
}
free(orig);
return 0;
out:
free(orig);
return -1;
}
/**
* Dechunk more data from the input buffer `mb'.
* @returns dechunked data in a new buffer, or NULL if no more data.
*/
static pmsg_t *
dechunk_data(rxdrv_t *rx, pmsg_t *mb)
{
struct attr *attr = rx->opaque;
const char *error_str, *src;
size_t size;
/*
* Prepare call to parse_chunk().
*/
size = pmsg_size(mb);
src = pmsg_read_base(mb);
while (size > 0) {
size_t ret;
g_assert(CHUNK_STATE_ERROR != attr->state);
/*
* Copy avoidance: if the data we got fits into the current chunk size,
* then we don't have to parse anything: all the data belong to the
* current chunk, so we can simply pass them to the upper layer.
*/
if (CHUNK_STATE_DATA == attr->state) {
pmsg_t *nmb;
nmb = pmsg_clone(mb);
if (size < attr->data_remain) {
/* The complete chunk data is forwarded to the upper layer */
mb->m_rptr += size;
attr->data_remain -= size;
} else {
/* Only the first ``data_remain'' bytes are forwarded */
mb->m_rptr += attr->data_remain;
nmb->m_wptr =
deconstify_pointer(&nmb->m_rptr[attr->data_remain]);
attr->data_remain = 0;
attr->state = CHUNK_STATE_DATA_CRLF;
}
if (GNET_PROPERTY(rx_debug) > 9)
g_debug("dechunk_data: returning chunk of %u bytes",
pmsg_size(nmb));
return nmb;
}
g_assert(size > 0);
g_assert(CHUNK_STATE_DATA != attr->state);
/*
* Parse chunk headers
*/
ret = parse_chunk(rx, src, size, &error_str);
if (0 == ret) {
/*
* We can't continue if we meet a dechunking error. Signal
* our user so that the connection is terminated.
*/
errno = EIO;
attr->cb->chunk_error(rx->owner,
"dechunk() failed: %s", error_str);
g_warning("dechunk_data(): %s", error_str);
break;
}
g_assert(ret <= size);
size -= ret;
mb->m_rptr += ret; /* Read that far */
}
return NULL; /* No more data */
}
