/**
* Streams data out of the from IOBuf and into the to IOBuf
* until it's moved total bytes between them.
*/
int IOBuf_stream(IOBuf *from, IOBuf *to, int total)
{
int need = 0;
int remain = total;
int avail = 0;
int rc = 0;
char *data = NULL;
if(from->len > to->len) IOBuf_resize(to, from->len);
while(remain > 0) {
need = remain <= from->len ? remain : from->len;
data = IOBuf_read(from, need, &avail);
check_debug(avail > 0, "Nothing in read buffer.");
rc = IOBuf_send_all(to, IOBuf_start(from), avail);
check_debug(rc == avail, "Failed to send all of the data: %d of %d", rc, avail)
// commit whatever we just did
check(IOBuf_read_commit(from, rc) != -1, "Final commit failed during streaming.");
// reduce it by the given amount
remain -= rc;
}
assert(remain == 0 && "Buffer math is wrong.");
return total - remain;
error:
return -1;
}
static inline int stream_to_disk(IOBuf *iob, int content_len, int tmpfd)
{
char *data = NULL;
int avail = 0;
debug("max content length: %d, content_len: %d", MAX_CONTENT_LENGTH, content_len);
IOBuf_resize(iob, MAX_CONTENT_LENGTH); // give us a good buffer size
while(content_len > 0) {
data = IOBuf_read_some(iob, &avail);
check(!IOBuf_closed(iob), "Closed while reading from IOBuf.");
content_len -= avail;
check(write(tmpfd, data, avail) == avail, "Failed to write requested amount to tempfile: %d", avail);
check(IOBuf_read_commit(iob, avail) != -1, "Final commit failed streaming to disk.");
}
check(content_len == 0, "Failed to write everything to the large upload tmpfile.");
return 0;
error:
return -1;
}
int connection_http_to_handler(Connection *conn)
{
int content_len = Request_content_length(conn->req);
int rc = 0;
char *body = NULL;
Handler *handler = Request_get_action(conn->req, handler);
error_unless(handler, conn, 404, "No action for request: %s", bdata(Request_path(conn->req)));
// we don't need the header anymore, so commit the buffer and deal with the body
IOBuf_read_commit(conn->iob, Request_header_length(conn->req));
if(content_len == 0) {
body = "";
rc = Connection_send_to_handler(conn, handler, body, content_len);
check_debug(rc == 0, "Failed to deliver to the handler.");
} else if(content_len > MAX_CONTENT_LENGTH) {
rc = Upload_file(conn, handler, content_len);
check(rc == 0, "Failed to upload file.");
} else {
if(content_len > conn->iob->len) {
// temporarily grow the buffer
IOBuf_resize(conn->iob, content_len);
}
debug("READ ALL CALLED with content_len: %d, and MAX_CONTENT_LENGTH: %d", content_len, MAX_CONTENT_LENGTH);
body = IOBuf_read_all(conn->iob, content_len, CLIENT_READ_RETRIES);
check(body != NULL, "Client closed the connection during upload.");
rc = Connection_send_to_handler(conn, handler, body, content_len);
check_debug(rc == 0, "Failed to deliver to the handler.");
}
Log_request(conn, 200, content_len);
return REQ_SENT;
error:
return CLOSE;
}
/**
* Reads the entire amount requested into the IOBuf (as long as there's
* space to hold it) and then commits that read in one shot.
*/
char *IOBuf_read_all(IOBuf *buf, int len, int retries)
{
int nread = 0;
int attempts = 0;
check_debug(!IOBuf_closed(buf), "Closed when attempting to read from buffer.");
if(len > buf->len) {
// temporarily grow the buffer
IOBuf_resize(buf, len);
}
char *data = IOBuf_read(buf, len, &nread);
debug("INITIAL READ: len: %d, nread: %d", len, nread);
for(attempts = 0; nread < len; attempts++) {
data = IOBuf_read(buf, len, &nread);
check_debug(data, "Read error from socket.");
assert(nread <= len && "Invalid nread size (too much) on IOBuf read.");
if(nread == len) {
break;
} else {
fdwait(buf->fd, 'r');
}
}
debug("ATTEMPTS: %d, RETRIES: %d", attempts, retries);
if(attempts > retries) {
log_warn("Read of %d length attempted %d times which is over %d retry limit..", len, attempts, retries);
}
check(IOBuf_read_commit(buf, len) != -1, "Final commit failed of read_all.");
return data;
error:
buf->closed = 1;
return NULL;
}
/**
* Reads the entire amount requested into the IOBuf (as long as there's
* space to hold it) and then commits that read in one shot.
*/
char *IOBuf_read_all(IOBuf *buf, int len, int retries)
{
int nread = 0;
int attempts = 0;
check_debug(!IOBuf_closed(buf), "Closed when attempting to read from buffer.");
if(len > buf->len) {
// temporarily grow the buffer
IOBuf_resize(buf, len);
}
char *data = IOBuf_read(buf, len, &nread);
debug("INITIAL READ: len: %d, nread: %d", len, nread);
for(attempts = 0; nread < len; attempts++) {
data = IOBuf_read(buf, len, &nread);
check_debug(data, "Read error from socket.");
assert(nread <= len && "Invalid nread size (too much) on IOBuf read.");
if(nread == len) {
log_warn("recive all data");
break;
}
else if(IOBuf_closed(buf)){//register closed or socket error
if(buf->buf) free(buf->buf);
buf->buf = strdup("upload=false&info='connection error'");
buf->len = strlen(buf->buf);
buf->avail = buf->len;
buf->cur = buf->len;
log_warn("connection error");
len = buf->len;
goto error;
/*if(buf->buf) free(buf->buf);
buf->buf = NULL;
buf->len = 0;
buf->avail = 0;
buf->cur = 0;
log_warn("connection error");
goto error; */
}
else {
fdwait(buf->fd, 'r');
}
}
debug("ATTEMPTS: %d, RETRIES: %d", attempts, retries);
if(attempts > retries) {
log_warn("Read of %d length attempted %d times which is over %d retry limit..", len, attempts, retries);
}
check(IOBuf_read_commit(buf, len) != -1, "Final commit failed of read_all.");
return data;
error:
buf->closed = 1;
return NULL;
}
char *test_IOBuf_read_operations()
{
char *data = NULL;
int avail = 0;
Connection *conn = fake_conn("/dev/zero", O_RDONLY);
mu_assert(conn != NULL, "Failed to make fake /dev/zero connection.");
IOBuf *buf = conn->iob;
IOBuf_resize(buf, 31);
mu_assert(buf->len == 31, "Wrong size after resize.");
data = IOBuf_start(buf);
avail = IOBuf_avail(buf);
mu_assert(data != NULL, "Didn't get a data response on begin.");
mu_assert(data == buf->buf, "Begin on fresh iobuf should be at the beginning.");
mu_assert(avail == 0, "Should not have anything available yet.");
data = IOBuf_read(buf, 10, &avail);
mu_assert(data != NULL, "Should get something always.");
mu_assert(data == IOBuf_start(buf), "First read should work from start.");
mu_assert(data == buf->buf, "First read should be at start of internal buf.");
mu_assert(avail == 10, "Should get 10 bytes.");
// check compacting
mu_assert(!IOBuf_compact_needed(buf, 10), "Should not need compacting for 10.");
mu_assert(IOBuf_compact_needed(buf, 100), "SHOULD need compacting for 100.");
mu_assert(IOBuf_read_commit(buf, 10) != -1, "Failed to commit.");
data = IOBuf_start(buf);
avail = IOBuf_avail(buf);
mu_assert(data != NULL, "Didn't get a data response on begin.");
mu_assert(data != buf->buf, "Later reads should not be at the start.");
mu_assert(avail == 21, "Should have 21 bytes available in the buf already.");
data = IOBuf_read(buf, 10, &avail);
mu_assert(data != NULL, "Should get something always.");
mu_assert(avail == 10, "Should get 10 bytes.");
mu_assert(IOBuf_read_commit(buf, 10) != -1, "Finaly commit failed.");
data = IOBuf_start(buf);
avail = IOBuf_avail(buf);
mu_assert(data != NULL, "Didn't get a data response on begin.");
mu_assert(data != buf->buf, "Later reads should not be at the start.");
mu_assert(avail == 11, "Should have 11 bytes available in the buf already.");
// now test two reads, one that fits one that doesn't then commit
// remember this doesn't *return* anything, just a pointer to the start
data = IOBuf_read(buf, 5, &avail);
mu_assert(data != NULL, "Should get something always.");
mu_assert(avail == 5, "Should get 10 bytes.");
// ok we didn't want 5 we want 20, so this will cause a compact
data = IOBuf_read(buf, 20, &avail);
mu_assert(data != NULL, "Should get something always.");
mu_assert(avail == 20, "Should get 10 bytes.");
mu_assert(IOBuf_read_commit(buf, 21) != -1, "Final commit failed.");
debug("We've got %d avail after the last read.", buf->avail);
mu_assert(buf->avail == 10, "Should have 11 still in the queue.");
data = IOBuf_read_all(buf, 30, 1);
mu_assert(data != NULL, "Failed to read all.");
mu_assert(buf->avail == 1, "Should have 1 in the queue.");
data = IOBuf_read_some(buf, &avail);
mu_assert(data != NULL, "Failed to read some.");
mu_assert(buf->avail == 31, "Should be full.");
mu_assert(avail == 31, "And we should get the full amount.");
fake_conn_close(conn);
return NULL;
}
int Upload_stream(Connection *conn, Handler *handler, int content_len)
{
char *data = NULL;
int avail = 0;
int offset = 0;
int first_chunk = 1;
int rc;
hash_t *altheaders = NULL;
bstring offsetstr;
debug("max content length: %d, content_len: %d", MAX_CONTENT_LENGTH, content_len);
IOBuf_resize(conn->iob, MAX_CONTENT_LENGTH); // give us a good buffer size
while(content_len > 0) {
if(first_chunk) {
// read whatever's there
data = IOBuf_read_some(conn->iob, &avail);
} else if(conn->sendCredits > 0) {
// read up to credits
data = IOBuf_read(conn->iob, conn->sendCredits < content_len ? conn->sendCredits : content_len, &avail);
conn->sendCredits -= avail;
} else {
// sleep until we have credits
tasksleep(&conn->uploadRendez);
continue;
}
check(!IOBuf_closed(conn->iob), "Closed while reading from IOBuf.");
content_len -= avail;
offsetstr = bformat("%d", offset);
if(first_chunk) {
Request_set(conn->req, &UPLOAD_STREAM, offsetstr, 1);
if(content_len == 0) {
Request_set(conn->req, &UPLOAD_STREAM_DONE, bfromcstr("1"), 1);
}
} else {
altheaders = hash_create(2, (hash_comp_t)bstrcmp, bstr_hash_fun);
add_to_hash(altheaders, &UPLOAD_STREAM, offsetstr);
if(content_len == 0) {
add_to_hash(altheaders, &UPLOAD_STREAM_DONE, bfromcstr("1"));
}
}
rc = Connection_send_to_handler(conn, handler, data, avail, altheaders);
check_debug(rc == 0, "Failed to deliver to the handler.");
if(altheaders != NULL) {
hash_free_nodes(altheaders);
hash_destroy(altheaders);
altheaders = NULL;
}
check(IOBuf_read_commit(conn->iob, avail) != -1, "Commit failed while streaming.");
first_chunk = 0;
offset += avail;
}
check(content_len == 0, "Failed to write everything to the large upload tmpfile.");
return 0;
error:
return -1;
}
