static inline int stream_to_disk(IOBuf *iob, int content_len, int tmpfd) { char *data = NULL; int avail = 0; debug("max content length: %d, content_len: %d", MAX_CONTENT_LENGTH, content_len); IOBuf_resize(iob, MAX_CONTENT_LENGTH); // give us a good buffer size while(content_len > 0) { data = IOBuf_read_some(iob, &avail); check(!IOBuf_closed(iob), "Closed while reading from IOBuf."); content_len -= avail; check(write(tmpfd, data, avail) == avail, "Failed to write requested amount to tempfile: %d", avail); check(IOBuf_read_commit(iob, avail) != -1, "Final commit failed streaming to disk."); } check(content_len == 0, "Failed to write everything to the large upload tmpfile."); return 0; error: return -1; }
int Connection_read_header(Connection *conn, Request *req) { char *data = IOBuf_start(conn->iob); int avail = IOBuf_avail(conn->iob); int rc = 0; size_t nparsed = 0; int tries = 0; Request_start(req); for(tries = 0; rc == 0 && tries < CLIENT_READ_RETRIES; tries++) { if(avail > 0) { rc = Request_parse(req, data, avail, &nparsed); } if(rc == 0) { data = IOBuf_read_some(conn->iob, &avail); check_debug(!IOBuf_closed(conn->iob), "Client closed during read."); } } error_unless(tries < CLIENT_READ_RETRIES, conn, 400, "Too many small packet read attempts."); error_unless(rc == 1, conn, 400, "Error parsing request."); // add the x-forwarded-for header Request_set(conn->req, bstrcpy(&HTTP_X_FORWARDED_FOR), blk2bstr(conn->remote, IPADDR_SIZE), 1); check_should_close(conn, conn->req); return nparsed; error: return -1; }
int Connection_read_wspacket(Connection *conn) { bstring payload=NULL; uint8_t *dataU=NULL; char *data = IOBuf_start(conn->iob); int avail = IOBuf_avail(conn->iob); int64_t packet_length=-1; int smaller_packet_length; int header_length; char key[4]; int i; int data_length; int tries = 0; int rc=0; int fin; int inprogFlags=0; int isControl; int flags; again: dataU=NULL; data = IOBuf_start(conn->iob); avail = IOBuf_avail(conn->iob); packet_length=-1; smaller_packet_length=0; header_length=0; i=0; data_length=0; tries = 0; rc=0; fin=0; for(tries = 0; packet_length == -1 && tries < 8*CLIENT_READ_RETRIES; tries++) { if(avail > 0) { packet_length = Websocket_packet_length((uint8_t *)data, avail); } if(packet_length == -1) { data = IOBuf_read_some(conn->iob, &avail); check_debug(!IOBuf_closed(conn->iob), "Client closed during read."); } } check(packet_length > 0,"Error receiving websocket packet header.") check_debug(packet_length <= INT_MAX,"Websocket packet longer than MAXINT."); /* TODO properly terminate WS connection */ smaller_packet_length = (int)packet_length; /* TODO check for maximum length */ header_length=Websocket_header_length((uint8_t *) data, avail); data_length=smaller_packet_length-header_length; dataU = (uint8_t *)IOBuf_read_all(conn->iob,header_length,8*CLIENT_READ_RETRIES); memcpy(key,dataU+header_length-4,4); flags=dataU[0]; if (payload==NULL) { inprogFlags=flags; } fin=(WS_fin(dataU)); isControl=(WS_is_control(dataU)); { const char *error=WS_validate_packet(dataU,payload!=NULL); check(error==NULL,"%s",error); } dataU = (uint8_t *)IOBuf_read_all(conn->iob,data_length, 8*CLIENT_READ_RETRIES); check(dataU != NULL, "Client closed the connection during websocket packet."); for(i=0;i<data_length;++i) { dataU[i]^=key[i%4]; } if(isControl) /* Control frames get sent right-away */ { Request_set(conn->req,bfromcstr("FLAGS"),bformat("0x%X",flags|0x80),1); rc = Connection_send_to_handler(conn, conn->handler, (void *)dataU,data_length); check_debug(rc == 0, "Failed to deliver to the handler."); } else { if(fin) { Request_set(conn->req,bfromcstr("FLAGS"),bformat("0x%X",inprogFlags|0x80),1); } if (payload == NULL) { if (fin) { rc = Connection_send_to_handler(conn, conn->handler, (void *)dataU,data_length); check_debug(rc == 0, "Failed to deliver to the handler."); } else { payload = blk2bstr(dataU,data_length); check(payload != NULL,"Allocation failed"); } } else { check(BSTR_OK == bcatblk(payload,dataU,data_length), "Concatenation failed"); if (fin) { rc = Connection_send_to_handler(conn, conn->handler, bdata(payload),blength(payload)); check_debug(rc == 0, "Failed to deliver to the handler."); bdestroy(payload); payload=NULL; } } } if (payload != NULL) { goto again; } return packet_length; error: bdestroy(payload); return -1; }
char *test_IOBuf_read_operations() { char *data = NULL; int avail = 0; Connection *conn = fake_conn("/dev/zero", O_RDONLY); mu_assert(conn != NULL, "Failed to make fake /dev/zero connection."); IOBuf *buf = conn->iob; IOBuf_resize(buf, 31); mu_assert(buf->len == 31, "Wrong size after resize."); data = IOBuf_start(buf); avail = IOBuf_avail(buf); mu_assert(data != NULL, "Didn't get a data response on begin."); mu_assert(data == buf->buf, "Begin on fresh iobuf should be at the beginning."); mu_assert(avail == 0, "Should not have anything available yet."); data = IOBuf_read(buf, 10, &avail); mu_assert(data != NULL, "Should get something always."); mu_assert(data == IOBuf_start(buf), "First read should work from start."); mu_assert(data == buf->buf, "First read should be at start of internal buf."); mu_assert(avail == 10, "Should get 10 bytes."); // check compacting mu_assert(!IOBuf_compact_needed(buf, 10), "Should not need compacting for 10."); mu_assert(IOBuf_compact_needed(buf, 100), "SHOULD need compacting for 100."); mu_assert(IOBuf_read_commit(buf, 10) != -1, "Failed to commit."); data = IOBuf_start(buf); avail = IOBuf_avail(buf); mu_assert(data != NULL, "Didn't get a data response on begin."); mu_assert(data != buf->buf, "Later reads should not be at the start."); mu_assert(avail == 21, "Should have 21 bytes available in the buf already."); data = IOBuf_read(buf, 10, &avail); mu_assert(data != NULL, "Should get something always."); mu_assert(avail == 10, "Should get 10 bytes."); mu_assert(IOBuf_read_commit(buf, 10) != -1, "Finaly commit failed."); data = IOBuf_start(buf); avail = IOBuf_avail(buf); mu_assert(data != NULL, "Didn't get a data response on begin."); mu_assert(data != buf->buf, "Later reads should not be at the start."); mu_assert(avail == 11, "Should have 11 bytes available in the buf already."); // now test two reads, one that fits one that doesn't then commit // remember this doesn't *return* anything, just a pointer to the start data = IOBuf_read(buf, 5, &avail); mu_assert(data != NULL, "Should get something always."); mu_assert(avail == 5, "Should get 10 bytes."); // ok we didn't want 5 we want 20, so this will cause a compact data = IOBuf_read(buf, 20, &avail); mu_assert(data != NULL, "Should get something always."); mu_assert(avail == 20, "Should get 10 bytes."); mu_assert(IOBuf_read_commit(buf, 21) != -1, "Final commit failed."); debug("We've got %d avail after the last read.", buf->avail); mu_assert(buf->avail == 10, "Should have 11 still in the queue."); data = IOBuf_read_all(buf, 30, 1); mu_assert(data != NULL, "Failed to read all."); mu_assert(buf->avail == 1, "Should have 1 in the queue."); data = IOBuf_read_some(buf, &avail); mu_assert(data != NULL, "Failed to read some."); mu_assert(buf->avail == 31, "Should be full."); mu_assert(avail == 31, "And we should get the full amount."); fake_conn_close(conn); return NULL; }
int Upload_stream(Connection *conn, Handler *handler, int content_len) { char *data = NULL; int avail = 0; int offset = 0; int first_chunk = 1; int rc; hash_t *altheaders = NULL; bstring offsetstr; debug("max content length: %d, content_len: %d", MAX_CONTENT_LENGTH, content_len); IOBuf_resize(conn->iob, MAX_CONTENT_LENGTH); // give us a good buffer size while(content_len > 0) { if(first_chunk) { // read whatever's there data = IOBuf_read_some(conn->iob, &avail); } else if(conn->sendCredits > 0) { // read up to credits data = IOBuf_read(conn->iob, conn->sendCredits < content_len ? conn->sendCredits : content_len, &avail); conn->sendCredits -= avail; } else { // sleep until we have credits tasksleep(&conn->uploadRendez); continue; } check(!IOBuf_closed(conn->iob), "Closed while reading from IOBuf."); content_len -= avail; offsetstr = bformat("%d", offset); if(first_chunk) { Request_set(conn->req, &UPLOAD_STREAM, offsetstr, 1); if(content_len == 0) { Request_set(conn->req, &UPLOAD_STREAM_DONE, bfromcstr("1"), 1); } } else { altheaders = hash_create(2, (hash_comp_t)bstrcmp, bstr_hash_fun); add_to_hash(altheaders, &UPLOAD_STREAM, offsetstr); if(content_len == 0) { add_to_hash(altheaders, &UPLOAD_STREAM_DONE, bfromcstr("1")); } } rc = Connection_send_to_handler(conn, handler, data, avail, altheaders); check_debug(rc == 0, "Failed to deliver to the handler."); if(altheaders != NULL) { hash_free_nodes(altheaders); hash_destroy(altheaders); altheaders = NULL; } check(IOBuf_read_commit(conn->iob, avail) != -1, "Commit failed while streaming."); first_chunk = 0; offset += avail; } check(content_len == 0, "Failed to write everything to the large upload tmpfile."); return 0; error: return -1; }