/* skip up to length bytes in a chunkqueue, return number of bytes skipped */ goffset li_chunkqueue_skip(liChunkQueue *cq, goffset length) { liChunk *c; goffset bytes = 0; goffset we_have; while ( (NULL != (c = li_chunkqueue_first_chunk(cq))) && (0 == (we_have = li_chunk_length(c)) || length > 0) ) { if (we_have <= length) { /* skip (delete) complete chunk */ if (c->type == STRING_CHUNK) cqlimit_update(cq, - (goffset)c->data.str->len); else if (c->type == MEM_CHUNK) cqlimit_update(cq, - (goffset)c->mem->len); else if (c->type == BUFFER_CHUNK) cqlimit_update(cq, - (goffset)c->data.buffer.length); chunk_free(cq, c); bytes += we_have; length -= we_have; } else { /* skip first part of a chunk */ c->offset += length; bytes += length; length = 0; } } cq->bytes_out += bytes; cq->length -= bytes; return bytes; }
liNetworkStatus li_network_write_writev(int fd, liChunkQueue *cq, goffset *write_max, GError **err) { if (cq->length == 0) return LI_NETWORK_STATUS_FATAL_ERROR; do { switch (li_chunkqueue_first_chunk(cq)->type) { case STRING_CHUNK: case MEM_CHUNK: case BUFFER_CHUNK: LI_NETWORK_FALLBACK(li_network_backend_writev, write_max); break; case FILE_CHUNK: LI_NETWORK_FALLBACK(li_network_backend_write, write_max); break; default: return LI_NETWORK_STATUS_FATAL_ERROR; } if (cq->length == 0) return LI_NETWORK_STATUS_SUCCESS; } while (*write_max > 0); return LI_NETWORK_STATUS_SUCCESS; }
static void bod_handle_data(bod_state *state) { liChunkQueue *out = state->stream.out; liChunkQueue *in; if (out->is_closed) { li_stream_disconnect(&state->stream); bod_close(state); return; } in = (state->stream.source != NULL) ? state->stream.source->out : NULL; if (NULL == in) goto out; if (NULL == state->vr) { li_chunkqueue_steal_all(out, in); goto out; } while (in->length > 0) { liChunk *c = li_chunkqueue_first_chunk(in); liChunkIter ci; off_t length, data_len; char *data = NULL; GError *err; assert(UNUSED_CHUNK != c->type); switch (c->type) { case UNUSED_CHUNK: /* shouldn't happen anyway, but stealing it is ok here too */ case FILE_CHUNK: if (state->split_on_file_chunks) { bod_close(state); } else { bod_flush(state); } li_chunkqueue_steal_chunk(out, in); break; case STRING_CHUNK: case MEM_CHUNK: case BUFFER_CHUNK: if (!bod_open(state)) return; length = li_chunk_length(c); ci = li_chunkqueue_iter(in); err = NULL; if (LI_HANDLER_GO_ON != li_chunkiter_read(ci, 0, length, &data, &data_len, &err)) { if (NULL != err) { VR_ERROR(state->vr, "%s", err->message); g_error_free(err); } bod_error(state); return; } while ( data_len > 0 ) { ssize_t r; r = pwrite(state->tempfile->fd, data, data_len, state->write_pos); if (r < 0) { switch (errno) { case EINTR: continue; default: break; } VR_ERROR(state->vr, "pwrite failed: %s", g_strerror(errno)); bod_stop(state); /* write failures are not critical */ return; } data += r; data_len -= r; state->write_pos += r; } li_chunkqueue_skip(in, length); break; } } bod_autoflush(state); out: if (NULL == in || in->is_closed) { out->is_closed = TRUE; bod_close(state); /* close/flush ignores out->is_closed */ li_stream_notify(&state->stream); /* if no flush happened we still notify */ } }
liHandlerResult li_filter_buffer_on_disk(liVRequest *vr, liChunkQueue *out, liChunkQueue *in, bod_state *state) { UNUSED(vr); if (out->is_closed) { in->is_closed = TRUE; li_chunkqueue_skip_all(in); bod_close(state); return LI_HANDLER_GO_ON; } while (in->length > 0) { liChunk *c = li_chunkqueue_first_chunk(in); liChunkIter ci; off_t length, data_len; char *data = NULL; GError *err; switch (c->type) { case UNUSED_CHUNK: return LI_HANDLER_ERROR; case FILE_CHUNK: bod_flush(out, state); if (state->split_on_file_chunks) { bod_close(state); } li_chunkqueue_steal_chunk(out, in); break; case STRING_CHUNK: case MEM_CHUNK: case BUFFER_CHUNK: if (!bod_open(vr, state)) return LI_HANDLER_ERROR; length = li_chunk_length(c); ci = li_chunkqueue_iter(in); err = NULL; if (LI_HANDLER_GO_ON != li_chunkiter_read(ci, 0, length, &data, &data_len, &err)) { if (NULL != err) { VR_ERROR(vr, "%s", err->message); g_error_free(err); } return LI_HANDLER_ERROR; } while ( data_len > 0 ) { ssize_t r; r = pwrite(state->tempfile->fd, data, data_len, state->write_pos); if (r < 0) { switch (errno) { case EINTR: continue; default: break; } VR_ERROR(vr, "pwrite failed: %s", g_strerror(errno)); return LI_HANDLER_ERROR; } data += r; data_len -= r; state->write_pos += r; } li_chunkqueue_skip(in, length); break; } } bod_autoflush(out, state); if (in->is_closed) { bod_flush(out, state); out->is_closed = TRUE; bod_close(state); return LI_HANDLER_GO_ON; } return LI_HANDLER_GO_ON; }
/* steal up to length bytes from in and put them into out, return number of bytes stolen */ goffset li_chunkqueue_steal_len(liChunkQueue *out, liChunkQueue *in, goffset length) { liChunk *c, *cnew; GList* l; goffset bytes = 0, meminbytes = 0, memoutbytes = 0; goffset we_have; while ( (NULL != (c = li_chunkqueue_first_chunk(in))) && length > 0 ) { we_have = li_chunk_length(c); if (!we_have) { /* remove empty chunks */ if (c->type == STRING_CHUNK) meminbytes -= c->data.str->len; else if (c->type == MEM_CHUNK) meminbytes -= c->mem->len; else if (c->type == BUFFER_CHUNK) meminbytes -= c->data.buffer.length; chunk_free(in, c); continue; } if (we_have <= length) { /* move complete chunk */ l = g_queue_pop_head_link(&in->queue); g_queue_push_tail_link(&out->queue, l); bytes += we_have; if (c->type == STRING_CHUNK) { meminbytes -= c->data.str->len; memoutbytes += c->data.str->len; } else if (c->type == MEM_CHUNK) { meminbytes -= c->mem->len; memoutbytes += c->mem->len; } else if (c->type == BUFFER_CHUNK) { meminbytes -= c->data.buffer.length; memoutbytes += c->data.buffer.length; } length -= we_have; } else { /* copy first part of a chunk */ cnew = chunk_new(); switch (c->type) { case UNUSED_CHUNK: /* impossible, has length 0 */ /* remove "empty" chunks */ chunk_free(in, c); chunk_free(NULL, cnew); continue; case STRING_CHUNK: /* change type to MEM_CHUNK, as we copy it anyway */ cnew->type = MEM_CHUNK; cnew->mem = g_byte_array_sized_new(length); g_byte_array_append(cnew->mem, (guint8*) c->data.str->str + c->offset, length); memoutbytes += length; break; case MEM_CHUNK: cnew->type = MEM_CHUNK; cnew->mem = g_byte_array_sized_new(length); g_byte_array_append(cnew->mem, (guint8*) c->mem->data + c->offset, length); memoutbytes += length; break; case FILE_CHUNK: cnew->type = FILE_CHUNK; li_chunkfile_acquire(c->data.file.file); cnew->data.file.file = c->data.file.file; cnew->data.file.start = c->data.file.start + c->offset; cnew->data.file.length = length; break; case BUFFER_CHUNK: cnew->type = BUFFER_CHUNK; li_buffer_acquire(c->data.buffer.buffer); cnew->data.buffer.buffer = c->data.buffer.buffer; cnew->data.buffer.offset = c->data.buffer.offset + c->offset; cnew->data.buffer.length = length; memoutbytes += length; break; } c->offset += length; bytes += length; length = 0; g_queue_push_tail_link(&out->queue, &cnew->cq_link); } } in->bytes_out += bytes; in->length -= bytes; out->bytes_in += bytes; out->length += bytes; cqlimit_update(out, memoutbytes); cqlimit_update(in, meminbytes); return bytes; }