static gpointer pull_buffer_impl( XferElement *elt, size_t *size) { XferFilterCrc *self = (XferFilterCrc *)elt; char *buf; XMsg *msg; if (elt->cancelled) { /* drain our upstream only if we're expecting an EOF */ if (elt->expect_eof) { xfer_element_drain_buffers(XFER_ELEMENT(self)->upstream); } /* return an EOF */ *size = 0; return NULL; } /* get a buffer from upstream, crc it, and hand it back */ buf = xfer_element_pull_buffer(XFER_ELEMENT(self)->upstream, size); if (buf) { crc32_add((uint8_t *)buf, *size, &elt->crc); } else { g_debug("sending XMSG_CRC message"); g_debug("crc pull_buffer CRC: %08x", crc32_finish(&elt->crc)); msg = xmsg_new(elt, XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(elt->xfer, msg); } return buf; }
static void push_buffer_impl( XferElement *elt, gpointer buf, size_t len) { XferFilterCrc *self = (XferFilterCrc *)elt; XMsg *msg; /* drop the buffer if we've been cancelled */ if (elt->cancelled) { amfree(buf); return; } /* crc the given buffer and pass it downstream */ if (buf) { crc32_add((uint8_t *)buf, len, &elt->crc); } else { g_debug("sending XMSG_CRC message to %p", elt); g_debug("crc push_buffer CRC: %08x", crc32_finish(&elt->crc)); msg = xmsg_new(elt, XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(elt->xfer, msg); } xfer_element_push_buffer(XFER_ELEMENT(self)->downstream, buf, len); }
static void pull_and_write(XferElementGlue *self) { XferElement *elt = XFER_ELEMENT(self); int fd = get_write_fd(self); XMsg *msg; size_t written; self->write_fdp = NULL; while (!elt->cancelled) { size_t len; char *buf; /* get a buffer from upstream */ buf = xfer_element_pull_buffer(elt->upstream, &len); if (!buf) break; /* write it */ if (!elt->downstream->drain_mode) { written = full_write(fd, buf, len); if (written < len) { if (elt->downstream->must_drain) { g_debug("Error writing to fd %d: %s", fd, strerror(errno)); } else if (elt->downstream->ignore_broken_pipe && errno == EPIPE) { } else { if (!elt->cancelled) { xfer_cancel_with_error(elt, _("Error writing to fd %d: %s"), fd, strerror(errno)); xfer_cancel(elt->xfer); wait_until_xfer_cancelled(elt->xfer); } amfree(buf); break; } elt->downstream->drain_mode = TRUE; } } crc32_add((uint8_t *)buf, len, &elt->crc); amfree(buf); } if (elt->cancelled && elt->expect_eof) xfer_element_drain_buffers(elt->upstream); g_debug("sending XMSG_CRC message %p", elt->downstream); g_debug("pull_and_write CRC: %08x size %lld", crc32_finish(&elt->crc), (long long)elt->crc.size); msg = xmsg_new(elt->downstream, XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(elt->xfer, msg); /* close the fd we've been writing, as an EOF signal to downstream, and * set it to -1 to avoid accidental re-use */ close_write_fd(self); }
static void read_and_push( XferElementGlue *self) { XferElement *elt = XFER_ELEMENT(self); int fd = get_read_fd(self); XMsg *msg; crc32_init(&elt->crc); while (!elt->cancelled) { char *buf = g_malloc(GLUE_BUFFER_SIZE); gsize len; int read_error; /* read a buffer from upstream */ len = read_fully(fd, buf, GLUE_BUFFER_SIZE, &read_error); if (len < GLUE_BUFFER_SIZE) { if (read_error) { if (!elt->cancelled) { xfer_cancel_with_error(elt, _("Error reading from fd %d: %s"), fd, strerror(read_error)); g_debug("element-glue: error reading from fd %d: %s", fd, strerror(read_error)); wait_until_xfer_cancelled(elt->xfer); } amfree(buf); break; } else if (len == 0) { /* we only count a zero-length read as EOF */ amfree(buf); break; } } crc32_add((uint8_t *)buf, len, &elt->crc); xfer_element_push_buffer(elt->downstream, buf, len); } if (elt->cancelled && elt->expect_eof) xfer_element_drain_fd(fd); /* send an EOF indication downstream */ xfer_element_push_buffer(elt->downstream, NULL, 0); /* close the read fd, since it's at EOF */ close_read_fd(self); g_debug("sending XMSG_CRC message"); g_debug("read_and_push CRC: %08x size %lld", crc32_finish(&elt->crc), (long long)elt->crc.size); msg = xmsg_new(elt->upstream, XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(elt->xfer, msg); }
static void push_buffer_impl( XferElement *elt, gpointer buf, size_t len) { XferDestNull *self = (XferDestNull *)elt; if (buf) { crc32_add(buf, len, &elt->crc); } else { XMsg *msg = xmsg_new((XferElement *)self, XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(XFER_ELEMENT(self)->xfer, msg); return; } if (self->do_verify && !elt->cancelled) { if (!simpleprng_verify_buffer(&self->prng, buf, len)) { xfer_cancel_with_error(elt, "verification of incoming bytestream failed; see stderr for details"), wait_until_xfer_cancelled(elt->xfer); amfree(buf); return; } } self->byte_position += len; if (!self->sent_info) { /* send a superfluous message (this is a testing XferElement, * after all) */ XMsg *msg = xmsg_new((XferElement *)self, XMSG_INFO, 0); msg->message = g_strdup("Is this thing on?"); xfer_queue_message(XFER_ELEMENT(self)->xfer, msg); self->sent_info = TRUE; } amfree(buf); }
static void read_and_write(XferElementGlue *self) { XferElement *elt = XFER_ELEMENT(self); /* dynamically allocate a buffer, in case this thread has * a limited amount of stack allocated */ char *buf = g_malloc(GLUE_BUFFER_SIZE); int rfd = get_read_fd(self); int wfd = get_write_fd(self); XMsg *msg; crc32_init(&elt->crc); g_debug("read_and_write: read from %d, write to %d", rfd, wfd); while (!elt->cancelled) { size_t len; /* read from upstream */ len = read_fully(rfd, buf, GLUE_BUFFER_SIZE, NULL); if (len < GLUE_BUFFER_SIZE) { if (errno) { if (!elt->cancelled) { xfer_cancel_with_error(elt, _("Error reading from fd %d: %s"), rfd, strerror(errno)); wait_until_xfer_cancelled(elt->xfer); } break; } else if (len == 0) { /* we only count a zero-length read as EOF */ break; } } /* write the buffer fully */ if (!elt->downstream->drain_mode && full_write(wfd, buf, len) < len) { if (elt->downstream->must_drain) { g_debug("Could not write to fd %d: %s", wfd, strerror(errno)); } else if (elt->downstream->ignore_broken_pipe && errno == EPIPE) { } else { if (!elt->cancelled) { xfer_cancel_with_error(elt, _("Could not write to fd %d: %s"), wfd, strerror(errno)); wait_until_xfer_cancelled(elt->xfer); } break; } } crc32_add((uint8_t *)buf, len, &elt->crc); } if (elt->cancelled && elt->expect_eof) xfer_element_drain_fd(rfd); /* close the read fd. If it's not at EOF, then upstream will get EPIPE, which will hopefully * kill it and complete the cancellation */ close_read_fd(self); /* close the fd we've been writing, as an EOF signal to downstream */ close_write_fd(self); g_debug("read_and_write upstream CRC: %08x size %lld", crc32_finish(&elt->crc), (long long)elt->crc.size); g_debug("sending XMSG_CRC message"); msg = xmsg_new(elt->upstream, XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(elt->xfer, msg); g_debug("read_and_write downstream CRC: %08x size %lld", crc32_finish(&elt->crc), (long long)elt->crc.size); g_debug("sending XMSG_CRC message"); msg = xmsg_new(elt->downstream, XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(elt->xfer, msg); amfree(buf); }
static void push_buffer_impl( XferElement *elt, gpointer buf, size_t len) { XferElementGlue *self = (XferElementGlue *)elt; XMsg *msg; /* accept first, if required */ if (self->on_push & PUSH_ACCEPT_FIRST) { /* don't accept the next time around */ self->on_push &= ~PUSH_ACCEPT_FIRST; if (elt->cancelled) { return; } if ((self->output_data_socket = do_directtcp_accept(self, &self->output_listen_socket)) == -1) { /* do_directtcp_accept already signalled an error; xfer * is cancelled */ return; } /* write to this new socket */ self->write_fdp = &self->output_data_socket; } /* or connect first, if required */ if (self->on_push & PUSH_CONNECT_FIRST) { /* don't accept the next time around */ self->on_push &= ~PUSH_CONNECT_FIRST; if (elt->cancelled) { return; } if ((self->output_data_socket = do_directtcp_connect(self, elt->downstream->input_listen_addrs)) == -1) { /* do_directtcp_connect already signalled an error; xfer * is cancelled */ return; } /* read from this new socket */ self->write_fdp = &self->output_data_socket; } switch (self->on_push) { case PUSH_TO_RING_BUFFER: /* just drop packets if the transfer has been cancelled */ if (elt->cancelled) { amfree(buf); return; } /* make sure there's at least one element free */ amsemaphore_down(self->ring_free_sem); /* set it */ self->ring[self->ring_head].buf = buf; self->ring[self->ring_head].size = len; self->ring_head = (self->ring_head + 1) % GLUE_RING_BUFFER_SIZE; /* and mark this element as available for reading */ amsemaphore_up(self->ring_used_sem); return; case PUSH_TO_FD: { int fd = get_write_fd(self); /* if the fd is already closed, it's possible upstream bailed out * so quickly that we didn't even get a look at the fd. In this * case we can assume the xfer has been cancelled and just discard * the data. */ if (fd == -1) return; if (elt->cancelled) { if (!elt->expect_eof || !buf) { close_write_fd(self); /* hack to ensure we won't close the fd again, if we get another push */ elt->expect_eof = TRUE; } amfree(buf); return; } /* write the full buffer to the fd, or close on EOF */ if (buf) { if (!elt->downstream->drain_mode && full_write(fd, buf, len) < len) { if (elt->downstream->must_drain) { g_debug("Error writing to fd %d: %s", fd, strerror(errno)); } else if (elt->downstream->ignore_broken_pipe && errno == EPIPE) { } else { if (!elt->cancelled) { xfer_cancel_with_error(elt, _("Error writing to fd %d: %s"), fd, strerror(errno)); wait_until_xfer_cancelled(elt->xfer); } /* nothing special to do to handle a cancellation */ } elt->downstream->drain_mode = TRUE; } crc32_add((uint8_t *)buf, len, &elt->crc); amfree(buf); } else { g_debug("sending XMSG_CRC message"); g_debug("push_to_fd CRC: %08x", crc32_finish(&elt->crc)); msg = xmsg_new(elt->downstream, XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(elt->xfer, msg); close_write_fd(self); } return; } default: case PUSH_INVALID: g_assert_not_reached(); break; } }
/* Write an entire chunk. Called with the state_mutex held */ static gboolean holding_thread_write_chunk( XferDestHolding *self) { XferElement *elt = XFER_ELEMENT(self); self->chunk_status = CHUNK_OK; g_mutex_lock(self->ring_mutex); while (1) { gsize to_write; size_t count; /* wait for at least one block, and (if necessary) prebuffer */ to_write = holding_thread_wait_for_block(self); to_write = MIN(to_write, HOLDING_BLOCK_BYTES); if (elt->cancelled) break; if (to_write == 0) { self->chunk_status = CHUNK_EOF; break; } if (self->chunk_status == CHUNK_EOC) { break; } to_write = MIN(to_write, self->use_bytes); DBG(8, "writing %ju bytes to holding", (uintmax_t)to_write); /* note that it's OK to reference these ring_* vars here, as they * are static at this point */ g_mutex_unlock(self->ring_mutex); count = db_full_write(self->fd, self->ring_buffer + self->ring_tail, (guint)to_write); g_mutex_lock(self->ring_mutex); if (count != to_write) { if (count > 0) { if (ftruncate(self->fd, self->chunk_offset) != 0) { g_debug("ftruncate failed: %s", strerror(errno)); g_mutex_unlock(self->ring_mutex); return FALSE; } } self->chunk_status = CHUNK_NO_ROOM; break; } crc32_add((uint8_t *)(self->ring_buffer + self->ring_tail), to_write, &elt->crc); self->chunk_offset += count; self->data_bytes_written += count; self->use_bytes -= count; holding_thread_consume_block(self, count); if (self->use_bytes <= 0) { self->chunk_status = CHUNK_EOC; /* loop to see if more data is available * chunk_status might become CHUNK_EOF if at end of input file */ } } g_mutex_unlock(self->ring_mutex); /* if we write all of the blocks, but the finish_file fails, then likely * there was some buffering going on in the holding driver, and the blocks * did not all make it to permanent storage -- so it's a failed part. Note * that we try to finish_file even if the part failed, just to be thorough. */ if (elt->cancelled) { return FALSE; } return TRUE; }
static gpointer pull_buffer_impl( XferElement *elt, size_t *size) { XferSourceRecovery *self = XFER_SOURCE_RECOVERY(elt); gpointer buf = NULL; int result; int devsize; XMsg *msg; g_assert(elt->output_mech == XFER_MECH_PULL_BUFFER); g_mutex_lock(self->start_part_mutex); if (elt->size == 0) { if (elt->offset == 0 && elt->orig_size == 0) { self->paused = TRUE; } else { DBG(2, "xfer-source-recovery sending XMSG_CRC message"); DBG(2, "xfer-source-recovery CRC: %08x size %lld", crc32_finish(&elt->crc), (long long)elt->crc.size); msg = xmsg_new(XFER_ELEMENT(self), XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(elt->xfer, msg); /* the device has signalled EOF (really end-of-part), so clean up instance * variables and report the EOP to the caller in the form of an xmsg */ DBG(2, "pull_buffer hit EOF; sending XMSG_SEGMENT_DONE"); msg = xmsg_new(XFER_ELEMENT(self), XMSG_SEGMENT_DONE, 0); msg->size = self->part_size; if (self->part_timer) { msg->duration = g_timer_elapsed(self->part_timer, NULL); g_timer_destroy(self->part_timer); self->part_timer = NULL; } msg->partnum = 0; msg->fileno = self->device->file; msg->successful = TRUE; msg->eof = FALSE; self->paused = TRUE; device_clear_bytes_read(self->device); self->bytes_read += self->part_size; self->part_size = 0; self->block_size = 0; /* don't queue the XMSG_PART_DONE until we've adjusted all of our * instance variables appropriately */ xfer_queue_message(elt->xfer, msg); if (self->device->is_eof) { DBG(2, "pull_buffer hit EOF; sending XMSG_PART_DONE"); msg = xmsg_new(XFER_ELEMENT(self), XMSG_PART_DONE, 0); msg->size = self->part_size; if (self->part_timer) { msg->duration = g_timer_elapsed(self->part_timer, NULL); g_timer_destroy(self->part_timer); self->part_timer = NULL; } msg->partnum = 0; msg->fileno = self->device->file; msg->successful = TRUE; msg->eof = FALSE; xfer_queue_message(elt->xfer, msg); } } } while (1) { /* make sure we have a device */ while (self->paused && !elt->cancelled) g_cond_wait(self->start_part_cond, self->start_part_mutex); /* indicate EOF on an cancel or when there are no more parts */ if (elt->cancelled) { goto error; } if (self->done) goto error; /* start the timer if this is the first pull_buffer of this part */ if (!self->part_timer) { DBG(2, "first pull_buffer of new part"); self->part_timer = g_timer_new(); } if (elt->size == 0) { result = -1; } else { /* loop until we read a full block, in case the blocks are larger * than expected */ if (self->block_size == 0) self->block_size = (size_t)self->device->block_size; do { int max_block; buf = g_malloc(self->block_size); if (buf == NULL) { xfer_cancel_with_error(elt, _("%s: cannot allocate memory"), self->device->device_name); g_mutex_unlock(self->start_part_mutex); wait_until_xfer_cancelled(elt->xfer); goto error_unlocked; } devsize = (int)self->block_size; if (elt->size < 0) max_block = -1; else max_block = (elt->size+self->block_size-1)/self->block_size; result = device_read_block(self->device, buf, &devsize, max_block); *size = devsize; if (result == 0) { g_assert(*size > self->block_size); self->block_size = devsize; amfree(buf); } } while (result == 0); if (result > 0 && (elt->offset || (elt->size > 0 && (long long unsigned)elt->size < *size))) { gpointer buf1 = g_malloc(self->block_size); if ((long long unsigned)elt->offset > *size) { g_debug("offset > *size"); } else if ((long long unsigned)elt->offset == *size) { g_debug("offset == *size"); } *size -= elt->offset; if (elt->size > 0 && (size_t)elt->size < *size) *size = elt->size; memmove(buf1, buf + elt->offset, *size); elt->offset = 0; g_free(buf); buf = buf1; } if (result > 0) elt->size -= *size; } /* if this block was successful, return it */ if (result > 0) { self->part_size += *size; break; } if (result < 0) { amfree(buf); /* if we're not at EOF, it's an error */ if (!self->device->is_eof && elt->size != 0) { xfer_cancel_with_error(elt, _("error reading from %s: %s"), self->device->device_name, device_error_or_status(self->device)); g_mutex_unlock(self->start_part_mutex); wait_until_xfer_cancelled(elt->xfer); goto error_unlocked; } DBG(2, "xfer-source-recovery sending XMSG_CRC message"); DBG(2, "xfer-source-recovery CRC: %08x size %lld", crc32_finish(&elt->crc), (long long)elt->crc.size); msg = xmsg_new(XFER_ELEMENT(self), XMSG_CRC, 0); msg->crc = crc32_finish(&elt->crc); msg->size = elt->crc.size; xfer_queue_message(elt->xfer, msg); /* the device has signalled EOF (really end-of-part), so clean up instance * variables and report the EOP to the caller in the form of an xmsg */ DBG(2, "pull_buffer hit EOF; sending XMSG_PART_DONE"); msg = xmsg_new(XFER_ELEMENT(self), XMSG_PART_DONE, 0); msg->size = self->part_size; msg->duration = g_timer_elapsed(self->part_timer, NULL); msg->partnum = 0; msg->fileno = self->device->file; msg->successful = TRUE; msg->eof = FALSE; self->paused = TRUE; self->bytes_read += self->part_size; device_clear_bytes_read(self->device); self->part_size = 0; self->block_size = 0; if (self->part_timer) { g_timer_destroy(self->part_timer); self->part_timer = NULL; } /* don't queue the XMSG_PART_DONE until we've adjusted all of our * instance variables appropriately */ xfer_queue_message(elt->xfer, msg); if (elt->size == 0) { g_mutex_unlock(self->start_part_mutex); return NULL; } } } g_mutex_unlock(self->start_part_mutex); if (buf) { crc32_add(buf, *size, &elt->crc); } return buf; error: g_mutex_unlock(self->start_part_mutex); error_unlocked: *size = 0; return NULL; }