static int
do_directtcp_accept(
XferElementGlue *self,
int *socketp)
{
int sock;
g_assert(*socketp != -1);
if ((sock = interruptible_accept(*socketp, NULL, NULL,
prolong_accept, self)) == -1) {
/* if the accept was interrupted due to a cancellation, then do not
* add a further error message */
if (errno == 0 && XFER_ELEMENT(self)->cancelled)
return -1;
xfer_cancel_with_error(XFER_ELEMENT(self),
_("Error accepting incoming connection: %s"), strerror(errno));
wait_until_xfer_cancelled(XFER_ELEMENT(self)->xfer);
return -1;
}
/* close the listening socket now, for good measure */
close(*socketp);
*socketp = -1;
g_debug("do_directtcp_accept: %d", sock);
return sock;
}
static gpointer
pull_buffer_impl(
XferElement *elt,
size_t *size)
{
XferFilterCrc *self = (XferFilterCrc *)elt;
char *buf;
XMsg *msg;
if (elt->cancelled) {
/* drain our upstream only if we're expecting an EOF */
if (elt->expect_eof) {
xfer_element_drain_buffers(XFER_ELEMENT(self)->upstream);
}
/* return an EOF */
*size = 0;
return NULL;
}
/* get a buffer from upstream, crc it, and hand it back */
buf = xfer_element_pull_buffer(XFER_ELEMENT(self)->upstream, size);
if (buf) {
crc32_add((uint8_t *)buf, *size, &elt->crc);
} else {
g_debug("sending XMSG_CRC message");
g_debug("crc pull_buffer CRC: %08x",
crc32_finish(&elt->crc));
msg = xmsg_new(elt, XMSG_CRC, 0);
msg->crc = crc32_finish(&elt->crc);
msg->size = elt->crc.size;
xfer_queue_message(elt->xfer, msg);
}
return buf;
}
static void
start_part_impl(
XferSourceRecovery *self,
Device *device)
{
XferElement *elt = XFER_ELEMENT(self);
g_assert(!device || device->in_file);
DBG(2, "start_part called");
if (self->device_bad) {
/* use_device didn't like the device it got, but the xfer cancellation
* has not completed yet, so do nothing */
return;
}
g_mutex_lock(self->start_part_mutex);
/* make sure we're ready to go */
g_assert(self->paused || self->done);
self->done = FALSE;
if (XFER_ELEMENT(self)->output_mech == XFER_MECH_DIRECTTCP_CONNECT
|| XFER_ELEMENT(self)->output_mech == XFER_MECH_DIRECTTCP_LISTEN) {
g_assert(self->conn != NULL);
}
/* if we already have a device, it should have been given to use_device */
if (device && self->device) {
g_assert(self->device == device);
} else if (device) {
self->device = device;
g_object_ref(device);
}
if (!device)
self->done = TRUE;
if (elt->offset == 0 && elt->orig_size == 0) {
self->done = TRUE;
g_mutex_unlock(self->start_part_mutex);
return;
}
if (elt->size == 0) {
self->done = TRUE;
g_mutex_unlock(self->start_part_mutex);
return;
}
self->paused = FALSE;
DBG(2, "triggering condition variable");
g_cond_broadcast(self->start_part_cond);
g_mutex_unlock(self->start_part_mutex);
}
static void
use_device_impl(
XferSourceRecovery *xdtself,
Device *device)
{
XferSourceRecovery *self = XFER_SOURCE_RECOVERY(xdtself);
g_assert(self->paused);
/* short-circuit if nothing is changing */
if (self->device == device)
return;
if (self->device)
g_object_unref(self->device);
self->device = NULL;
/* if we already have a connection, then make this device use it */
if (self->conn) {
if (!device_use_connection(device, self->conn)) {
/* queue up an error for later, and set device_bad.
* start_part will see this and fail silently */
self->device_bad = TRUE;
xfer_cancel_with_error(XFER_ELEMENT(self),
_("Cannot continue onto new volume: %s"),
device_error_or_status(device));
return;
}
}
self->device = device;
g_object_ref(device);
}
static void
pull_and_push(XferElementGlue *self)
{
XferElement *elt = XFER_ELEMENT(self);
gboolean eof_sent = FALSE;
while (!elt->cancelled) {
char *buf;
size_t len;
/* get a buffer from upstream */
buf = xfer_element_pull_buffer(elt->upstream, &len);
/* and push it downstream */
xfer_element_push_buffer(elt->downstream, buf, len);
if (!buf) {
eof_sent = TRUE;
break;
}
}
if (elt->cancelled && elt->expect_eof)
xfer_element_drain_buffers(elt->upstream);
if (!eof_sent)
xfer_element_push_buffer(elt->downstream, NULL, 0);
}
static void
push_buffer_impl(
XferElement *elt,
gpointer buf,
size_t len)
{
XferFilterCrc *self = (XferFilterCrc *)elt;
XMsg *msg;
/* drop the buffer if we've been cancelled */
if (elt->cancelled) {
amfree(buf);
return;
}
/* crc the given buffer and pass it downstream */
if (buf) {
crc32_add((uint8_t *)buf, len, &elt->crc);
} else {
g_debug("sending XMSG_CRC message to %p", elt);
g_debug("crc push_buffer CRC: %08x",
crc32_finish(&elt->crc));
msg = xmsg_new(elt, XMSG_CRC, 0);
msg->crc = crc32_finish(&elt->crc);
msg->size = elt->crc.size;
xfer_queue_message(elt->xfer, msg);
}
xfer_element_push_buffer(XFER_ELEMENT(self)->downstream, buf, len);
}
/* Wait for at least one block, or EOF, to be available in the ring buffer.
* Called with the ring mutex held. */
static gsize
holding_thread_wait_for_block(
XferDestHolding *self)
{
XferElement *elt = XFER_ELEMENT(self);
gsize bytes_needed = HOLDING_BLOCK_BYTES;
gsize usable;
while (1) {
/* are we ready? */
if (elt->cancelled)
break;
if (self->ring_count >= bytes_needed)
break;
if (self->ring_head_at_eof)
break;
/* nope - so wait */
g_cond_wait(self->ring_add_cond, self->ring_mutex);
}
usable = MIN(self->ring_count, bytes_needed);
return usable;
}
static void
send_xfer_done(
XferElementGlue *self)
{
xfer_queue_message(XFER_ELEMENT(self)->xfer,
xmsg_new((XferElement *)self, XMSG_DONE, 0));
}
XferElement *
xfer_dest_holding(
size_t max_memory)
{
XferDestHolding *self = (XferDestHolding *)g_object_new(XFER_DEST_HOLDING_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
char *env;
/* max_memory get rounded up to the next multiple of block_size */
max_memory = ((max_memory + HOLDING_BLOCK_BYTES - 1)
/ HOLDING_BLOCK_BYTES) * HOLDING_BLOCK_BYTES;
self->paused = TRUE;
/* set up a ring buffer of size max_memory */
self->ring_length = max_memory;
self->ring_buffer = g_malloc(max_memory);
self->ring_head = self->ring_tail = 0;
self->ring_count = 0;
self->ring_head_at_eof = 0;
/* set up a fake ENOSPC for testing purposes. Note that this counts
* headers as well as data written to disk. */
env = getenv("CHUNKER_FAKE_ENOSPC_AT");
if (env) {
fake_enospc_at_byte = (off_t)atoi(env); /* these values are never > MAXINT */
db_full_write = full_write_with_fake_enospc;
DBG(1,"will trigger fake ENOSPC at byte %d", (int)fake_enospc_at_byte);
} else {
db_full_write = full_write;
}
return elt;
}
static void
pull_and_write(XferElementGlue *self)
{
XferElement *elt = XFER_ELEMENT(self);
int fd = get_write_fd(self);
XMsg *msg;
size_t written;
self->write_fdp = NULL;
while (!elt->cancelled) {
size_t len;
char *buf;
/* get a buffer from upstream */
buf = xfer_element_pull_buffer(elt->upstream, &len);
if (!buf)
break;
/* write it */
if (!elt->downstream->drain_mode) {
written = full_write(fd, buf, len);
if (written < len) {
if (elt->downstream->must_drain) {
g_debug("Error writing to fd %d: %s", fd, strerror(errno));
} else if (elt->downstream->ignore_broken_pipe && errno == EPIPE) {
} else {
if (!elt->cancelled) {
xfer_cancel_with_error(elt,
_("Error writing to fd %d: %s"), fd, strerror(errno));
xfer_cancel(elt->xfer);
wait_until_xfer_cancelled(elt->xfer);
}
amfree(buf);
break;
}
elt->downstream->drain_mode = TRUE;
}
}
crc32_add((uint8_t *)buf, len, &elt->crc);
amfree(buf);
}
if (elt->cancelled && elt->expect_eof)
xfer_element_drain_buffers(elt->upstream);
g_debug("sending XMSG_CRC message %p", elt->downstream);
g_debug("pull_and_write CRC: %08x size %lld",
crc32_finish(&elt->crc), (long long)elt->crc.size);
msg = xmsg_new(elt->downstream, XMSG_CRC, 0);
msg->crc = crc32_finish(&elt->crc);
msg->size = elt->crc.size;
xfer_queue_message(elt->xfer, msg);
/* close the fd we've been writing, as an EOF signal to downstream, and
* set it to -1 to avoid accidental re-use */
close_write_fd(self);
}
static void
child_watch_callback(
pid_t pid,
gint status,
gpointer data)
{
XferFilterProcess *self = XFER_FILTER_PROCESS(data);
XferElement *elt = (XferElement *)self;
XMsg *msg;
char *errmsg = NULL;
g_assert(pid == self->child_pid);
self->child_pid = -1; /* it's gone now.. */
if (WIFEXITED(status)) {
int exitcode = WEXITSTATUS(status);
g_debug("%s: process exited with status %d", xfer_element_repr(elt), exitcode);
if (exitcode != 0) {
errmsg = g_strdup_printf("%s exited with status %d",
self->argv[0], exitcode);
}
} else if (WIFSIGNALED(status)) {
int signal = WTERMSIG(status);
if (signal != SIGKILL || !self->child_killed) {
errmsg = g_strdup_printf("%s died on signal %d", self->argv[0], signal);
g_debug("%s: %s", xfer_element_repr(elt), errmsg);
}
}
if (errmsg) {
msg = xmsg_new(XFER_ELEMENT(self), XMSG_INFO, 0);
msg->message = g_strdup("ERROR");
xfer_queue_message(XFER_ELEMENT(self)->xfer, msg);
} else {
msg = xmsg_new(XFER_ELEMENT(self), XMSG_INFO, 0);
msg->message = g_strdup("SUCCESS");
xfer_queue_message(XFER_ELEMENT(self)->xfer, msg);
}
/* if this is an error exit, send an XMSG_ERROR and cancel */
if (!elt->cancelled) {
if (errmsg) {
msg = xmsg_new(XFER_ELEMENT(self), XMSG_ERROR, 0);
msg->message = errmsg;
xfer_queue_message(XFER_ELEMENT(self)->xfer, msg);
xfer_cancel(elt->xfer);
} else if (elt->cancel_on_success) {
xfer_cancel(elt->xfer);
}
}
/* this element is as good as cancelled already, so fall through to XMSG_DONE */
xfer_queue_message(XFER_ELEMENT(self)->xfer, xmsg_new(XFER_ELEMENT(self), XMSG_DONE, 0));
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_source_directtcp_listen(void)
{
XferSourceDirectTCPListen *xsr = (XferSourceDirectTCPListen *)g_object_new(XFER_SOURCE_DIRECTTCP_LISTEN_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(xsr);
return elt;
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_element_glue(void)
{
XferElementGlue *self = (XferElementGlue *)g_object_new(XFER_ELEMENT_GLUE_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
return elt;
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_filter_crc(void)
{
XferFilterCrc *xfx = (XferFilterCrc *)g_object_new(XFER_FILTER_CRC_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(xfx);
return elt;
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_dest_directtcp_listen(void)
{
XferDestDirectTCPListen *self = (XferDestDirectTCPListen *)
g_object_new(XFER_DEST_DIRECTTCP_LISTEN_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
return elt;
}
static void
read_and_push(
XferElementGlue *self)
{
XferElement *elt = XFER_ELEMENT(self);
int fd = get_read_fd(self);
XMsg *msg;
crc32_init(&elt->crc);
while (!elt->cancelled) {
char *buf = g_malloc(GLUE_BUFFER_SIZE);
gsize len;
int read_error;
/* read a buffer from upstream */
len = read_fully(fd, buf, GLUE_BUFFER_SIZE, &read_error);
if (len < GLUE_BUFFER_SIZE) {
if (read_error) {
if (!elt->cancelled) {
xfer_cancel_with_error(elt,
_("Error reading from fd %d: %s"), fd, strerror(read_error));
g_debug("element-glue: error reading from fd %d: %s",
fd, strerror(read_error));
wait_until_xfer_cancelled(elt->xfer);
}
amfree(buf);
break;
} else if (len == 0) { /* we only count a zero-length read as EOF */
amfree(buf);
break;
}
}
crc32_add((uint8_t *)buf, len, &elt->crc);
xfer_element_push_buffer(elt->downstream, buf, len);
}
if (elt->cancelled && elt->expect_eof)
xfer_element_drain_fd(fd);
/* send an EOF indication downstream */
xfer_element_push_buffer(elt->downstream, NULL, 0);
/* close the read fd, since it's at EOF */
close_read_fd(self);
g_debug("sending XMSG_CRC message");
g_debug("read_and_push CRC: %08x size %lld",
crc32_finish(&elt->crc), (long long)elt->crc.size);
msg = xmsg_new(elt->upstream, XMSG_CRC, 0);
msg->crc = crc32_finish(&elt->crc);
msg->size = elt->crc.size;
xfer_queue_message(elt->xfer, msg);
}
gboolean
xfer_source_recovery_cancel(
XferElement *elt,
gboolean expect_eof G_GNUC_UNUSED)
{
XferElementClass *klass;
g_assert(IS_XFER_SOURCE_RECOVERY(elt));
klass = XFER_ELEMENT_GET_CLASS(elt);
return klass->cancel(XFER_ELEMENT(elt), 0);
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_dest_buffer(
gsize max_size)
{
XferDestBuffer *self = (XferDestBuffer *)g_object_new(XFER_DEST_BUFFER_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
self->max_size = max_size;
return elt;
}
/* create an element of this class; prototype is in xfer-device.h */
XferElement *
xfer_source_recovery(Device *first_device)
{
XferSourceRecovery *self = (XferSourceRecovery *)g_object_new(XFER_SOURCE_RECOVERY_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
g_assert(first_device != NULL);
g_object_ref(first_device);
self->device = first_device;
return elt;
}
static void
push_buffer_impl(
XferElement *elt,
gpointer buf,
size_t len)
{
XferDestNull *self = (XferDestNull *)elt;
if (buf) {
crc32_add(buf, len, &elt->crc);
} else {
XMsg *msg = xmsg_new((XferElement *)self, XMSG_CRC, 0);
msg->crc = crc32_finish(&elt->crc);
msg->size = elt->crc.size;
xfer_queue_message(XFER_ELEMENT(self)->xfer, msg);
return;
}
if (self->do_verify && !elt->cancelled) {
if (!simpleprng_verify_buffer(&self->prng, buf, len)) {
xfer_cancel_with_error(elt,
"verification of incoming bytestream failed; see stderr for details"),
wait_until_xfer_cancelled(elt->xfer);
amfree(buf);
return;
}
}
self->byte_position += len;
if (!self->sent_info) {
/* send a superfluous message (this is a testing XferElement,
* after all) */
XMsg *msg = xmsg_new((XferElement *)self, XMSG_INFO, 0);
msg->message = g_strdup("Is this thing on?");
xfer_queue_message(XFER_ELEMENT(self)->xfer, msg);
self->sent_info = TRUE;
}
amfree(buf);
}
/* create an element of this class; prototype is in xfer-device.h */
XferElement *
xfer_source_device(
Device *device)
{
XferSourceDevice *self = (XferSourceDevice *)g_object_new(XFER_SOURCE_DEVICE_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
g_assert(device != NULL);
self->device = device;
return elt;
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_source_random(
guint64 length,
guint32 prng_seed)
{
XferSourceRandom *xsr = (XferSourceRandom *)g_object_new(XFER_SOURCE_RANDOM_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(xsr);
xsr->length = length;
xsr->limited_length = (length != 0);
simpleprng_seed(&xsr->prng, prng_seed);
return elt;
}
/* create an element of this class; prototype is in xfer-device.h */
XferElement *
xfer_dest_device(
Device *device,
gboolean cancel_at_leom)
{
XferDestDevice *self = (XferDestDevice *)g_object_new(XFER_DEST_DEVICE_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
g_assert(device != NULL);
self->device = device;
self->cancel_at_leom = cancel_at_leom;
return elt;
}
static int
_get_read_fd(XferElementGlue *self)
{
assert(self->read_fdp);
if (self->read_fdp == &neighboring_element_fd) {
XferElement *elt = XFER_ELEMENT(self);
self->read_fd = xfer_element_swap_output_fd(elt->upstream, -1);
} else {
self->read_fd = *self->read_fdp;
*self->read_fdp = -1;
}
self->read_fdp = NULL;
return self->read_fd;
}
static gpointer
directtcp_listen_thread(
gpointer data)
{
XferSourceRecovery *self = XFER_SOURCE_RECOVERY(data);
XferElement *elt = XFER_ELEMENT(self);
int result;
DBG(1, "(this is directtcp_listen_thread)");
/* we need to make an outgoing connection to downstream; we do this while
* holding the start_part_mutex, so that a part doesn't get started until
* we're finished with the device */
g_mutex_lock(self->start_part_mutex);
if (elt->cancelled) {
g_mutex_unlock(self->start_part_mutex);
goto send_done;
}
g_assert(self->device != NULL); /* have a device */
g_assert(elt->downstream->input_listen_addrs != NULL); /* downstream listening */
DBG(2, "making DirectTCP connection on device %s", self->device->device_name);
result = device_connect(self->device, FALSE,
elt->downstream->input_listen_addrs,
&self->conn, &elt->cancelled,
self->start_part_mutex, self->abort_cond);
if (result == 1 && !elt->cancelled) {
xfer_cancel_with_error(elt,
_("error making DirectTCP connection: %s"),
device_error_or_status(self->device));
g_mutex_unlock(self->start_part_mutex);
wait_until_xfer_cancelled(elt->xfer);
goto send_done;
} else if (result == 2 || elt->cancelled) {
g_mutex_unlock(self->start_part_mutex);
wait_until_xfer_cancelled(elt->xfer);
goto send_done;
}
DBG(2, "DirectTCP connect succeeded");
return directtcp_common_thread(self);
send_done:
xfer_queue_message(elt->xfer, xmsg_new(elt, XMSG_DONE, 0));
return NULL;
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_dest_fd(
int fd)
{
XferDestFd *self = (XferDestFd *)g_object_new(XFER_DEST_FD_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
int old_fd;
g_assert(fd >= 0);
/* we keep a *copy* of this fd, because our caller will close it to
* indicate EOF */
old_fd = xfer_element_swap_input_fd(elt, dup(fd));
g_assert(old_fd == -1);
return elt;
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_dest_null(
guint32 prng_seed)
{
XferDestNull *self = (XferDestNull *)g_object_new(XFER_DEST_NULL_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
if (prng_seed) {
self->do_verify = TRUE;
simpleprng_seed(&self->prng, prng_seed);
} else {
self->do_verify = FALSE;
}
crc32_init(&elt->crc);
return elt;
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_source_fd(
int fd)
{
XferSourceFd *self = (XferSourceFd *)g_object_new(XFER_SOURCE_FD_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(self);
int old_fd;
g_assert(fd >= 0);
/* we read from a *copy* of this file descriptor, as the downstream element
* will close output_fd on EOF */
old_fd = xfer_element_swap_output_fd(elt, dup(fd));
g_assert(old_fd == -1);
return elt;
}
static gboolean
do_block(
XferDestDevice *self,
guint size,
gpointer data)
{
XferElement *elt = XFER_ELEMENT(self);
if (device_write_block(self->device, size, data) != WRITE_SUCCEED) {
xfer_cancel_with_error(elt, "%s: %s",
self->device->device_name, device_error_or_status(self->device));
wait_until_xfer_cancelled(elt->xfer);
return FALSE;
}
/* check for LEOM */
if (self->cancel_at_leom && self->device->is_eom) {
xfer_cancel_with_error(elt, "%s: LEOM detected", self->device->device_name);
wait_until_xfer_cancelled(elt->xfer);
return FALSE;
}
return TRUE;
}
/* create an element of this class; prototype is in xfer-element.h */
XferElement *
xfer_filter_process(
gchar **argv,
gboolean need_root,
gboolean must_drain,
gboolean cancel_on_success,
gboolean ignore_broken_pipe)
{
XferFilterProcess *xfp = (XferFilterProcess *)g_object_new(XFER_FILTER_PROCESS_TYPE, NULL);
XferElement *elt = XFER_ELEMENT(xfp);
if (!argv || !*argv)
error("xfer_filter_process got a NULL or empty argv");
xfp->argv = argv;
xfp->need_root = need_root;
if (pipe(xfp->pipe_err) < 0) {
g_critical(_("Can't create pipe: %s"), strerror(errno));
}
elt->must_drain = must_drain;
elt->cancel_on_success = cancel_on_success;
elt->ignore_broken_pipe = ignore_broken_pipe;
return elt;
}
