static void read_cb(int fd, void *arg) { struct argos_net_conn *conn = arg; /* * We only want to do reads if conn->state == NET_CONN_CONNECTED, but we * don't assert() this because its possible for this socket to be selected * simultaneously for both a read and a write and then for our state to * change during the write attempt. */ if (conn->state != ARGOS_NET_CONN_CONNECTED) return; ssize_t len = recv(conn->sock, buffer_tail(conn->inbuf), buffer_remaining(conn->inbuf), 0); if (len == -1) { if (IS_NETWORK_ERROR(errno)) { /* network error; reset our connection */ orion_log_warn_errno("recv"); reset_connection(conn, 0); } else if (errno == EINTR) { /* don't care; ignore it */ } else { /* anything else is a fatal error */ orion_log_crit_errno("recv"); kill_connection(conn); orion_log_crit("unexpected recv() error; connection is now dead"); } } else if (len == 0) { /* EOF received (maybe other end is shutting down?) */ orion_log_info("EOF received from remote end - closing socket"); if (buffer_len(conn->inbuf) > 0) orion_log_warn("incomplete message received (inbuflen=%d)", buffer_len(conn->inbuf)); reset_connection(conn, 1 /* flush buffers */); } else { /* ok, we read some data into the inbuf; update the buffer */ int rv = buffer_expand(conn->inbuf, len); if (rv == -1) KABOOM("buffer_expand"); conn->bytes_recv += len; /* now process (i.e. look for complete messages in) the inbuf */ process_inbuf(conn); } }
int tcpclient_sendall(tcpclient_t *client, const char *buf, size_t len) { buffer_t *sendq = &client->send_queue; if (client->addr == NULL) { stats_error_log("tcpclient[%s]: Cannot send before connect!", client->name); return 1; } else { // Does nothing if we're already connected, triggers a // reconnect if backoff has expired. tcpclient_connect(client); } if (buffer_datacount(&client->send_queue) >= client->config->max_send_queue) { if (client->failing == 0) { stats_error_log("tcpclient[%s]: send queue for %s client is full (at %zd bytes, max is %" PRIu64 " bytes), dropping data", client->name, tcpclient_state_name[client->state], buffer_datacount(&client->send_queue), client->config->max_send_queue); client->failing = 1; } return 2; } if (buffer_spacecount(sendq) < len) { if (buffer_realign(sendq) != 0) { stats_error_log("tcpclient[%s]: Unable to realign send queue", client->name); return 3; } } while (buffer_spacecount(sendq) < len) { if (buffer_expand(sendq) != 0) { stats_error_log("tcpclient[%s]: Unable to allocate additional memory for send queue, dropping data", client->name); return 4; } } memcpy(buffer_tail(sendq), buf, len); buffer_produced(sendq, len); if (client->state == STATE_CONNECTED) { client->write_watcher.started = true; ev_io_start(client->loop, &client->write_watcher.watcher); } return 0; }
static GstFlowReturn send_buffers_after(AudioTrim *filter, gint64 after) { GstFlowReturn ret = GST_FLOW_OK; GList *b = filter->buffers; while(b) { GstBuffer *buf = b->data; b->data = NULL; if ((gint64)GST_BUFFER_OFFSET_END(buf) <= after) { gst_buffer_unref(buf); } else { if ((gint64)GST_BUFFER_OFFSET(buf) < after) { GstBuffer *tail = buffer_tail(filter, buf, after); gst_buffer_unref(buf); buf = tail; } ret = gst_pad_push(filter->srcpad, buf); if (ret != GST_FLOW_OK) break; } b = g_list_next(b); } release_buffers(filter); return ret; }
static int compress_and_xfer(struct argos_net_conn *conn, u_char force) { /* * if the packet-buf is empty, quit right off; this check is not necessary * for correctness, but its nice to check it early for efficiency and also * to avoid some spam in the logs (e.g. lots of 'compression timeout' * messages) since this case is quite common. */ if (buffer_len(conn->pktbuf) == 0) return 0; #if ARGOS_NET_USE_COMPRESSION == ARGOS_NET_COMPRESS_NONE size_t to_xfer = min(buffer_len(conn->pktbuf), buffer_remaining(conn->outbuf)); if (to_xfer == 0) return 0; #if ARGOS_NET_TRACE_IO struct timeval start; if (gettimeofday(&start, NULL) != 0) { orion_log_crit_errno("gettimeofday"); return 0; } #endif /* #if ARGOS_NET_TRACE_IO */ memcpy(buffer_tail(conn->outbuf), buffer_head(conn->pktbuf), to_xfer); #if ARGOS_NET_TRACE_IO struct timeval end; if (gettimeofday(&end, NULL) != 0) { orion_log_crit_errno("gettimeofday"); return 0; } struct timeval elapsed; orion_time_subtract(&end, &start, &elapsed); float elapsed_msec = elapsed.tv_sec*1000 + (float)elapsed.tv_usec/1000; orion_log_debug("memcpy'ed %u bytes in %.2f ms (%.2f MB/s)", to_xfer, elapsed_msec, ((to_xfer/elapsed_msec)*1000)/(1024*1024)); #endif /* #if ARGOS_NET_TRACE_IO */ if (buffer_expand(conn->outbuf, to_xfer) < 0) KABOOM("buffer_expand"); if (buffer_discard(conn->pktbuf, to_xfer) < 0) KABOOM("buffer_discard"); #else /* #if ARGOS_NET_USE_COMPRESSION == ARGOS_NET_COMPRESS_NONE */ /* * In general, we want large blocks of packets to compress (because that * makes the compression more space-efficient). So if there isn't much data * in pktbuf, just return without doing anything. Note that we also need to * check to make sure there is enough room in the outbuf for us to put the * packets once they are compressed. */ size_t minlen; if (force) { minlen = 1; } else { /* * If this conn has a small pktbuf (such that even with totally full, * the COMPRESS_HARD_MIN won't be met), then we need to adjust to a * limit that is actually attainable; we use 75% of the buffer size. */ minlen = (3*buffer_size(conn->pktbuf))/4; /* usually, this is the value that we end up with for minlen: */ minlen = min(minlen, COMPRESS_HARD_MIN); /* * one more special case: if argos_net_shutdown() was called on this * connection then there is no minimum compression size - we just want * to drain the buffers no matter how much is in there */ if (conn->shutdown) minlen = 1; } /* quit if we don't have at least 'minlen' bytes of packet data to compress */ if (buffer_len(conn->pktbuf) < minlen) return 0; /* check the total space available in the connection outbuf */ size_t total_space = buffer_remaining(conn->outbuf); if (total_space < sizeof(struct argos_net_compress_msg)) return 0; /* not enough space available */ /* this is the total space available for the compressed data */ size_t space = total_space - sizeof(struct argos_net_compress_msg); /* don't exceed the maximum compression-block size */ space = min(space, ARGOS_NET_MAX_COMPRESS_LEN); /* * given the space available, calculate how much packet data we can safely * consume (considering worst-cast input:output size ratios for whatever * compression algorithm we are using). */ ssize_t ok_to_consume; #if ARGOS_NET_USE_COMPRESSION == ARGOS_NET_COMPRESS_LZO /* this is the inversion of the function given in the LZO faq file */ ok_to_consume = (16*(space - 64 - 3))/17; #elif ARGOS_NET_USE_COMPRESSION == ARGOS_NET_COMPRESS_QUICKLZ /* this is the inversion of the function given in the QuickLZ manual */ ok_to_consume = space - 400; #else #error "unknown value for ARGOS_NET_USE_COMPRESSION" #endif if (ok_to_consume <= 0) return 0; /* not enough space available */ /* number of bytes that will actually be compressed and transferred */ size_t readlen = min(ok_to_consume, buffer_len(conn->pktbuf)); assert(readlen > 0); /* don't exceed the maximum compression-block size */ readlen = min(readlen, COMPRESS_HARD_MAX); /* where the compressed data should be written */ u_char *write_ptr = buffer_tail(conn->outbuf) + sizeof(struct argos_net_compress_msg); struct argos_net_compress_msg *msg = (struct argos_net_compress_msg*)buffer_tail(conn->outbuf); msg->msgtype = htons(ARGOS_NET_COMPRESS_MSGTYPE); /* have to defer filling in msglen field */ msg->algorithm = ARGOS_NET_USE_COMPRESSION; msg->orig_len = htonl(readlen); #if ARGOS_NET_TRACE_IO /* measure the elapsed process time to try to detect cpu starvation */ struct itimerval itimer_start; bzero(&itimer_start, sizeof(itimer_start)); itimer_start.it_value.tv_sec = 100; /* arbitrary large value */ struct timeval start; if (gettimeofday(&start, NULL) != 0) { orion_log_crit_errno("gettimeofday"); return 0; } if (setitimer(ITIMER_PROF, &itimer_start, NULL) != 0) { orion_log_crit_errno("setitimer"); return 0; } #endif /* #if ARGOS_NET_TRACE_IO */ #if ARGOS_NET_USE_COMPRESSION == ARGOS_NET_COMPRESS_LZO lzo_uint lzo_outlen; int rv = lzo1x_1_compress(buffer_head(conn->pktbuf), readlen, write_ptr, &lzo_outlen, conn->lzo_wrk_space); if (rv != LZO_E_OK) { /* according to LZO documentation "this should NEVER happen" */ orion_log_crit("LZO compression library internal error: %d", rv); return 0; } uint32_t outlen = lzo_outlen; #elif ARGOS_NET_USE_COMPRESSION == ARGOS_NET_COMPRESS_QUICKLZ uint32_t outlen = qlz_compress(buffer_head(conn->pktbuf), (char*)write_ptr, readlen, conn->qlz_scratch); #else #error "unknown value for ARGOS_NET_USE_COMPRESSION" #endif #if ARGOS_NET_TRACE_IO /* call this before gettimeofday */ struct itimerval itimer_end; if (getitimer(ITIMER_PROF, &itimer_end) != 0) { orion_log_crit_errno("getitimer"); return 0; } #endif /* #if ARGOS_NET_TRACE_IO */ struct timeval end; if (gettimeofday(&end, NULL) != 0) { orion_log_crit_errno("gettimeofday"); return 0; } #if ARGOS_NET_TRACE_IO struct timeval real_elapsed; orion_time_subtract(&end, &start, &real_elapsed); float real_msec = real_elapsed.tv_sec*1000 + (float)real_elapsed.tv_usec/1000; struct timeval process_elapsed; orion_time_subtract(&itimer_start.it_value, &itimer_end.it_value, &process_elapsed); float process_msec = process_elapsed.tv_sec*1000 + (float)process_elapsed.tv_usec/1000; orion_log_debug("compressed %u bytes to %u (%.2f%%) in %.2f ms" " (%.2f MB/s); %.2f ms process time", readlen, outlen, (float)outlen*100/readlen, real_msec, ((readlen/real_msec)*1000)/(1024*1024), process_msec); #endif /* #if ARGOS_NET_TRACE_IO */ size_t total_len = sizeof(struct argos_net_compress_msg) + outlen; if (buffer_expand(conn->outbuf, total_len) < 0) KABOOM("buffer_expand"); if (buffer_discard(conn->pktbuf, readlen) < 0) KABOOM("buffer_discard"); /* write back into the msglen field now that we know the total length */ msg->msglen = htonl(sizeof(struct argos_net_compress_msg) + outlen); /* check for incompressible block (this is normal for small blocks) */ if (outlen > readlen) { if (readlen < 4096) orion_log_debug("incompressible block: inlen=%d, outlen=%d", readlen, outlen); else orion_log_warn("incompressible block: inlen=%d, outlen=%d", readlen, outlen); } #endif /* #if ARGOS_NET_USE_COMPRESSION == ARGOS_NET_COMPRESS_NONE */ /* cancel any currently schedule compression timeout event */ if (conn->compress_evt_reg != NULL) { if (async_cancel(conn->compress_evt_reg) != 0) orion_log_crit_errno("async_cancel"); conn->compress_evt_reg = NULL; } return 1; }
/* chain function * this function does the actual processing */ static GstFlowReturn audio_trim_chain (GstPad * pad, GstBuffer * buf) { AudioTrim *filter; g_assert(GST_BUFFER_OFFSET(buf) != GST_BUFFER_OFFSET_NONE); g_assert(GST_BUFFER_OFFSET_END(buf) != GST_BUFFER_OFFSET_NONE); filter = AUDIO_TRIM (GST_OBJECT_PARENT (pad)); while(buf) { g_assert(GST_IS_BUFFER(buf)); switch(filter->trim_state) { case AUDIO_TRIM_NOT_STARTED: filter->ref_time = (GST_BUFFER_OFFSET(buf) + time_to_sample(filter, filter->start_skip)); if (filter->empty_start_packet) { GstFlowReturn ret; GstBuffer *first; first = gst_buffer_new_and_alloc (sizeof(gfloat)); *(gfloat*)GST_BUFFER_DATA(first) = 0.0; GST_BUFFER_SIZE(first) = 4; GST_BUFFER_OFFSET(first) = GST_BUFFER_OFFSET(buf); GST_BUFFER_OFFSET_END(first) = GST_BUFFER_OFFSET(buf); GST_BUFFER_TIMESTAMP(first) = GST_BUFFER_TIMESTAMP(buf); GST_BUFFER_DURATION(first) = 0; GST_BUFFER_CAPS(first) = gst_caps_ref(GST_BUFFER_CAPS(buf)); ret = gst_pad_push(filter->srcpad, first); if (ret != GST_FLOW_OK) { gst_buffer_unref(buf); return ret; } } filter->trim_state = AUDIO_TRIM_START_SKIP; break; case AUDIO_TRIM_START_SKIP: if (GST_BUFFER_OFFSET_END(buf) <= filter->ref_time) { gst_buffer_unref(buf); /* Ignore buffer completely */ } else { GstBuffer *tail = buffer_tail(filter, buf, filter->ref_time); if (buf) gst_buffer_unref(buf); buf = tail; filter->trim_state = AUDIO_TRIM_START_SILENCE; } break; case AUDIO_TRIM_START_SILENCE: { guint64 offset = find_not_silence(filter, buf); if (offset == GST_BUFFER_OFFSET_NONE) { while(filter->buffered > filter->pre_silence) { GstBuffer *old = filter->buffers->data; filter->buffered -= GST_BUFFER_DURATION(old); gst_buffer_unref(old); filter->buffers = g_list_delete_link(filter->buffers, filter->buffers); } save_buffer(filter, buf); buf = NULL; } else { GstBuffer *head; GstBuffer *tail; GstFlowReturn ret; gint64 clip_start; clip_start = offset - time_to_sample(filter, filter->pre_silence); ret = send_buffers_after(filter, clip_start); if (ret != GST_FLOW_OK) { gst_buffer_unref(buf); return ret; } head = buffer_slice(filter, buf, clip_start, offset); if (head) { ret = gst_pad_push(filter->srcpad, head); if (ret != GST_FLOW_OK) { gst_buffer_unref(buf); return ret; } } tail = buffer_tail(filter, buf, offset); filter->sound_duration = sample_to_time(filter, GST_BUFFER_OFFSET_END(buf) - clip_start); filter->ref_time = clip_start; gst_buffer_unref(buf); buf = tail; filter->trim_state = AUDIO_TRIM_NOT_SILENCE; g_debug("Got sound"); } } break; case AUDIO_TRIM_NOT_SILENCE: { GstFlowReturn ret; filter->sound_duration += GST_BUFFER_DURATION(buf); while(filter->buffered > filter->max_silence_duration) { GstBuffer *old = filter->buffers->data; filter->buffered -= GST_BUFFER_DURATION(old); filter->buffers = g_list_delete_link(filter->buffers,filter->buffers); ret = gst_pad_push(filter->srcpad, old); if (ret != GST_FLOW_OK) { gst_buffer_unref(buf); return ret; } } save_buffer(filter, buf); buf = 0; } break; default: gst_buffer_unref(buf); buf = NULL; } } return GST_FLOW_OK; }