static irqreturn_t msm_hsl_irq(int irq, void *dev_id) { struct uart_port *port = dev_id; struct msm_hsl_port *msm_hsl_port = UART_TO_MSM(port); unsigned int misr; unsigned long flags; spin_lock_irqsave(&port->lock, flags); clk_en(port, 1); misr = msm_hsl_read(port, UARTDM_MISR_ADDR); msm_hsl_write(port, 0, UARTDM_IMR_ADDR); /* disable interrupt */ if (misr & (UARTDM_ISR_RXSTALE_BMSK | UARTDM_ISR_RXLEV_BMSK)) { handle_rx(port, misr); if (misr & (UARTDM_ISR_RXSTALE_BMSK)) msm_hsl_write(port, RESET_STALE_INT, UARTDM_CR_ADDR); msm_hsl_write(port, 6500, UARTDM_DMRX_ADDR); msm_hsl_write(port, STALE_EVENT_ENABLE, UARTDM_CR_ADDR); } if (misr & UARTDM_ISR_TXLEV_BMSK) handle_tx(port); if (misr & UARTDM_ISR_DELTA_CTS_BMSK) handle_delta_cts(port); /* restore interrupt */ msm_hsl_write(port, msm_hsl_port->imr, UARTDM_IMR_ADDR); clk_en(port, 0); spin_unlock_irqrestore(&port->lock, flags); return IRQ_HANDLED; }
int handle_rx_wrapper(struct __sk_buff *skb) { int md_id = skb->cb[0]; struct metadata *md = metadata.lookup(&md_id); if (!md) { bpf_trace_printk("metadata lookup failed\n"); return TC_ACT_SHOT; } // copy to stack in cases llvm spills map pointers to stack //struct metadata local_md = *md; //local_md.flags = 0; //local_md.redir_ifc = 0; //local_md.clone_ifc = 0; md->flags = 0; md->redir_ifc = 0; md->clone_ifc = 0; int rc = handle_rx(skb, md); // TODO: implementation switch (rc) { case RX_OK: break; case RX_REDIRECT: break; case RX_RECIRCULATE: modules.call(skb, 1); break; case RX_DROP: return TC_ACT_SHOT; } //metadata.update(&md_id, &local_md); modules.call(skb, 0); return TC_ACT_SHOT; }
static irqreturn_t msm_irq(int irq, void *dev_id) { unsigned long flags; struct uart_port *port = dev_id; struct msm_port *msm_port = UART_TO_MSM(port); unsigned int misr; spin_lock_irqsave(&port->lock, flags); clk_enable(msm_port->clk); misr = msm_read(port, UART_MISR); msm_write(port, 0, UART_IMR); /* disable interrupt */ if (misr & (UART_IMR_RXLEV | UART_IMR_RXSTALE)) handle_rx(port); if (misr & UART_IMR_TXLEV) handle_tx(port); if (misr & UART_IMR_DELTA_CTS) handle_delta_cts(port); msm_write(port, msm_port->imr, UART_IMR); /* restore interrupt */ clk_disable(msm_port->clk); spin_unlock_irqrestore(&port->lock, flags); return IRQ_HANDLED; }
static irqreturn_t msm_hsl_irq(int irq, void *dev_id) { struct uart_port *port = dev_id; struct msm_hsl_port *msm_hsl_port = UART_TO_MSM(port); unsigned int vid; unsigned int misr; unsigned long flags; spin_lock_irqsave(&port->lock, flags); vid = msm_hsl_port->ver_id; misr = msm_hsl_read(port, regmap[vid][UARTDM_MISR]); msm_hsl_write(port, 0, regmap[vid][UARTDM_IMR]); if (misr & (UARTDM_ISR_RXSTALE_BMSK | UARTDM_ISR_RXLEV_BMSK)) { handle_rx(port, misr); if (misr & (UARTDM_ISR_RXSTALE_BMSK)) msm_hsl_write(port, RESET_STALE_INT, regmap[vid][UARTDM_CR]); msm_hsl_write(port, 6500, regmap[vid][UARTDM_DMRX]); msm_hsl_write(port, STALE_EVENT_ENABLE, regmap[vid][UARTDM_CR]); } if (misr & UARTDM_ISR_TXLEV_BMSK) handle_tx(port); if (misr & UARTDM_ISR_DELTA_CTS_BMSK) handle_delta_cts(port); msm_hsl_write(port, msm_hsl_port->imr, regmap[vid][UARTDM_IMR]); spin_unlock_irqrestore(&port->lock, flags); return IRQ_HANDLED; }
static irqreturn_t msm_irq(int irq, void *dev_id) { struct uart_port *port = dev_id; struct msm_port *msm_port = UART_TO_MSM(port); unsigned int misr; spin_lock(&port->lock); misr = msm_read(port, UART_MISR); msm_write(port, 0, UART_IMR); /* disable interrupt */ if (misr & (UART_IMR_RXLEV | UART_IMR_RXSTALE)) { if (msm_port->is_uartdm) handle_rx_dm(port, misr); else handle_rx(port); } if (misr & UART_IMR_TXLEV) handle_tx(port); if (misr & UART_IMR_DELTA_CTS) handle_delta_cts(port); msm_write(port, msm_port->imr, UART_IMR); /* restore interrupt */ spin_unlock(&port->lock); return IRQ_HANDLED; }
static void loop(struct vi_state* vi_state) { ef_event evs[EF_VI_EVENT_POLL_MIN_EVS]; ef_vi* vi = &vi_state->vi; int i; pthread_mutex_lock(&ready_mutex); ++ready_cnt; pthread_cond_signal(&ready_cond); pthread_mutex_unlock(&ready_mutex); while( 1 ) { int n_ev = ef_eventq_poll(vi, evs, sizeof(evs) / sizeof(evs[0])); for( i = 0; i < n_ev; ++i ) switch( EF_EVENT_TYPE(evs[i]) ) { case EF_EVENT_TYPE_RX: /* This code does not handle jumbos. */ assert(EF_EVENT_RX_SOP(evs[i]) != 0); assert(EF_EVENT_RX_CONT(evs[i]) == 0); handle_rx(vi_state, EF_EVENT_RX_RQ_ID(evs[i]), EF_EVENT_RX_BYTES(evs[i]) - ef_vi_receive_prefix_len(vi)); break; case EF_EVENT_TYPE_RX_DISCARD: handle_rx_discard(vi_state, EF_EVENT_RX_DISCARD_RQ_ID(evs[i]), EF_EVENT_RX_DISCARD_TYPE(evs[i])); break; default: LOGE("ERROR: unexpected event %d\n", (int) EF_EVENT_TYPE(evs[i])); break; } vi_refill_rx_ring(vi_state); } }
static void handle_batched_rx(int rx_vi_i, int pkt_buf_i) { void* dma_ptr = (char*) pkt_buf_from_id(pkt_buf_i) + RX_DMA_OFF + addr_offset_from_id(pkt_buf_i); uint16_t len; TRY( ef_vi_receive_get_bytes(&vis[rx_vi_i].vi, dma_ptr ,&len) ); handle_rx(rx_vi_i, pkt_buf_i, len); }
static irqreturn_t vt8500_irq(int irq, void *dev_id) { struct uart_port *port = dev_id; unsigned long isr; spin_lock(&port->lock); isr = vt8500_read(port, VT8500_URISR); /* Acknowledge active status bits */ vt8500_write(port, isr, VT8500_URISR); if (isr & RX_FIFO_INTS) handle_rx(port); if (isr & TX_FIFO_INTS) handle_tx(port); if (isr & TCTS) handle_delta_cts(port); spin_unlock(&port->lock); return IRQ_HANDLED; }
/* The main loop. Poll each VI handling various types of events and * then try to refill them. */ static void main_loop(void) { int i, j, k; while( 1 ) { for( i = 0; i < 2; ++i ) { ef_vi* vi = &vis[i].vi; ef_event evs[EF_VI_EVENT_POLL_MIN_EVS]; int n_ev = ef_eventq_poll(vi, evs, sizeof(evs) / sizeof(evs[0])); for( j = 0; j < n_ev; ++j ) { switch( EF_EVENT_TYPE(evs[j]) ) { case EF_EVENT_TYPE_RX: /* This code does not handle jumbos. */ assert(EF_EVENT_RX_SOP(evs[j]) != 0); assert(EF_EVENT_RX_CONT(evs[j]) == 0); handle_rx(i, EF_EVENT_RX_RQ_ID(evs[j]), EF_EVENT_RX_BYTES(evs[j]) - ef_vi_receive_prefix_len(vi)); break; case EF_EVENT_TYPE_TX: { ef_request_id ids[EF_VI_TRANSMIT_BATCH]; int ntx = ef_vi_transmit_unbundle(vi, &evs[j], ids); for( k = 0; k < ntx; ++k ) complete_tx(i, ids[k]); break; } case EF_EVENT_TYPE_RX_DISCARD: handle_rx_discard(EF_EVENT_RX_DISCARD_RQ_ID(evs[j]), EF_EVENT_RX_DISCARD_TYPE(evs[j])); break; default: LOGE("ERROR: unexpected event %d\n", (int) EF_EVENT_TYPE(evs[j])); break; } } vi_refill_rx_ring(i); } } }
/* Expects to be always run from workqueue - which acts as * read-size critical section for our kind of RCU. */ static void handle_rx(struct vhost_net *net) { struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX]; struct vhost_virtqueue *vq = &nvq->vq; unsigned uninitialized_var(in), log; struct vhost_log *vq_log; struct msghdr msg = { .msg_name = NULL, .msg_namelen = 0, .msg_control = NULL, /* FIXME: get and handle RX aux data. */ .msg_controllen = 0, .msg_flags = MSG_DONTWAIT, }; struct virtio_net_hdr_mrg_rxbuf hdr = { .hdr.flags = 0, .hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE }; size_t total_len = 0; int err, mergeable; s16 headcount; size_t vhost_hlen, sock_hlen; size_t vhost_len, sock_len; struct socket *sock; mutex_lock(&vq->mutex); sock = vq->private_data; if (!sock) goto out; vhost_disable_notify(&net->dev, vq); vhost_hlen = nvq->vhost_hlen; sock_hlen = nvq->sock_hlen; vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ? vq->log : NULL; mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF); while ((sock_len = peek_head_len(sock->sk))) { sock_len += sock_hlen; vhost_len = sock_len + vhost_hlen; headcount = get_rx_bufs(vq, vq->heads, vhost_len, &in, vq_log, &log, likely(mergeable) ? UIO_MAXIOV : 1); /* On error, stop handling until the next kick. */ if (unlikely(headcount < 0)) break; /* On overrun, truncate and discard */ if (unlikely(headcount > UIO_MAXIOV)) { iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1); err = sock->ops->recvmsg(NULL, sock, &msg, 1, MSG_DONTWAIT | MSG_TRUNC); pr_debug("Discarded rx packet: len %zd\n", sock_len); continue; } /* OK, now we need to know about added descriptors. */ if (!headcount) { if (unlikely(vhost_enable_notify(&net->dev, vq))) { /* They have slipped one in as we were * doing that: check again. */ vhost_disable_notify(&net->dev, vq); continue; } /* Nothing new? Wait for eventfd to tell us * they refilled. */ break; } /* We don't need to be notified again. */ if (unlikely((vhost_hlen))) /* Skip header. TODO: support TSO. */ move_iovec_hdr(vq->iov, nvq->hdr, vhost_hlen, in); else /* Copy the header for use in VIRTIO_NET_F_MRG_RXBUF: * needed because recvmsg can modify msg_iov. */ copy_iovec_hdr(vq->iov, nvq->hdr, sock_hlen, in); iov_iter_init(&msg.msg_iter, READ, vq->iov, in, sock_len); err = sock->ops->recvmsg(NULL, sock, &msg, sock_len, MSG_DONTWAIT | MSG_TRUNC); /* Userspace might have consumed the packet meanwhile: * it's not supposed to do this usually, but might be hard * to prevent. Discard data we got (if any) and keep going. */ if (unlikely(err != sock_len)) { pr_debug("Discarded rx packet: " " len %d, expected %zd\n", err, sock_len); vhost_discard_vq_desc(vq, headcount); continue; } if (unlikely(vhost_hlen) && memcpy_toiovecend(nvq->hdr, (unsigned char *)&hdr, 0, vhost_hlen)) { vq_err(vq, "Unable to write vnet_hdr at addr %p\n", vq->iov->iov_base); break; } /* TODO: Should check and handle checksum. */ if (likely(mergeable) && memcpy_toiovecend(nvq->hdr, (unsigned char *)&headcount, offsetof(typeof(hdr), num_buffers), sizeof hdr.num_buffers)) { vq_err(vq, "Failed num_buffers write"); vhost_discard_vq_desc(vq, headcount); break; } vhost_add_used_and_signal_n(&net->dev, vq, vq->heads, headcount); if (unlikely(vq_log)) vhost_log_write(vq, vq_log, log, vhost_len); total_len += vhost_len; if (unlikely(total_len >= VHOST_NET_WEIGHT)) { vhost_poll_queue(&vq->poll); break; } } out: mutex_unlock(&vq->mutex); } static void handle_tx_kick(struct vhost_work *work) { struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, poll.work); struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); handle_tx(net); } static void handle_rx_kick(struct vhost_work *work) { struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, poll.work); struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); handle_rx(net); } static void handle_tx_net(struct vhost_work *work) { struct vhost_net *net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_TX].work); handle_tx(net); } static void handle_rx_net(struct vhost_work *work) { struct vhost_net *net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_RX].work); handle_rx(net); } static int vhost_net_open(struct inode *inode, struct file *f) { struct vhost_net *n; struct vhost_dev *dev; struct vhost_virtqueue **vqs; int i; n = kmalloc(sizeof *n, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); if (!n) { n = vmalloc(sizeof *n); if (!n) return -ENOMEM; } vqs = kmalloc(VHOST_NET_VQ_MAX * sizeof(*vqs), GFP_KERNEL); if (!vqs) { kvfree(n); return -ENOMEM; } dev = &n->dev; vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq; vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq; n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick; n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick; for (i = 0; i < VHOST_NET_VQ_MAX; i++) { n->vqs[i].ubufs = NULL; n->vqs[i].ubuf_info = NULL; n->vqs[i].upend_idx = 0; n->vqs[i].done_idx = 0; n->vqs[i].vhost_hlen = 0; n->vqs[i].sock_hlen = 0; } vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX); vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev); vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev); f->private_data = n; return 0; } static void vhost_net_disable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct vhost_net_virtqueue *nvq = container_of(vq, struct vhost_net_virtqueue, vq); struct vhost_poll *poll = n->poll + (nvq - n->vqs); if (!vq->private_data) return; vhost_poll_stop(poll); } static int vhost_net_enable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct vhost_net_virtqueue *nvq = container_of(vq, struct vhost_net_virtqueue, vq); struct vhost_poll *poll = n->poll + (nvq - n->vqs); struct socket *sock; sock = vq->private_data; if (!sock) return 0; return vhost_poll_start(poll, sock->file); } static struct socket *vhost_net_stop_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct socket *sock; mutex_lock(&vq->mutex); sock = vq->private_data; vhost_net_disable_vq(n, vq); vq->private_data = NULL; mutex_unlock(&vq->mutex); return sock; } static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock, struct socket **rx_sock) { *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq); *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq); } static void vhost_net_flush_vq(struct vhost_net *n, int index) { vhost_poll_flush(n->poll + index); vhost_poll_flush(&n->vqs[index].vq.poll); }
static void handle_tx(struct vhost_net *net) { struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_TX]; unsigned out, in, s; int head; struct msghdr msg = { .msg_name = NULL, .msg_namelen = 0, .msg_control = NULL, .msg_controllen = 0, .msg_iov = vq->iov, .msg_flags = MSG_DONTWAIT, }; size_t len, total_len = 0; int err, wmem; size_t hdr_size; struct socket *sock = rcu_dereference(vq->private_data); if (!sock) return; wmem = atomic_read(&sock->sk->sk_wmem_alloc); if (wmem >= sock->sk->sk_sndbuf) { mutex_lock(&vq->mutex); tx_poll_start(net, sock); mutex_unlock(&vq->mutex); return; } use_mm(net->dev.mm); mutex_lock(&vq->mutex); vhost_disable_notify(vq); if (wmem < sock->sk->sk_sndbuf / 2) tx_poll_stop(net); hdr_size = vq->hdr_size; for (;;) { head = vhost_get_vq_desc(&net->dev, vq, vq->iov, ARRAY_SIZE(vq->iov), &out, &in, NULL, NULL); /* On error, stop handling until the next kick. */ if (unlikely(head < 0)) break; /* Nothing new? Wait for eventfd to tell us they refilled. */ if (head == vq->num) { wmem = atomic_read(&sock->sk->sk_wmem_alloc); if (wmem >= sock->sk->sk_sndbuf * 3 / 4) { tx_poll_start(net, sock); set_bit(SOCK_ASYNC_NOSPACE, &sock->flags); break; } if (unlikely(vhost_enable_notify(vq))) { vhost_disable_notify(vq); continue; } break; } if (in) { vq_err(vq, "Unexpected descriptor format for TX: " "out %d, int %d\n", out, in); break; } /* Skip header. TODO: support TSO. */ s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, out); msg.msg_iovlen = out; len = iov_length(vq->iov, out); /* Sanity check */ if (!len) { vq_err(vq, "Unexpected header len for TX: " "%zd expected %zd\n", iov_length(vq->hdr, s), hdr_size); break; } /* TODO: Check specific error and bomb out unless ENOBUFS? */ err = sock->ops->sendmsg(NULL, sock, &msg, len); if (unlikely(err < 0)) { vhost_discard_vq_desc(vq); tx_poll_start(net, sock); break; } if (err != len) pr_debug("Truncated TX packet: " " len %d != %zd\n", err, len); vhost_add_used_and_signal(&net->dev, vq, head, 0); total_len += len; if (unlikely(total_len >= VHOST_NET_WEIGHT)) { vhost_poll_queue(&vq->poll); break; } } mutex_unlock(&vq->mutex); unuse_mm(net->dev.mm); } static void handle_rx(struct vhost_net *net) { struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX]; unsigned out, in, log, s; int head; struct vhost_log *vq_log; struct msghdr msg = { .msg_name = NULL, .msg_namelen = 0, .msg_control = NULL, /* FIXME: get and handle RX aux data. */ .msg_controllen = 0, .msg_iov = vq->iov, .msg_flags = MSG_DONTWAIT, }; struct virtio_net_hdr hdr = { .flags = 0, .gso_type = VIRTIO_NET_HDR_GSO_NONE }; size_t len, total_len = 0; int err; size_t hdr_size; struct socket *sock = rcu_dereference(vq->private_data); if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue)) return; use_mm(net->dev.mm); mutex_lock(&vq->mutex); vhost_disable_notify(vq); hdr_size = vq->hdr_size; vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ? vq->log : NULL; for (;;) { head = vhost_get_vq_desc(&net->dev, vq, vq->iov, ARRAY_SIZE(vq->iov), &out, &in, vq_log, &log); /* On error, stop handling until the next kick. */ if (unlikely(head < 0)) break; /* OK, now we need to know about added descriptors. */ if (head == vq->num) { if (unlikely(vhost_enable_notify(vq))) { /* They have slipped one in as we were * doing that: check again. */ vhost_disable_notify(vq); continue; } /* Nothing new? Wait for eventfd to tell us * they refilled. */ break; } /* We don't need to be notified again. */ if (out) { vq_err(vq, "Unexpected descriptor format for RX: " "out %d, int %d\n", out, in); break; } /* Skip header. TODO: support TSO/mergeable rx buffers. */ s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, in); msg.msg_iovlen = in; len = iov_length(vq->iov, in); /* Sanity check */ if (!len) { vq_err(vq, "Unexpected header len for RX: " "%zd expected %zd\n", iov_length(vq->hdr, s), hdr_size); break; } err = sock->ops->recvmsg(NULL, sock, &msg, len, MSG_DONTWAIT | MSG_TRUNC); /* TODO: Check specific error and bomb out unless EAGAIN? */ if (err < 0) { vhost_discard_vq_desc(vq); break; } /* TODO: Should check and handle checksum. */ if (err > len) { pr_debug("Discarded truncated rx packet: " " len %d > %zd\n", err, len); vhost_discard_vq_desc(vq); continue; } len = err; err = memcpy_toiovec(vq->hdr, (unsigned char *)&hdr, hdr_size); if (err) { vq_err(vq, "Unable to write vnet_hdr at addr %p: %d\n", vq->iov->iov_base, err); break; } len += hdr_size; vhost_add_used_and_signal(&net->dev, vq, head, len); if (unlikely(vq_log)) vhost_log_write(vq, vq_log, log, len); total_len += len; if (unlikely(total_len >= VHOST_NET_WEIGHT)) { vhost_poll_queue(&vq->poll); break; } } mutex_unlock(&vq->mutex); unuse_mm(net->dev.mm); } static void handle_tx_kick(struct work_struct *work) { struct vhost_virtqueue *vq; struct vhost_net *net; vq = container_of(work, struct vhost_virtqueue, poll.work); net = container_of(vq->dev, struct vhost_net, dev); handle_tx(net); } static void handle_rx_kick(struct work_struct *work) { struct vhost_virtqueue *vq; struct vhost_net *net; vq = container_of(work, struct vhost_virtqueue, poll.work); net = container_of(vq->dev, struct vhost_net, dev); handle_rx(net); } static void handle_tx_net(struct work_struct *work) { struct vhost_net *net; net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_TX].work); handle_tx(net); } static void handle_rx_net(struct work_struct *work) { struct vhost_net *net; net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_RX].work); handle_rx(net); } static int vhost_net_open(struct inode *inode, struct file *f) { struct vhost_net *n = kmalloc(sizeof *n, GFP_KERNEL); int r; if (!n) return -ENOMEM; n->vqs[VHOST_NET_VQ_TX].handle_kick = handle_tx_kick; n->vqs[VHOST_NET_VQ_RX].handle_kick = handle_rx_kick; r = vhost_dev_init(&n->dev, n->vqs, VHOST_NET_VQ_MAX); if (r < 0) { kfree(n); return r; } vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT); vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN); n->tx_poll_state = VHOST_NET_POLL_DISABLED; f->private_data = n; return 0; } static void vhost_net_disable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { if (!vq->private_data) return; if (vq == n->vqs + VHOST_NET_VQ_TX) { tx_poll_stop(n); n->tx_poll_state = VHOST_NET_POLL_DISABLED; } else vhost_poll_stop(n->poll + VHOST_NET_VQ_RX); } static void vhost_net_enable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct socket *sock = vq->private_data; if (!sock) return; if (vq == n->vqs + VHOST_NET_VQ_TX) { n->tx_poll_state = VHOST_NET_POLL_STOPPED; tx_poll_start(n, sock); } else vhost_poll_start(n->poll + VHOST_NET_VQ_RX, sock->file); } static struct socket *vhost_net_stop_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct socket *sock; mutex_lock(&vq->mutex); sock = vq->private_data; vhost_net_disable_vq(n, vq); rcu_assign_pointer(vq->private_data, NULL); mutex_unlock(&vq->mutex); return sock; } static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock, struct socket **rx_sock) { *tx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_TX); *rx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_RX); } static void vhost_net_flush_vq(struct vhost_net *n, int index) { vhost_poll_flush(n->poll + index); vhost_poll_flush(&n->dev.vqs[index].poll); } static void vhost_net_flush(struct vhost_net *n) { vhost_net_flush_vq(n, VHOST_NET_VQ_TX); vhost_net_flush_vq(n, VHOST_NET_VQ_RX); }
/****************************************************************************** * bcr_receive_thread() * This function is started as a thread. It waits in a select() for input from * the two serial interfaces where the barcode readers are connected. * When any of the barcode reader sent some data, the received characters are * entered into a ringbuffer which is a member of the PORT_STAT structure * representing the barcode reader. *****************************************************************************/ void * bcr_receive_thread(void* args) { fd_set rfds; int ready_fd; char rxbuf[100]; int read_stat; int res; while (1) { FD_ZERO(&rfds); FD_SET(sport1_stat.fd, &rfds); // add serial port 1 file descriptor to set FD_SET(sport2_stat.fd, &rfds); // add serial port 2 file descriptor to set //ready_fd = select(FD_SETSIZE, &rfds, NULL, NULL, &timeout); ready_fd = select(FD_SETSIZE, &rfds, NULL, NULL, NULL); if (ready_fd < 0) { /* select returns an error */ sys_log(ILOG_ERR, "select(): %s", strerror(errno)); } else if (ready_fd == 0) { /* select timed out */ sys_log(ILOG_WARNING, "timeout in select()"); } else { if (FD_ISSET(sport1_stat.fd, &rfds)) { if ((read_stat = read(sport1_stat.fd, &rxbuf, 100)) < 0) { /* there was an error while reading */ sys_log(ILOG_ERR, "error while reading %s: %s" ,sport1_stat.dev_name, strerror(errno)); } else if (read_stat == 0) { /* no character to read, empty buffer */ } else { /* character successfully read */ rxbuf[read_stat] = '\0'; sys_debug_log(4, "RXed %d chars on %s: %s", read_stat, sport1_stat.dev_name, rxbuf); if ((res = handle_rx(&sport1_stat, rxbuf, read_stat))) { sys_log(ILOG_ERR, "rx buffer overflow on port %s", sport1_stat.dev_name); } else { sys_debug_log(4, "handle_rx(%s) returned %d", sport1_stat.dev_name, res); } } } if (FD_ISSET(sport2_stat.fd, &rfds)) { if ((read_stat = read(sport2_stat.fd, &rxbuf, 100)) < 0) { /* there was an error while reading */ sys_log(ILOG_ERR, "error while reading %s: %s", sport2_stat.dev_name, strerror(errno)); } else if (read_stat == 0) { /* no character read, empty buffer */ } else { /* character successfully read */ rxbuf[read_stat] = '\0'; if (debug > 4) { sys_debug_log(4, "RXed %d chars on %s: %s", read_stat, sport2_stat.dev_name, rxbuf); } if ((res = handle_rx(&sport2_stat, rxbuf, read_stat))) { sys_log(ILOG_ERR, "rx buffer overflow on port %s", sport2_stat.dev_name); } else { sys_debug_log(4, "handle_rx(%s) returned %d", sport2_stat.dev_name, res); } } } } } /* while (1) */ sys_debug_log(3, "exiting receive_thread()"); return NULL; }
/* Expects to be always run from workqueue - which acts as * read-size critical section for our kind of RCU. */ static void handle_rx_big(struct vhost_net *net) { struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX]; unsigned out, in, log, s; int head; struct vhost_log *vq_log; struct msghdr msg = { .msg_name = NULL, .msg_namelen = 0, .msg_control = NULL, /* FIXME: get and handle RX aux data. */ .msg_controllen = 0, .msg_iov = vq->iov, .msg_flags = MSG_DONTWAIT, }; struct virtio_net_hdr hdr = { .flags = 0, .gso_type = VIRTIO_NET_HDR_GSO_NONE }; size_t len, total_len = 0; int err; size_t hdr_size; struct socket *sock = rcu_dereference(vq->private_data); if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue)) return; use_mm(net->dev.mm); mutex_lock(&vq->mutex); vhost_disable_notify(vq); hdr_size = vq->vhost_hlen; vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ? vq->log : NULL; for (;;) { head = vhost_get_vq_desc(&net->dev, vq, vq->iov, ARRAY_SIZE(vq->iov), &out, &in, vq_log, &log); /* On error, stop handling until the next kick. */ if (unlikely(head < 0)) break; /* OK, now we need to know about added descriptors. */ if (head == vq->num) { if (unlikely(vhost_enable_notify(vq))) { /* They have slipped one in as we were * doing that: check again. */ vhost_disable_notify(vq); continue; } /* Nothing new? Wait for eventfd to tell us * they refilled. */ break; } /* We don't need to be notified again. */ if (out) { vq_err(vq, "Unexpected descriptor format for RX: " "out %d, int %d\n", out, in); break; } /* Skip header. TODO: support TSO/mergeable rx buffers. */ s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, in); msg.msg_iovlen = in; len = iov_length(vq->iov, in); /* Sanity check */ if (!len) { vq_err(vq, "Unexpected header len for RX: " "%zd expected %zd\n", iov_length(vq->hdr, s), hdr_size); break; } err = sock->ops->recvmsg(NULL, sock, &msg, len, MSG_DONTWAIT | MSG_TRUNC); /* TODO: Check specific error and bomb out unless EAGAIN? */ if (err < 0) { vhost_discard_vq_desc(vq, 1); break; } /* TODO: Should check and handle checksum. */ if (err > len) { pr_debug("Discarded truncated rx packet: " " len %d > %zd\n", err, len); vhost_discard_vq_desc(vq, 1); continue; } len = err; err = memcpy_toiovec(vq->hdr, (unsigned char *)&hdr, hdr_size); if (err) { vq_err(vq, "Unable to write vnet_hdr at addr %p: %d\n", vq->iov->iov_base, err); break; } len += hdr_size; vhost_add_used_and_signal(&net->dev, vq, head, len); if (unlikely(vq_log)) vhost_log_write(vq, vq_log, log, len); total_len += len; if (unlikely(total_len >= VHOST_NET_WEIGHT)) { vhost_poll_queue(&vq->poll); break; } } mutex_unlock(&vq->mutex); unuse_mm(net->dev.mm); } /* Expects to be always run from workqueue - which acts as * read-size critical section for our kind of RCU. */ static void handle_rx_mergeable(struct vhost_net *net) { struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX]; unsigned uninitialized_var(in), log; struct vhost_log *vq_log; struct msghdr msg = { .msg_name = NULL, .msg_namelen = 0, .msg_control = NULL, /* FIXME: get and handle RX aux data. */ .msg_controllen = 0, .msg_iov = vq->iov, .msg_flags = MSG_DONTWAIT, }; struct virtio_net_hdr_mrg_rxbuf hdr = { .hdr.flags = 0, .hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE }; size_t total_len = 0; int err, headcount; size_t vhost_hlen, sock_hlen; size_t vhost_len, sock_len; struct socket *sock = rcu_dereference(vq->private_data); if (!sock || skb_queue_empty(&sock->sk->sk_receive_queue)) return; use_mm(net->dev.mm); mutex_lock(&vq->mutex); vhost_disable_notify(vq); vhost_hlen = vq->vhost_hlen; sock_hlen = vq->sock_hlen; vq_log = unlikely(vhost_has_feature(&net->dev, VHOST_F_LOG_ALL)) ? vq->log : NULL; while ((sock_len = peek_head_len(sock->sk))) { sock_len += sock_hlen; vhost_len = sock_len + vhost_hlen; headcount = get_rx_bufs(vq, vq->heads, vhost_len, &in, vq_log, &log); /* On error, stop handling until the next kick. */ if (unlikely(headcount < 0)) break; /* OK, now we need to know about added descriptors. */ if (!headcount) { if (unlikely(vhost_enable_notify(vq))) { /* They have slipped one in as we were * doing that: check again. */ vhost_disable_notify(vq); continue; } /* Nothing new? Wait for eventfd to tell us * they refilled. */ break; } /* We don't need to be notified again. */ if (unlikely((vhost_hlen))) /* Skip header. TODO: support TSO. */ move_iovec_hdr(vq->iov, vq->hdr, vhost_hlen, in); else /* Copy the header for use in VIRTIO_NET_F_MRG_RXBUF: * needed because sendmsg can modify msg_iov. */ copy_iovec_hdr(vq->iov, vq->hdr, sock_hlen, in); msg.msg_iovlen = in; err = sock->ops->recvmsg(NULL, sock, &msg, sock_len, MSG_DONTWAIT | MSG_TRUNC); /* Userspace might have consumed the packet meanwhile: * it's not supposed to do this usually, but might be hard * to prevent. Discard data we got (if any) and keep going. */ if (unlikely(err != sock_len)) { pr_debug("Discarded rx packet: " " len %d, expected %zd\n", err, sock_len); vhost_discard_vq_desc(vq, headcount); continue; } if (unlikely(vhost_hlen) && memcpy_toiovecend(vq->hdr, (unsigned char *)&hdr, 0, vhost_hlen)) { vq_err(vq, "Unable to write vnet_hdr at addr %p\n", vq->iov->iov_base); break; } /* TODO: Should check and handle checksum. */ if (vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF) && memcpy_toiovecend(vq->hdr, (unsigned char *)&headcount, offsetof(typeof(hdr), num_buffers), sizeof hdr.num_buffers)) { vq_err(vq, "Failed num_buffers write"); vhost_discard_vq_desc(vq, headcount); break; } vhost_add_used_and_signal_n(&net->dev, vq, vq->heads, headcount); if (unlikely(vq_log)) vhost_log_write(vq, vq_log, log, vhost_len); total_len += vhost_len; if (unlikely(total_len >= VHOST_NET_WEIGHT)) { vhost_poll_queue(&vq->poll); break; } } mutex_unlock(&vq->mutex); unuse_mm(net->dev.mm); } static void handle_rx(struct vhost_net *net) { if (vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF)) handle_rx_mergeable(net); else handle_rx_big(net); } static void handle_tx_kick(struct vhost_work *work) { struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, poll.work); struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); handle_tx(net); } static void handle_rx_kick(struct vhost_work *work) { struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, poll.work); struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); handle_rx(net); } static void handle_tx_net(struct vhost_work *work) { struct vhost_net *net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_TX].work); handle_tx(net); } static void handle_rx_net(struct vhost_work *work) { struct vhost_net *net = container_of(work, struct vhost_net, poll[VHOST_NET_VQ_RX].work); handle_rx(net); } static int vhost_net_open(struct inode *inode, struct file *f) { struct vhost_net *n = kmalloc(sizeof *n, GFP_KERNEL); struct vhost_dev *dev; int r; if (!n) return -ENOMEM; dev = &n->dev; n->vqs[VHOST_NET_VQ_TX].handle_kick = handle_tx_kick; n->vqs[VHOST_NET_VQ_RX].handle_kick = handle_rx_kick; r = vhost_dev_init(dev, n->vqs, VHOST_NET_VQ_MAX); if (r < 0) { kfree(n); return r; } vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev); vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev); n->tx_poll_state = VHOST_NET_POLL_DISABLED; f->private_data = n; return 0; } static void vhost_net_disable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { if (!vq->private_data) return; if (vq == n->vqs + VHOST_NET_VQ_TX) { tx_poll_stop(n); n->tx_poll_state = VHOST_NET_POLL_DISABLED; } else vhost_poll_stop(n->poll + VHOST_NET_VQ_RX); } static void vhost_net_enable_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct socket *sock; sock = rcu_dereference_protected(vq->private_data, lockdep_is_held(&vq->mutex)); if (!sock) return; if (vq == n->vqs + VHOST_NET_VQ_TX) { n->tx_poll_state = VHOST_NET_POLL_STOPPED; tx_poll_start(n, sock); } else vhost_poll_start(n->poll + VHOST_NET_VQ_RX, sock->file); } static struct socket *vhost_net_stop_vq(struct vhost_net *n, struct vhost_virtqueue *vq) { struct socket *sock; mutex_lock(&vq->mutex); sock = rcu_dereference_protected(vq->private_data, lockdep_is_held(&vq->mutex)); vhost_net_disable_vq(n, vq); rcu_assign_pointer(vq->private_data, NULL); mutex_unlock(&vq->mutex); return sock; } static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock, struct socket **rx_sock) { *tx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_TX); *rx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_RX); } static void vhost_net_flush_vq(struct vhost_net *n, int index) { vhost_poll_flush(n->poll + index); vhost_poll_flush(&n->dev.vqs[index].poll); } static void vhost_net_flush(struct vhost_net *n) { vhost_net_flush_vq(n, VHOST_NET_VQ_TX); vhost_net_flush_vq(n, VHOST_NET_VQ_RX); } static int vhost_net_release(struct inode *inode, struct file *f) { struct vhost_net *n = f->private_data; struct socket *tx_sock; struct socket *rx_sock; vhost_net_stop(n, &tx_sock, &rx_sock); vhost_net_flush(n); vhost_dev_cleanup(&n->dev); if (tx_sock) fput(tx_sock->file); if (rx_sock) fput(rx_sock->file); /* We do an extra flush before freeing memory, * since jobs can re-queue themselves. */ vhost_net_flush(n); kfree(n); return 0; } static struct socket *get_raw_socket(int fd) { struct { struct sockaddr_ll sa; char buf[MAX_ADDR_LEN]; } uaddr; int uaddr_len = sizeof uaddr, r; struct socket *sock = sockfd_lookup(fd, &r); if (!sock) return ERR_PTR(-ENOTSOCK); /* Parameter checking */ if (sock->sk->sk_type != SOCK_RAW) { r = -ESOCKTNOSUPPORT; goto err; } r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa, &uaddr_len, 0); if (r) goto err; if (uaddr.sa.sll_family != AF_PACKET) { r = -EPFNOSUPPORT; goto err; } return sock; err: fput(sock->file); return ERR_PTR(r); } static struct socket *get_tap_socket(int fd) { struct file *file = fget(fd); struct socket *sock; if (!file) return ERR_PTR(-EBADF); sock = tun_get_socket(file); if (!IS_ERR(sock)) return sock; sock = macvtap_get_socket(file); if (IS_ERR(sock)) fput(file); return sock; } static struct socket *get_socket(int fd) { struct socket *sock; /* special case to disable backend */ if (fd == -1) return NULL; sock = get_raw_socket(fd); if (!IS_ERR(sock)) return sock; sock = get_tap_socket(fd); if (!IS_ERR(sock)) return sock; return ERR_PTR(-ENOTSOCK); } static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd) { struct socket *sock, *oldsock; struct vhost_virtqueue *vq; int r; mutex_lock(&n->dev.mutex); r = vhost_dev_check_owner(&n->dev); if (r) goto err; if (index >= VHOST_NET_VQ_MAX) { r = -ENOBUFS; goto err; } vq = n->vqs + index; mutex_lock(&vq->mutex); /* Verify that ring has been setup correctly. */ if (!vhost_vq_access_ok(vq)) { r = -EFAULT; goto err_vq; } sock = get_socket(fd); if (IS_ERR(sock)) { r = PTR_ERR(sock); goto err_vq; } /* start polling new socket */ oldsock = rcu_dereference_protected(vq->private_data, lockdep_is_held(&vq->mutex)); if (sock != oldsock) { vhost_net_disable_vq(n, vq); rcu_assign_pointer(vq->private_data, sock); vhost_net_enable_vq(n, vq); } mutex_unlock(&vq->mutex); if (oldsock) { vhost_net_flush_vq(n, index); fput(oldsock->file); } mutex_unlock(&n->dev.mutex); return 0; err_vq: mutex_unlock(&vq->mutex); err: mutex_unlock(&n->dev.mutex); return r; } static long vhost_net_reset_owner(struct vhost_net *n) { struct socket *tx_sock = NULL; struct socket *rx_sock = NULL; long err; mutex_lock(&n->dev.mutex); err = vhost_dev_check_owner(&n->dev); if (err) goto done; vhost_net_stop(n, &tx_sock, &rx_sock); vhost_net_flush(n); err = vhost_dev_reset_owner(&n->dev); done: mutex_unlock(&n->dev.mutex); if (tx_sock) fput(tx_sock->file); if (rx_sock) fput(rx_sock->file); return err; } static int vhost_net_set_features(struct vhost_net *n, u64 features) { size_t vhost_hlen, sock_hlen, hdr_len; int i; hdr_len = (features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr); if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) { /* vhost provides vnet_hdr */ vhost_hlen = hdr_len; sock_hlen = 0; } else { /* socket provides vnet_hdr */ vhost_hlen = 0; sock_hlen = hdr_len; } mutex_lock(&n->dev.mutex); if ((features & (1 << VHOST_F_LOG_ALL)) && !vhost_log_access_ok(&n->dev)) { mutex_unlock(&n->dev.mutex); return -EFAULT; } n->dev.acked_features = features; smp_wmb(); for (i = 0; i < VHOST_NET_VQ_MAX; ++i) { mutex_lock(&n->vqs[i].mutex); n->vqs[i].vhost_hlen = vhost_hlen; n->vqs[i].sock_hlen = sock_hlen; mutex_unlock(&n->vqs[i].mutex); } vhost_net_flush(n); mutex_unlock(&n->dev.mutex); return 0; }