int ipaugenblick_read_updates(void) { struct rte_mbuf *mbuf = NULL; unsigned char cmd = 0; if(rte_ring_dequeue(client_ring,(void **)&mbuf)) { syslog(LOG_ERR,"%s %d\n",__FILE__,__LINE__); return -1; } unsigned char *p = rte_pktmbuf_mtod(mbuf, unsigned char *); switch(*p) { case IPAUGENBLICK_NEW_IFACES: if(ipaugenblick_update_cbk) { cmd = 1; p++; ipaugenblick_update_cbk(cmd,p,rte_pktmbuf_data_len(mbuf) - 1); } break; case IPAUGENBLICK_NEW_ADDRESSES: if(ipaugenblick_update_cbk) { cmd = 3; p++; ipaugenblick_update_cbk(cmd,p,rte_pktmbuf_data_len(mbuf) - 1); } break; case IPAUGENBLICK_END_OF_RECORD: return 0; } return -1; }
static void on_client_connect(int client_idx) { struct rte_mbuf *buffer = get_buffer(); if(!buffer) { return; } unsigned char *data = rte_pktmbuf_mtod(buffer, unsigned char *); *data = IPAUGENBLICK_NEW_IFACES; data++; rte_pktmbuf_data_len(buffer) = get_all_devices(data) + 1; rte_ring_enqueue(ipaugenblick_clients[client_idx].client_ring,(void *)buffer); buffer = get_buffer(); if(!buffer) { return; } data = rte_pktmbuf_mtod(buffer, unsigned char *); *data = IPAUGENBLICK_NEW_ADDRESSES; data++; rte_pktmbuf_data_len(buffer) = get_all_addresses(data) + 1; rte_ring_enqueue(ipaugenblick_clients[client_idx].client_ring,(void *)buffer); buffer = get_buffer(); if(!buffer) { return; } data = rte_pktmbuf_mtod(buffer, unsigned char *); *data = IPAUGENBLICK_END_OF_RECORD; rte_ring_enqueue(ipaugenblick_clients[client_idx].client_ring,(void *)buffer); }
/* * Allocate mbuf for flow stat (and latency) info sending * m - Original mbuf (can be complicated mbuf data structure) * fsp_head - return pointer in which the flow stat info should be filled * is_const - is the given mbuf const * return new mbuf structure in which the fsp_head can be written. If needed, orginal mbuf is freed. */ rte_mbuf_t * CGenNodeStateless::alloc_flow_stat_mbuf(rte_mbuf_t *m, struct flow_stat_payload_header *&fsp_head , bool is_const) { rte_mbuf_t *m_ret = NULL, *m_lat = NULL; uint16_t fsp_head_size = sizeof(struct flow_stat_payload_header); if (is_const) { // const mbuf case if (rte_pktmbuf_data_len(m) > 128) { m_ret = CGlobalInfo::pktmbuf_alloc_small(get_socket_id()); assert(m_ret); // alloc mbuf just for the latency header m_lat = CGlobalInfo::pktmbuf_alloc( get_socket_id(), fsp_head_size); assert(m_lat); fsp_head = (struct flow_stat_payload_header *)rte_pktmbuf_append(m_lat, fsp_head_size); rte_pktmbuf_attach(m_ret, m); rte_pktmbuf_trim(m_ret, sizeof(struct flow_stat_payload_header)); utl_rte_pktmbuf_add_after2(m_ret, m_lat); // ref count was updated when we took the (const) mbuf, and again in rte_pktmbuf_attach // so need do decrease now, to avoid leak. rte_pktmbuf_refcnt_update(m, -1); return m_ret; } else { // Short packet. Just copy all bytes. m_ret = CGlobalInfo::pktmbuf_alloc( get_socket_id(), rte_pktmbuf_data_len(m) ); assert(m_ret); char *p = rte_pktmbuf_mtod(m, char*); char *p_new = rte_pktmbuf_append(m_ret, rte_pktmbuf_data_len(m)); memcpy(p_new , p, rte_pktmbuf_data_len(m)); fsp_head = (struct flow_stat_payload_header *)(p_new + rte_pktmbuf_data_len(m) - fsp_head_size); rte_pktmbuf_free(m); return m_ret; } } else { // Field engine (vm) if (rte_pktmbuf_is_contiguous(m)) {
inline int ipaugenblick_sendto_bulk(int sock,struct data_and_descriptor *bufs_and_desc,int *offsets,int *lengths,unsigned int *ipaddrs,unsigned short *ports,int buffer_count) { int rc,idx,total_length = 0; struct rte_mbuf *mbufs[buffer_count]; for(idx = 0;idx < buffer_count;idx++) { char *p_addr; struct sockaddr_in *p_addr_in; /* TODO: set offsets */ mbufs[idx] = (struct rte_mbuf *)bufs_and_desc[idx].pdesc; rte_pktmbuf_data_len(mbufs[idx]) = lengths[idx]; total_length += lengths[idx]; p_addr = rte_pktmbuf_mtod(mbufs[idx],char *); rte_pktmbuf_data_len(mbufs[idx]) = lengths[idx]; p_addr -= sizeof(struct sockaddr_in); p_addr_in = (struct sockaddr_in *)p_addr; p_addr_in->sin_family = AF_INET; p_addr_in->sin_port = htons(ports[idx]); p_addr_in->sin_addr.s_addr = ipaddrs[idx]; } ipaugenblick_stats_send_called++; ipaugenblick_stats_buffers_sent += buffer_count; rte_atomic16_set(&(local_socket_descriptors[sock & SOCKET_READY_MASK].socket->write_ready_to_app),0); rc = ipaugenblick_enqueue_tx_bufs_bulk(sock,mbufs,buffer_count); if(rc == 0) rte_atomic32_sub(&(local_socket_descriptors[sock & SOCKET_READY_MASK].socket->tx_space),total_length); ipaugenblick_stats_send_failure += (rc != 0); return rc; }
/* Sends 'num_pkts' 'packets' and 'request' data to datapath. */ int dpdk_link_send_bulk(struct dpif_dpdk_message *request, const struct ofpbuf *const *packets, size_t num_pkts) { struct rte_mbuf *mbufs[PKT_BURST_SIZE] = {NULL}; uint8_t *mbuf_data = NULL; int i = 0; int ret = 0; if (num_pkts > PKT_BURST_SIZE) { return EINVAL; } DPDK_DEBUG() for (i = 0; i < num_pkts; i++) { mbufs[i] = rte_pktmbuf_alloc(mp); if (!mbufs[i]) { return ENOBUFS; } mbuf_data = rte_pktmbuf_mtod(mbufs[i], uint8_t *); rte_memcpy(mbuf_data, &request[i], sizeof(request[i])); if (request->type == DPIF_DPDK_PACKET_FAMILY) { mbuf_data = mbuf_data + sizeof(request[i]); if (likely(packets[i]->size <= (mbufs[i]->buf_len - sizeof(request[i])))) { rte_memcpy(mbuf_data, packets[i]->data, packets[i]->size); rte_pktmbuf_data_len(mbufs[i]) = sizeof(request[i]) + packets[i]->size; rte_pktmbuf_pkt_len(mbufs[i]) = rte_pktmbuf_data_len(mbufs[i]); } else { RTE_LOG(ERR, APP, "%s, %d: %s", __FUNCTION__, __LINE__, "memcpy prevented: packet size exceeds available mbuf space"); for (i = 0; i < num_pkts; i++) { rte_pktmbuf_free(mbufs[i]); } return ENOMEM; } } else { rte_pktmbuf_data_len(mbufs[i]) = sizeof(request[i]); rte_pktmbuf_pkt_len(mbufs[i]) = rte_pktmbuf_data_len(mbufs[i]); } } ret = rte_ring_sp_enqueue_bulk(message_ring, (void * const *)mbufs, num_pkts); if (ret == -ENOBUFS) { for (i = 0; i < num_pkts; i++) { rte_pktmbuf_free(mbufs[i]); } ret = ENOBUFS; } else if (unlikely(ret == -EDQUOT)) { ret = EDQUOT; } return ret; }
static void deliverframe(struct virtif_user *viu) { struct rte_mbuf *m, *m0; struct iovec iov[STACK_IOV]; struct iovec *iovp, *iovp0; assert(viu->viu_nbufpkts > 0 && viu->viu_bufidx < MAX_PKT_BURST); m0 = viu->viu_m_pkts[viu->viu_bufidx]; assert(m0 != NULL); viu->viu_bufidx++; viu->viu_nbufpkts--; if (m0->pkt.nb_segs > STACK_IOV) { iovp = malloc(sizeof(*iovp) * m0->pkt.nb_segs); if (iovp == NULL) return; /* drop */ } else { iovp = iov; } iovp0 = iovp; for (m = m0; m; m = m->pkt.next, iovp++) { iovp->iov_base = rte_pktmbuf_mtod(m, void *); iovp->iov_len = rte_pktmbuf_data_len(m); } VIF_DELIVERPKT(viu->viu_virtifsc, iovp0, iovp-iovp0); rte_pktmbuf_free(m0); if (iovp0 != iov) free(iovp0); }
/* * Main thread that does the work, reading from INPUT_PORT * and writing to OUTPUT_PORT */ static __attribute__((noreturn)) void lcore_main(void) { uint8_t port = 0; if (rte_eth_dev_socket_id(port) > 0 && rte_eth_dev_socket_id(port) != (int)rte_socket_id()) printf("WARNING, port %u is on remote NUMA node to " "polling thread.\n\tPerformance will " "not be optimal.\n", port); printf("\nCore %u forwarding packets. [Ctrl+C to quit]\n", rte_lcore_id()); for (;;) { struct rte_mbuf *bufs[BURST_SIZE]; const uint16_t nb_rx = rte_eth_rx_burst(port, 0, bufs, BURST_SIZE); uint16_t buf; if (unlikely(nb_rx == 0)) continue; for (buf = 0; buf < nb_rx; buf++) { struct rte_mbuf *mbuf = bufs[buf]; unsigned int len = rte_pktmbuf_data_len(mbuf); rte_pktmbuf_dump(stdout, mbuf, len); rte_pktmbuf_free(mbuf); } } }
static inline void send_burst_nodrop(struct rte_port_fd_writer_nodrop *p) { uint64_t n_retries; uint32_t i; n_retries = 0; for (i = 0; (i < p->tx_buf_count) && (n_retries < p->n_retries); i++) { struct rte_mbuf *pkt = p->tx_buf[i]; void *pkt_data = rte_pktmbuf_mtod(pkt, void*); size_t n_bytes = rte_pktmbuf_data_len(pkt); for ( ; n_retries < p->n_retries; n_retries++) { ssize_t ret; ret = write(p->fd, pkt_data, n_bytes); if (ret) break; } } RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - i); for (i = 0; i < p->tx_buf_count; i++) rte_pktmbuf_free(p->tx_buf[i]); p->tx_buf_count = 0; }
/* * test data manipulation in mbuf with non-ascii data */ static int test_pktmbuf_with_non_ascii_data(void) { struct rte_mbuf *m = NULL; char *data; m = rte_pktmbuf_alloc(pktmbuf_pool); if (m == NULL) GOTO_FAIL("Cannot allocate mbuf"); if (rte_pktmbuf_pkt_len(m) != 0) GOTO_FAIL("Bad length"); data = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN); if (data == NULL) GOTO_FAIL("Cannot append data"); if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) GOTO_FAIL("Bad pkt length"); if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) GOTO_FAIL("Bad data length"); memset(data, 0xff, rte_pktmbuf_pkt_len(m)); if (!rte_pktmbuf_is_contiguous(m)) GOTO_FAIL("Buffer should be continuous"); rte_pktmbuf_dump(m, MBUF_TEST_DATA_LEN); rte_pktmbuf_free(m); return 0; fail: if(m) { rte_pktmbuf_free(m); } return -1; }
int arp_in (struct rte_mbuf *mbuf) { assert(mbuf->buf_len >= sizeof(struct arp)); struct arp *arp_pkt; struct ether_hdr *eth; uint32_t ip_add = 0; eth = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); assert(rte_pktmbuf_data_len(mbuf) >= (sizeof(struct arp) + sizeof(struct ether_hdr))); arp_pkt = rte_pktmbuf_mtod(mbuf, char *) + sizeof(struct ether_hdr); switch(ntohs(arp_pkt->opcode)) { case ARP_REQ :// send_arp_reply(arp_pkt->dst_pr_add, arp_pkt->src_pr_add); break; /* uint32_t ip_add = GetIntAddFromChar(arp_pkt->src_pr_add, 0); add_mac((ip_add), arp_pkt->src_hw_add); logger(ARP, NORMAL, "seen arp packet\n"); break; */ case ARP_REPLY :// ip_add = GetIntAddFromChar(arp_pkt->src_pr_add, 0); add_mac((ip_add), arp_pkt->src_hw_add); break; // default : assert(0); } }
void dump_ring(CLQManager *a_pclsCLQ, char *a_pszQName, uint32_t a_unStartIdx, uint32_t a_unEndIdx) { struct rte_ring *pstRing = NULL; struct rte_mbuf *m = NULL; int ret = 0; uint32_t unMask = 0; uint32_t unStartIdx = 0; uint32_t unEndIdx = 0; uint32_t unIdx = 0; ret = a_pclsCLQ->CreateRing(a_pszQName, &pstRing); if(ret != E_Q_EXIST) { printf("There is no Queue (%s) \n", a_pszQName); if(ret == 0) { a_pclsCLQ->DeleteQueue(a_pszQName); } } if(pstRing != NULL) { unMask = pstRing->prod.mask; unStartIdx = pstRing->cons.tail; unEndIdx = pstRing->prod.tail; if(a_unEndIdx > unEndIdx) { printf("Invalid End idx %u\n", a_unEndIdx); return ; } if(a_unEndIdx < a_unStartIdx) { printf("Invalid Start Idx %u, End Idx %u\n", a_unStartIdx, a_unEndIdx); return ; } if(a_unStartIdx) unStartIdx = a_unStartIdx; if(a_unEndIdx) unEndIdx = a_unEndIdx; printf("Start Idx %u, End Idx %u\n", unStartIdx, unEndIdx); for(uint32_t i = unStartIdx; i < unEndIdx ; i++) { unIdx = i & unMask ; m = (struct rte_mbuf*)pstRing->ring[unIdx]; printf("idx : [%8u], total_len : [%5u], data_len : [%5u], seg_cnt : [%2u], Data : %s\n" , i , rte_pktmbuf_pkt_len(m) , rte_pktmbuf_data_len(m) , m->nb_segs , rte_pktmbuf_mtod(m, char*) ); } }
static inline struct rte_mbuf *ipaugenblick_get_from_shadow(int sock) { struct rte_mbuf *mbuf = NULL; if(local_socket_descriptors[sock].shadow) { mbuf = local_socket_descriptors[sock].shadow; local_socket_descriptors[sock].shadow = NULL; rte_pktmbuf_data_len(mbuf) = local_socket_descriptors[sock].shadow_len_remainder; mbuf->data_off += local_socket_descriptors[sock].shadow_len_delievered; if(mbuf->next) { rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf) + rte_pktmbuf_pkt_len(mbuf->next); } else { rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf); } mbuf->next = local_socket_descriptors[sock].shadow_next; local_socket_descriptors[sock].shadow_next = NULL; } return mbuf; }
static inline void ipaugenblick_try_read_exact_amount(struct rte_mbuf *mbuf,int sock,int *total_len,int *first_segment_len) { struct rte_mbuf *tmp = mbuf,*prev = NULL; int curr_len = 0; while((tmp)&&((rte_pktmbuf_data_len(tmp) + curr_len) < *total_len)) { curr_len += rte_pktmbuf_data_len(tmp); // printf("%s %d %d\n",__FILE__,__LINE__,rte_pktmbuf_data_len(tmp)); prev = tmp; tmp = tmp->next; } if(tmp) { // printf("%s %d %d\n",__FILE__,__LINE__,rte_pktmbuf_data_len(tmp)); if((curr_len + rte_pktmbuf_data_len(tmp)) > *total_len) { /* more data remains */ local_socket_descriptors[sock].shadow = tmp; local_socket_descriptors[sock].shadow_next = tmp->next; local_socket_descriptors[sock].shadow_len_remainder = (curr_len + rte_pktmbuf_data_len(tmp)) - *total_len; local_socket_descriptors[sock].shadow_len_delievered = rte_pktmbuf_data_len(tmp) - local_socket_descriptors[sock].shadow_len_remainder; rte_pktmbuf_data_len(tmp) = local_socket_descriptors[sock].shadow_len_delievered; *first_segment_len = local_socket_descriptors[sock].shadow_len_delievered; } /* if less data than required, tmp is NULL. we're here that means exact amount is read */ else { *first_segment_len = rte_pktmbuf_data_len(mbuf); /* store next mbuf, if there is */ if(tmp->next) { local_socket_descriptors[sock].shadow = tmp->next; local_socket_descriptors[sock].shadow_next = tmp->next->next; if(local_socket_descriptors[sock].shadow_next) { local_socket_descriptors[sock].shadow_len_remainder = rte_pktmbuf_data_len(local_socket_descriptors[sock].shadow_next); local_socket_descriptors[sock].shadow_len_delievered = 0; } } } tmp->next = NULL; if(curr_len == *total_len) { if(prev) prev->next = NULL; } } else { *total_len = curr_len; *first_segment_len = rte_pktmbuf_data_len(mbuf); } }
int ether_out(unsigned char *dst_mac, char *src_mac, uint16_t ether_type, struct rte_mbuf *mbuf) { int i = 0; struct ether_hdr *eth; eth = (struct ether_hdr *)rte_pktmbuf_prepend (mbuf, sizeof(struct ether_hdr)); eth->ether_type = htons(ether_type); for(i=0;i<6;i++) { eth->d_addr.addr_bytes[i] = dst_mac[i]; } // for(i=0;i<6;i++) { eth->s_addr.addr_bytes[0] = 0x6a; eth->s_addr.addr_bytes[1] = 0x9c; eth->s_addr.addr_bytes[2] = 0xba; eth->s_addr.addr_bytes[3] = 0xa0; eth->s_addr.addr_bytes[4] = 0x96; eth->s_addr.addr_bytes[5] = 0x24; // } // fix this this should be automatically detect the network interface id. send_packet_out(mbuf, 0); static int counter_id = -1; if(counter_id == -1) { counter_id = create_counter("sent_rate"); } int data_len = rte_pktmbuf_data_len(mbuf); counter_abs(counter_id, data_len); { static int counter_id = -1; if(counter_id == -1) { counter_id = create_counter("wire_sent"); } int data_len = rte_pktmbuf_data_len(mbuf); counter_inc(counter_id, data_len); } (void) src_mac; // jusat to avoid warning src_mac = NULL; return 0; }
/** * Process a crypto operation and complete a JOB_AES_HMAC job structure for * submission to the multi buffer library for processing. * * @param qp queue pair * @param job JOB_AES_HMAC structure to fill * @param m mbuf to process * * @return * - Completed JOB_AES_HMAC structure pointer on success * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible */ static JOB_AES_HMAC * process_crypto_op(struct aesni_mb_qp *qp, struct rte_crypto_op *op, struct aesni_mb_session *session) { JOB_AES_HMAC *job; struct rte_mbuf *m_src = op->sym->m_src, *m_dst; uint16_t m_offset = 0; job = (*qp->ops->job.get_next)(&qp->mb_mgr); if (unlikely(job == NULL)) return job; /* Set crypto operation */ job->chain_order = session->chain_order; /* Set cipher parameters */ job->cipher_direction = session->cipher.direction; job->cipher_mode = session->cipher.mode; job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes; job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode; job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode; /* Set authentication parameters */ job->hash_alg = session->auth.algo; if (job->hash_alg == AES_XCBC) { job->_k1_expanded = session->auth.xcbc.k1_expanded; job->_k2 = session->auth.xcbc.k2; job->_k3 = session->auth.xcbc.k3; } else { job->hashed_auth_key_xor_ipad = session->auth.pads.inner; job->hashed_auth_key_xor_opad = session->auth.pads.outer; } /* Mutable crypto operation parameters */ if (op->sym->m_dst) { m_src = m_dst = op->sym->m_dst; /* append space for output data to mbuf */ char *odata = rte_pktmbuf_append(m_dst, rte_pktmbuf_data_len(op->sym->m_src)); if (odata == NULL) { MB_LOG_ERR("failed to allocate space in destination " "mbuf for source data"); return NULL; } memcpy(odata, rte_pktmbuf_mtod(op->sym->m_src, void*), rte_pktmbuf_data_len(op->sym->m_src)); } else {
/* TCP or connected UDP */ inline int ipaugenblick_send(int sock,void *pdesc,int offset,int length) { int rc; struct rte_mbuf *mbuf = (struct rte_mbuf *)pdesc; ipaugenblick_stats_send_called++; rte_pktmbuf_data_len(mbuf) = length; mbuf->next = NULL; rte_atomic16_set(&(local_socket_descriptors[sock & SOCKET_READY_MASK].socket->write_ready_to_app),0); rc = ipaugenblick_enqueue_tx_buf(sock,mbuf); if(rc == 0) rte_atomic32_sub(&(local_socket_descriptors[sock & SOCKET_READY_MASK].socket->tx_space),length); ipaugenblick_stats_send_failure += (rc != 0); return rc; }
/* * Function sends unmatched packets to vswitchd. */ void send_packet_to_vswitchd(struct rte_mbuf *mbuf, struct dpdk_upcall *info) { int rslt = 0; void *mbuf_ptr = NULL; const uint64_t dpif_send_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * DPIF_SEND_US; uint64_t cur_tsc = 0; uint64_t diff_tsc = 0; static uint64_t prev_tsc = 0; /* send one packet, delete information about segments */ rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf); /* allocate space before the packet for the upcall info */ mbuf_ptr = rte_pktmbuf_prepend(mbuf, sizeof(*info)); if (mbuf_ptr == NULL) { printf("Cannot prepend upcall info\n"); rte_pktmbuf_free(mbuf); stats_vswitch_tx_drop_increment(INC_BY_1); stats_vport_tx_drop_increment(VSWITCHD, INC_BY_1); return; } rte_memcpy(mbuf_ptr, info, sizeof(*info)); /* send the packet and the upcall info to the daemon */ rslt = rte_ring_mp_enqueue(vswitchd_packet_ring, mbuf); if (rslt < 0) { if (rslt == -ENOBUFS) { rte_pktmbuf_free(mbuf); stats_vswitch_tx_drop_increment(INC_BY_1); stats_vport_tx_drop_increment(VSWITCHD, INC_BY_1); return; } else { stats_vport_overrun_increment(VSWITCHD, INC_BY_1); } } stats_vport_tx_increment(VSWITCHD, INC_BY_1); cur_tsc = rte_rdtsc(); diff_tsc = cur_tsc - prev_tsc; prev_tsc = cur_tsc; /* Only signal the daemon after 100 milliseconds */ if (unlikely(diff_tsc > dpif_send_tsc)) send_signal_to_dpif(); }
/** * Given a dpdk mbuf, fill in the Netmap slot in ring r and its associated * buffer with the data held by the mbuf. * Note that mbuf chains are not supported. */ static void mbuf_to_slot(struct rte_mbuf *mbuf, struct netmap_ring *r, uint32_t index) { char *data; uint16_t length; data = rte_pktmbuf_mtod(mbuf, char *); length = rte_pktmbuf_data_len(mbuf); if (length > r->nr_buf_size) length = 0; r->slot[index].len = length; rte_memcpy(NETMAP_BUF(r, r->slot[index].buf_idx), data, length); }
static inline void tx_xmit_pkt(struct fm10k_tx_queue *q, struct rte_mbuf *mb) { uint16_t last_id; uint8_t flags; /* always set the LAST flag on the last descriptor used to * transmit the packet */ flags = FM10K_TXD_FLAG_LAST; last_id = q->next_free + mb->nb_segs - 1; if (last_id >= q->nb_desc) last_id = last_id - q->nb_desc; /* but only set the RS flag on the last descriptor if rs_thresh * descriptors will be used since the RS flag was last set */ if ((q->nb_used + mb->nb_segs) >= q->rs_thresh) { flags |= FM10K_TXD_FLAG_RS; fifo_insert(&q->rs_tracker, last_id); q->nb_used = 0; } else { q->nb_used = q->nb_used + mb->nb_segs; } q->hw_ring[last_id].flags = flags; q->nb_free -= mb->nb_segs; /* set checksum flags on first descriptor of packet. SCTP checksum * offload is not supported, but we do not explicitly check for this * case in favor of greatly simplified processing. */ if (mb->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK)) q->hw_ring[q->next_free].flags |= FM10K_TXD_FLAG_CSUM; /* set vlan if requested */ if (mb->ol_flags & PKT_TX_VLAN_PKT) q->hw_ring[q->next_free].vlan = mb->vlan_tci; /* fill up the rings */ for (; mb != NULL; mb = mb->next) { q->sw_ring[q->next_free] = mb; q->hw_ring[q->next_free].buffer_addr = rte_cpu_to_le_64(MBUF_DMA_ADDR(mb)); q->hw_ring[q->next_free].buflen = rte_cpu_to_le_16(rte_pktmbuf_data_len(mb)); if (++q->next_free == q->nb_desc) q->next_free = 0; } }
/* Blocking function that waits for 'reply' from datapath. */ int dpdk_link_recv_reply(struct dpif_dpdk_message *reply) { struct rte_mbuf *mbuf = NULL; void *pktmbuf_data = NULL; int pktmbuf_len = 0; DPDK_DEBUG() while (rte_ring_sc_dequeue(reply_ring, (void **)&mbuf) != 0); pktmbuf_data = rte_pktmbuf_mtod(mbuf, void *); pktmbuf_len = rte_pktmbuf_data_len(mbuf); rte_memcpy(reply, pktmbuf_data, pktmbuf_len); rte_pktmbuf_free(mbuf); return 0; }
static int usock_mbuf_write(struct vr_usocket *usockp, struct rte_mbuf *mbuf) { unsigned int i, pkt_len; struct msghdr mhdr; struct rte_mbuf *m; struct iovec *iov; if (!mbuf) return 0; pkt_len = rte_pktmbuf_pkt_len(mbuf); if (!pkt_len) return 0; iov = usockp->usock_iovec; m = mbuf; for (i = 0; (m && (i < PKT0_MAX_IOV_LEN)); i++) { iov->iov_base = rte_pktmbuf_mtod(m, unsigned char *); iov->iov_len = rte_pktmbuf_data_len(m); m = m->next; iov++; } if ((i == PKT0_MAX_IOV_LEN) && m) usockp->usock_pkt_truncated++; mhdr.msg_name = NULL; mhdr.msg_namelen = 0; mhdr.msg_iov = usockp->usock_iovec; mhdr.msg_iovlen = i; mhdr.msg_control = NULL; mhdr.msg_controllen = 0; mhdr.msg_flags = 0; #ifdef VR_DPDK_USOCK_DUMP RTE_LOG(DEBUG, USOCK, "%s[%lx]: FD %d sending message\n", __func__, pthread_self(), usockp->usock_fd); rte_hexdump(stdout, "usock message dump:", &mhdr, sizeof(mhdr)); #endif return sendmsg(usockp->usock_fd, &mhdr, MSG_DONTWAIT); }
static void deliverframe(struct virtif_user *viu) { struct mbuf *m; struct rte_mbuf *rm, *rm0; struct vif_mextdata mext[STACK_MEXTDATA]; struct vif_mextdata *mextp, *mextp0 = NULL; assert(viu->viu_nbufpkts > 0 && viu->viu_bufidx < MAX_PKT_BURST); rm0 = viu->viu_m_pkts[viu->viu_bufidx]; assert(rm0 != NULL); viu->viu_bufidx++; viu->viu_nbufpkts--; if (rm0->pkt.nb_segs > STACK_MEXTDATA) { mextp = malloc(sizeof(*mextp) * rm0->pkt.nb_segs); if (mextp == NULL) goto drop; } else { mextp = mext; } mextp0 = mextp; for (rm = rm0; rm; rm = rm->pkt.next, mextp++) { mextp->mext_data = rte_pktmbuf_mtod(rm, void *); mextp->mext_dlen = rte_pktmbuf_data_len(rm); mextp->mext_arg = rm; } if (VIF_MBUF_EXTALLOC(mextp0, mextp - mextp0, &m) != 0) goto drop; VIF_DELIVERMBUF(viu->viu_virtifsc, m); if (mextp0 != mext) free(mextp0); return; drop: if (mextp0 != mext) free(mextp0); rte_pktmbuf_free(rm0); }
/* UDP or RAW */ inline int ipaugenblick_sendto(int sock,void *pdesc,int offset,int length,unsigned int ipaddr,unsigned short port) { int rc; struct rte_mbuf *mbuf = (struct rte_mbuf *)pdesc; char *p_addr = rte_pktmbuf_mtod(mbuf,char *); struct sockaddr_in *p_addr_in; ipaugenblick_stats_send_called++; rte_pktmbuf_data_len(mbuf) = length; p_addr -= sizeof(struct sockaddr_in); p_addr_in = (struct sockaddr_in *)p_addr; p_addr_in->sin_family = AF_INET; p_addr_in->sin_port = htons(port); p_addr_in->sin_addr.s_addr = ipaddr; rte_atomic16_set(&(local_socket_descriptors[sock & SOCKET_READY_MASK].socket->write_ready_to_app),0); rc = ipaugenblick_enqueue_tx_buf(sock,mbuf); if(rc == 0) rte_atomic32_sub(&(local_socket_descriptors[sock & SOCKET_READY_MASK].socket->tx_space),length); ipaugenblick_stats_send_failure += (rc != 0); return rc; }
inline int ipaugenblick_send_bulk(int sock,struct data_and_descriptor *bufs_and_desc,int *offsets,int *lengths,int buffer_count) { int rc,idx,total_length = 0; struct rte_mbuf *mbufs[buffer_count]; for(idx = 0;idx < buffer_count;idx++) { /* TODO: set offsets */ mbufs[idx] = (struct rte_mbuf *)bufs_and_desc[idx].pdesc; rte_pktmbuf_data_len(mbufs[idx]) = lengths[idx]; total_length += lengths[idx]; } ipaugenblick_stats_send_called++; ipaugenblick_stats_buffers_sent += buffer_count; rte_atomic16_set(&(local_socket_descriptors[sock & SOCKET_READY_MASK].socket->write_ready_to_app),0); rc = ipaugenblick_enqueue_tx_bufs_bulk(sock,mbufs,buffer_count); if(rc == 0) rte_atomic32_sub(&(local_socket_descriptors[sock & SOCKET_READY_MASK].socket->tx_space),total_length); ipaugenblick_stats_send_failure += (rc != 0); return rc; }
/* * Function sends unmatched packets to vswitchd. */ static void send_packet_to_vswitchd(struct rte_mbuf *mbuf, struct dpdk_upcall *info) { int rslt = 0; struct statistics *vswd_stat = NULL; void *mbuf_ptr = NULL; vswd_stat = &vport_stats[VSWITCHD]; /* send one packet, delete information about segments */ rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf); /* allocate space before the packet for the upcall info */ mbuf_ptr = rte_pktmbuf_prepend(mbuf, sizeof(*info)); if (mbuf_ptr == NULL) { printf("Cannot prepend upcall info\n"); rte_pktmbuf_free(mbuf); switch_tx_drop++; vswd_stat->tx_drop++; return; } rte_memcpy(mbuf_ptr, info, sizeof(*info)); /* send the packet and the upcall info to the daemon */ rslt = rte_ring_sp_enqueue(vswitch_packet_ring, mbuf); if (rslt < 0) { if (rslt == -ENOBUFS) { rte_pktmbuf_free(mbuf); switch_tx_drop++; vswd_stat->tx_drop++; return; } else { overruns++; } } vswd_stat->tx++; }
static inline void send_burst(struct rte_port_fd_writer *p) { uint32_t i; for (i = 0; i < p->tx_buf_count; i++) { struct rte_mbuf *pkt = p->tx_buf[i]; void *pkt_data = rte_pktmbuf_mtod(pkt, void*); size_t n_bytes = rte_pktmbuf_data_len(pkt); ssize_t ret; ret = write(p->fd, pkt_data, n_bytes); if (ret < 0) break; } RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - i); for (i = 0; i < p->tx_buf_count; i++) rte_pktmbuf_free(p->tx_buf[i]); p->tx_buf_count = 0; }
/* Blocking function that waits for a packet from datapath. 'pkt' will get * populated with packet data. */ int dpdk_link_recv_packet(struct ofpbuf **pkt, struct dpif_dpdk_upcall *info) { struct rte_mbuf *mbuf = NULL; uint16_t pktmbuf_len = 0; void *pktmbuf_data = NULL; DPDK_DEBUG() if (rte_ring_sc_dequeue(packet_ring, (void **)&mbuf) != 0) { return EAGAIN; } pktmbuf_data = rte_pktmbuf_mtod(mbuf, void *); pktmbuf_len = rte_pktmbuf_data_len(mbuf); rte_memcpy(info, pktmbuf_data, sizeof(*info)); pktmbuf_data = (uint8_t *)pktmbuf_data + sizeof(*info); *pkt = ofpbuf_clone_data(pktmbuf_data, pktmbuf_len - sizeof(*info)); rte_pktmbuf_free(mbuf); return 0; }
static uint32_t send_pkts(uint8_t port, struct rte_mempool* pool) { static uint64_t seq = 0; // alloc bufs struct rte_mbuf* bufs[BATCH_SIZE]; uint32_t i; for (i = 0; i < BATCH_SIZE; i++) { struct rte_mbuf* buf = rte_pktmbuf_alloc(pool); rte_pktmbuf_data_len(buf) = 60; rte_pktmbuf_pkt_len(buf) = 60; bufs[i] = buf; // write seq number uint64_t* pkt = rte_pktmbuf_mtod(buf, uint64_t*); pkt[0] = seq++; } // send pkts uint32_t sent = 0; while (1) { sent += rte_eth_tx_burst(port, 0, bufs + sent, BATCH_SIZE - sent); if (sent >= BATCH_SIZE) { return sent; } } }
/* * Send a reply message to the vswitchd */ static void send_reply_to_vswitchd(struct dpdk_message *reply) { struct rte_mbuf *mbuf = NULL; void *pktmbuf_data = NULL; int rslt = 0; /* Preparing the buffer to send */ mbuf = rte_pktmbuf_alloc(pktmbuf_pool); if (!mbuf) { RTE_LOG(WARNING, APP, "Error : Unable to allocate an mbuf " ": %s : %d", __FUNCTION__, __LINE__); stats_vswitch_tx_drop_increment(INC_BY_1); stats_vport_rx_drop_increment(VSWITCHD, INC_BY_1); return; } pktmbuf_data = rte_pktmbuf_mtod(mbuf, void *); rte_memcpy(pktmbuf_data, reply, sizeof(*reply)); rte_pktmbuf_data_len(mbuf) = sizeof(*reply); /* Sending the buffer to vswitchd */ rslt = rte_ring_mp_enqueue(vswitchd_reply_ring, (void *)mbuf); if (rslt < 0) { if (rslt == -ENOBUFS) { rte_pktmbuf_free(mbuf); stats_vswitch_tx_drop_increment(INC_BY_1); stats_vport_rx_drop_increment(VSWITCHD, INC_BY_1); } else { stats_vport_overrun_increment(VSWITCHD, INC_BY_1); stats_vport_rx_increment(VSWITCHD, INC_BY_1); } } else { stats_vport_rx_increment(VSWITCHD, INC_BY_1); } }
virtio_dev_rx(struct virtio_net *dev, struct rte_mbuf **pkts, uint32_t count) { struct vhost_virtqueue *vq; struct vring_desc *desc; struct rte_mbuf *buff; /* The virtio_hdr is initialised to 0. */ struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0,0,0,0,0,0},0}; uint64_t buff_addr = 0; uint64_t buff_hdr_addr = 0; uint32_t head[MAX_PKT_BURST], packet_len = 0; uint32_t head_idx, packet_success = 0; uint16_t avail_idx, res_cur_idx; uint16_t res_base_idx, res_end_idx; uint16_t free_entries; uint8_t success = 0; LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_rx()\n", dev->device_fh); vq = dev->virtqueue_rx; count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count; /* As many data cores may want access to available buffers, they need to be reserved. */ do { res_base_idx = vq->last_used_idx_res; avail_idx = *((volatile uint16_t *)&vq->avail->idx); free_entries = (avail_idx - res_base_idx); /*check that we have enough buffers*/ if (unlikely(count > free_entries)) count = free_entries; if (count == 0) return 0; res_end_idx = res_base_idx + count; /* vq->last_used_idx_res is atomically updated. */ success = rte_atomic16_cmpset(&vq->last_used_idx_res, res_base_idx, res_end_idx); } while (unlikely(success == 0)); res_cur_idx = res_base_idx; LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n", dev->device_fh, res_cur_idx, res_end_idx); /* Prefetch available ring to retrieve indexes. */ rte_prefetch0(&vq->avail->ring[res_cur_idx & (vq->size - 1)]); /* Retrieve all of the head indexes first to avoid caching issues. */ for (head_idx = 0; head_idx < count; head_idx++) head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) & (vq->size - 1)]; /*Prefetch descriptor index. */ rte_prefetch0(&vq->desc[head[packet_success]]); while (res_cur_idx != res_end_idx) { /* Get descriptor from available ring */ desc = &vq->desc[head[packet_success]]; /* Prefetch descriptor address. */ rte_prefetch0(desc); buff = pkts[packet_success]; /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */ buff_addr = gpa_to_vva(dev, desc->addr); /* Prefetch buffer address. */ rte_prefetch0((void*)(uintptr_t)buff_addr); { /* Copy virtio_hdr to packet and increment buffer address */ buff_hdr_addr = buff_addr; packet_len = rte_pktmbuf_data_len(buff) + vq->vhost_hlen; /* * If the descriptors are chained the header and data are placed in * separate buffers. */ if (desc->flags & VRING_DESC_F_NEXT) { desc->len = vq->vhost_hlen; desc = &vq->desc[desc->next]; /* Buffer address translation. */ buff_addr = gpa_to_vva(dev, desc->addr); desc->len = rte_pktmbuf_data_len(buff); } else { buff_addr += vq->vhost_hlen; desc->len = packet_len; } } /* Update used ring with desc information */ vq->used->ring[res_cur_idx & (vq->size - 1)].id = head[packet_success]; vq->used->ring[res_cur_idx & (vq->size - 1)].len = packet_len; /* Copy mbuf data to buffer */ rte_memcpy((void *)(uintptr_t)buff_addr, (const void*)buff->pkt.data, rte_pktmbuf_data_len(buff)); res_cur_idx++; packet_success++; /* mergeable is disabled then a header is required per buffer. */ rte_memcpy((void *)(uintptr_t)buff_hdr_addr, (const void*)&virtio_hdr, vq->vhost_hlen); if (res_cur_idx < res_end_idx) { /* Prefetch descriptor index. */ rte_prefetch0(&vq->desc[head[packet_success]]); } } rte_compiler_barrier(); /* Wait until it's our turn to add our buffer to the used ring. */ while (unlikely(vq->last_used_idx != res_base_idx)) rte_pause(); *(volatile uint16_t *)&vq->used->idx += count; vq->last_used_idx = res_end_idx; return count; }