/* Init eth TX queue */ struct vr_dpdk_queue * vr_dpdk_ethdev_tx_queue_init(unsigned lcore_id, struct vr_interface *vif, unsigned queue_or_lcore_id) { uint8_t port_id; uint16_t tx_queue_id = queue_or_lcore_id; unsigned int vif_idx = vif->vif_idx, dpdk_queue_index; const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id); struct vr_dpdk_ethdev *ethdev; struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id]; struct vr_dpdk_queue *tx_queue; struct vr_dpdk_queue_params *tx_queue_params; ethdev = (struct vr_dpdk_ethdev *)vif->vif_os; port_id = ethdev->ethdev_port_id; if (lcore->lcore_hw_queue_to_dpdk_index[vif->vif_idx]) { dpdk_queue_index = lcore->lcore_hw_queue_to_dpdk_index[vif->vif_idx][tx_queue_id]; } else { dpdk_queue_index = 0; } tx_queue = &lcore->lcore_tx_queues[vif_idx][dpdk_queue_index]; tx_queue_params = &lcore->lcore_tx_queue_params[vif_idx][dpdk_queue_index]; /* init queue */ tx_queue->txq_ops = rte_port_ethdev_writer_ops; tx_queue->q_queue_h = NULL; tx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx); /* create the queue */ struct rte_port_ethdev_writer_params writer_params = { .port_id = port_id, .queue_id = tx_queue_id, .tx_burst_sz = VR_DPDK_TX_BURST_SZ, }; tx_queue->q_queue_h = tx_queue->txq_ops.f_create(&writer_params, socket_id); if (tx_queue->q_queue_h == NULL) { RTE_LOG(ERR, VROUTER, " error creating eth device %" PRIu8 " TX queue %" PRIu16 "\n", port_id, tx_queue_id); return NULL; } /* store queue params */ tx_queue_params->qp_release_op = &dpdk_ethdev_tx_queue_release; tx_queue_params->qp_ethdev.queue_id = tx_queue_id; tx_queue_params->qp_ethdev.port_id = port_id; /* for the queue 0 add queue params to the list of bonds to TX */ if (ethdev->ethdev_nb_slaves > 0 && tx_queue_id == 0) { /* make sure queue params have been stored */ rte_wmb(); lcore->lcore_bonds_to_tx[lcore->lcore_nb_bonds_to_tx++] = tx_queue_params; RTE_VERIFY(lcore->lcore_nb_bonds_to_tx <= VR_DPDK_MAX_BONDS); } return tx_queue; }
/* * vr_dpdk_virtio_rx_queue_init - initializes a virtio RX queue. * * Returns a pointer to the RX queue on success, NULL otherwise. */ struct vr_dpdk_queue * vr_dpdk_virtio_rx_queue_init(unsigned int lcore_id, struct vr_interface *vif, unsigned int queue_or_lcore_id) { uint16_t queue_id = queue_or_lcore_id; struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id]; const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id); unsigned int vif_idx = vif->vif_idx; struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif_idx]; struct vr_dpdk_queue_params *rx_queue_params = &lcore->lcore_rx_queue_params[vif_idx]; /* Check input parameters */ if (queue_id >= vr_dpdk_virtio_nrxqs(vif)) { RTE_LOG(ERR, VROUTER, " error creating virtio device %s RX queue %" PRIu16 "\n", vif->vif_name, queue_id); return NULL; } /* init queue */ rx_queue->rxq_ops = vr_dpdk_virtio_reader_ops; rx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx); /* init virtio queue */ vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_ready_state = VQ_NOT_READY; vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_last_used_idx = 0; vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_last_used_idx_res = 0; vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_vif_idx = vif->vif_idx; /* create the queue */ struct dpdk_virtio_reader_params reader_params = { .rx_virtioq = &vr_dpdk_virtio_rxqs[vif_idx][queue_id], }; rx_queue->q_queue_h = rx_queue->rxq_ops.f_create(&reader_params, socket_id); if (rx_queue->q_queue_h == NULL) { RTE_LOG(ERR, VROUTER, " error creating virtio device %s RX queue %" PRIu16 "\n", vif->vif_name, queue_id); return NULL; } /* store queue params */ rx_queue_params->qp_release_op = &dpdk_virtio_rx_queue_release; return rx_queue; }
/* Init eth RX queue */ struct vr_dpdk_queue * vr_dpdk_ethdev_rx_queue_init(unsigned lcore_id, struct vr_interface *vif, unsigned queue_or_lcore_id) { uint16_t rx_queue_id = queue_or_lcore_id; uint8_t port_id; unsigned int vif_idx = vif->vif_idx; const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id); struct vr_dpdk_ethdev *ethdev; struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id]; struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif_idx]; struct vr_dpdk_queue_params *rx_queue_params = &lcore->lcore_rx_queue_params[vif_idx]; ethdev = (struct vr_dpdk_ethdev *)vif->vif_os; port_id = ethdev->ethdev_port_id; /* init queue */ rx_queue->rxq_ops = rte_port_ethdev_reader_ops; rx_queue->q_queue_h = NULL; rx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx); /* create the queue */ struct rte_port_ethdev_reader_params reader_params = { .port_id = port_id, .queue_id = rx_queue_id, }; rx_queue->q_queue_h = rx_queue->rxq_ops.f_create(&reader_params, socket_id); if (rx_queue->q_queue_h == NULL) { RTE_LOG(ERR, VROUTER, " error creating eth device %" PRIu8 " RX queue %" PRIu16 "\n", port_id, rx_queue_id); return NULL; } /* store queue params */ rx_queue_params->qp_release_op = &dpdk_ethdev_rx_queue_release; rx_queue_params->qp_ethdev.queue_id = rx_queue_id; rx_queue_params->qp_ethdev.port_id = port_id; return rx_queue; }
/* * vr_dpdk_virtio_tx_queue_init - initializes a virtio TX queue. * * Returns a pointer to the TX queue on success, NULL otherwise. */ struct vr_dpdk_queue * vr_dpdk_virtio_tx_queue_init(unsigned int lcore_id, struct vr_interface *vif, unsigned int queue_or_lcore_id) { uint16_t queue_id = queue_or_lcore_id; struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id]; const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id); unsigned int vif_idx = vif->vif_idx; struct vr_dpdk_queue *tx_queue = &lcore->lcore_tx_queues[vif_idx]; struct vr_dpdk_queue_params *tx_queue_params = &lcore->lcore_tx_queue_params[vif_idx]; /* Check input parameters */ /* virtio TX is thread safe, so just use one of the rings */ queue_id = queue_id % vr_dpdk_virtio_ntxqs(vif); /* init queue */ tx_queue->txq_ops = vr_dpdk_virtio_writer_ops; tx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx); /* init virtio queue */ vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_ready_state = VQ_NOT_READY; vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_last_used_idx = 0; vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_last_used_idx_res = 0; vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_vif_idx = vif->vif_idx; /* create the queue */ struct dpdk_virtio_writer_params writer_params = { .tx_virtioq = &vr_dpdk_virtio_txqs[vif_idx][queue_id], }; tx_queue->q_queue_h = tx_queue->txq_ops.f_create(&writer_params, socket_id); if (tx_queue->q_queue_h == NULL) { RTE_LOG(ERR, VROUTER, " error creating virtio device %s TX queue %" PRIu16 "\n", vif->vif_name, queue_id); return NULL; } /* store queue params */ tx_queue_params->qp_release_op = &dpdk_virtio_tx_queue_release; return tx_queue; }
/* Init KNI RX queue */ struct vr_dpdk_queue * vr_dpdk_kni_rx_queue_init(unsigned lcore_id, struct vr_interface *vif, unsigned host_lcore_id) { struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id]; const unsigned socket_id = rte_lcore_to_socket_id(lcore_id); uint8_t port_id = 0; unsigned vif_idx = vif->vif_idx; struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif_idx]; struct vr_dpdk_queue_params *rx_queue_params = &lcore->lcore_rx_queue_params[vif_idx]; if (vif->vif_type == VIF_TYPE_HOST) { port_id = (((struct vr_dpdk_ethdev *)(vif->vif_bridge->vif_os))-> ethdev_port_id); } /* init queue */ rx_queue->rxq_ops = dpdk_knidev_reader_ops; rx_queue->q_queue_h = NULL; rx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx); /* create the queue */ struct dpdk_knidev_reader_params reader_params = { .kni = vif->vif_os, }; rx_queue->q_queue_h = rx_queue->rxq_ops.f_create(&reader_params, socket_id); if (rx_queue->q_queue_h == NULL) { RTE_LOG(ERR, VROUTER, " error creating KNI device %s RX queue" " at eth device %" PRIu8 "\n", vif->vif_name, port_id); return NULL; } /* store queue params */ rx_queue_params->qp_release_op = &dpdk_kni_rx_queue_release; return rx_queue; } /* Release KNI TX queue */ static void dpdk_kni_tx_queue_release(unsigned lcore_id, struct vr_interface *vif) { struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id]; struct vr_dpdk_queue *tx_queue = &lcore->lcore_tx_queues[vif->vif_idx]; struct vr_dpdk_queue_params *tx_queue_params = &lcore->lcore_tx_queue_params[vif->vif_idx]; tx_queue->txq_ops.f_tx = NULL; rte_wmb(); /* flush and free the queue */ if (tx_queue->txq_ops.f_free(tx_queue->q_queue_h)) { RTE_LOG(ERR, VROUTER, " error freeing lcore %u KNI device TX queue\n", lcore_id); } /* reset the queue */ vrouter_put_interface(tx_queue->q_vif); memset(tx_queue, 0, sizeof(*tx_queue)); memset(tx_queue_params, 0, sizeof(*tx_queue_params)); } /* Init KNI TX queue */ struct vr_dpdk_queue * vr_dpdk_kni_tx_queue_init(unsigned lcore_id, struct vr_interface *vif, unsigned host_lcore_id) { struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id]; const unsigned socket_id = rte_lcore_to_socket_id(lcore_id); uint8_t port_id = 0; unsigned vif_idx = vif->vif_idx; struct vr_dpdk_queue *tx_queue = &lcore->lcore_tx_queues[vif_idx]; struct vr_dpdk_queue_params *tx_queue_params = &lcore->lcore_tx_queue_params[vif_idx]; struct vr_dpdk_ethdev *ethdev; if (vif->vif_type == VIF_TYPE_HOST) { ethdev = vif->vif_bridge->vif_os; if (ethdev == NULL) { RTE_LOG(ERR, VROUTER, " error creating KNI device %s TX queue:" " bridge vif %u ethdev is not initialized\n", vif->vif_name, vif->vif_bridge->vif_idx); return NULL; } port_id = ethdev->ethdev_port_id; } /* init queue */ tx_queue->txq_ops = dpdk_knidev_writer_ops; tx_queue->q_queue_h = NULL; tx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx); /* create the queue */ struct dpdk_knidev_writer_params writer_params = { .kni = vif->vif_os, .tx_burst_sz = VR_DPDK_TX_BURST_SZ, }; tx_queue->q_queue_h = tx_queue->txq_ops.f_create(&writer_params, socket_id); if (tx_queue->q_queue_h == NULL) { RTE_LOG(ERR, VROUTER, " error creating KNI device %s TX queue" " at eth device %" PRIu8 "\n", vif->vif_name, port_id); return NULL; } /* store queue params */ tx_queue_params->qp_release_op = &dpdk_kni_tx_queue_release; return tx_queue; } /* Change KNI MTU size callback */ static int dpdk_knidev_change_mtu(uint8_t port_id, unsigned new_mtu) { struct vrouter *router = vrouter_get(0); struct vr_interface *vif; int i, ret; uint8_t ethdev_port_id, slave_port_id; struct vr_dpdk_ethdev *ethdev = NULL; RTE_LOG(INFO, VROUTER, "Changing eth device %" PRIu8 " MTU to %u\n", port_id, new_mtu); if (port_id >= rte_eth_dev_count()) { RTE_LOG(ERR, VROUTER, "Error changing eth device %"PRIu8" MTU: invalid eth device\n", port_id); return -EINVAL; } /* * TODO: DPDK bond PMD does not implement mtu_set op, so we need to * set the MTU manually for all the slaves. */ /* Bond vif uses first slave port ID. */ if (router->vr_eth_if) { ethdev = (struct vr_dpdk_ethdev *)router->vr_eth_if->vif_os; if (ethdev && ethdev->ethdev_nb_slaves > 0) { for (i = 0; i < ethdev->ethdev_nb_slaves; i++) { if (port_id == ethdev->ethdev_slaves[i]) break; } /* Clear ethdev if no port match. */ if (i >= ethdev->ethdev_nb_slaves) ethdev = NULL; } } if (ethdev && ethdev->ethdev_nb_slaves > 0) { for (i = 0; i < ethdev->ethdev_nb_slaves; i++) { slave_port_id = ethdev->ethdev_slaves[i]; RTE_LOG(INFO, VROUTER, " changing bond member eth device %" PRIu8 " MTU to %u\n", slave_port_id, new_mtu); ret = rte_eth_dev_set_mtu(slave_port_id, new_mtu); if (ret < 0) { RTE_LOG(ERR, VROUTER, " error changing bond member eth device %" PRIu8 " MTU: %s (%d)\n", slave_port_id, rte_strerror(-ret), -ret); return ret; } } } else { ret = rte_eth_dev_set_mtu(port_id, new_mtu); if (ret < 0) { RTE_LOG(ERR, VROUTER, "Error changing eth device %" PRIu8 " MTU: %s (%d)\n", port_id, rte_strerror(-ret), -ret); } return ret; } /* On success, inform vrouter about new MTU */ for (i = 0; i < router->vr_max_interfaces; i++) { vif = __vrouter_get_interface(router, i); if (vif && (vif->vif_type == VIF_TYPE_PHYSICAL)) { ethdev_port_id = (((struct vr_dpdk_ethdev *)(vif->vif_os))-> ethdev_port_id); if (ethdev_port_id == port_id) { /* Ethernet header size */ new_mtu += sizeof(struct vr_eth); if (vr_dpdk.vlan_tag != VLAN_ID_INVALID) { /* 802.1q header size */ new_mtu += sizeof(uint32_t); } vif->vif_mtu = new_mtu; if (vif->vif_bridge) vif->vif_bridge->vif_mtu = new_mtu; } } } return 0; } /* Configure KNI state callback */ static int dpdk_knidev_config_network_if(uint8_t port_id, uint8_t if_up) { int ret = 0; RTE_LOG(INFO, VROUTER, "Configuring eth device %" PRIu8 " %s\n", port_id, if_up ? "UP" : "DOWN"); if (port_id >= rte_eth_dev_count() || port_id >= RTE_MAX_ETHPORTS) { RTE_LOG(ERR, VROUTER, "Invalid eth device %" PRIu8 "\n", port_id); return -EINVAL; } if (if_up) ret = rte_eth_dev_start(port_id); else rte_eth_dev_stop(port_id); if (ret < 0) { RTE_LOG(ERR, VROUTER, "Configuring eth device %" PRIu8 " UP" "failed (%d)\n", port_id, ret); } return ret; } /* Init KNI */ int vr_dpdk_knidev_init(uint8_t port_id, struct vr_interface *vif) { int i; struct rte_eth_dev_info dev_info; struct rte_kni_conf kni_conf; struct rte_kni_ops kni_ops; struct rte_kni *kni; struct rte_config *rte_conf = rte_eal_get_configuration(); if (!vr_dpdk.kni_inited) { /* * If the host does not support KNIs (i.e. RedHat), we'll get * a panic here. */ rte_kni_init(VR_DPDK_MAX_KNI_INTERFACES); vr_dpdk.kni_inited = true; } /* get eth device info */ memset(&dev_info, 0, sizeof(dev_info)); rte_eth_dev_info_get(port_id, &dev_info); /* create KNI configuration */ memset(&kni_conf, 0, sizeof(kni_conf)); strncpy(kni_conf.name, (char *)vif->vif_name, sizeof(kni_conf.name) - 1); kni_conf.addr = dev_info.pci_dev->addr; kni_conf.id = dev_info.pci_dev->id; kni_conf.group_id = port_id; kni_conf.mbuf_size = VR_DPDK_MAX_PACKET_SZ; /* * Due to DPDK commit 41a6ebd, now to prevent packet reordering in KNI * we have to bind KNI kernel thread to a first online unused CPU. */ for (i = 0; i < RTE_MAX_LCORE; i++) { if (lcore_config[i].detected && rte_conf->lcore_role[VR_DPDK_FWD_LCORE_ID + i] == ROLE_OFF) { kni_conf.force_bind = 1; kni_conf.core_id = i; RTE_LOG(INFO, VROUTER, " bind KNI kernel thread to CPU %d\n", i); break; } } /* KNI options * * Changing state of the KNI interface can change state of the physical * interface. This is useful for the vhost, but not for the VLAN * forwarding interface. */ if (vif->vif_type == VIF_TYPE_VLAN) { memset(&kni_ops, 0, sizeof(kni_ops)); } else { kni_ops.port_id = port_id; kni_ops.change_mtu = dpdk_knidev_change_mtu; kni_ops.config_network_if = dpdk_knidev_config_network_if; } /* allocate KNI device */ kni = rte_kni_alloc(vr_dpdk.rss_mempool, &kni_conf, &kni_ops); if (kni == NULL) { RTE_LOG(ERR, VROUTER, " error allocation KNI device %s" " at eth device %" PRIu8 "\n", vif->vif_name, port_id); return -ENOMEM; } /* store pointer to KNI for further use */ vif->vif_os = kni; /* add interface to the table of KNIs */ for (i = 0; i < VR_DPDK_MAX_KNI_INTERFACES; i++) { if (vr_dpdk.knis[i] == NULL) { vr_dpdk.knis[i] = vif->vif_os; break; } } return 0; }