/* Get generic traffic manager operations structure from a port. */
const struct rte_tm_ops *
rte_tm_ops_get(uint16_t port_id, struct rte_tm_error *error)
{
struct rte_eth_dev *dev = &rte_eth_devices[port_id];
const struct rte_tm_ops *ops;
if (!rte_eth_dev_is_valid_port(port_id)) {
rte_tm_error_set(error,
ENODEV,
RTE_TM_ERROR_TYPE_UNSPECIFIED,
NULL,
rte_strerror(ENODEV));
return NULL;
}
if ((dev->dev_ops->tm_ops_get == NULL) ||
(dev->dev_ops->tm_ops_get(dev, &ops) != 0) ||
(ops == NULL)) {
rte_tm_error_set(error,
ENOSYS,
RTE_TM_ERROR_TYPE_UNSPECIFIED,
NULL,
rte_strerror(ENOSYS));
return NULL;
}
return ops;
}
int
valid_bonded_port_id(uint8_t port_id)
{
if (!rte_eth_dev_is_valid_port(port_id))
return -1;
return valid_bonded_ethdev(&rte_eth_devices[port_id]);
}
int
valid_slave_port_id(uint8_t port_id)
{
/* Verify that port id's are valid */
if (!rte_eth_dev_is_valid_port(port_id))
return -1;
/* Verify that port_id refers to a non bonded port */
if (!valid_bonded_ethdev(&rte_eth_devices[port_id]))
return -1;
return 0;
}
struct sfc_txq *
sfc_txq_by_dp_txq(const struct sfc_dp_txq *dp_txq)
{
const struct sfc_dp_queue *dpq = &dp_txq->dpq;
struct rte_eth_dev *eth_dev;
struct sfc_adapter *sa;
struct sfc_txq *txq;
SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id));
eth_dev = &rte_eth_devices[dpq->port_id];
sa = eth_dev->data->dev_private;
SFC_ASSERT(dpq->queue_id < sa->txq_count);
txq = sa->txq_info[dpq->queue_id].txq;
SFC_ASSERT(txq != NULL);
return txq;
}
static inline int
port_init_common(uint8_t port, const struct rte_eth_conf *port_conf,
struct rte_mempool *mp)
{
const uint16_t rx_ring_size = 512, tx_ring_size = 512;
int retval;
uint16_t q;
struct rte_eth_dev_info dev_info;
if (!rte_eth_dev_is_valid_port(port))
return -1;
retval = rte_eth_dev_configure(port, 0, 0, port_conf);
rte_eth_dev_info_get(port, &dev_info);
default_params.rx_rings = RTE_MIN(dev_info.max_rx_queues,
MAX_NUM_RX_QUEUE);
default_params.tx_rings = 1;
/* Configure the Ethernet device. */
retval = rte_eth_dev_configure(port, default_params.rx_rings,
default_params.tx_rings, port_conf);
if (retval != 0)
return retval;
for (q = 0; q < default_params.rx_rings; q++) {
retval = rte_eth_rx_queue_setup(port, q, rx_ring_size,
rte_eth_dev_socket_id(port), NULL, mp);
if (retval < 0)
return retval;
}
/* Allocate and set up 1 TX queue per Ethernet port. */
for (q = 0; q < default_params.tx_rings; q++) {
retval = rte_eth_tx_queue_setup(port, q, tx_ring_size,
rte_eth_dev_socket_id(port), NULL);
if (retval < 0)
return retval;
}
/* Start the Ethernet port. */
retval = rte_eth_dev_start(port);
if (retval < 0)
return retval;
/* Display the port MAC address. */
struct ether_addr addr;
rte_eth_macaddr_get(port, &addr);
printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8
" %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n",
(unsigned int)port,
addr.addr_bytes[0], addr.addr_bytes[1],
addr.addr_bytes[2], addr.addr_bytes[3],
addr.addr_bytes[4], addr.addr_bytes[5]);
/* Enable RX in promiscuous mode for the Ethernet device. */
rte_eth_promiscuous_enable(port);
return 0;
}
/* Init KNI RX queue */
struct vr_dpdk_queue *
vr_dpdk_kni_rx_queue_init(unsigned lcore_id, struct vr_interface *vif,
unsigned host_lcore_id)
{
struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
const unsigned socket_id = rte_lcore_to_socket_id(lcore_id);
uint8_t port_id = 0;
unsigned vif_idx = vif->vif_idx;
struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif_idx];
struct vr_dpdk_queue_params *rx_queue_params
= &lcore->lcore_rx_queue_params[vif_idx];
if (vif->vif_type == VIF_TYPE_HOST) {
port_id = (((struct vr_dpdk_ethdev *)(vif->vif_bridge->vif_os))->
ethdev_port_id);
}
/* init queue */
rx_queue->rxq_ops = dpdk_knidev_reader_ops;
rx_queue->q_queue_h = NULL;
rx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx);
/* create the queue */
struct dpdk_knidev_reader_params reader_params = {
.kni = vif->vif_os,
};
rx_queue->q_queue_h = rx_queue->rxq_ops.f_create(&reader_params, socket_id);
if (rx_queue->q_queue_h == NULL) {
RTE_LOG(ERR, VROUTER, " error creating KNI device %s RX queue"
" at eth device %" PRIu8 "\n", vif->vif_name, port_id);
return NULL;
}
/* store queue params */
rx_queue_params->qp_release_op = &dpdk_kni_rx_queue_release;
return rx_queue;
}
/* Release KNI TX queue */
static void
dpdk_kni_tx_queue_release(unsigned lcore_id, struct vr_interface *vif)
{
struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
struct vr_dpdk_queue *tx_queue = &lcore->lcore_tx_queues[vif->vif_idx];
struct vr_dpdk_queue_params *tx_queue_params
= &lcore->lcore_tx_queue_params[vif->vif_idx];
tx_queue->txq_ops.f_tx = NULL;
rte_wmb();
/* flush and free the queue */
if (tx_queue->txq_ops.f_free(tx_queue->q_queue_h)) {
RTE_LOG(ERR, VROUTER, " error freeing lcore %u KNI device TX queue\n",
lcore_id);
}
/* reset the queue */
vrouter_put_interface(tx_queue->q_vif);
memset(tx_queue, 0, sizeof(*tx_queue));
memset(tx_queue_params, 0, sizeof(*tx_queue_params));
}
/* Init KNI TX queue */
struct vr_dpdk_queue *
vr_dpdk_kni_tx_queue_init(unsigned lcore_id, struct vr_interface *vif,
unsigned host_lcore_id)
{
struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
const unsigned socket_id = rte_lcore_to_socket_id(lcore_id);
uint8_t port_id = 0;
unsigned vif_idx = vif->vif_idx;
struct vr_dpdk_queue *tx_queue = &lcore->lcore_tx_queues[vif_idx];
struct vr_dpdk_queue_params *tx_queue_params
= &lcore->lcore_tx_queue_params[vif_idx];
struct vr_dpdk_ethdev *ethdev;
if (vif->vif_type == VIF_TYPE_HOST) {
ethdev = vif->vif_bridge->vif_os;
if (ethdev == NULL) {
RTE_LOG(ERR, VROUTER, " error creating KNI device %s TX queue:"
" bridge vif %u ethdev is not initialized\n",
vif->vif_name, vif->vif_bridge->vif_idx);
return NULL;
}
port_id = ethdev->ethdev_port_id;
}
/* init queue */
tx_queue->txq_ops = dpdk_knidev_writer_ops;
tx_queue->q_queue_h = NULL;
tx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx);
/* create the queue */
struct dpdk_knidev_writer_params writer_params = {
.kni = vif->vif_os,
.tx_burst_sz = VR_DPDK_TX_BURST_SZ,
};
tx_queue->q_queue_h = tx_queue->txq_ops.f_create(&writer_params, socket_id);
if (tx_queue->q_queue_h == NULL) {
RTE_LOG(ERR, VROUTER, " error creating KNI device %s TX queue"
" at eth device %" PRIu8 "\n", vif->vif_name, port_id);
return NULL;
}
/* store queue params */
tx_queue_params->qp_release_op = &dpdk_kni_tx_queue_release;
return tx_queue;
}
/* Change KNI MTU size callback */
static int
dpdk_knidev_change_mtu(uint8_t port_id, unsigned new_mtu)
{
struct vrouter *router = vrouter_get(0);
struct vr_interface *vif;
int i, ret;
uint8_t slave_port_id;
struct vr_dpdk_ethdev *ethdev = NULL;
if (port_id >= rte_eth_dev_count()) {
RTE_LOG(ERR, VROUTER,
"Error changing eth device %"PRIu8" MTU: invalid eth device\n",
port_id);
return -EINVAL;
}
/*
* TODO: DPDK bond PMD does not implement mtu_set op, so we need to
* set the MTU manually for all the slaves.
*/
/* Bond vif uses first slave port ID. */
if (router->vr_eth_if)
ethdev = (struct vr_dpdk_ethdev *)router->vr_eth_if->vif_os;
if (ethdev && vr_dpdk_ethdev_bond_port_match(port_id, ethdev)) {
RTE_LOG(INFO, VROUTER, "Changing bond eth device %" PRIu8 " MTU\n",
ethdev->ethdev_port_id);
rte_eth_devices[ethdev->ethdev_port_id].data->mtu = new_mtu;
for (i = 0; i < ethdev->ethdev_nb_slaves; i++) {
slave_port_id = ethdev->ethdev_slaves[i];
RTE_LOG(INFO, VROUTER,
" changing bond member eth device %" PRIu8 " MTU to %u\n",
slave_port_id, new_mtu);
ret = rte_eth_dev_set_mtu(slave_port_id, new_mtu);
if (ret < 0) {
RTE_LOG(ERR, VROUTER,
" error changing bond member eth device %" PRIu8 " MTU: %s (%d)\n",
slave_port_id, rte_strerror(-ret), -ret);
return ret;
}
}
} else {
RTE_LOG(INFO, VROUTER, "Changing eth device %" PRIu8 " MTU to %u\n",
port_id, new_mtu);
ret = rte_eth_dev_set_mtu(port_id, new_mtu);
if (ret < 0) {
RTE_LOG(ERR, VROUTER,
"Error changing eth device %" PRIu8 " MTU: %s (%d)\n",
port_id, rte_strerror(-ret), -ret);
return ret;
}
}
/* On success, inform vrouter about new MTU */
for (i = 0; i < router->vr_max_interfaces; i++) {
vif = __vrouter_get_interface(router, i);
if (vif && (vif->vif_type == VIF_TYPE_PHYSICAL)) {
/* Ethernet header size */
new_mtu += sizeof(struct vr_eth);
if (vr_dpdk.vlan_tag != VLAN_ID_INVALID) {
/* 802.1q header size */
new_mtu += sizeof(uint32_t);
}
vif->vif_mtu = new_mtu;
if (vif->vif_bridge)
vif->vif_bridge->vif_mtu = new_mtu;
}
}
return 0;
}
/* Configure KNI state callback */
static int
dpdk_knidev_config_network_if(uint8_t port_id, uint8_t if_up)
{
struct vrouter *router = vrouter_get(0);
struct vr_dpdk_ethdev *ethdev = NULL;
int ret = 0;
if (port_id >= rte_eth_dev_count() || port_id >= RTE_MAX_ETHPORTS) {
RTE_LOG(ERR, VROUTER, "%s: Invalid eth device %" PRIu8 "\n",
__func__, port_id);
return -EINVAL;
}
if (router->vr_eth_if)
ethdev = (struct vr_dpdk_ethdev *)router->vr_eth_if->vif_os;
if (ethdev && vr_dpdk_ethdev_bond_port_match(port_id, ethdev))
port_id = ethdev->ethdev_port_id;
RTE_LOG(INFO, VROUTER, "Configuring eth device %" PRIu8 " %s\n",
port_id, if_up ? "UP" : "DOWN");
if (if_up)
ret = rte_eth_dev_start(port_id);
else
rte_eth_dev_stop(port_id);
if (ret < 0) {
RTE_LOG(ERR, VROUTER, "Configuring eth device %" PRIu8 " UP "
"failed (%d)\n", port_id, ret);
}
return ret;
}
/*
* vr_dpdk_knidev_init - initializes Kernel Network Interface device using
* specified Ethernet device port.
*
* Returns 0 on success, < 0 otherwise.
*/
int
vr_dpdk_knidev_init(uint8_t port_id, struct vr_interface *vif)
{
int i;
struct rte_eth_dev_info dev_info;
struct rte_kni_conf kni_conf;
struct rte_kni_ops kni_ops;
struct rte_kni *kni;
struct rte_config *rte_conf = rte_eal_get_configuration();
if (!vr_dpdk.kni_inited) {
/*
* If the host does not support KNIs (i.e. RedHat), we'll get
* a panic here.
*/
rte_kni_init(VR_DPDK_MAX_KNI_INTERFACES);
vr_dpdk.kni_inited = true;
}
/* Check if port is valid. */
if (!rte_eth_dev_is_valid_port(port_id)) {
RTE_LOG(ERR, VROUTER, " error initializing KNI device %s: invalid eth device %"
PRIu8"\n", vif->vif_name, port_id);
return -EINVAL;
}
/* get eth device info */
memset(&dev_info, 0, sizeof(dev_info));
rte_eth_dev_info_get(port_id, &dev_info);
/* create KNI configuration */
memset(&kni_conf, 0, sizeof(kni_conf));
strncpy(kni_conf.name, (char *)vif->vif_name, sizeof(kni_conf.name) - 1);
kni_conf.addr = dev_info.pci_dev->addr;
kni_conf.id = dev_info.pci_dev->id;
kni_conf.group_id = port_id;
kni_conf.mbuf_size = vr_packet_sz;
/*
* Due to DPDK commit 41a6ebd, now to prevent packet reordering in KNI
* we have to bind KNI kernel thread to a first online unused CPU.
*/
for (i = 0; i < RTE_MAX_LCORE; i++) {
if (lcore_config[i].detected
&& rte_conf->lcore_role[VR_DPDK_FWD_LCORE_ID + i] == ROLE_OFF) {
kni_conf.force_bind = 1;
kni_conf.core_id = i;
RTE_LOG(INFO, VROUTER, " bind KNI kernel thread to CPU %d\n", i);
break;
}
}
/* KNI options
*
* Changing state of the KNI interface can change state of the physical
* interface. This is useful for the vhost, but not for the VLAN
* forwarding interface.
*/
if (vif->vif_type == VIF_TYPE_VLAN) {
memset(&kni_ops, 0, sizeof(kni_ops));
} else {
kni_ops.port_id = port_id;
kni_ops.change_mtu = dpdk_knidev_change_mtu;
kni_ops.config_network_if = dpdk_knidev_config_network_if;
}
/* allocate KNI device */
kni = rte_kni_alloc(vr_dpdk.rss_mempool, &kni_conf, &kni_ops);
if (kni == NULL) {
RTE_LOG(ERR, VROUTER, " error allocation KNI device %s"
" at eth device %" PRIu8 "\n", vif->vif_name, port_id);
return -ENOMEM;
}
/* store pointer to KNI for further use */
vif->vif_os = kni;
/* add interface to the table of KNIs */
for (i = 0; i < VR_DPDK_MAX_KNI_INTERFACES; i++) {
if (vr_dpdk.knis[i] == NULL) {
vr_dpdk.knis[i] = vif->vif_os;
break;
}
}
return 0;
}
/*
* vr_dpdk_knidev_release - release KNI interface and remove it from the
* global list.
* Returns 0 on success, < 0 otherwise.
*/
int
vr_dpdk_knidev_release(struct vr_interface *vif)
{
int i;
struct rte_kni *kni = vif->vif_os;
vif->vif_os = NULL;
/* delete the interface from the table of KNIs */
for (i = 0; i < VR_DPDK_MAX_KNI_INTERFACES; i++) {
if (vr_dpdk.knis[i] == kni) {
vr_dpdk.knis[i] = NULL;
break;
}
}
rte_wmb();
return rte_kni_release(kni);
}
static void
slave_port_init(uint16_t portid, struct rte_mempool *mbuf_pool)
{
int retval;
uint16_t nb_rxd = RTE_RX_DESC_DEFAULT;
uint16_t nb_txd = RTE_TX_DESC_DEFAULT;
struct rte_eth_dev_info dev_info;
struct rte_eth_rxconf rxq_conf;
struct rte_eth_txconf txq_conf;
struct rte_eth_conf local_port_conf = port_conf;
if (!rte_eth_dev_is_valid_port(portid))
rte_exit(EXIT_FAILURE, "Invalid port\n");
rte_eth_dev_info_get(portid, &dev_info);
if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
local_port_conf.txmode.offloads |=
DEV_TX_OFFLOAD_MBUF_FAST_FREE;
local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
dev_info.flow_type_rss_offloads;
if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
port_conf.rx_adv_conf.rss_conf.rss_hf) {
printf("Port %u modified RSS hash function based on hardware support,"
"requested:%#"PRIx64" configured:%#"PRIx64"\n",
portid,
port_conf.rx_adv_conf.rss_conf.rss_hf,
local_port_conf.rx_adv_conf.rss_conf.rss_hf);
}
retval = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);
if (retval != 0)
rte_exit(EXIT_FAILURE, "port %u: configuration failed (res=%d)\n",
portid, retval);
retval = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd);
if (retval != 0)
rte_exit(EXIT_FAILURE, "port %u: rte_eth_dev_adjust_nb_rx_tx_desc "
"failed (res=%d)\n", portid, retval);
/* RX setup */
rxq_conf = dev_info.default_rxconf;
rxq_conf.offloads = local_port_conf.rxmode.offloads;
retval = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
rte_eth_dev_socket_id(portid),
&rxq_conf,
mbuf_pool);
if (retval < 0)
rte_exit(retval, " port %u: RX queue 0 setup failed (res=%d)",
portid, retval);
/* TX setup */
txq_conf = dev_info.default_txconf;
txq_conf.offloads = local_port_conf.txmode.offloads;
retval = rte_eth_tx_queue_setup(portid, 0, nb_txd,
rte_eth_dev_socket_id(portid), &txq_conf);
if (retval < 0)
rte_exit(retval, "port %u: TX queue 0 setup failed (res=%d)",
portid, retval);
retval = rte_eth_dev_start(portid);
if (retval < 0)
rte_exit(retval,
"Start port %d failed (res=%d)",
portid, retval);
struct ether_addr addr;
rte_eth_macaddr_get(portid, &addr);
printf("Port %u MAC: ", portid);
PRINT_MAC(addr);
printf("\n");
}
/*
* Initializes a given port using global settings and with the RX buffers
* coming from the mbuf_pool passed as a parameter.
*/
static inline int
port_init(uint16_t port, struct rte_mempool *mbuf_pool)
{
struct rte_eth_conf port_conf = port_conf_default;
const uint16_t rx_rings = 1, tx_rings = 1;
uint16_t nb_rxd = RX_RING_SIZE;
uint16_t nb_txd = TX_RING_SIZE;
int retval;
uint16_t q;
struct rte_eth_dev_info dev_info;
struct rte_eth_txconf txconf;
if (!rte_eth_dev_is_valid_port(port))
return -1;
rte_eth_dev_info_get(port, &dev_info);
if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
port_conf.txmode.offloads |=
DEV_TX_OFFLOAD_MBUF_FAST_FREE;
/* Configure the Ethernet device. */
retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
if (retval != 0)
return retval;
retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd);
if (retval != 0)
return retval;
/* Allocate and set up 1 RX queue per Ethernet port. */
for (q = 0; q < rx_rings; q++) {
retval = rte_eth_rx_queue_setup(port, q, nb_rxd,
rte_eth_dev_socket_id(port), NULL, mbuf_pool);
if (retval < 0)
return retval;
}
txconf = dev_info.default_txconf;
txconf.offloads = port_conf.txmode.offloads;
/* Allocate and set up 1 TX queue per Ethernet port. */
for (q = 0; q < tx_rings; q++) {
retval = rte_eth_tx_queue_setup(port, q, nb_txd,
rte_eth_dev_socket_id(port), &txconf);
if (retval < 0)
return retval;
}
/* Start the Ethernet port. */
retval = rte_eth_dev_start(port);
if (retval < 0)
return retval;
/* Display the port MAC address. */
struct ether_addr addr;
rte_eth_macaddr_get(port, &addr);
printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8
" %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n",
port,
addr.addr_bytes[0], addr.addr_bytes[1],
addr.addr_bytes[2], addr.addr_bytes[3],
addr.addr_bytes[4], addr.addr_bytes[5]);
/* Enable RX in promiscuous mode for the Ethernet device. */
rte_eth_promiscuous_enable(port);
return 0;
}
