/**
* Set up the DPDK rings which will be used to pass packets, via
* pointers, between the multi-process server and client processes.
* Each client needs one RX queue.
*/
static int
init_shm_rings(void)
{
unsigned i;
unsigned socket_id;
const char * q_name;
const unsigned ringsize = CLIENT_QUEUE_RINGSIZE;
clients = rte_malloc("client details",
sizeof(*clients) * num_clients, 0);
if (clients == NULL)
rte_exit(EXIT_FAILURE, "Cannot allocate memory for client program details\n");
for (i = 0; i < num_clients; i++) {
/* Create an RX queue for each client */
socket_id = rte_socket_id();
q_name = get_rx_queue_name(i);
clients[i].rx_q = rte_ring_create(q_name,
ringsize, socket_id,
RING_F_SP_ENQ | RING_F_SC_DEQ ); /* single prod, single cons */
if (clients[i].rx_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create rx ring queue for client %u\n", i);
}
return 0;
}
/**
* Set up the DPDK rings which will be used to pass packets, via
* pointers, between the multi-process distributor and node processes.
* Each node needs one RX queue.
*/
static int
init_shm_rings(void)
{
unsigned int i;
unsigned int socket_id;
const char *q_name;
const unsigned int ringsize = NODE_QUEUE_RINGSIZE;
nodes = rte_malloc("node details",
sizeof(*nodes) * num_nodes, 0);
if (nodes == NULL)
rte_exit(EXIT_FAILURE, "Cannot allocate memory for "
"node program details\n");
for (i = 0; i < num_nodes; i++) {
/* Create an RX queue for each node */
socket_id = rte_socket_id();
q_name = get_rx_queue_name(i);
nodes[i].rx_q = rte_ring_create(q_name,
ringsize, socket_id,
RING_F_SP_ENQ | RING_F_SC_DEQ);
if (nodes[i].rx_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create rx ring queue "
"for node %u\n", i);
}
return 0;
}
/**
* Set up the DPDK rings which will be used to pass packets, via
* pointers, between the multi-process server and client processes.
* Each client needs one RX queue.
*/
static int
init_shm_rings(void)
{
unsigned i;
unsigned socket_id;
const char * q_name;
const unsigned ringsize = CLIENT_QUEUE_RINGSIZE;
int retval;
clients = rte_malloc("client details",
sizeof(*clients) * num_clients, 0);
if (clients == NULL)
rte_exit(EXIT_FAILURE, "Cannot allocate memory for client program details\n");
for (i = 0; i < num_clients; i++) {
/* Create an RX queue for each client */
socket_id = rte_socket_id();
q_name = get_rx_queue_name(i);
if (rte_eal_process_type() == RTE_PROC_SECONDARY)
{
clients[i].rx_q = rte_ring_lookup(get_rx_queue_name(i));
}
else
{
clients[i].rx_q = rte_ring_create(q_name,
ringsize, socket_id,
RING_F_SP_ENQ | RING_F_SC_DEQ ); /* single prod, single cons */
}
if (clients[i].rx_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create rx ring queue for client %u\n", i);
/*
int port = rte_eth_from_rings("PORT1", &clients[i].rx_q, 1, &clients[i].rx_q, 1, rte_socket_id());
RTE_LOG(INFO, APP, "Ring port Number: %d\n", port);
retval = init_port(port);
*/
}
if (retval < 0) return retval;
return 0;
}
/**
* Set up the DPDK rings which will be used to pass packets, via
* pointers, between the multi-process server and client processes.
* Each client needs one RX queue.
*/
static int
init_shm_rings(void)
{
unsigned i;
const unsigned ringsize = CLIENT_QUEUE_RINGSIZE;
clients = rte_malloc("client details",
sizeof(*clients) * num_clients, 0);
if (clients == NULL)
rte_exit(EXIT_FAILURE, "Cannot allocate memory for client program details\n");
port_queues = rte_malloc("port_txq details",
sizeof(*port_queues) * ports->num_ports, 0);
if (port_queues == NULL)
rte_exit(EXIT_FAILURE, "Cannot allocate memory for port tx_q details\n");
for (i = 0; i < num_clients; i++) {
/* Create an RX queue for each client */
clients[i].rx_q = rte_ring_create(get_rx_queue_name(i),
ringsize, SOCKET0,
NO_FLAGS); /* multi producer multi consumer*/
if (clients[i].rx_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create rx ring for client %u\n", i);
clients[i].tx_q = rte_ring_create(get_tx_queue_name(i),
ringsize, SOCKET0,
NO_FLAGS); /* multi producer multi consumer*/
if (clients[i].tx_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create tx ring for client %u\n", i);
}
for (i = 0; i < ports->num_ports; i++) {
/* Create an RX queue for each ports */
port_queues[i].tx_q = rte_ring_create(get_port_tx_queue_name(i),
ringsize, SOCKET0,
RING_F_SC_DEQ); /* multi producer single consumer*/
if (port_queues[i].tx_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create tx ring for port %u\n", i);
}
vswitch_packet_ring = rte_ring_create(PACKET_RING_NAME,
DAEMON_PKT_QUEUE_RINGSIZE, SOCKET0, NO_FLAGS);
if (vswitch_packet_ring == NULL)
rte_exit(EXIT_FAILURE, "Cannot create packet ring for vswitchd");
return 0;
}
/**
* Set up the DPDK rings which will be used to pass packets, via
* pointers, between the multi-process server and NF processes.
* Each NF needs one RX queue.
*/
static int
init_shm_rings(void) {
unsigned i;
unsigned socket_id;
const char * rq_name;
const char * tq_name;
const char * msg_q_name;
const unsigned ringsize = NF_QUEUE_RINGSIZE;
const unsigned msgringsize = NF_MSG_QUEUE_SIZE;
// use calloc since we allocate for all possible NFs
// ensure that all fields are init to 0 to avoid reading garbage
// TODO plopreiato, move to creation when a NF starts
for (i = 0; i < MAX_NFS; i++) {
/* Create an RX queue for each NF */
socket_id = rte_socket_id();
rq_name = get_rx_queue_name(i);
tq_name = get_tx_queue_name(i);
msg_q_name = get_msg_queue_name(i);
nfs[i].instance_id = i;
nfs[i].rx_q = rte_ring_create(rq_name,
ringsize, socket_id,
RING_F_SC_DEQ); /* multi prod, single cons */
nfs[i].tx_q = rte_ring_create(tq_name,
ringsize, socket_id,
RING_F_SC_DEQ); /* multi prod, single cons */
nfs[i].msg_q = rte_ring_create(msg_q_name,
msgringsize, socket_id,
RING_F_SC_DEQ); /* multi prod, single cons */
if (nfs[i].rx_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create rx ring queue for NF %u\n", i);
if (nfs[i].tx_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create tx ring queue for NF %u\n", i);
if (nfs[i].msg_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create msg queue for NF %u\n", i);
}
return 0;
}
/**
* Set up the DPDK rings which will be used to pass packets, via
* pointers, between the multi-process server and client processes.
* Each client needs one RX queue.
*/
static int
init_shm_rings(void)
{
unsigned i;
unsigned socket_id;
const char * q_name;
#ifdef INTERRUPT_FIFO
const char * fifo_name;
#endif
#ifdef INTERRUPT_SEM
const char * sem_name;
sem_t *mutex;
#endif
#if defined(INTERRUPT_FIFO) || defined(INTERRUPT_SEM)
#ifdef DPDK_FLAG
const char *irq_flag_name;
const unsigned flagsize = 4;
#else
key_t key;
int shmid;
char *shm;
#endif
#endif
const unsigned ringsize = CLIENT_QUEUE_RINGSIZE;
clients = rte_malloc("client details",
sizeof(*clients) * num_clients, 0);
if (clients == NULL)
rte_exit(EXIT_FAILURE, "Cannot allocate memory for client program details\n");
for (i = 0; i < num_clients; i++) {
/* Create an RX queue for each client */
socket_id = rte_socket_id();
q_name = get_rx_queue_name(i);
clients[i].rx_q = rte_ring_create(q_name,
ringsize, socket_id,
RING_F_SP_ENQ | RING_F_SC_DEQ ); /* single prod, single cons */
//verify two functions
uint16_t ring_cur_entries = rte_ring_count(clients[i].rx_q);
uint16_t ring_free_entries = rte_ring_free_count(clients[i].rx_q);
fprintf(stderr, "ring_cur_entries=%d, ring_free_entries=%d\n", ring_cur_entries, ring_free_entries);
if (clients[i].rx_q == NULL)
rte_exit(EXIT_FAILURE, "Cannot create rx ring queue for client %u\n", i);
//add by wei, create FIFO pipe
#ifdef INTERRUPT_FIFO
umask(0);
fifo_name = get_fifo_name(i);
clients[i].fifo_name = fifo_name;
mknod(fifo_name, S_IFIFO|0666, 0);
clients[i].fifo_fp = fopen(fifo_name, "w");
if(clients[i].fifo_fp == NULL) {
fprintf(stderr, "can not create FIFO for client %d\n", i);
exit(1);
}
#endif
#ifdef INTERRUPT_SEM
sem_name = get_sem_name(i);
clients[i].sem_name = sem_name;
fprintf(stderr, "sem_name=%s for client %d\n", sem_name, i);
mutex = sem_open(sem_name, O_CREAT, 06666, 0);
if(mutex == SEM_FAILED) {
fprintf(stderr, "can not create semaphore for client %d\n", i);
sem_unlink(sem_name);
exit(1);
}
clients[i].mutex = mutex;
#endif
#if defined(INTERRUPT_FIFO) || defined(INTERRUPT_SEM)
#ifdef DPDK_FLAG
irq_flag_name = get_irq_flag_name(i);
clients[i].irq_flag = (int *)rte_ring_create(irq_flag_name,
flagsize, socket_id,
RING_F_SP_ENQ | RING_F_SC_DEQ ); /* single prod, single cons */
#else
key = get_rx_shmkey(i);
if ((shmid = shmget(key, SHMSZ, IPC_CREAT | 0666)) < 0) {
fprintf(stderr, "can not create the shared memory segment for client %d\n", i);
exit(1);
}
if ((shm = shmat(shmid, NULL, 0)) == (char *) -1) {
fprintf(stderr, "can not attach the shared segment to the server space for client %d\n", i);
exit(1);
}
clients[i].shm_server = (int *)shm;
#endif
#endif
}
return 0;
}
/*
* Application main function - loops through
* receiving and processing packets. Never returns
*/
int
main(int argc, char *argv[])
{
struct rte_ring *rx_ring = NULL;
struct rte_ring *tx_ring = NULL;
int retval = 0;
void *pkts[PKT_READ_SIZE];
int rslt = 0;
if ((retval = rte_eal_init(argc, argv)) < 0) {
return -1;
}
argc -= retval;
argv += retval;
if (parse_app_args(argc, argv) < 0) {
rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n");
}
rx_ring = rte_ring_lookup(get_rx_queue_name(client_id));
if (rx_ring == NULL) {
rte_exit(EXIT_FAILURE,
"Cannot get RX ring - is server process running?\n");
}
tx_ring = rte_ring_lookup(get_tx_queue_name(client_id));
if (tx_ring == NULL) {
rte_exit(EXIT_FAILURE,
"Cannot get TX ring - is server process running?\n");
}
RTE_LOG(INFO, APP, "Finished Process Init.\n");
printf("\nClient process %d handling packets\n", client_id);
printf("[Press Ctrl-C to quit ...]\n");
for (;;) {
unsigned rx_pkts = PKT_READ_SIZE;
/* Try dequeuing max possible packets first, if that fails, get the
* most we can. Loop body should only execute once, maximum.
*/
while (unlikely(rte_ring_dequeue_bulk(rx_ring, pkts, rx_pkts) != 0) &&
rx_pkts > 0) {
rx_pkts = (uint16_t)RTE_MIN(rte_ring_count(rx_ring), PKT_READ_SIZE);
}
if (rx_pkts > 0) {
pkt++;
/* blocking enqueue */
do {
rslt = rte_ring_enqueue_bulk(tx_ring, pkts, rx_pkts);
} while (rslt == -ENOBUFS);
} else {
no_pkt++;
}
if (!(pkt % 100000)) {
printf("pkt %d %d\n", pkt, no_pkt);
pkt = no_pkt = 0;
}
}
}
/**
* CALLED BY NF:
* Initialises everything we need
*
* Returns the number of arguments parsed by both rte_eal_init and
* parse_nflib_args offset by 1. This is used by getopt in the NF's
* code. The offsetting by one accounts for getopt parsing "--" which
* increments optind by 1 each time.
*/
int
onvm_nf_init(int argc, char *argv[], const char *nf_tag) {
const struct rte_memzone *mz;
const struct rte_memzone *mz_scp;
struct rte_mempool *mp;
struct onvm_service_chain **scp;
int retval_eal, retval_parse, retval_final;
if ((retval_eal = rte_eal_init(argc, argv)) < 0)
return -1;
/* Modify argc and argv to conform to getopt rules for parse_nflib_args */
argc -= retval_eal; argv += retval_eal;
/* Reset getopt global variables opterr and optind to their default values */
opterr = 0; optind = 1;
if ((retval_parse = parse_nflib_args(argc, argv)) < 0)
rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n");
/*
* Calculate the offset that the nf will use to modify argc and argv for its
* getopt call. This is the sum of the number of arguments parsed by
* rte_eal_init and parse_nflib_args. This will be decremented by 1 to assure
* getopt is looking at the correct index since optind is incremented by 1 each
* time "--" is parsed.
* This is the value that will be returned if initialization succeeds.
*/
retval_final = (retval_eal + retval_parse) - 1;
/* Reset getopt global variables opterr and optind to their default values */
opterr = 0; optind = 1;
/* Lookup mempool for nf_info struct */
nf_info_mp = rte_mempool_lookup(_NF_MEMPOOL_NAME);
if (nf_info_mp == NULL)
rte_exit(EXIT_FAILURE, "No Client Info mempool - bye\n");
/* Initialize the info struct */
nf_info = ovnm_nf_info_init(nf_tag);
mp = rte_mempool_lookup(PKTMBUF_POOL_NAME);
if (mp == NULL)
rte_exit(EXIT_FAILURE, "Cannot get mempool for mbufs\n");
mz = rte_memzone_lookup(MZ_CLIENT_INFO);
if (mz == NULL)
rte_exit(EXIT_FAILURE, "Cannot get tx info structure\n");
tx_stats = mz->addr;
mz_scp = rte_memzone_lookup(MZ_SCP_INFO);
if (mz_scp == NULL)
rte_exit(EXIT_FAILURE, "Cannot get service chain info structre\n");
scp = mz_scp->addr;
default_chain = *scp;
onvm_sc_print(default_chain);
nf_info_ring = rte_ring_lookup(_NF_QUEUE_NAME);
if (nf_info_ring == NULL)
rte_exit(EXIT_FAILURE, "Cannot get nf_info ring");
/* Put this NF's info struct onto queue for manager to process startup */
if (rte_ring_enqueue(nf_info_ring, nf_info) < 0) {
rte_mempool_put(nf_info_mp, nf_info); // give back mermory
rte_exit(EXIT_FAILURE, "Cannot send nf_info to manager");
}
/* Wait for a client id to be assigned by the manager */
RTE_LOG(INFO, APP, "Waiting for manager to assign an ID...\n");
for (; nf_info->status == (uint16_t)NF_WAITING_FOR_ID ;) {
sleep(1);
}
/* This NF is trying to declare an ID already in use. */
if (nf_info->status == NF_ID_CONFLICT) {
rte_mempool_put(nf_info_mp, nf_info);
rte_exit(NF_ID_CONFLICT, "Selected ID already in use. Exiting...\n");
} else if(nf_info->status == NF_NO_IDS) {
rte_mempool_put(nf_info_mp, nf_info);
rte_exit(NF_NO_IDS, "There are no ids available for this NF\n");
} else if(nf_info->status != NF_STARTING) {
rte_mempool_put(nf_info_mp, nf_info);
rte_exit(EXIT_FAILURE, "Error occurred during manager initialization\n");
}
RTE_LOG(INFO, APP, "Using Instance ID %d\n", nf_info->instance_id);
RTE_LOG(INFO, APP, "Using Service ID %d\n", nf_info->service_id);
/* Now, map rx and tx rings into client space */
rx_ring = rte_ring_lookup(get_rx_queue_name(nf_info->instance_id));
if (rx_ring == NULL)
rte_exit(EXIT_FAILURE, "Cannot get RX ring - is server process running?\n");
tx_ring = rte_ring_lookup(get_tx_queue_name(nf_info->instance_id));
if (tx_ring == NULL)
rte_exit(EXIT_FAILURE, "Cannot get TX ring - is server process running?\n");
/* Tell the manager we're ready to recieve packets */
nf_info->status = NF_RUNNING;
RTE_LOG(INFO, APP, "Finished Process Init.\n");
return retval_final;
}
/*
* Application main function - loops through
* receiving and processing packets. Never returns
*/
int
main(int argc, char *argv[])
{
const struct rte_memzone *mz;
struct rte_ring *rx_ring;
struct rte_mempool *mp;
struct port_info *ports;
int need_flush = 0; /* indicates whether we have unsent packets */
int retval;
void *pkts[PKT_READ_SIZE];
uint16_t sent;
if ((retval = rte_eal_init(argc, argv)) < 0)
return -1;
argc -= retval;
argv += retval;
if (parse_app_args(argc, argv) < 0)
rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n");
if (rte_eth_dev_count() == 0)
rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");
rx_ring = rte_ring_lookup(get_rx_queue_name(client_id));
if (rx_ring == NULL)
rte_exit(EXIT_FAILURE, "Cannot get RX ring - is server process running?\n");
mp = rte_mempool_lookup(PKTMBUF_POOL_NAME);
if (mp == NULL)
rte_exit(EXIT_FAILURE, "Cannot get mempool for mbufs\n");
mz = rte_memzone_lookup(MZ_PORT_INFO);
if (mz == NULL)
rte_exit(EXIT_FAILURE, "Cannot get port info structure\n");
ports = mz->addr;
tx_stats = &(ports->tx_stats[client_id]);
configure_output_ports(ports);
RTE_LOG(INFO, APP, "Finished Process Init.\n");
printf("\nClient process %d handling packets\n", client_id);
printf("[Press Ctrl-C to quit ...]\n");
for (;;) {
uint16_t i, rx_pkts = PKT_READ_SIZE;
uint8_t port;
/* try dequeuing max possible packets first, if that fails, get the
* most we can. Loop body should only execute once, maximum */
while (rx_pkts > 0 &&
unlikely(rte_ring_dequeue_bulk(rx_ring, pkts, rx_pkts) != 0))
rx_pkts = (uint16_t)RTE_MIN(rte_ring_count(rx_ring), PKT_READ_SIZE);
if (unlikely(rx_pkts == 0)) {
if (need_flush)
for (port = 0; port < ports->num_ports; port++) {
sent = rte_eth_tx_buffer_flush(ports->id[port], client_id,
tx_buffer[port]);
if (unlikely(sent))
tx_stats->tx[port] += sent;
}
need_flush = 0;
continue;
}
for (i = 0; i < rx_pkts; i++)
handle_packet(pkts[i]);
need_flush = 1;
}
}
