static void
test_ofp_packet_input_to_sp(void)
{
odp_packet_t pkt;
odp_event_t ev;
int res;
my_test_val = TEST_FORWARD_HOOK;
/* Call ofp_packet_input using a pkt with destination ip
* that does NOT match the local ip on ifnet and NO route is found.
* The packet is forwarded to slow path queue. */
if (create_odp_packet_ip4(&pkt, test_frame, sizeof(test_frame), 0, 0)) {
CU_FAIL("Fail to create packet");
return;
}
res = ofp_packet_input(pkt, interface_queue[port],
ofp_eth_vlan_processing);
CU_ASSERT_EQUAL(res, OFP_PKT_PROCESSED);
CU_ASSERT_NOT_EQUAL(ev = odp_queue_deq(ifnet->spq_def),
ODP_EVENT_INVALID);
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->spq_def), ODP_EVENT_INVALID);
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->outq_def), ODP_EVENT_INVALID);
if (memcmp(odp_packet_data(odp_packet_from_event(ev)),
in_pkt_data, sizeof(test_frame)))
CU_FAIL("corrupt data sent to slow path");
odp_packet_free(odp_packet_from_event(ev));
CU_PASS("ofp_packet_input_to_sp");
}
static void
test_ofp_packet_input_local_UDPv4_hook(void)
{
odp_packet_t pkt;
int res;
/* Call ofp_packet_input with a pkt with destination ip
* that matches the local ip on ifnet.
* The packet is terminated in local UDPv4 hook */
my_test_val = TEST_LOCAL_UDPv4_HOOK;
ifnet->ip_addr = dst_ipaddr;
if (create_odp_packet_ip4(&pkt, test_frame, sizeof(test_frame),
dst_ipaddr, 0)) {
CU_FAIL("Fail to create packet");
return;
}
res = ofp_packet_input(pkt, interface_queue[port],
ofp_eth_vlan_processing);
CU_ASSERT_EQUAL(res, OFP_TEST_LOCAL_UDPv4_HOOK);
#ifdef SP
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->spq_def), ODP_EVENT_INVALID);
#endif /* SP */
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->outq_def), ODP_EVENT_INVALID);
ifnet->ip_addr = 0;
CU_PASS("ofp_packet_input_local_UDPv4_hook");
}
static void *pkt_io_recv(void *arg)
{
odp_pktio_t pktio;
odp_packet_t pkt, pkt_tbl[OFP_PKT_BURST_SIZE];
int pkt_idx, pkt_cnt;
struct pktio_thr_arg *thr_args;
ofp_pkt_processing_func pkt_func;
thr_args = arg;
pkt_func = thr_args->pkt_func;
if (odp_init_local(ODP_THREAD_WORKER)) {
OFP_ERR("Error: ODP local init failed.\n");
return NULL;
}
if (ofp_init_local()) {
OFP_ERR("Error: OFP local init failed.\n");
return NULL;
}
pktio = ofp_port_pktio_get(thr_args->port);
OFP_DBG("PKT-IO receive starting on port: %d, pktio-id: %"PRIX64"\n",
thr_args->port, odp_pktio_to_u64(pktio));
while (1) {
pkt_cnt = odp_pktio_recv(pktio, pkt_tbl, OFP_PKT_BURST_SIZE);
for (pkt_idx = 0; pkt_idx < pkt_cnt; pkt_idx++) {
pkt = pkt_tbl[pkt_idx];
if (odp_unlikely(odp_packet_has_error(pkt))) {
OFP_DBG("Packet with error dropped.\n");
odp_packet_free(pkt);
continue;
}
ofp_packet_input(pkt, ODP_QUEUE_INVALID, pkt_func);
}
#ifdef OFP_SEND_PKT_BURST
ofp_send_pending_pkt_burst();
#endif /*OFP_SEND_PKT_BURST*/
}
/* Never reached */
return NULL;
}
static void test_ofp_packet_input_gre_orig_pkt_to_sp(void)
{
odp_packet_t pkt;
int res;
#ifdef SP
odp_event_t ev;
#endif
my_test_val = TEST_LOCAL_HOOK_GRE_APP;
/* Call ofp_packet_input using a GRE pkt with destination ip
* that matches the local ip on ifnet, tunnel not found,
* packet offered to GRE hook, returns continue.
* Full packet sent to slowpath */
ifnet->ip_addr = local_ip;
if (create_odp_packet_ip4(&pkt, gre_frame, sizeof(gre_frame),
local_ip, tun_rem_ip + 1)) {
CU_FAIL("Fail to create packet");
return;
}
res = ofp_packet_input(pkt, interface_queue[port],
ofp_eth_vlan_processing);
#ifdef SP
CU_ASSERT_EQUAL(res, OFP_PKT_PROCESSED);
CU_ASSERT_NOT_EQUAL_FATAL(ev = odp_queue_deq(ifnet->spq_def),
ODP_EVENT_INVALID);
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->spq_def), ODP_EVENT_INVALID);
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->outq_def), ODP_EVENT_INVALID);
if (memcmp(odp_packet_data(odp_packet_from_event(ev)),
in_pkt_data, sizeof(gre_frame)))
CU_FAIL("corrupt data sent to slow path");
odp_packet_free(odp_packet_from_event(ev));
ifnet->ip_addr = 0;
CU_PASS("ofp_packet_input_gre_orig_pkt_to_sp");
#else
CU_ASSERT_EQUAL(res, OFP_PKT_DROP);
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->outq_def), ODP_EVENT_INVALID);
#endif
}
static void
test_ofp_packet_input_send_arp(void)
{
odp_packet_t pkt;
odp_event_t ev;
int res;
/* Call ofp_packet_input using a pkt with destination ip
* that does NOT match the local ip on ifnet and a route is found.
* No ARP is found for gateway IP so an ARP req is sent.
* Function returns OFP_PKT_DROP and packet can be reused.*/
my_test_val = TEST_FORWARD_HOOK;
test_ofp_add_route(port, vrf, vlan, dst_ipaddr, 24, 4,
dst_ipaddr + 1);
if (create_odp_packet_ip4(&pkt, test_frame, sizeof(test_frame),
dst_ipaddr, 0)) {
CU_FAIL("Fail to create packet");
return;
}
res = ofp_packet_input(pkt, interface_queue[port],
ofp_eth_vlan_processing);
CU_ASSERT_EQUAL(res, OFP_PKT_PROCESSED);
odp_packet_free(pkt);
CU_ASSERT_NOT_EQUAL(ev = odp_queue_deq(ifnet->outq_def),
ODP_EVENT_INVALID);
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->outq_def), ODP_EVENT_INVALID);
#ifdef SP
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->spq_def), ODP_EVENT_INVALID);
#endif /* SP */
pkt = odp_packet_from_event(ev);
CU_ASSERT_NOT_EQUAL(pkt, ODP_PACKET_INVALID);
CU_ASSERT_EQUAL(odp_packet_has_arp(pkt), 1);
CU_ASSERT_EQUAL(odp_packet_has_vlan(pkt), 0);
CU_ASSERT_EQUAL(odp_packet_len(pkt), sizeof(struct ofp_arphdr) +
sizeof(struct ofp_ether_header));
odp_packet_free(odp_packet_from_event(ev));
ofp_arp_init_tables(); /* to clean saved packet */
CU_PASS("ofp_packet_input_send_arp");
}
static void *pkt_io_recv(void *arg)
{
odp_pktin_queue_t pktin;
odp_packet_t pkt, pkt_tbl[PKT_BURST_SIZE];
int pkt_idx, pkt_cnt;
struct pktio_thr_arg *thr_args;
ofp_pkt_processing_func pkt_func;
thr_args = arg;
pkt_func = thr_args->pkt_func;
pktin = thr_args->pktin;
if (ofp_init_local()) {
OFP_ERR("Error: OFP local init failed.\n");
return NULL;
}
OFP_DBG("PKT-IO receive starting on cpu: %d", odp_cpu_id());
while (1) {
pkt_cnt = odp_pktin_recv(pktin, pkt_tbl, PKT_BURST_SIZE);
for (pkt_idx = 0; pkt_idx < pkt_cnt; pkt_idx++) {
pkt = pkt_tbl[pkt_idx];
if (odp_unlikely(odp_packet_has_error(pkt))) {
OFP_DBG("Packet with error dropped.\n");
odp_packet_free(pkt);
continue;
}
ofp_packet_input(pkt, ODP_QUEUE_INVALID, pkt_func);
}
ofp_send_pending_pkt();
}
/* Never reached */
return NULL;
}
int default_event_dispatcher(void *arg)
{
odp_event_t ev;
odp_packet_t pkt;
odp_queue_t in_queue;
int event_idx = 0;
int event_cnt = 0;
ofp_pkt_processing_func pkt_func = (ofp_pkt_processing_func)arg;
odp_bool_t *is_running = NULL;
if (ofp_init_local()) {
OFP_ERR("ofp_init_local failed");
return -1;
}
int rx_burst = global_param->evt_rx_burst_size;
odp_event_t events[rx_burst];
is_running = ofp_get_processing_state();
if (is_running == NULL) {
OFP_ERR("ofp_get_processing_state failed");
ofp_term_local();
return -1;
}
/* PER CORE DISPATCHER */
while (*is_running) {
event_cnt = odp_schedule_multi(&in_queue, ODP_SCHED_WAIT,
events, rx_burst);
for (event_idx = 0; event_idx < event_cnt; event_idx++) {
odp_event_type_t ev_type;
ev = events[event_idx];
if (ev == ODP_EVENT_INVALID)
continue;
ev_type = odp_event_type(ev);
if (odp_likely(ev_type == ODP_EVENT_PACKET)) {
pkt = odp_packet_from_event(ev);
#if 0
if (odp_unlikely(odp_packet_has_error(pkt))) {
OFP_DBG("Dropping packet with error");
odp_packet_free(pkt);
continue;
}
#endif
ofp_packet_input(pkt, in_queue, pkt_func);
continue;
}
if (ev_type == ODP_EVENT_TIMEOUT) {
ofp_timer_handle(ev);
continue;
}
OFP_ERR("Unexpected event type: %u", ev_type);
odp_event_free(ev);
}
ofp_send_pending_pkt();
}
if (ofp_term_local())
OFP_ERR("ofp_term_local failed");
return 0;
}
static ngx_int_t
ngx_select_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
ngx_uint_t flags)
{
odp_packet_t pkts[OFP_PKT_RX_BURST_SIZE];
odp_packet_t odp_pkt;
int pkt_cnt = 0;
int pkt_idx = 0;
pkt_cnt = odp_pktin_recv(in_queue, pkts, OFP_PKT_RX_BURST_SIZE);
for (pkt_idx = 0; pkt_idx < pkt_cnt; pkt_idx++) {
odp_pkt = pkts[pkt_idx];
ofp_packet_input(odp_pkt, ODP_QUEUE_INVALID, ofp_eth_vlan_processing);
}
if (pkt_cnt < 1) ofp_send_pending_pkt();
static uint32_t count = 0;
/*odp_queue_deq has a lock that impacts performance*/
if (count ++ > 50) {
count = 0;
odp_queue_t timer_queue = ofp_timer_queue_cpu(-1);
odp_event_t event = odp_queue_deq(timer_queue);
if (event != ODP_EVENT_INVALID) {
if (odp_event_type(event) == ODP_EVENT_TIMEOUT) {
ofp_timer_handle(event);
} else {
odp_buffer_free(odp_buffer_from_event(event));
}
}
}
#if OFP_NOTIFY
return NGX_OK;
#endif
int ready, nready;
ngx_err_t err;
ngx_uint_t i, found;
ngx_event_t *ev;
ngx_queue_t *queue;
struct timeval tv, *tp;
ngx_connection_t *c;
if (max_fd == -1) {
for (i = 0; i < nevents; i++) {
c = event_index[i]->data;
ngx_log_debug(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"change max_fd: %i, c->fd : %d", max_fd, c->fd);
if (max_fd < c->fd) {
max_fd = c->fd;
}
}
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"change max_fd: %i", max_fd);
}
#if (NGX_DEBUG)
if (cycle->log->log_level & NGX_LOG_DEBUG_ALL) {
for (i = 0; i < nevents; i++) {
ev = event_index[i];
c = ev->data;
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"select event: fd:%d wr:%d", c->fd, ev->write);
}
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"max_fd: %i", max_fd);
}
#endif
if (timer == NGX_TIMER_INFINITE) {
tp = NULL;
} else {
tv.tv_sec = (long) (timer / 1000);
tv.tv_usec = (long) ((timer % 1000) * 1000);
tp = &tv;
}
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"select timer: %M", timer);
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"select max_fd %d, tp %p ", max_fd, tp);
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
" master %p : work %p", (ofp_fd_set *)&master_read_fd_set,
(ofp_fd_set *)&work_read_fd_set);
ready = select(max_fd + 1, &master_read_fd_set, &master_write_fd_set, NULL, tp);
err = (ready == -1) ? ngx_errno : 0;
if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) {
ngx_time_update();
}
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"select ready %d", ready);
if (err) {
ngx_uint_t level;
if (err == NGX_EINTR) {
if (ngx_event_timer_alarm) {
ngx_event_timer_alarm = 0;
return NGX_OK;
}
level = NGX_LOG_INFO;
} else {
level = NGX_LOG_ALERT;
}
ngx_log_error(level, cycle->log, err, "select() failed");
if (err == NGX_EBADF) {
/*ngx_select_repair_fd_sets(cycle);*/
}
return NGX_ERROR;
}
if (ready == 0) {
return NGX_OK;
if (timer != NGX_TIMER_INFINITE) {
return NGX_OK;
}
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"select() returned no events without timeout");
return NGX_ERROR;
}
nready = 0;
for (i = 0; i < nevents; i++) {
ev = event_index[i];
c = ev->data;
found = 0;
if (ev->write) {
if (FD_ISSET(c->fd, &master_write_fd_set)) {
found = 1;
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"select write %d", c->fd);
}
} else {
if (FD_ISSET(c->fd, &master_read_fd_set)) {
found = 1;
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"select read %d", c->fd);
}
}
if (found) {
ev->ready = 1;
queue = ev->accept ? &ngx_posted_accept_events
: &ngx_posted_events;
ngx_post_event(ev, queue);
nready++;
}
}
if (ready != nready) {
/*ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"select ready != events: %d:%d", ready, nready);
ngx_select_repair_fd_sets(cycle);*/
}
return NGX_OK;
}
void *default_event_dispatcher(void *arg)
{
odp_event_t ev;
odp_packet_t pkt;
odp_queue_t in_queue;
odp_event_t events[OFP_EVT_RX_BURST_SIZE];
int event_idx = 0;
int event_cnt = 0;
ofp_pkt_processing_func pkt_func = (ofp_pkt_processing_func)arg;
odp_bool_t *is_running = NULL;
#if ODP_VERSION < 106
if (odp_init_local(ODP_THREAD_WORKER)) {
OFP_ERR("odp_init_local failed");
return NULL;
}
#endif
if (ofp_init_local()) {
OFP_ERR("ofp_init_local failed");
return NULL;
}
is_running = ofp_get_processing_state();
if (is_running == NULL) {
OFP_ERR("ofp_get_processing_state failed");
ofp_term_local();
return NULL;
}
/* PER CORE DISPATCHER */
while (*is_running) {
event_cnt = odp_schedule_multi(&in_queue, ODP_SCHED_WAIT,
events, OFP_EVT_RX_BURST_SIZE);
for (event_idx = 0; event_idx < event_cnt; event_idx++) {
ev = events[event_idx];
if (ev == ODP_EVENT_INVALID)
continue;
if (odp_event_type(ev) == ODP_EVENT_TIMEOUT) {
ofp_timer_handle(ev);
continue;
}
if (odp_event_type(ev) == ODP_EVENT_PACKET) {
pkt = odp_packet_from_event(ev);
#if 0
if (odp_unlikely(odp_packet_has_error(pkt))) {
OFP_DBG("Dropping packet with error");
odp_packet_free(pkt);
continue;
}
#endif
ofp_packet_input(pkt, in_queue, pkt_func);
continue;
}
OFP_ERR("Unexpected event type: %u", odp_event_type(ev));
/* Free events by type */
if (odp_event_type(ev) == ODP_EVENT_BUFFER) {
odp_buffer_free(odp_buffer_from_event(ev));
continue;
}
if (odp_event_type(ev) == ODP_EVENT_CRYPTO_COMPL) {
odp_crypto_compl_free(
odp_crypto_compl_from_event(ev));
continue;
}
}
ofp_send_pending_pkt();
}
if (ofp_term_local())
OFP_ERR("ofp_term_local failed");
return NULL;
}
static void
test_ofp_packet_input_gre_processed_inner_pkt_forwarded(void)
{
odp_packet_t pkt;
odp_event_t ev;
int res;
struct ofp_ether_header *eth;
struct ofp_ip *ip;
struct ofp_ip *ip_encap;
uint32_t dst_ip;
uint8_t dst_mac_addr[6] = {0x11, 0x22, 0x33, 0x44, 0x55, 0x66};
my_test_val = TEST_LOCAL_HOOK_GRE;
/* Call ofp_packet_input using a GRE pkt with destination ip
* that matches the local ip on ifnet, tunnel found, GRE processed.
* Inner packet does not match local ip, route found,
* packet forwarded */
ifnet->ip_addr = local_ip;
if (create_odp_packet_ip4(&pkt, gre_frame, sizeof(gre_frame),
local_ip, tun_rem_ip)) {
CU_FAIL("Fail to create packet");
return;
}
ip_encap = (struct ofp_ip *)&in_pkt_data[38];
dst_ip = local_ip + 10;
test_ofp_add_route(port, vrf, vlan, ip_encap->ip_dst.s_addr, 24, 4,
dst_ip);
ofp_arp_ipv4_insert(dst_ip, dst_mac_addr, ifnet);
res = ofp_packet_input(pkt, interface_queue[port],
ofp_eth_vlan_processing);
CU_ASSERT_EQUAL(res, OFP_PKT_PROCESSED);
CU_ASSERT_NOT_EQUAL_FATAL(ev = odp_queue_deq(ifnet->outq_def),
ODP_EVENT_INVALID);
#ifdef SP
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->spq_def), ODP_EVENT_INVALID);
#endif
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->outq_def), ODP_EVENT_INVALID);
pkt = odp_packet_from_event(ev);
eth = odp_packet_data(pkt);
ip = odp_packet_l3_ptr(pkt, NULL);
if (memcmp(eth->ether_dhost, dst_mac_addr, OFP_ETHER_ADDR_LEN))
CU_FAIL("Bad destination mac address.");
if (memcmp(eth->ether_shost, ifnet->mac, OFP_ETHER_ADDR_LEN))
CU_FAIL("Bad source mac address.");
CU_ASSERT_EQUAL(ip->ip_src.s_addr, ip_encap->ip_src.s_addr);
CU_ASSERT_EQUAL(ip->ip_dst.s_addr, ip_encap->ip_dst.s_addr);
if (memcmp(ip + (ip->ip_hl << 2), ip_encap + (ip->ip_hl << 2),
odp_be_to_cpu_16(ip_encap->ip_len) - (ip->ip_hl << 2)))
CU_FAIL("corrupt l3 + data");
odp_packet_free(odp_packet_from_event(ev));
ifnet->ip_addr = 0;
CU_PASS("ofp_packet_input_gre_processed_inner_pkt_to_sp");
}
static void
test_ofp_packet_input_forwarding_to_output(void)
{
odp_packet_t pkt;
odp_event_t ev;
int res;
/* Call ofp_packet_input using a pkt with destination ip
* that does NOT match the local ip on ifnet and a route is found.
* ARP is found for gateway IP.
* Function returns OFP_PKT_PROCESSED and
* packet is forwarded to ofp_ip_output.*/
unsigned char ll_addr[13] = "123456789012";
my_test_val = TEST_FORWARD_HOOK;
CU_ASSERT_EQUAL(
ofp_ipv4_lookup_mac(dst_ipaddr + 1, ll_addr, ifnet), -1);
CU_ASSERT_EQUAL(
ofp_arp_ipv4_insert(dst_ipaddr + 1, ll_addr, ifnet), 0);
if (create_odp_packet_ip4(&pkt, test_frame, sizeof(test_frame),
dst_ipaddr, 0)) {
CU_FAIL("Fail to create packet");
return;
}
res = ofp_packet_input(pkt, interface_queue[port],
ofp_eth_vlan_processing);
CU_ASSERT_EQUAL(res, OFP_PKT_PROCESSED);
CU_ASSERT_NOT_EQUAL(ev = odp_queue_deq(ifnet->outq_def),
ODP_EVENT_INVALID);
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->outq_def), ODP_EVENT_INVALID);
#ifdef SP
CU_ASSERT_EQUAL(odp_queue_deq(ifnet->spq_def), ODP_EVENT_INVALID);
#endif /* SP */
CU_ASSERT_EQUAL(odp_packet_len(pkt), sizeof(test_frame));
pkt = odp_packet_from_event(ev);
struct ofp_ip *ip_in_pkt_data =
(struct ofp_ip *)(in_pkt_data + OFP_ETHER_HDR_LEN);
ip_in_pkt_data->ip_ttl--;
#ifdef OFP_PERFORMANCE
/*checksum is not filled on ip_output*/
ip_in_pkt_data->ip_sum =
((struct ofp_ip *)odp_packet_l3_ptr(pkt, NULL))->ip_sum;
#else
ip_in_pkt_data->ip_sum = 0;
ip_in_pkt_data->ip_sum = ofp_cksum_buffer((uint16_t *)ip_in_pkt_data,
ip_in_pkt_data->ip_hl<<2);
#endif
if (memcmp((uint8_t *)odp_packet_data(pkt) + odp_packet_l3_offset(pkt),
in_pkt_data + OFP_ETHER_HDR_LEN,
sizeof(test_frame) - OFP_ETHER_HDR_LEN))
CU_FAIL("corrupt l3 + data forwarded");
struct ofp_ether_header *eth =
(struct ofp_ether_header *)odp_packet_l2_ptr(pkt, NULL);
if (memcmp(eth->ether_dhost, ll_addr, OFP_ETHER_ADDR_LEN))
CU_FAIL("Bad destination mac address on the forwarded packet");
CU_ASSERT_EQUAL(eth->ether_type, odp_cpu_to_be_16(OFP_ETHERTYPE_IP));
CU_PASS("ofp_packet_input_forwarding_to_output");
}
