phys_addr_t odp_mem_virt2phy(const void *virtaddr)
{
int fd;
uint64_t page, physaddr;
unsigned long virt_pfn;
int page_size;
off_t offset;
page_size = getpagesize();
fd = open("/proc/self/pagemap", O_RDONLY);
if (fd < 0) {
ODP_ERR("cannot open /proc/self/pagemap: %s\n",
strerror(errno));
return ODP_BAD_PHYS_ADDR;
}
virt_pfn = (unsigned long)virtaddr / page_size;
offset = sizeof(uint64_t) * virt_pfn;
if (lseek(fd, offset, SEEK_SET) == (off_t)-1) {
ODP_ERR("seek error in /proc/self/pagemap: %s\n",
strerror(errno));
close(fd);
return ODP_BAD_PHYS_ADDR;
}
if (read(fd, &page, sizeof(uint64_t)) < 0) {
ODP_ERR("cannot read /proc/self/pagemap: %s\n",
strerror(errno));
close(fd);
return ODP_BAD_PHYS_ADDR;
}
physaddr = ((page & 0x7fffffffffffffULL) * page_size) +
((unsigned long)virtaddr % page_size);
close(fd);
return physaddr;
}
static int sock_mmap_close(pktio_entry_t *entry)
{
pkt_sock_mmap_t *const pkt_sock = &entry->s.pkt_sock_mmap;
mmap_unmap_sock(pkt_sock);
if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) {
__odp_errno = errno;
ODP_ERR("close(sockfd): %s\n", strerror(errno));
return -1;
}
return 0;
}
static int netmap_do_ioctl(pktio_entry_t *pktio_entry, unsigned long cmd,
int subcmd)
{
pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm;
struct ethtool_value eval;
struct ifreq ifr;
int err;
int fd = pkt_nm->sockfd;
memset(&ifr, 0, sizeof(ifr));
snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s",
pktio_entry->s.name);
switch (cmd) {
case SIOCSIFFLAGS:
ifr.ifr_flags = pkt_nm->if_flags & 0xffff;
break;
case SIOCETHTOOL:
eval.cmd = subcmd;
eval.data = 0;
ifr.ifr_data = (caddr_t)&eval;
break;
default:
break;
}
err = ioctl(fd, cmd, &ifr);
if (err)
goto done;
switch (cmd) {
case SIOCGIFFLAGS:
pkt_nm->if_flags = (ifr.ifr_flags << 16) |
(0xffff & ifr.ifr_flags);
break;
case SIOCETHTOOL:
if (subcmd == ETHTOOL_GLINK)
return eval.data;
break;
default:
break;
}
done:
if (err)
ODP_ERR("ioctl err %d %lu: %s\n", err, cmd, strerror(errno));
return err;
}
static int netmap_close(pktio_entry_t *pktio_entry)
{
pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm;
netmap_close_descriptors(pktio_entry);
if (pkt_nm->sockfd != -1 && close(pkt_nm->sockfd) != 0) {
__odp_errno = errno;
ODP_ERR("close(sockfd): %s\n", strerror(errno));
return -1;
}
return 0;
}
int odp_pktio_term_global(void)
{
pktio_entry_t *pktio_entry;
int ret = 0;
int id;
int pktio_if;
for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if)
if (pktio_if_ops[pktio_if]->term)
if (pktio_if_ops[pktio_if]->term())
ODP_ERR("failed to terminate pktio type %d",
pktio_if);
for (id = 1; id <= ODP_CONFIG_PKTIO_ENTRIES; ++id) {
pktio_entry = &pktio_tbl->entries[id - 1];
odp_queue_destroy(pktio_entry->s.outq_default);
}
ret = odp_shm_free(odp_shm_lookup("odp_pktio_entries"));
if (ret < 0)
ODP_ERR("shm free failed for odp_pktio_entries");
return ret;
}
int odp_thread_term_local(void)
{
int num;
int id = this_thread->thr;
odp_spinlock_lock(&thread_globals->lock);
num = free_id(id);
odp_spinlock_unlock(&thread_globals->lock);
if (num < 0) {
ODP_ERR("failed to free thread id %i", id);
return -1;
}
return num; /* return a number of threads left */
}
int odp_pktio_restart(odp_pktio_t id)
{
pktio_entry_t *entry;
uint8_t port_id;
int ret;
entry = get_pktio_entry(id);
if (entry == NULL) {
ODP_DBG("pktio entry %d does not exist\n",
id->unused_dummy_var);
return -1;
}
if (odp_unlikely(is_free(entry))) {
ODP_DBG("already freed pktio\n");
return -1;
}
if (odp_pktio_is_not_hns_eth(entry)) {
ODP_DBG("pktio entry %d is not ODP UMD pktio\n",
id->unused_dummy_var);
return -1;
}
port_id = entry->s.pkt_odp.portid;
if (!odp_eth_dev_is_valid_port(port_id)) {
ODP_DBG("pktio entry %d ODP UMD Invalid port_id=%d\n",
id->unused_dummy_var, port_id);
return -1;
}
/* Stop device */
odp_eth_dev_stop(port_id);
/* Start device */
ret = odp_eth_dev_start(port_id);
if (ret < 0) {
ODP_ERR("odp_eth_dev_start:err=%d, port=%u\n",
ret, (unsigned)port_id);
return -1;
}
ODP_DBG("odp pmd restart done\n\n");
return 0;
}
odp_queue_t odp_queue_create(const char *name, odp_queue_type_t type,
odp_queue_param_t *param)
{
uint32_t i;
queue_entry_t *queue;
odp_queue_t handle = ODP_QUEUE_INVALID;
for (i = 0; i < ODP_CONFIG_QUEUES; i++) {
queue = &queue_tbl->queue[i];
if (queue->s.status != QUEUE_STATUS_FREE)
continue;
LOCK(&queue->s.lock);
if (queue->s.status == QUEUE_STATUS_FREE) {
queue_init(queue, name, type, param);
if (type == ODP_QUEUE_TYPE_SCHED ||
type == ODP_QUEUE_TYPE_PKTIN)
queue->s.status = QUEUE_STATUS_NOTSCHED;
else
queue->s.status = QUEUE_STATUS_READY;
handle = queue->s.handle;
UNLOCK(&queue->s.lock);
break;
}
UNLOCK(&queue->s.lock);
}
if (handle != ODP_QUEUE_INVALID &&
(type == ODP_QUEUE_TYPE_SCHED || type == ODP_QUEUE_TYPE_PKTIN)) {
odp_buffer_t buf;
buf = odp_schedule_buffer_alloc(handle);
if (buf == ODP_BUFFER_INVALID) {
ODP_ERR("queue_init: sched buf alloc failed\n");
return ODP_QUEUE_INVALID;
}
queue->s.sched_buf = buf;
odp_schedule_mask_set(handle, queue->s.param.sched.prio);
}
return handle;
}
int odp_pktio_inq_setdef(odp_pktio_t id, odp_queue_t queue)
{
pktio_entry_t *pktio_entry = get_pktio_entry(id);
queue_entry_t *qentry;
if (pktio_entry == NULL || queue == ODP_QUEUE_INVALID)
return -1;
qentry = queue_to_qentry(queue);
if (qentry->s.type != ODP_QUEUE_TYPE_PKTIN)
return -1;
lock_entry(pktio_entry);
pktio_entry->s.inq_default = queue;
unlock_entry(pktio_entry);
switch (qentry->s.type) {
/* Change to ODP_QUEUE_TYPE_POLL when ODP_QUEUE_TYPE_PKTIN is removed */
case ODP_QUEUE_TYPE_PKTIN:
/* User polls the input queue */
queue_lock(qentry);
qentry->s.pktin = id;
queue_unlock(qentry);
/* Uncomment when ODP_QUEUE_TYPE_PKTIN is removed
break;
case ODP_QUEUE_TYPE_SCHED:
*/
/* Packet input through the scheduler */
if (schedule_pktio_start(id, ODP_SCHED_PRIO_LOWEST)) {
ODP_ERR("Schedule pktio start failed\n");
return -1;
}
break;
default:
ODP_ABORT("Bad queue type\n");
}
return 0;
}
static int set_pkt_sock_fanout_mmap(pkt_sock_mmap_t *const pkt_sock,
int sock_group_idx)
{
int sockfd = pkt_sock->sockfd;
int val;
int err;
uint16_t fanout_group;
fanout_group = (uint16_t)(sock_group_idx & 0xffff);
val = (PACKET_FANOUT_HASH << 16) | fanout_group;
err = setsockopt(sockfd, SOL_PACKET, PACKET_FANOUT, &val, sizeof(val));
if (err != 0) {
__odp_errno = errno;
ODP_ERR("setsockopt(PACKET_FANOUT): %s\n", strerror(errno));
return -1;
}
return 0;
}
static int mmap_sock(pkt_sock_mmap_t *pkt_sock)
{
int i;
int sock = pkt_sock->sockfd;
/* map rx + tx buffer to userspace : they are in this order */
pkt_sock->mmap_len =
pkt_sock->rx_ring.req.tp_block_size *
pkt_sock->rx_ring.req.tp_block_nr +
pkt_sock->tx_ring.req.tp_block_size *
pkt_sock->tx_ring.req.tp_block_nr;
pkt_sock->mmap_base =
mmap(NULL, pkt_sock->mmap_len, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_LOCKED | MAP_POPULATE, sock, 0);
if (pkt_sock->mmap_base == MAP_FAILED) {
__odp_errno = errno;
ODP_ERR("mmap rx&tx buffer failed: %s\n", strerror(errno));
return -1;
}
pkt_sock->rx_ring.mm_space = pkt_sock->mmap_base;
memset(pkt_sock->rx_ring.rd, 0, pkt_sock->rx_ring.rd_len);
for (i = 0; i < pkt_sock->rx_ring.rd_num; ++i) {
pkt_sock->rx_ring.rd[i].iov_base =
pkt_sock->rx_ring.mm_space
+ (i * pkt_sock->rx_ring.flen);
pkt_sock->rx_ring.rd[i].iov_len = pkt_sock->rx_ring.flen;
}
pkt_sock->tx_ring.mm_space =
pkt_sock->mmap_base + pkt_sock->rx_ring.mm_len;
memset(pkt_sock->tx_ring.rd, 0, pkt_sock->tx_ring.rd_len);
for (i = 0; i < pkt_sock->tx_ring.rd_num; ++i) {
pkt_sock->tx_ring.rd[i].iov_base =
pkt_sock->tx_ring.mm_space
+ (i * pkt_sock->tx_ring.flen);
pkt_sock->tx_ring.rd[i].iov_len = pkt_sock->tx_ring.flen;
}
return 0;
}
odp_queue_t odp_queue_create(const char *name, odp_queue_type_t type,
odp_queue_param_t *param)
{
uint32_t i;
queue_entry_t *queue;
odp_queue_t handle = ODP_QUEUE_INVALID;
for (i = 0; i < ODP_CONFIG_QUEUES; i++) {
queue = &queue_tbl->queue[i];
if (LOAD_S32(queue->s.status) != QUEUE_STATUS_FREE)
continue;
LOCK(queue);
INVALIDATE(queue);
if (queue->s.status == QUEUE_STATUS_FREE) {
queue_init(queue, name, type, param);
if (type == ODP_QUEUE_TYPE_SCHED ||
type == ODP_QUEUE_TYPE_PKTIN)
queue->s.status = QUEUE_STATUS_NOTSCHED;
else
queue->s.status = QUEUE_STATUS_READY;
handle = queue->s.handle;
UNLOCK(queue);
break;
}
UNLOCK(queue);
}
if (handle != ODP_QUEUE_INVALID &&
(type == ODP_QUEUE_TYPE_SCHED || type == ODP_QUEUE_TYPE_PKTIN)) {
if (schedule_queue_init(queue)) {
ODP_ERR("schedule queue init failed\n");
return ODP_QUEUE_INVALID;
}
}
return handle;
}
/* Stress func for Multi producer only */
static int producer_fn(void)
{
unsigned i;
void **src = NULL;
/* alloc dummy object pointers */
src = malloc(MAX_BULK*2*sizeof(void *));
if (src == NULL) {
ODP_ERR("failed to allocate producer memory.\n");
return -1;
}
for (i = 0; i < MAX_BULK; i++)
src[i] = (void *)(unsigned long)i;
do {
i = odp_ring_mp_enqueue_bulk(r_stress, src, MAX_BULK);
if (i == 0)
return 0;
} while (1);
}
static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev)
{
int ret;
pkt_sock->ll.sll_family = PF_PACKET;
pkt_sock->ll.sll_protocol = htons(ETH_P_ALL);
pkt_sock->ll.sll_ifindex = if_nametoindex(netdev);
pkt_sock->ll.sll_hatype = 0;
pkt_sock->ll.sll_pkttype = 0;
pkt_sock->ll.sll_halen = 0;
ret = bind(pkt_sock->sockfd, (struct sockaddr *)&pkt_sock->ll,
sizeof(pkt_sock->ll));
if (ret == -1) {
__odp_errno = errno;
ODP_ERR("bind(to IF): %s\n", strerror(errno));
return -1;
}
return 0;
}
int queue_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num)
{
int sched = 0;
int i;
odp_buffer_hdr_t *tail;
for (i = 0; i < num - 1; i++)
buf_hdr[i]->next = buf_hdr[i+1];
tail = buf_hdr[num-1];
buf_hdr[num-1]->next = NULL;
LOCK(queue);
int status = LOAD_S32(queue->s.status);
if (odp_unlikely(status < QUEUE_STATUS_READY)) {
UNLOCK(queue);
ODP_ERR("Bad queue status\n");
return -1;
}
/* Empty queue */
if (LOAD_PTR(queue->s.head) == NULL)
STORE_PTR(queue->s.head, buf_hdr[0]);
else
STORE_PTR(((typeof(queue->s.tail))LOAD_PTR(queue->s.tail))->next, buf_hdr[0]);
STORE_PTR(queue->s.tail, tail);
if (status == QUEUE_STATUS_NOTSCHED) {
STORE_PTR(queue->s.status, QUEUE_STATUS_SCHED);
sched = 1; /* retval: schedule queue */
}
UNLOCK(queue);
/* Add queue to scheduling */
if (sched)
schedule_queue(queue);
return num; /* All events enqueued */
}
int queue_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num)
{
int sched = 0;
int i;
odp_buffer_hdr_t *tail;
for (i = 0; i < num - 1; i++)
buf_hdr[i]->next = buf_hdr[i+1];
tail = buf_hdr[num-1];
buf_hdr[num-1]->next = NULL;
LOCK(&queue->s.lock);
if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
UNLOCK(&queue->s.lock);
ODP_ERR("Bad queue status\n");
return -1;
}
/* Empty queue */
if (queue->s.head == NULL)
queue->s.head = buf_hdr[0];
else
queue->s.tail->next = buf_hdr[0];
queue->s.tail = tail;
if (queue->s.status == QUEUE_STATUS_NOTSCHED) {
queue->s.status = QUEUE_STATUS_SCHED;
sched = 1; /* retval: schedule queue */
}
UNLOCK(&queue->s.lock);
/* Add queue to scheduling */
if (sched && schedule_queue(queue))
ODP_ABORT("schedule_queue failed\n");
return num; /* All events enqueued */
}
static void init_pktio_entry(pktio_entry_t *entry, odp_pktio_params_t *params)
{
set_taken(entry);
entry->s.inq_default = ODP_QUEUE_INVALID;
switch (params->type) {
case ODP_PKTIO_TYPE_SOCKET_BASIC:
case ODP_PKTIO_TYPE_SOCKET_MMSG:
case ODP_PKTIO_TYPE_SOCKET_MMAP:
memset(&entry->s.pkt_sock, 0, sizeof(entry->s.pkt_sock));
memset(&entry->s.pkt_sock_mmap, 0,
sizeof(entry->s.pkt_sock_mmap));
break;
#ifdef ODP_HAVE_NETMAP
case ODP_PKTIO_TYPE_NETMAP:
memset(&entry->s.pkt_nm, 0, sizeof(entry->s.pkt_nm));
break;
#endif
default:
ODP_ERR("Packet I/O type not supported. Please recompile\n");
break;
}
/* Save pktio parameters, type is the most useful */
memcpy(&entry->s.params, params, sizeof(*params));
}
/**
* Packet IO loopback worker thread using bursts from/to IO resources
*
* @param arg thread arguments of type 'thread_args_t *'
*/
static void *pktio_ifburst_thread(void *arg)
{
int thr;
odp_buffer_pool_t pkt_pool;
odp_pktio_t pktio;
thread_args_t *thr_args;
int pkts, pkts_ok;
odp_packet_t pkt_tbl[MAX_PKT_BURST];
unsigned long pkt_cnt = 0;
unsigned long err_cnt = 0;
unsigned long tmp = 0;
odp_pktio_params_t params;
socket_params_t *sock_params = ¶ms.sock_params;
thr = odp_thread_id();
thr_args = arg;
printf("Pktio thread [%02i] starts, pktio_dev:%s\n", thr,
thr_args->pktio_dev);
/* Lookup the packet pool */
pkt_pool = odp_buffer_pool_lookup("packet_pool");
if (pkt_pool == ODP_BUFFER_POOL_INVALID || pkt_pool != thr_args->pool) {
ODP_ERR(" [%02i] Error: pkt_pool not found\n", thr);
return NULL;
}
/* Open a packet IO instance for this thread */
sock_params->type = thr_args->type;
sock_params->fanout = thr_args->fanout;
pktio = odp_pktio_open(thr_args->pktio_dev, pkt_pool, ¶ms);
if (pktio == ODP_PKTIO_INVALID) {
ODP_ERR(" [%02i] Error: pktio create failed.\n", thr);
return NULL;
}
printf(" [%02i] created pktio:%02i, burst mode\n",
thr, pktio);
/* Loop packets */
for (;;) {
pkts = odp_pktio_recv(pktio, pkt_tbl, MAX_PKT_BURST);
if (pkts > 0) {
/* Drop packets with errors */
pkts_ok = drop_err_pkts(pkt_tbl, pkts);
if (pkts_ok > 0) {
/* Swap Eth MACs and IP-addrs */
swap_pkt_addrs(pkt_tbl, pkts_ok);
odp_pktio_send(pktio, pkt_tbl, pkts_ok);
}
if (odp_unlikely(pkts_ok != pkts))
ODP_ERR("Dropped frames:%u - err_cnt:%lu\n",
pkts-pkts_ok, ++err_cnt);
/* Print packet counts every once in a while */
tmp += pkts_ok;
if (odp_unlikely((tmp >= 100000) || /* OR first print:*/
((pkt_cnt == 0) && ((tmp-1) < MAX_PKT_BURST)))) {
pkt_cnt += tmp;
printf(" [%02i] pkt_cnt:%lu\n", thr, pkt_cnt);
fflush(NULL);
tmp = 0;
}
}
}
/* unreachable */
}
static int sock_mmap_open(odp_pktio_t id ODP_UNUSED,
pktio_entry_t *pktio_entry,
const char *netdev, odp_pool_t pool)
{
int if_idx;
int ret = 0;
odp_pktio_stats_t cur_stats;
if (disable_pktio)
return -1;
pkt_sock_mmap_t *const pkt_sock = &pktio_entry->s.pkt_sock_mmap;
int fanout = 1;
/* Init pktio entry */
memset(pkt_sock, 0, sizeof(*pkt_sock));
/* set sockfd to -1, because a valid socked might be initialized to 0 */
pkt_sock->sockfd = -1;
if (pool == ODP_POOL_INVALID)
return -1;
/* Store eth buffer offset for pkt buffers from this pool */
pkt_sock->frame_offset = 0;
pkt_sock->pool = pool;
pkt_sock->sockfd = mmap_pkt_socket();
if (pkt_sock->sockfd == -1)
goto error;
ret = mmap_bind_sock(pkt_sock, netdev);
if (ret != 0)
goto error;
ret = mmap_setup_ring(pkt_sock->sockfd, &pkt_sock->tx_ring,
PACKET_TX_RING, pool, fanout);
if (ret != 0)
goto error;
ret = mmap_setup_ring(pkt_sock->sockfd, &pkt_sock->rx_ring,
PACKET_RX_RING, pool, fanout);
if (ret != 0)
goto error;
ret = mmap_sock(pkt_sock);
if (ret != 0)
goto error;
ret = mac_addr_get_fd(pkt_sock->sockfd, netdev, pkt_sock->if_mac);
if (ret != 0)
goto error;
if_idx = if_nametoindex(netdev);
if (if_idx == 0) {
__odp_errno = errno;
ODP_ERR("if_nametoindex(): %s\n", strerror(errno));
goto error;
}
pkt_sock->fanout = fanout;
if (fanout) {
ret = set_pkt_sock_fanout_mmap(pkt_sock, if_idx);
if (ret != 0)
goto error;
}
ret = ethtool_stats_get_fd(pktio_entry->s.pkt_sock_mmap.sockfd,
pktio_entry->s.name,
&cur_stats);
if (ret != 0) {
ret = sysfs_stats(pktio_entry, &cur_stats);
if (ret != 0) {
pktio_entry->s.stats_type = STATS_UNSUPPORTED;
ODP_DBG("pktio: %s unsupported stats\n",
pktio_entry->s.name);
} else {
pktio_entry->s.stats_type = STATS_SYSFS;
}
} else {
pktio_entry->s.stats_type = STATS_ETHTOOL;
}
ret = sock_stats_reset_fd(pktio_entry,
pktio_entry->s.pkt_sock_mmap.sockfd);
if (ret != 0)
goto error;
return 0;
error:
sock_mmap_close(pktio_entry);
return -1;
}
static int ethtool_stats(int fd, struct ifreq *ifr, odp_pktio_stats_t *stats)
{
struct ethtool_gstrings *strings;
struct ethtool_stats *estats;
unsigned int n_stats, i;
int err;
int cnts;
strings = get_stringset(fd, ifr);
if (!strings)
return -1;
n_stats = strings->len;
if (n_stats < 1) {
ODP_ERR("no stats available\n");
free(strings);
return -1;
}
estats = calloc(1, n_stats * sizeof(uint64_t) +
sizeof(struct ethtool_stats));
if (!estats) {
free(strings);
return -1;
}
estats->cmd = ETHTOOL_GSTATS;
estats->n_stats = n_stats;
ifr->ifr_data = estats;
err = ioctl(fd, SIOCETHTOOL, ifr);
if (err < 0) {
__odp_errno = errno;
free(strings);
free(estats);
return -1;
}
cnts = 0;
for (i = 0; i < n_stats; i++) {
char *cnt = (char *)&strings->data[i * ETH_GSTRING_LEN];
uint64_t val = estats->data[i];
if (!strcmp(cnt, "rx_octets")) {
stats->in_octets = val;
cnts++;
} else if (!strcmp(cnt, "rx_ucast_packets")) {
stats->in_ucast_pkts = val;
cnts++;
} else if (!strcmp(cnt, "rx_discards")) {
stats->in_discards = val;
cnts++;
} else if (!strcmp(cnt, "rx_errors")) {
stats->in_errors = val;
cnts++;
} else if (!strcmp(cnt, "tx_octets")) {
stats->out_octets = val;
cnts++;
} else if (!strcmp(cnt, "tx_ucast_packets")) {
stats->out_ucast_pkts = val;
cnts++;
} else if (!strcmp(cnt, "tx_discards")) {
stats->out_discards = val;
cnts++;
} else if (!strcmp(cnt, "tx_errors")) {
stats->out_errors = val;
cnts++;
}
}
free(strings);
free(estats);
/* Ethtool strings came from kernel driver. Name of that
* strings is not universal. Current function needs to be updated
* if your driver has different names for counters */
if (cnts < 8)
return -1;
return 0;
}
static inline unsigned pkt_mmap_v2_tx(int sock, struct ring *ring,
const odp_packet_t pkt_table[],
unsigned len)
{
union frame_map ppd;
uint32_t pkt_len;
unsigned first_frame_num, frame_num, frame_count;
int ret;
uint8_t *buf;
unsigned n, i = 0;
unsigned nb_tx = 0;
int send_errno;
int total_len = 0;
first_frame_num = ring->frame_num;
frame_num = first_frame_num;
frame_count = ring->rd_num;
while (i < len) {
ppd.raw = ring->rd[frame_num].iov_base;
if (!odp_unlikely(mmap_tx_kernel_ready(ppd.raw)))
break;
pkt_len = odp_packet_len(pkt_table[i]);
ppd.v2->tp_h.tp_snaplen = pkt_len;
ppd.v2->tp_h.tp_len = pkt_len;
total_len += pkt_len;
buf = (uint8_t *)ppd.raw + TPACKET2_HDRLEN -
sizeof(struct sockaddr_ll);
odp_packet_copy_to_mem(pkt_table[i], 0, pkt_len, buf);
mmap_tx_user_ready(ppd.raw);
if (++frame_num >= frame_count)
frame_num = 0;
i++;
}
ret = sendto(sock, NULL, 0, MSG_DONTWAIT, NULL, 0);
send_errno = errno;
/* On success, the return value indicates the number of bytes sent. On
* failure a value of -1 is returned, even if the failure occurred
* after some of the packets in the ring have already been sent, so we
* need to inspect the packet status to determine which were sent. */
if (odp_likely(ret == total_len)) {
nb_tx = i;
ring->frame_num = frame_num;
} else if (ret == -1) {
for (frame_num = first_frame_num, n = 0; n < i; ++n) {
struct tpacket2_hdr *hdr = ring->rd[frame_num].iov_base;
if (odp_likely(hdr->tp_status == TP_STATUS_AVAILABLE ||
hdr->tp_status == TP_STATUS_SENDING)) {
nb_tx++;
} else {
/* The remaining frames weren't sent, clear
* their status to indicate we're not waiting
* for the kernel to process them. */
hdr->tp_status = TP_STATUS_AVAILABLE;
}
if (++frame_num >= frame_count)
frame_num = 0;
}
ring->frame_num = (first_frame_num + nb_tx) % frame_count;
if (nb_tx == 0 && SOCK_ERR_REPORT(send_errno)) {
__odp_errno = send_errno;
/* ENOBUFS indicates that the transmit queue is full,
* which will happen regularly when overloaded so don't
* print it */
if (errno != ENOBUFS)
ODP_ERR("sendto(pkt mmap): %s\n",
strerror(send_errno));
return -1;
}
} else {
/* Short send, return value is number of bytes sent so use this
* to determine number of complete frames sent. */
for (n = 0; n < i && ret > 0; ++n) {
ret -= odp_packet_len(pkt_table[n]);
if (ret >= 0)
nb_tx++;
}
ring->frame_num = (first_frame_num + nb_tx) % frame_count;
}
for (i = 0; i < nb_tx; ++i)
odp_packet_free(pkt_table[i]);
return nb_tx;
}
static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry,
const char *netdev, odp_pool_t pool)
{
int i;
int err;
int sockfd;
int mtu;
uint32_t buf_size;
pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm;
struct nm_desc *desc;
struct netmap_ring *ring;
odp_pktin_hash_proto_t hash_proto;
odp_pktio_stats_t cur_stats;
if (getenv("ODP_PKTIO_DISABLE_NETMAP"))
return -1;
if (pool == ODP_POOL_INVALID)
return -1;
/* Init pktio entry */
memset(pkt_nm, 0, sizeof(*pkt_nm));
pkt_nm->sockfd = -1;
pkt_nm->pool = pool;
/* max frame len taking into account the l2-offset */
pkt_nm->max_frame_len = ODP_CONFIG_PACKET_BUF_LEN_MAX -
odp_buffer_pool_headroom(pool) -
odp_buffer_pool_tailroom(pool);
snprintf(pktio_entry->s.name, sizeof(pktio_entry->s.name), "%s",
netdev);
snprintf(pkt_nm->nm_name, sizeof(pkt_nm->nm_name), "netmap:%s",
netdev);
/* Dummy open here to check if netmap module is available and to read
* capability info. */
desc = nm_open(pkt_nm->nm_name, NULL, 0, NULL);
if (desc == NULL) {
ODP_ERR("nm_open(%s) failed\n", pkt_nm->nm_name);
goto error;
}
if (desc->nifp->ni_rx_rings > NM_MAX_DESC) {
ODP_ERR("Unable to store all rx rings\n");
nm_close(desc);
goto error;
}
pkt_nm->num_rx_rings = desc->nifp->ni_rx_rings;
pkt_nm->capa.max_input_queues = PKTIO_MAX_QUEUES;
if (desc->nifp->ni_rx_rings < PKTIO_MAX_QUEUES)
pkt_nm->capa.max_input_queues = desc->nifp->ni_rx_rings;
if (desc->nifp->ni_tx_rings > NM_MAX_DESC) {
ODP_ERR("Unable to store all tx rings\n");
nm_close(desc);
goto error;
}
pkt_nm->num_tx_rings = desc->nifp->ni_tx_rings;
pkt_nm->capa.max_output_queues = PKTIO_MAX_QUEUES;
if (desc->nifp->ni_tx_rings < PKTIO_MAX_QUEUES)
pkt_nm->capa.max_output_queues = desc->nifp->ni_tx_rings;
ring = NETMAP_RXRING(desc->nifp, desc->cur_rx_ring);
buf_size = ring->nr_buf_size;
nm_close(desc);
sockfd = socket(AF_INET, SOCK_DGRAM, 0);
if (sockfd == -1) {
ODP_ERR("Cannot get device control socket\n");
goto error;
}
pkt_nm->sockfd = sockfd;
/* Use either interface MTU (+ ethernet header length) or netmap buffer
* size as MTU, whichever is smaller. */
mtu = mtu_get_fd(pktio_entry->s.pkt_nm.sockfd, pktio_entry->s.name) +
ODPH_ETHHDR_LEN;
if (mtu < 0) {
ODP_ERR("Unable to read interface MTU\n");
goto error;
}
pkt_nm->mtu = ((uint32_t)mtu < buf_size) ? (uint32_t)mtu : buf_size;
/* Check if RSS is supported. If not, set 'max_input_queues' to 1. */
if (rss_conf_get_supported_fd(sockfd, netdev, &hash_proto) == 0) {
ODP_DBG("RSS not supported\n");
pkt_nm->capa.max_input_queues = 1;
}
err = netmap_do_ioctl(pktio_entry, SIOCGIFFLAGS, 0);
if (err)
goto error;
if ((pkt_nm->if_flags & IFF_UP) == 0)
ODP_DBG("%s is down\n", pktio_entry->s.name);
err = mac_addr_get_fd(sockfd, netdev, pkt_nm->if_mac);
if (err)
goto error;
for (i = 0; i < PKTIO_MAX_QUEUES; i++) {
odp_ticketlock_init(&pkt_nm->rx_desc_ring[i].s.lock);
odp_ticketlock_init(&pkt_nm->tx_desc_ring[i].s.lock);
}
/* netmap uses only ethtool to get statistics counters */
err = ethtool_stats_get_fd(pktio_entry->s.pkt_nm.sockfd,
pktio_entry->s.name,
&cur_stats);
if (err) {
ODP_ERR(
"netmap pktio %s does not support statistics counters\n",
pktio_entry->s.name);
pktio_entry->s.stats_type = STATS_UNSUPPORTED;
} else {
pktio_entry->s.stats_type = STATS_ETHTOOL;
}
(void)netmap_stats_reset(pktio_entry);
return 0;
error:
netmap_close(pktio_entry);
return -1;
}
static int netmap_start(pktio_entry_t *pktio_entry)
{
pkt_netmap_t *pkt_nm = &pktio_entry->s.pkt_nm;
netmap_ring_t *desc_ring;
struct nm_desc base_desc;
unsigned i;
unsigned j;
unsigned num_rx_desc = 0;
uint64_t flags;
odp_pktin_mode_t in_mode = pktio_entry->s.param.in_mode;
odp_pktout_mode_t out_mode = pktio_entry->s.param.out_mode;
/* If no pktin/pktout queues have been configured. Configure one
* for each direction. */
if (!pktio_entry->s.num_in_queue &&
in_mode != ODP_PKTIN_MODE_DISABLED) {
odp_pktin_queue_param_t param;
odp_pktin_queue_param_init(¶m);
param.num_queues = 1;
if (odp_pktin_queue_config(pktio_entry->s.handle, ¶m))
return -1;
}
if (!pktio_entry->s.num_out_queue &&
out_mode == ODP_PKTOUT_MODE_DIRECT) {
odp_pktout_queue_param_t param;
odp_pktout_queue_param_init(¶m);
param.num_queues = 1;
if (odp_pktout_queue_config(pktio_entry->s.handle, ¶m))
return -1;
}
if (pkt_nm->num_rx_desc_rings == pktio_entry->s.num_in_queue &&
pkt_nm->num_tx_desc_rings == pktio_entry->s.num_out_queue)
return (netmap_wait_for_link(pktio_entry) == 1) ? 0 : -1;
netmap_close_descriptors(pktio_entry);
/* Map pktin/pktout queues to netmap rings */
if (pktio_entry->s.num_in_queue) {
/* In single queue case only one netmap descriptor is
* required. */
num_rx_desc = (pktio_entry->s.num_in_queue == 1) ? 1 :
pkt_nm->num_rx_rings;
map_netmap_rings(pkt_nm->rx_desc_ring,
pktio_entry->s.num_in_queue, num_rx_desc);
}
if (pktio_entry->s.num_out_queue)
/* Enough to map only one netmap tx ring per pktout queue */
map_netmap_rings(pkt_nm->tx_desc_ring,
pktio_entry->s.num_out_queue,
pktio_entry->s.num_out_queue);
base_desc.self = &base_desc;
base_desc.mem = NULL;
memcpy(base_desc.req.nr_name, pktio_entry->s.name,
sizeof(pktio_entry->s.name));
base_desc.req.nr_flags &= ~NR_REG_MASK;
if (num_rx_desc == 1)
base_desc.req.nr_flags |= NR_REG_ALL_NIC;
else
base_desc.req.nr_flags |= NR_REG_ONE_NIC;
base_desc.req.nr_ringid = 0;
/* Only the first rx descriptor does mmap */
desc_ring = pkt_nm->rx_desc_ring;
flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL;
desc_ring[0].s.desc[0] = nm_open(pkt_nm->nm_name, NULL, flags,
&base_desc);
if (desc_ring[0].s.desc[0] == NULL) {
ODP_ERR("nm_start(%s) failed\n", pkt_nm->nm_name);
goto error;
}
/* Open rest of the rx descriptors (one per netmap ring) */
flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL | NM_OPEN_NO_MMAP;
for (i = 0; i < pktio_entry->s.num_in_queue; i++)
for (j = desc_ring[i].s.first; j <= desc_ring[i].s.last; j++) {
if (i == 0 && j == 0) { /* First already opened */
if (num_rx_desc > 1)
continue;
else
break;
}
base_desc.req.nr_ringid = j;
desc_ring[i].s.desc[j] = nm_open(pkt_nm->nm_name, NULL,
flags, &base_desc);
if (desc_ring[i].s.desc[j] == NULL) {
ODP_ERR("nm_start(%s) failed\n",
pkt_nm->nm_name);
goto error;
}
}
/* Open tx descriptors */
desc_ring = pkt_nm->tx_desc_ring;
flags = NM_OPEN_IFNAME | NM_OPEN_NO_MMAP;
base_desc.req.nr_flags &= !NR_REG_ALL_NIC;
base_desc.req.nr_flags |= NR_REG_ONE_NIC;
for (i = 0; i < pktio_entry->s.num_out_queue; i++)
for (j = desc_ring[i].s.first; j <= desc_ring[i].s.last; j++) {
base_desc.req.nr_ringid = j;
desc_ring[i].s.desc[j] = nm_open(pkt_nm->nm_name, NULL,
flags, &base_desc);
if (desc_ring[i].s.desc[j] == NULL) {
ODP_ERR("nm_start(%s) failed\n",
pkt_nm->nm_name);
goto error;
}
}
pkt_nm->num_rx_desc_rings = pktio_entry->s.num_in_queue;
pkt_nm->num_tx_desc_rings = pktio_entry->s.num_out_queue;
/* Wait for the link to come up */
return (netmap_wait_for_link(pktio_entry) == 1) ? 0 : -1;
error:
netmap_close_descriptors(pktio_entry);
return -1;
}
odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool,
odp_pktio_params_t *params)
{
odp_pktio_t id;
pktio_entry_t *pktio_entry;
int res;
if (params == NULL) {
ODP_ERR("Invalid pktio params\n");
return ODP_PKTIO_INVALID;
}
switch (params->type) {
case ODP_PKTIO_TYPE_SOCKET_BASIC:
case ODP_PKTIO_TYPE_SOCKET_MMSG:
case ODP_PKTIO_TYPE_SOCKET_MMAP:
ODP_DBG("Allocating socket pktio\n");
break;
#ifdef ODP_HAVE_NETMAP
case ODP_PKTIO_TYPE_NETMAP:
ODP_DBG("Allocating netmap pktio\n");
break;
#endif
default:
ODP_ERR("Invalid pktio type: %02x\n", params->type);
return ODP_PKTIO_INVALID;
}
id = alloc_lock_pktio_entry(params);
if (id == ODP_PKTIO_INVALID) {
ODP_ERR("No resources available.\n");
return ODP_PKTIO_INVALID;
}
/* if successful, alloc_pktio_entry() returns with the entry locked */
pktio_entry = get_entry(id);
switch (params->type) {
case ODP_PKTIO_TYPE_SOCKET_BASIC:
case ODP_PKTIO_TYPE_SOCKET_MMSG:
res = setup_pkt_sock(&pktio_entry->s.pkt_sock, dev, pool);
if (res == -1) {
close_pkt_sock(&pktio_entry->s.pkt_sock);
free_pktio_entry(id);
id = ODP_PKTIO_INVALID;
}
break;
case ODP_PKTIO_TYPE_SOCKET_MMAP:
res = setup_pkt_sock_mmap(&pktio_entry->s.pkt_sock_mmap, dev,
pool, params->sock_params.fanout);
if (res == -1) {
close_pkt_sock_mmap(&pktio_entry->s.pkt_sock_mmap);
free_pktio_entry(id);
id = ODP_PKTIO_INVALID;
}
break;
#ifdef ODP_HAVE_NETMAP
case ODP_PKTIO_TYPE_NETMAP:
res = setup_pkt_netmap(&pktio_entry->s.pkt_nm, dev,
pool, ¶ms->nm_params);
if (res == -1) {
close_pkt_netmap(&pktio_entry->s.pkt_nm);
free_pktio_entry(id);
id = ODP_PKTIO_INVALID;
}
break;
#endif
default:
free_pktio_entry(id);
id = ODP_PKTIO_INVALID;
ODP_ERR("This type of I/O is not supported. Please recompile.\n");
break;
}
unlock_entry(pktio_entry);
return id;
}
int odp_hugepage_info_init(void)
{
unsigned i, num_sizes = 0;
struct dirent *dirent = NULL;
DIR *dir = NULL;
struct odp_hugepage_type *hpt = NULL;
dir = opendir(sys_dir_path);
if (!dir)
ODP_PRINT("Cannot open directory %s to "
"read system hugepage info\n", sys_dir_path);
dirent = readdir(dir);
/* loop get different kinds of hugepages in
* the system hugepage directory */
while (dirent) {
if (strncmp(dirent->d_name, "hugepages-",
ODP_SYS_HGPG_STR_LEN) == 0) {
hpt = &local_config.odp_hugepage_type[num_sizes];
hpt->hugepage_sz =
odp_str_to_size(&dirent->d_name[
ODP_SYS_HGPG_STR_LEN]);
hpt->hugedir = get_hugepage_dir(hpt->hugepage_sz);
/* first, check if we have a mountpoint */
if (!hpt->hugedir) {
int32_t num_pages;
num_pages = get_num_hugepages(dirent->d_name);
if (num_pages > 0) {
ODP_PRINT("%d hugepages of"
" size %lu reserved, ",
num_pages,
hpt->hugepage_sz);
ODP_PRINT("but no mounted hugetlbfs"
" found for that size\n");
}
} else {
/* try to obtain a writelock */
hpt->lock_descriptor = open(hpt->hugedir,
O_RDONLY);
/* if blocking lock failed */
if (flock(hpt->lock_descriptor,
LOCK_EX) == -1) {
ODP_ERR("Failed to lock hugepage"
" directory!\n");
closedir(dir);
return -1;
}
/* clear out the hugepages
* dir from unused pages */
if (clear_hugedir(hpt->hugedir) == -1) {
closedir(dir);
return -1;
}
/* for now, put all pages into socket 0,
* later they will be sohodpd */
hpt->num_pages[0] =
get_num_hugepages(dirent->d_name);
hpt->num_pages[0] =
ODP_MIN(hpt->num_pages[0],
ODP_PAGE_MEMORY_MAX /
hpt->hugepage_sz);
num_sizes++;
}
}
dirent = readdir(dir);
}
closedir(dir);
local_config.num_hugepage_types = num_sizes;
/* sort the page directory entries by size, largest to smallest */
for (i = 0; i < num_sizes; i++) {
unsigned int j;
for (j = i + 1; j < num_sizes; j++)
if (local_config.odp_hugepage_type[j - 1].hugepage_sz <
local_config.odp_hugepage_type[j].hugepage_sz)
swap_hpt(
&local_config.odp_hugepage_type[j - 1],
&local_config.odp_hugepage_type[j]);
}
/* now we have all info, check we have at least one valid size */
for (i = 0; i < num_sizes; i++)
if (local_config.odp_hugepage_type[i].hugedir &&
(local_config.odp_hugepage_type[i].num_pages[0] > 0))
return 0;
/* no valid hugepage mounts available, return error */
return -1;
}
static int find_hpt_numasocket(struct hugepage_file *hugepg_tbl,
struct odp_hugepage_type *hpt)
{
int socket_id;
char *end, *nodestr;
unsigned i, hp_count = 0;
uint64_t virt_addr;
char buf[ODP_BUFF_SIZE];
char hugedir_str[ODP_PATH_MAX];
FILE *f;
f = fopen("/proc/self/numa_maps", "r");
if (!f)
return 0;
snprintf(hugedir_str, sizeof(hugedir_str), "%s/%s",
hpt->hugedir, HUGEFILE_PREFIX_DEFAULT);
while (fgets(buf, sizeof(buf), f)) {
if (!strstr(buf, " huge ") && !strstr(buf, hugedir_str))
continue;
virt_addr = strtoull(buf, &end, 16);
if ((virt_addr == 0) || (end == buf)) {
ODP_ERR("error in numa_maps parsing\n");
goto error;
}
nodestr = strstr(buf, " N");
if (!nodestr) {
ODP_ERR("error in numa_maps parsing\n");
goto error;
}
nodestr += 2;
end = strstr(nodestr, "=");
if (!end) {
ODP_ERR("error in numa_maps parsing\n");
goto error;
}
end[0] = '\0';
end = NULL;
socket_id = strtoul(nodestr, &end, 0);
if ((nodestr[0] == '\0') || (end == NULL) || (*end != '\0')) {
ODP_ERR("error in numa_maps parsing\n");
goto error;
}
for (i = 0; i < hpt->num_pages[0]; i++) {
void *va = (void *)(unsigned long)virt_addr;
if (hugepg_tbl[i].orig_va == va) {
hugepg_tbl[i].socket_id = socket_id;
hp_count++;
}
}
}
if (hp_count < hpt->num_pages[0])
goto error;
fclose(f);
return 0;
error:
fclose(f);
return -1;
}
/**
* ODP packet example main function
*/
int main(int argc, char *argv[])
{
odp_linux_pthread_t thread_tbl[MAX_WORKERS];
odp_buffer_pool_t pool;
int thr_id;
int num_workers;
void *pool_base;
int i;
int first_core;
int core_count;
/* Init ODP before calling anything else */
if (odp_init_global()) {
ODP_ERR("Error: ODP global init failed.\n");
exit(EXIT_FAILURE);
}
/* Reserve memory for args from shared mem */
args = odp_shm_reserve("shm_args", sizeof(args_t), ODP_CACHE_LINE_SIZE);
if (args == NULL) {
ODP_ERR("Error: shared mem alloc failed.\n");
exit(EXIT_FAILURE);
}
memset(args, 0, sizeof(*args));
/* Parse and store the application arguments */
parse_args(argc, argv, &args->appl);
/* Print both system and application information */
print_info(NO_PATH(argv[0]), &args->appl);
core_count = odp_sys_core_count();
num_workers = core_count;
if (args->appl.core_count)
num_workers = args->appl.core_count;
if (num_workers > MAX_WORKERS)
num_workers = MAX_WORKERS;
printf("Num worker threads: %i\n", num_workers);
/*
* By default core #0 runs Linux kernel background tasks.
* Start mapping thread from core #1
*/
first_core = 1;
if (core_count == 1)
first_core = 0;
printf("First core: %i\n\n", first_core);
/* Init this thread */
thr_id = odp_thread_create(0);
odp_init_local(thr_id);
/* Create packet pool */
pool_base = odp_shm_reserve("shm_packet_pool",
SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE);
if (pool_base == NULL) {
ODP_ERR("Error: packet pool mem alloc failed.\n");
exit(EXIT_FAILURE);
}
pool = odp_buffer_pool_create("packet_pool", pool_base,
SHM_PKT_POOL_SIZE,
SHM_PKT_POOL_BUF_SIZE,
ODP_CACHE_LINE_SIZE,
ODP_BUFFER_TYPE_PACKET);
if (pool == ODP_BUFFER_POOL_INVALID) {
ODP_ERR("Error: packet pool create failed.\n");
exit(EXIT_FAILURE);
}
odp_buffer_pool_print(pool);
/* Create and init worker threads */
memset(thread_tbl, 0, sizeof(thread_tbl));
for (i = 0; i < num_workers; ++i) {
void *(*thr_run_func) (void *);
int core;
int if_idx;
core = (first_core + i) % core_count;
if_idx = i % args->appl.if_count;
args->thread[i].pktio_dev = args->appl.if_names[if_idx];
args->thread[i].pool = pool;
args->thread[i].mode = args->appl.mode;
args->thread[i].type = args->appl.type;
args->thread[i].fanout = args->appl.fanout;
if (args->appl.mode == APPL_MODE_PKT_BURST)
thr_run_func = pktio_ifburst_thread;
else /* APPL_MODE_PKT_QUEUE */
thr_run_func = pktio_queue_thread;
/*
* Create threads one-by-one instead of all-at-once,
* because each thread might get different arguments.
* Calls odp_thread_create(cpu) for each thread
*/
odp_linux_pthread_create(thread_tbl, 1, core, thr_run_func,
&args->thread[i]);
}
/* Master thread waits for other threads to exit */
odp_linux_pthread_join(thread_tbl, num_workers);
printf("Exit\n\n");
return 0;
}
static const struct odp_mm_district *mm_district_reserve_aligned(
const char *name, const char *orig_name,
size_t len,
int socket_id, unsigned flags,
unsigned align,
unsigned bound)
{
struct odp_sys_layout *mcfg;
unsigned i = 0;
int mmfrag_idx = -1;
uint64_t addr_offset, seg_offset = 0;
size_t requested_len;
size_t mmfrag_len = 0;
phys_addr_t mmfrag_physaddr;
void *mmfrag_addr;
struct odp_mm_district *md = NULL;
/* get pointer to global configuration */
mcfg = odp_get_configuration()->sys_layout;
/* no more room in config */
if (mcfg->mm_district_idx >= ODP_MAX_MM_DISTRICT) {
ODP_ERR("%s: No more room in config\n", name);
odp_err = ENOSPC;
return NULL;
}
/* zone already exist */
if (mm_district_lookup(name)) {
ODP_ERR("mm_district <%s> already exists\n", name);
odp_err = EEXIST;
return NULL;
}
if (!orig_name) {
ODP_ERR("Invalid param: orig_name\n");
odp_err = EINVAL;
return NULL;
}
md = free_mm_district_lookup(orig_name);
if (md)
if (len <= md->len) {
free_mm_district_fetch(md);
return md;
}
/* if alignment is not a power of two */
if (align && !odp_is_power_of_2(align)) {
ODP_ERR("Invalid alignment: %u\n", align);
odp_err = EINVAL;
return NULL;
}
/* alignment less than cache size is not allowed */
if (align < ODP_CACHE_LINE_SIZE)
align = ODP_CACHE_LINE_SIZE;
/* align length on cache boundary. Check for overflow before doing so */
if (len > MEM_SIZE_MAX - ODP_CACHE_LINE_MASK) {
odp_err = EINVAL; /* requested size too big */
return NULL;
}
len += ODP_CACHE_LINE_MASK;
len &= ~((size_t)ODP_CACHE_LINE_MASK);
/* save minimal requested length */
requested_len = ODP_MAX((size_t)ODP_CACHE_LINE_SIZE, len);
/* check that boundary condition is valid */
if ((bound != 0) && ((requested_len > bound) ||
!odp_is_power_of_2(bound))) {
odp_err = EINVAL;
return NULL;
}
/* find the smallest segment matching requirements */
for (i = 0; i < ODP_MAX_MMFRAG; i++) {
/* last segment */
if (!free_mmfrag[i].addr)
break;
/* empty segment, skip it */
if (free_mmfrag[i].len == 0)
continue;
/* bad socket ID */
if ((socket_id != SOCKET_ID_ANY) &&
(free_mmfrag[i].socket_id != SOCKET_ID_ANY) &&
(socket_id != free_mmfrag[i].socket_id))
continue;
/*
* calculate offset to closest alignment that
* meets boundary conditions.
*/
addr_offset = align_phys_boundary(free_mmfrag + i,
requested_len, align, bound);
/* check len */
if ((requested_len + addr_offset) > free_mmfrag[i].len)
continue;
/* check flags for hugepage sizes */
if ((flags & ODP_MEMZONE_2MB) &&
(free_mmfrag[i].hugepage_sz == ODP_PGSIZE_1G))
continue;
if ((flags & ODP_MEMZONE_1GB) &&
(free_mmfrag[i].hugepage_sz == ODP_PGSIZE_2M))
continue;
if ((flags & ODP_MEMZONE_16MB) &&
(free_mmfrag[i].hugepage_sz == ODP_PGSIZE_16G))
continue;
if ((flags & ODP_MEMZONE_16GB) &&
(free_mmfrag[i].hugepage_sz == ODP_PGSIZE_16M))
continue;
/* this segment is the best until now */
if (mmfrag_idx == -1) {
mmfrag_idx = i;
mmfrag_len = free_mmfrag[i].len;
seg_offset = addr_offset;
}
/* find the biggest contiguous zone */
else if (len == 0) {
if (free_mmfrag[i].len > mmfrag_len) {
mmfrag_idx = i;
mmfrag_len = free_mmfrag[i].len;
seg_offset = addr_offset;
}
}
/*
* find the smallest (we already checked that current
* zone length is > len
*/
else if ((free_mmfrag[i].len + align < mmfrag_len) ||
((free_mmfrag[i].len <= mmfrag_len + align) &&
(addr_offset < seg_offset))) {
mmfrag_idx = i;
mmfrag_len = free_mmfrag[i].len;
seg_offset = addr_offset;
}
}
/* no segment found */
if (mmfrag_idx == -1) {
/*
* If ODP_MEMZONE_SIZE_HINT_ONLY flag is specified,
* try allocating again without the size parameter
* otherwise -fail.
*/
if ((flags & ODP_MEMZONE_SIZE_HINT_ONLY) &&
((flags & ODP_MEMZONE_1GB) ||
(flags & ODP_MEMZONE_2MB) ||
(flags & ODP_MEMZONE_16MB) ||
(flags & ODP_MEMZONE_16GB)))
return mm_district_reserve_aligned(name, orig_name,
len, socket_id, 0,
align, bound);
odp_err = ENOMEM;
return NULL;
}
/* save aligned physical and virtual addresses */
mmfrag_physaddr = free_mmfrag[mmfrag_idx].phys_addr + seg_offset;
mmfrag_addr = ODP_PTR_ADD(free_mmfrag[mmfrag_idx].addr,
(uintptr_t)seg_offset);
/* if we are looking for a biggest mm_district */
if (len == 0) {
if (bound == 0)
requested_len = mmfrag_len - seg_offset;
else
requested_len =
ODP_ALIGN_CEIL(mmfrag_physaddr + 1, bound)
- mmfrag_physaddr;
}
/* set length to correct value */
len = (size_t)seg_offset + requested_len;
/* update our internal state */
free_mmfrag[mmfrag_idx].len -= len;
free_mmfrag[mmfrag_idx].phys_addr += len;
free_mmfrag[mmfrag_idx].addr =
(char *)free_mmfrag[mmfrag_idx].addr + len;
/* fill the zone in config */
struct odp_mm_district *mz =
&mcfg->mm_district[mcfg->mm_district_idx++];
snprintf(mz->orig_name, sizeof(mz->orig_name), "%s", orig_name);
snprintf(mz->name, sizeof(mz->name), "%s", name);
mz->phys_addr = mmfrag_physaddr;
mz->phys_addr_end = mmfrag_physaddr + requested_len;
mz->excursion_addr = mmfrag_addr - mmfrag_physaddr;
mz->addr = mmfrag_addr;
mz->len = requested_len;
mz->hugepage_sz = free_mmfrag[mmfrag_idx].hugepage_sz;
mz->socket_id = free_mmfrag[mmfrag_idx].socket_id;
mz->flags = 0;
mz->mmfrag_id = mmfrag_idx;
return mz;
}
/**
* Packet IO loopback worker thread using ODP queues
*
* @param arg thread arguments of type 'thread_args_t *'
*/
static void *pktio_queue_thread(void *arg)
{
int thr;
odp_buffer_pool_t pkt_pool;
odp_pktio_t pktio;
thread_args_t *thr_args;
odp_queue_t outq_def;
odp_queue_t inq_def;
char inq_name[ODP_QUEUE_NAME_LEN];
odp_queue_param_t qparam;
odp_packet_t pkt;
odp_buffer_t buf;
int ret;
unsigned long pkt_cnt = 0;
unsigned long err_cnt = 0;
odp_pktio_params_t params;
socket_params_t *sock_params = ¶ms.sock_params;
thr = odp_thread_id();
thr_args = arg;
printf("Pktio thread [%02i] starts, pktio_dev:%s\n", thr,
thr_args->pktio_dev);
/* Lookup the packet pool */
pkt_pool = odp_buffer_pool_lookup("packet_pool");
if (pkt_pool == ODP_BUFFER_POOL_INVALID || pkt_pool != thr_args->pool) {
ODP_ERR(" [%02i] Error: pkt_pool not found\n", thr);
return NULL;
}
/* Open a packet IO instance for this thread */
sock_params->type = thr_args->type;
sock_params->fanout = thr_args->fanout;
pktio = odp_pktio_open(thr_args->pktio_dev, pkt_pool, ¶ms);
if (pktio == ODP_PKTIO_INVALID) {
ODP_ERR(" [%02i] Error: pktio create failed\n", thr);
return NULL;
}
/*
* Create and set the default INPUT queue associated with the 'pktio'
* resource
*/
qparam.sched.prio = ODP_SCHED_PRIO_DEFAULT;
qparam.sched.sync = ODP_SCHED_SYNC_ATOMIC;
qparam.sched.group = ODP_SCHED_GROUP_DEFAULT;
snprintf(inq_name, sizeof(inq_name), "%i-pktio_inq_def", (int)pktio);
inq_name[ODP_QUEUE_NAME_LEN - 1] = '\0';
inq_def = odp_queue_create(inq_name, ODP_QUEUE_TYPE_PKTIN, &qparam);
if (inq_def == ODP_QUEUE_INVALID) {
ODP_ERR(" [%02i] Error: pktio queue creation failed\n", thr);
return NULL;
}
ret = odp_pktio_inq_setdef(pktio, inq_def);
if (ret != 0) {
ODP_ERR(" [%02i] Error: default input-Q setup\n", thr);
return NULL;
}
printf(" [%02i] created pktio:%02i, queue mode (ATOMIC queues)\n"
" default pktio%02i-INPUT queue:%u\n",
thr, pktio, pktio, inq_def);
/* Loop packets */
for (;;) {
odp_pktio_t pktio_tmp;
#if 1
/* Use schedule to get buf from any input queue */
buf = odp_schedule(NULL, ODP_SCHED_WAIT);
#else
/* Always dequeue from the same input queue */
buf = odp_queue_deq(inq_def);
if (!odp_buffer_is_valid(buf))
continue;
#endif
pkt = odp_packet_from_buffer(buf);
/* Drop packets with errors */
if (odp_unlikely(drop_err_pkts(&pkt, 1) == 0)) {
ODP_ERR("Drop frame - err_cnt:%lu\n", ++err_cnt);
continue;
}
pktio_tmp = odp_pktio_get_input(pkt);
outq_def = odp_pktio_outq_getdef(pktio_tmp);
if (outq_def == ODP_QUEUE_INVALID) {
ODP_ERR(" [%02i] Error: def output-Q query\n", thr);
return NULL;
}
/* Swap Eth MACs and possibly IP-addrs before sending back */
swap_pkt_addrs(&pkt, 1);
/* Enqueue the packet for output */
odp_queue_enq(outq_def, buf);
/* Print packet counts every once in a while */
if (odp_unlikely(pkt_cnt++ % 100000 == 0)) {
printf(" [%02i] pkt_cnt:%lu\n", thr, pkt_cnt);
fflush(NULL);
}
}
/* unreachable */
}
int setup_pkt_odp(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry,
const char *netdev, odp_pool_t pool)
{
uint8_t portid = 0;
uint16_t nbrxq = 0;
uint16_t nbtxq = 0;
int ret, i;
struct odp_eth_dev *dev;
pkt_odp_t *const pkt_odp = &pktio_entry->s.pkt_odp;
if (!_odp_netdev_is_valid(netdev)) {
ODP_ERR("netdev %s, format err! should be "
"pktio_x! x is digital number\n", netdev);
return -1;
}
portid = atoi(netdev + strlen(PKTIO_DEV_NAME));
pkt_odp->portid = portid;
pkt_odp->pool = pool;
pkt_odp->queueid = 0;
/* On init set it up only to 1 rx and tx queue.*/
dev = odp_eth_dev_allocated_id(portid);
if (!dev) {
ODP_ERR("netdev %s, odp_eth_dev_allocated_id err!\n", netdev);
return -1;
}
nbtxq = dev->q_num;
nbrxq = dev->q_num;
ret = odp_eth_dev_configure(portid, nbrxq, nbtxq, &port_conf);
if (ret < 0) {
ODP_ERR("Cannot configure device: err=%d, port=%u\n",
ret, (unsigned int)portid);
return -1;
}
/* init one RX queue on each port */
for (i = 0; i < nbrxq; i++) {
ret = odp_eth_rx_queue_setup(portid, i, nb_rxd,
0, NULL/*&rx_conf*/, (void *)pool);
if (ret < 0) {
ODP_ERR("rxq:err=%d, port=%u\n",
ret, (unsigned int)portid);
return -1;
}
}
/* init one TX queue on each port */
for (i = 0; i < nbtxq; i++) {
ret = odp_eth_tx_queue_setup(portid, i, nb_txd,
0, NULL/*&tx_conf*/, (void *)pool);
if (ret < 0) {
ODP_ERR("txq:err=%d, port=%u\n",
ret, (unsigned int)portid);
return -1;
}
}
/* Start device */
ret = odp_eth_dev_start(portid);
if (ret < 0) {
ODP_ERR("odp_eth_dev_start:err=%d, port=%u\n",
ret, (unsigned int)portid);
return -1;
}
return 0;
}
