int queue_enq(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr)
{
int sched = 0;
LOCK(&queue->s.lock);
if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
UNLOCK(&queue->s.lock);
ODP_ERR("Bad queue status\n");
return -1;
}
if (queue->s.head == NULL) {
/* Empty queue */
queue->s.head = buf_hdr;
queue->s.tail = buf_hdr;
buf_hdr->next = NULL;
} else {
queue->s.tail->next = buf_hdr;
queue->s.tail = buf_hdr;
buf_hdr->next = NULL;
}
if (queue->s.status == QUEUE_STATUS_NOTSCHED) {
queue->s.status = QUEUE_STATUS_SCHED;
sched = 1; /* retval: schedule queue */
}
UNLOCK(&queue->s.lock);
/* Add queue to scheduling */
if (sched && schedule_queue(queue))
ODP_ABORT("schedule_queue failed\n");
return 0;
}
int schedule_pktio_start(odp_pktio_t pktio, int prio)
{
odp_buffer_t buf;
sched_cmd_t *sched_cmd;
odp_queue_t pri_queue;
buf = odp_buffer_alloc(sched->pool);
if (buf == ODP_BUFFER_INVALID)
return -1;
sched_cmd = odp_buffer_addr(buf);
sched_cmd->cmd = SCHED_CMD_POLL_PKTIN;
sched_cmd->pktio = pktio;
sched_cmd->pe = get_pktio_entry(pktio);
sched_cmd->prio = prio;
pri_queue = pri_set_pktio(pktio, prio);
if (odp_queue_enq(pri_queue, odp_buffer_to_event(buf)))
ODP_ABORT("schedule_pktio_start failed\n");
return 0;
}
void _odp_timer_fini(odp_timer_pool *tp)
{
if (timer_delete(tp->timerid) != 0)
ODP_ABORT("timer_delete() returned error %s\n",
strerror(errno));
if(_odp_timer_pool_global == tp)
_odp_timer_pool_global = NULL;
}
void odp_schedule_release_atomic(void)
{
if (sched_local.pri_queue != ODP_QUEUE_INVALID &&
sched_local.num == 0) {
/* Release current atomic queue */
if (odp_queue_enq(sched_local.pri_queue, sched_local.cmd_ev))
ODP_ABORT("odp_schedule_release_atomic failed\n");
sched_local.pri_queue = ODP_QUEUE_INVALID;
}
}
void _odp_timer_init(odp_timer_pool *tp)
{
struct sigevent sigev;
struct itimerspec ispec;
uint64_t res, sec, nsec;
if(_odp_timer_pool_global != NULL){
ODP_ABORT("Cannot have more than one timer at once");
}
ODP_DBG("Creating POSIX timer for timer pool %s, period %"
PRIu64" ns\n", tp->name, tp->param.res_ns);
memset(&sigev, 0, sizeof(sigev));
memset(&ispec, 0, sizeof(ispec));
_odp_timer_pool_global = tp;
sigev.sigev_notify = SIGEV_CALLBACK;
sigev.sigev_notify_function = timer_notify;
sigev.sigev_value.sival_ptr = tp;
if (timer_create(CLOCK_MONOTONIC, &sigev, &tp->timerid))
ODP_ABORT("timer_create() returned error %s\n",
strerror(errno));
res = tp->param.res_ns;
sec = res / ODP_TIME_SEC_IN_NS;
nsec = res - sec * ODP_TIME_SEC_IN_NS;
ispec.it_interval.tv_sec = (time_t)sec;
ispec.it_interval.tv_nsec = (long)nsec;
ispec.it_value.tv_sec = (time_t)sec;
ispec.it_value.tv_nsec = (long)nsec;
if (timer_settime(tp->timerid, 0, &ispec, NULL))
ODP_ABORT("timer_settime() returned error %s\n",
strerror(errno));
}
int odp_pktio_inq_setdef(odp_pktio_t id, odp_queue_t queue)
{
pktio_entry_t *pktio_entry = get_pktio_entry(id);
queue_entry_t *qentry;
if (pktio_entry == NULL || queue == ODP_QUEUE_INVALID)
return -1;
qentry = queue_to_qentry(queue);
if (qentry->s.type != ODP_QUEUE_TYPE_PKTIN)
return -1;
lock_entry(pktio_entry);
pktio_entry->s.inq_default = queue;
unlock_entry(pktio_entry);
switch (qentry->s.type) {
/* Change to ODP_QUEUE_TYPE_POLL when ODP_QUEUE_TYPE_PKTIN is removed */
case ODP_QUEUE_TYPE_PKTIN:
/* User polls the input queue */
queue_lock(qentry);
qentry->s.pktin = id;
queue_unlock(qentry);
/* Uncomment when ODP_QUEUE_TYPE_PKTIN is removed
break;
case ODP_QUEUE_TYPE_SCHED:
*/
/* Packet input through the scheduler */
if (schedule_pktio_start(id, ODP_SCHED_PRIO_LOWEST)) {
ODP_ERR("Schedule pktio start failed\n");
return -1;
}
break;
default:
ODP_ABORT("Bad queue type\n");
}
return 0;
}
void odp_event_free(odp_event_t event)
{
switch (odp_event_type(event)) {
case ODP_EVENT_BUFFER:
odp_buffer_free(odp_buffer_from_event(event));
break;
case ODP_EVENT_PACKET:
odp_packet_free(odp_packet_from_event(event));
break;
case ODP_EVENT_TIMEOUT:
odp_timeout_free(odp_timeout_from_event(event));
break;
case ODP_EVENT_CRYPTO_COMPL:
odp_crypto_compl_free(odp_crypto_compl_from_event(event));
break;
default:
ODP_ABORT("Invalid event type: %d\n", odp_event_type(event));
}
}
int odp_queue_destroy(odp_queue_t handle)
{
queue_entry_t *queue;
queue = queue_to_qentry(handle);
LOCK(queue);
INVALIDATE(queue);
if (queue->s.status == QUEUE_STATUS_FREE) {
UNLOCK(queue);
ODP_ERR("queue \"%s\" already free\n", queue->s.name);
return -1;
}
if (queue->s.status == QUEUE_STATUS_DESTROYED) {
UNLOCK(queue);
ODP_ERR("queue \"%s\" already destroyed\n", queue->s.name);
return -1;
}
if (queue->s.head != NULL) {
UNLOCK(queue);
ODP_ERR("queue \"%s\" not empty\n", queue->s.name);
return -1;
}
switch (queue->s.status) {
case QUEUE_STATUS_READY:
queue->s.status = QUEUE_STATUS_FREE;
break;
case QUEUE_STATUS_NOTSCHED:
queue->s.status = QUEUE_STATUS_FREE;
schedule_queue_destroy(queue);
break;
case QUEUE_STATUS_SCHED:
/* Queue is still in scheduling */
queue->s.status = QUEUE_STATUS_DESTROYED;
break;
default:
ODP_ABORT("Unexpected queue status\n");
}
UNLOCK(queue);
return 0;
}
static void mmap_fill_ring(struct ring *ring, odp_pool_t pool_hdl, int fanout)
{
/*@todo add Huge Pages support*/
int pz = getpagesize();
uint32_t pool_id;
pool_entry_t *pool_entry;
if (pool_hdl == ODP_POOL_INVALID)
ODP_ABORT("Invalid pool handle\n");
pool_id = pool_handle_to_index(pool_hdl);
pool_entry = get_pool_entry(pool_id);
/* Frame has to capture full packet which can fit to the pool block.*/
ring->req.tp_frame_size = (pool_entry->s.blk_size +
TPACKET_HDRLEN + TPACKET_ALIGNMENT +
+ (pz - 1)) & (-pz);
/* Calculate how many pages do we need to hold all pool packets
* and align size to page boundary.
*/
ring->req.tp_block_size = (ring->req.tp_frame_size *
pool_entry->s.buf_num + (pz - 1)) & (-pz);
if (!fanout) {
/* Single socket is in use. Use 1 block with buf_num frames. */
ring->req.tp_block_nr = 1;
} else {
/* Fanout is in use, more likely taffic split accodring to
* number of cpu threads. Use cpu blocks and buf_num frames. */
ring->req.tp_block_nr = odp_cpu_count();
}
ring->req.tp_frame_nr = ring->req.tp_block_size /
ring->req.tp_frame_size * ring->req.tp_block_nr;
ring->mm_len = ring->req.tp_block_size * ring->req.tp_block_nr;
ring->rd_num = ring->req.tp_frame_nr;
ring->flen = ring->req.tp_frame_size;
}
int queue_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num)
{
int sched = 0;
int i;
odp_buffer_hdr_t *tail;
for (i = 0; i < num - 1; i++)
buf_hdr[i]->next = buf_hdr[i+1];
tail = buf_hdr[num-1];
buf_hdr[num-1]->next = NULL;
LOCK(&queue->s.lock);
if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
UNLOCK(&queue->s.lock);
ODP_ERR("Bad queue status\n");
return -1;
}
/* Empty queue */
if (queue->s.head == NULL)
queue->s.head = buf_hdr[0];
else
queue->s.tail->next = buf_hdr[0];
queue->s.tail = tail;
if (queue->s.status == QUEUE_STATUS_NOTSCHED) {
queue->s.status = QUEUE_STATUS_SCHED;
sched = 1; /* retval: schedule queue */
}
UNLOCK(&queue->s.lock);
/* Add queue to scheduling */
if (sched && schedule_queue(queue))
ODP_ABORT("schedule_queue failed\n");
return num; /* All events enqueued */
}
/*
* Schedule queues
*
* TODO: SYNC_ORDERED not implemented yet
*/
static int schedule(odp_queue_t *out_queue, odp_event_t out_ev[],
unsigned int max_num, unsigned int max_deq)
{
int i, j;
int thr;
int ret;
if (sched_local.num) {
ret = copy_events(out_ev, max_num);
if (out_queue)
*out_queue = queue_handle(sched_local.qe);
return ret;
}
odp_schedule_release_atomic();
if (odp_unlikely(sched_local.pause))
return 0;
thr = odp_thread_id();
for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) {
int id;
if (sched->pri_mask[i] == 0)
continue;
id = thr & (QUEUES_PER_PRIO-1);
for (j = 0; j < QUEUES_PER_PRIO; j++, id++) {
odp_queue_t pri_q;
odp_event_t ev;
odp_buffer_t buf;
sched_cmd_t *sched_cmd;
queue_entry_t *qe;
int num;
if (id >= QUEUES_PER_PRIO)
id = 0;
if (odp_unlikely((sched->pri_mask[i] & (1 << id)) == 0))
continue;
pri_q = sched->pri_queue[i][id];
ev = odp_queue_deq(pri_q);
buf = odp_buffer_from_event(ev);
if (buf == ODP_BUFFER_INVALID)
continue;
sched_cmd = odp_buffer_addr(buf);
if (sched_cmd->cmd == SCHED_CMD_POLL_PKTIN) {
/* Poll packet input */
if (pktin_poll(sched_cmd->pe)) {
/* Stop scheduling the pktio */
pri_clr_pktio(sched_cmd->pktio,
sched_cmd->prio);
odp_buffer_free(buf);
} else {
/* Continue scheduling the pktio */
if (odp_queue_enq(pri_q, ev))
ODP_ABORT("schedule failed\n");
}
continue;
}
qe = sched_cmd->qe;
num = queue_deq_multi(qe, sched_local.buf_hdr, max_deq);
if (num < 0) {
/* Destroyed queue */
queue_destroy_finalize(qe);
continue;
}
if (num == 0) {
/* Remove empty queue from scheduling */
continue;
}
sched_local.num = num;
sched_local.index = 0;
sched_local.qe = qe;
ret = copy_events(out_ev, max_num);
if (queue_is_atomic(qe)) {
/* Hold queue during atomic access */
sched_local.pri_queue = pri_q;
sched_local.cmd_ev = ev;
} else {
/* Continue scheduling the queue */
if (odp_queue_enq(pri_q, ev))
ODP_ABORT("schedule failed\n");
}
/* Output the source queue handle */
if (out_queue)
*out_queue = queue_handle(qe);
return ret;
}
}
return 0;
}
