int queue_enq(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr)
{
int sched = 0;
LOCK(queue);
int status = LOAD_S32(queue->s.status);
if (odp_unlikely(status < QUEUE_STATUS_READY)) {
UNLOCK(queue);
ODP_ERR("Bad queue status\n");
return -1;
}
if (LOAD_PTR(queue->s.head) == NULL) {
/* Empty queue */
STORE_PTR(queue->s.head, buf_hdr);
STORE_PTR(queue->s.tail, buf_hdr);
buf_hdr->next = NULL;
} else {
STORE_PTR(((typeof(queue->s.tail))LOAD_PTR(queue->s.tail))->next, buf_hdr);
STORE_PTR(queue->s.tail, buf_hdr);
buf_hdr->next = NULL;
}
if (status == QUEUE_STATUS_NOTSCHED) {
STORE_S32(queue->s.status, QUEUE_STATUS_SCHED);
sched = 1; /* retval: schedule queue */
}
UNLOCK(queue);
/* Add queue to scheduling */
if (sched)
schedule_queue(queue);
return 0;
}
odp_buffer_hdr_t *queue_deq(queue_entry_t *queue)
{
odp_buffer_hdr_t *buf_hdr;
if (LOAD_PTR(queue->s.head) == NULL)
return NULL;
LOCK(queue);
buf_hdr = LOAD_PTR(queue->s.head);
if (buf_hdr == NULL) {
UNLOCK(queue);
return NULL;
}
INVALIDATE(buf_hdr);
STORE_PTR(queue->s.head, buf_hdr->next);
if (buf_hdr->next == NULL) {
/* Queue is now empty */
STORE_PTR(queue->s.tail, NULL);
if (LOAD_S32(queue->s.status) == QUEUE_STATUS_SCHED)
STORE_S32(queue->s.status, QUEUE_STATUS_NOTSCHED);
}
buf_hdr->next = NULL;
UNLOCK(queue);
return buf_hdr;
}
int odp_queue_term_global(void)
{
int ret = 0;
int rc = 0;
queue_entry_t *queue;
int i;
for (i = 0; i < ODP_CONFIG_QUEUES; i++) {
queue = &queue_tbl->queue[i];
LOCK(queue);
if (LOAD_S32(queue->s.status) != QUEUE_STATUS_FREE) {
ODP_ERR("Not destroyed queue: %s\n", queue->s.name);
rc = -1;
}
UNLOCK(queue);
}
ret = odp_shm_free(odp_shm_lookup("odp_queues"));
if (ret < 0) {
ODP_ERR("shm free failed for odp_queues");
rc = -1;
}
return rc;
}
void queue_destroy_finalize(queue_entry_t *queue)
{
LOCK(queue);
if (LOAD_S32(queue->s.status) == QUEUE_STATUS_DESTROYED) {
INVALIDATE(queue);
queue->s.status = QUEUE_STATUS_FREE;
schedule_queue_destroy(queue);
}
UNLOCK(queue);
}
/* Only one writer can recurse the lock */
void odp_rwlock_recursive_write_lock(odp_rwlock_recursive_t *rlock)
{
int thr = odp_thread_id();
if (LOAD_S32(rlock->wr_owner) == thr) {
rlock->wr_cnt++;
return;
}
odp_rwlock_write_lock(&rlock->lock);
STORE_U32(rlock->wr_owner, thr);
rlock->wr_cnt = 1;
}
int queue_deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num)
{
odp_buffer_hdr_t *hdr;
int i;
LOCK(queue);
int status = LOAD_S32(queue->s.status);
if (odp_unlikely(status < QUEUE_STATUS_READY)) {
/* Bad queue, or queue has been destroyed.
* Scheduler finalizes queue destroy after this. */
UNLOCK(queue);
return -1;
}
hdr = LOAD_PTR(queue->s.head);
if (hdr == NULL) {
/* Already empty queue */
if (status == QUEUE_STATUS_SCHED)
STORE_S32(queue->s.status, QUEUE_STATUS_NOTSCHED);
UNLOCK(queue);
return 0;
}
for (i = 0; i < num && hdr; i++) {
INVALIDATE(hdr);
buf_hdr[i] = hdr;
hdr = hdr->next;
buf_hdr[i]->next = NULL;
}
STORE_PTR(queue->s.head, hdr);
if (hdr == NULL) {
/* Queue is now empty */
STORE_PTR(queue->s.tail, NULL);
}
UNLOCK(queue);
return i;
}
odp_queue_t odp_queue_create(const char *name, odp_queue_type_t type,
odp_queue_param_t *param)
{
uint32_t i;
queue_entry_t *queue;
odp_queue_t handle = ODP_QUEUE_INVALID;
for (i = 0; i < ODP_CONFIG_QUEUES; i++) {
queue = &queue_tbl->queue[i];
if (LOAD_S32(queue->s.status) != QUEUE_STATUS_FREE)
continue;
LOCK(queue);
INVALIDATE(queue);
if (queue->s.status == QUEUE_STATUS_FREE) {
queue_init(queue, name, type, param);
if (type == ODP_QUEUE_TYPE_SCHED ||
type == ODP_QUEUE_TYPE_PKTIN)
queue->s.status = QUEUE_STATUS_NOTSCHED;
else
queue->s.status = QUEUE_STATUS_READY;
handle = queue->s.handle;
UNLOCK(queue);
break;
}
UNLOCK(queue);
}
if (handle != ODP_QUEUE_INVALID &&
(type == ODP_QUEUE_TYPE_SCHED || type == ODP_QUEUE_TYPE_PKTIN)) {
if (schedule_queue_init(queue)) {
ODP_ERR("schedule queue init failed\n");
return ODP_QUEUE_INVALID;
}
}
return handle;
}
int queue_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num)
{
int sched = 0;
int i;
odp_buffer_hdr_t *tail;
for (i = 0; i < num - 1; i++)
buf_hdr[i]->next = buf_hdr[i+1];
tail = buf_hdr[num-1];
buf_hdr[num-1]->next = NULL;
LOCK(queue);
int status = LOAD_S32(queue->s.status);
if (odp_unlikely(status < QUEUE_STATUS_READY)) {
UNLOCK(queue);
ODP_ERR("Bad queue status\n");
return -1;
}
/* Empty queue */
if (LOAD_PTR(queue->s.head) == NULL)
STORE_PTR(queue->s.head, buf_hdr[0]);
else
STORE_PTR(((typeof(queue->s.tail))LOAD_PTR(queue->s.tail))->next, buf_hdr[0]);
STORE_PTR(queue->s.tail, tail);
if (status == QUEUE_STATUS_NOTSCHED) {
STORE_PTR(queue->s.status, QUEUE_STATUS_SCHED);
sched = 1; /* retval: schedule queue */
}
UNLOCK(queue);
/* Add queue to scheduling */
if (sched)
schedule_queue(queue);
return num; /* All events enqueued */
}
