static inline void
mbox_send_request(struct mbox *m, struct octeontx_mbox_hdr *hdr,
const void *txmsg, uint16_t txsize)
{
struct mbox_ram_hdr old_hdr;
struct mbox_ram_hdr new_hdr = { {0} };
uint64_t *ram_mbox_hdr = (uint64_t *)m->ram_mbox_base;
uint8_t *ram_mbox_msg = m->ram_mbox_base + sizeof(struct mbox_ram_hdr);
/*
* Initialize the channel with the tag left by last send.
* On success full mbox send complete, PF increments the tag by one.
* The sender can validate integrity of PF message with this scheme
*/
old_hdr.u64 = rte_read64(ram_mbox_hdr);
m->tag_own = (old_hdr.tag + 2) & (~0x1ul); /* next even number */
/* Copy msg body */
if (txmsg)
mbox_msgcpy(ram_mbox_msg, txmsg, txsize);
/* Prepare new hdr */
new_hdr.chan_state = MBOX_CHAN_STATE_REQ;
new_hdr.coproc = hdr->coproc;
new_hdr.msg = hdr->msg;
new_hdr.vfid = hdr->vfid;
new_hdr.tag = m->tag_own;
new_hdr.len = txsize;
/* Write the msg header */
rte_write64(new_hdr.u64, ram_mbox_hdr);
rte_smp_wmb();
/* Notify PF about the new msg - write to MBOX reg generates PF IRQ */
rte_write64(0, m->reg);
}
int32_t
rte_service_component_register(const struct rte_service_spec *spec,
uint32_t *id_ptr)
{
uint32_t i;
int32_t free_slot = -1;
if (spec->callback == NULL || strlen(spec->name) == 0)
return -EINVAL;
for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
if (!service_valid(i)) {
free_slot = i;
break;
}
}
if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))
return -ENOSPC;
struct rte_service_spec_impl *s = &rte_services[free_slot];
s->spec = *spec;
s->internal_flags |= SERVICE_F_REGISTERED | SERVICE_F_START_CHECK;
rte_smp_wmb();
rte_service_count++;
if (id_ptr)
*id_ptr = free_slot;
return 0;
}
static __rte_always_inline void
flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq)
{
uint16_t used_idx = vq->last_used_idx & (vq->size - 1);
if (used_idx + vq->shadow_used_idx <= vq->size) {
do_flush_shadow_used_ring(dev, vq, used_idx, 0,
vq->shadow_used_idx);
} else {
uint16_t size;
/* update used ring interval [used_idx, vq->size] */
size = vq->size - used_idx;
do_flush_shadow_used_ring(dev, vq, used_idx, 0, size);
/* update the left half used ring interval [0, left_size] */
do_flush_shadow_used_ring(dev, vq, 0, size,
vq->shadow_used_idx - size);
}
vq->last_used_idx += vq->shadow_used_idx;
rte_smp_wmb();
*(volatile uint16_t *)&vq->used->idx += vq->shadow_used_idx;
vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
sizeof(vq->used->idx));
}
/*
* An rarp packet is constructed and broadcasted to notify switches about
* the new location of the migrated VM, so that packets from outside will
* not be lost after migration.
*
* However, we don't actually "send" a rarp packet here, instead, we set
* a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
*/
int
user_send_rarp(int vid, struct VhostUserMsg *msg)
{
struct virtio_net *dev;
uint8_t *mac = (uint8_t *)&msg->payload.u64;
dev = get_device(vid);
if (!dev)
return -1;
RTE_LOG(DEBUG, VHOST_CONFIG,
":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n",
mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
memcpy(dev->mac.addr_bytes, mac, 6);
/*
* Set the flag to inject a RARP broadcast packet at
* rte_vhost_dequeue_burst().
*
* rte_smp_wmb() is for making sure the mac is copied
* before the flag is set.
*/
rte_smp_wmb();
rte_atomic16_set(&dev->broadcast_rarp, 1);
return 0;
}
int32_t
rte_service_runstate_set(uint32_t id, uint32_t runstate)
{
struct rte_service_spec_impl *s;
SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
if (runstate)
s->app_runstate = RUNSTATE_RUNNING;
else
s->app_runstate = RUNSTATE_STOPPED;
rte_smp_wmb();
return 0;
}
/*
* Notify host that there are data pending on our TX bufring.
*
* Since this in userspace, rely on the monitor page.
* Can't do a hypercall from userspace.
*/
void
rte_vmbus_chan_signal_tx(const struct vmbus_channel *chan)
{
const struct rte_vmbus_device *dev = chan->device;
const struct vmbus_br *tbr = &chan->txbr;
/* Make sure all updates are done before signaling host */
rte_smp_wmb();
/* If host is ignoring interrupts? */
if (tbr->vbr->imask)
return;
vmbus_set_event(dev, chan);
}
int32_t rte_service_lcore_reset_all(void)
{
/* loop over cores, reset all to mask 0 */
uint32_t i;
for (i = 0; i < RTE_MAX_LCORE; i++) {
lcore_states[i].service_mask = 0;
lcore_states[i].is_service_core = 0;
lcore_states[i].runstate = RUNSTATE_STOPPED;
}
for (i = 0; i < RTE_SERVICE_NUM_MAX; i++)
rte_atomic32_set(&rte_services[i].num_mapped_cores, 0);
rte_smp_wmb();
return 0;
}
int32_t
rte_service_lcore_del(uint32_t lcore)
{
if (lcore >= RTE_MAX_LCORE)
return -EINVAL;
struct core_state *cs = &lcore_states[lcore];
if (!cs->is_service_core)
return -EINVAL;
if (cs->runstate != RUNSTATE_STOPPED)
return -EBUSY;
set_lcore_state(lcore, ROLE_RTE);
rte_smp_wmb();
return 0;
}
int32_t
rte_service_lcore_add(uint32_t lcore)
{
if (lcore >= RTE_MAX_LCORE)
return -EINVAL;
if (lcore_states[lcore].is_service_core)
return -EALREADY;
set_lcore_state(lcore, ROLE_SERVICE);
/* ensure that after adding a core the mask and state are defaults */
lcore_states[lcore].service_mask = 0;
lcore_states[lcore].runstate = RUNSTATE_STOPPED;
rte_smp_wmb();
return rte_eal_wait_lcore(lcore);
}
vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
{
uint64_t page;
if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) ||
!dev->log_base || !len))
return;
if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
return;
/* To make sure guest memory updates are committed before logging */
rte_smp_wmb();
page = addr / VHOST_LOG_PAGE;
while (page * VHOST_LOG_PAGE < addr + len) {
vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
page += 1;
}
}
int32_t
rte_service_component_unregister(uint32_t id)
{
uint32_t i;
struct rte_service_spec_impl *s;
SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
rte_service_count--;
rte_smp_wmb();
s->internal_flags &= ~(SERVICE_F_REGISTERED);
/* clear the run-bit in all cores */
for (i = 0; i < RTE_MAX_LCORE; i++)
lcore_states[i].service_mask &= ~(UINT64_C(1) << id);
memset(&rte_services[id], 0, sizeof(struct rte_service_spec_impl));
return 0;
}
static int32_t
service_update(struct rte_service_spec *service, uint32_t lcore,
uint32_t *set, uint32_t *enabled)
{
uint32_t i;
int32_t sid = -1;
for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
if ((struct rte_service_spec *)&rte_services[i] == service &&
service_valid(i)) {
sid = i;
break;
}
}
if (sid == -1 || lcore >= RTE_MAX_LCORE)
return -EINVAL;
if (!lcore_states[lcore].is_service_core)
return -EINVAL;
uint64_t sid_mask = UINT64_C(1) << sid;
if (set) {
if (*set) {
lcore_states[lcore].service_mask |= sid_mask;
rte_atomic32_inc(&rte_services[sid].num_mapped_cores);
} else {
lcore_states[lcore].service_mask &= ~(sid_mask);
rte_atomic32_dec(&rte_services[sid].num_mapped_cores);
}
}
if (enabled)
*enabled = !!(lcore_states[lcore].service_mask & (sid_mask));
rte_smp_wmb();
return 0;
}
int
perf_launch_lcores(struct evt_test *test, struct evt_options *opt,
int (*worker)(void *))
{
int ret, lcore_id;
struct test_perf *t = evt_test_priv(test);
int port_idx = 0;
/* launch workers */
RTE_LCORE_FOREACH_SLAVE(lcore_id) {
if (!(opt->wlcores[lcore_id]))
continue;
ret = rte_eal_remote_launch(worker,
&t->worker[port_idx], lcore_id);
if (ret) {
evt_err("failed to launch worker %d", lcore_id);
return ret;
}
port_idx++;
}
/* launch producers */
RTE_LCORE_FOREACH_SLAVE(lcore_id) {
if (!(opt->plcores[lcore_id]))
continue;
ret = rte_eal_remote_launch(perf_producer_wrapper,
&t->prod[port_idx], lcore_id);
if (ret) {
evt_err("failed to launch perf_producer %d", lcore_id);
return ret;
}
port_idx++;
}
const uint64_t total_pkts = opt->nb_pkts *
evt_nr_active_lcores(opt->plcores);
uint64_t dead_lock_cycles = rte_get_timer_cycles();
int64_t dead_lock_remaining = total_pkts;
const uint64_t dead_lock_sample = rte_get_timer_hz() * 5;
uint64_t perf_cycles = rte_get_timer_cycles();
int64_t perf_remaining = total_pkts;
const uint64_t perf_sample = rte_get_timer_hz();
static float total_mpps;
static uint64_t samples;
const uint64_t freq_mhz = rte_get_timer_hz() / 1000000;
int64_t remaining = t->outstand_pkts - processed_pkts(t);
while (t->done == false) {
const uint64_t new_cycles = rte_get_timer_cycles();
if ((new_cycles - perf_cycles) > perf_sample) {
const uint64_t latency = total_latency(t);
const uint64_t pkts = processed_pkts(t);
remaining = t->outstand_pkts - pkts;
float mpps = (float)(perf_remaining-remaining)/1000000;
perf_remaining = remaining;
perf_cycles = new_cycles;
total_mpps += mpps;
++samples;
if (opt->fwd_latency && pkts > 0) {
printf(CLGRN"\r%.3f mpps avg %.3f mpps [avg fwd latency %.3f us] "CLNRM,
mpps, total_mpps/samples,
(float)(latency/pkts)/freq_mhz);
} else {
printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM,
mpps, total_mpps/samples);
}
fflush(stdout);
if (remaining <= 0) {
t->result = EVT_TEST_SUCCESS;
if (opt->prod_type == EVT_PROD_TYPE_SYNT) {
t->done = true;
rte_smp_wmb();
break;
}
}
}
if (new_cycles - dead_lock_cycles > dead_lock_sample &&
opt->prod_type == EVT_PROD_TYPE_SYNT) {
remaining = t->outstand_pkts - processed_pkts(t);
if (dead_lock_remaining == remaining) {
rte_event_dev_dump(opt->dev_id, stdout);
evt_err("No schedules for seconds, deadlock");
t->done = true;
rte_smp_wmb();
break;
}
dead_lock_remaining = remaining;
dead_lock_cycles = new_cycles;
}
}
printf("\n");
return 0;
}
/*
* Write scattered channel packet to TX bufring.
*
* The offset of this channel packet is written as a 64bits value
* immediately after this channel packet.
*
* The write goes through three stages:
* 1. Reserve space in ring buffer for the new data.
* Writer atomically moves priv_write_index.
* 2. Copy the new data into the ring.
* 3. Update the tail of the ring (visible to host) that indicates
* next read location. Writer updates write_index
*/
int
vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
bool *need_sig)
{
struct vmbus_bufring *vbr = tbr->vbr;
uint32_t ring_size = tbr->dsize;
uint32_t old_windex, next_windex, windex, total;
uint64_t save_windex;
int i;
total = 0;
for (i = 0; i < iovlen; i++)
total += iov[i].iov_len;
total += sizeof(save_windex);
/* Reserve space in ring */
do {
uint32_t avail;
/* Get current free location */
old_windex = tbr->windex;
/* Prevent compiler reordering this with calculation */
rte_compiler_barrier();
avail = vmbus_br_availwrite(tbr, old_windex);
/* If not enough space in ring, then tell caller. */
if (avail <= total)
return -EAGAIN;
next_windex = vmbus_br_idxinc(old_windex, total, ring_size);
/* Atomic update of next write_index for other threads */
} while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex));
/* Space from old..new is now reserved */
windex = old_windex;
for (i = 0; i < iovlen; i++) {
windex = vmbus_txbr_copyto(tbr, windex,
iov[i].iov_base, iov[i].iov_len);
}
/* Set the offset of the current channel packet. */
save_windex = ((uint64_t)old_windex) << 32;
windex = vmbus_txbr_copyto(tbr, windex, &save_windex,
sizeof(save_windex));
/* The region reserved should match region used */
RTE_ASSERT(windex == next_windex);
/* Ensure that data is available before updating host index */
rte_smp_wmb();
/* Checkin for our reservation. wait for our turn to update host */
while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex))
rte_pause();
/* If host had read all data before this, then need to signal */
*need_sig |= vmbus_txbr_need_signal(tbr, old_windex);
return 0;
}
