void oct_tx_process_hw_work(cvmx_wqe_t *work, uint32_t outport)
{
uint64_t queue = cvmx_pko_get_base_queue(outport);
cvmx_pko_send_packet_prepare(outport, queue, CVMX_PKO_LOCK_CMD_QUEUE);
/* Build a PKO pointer to this packet */
cvmx_pko_command_word0_t pko_command;
pko_command.u64 = 0;
pko_command.s.segs = work->word2.s.bufs;
pko_command.s.total_bytes = cvmx_wqe_get_len(work);
/* Send the packet */
cvmx_pko_return_value_t send_status = cvmx_pko_send_packet_finish(outport, queue, pko_command, work->packet_ptr, CVMX_PKO_LOCK_CMD_QUEUE);
if (send_status != CVMX_PKO_SUCCESS)
{
printf("Failed to send packet using cvmx_pko_send_packet2\n");
cvmx_helper_free_packet_data(work);
STAT_TX_HW_SEND_ERR;
}
else
{
STAT_TX_SEND_OVER;
}
cvmx_fpa_free(work, wqe_pool, 0);
}
void oct_tx_process_hw(mbuf_t *mbuf, uint32_t outport)
{
uint64_t queue;
/* Build a PKO pointer to this packet */
cvmx_pko_return_value_t send_status;
cvmx_pko_command_word0_t pko_command;
queue = cvmx_pko_get_base_queue(outport);
cvmx_pko_send_packet_prepare(outport, queue, CVMX_PKO_LOCK_CMD_QUEUE);
pko_command.u64 = 0;
pko_command.s.segs = 1;
pko_command.s.total_bytes = mbuf->pkt_totallen;
/* Send the packet */
send_status = cvmx_pko_send_packet_finish(outport, queue, pko_command, mbuf->packet_ptr, CVMX_PKO_LOCK_CMD_QUEUE);
if (send_status != CVMX_PKO_SUCCESS)
{
STAT_TX_HW_SEND_ERR;
PACKET_DESTROY_DATA(mbuf);
}
else
{
STAT_TX_SEND_OVER;
}
MBUF_FREE(mbuf);
}
/* IN PROGRESS */
void send_packet()
{
uint8_t *buf, *pbuf;
uint64_t queue, length, buf_phys_addr;
cvmx_pko_command_word0_t pko_command;
cvmx_pko_return_value_t status;
cvmx_buf_ptr_t hw_buffer;
buf = (uint8_t *) cvmx_fpa_alloc(packet_pool);
if (buf == NULL) {
printf("ERROR: allocation from pool %" PRIu64 " failed!\n", packet_pool);
return;
} else {
printf("Packet allocation successful!\n");
}
pbuf = build_packet(buf, PAYLOAD_SIZE);
length = (uint64_t) (pbuf - buf);
printf("buf : %p\n", buf);
printf("pbuf: %p\n", pbuf);
printf("diff: %" PRIu64 "\n", length);
pko_command.u64 = 0;
pko_command.s.segs = 1;
pko_command.s.total_bytes = length;
pko_command.s.dontfree = 1;
buf_phys_addr = cvmx_ptr_to_phys(buf);
printf("buf_phys_addr: %" PRIu64 "\n", buf_phys_addr);
hw_buffer.s.i = 0;
hw_buffer.s.back = 0;
hw_buffer.s.pool = packet_pool; // the pool that the buffer came from
hw_buffer.s.size = length; // the size of the segment pointed to by addr (in bytes)
hw_buffer.s.addr = cvmx_ptr_to_phys(buf); // pointer to the first byte of the data
queue = cvmx_pko_get_base_queue(xaui_ipd_port);
printf("queue: %" PRIu64 "\n", queue);
cvmx_pko_send_packet_prepare(xaui_ipd_port, queue, CVMX_PKO_LOCK_NONE);
// THROWS EXCEPTION HERE
status = cvmx_pko_send_packet_finish(xaui_ipd_port, queue, pko_command, hw_buffer, CVMX_PKO_LOCK_NONE);
if (status == CVMX_PKO_SUCCESS) {
printf("Succesfully sent packet!\n");
cvmx_fpa_free(buf, packet_pool, 0);
}
}
void S3_send_packet(cvmx_wqe_t * work)
{
uint64_t port;
cvmx_buf_ptr_t packet_ptr;
cvmx_pko_command_word0_t pko_command;
/* Build a PKO pointer to this packet */
pko_command.u64 = 0;
/* Errata PKI-100 fix. We need to fix chain pointers on segmneted
packets. Although the size is also wrong on a single buffer packet,
PKO doesn't care so we ignore it */
if (cvmx_unlikely(work->word2.s.bufs > 1))
cvmx_helper_fix_ipd_packet_chain(work);
port = work->ipprt;
if( port >= portbase + portnum)
port = work->ipprt - portnum;
else
port = work->ipprt + portnum;
int queue = cvmx_pko_get_base_queue(port);
cvmx_pko_send_packet_prepare(port, queue, CVMX_PKO_LOCK_ATOMIC_TAG);
pko_command.s.total_bytes = work->len;
pko_command.s.segs = work->word2.s.bufs;
pko_command.s.ipoffp1 = 14 + 1;
packet_ptr = work->packet_ptr;
//cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 0);
cvm_common_free_fpa_buffer(work, CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE / CVMX_CACHE_LINE_SIZE);
work = NULL;
/*
* Send the packet and wait for the tag switch to complete before
* accessing the output queue. This ensures the locking required
* for the queue.
*
*/
if (cvmx_pko_send_packet_finish(port, queue, pko_command, packet_ptr, CVMX_PKO_LOCK_ATOMIC_TAG))
{
printf("Failed to send packet using cvmx_pko_send_packet_finish\
n");
}
}
void oct_tx_process_sw(mbuf_t *mbuf, uint8_t outport)
{
uint64_t queue;
cvmx_pko_return_value_t send_status;
uint8_t *dont_free_cookie = NULL;
queue = cvmx_pko_get_base_queue(outport);
cvmx_pko_send_packet_prepare(outport, queue, CVMX_PKO_LOCK_CMD_QUEUE);
tx_done_t *tx_done = &(oct_stx[LOCAL_CPU_ID]->tx_done[outport]);
if(tx_done->tx_entries < (OCT_PKO_TX_DESC_NUM - 1))
{
dont_free_cookie = oct_pend_tx_done_add(tx_done, (void *)mbuf);
}
else
{
PACKET_DESTROY_ALL(mbuf);
STAT_TX_SW_DESC_ERR;
return;
}
/*command word0*/
cvmx_pko_command_word0_t pko_command;
pko_command.u64 = 0;
pko_command.s.segs = 1;
pko_command.s.total_bytes = mbuf->pkt_totallen;
pko_command.s.rsp = 1;
pko_command.s.dontfree = 1;
/*command word1*/
cvmx_buf_ptr_t packet;
packet.u64 = 0;
packet.s.size = mbuf->pkt_totallen;
packet.s.addr = (uint64_t)mbuf->pkt_ptr;
/*command word2*/
cvmx_pko_command_word2_t tx_ptr_word;
tx_ptr_word.u64 = 0;
tx_ptr_word.s.ptr = (uint64_t)cvmx_ptr_to_phys(dont_free_cookie);
/* Send the packet */
send_status = cvmx_pko_send_packet_finish3(outport, queue, pko_command, packet, tx_ptr_word.u64, CVMX_PKO_LOCK_CMD_QUEUE);
if(send_status != CVMX_PKO_SUCCESS)
{
if(dont_free_cookie)
{
oct_pend_tx_done_remove(tx_done);
}
PACKET_DESTROY_ALL(mbuf);
STAT_TX_SW_SEND_ERR;
return;
}
else
{
STAT_TX_SEND_OVER;
}
}
/**
* Packet transmit
*
* @param m Packet to send
* @param dev Device info structure
* @return Always returns zero
*/
int cvm_oct_xmit(struct mbuf *m, struct ifnet *ifp)
{
cvmx_pko_command_word0_t pko_command;
cvmx_buf_ptr_t hw_buffer;
int dropped;
int qos;
cvm_oct_private_t *priv = (cvm_oct_private_t *)ifp->if_softc;
int32_t in_use;
int32_t buffers_to_free;
cvmx_wqe_t *work;
/* Prefetch the private data structure.
It is larger that one cache line */
CVMX_PREFETCH(priv, 0);
/* Start off assuming no drop */
dropped = 0;
/* The check on CVMX_PKO_QUEUES_PER_PORT_* is designed to completely
remove "qos" in the event neither interface supports multiple queues
per port */
if ((CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 > 1) ||
(CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 > 1)) {
qos = GET_MBUF_QOS(m);
if (qos <= 0)
qos = 0;
else if (qos >= cvmx_pko_get_num_queues(priv->port))
qos = 0;
} else
qos = 0;
/* The CN3XXX series of parts has an errata (GMX-401) which causes the
GMX block to hang if a collision occurs towards the end of a
<68 byte packet. As a workaround for this, we pad packets to be
68 bytes whenever we are in half duplex mode. We don't handle
the case of having a small packet but no room to add the padding.
The kernel should always give us at least a cache line */
if (__predict_false(m->m_pkthdr.len < 64) && OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
cvmx_gmxx_prtx_cfg_t gmx_prt_cfg;
int interface = INTERFACE(priv->port);
int index = INDEX(priv->port);
if (interface < 2) {
/* We only need to pad packet in half duplex mode */
gmx_prt_cfg.u64 = cvmx_read_csr(CVMX_GMXX_PRTX_CFG(index, interface));
if (gmx_prt_cfg.s.duplex == 0) {
static uint8_t pad[64];
if (!m_append(m, sizeof pad - m->m_pkthdr.len, pad))
printf("%s: unable to padd small packet.", __func__);
}
}
}
#ifdef OCTEON_VENDOR_RADISYS
/*
* The RSYS4GBE will hang if asked to transmit a packet less than 60 bytes.
*/
if (__predict_false(m->m_pkthdr.len < 60) &&
cvmx_sysinfo_get()->board_type == CVMX_BOARD_TYPE_CUST_RADISYS_RSYS4GBE) {
static uint8_t pad[60];
if (!m_append(m, sizeof pad - m->m_pkthdr.len, pad))
printf("%s: unable to pad small packet.", __func__);
}
#endif
/*
* If the packet is not fragmented.
*/
if (m->m_pkthdr.len == m->m_len) {
/* Build the PKO buffer pointer */
hw_buffer.u64 = 0;
hw_buffer.s.addr = cvmx_ptr_to_phys(m->m_data);
hw_buffer.s.pool = 0;
hw_buffer.s.size = m->m_len;
/* Build the PKO command */
pko_command.u64 = 0;
pko_command.s.segs = 1;
pko_command.s.dontfree = 1; /* Do not put this buffer into the FPA. */
work = NULL;
} else {
struct mbuf *n;
unsigned segs;
uint64_t *gp;
/*
* The packet is fragmented, we need to send a list of segments
* in memory we borrow from the WQE pool.
*/
work = cvmx_fpa_alloc(CVMX_FPA_WQE_POOL);
if (work == NULL) {
m_freem(m);
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
return 1;
}
segs = 0;
gp = (uint64_t *)work;
for (n = m; n != NULL; n = n->m_next) {
if (segs == CVMX_FPA_WQE_POOL_SIZE / sizeof (uint64_t))
panic("%s: too many segments in packet; call m_collapse().", __func__);
/* Build the PKO buffer pointer */
hw_buffer.u64 = 0;
hw_buffer.s.i = 1; /* Do not put this buffer into the FPA. */
hw_buffer.s.addr = cvmx_ptr_to_phys(n->m_data);
hw_buffer.s.pool = 0;
hw_buffer.s.size = n->m_len;
*gp++ = hw_buffer.u64;
segs++;
}
/* Build the PKO buffer gather list pointer */
hw_buffer.u64 = 0;
hw_buffer.s.addr = cvmx_ptr_to_phys(work);
hw_buffer.s.pool = CVMX_FPA_WQE_POOL;
hw_buffer.s.size = segs;
/* Build the PKO command */
pko_command.u64 = 0;
pko_command.s.segs = segs;
pko_command.s.gather = 1;
pko_command.s.dontfree = 0; /* Put the WQE above back into the FPA. */
}
/* Finish building the PKO command */
pko_command.s.n2 = 1; /* Don't pollute L2 with the outgoing packet */
pko_command.s.reg0 = priv->fau+qos*4;
pko_command.s.total_bytes = m->m_pkthdr.len;
pko_command.s.size0 = CVMX_FAU_OP_SIZE_32;
pko_command.s.subone0 = 1;
/* Check if we can use the hardware checksumming */
if ((m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) != 0) {
/* Use hardware checksum calc */
pko_command.s.ipoffp1 = ETHER_HDR_LEN + 1;
}
/*
* XXX
* Could use a different free queue (and different FAU address) per
* core instead of per QoS, to reduce contention here.
*/
IF_LOCK(&priv->tx_free_queue[qos]);
/* Get the number of mbufs in use by the hardware */
in_use = cvmx_fau_fetch_and_add32(priv->fau+qos*4, 1);
buffers_to_free = cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
cvmx_pko_send_packet_prepare(priv->port, priv->queue + qos, CVMX_PKO_LOCK_CMD_QUEUE);
/* Drop this packet if we have too many already queued to the HW */
if (_IF_QFULL(&priv->tx_free_queue[qos])) {
dropped = 1;
}
/* Send the packet to the output queue */
else
if (__predict_false(cvmx_pko_send_packet_finish(priv->port, priv->queue + qos, pko_command, hw_buffer, CVMX_PKO_LOCK_CMD_QUEUE))) {
DEBUGPRINT("%s: Failed to send the packet\n", if_name(ifp));
dropped = 1;
}
if (__predict_false(dropped)) {
m_freem(m);
cvmx_fau_atomic_add32(priv->fau+qos*4, -1);
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
} else {
/* Put this packet on the queue to be freed later */
_IF_ENQUEUE(&priv->tx_free_queue[qos], m);
/* Pass it to any BPF listeners. */
ETHER_BPF_MTAP(ifp, m);
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
if_inc_counter(ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len);
}
/* Free mbufs not in use by the hardware */
if (_IF_QLEN(&priv->tx_free_queue[qos]) > in_use) {
while (_IF_QLEN(&priv->tx_free_queue[qos]) > in_use) {
_IF_DEQUEUE(&priv->tx_free_queue[qos], m);
m_freem(m);
}
}
IF_UNLOCK(&priv->tx_free_queue[qos]);
return dropped;
}
void oct_tx_process_mbuf(mbuf_t *mbuf, uint8_t port)
{
uint64_t queue;
cvmx_pko_return_value_t send_status;
if(port > OCT_PHY_PORT_MAX)
{
printf("Send port is invalid");
PACKET_DESTROY_ALL(mbuf);
STAT_TX_SEND_PORT_ERR;
return;
}
queue = cvmx_pko_get_base_queue(port);
cvmx_pko_send_packet_prepare(port, queue, CVMX_PKO_LOCK_CMD_QUEUE);
if(PKTBUF_IS_HW(mbuf))
{
/* Build a PKO pointer to this packet */
cvmx_pko_command_word0_t pko_command;
pko_command.u64 = 0;
pko_command.s.segs = 1;
pko_command.s.total_bytes = mbuf->pkt_totallen;
/* Send the packet */
send_status = cvmx_pko_send_packet_finish(port, queue, pko_command, mbuf->packet_ptr, CVMX_PKO_LOCK_CMD_QUEUE);
if (send_status != CVMX_PKO_SUCCESS)
{
printf("Failed to send packet using cvmx_pko_send_packet2\n");
STAT_TX_HW_SEND_ERR;
PACKET_DESTROY_DATA(mbuf);
}
MBUF_FREE(mbuf);
}
else if(PKTBUF_IS_SW(mbuf))
{
uint8_t *dont_free_cookie = NULL;
tx_done_t *tx_done = &(oct_stx[local_cpu_id]->tx_done[port]);
if(tx_done->tx_entries < (OCT_PKO_TX_DESC_NUM - 1))
{
dont_free_cookie = oct_pend_tx_done_add(tx_done, (void *)mbuf);
}
else
{
PACKET_DESTROY_ALL(mbuf);
STAT_TX_SW_DESC_ERR;
return;
}
/*command word0*/
cvmx_pko_command_word0_t pko_command;
pko_command.u64 = 0;
pko_command.s.segs = 1;
pko_command.s.total_bytes = mbuf->pkt_totallen;
pko_command.s.rsp = 1;
pko_command.s.dontfree = 1;
/*command word1*/
cvmx_buf_ptr_t packet;
packet.u64 = 0;
packet.s.size = mbuf->pkt_totallen;
packet.s.addr = (uint64_t)mbuf->pkt_ptr;
/*command word2*/
cvmx_pko_command_word2_t tx_ptr_word;
tx_ptr_word.u64 = 0;
tx_ptr_word.s.ptr = (uint64_t)cvmx_ptr_to_phys(dont_free_cookie);
/* Send the packet */
send_status = cvmx_pko_send_packet_finish3(port, queue, pko_command, packet, tx_ptr_word.u64, CVMX_PKO_LOCK_CMD_QUEUE);
if(send_status != CVMX_PKO_SUCCESS)
{
if(dont_free_cookie)
{
oct_pend_tx_done_remove(tx_done);
}
printf("Failed to send packet using cvmx_pko_send_packet3\n");
PACKET_DESTROY_ALL(mbuf);
STAT_TX_SW_SEND_ERR;
return;
}
}
else
{
printf("pkt space %d is wrong, please check it\n", PKTBUF_SPACE_GET(mbuf));
}
STAT_TX_SEND_OVER;
}
