/**
* ipath_free_all_qps - remove all QPs from the table
* @qpt: the QP table to empty
*/
void ipath_free_all_qps(struct ipath_qp_table *qpt)
{
unsigned long flags;
struct ipath_qp *qp, *nqp;
u32 n;
for (n = 0; n < qpt->max; n++) {
spin_lock_irqsave(&qpt->lock, flags);
qp = qpt->table[n];
qpt->table[n] = NULL;
spin_unlock_irqrestore(&qpt->lock, flags);
while (qp) {
nqp = qp->next;
free_qpn(qpt, qp->ibqp.qp_num);
if (!atomic_dec_and_test(&qp->refcount) ||
!ipath_destroy_qp(&qp->ibqp))
ipath_dbg("QP memory leak!\n");
qp = nqp;
}
}
for (n = 0; n < ARRAY_SIZE(qpt->map); n++) {
if (qpt->map[n].page)
free_page((unsigned long)qpt->map[n].page);
}
}
void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc)
{
struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
ipath_dbg("Send queue error on QP%d/%d: err: %d\n",
qp->ibqp.qp_num, qp->remote_qpn, wc->status);
spin_lock(&dev->pending_lock);
/* XXX What if its already removed by the timeout code? */
if (!list_empty(&qp->timerwait))
list_del_init(&qp->timerwait);
if (!list_empty(&qp->piowait))
list_del_init(&qp->piowait);
spin_unlock(&dev->pending_lock);
ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
if (++qp->s_last >= qp->s_size)
qp->s_last = 0;
wc->status = IB_WC_WR_FLUSH_ERR;
while (qp->s_last != qp->s_head) {
wqe = get_swqe_ptr(qp, qp->s_last);
wc->wr_id = wqe->wr.wr_id;
wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
if (++qp->s_last >= qp->s_size)
qp->s_last = 0;
}
qp->s_cur = qp->s_tail = qp->s_head;
qp->state = IB_QPS_SQE;
}
/**
* ipath_qcheck - print delta of egrfull/hdrqfull errors for kernel ports
* @dd: the infinipath device
*
* print the delta of egrfull/hdrqfull errors for kernel ports no more than
* every 5 seconds. User processes are printed at close, but kernel doesn't
* close, so... Separate routine so may call from other places someday, and
* so function name when printed by _IPATH_INFO is meaningfull
*/
static void ipath_qcheck(struct ipath_devdata *dd)
{
static u64 last_tot_hdrqfull;
size_t blen = 0;
char buf[128];
*buf = 0;
if (dd->ipath_pd[0]->port_hdrqfull != dd->ipath_p0_hdrqfull) {
blen = snprintf(buf, sizeof buf, "port 0 hdrqfull %u",
dd->ipath_pd[0]->port_hdrqfull -
dd->ipath_p0_hdrqfull);
dd->ipath_p0_hdrqfull = dd->ipath_pd[0]->port_hdrqfull;
}
if (ipath_stats.sps_etidfull != dd->ipath_last_tidfull) {
blen += snprintf(buf + blen, sizeof buf - blen,
"%srcvegrfull %llu",
blen ? ", " : "",
(unsigned long long)
(ipath_stats.sps_etidfull -
dd->ipath_last_tidfull));
dd->ipath_last_tidfull = ipath_stats.sps_etidfull;
}
/*
* this is actually the number of hdrq full interrupts, not actual
* events, but at the moment that's mostly what I'm interested in.
* Actual count, etc. is in the counters, if needed. For production
* users this won't ordinarily be printed.
*/
if ((ipath_debug & (__IPATH_PKTDBG | __IPATH_DBG)) &&
ipath_stats.sps_hdrqfull != last_tot_hdrqfull) {
blen += snprintf(buf + blen, sizeof buf - blen,
"%shdrqfull %llu (all ports)",
blen ? ", " : "",
(unsigned long long)
(ipath_stats.sps_hdrqfull -
last_tot_hdrqfull));
last_tot_hdrqfull = ipath_stats.sps_hdrqfull;
}
if (blen)
ipath_dbg("%s\n", buf);
if (dd->ipath_port0head != (u32)
le64_to_cpu(*dd->ipath_hdrqtailptr)) {
if (dd->ipath_lastport0rcv_cnt ==
ipath_stats.sps_port0pkts) {
ipath_cdbg(PKT, "missing rcv interrupts? "
"port0 hd=%llx tl=%x; port0pkts %llx\n",
(unsigned long long)
le64_to_cpu(*dd->ipath_hdrqtailptr),
dd->ipath_port0head,
(unsigned long long)
ipath_stats.sps_port0pkts);
}
dd->ipath_lastport0rcv_cnt = ipath_stats.sps_port0pkts;
}
}
static void ipath_chk_errormask(struct ipath_devdata *dd)
{
static u32 fixed;
u32 ctrl;
unsigned long errormask;
unsigned long hwerrs;
if (!dd->ipath_errormask || !(dd->ipath_flags & IPATH_INITTED))
return;
errormask = ipath_read_kreg64(dd, dd->ipath_kregs->kr_errormask);
if (errormask == dd->ipath_errormask)
return;
fixed++;
hwerrs = ipath_read_kreg64(dd, dd->ipath_kregs->kr_hwerrstatus);
ctrl = ipath_read_kreg32(dd, dd->ipath_kregs->kr_control);
ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
dd->ipath_errormask);
if ((hwerrs & dd->ipath_hwerrmask) ||
(ctrl & INFINIPATH_C_FREEZEMODE)) {
/* force re-interrupt of pending events, just in case */
ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear, 0ULL);
ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear, 0ULL);
ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, 0ULL);
dev_info(&dd->pcidev->dev,
"errormask fixed(%u) %lx -> %lx, ctrl %x hwerr %lx\n",
fixed, errormask, (unsigned long)dd->ipath_errormask,
ctrl, hwerrs);
} else
ipath_dbg("errormask fixed(%u) %lx -> %lx, no freeze\n",
fixed, errormask,
(unsigned long)dd->ipath_errormask);
}
/**
* ipath_get_faststats - get word counters from chip before they overflow
* @opaque - contains a pointer to the infinipath device ipath_devdata
*
* called from add_timer
*/
void ipath_get_faststats(unsigned long opaque)
{
struct ipath_devdata *dd = (struct ipath_devdata *) opaque;
u32 val;
static unsigned cnt;
unsigned long flags;
/*
* don't access the chip while running diags, or memory diags can
* fail
*/
if (!dd->ipath_kregbase || !(dd->ipath_flags & IPATH_INITTED) ||
ipath_diag_inuse)
/* but re-arm the timer, for diags case; won't hurt other */
goto done;
/*
* We now try to maintain a "active timer", based on traffic
* exceeding a threshold, so we need to check the word-counts
* even if they are 64-bit.
*/
ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordsendcnt);
ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordrcvcnt);
spin_lock_irqsave(&dd->ipath_eep_st_lock, flags);
if (dd->ipath_traffic_wds >= IPATH_TRAFFIC_ACTIVE_THRESHOLD)
atomic_add(5, &dd->ipath_active_time); /* S/B #define */
dd->ipath_traffic_wds = 0;
spin_unlock_irqrestore(&dd->ipath_eep_st_lock, flags);
if (dd->ipath_flags & IPATH_32BITCOUNTERS) {
ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktsendcnt);
ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktrcvcnt);
}
ipath_qcheck(dd);
/*
* deal with repeat error suppression. Doesn't really matter if
* last error was almost a full interval ago, or just a few usecs
* ago; still won't get more than 2 per interval. We may want
* longer intervals for this eventually, could do with mod, counter
* or separate timer. Also see code in ipath_handle_errors() and
* ipath_handle_hwerrors().
*/
if (dd->ipath_lasterror)
dd->ipath_lasterror = 0;
if (dd->ipath_lasthwerror)
dd->ipath_lasthwerror = 0;
if (dd->ipath_maskederrs
&& time_after(jiffies, dd->ipath_unmasktime)) {
char ebuf[256];
int iserr;
iserr = ipath_decode_err(ebuf, sizeof ebuf,
dd->ipath_maskederrs);
if (dd->ipath_maskederrs &
~(INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL |
INFINIPATH_E_PKTERRS ))
ipath_dev_err(dd, "Re-enabling masked errors "
"(%s)\n", ebuf);
else {
/*
* rcvegrfull and rcvhdrqfull are "normal", for some
* types of processes (mostly benchmarks) that send
* huge numbers of messages, while not processing
* them. So only complain about these at debug
* level.
*/
if (iserr)
ipath_dbg("Re-enabling queue full errors (%s)\n",
ebuf);
else
ipath_cdbg(ERRPKT, "Re-enabling packet"
" problem interrupt (%s)\n", ebuf);
}
/* re-enable masked errors */
dd->ipath_errormask |= dd->ipath_maskederrs;
ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
dd->ipath_errormask);
dd->ipath_maskederrs = 0;
}
/* limit qfull messages to ~one per minute per port */
if ((++cnt & 0x10)) {
for (val = dd->ipath_cfgports - 1; ((int)val) >= 0;
val--) {
if (dd->ipath_lastegrheads[val] != -1)
dd->ipath_lastegrheads[val] = -1;
if (dd->ipath_lastrcvhdrqtails[val] != -1)
dd->ipath_lastrcvhdrqtails[val] = -1;
}
}
ipath_chk_errormask(dd);
done:
mod_timer(&dd->ipath_stats_timer, jiffies + HZ * 5);
}
/**
* ipath_diagpkt_write - write an IB packet
* @fp: the diag data device file pointer
* @data: ipath_diag_pkt structure saying where to get the packet
* @count: size of data to write
* @off: unused by this code
*/
static ssize_t ipath_diagpkt_write(struct file *fp,
const char __user *data,
size_t count, loff_t *off)
{
u32 __iomem *piobuf;
u32 plen, clen, pbufn;
struct ipath_diag_pkt dp;
u32 *tmpbuf = NULL;
struct ipath_devdata *dd;
ssize_t ret = 0;
u64 val;
if (count < sizeof(dp)) {
ret = -EINVAL;
goto bail;
}
if (copy_from_user(&dp, data, sizeof(dp))) {
ret = -EFAULT;
goto bail;
}
/* send count must be an exact number of dwords */
if (dp.len & 3) {
ret = -EINVAL;
goto bail;
}
clen = dp.len >> 2;
dd = ipath_lookup(dp.unit);
if (!dd || !(dd->ipath_flags & IPATH_PRESENT) ||
!dd->ipath_kregbase) {
ipath_cdbg(VERBOSE, "illegal unit %u for diag data send\n",
dp.unit);
ret = -ENODEV;
goto bail;
}
if (ipath_diag_inuse && !diag_set_link &&
!(dd->ipath_flags & IPATH_LINKACTIVE)) {
diag_set_link = 1;
ipath_cdbg(VERBOSE, "Trying to set to set link active for "
"diag pkt\n");
ipath_set_linkstate(dd, IPATH_IB_LINKARM);
ipath_set_linkstate(dd, IPATH_IB_LINKACTIVE);
}
if (!(dd->ipath_flags & IPATH_INITTED)) {
/* no hardware, freeze, etc. */
ipath_cdbg(VERBOSE, "unit %u not usable\n", dd->ipath_unit);
ret = -ENODEV;
goto bail;
}
val = dd->ipath_lastibcstat & IPATH_IBSTATE_MASK;
if (val != IPATH_IBSTATE_INIT && val != IPATH_IBSTATE_ARM &&
val != IPATH_IBSTATE_ACTIVE) {
ipath_cdbg(VERBOSE, "unit %u not ready (state %llx)\n",
dd->ipath_unit, (unsigned long long) val);
ret = -EINVAL;
goto bail;
}
/* need total length before first word written */
/* +1 word is for the qword padding */
plen = sizeof(u32) + dp.len;
if ((plen + 4) > dd->ipath_ibmaxlen) {
ipath_dbg("Pkt len 0x%x > ibmaxlen %x\n",
plen - 4, dd->ipath_ibmaxlen);
ret = -EINVAL;
goto bail; /* before writing pbc */
}
tmpbuf = vmalloc(plen);
if (!tmpbuf) {
dev_info(&dd->pcidev->dev, "Unable to allocate tmp buffer, "
"failing\n");
ret = -ENOMEM;
goto bail;
}
if (copy_from_user(tmpbuf,
(const void __user *) (unsigned long) dp.data,
dp.len)) {
ret = -EFAULT;
goto bail;
}
piobuf = ipath_getpiobuf(dd, &pbufn);
if (!piobuf) {
ipath_cdbg(VERBOSE, "No PIO buffers avail unit for %u\n",
dd->ipath_unit);
ret = -EBUSY;
goto bail;
}
plen >>= 2; /* in dwords */
if (ipath_debug & __IPATH_PKTDBG)
ipath_cdbg(VERBOSE, "unit %u 0x%x+1w pio%d\n",
dd->ipath_unit, plen - 1, pbufn);
/* we have to flush after the PBC for correctness on some cpus
* or WC buffer can be written out of order */
writeq(plen, piobuf);
ipath_flush_wc();
/* copy all by the trigger word, then flush, so it's written
* to chip before trigger word, then write trigger word, then
* flush again, so packet is sent. */
__iowrite32_copy(piobuf + 2, tmpbuf, clen - 1);
ipath_flush_wc();
__raw_writel(tmpbuf[clen - 1], piobuf + clen + 1);
ipath_flush_wc();
ret = sizeof(dp);
bail:
vfree(tmpbuf);
return ret;
}
/* Transfer date to/from uC Program RAM of IB or PCIe SerDes */
static int ipath_sd7220_ram_xfer(struct ipath_devdata *dd, int sdnum, u32 loc,
u8 *buf, int cnt, int rd_notwr)
{
u16 trans;
u64 transval;
u64 csbit;
int owned;
int tries;
int sofar;
int addr;
int ret;
unsigned long flags;
const char *op;
/* Pick appropriate transaction reg and "Chip select" for this serdes */
switch (sdnum) {
case IB_7220_SERDES :
csbit = 1ULL << EPB_IB_UC_CS_SHF;
trans = dd->ipath_kregs->kr_ib_epbtrans;
break;
case PCIE_SERDES0 :
case PCIE_SERDES1 :
/* PCIe SERDES has uC "chip select" in different bit, too */
csbit = 1ULL << EPB_PCIE_UC_CS_SHF;
trans = dd->ipath_kregs->kr_pcie_epbtrans;
break;
default :
return -1;
}
op = rd_notwr ? "Rd" : "Wr";
spin_lock_irqsave(&dd->ipath_sdepb_lock, flags);
owned = epb_access(dd, sdnum, 1);
if (owned < 0) {
spin_unlock_irqrestore(&dd->ipath_sdepb_lock, flags);
ipath_dbg("Could not get %s access to %s EPB: %X, loc %X\n",
op, (sdnum == IB_7220_SERDES) ? "IB" : "PCIe",
owned, loc);
return -1;
}
/*
* In future code, we may need to distinguish several address ranges,
* and select various memories based on this. For now, just trim
* "loc" (location including address and memory select) to
* "addr" (address within memory). we will only support PRAM
* The memory is 8KB.
*/
addr = loc & 0x1FFF;
for (tries = EPB_TRANS_TRIES; tries; --tries) {
transval = ipath_read_kreg32(dd, trans);
if (transval & EPB_TRANS_RDY)
break;
udelay(5);
}
sofar = 0;
if (tries <= 0)
ipath_dbg("No initial RDY on EPB access request\n");
else {
/*
* Every "memory" access is doubly-indirect.
* We set two bytes of address, then read/write
* one or mores bytes of data.
*/
/* First, we set control to "Read" or "Write" */
transval = csbit | EPB_UC_CTL |
(rd_notwr ? EPB_ROM_R : EPB_ROM_W);
tries = epb_trans(dd, trans, transval, &transval);
if (tries <= 0)
ipath_dbg("No EPB response to uC %s cmd\n", op);
while (tries > 0 && sofar < cnt) {
if (!sofar) {
/* Only set address at start of chunk */
int addrbyte = (addr + sofar) >> 8;
transval = csbit | EPB_MADDRH | addrbyte;
tries = epb_trans(dd, trans, transval,
&transval);
if (tries <= 0) {
ipath_dbg("No EPB response ADDRH\n");
break;
}
addrbyte = (addr + sofar) & 0xFF;
transval = csbit | EPB_MADDRL | addrbyte;
tries = epb_trans(dd, trans, transval,
&transval);
if (tries <= 0) {
ipath_dbg("No EPB response ADDRL\n");
break;
}
}
if (rd_notwr)
transval = csbit | EPB_ROMDATA | EPB_RD;
else
transval = csbit | EPB_ROMDATA | buf[sofar];
tries = epb_trans(dd, trans, transval, &transval);
if (tries <= 0) {
ipath_dbg("No EPB response DATA\n");
break;
}
if (rd_notwr)
buf[sofar] = transval & EPB_DATA_MASK;
++sofar;
}
/* Finally, clear control-bit for Read or Write */
transval = csbit | EPB_UC_CTL;
tries = epb_trans(dd, trans, transval, &transval);
if (tries <= 0)
ipath_dbg("No EPB response to drop of uC %s cmd\n", op);
}
/*
* Below is portion of IBA7220-specific bringup_serdes() that actually
* deals with registers and memory within the SerDes itself.
* Post IB uC code version 1.32.17, was_reset being 1 is not really
* informative, so we double-check.
*/
int ipath_sd7220_init(struct ipath_devdata *dd, int was_reset)
{
int ret = 1; /* default to failure */
int first_reset;
int val_stat;
if (!was_reset) {
/* entered with reset not asserted, we need to do it */
ipath_ibsd_reset(dd, 1);
ipath_sd_trimdone_monitor(dd, "Driver-reload");
}
/* Substitute our deduced value for was_reset */
ret = ipath_ibsd_ucode_loaded(dd);
if (ret < 0) {
ret = 1;
goto done;
}
first_reset = !ret; /* First reset if IBSD uCode not yet loaded */
/*
* Alter some regs per vendor latest doc, reset-defaults
* are not right for IB.
*/
ret = ipath_sd_early(dd);
if (ret < 0) {
ipath_dev_err(dd, "Failed to set IB SERDES early defaults\n");
ret = 1;
goto done;
}
/*
* Set DAC manual trim IB.
* We only do this once after chip has been reset (usually
* same as once per system boot).
*/
if (first_reset) {
ret = ipath_sd_dactrim(dd);
if (ret < 0) {
ipath_dev_err(dd, "Failed IB SERDES DAC trim\n");
ret = 1;
goto done;
}
}
/*
* Set various registers (DDS and RXEQ) that will be
* controlled by IBC (in 1.2 mode) to reasonable preset values
* Calling the "internal" version avoids the "check for needed"
* and "trimdone monitor" that might be counter-productive.
*/
ret = ipath_internal_presets(dd);
if (ret < 0) {
ipath_dev_err(dd, "Failed to set IB SERDES presets\n");
ret = 1;
goto done;
}
ret = ipath_sd_trimself(dd, 0x80);
if (ret < 0) {
ipath_dev_err(dd, "Failed to set IB SERDES TRIMSELF\n");
ret = 1;
goto done;
}
/* Load image, then try to verify */
ret = 0; /* Assume success */
if (first_reset) {
int vfy;
int trim_done;
ipath_dbg("SerDes uC was reset, reloading PRAM\n");
ret = ipath_sd7220_ib_load(dd);
if (ret < 0) {
ipath_dev_err(dd, "Failed to load IB SERDES image\n");
ret = 1;
goto done;
}
/* Loaded image, try to verify */
vfy = ipath_sd7220_ib_vfy(dd);
if (vfy != ret) {
ipath_dev_err(dd, "SERDES PRAM VFY failed\n");
ret = 1;
goto done;
}
/*
* Loaded and verified. Almost good...
* hold "success" in ret
*/
ret = 0;
/*
* Prev steps all worked, continue bringup
* De-assert RESET to uC, only in first reset, to allow
* trimming.
*
* Since our default setup sets START_EQ1 to
* PRESET, we need to clear that for this very first run.
*/
ret = ibsd_mod_allchnls(dd, START_EQ1(0), 0, 0x38);
if (ret < 0) {
ipath_dev_err(dd, "Failed clearing START_EQ1\n");
ret = 1;
goto done;
}
ipath_ibsd_reset(dd, 0);
/*
* If this is not the first reset, trimdone should be set
* already.
*/
trim_done = ipath_sd_trimdone_poll(dd);
/*
* Whether or not trimdone succeeded, we need to put the
* uC back into reset to avoid a possible fight with the
* IBC state-machine.
*/
ipath_ibsd_reset(dd, 1);
if (!trim_done) {
ipath_dev_err(dd, "No TRIMDONE seen\n");
ret = 1;
goto done;
}
ipath_sd_trimdone_monitor(dd, "First-reset");
/* Remember so we do not re-do the load, dactrim, etc. */
dd->serdes_first_init_done = 1;
}
/*
* Setup for channel training and load values for
* RxEq and DDS in tables used by IBC in IB1.2 mode
*/
val_stat = ipath_sd_setvals(dd);
if (val_stat < 0)
ret = 1;
done:
/* start relock timer regardless, but start at 1 second */
ipath_set_relock_poll(dd, -1);
return ret;
}
/**
* ipath_diagpkt_write - write an IB packet
* @fp: the diag data device file pointer
* @data: ipath_diag_pkt structure saying where to get the packet
* @count: size of data to write
* @off: unused by this code
*/
static ssize_t ipath_diagpkt_write(struct file *fp,
const char __user *data,
size_t count, loff_t *off)
{
u32 __iomem *piobuf;
u32 plen, clen, pbufn;
struct ipath_diag_pkt odp;
struct ipath_diag_xpkt dp;
u32 *tmpbuf = NULL;
struct ipath_devdata *dd;
ssize_t ret = 0;
u64 val;
u32 l_state, lt_state; /* LinkState, LinkTrainingState */
if (count < sizeof(odp)) {
ret = -EINVAL;
goto bail;
}
if (count == sizeof(dp)) {
if (copy_from_user(&dp, data, sizeof(dp))) {
ret = -EFAULT;
goto bail;
}
} else if (copy_from_user(&odp, data, sizeof(odp))) {
ret = -EFAULT;
goto bail;
}
/*
* Due to padding/alignment issues (lessened with new struct)
* the old and new structs are the same length. We need to
* disambiguate them, which we can do because odp.len has never
* been less than the total of LRH+BTH+DETH so far, while
* dp.unit (same offset) unit is unlikely to get that high.
* Similarly, dp.data, the pointer to user at the same offset
* as odp.unit, is almost certainly at least one (512byte)page
* "above" NULL. The if-block below can be omitted if compatibility
* between a new driver and older diagnostic code is unimportant.
* compatibility the other direction (new diags, old driver) is
* handled in the diagnostic code, with a warning.
*/
if (dp.unit >= 20 && dp.data < 512) {
/* very probable version mismatch. Fix it up */
memcpy(&odp, &dp, sizeof(odp));
/* We got a legacy dp, copy elements to dp */
dp.unit = odp.unit;
dp.data = odp.data;
dp.len = odp.len;
dp.pbc_wd = 0; /* Indicate we need to compute PBC wd */
}
/* send count must be an exact number of dwords */
if (dp.len & 3) {
ret = -EINVAL;
goto bail;
}
clen = dp.len >> 2;
dd = ipath_lookup(dp.unit);
if (!dd || !(dd->ipath_flags & IPATH_PRESENT) ||
!dd->ipath_kregbase) {
ipath_cdbg(VERBOSE, "illegal unit %u for diag data send\n",
dp.unit);
ret = -ENODEV;
goto bail;
}
if (ipath_diag_inuse && !diag_set_link &&
!(dd->ipath_flags & IPATH_LINKACTIVE)) {
diag_set_link = 1;
ipath_cdbg(VERBOSE, "Trying to set to set link active for "
"diag pkt\n");
ipath_set_linkstate(dd, IPATH_IB_LINKARM);
ipath_set_linkstate(dd, IPATH_IB_LINKACTIVE);
}
if (!(dd->ipath_flags & IPATH_INITTED)) {
/* no hardware, freeze, etc. */
ipath_cdbg(VERBOSE, "unit %u not usable\n", dd->ipath_unit);
ret = -ENODEV;
goto bail;
}
/*
* Want to skip check for l_state if using custom PBC,
* because we might be trying to force an SM packet out.
* first-cut, skip _all_ state checking in that case.
*/
val = ipath_ib_state(dd, dd->ipath_lastibcstat);
lt_state = ipath_ib_linktrstate(dd, dd->ipath_lastibcstat);
l_state = ipath_ib_linkstate(dd, dd->ipath_lastibcstat);
if (!dp.pbc_wd && (lt_state != INFINIPATH_IBCS_LT_STATE_LINKUP ||
(val != dd->ib_init && val != dd->ib_arm &&
val != dd->ib_active))) {
ipath_cdbg(VERBOSE, "unit %u not ready (state %llx)\n",
dd->ipath_unit, (unsigned long long) val);
ret = -EINVAL;
goto bail;
}
/* need total length before first word written */
/* +1 word is for the qword padding */
plen = sizeof(u32) + dp.len;
if ((plen + 4) > dd->ipath_ibmaxlen) {
ipath_dbg("Pkt len 0x%x > ibmaxlen %x\n",
plen - 4, dd->ipath_ibmaxlen);
ret = -EINVAL;
goto bail; /* before writing pbc */
}
tmpbuf = vmalloc(plen);
if (!tmpbuf) {
dev_info(&dd->pcidev->dev, "Unable to allocate tmp buffer, "
"failing\n");
ret = -ENOMEM;
goto bail;
}
if (copy_from_user(tmpbuf,
(const void __user *) (unsigned long) dp.data,
dp.len)) {
ret = -EFAULT;
goto bail;
}
plen >>= 2; /* in dwords */
piobuf = ipath_getpiobuf(dd, plen, &pbufn);
if (!piobuf) {
ipath_cdbg(VERBOSE, "No PIO buffers avail unit for %u\n",
dd->ipath_unit);
ret = -EBUSY;
goto bail;
}
/* disarm it just to be extra sure */
ipath_disarm_piobufs(dd, pbufn, 1);
if (ipath_debug & __IPATH_PKTDBG)
ipath_cdbg(VERBOSE, "unit %u 0x%x+1w pio%d\n",
dd->ipath_unit, plen - 1, pbufn);
if (dp.pbc_wd == 0)
dp.pbc_wd = plen;
writeq(dp.pbc_wd, piobuf);
/*
* Copy all by the trigger word, then flush, so it's written
* to chip before trigger word, then write trigger word, then
* flush again, so packet is sent.
*/
if (dd->ipath_flags & IPATH_PIO_FLUSH_WC) {
ipath_flush_wc();
__iowrite32_copy(piobuf + 2, tmpbuf, clen - 1);
ipath_flush_wc();
__raw_writel(tmpbuf[clen - 1], piobuf + clen + 1);
} else
__iowrite32_copy(piobuf + 2, tmpbuf, clen);
ipath_flush_wc();
ret = sizeof(dp);
bail:
vfree(tmpbuf);
return ret;
}
int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err)
{
struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
struct ib_wc wc;
int ret = 0;
ipath_dbg("QP%d/%d in error state (%d)\n",
qp->ibqp.qp_num, qp->remote_qpn, err);
spin_lock(&dev->pending_lock);
/* XXX What if its already removed by the timeout code? */
if (!list_empty(&qp->timerwait))
list_del_init(&qp->timerwait);
if (!list_empty(&qp->piowait))
list_del_init(&qp->piowait);
spin_unlock(&dev->pending_lock);
wc.vendor_err = 0;
wc.byte_len = 0;
wc.imm_data = 0;
wc.qp = &qp->ibqp;
wc.src_qp = 0;
wc.wc_flags = 0;
wc.pkey_index = 0;
wc.slid = 0;
wc.sl = 0;
wc.dlid_path_bits = 0;
wc.port_num = 0;
if (qp->r_wrid_valid) {
qp->r_wrid_valid = 0;
wc.wr_id = qp->r_wr_id;
wc.opcode = IB_WC_RECV;
wc.status = err;
ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
}
wc.status = IB_WC_WR_FLUSH_ERR;
while (qp->s_last != qp->s_head) {
struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
wc.wr_id = wqe->wr.wr_id;
wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
if (++qp->s_last >= qp->s_size)
qp->s_last = 0;
ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 1);
}
qp->s_cur = qp->s_tail = qp->s_head;
qp->s_hdrwords = 0;
qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
if (qp->r_rq.wq) {
struct ipath_rwq *wq;
u32 head;
u32 tail;
spin_lock(&qp->r_rq.lock);
/* sanity check pointers before trusting them */
wq = qp->r_rq.wq;
head = wq->head;
if (head >= qp->r_rq.size)
head = 0;
tail = wq->tail;
if (tail >= qp->r_rq.size)
tail = 0;
wc.opcode = IB_WC_RECV;
while (tail != head) {
wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;
if (++tail >= qp->r_rq.size)
tail = 0;
ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
}
wq->tail = tail;
spin_unlock(&qp->r_rq.lock);
} else if (qp->ibqp.event_handler)
ret = 1;
return ret;
}
/**
* ipath_diagpkt_write - write an IB packet
* @fp: the diag data device file pointer
* @data: ipath_diag_pkt structure saying where to get the packet
* @count: size of data to write
* @off: unused by this code
*/
static ssize_t ipath_diagpkt_write(struct file *fp,
const char __user *data,
size_t count, loff_t *off)
{
u32 __iomem *piobuf;
u32 plen, pbufn, maxlen_reserve;
struct ipath_diag_pkt odp;
struct ipath_diag_xpkt dp;
u32 *tmpbuf = NULL;
struct ipath_devdata *dd;
ssize_t ret = 0;
u64 val;
u32 l_state, lt_state; /* LinkState, LinkTrainingState */
if (count == sizeof(dp)) {
if (copy_from_user(&dp, data, sizeof(dp))) {
ret = -EFAULT;
goto bail;
}
} else if (count == sizeof(odp)) {
if (copy_from_user(&odp, data, sizeof(odp))) {
ret = -EFAULT;
goto bail;
}
} else {
ret = -EINVAL;
goto bail;
}
/* send count must be an exact number of dwords */
if (dp.len & 3) {
ret = -EINVAL;
goto bail;
}
plen = dp.len >> 2;
dd = ipath_lookup(dp.unit);
if (!dd || !(dd->ipath_flags & IPATH_PRESENT) ||
!dd->ipath_kregbase) {
ipath_cdbg(VERBOSE, "illegal unit %u for diag data send\n",
dp.unit);
ret = -ENODEV;
goto bail;
}
if (ipath_diag_inuse && !diag_set_link &&
!(dd->ipath_flags & IPATH_LINKACTIVE)) {
diag_set_link = 1;
ipath_cdbg(VERBOSE, "Trying to set to set link active for "
"diag pkt\n");
ipath_set_linkstate(dd, IPATH_IB_LINKARM);
ipath_set_linkstate(dd, IPATH_IB_LINKACTIVE);
}
if (!(dd->ipath_flags & IPATH_INITTED)) {
/* no hardware, freeze, etc. */
ipath_cdbg(VERBOSE, "unit %u not usable\n", dd->ipath_unit);
ret = -ENODEV;
goto bail;
}
/*
* Want to skip check for l_state if using custom PBC,
* because we might be trying to force an SM packet out.
* first-cut, skip _all_ state checking in that case.
*/
val = ipath_ib_state(dd, dd->ipath_lastibcstat);
lt_state = ipath_ib_linktrstate(dd, dd->ipath_lastibcstat);
l_state = ipath_ib_linkstate(dd, dd->ipath_lastibcstat);
if (!dp.pbc_wd && (lt_state != INFINIPATH_IBCS_LT_STATE_LINKUP ||
(val != dd->ib_init && val != dd->ib_arm &&
val != dd->ib_active))) {
ipath_cdbg(VERBOSE, "unit %u not ready (state %llx)\n",
dd->ipath_unit, (unsigned long long) val);
ret = -EINVAL;
goto bail;
}
/*
* need total length before first word written, plus 2 Dwords. One Dword
* is for padding so we get the full user data when not aligned on
* a word boundary. The other Dword is to make sure we have room for the
* ICRC which gets tacked on later.
*/
maxlen_reserve = 2 * sizeof(u32);
if (dp.len > dd->ipath_ibmaxlen - maxlen_reserve) {
ipath_dbg("Pkt len 0x%x > ibmaxlen %x\n",
dp.len, dd->ipath_ibmaxlen);
ret = -EINVAL;
goto bail;
}
plen = sizeof(u32) + dp.len;
tmpbuf = vmalloc(plen);
if (!tmpbuf) {
dev_info(&dd->pcidev->dev, "Unable to allocate tmp buffer, "
"failing\n");
ret = -ENOMEM;
goto bail;
}
if (copy_from_user(tmpbuf,
(const void __user *) (unsigned long) dp.data,
dp.len)) {
ret = -EFAULT;
goto bail;
}
plen >>= 2; /* in dwords */
piobuf = ipath_getpiobuf(dd, plen, &pbufn);
if (!piobuf) {
ipath_cdbg(VERBOSE, "No PIO buffers avail unit for %u\n",
dd->ipath_unit);
ret = -EBUSY;
goto bail;
}
/* disarm it just to be extra sure */
ipath_disarm_piobufs(dd, pbufn, 1);
if (ipath_debug & __IPATH_PKTDBG)
ipath_cdbg(VERBOSE, "unit %u 0x%x+1w pio%d\n",
dd->ipath_unit, plen - 1, pbufn);
if (dp.pbc_wd == 0)
dp.pbc_wd = plen;
writeq(dp.pbc_wd, piobuf);
/*
* Copy all by the trigger word, then flush, so it's written
* to chip before trigger word, then write trigger word, then
* flush again, so packet is sent.
*/
if (dd->ipath_flags & IPATH_PIO_FLUSH_WC) {
ipath_flush_wc();
__iowrite32_copy(piobuf + 2, tmpbuf, plen - 1);
ipath_flush_wc();
__raw_writel(tmpbuf[plen - 1], piobuf + plen + 1);
} else
__iowrite32_copy(piobuf + 2, tmpbuf, plen);
ipath_flush_wc();
ret = sizeof(dp);
bail:
vfree(tmpbuf);
return ret;
}
/**
* ipath_get_faststats - get word counters from chip before they overflow
* @opaque - contains a pointer to the infinipath device ipath_devdata
*
* called from add_timer
*/
void ipath_get_faststats(unsigned long opaque)
{
struct ipath_devdata *dd = (struct ipath_devdata *) opaque;
u32 val;
static unsigned cnt;
/*
* don't access the chip while running diags, or memory diags can
* fail
*/
if (!dd->ipath_kregbase || !(dd->ipath_flags & IPATH_PRESENT) ||
ipath_diag_inuse)
/* but re-arm the timer, for diags case; won't hurt other */
goto done;
if (dd->ipath_flags & IPATH_32BITCOUNTERS) {
ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordsendcnt);
ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordrcvcnt);
ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktsendcnt);
ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktrcvcnt);
}
ipath_qcheck(dd);
/*
* deal with repeat error suppression. Doesn't really matter if
* last error was almost a full interval ago, or just a few usecs
* ago; still won't get more than 2 per interval. We may want
* longer intervals for this eventually, could do with mod, counter
* or separate timer. Also see code in ipath_handle_errors() and
* ipath_handle_hwerrors().
*/
if (dd->ipath_lasterror)
dd->ipath_lasterror = 0;
if (dd->ipath_lasthwerror)
dd->ipath_lasthwerror = 0;
if ((dd->ipath_maskederrs & ~dd->ipath_ignorederrs)
&& time_after(jiffies, dd->ipath_unmasktime)) {
char ebuf[256];
ipath_decode_err(ebuf, sizeof ebuf,
(dd->ipath_maskederrs & ~dd->
ipath_ignorederrs));
if ((dd->ipath_maskederrs & ~dd->ipath_ignorederrs) &
~(INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL))
ipath_dev_err(dd, "Re-enabling masked errors "
"(%s)\n", ebuf);
else {
/*
* rcvegrfull and rcvhdrqfull are "normal", for some
* types of processes (mostly benchmarks) that send
* huge numbers of messages, while not processing
* them. So only complain about these at debug
* level.
*/
ipath_dbg("Disabling frequent queue full errors "
"(%s)\n", ebuf);
}
dd->ipath_maskederrs = dd->ipath_ignorederrs;
ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
~dd->ipath_maskederrs);
}
/* limit qfull messages to ~one per minute per port */
if ((++cnt & 0x10)) {
for (val = dd->ipath_cfgports - 1; ((int)val) >= 0;
val--) {
if (dd->ipath_lastegrheads[val] != -1)
dd->ipath_lastegrheads[val] = -1;
if (dd->ipath_lastrcvhdrqtails[val] != -1)
dd->ipath_lastrcvhdrqtails[val] = -1;
}
}
if (dd->ipath_nosma_bufs) {
dd->ipath_nosma_secs += 5;
if (dd->ipath_nosma_secs >= 30) {
ipath_cdbg(SMA, "No SMA bufs avail %u seconds; "
"cancelling pending sends\n",
dd->ipath_nosma_secs);
/*
* issue an abort as well, in case we have a packet
* stuck in launch fifo. This could corrupt an
* outgoing user packet in the worst case,
* but this is a pretty catastrophic, anyway.
*/
ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
INFINIPATH_S_ABORT);
ipath_disarm_piobufs(dd, dd->ipath_lastport_piobuf,
dd->ipath_piobcnt2k +
dd->ipath_piobcnt4k -
dd->ipath_lastport_piobuf);
/* start again, if necessary */
dd->ipath_nosma_secs = 0;
} else
ipath_cdbg(SMA, "No SMA bufs avail %u tries, "
"after %u seconds\n",
dd->ipath_nosma_bufs,
dd->ipath_nosma_secs);
}
done:
mod_timer(&dd->ipath_stats_timer, jiffies + HZ * 5);
}
