Exemple #1
0
/**
 * ipath_qcheck - print delta of egrfull/hdrqfull errors for kernel ports
 * @dd: the infinipath device
 *
 * print the delta of egrfull/hdrqfull errors for kernel ports no more than
 * every 5 seconds.  User processes are printed at close, but kernel doesn't
 * close, so...  Separate routine so may call from other places someday, and
 * so function name when printed by _IPATH_INFO is meaningfull
 */
static void ipath_qcheck(struct ipath_devdata *dd)
{
	static u64 last_tot_hdrqfull;
	size_t blen = 0;
	char buf[128];

	*buf = 0;
	if (dd->ipath_pd[0]->port_hdrqfull != dd->ipath_p0_hdrqfull) {
		blen = snprintf(buf, sizeof buf, "port 0 hdrqfull %u",
				dd->ipath_pd[0]->port_hdrqfull -
				dd->ipath_p0_hdrqfull);
		dd->ipath_p0_hdrqfull = dd->ipath_pd[0]->port_hdrqfull;
	}
	if (ipath_stats.sps_etidfull != dd->ipath_last_tidfull) {
		blen += snprintf(buf + blen, sizeof buf - blen,
				 "%srcvegrfull %llu",
				 blen ? ", " : "",
				 (unsigned long long)
				 (ipath_stats.sps_etidfull -
				  dd->ipath_last_tidfull));
		dd->ipath_last_tidfull = ipath_stats.sps_etidfull;
	}

	/*
	 * this is actually the number of hdrq full interrupts, not actual
	 * events, but at the moment that's mostly what I'm interested in.
	 * Actual count, etc. is in the counters, if needed.  For production
	 * users this won't ordinarily be printed.
	 */

	if ((ipath_debug & (__IPATH_PKTDBG | __IPATH_DBG)) &&
	    ipath_stats.sps_hdrqfull != last_tot_hdrqfull) {
		blen += snprintf(buf + blen, sizeof buf - blen,
				 "%shdrqfull %llu (all ports)",
				 blen ? ", " : "",
				 (unsigned long long)
				 (ipath_stats.sps_hdrqfull -
				  last_tot_hdrqfull));
		last_tot_hdrqfull = ipath_stats.sps_hdrqfull;
	}
	if (blen)
		ipath_dbg("%s\n", buf);

	if (dd->ipath_port0head != (u32)
	    le64_to_cpu(*dd->ipath_hdrqtailptr)) {
		if (dd->ipath_lastport0rcv_cnt ==
		    ipath_stats.sps_port0pkts) {
			ipath_cdbg(PKT, "missing rcv interrupts? "
				   "port0 hd=%llx tl=%x; port0pkts %llx\n",
				   (unsigned long long)
				   le64_to_cpu(*dd->ipath_hdrqtailptr),
				   dd->ipath_port0head,
				   (unsigned long long)
				   ipath_stats.sps_port0pkts);
		}
		dd->ipath_lastport0rcv_cnt = ipath_stats.sps_port0pkts;
	}
}
Exemple #2
0
static void __ipath_release_user_pages(struct page **p, size_t num_pages,
				   int dirty)
{
	size_t i;

	for (i = 0; i < num_pages; i++) {
		ipath_cdbg(MM, "%lu/%lu put_page %p\n", (unsigned long) i,
			   (unsigned long) num_pages, p[i]);
		if (dirty)
			set_page_dirty_lock(p[i]);
		put_page(p[i]);
	}
}
static int ipath_resync_ibepb(struct ipath_devdata *dd)
{
	int ret, pat, tries, chn;
	u32 loc;

	ret = -1;
	chn = 0;
	for (tries = 0; tries < (4 * IBSD_RESYNC_TRIES); ++tries) {
		loc = IB_PGUDP(chn);
		ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, loc, 0, 0);
		if (ret < 0) {
			ipath_dev_err(dd, "Failed read in resync\n");
			continue;
		}
		if (ret != 0xF0 && ret != 0x55 && tries == 0)
			ipath_dev_err(dd, "unexpected pattern in resync\n");
		pat = ret ^ 0xA5; /* alternate F0 and 55 */
		ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, loc, pat, 0xFF);
		if (ret < 0) {
			ipath_dev_err(dd, "Failed write in resync\n");
			continue;
		}
		ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, loc, 0, 0);
		if (ret < 0) {
			ipath_dev_err(dd, "Failed re-read in resync\n");
			continue;
		}
		if (ret != pat) {
			ipath_dev_err(dd, "Failed compare1 in resync\n");
			continue;
		}
		loc = IB_CMUDONE(chn);
		ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, loc, 0, 0);
		if (ret < 0) {
			ipath_dev_err(dd, "Failed CMUDONE rd in resync\n");
			continue;
		}
		if ((ret & 0x70) != ((chn << 4) | 0x40)) {
			ipath_dev_err(dd, "Bad CMUDONE value %02X, chn %d\n",
				ret, chn);
			continue;
		}
		if (++chn == 4)
			break;  /* Success */
	}
	ipath_cdbg(VERBOSE, "Resync in %d tries\n", tries);
	return (ret > 0) ? 0 : ret;
}
Exemple #4
0
static int __get_user_pages(unsigned long start_page, size_t num_pages,
			struct page **p, struct vm_area_struct **vma)
{
	unsigned long lock_limit;
	size_t got;
	int ret;

	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >>
		PAGE_SHIFT;

	if (num_pages > lock_limit) {
		ret = -ENOMEM;
		goto bail;
	}

	ipath_cdbg(VERBOSE, "pin %lx pages from vaddr %lx\n",
		   (unsigned long) num_pages, start_page);

	for (got = 0; got < num_pages; got += ret) {
		ret = get_user_pages(current, current->mm,
				     start_page + got * PAGE_SIZE,
				     num_pages - got, 1, 1,
				     p + got, vma);
		if (ret < 0)
			goto bail_release;
	}

	current->mm->locked_vm += num_pages;

	ret = 0;
	goto bail;

bail_release:
	__ipath_release_user_pages(p, got, 0);
bail:
	return ret;
}
Exemple #5
0
/**
 * ipath_get_faststats - get word counters from chip before they overflow
 * @opaque - contains a pointer to the infinipath device ipath_devdata
 *
 * called from add_timer
 */
void ipath_get_faststats(unsigned long opaque)
{
	struct ipath_devdata *dd = (struct ipath_devdata *) opaque;
	u32 val;
	static unsigned cnt;
	unsigned long flags;

	/*
	 * don't access the chip while running diags, or memory diags can
	 * fail
	 */
	if (!dd->ipath_kregbase || !(dd->ipath_flags & IPATH_INITTED) ||
	    ipath_diag_inuse)
		/* but re-arm the timer, for diags case; won't hurt other */
		goto done;

	/*
	 * We now try to maintain a "active timer", based on traffic
	 * exceeding a threshold, so we need to check the word-counts
	 * even if they are 64-bit.
	 */
	ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordsendcnt);
	ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordrcvcnt);
	spin_lock_irqsave(&dd->ipath_eep_st_lock, flags);
	if (dd->ipath_traffic_wds  >= IPATH_TRAFFIC_ACTIVE_THRESHOLD)
		atomic_add(5, &dd->ipath_active_time); /* S/B #define */
	dd->ipath_traffic_wds = 0;
	spin_unlock_irqrestore(&dd->ipath_eep_st_lock, flags);

	if (dd->ipath_flags & IPATH_32BITCOUNTERS) {
		ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktsendcnt);
		ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktrcvcnt);
	}

	ipath_qcheck(dd);

	/*
	 * deal with repeat error suppression.  Doesn't really matter if
	 * last error was almost a full interval ago, or just a few usecs
	 * ago; still won't get more than 2 per interval.  We may want
	 * longer intervals for this eventually, could do with mod, counter
	 * or separate timer.  Also see code in ipath_handle_errors() and
	 * ipath_handle_hwerrors().
	 */

	if (dd->ipath_lasterror)
		dd->ipath_lasterror = 0;
	if (dd->ipath_lasthwerror)
		dd->ipath_lasthwerror = 0;
	if (dd->ipath_maskederrs
	    && time_after(jiffies, dd->ipath_unmasktime)) {
		char ebuf[256];
		int iserr;
		iserr = ipath_decode_err(ebuf, sizeof ebuf,
			dd->ipath_maskederrs);
		if (dd->ipath_maskederrs &
				~(INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL |
				INFINIPATH_E_PKTERRS ))
			ipath_dev_err(dd, "Re-enabling masked errors "
				      "(%s)\n", ebuf);
		else {
			/*
			 * rcvegrfull and rcvhdrqfull are "normal", for some
			 * types of processes (mostly benchmarks) that send
			 * huge numbers of messages, while not processing
			 * them.  So only complain about these at debug
			 * level.
			 */
			if (iserr)
					ipath_dbg("Re-enabling queue full errors (%s)\n",
							ebuf);
			else
				ipath_cdbg(ERRPKT, "Re-enabling packet"
						" problem interrupt (%s)\n", ebuf);
		}

		/* re-enable masked errors */
		dd->ipath_errormask |= dd->ipath_maskederrs;
		ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
			dd->ipath_errormask);
		dd->ipath_maskederrs = 0;
	}

	/* limit qfull messages to ~one per minute per port */
	if ((++cnt & 0x10)) {
		for (val = dd->ipath_cfgports - 1; ((int)val) >= 0;
		     val--) {
			if (dd->ipath_lastegrheads[val] != -1)
				dd->ipath_lastegrheads[val] = -1;
			if (dd->ipath_lastrcvhdrqtails[val] != -1)
				dd->ipath_lastrcvhdrqtails[val] = -1;
		}
	}

	ipath_chk_errormask(dd);
done:
	mod_timer(&dd->ipath_stats_timer, jiffies + HZ * 5);
}
Exemple #6
0
/**
 * ipath_diagpkt_write - write an IB packet
 * @fp: the diag data device file pointer
 * @data: ipath_diag_pkt structure saying where to get the packet
 * @count: size of data to write
 * @off: unused by this code
 */
static ssize_t ipath_diagpkt_write(struct file *fp,
				   const char __user *data,
				   size_t count, loff_t *off)
{
	u32 __iomem *piobuf;
	u32 plen, clen, pbufn;
	struct ipath_diag_pkt dp;
	u32 *tmpbuf = NULL;
	struct ipath_devdata *dd;
	ssize_t ret = 0;
	u64 val;

	if (count < sizeof(dp)) {
		ret = -EINVAL;
		goto bail;
	}

	if (copy_from_user(&dp, data, sizeof(dp))) {
		ret = -EFAULT;
		goto bail;
	}

	/* send count must be an exact number of dwords */
	if (dp.len & 3) {
		ret = -EINVAL;
		goto bail;
	}

	clen = dp.len >> 2;

	dd = ipath_lookup(dp.unit);
	if (!dd || !(dd->ipath_flags & IPATH_PRESENT) ||
	    !dd->ipath_kregbase) {
		ipath_cdbg(VERBOSE, "illegal unit %u for diag data send\n",
			   dp.unit);
		ret = -ENODEV;
		goto bail;
	}

	if (ipath_diag_inuse && !diag_set_link &&
	    !(dd->ipath_flags & IPATH_LINKACTIVE)) {
		diag_set_link = 1;
		ipath_cdbg(VERBOSE, "Trying to set to set link active for "
			   "diag pkt\n");
		ipath_set_linkstate(dd, IPATH_IB_LINKARM);
		ipath_set_linkstate(dd, IPATH_IB_LINKACTIVE);
	}

	if (!(dd->ipath_flags & IPATH_INITTED)) {
		/* no hardware, freeze, etc. */
		ipath_cdbg(VERBOSE, "unit %u not usable\n", dd->ipath_unit);
		ret = -ENODEV;
		goto bail;
	}
	val = dd->ipath_lastibcstat & IPATH_IBSTATE_MASK;
	if (val != IPATH_IBSTATE_INIT && val != IPATH_IBSTATE_ARM &&
	    val != IPATH_IBSTATE_ACTIVE) {
		ipath_cdbg(VERBOSE, "unit %u not ready (state %llx)\n",
			   dd->ipath_unit, (unsigned long long) val);
		ret = -EINVAL;
		goto bail;
	}

	/* need total length before first word written */
	/* +1 word is for the qword padding */
	plen = sizeof(u32) + dp.len;

	if ((plen + 4) > dd->ipath_ibmaxlen) {
		ipath_dbg("Pkt len 0x%x > ibmaxlen %x\n",
			  plen - 4, dd->ipath_ibmaxlen);
		ret = -EINVAL;
		goto bail;	/* before writing pbc */
	}
	tmpbuf = vmalloc(plen);
	if (!tmpbuf) {
		dev_info(&dd->pcidev->dev, "Unable to allocate tmp buffer, "
			 "failing\n");
		ret = -ENOMEM;
		goto bail;
	}

	if (copy_from_user(tmpbuf,
			   (const void __user *) (unsigned long) dp.data,
			   dp.len)) {
		ret = -EFAULT;
		goto bail;
	}

	piobuf = ipath_getpiobuf(dd, &pbufn);
	if (!piobuf) {
		ipath_cdbg(VERBOSE, "No PIO buffers avail unit for %u\n",
			   dd->ipath_unit);
		ret = -EBUSY;
		goto bail;
	}

	plen >>= 2;		/* in dwords */

	if (ipath_debug & __IPATH_PKTDBG)
		ipath_cdbg(VERBOSE, "unit %u 0x%x+1w pio%d\n",
			   dd->ipath_unit, plen - 1, pbufn);

	/* we have to flush after the PBC for correctness on some cpus
	 * or WC buffer can be written out of order */
	writeq(plen, piobuf);
	ipath_flush_wc();
	/* copy all by the trigger word, then flush, so it's written
	 * to chip before trigger word, then write trigger word, then
	 * flush again, so packet is sent. */
	__iowrite32_copy(piobuf + 2, tmpbuf, clen - 1);
	ipath_flush_wc();
	__raw_writel(tmpbuf[clen - 1], piobuf + clen + 1);
	ipath_flush_wc();

	ret = sizeof(dp);

bail:
	vfree(tmpbuf);
	return ret;
}
static void ipath_sd_trimdone_monitor(struct ipath_devdata *dd,
       const char *where)
{
	int ret, chn, baduns;
	u64 val;

	if (!where)
		where = "?";

	/* give time for reset to settle out in EPB */
	udelay(2);

	ret = ipath_resync_ibepb(dd);
	if (ret < 0)
		ipath_dev_err(dd, "not able to re-sync IB EPB (%s)\n", where);

	/* Do "sacrificial read" to get EPB in sane state after reset */
	ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, IB_CTRL2(0), 0, 0);
	if (ret < 0)
		ipath_dev_err(dd, "Failed TRIMDONE 1st read, (%s)\n", where);

	/* Check/show "summary" Trim-done bit in IBCStatus */
	val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
	if (val & (1ULL << 11))
		ipath_cdbg(VERBOSE, "IBCS TRIMDONE set (%s)\n", where);
	else
		ipath_dev_err(dd, "IBCS TRIMDONE clear (%s)\n", where);

	udelay(2);

	ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES, IB_MPREG6, 0x80, 0x80);
	if (ret < 0)
		ipath_dev_err(dd, "Failed Dummy RMW, (%s)\n", where);
	udelay(10);

	baduns = 0;

	for (chn = 3; chn >= 0; --chn) {
		/* Read CTRL reg for each channel to check TRIMDONE */
		ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
			IB_CTRL2(chn), 0, 0);
		if (ret < 0)
			ipath_dev_err(dd, "Failed checking TRIMDONE, chn %d"
				" (%s)\n", chn, where);

		if (!(ret & 0x10)) {
			int probe;
			baduns |= (1 << chn);
			ipath_dev_err(dd, "TRIMDONE cleared on chn %d (%02X)."
				" (%s)\n", chn, ret, where);
			probe = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
				IB_PGUDP(0), 0, 0);
			ipath_dev_err(dd, "probe is %d (%02X)\n",
				probe, probe);
			probe = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
				IB_CTRL2(chn), 0, 0);
			ipath_dev_err(dd, "re-read: %d (%02X)\n",
				probe, probe);
			ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
				IB_CTRL2(chn), 0x10, 0x10);
			if (ret < 0)
				ipath_dev_err(dd,
					"Err on TRIMDONE rewrite1\n");
		}
	}
	for (chn = 3; chn >= 0; --chn) {
		/* Read CTRL reg for each channel to check TRIMDONE */
		if (baduns & (1 << chn)) {
			ipath_dev_err(dd,
				"Reseting TRIMDONE on chn %d (%s)\n",
				chn, where);
			ret = ipath_sd7220_reg_mod(dd, IB_7220_SERDES,
				IB_CTRL2(chn), 0x10, 0x10);
			if (ret < 0)
				ipath_dev_err(dd, "Failed re-setting "
					"TRIMDONE, chn %d (%s)\n",
					chn, where);
		}
	}
}
Exemple #8
0
/**
 * ipath_diagpkt_write - write an IB packet
 * @fp: the diag data device file pointer
 * @data: ipath_diag_pkt structure saying where to get the packet
 * @count: size of data to write
 * @off: unused by this code
 */
static ssize_t ipath_diagpkt_write(struct file *fp,
				   const char __user *data,
				   size_t count, loff_t *off)
{
	u32 __iomem *piobuf;
	u32 plen, clen, pbufn;
	struct ipath_diag_pkt odp;
	struct ipath_diag_xpkt dp;
	u32 *tmpbuf = NULL;
	struct ipath_devdata *dd;
	ssize_t ret = 0;
	u64 val;
	u32 l_state, lt_state; /* LinkState, LinkTrainingState */

	if (count < sizeof(odp)) {
		ret = -EINVAL;
		goto bail;
	}

	if (count == sizeof(dp)) {
		if (copy_from_user(&dp, data, sizeof(dp))) {
			ret = -EFAULT;
			goto bail;
		}
	} else if (copy_from_user(&odp, data, sizeof(odp))) {
		ret = -EFAULT;
		goto bail;
	}

	/*
	 * Due to padding/alignment issues (lessened with new struct)
	 * the old and new structs are the same length. We need to
	 * disambiguate them, which we can do because odp.len has never
	 * been less than the total of LRH+BTH+DETH so far, while
	 * dp.unit (same offset) unit is unlikely to get that high.
	 * Similarly, dp.data, the pointer to user at the same offset
	 * as odp.unit, is almost certainly at least one (512byte)page
	 * "above" NULL. The if-block below can be omitted if compatibility
	 * between a new driver and older diagnostic code is unimportant.
	 * compatibility the other direction (new diags, old driver) is
	 * handled in the diagnostic code, with a warning.
	 */
	if (dp.unit >= 20 && dp.data < 512) {
		/* very probable version mismatch. Fix it up */
		memcpy(&odp, &dp, sizeof(odp));
		/* We got a legacy dp, copy elements to dp */
		dp.unit = odp.unit;
		dp.data = odp.data;
		dp.len = odp.len;
		dp.pbc_wd = 0; /* Indicate we need to compute PBC wd */
	}

	/* send count must be an exact number of dwords */
	if (dp.len & 3) {
		ret = -EINVAL;
		goto bail;
	}

	clen = dp.len >> 2;

	dd = ipath_lookup(dp.unit);
	if (!dd || !(dd->ipath_flags & IPATH_PRESENT) ||
	    !dd->ipath_kregbase) {
		ipath_cdbg(VERBOSE, "illegal unit %u for diag data send\n",
			   dp.unit);
		ret = -ENODEV;
		goto bail;
	}

	if (ipath_diag_inuse && !diag_set_link &&
	    !(dd->ipath_flags & IPATH_LINKACTIVE)) {
		diag_set_link = 1;
		ipath_cdbg(VERBOSE, "Trying to set to set link active for "
			   "diag pkt\n");
		ipath_set_linkstate(dd, IPATH_IB_LINKARM);
		ipath_set_linkstate(dd, IPATH_IB_LINKACTIVE);
	}

	if (!(dd->ipath_flags & IPATH_INITTED)) {
		/* no hardware, freeze, etc. */
		ipath_cdbg(VERBOSE, "unit %u not usable\n", dd->ipath_unit);
		ret = -ENODEV;
		goto bail;
	}
	/*
	 * Want to skip check for l_state if using custom PBC,
	 * because we might be trying to force an SM packet out.
	 * first-cut, skip _all_ state checking in that case.
	 */
	val = ipath_ib_state(dd, dd->ipath_lastibcstat);
	lt_state = ipath_ib_linktrstate(dd, dd->ipath_lastibcstat);
	l_state = ipath_ib_linkstate(dd, dd->ipath_lastibcstat);
	if (!dp.pbc_wd && (lt_state != INFINIPATH_IBCS_LT_STATE_LINKUP ||
	    (val != dd->ib_init && val != dd->ib_arm &&
	    val != dd->ib_active))) {
		ipath_cdbg(VERBOSE, "unit %u not ready (state %llx)\n",
			   dd->ipath_unit, (unsigned long long) val);
		ret = -EINVAL;
		goto bail;
	}

	/* need total length before first word written */
	/* +1 word is for the qword padding */
	plen = sizeof(u32) + dp.len;

	if ((plen + 4) > dd->ipath_ibmaxlen) {
		ipath_dbg("Pkt len 0x%x > ibmaxlen %x\n",
			  plen - 4, dd->ipath_ibmaxlen);
		ret = -EINVAL;
		goto bail;	/* before writing pbc */
	}
	tmpbuf = vmalloc(plen);
	if (!tmpbuf) {
		dev_info(&dd->pcidev->dev, "Unable to allocate tmp buffer, "
			 "failing\n");
		ret = -ENOMEM;
		goto bail;
	}

	if (copy_from_user(tmpbuf,
			   (const void __user *) (unsigned long) dp.data,
			   dp.len)) {
		ret = -EFAULT;
		goto bail;
	}

	plen >>= 2;		/* in dwords */

	piobuf = ipath_getpiobuf(dd, plen, &pbufn);
	if (!piobuf) {
		ipath_cdbg(VERBOSE, "No PIO buffers avail unit for %u\n",
			   dd->ipath_unit);
		ret = -EBUSY;
		goto bail;
	}
	/* disarm it just to be extra sure */
	ipath_disarm_piobufs(dd, pbufn, 1);

	if (ipath_debug & __IPATH_PKTDBG)
		ipath_cdbg(VERBOSE, "unit %u 0x%x+1w pio%d\n",
			   dd->ipath_unit, plen - 1, pbufn);

	if (dp.pbc_wd == 0)
		dp.pbc_wd = plen;
	writeq(dp.pbc_wd, piobuf);
	/*
	 * Copy all by the trigger word, then flush, so it's written
	 * to chip before trigger word, then write trigger word, then
	 * flush again, so packet is sent.
	 */
	if (dd->ipath_flags & IPATH_PIO_FLUSH_WC) {
		ipath_flush_wc();
		__iowrite32_copy(piobuf + 2, tmpbuf, clen - 1);
		ipath_flush_wc();
		__raw_writel(tmpbuf[clen - 1], piobuf + clen + 1);
	} else
		__iowrite32_copy(piobuf + 2, tmpbuf, clen);

	ipath_flush_wc();

	ret = sizeof(dp);

bail:
	vfree(tmpbuf);
	return ret;
}
Exemple #9
0
/**
 * ipath_diagpkt_write - write an IB packet
 * @fp: the diag data device file pointer
 * @data: ipath_diag_pkt structure saying where to get the packet
 * @count: size of data to write
 * @off: unused by this code
 */
static ssize_t ipath_diagpkt_write(struct file *fp,
				   const char __user *data,
				   size_t count, loff_t *off)
{
	u32 __iomem *piobuf;
	u32 plen, pbufn, maxlen_reserve;
	struct ipath_diag_pkt odp;
	struct ipath_diag_xpkt dp;
	u32 *tmpbuf = NULL;
	struct ipath_devdata *dd;
	ssize_t ret = 0;
	u64 val;
	u32 l_state, lt_state; /* LinkState, LinkTrainingState */


	if (count == sizeof(dp)) {
		if (copy_from_user(&dp, data, sizeof(dp))) {
			ret = -EFAULT;
			goto bail;
		}
	} else if (count == sizeof(odp)) {
		if (copy_from_user(&odp, data, sizeof(odp))) {
			ret = -EFAULT;
			goto bail;
		}
	} else {
		ret = -EINVAL;
		goto bail;
	}

	/* send count must be an exact number of dwords */
	if (dp.len & 3) {
		ret = -EINVAL;
		goto bail;
	}

	plen = dp.len >> 2;

	dd = ipath_lookup(dp.unit);
	if (!dd || !(dd->ipath_flags & IPATH_PRESENT) ||
	    !dd->ipath_kregbase) {
		ipath_cdbg(VERBOSE, "illegal unit %u for diag data send\n",
			   dp.unit);
		ret = -ENODEV;
		goto bail;
	}

	if (ipath_diag_inuse && !diag_set_link &&
	    !(dd->ipath_flags & IPATH_LINKACTIVE)) {
		diag_set_link = 1;
		ipath_cdbg(VERBOSE, "Trying to set to set link active for "
			   "diag pkt\n");
		ipath_set_linkstate(dd, IPATH_IB_LINKARM);
		ipath_set_linkstate(dd, IPATH_IB_LINKACTIVE);
	}

	if (!(dd->ipath_flags & IPATH_INITTED)) {
		/* no hardware, freeze, etc. */
		ipath_cdbg(VERBOSE, "unit %u not usable\n", dd->ipath_unit);
		ret = -ENODEV;
		goto bail;
	}
	/*
	 * Want to skip check for l_state if using custom PBC,
	 * because we might be trying to force an SM packet out.
	 * first-cut, skip _all_ state checking in that case.
	 */
	val = ipath_ib_state(dd, dd->ipath_lastibcstat);
	lt_state = ipath_ib_linktrstate(dd, dd->ipath_lastibcstat);
	l_state = ipath_ib_linkstate(dd, dd->ipath_lastibcstat);
	if (!dp.pbc_wd && (lt_state != INFINIPATH_IBCS_LT_STATE_LINKUP ||
	    (val != dd->ib_init && val != dd->ib_arm &&
	    val != dd->ib_active))) {
		ipath_cdbg(VERBOSE, "unit %u not ready (state %llx)\n",
			   dd->ipath_unit, (unsigned long long) val);
		ret = -EINVAL;
		goto bail;
	}

	/*
	 * need total length before first word written, plus 2 Dwords. One Dword
	 * is for padding so we get the full user data when not aligned on
	 * a word boundary. The other Dword is to make sure we have room for the
	 * ICRC which gets tacked on later.
	 */
	maxlen_reserve = 2 * sizeof(u32);
	if (dp.len > dd->ipath_ibmaxlen - maxlen_reserve) {
		ipath_dbg("Pkt len 0x%x > ibmaxlen %x\n",
			  dp.len, dd->ipath_ibmaxlen);
		ret = -EINVAL;
		goto bail;
	}

	plen = sizeof(u32) + dp.len;

	tmpbuf = vmalloc(plen);
	if (!tmpbuf) {
		dev_info(&dd->pcidev->dev, "Unable to allocate tmp buffer, "
			 "failing\n");
		ret = -ENOMEM;
		goto bail;
	}

	if (copy_from_user(tmpbuf,
			   (const void __user *) (unsigned long) dp.data,
			   dp.len)) {
		ret = -EFAULT;
		goto bail;
	}

	plen >>= 2;		/* in dwords */

	piobuf = ipath_getpiobuf(dd, plen, &pbufn);
	if (!piobuf) {
		ipath_cdbg(VERBOSE, "No PIO buffers avail unit for %u\n",
			   dd->ipath_unit);
		ret = -EBUSY;
		goto bail;
	}
	/* disarm it just to be extra sure */
	ipath_disarm_piobufs(dd, pbufn, 1);

	if (ipath_debug & __IPATH_PKTDBG)
		ipath_cdbg(VERBOSE, "unit %u 0x%x+1w pio%d\n",
			   dd->ipath_unit, plen - 1, pbufn);

	if (dp.pbc_wd == 0)
		dp.pbc_wd = plen;
	writeq(dp.pbc_wd, piobuf);
	/*
	 * Copy all by the trigger word, then flush, so it's written
	 * to chip before trigger word, then write trigger word, then
	 * flush again, so packet is sent.
	 */
	if (dd->ipath_flags & IPATH_PIO_FLUSH_WC) {
		ipath_flush_wc();
		__iowrite32_copy(piobuf + 2, tmpbuf, plen - 1);
		ipath_flush_wc();
		__raw_writel(tmpbuf[plen - 1], piobuf + plen + 1);
	} else
		__iowrite32_copy(piobuf + 2, tmpbuf, plen);

	ipath_flush_wc();

	ret = sizeof(dp);

bail:
	vfree(tmpbuf);
	return ret;
}
/**
 * ipath_get_faststats - get word counters from chip before they overflow
 * @opaque - contains a pointer to the infinipath device ipath_devdata
 *
 * called from add_timer
 */
void ipath_get_faststats(unsigned long opaque)
{
	struct ipath_devdata *dd = (struct ipath_devdata *) opaque;
	u32 val;
	static unsigned cnt;

	/*
	 * don't access the chip while running diags, or memory diags can
	 * fail
	 */
	if (!dd->ipath_kregbase || !(dd->ipath_flags & IPATH_PRESENT) ||
	    ipath_diag_inuse)
		/* but re-arm the timer, for diags case; won't hurt other */
		goto done;

	if (dd->ipath_flags & IPATH_32BITCOUNTERS) {
		ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordsendcnt);
		ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordrcvcnt);
		ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktsendcnt);
		ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktrcvcnt);
	}

	ipath_qcheck(dd);

	/*
	 * deal with repeat error suppression.  Doesn't really matter if
	 * last error was almost a full interval ago, or just a few usecs
	 * ago; still won't get more than 2 per interval.  We may want
	 * longer intervals for this eventually, could do with mod, counter
	 * or separate timer.  Also see code in ipath_handle_errors() and
	 * ipath_handle_hwerrors().
	 */

	if (dd->ipath_lasterror)
		dd->ipath_lasterror = 0;
	if (dd->ipath_lasthwerror)
		dd->ipath_lasthwerror = 0;
	if ((dd->ipath_maskederrs & ~dd->ipath_ignorederrs)
	    && time_after(jiffies, dd->ipath_unmasktime)) {
		char ebuf[256];
		ipath_decode_err(ebuf, sizeof ebuf,
				 (dd->ipath_maskederrs & ~dd->
				  ipath_ignorederrs));
		if ((dd->ipath_maskederrs & ~dd->ipath_ignorederrs) &
		    ~(INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL))
			ipath_dev_err(dd, "Re-enabling masked errors "
				      "(%s)\n", ebuf);
		else {
			/*
			 * rcvegrfull and rcvhdrqfull are "normal", for some
			 * types of processes (mostly benchmarks) that send
			 * huge numbers of messages, while not processing
			 * them.  So only complain about these at debug
			 * level.
			 */
			ipath_dbg("Disabling frequent queue full errors "
				  "(%s)\n", ebuf);
		}
		dd->ipath_maskederrs = dd->ipath_ignorederrs;
		ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
				 ~dd->ipath_maskederrs);
	}

	/* limit qfull messages to ~one per minute per port */
	if ((++cnt & 0x10)) {
		for (val = dd->ipath_cfgports - 1; ((int)val) >= 0;
		     val--) {
			if (dd->ipath_lastegrheads[val] != -1)
				dd->ipath_lastegrheads[val] = -1;
			if (dd->ipath_lastrcvhdrqtails[val] != -1)
				dd->ipath_lastrcvhdrqtails[val] = -1;
		}
	}

	if (dd->ipath_nosma_bufs) {
		dd->ipath_nosma_secs += 5;
		if (dd->ipath_nosma_secs >= 30) {
			ipath_cdbg(SMA, "No SMA bufs avail %u seconds; "
				   "cancelling pending sends\n",
				   dd->ipath_nosma_secs);
			/*
			 * issue an abort as well, in case we have a packet
			 * stuck in launch fifo.  This could corrupt an
			 * outgoing user packet in the worst case,
			 * but this is a pretty catastrophic, anyway.
			 */
			ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
					 INFINIPATH_S_ABORT);
			ipath_disarm_piobufs(dd, dd->ipath_lastport_piobuf,
					     dd->ipath_piobcnt2k +
					     dd->ipath_piobcnt4k -
					     dd->ipath_lastport_piobuf);
			/* start again, if necessary */
			dd->ipath_nosma_secs = 0;
		} else
			ipath_cdbg(SMA, "No SMA bufs avail %u tries, "
				   "after %u seconds\n",
				   dd->ipath_nosma_bufs,
				   dd->ipath_nosma_secs);
	}

done:
	mod_timer(&dd->ipath_stats_timer, jiffies + HZ * 5);
}