int
_sema_timedwait(struct sema *sema, int timo, const char *file, int line)
{
int ret, timed_out;
mtx_lock(&sema->sema_mtx);
/*
* A spurious wakeup will cause the timeout interval to start over.
* This isn't a big deal as long as spurious wakeups don't occur
* continuously, since the timeout period is merely a lower bound on how
* long to wait.
*/
for (timed_out = 0; sema->sema_value == 0 && timed_out == 0;) {
sema->sema_waiters++;
timed_out = cv_timedwait(&sema->sema_cv, &sema->sema_mtx, timo);
sema->sema_waiters--;
}
if (sema->sema_value > 0) {
/* Success. */
sema->sema_value--;
ret = 1;
CTR6(KTR_LOCK, "%s(%p) \"%s\" v = %d at %s:%d", __func__, sema,
cv_wmesg(&sema->sema_cv), sema->sema_value, file, line);
} else {
ret = 0;
CTR5(KTR_LOCK, "%s(%p) \"%s\" fail at %s:%d", __func__, sema,
cv_wmesg(&sema->sema_cv), file, line);
}
mtx_unlock(&sema->sema_mtx);
return (ret);
}
/*
* One sided detach. The tcpcb is going away and we need to unhook the toepcb
* hanging off it. If the TOE driver is also done with the toepcb we'll release
* all offload resources.
*/
static void
toepcb_detach(struct inpcb *inp)
{
struct toepcb *toep;
struct tcpcb *tp;
KASSERT(inp, ("%s: inp is NULL", __func__));
INP_WLOCK_ASSERT(inp);
tp = intotcpcb(inp);
toep = tp->t_toe;
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
KASSERT(toep->tp_flags & TP_ATTACHED, ("%s: not attached", __func__));
CTR6(KTR_CXGB, "%s: %s %u, toep %p, inp %p, tp %p", __func__,
tp->t_state == TCPS_SYN_SENT ? "atid" : "tid", toep->tp_tid,
toep, inp, tp);
tp->t_toe = NULL;
tp->t_flags &= ~TF_TOE;
toep->tp_flags &= ~TP_ATTACHED;
if (toep->tp_flags & TP_CPL_DONE)
t3_release_offload_resources(toep);
}
static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
{
int len = vma->vm_end - vma->vm_start;
u32 key = vma->vm_pgoff << PAGE_SHIFT;
struct c4iw_rdev *rdev;
int ret = 0;
struct c4iw_mm_entry *mm;
struct c4iw_ucontext *ucontext;
u64 addr, paddr;
u64 va_regs_res = 0, va_udbs_res = 0;
u64 len_regs_res = 0, len_udbs_res = 0;
CTR3(KTR_IW_CXGBE, "%s:1 ctx %p vma %p", __func__, context, vma);
CTR4(KTR_IW_CXGBE, "%s:1a pgoff 0x%lx key 0x%x len %d", __func__,
vma->vm_pgoff, key, len);
if (vma->vm_start & (PAGE_SIZE-1)) {
CTR3(KTR_IW_CXGBE, "%s:2 unaligned vm_start %u vma %p",
__func__, vma->vm_start, vma);
return -EINVAL;
}
rdev = &(to_c4iw_dev(context->device)->rdev);
ucontext = to_c4iw_ucontext(context);
mm = remove_mmap(ucontext, key, len);
if (!mm) {
CTR4(KTR_IW_CXGBE, "%s:3 ucontext %p key %u len %u", __func__,
ucontext, key, len);
return -EINVAL;
}
addr = mm->addr;
kfree(mm);
va_regs_res = (u64)rman_get_virtual(rdev->adap->regs_res);
len_regs_res = (u64)rman_get_size(rdev->adap->regs_res);
va_udbs_res = (u64)rman_get_virtual(rdev->adap->udbs_res);
len_udbs_res = (u64)rman_get_size(rdev->adap->udbs_res);
CTR6(KTR_IW_CXGBE,
"%s:4 addr %p, masync region %p:%p, udb region %p:%p", __func__,
addr, va_regs_res, va_regs_res+len_regs_res, va_udbs_res,
va_udbs_res+len_udbs_res);
if (addr >= va_regs_res && addr < va_regs_res + len_regs_res) {
CTR4(KTR_IW_CXGBE, "%s:5 MA_SYNC addr %p region %p, reglen %u",
__func__, addr, va_regs_res, len_regs_res);
/*
* MA_SYNC register...
*/
paddr = vtophys(addr);
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
ret = io_remap_pfn_range(vma, vma->vm_start,
paddr >> PAGE_SHIFT,
len, vma->vm_page_prot);
} else {
void
pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
vm_offset_t b_offset, int xfersize)
{
CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma,
a_offset, mb, b_offset, xfersize);
MMU_COPY_PAGES(mmu_obj, ma, a_offset, mb, b_offset, xfersize);
}
void
pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t p,
vm_prot_t prot, boolean_t wired)
{
CTR6(KTR_PMAP, "pmap_enter(%p, %#x, %#x, %p, %#x, %u)", pmap, va,
access, p, prot, wired);
MMU_ENTER(mmu_obj, pmap, va, p, prot, wired);
}
void
pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
vm_pindex_t pindex, vm_size_t size)
{
CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr,
object, pindex, size);
MMU_OBJECT_INIT_PT(mmu_obj, pmap, addr, object, pindex, size);
}
int
pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t p, vm_prot_t prot,
u_int flags, int8_t psind)
{
CTR6(KTR_PMAP, "pmap_enter(%p, %#x, %p, %#x, %x, %d)", pmap, va,
p, prot, flags, psind);
return (MMU_ENTER(mmu_obj, pmap, va, p, prot, flags, psind));
}
void
pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
vm_size_t len, vm_offset_t src_addr)
{
CTR6(KTR_PMAP, "%s(%p, %p, %#x, %#x, %#x)", __func__, dst_pmap,
src_pmap, dst_addr, len, src_addr);
MMU_COPY(mmu_obj, dst_pmap, src_pmap, dst_addr, len, src_addr);
}
void
pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
vm_page_t m_start, vm_prot_t prot)
{
CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start,
end, m_start, prot);
MMU_ENTER_OBJECT(mmu_obj, pmap, start, end, m_start, prot);
}
void
send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
{
struct wrqe *wr;
struct cpl_abort_req *req;
int tid = toep->tid;
struct inpcb *inp = toep->inp;
struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */
INP_WLOCK_ASSERT(inp);
CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s",
__func__, toep->tid,
inp->inp_flags & INP_DROPPED ? "inp dropped" :
tcpstates[tp->t_state],
toep->flags, inp->inp_flags,
toep->flags & TPF_ABORT_SHUTDOWN ?
" (abort already in progress)" : "");
if (toep->flags & TPF_ABORT_SHUTDOWN)
return; /* abort already in progress */
toep->flags |= TPF_ABORT_SHUTDOWN;
KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %d.", __func__, tid));
wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
if (wr == NULL) {
/* XXX */
panic("%s: allocation failure.", __func__);
}
req = wrtod(wr);
INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid);
if (inp->inp_flags & INP_DROPPED)
req->rsvd0 = htobe32(snd_nxt);
else
req->rsvd0 = htobe32(tp->snd_nxt);
req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT);
req->cmd = CPL_ABORT_SEND_RST;
/*
* XXX: What's the correct way to tell that the inp hasn't been detached
* from its socket? Should I even be flushing the snd buffer here?
*/
if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
struct socket *so = inp->inp_socket;
if (so != NULL) /* because I'm not sure. See comment above */
sbflush(&so->so_snd);
}
t4_l2t_send(sc, wr, toep->l2te);
}
void
_sema_post(struct sema *sema, const char *file, int line)
{
mtx_lock(&sema->sema_mtx);
sema->sema_value++;
if (sema->sema_waiters && sema->sema_value > 0)
cv_signal(&sema->sema_cv);
CTR6(KTR_LOCK, "%s(%p) \"%s\" v = %d at %s:%d", __func__, sema,
cv_wmesg(&sema->sema_cv), sema->sema_value, file, line);
mtx_unlock(&sema->sema_mtx);
}
void
_sema_wait(struct sema *sema, const char *file, int line)
{
mtx_lock(&sema->sema_mtx);
while (sema->sema_value == 0) {
sema->sema_waiters++;
cv_wait(&sema->sema_cv, &sema->sema_mtx);
sema->sema_waiters--;
}
sema->sema_value--;
CTR6(KTR_LOCK, "%s(%p) \"%s\" v = %d at %s:%d", __func__, sema,
cv_wmesg(&sema->sema_cv), sema->sema_value, file, line);
mtx_unlock(&sema->sema_mtx);
}
int
_sema_trywait(struct sema *sema, const char *file, int line)
{
int ret;
mtx_lock(&sema->sema_mtx);
if (sema->sema_value > 0) {
/* Success. */
sema->sema_value--;
ret = 1;
CTR6(KTR_LOCK, "%s(%p) \"%s\" v = %d at %s:%d", __func__, sema,
cv_wmesg(&sema->sema_cv), sema->sema_value, file, line);
} else {
ret = 0;
CTR5(KTR_LOCK, "%s(%p) \"%s\" fail at %s:%d", __func__, sema,
cv_wmesg(&sema->sema_cv), file, line);
}
mtx_unlock(&sema->sema_mtx);
return (ret);
}
int build_phys_page_list(struct ib_phys_buf *buffer_list,
int num_phys_buf,
u64 *iova_start,
u64 *total_size,
int *npages,
int *shift,
__be64 **page_list)
{
u64 mask;
int i, j, n;
mask = 0;
*total_size = 0;
for (i = 0; i < num_phys_buf; ++i) {
if (i != 0 && buffer_list[i].addr & ~PAGE_MASK)
return (-EINVAL);
if (i != 0 && i != num_phys_buf - 1 &&
(buffer_list[i].size & ~PAGE_MASK))
return (-EINVAL);
*total_size += buffer_list[i].size;
if (i > 0)
mask |= buffer_list[i].addr;
else
mask |= buffer_list[i].addr & PAGE_MASK;
if (i != num_phys_buf - 1)
mask |= buffer_list[i].addr + buffer_list[i].size;
else
mask |= (buffer_list[i].addr + buffer_list[i].size +
PAGE_SIZE - 1) & PAGE_MASK;
}
if (*total_size > 0xFFFFFFFFULL)
return (-ENOMEM);
/* Find largest page shift we can use to cover buffers */
for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift))
if ((1ULL << *shift) & mask)
break;
buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1);
buffer_list[0].addr &= ~0ull << *shift;
*npages = 0;
for (i = 0; i < num_phys_buf; ++i)
*npages += (buffer_list[i].size +
(1ULL << *shift) - 1) >> *shift;
if (!*npages)
return (-EINVAL);
*page_list = kmalloc(sizeof(u64) * *npages, M_NOWAIT);
if (!*page_list)
return (-ENOMEM);
n = 0;
for (i = 0; i < num_phys_buf; ++i)
for (j = 0;
j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift;
++j)
(*page_list)[n++] = htobe64(buffer_list[i].addr +
((u64) j << *shift));
CTR6(KTR_IW_CXGB, "%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d",
__FUNCTION__, (unsigned long long) *iova_start,
(unsigned long long) mask, *shift, (unsigned long long) *total_size,
*npages);
return 0;
}
static inline int
syscallenter(struct thread *td, struct syscall_args *sa)
{
struct proc *p;
int error, traced;
PCPU_INC(cnt.v_syscall);
p = td->td_proc;
td->td_pticks = 0;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
if (p->p_flag & P_TRACED) {
traced = 1;
PROC_LOCK(p);
td->td_dbgflags &= ~TDB_USERWR;
td->td_dbgflags |= TDB_SCE;
PROC_UNLOCK(p);
} else
traced = 0;
error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
#ifdef KTRACE
if (KTRPOINT(td, KTR_SYSCALL))
ktrsyscall(sa->code, sa->narg, sa->args);
#endif
CTR6(KTR_SYSC,
"syscall: td=%p pid %d %s (%#lx, %#lx, %#lx)",
td, td->td_proc->p_pid, syscallname(p, sa->code),
sa->args[0], sa->args[1], sa->args[2]);
if (error == 0) {
STOPEVENT(p, S_SCE, sa->narg);
if (p->p_flag & P_TRACED && p->p_stops & S_PT_SCE) {
PROC_LOCK(p);
ptracestop((td), SIGTRAP);
PROC_UNLOCK(p);
}
if (td->td_dbgflags & TDB_USERWR) {
/*
* Reread syscall number and arguments if
* debugger modified registers or memory.
*/
error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
#ifdef KTRACE
if (KTRPOINT(td, KTR_SYSCALL))
ktrsyscall(sa->code, sa->narg, sa->args);
#endif
if (error != 0)
goto retval;
}
#ifdef CAPABILITY_MODE
/*
* In capability mode, we only allow access to system calls
* flagged with SYF_CAPENABLED.
*/
if (IN_CAPABILITY_MODE(td) &&
!(sa->callp->sy_flags & SYF_CAPENABLED)) {
error = ECAPMODE;
goto retval;
}
#endif
error = syscall_thread_enter(td, sa->callp);
if (error != 0)
goto retval;
#ifdef KDTRACE_HOOKS
/*
* If the systrace module has registered it's probe
* callback and if there is a probe active for the
* syscall 'entry', process the probe.
*/
if (systrace_probe_func != NULL && sa->callp->sy_entry != 0)
(*systrace_probe_func)(sa->callp->sy_entry, sa->code,
sa->callp, sa->args, 0);
#endif
AUDIT_SYSCALL_ENTER(sa->code, td);
error = (sa->callp->sy_call)(td, sa->args);
AUDIT_SYSCALL_EXIT(error, td);
/* Save the latest error return value. */
td->td_errno = error;
#ifdef KDTRACE_HOOKS
/*
* If the systrace module has registered it's probe
* callback and if there is a probe active for the
* syscall 'return', process the probe.
*/
if (systrace_probe_func != NULL && sa->callp->sy_return != 0)
(*systrace_probe_func)(sa->callp->sy_return, sa->code,
sa->callp, NULL, (error) ? -1 : td->td_retval[0]);
#endif
syscall_thread_exit(td, sa->callp);
CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx",
p, error, td->td_retval[0], td->td_retval[1]);
}
retval:
if (traced) {
PROC_LOCK(p);
td->td_dbgflags &= ~TDB_SCE;
PROC_UNLOCK(p);
}
(p->p_sysent->sv_set_syscall_retval)(td, error);
return (error);
}
static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
struct t4_cq *rcq, struct t4_cq *scq,
struct c4iw_dev_ucontext *uctx)
{
struct adapter *sc = rdev->adap;
int user = (uctx != &rdev->uctx);
struct fw_ri_res_wr *res_wr;
struct fw_ri_res *res;
int wr_len;
struct c4iw_wr_wait wr_wait;
int ret;
int eqsize;
struct wrqe *wr;
wq->sq.qid = c4iw_get_qpid(rdev, uctx);
if (!wq->sq.qid)
return -ENOMEM;
wq->rq.qid = c4iw_get_qpid(rdev, uctx);
if (!wq->rq.qid)
goto err1;
if (!user) {
wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq,
GFP_KERNEL);
if (!wq->sq.sw_sq)
goto err2;
wq->rq.sw_rq = kzalloc(wq->rq.size * sizeof *wq->rq.sw_rq,
GFP_KERNEL);
if (!wq->rq.sw_rq)
goto err3;
}
/* RQT must be a power of 2. */
wq->rq.rqt_size = roundup_pow_of_two(wq->rq.size);
wq->rq.rqt_hwaddr = c4iw_rqtpool_alloc(rdev, wq->rq.rqt_size);
if (!wq->rq.rqt_hwaddr)
goto err4;
if (alloc_host_sq(rdev, &wq->sq))
goto err5;
memset(wq->sq.queue, 0, wq->sq.memsize);
pci_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr);
wq->rq.queue = contigmalloc(wq->rq.memsize,
M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
if (wq->rq.queue)
wq->rq.dma_addr = vtophys(wq->rq.queue);
else
goto err6;
CTR5(KTR_IW_CXGBE,
"%s sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx", __func__,
wq->sq.queue, (unsigned long long)vtophys(wq->sq.queue),
wq->rq.queue, (unsigned long long)vtophys(wq->rq.queue));
memset(wq->rq.queue, 0, wq->rq.memsize);
pci_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr);
wq->db = (void *)((unsigned long)rman_get_virtual(sc->regs_res) +
MYPF_REG(SGE_PF_KDOORBELL));
wq->gts = (void *)((unsigned long)rman_get_virtual(rdev->adap->regs_res)
+ MYPF_REG(SGE_PF_GTS));
if (user) {
wq->sq.udb = (u64)((char*)rman_get_virtual(rdev->adap->udbs_res) +
(wq->sq.qid << rdev->qpshift));
wq->sq.udb &= PAGE_MASK;
wq->rq.udb = (u64)((char*)rman_get_virtual(rdev->adap->udbs_res) +
(wq->rq.qid << rdev->qpshift));
wq->rq.udb &= PAGE_MASK;
}
wq->rdev = rdev;
wq->rq.msn = 1;
/* build fw_ri_res_wr */
wr_len = sizeof *res_wr + 2 * sizeof *res;
wr = alloc_wrqe(wr_len, &sc->sge.mgmtq);
if (wr == NULL)
return (0);
res_wr = wrtod(wr);
memset(res_wr, 0, wr_len);
res_wr->op_nres = cpu_to_be32(
V_FW_WR_OP(FW_RI_RES_WR) |
V_FW_RI_RES_WR_NRES(2) |
F_FW_WR_COMPL);
res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16));
res_wr->cookie = (unsigned long) &wr_wait;
res = res_wr->res;
res->u.sqrq.restype = FW_RI_RES_TYPE_SQ;
res->u.sqrq.op = FW_RI_RES_OP_WRITE;
/* eqsize is the number of 64B entries plus the status page size. */
eqsize = wq->sq.size * T4_SQ_NUM_SLOTS + spg_creds;
res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */
V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */
V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */
V_FW_RI_RES_WR_IQID(scq->cqid));
res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
V_FW_RI_RES_WR_DCAEN(0) |
V_FW_RI_RES_WR_DCACPU(0) |
V_FW_RI_RES_WR_FBMIN(2) |
V_FW_RI_RES_WR_FBMAX(2) |
V_FW_RI_RES_WR_CIDXFTHRESHO(0) |
V_FW_RI_RES_WR_CIDXFTHRESH(0) |
V_FW_RI_RES_WR_EQSIZE(eqsize));
res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid);
res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr);
res++;
res->u.sqrq.restype = FW_RI_RES_TYPE_RQ;
res->u.sqrq.op = FW_RI_RES_OP_WRITE;
/* eqsize is the number of 64B entries plus the status page size. */
eqsize = wq->rq.size * T4_RQ_NUM_SLOTS + spg_creds ;
res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */
V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */
V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */
V_FW_RI_RES_WR_IQID(rcq->cqid));
res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
V_FW_RI_RES_WR_DCAEN(0) |
V_FW_RI_RES_WR_DCACPU(0) |
V_FW_RI_RES_WR_FBMIN(2) |
V_FW_RI_RES_WR_FBMAX(2) |
V_FW_RI_RES_WR_CIDXFTHRESHO(0) |
V_FW_RI_RES_WR_CIDXFTHRESH(0) |
V_FW_RI_RES_WR_EQSIZE(eqsize));
res->u.sqrq.eqid = cpu_to_be32(wq->rq.qid);
res->u.sqrq.eqaddr = cpu_to_be64(wq->rq.dma_addr);
c4iw_init_wr_wait(&wr_wait);
t4_wrq_tx(sc, wr);
ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, __func__);
if (ret)
goto err7;
CTR6(KTR_IW_CXGBE,
"%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%llx rqudb 0x%llx",
__func__, wq->sq.qid, wq->rq.qid, wq->db,
(unsigned long long)wq->sq.udb, (unsigned long long)wq->rq.udb);
return 0;
err7:
contigfree(wq->rq.queue, wq->rq.memsize, M_DEVBUF);
err6:
dealloc_sq(rdev, &wq->sq);
err5:
c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size);
err4:
kfree(wq->rq.sw_rq);
err3:
kfree(wq->sq.sw_sq);
err2:
c4iw_put_qpid(rdev, wq->rq.qid, uctx);
err1:
c4iw_put_qpid(rdev, wq->sq.qid, uctx);
return -ENOMEM;
}
/*
* Abort handler.
*
* FAR, FSR, and everything what can be lost after enabling
* interrupts must be grabbed before the interrupts will be
* enabled. Note that when interrupts will be enabled, we
* could even migrate to another CPU ...
*
* TODO: move quick cases to ASM
*/
void
abort_handler(struct trapframe *tf, int prefetch)
{
struct thread *td;
vm_offset_t far, va;
int idx, rv;
uint32_t fsr;
struct ksig ksig;
struct proc *p;
struct pcb *pcb;
struct vm_map *map;
struct vmspace *vm;
vm_prot_t ftype;
bool usermode;
#ifdef INVARIANTS
void *onfault;
#endif
td = curthread;
fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get();
#if __ARM_ARCH >= 7
far = (prefetch) ? cp15_ifar_get() : cp15_dfar_get();
#else
far = (prefetch) ? TRAPF_PC(tf) : cp15_dfar_get();
#endif
idx = FSR_TO_FAULT(fsr);
usermode = TRAPF_USERMODE(tf); /* Abort came from user mode? */
if (usermode)
td->td_frame = tf;
CTR6(KTR_TRAP, "%s: fsr %#x (idx %u) far %#x prefetch %u usermode %d",
__func__, fsr, idx, far, prefetch, usermode);
/*
* Firstly, handle aborts that are not directly related to mapping.
*/
if (__predict_false(idx == FAULT_EA_IMPREC)) {
abort_imprecise(tf, fsr, prefetch, usermode);
return;
}
if (__predict_false(idx == FAULT_DEBUG)) {
abort_debug(tf, fsr, prefetch, usermode, far);
return;
}
/*
* ARM has a set of unprivileged load and store instructions
* (LDRT/LDRBT/STRT/STRBT ...) which are supposed to be used in other
* than user mode and OS should recognize their aborts and behave
* appropriately. However, there is no way how to do that reasonably
* in general unless we restrict the handling somehow.
*
* For now, these instructions are used only in copyin()/copyout()
* like functions where usermode buffers are checked in advance that
* they are not from KVA space. Thus, no action is needed here.
*/
#ifdef ARM_NEW_PMAP
rv = pmap_fault(PCPU_GET(curpmap), far, fsr, idx, usermode);
if (rv == 0) {
return;
} else if (rv == EFAULT) {
call_trapsignal(td, SIGSEGV, SEGV_MAPERR, far);
userret(td, tf);
return;
}
#endif
/*
* Now, when we handled imprecise and debug aborts, the rest of
* aborts should be really related to mapping.
*/
PCPU_INC(cnt.v_trap);
#ifdef KDB
if (kdb_active) {
kdb_reenter();
goto out;
}
#endif
if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
/*
* Due to both processor errata and lazy TLB invalidation when
* access restrictions are removed from virtual pages, memory
* accesses that are allowed by the physical mapping layer may
* nonetheless cause one spurious page fault per virtual page.
* When the thread is executing a "no faulting" section that
* is bracketed by vm_fault_{disable,enable}_pagefaults(),
* every page fault is treated as a spurious page fault,
* unless it accesses the same virtual address as the most
* recent page fault within the same "no faulting" section.
*/
if (td->td_md.md_spurflt_addr != far ||
(td->td_pflags & TDP_RESETSPUR) != 0) {
td->td_md.md_spurflt_addr = far;
td->td_pflags &= ~TDP_RESETSPUR;
tlb_flush_local(far & ~PAGE_MASK);
return;
}
} else {
/*
* If we get a page fault while in a critical section, then
* it is most likely a fatal kernel page fault. The kernel
* is already going to panic trying to get a sleep lock to
* do the VM lookup, so just consider it a fatal trap so the
* kernel can print out a useful trap message and even get
* to the debugger.
*
* If we get a page fault while holding a non-sleepable
* lock, then it is most likely a fatal kernel page fault.
* If WITNESS is enabled, then it's going to whine about
* bogus LORs with various VM locks, so just skip to the
* fatal trap handling directly.
*/
if (td->td_critnest != 0 ||
WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
"Kernel page fault") != 0) {
abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
return;
}
}
/* Re-enable interrupts if they were enabled previously. */
if (td->td_md.md_spinlock_count == 0) {
if (__predict_true(tf->tf_spsr & PSR_I) == 0)
enable_interrupts(PSR_I);
if (__predict_true(tf->tf_spsr & PSR_F) == 0)
enable_interrupts(PSR_F);
}
p = td->td_proc;
if (usermode) {
td->td_pticks = 0;
if (td->td_cowgen != p->p_cowgen)
thread_cow_update(td);
}
/* Invoke the appropriate handler, if necessary. */
if (__predict_false(aborts[idx].func != NULL)) {
if ((aborts[idx].func)(tf, idx, fsr, far, prefetch, td, &ksig))
goto do_trapsignal;
goto out;
}
/*
* Don't pass faulting cache operation to vm_fault(). We don't want
* to handle all vm stuff at this moment.
*/
pcb = td->td_pcb;
if (__predict_false(pcb->pcb_onfault == cachebailout)) {
tf->tf_r0 = far; /* return failing address */
tf->tf_pc = (register_t)pcb->pcb_onfault;
return;
}
/* Handle remaining I-cache aborts. */
if (idx == FAULT_ICACHE) {
if (abort_icache(tf, idx, fsr, far, prefetch, td, &ksig))
goto do_trapsignal;
goto out;
}
/*
* At this point, we're dealing with one of the following aborts:
*
* FAULT_TRAN_xx - Translation
* FAULT_PERM_xx - Permission
*
* These are the main virtual memory-related faults signalled by
* the MMU.
*/
/* fusubailout is used by [fs]uswintr to avoid page faulting. */
pcb = td->td_pcb;
if (__predict_false(pcb->pcb_onfault == fusubailout)) {
tf->tf_r0 = EFAULT;
tf->tf_pc = (register_t)pcb->pcb_onfault;
return;
}
va = trunc_page(far);
if (va >= KERNBASE) {
/*
* Don't allow user-mode faults in kernel address space.
*/
if (usermode)
goto nogo;
map = kernel_map;
} else {
/*
* This is a fault on non-kernel virtual memory. If curproc
* is NULL or curproc->p_vmspace is NULL the fault is fatal.
*/
vm = (p != NULL) ? p->p_vmspace : NULL;
if (vm == NULL)
goto nogo;
map = &vm->vm_map;
if (!usermode && (td->td_intr_nesting_level != 0 ||
pcb->pcb_onfault == NULL)) {
abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
return;
}
}
ftype = (fsr & FSR_WNR) ? VM_PROT_WRITE : VM_PROT_READ;
if (prefetch)
ftype |= VM_PROT_EXECUTE;
#ifdef DEBUG
last_fault_code = fsr;
#endif
#ifndef ARM_NEW_PMAP
if (pmap_fault_fixup(vmspace_pmap(td->td_proc->p_vmspace), va, ftype,
usermode)) {
goto out;
}
#endif
#ifdef INVARIANTS
onfault = pcb->pcb_onfault;
pcb->pcb_onfault = NULL;
#endif
/* Fault in the page. */
rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
#ifdef INVARIANTS
pcb->pcb_onfault = onfault;
#endif
if (__predict_true(rv == KERN_SUCCESS))
goto out;
nogo:
if (!usermode) {
if (td->td_intr_nesting_level == 0 &&
pcb->pcb_onfault != NULL) {
tf->tf_r0 = rv;
tf->tf_pc = (int)pcb->pcb_onfault;
return;
}
CTR2(KTR_TRAP, "%s: vm_fault() failed with %d", __func__, rv);
abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
return;
}
ksig.sig = SIGSEGV;
ksig.code = (rv == KERN_PROTECTION_FAILURE) ? SEGV_ACCERR : SEGV_MAPERR;
ksig.addr = far;
do_trapsignal:
call_trapsignal(td, ksig.sig, ksig.code, ksig.addr);
out:
if (usermode)
userret(td, tf);
}
/*
* __mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
*
* We call this if the lock is either contested (i.e. we need to go to
* sleep waiting for it), or if we need to recurse on it.
*/
void
__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts,
const char *file, int line)
{
struct mtx *m;
struct turnstile *ts;
uintptr_t v;
#ifdef ADAPTIVE_MUTEXES
volatile struct thread *owner;
#endif
#ifdef KTR
int cont_logged = 0;
#endif
#ifdef LOCK_PROFILING
int contested = 0;
uint64_t waittime = 0;
#endif
#if defined(ADAPTIVE_MUTEXES) || defined(KDTRACE_HOOKS)
struct lock_delay_arg lda;
#endif
#ifdef KDTRACE_HOOKS
u_int sleep_cnt = 0;
int64_t sleep_time = 0;
int64_t all_time = 0;
#endif
if (SCHEDULER_STOPPED())
return;
#if defined(ADAPTIVE_MUTEXES)
lock_delay_arg_init(&lda, &mtx_delay);
#elif defined(KDTRACE_HOOKS)
lock_delay_arg_init(&lda, NULL);
#endif
m = mtxlock2mtx(c);
if (mtx_owned(m)) {
KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 ||
(opts & MTX_RECURSE) != 0,
("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
m->lock_object.lo_name, file, line));
opts &= ~MTX_RECURSE;
m->mtx_recurse++;
atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
if (LOCK_LOG_TEST(&m->lock_object, opts))
CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
return;
}
opts &= ~MTX_RECURSE;
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&m->lock_object,
&contested, &waittime);
if (LOCK_LOG_TEST(&m->lock_object, opts))
CTR4(KTR_LOCK,
"_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
#ifdef KDTRACE_HOOKS
all_time -= lockstat_nsecs(&m->lock_object);
#endif
for (;;) {
if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid))
break;
#ifdef KDTRACE_HOOKS
lda.spin_cnt++;
#endif
#ifdef ADAPTIVE_MUTEXES
/*
* If the owner is running on another CPU, spin until the
* owner stops running or the state of the lock changes.
*/
v = m->mtx_lock;
if (v != MTX_UNOWNED) {
owner = (struct thread *)(v & ~MTX_FLAGMASK);
if (TD_IS_RUNNING(owner)) {
if (LOCK_LOG_TEST(&m->lock_object, 0))
CTR3(KTR_LOCK,
"%s: spinning on %p held by %p",
__func__, m, owner);
KTR_STATE1(KTR_SCHED, "thread",
sched_tdname((struct thread *)tid),
"spinning", "lockname:\"%s\"",
m->lock_object.lo_name);
while (mtx_owner(m) == owner &&
TD_IS_RUNNING(owner))
lock_delay(&lda);
KTR_STATE0(KTR_SCHED, "thread",
sched_tdname((struct thread *)tid),
"running");
continue;
}
}
#endif
ts = turnstile_trywait(&m->lock_object);
v = m->mtx_lock;
/*
* Check if the lock has been released while spinning for
* the turnstile chain lock.
*/
if (v == MTX_UNOWNED) {
turnstile_cancel(ts);
continue;
}
#ifdef ADAPTIVE_MUTEXES
/*
* The current lock owner might have started executing
* on another CPU (or the lock could have changed
* owners) while we were waiting on the turnstile
* chain lock. If so, drop the turnstile lock and try
* again.
*/
owner = (struct thread *)(v & ~MTX_FLAGMASK);
if (TD_IS_RUNNING(owner)) {
turnstile_cancel(ts);
continue;
}
#endif
/*
* If the mutex isn't already contested and a failure occurs
* setting the contested bit, the mutex was either released
* or the state of the MTX_RECURSED bit changed.
*/
if ((v & MTX_CONTESTED) == 0 &&
!atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) {
turnstile_cancel(ts);
continue;
}
/*
* We definitely must sleep for this lock.
*/
mtx_assert(m, MA_NOTOWNED);
#ifdef KTR
if (!cont_logged) {
CTR6(KTR_CONTENTION,
"contention: %p at %s:%d wants %s, taken by %s:%d",
(void *)tid, file, line, m->lock_object.lo_name,
WITNESS_FILE(&m->lock_object),
WITNESS_LINE(&m->lock_object));
cont_logged = 1;
}
#endif
/*
* Block on the turnstile.
*/
#ifdef KDTRACE_HOOKS
sleep_time -= lockstat_nsecs(&m->lock_object);
#endif
turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
#ifdef KDTRACE_HOOKS
sleep_time += lockstat_nsecs(&m->lock_object);
sleep_cnt++;
#endif
}
#ifdef KDTRACE_HOOKS
all_time += lockstat_nsecs(&m->lock_object);
#endif
#ifdef KTR
if (cont_logged) {
CTR4(KTR_CONTENTION,
"contention end: %s acquired by %p at %s:%d",
m->lock_object.lo_name, (void *)tid, file, line);
}
#endif
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, m, contested,
waittime, file, line);
#ifdef KDTRACE_HOOKS
if (sleep_time)
LOCKSTAT_RECORD1(adaptive__block, m, sleep_time);
/*
* Only record the loops spinning and not sleeping.
*/
if (lda.spin_cnt > sleep_cnt)
LOCKSTAT_RECORD1(adaptive__spin, m, all_time - sleep_time);
#endif
}
static struct ddp_buffer *
alloc_ddp_buffer(struct tom_data *td, vm_page_t *pages, int npages, int offset,
int len)
{
int i, hcf, seglen, idx, ppod, nppods;
struct ddp_buffer *db;
/*
* The DDP page size is unrelated to the VM page size. We combine
* contiguous physical pages into larger segments to get the best DDP
* page size possible. This is the largest of the four sizes in
* A_ULP_RX_TDDP_PSZ that evenly divides the HCF of the segment sizes in
* the page list.
*/
hcf = 0;
for (i = 0; i < npages; i++) {
seglen = PAGE_SIZE;
while (i < npages - 1 &&
pages[i]->phys_addr + PAGE_SIZE == pages[i + 1]->phys_addr) {
seglen += PAGE_SIZE;
i++;
}
hcf = calculate_hcf(hcf, seglen);
if (hcf < t4_ddp_pgsz[1]) {
idx = 0;
goto have_pgsz; /* give up, short circuit */
}
}
if (hcf % t4_ddp_pgsz[0] != 0) {
/* hmmm. This could only happen when PAGE_SIZE < 4K */
KASSERT(PAGE_SIZE < 4096,
("%s: PAGE_SIZE %d, hcf %d", __func__, PAGE_SIZE, hcf));
CTR3(KTR_CXGBE, "%s: PAGE_SIZE %d, hcf %d",
__func__, PAGE_SIZE, hcf);
return (NULL);
}
for (idx = nitems(t4_ddp_pgsz) - 1; idx > 0; idx--) {
if (hcf % t4_ddp_pgsz[idx] == 0)
break;
}
have_pgsz:
MPASS(idx <= M_PPOD_PGSZ);
db = malloc(sizeof(*db), M_CXGBE, M_NOWAIT);
if (db == NULL) {
CTR1(KTR_CXGBE, "%s: malloc failed.", __func__);
return (NULL);
}
nppods = pages_to_nppods(npages, t4_ddp_pgsz[idx]);
if (alloc_ppods(td, nppods, &db->ppod_addr) != 0) {
free(db, M_CXGBE);
CTR4(KTR_CXGBE, "%s: no pods, nppods %d, resid %d, pgsz %d",
__func__, nppods, len, t4_ddp_pgsz[idx]);
return (NULL);
}
ppod = (db->ppod_addr - td->ppod_start) / PPOD_SIZE;
db->tag = V_PPOD_PGSZ(idx) | V_PPOD_TAG(ppod);
db->nppods = nppods;
db->npages = npages;
db->pages = pages;
db->offset = offset;
db->len = len;
CTR6(KTR_CXGBE, "New DDP buffer. "
"ddp_pgsz %d, ppod 0x%x, npages %d, nppods %d, offset %d, len %d",
t4_ddp_pgsz[idx], ppod, db->npages, db->nppods, db->offset,
db->len);
return (db);
}
void
send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
{
struct wrqe *wr;
struct fw_flowc_wr *flowc;
unsigned int nparams = ftxp ? 8 : 6, flowclen;
struct vi_info *vi = toep->vi;
struct port_info *pi = vi->pi;
struct adapter *sc = pi->adapter;
unsigned int pfvf = G_FW_VIID_PFN(vi->viid) << S_FW_VIID_PFN;
struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
("%s: flowc for tid %u sent already", __func__, toep->tid));
flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);
if (wr == NULL) {
/* XXX */
panic("%s: allocation failure.", __func__);
}
flowc = wrtod(wr);
memset(flowc, 0, wr->wr_len);
flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
V_FW_FLOWC_WR_NPARAMS(nparams));
flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
V_FW_WR_FLOWID(toep->tid));
flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
flowc->mnemval[0].val = htobe32(pfvf);
flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
flowc->mnemval[1].val = htobe32(pi->tx_chan);
flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
flowc->mnemval[2].val = htobe32(pi->tx_chan);
flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id);
if (ftxp) {
uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf);
flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
flowc->mnemval[4].val = htobe32(ftxp->snd_nxt);
flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt);
flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
flowc->mnemval[6].val = htobe32(sndbuf);
flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
flowc->mnemval[7].val = htobe32(ftxp->mss);
CTR6(KTR_CXGBE,
"%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x",
__func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt,
ftxp->rcv_nxt);
} else {
flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF;
flowc->mnemval[4].val = htobe32(512);
flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS;
flowc->mnemval[5].val = htobe32(512);
CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
}
txsd->tx_credits = howmany(flowclen, 16);
txsd->plen = 0;
KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
("%s: not enough credits (%d)", __func__, toep->tx_credits));
toep->tx_credits -= txsd->tx_credits;
if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
toep->txsd_pidx = 0;
toep->txsd_avail--;
toep->flags |= TPF_FLOWC_WR_SENT;
t4_wrq_tx(sc, wr);
}
/*
* _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
*
* We call this if the lock is either contested (i.e. we need to go to
* sleep waiting for it), or if we need to recurse on it.
*/
void
_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
int line)
{
struct turnstile *ts;
uintptr_t v;
#ifdef ADAPTIVE_MUTEXES
volatile struct thread *owner;
#endif
#ifdef KTR
int cont_logged = 0;
#endif
#ifdef LOCK_PROFILING
int contested = 0;
uint64_t waittime = 0;
#endif
#ifdef KDTRACE_HOOKS
uint64_t spin_cnt = 0;
uint64_t sleep_cnt = 0;
int64_t sleep_time = 0;
#endif
if (mtx_owned(m)) {
KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
m->lock_object.lo_name, file, line));
m->mtx_recurse++;
atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
if (LOCK_LOG_TEST(&m->lock_object, opts))
CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
return;
}
lock_profile_obtain_lock_failed(&m->lock_object,
&contested, &waittime);
if (LOCK_LOG_TEST(&m->lock_object, opts))
CTR4(KTR_LOCK,
"_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
while (!_obtain_lock(m, tid)) {
#ifdef KDTRACE_HOOKS
spin_cnt++;
#endif
#ifdef ADAPTIVE_MUTEXES
/*
* If the owner is running on another CPU, spin until the
* owner stops running or the state of the lock changes.
*/
v = m->mtx_lock;
if (v != MTX_UNOWNED) {
owner = (struct thread *)(v & ~MTX_FLAGMASK);
if (TD_IS_RUNNING(owner)) {
if (LOCK_LOG_TEST(&m->lock_object, 0))
CTR3(KTR_LOCK,
"%s: spinning on %p held by %p",
__func__, m, owner);
while (mtx_owner(m) == owner &&
TD_IS_RUNNING(owner)) {
cpu_spinwait();
#ifdef KDTRACE_HOOKS
spin_cnt++;
#endif
}
continue;
}
}
#endif
ts = turnstile_trywait(&m->lock_object);
v = m->mtx_lock;
/*
* Check if the lock has been released while spinning for
* the turnstile chain lock.
*/
if (v == MTX_UNOWNED) {
turnstile_cancel(ts);
continue;
}
#ifdef ADAPTIVE_MUTEXES
/*
* The current lock owner might have started executing
* on another CPU (or the lock could have changed
* owners) while we were waiting on the turnstile
* chain lock. If so, drop the turnstile lock and try
* again.
*/
owner = (struct thread *)(v & ~MTX_FLAGMASK);
if (TD_IS_RUNNING(owner)) {
turnstile_cancel(ts);
continue;
}
#endif
/*
* If the mutex isn't already contested and a failure occurs
* setting the contested bit, the mutex was either released
* or the state of the MTX_RECURSED bit changed.
*/
if ((v & MTX_CONTESTED) == 0 &&
!atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) {
turnstile_cancel(ts);
continue;
}
/*
* We definitely must sleep for this lock.
*/
mtx_assert(m, MA_NOTOWNED);
#ifdef KTR
if (!cont_logged) {
CTR6(KTR_CONTENTION,
"contention: %p at %s:%d wants %s, taken by %s:%d",
(void *)tid, file, line, m->lock_object.lo_name,
WITNESS_FILE(&m->lock_object),
WITNESS_LINE(&m->lock_object));
cont_logged = 1;
}
#endif
/*
* Block on the turnstile.
*/
#ifdef KDTRACE_HOOKS
sleep_time -= lockstat_nsecs();
#endif
turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
#ifdef KDTRACE_HOOKS
sleep_time += lockstat_nsecs();
sleep_cnt++;
#endif
}
#ifdef KTR
if (cont_logged) {
CTR4(KTR_CONTENTION,
"contention end: %s acquired by %p at %s:%d",
m->lock_object.lo_name, (void *)tid, file, line);
}
#endif
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested,
waittime, file, line);
#ifdef KDTRACE_HOOKS
if (sleep_time)
LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time);
/*
* Only record the loops spinning and not sleeping.
*/
if (spin_cnt > sleep_cnt)
LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (spin_cnt - sleep_cnt));
#endif
}
/*
* TCP RST from the peer, timeout, or some other such critical error.
*/
static int
do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct sge_wrq *ofld_txq = toep->ofld_txq;
struct inpcb *inp;
struct tcpcb *tp;
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_ABORT_REQ_RSS,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
if (toep->flags & TPF_SYNQE)
return (do_abort_req_synqe(iq, rss, m));
KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
if (negative_advice(cpl->status)) {
CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)",
__func__, cpl->status, tid, toep->flags);
return (0); /* Ignore negative advice */
}
inp = toep->inp;
INP_INFO_WLOCK(&V_tcbinfo); /* for tcp_close */
INP_WLOCK(inp);
tp = intotcpcb(inp);
CTR6(KTR_CXGBE,
"%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d",
__func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
inp->inp_flags, cpl->status);
/*
* If we'd initiated an abort earlier the reply to it is responsible for
* cleaning up resources. Otherwise we tear everything down right here
* right now. We owe the T4 a CPL_ABORT_RPL no matter what.
*/
if (toep->flags & TPF_ABORT_SHUTDOWN) {
INP_WUNLOCK(inp);
goto done;
}
toep->flags |= TPF_ABORT_SHUTDOWN;
if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
struct socket *so = inp->inp_socket;
if (so != NULL)
so_error_set(so, abort_status_to_errno(tp,
cpl->status));
tp = tcp_close(tp);
if (tp == NULL)
INP_WLOCK(inp); /* re-acquire */
}
final_cpl_received(toep);
done:
INP_INFO_WUNLOCK(&V_tcbinfo);
send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST);
return (0);
}
