int fairq_enqueue(struct fairq_if *fif, struct fairq_class *cl, struct mbuf *m, struct pf_mtag *t) { struct ifclassq *ifq = fif->fif_ifq; int len, ret; IFCQ_LOCK_ASSERT_HELD(ifq); VERIFY(cl == NULL || cl->cl_fif == fif); if (cl == NULL) { #if PF_ALTQ cl = fairq_clh_to_clp(fif, t->pftag_qid); #else /* !PF_ALTQ */ cl = fairq_clh_to_clp(fif, 0); #endif /* !PF_ALTQ */ if (cl == NULL) { cl = fif->fif_default; if (cl == NULL) { IFCQ_CONVERT_LOCK(ifq); m_freem(m); return (ENOBUFS); } } } cl->cl_flags |= FARF_HAS_PACKETS; len = m_pktlen(m); ret = fairq_addq(cl, m, t); if (ret != 0) { if (ret == CLASSQEQ_SUCCESS_FC) { /* packet enqueued, return advisory feedback */ ret = EQFULL; } else { VERIFY(ret == CLASSQEQ_DROPPED || ret == CLASSQEQ_DROPPED_FC || ret == CLASSQEQ_DROPPED_SP); /* packet has been freed in fairq_addq */ PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); IFCQ_DROP_ADD(ifq, 1, len); switch (ret) { case CLASSQEQ_DROPPED: return (ENOBUFS); case CLASSQEQ_DROPPED_FC: return (EQFULL); case CLASSQEQ_DROPPED_SP: return (EQSUSPENDED); } /* NOT REACHED */ } } IFCQ_INC_LEN(ifq); IFCQ_INC_BYTES(ifq, len); /* successfully queued. */ return (ret); }
static void tcq_purgeq(struct tcq_if *tif, struct tcq_class *cl, u_int32_t flow, u_int32_t *packets, u_int32_t *bytes) { struct ifclassq *ifq = tif->tif_ifq; u_int32_t cnt = 0, len = 0, qlen; IFCQ_LOCK_ASSERT_HELD(ifq); if ((qlen = qlen(&cl->cl_q)) == 0) goto done; /* become regular mutex before freeing mbufs */ IFCQ_CONVERT_LOCK(ifq); #if CLASSQ_RIO if (q_is_rio(&cl->cl_q)) rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len); else #endif /* CLASSQ_RIO */ #if CLASSQ_RED if (q_is_red(&cl->cl_q)) red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len); else #endif /* CLASSQ_RED */ #if CLASSQ_BLUE if (q_is_blue(&cl->cl_q)) blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len); else #endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len); else _flushq_flow(&cl->cl_q, flow, &cnt, &len); if (cnt > 0) { VERIFY(qlen(&cl->cl_q) == (qlen - cnt)); PKTCNTR_ADD(&cl->cl_dropcnt, cnt, len); IFCQ_DROP_ADD(ifq, cnt, len); VERIFY(((signed)IFCQ_LEN(ifq) - cnt) >= 0); IFCQ_LEN(ifq) -= cnt; if (pktsched_verbose) { log(LOG_DEBUG, "%s: %s purge qid=%d pri=%d " "qlen=[%d,%d] cnt=%d len=%d flow=0x%x\n", if_name(TCQIF_IFP(tif)), tcq_style(tif), cl->cl_handle, cl->cl_pri, qlen, qlen(&cl->cl_q), cnt, len, flow); } } done: if (packets != NULL) *packets = cnt; if (bytes != NULL) *bytes = len; }
static void sfb_fclist_append(struct sfb *sp, struct sfb_fcl *fcl) { IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); VERIFY(STAILQ_EMPTY(&fcl->fclist) || fcl->cnt > 0); sp->sfb_stats.flow_feedback += fcl->cnt; fcl->cnt = 0; flowadv_add(&fcl->fclist); VERIFY(fcl->cnt == 0 && STAILQ_EMPTY(&fcl->fclist)); }
/* * priq_enqueue_ifclassq is an enqueue function to be registered to * (*ifcq_enqueue) in struct ifclassq. */ static int priq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) { u_int32_t i; IFCQ_LOCK_ASSERT_HELD(ifq); if (!(m->m_flags & M_PKTHDR)) { /* should not happen */ log(LOG_ERR, "%s: packet does not have pkthdr\n", if_name(ifq->ifcq_ifp)); IFCQ_CONVERT_LOCK(ifq); m_freem(m); return (ENOBUFS); } i = MBUF_SCIDX(mbuf_get_service_class(m)); VERIFY((u_int32_t)i < IFCQ_SC_MAX); return (priq_enqueue(ifq->ifcq_disc, ifq->ifcq_disc_slots[i].cl, m, m_pftag(m))); }
static inline int fairq_addq(struct fairq_class *cl, struct mbuf *m, struct pf_mtag *t) { struct ifclassq *ifq = cl->cl_fif->fif_ifq; fairq_bucket_t *b; u_int32_t hash = m->m_pkthdr.pkt_flowid; u_int32_t hindex; u_int64_t bw; IFCQ_LOCK_ASSERT_HELD(ifq); /* * If the packet doesn't have any keep state put it on the end of * our queue. XXX this can result in out of order delivery. */ if (hash == 0) { if (cl->cl_head) b = cl->cl_head->prev; else b = &cl->cl_buckets[0]; } else { hindex = (hash & cl->cl_nbucket_mask); b = &cl->cl_buckets[hindex]; } /* * Add the bucket to the end of the circular list of active buckets. * * As a special case we add the bucket to the beginning of the list * instead of the end if it was not previously on the list and if * its traffic is less then the hog level. */ if (b->in_use == 0) { b->in_use = 1; if (cl->cl_head == NULL) { cl->cl_head = b; b->next = b; b->prev = b; } else { b->next = cl->cl_head; b->prev = cl->cl_head->prev; b->prev->next = b; b->next->prev = b; if (b->bw_delta && cl->cl_hogs_m1) { bw = b->bw_bytes * machclk_freq / b->bw_delta; if (bw < cl->cl_hogs_m1) cl->cl_head = b; } } } #if CLASSQ_RIO if (cl->cl_qtype == Q_RIO) return (rio_addq(cl->cl_rio, &b->queue, m, t)); else #endif /* CLASSQ_RIO */ #if CLASSQ_RED if (cl->cl_qtype == Q_RED) return (red_addq(cl->cl_red, &b->queue, m, t)); else #endif /* CLASSQ_RED */ #if CLASSQ_BLUE if (cl->cl_qtype == Q_BLUE) return (blue_addq(cl->cl_blue, &b->queue, m, t)); else #endif /* CLASSQ_BLUE */ if (cl->cl_qtype == Q_SFB) { if (cl->cl_sfb == NULL) { struct ifnet *ifp = FAIRQIF_IFP(cl->cl_fif); VERIFY(cl->cl_flags & FARF_LAZY); IFCQ_CONVERT_LOCK(ifq); cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, cl->cl_qlimit, cl->cl_qflags); if (cl->cl_sfb == NULL) { /* fall back to droptail */ cl->cl_qtype = Q_DROPTAIL; cl->cl_flags &= ~FARF_SFB; cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL); log(LOG_ERR, "%s: %s SFB lazy allocation " "failed for qid=%d pri=%d, falling back " "to DROPTAIL\n", if_name(ifp), fairq_style(cl->cl_fif), cl->cl_handle, cl->cl_pri); } } if (cl->cl_sfb != NULL) return (sfb_addq(cl->cl_sfb, &b->queue, m, t)); } else if (qlen(&b->queue) >= qlimit(&b->queue)) { IFCQ_CONVERT_LOCK(ifq); m_freem(m); return (CLASSQEQ_DROPPED); } #if PF_ECN if (cl->cl_flags & FARF_CLEARDSCP) write_dsfield(m, t, 0); #endif /* PF_ECN */ _addq(&b->queue, m); return (0); }
static u_int32_t sfb_random(struct sfb *sp) { IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); return (RandomULong()); }
static inline int priq_addq(struct priq_class *cl, struct mbuf *m, struct pf_mtag *t) { struct priq_if *pif = cl->cl_pif; struct ifclassq *ifq = pif->pif_ifq; IFCQ_LOCK_ASSERT_HELD(ifq); #if CLASSQ_RIO if (q_is_rio(&cl->cl_q)) return (rio_addq(cl->cl_rio, &cl->cl_q, m, t)); else #endif /* CLASSQ_RIO */ #if CLASSQ_RED if (q_is_red(&cl->cl_q)) return (red_addq(cl->cl_red, &cl->cl_q, m, t)); else #endif /* CLASSQ_RED */ #if CLASSQ_BLUE if (q_is_blue(&cl->cl_q)) return (blue_addq(cl->cl_blue, &cl->cl_q, m, t)); else #endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q)) { if (cl->cl_sfb == NULL) { struct ifnet *ifp = PRIQIF_IFP(pif); VERIFY(cl->cl_flags & PRCF_LAZY); cl->cl_flags &= ~PRCF_LAZY; IFCQ_CONVERT_LOCK(ifq); cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, qlimit(&cl->cl_q), cl->cl_qflags); if (cl->cl_sfb == NULL) { /* fall back to droptail */ qtype(&cl->cl_q) = Q_DROPTAIL; cl->cl_flags &= ~PRCF_SFB; cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL); log(LOG_ERR, "%s: %s SFB lazy allocation " "failed for qid=%d pri=%d, falling back " "to DROPTAIL\n", if_name(ifp), priq_style(pif), cl->cl_handle, cl->cl_pri); } else if (pif->pif_throttle != IFNET_THROTTLE_OFF) { /* if there's pending throttling, set it */ cqrq_throttle_t tr = { 1, pif->pif_throttle }; int err = priq_throttle(pif, &tr); if (err == EALREADY) err = 0; if (err != 0) { tr.level = IFNET_THROTTLE_OFF; (void) priq_throttle(pif, &tr); } } } if (cl->cl_sfb != NULL) return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t)); } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) { IFCQ_CONVERT_LOCK(ifq); m_freem(m); return (CLASSQEQ_DROPPED); } #if PF_ECN if (cl->cl_flags & PRCF_CLEARDSCP) write_dsfield(m, t, 0); #endif /* PF_ECN */ _addq(&cl->cl_q, m); return (0); }
int priq_enqueue(struct priq_if *pif, struct priq_class *cl, struct mbuf *m, struct pf_mtag *t) { struct ifclassq *ifq = pif->pif_ifq; u_int32_t pri; int len, ret; IFCQ_LOCK_ASSERT_HELD(ifq); VERIFY(cl == NULL || cl->cl_pif == pif); if (cl == NULL) { #if PF_ALTQ cl = priq_clh_to_clp(pif, t->pftag_qid); #else /* !PF_ALTQ */ cl = priq_clh_to_clp(pif, 0); #endif /* !PF_ALTQ */ if (cl == NULL) { cl = pif->pif_default; if (cl == NULL) { IFCQ_CONVERT_LOCK(ifq); m_freem(m); return (ENOBUFS); } } } pri = cl->cl_pri; VERIFY(pri < PRIQ_MAXPRI); len = m_pktlen(m); ret = priq_addq(cl, m, t); if (ret != 0) { if (ret == CLASSQEQ_SUCCESS_FC) { /* packet enqueued, return advisory feedback */ ret = EQFULL; } else { VERIFY(ret == CLASSQEQ_DROPPED || ret == CLASSQEQ_DROPPED_FC || ret == CLASSQEQ_DROPPED_SP); /* packet has been freed in priq_addq */ PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); IFCQ_DROP_ADD(ifq, 1, len); switch (ret) { case CLASSQEQ_DROPPED: return (ENOBUFS); case CLASSQEQ_DROPPED_FC: return (EQFULL); case CLASSQEQ_DROPPED_SP: return (EQSUSPENDED); } /* NOT REACHED */ } } IFCQ_INC_LEN(ifq); IFCQ_INC_BYTES(ifq, len); /* class is now active; indicate it as such */ if (!pktsched_bit_tst(pri, &pif->pif_bitmap)) pktsched_bit_set(pri, &pif->pif_bitmap); /* successfully queued. */ return (ret); }