static int
codel_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr)
{
struct codel_if *cif = (struct codel_if *) ifq->altq_disc;
IFQ_LOCK_ASSERT(ifq);
/* grab class set by classifier */
if ((m->m_flags & M_PKTHDR) == 0) {
/* should not happen */
printf("altq: packet for %s does not have pkthdr\n",
ifq->altq_ifp->if_xname);
m_freem(m);
PKTCNTR_ADD(&cif->cl_stats.cl_dropcnt, m_pktlen(m));
return (ENOBUFS);
}
if (codel_addq(&cif->codel, cif->cl_q, m)) {
PKTCNTR_ADD(&cif->cl_stats.cl_dropcnt, m_pktlen(m));
return (ENOBUFS);
}
IFQ_INC_LEN(ifq);
return (0);
}
static struct mbuf *
tcq_dequeue_cl(struct tcq_if *tif, struct tcq_class *cl,
mbuf_svc_class_t sc, cqdq_op_t op)
{
struct ifclassq *ifq = tif->tif_ifq;
struct mbuf *m;
IFCQ_LOCK_ASSERT_HELD(ifq);
if (cl == NULL) {
cl = tcq_clh_to_clp(tif, MBUF_SCIDX(sc));
if (cl == NULL)
return (NULL);
}
if (qempty(&cl->cl_q))
return (NULL);
VERIFY(!IFCQ_IS_EMPTY(ifq));
if (op == CLASSQDQ_POLL)
return (tcq_pollq(cl));
m = tcq_getq(cl);
if (m != NULL) {
IFCQ_DEC_LEN(ifq);
IFCQ_DEC_BYTES(ifq, m_pktlen(m));
if (qempty(&cl->cl_q))
cl->cl_period++;
PKTCNTR_ADD(&cl->cl_xmitcnt, 1, m_pktlen(m));
IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m));
}
return (m);
}
/* Network Interface functions */
static errno_t
utun_output(
ifnet_t interface,
mbuf_t data)
{
struct utun_pcb *pcb = ifnet_softc(interface);
errno_t result;
if (m_pktlen(data) >= 4) {
bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
}
if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
/* flush data */
mbuf_freem(data);
return 0;
}
// otherwise, fall thru to ctl_enqueumbuf
if (pcb->utun_ctlref) {
int length;
// only pass packets to utun-crypto if crypto is enabled and 'suspend data traffic' is not.
if ((pcb->utun_flags & (UTUN_FLAGS_CRYPTO | UTUN_FLAGS_CRYPTO_STOP_DATA_TRAFFIC)) == UTUN_FLAGS_CRYPTO) {
if (utun_pkt_crypto_output(pcb, &data) == 0) {
return 0;
}
}
/*
* The ABI requires the protocol in network byte order
*/
if (m_pktlen(data) >= 4)
*(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
length = mbuf_pkthdr_len(data);
result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
if (result != 0) {
mbuf_freem(data);
printf("utun_output - ctl_enqueuembuf failed: %d\n", result);
ifnet_stat_increment_out(interface, 0, 0, 1);
}
else {
if (!pcb->utun_ext_ifdata_stats)
ifnet_stat_increment_out(interface, 1, length, 0);
}
}
else
mbuf_freem(data);
return 0;
}
static errno_t
utun_output(ifnet_t interface,
mbuf_t data)
{
struct utun_pcb *pcb = ifnet_softc(interface);
errno_t result;
VERIFY(interface == pcb->utun_ifp);
if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
}
if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
/* flush data */
mbuf_freem(data);
return 0;
}
// otherwise, fall thru to ctl_enqueumbuf
if (pcb->utun_ctlref) {
int length;
/*
* The ABI requires the protocol in network byte order
*/
if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
*(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
}
length = mbuf_pkthdr_len(data);
result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
if (result != 0) {
mbuf_freem(data);
printf("utun_output - ctl_enqueuembuf failed: %d\n", result);
ifnet_stat_increment_out(interface, 0, 0, 1);
}
else {
if (!pcb->utun_ext_ifdata_stats)
ifnet_stat_increment_out(interface, 1, length, 0);
}
}
else
mbuf_freem(data);
return 0;
}
static int
codel_request(struct ifaltq *ifq, int req, void *arg)
{
struct codel_if *cif = (struct codel_if *)ifq->altq_disc;
struct mbuf *m;
IFQ_LOCK_ASSERT(ifq);
switch (req) {
case ALTRQ_PURGE:
if (!ALTQ_IS_ENABLED(cif->cif_ifq))
break;
if (qempty(cif->cl_q))
break;
while ((m = _getq(cif->cl_q)) != NULL) {
PKTCNTR_ADD(&cif->cl_stats.cl_dropcnt, m_pktlen(m));
m_freem(m);
IFQ_DEC_LEN(cif->cif_ifq);
}
cif->cif_ifq->ifq_len = 0;
break;
}
return (0);
}
/*
* priq_dequeue is a dequeue function to be registered to
* (*altq_dequeue) in struct ifaltq.
*
* note: ALTDQ_POLL returns the next packet without removing the packet
* from the queue. ALTDQ_REMOVE is a normal dequeue operation.
* ALTDQ_REMOVE must return the same packet if called immediately
* after ALTDQ_POLL.
*/
static struct mbuf *
priq_dequeue(struct ifaltq *ifq, int op)
{
struct priq_if *pif = (struct priq_if *)ifq->altq_disc;
struct priq_class *cl;
struct mbuf *m;
int pri;
if (IFQ_IS_EMPTY(ifq))
/* no packet in the queue */
return (NULL);
for (pri = pif->pif_maxpri; pri >= 0; pri--) {
if ((cl = pif->pif_classes[pri]) != NULL &&
!qempty(cl->cl_q)) {
if (op == ALTDQ_POLL)
return (priq_pollq(cl));
m = priq_getq(cl);
if (m != NULL) {
IFQ_DEC_LEN(ifq);
if (qempty(cl->cl_q))
cl->cl_period++;
PKTCNTR_ADD(&cl->cl_xmitcnt, m_pktlen(m));
}
return (m);
}
}
return (NULL);
}
int
fairq_enqueue(struct fairq_if *fif, struct fairq_class *cl, struct mbuf *m,
struct pf_mtag *t)
{
struct ifclassq *ifq = fif->fif_ifq;
int len, ret;
IFCQ_LOCK_ASSERT_HELD(ifq);
VERIFY(cl == NULL || cl->cl_fif == fif);
if (cl == NULL) {
#if PF_ALTQ
cl = fairq_clh_to_clp(fif, t->pftag_qid);
#else /* !PF_ALTQ */
cl = fairq_clh_to_clp(fif, 0);
#endif /* !PF_ALTQ */
if (cl == NULL) {
cl = fif->fif_default;
if (cl == NULL) {
IFCQ_CONVERT_LOCK(ifq);
m_freem(m);
return (ENOBUFS);
}
}
}
cl->cl_flags |= FARF_HAS_PACKETS;
len = m_pktlen(m);
ret = fairq_addq(cl, m, t);
if (ret != 0) {
if (ret == CLASSQEQ_SUCCESS_FC) {
/* packet enqueued, return advisory feedback */
ret = EQFULL;
} else {
VERIFY(ret == CLASSQEQ_DROPPED ||
ret == CLASSQEQ_DROPPED_FC ||
ret == CLASSQEQ_DROPPED_SP);
/* packet has been freed in fairq_addq */
PKTCNTR_ADD(&cl->cl_dropcnt, 1, len);
IFCQ_DROP_ADD(ifq, 1, len);
switch (ret) {
case CLASSQEQ_DROPPED:
return (ENOBUFS);
case CLASSQEQ_DROPPED_FC:
return (EQFULL);
case CLASSQEQ_DROPPED_SP:
return (EQSUSPENDED);
}
/* NOT REACHED */
}
}
IFCQ_INC_LEN(ifq);
IFCQ_INC_BYTES(ifq, len);
/* successfully queued. */
return (ret);
}
/*
* priq_enqueue is an enqueue function to be registered to
* (*altq_enqueue) in struct ifaltq.
*/
static int
priq_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m,
struct altq_pktattr *pktattr)
{
struct ifaltq *ifq = ifsq->ifsq_altq;
struct priq_if *pif = (struct priq_if *)ifq->altq_disc;
struct priq_class *cl;
int error;
int len;
if (ifsq_get_index(ifsq) != PRIQ_SUBQ_INDEX) {
/*
* Race happened, the unrelated subqueue was
* picked during the packet scheduler transition.
*/
ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL);
m_freem(m);
return ENOBUFS;
}
crit_enter();
/* grab class set by classifier */
if ((m->m_flags & M_PKTHDR) == 0) {
/* should not happen */
if_printf(ifq->altq_ifp, "altq: packet does not have pkthdr\n");
m_freem(m);
error = ENOBUFS;
goto done;
}
if (m->m_pkthdr.fw_flags & PF_MBUF_STRUCTURE)
cl = clh_to_clp(pif, m->m_pkthdr.pf.qid);
else
cl = NULL;
if (cl == NULL) {
cl = pif->pif_default;
if (cl == NULL) {
m_freem(m);
error = ENOBUFS;
goto done;
}
}
cl->cl_pktattr = NULL;
len = m_pktlen(m);
if (priq_addq(cl, m) != 0) {
/* drop occurred. mbuf was freed in priq_addq. */
PKTCNTR_ADD(&cl->cl_dropcnt, len);
error = ENOBUFS;
goto done;
}
ifsq->ifq_len++;
error = 0;
done:
crit_exit();
return (error);
}
static int
cbq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr)
{
cbq_state_t *cbqp = (cbq_state_t *)ifq->altq_disc;
struct rm_class *cl;
struct m_tag *t;
int len;
IFQ_LOCK_ASSERT(ifq);
/* grab class set by classifier */
if ((m->m_flags & M_PKTHDR) == 0) {
/* should not happen */
#if defined(__NetBSD__) || defined(__OpenBSD__)\
|| (defined(__FreeBSD__) && __FreeBSD_version >= 501113)
printf("altq: packet for %s does not have pkthdr\n",
ifq->altq_ifp->if_xname);
#else
printf("altq: packet for %s%d does not have pkthdr\n",
ifq->altq_ifp->if_name, ifq->altq_ifp->if_unit);
#endif
m_freem(m);
return (ENOBUFS);
}
cl = NULL;
if ((t = m_tag_find(m, PACKET_TAG_PF_QID, NULL)) != NULL)
cl = clh_to_clp(cbqp, ((struct altq_tag *)(t+1))->qid);
#ifdef ALTQ3_COMPAT
else if ((ifq->altq_flags & ALTQF_CLASSIFY) && pktattr != NULL)
cl = pktattr->pattr_class;
#endif
if (cl == NULL) {
cl = cbqp->ifnp.default_;
if (cl == NULL) {
m_freem(m);
return (ENOBUFS);
}
}
#ifdef ALTQ3_COMPAT
if (pktattr != NULL)
cl->pktattr_ = pktattr; /* save proto hdr used by ECN */
else
#endif
cl->pktattr_ = NULL;
len = m_pktlen(m);
if (rmc_queue_packet(cl, m) != 0) {
/* drop occurred. some mbuf was freed in rmc_queue_packet. */
PKTCNTR_ADD(&cl->stats_.drop_cnt, len);
return (ENOBUFS);
}
/* successfully queued. */
++cbqp->cbq_qlen;
IFQ_INC_LEN(ifq);
return (0);
}
static errno_t
utun_ctl_send(
__unused kern_ctl_ref kctlref,
__unused u_int32_t unit,
void *unitinfo,
mbuf_t m,
__unused int flags)
{
/*
* The userland ABI requires the first four bytes have the protocol family
* in network byte order: swap them
*/
if (m_pktlen(m) >= 4)
*(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
else
printf("%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m) );
return utun_pkt_input((struct utun_pcb *)unitinfo, m);
}
/*
* priq_enqueue is an enqueue function to be registered to
* (*altq_enqueue) in struct ifaltq.
*/
static int
priq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr)
{
struct priq_if *pif = (struct priq_if *)ifq->altq_disc;
struct priq_class *cl;
struct pf_mtag *t;
int len;
IFQ_LOCK_ASSERT(ifq);
/* grab class set by classifier */
if ((m->m_flags & M_PKTHDR) == 0) {
/* should not happen */
#if defined(__NetBSD__) || defined(__OpenBSD__)\
|| (defined(__FreeBSD__) && __FreeBSD_version >= 501113)
printf("altq: packet for %s does not have pkthdr\n",
ifq->altq_ifp->if_xname);
#else
printf("altq: packet for %s%d does not have pkthdr\n",
ifq->altq_ifp->if_name, ifq->altq_ifp->if_unit);
#endif
m_freem(m);
return (ENOBUFS);
}
cl = NULL;
if ((t = pf_find_mtag(m)) != NULL)
cl = clh_to_clp(pif, t->qid);
#ifdef ALTQ3_COMPAT
else if ((ifq->altq_flags & ALTQF_CLASSIFY) && pktattr != NULL)
cl = pktattr->pattr_class;
#endif
if (cl == NULL) {
cl = pif->pif_default;
if (cl == NULL) {
m_freem(m);
return (ENOBUFS);
}
}
#ifdef ALTQ3_COMPAT
if (pktattr != NULL)
cl->cl_pktattr = pktattr; /* save proto hdr used by ECN */
else
#endif
cl->cl_pktattr = NULL;
len = m_pktlen(m);
if (priq_addq(cl, m) != 0) {
/* drop occurred. mbuf was freed in priq_addq. */
PKTCNTR_ADD(&cl->cl_dropcnt, len);
return (ENOBUFS);
}
IFQ_INC_LEN(ifq);
/* successfully queued. */
return (0);
}
struct mbuf *
tbr_dequeue(struct ifaltq_subque *ifsq, int op)
{
struct ifaltq *ifq = ifsq->ifsq_altq;
struct tb_regulator *tbr;
struct mbuf *m;
int64_t interval;
uint64_t now;
if (ifsq_get_index(ifsq) != ALTQ_SUBQ_INDEX_DEFAULT) {
/*
* Race happened, the unrelated subqueue was
* picked during the packet scheduler transition.
*/
ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL);
return NULL;
}
crit_enter();
tbr = ifq->altq_tbr;
if (op == ALTDQ_REMOVE && tbr->tbr_lastop == ALTDQ_POLL) {
/* if this is a remove after poll, bypass tbr check */
} else {
/* update token only when it is negative */
if (tbr->tbr_token <= 0) {
now = read_machclk();
interval = now - tbr->tbr_last;
if (interval >= tbr->tbr_filluptime)
tbr->tbr_token = tbr->tbr_depth;
else {
tbr->tbr_token += interval * tbr->tbr_rate;
if (tbr->tbr_token > tbr->tbr_depth)
tbr->tbr_token = tbr->tbr_depth;
}
tbr->tbr_last = now;
}
/* if token is still negative, don't allow dequeue */
if (tbr->tbr_token <= 0) {
crit_exit();
return (NULL);
}
}
if (ifq_is_enabled(ifq))
m = (*ifsq->ifsq_dequeue)(ifsq, op);
else
m = ifsq_classic_dequeue(ifsq, op);
if (m != NULL && op == ALTDQ_REMOVE)
tbr->tbr_token -= TBR_SCALE(m_pktlen(m));
tbr->tbr_lastop = op;
crit_exit();
return (m);
}
static void
priq_purgeq(struct priq_class *cl)
{
struct mbuf *m;
if (qempty(cl->cl_q))
return;
while ((m = _getq(cl->cl_q)) != NULL) {
PKTCNTR_ADD(&cl->cl_dropcnt, m_pktlen(m));
m_freem(m);
}
KKASSERT(qlen(cl->cl_q) == 0);
}
errno_t
utun_pkt_input (struct utun_pcb *pcb, mbuf_t m)
{
errno_t result;
protocol_family_t protocol = 0;
mbuf_pkthdr_setrcvif(m, pcb->utun_ifp);
if (m_pktlen(m) >= 4) {
protocol = *(u_int32_t *)mbuf_data(m);
bpf_tap_in(pcb->utun_ifp, DLT_NULL, m, 0, 0);
}
if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
/* flush data */
mbuf_freem(m);
return 0;
}
// quick exit for keepalive packets
if (protocol == AF_UTUN && pcb->utun_flags & UTUN_FLAGS_CRYPTO) {
if (utun_pkt_crypto_output(pcb, &m) == 0) {
return 0;
}
printf("%s: utun_pkt_crypto_output failed, flags %x\n", __FUNCTION__, pcb->utun_flags);
return EINVAL;
}
if (!pcb->utun_ext_ifdata_stats) {
struct ifnet_stat_increment_param incs;
bzero(&incs, sizeof(incs));
incs.packets_in = 1;
incs.bytes_in = mbuf_pkthdr_len(m);
result = ifnet_input(pcb->utun_ifp, m, &incs);
} else {
result = ifnet_input(pcb->utun_ifp, m, NULL);
}
if (result != 0) {
ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
mbuf_freem(m);
}
return 0;
}
/*
* priq_dequeue is a dequeue function to be registered to
* (*altq_dequeue) in struct ifaltq.
*
* note: ALTDQ_POLL returns the next packet without removing the packet
* from the queue. ALTDQ_REMOVE is a normal dequeue operation.
* ALTDQ_REMOVE must return the same packet if called immediately
* after ALTDQ_POLL.
*/
static struct mbuf *
priq_dequeue(struct ifaltq_subque *ifsq, struct mbuf *mpolled, int op)
{
struct ifaltq *ifq = ifsq->ifsq_altq;
struct priq_if *pif = (struct priq_if *)ifq->altq_disc;
struct priq_class *cl;
struct mbuf *m;
int pri;
if (ifsq_get_index(ifsq) != PRIQ_SUBQ_INDEX) {
/*
* Race happened, the unrelated subqueue was
* picked during the packet scheduler transition.
*/
ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL);
return NULL;
}
if (ifsq_is_empty(ifsq)) {
/* no packet in the queue */
KKASSERT(mpolled == NULL);
return (NULL);
}
crit_enter();
m = NULL;
for (pri = pif->pif_maxpri; pri >= 0; pri--) {
if ((cl = pif->pif_classes[pri]) != NULL && !qempty(cl->cl_q)) {
if (op == ALTDQ_POLL) {
m = priq_pollq(cl);
break;
}
m = priq_getq(cl);
if (m != NULL) {
ifsq->ifq_len--;
if (qempty(cl->cl_q))
cl->cl_period++;
PKTCNTR_ADD(&cl->cl_xmitcnt, m_pktlen(m));
}
break;
}
}
crit_exit();
KKASSERT(mpolled == NULL || mpolled == m);
return (m);
}
/*
* priq_enqueue is an enqueue function to be registered to
* (*altq_enqueue) in struct ifaltq.
*/
static int
priq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr)
{
struct priq_if *pif = (struct priq_if *)ifq->altq_disc;
struct priq_class *cl;
struct m_tag *t;
int len;
/* grab class set by classifier */
if ((m->m_flags & M_PKTHDR) == 0) {
/* should not happen */
printf("altq: packet for %s does not have pkthdr\n",
ifq->altq_ifp->if_xname);
m_freem(m);
return (ENOBUFS);
}
cl = NULL;
if ((t = m_tag_find(m, PACKET_TAG_ALTQ_QID, NULL)) != NULL)
cl = clh_to_clp(pif, ((struct altq_tag *)(t+1))->qid);
#ifdef ALTQ3_COMPAT
else if ((ifq->altq_flags & ALTQF_CLASSIFY) && pktattr != NULL)
cl = pktattr->pattr_class;
#endif
if (cl == NULL) {
cl = pif->pif_default;
if (cl == NULL) {
m_freem(m);
return (ENOBUFS);
}
}
#ifdef ALTQ3_COMPAT
if (pktattr != NULL)
cl->cl_pktattr = pktattr; /* save proto hdr used by ECN */
else
#endif
cl->cl_pktattr = NULL;
len = m_pktlen(m);
if (priq_addq(cl, m) != 0) {
/* drop occurred. mbuf was freed in priq_addq. */
PKTCNTR_ADD(&cl->cl_dropcnt, len);
return (ENOBUFS);
}
IFQ_INC_LEN(ifq);
/* successfully queued. */
return (0);
}
void
pktsched_pkt_encap(pktsched_pkt_t *pkt, classq_pkt_type_t ptype, void *pp)
{
pkt->pktsched_ptype = ptype;
pkt->pktsched_pkt = pp;
switch (ptype) {
case QP_MBUF:
pkt->pktsched_plen =
(uint32_t)m_pktlen((struct mbuf *)pkt->pktsched_pkt);
break;
default:
VERIFY(0);
/* NOTREACHED */
}
}
static int
codel_should_drop(struct codel *c, class_queue_t *q, struct mbuf *m,
u_int64_t now)
{
struct m_tag *mtag;
uint64_t *enqueue_time;
if (m == NULL) {
c->vars.first_above_time = 0;
return (0);
}
mtag = m_tag_locate(m, MTAG_CODEL, 0, NULL);
if (mtag == NULL) {
/* Only one warning per second. */
if (ppsratecheck(&c->last_log, &c->last_pps, 1))
printf("%s: could not found the packet mtag!\n",
__func__);
c->vars.first_above_time = 0;
return (0);
}
enqueue_time = (uint64_t *)(mtag + 1);
c->vars.ldelay = now - *enqueue_time;
c->stats.maxpacket = MAX(c->stats.maxpacket, m_pktlen(m));
if (codel_time_before(c->vars.ldelay, c->params.target) ||
qsize(q) <= c->stats.maxpacket) {
/* went below - stay below for at least interval */
c->vars.first_above_time = 0;
return (0);
}
if (c->vars.first_above_time == 0) {
/* just went above from below. If we stay above
* for at least interval we'll say it's ok to drop
*/
c->vars.first_above_time = now + c->params.interval;
return (0);
}
if (codel_time_after(now, c->vars.first_above_time))
return (1);
return (0);
}
/*
* note: CLASSQDQ_POLL returns the next packet without removing the packet
* from the queue. CLASSQDQ_REMOVE is a normal dequeue operation.
* CLASSQDQ_REMOVE must return the same packet if called immediately
* after CLASSQDQ_POLL.
*/
struct mbuf *
priq_dequeue(struct priq_if *pif, cqdq_op_t op)
{
struct ifclassq *ifq = pif->pif_ifq;
struct priq_class *cl;
struct mbuf *m;
u_int32_t pri, len;
IFCQ_LOCK_ASSERT_HELD(ifq);
if (pif->pif_bitmap == 0) {
/* no active class; nothing to dequeue */
return (NULL);
}
VERIFY(!IFCQ_IS_EMPTY(ifq));
pri = pktsched_fls(pif->pif_bitmap) - 1; /* zero based */
VERIFY(pri < PRIQ_MAXPRI);
cl = pif->pif_classes[pri];
VERIFY(cl != NULL && !qempty(&cl->cl_q));
if (op == CLASSQDQ_POLL)
return (priq_pollq(cl));
m = priq_getq(cl);
VERIFY(m != NULL); /* qalg must be work conserving */
len = m_pktlen(m);
IFCQ_DEC_LEN(ifq);
IFCQ_DEC_BYTES(ifq, len);
if (qempty(&cl->cl_q)) {
cl->cl_period++;
/* class is now inactive; indicate it as such */
pktsched_bit_clr(pri, &pif->pif_bitmap);
}
PKTCNTR_ADD(&cl->cl_xmitcnt, 1, len);
IFCQ_XMIT_ADD(ifq, 1, len);
return (m);
}
static errno_t
utun_pkt_input (struct utun_pcb *pcb, mbuf_t m)
{
errno_t result;
protocol_family_t protocol = 0;
mbuf_pkthdr_setrcvif(m, pcb->utun_ifp);
if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
protocol = *(u_int32_t *)mbuf_data(m);
bpf_tap_in(pcb->utun_ifp, DLT_NULL, m, 0, 0);
}
if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
/* flush data */
mbuf_freem(m);
return 0;
}
if (!pcb->utun_ext_ifdata_stats) {
struct ifnet_stat_increment_param incs;
bzero(&incs, sizeof(incs));
incs.packets_in = 1;
incs.bytes_in = mbuf_pkthdr_len(m);
result = ifnet_input(pcb->utun_ifp, m, &incs);
} else {
result = ifnet_input(pcb->utun_ifp, m, NULL);
}
if (result != 0) {
ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
mbuf_freem(m);
}
return 0;
}
static int
cbq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr)
{
cbq_state_t *cbqp = (cbq_state_t *)ifq->altq_disc;
struct rm_class *cl;
int len;
/* grab class set by classifier */
if ((m->m_flags & M_PKTHDR) == 0) {
/* should not happen */
printf("altq: packet for %s does not have pkthdr\n",
ifq->altq_ifp->if_xname);
m_freem(m);
return (ENOBUFS);
}
if ((cl = clh_to_clp(cbqp, m->m_pkthdr.pf.qid)) == NULL) {
cl = cbqp->ifnp.default_;
if (cl == NULL) {
m_freem(m);
return (ENOBUFS);
}
cl->pktattr_ = NULL;
}
len = m_pktlen(m);
if (rmc_queue_packet(cl, m) != 0) {
/* drop occurred. some mbuf was freed in rmc_queue_packet. */
PKTCNTR_ADD(&cl->stats_.drop_cnt, len);
return (ENOBUFS);
}
/* successfully queued. */
++cbqp->cbq_qlen;
IFQ_INC_LEN(ifq);
return (0);
}
static struct mbuf *
codel_dequeue(struct ifaltq *ifq, int op)
{
struct codel_if *cif = (struct codel_if *)ifq->altq_disc;
struct mbuf *m;
IFQ_LOCK_ASSERT(ifq);
if (IFQ_IS_EMPTY(ifq))
return (NULL);
if (op == ALTDQ_POLL)
return (qhead(cif->cl_q));
m = codel_getq(&cif->codel, cif->cl_q);
if (m != NULL) {
IFQ_DEC_LEN(ifq);
PKTCNTR_ADD(&cif->cl_stats.cl_xmitcnt, m_pktlen(m));
return (m);
}
return (NULL);
}
struct mbuf *
codel_getq(struct codel *c, class_queue_t *q)
{
struct mbuf *m;
u_int64_t now;
int drop;
if ((m = _getq(q)) == NULL) {
c->vars.dropping = 0;
return (m);
}
now = read_machclk();
drop = codel_should_drop(c, q, m, now);
if (c->vars.dropping) {
if (!drop) {
/* sojourn time below target - leave dropping state */
c->vars.dropping = 0;
} else if (codel_time_after_eq(now, c->vars.drop_next)) {
/* It's time for the next drop. Drop the current
* packet and dequeue the next. The dequeue might
* take us out of dropping state.
* If not, schedule the next drop.
* A large backlog might result in drop rates so high
* that the next drop should happen now,
* hence the while loop.
*/
while (c->vars.dropping &&
codel_time_after_eq(now, c->vars.drop_next)) {
c->vars.count++; /* don't care of possible wrap
* since there is no more
* divide */
codel_Newton_step(&c->vars);
/* TODO ECN */
PKTCNTR_ADD(&c->stats.drop_cnt, m_pktlen(m));
m_freem(m);
m = _getq(q);
if (!codel_should_drop(c, q, m, now))
/* leave dropping state */
c->vars.dropping = 0;
else
/* and schedule the next drop */
c->vars.drop_next =
codel_control_law(c->vars.drop_next,
c->params.interval,
c->vars.rec_inv_sqrt);
}
}
} else if (drop) {
/* TODO ECN */
PKTCNTR_ADD(&c->stats.drop_cnt, m_pktlen(m));
m_freem(m);
m = _getq(q);
drop = codel_should_drop(c, q, m, now);
c->vars.dropping = 1;
/* if min went above target close to when we last went below it
* assume that the drop rate that controlled the queue on the
* last cycle is a good starting point to control it now.
*/
if (codel_time_before(now - c->vars.drop_next,
16 * c->params.interval)) {
c->vars.count = (c->vars.count - c->vars.lastcount) | 1;
/* we dont care if rec_inv_sqrt approximation
* is not very precise :
* Next Newton steps will correct it quadratically.
*/
codel_Newton_step(&c->vars);
} else {
c->vars.count = 1;
c->vars.rec_inv_sqrt = ~0U >> REC_INV_SQRT_SHIFT;
}
c->vars.lastcount = c->vars.count;
c->vars.drop_next = codel_control_law(now, c->params.interval,
c->vars.rec_inv_sqrt);
}
return (m);
}
int
priq_enqueue(struct priq_if *pif, struct priq_class *cl, struct mbuf *m,
struct pf_mtag *t)
{
struct ifclassq *ifq = pif->pif_ifq;
u_int32_t pri;
int len, ret;
IFCQ_LOCK_ASSERT_HELD(ifq);
VERIFY(cl == NULL || cl->cl_pif == pif);
if (cl == NULL) {
#if PF_ALTQ
cl = priq_clh_to_clp(pif, t->pftag_qid);
#else /* !PF_ALTQ */
cl = priq_clh_to_clp(pif, 0);
#endif /* !PF_ALTQ */
if (cl == NULL) {
cl = pif->pif_default;
if (cl == NULL) {
IFCQ_CONVERT_LOCK(ifq);
m_freem(m);
return (ENOBUFS);
}
}
}
pri = cl->cl_pri;
VERIFY(pri < PRIQ_MAXPRI);
len = m_pktlen(m);
ret = priq_addq(cl, m, t);
if (ret != 0) {
if (ret == CLASSQEQ_SUCCESS_FC) {
/* packet enqueued, return advisory feedback */
ret = EQFULL;
} else {
VERIFY(ret == CLASSQEQ_DROPPED ||
ret == CLASSQEQ_DROPPED_FC ||
ret == CLASSQEQ_DROPPED_SP);
/* packet has been freed in priq_addq */
PKTCNTR_ADD(&cl->cl_dropcnt, 1, len);
IFCQ_DROP_ADD(ifq, 1, len);
switch (ret) {
case CLASSQEQ_DROPPED:
return (ENOBUFS);
case CLASSQEQ_DROPPED_FC:
return (EQFULL);
case CLASSQEQ_DROPPED_SP:
return (EQSUSPENDED);
}
/* NOT REACHED */
}
}
IFCQ_INC_LEN(ifq);
IFCQ_INC_BYTES(ifq, len);
/* class is now active; indicate it as such */
if (!pktsched_bit_tst(pri, &pif->pif_bitmap))
pktsched_bit_set(pri, &pif->pif_bitmap);
/* successfully queued. */
return (ret);
}
/*
* note: CLASSQDQ_POLL returns the next packet without removing the packet
* from the queue. CLASSQDQ_REMOVE is a normal dequeue operation.
* CLASSQDQ_REMOVE must return the same packet if called immediately
* after CLASSQDQ_POLL.
*/
struct mbuf *
fairq_dequeue(struct fairq_if *fif, cqdq_op_t op)
{
struct ifclassq *ifq = fif->fif_ifq;
struct fairq_class *cl;
struct fairq_class *best_cl;
struct mbuf *best_m;
struct mbuf *m;
u_int64_t cur_time = read_machclk();
u_int32_t best_scale;
u_int32_t scale;
int pri;
int hit_limit;
IFCQ_LOCK_ASSERT_HELD(ifq);
if (IFCQ_IS_EMPTY(ifq)) {
/* no packet in the queue */
return (NULL);
}
if (fif->fif_poll_cache && op == CLASSQDQ_REMOVE) {
best_cl = fif->fif_poll_cache;
m = fairq_getq(best_cl, cur_time);
fif->fif_poll_cache = NULL;
if (m != NULL) {
IFCQ_DEC_LEN(ifq);
IFCQ_DEC_BYTES(ifq, m_pktlen(m));
IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m));
PKTCNTR_ADD(&best_cl->cl_xmitcnt, 1, m_pktlen(m));
}
} else {
best_cl = NULL;
best_m = NULL;
best_scale = 0xFFFFFFFFU;
for (pri = fif->fif_maxpri; pri >= 0; pri--) {
if ((cl = fif->fif_classes[pri]) == NULL)
continue;
if ((cl->cl_flags & FARF_HAS_PACKETS) == 0)
continue;
m = fairq_pollq(cl, cur_time, &hit_limit);
if (m == NULL) {
cl->cl_flags &= ~FARF_HAS_PACKETS;
continue;
}
/*
* We can halt the search immediately if the queue
* did not hit its bandwidth limit.
*/
if (hit_limit == 0) {
best_cl = cl;
best_m = m;
break;
}
/*
* Otherwise calculate the scale factor and select
* the queue with the lowest scale factor. This
* apportions any unused bandwidth weighted by
* the relative bandwidth specification.
*/
scale = cl->cl_bw_current * 100 / cl->cl_bandwidth;
if (scale < best_scale) {
best_cl = cl;
best_m = m;
best_scale = scale;
}
}
if (op == CLASSQDQ_POLL) {
fif->fif_poll_cache = best_cl;
m = best_m;
} else if (best_cl != NULL) {
m = fairq_getq(best_cl, cur_time);
if (m != NULL) {
IFCQ_DEC_LEN(ifq);
IFCQ_DEC_BYTES(ifq, m_pktlen(m));
IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m));
PKTCNTR_ADD(&best_cl->cl_xmitcnt, 1,
m_pktlen(m));
}
} else {
m = NULL;
}
}
return (m);
}
struct mbuf *
ip6_forward(struct mbuf *m, struct route_in6 *ip6forward_rt,
int srcrt)
{
struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
struct sockaddr_in6 *dst;
struct rtentry *rt;
int error, type = 0, code = 0;
boolean_t proxy = FALSE;
struct mbuf *mcopy = NULL;
struct ifnet *ifp, *rcvifp, *origifp; /* maybe unnecessary */
u_int32_t inzone, outzone, len;
struct in6_addr src_in6, dst_in6;
uint64_t curtime = net_uptime();
#if IPSEC
struct secpolicy *sp = NULL;
#endif
unsigned int ifscope = IFSCOPE_NONE;
#if PF
struct pf_mtag *pf_mtag;
#endif /* PF */
/*
* In the prefix proxying case, the route to the proxied node normally
* gets created by nd6_prproxy_ns_output(), as part of forwarding a
* NS (NUD/AR) packet to the proxied node. In the event that such
* packet did not arrive in time before the correct route gets created,
* ip6_input() would have performed a rtalloc() which most likely will
* create the wrong cloned route; this route points back to the same
* interface as the inbound interface, since the parent non-scoped
* prefix route points there. Therefore we check if that is the case
* and perform the necessary fixup to get the correct route installed.
*/
if (!srcrt && nd6_prproxy &&
(rt = ip6forward_rt->ro_rt) != NULL && (rt->rt_flags & RTF_PROXY)) {
nd6_proxy_find_fwdroute(m->m_pkthdr.rcvif, ip6forward_rt);
if ((rt = ip6forward_rt->ro_rt) != NULL)
ifscope = rt->rt_ifp->if_index;
}
#if PF
pf_mtag = pf_find_mtag(m);
if (pf_mtag != NULL && pf_mtag->pftag_rtableid != IFSCOPE_NONE)
ifscope = pf_mtag->pftag_rtableid;
/*
* If the caller provides a route which is on a different interface
* than the one specified for scoped forwarding, discard the route
* and do a lookup below.
*/
if (ifscope != IFSCOPE_NONE && (rt = ip6forward_rt->ro_rt) != NULL) {
RT_LOCK(rt);
if (rt->rt_ifp->if_index != ifscope) {
RT_UNLOCK(rt);
ROUTE_RELEASE(ip6forward_rt);
rt = NULL;
} else {
RT_UNLOCK(rt);
}
}
#endif /* PF */
#if IPSEC
/*
* Check AH/ESP integrity.
*/
/*
* Don't increment ip6s_cantforward because this is the check
* before forwarding packet actually.
*/
if (ipsec_bypass == 0) {
if (ipsec6_in_reject(m, NULL)) {
IPSEC_STAT_INCREMENT(ipsec6stat.in_polvio);
m_freem(m);
return (NULL);
}
}
#endif /*IPSEC*/
/*
* Do not forward packets to multicast destination.
* Do not forward packets with unspecified source. It was discussed
* in July 2000, on ipngwg mailing list.
*/
if ((m->m_flags & (M_BCAST|M_MCAST)) != 0 ||
IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst) ||
IN6_IS_ADDR_UNSPECIFIED(&ip6->ip6_src)) {
ip6stat.ip6s_cantforward++;
/* XXX in6_ifstat_inc(rt->rt_ifp, ifs6_in_discard) */
if (ip6_log_time + ip6_log_interval < curtime) {
ip6_log_time = curtime;
log(LOG_DEBUG,
"cannot forward "
"from %s to %s nxt %d received on %s\n",
ip6_sprintf(&ip6->ip6_src),
ip6_sprintf(&ip6->ip6_dst),
ip6->ip6_nxt,
if_name(m->m_pkthdr.rcvif));
}
m_freem(m);
return (NULL);
}
if (ip6->ip6_hlim <= IPV6_HLIMDEC) {
/* XXX in6_ifstat_inc(rt->rt_ifp, ifs6_in_discard) */
icmp6_error_flag(m, ICMP6_TIME_EXCEEDED,
ICMP6_TIME_EXCEED_TRANSIT, 0, 0);
return (NULL);
}
/*
* See if the destination is a proxied address, and if so pretend
* that it's for us. This is mostly to handle NUD probes against
* the proxied addresses. We filter for ICMPv6 here and will let
* icmp6_input handle the rest.
*/
if (!srcrt && nd6_prproxy) {
VERIFY(!IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst));
proxy = nd6_prproxy_isours(m, ip6, ip6forward_rt, ifscope);
/*
* Don't update hop limit while proxying; RFC 4389 4.1.
* Also skip IPsec forwarding path processing as this
* packet is not to be forwarded.
*/
if (proxy)
goto skip_ipsec;
}
ip6->ip6_hlim -= IPV6_HLIMDEC;
/*
* Save at most ICMPV6_PLD_MAXLEN (= the min IPv6 MTU -
* size of IPv6 + ICMPv6 headers) bytes of the packet in case
* we need to generate an ICMP6 message to the src.
* Thanks to M_EXT, in most cases copy will not occur.
*
* It is important to save it before IPsec processing as IPsec
* processing may modify the mbuf.
*/
mcopy = m_copy(m, 0, imin(m->m_pkthdr.len, ICMPV6_PLD_MAXLEN));
#if IPSEC
if (ipsec_bypass != 0)
goto skip_ipsec;
/* get a security policy for this packet */
sp = ipsec6_getpolicybyaddr(m, IPSEC_DIR_OUTBOUND, IP_FORWARDING,
&error);
if (sp == NULL) {
IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
ip6stat.ip6s_cantforward++;
if (mcopy) {
#if 0
/* XXX: what icmp ? */
#else
m_freem(mcopy);
#endif
}
m_freem(m);
return (NULL);
}
error = 0;
/* check policy */
switch (sp->policy) {
case IPSEC_POLICY_DISCARD:
case IPSEC_POLICY_GENERATE:
/*
* This packet is just discarded.
*/
IPSEC_STAT_INCREMENT(ipsec6stat.out_polvio);
ip6stat.ip6s_cantforward++;
key_freesp(sp, KEY_SADB_UNLOCKED);
if (mcopy) {
#if 0
/* XXX: what icmp ? */
#else
m_freem(mcopy);
#endif
}
m_freem(m);
return (NULL);
case IPSEC_POLICY_BYPASS:
case IPSEC_POLICY_NONE:
/* no need to do IPsec. */
key_freesp(sp, KEY_SADB_UNLOCKED);
goto skip_ipsec;
case IPSEC_POLICY_IPSEC:
if (sp->req == NULL) {
/* XXX should be panic ? */
printf("ip6_forward: No IPsec request specified.\n");
ip6stat.ip6s_cantforward++;
key_freesp(sp, KEY_SADB_UNLOCKED);
if (mcopy) {
#if 0
/* XXX: what icmp ? */
#else
m_freem(mcopy);
#endif
}
m_freem(m);
return (NULL);
}
/* do IPsec */
break;
case IPSEC_POLICY_ENTRUST:
default:
/* should be panic ?? */
printf("ip6_forward: Invalid policy found. %d\n", sp->policy);
key_freesp(sp, KEY_SADB_UNLOCKED);
goto skip_ipsec;
}
{
struct ipsec_output_state state;
/*
* All the extension headers will become inaccessible
* (since they can be encrypted).
* Don't panic, we need no more updates to extension headers
* on inner IPv6 packet (since they are now encapsulated).
*
* IPv6 [ESP|AH] IPv6 [extension headers] payload
*/
bzero(&state, sizeof(state));
state.m = m;
state.dst = NULL; /* update at ipsec6_output_tunnel() */
error = ipsec6_output_tunnel(&state, sp, 0);
key_freesp(sp, KEY_SADB_UNLOCKED);
if (state.tunneled == 4) {
ROUTE_RELEASE(&state.ro);
return (NULL); /* packet is gone - sent over IPv4 */
}
m = state.m;
ROUTE_RELEASE(&state.ro);
if (error) {
/* mbuf is already reclaimed in ipsec6_output_tunnel. */
switch (error) {
case EHOSTUNREACH:
case ENETUNREACH:
case EMSGSIZE:
case ENOBUFS:
case ENOMEM:
break;
default:
printf("ip6_output (ipsec): error code %d\n", error);
/* fall through */
case ENOENT:
/* don't show these error codes to the user */
break;
}
ip6stat.ip6s_cantforward++;
if (mcopy) {
#if 0
/* XXX: what icmp ? */
#else
m_freem(mcopy);
#endif
}
m_freem(m);
return (NULL);
}
}
#endif /* IPSEC */
skip_ipsec:
dst = (struct sockaddr_in6 *)&ip6forward_rt->ro_dst;
if ((rt = ip6forward_rt->ro_rt) != NULL) {
RT_LOCK(rt);
/* Take an extra ref for ourselves */
RT_ADDREF_LOCKED(rt);
}
VERIFY(rt == NULL || rt == ip6forward_rt->ro_rt);
if (!srcrt) {
/*
* ip6forward_rt->ro_dst.sin6_addr is equal to ip6->ip6_dst
*/
if (ROUTE_UNUSABLE(ip6forward_rt)) {
if (rt != NULL) {
/* Release extra ref */
RT_REMREF_LOCKED(rt);
RT_UNLOCK(rt);
}
ROUTE_RELEASE(ip6forward_rt);
/* this probably fails but give it a try again */
rtalloc_scoped_ign((struct route *)ip6forward_rt,
RTF_PRCLONING, ifscope);
if ((rt = ip6forward_rt->ro_rt) != NULL) {
RT_LOCK(rt);
/* Take an extra ref for ourselves */
RT_ADDREF_LOCKED(rt);
}
}
if (rt == NULL) {
ip6stat.ip6s_noroute++;
in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_noroute);
if (mcopy)
icmp6_error(mcopy, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_NOROUTE, 0);
m_freem(m);
return (NULL);
}
RT_LOCK_ASSERT_HELD(rt);
} else if (ROUTE_UNUSABLE(ip6forward_rt) ||
!IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst, &dst->sin6_addr)) {
if (rt != NULL) {
/* Release extra ref */
RT_REMREF_LOCKED(rt);
RT_UNLOCK(rt);
}
ROUTE_RELEASE(ip6forward_rt);
bzero(dst, sizeof(*dst));
dst->sin6_len = sizeof(struct sockaddr_in6);
dst->sin6_family = AF_INET6;
dst->sin6_addr = ip6->ip6_dst;
rtalloc_scoped_ign((struct route *)ip6forward_rt,
RTF_PRCLONING, ifscope);
if ((rt = ip6forward_rt->ro_rt) == NULL) {
ip6stat.ip6s_noroute++;
in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_noroute);
if (mcopy)
icmp6_error(mcopy, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_NOROUTE, 0);
m_freem(m);
return (NULL);
}
RT_LOCK(rt);
/* Take an extra ref for ourselves */
RT_ADDREF_LOCKED(rt);
}
/*
* Source scope check: if a packet can't be delivered to its
* destination for the reason that the destination is beyond the scope
* of the source address, discard the packet and return an icmp6
* destination unreachable error with Code 2 (beyond scope of source
* address) unless we are proxying (source address is link local
* for NUDs.) We use a local copy of ip6_src, since in6_setscope()
* will possibly modify its first argument.
* [draft-ietf-ipngwg-icmp-v3-04.txt, Section 3.1]
*/
src_in6 = ip6->ip6_src;
if (in6_setscope(&src_in6, rt->rt_ifp, &outzone)) {
/* XXX: this should not happen */
ip6stat.ip6s_cantforward++;
ip6stat.ip6s_badscope++;
m_freem(m);
return (NULL);
}
if (in6_setscope(&src_in6, m->m_pkthdr.rcvif, &inzone)) {
ip6stat.ip6s_cantforward++;
ip6stat.ip6s_badscope++;
m_freem(m);
return (NULL);
}
if (inzone != outzone && !proxy) {
ip6stat.ip6s_cantforward++;
ip6stat.ip6s_badscope++;
in6_ifstat_inc(rt->rt_ifp, ifs6_in_discard);
if (ip6_log_time + ip6_log_interval < curtime) {
ip6_log_time = curtime;
log(LOG_DEBUG,
"cannot forward "
"src %s, dst %s, nxt %d, rcvif %s, outif %s\n",
ip6_sprintf(&ip6->ip6_src),
ip6_sprintf(&ip6->ip6_dst),
ip6->ip6_nxt,
if_name(m->m_pkthdr.rcvif), if_name(rt->rt_ifp));
}
/* Release extra ref */
RT_REMREF_LOCKED(rt);
RT_UNLOCK(rt);
if (mcopy) {
icmp6_error(mcopy, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_BEYONDSCOPE, 0);
}
m_freem(m);
return (NULL);
}
/*
* Destination scope check: if a packet is going to break the scope
* zone of packet's destination address, discard it. This case should
* usually be prevented by appropriately-configured routing table, but
* we need an explicit check because we may mistakenly forward the
* packet to a different zone by (e.g.) a default route.
*/
dst_in6 = ip6->ip6_dst;
if (in6_setscope(&dst_in6, m->m_pkthdr.rcvif, &inzone) != 0 ||
in6_setscope(&dst_in6, rt->rt_ifp, &outzone) != 0 ||
inzone != outzone) {
ip6stat.ip6s_cantforward++;
ip6stat.ip6s_badscope++;
m_freem(m);
return (NULL);
}
if (m->m_pkthdr.len > rt->rt_ifp->if_mtu) {
in6_ifstat_inc(rt->rt_ifp, ifs6_in_toobig);
if (mcopy) {
uint32_t mtu;
#if IPSEC
struct secpolicy *sp2;
int ipsecerror;
size_t ipsechdrsiz;
#endif
mtu = rt->rt_ifp->if_mtu;
#if IPSEC
/*
* When we do IPsec tunnel ingress, we need to play
* with the link value (decrement IPsec header size
* from mtu value). The code is much simpler than v4
* case, as we have the outgoing interface for
* encapsulated packet as "rt->rt_ifp".
*/
sp2 = ipsec6_getpolicybyaddr(mcopy, IPSEC_DIR_OUTBOUND,
IP_FORWARDING, &ipsecerror);
if (sp2) {
ipsechdrsiz = ipsec6_hdrsiz(mcopy,
IPSEC_DIR_OUTBOUND, NULL);
if (ipsechdrsiz < mtu)
mtu -= ipsechdrsiz;
key_freesp(sp2, KEY_SADB_UNLOCKED);
}
/*
* if mtu becomes less than minimum MTU,
* tell minimum MTU (and I'll need to fragment it).
*/
if (mtu < IPV6_MMTU)
mtu = IPV6_MMTU;
#endif
/* Release extra ref */
RT_REMREF_LOCKED(rt);
RT_UNLOCK(rt);
icmp6_error(mcopy, ICMP6_PACKET_TOO_BIG, 0, mtu);
} else {
/* Release extra ref */
RT_REMREF_LOCKED(rt);
RT_UNLOCK(rt);
}
m_freem(m);
return (NULL);
}
if (rt->rt_flags & RTF_GATEWAY)
dst = (struct sockaddr_in6 *)(void *)rt->rt_gateway;
/*
* If we are to forward the packet using the same interface
* as one we got the packet from, perhaps we should send a redirect
* to sender to shortcut a hop.
* Only send redirect if source is sending directly to us,
* and if packet was not source routed (or has any options).
* Also, don't send redirect if forwarding using a route
* modified by a redirect.
*/
if (!proxy &&
ip6_sendredirects && rt->rt_ifp == m->m_pkthdr.rcvif && !srcrt &&
(rt->rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) == 0) {
if ((rt->rt_ifp->if_flags & IFF_POINTOPOINT) != 0) {
/*
* If the incoming interface is equal to the outgoing
* one, and the link attached to the interface is
* point-to-point, then it will be highly probable
* that a routing loop occurs. Thus, we immediately
* drop the packet and send an ICMPv6 error message.
*
* type/code is based on suggestion by Rich Draves.
* not sure if it is the best pick.
*/
RT_REMREF_LOCKED(rt); /* Release extra ref */
RT_UNLOCK(rt);
icmp6_error(mcopy, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_ADDR, 0);
m_freem(m);
return (NULL);
}
type = ND_REDIRECT;
}
#if IPFW2
/*
* Check with the firewall...
*/
if (ip6_fw_enable && ip6_fw_chk_ptr) {
u_short port = 0;
ifp = rt->rt_ifp;
/* Drop the lock but retain the extra ref */
RT_UNLOCK(rt);
/* If ipfw says divert, we have to just drop packet */
if (ip6_fw_chk_ptr(&ip6, ifp, &port, &m)) {
m_freem(m);
goto freecopy;
}
if (!m) {
goto freecopy;
}
/* We still have the extra ref on rt */
RT_LOCK(rt);
}
#endif
/*
* Fake scoped addresses. Note that even link-local source or
* destinaion can appear, if the originating node just sends the
* packet to us (without address resolution for the destination).
* Since both icmp6_error and icmp6_redirect_output fill the embedded
* link identifiers, we can do this stuff after making a copy for
* returning an error.
*/
if ((rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
/*
* See corresponding comments in ip6_output.
* XXX: but is it possible that ip6_forward() sends a packet
* to a loopback interface? I don't think so, and thus
* I bark here. ([email protected])
* XXX: it is common to route invalid packets to loopback.
* also, the codepath will be visited on use of ::1 in
* rthdr. (itojun)
*/
#if 1
if ((0))
#else
if ((rt->rt_flags & (RTF_BLACKHOLE|RTF_REJECT)) == 0)
#endif
{
printf("ip6_forward: outgoing interface is loopback. "
"src %s, dst %s, nxt %d, rcvif %s, outif %s\n",
ip6_sprintf(&ip6->ip6_src),
ip6_sprintf(&ip6->ip6_dst),
ip6->ip6_nxt, if_name(m->m_pkthdr.rcvif),
if_name(rt->rt_ifp));
}
/* we can just use rcvif in forwarding. */
origifp = rcvifp = m->m_pkthdr.rcvif;
} else if (nd6_prproxy) {
/*
* In the prefix proxying case, we need to inform nd6_output()
* about the inbound interface, so that any subsequent NS
* packets generated by nd6_prproxy_ns_output() will not be
* sent back to that same interface.
*/
origifp = rcvifp = m->m_pkthdr.rcvif;
} else {
rcvifp = m->m_pkthdr.rcvif;
origifp = rt->rt_ifp;
}
/*
* clear embedded scope identifiers if necessary.
* in6_clearscope will touch the addresses only when necessary.
*/
in6_clearscope(&ip6->ip6_src);
in6_clearscope(&ip6->ip6_dst);
ifp = rt->rt_ifp;
/* Drop the lock but retain the extra ref */
RT_UNLOCK(rt);
/*
* If this is to be processed locally, let ip6_input have it.
*/
if (proxy) {
VERIFY(m->m_pkthdr.pkt_flags & PKTF_PROXY_DST);
/* Release extra ref */
RT_REMREF(rt);
if (mcopy != NULL)
m_freem(mcopy);
return (m);
}
#if PF
/* Invoke outbound packet filter */
error = pf_af_hook(ifp, NULL, &m, AF_INET6, FALSE, NULL);
if (error != 0 || m == NULL) {
if (m != NULL) {
panic("%s: unexpected packet %p\n", __func__, m);
/* NOTREACHED */
}
/* Already freed by callee */
goto senderr;
}
ip6 = mtod(m, struct ip6_hdr *);
#endif /* PF */
/* Mark this packet as being forwarded from another interface */
m->m_pkthdr.pkt_flags |= PKTF_FORWARDED;
len = m_pktlen(m);
error = nd6_output(ifp, origifp, m, dst, rt, NULL);
if (error) {
in6_ifstat_inc(ifp, ifs6_out_discard);
ip6stat.ip6s_cantforward++;
} else {
/*
* Increment stats on the source interface; the ones
* for destination interface has been taken care of
* during output above by virtue of PKTF_FORWARDED.
*/
rcvifp->if_fpackets++;
rcvifp->if_fbytes += len;
ip6stat.ip6s_forward++;
in6_ifstat_inc(ifp, ifs6_out_forward);
if (type)
ip6stat.ip6s_redirectsent++;
else {
if (mcopy) {
goto freecopy;
}
}
}
#if PF
senderr:
#endif /* PF */
if (mcopy == NULL) {
/* Release extra ref */
RT_REMREF(rt);
return (NULL);
}
switch (error) {
case 0:
#if 1
if (type == ND_REDIRECT) {
icmp6_redirect_output(mcopy, rt);
/* Release extra ref */
RT_REMREF(rt);
return (NULL);
}
#endif
goto freecopy;
case EMSGSIZE:
/* xxx MTU is constant in PPP? */
goto freecopy;
case ENOBUFS:
/* Tell source to slow down like source quench in IP? */
goto freecopy;
case ENETUNREACH: /* shouldn't happen, checked above */
case EHOSTUNREACH:
case ENETDOWN:
case EHOSTDOWN:
default:
type = ICMP6_DST_UNREACH;
code = ICMP6_DST_UNREACH_ADDR;
break;
}
icmp6_error(mcopy, type, code, 0);
/* Release extra ref */
RT_REMREF(rt);
return (NULL);
freecopy:
m_freem(mcopy);
/* Release extra ref */
RT_REMREF(rt);
return (NULL);
}
