/* * Copyright (c) 2008 The DragonFly Project. All rights reserved. * * This code is derived from software contributed to The DragonFly Project * by Matthew Dillon * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * 3. Neither the name of The DragonFly Project nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific, prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $DragonFly: src/sys/net/altq/altq_fairq.c,v 1.2 2008/05/14 11:59:23 sephe Exp $ */ /* * Matt: I gutted altq_priq.c and used it as a skeleton on which to build * fairq. The fairq algorithm is completely different then priq, of course, * but because I used priq's skeleton I believe I should include priq's * copyright. * * Copyright (C) 2000-2003 * Sony Computer Science Laboratories Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * FAIRQ - take traffic classified by keep state (hashed into * pf->state_hash) and bucketize it. Fairly extract * the first packet from each bucket in a round-robin fashion. * * TODO - better overall qlimit support (right now it is per-bucket). * - NOTE: red etc is per bucket, not overall. * - better service curve support. * * EXAMPLE: * * altq on em0 fairq bandwidth 650Kb queue { std, bulk } * queue std priority 3 bandwidth 200Kb \ * fairq (buckets 64, default, hogs 1Kb) qlimit 50 * queue bulk priority 2 bandwidth 100Kb \ * fairq (buckets 64, hogs 1Kb) qlimit 50 * * NOTE: When the aggregate bandwidth is less than the link bandwidth * any remaining bandwidth is dynamically assigned using the * existing bandwidth specs as weightings. * * pass out on em0 from any to any keep state queue std * pass out on em0 inet proto tcp ..... port ... keep state queue bulk */ #include "opt_altq.h" #include "opt_inet.h" #include "opt_inet6.h" #ifdef ALTQ_FAIRQ /* fairq is enabled in the kernel conf */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define FAIRQ_SUBQ_INDEX ALTQ_SUBQ_INDEX_DEFAULT #define FAIRQ_LOCK(ifq) \ ALTQ_SQ_LOCK(&(ifq)->altq_subq[FAIRQ_SUBQ_INDEX]) #define FAIRQ_UNLOCK(ifq) \ ALTQ_SQ_UNLOCK(&(ifq)->altq_subq[FAIRQ_SUBQ_INDEX]) /* * function prototypes */ static int fairq_clear_interface(struct fairq_if *); static int fairq_request(struct ifaltq_subque *, int, void *); static void fairq_purge(struct fairq_if *); static struct fairq_class *fairq_class_create(struct fairq_if *, int, int, u_int, struct fairq_opts *, int); static int fairq_class_destroy(struct fairq_class *); static int fairq_enqueue(struct ifaltq_subque *, struct mbuf *, struct altq_pktattr *); static struct mbuf *fairq_dequeue(struct ifaltq_subque *, int); static int fairq_addq(struct fairq_class *, struct mbuf *, int hash); static struct mbuf *fairq_getq(struct fairq_class *, uint64_t); static struct mbuf *fairq_pollq(struct fairq_class *, uint64_t, int *); static fairq_bucket_t *fairq_selectq(struct fairq_class *); static void fairq_purgeq(struct fairq_class *); static void get_class_stats(struct fairq_classstats *, struct fairq_class *); static struct fairq_class *clh_to_clp(struct fairq_if *, uint32_t); int fairq_pfattach(struct pf_altq *a, struct ifaltq *ifq) { return altq_attach(ifq, ALTQT_FAIRQ, a->altq_disc, ifq_mapsubq_default, fairq_enqueue, fairq_dequeue, fairq_request, NULL, NULL); } int fairq_add_altq(struct pf_altq *a) { struct fairq_if *pif; struct ifnet *ifp; ifnet_lock(); if ((ifp = ifunit(a->ifname)) == NULL) { ifnet_unlock(); return (EINVAL); } if (!ifq_is_ready(&ifp->if_snd)) { ifnet_unlock(); return (ENODEV); } pif = kmalloc(sizeof(*pif), M_ALTQ, M_WAITOK | M_ZERO); pif->pif_bandwidth = a->ifbandwidth; pif->pif_maxpri = -1; pif->pif_ifq = &ifp->if_snd; ifq_purge_all(&ifp->if_snd); ifnet_unlock(); /* keep the state in pf_altq */ a->altq_disc = pif; return (0); } int fairq_remove_altq(struct pf_altq *a) { struct fairq_if *pif; if ((pif = a->altq_disc) == NULL) return (EINVAL); a->altq_disc = NULL; fairq_clear_interface(pif); kfree(pif, M_ALTQ); return (0); } static int fairq_add_queue_locked(struct pf_altq *a, struct fairq_if *pif) { struct fairq_class *cl; KKASSERT(a->priority < FAIRQ_MAXPRI); KKASSERT(a->qid != 0); if (pif->pif_classes[a->priority] != NULL) return (EBUSY); if (clh_to_clp(pif, a->qid) != NULL) return (EBUSY); cl = fairq_class_create(pif, a->priority, a->qlimit, a->bandwidth, &a->pq_u.fairq_opts, a->qid); if (cl == NULL) return (ENOMEM); return (0); } int fairq_add_queue(struct pf_altq *a) { struct fairq_if *pif; struct ifaltq *ifq; int error; /* check parameters */ if (a->priority >= FAIRQ_MAXPRI) return (EINVAL); if (a->qid == 0) return (EINVAL); /* XXX not MP safe */ if ((pif = a->altq_disc) == NULL) return (EINVAL); ifq = pif->pif_ifq; FAIRQ_LOCK(ifq); error = fairq_add_queue_locked(a, pif); FAIRQ_UNLOCK(ifq); return error; } static int fairq_remove_queue_locked(struct pf_altq *a, struct fairq_if *pif) { struct fairq_class *cl; if ((cl = clh_to_clp(pif, a->qid)) == NULL) return (EINVAL); return (fairq_class_destroy(cl)); } int fairq_remove_queue(struct pf_altq *a) { struct fairq_if *pif; struct ifaltq *ifq; int error; /* XXX not MP safe */ if ((pif = a->altq_disc) == NULL) return (EINVAL); ifq = pif->pif_ifq; FAIRQ_LOCK(ifq); error = fairq_remove_queue_locked(a, pif); FAIRQ_UNLOCK(ifq); return error; } int fairq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) { struct fairq_if *pif; struct fairq_class *cl; struct fairq_classstats stats; struct ifaltq *ifq; int error = 0; if (*nbytes < sizeof(stats)) return (EINVAL); ifnet_lock(); /* XXX not MP safe */ if ((pif = altq_lookup(a->ifname, ALTQT_FAIRQ)) == NULL) { ifnet_unlock(); return (EBADF); } ifq = pif->pif_ifq; FAIRQ_LOCK(ifq); if ((cl = clh_to_clp(pif, a->qid)) == NULL) { FAIRQ_UNLOCK(ifq); ifnet_unlock(); return (EINVAL); } get_class_stats(&stats, cl); FAIRQ_UNLOCK(ifq); ifnet_unlock(); if ((error = copyout((caddr_t)&stats, ubuf, sizeof(stats))) != 0) return (error); *nbytes = sizeof(stats); return (0); } /* * bring the interface back to the initial state by discarding * all the filters and classes. */ static int fairq_clear_interface(struct fairq_if *pif) { struct fairq_class *cl; int pri; /* clear out the classes */ for (pri = 0; pri <= pif->pif_maxpri; pri++) { if ((cl = pif->pif_classes[pri]) != NULL) fairq_class_destroy(cl); } return (0); } static int fairq_request(struct ifaltq_subque *ifsq, int req, void *arg) { struct ifaltq *ifq = ifsq->ifsq_altq; struct fairq_if *pif = (struct fairq_if *)ifq->altq_disc; crit_enter(); switch (req) { case ALTRQ_PURGE: if (ifsq_get_index(ifsq) == FAIRQ_SUBQ_INDEX) { fairq_purge(pif); } else { /* * Race happened, the unrelated subqueue was * picked during the packet scheduler transition. */ ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); } break; } crit_exit(); return (0); } /* discard all the queued packets on the interface */ static void fairq_purge(struct fairq_if *pif) { struct fairq_class *cl; int pri; for (pri = 0; pri <= pif->pif_maxpri; pri++) { if ((cl = pif->pif_classes[pri]) != NULL && cl->cl_head) fairq_purgeq(cl); } if (ifq_is_enabled(pif->pif_ifq)) ALTQ_SQ_CNTR_RESET(&pif->pif_ifq->altq_subq[FAIRQ_SUBQ_INDEX]); } static struct fairq_class * fairq_class_create(struct fairq_if *pif, int pri, int qlimit, u_int bandwidth, struct fairq_opts *opts, int qid) { struct fairq_class *cl; int flags = opts->flags; u_int nbuckets = opts->nbuckets; int i; #ifndef ALTQ_RED if (flags & FARF_RED) { #ifdef ALTQ_DEBUG kprintf("fairq_class_create: RED not configured for FAIRQ!\n"); #endif return (NULL); } #endif if (nbuckets == 0) nbuckets = 256; if (nbuckets > FAIRQ_MAX_BUCKETS) nbuckets = FAIRQ_MAX_BUCKETS; /* enforce power-of-2 size */ while ((nbuckets ^ (nbuckets - 1)) != ((nbuckets << 1) - 1)) ++nbuckets; if ((cl = pif->pif_classes[pri]) != NULL) { /* modify the class instead of creating a new one */ crit_enter(); if (cl->cl_head) fairq_purgeq(cl); crit_exit(); #ifdef ALTQ_RIO if (cl->cl_qtype == Q_RIO) rio_destroy((rio_t *)cl->cl_red); #endif #ifdef ALTQ_RED if (cl->cl_qtype == Q_RED) red_destroy(cl->cl_red); #endif } else { cl = kmalloc(sizeof(*cl), M_ALTQ, M_WAITOK | M_ZERO); cl->cl_nbuckets = nbuckets; cl->cl_nbucket_mask = nbuckets - 1; cl->cl_buckets = kmalloc(sizeof(*cl->cl_buckets) * cl->cl_nbuckets, M_ALTQ, M_WAITOK | M_ZERO); cl->cl_head = NULL; } pif->pif_classes[pri] = cl; if (flags & FARF_DEFAULTCLASS) pif->pif_default = cl; if (qlimit == 0) qlimit = 50; /* use default */ cl->cl_qlimit = qlimit; for (i = 0; i < cl->cl_nbuckets; ++i) { qlimit(&cl->cl_buckets[i].queue) = qlimit; } cl->cl_bandwidth = bandwidth / 8; /* cvt to bytes per second */ cl->cl_qtype = Q_DROPTAIL; cl->cl_flags = flags & FARF_USERFLAGS; cl->cl_pri = pri; if (pri > pif->pif_maxpri) pif->pif_maxpri = pri; cl->cl_pif = pif; cl->cl_handle = qid; cl->cl_hogs_m1 = opts->hogs_m1 / 8; cl->cl_lssc_m1 = opts->lssc_m1 / 8; /* NOT YET USED */ cl->cl_bw_current = 0; #ifdef ALTQ_RED if (flags & (FARF_RED|FARF_RIO)) { int red_flags, red_pkttime; red_flags = 0; if (flags & FARF_ECN) red_flags |= REDF_ECN; #ifdef ALTQ_RIO if (flags & FARF_CLEARDSCP) red_flags |= RIOF_CLEARDSCP; #endif if (pif->pif_bandwidth < 8) red_pkttime = 1000 * 1000 * 1000; /* 1 sec */ else red_pkttime = (int64_t)pif->pif_ifq->altq_ifp->if_mtu * (1000 * 1000 * 1000) / (pif->pif_bandwidth / 8 + 1); #ifdef ALTQ_RIO if (flags & FARF_RIO) { cl->cl_red = (red_t *)rio_alloc(0, NULL, red_flags, red_pkttime); if (cl->cl_red != NULL) cl->cl_qtype = Q_RIO; } else #endif if (flags & FARF_RED) { cl->cl_red = red_alloc(0, 0, cl->cl_qlimit * 10 / 100, cl->cl_qlimit * 30 / 100, red_flags, red_pkttime); if (cl->cl_red != NULL) cl->cl_qtype = Q_RED; } } #endif /* ALTQ_RED */ return (cl); } static int fairq_class_destroy(struct fairq_class *cl) { struct fairq_if *pif; int pri; crit_enter(); if (cl->cl_head) fairq_purgeq(cl); pif = cl->cl_pif; pif->pif_classes[cl->cl_pri] = NULL; if (pif->pif_poll_cache == cl) pif->pif_poll_cache = NULL; if (pif->pif_maxpri == cl->cl_pri) { for (pri = cl->cl_pri; pri >= 0; pri--) if (pif->pif_classes[pri] != NULL) { pif->pif_maxpri = pri; break; } if (pri < 0) pif->pif_maxpri = -1; } crit_exit(); if (cl->cl_red != NULL) { #ifdef ALTQ_RIO if (cl->cl_qtype == Q_RIO) rio_destroy((rio_t *)cl->cl_red); #endif #ifdef ALTQ_RED if (cl->cl_qtype == Q_RED) red_destroy(cl->cl_red); #endif } kfree(cl->cl_buckets, M_ALTQ); cl->cl_head = NULL; /* sanity */ cl->cl_buckets = NULL; /* sanity */ kfree(cl, M_ALTQ); return (0); } /* * fairq_enqueue is an enqueue function to be registered to * (*ifsq_enqueue) in struct ifaltq_subque. */ static int fairq_enqueue(struct ifaltq_subque *ifsq, struct mbuf *m, struct altq_pktattr *pktattr) { struct ifaltq *ifq = ifsq->ifsq_altq; struct fairq_if *pif = (struct fairq_if *)ifq->altq_disc; struct fairq_class *cl; int error; int len; int hash; if (ifsq_get_index(ifsq) != FAIRQ_SUBQ_INDEX) { /* * Race happened, the unrelated subqueue was * picked during the packet scheduler transition. */ ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); m_freem(m); return ENOBUFS; } crit_enter(); /* grab class set by classifier */ M_ASSERTPKTHDR(m); if (m->m_pkthdr.fw_flags & PF_MBUF_STRUCTURE) { cl = clh_to_clp(pif, m->m_pkthdr.pf.qid); if (m->m_pkthdr.pf.flags & PF_TAG_STATE_HASHED) hash = (int)m->m_pkthdr.pf.state_hash; else hash = 0; } else { cl = NULL; hash = 0; } if (cl == NULL) { cl = pif->pif_default; if (cl == NULL) { m_freem(m); error = ENOBUFS; goto done; } } cl->cl_flags |= FARF_HAS_PACKETS; cl->cl_pktattr = NULL; len = m_pktlen(m); if (fairq_addq(cl, m, hash) != 0) { /* drop occurred. mbuf was freed in fairq_addq. */ PKTCNTR_ADD(&cl->cl_dropcnt, len); error = ENOBUFS; goto done; } ALTQ_SQ_PKTCNT_INC(ifsq); error = 0; done: crit_exit(); return (error); } /* * fairq_dequeue is a dequeue function to be registered to * (*ifsq_dequeue) in struct ifaltq_subque. * * note: ALTDQ_POLL returns the next packet without removing the packet * from the queue. ALTDQ_REMOVE is a normal dequeue operation. */ static struct mbuf * fairq_dequeue(struct ifaltq_subque *ifsq, int op) { struct ifaltq *ifq = ifsq->ifsq_altq; struct fairq_if *pif = (struct fairq_if *)ifq->altq_disc; struct fairq_class *cl; struct fairq_class *best_cl; struct mbuf *best_m; struct mbuf *m; uint64_t cur_time = read_machclk(); uint64_t best_scale; uint64_t scale; int pri; int hit_limit; if (ifsq_get_index(ifsq) != FAIRQ_SUBQ_INDEX) { /* * Race happened, the unrelated subqueue was * picked during the packet scheduler transition. */ ifsq_classic_request(ifsq, ALTRQ_PURGE, NULL); return NULL; } if (ifsq_is_empty(ifsq)) { /* no packet in the queue */ return (NULL); } crit_enter(); if (pif->pif_poll_cache && op == ALTDQ_REMOVE) { best_cl = pif->pif_poll_cache; m = fairq_getq(best_cl, cur_time); pif->pif_poll_cache = NULL; if (m) { ALTQ_SQ_PKTCNT_DEC(ifsq); PKTCNTR_ADD(&best_cl->cl_xmitcnt, m_pktlen(m)); } } else { best_cl = NULL; best_m = NULL; best_scale = 0xFFFFFFFFFFFFFFFFLLU; for (pri = pif->pif_maxpri; pri >= 0; pri--) { if ((cl = pif->pif_classes[pri]) == NULL) continue; if ((cl->cl_flags & FARF_HAS_PACKETS) == 0) continue; m = fairq_pollq(cl, cur_time, &hit_limit); if (m == NULL) { cl->cl_flags &= ~FARF_HAS_PACKETS; continue; } /* * We can halt the search immediately if the queue * did not hit its bandwidth limit. */ if (hit_limit == 0) { best_cl = cl; best_m = m; break; } /* * Otherwise calculate the scale factor and select * the queue with the lowest scale factor. This * apportions any unused bandwidth weighted by * the relative bandwidth specification. * * scale = (bw / max) with a multiple of 256. * * The calculation is refactored to reduce the * chance of overflow. */ scale = cl->cl_bw_current * 16 / (cl->cl_bandwidth / 16 + 1); if (best_scale > scale) { best_cl = cl; best_m = m; best_scale = scale; } } if (op == ALTDQ_POLL) { #ifdef foo /* * Don't use poll cache; the poll/dequeue * model is no longer applicable to SMP * system. e.g. * CPU-A CPU-B * : : * poll : * : poll * dequeue (+) : * * The dequeue at (+) will hit the poll * cache set by CPU-B. */ pif->pif_poll_cache = best_cl; #endif m = best_m; } else if (best_cl) { m = fairq_getq(best_cl, cur_time); KKASSERT(best_m == m); ALTQ_SQ_PKTCNT_DEC(ifsq); PKTCNTR_ADD(&best_cl->cl_xmitcnt, m_pktlen(m)); } else { m = NULL; } } crit_exit(); return (m); } static int fairq_addq(struct fairq_class *cl, struct mbuf *m, int hash) { fairq_bucket_t *b; u_int hindex; uint64_t bw; /* * If the packet doesn't have any keep state put it on the end of * our queue. XXX this can result in out of order delivery. */ if (hash == 0) { if (cl->cl_head) b = cl->cl_head->prev; else b = &cl->cl_buckets[0]; } else { hindex = hash & cl->cl_nbucket_mask; b = &cl->cl_buckets[hindex]; } /* * Add the bucket to the end of the circular list of active buckets. * * As a special case we add the bucket to the beginning of the list * instead of the end if it was not previously on the list and if * its traffic is less then the hog level. */ if (b->in_use == 0) { b->in_use = 1; if (cl->cl_head == NULL) { cl->cl_head = b; cl->cl_advanced = 1; b->next = b; b->prev = b; } else { b->next = cl->cl_head; b->prev = cl->cl_head->prev; b->prev->next = b; b->next->prev = b; if (b->bw_delta && cl->cl_hogs_m1) { bw = b->bw_bytes * machclk_freq / b->bw_delta; if (bw < cl->cl_hogs_m1) { cl->cl_head = b; cl->cl_advanced = 1; } } } } #ifdef ALTQ_RIO if (cl->cl_qtype == Q_RIO) return rio_addq((rio_t *)cl->cl_red, &b->queue, m, cl->cl_pktattr); #endif #ifdef ALTQ_RED if (cl->cl_qtype == Q_RED) return red_addq(cl->cl_red, &b->queue, m, cl->cl_pktattr); #endif if (qlen(&b->queue) >= qlimit(&b->queue)) { m_freem(m); return (-1); } if (cl->cl_flags & FARF_CLEARDSCP) write_dsfield(m, cl->cl_pktattr, 0); _addq(&b->queue, m); return (0); } static struct mbuf * fairq_getq(struct fairq_class *cl, uint64_t cur_time) { fairq_bucket_t *b; struct mbuf *m; b = fairq_selectq(cl); if (b == NULL) m = NULL; #ifdef ALTQ_RIO else if (cl->cl_qtype == Q_RIO) m = rio_getq((rio_t *)cl->cl_red, &b->queue); #endif #ifdef ALTQ_RED else if (cl->cl_qtype == Q_RED) m = red_getq(cl->cl_red, &b->queue); #endif else m = _getq(&b->queue); /* * Calculate the BW change */ if (m != NULL) { uint64_t delta; /* * Per-class bandwidth calculation */ delta = (cur_time - cl->cl_last_time); if (delta > machclk_freq * 8) delta = machclk_freq * 8; cl->cl_bw_delta += delta; cl->cl_bw_bytes += m->m_pkthdr.len; cl->cl_last_time = cur_time; /* * Cap delta at ~machclk_freq to avoid overflows. */ if (cl->cl_bw_delta > machclk_freq) { uint64_t f = cl->cl_bw_delta * 32 / machclk_freq; cl->cl_bw_delta = cl->cl_bw_delta * 16 / f; cl->cl_bw_bytes = cl->cl_bw_bytes * 16 / f; } /* * Per-bucket bandwidth calculation. */ delta = (cur_time - b->last_time); if (delta > machclk_freq * 8) delta = machclk_freq * 8; b->bw_delta += delta; b->bw_bytes += m->m_pkthdr.len; b->last_time = cur_time; /* * Cap bw_delta at ~machclk_freq to avoid overflows. */ if (b->bw_delta > machclk_freq) { uint64_t f = b->bw_delta * 32 / machclk_freq; b->bw_delta = b->bw_delta * 16 / f; b->bw_bytes = b->bw_bytes * 16 / f; } } return(m); } /* * Figure out what the next packet would be if there were no limits. If * this class hits its bandwidth limit *hit_limit is set to no-zero, otherwise * it is set to 0. A non-NULL mbuf is returned either way. */ static struct mbuf * fairq_pollq(struct fairq_class *cl, uint64_t cur_time, int *hit_limit) { fairq_bucket_t *b; struct mbuf *m; uint64_t delta; uint64_t bw; *hit_limit = 0; b = fairq_selectq(cl); if (b == NULL) return(NULL); m = qhead(&b->queue); if (m == NULL) return(NULL); cl->cl_advanced = 1; /* so next select/get doesn't re-advance */ /* * Did this packet exceed the class bandwidth? * * Calculate the bandwidth component of the packet in bytes/sec. * Avoid overflows when machclk_freq is very high. */ delta = cur_time - cl->cl_last_time; if (delta > machclk_freq * 8) delta = machclk_freq * 8; cl->cl_bw_delta += delta; cl->cl_last_time = cur_time; if (cl->cl_bw_delta) { bw = (cl->cl_bw_bytes + m->m_pkthdr.len) * machclk_freq / cl->cl_bw_delta; if (bw > cl->cl_bandwidth) *hit_limit = 1; cl->cl_bw_current = bw; #if 0 kprintf("BW %6lld relative to %6llu %d queue %p\n", bw, cl->cl_bandwidth, *hit_limit, b); #endif } return(m); } /* * Locate the next queue we want to pull a packet out of. This code * is also responsible for removing empty buckets from the circular list. */ static fairq_bucket_t * fairq_selectq(struct fairq_class *cl) { fairq_bucket_t *b; uint64_t bw; while ((b = cl->cl_head) != NULL) { /* * Remove empty queues from consideration */ if (qempty(&b->queue)) { b->in_use = 0; cl->cl_head = b->next; cl->cl_advanced = 1; if (cl->cl_head == b) { cl->cl_head = NULL; } else { b->next->prev = b->prev; b->prev->next = b->next; } continue; } /* * Advance the round robin. Queues with bandwidths less * then the hog bandwidth are allowed to burst. * * Don't advance twice if the previous head emptied. */ if (cl->cl_advanced) { cl->cl_advanced = 0; break; } if (cl->cl_hogs_m1 == 0) { cl->cl_head = b->next; } else if (b->bw_delta) { bw = b->bw_bytes * machclk_freq / b->bw_delta; if (bw >= cl->cl_hogs_m1) cl->cl_head = b->next; } /* * Return the (possibly new) head. */ b = cl->cl_head; break; } return(b); } static void fairq_purgeq(struct fairq_class *cl) { fairq_bucket_t *b; struct mbuf *m; while ((b = fairq_selectq(cl)) != NULL) { while ((m = _getq(&b->queue)) != NULL) { PKTCNTR_ADD(&cl->cl_dropcnt, m_pktlen(m)); m_freem(m); } KKASSERT(qlen(&b->queue) == 0); } } static void get_class_stats(struct fairq_classstats *sp, struct fairq_class *cl) { fairq_bucket_t *b; sp->class_handle = cl->cl_handle; sp->qlimit = cl->cl_qlimit; sp->xmit_cnt = cl->cl_xmitcnt; sp->drop_cnt = cl->cl_dropcnt; sp->qtype = cl->cl_qtype; sp->qlength = 0; if (cl->cl_head) { b = cl->cl_head; do { sp->qlength += qlen(&b->queue); b = b->next; } while (b != cl->cl_head); } #ifdef ALTQ_RED if (cl->cl_qtype == Q_RED) red_getstats(cl->cl_red, &sp->red[0]); #endif #ifdef ALTQ_RIO if (cl->cl_qtype == Q_RIO) rio_getstats((rio_t *)cl->cl_red, &sp->red[0]); #endif } /* convert a class handle to the corresponding class pointer */ static struct fairq_class * clh_to_clp(struct fairq_if *pif, uint32_t chandle) { struct fairq_class *cl; int idx; if (chandle == 0) return (NULL); for (idx = pif->pif_maxpri; idx >= 0; idx--) if ((cl = pif->pif_classes[idx]) != NULL && cl->cl_handle == chandle) return (cl); return (NULL); } #endif /* ALTQ_FAIRQ */