161 static int netmap_vp_reg(struct netmap_adapter *na, int onoff);
353 nm_free_bdgfwd(struct netmap_adapter *na)  in nm_free_bdgfwd()  argument
359 	nrings = na->num_tx_rings;  in nm_free_bdgfwd()
360 	kring = na->tx_rings;  in nm_free_bdgfwd()
374 nm_alloc_bdgfwd(struct netmap_adapter *na)  in nm_alloc_bdgfwd()  argument
386 	nrings = netmap_real_rings(na, NR_TX);  in nm_alloc_bdgfwd()
387 	kring = na->tx_rings;  in nm_alloc_bdgfwd()
395 			nm_free_bdgfwd(na);  in nm_alloc_bdgfwd()
477 netmap_vp_bdg_ctl(struct netmap_adapter *na, struct nmreq *nmr, int attach)  in netmap_vp_bdg_ctl()  argument
479 	struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter *)na;  in netmap_vp_bdg_ctl()
485 		netmap_set_all_rings(na, 0 /* disable */);  in netmap_vp_bdg_ctl()
488 		netmap_set_all_rings(na, 1 /* enable */);  in netmap_vp_bdg_ctl()
491 	netmap_adapter_put(na);  in netmap_vp_bdg_ctl()
497 netmap_vp_dtor(struct netmap_adapter *na)  in netmap_vp_dtor()  argument
499 	struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter*)na;  in netmap_vp_dtor()
502 	ND("%s has %d references", na->name, na->na_refcount);  in netmap_vp_dtor()
596 netmap_get_bdg_na(struct nmreq *nmr, struct netmap_adapter **na, int create)  in netmap_get_bdg_na()  argument
607 	*na = NULL;     /* default return value */  in netmap_get_bdg_na()
639 			*na = &vpna->up;  in netmap_get_bdg_na()
718 	*na = &vpna->up;  in netmap_get_bdg_na()
719 	netmap_adapter_get(*na);  in netmap_get_bdg_na()
733 	struct netmap_adapter *na;  in nm_bdg_ctl_attach()  local
738 	error = netmap_get_bdg_na(nmr, &na, 1 /* create if not exists */);  in nm_bdg_ctl_attach()
742 	if (na == NULL) { /* VALE prefix missing */  in nm_bdg_ctl_attach()
747 	if (NETMAP_OWNED_BY_ANY(na)) {  in nm_bdg_ctl_attach()
752 	if (na->nm_bdg_ctl) {  in nm_bdg_ctl_attach()
756 		error = na->nm_bdg_ctl(na, nmr, 1);  in nm_bdg_ctl_attach()
759 		ND("registered %s to netmap-mode", na->name);  in nm_bdg_ctl_attach()
765 	netmap_adapter_put(na);  in nm_bdg_ctl_attach()
776 	struct netmap_adapter *na;  in nm_bdg_ctl_detach()  local
780 	error = netmap_get_bdg_na(nmr, &na, 0 /* don't create */);  in nm_bdg_ctl_detach()
785 	if (na == NULL) { /* VALE prefix missing */  in nm_bdg_ctl_detach()
790 	if (na->nm_bdg_ctl) {  in nm_bdg_ctl_detach()
794 		error = na->nm_bdg_ctl(na, nmr, 0);  in nm_bdg_ctl_detach()
797 	netmap_adapter_put(na);  in nm_bdg_ctl_detach()
817 	struct netmap_adapter *na;  in netmap_bdg_ctl()  local
937 		error = netmap_get_bdg_na(nmr, &na, 0);  in netmap_bdg_ctl()
938 		if (na && !error) {  in netmap_bdg_ctl()
939 			vpna = (struct netmap_vp_adapter *)na;  in netmap_bdg_ctl()
942 				vpna->mfs = NETMAP_BUF_SIZE(na);  in netmap_bdg_ctl()
944 			netmap_adapter_put(na);  in netmap_bdg_ctl()
984 netmap_vp_krings_create(struct netmap_adapter *na)  in netmap_vp_krings_create()  argument
989 	u_int nrx = netmap_real_rings(na, NR_RX);  in netmap_vp_krings_create()
994 	tailroom = sizeof(uint32_t) * na->num_rx_desc * nrx;  in netmap_vp_krings_create()
996 	error = netmap_krings_create(na, tailroom);  in netmap_vp_krings_create()
1000 	leases = na->tailroom;  in netmap_vp_krings_create()
1003 		na->rx_rings[i].nkr_leases = leases;  in netmap_vp_krings_create()
1004 		leases += na->num_rx_desc;  in netmap_vp_krings_create()
1007 	error = nm_alloc_bdgfwd(na);  in netmap_vp_krings_create()
1009 		netmap_krings_delete(na);  in netmap_vp_krings_create()
1019 netmap_vp_krings_delete(struct netmap_adapter *na)  in netmap_vp_krings_delete()  argument
1021 	nm_free_bdgfwd(na);  in netmap_vp_krings_delete()
1022 	netmap_krings_delete(na);  in netmap_vp_krings_delete()
1028 	struct netmap_vp_adapter *na, u_int ring_nr);
1041 	struct netmap_vp_adapter *na =  in nm_bdg_preflush()  local
1042 		(struct netmap_vp_adapter*)kring->na;  in nm_bdg_preflush()
1049 	struct nm_bridge *b = na->na_bdg;  in nm_bdg_preflush()
1056 	if (na->up.na_flags & NAF_BDG_MAYSLEEP)  in nm_bdg_preflush()
1077 			(void *)(uintptr_t)slot->ptr : NMB(&na->up, slot);  in nm_bdg_preflush()
1082 			buf = ft[ft_i].ft_buf = NETMAP_BUF_BASE(&na->up);  in nm_bdg_preflush()
1097 			ft_i = nm_bdg_flush(ft, ft_i, na, ring_nr);  in nm_bdg_preflush()
1106 		ft_i = nm_bdg_flush(ft, ft_i, na, ring_nr);  in nm_bdg_preflush()
1156 netmap_vp_reg(struct netmap_adapter *na, int onoff)  in netmap_vp_reg()  argument
1159 		(struct netmap_vp_adapter*)na;  in netmap_vp_reg()
1167 		na->na_flags |= NAF_NETMAP_ON;  in netmap_vp_reg()
1172 		na->na_flags &= ~NAF_NETMAP_ON;  in netmap_vp_reg()
1188 		struct netmap_vp_adapter *na)  in netmap_bdg_learning()  argument
1192 	struct nm_hash_ent *ht = na->na_bdg->ht;  in netmap_bdg_learning()
1194 	u_int dst, mysrc = na->bdg_port;  in netmap_bdg_learning()
1198 	if (buf_len >= 14 + na->virt_hdr_len) {  in netmap_bdg_learning()
1200 		buf += na->virt_hdr_len;  in netmap_bdg_learning()
1201 		buf_len -= na->virt_hdr_len;  in netmap_bdg_learning()
1202 	} else if (buf_len == na->virt_hdr_len && ft->ft_flags & NS_MOREFRAG) {  in netmap_bdg_learning()
1219 	if (((buf[6] & 1) == 0) && (na->last_smac != smac)) { /* valid src */  in netmap_bdg_learning()
1223 		na->last_smac = ht[sh].mac = smac;	/* XXX expire ? */  in netmap_bdg_learning()
1305 			k->na->name,  in nm_kr_lease()
1318 nm_bdg_flush(struct nm_bdg_fwd *ft, u_int n, struct netmap_vp_adapter *na,  in nm_bdg_flush()  argument
1323 	struct nm_bridge *b = na->na_bdg;  in nm_bdg_flush()
1324 	u_int i, j, me = na->bdg_port;  in nm_bdg_flush()
1344 		if (unlikely(na->virt_hdr_len > ft[i].ft_len))  in nm_bdg_flush()
1346 		dst_port = b->bdg_ops.lookup(&ft[i], &dst_ring, na);  in nm_bdg_flush()
1444 		if (unlikely(dst_na->virt_hdr_len != na->virt_hdr_len)) {  in nm_bdg_flush()
1445 			RD(3, "virt_hdr_mismatch, src %d dst %d", na->virt_hdr_len, dst_na->virt_hdr_len);  in nm_bdg_flush()
1451 			if (dst_na->mfs < na->mfs) {  in nm_bdg_flush()
1462 				needed = (needed * na->mfs) /  in nm_bdg_flush()
1464 				ND(3, "srcmtu=%u, dstmtu=%u, x=%u", na->mfs, dst_na->mfs, needed);  in nm_bdg_flush()
1534 				bdg_mismatch_datapath(na, dst_na, ft_p, ring, &j, lim, &howmany);  in nm_bdg_flush()
1551 						     copy_len > NETMAP_BUF_SIZE(&na->up))) {  in nm_bdg_flush()
1652 	struct netmap_vp_adapter *na =  in netmap_vp_txsync()  local
1653 		(struct netmap_vp_adapter *)kring->na;  in netmap_vp_txsync()
1662 	if (!na->na_bdg) {  in netmap_vp_txsync()
1679 		D("%s ring %d flags %d", na->up.name, kring->ring_id, flags);  in netmap_vp_txsync()
1690 	struct netmap_adapter *na = kring->na;  in netmap_vp_rxsync_locked()  local
1711 			void *addr = NMB(na, slot);  in netmap_vp_rxsync_locked()
1713 			if (addr == NETMAP_BUF_BASE(kring->na)) { /* bad buf */  in netmap_vp_rxsync_locked()
1751 netmap_vp_bdg_attach(const char *name, struct netmap_adapter *na)  in netmap_vp_bdg_attach()  argument
1753 	struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter *)na;  in netmap_vp_bdg_attach()
1757 	na->na_vp = vpna;  in netmap_vp_bdg_attach()
1758 	strncpy(na->name, name, sizeof(na->name));  in netmap_vp_bdg_attach()
1759 	na->na_hostvp = NULL;  in netmap_vp_bdg_attach()
1770 	struct netmap_adapter *na;  in netmap_vp_create()  local
1778  	na = &vpna->up;  in netmap_vp_create()
1780 	na->ifp = ifp;  in netmap_vp_create()
1781 	strncpy(na->name, nmr->nr_name, sizeof(na->name));  in netmap_vp_create()
1784 	na->num_tx_rings = nmr->nr_tx_rings;  in netmap_vp_create()
1785 	nm_bound_var(&na->num_tx_rings, 1, 1, NM_BDG_MAXRINGS, NULL);  in netmap_vp_create()
1786 	nmr->nr_tx_rings = na->num_tx_rings; // write back  in netmap_vp_create()
1787 	na->num_rx_rings = nmr->nr_rx_rings;  in netmap_vp_create()
1788 	nm_bound_var(&na->num_rx_rings, 1, 1, NM_BDG_MAXRINGS, NULL);  in netmap_vp_create()
1789 	nmr->nr_rx_rings = na->num_rx_rings; // write back  in netmap_vp_create()
1792 	na->num_tx_desc = nmr->nr_tx_slots;  in netmap_vp_create()
1805 	na->num_rx_desc = nmr->nr_rx_slots;  in netmap_vp_create()
1814 	na->na_flags |= NAF_BDG_MAYSLEEP;  in netmap_vp_create()
1819 		na->na_flags |= NAF_NATIVE;  in netmap_vp_create()
1820 	na->nm_txsync = netmap_vp_txsync;  in netmap_vp_create()
1821 	na->nm_rxsync = netmap_vp_rxsync;  in netmap_vp_create()
1822 	na->nm_register = netmap_vp_reg;  in netmap_vp_create()
1823 	na->nm_krings_create = netmap_vp_krings_create;  in netmap_vp_create()
1824 	na->nm_krings_delete = netmap_vp_krings_delete;  in netmap_vp_create()
1825 	na->nm_dtor = netmap_vp_dtor;  in netmap_vp_create()
1826 	na->nm_mem = netmap_mem_private_new(na->name,  in netmap_vp_create()
1827 			na->num_tx_rings, na->num_tx_desc,  in netmap_vp_create()
1828 			na->num_rx_rings, na->num_rx_desc,  in netmap_vp_create()
1830 	if (na->nm_mem == NULL)  in netmap_vp_create()
1832 	na->nm_bdg_attach = netmap_vp_bdg_attach;  in netmap_vp_create()
1834 	error = netmap_attach_common(na);  in netmap_vp_create()
1841 	if (na->nm_mem != NULL)  in netmap_vp_create()
1842 		netmap_mem_delete(na->nm_mem);  in netmap_vp_create()
1879 netmap_bwrap_dtor(struct netmap_adapter *na)  in netmap_bwrap_dtor()  argument
1881 	struct netmap_bwrap_adapter *bna = (struct netmap_bwrap_adapter*)na;  in netmap_bwrap_dtor()
1884 	ND("na %p", na);  in netmap_bwrap_dtor()
1889 	na->ifp = NULL;  in netmap_bwrap_dtor()
1925 	struct netmap_adapter *na = kring->na;  in netmap_bwrap_intr_notify()  local
1926 	struct netmap_bwrap_adapter *bna = na->na_private;  in netmap_bwrap_intr_notify()
1933 	    D("%s %s 0x%x", na->name, kring->name, flags);  in netmap_bwrap_intr_notify()
1935 	if (!nm_netmap_on(na))  in netmap_bwrap_intr_notify()
1945 	    D("%s head %d cur %d tail %d",  na->name,  in netmap_bwrap_intr_notify()
1956 			na->name);  in netmap_bwrap_intr_notify()
1981 netmap_bwrap_register(struct netmap_adapter *na, int onoff)  in netmap_bwrap_register()  argument
1984 		(struct netmap_bwrap_adapter *)na;  in netmap_bwrap_register()
1990 	ND("%s %s", na->name, onoff ? "on" : "off");  in netmap_bwrap_register()
2000 		hwna->na_lut = na->na_lut;  in netmap_bwrap_register()
2007 			hostna->up.na_lut = na->na_lut;  in netmap_bwrap_register()
2018 			for (i = 0; i < nma_get_nrings(na, r) + 1; i++) {  in netmap_bwrap_register()
2019 				NMR(hwna, t)[i].nkr_num_slots = NMR(na, r)[i].nkr_num_slots;  in netmap_bwrap_register()
2020 				NMR(hwna, t)[i].ring = NMR(na, r)[i].ring;  in netmap_bwrap_register()
2031 	netmap_vp_reg(na, onoff);  in netmap_bwrap_register()
2066 netmap_bwrap_config(struct netmap_adapter *na, u_int *txr, u_int *txd,  in netmap_bwrap_config()  argument
2070 		(struct netmap_bwrap_adapter *)na;  in netmap_bwrap_config()
2087 netmap_bwrap_krings_create(struct netmap_adapter *na)  in netmap_bwrap_krings_create()  argument
2090 		(struct netmap_bwrap_adapter *)na;  in netmap_bwrap_krings_create()
2095 	ND("%s", na->name);  in netmap_bwrap_krings_create()
2098 	error = netmap_vp_krings_create(na);  in netmap_bwrap_krings_create()
2105 		netmap_vp_krings_delete(na);  in netmap_bwrap_krings_create()
2113 	if (na->na_flags & NAF_HOST_RINGS) {  in netmap_bwrap_krings_create()
2118 		hostna->tx_rings = &na->tx_rings[na->num_tx_rings];  in netmap_bwrap_krings_create()
2119 		hostna->tx_rings[0].na = hostna;  in netmap_bwrap_krings_create()
2120 		hostna->rx_rings = &na->rx_rings[na->num_rx_rings];  in netmap_bwrap_krings_create()
2121 		hostna->rx_rings[0].na = hostna;  in netmap_bwrap_krings_create()
2129 netmap_bwrap_krings_delete(struct netmap_adapter *na)  in netmap_bwrap_krings_delete()  argument
2132 		(struct netmap_bwrap_adapter *)na;  in netmap_bwrap_krings_delete()
2135 	ND("%s", na->name);  in netmap_bwrap_krings_delete()
2138 	netmap_vp_krings_delete(na);  in netmap_bwrap_krings_delete()
2146 	struct netmap_adapter *na = kring->na;  in netmap_bwrap_notify()  local
2147 	struct netmap_bwrap_adapter *bna = na->na_private;  in netmap_bwrap_notify()
2156 			(na ? na->name : "NULL!"),  in netmap_bwrap_notify()
2168 		na->name, ring_n,  in netmap_bwrap_notify()
2187 		na->name, ring_n,  in netmap_bwrap_notify()
2206 netmap_bwrap_bdg_ctl(struct netmap_adapter *na, struct nmreq *nmr, int attach)  in netmap_bwrap_bdg_ctl()  argument
2209 	struct netmap_bwrap_adapter *bna = (struct netmap_bwrap_adapter*)na;  in netmap_bwrap_bdg_ctl()
2213 		if (NETMAP_OWNED_BY_ANY(na)) {  in netmap_bwrap_bdg_ctl()
2223 		error = netmap_do_regif(npriv, na, nmr->nr_ringid, nmr->nr_flags);  in netmap_bwrap_bdg_ctl()
2230 		na->na_flags |= NAF_BUSY;  in netmap_bwrap_bdg_ctl()
2234 		if (na->active_fds == 0) /* not registered */  in netmap_bwrap_bdg_ctl()
2256 			na->na_flags &= ~NAF_BUSY;  in netmap_bwrap_bdg_ctl()
2268 	struct netmap_adapter *na = NULL;  in netmap_bwrap_attach()  local
2284 	na = &bna->up.up;  in netmap_bwrap_attach()
2285 	na->na_private = bna;  in netmap_bwrap_attach()
2286 	strncpy(na->name, nr_name, sizeof(na->name));  in netmap_bwrap_attach()
2293 		nma_set_nrings(na, t, nma_get_nrings(hwna, r));  in netmap_bwrap_attach()
2294 		nma_set_ndesc(na, t, nma_get_ndesc(hwna, r));  in netmap_bwrap_attach()
2296 	na->nm_dtor = netmap_bwrap_dtor;  in netmap_bwrap_attach()
2297 	na->nm_register = netmap_bwrap_register;  in netmap_bwrap_attach()
2300 	na->nm_config = netmap_bwrap_config;  in netmap_bwrap_attach()
2301 	na->nm_krings_create = netmap_bwrap_krings_create;  in netmap_bwrap_attach()
2302 	na->nm_krings_delete = netmap_bwrap_krings_delete;  in netmap_bwrap_attach()
2303 	na->nm_notify = netmap_bwrap_notify;  in netmap_bwrap_attach()
2304 	na->nm_bdg_ctl = netmap_bwrap_bdg_ctl;  in netmap_bwrap_attach()
2305 	na->pdev = hwna->pdev;  in netmap_bwrap_attach()
2306 	na->nm_mem = netmap_mem_private_new(na->name,  in netmap_bwrap_attach()
2307 			na->num_tx_rings, na->num_tx_desc,  in netmap_bwrap_attach()
2308 			na->num_rx_rings, na->num_rx_desc,  in netmap_bwrap_attach()
2310 	na->na_flags |= NAF_MEM_OWNER;  in netmap_bwrap_attach()
2311 	if (na->nm_mem == NULL)  in netmap_bwrap_attach()
2322 			na->na_flags |= NAF_SW_ONLY;  in netmap_bwrap_attach()
2323 		na->na_flags |= NAF_HOST_RINGS;  in netmap_bwrap_attach()
2335 		hostna->nm_mem = na->nm_mem;  in netmap_bwrap_attach()
2338 		na->na_hostvp = hwna->na_hostvp =  in netmap_bwrap_attach()
2344 		na->name, ifp->if_xname,  in netmap_bwrap_attach()
2345 		na->num_tx_rings, na->num_tx_desc,  in netmap_bwrap_attach()
2346 		na->num_rx_rings, na->num_rx_desc);  in netmap_bwrap_attach()
2348 	error = netmap_attach_common(na);  in netmap_bwrap_attach()
2358 	na->ifp = hwna->ifp;  in netmap_bwrap_attach()
2364 	hwna->nm_mem = na->nm_mem;  in netmap_bwrap_attach()
2368 	netmap_mem_delete(na->nm_mem);  in netmap_bwrap_attach()