113 	struct toepcb *toep;  in alloc_toepcb()  local
135 	toep = malloc(len, M_CXGBE, M_ZERO | flags);  in alloc_toepcb()
136 	if (toep == NULL)  in alloc_toepcb()
139 	refcount_init(&toep->refcount, 1);  in alloc_toepcb()
140 	toep->td = sc->tom_softc;  in alloc_toepcb()
141 	toep->vi = vi;  in alloc_toepcb()
142 	toep->tid = -1;  in alloc_toepcb()
143 	toep->tx_total = tx_credits;  in alloc_toepcb()
144 	toep->tx_credits = tx_credits;  in alloc_toepcb()
145 	mbufq_init(&toep->ulp_pduq, INT_MAX);  in alloc_toepcb()
146 	mbufq_init(&toep->ulp_pdu_reclaimq, INT_MAX);  in alloc_toepcb()
147 	toep->txsd_total = txsd_total;  in alloc_toepcb()
148 	toep->txsd_avail = txsd_total;  in alloc_toepcb()
149 	toep->txsd_pidx = 0;  in alloc_toepcb()
150 	toep->txsd_cidx = 0;  in alloc_toepcb()
151 	aiotx_init_toep(toep);  in alloc_toepcb()
153 	return (toep);  in alloc_toepcb()
160 init_toepcb(struct vi_info *vi, struct toepcb *toep)  in init_toepcb()  argument
162 	struct conn_params *cp = &toep->params;  in init_toepcb()
174 			    toep->tid);  in init_toepcb()
181 	toep->ofld_txq = &sc->sge.ofld_txq[cp->txq_idx];  in init_toepcb()
182 	toep->ofld_rxq = &sc->sge.ofld_rxq[cp->rxq_idx];  in init_toepcb()
183 	toep->ctrlq = &sc->sge.ctrlq[pi->port_id];  in init_toepcb()
185 	tls_init_toep(toep);  in init_toepcb()
186 	if (ulp_mode(toep) == ULP_MODE_TCPDDP)  in init_toepcb()
187 		ddp_init_toep(toep);  in init_toepcb()
189 	toep->flags |= TPF_INITIALIZED;  in init_toepcb()
195 hold_toepcb(struct toepcb *toep)  in hold_toepcb()  argument
198 	refcount_acquire(&toep->refcount);  in hold_toepcb()
199 	return (toep);  in hold_toepcb()
203 free_toepcb(struct toepcb *toep)  in free_toepcb()  argument
206 	if (refcount_release(&toep->refcount) == 0)  in free_toepcb()
209 	KASSERT(!(toep->flags & TPF_ATTACHED),  in free_toepcb()
211 	KASSERT(!(toep->flags & TPF_CPL_PENDING),  in free_toepcb()
214 	if (toep->flags & TPF_INITIALIZED) {  in free_toepcb()
215 		if (ulp_mode(toep) == ULP_MODE_TCPDDP)  in free_toepcb()
216 			ddp_uninit_toep(toep);  in free_toepcb()
217 		tls_uninit_toep(toep);  in free_toepcb()
219 	free(toep, M_CXGBE);  in free_toepcb()
226 offload_socket(struct socket *so, struct toepcb *toep)  in offload_socket()  argument
228 	struct tom_data *td = toep->td;  in offload_socket()
251 	tp->t_toe = toep;  in offload_socket()
255 	toep->inp = inp;  in offload_socket()
256 	toep->flags |= TPF_ATTACHED;  in offload_socket()
261 	TAILQ_INSERT_HEAD(&td->toep_list, toep, link);  in offload_socket()
271 	struct toepcb *toep = tp->t_toe;  in undo_offload_socket()  local
272 	struct tom_data *td = toep->td;  in undo_offload_socket()
290 	toep->inp = NULL;  in undo_offload_socket()
291 	toep->flags &= ~TPF_ATTACHED;  in undo_offload_socket()
296 	TAILQ_REMOVE(&td->toep_list, toep, link);  in undo_offload_socket()
301 release_offload_resources(struct toepcb *toep)  in release_offload_resources()  argument
303 	struct tom_data *td = toep->td;  in release_offload_resources()
305 	int tid = toep->tid;  in release_offload_resources()
307 	KASSERT(!(toep->flags & TPF_CPL_PENDING),  in release_offload_resources()
308 	    ("%s: %p has CPL pending.", __func__, toep));  in release_offload_resources()
309 	KASSERT(!(toep->flags & TPF_ATTACHED),  in release_offload_resources()
310 	    ("%s: %p is still attached.", __func__, toep));  in release_offload_resources()
313 	    __func__, toep, tid, toep->l2te, toep->ce);  in release_offload_resources()
320 	MPASS(mbufq_len(&toep->ulp_pduq) == 0);  in release_offload_resources()
321 	MPASS(mbufq_len(&toep->ulp_pdu_reclaimq) == 0);  in release_offload_resources()
323 	if (ulp_mode(toep) == ULP_MODE_TCPDDP)  in release_offload_resources()
324 		ddp_assert_empty(toep);  in release_offload_resources()
326 	MPASS(TAILQ_EMPTY(&toep->aiotx_jobq));  in release_offload_resources()
328 	if (toep->l2te)  in release_offload_resources()
329 		t4_l2t_release(toep->l2te);  in release_offload_resources()
332 		remove_tid(sc, tid, toep->ce ? 2 : 1);  in release_offload_resources()
333 		release_tid(sc, tid, toep->ctrlq);  in release_offload_resources()
336 	if (toep->ce)  in release_offload_resources()
337 		t4_release_lip(sc, toep->ce);  in release_offload_resources()
339 	if (toep->params.tc_idx != -1)  in release_offload_resources()
340 		t4_release_cl_rl(sc, toep->vi->pi->port_id, toep->params.tc_idx);  in release_offload_resources()
343 	TAILQ_REMOVE(&td->toep_list, toep, link);  in release_offload_resources()
346 	free_toepcb(toep);  in release_offload_resources()
363 	struct toepcb *toep = tp->t_toe;  in t4_pcb_detach()  local
367 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));  in t4_pcb_detach()
368 	KASSERT(toep->flags & TPF_ATTACHED,  in t4_pcb_detach()
374 		    __func__, toep->tid, toep, toep->flags, inp,  in t4_pcb_detach()
379 		    toep->tid, tcpstates[tp->t_state], toep, toep->flags, inp,  in t4_pcb_detach()
387 	toep->flags &= ~TPF_ATTACHED;  in t4_pcb_detach()
389 	if (!(toep->flags & TPF_CPL_PENDING))  in t4_pcb_detach()
390 		release_offload_resources(toep);  in t4_pcb_detach()
400 	struct toepcb *toep = tp->t_toe;  in t4_ctloutput()  local
411 		toep->params.nagle = tp->t_flags & TF_NODELAY ? 0 : 1;  in t4_ctloutput()
412 		t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS,  in t4_ctloutput()
413 		    V_TF_NAGLE(1), V_TF_NAGLE(toep->params.nagle), 0, 0);  in t4_ctloutput()
806 	struct toepcb *toep = tp->t_toe;  in t4_tcp_info()  local
811 	fill_tcp_info(sc, toep->tid, ti);  in t4_tcp_info()
819 final_cpl_received(struct toepcb *toep)  in final_cpl_received()  argument
821 	struct inpcb *inp = toep->inp;  in final_cpl_received()
825 	KASSERT(toep->flags & TPF_CPL_PENDING,  in final_cpl_received()
829 	    __func__, toep->tid, toep, toep->flags, inp, inp->inp_flags);  in final_cpl_received()
831 	if (ulp_mode(toep) == ULP_MODE_TCPDDP)  in final_cpl_received()
832 		release_ddp_resources(toep);  in final_cpl_received()
833 	toep->inp = NULL;  in final_cpl_received()
834 	toep->flags &= ~TPF_CPL_PENDING;  in final_cpl_received()
835 	mbufq_drain(&toep->ulp_pdu_reclaimq);  in final_cpl_received()
837 	if (!(toep->flags & TPF_ATTACHED))  in final_cpl_received()
838 		release_offload_resources(toep);  in final_cpl_received()
1789 	struct toepcb *toep = tp->t_toe;  in t4_aio_queue_tom()  local
1792 	if (ulp_mode(toep) == ULP_MODE_TCPDDP) {  in t4_aio_queue_tom()