1 /*-
2 * Copyright (c) 2002-2007 Neterion, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29 #ifdef XGE_DEBUG_FP
30 #include <dev/nxge/include/xgehal-device.h>
31 #endif
32
33 #include <dev/nxge/include/xgehal-ring.h>
34 #include <dev/nxge/include/xgehal-fifo.h>
35
36 /**
37 * xge_hal_device_bar0 - Get BAR0 mapped address.
38 * @hldev: HAL device handle.
39 *
40 * Returns: BAR0 address of the specified device.
41 */
42 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
xge_hal_device_bar0(xge_hal_device_t * hldev)43 xge_hal_device_bar0(xge_hal_device_t *hldev)
44 {
45 return hldev->bar0;
46 }
47
48 /**
49 * xge_hal_device_isrbar0 - Get BAR0 mapped address.
50 * @hldev: HAL device handle.
51 *
52 * Returns: BAR0 address of the specified device.
53 */
54 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
xge_hal_device_isrbar0(xge_hal_device_t * hldev)55 xge_hal_device_isrbar0(xge_hal_device_t *hldev)
56 {
57 return hldev->isrbar0;
58 }
59
60 /**
61 * xge_hal_device_bar1 - Get BAR1 mapped address.
62 * @hldev: HAL device handle.
63 *
64 * Returns: BAR1 address of the specified device.
65 */
66 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
xge_hal_device_bar1(xge_hal_device_t * hldev)67 xge_hal_device_bar1(xge_hal_device_t *hldev)
68 {
69 return hldev->bar1;
70 }
71
72 /**
73 * xge_hal_device_bar0_set - Set BAR0 mapped address.
74 * @hldev: HAL device handle.
75 * @bar0: BAR0 mapped address.
76 * * Set BAR0 address in the HAL device object.
77 */
78 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_bar0_set(xge_hal_device_t * hldev,char * bar0)79 xge_hal_device_bar0_set(xge_hal_device_t *hldev, char *bar0)
80 {
81 xge_assert(bar0);
82 hldev->bar0 = bar0;
83 }
84
85 /**
86 * xge_hal_device_isrbar0_set - Set BAR0 mapped address.
87 * @hldev: HAL device handle.
88 * @isrbar0: BAR0 mapped address.
89 * * Set BAR0 address in the HAL device object.
90 */
91 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_isrbar0_set(xge_hal_device_t * hldev,char * isrbar0)92 xge_hal_device_isrbar0_set(xge_hal_device_t *hldev, char *isrbar0)
93 {
94 xge_assert(isrbar0);
95 hldev->isrbar0 = isrbar0;
96 }
97
98 /**
99 * xge_hal_device_bar1_set - Set BAR1 mapped address.
100 * @hldev: HAL device handle.
101 * @channelh: Channel handle.
102 * @bar1: BAR1 mapped address.
103 *
104 * Set BAR1 address for the given channel.
105 */
106 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_bar1_set(xge_hal_device_t * hldev,xge_hal_channel_h channelh,char * bar1)107 xge_hal_device_bar1_set(xge_hal_device_t *hldev, xge_hal_channel_h channelh,
108 char *bar1)
109 {
110 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
111
112 xge_assert(bar1);
113 xge_assert(fifo);
114
115 /* Initializing the BAR1 address as the start of
116 * the FIFO queue pointer and as a location of FIFO control
117 * word. */
118 fifo->hw_pair =
119 (xge_hal_fifo_hw_pair_t *) (bar1 +
120 (fifo->channel.post_qid * XGE_HAL_FIFO_HW_PAIR_OFFSET));
121 hldev->bar1 = bar1;
122 }
123
124
125 /**
126 * xge_hal_device_rev - Get Device revision number.
127 * @hldev: HAL device handle.
128 *
129 * Returns: Device revision number
130 */
131 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE int
xge_hal_device_rev(xge_hal_device_t * hldev)132 xge_hal_device_rev(xge_hal_device_t *hldev)
133 {
134 return hldev->revision;
135 }
136
137
138 /**
139 * xge_hal_device_begin_irq - Begin IRQ processing.
140 * @hldev: HAL device handle.
141 * @reason: "Reason" for the interrupt, the value of Xframe's
142 * general_int_status register.
143 *
144 * The function performs two actions, It first checks whether (shared IRQ) the
145 * interrupt was raised by the device. Next, it masks the device interrupts.
146 *
147 * Note:
148 * xge_hal_device_begin_irq() does not flush MMIO writes through the
149 * bridge. Therefore, two back-to-back interrupts are potentially possible.
150 * It is the responsibility of the ULD to make sure that only one
151 * xge_hal_device_continue_irq() runs at a time.
152 *
153 * Returns: 0, if the interrupt is not "ours" (note that in this case the
154 * device remain enabled).
155 * Otherwise, xge_hal_device_begin_irq() returns 64bit general adapter
156 * status.
157 * See also: xge_hal_device_handle_irq()
158 */
159 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_begin_irq(xge_hal_device_t * hldev,u64 * reason)160 xge_hal_device_begin_irq(xge_hal_device_t *hldev, u64 *reason)
161 {
162 u64 val64;
163 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
164
165 hldev->stats.sw_dev_info_stats.total_intr_cnt++;
166
167 val64 = xge_os_pio_mem_read64(hldev->pdev,
168 hldev->regh0, &isrbar0->general_int_status);
169 if (xge_os_unlikely(!val64)) {
170 /* not Xframe interrupt */
171 hldev->stats.sw_dev_info_stats.not_xge_intr_cnt++;
172 *reason = 0;
173 return XGE_HAL_ERR_WRONG_IRQ;
174 }
175
176 if (xge_os_unlikely(val64 == XGE_HAL_ALL_FOXES)) {
177 u64 adapter_status =
178 xge_os_pio_mem_read64(hldev->pdev, hldev->regh0,
179 &isrbar0->adapter_status);
180 if (adapter_status == XGE_HAL_ALL_FOXES) {
181 (void) xge_queue_produce(hldev->queueh,
182 XGE_HAL_EVENT_SLOT_FREEZE,
183 hldev,
184 1, /* critical: slot freeze */
185 sizeof(u64),
186 (void*)&adapter_status);
187 *reason = 0;
188 return XGE_HAL_ERR_CRITICAL;
189 }
190 }
191
192 *reason = val64;
193
194 /* separate fast path, i.e. no errors */
195 if (val64 & XGE_HAL_GEN_INTR_RXTRAFFIC) {
196 hldev->stats.sw_dev_info_stats.rx_traffic_intr_cnt++;
197 return XGE_HAL_OK;
198 }
199 if (val64 & XGE_HAL_GEN_INTR_TXTRAFFIC) {
200 hldev->stats.sw_dev_info_stats.tx_traffic_intr_cnt++;
201 return XGE_HAL_OK;
202 }
203
204 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
205 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXPIC)) {
206 xge_hal_status_e status;
207 hldev->stats.sw_dev_info_stats.txpic_intr_cnt++;
208 status = __hal_device_handle_txpic(hldev, val64);
209 if (status != XGE_HAL_OK) {
210 return status;
211 }
212 }
213
214 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXDMA)) {
215 xge_hal_status_e status;
216 hldev->stats.sw_dev_info_stats.txdma_intr_cnt++;
217 status = __hal_device_handle_txdma(hldev, val64);
218 if (status != XGE_HAL_OK) {
219 return status;
220 }
221 }
222
223 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXMAC)) {
224 xge_hal_status_e status;
225 hldev->stats.sw_dev_info_stats.txmac_intr_cnt++;
226 status = __hal_device_handle_txmac(hldev, val64);
227 if (status != XGE_HAL_OK) {
228 return status;
229 }
230 }
231
232 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXXGXS)) {
233 xge_hal_status_e status;
234 hldev->stats.sw_dev_info_stats.txxgxs_intr_cnt++;
235 status = __hal_device_handle_txxgxs(hldev, val64);
236 if (status != XGE_HAL_OK) {
237 return status;
238 }
239 }
240
241 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXPIC)) {
242 xge_hal_status_e status;
243 hldev->stats.sw_dev_info_stats.rxpic_intr_cnt++;
244 status = __hal_device_handle_rxpic(hldev, val64);
245 if (status != XGE_HAL_OK) {
246 return status;
247 }
248 }
249
250 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXDMA)) {
251 xge_hal_status_e status;
252 hldev->stats.sw_dev_info_stats.rxdma_intr_cnt++;
253 status = __hal_device_handle_rxdma(hldev, val64);
254 if (status != XGE_HAL_OK) {
255 return status;
256 }
257 }
258
259 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXMAC)) {
260 xge_hal_status_e status;
261 hldev->stats.sw_dev_info_stats.rxmac_intr_cnt++;
262 status = __hal_device_handle_rxmac(hldev, val64);
263 if (status != XGE_HAL_OK) {
264 return status;
265 }
266 }
267
268 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXXGXS)) {
269 xge_hal_status_e status;
270 hldev->stats.sw_dev_info_stats.rxxgxs_intr_cnt++;
271 status = __hal_device_handle_rxxgxs(hldev, val64);
272 if (status != XGE_HAL_OK) {
273 return status;
274 }
275 }
276
277 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_MC)) {
278 xge_hal_status_e status;
279 hldev->stats.sw_dev_info_stats.mc_intr_cnt++;
280 status = __hal_device_handle_mc(hldev, val64);
281 if (status != XGE_HAL_OK) {
282 return status;
283 }
284 }
285
286 return XGE_HAL_OK;
287 }
288
289 /**
290 * xge_hal_device_clear_rx - Acknowledge (that is, clear) the
291 * condition that has caused the RX interrupt.
292 * @hldev: HAL device handle.
293 *
294 * Acknowledge (that is, clear) the condition that has caused
295 * the Rx interrupt.
296 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
297 * xge_hal_device_clear_tx(), xge_hal_device_mask_rx().
298 */
299 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_clear_rx(xge_hal_device_t * hldev)300 xge_hal_device_clear_rx(xge_hal_device_t *hldev)
301 {
302 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
303
304 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
305 0xFFFFFFFFFFFFFFFFULL,
306 &isrbar0->rx_traffic_int);
307 }
308
309 /**
310 * xge_hal_device_clear_tx - Acknowledge (that is, clear) the
311 * condition that has caused the TX interrupt.
312 * @hldev: HAL device handle.
313 *
314 * Acknowledge (that is, clear) the condition that has caused
315 * the Tx interrupt.
316 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
317 * xge_hal_device_clear_rx(), xge_hal_device_mask_tx().
318 */
319 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_clear_tx(xge_hal_device_t * hldev)320 xge_hal_device_clear_tx(xge_hal_device_t *hldev)
321 {
322 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
323
324 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
325 0xFFFFFFFFFFFFFFFFULL,
326 &isrbar0->tx_traffic_int);
327 }
328
329 /**
330 * xge_hal_device_poll_rx_channel - Poll Rx channel for completed
331 * descriptors and process the same.
332 * @channel: HAL channel.
333 * @got_rx: Buffer to return the flag set if receive interrupt is occured
334 *
335 * The function polls the Rx channel for the completed descriptors and calls
336 * the upper-layer driver (ULD) via supplied completion callback.
337 *
338 * Returns: XGE_HAL_OK, if the polling is completed successful.
339 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
340 * descriptors available which are yet to be processed.
341 *
342 * See also: xge_hal_device_poll_tx_channel()
343 */
344 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_rx_channel(xge_hal_channel_t * channel,int * got_rx)345 xge_hal_device_poll_rx_channel(xge_hal_channel_t *channel, int *got_rx)
346 {
347 xge_hal_status_e ret = XGE_HAL_OK;
348 xge_hal_dtr_h first_dtrh;
349 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
350 u8 t_code;
351 int got_bytes;
352
353 /* for each opened rx channel */
354 got_bytes = *got_rx = 0;
355 ((xge_hal_ring_t *)channel)->cmpl_cnt = 0;
356 channel->poll_bytes = 0;
357 if ((ret = xge_hal_ring_dtr_next_completed (channel, &first_dtrh,
358 &t_code)) == XGE_HAL_OK) {
359 if (channel->callback(channel, first_dtrh,
360 t_code, channel->userdata) != XGE_HAL_OK) {
361 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
362 got_bytes += channel->poll_bytes + 1;
363 ret = XGE_HAL_COMPLETIONS_REMAIN;
364 } else {
365 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
366 got_bytes += channel->poll_bytes + 1;
367 }
368 }
369
370 if (*got_rx) {
371 hldev->irq_workload_rxd[channel->post_qid] += *got_rx;
372 hldev->irq_workload_rxcnt[channel->post_qid] ++;
373 }
374 hldev->irq_workload_rxlen[channel->post_qid] += got_bytes;
375
376 return ret;
377 }
378
379 /**
380 * xge_hal_device_poll_tx_channel - Poll Tx channel for completed
381 * descriptors and process the same.
382 * @channel: HAL channel.
383 * @got_tx: Buffer to return the flag set if transmit interrupt is occured
384 *
385 * The function polls the Tx channel for the completed descriptors and calls
386 * the upper-layer driver (ULD) via supplied completion callback.
387 *
388 * Returns: XGE_HAL_OK, if the polling is completed successful.
389 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
390 * descriptors available which are yet to be processed.
391 *
392 * See also: xge_hal_device_poll_rx_channel().
393 */
394 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_tx_channel(xge_hal_channel_t * channel,int * got_tx)395 xge_hal_device_poll_tx_channel(xge_hal_channel_t *channel, int *got_tx)
396 {
397 xge_hal_dtr_h first_dtrh;
398 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
399 u8 t_code;
400 int got_bytes;
401
402 /* for each opened tx channel */
403 got_bytes = *got_tx = 0;
404 channel->poll_bytes = 0;
405 if (xge_hal_fifo_dtr_next_completed (channel, &first_dtrh,
406 &t_code) == XGE_HAL_OK) {
407 if (channel->callback(channel, first_dtrh,
408 t_code, channel->userdata) != XGE_HAL_OK) {
409 (*got_tx)++;
410 got_bytes += channel->poll_bytes + 1;
411 return XGE_HAL_COMPLETIONS_REMAIN;
412 }
413 (*got_tx)++;
414 got_bytes += channel->poll_bytes + 1;
415 }
416
417 if (*got_tx) {
418 hldev->irq_workload_txd[channel->post_qid] += *got_tx;
419 hldev->irq_workload_txcnt[channel->post_qid] ++;
420 }
421 hldev->irq_workload_txlen[channel->post_qid] += got_bytes;
422
423 return XGE_HAL_OK;
424 }
425
426 /**
427 * xge_hal_device_poll_rx_channels - Poll Rx channels for completed
428 * descriptors and process the same.
429 * @hldev: HAL device handle.
430 * @got_rx: Buffer to return flag set if receive is ready
431 *
432 * The function polls the Rx channels for the completed descriptors and calls
433 * the upper-layer driver (ULD) via supplied completion callback.
434 *
435 * Returns: XGE_HAL_OK, if the polling is completed successful.
436 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
437 * descriptors available which are yet to be processed.
438 *
439 * See also: xge_hal_device_poll_tx_channels(), xge_hal_device_continue_irq().
440 */
441 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_rx_channels(xge_hal_device_t * hldev,int * got_rx)442 xge_hal_device_poll_rx_channels(xge_hal_device_t *hldev, int *got_rx)
443 {
444 xge_list_t *item;
445 xge_hal_channel_t *channel;
446
447 /* for each opened rx channel */
448 xge_list_for_each(item, &hldev->ring_channels) {
449 if (hldev->terminating)
450 return XGE_HAL_OK;
451 channel = xge_container_of(item, xge_hal_channel_t, item);
452 (void) xge_hal_device_poll_rx_channel(channel, got_rx);
453 }
454
455 return XGE_HAL_OK;
456 }
457
458 /**
459 * xge_hal_device_poll_tx_channels - Poll Tx channels for completed
460 * descriptors and process the same.
461 * @hldev: HAL device handle.
462 * @got_tx: Buffer to return flag set if transmit is ready
463 *
464 * The function polls the Tx channels for the completed descriptors and calls
465 * the upper-layer driver (ULD) via supplied completion callback.
466 *
467 * Returns: XGE_HAL_OK, if the polling is completed successful.
468 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
469 * descriptors available which are yet to be processed.
470 *
471 * See also: xge_hal_device_poll_rx_channels(), xge_hal_device_continue_irq().
472 */
473 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_tx_channels(xge_hal_device_t * hldev,int * got_tx)474 xge_hal_device_poll_tx_channels(xge_hal_device_t *hldev, int *got_tx)
475 {
476 xge_list_t *item;
477 xge_hal_channel_t *channel;
478
479 /* for each opened tx channel */
480 xge_list_for_each(item, &hldev->fifo_channels) {
481 if (hldev->terminating)
482 return XGE_HAL_OK;
483 channel = xge_container_of(item, xge_hal_channel_t, item);
484 (void) xge_hal_device_poll_tx_channel(channel, got_tx);
485 }
486
487 return XGE_HAL_OK;
488 }
489
490 /**
491 * xge_hal_device_mask_tx - Mask Tx interrupts.
492 * @hldev: HAL device handle.
493 *
494 * Mask Tx device interrupts.
495 *
496 * See also: xge_hal_device_unmask_tx(), xge_hal_device_mask_rx(),
497 * xge_hal_device_clear_tx().
498 */
499 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_mask_tx(xge_hal_device_t * hldev)500 xge_hal_device_mask_tx(xge_hal_device_t *hldev)
501 {
502 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
503
504 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
505 0xFFFFFFFFFFFFFFFFULL,
506 &isrbar0->tx_traffic_mask);
507 }
508
509 /**
510 * xge_hal_device_mask_rx - Mask Rx interrupts.
511 * @hldev: HAL device handle.
512 *
513 * Mask Rx device interrupts.
514 *
515 * See also: xge_hal_device_unmask_rx(), xge_hal_device_mask_tx(),
516 * xge_hal_device_clear_rx().
517 */
518 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_mask_rx(xge_hal_device_t * hldev)519 xge_hal_device_mask_rx(xge_hal_device_t *hldev)
520 {
521 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
522
523 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
524 0xFFFFFFFFFFFFFFFFULL,
525 &isrbar0->rx_traffic_mask);
526 }
527
528 /**
529 * xge_hal_device_mask_all - Mask all device interrupts.
530 * @hldev: HAL device handle.
531 *
532 * Mask all device interrupts.
533 *
534 * See also: xge_hal_device_unmask_all()
535 */
536 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_mask_all(xge_hal_device_t * hldev)537 xge_hal_device_mask_all(xge_hal_device_t *hldev)
538 {
539 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
540
541 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
542 0xFFFFFFFFFFFFFFFFULL,
543 &isrbar0->general_int_mask);
544 }
545
546 /**
547 * xge_hal_device_unmask_tx - Unmask Tx interrupts.
548 * @hldev: HAL device handle.
549 *
550 * Unmask Tx device interrupts.
551 *
552 * See also: xge_hal_device_mask_tx(), xge_hal_device_clear_tx().
553 */
554 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_unmask_tx(xge_hal_device_t * hldev)555 xge_hal_device_unmask_tx(xge_hal_device_t *hldev)
556 {
557 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
558
559 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
560 0x0ULL,
561 &isrbar0->tx_traffic_mask);
562 }
563
564 /**
565 * xge_hal_device_unmask_rx - Unmask Rx interrupts.
566 * @hldev: HAL device handle.
567 *
568 * Unmask Rx device interrupts.
569 *
570 * See also: xge_hal_device_mask_rx(), xge_hal_device_clear_rx().
571 */
572 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_unmask_rx(xge_hal_device_t * hldev)573 xge_hal_device_unmask_rx(xge_hal_device_t *hldev)
574 {
575 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
576
577 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
578 0x0ULL,
579 &isrbar0->rx_traffic_mask);
580 }
581
582 /**
583 * xge_hal_device_unmask_all - Unmask all device interrupts.
584 * @hldev: HAL device handle.
585 *
586 * Unmask all device interrupts.
587 *
588 * See also: xge_hal_device_mask_all()
589 */
590 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_unmask_all(xge_hal_device_t * hldev)591 xge_hal_device_unmask_all(xge_hal_device_t *hldev)
592 {
593 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
594
595 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
596 0x0ULL,
597 &isrbar0->general_int_mask);
598 }
599
600
601 /**
602 * xge_hal_device_continue_irq - Continue handling IRQ: process all
603 * completed descriptors.
604 * @hldev: HAL device handle.
605 *
606 * Process completed descriptors and unmask the device interrupts.
607 *
608 * The xge_hal_device_continue_irq() walks all open channels
609 * and calls upper-layer driver (ULD) via supplied completion
610 * callback. Note that the completion callback is specified at channel open
611 * time, see xge_hal_channel_open().
612 *
613 * Note that the xge_hal_device_continue_irq is part of the _fast_ path.
614 * To optimize the processing, the function does _not_ check for
615 * errors and alarms.
616 *
617 * The latter is done in a polling fashion, via xge_hal_device_poll().
618 *
619 * Returns: XGE_HAL_OK.
620 *
621 * See also: xge_hal_device_handle_irq(), xge_hal_device_poll(),
622 * xge_hal_ring_dtr_next_completed(),
623 * xge_hal_fifo_dtr_next_completed(), xge_hal_channel_callback_f{}.
624 */
625 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_continue_irq(xge_hal_device_t * hldev)626 xge_hal_device_continue_irq(xge_hal_device_t *hldev)
627 {
628 int got_rx = 1, got_tx = 1;
629 int isr_polling_cnt = hldev->config.isr_polling_cnt;
630 int count = 0;
631
632 do
633 {
634 if (got_rx)
635 (void) xge_hal_device_poll_rx_channels(hldev, &got_rx);
636 if (got_tx && hldev->tti_enabled)
637 (void) xge_hal_device_poll_tx_channels(hldev, &got_tx);
638
639 if (!got_rx && !got_tx)
640 break;
641
642 count += (got_rx + got_tx);
643 }while (isr_polling_cnt--);
644
645 if (!count)
646 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
647
648 return XGE_HAL_OK;
649 }
650
651 /**
652 * xge_hal_device_handle_irq - Handle device IRQ.
653 * @hldev: HAL device handle.
654 *
655 * Perform the complete handling of the line interrupt. The function
656 * performs two calls.
657 * First it uses xge_hal_device_begin_irq() to check the reason for
658 * the interrupt and mask the device interrupts.
659 * Second, it calls xge_hal_device_continue_irq() to process all
660 * completed descriptors and re-enable the interrupts.
661 *
662 * Returns: XGE_HAL_OK - success;
663 * XGE_HAL_ERR_WRONG_IRQ - (shared) IRQ produced by other device.
664 *
665 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq().
666 */
667 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_handle_irq(xge_hal_device_t * hldev)668 xge_hal_device_handle_irq(xge_hal_device_t *hldev)
669 {
670 u64 reason;
671 xge_hal_status_e status;
672
673 xge_hal_device_mask_all(hldev);
674
675 status = xge_hal_device_begin_irq(hldev, &reason);
676 if (status != XGE_HAL_OK) {
677 xge_hal_device_unmask_all(hldev);
678 return status;
679 }
680
681 if (reason & XGE_HAL_GEN_INTR_RXTRAFFIC) {
682 xge_hal_device_clear_rx(hldev);
683 }
684
685 status = xge_hal_device_continue_irq(hldev);
686
687 xge_hal_device_clear_tx(hldev);
688
689 xge_hal_device_unmask_all(hldev);
690
691 return status;
692 }
693
694 #if defined(XGE_HAL_CONFIG_LRO)
695
696
697 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
__hal_lro_check_for_session_match(lro_t * lro,tcplro_t * tcp,iplro_t * ip)698 __hal_lro_check_for_session_match(lro_t *lro, tcplro_t *tcp, iplro_t *ip)
699 {
700
701 /* Match Source address field */
702 if ((lro->ip_hdr->saddr != ip->saddr))
703 return XGE_HAL_FAIL;
704
705 /* Match Destination address field */
706 if ((lro->ip_hdr->daddr != ip->daddr))
707 return XGE_HAL_FAIL;
708
709 /* Match Source Port field */
710 if ((lro->tcp_hdr->source != tcp->source))
711 return XGE_HAL_FAIL;
712
713 /* Match Destination Port field */
714 if ((lro->tcp_hdr->dest != tcp->dest))
715 return XGE_HAL_FAIL;
716
717 return XGE_HAL_OK;
718 }
719
720 /*
721 * __hal_tcp_seg_len: Find the tcp seg len.
722 * @ip: ip header.
723 * @tcp: tcp header.
724 * returns: Tcp seg length.
725 */
726 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL u16
__hal_tcp_seg_len(iplro_t * ip,tcplro_t * tcp)727 __hal_tcp_seg_len(iplro_t *ip, tcplro_t *tcp)
728 {
729 u16 ret;
730
731 ret = (xge_os_ntohs(ip->tot_len) -
732 ((ip->version_ihl & 0x0F)<<2) -
733 ((tcp->doff_res)>>2));
734 return (ret);
735 }
736
737 /*
738 * __hal_ip_lro_capable: Finds whether ip is lro capable.
739 * @ip: ip header.
740 * @ext_info: descriptor info.
741 */
742 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_ip_lro_capable(iplro_t * ip,xge_hal_dtr_info_t * ext_info)743 __hal_ip_lro_capable(iplro_t *ip,
744 xge_hal_dtr_info_t *ext_info)
745 {
746
747 #ifdef XGE_LL_DEBUG_DUMP_PKT
748 {
749 u16 i;
750 u8 ch, *iph = (u8 *)ip;
751
752 xge_debug_ring(XGE_TRACE, "Dump Ip:" );
753 for (i =0; i < 40; i++) {
754 ch = ntohs(*((u8 *)(iph + i)) );
755 printf("i:%d %02x, ",i,ch);
756 }
757 }
758 #endif
759
760 if (ip->version_ihl != IP_FAST_PATH_HDR_MASK) {
761 xge_debug_ring(XGE_ERR, "iphdr !=45 :%d",ip->version_ihl);
762 return XGE_HAL_FAIL;
763 }
764
765 if (ext_info->proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) {
766 xge_debug_ring(XGE_ERR, "IP fragmented");
767 return XGE_HAL_FAIL;
768 }
769
770 return XGE_HAL_OK;
771 }
772
773 /*
774 * __hal_tcp_lro_capable: Finds whether tcp is lro capable.
775 * @ip: ip header.
776 * @tcp: tcp header.
777 */
778 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_tcp_lro_capable(iplro_t * ip,tcplro_t * tcp,lro_t * lro,int * ts_off)779 __hal_tcp_lro_capable(iplro_t *ip, tcplro_t *tcp, lro_t *lro, int *ts_off)
780 {
781 #ifdef XGE_LL_DEBUG_DUMP_PKT
782 {
783 u8 ch;
784 u16 i;
785
786 xge_debug_ring(XGE_TRACE, "Dump Tcp:" );
787 for (i =0; i < 20; i++) {
788 ch = ntohs(*((u8 *)((u8 *)tcp + i)) );
789 xge_os_printf("i:%d %02x, ",i,ch);
790 }
791 }
792 #endif
793 if ((TCP_FAST_PATH_HDR_MASK2 != tcp->ctrl) &&
794 (TCP_FAST_PATH_HDR_MASK3 != tcp->ctrl))
795 goto _exit_fail;
796
797 *ts_off = -1;
798 if (TCP_FAST_PATH_HDR_MASK1 != tcp->doff_res) {
799 u16 tcp_hdr_len = tcp->doff_res >> 2; /* TCP header len */
800 u16 off = 20; /* Start of tcp options */
801 int i, diff;
802
803 /* Does Packet can contain time stamp */
804 if (tcp_hdr_len < 32) {
805 /*
806 * If the session is not opened, we can consider
807 * this packet for LRO
808 */
809 if (lro == NULL)
810 return XGE_HAL_OK;
811
812 goto _exit_fail;
813 }
814
815 /* Ignore No-operation 0x1 */
816 while (((u8 *)tcp)[off] == 0x1)
817 off++;
818
819 /* Next option == Timestamp */
820 if (((u8 *)tcp)[off] != 0x8) {
821 /*
822 * If the session ie not opened, we can consider
823 * this packet for LRO
824 */
825 if (lro == NULL)
826 return XGE_HAL_OK;
827
828 goto _exit_fail;
829 }
830
831 *ts_off = off;
832 if (lro == NULL)
833 return XGE_HAL_OK;
834
835 /*
836 * Now the session is opened. If the LRO frame doesn't
837 * have time stamp, we cannot consider current packet for
838 * LRO.
839 */
840 if (lro->ts_off == -1) {
841 xge_debug_ring(XGE_ERR, "Pkt received with time stamp after session opened with no time stamp : %02x %02x", tcp->doff_res, tcp->ctrl);
842 return XGE_HAL_FAIL;
843 }
844
845 /*
846 * If the difference is greater than three, then there are
847 * more options possible.
848 * else, there are two cases:
849 * case 1: remaining are padding bytes.
850 * case 2: remaining can contain options or padding
851 */
852 off += ((u8 *)tcp)[off+1];
853 diff = tcp_hdr_len - off;
854 if (diff > 3) {
855 /*
856 * Probably contains more options.
857 */
858 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl);
859 return XGE_HAL_FAIL;
860 }
861
862 for (i = 0; i < diff; i++) {
863 u8 byte = ((u8 *)tcp)[off+i];
864
865 /* Ignore No-operation 0x1 */
866 if ((byte == 0x0) || (byte == 0x1))
867 continue;
868 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl);
869 return XGE_HAL_FAIL;
870 }
871
872 /*
873 * Update the time stamp of LRO frame.
874 */
875 xge_os_memcpy(((char *)lro->tcp_hdr + lro->ts_off + 2),
876 (char *)((char *)tcp + (*ts_off) + 2), 8);
877 }
878
879 return XGE_HAL_OK;
880
881 _exit_fail:
882 xge_debug_ring(XGE_TRACE, "tcphdr not fastpth %02x %02x", tcp->doff_res, tcp->ctrl);
883 return XGE_HAL_FAIL;
884
885 }
886
887 /*
888 * __hal_lro_capable: Finds whether frame is lro capable.
889 * @buffer: Ethernet frame.
890 * @ip: ip frame.
891 * @tcp: tcp frame.
892 * @ext_info: Descriptor info.
893 */
894 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_lro_capable(u8 * buffer,iplro_t ** ip,tcplro_t ** tcp,xge_hal_dtr_info_t * ext_info)895 __hal_lro_capable( u8 *buffer,
896 iplro_t **ip,
897 tcplro_t **tcp,
898 xge_hal_dtr_info_t *ext_info)
899 {
900 u8 ip_off, ip_length;
901
902 if (!(ext_info->proto & XGE_HAL_FRAME_PROTO_TCP)) {
903 xge_debug_ring(XGE_ERR, "Cant do lro %d", ext_info->proto);
904 return XGE_HAL_FAIL;
905 }
906
907 if ( !*ip )
908 {
909 #ifdef XGE_LL_DEBUG_DUMP_PKT
910 {
911 u8 ch;
912 u16 i;
913
914 xge_os_printf("Dump Eth:" );
915 for (i =0; i < 60; i++) {
916 ch = ntohs(*((u8 *)(buffer + i)) );
917 xge_os_printf("i:%d %02x, ",i,ch);
918 }
919 }
920 #endif
921
922 switch (ext_info->frame) {
923 case XGE_HAL_FRAME_TYPE_DIX:
924 ip_off = XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE;
925 break;
926 case XGE_HAL_FRAME_TYPE_LLC:
927 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE +
928 XGE_HAL_HEADER_802_2_SIZE);
929 break;
930 case XGE_HAL_FRAME_TYPE_SNAP:
931 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE +
932 XGE_HAL_HEADER_SNAP_SIZE);
933 break;
934 default: // XGE_HAL_FRAME_TYPE_IPX, etc.
935 return XGE_HAL_FAIL;
936 }
937
938
939 if (ext_info->proto & XGE_HAL_FRAME_PROTO_VLAN_TAGGED) {
940 ip_off += XGE_HAL_HEADER_VLAN_SIZE;
941 }
942
943 /* Grab ip, tcp headers */
944 *ip = (iplro_t *)((char*)buffer + ip_off);
945 } /* !*ip */
946
947 ip_length = (u8)((*ip)->version_ihl & 0x0F);
948 ip_length = ip_length <<2;
949 *tcp = (tcplro_t *)((char *)*ip + ip_length);
950
951 xge_debug_ring(XGE_TRACE, "ip_length:%d ip:"XGE_OS_LLXFMT
952 " tcp:"XGE_OS_LLXFMT"", (int)ip_length,
953 (unsigned long long)(ulong_t)*ip, (unsigned long long)(ulong_t)*tcp);
954
955 return XGE_HAL_OK;
956
957 }
958
959
960 /*
961 * __hal_open_lro_session: Open a new LRO session.
962 * @buffer: Ethernet frame.
963 * @ip: ip header.
964 * @tcp: tcp header.
965 * @lro: lro pointer
966 * @ext_info: Descriptor info.
967 * @hldev: Hal context.
968 * @ring_lro: LRO descriptor per rx ring.
969 * @slot: Bucket no.
970 * @tcp_seg_len: Length of tcp segment.
971 * @ts_off: time stamp offset in the packet.
972 */
973 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
__hal_open_lro_session(u8 * buffer,iplro_t * ip,tcplro_t * tcp,lro_t ** lro,xge_hal_device_t * hldev,xge_hal_lro_desc_t * ring_lro,int slot,u32 tcp_seg_len,int ts_off)974 __hal_open_lro_session (u8 *buffer, iplro_t *ip, tcplro_t *tcp, lro_t **lro,
975 xge_hal_device_t *hldev, xge_hal_lro_desc_t *ring_lro, int slot,
976 u32 tcp_seg_len, int ts_off)
977 {
978
979 lro_t *lro_new = &ring_lro->lro_pool[slot];
980
981 lro_new->in_use = 1;
982 lro_new->ll_hdr = buffer;
983 lro_new->ip_hdr = ip;
984 lro_new->tcp_hdr = tcp;
985 lro_new->tcp_next_seq_num = tcp_seg_len + xge_os_ntohl(
986 tcp->seq);
987 lro_new->tcp_seq_num = tcp->seq;
988 lro_new->tcp_ack_num = tcp->ack_seq;
989 lro_new->sg_num = 1;
990 lro_new->total_length = xge_os_ntohs(ip->tot_len);
991 lro_new->frags_len = 0;
992 lro_new->ts_off = ts_off;
993
994 hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
995 hldev->stats.sw_dev_info_stats.tot_lro_sessions++;
996
997 *lro = ring_lro->lro_recent = lro_new;
998 return;
999 }
1000 /*
1001 * __hal_lro_get_free_slot: Get a free LRO bucket.
1002 * @ring_lro: LRO descriptor per ring.
1003 */
1004 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
__hal_lro_get_free_slot(xge_hal_lro_desc_t * ring_lro)1005 __hal_lro_get_free_slot (xge_hal_lro_desc_t *ring_lro)
1006 {
1007 int i;
1008
1009 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
1010 lro_t *lro_temp = &ring_lro->lro_pool[i];
1011
1012 if (!lro_temp->in_use)
1013 return i;
1014 }
1015 return -1;
1016 }
1017
1018 /*
1019 * __hal_get_lro_session: Gets matching LRO session or creates one.
1020 * @eth_hdr: Ethernet header.
1021 * @ip: ip header.
1022 * @tcp: tcp header.
1023 * @lro: lro pointer
1024 * @ext_info: Descriptor info.
1025 * @hldev: Hal context.
1026 * @ring_lro: LRO descriptor per rx ring
1027 */
1028 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_get_lro_session(u8 * eth_hdr,iplro_t * ip,tcplro_t * tcp,lro_t ** lro,xge_hal_dtr_info_t * ext_info,xge_hal_device_t * hldev,xge_hal_lro_desc_t * ring_lro,lro_t ** lro_end3)1029 __hal_get_lro_session (u8 *eth_hdr,
1030 iplro_t *ip,
1031 tcplro_t *tcp,
1032 lro_t **lro,
1033 xge_hal_dtr_info_t *ext_info,
1034 xge_hal_device_t *hldev,
1035 xge_hal_lro_desc_t *ring_lro,
1036 lro_t **lro_end3 /* Valid only when ret=END_3 */)
1037 {
1038 lro_t *lro_match;
1039 int i, free_slot = -1;
1040 u32 tcp_seg_len;
1041 int ts_off = -1;
1042
1043 *lro = lro_match = NULL;
1044 /*
1045 * Compare the incoming frame with the lro session left from the
1046 * previous call. There is a good chance that this incoming frame
1047 * matches the lro session.
1048 */
1049 if (ring_lro->lro_recent && ring_lro->lro_recent->in_use) {
1050 if (__hal_lro_check_for_session_match(ring_lro->lro_recent,
1051 tcp, ip)
1052 == XGE_HAL_OK)
1053 lro_match = ring_lro->lro_recent;
1054 }
1055
1056 if (!lro_match) {
1057 /*
1058 * Search in the pool of LROs for the session that matches
1059 * the incoming frame.
1060 */
1061 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
1062 lro_t *lro_temp = &ring_lro->lro_pool[i];
1063
1064 if (!lro_temp->in_use) {
1065 if (free_slot == -1)
1066 free_slot = i;
1067 continue;
1068 }
1069
1070 if (__hal_lro_check_for_session_match(lro_temp, tcp,
1071 ip) == XGE_HAL_OK) {
1072 lro_match = lro_temp;
1073 break;
1074 }
1075 }
1076 }
1077
1078
1079 if (lro_match) {
1080 /*
1081 * Matching LRO Session found
1082 */
1083 *lro = lro_match;
1084
1085 if (lro_match->tcp_next_seq_num != xge_os_ntohl(tcp->seq)) {
1086 xge_debug_ring(XGE_ERR, "**retransmit **"
1087 "found***");
1088 hldev->stats.sw_dev_info_stats.lro_out_of_seq_pkt_cnt++;
1089 return XGE_HAL_INF_LRO_END_2;
1090 }
1091
1092 if (XGE_HAL_OK != __hal_ip_lro_capable(ip, ext_info))
1093 {
1094 return XGE_HAL_INF_LRO_END_2;
1095 }
1096
1097 if (XGE_HAL_OK != __hal_tcp_lro_capable(ip, tcp, lro_match,
1098 &ts_off)) {
1099 /*
1100 * Close the current session and open a new
1101 * LRO session with this packet,
1102 * provided it has tcp payload
1103 */
1104 tcp_seg_len = __hal_tcp_seg_len(ip, tcp);
1105 if (tcp_seg_len == 0)
1106 {
1107 return XGE_HAL_INF_LRO_END_2;
1108 }
1109
1110 /* Get a free bucket */
1111 free_slot = __hal_lro_get_free_slot(ring_lro);
1112 if (free_slot == -1)
1113 {
1114 return XGE_HAL_INF_LRO_END_2;
1115 }
1116
1117 /*
1118 * Open a new LRO session
1119 */
1120 __hal_open_lro_session (eth_hdr, ip, tcp, lro_end3,
1121 hldev, ring_lro, free_slot, tcp_seg_len,
1122 ts_off);
1123
1124 return XGE_HAL_INF_LRO_END_3;
1125 }
1126
1127 /*
1128 * The frame is good, in-sequence, can be LRO-ed;
1129 * take its (latest) ACK - unless it is a dupack.
1130 * Note: to be exact need to check window size as well..
1131 */
1132 if (lro_match->tcp_ack_num == tcp->ack_seq &&
1133 lro_match->tcp_seq_num == tcp->seq) {
1134 hldev->stats.sw_dev_info_stats.lro_dup_pkt_cnt++;
1135 return XGE_HAL_INF_LRO_END_2;
1136 }
1137
1138 lro_match->tcp_seq_num = tcp->seq;
1139 lro_match->tcp_ack_num = tcp->ack_seq;
1140 lro_match->frags_len += __hal_tcp_seg_len(ip, tcp);
1141
1142 ring_lro->lro_recent = lro_match;
1143
1144 return XGE_HAL_INF_LRO_CONT;
1145 }
1146
1147 /* ********** New Session ***************/
1148 if (free_slot == -1)
1149 return XGE_HAL_INF_LRO_UNCAPABLE;
1150
1151 if (XGE_HAL_FAIL == __hal_ip_lro_capable(ip, ext_info))
1152 return XGE_HAL_INF_LRO_UNCAPABLE;
1153
1154 if (XGE_HAL_FAIL == __hal_tcp_lro_capable(ip, tcp, NULL, &ts_off))
1155 return XGE_HAL_INF_LRO_UNCAPABLE;
1156
1157 xge_debug_ring(XGE_TRACE, "Creating lro session.");
1158
1159 /*
1160 * Open a LRO session, provided the packet contains payload.
1161 */
1162 tcp_seg_len = __hal_tcp_seg_len(ip, tcp);
1163 if (tcp_seg_len == 0)
1164 return XGE_HAL_INF_LRO_UNCAPABLE;
1165
1166 __hal_open_lro_session (eth_hdr, ip, tcp, lro, hldev, ring_lro, free_slot,
1167 tcp_seg_len, ts_off);
1168
1169 return XGE_HAL_INF_LRO_BEGIN;
1170 }
1171
1172 /*
1173 * __hal_lro_under_optimal_thresh: Finds whether combined session is optimal.
1174 * @ip: ip header.
1175 * @tcp: tcp header.
1176 * @lro: lro pointer
1177 * @hldev: Hal context.
1178 */
1179 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_lro_under_optimal_thresh(iplro_t * ip,tcplro_t * tcp,lro_t * lro,xge_hal_device_t * hldev)1180 __hal_lro_under_optimal_thresh (iplro_t *ip,
1181 tcplro_t *tcp,
1182 lro_t *lro,
1183 xge_hal_device_t *hldev)
1184 {
1185 if (!lro) return XGE_HAL_FAIL;
1186
1187 if ((lro->total_length + __hal_tcp_seg_len(ip, tcp) ) >
1188 hldev->config.lro_frm_len) {
1189 xge_debug_ring(XGE_TRACE, "Max LRO frame len exceeded:"
1190 "max length %d ", hldev->config.lro_frm_len);
1191 hldev->stats.sw_dev_info_stats.lro_frm_len_exceed_cnt++;
1192 return XGE_HAL_FAIL;
1193 }
1194
1195 if (lro->sg_num == hldev->config.lro_sg_size) {
1196 xge_debug_ring(XGE_TRACE, "Max sg count exceeded:"
1197 "max sg %d ", hldev->config.lro_sg_size);
1198 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1199 return XGE_HAL_FAIL;
1200 }
1201
1202 return XGE_HAL_OK;
1203 }
1204
1205 /*
1206 * __hal_collapse_ip_hdr: Collapses ip header.
1207 * @ip: ip header.
1208 * @tcp: tcp header.
1209 * @lro: lro pointer
1210 * @hldev: Hal context.
1211 */
1212 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_collapse_ip_hdr(iplro_t * ip,tcplro_t * tcp,lro_t * lro,xge_hal_device_t * hldev)1213 __hal_collapse_ip_hdr ( iplro_t *ip,
1214 tcplro_t *tcp,
1215 lro_t *lro,
1216 xge_hal_device_t *hldev)
1217 {
1218
1219 lro->total_length += __hal_tcp_seg_len(ip, tcp);
1220
1221 /* May be we have to handle time stamps or more options */
1222
1223 return XGE_HAL_OK;
1224
1225 }
1226
1227 /*
1228 * __hal_collapse_tcp_hdr: Collapses tcp header.
1229 * @ip: ip header.
1230 * @tcp: tcp header.
1231 * @lro: lro pointer
1232 * @hldev: Hal context.
1233 */
1234 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_collapse_tcp_hdr(iplro_t * ip,tcplro_t * tcp,lro_t * lro,xge_hal_device_t * hldev)1235 __hal_collapse_tcp_hdr ( iplro_t *ip,
1236 tcplro_t *tcp,
1237 lro_t *lro,
1238 xge_hal_device_t *hldev)
1239 {
1240 lro->tcp_next_seq_num += __hal_tcp_seg_len(ip, tcp);
1241 return XGE_HAL_OK;
1242
1243 }
1244
1245 /*
1246 * __hal_append_lro: Appends new frame to existing LRO session.
1247 * @ip: ip header.
1248 * @tcp: IN tcp header, OUT tcp payload.
1249 * @seg_len: tcp payload length.
1250 * @lro: lro pointer
1251 * @hldev: Hal context.
1252 */
1253 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_append_lro(iplro_t * ip,tcplro_t ** tcp,u32 * seg_len,lro_t * lro,xge_hal_device_t * hldev)1254 __hal_append_lro(iplro_t *ip,
1255 tcplro_t **tcp,
1256 u32 *seg_len,
1257 lro_t *lro,
1258 xge_hal_device_t *hldev)
1259 {
1260 (void) __hal_collapse_ip_hdr(ip, *tcp, lro, hldev);
1261 (void) __hal_collapse_tcp_hdr(ip, *tcp, lro, hldev);
1262 // Update mbuf chain will be done in ll driver.
1263 // xge_hal_accumulate_large_rx on success of appending new frame to
1264 // lro will return to ll driver tcpdata pointer, and tcp payload length.
1265 // along with return code lro frame appended.
1266
1267 lro->sg_num++;
1268 *seg_len = __hal_tcp_seg_len(ip, *tcp);
1269 *tcp = (tcplro_t *)((char *)*tcp + (((*tcp)->doff_res)>>2));
1270
1271 return XGE_HAL_OK;
1272
1273 }
1274
1275 /**
1276 * __xge_hal_accumulate_large_rx: LRO a given frame
1277 * frames
1278 * @ring: rx ring number
1279 * @eth_hdr: ethernet header.
1280 * @ip_hdr: ip header (optional)
1281 * @tcp: tcp header.
1282 * @seglen: packet length.
1283 * @p_lro: lro pointer.
1284 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
1285 * @hldev: HAL device.
1286 * @lro_end3: for lro_end3 output
1287 *
1288 * LRO the newly received frame, i.e. attach it (if possible) to the
1289 * already accumulated (i.e., already LRO-ed) received frames (if any),
1290 * to form one super-sized frame for the subsequent processing
1291 * by the stack.
1292 */
1293 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
xge_hal_lro_process_rx(int ring,u8 * eth_hdr,u8 * ip_hdr,tcplro_t ** tcp,u32 * seglen,lro_t ** p_lro,xge_hal_dtr_info_t * ext_info,xge_hal_device_t * hldev,lro_t ** lro_end3)1294 xge_hal_lro_process_rx(int ring, u8 *eth_hdr, u8 *ip_hdr, tcplro_t **tcp,
1295 u32 *seglen, lro_t **p_lro,
1296 xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
1297 lro_t **lro_end3)
1298 {
1299 iplro_t *ip = (iplro_t *)ip_hdr;
1300 xge_hal_status_e ret;
1301 lro_t *lro;
1302
1303 xge_debug_ring(XGE_TRACE, "Entered accumu lro. ");
1304 if (XGE_HAL_OK != __hal_lro_capable(eth_hdr, &ip, (tcplro_t **)tcp,
1305 ext_info))
1306 return XGE_HAL_INF_LRO_UNCAPABLE;
1307
1308 /*
1309 * This function shall get matching LRO or else
1310 * create one and return it
1311 */
1312 ret = __hal_get_lro_session(eth_hdr, ip, (tcplro_t *)*tcp,
1313 p_lro, ext_info, hldev, &hldev->lro_desc[ring],
1314 lro_end3);
1315 xge_debug_ring(XGE_TRACE, "ret from get_lro:%d ",ret);
1316 lro = *p_lro;
1317 if (XGE_HAL_INF_LRO_CONT == ret) {
1318 if (XGE_HAL_OK == __hal_lro_under_optimal_thresh(ip,
1319 (tcplro_t *)*tcp, lro, hldev)) {
1320 (void) __hal_append_lro(ip,(tcplro_t **) tcp, seglen,
1321 lro, hldev);
1322 hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
1323
1324 if (lro->sg_num >= hldev->config.lro_sg_size) {
1325 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1326 ret = XGE_HAL_INF_LRO_END_1;
1327 }
1328
1329 } else ret = XGE_HAL_INF_LRO_END_2;
1330 }
1331
1332 /*
1333 * Since its time to flush,
1334 * update ip header so that it can be sent up
1335 */
1336 if ((ret == XGE_HAL_INF_LRO_END_1) ||
1337 (ret == XGE_HAL_INF_LRO_END_2) ||
1338 (ret == XGE_HAL_INF_LRO_END_3)) {
1339 lro->ip_hdr->tot_len = xge_os_htons((*p_lro)->total_length);
1340 lro->ip_hdr->check = xge_os_htons(0);
1341 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1342 (lro->ip_hdr->version_ihl & 0x0F));
1343 lro->tcp_hdr->ack_seq = lro->tcp_ack_num;
1344 }
1345
1346 return (ret);
1347 }
1348
1349 /**
1350 * xge_hal_accumulate_large_rx: LRO a given frame
1351 * frames
1352 * @buffer: Ethernet frame.
1353 * @tcp: tcp header.
1354 * @seglen: packet length.
1355 * @p_lro: lro pointer.
1356 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
1357 * @hldev: HAL device.
1358 * @lro_end3: for lro_end3 output
1359 *
1360 * LRO the newly received frame, i.e. attach it (if possible) to the
1361 * already accumulated (i.e., already LRO-ed) received frames (if any),
1362 * to form one super-sized frame for the subsequent processing
1363 * by the stack.
1364 */
1365 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
xge_hal_accumulate_large_rx(u8 * buffer,tcplro_t ** tcp,u32 * seglen,lro_t ** p_lro,xge_hal_dtr_info_t * ext_info,xge_hal_device_t * hldev,lro_t ** lro_end3)1366 xge_hal_accumulate_large_rx(u8 *buffer, tcplro_t **tcp, u32 *seglen,
1367 lro_t **p_lro, xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
1368 lro_t **lro_end3)
1369 {
1370 int ring = 0;
1371 return xge_hal_lro_process_rx(ring, buffer, NULL, tcp, seglen, p_lro,
1372 ext_info, hldev, lro_end3);
1373 }
1374
1375 /**
1376 * xge_hal_lro_close_session: Close LRO session
1377 * @lro: LRO Session.
1378 * @hldev: HAL Context.
1379 */
1380 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
xge_hal_lro_close_session(lro_t * lro)1381 xge_hal_lro_close_session (lro_t *lro)
1382 {
1383 lro->in_use = 0;
1384 }
1385
1386 /**
1387 * xge_hal_lro_next_session: Returns next LRO session in the list or NULL
1388 * if none exists.
1389 * @hldev: HAL Context.
1390 * @ring: rx ring number.
1391 */
1392 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t *
xge_hal_lro_next_session(xge_hal_device_t * hldev,int ring)1393 xge_hal_lro_next_session (xge_hal_device_t *hldev, int ring)
1394 {
1395 xge_hal_lro_desc_t *ring_lro = &hldev->lro_desc[ring];
1396 int i;
1397 int start_idx = ring_lro->lro_next_idx;
1398
1399 for(i = start_idx; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
1400 lro_t *lro = &ring_lro->lro_pool[i];
1401
1402 if (!lro->in_use)
1403 continue;
1404
1405 lro->ip_hdr->tot_len = xge_os_htons(lro->total_length);
1406 lro->ip_hdr->check = xge_os_htons(0);
1407 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1408 (lro->ip_hdr->version_ihl & 0x0F));
1409 ring_lro->lro_next_idx = i + 1;
1410 return lro;
1411 }
1412
1413 ring_lro->lro_next_idx = 0;
1414 return NULL;
1415
1416 }
1417
1418 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t *
xge_hal_lro_get_next_session(xge_hal_device_t * hldev)1419 xge_hal_lro_get_next_session(xge_hal_device_t *hldev)
1420 {
1421 int ring = 0; /* assume default ring=0 */
1422 return xge_hal_lro_next_session(hldev, ring);
1423 }
1424 #endif
1425