1 /*-
2 * Copyright (c) 2002-2007 Neterion, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/9/sys/dev/nxge/xgehal/xgehal-ring-fp.c 218909 2011-02-21 09:01:34Z brucec $
27 */
28
29 #ifdef XGE_DEBUG_FP
30 #include <dev/nxge/include/xgehal-ring.h>
31 #endif
32
33 __HAL_STATIC_RING __HAL_INLINE_RING xge_hal_ring_rxd_priv_t*
__hal_ring_rxd_priv(xge_hal_ring_t * ring,xge_hal_dtr_h dtrh)34 __hal_ring_rxd_priv(xge_hal_ring_t *ring, xge_hal_dtr_h dtrh)
35 {
36
37 xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
38 xge_hal_ring_rxd_priv_t *rxd_priv;
39
40 xge_assert(rxdp);
41
42 #if defined(XGE_HAL_USE_5B_MODE)
43 xge_assert(ring);
44 if (ring->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
45 xge_hal_ring_rxd_5_t *rxdp_5 = (xge_hal_ring_rxd_5_t *)dtrh;
46 #if defined (XGE_OS_PLATFORM_64BIT)
47 int memblock_idx = rxdp_5->host_control >> 16;
48 int i = rxdp_5->host_control & 0xFFFF;
49 rxd_priv = (xge_hal_ring_rxd_priv_t *)
50 ((char*)ring->mempool->memblocks_priv_arr[memblock_idx] + ring->rxd_priv_size * i);
51 #else
52 /* 32-bit case */
53 rxd_priv = (xge_hal_ring_rxd_priv_t *)rxdp_5->host_control;
54 #endif
55 } else
56 #endif
57 {
58 rxd_priv = (xge_hal_ring_rxd_priv_t *)
59 (ulong_t)rxdp->host_control;
60 }
61
62 xge_assert(rxd_priv);
63 xge_assert(rxd_priv->dma_object);
64
65 xge_assert(rxd_priv->dma_object->handle == rxd_priv->dma_handle);
66
67 xge_assert(rxd_priv->dma_object->addr + rxd_priv->dma_offset ==
68 rxd_priv->dma_addr);
69
70 return rxd_priv;
71 }
72
73 __HAL_STATIC_RING __HAL_INLINE_RING int
__hal_ring_block_memblock_idx(xge_hal_ring_block_t * block)74 __hal_ring_block_memblock_idx(xge_hal_ring_block_t *block)
75 {
76 return (int)*((u64 *)(void *)((char *)block +
77 XGE_HAL_RING_MEMBLOCK_IDX_OFFSET));
78 }
79
80 __HAL_STATIC_RING __HAL_INLINE_RING void
__hal_ring_block_memblock_idx_set(xge_hal_ring_block_t * block,int memblock_idx)81 __hal_ring_block_memblock_idx_set(xge_hal_ring_block_t*block, int memblock_idx)
82 {
83 *((u64 *)(void *)((char *)block +
84 XGE_HAL_RING_MEMBLOCK_IDX_OFFSET)) =
85 memblock_idx;
86 }
87
88
89 __HAL_STATIC_RING __HAL_INLINE_RING dma_addr_t
__hal_ring_block_next_pointer(xge_hal_ring_block_t * block)90 __hal_ring_block_next_pointer(xge_hal_ring_block_t *block)
91 {
92 return (dma_addr_t)*((u64 *)(void *)((char *)block +
93 XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET));
94 }
95
96 __HAL_STATIC_RING __HAL_INLINE_RING void
__hal_ring_block_next_pointer_set(xge_hal_ring_block_t * block,dma_addr_t dma_next)97 __hal_ring_block_next_pointer_set(xge_hal_ring_block_t *block,
98 dma_addr_t dma_next)
99 {
100 *((u64 *)(void *)((char *)block +
101 XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET)) = dma_next;
102 }
103
104 /**
105 * xge_hal_ring_dtr_private - Get ULD private per-descriptor data.
106 * @channelh: Channel handle.
107 * @dtrh: Descriptor handle.
108 *
109 * Returns: private ULD info associated with the descriptor.
110 * ULD requests per-descriptor space via xge_hal_channel_open().
111 *
112 * See also: xge_hal_fifo_dtr_private().
113 * Usage: See ex_rx_compl{}.
114 */
115 __HAL_STATIC_RING __HAL_INLINE_RING void*
xge_hal_ring_dtr_private(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh)116 xge_hal_ring_dtr_private(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
117 {
118 return (char *)__hal_ring_rxd_priv((xge_hal_ring_t *) channelh, dtrh) +
119 sizeof(xge_hal_ring_rxd_priv_t);
120 }
121
122 /**
123 * xge_hal_ring_dtr_reserve - Reserve ring descriptor.
124 * @channelh: Channel handle.
125 * @dtrh: Reserved descriptor. On success HAL fills this "out" parameter
126 * with a valid handle.
127 *
128 * Reserve Rx descriptor for the subsequent filling-in (by upper layer
129 * driver (ULD)) and posting on the corresponding channel (@channelh)
130 * via xge_hal_ring_dtr_post().
131 *
132 * Returns: XGE_HAL_OK - success.
133 * XGE_HAL_INF_OUT_OF_DESCRIPTORS - Currently no descriptors available.
134 *
135 * See also: xge_hal_fifo_dtr_reserve(), xge_hal_ring_dtr_free(),
136 * xge_hal_fifo_dtr_reserve_sp(), xge_hal_status_e{}.
137 * Usage: See ex_post_all_rx{}.
138 */
139 __HAL_STATIC_RING __HAL_INLINE_RING xge_hal_status_e
xge_hal_ring_dtr_reserve(xge_hal_channel_h channelh,xge_hal_dtr_h * dtrh)140 xge_hal_ring_dtr_reserve(xge_hal_channel_h channelh, xge_hal_dtr_h *dtrh)
141 {
142 xge_hal_status_e status;
143 #if defined(XGE_HAL_RX_MULTI_RESERVE_IRQ)
144 unsigned long flags;
145 #endif
146
147 #if defined(XGE_HAL_RX_MULTI_RESERVE)
148 xge_os_spin_lock(&((xge_hal_channel_t*)channelh)->reserve_lock);
149 #elif defined(XGE_HAL_RX_MULTI_RESERVE_IRQ)
150 xge_os_spin_lock_irq(&((xge_hal_channel_t*)channelh)->reserve_lock,
151 flags);
152 #endif
153
154 status = __hal_channel_dtr_alloc(channelh, dtrh);
155
156 #if defined(XGE_HAL_RX_MULTI_RESERVE)
157 xge_os_spin_unlock(&((xge_hal_channel_t*)channelh)->reserve_lock);
158 #elif defined(XGE_HAL_RX_MULTI_RESERVE_IRQ)
159 xge_os_spin_unlock_irq(&((xge_hal_channel_t*)channelh)->reserve_lock,
160 flags);
161 #endif
162
163 if (status == XGE_HAL_OK) {
164 xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)*dtrh;
165
166 /* instead of memset: reset this RxD */
167 rxdp->control_1 = rxdp->control_2 = 0;
168
169 #if defined(XGE_OS_MEMORY_CHECK)
170 __hal_ring_rxd_priv((xge_hal_ring_t *) channelh, rxdp)->allocated = 1;
171 #endif
172 }
173
174 return status;
175 }
176
177 /**
178 * xge_hal_ring_dtr_info_get - Get extended information associated with
179 * a completed receive descriptor for 1b mode.
180 * @channelh: Channel handle.
181 * @dtrh: Descriptor handle.
182 * @ext_info: See xge_hal_dtr_info_t{}. Returned by HAL.
183 *
184 * Retrieve extended information associated with a completed receive descriptor.
185 *
186 * See also: xge_hal_dtr_info_t{}, xge_hal_ring_dtr_1b_get(),
187 * xge_hal_ring_dtr_5b_get().
188 */
189 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_info_get(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh,xge_hal_dtr_info_t * ext_info)190 xge_hal_ring_dtr_info_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
191 xge_hal_dtr_info_t *ext_info)
192 {
193 /* cast to 1-buffer mode RxD: the code below relies on the fact
194 * that control_1 and control_2 are formatted the same way.. */
195 xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
196
197 ext_info->l3_cksum = XGE_HAL_RXD_GET_L3_CKSUM(rxdp->control_1);
198 ext_info->l4_cksum = XGE_HAL_RXD_GET_L4_CKSUM(rxdp->control_1);
199 ext_info->frame = XGE_HAL_RXD_GET_FRAME_TYPE(rxdp->control_1);
200 ext_info->proto = XGE_HAL_RXD_GET_FRAME_PROTO(rxdp->control_1);
201 ext_info->vlan = XGE_HAL_RXD_GET_VLAN_TAG(rxdp->control_2);
202
203 /* Herc only, a few extra cycles imposed on Xena and/or
204 * when RTH is not enabled.
205 * Alternatively, could check
206 * xge_hal_device_check_id(), hldev->config.rth_en, queue->rth_en */
207 ext_info->rth_it_hit = XGE_HAL_RXD_GET_RTH_IT_HIT(rxdp->control_1);
208 ext_info->rth_spdm_hit =
209 XGE_HAL_RXD_GET_RTH_SPDM_HIT(rxdp->control_1);
210 ext_info->rth_hash_type =
211 XGE_HAL_RXD_GET_RTH_HASH_TYPE(rxdp->control_1);
212 ext_info->rth_value = XGE_HAL_RXD_1_GET_RTH_VALUE(rxdp->control_2);
213 }
214
215 /**
216 * xge_hal_ring_dtr_info_nb_get - Get extended information associated
217 * with a completed receive descriptor for 3b or 5b
218 * modes.
219 * @channelh: Channel handle.
220 * @dtrh: Descriptor handle.
221 * @ext_info: See xge_hal_dtr_info_t{}. Returned by HAL.
222 *
223 * Retrieve extended information associated with a completed receive descriptor.
224 *
225 * See also: xge_hal_dtr_info_t{}, xge_hal_ring_dtr_1b_get(),
226 * xge_hal_ring_dtr_5b_get().
227 */
228 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_info_nb_get(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh,xge_hal_dtr_info_t * ext_info)229 xge_hal_ring_dtr_info_nb_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
230 xge_hal_dtr_info_t *ext_info)
231 {
232 /* cast to 1-buffer mode RxD: the code below relies on the fact
233 * that control_1 and control_2 are formatted the same way.. */
234 xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
235
236 ext_info->l3_cksum = XGE_HAL_RXD_GET_L3_CKSUM(rxdp->control_1);
237 ext_info->l4_cksum = XGE_HAL_RXD_GET_L4_CKSUM(rxdp->control_1);
238 ext_info->frame = XGE_HAL_RXD_GET_FRAME_TYPE(rxdp->control_1);
239 ext_info->proto = XGE_HAL_RXD_GET_FRAME_PROTO(rxdp->control_1);
240 ext_info->vlan = XGE_HAL_RXD_GET_VLAN_TAG(rxdp->control_2);
241 /* Herc only, a few extra cycles imposed on Xena and/or
242 * when RTH is not enabled. Same comment as above. */
243 ext_info->rth_it_hit = XGE_HAL_RXD_GET_RTH_IT_HIT(rxdp->control_1);
244 ext_info->rth_spdm_hit =
245 XGE_HAL_RXD_GET_RTH_SPDM_HIT(rxdp->control_1);
246 ext_info->rth_hash_type =
247 XGE_HAL_RXD_GET_RTH_HASH_TYPE(rxdp->control_1);
248 ext_info->rth_value = (u32)rxdp->buffer0_ptr;
249 }
250
251 /**
252 * xge_hal_ring_dtr_1b_set - Prepare 1-buffer-mode descriptor.
253 * @dtrh: Descriptor handle.
254 * @dma_pointer: DMA address of a single receive buffer this descriptor
255 * should carry. Note that by the time
256 * xge_hal_ring_dtr_1b_set
257 * is called, the receive buffer should be already mapped
258 * to the corresponding Xframe device.
259 * @size: Size of the receive @dma_pointer buffer.
260 *
261 * Prepare 1-buffer-mode Rx descriptor for posting
262 * (via xge_hal_ring_dtr_post()).
263 *
264 * This inline helper-function does not return any parameters and always
265 * succeeds.
266 *
267 * See also: xge_hal_ring_dtr_3b_set(), xge_hal_ring_dtr_5b_set().
268 * Usage: See ex_post_all_rx{}.
269 */
270 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_1b_set(xge_hal_dtr_h dtrh,dma_addr_t dma_pointer,int size)271 xge_hal_ring_dtr_1b_set(xge_hal_dtr_h dtrh, dma_addr_t dma_pointer, int size)
272 {
273 xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
274 rxdp->buffer0_ptr = dma_pointer;
275 rxdp->control_2 &= (~XGE_HAL_RXD_1_MASK_BUFFER0_SIZE);
276 rxdp->control_2 |= XGE_HAL_RXD_1_SET_BUFFER0_SIZE(size);
277
278 xge_debug_ring(XGE_TRACE, "xge_hal_ring_dtr_1b_set: rxdp %p control_2 %p buffer0_ptr %p",
279 (xge_hal_ring_rxd_1_t *)dtrh,
280 rxdp->control_2,
281 rxdp->buffer0_ptr);
282 }
283
284 /**
285 * xge_hal_ring_dtr_1b_get - Get data from the completed 1-buf
286 * descriptor.
287 * @channelh: Channel handle.
288 * @dtrh: Descriptor handle.
289 * @dma_pointer: DMA address of a single receive buffer _this_ descriptor
290 * carries. Returned by HAL.
291 * @pkt_length: Length (in bytes) of the data in the buffer pointed by
292 * @dma_pointer. Returned by HAL.
293 *
294 * Retrieve protocol data from the completed 1-buffer-mode Rx descriptor.
295 * This inline helper-function uses completed descriptor to populate receive
296 * buffer pointer and other "out" parameters. The function always succeeds.
297 *
298 * See also: xge_hal_ring_dtr_3b_get(), xge_hal_ring_dtr_5b_get().
299 * Usage: See ex_rx_compl{}.
300 */
301 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_1b_get(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh,dma_addr_t * dma_pointer,int * pkt_length)302 xge_hal_ring_dtr_1b_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
303 dma_addr_t *dma_pointer, int *pkt_length)
304 {
305 xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
306
307 *pkt_length = XGE_HAL_RXD_1_GET_BUFFER0_SIZE(rxdp->control_2);
308 *dma_pointer = rxdp->buffer0_ptr;
309
310 ((xge_hal_channel_t *)channelh)->poll_bytes += *pkt_length;
311 }
312
313 /**
314 * xge_hal_ring_dtr_3b_set - Prepare 3-buffer-mode descriptor.
315 * @dtrh: Descriptor handle.
316 * @dma_pointers: Array of DMA addresses. Contains exactly 3 receive buffers
317 * _this_ descriptor should carry.
318 * Note that by the time xge_hal_ring_dtr_3b_set
319 * is called, the receive buffers should be mapped
320 * to the corresponding Xframe device.
321 * @sizes: Array of receive buffer sizes. Contains 3 sizes: one size per
322 * buffer from @dma_pointers.
323 *
324 * Prepare 3-buffer-mode Rx descriptor for posting (via
325 * xge_hal_ring_dtr_post()).
326 * This inline helper-function does not return any parameters and always
327 * succeeds.
328 *
329 * See also: xge_hal_ring_dtr_1b_set(), xge_hal_ring_dtr_5b_set().
330 */
331 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_3b_set(xge_hal_dtr_h dtrh,dma_addr_t dma_pointers[],int sizes[])332 xge_hal_ring_dtr_3b_set(xge_hal_dtr_h dtrh, dma_addr_t dma_pointers[],
333 int sizes[])
334 {
335 xge_hal_ring_rxd_3_t *rxdp = (xge_hal_ring_rxd_3_t *)dtrh;
336 rxdp->buffer0_ptr = dma_pointers[0];
337 rxdp->control_2 &= (~XGE_HAL_RXD_3_MASK_BUFFER0_SIZE);
338 rxdp->control_2 |= XGE_HAL_RXD_3_SET_BUFFER0_SIZE(sizes[0]);
339 rxdp->buffer1_ptr = dma_pointers[1];
340 rxdp->control_2 &= (~XGE_HAL_RXD_3_MASK_BUFFER1_SIZE);
341 rxdp->control_2 |= XGE_HAL_RXD_3_SET_BUFFER1_SIZE(sizes[1]);
342 rxdp->buffer2_ptr = dma_pointers[2];
343 rxdp->control_2 &= (~XGE_HAL_RXD_3_MASK_BUFFER2_SIZE);
344 rxdp->control_2 |= XGE_HAL_RXD_3_SET_BUFFER2_SIZE(sizes[2]);
345 }
346
347 /**
348 * xge_hal_ring_dtr_3b_get - Get data from the completed 3-buf
349 * descriptor.
350 * @channelh: Channel handle.
351 * @dtrh: Descriptor handle.
352 * @dma_pointers: DMA addresses of the 3 receive buffers _this_ descriptor
353 * carries. The first two buffers contain ethernet and
354 * (IP + transport) headers. The 3rd buffer contains packet
355 * data.
356 * Returned by HAL.
357 * @sizes: Array of receive buffer sizes. Contains 3 sizes: one size per
358 * buffer from @dma_pointers. Returned by HAL.
359 *
360 * Retrieve protocol data from the completed 3-buffer-mode Rx descriptor.
361 * This inline helper-function uses completed descriptor to populate receive
362 * buffer pointer and other "out" parameters. The function always succeeds.
363 *
364 * See also: xge_hal_ring_dtr_3b_get(), xge_hal_ring_dtr_5b_get().
365 */
366 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_3b_get(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh,dma_addr_t dma_pointers[],int sizes[])367 xge_hal_ring_dtr_3b_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
368 dma_addr_t dma_pointers[], int sizes[])
369 {
370 xge_hal_ring_rxd_3_t *rxdp = (xge_hal_ring_rxd_3_t *)dtrh;
371
372 dma_pointers[0] = rxdp->buffer0_ptr;
373 sizes[0] = XGE_HAL_RXD_3_GET_BUFFER0_SIZE(rxdp->control_2);
374
375 dma_pointers[1] = rxdp->buffer1_ptr;
376 sizes[1] = XGE_HAL_RXD_3_GET_BUFFER1_SIZE(rxdp->control_2);
377
378 dma_pointers[2] = rxdp->buffer2_ptr;
379 sizes[2] = XGE_HAL_RXD_3_GET_BUFFER2_SIZE(rxdp->control_2);
380
381 ((xge_hal_channel_t *)channelh)->poll_bytes += sizes[0] + sizes[1] +
382 sizes[2];
383 }
384
385 /**
386 * xge_hal_ring_dtr_5b_set - Prepare 5-buffer-mode descriptor.
387 * @dtrh: Descriptor handle.
388 * @dma_pointers: Array of DMA addresses. Contains exactly 5 receive buffers
389 * _this_ descriptor should carry.
390 * Note that by the time xge_hal_ring_dtr_5b_set
391 * is called, the receive buffers should be mapped
392 * to the corresponding Xframe device.
393 * @sizes: Array of receive buffer sizes. Contains 5 sizes: one size per
394 * buffer from @dma_pointers.
395 *
396 * Prepare 3-buffer-mode Rx descriptor for posting (via
397 * xge_hal_ring_dtr_post()).
398 * This inline helper-function does not return any parameters and always
399 * succeeds.
400 *
401 * See also: xge_hal_ring_dtr_1b_set(), xge_hal_ring_dtr_3b_set().
402 */
403 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_5b_set(xge_hal_dtr_h dtrh,dma_addr_t dma_pointers[],int sizes[])404 xge_hal_ring_dtr_5b_set(xge_hal_dtr_h dtrh, dma_addr_t dma_pointers[],
405 int sizes[])
406 {
407 xge_hal_ring_rxd_5_t *rxdp = (xge_hal_ring_rxd_5_t *)dtrh;
408 rxdp->buffer0_ptr = dma_pointers[0];
409 rxdp->control_2 &= (~XGE_HAL_RXD_5_MASK_BUFFER0_SIZE);
410 rxdp->control_2 |= XGE_HAL_RXD_5_SET_BUFFER0_SIZE(sizes[0]);
411 rxdp->buffer1_ptr = dma_pointers[1];
412 rxdp->control_2 &= (~XGE_HAL_RXD_5_MASK_BUFFER1_SIZE);
413 rxdp->control_2 |= XGE_HAL_RXD_5_SET_BUFFER1_SIZE(sizes[1]);
414 rxdp->buffer2_ptr = dma_pointers[2];
415 rxdp->control_2 &= (~XGE_HAL_RXD_5_MASK_BUFFER2_SIZE);
416 rxdp->control_2 |= XGE_HAL_RXD_5_SET_BUFFER2_SIZE(sizes[2]);
417 rxdp->buffer3_ptr = dma_pointers[3];
418 rxdp->control_3 &= (~XGE_HAL_RXD_5_MASK_BUFFER3_SIZE);
419 rxdp->control_3 |= XGE_HAL_RXD_5_SET_BUFFER3_SIZE(sizes[3]);
420 rxdp->buffer4_ptr = dma_pointers[4];
421 rxdp->control_3 &= (~XGE_HAL_RXD_5_MASK_BUFFER4_SIZE);
422 rxdp->control_3 |= XGE_HAL_RXD_5_SET_BUFFER4_SIZE(sizes[4]);
423 }
424
425 /**
426 * xge_hal_ring_dtr_5b_get - Get data from the completed 5-buf
427 * descriptor.
428 * @channelh: Channel handle.
429 * @dtrh: Descriptor handle.
430 * @dma_pointers: DMA addresses of the 5 receive buffers _this_ descriptor
431 * carries. The first 4 buffers contains L2 (ethernet) through
432 * L5 headers. The 5th buffer contain received (applicaion)
433 * data. Returned by HAL.
434 * @sizes: Array of receive buffer sizes. Contains 5 sizes: one size per
435 * buffer from @dma_pointers. Returned by HAL.
436 *
437 * Retrieve protocol data from the completed 5-buffer-mode Rx descriptor.
438 * This inline helper-function uses completed descriptor to populate receive
439 * buffer pointer and other "out" parameters. The function always succeeds.
440 *
441 * See also: xge_hal_ring_dtr_3b_get(), xge_hal_ring_dtr_5b_get().
442 */
443 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_5b_get(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh,dma_addr_t dma_pointers[],int sizes[])444 xge_hal_ring_dtr_5b_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
445 dma_addr_t dma_pointers[], int sizes[])
446 {
447 xge_hal_ring_rxd_5_t *rxdp = (xge_hal_ring_rxd_5_t *)dtrh;
448
449 dma_pointers[0] = rxdp->buffer0_ptr;
450 sizes[0] = XGE_HAL_RXD_5_GET_BUFFER0_SIZE(rxdp->control_2);
451
452 dma_pointers[1] = rxdp->buffer1_ptr;
453 sizes[1] = XGE_HAL_RXD_5_GET_BUFFER1_SIZE(rxdp->control_2);
454
455 dma_pointers[2] = rxdp->buffer2_ptr;
456 sizes[2] = XGE_HAL_RXD_5_GET_BUFFER2_SIZE(rxdp->control_2);
457
458 dma_pointers[3] = rxdp->buffer3_ptr;
459 sizes[3] = XGE_HAL_RXD_5_GET_BUFFER3_SIZE(rxdp->control_3);
460
461 dma_pointers[4] = rxdp->buffer4_ptr;
462 sizes[4] = XGE_HAL_RXD_5_GET_BUFFER4_SIZE(rxdp->control_3);
463
464 ((xge_hal_channel_t *)channelh)->poll_bytes += sizes[0] + sizes[1] +
465 sizes[2] + sizes[3] + sizes[4];
466 }
467
468
469 /**
470 * xge_hal_ring_dtr_pre_post - FIXME.
471 * @channelh: Channel handle.
472 * @dtrh: Descriptor handle.
473 *
474 * TBD
475 */
476 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_pre_post(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh)477 xge_hal_ring_dtr_pre_post(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
478 {
479 xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
480 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
481 xge_hal_ring_rxd_priv_t *priv;
482 xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
483 #endif
484 #if defined(XGE_HAL_RX_MULTI_POST_IRQ)
485 unsigned long flags;
486 #endif
487
488 rxdp->control_2 |= XGE_HAL_RXD_NOT_COMPLETED;
489
490 #ifdef XGE_DEBUG_ASSERT
491 /* make sure Xena overwrites the (illegal) t_code on completion */
492 XGE_HAL_RXD_SET_T_CODE(rxdp->control_1, XGE_HAL_RXD_T_CODE_UNUSED_C);
493 #endif
494
495 xge_debug_ring(XGE_TRACE, "xge_hal_ring_dtr_pre_post: rxd 0x"XGE_OS_LLXFMT" posted %d post_qid %d",
496 (unsigned long long)(ulong_t)dtrh,
497 ((xge_hal_ring_t *)channelh)->channel.post_index,
498 ((xge_hal_ring_t *)channelh)->channel.post_qid);
499
500 #if defined(XGE_HAL_RX_MULTI_POST)
501 xge_os_spin_lock(&((xge_hal_channel_t*)channelh)->post_lock);
502 #elif defined(XGE_HAL_RX_MULTI_POST_IRQ)
503 xge_os_spin_lock_irq(&((xge_hal_channel_t*)channelh)->post_lock,
504 flags);
505 #endif
506
507 #if defined(XGE_DEBUG_ASSERT) && defined(XGE_HAL_RING_ENFORCE_ORDER)
508 {
509 xge_hal_channel_t *channel = (xge_hal_channel_t *)channelh;
510
511 if (channel->post_index != 0) {
512 xge_hal_dtr_h prev_dtrh;
513 xge_hal_ring_rxd_priv_t *rxdp_priv;
514
515 rxdp_priv = __hal_ring_rxd_priv((xge_hal_ring_t*)channel, rxdp);
516 prev_dtrh = channel->work_arr[channel->post_index - 1];
517
518 if (prev_dtrh != NULL &&
519 (rxdp_priv->dma_offset & (~0xFFF)) !=
520 rxdp_priv->dma_offset) {
521 xge_assert((char *)prev_dtrh +
522 ((xge_hal_ring_t*)channel)->rxd_size == dtrh);
523 }
524 }
525 }
526 #endif
527
528 __hal_channel_dtr_post(channelh, dtrh);
529
530 #if defined(XGE_HAL_RX_MULTI_POST)
531 xge_os_spin_unlock(&((xge_hal_channel_t*)channelh)->post_lock);
532 #elif defined(XGE_HAL_RX_MULTI_POST_IRQ)
533 xge_os_spin_unlock_irq(&((xge_hal_channel_t*)channelh)->post_lock,
534 flags);
535 #endif
536 }
537
538
539 /**
540 * xge_hal_ring_dtr_post_post - FIXME.
541 * @channelh: Channel handle.
542 * @dtrh: Descriptor handle.
543 *
544 * TBD
545 */
546 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_post_post(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh)547 xge_hal_ring_dtr_post_post(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
548 {
549 xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
550 xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
551 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
552 xge_hal_ring_rxd_priv_t *priv;
553 #endif
554 /* do POST */
555 rxdp->control_1 |= XGE_HAL_RXD_POSTED_4_XFRAME;
556
557 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
558 priv = __hal_ring_rxd_priv(ring, rxdp);
559 xge_os_dma_sync(ring->channel.pdev,
560 priv->dma_handle, priv->dma_addr,
561 priv->dma_offset, ring->rxd_size,
562 XGE_OS_DMA_DIR_TODEVICE);
563 #endif
564
565 xge_debug_ring(XGE_TRACE, "xge_hal_ring_dtr_post_post: rxdp %p control_1 %p",
566 (xge_hal_ring_rxd_1_t *)dtrh,
567 rxdp->control_1);
568
569 if (ring->channel.usage_cnt > 0)
570 ring->channel.usage_cnt--;
571 }
572
573 /**
574 * xge_hal_ring_dtr_post_post_wmb.
575 * @channelh: Channel handle.
576 * @dtrh: Descriptor handle.
577 *
578 * Similar as xge_hal_ring_dtr_post_post, but in addition it does memory barrier.
579 */
580 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_post_post_wmb(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh)581 xge_hal_ring_dtr_post_post_wmb(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
582 {
583 xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
584 xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
585 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
586 xge_hal_ring_rxd_priv_t *priv;
587 #endif
588 /* Do memory barrier before changing the ownership */
589 xge_os_wmb();
590
591 /* do POST */
592 rxdp->control_1 |= XGE_HAL_RXD_POSTED_4_XFRAME;
593
594 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
595 priv = __hal_ring_rxd_priv(ring, rxdp);
596 xge_os_dma_sync(ring->channel.pdev,
597 priv->dma_handle, priv->dma_addr,
598 priv->dma_offset, ring->rxd_size,
599 XGE_OS_DMA_DIR_TODEVICE);
600 #endif
601
602 if (ring->channel.usage_cnt > 0)
603 ring->channel.usage_cnt--;
604
605 xge_debug_ring(XGE_TRACE, "xge_hal_ring_dtr_post_post_wmb: rxdp %p control_1 %p rxds_with_host %d",
606 (xge_hal_ring_rxd_1_t *)dtrh,
607 rxdp->control_1, ring->channel.usage_cnt);
608
609 }
610
611 /**
612 * xge_hal_ring_dtr_post - Post descriptor on the ring channel.
613 * @channelh: Channel handle.
614 * @dtrh: Descriptor obtained via xge_hal_ring_dtr_reserve().
615 *
616 * Post descriptor on the 'ring' type channel.
617 * Prior to posting the descriptor should be filled in accordance with
618 * Host/Xframe interface specification for a given service (LL, etc.).
619 *
620 * See also: xge_hal_fifo_dtr_post_many(), xge_hal_fifo_dtr_post().
621 * Usage: See ex_post_all_rx{}.
622 */
623 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_post(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh)624 xge_hal_ring_dtr_post(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
625 {
626 xge_hal_ring_dtr_pre_post(channelh, dtrh);
627 xge_hal_ring_dtr_post_post(channelh, dtrh);
628 }
629
630 /**
631 * xge_hal_ring_dtr_next_completed - Get the _next_ completed
632 * descriptor.
633 * @channelh: Channel handle.
634 * @dtrh: Descriptor handle. Returned by HAL.
635 * @t_code: Transfer code, as per Xframe User Guide,
636 * Receive Descriptor Format. Returned by HAL.
637 *
638 * Retrieve the _next_ completed descriptor.
639 * HAL uses channel callback (*xge_hal_channel_callback_f) to notifiy
640 * upper-layer driver (ULD) of new completed descriptors. After that
641 * the ULD can use xge_hal_ring_dtr_next_completed to retrieve the rest
642 * completions (the very first completion is passed by HAL via
643 * xge_hal_channel_callback_f).
644 *
645 * Implementation-wise, the upper-layer driver is free to call
646 * xge_hal_ring_dtr_next_completed either immediately from inside the
647 * channel callback, or in a deferred fashion and separate (from HAL)
648 * context.
649 *
650 * Non-zero @t_code means failure to fill-in receive buffer(s)
651 * of the descriptor.
652 * For instance, parity error detected during the data transfer.
653 * In this case Xframe will complete the descriptor and indicate
654 * for the host that the received data is not to be used.
655 * For details please refer to Xframe User Guide.
656 *
657 * Returns: XGE_HAL_OK - success.
658 * XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors
659 * are currently available for processing.
660 *
661 * See also: xge_hal_channel_callback_f{},
662 * xge_hal_fifo_dtr_next_completed(), xge_hal_status_e{}.
663 * Usage: See ex_rx_compl{}.
664 */
665 __HAL_STATIC_RING __HAL_INLINE_RING xge_hal_status_e
xge_hal_ring_dtr_next_completed(xge_hal_channel_h channelh,xge_hal_dtr_h * dtrh,u8 * t_code)666 xge_hal_ring_dtr_next_completed(xge_hal_channel_h channelh, xge_hal_dtr_h *dtrh,
667 u8 *t_code)
668 {
669 xge_hal_ring_rxd_1_t *rxdp; /* doesn't matter 1, 3 or 5... */
670 xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
671 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
672 xge_hal_ring_rxd_priv_t *priv;
673 #endif
674
675 __hal_channel_dtr_try_complete(ring, dtrh);
676 rxdp = (xge_hal_ring_rxd_1_t *)*dtrh;
677 if (rxdp == NULL) {
678 return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
679 }
680
681 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
682 /* Note: 24 bytes at most means:
683 * - Control_3 in case of 5-buffer mode
684 * - Control_1 and Control_2
685 *
686 * This is the only length needs to be invalidated
687 * type of channels.*/
688 priv = __hal_ring_rxd_priv(ring, rxdp);
689 xge_os_dma_sync(ring->channel.pdev,
690 priv->dma_handle, priv->dma_addr,
691 priv->dma_offset, 24,
692 XGE_OS_DMA_DIR_FROMDEVICE);
693 #endif
694
695 /* check whether it is not the end */
696 if (!(rxdp->control_2 & XGE_HAL_RXD_NOT_COMPLETED) &&
697 !(rxdp->control_1 & XGE_HAL_RXD_POSTED_4_XFRAME)) {
698 #ifndef XGE_HAL_IRQ_POLLING
699 if (++ring->cmpl_cnt > ring->indicate_max_pkts) {
700 /* reset it. since we don't want to return
701 * garbage to the ULD */
702 *dtrh = 0;
703 return XGE_HAL_COMPLETIONS_REMAIN;
704 }
705 #endif
706
707 #ifdef XGE_DEBUG_ASSERT
708 #if defined(XGE_HAL_USE_5B_MODE)
709 #if !defined(XGE_OS_PLATFORM_64BIT)
710 if (ring->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
711 xge_assert(((xge_hal_ring_rxd_5_t *)
712 rxdp)->host_control!=0);
713 }
714 #endif
715
716 #else
717 xge_assert(rxdp->host_control!=0);
718 #endif
719 #endif
720
721 __hal_channel_dtr_complete(ring);
722
723 *t_code = (u8)XGE_HAL_RXD_GET_T_CODE(rxdp->control_1);
724
725 /* see XGE_HAL_SET_RXD_T_CODE() above.. */
726 xge_assert(*t_code != XGE_HAL_RXD_T_CODE_UNUSED_C);
727
728 xge_debug_ring(XGE_TRACE,
729 "compl_index %d post_qid %d t_code %d rxd 0x"XGE_OS_LLXFMT,
730 ((xge_hal_channel_t*)ring)->compl_index,
731 ((xge_hal_channel_t*)ring)->post_qid, *t_code,
732 (unsigned long long)(ulong_t)rxdp);
733
734 ring->channel.usage_cnt++;
735 if (ring->channel.stats.usage_max < ring->channel.usage_cnt)
736 ring->channel.stats.usage_max = ring->channel.usage_cnt;
737
738 return XGE_HAL_OK;
739 }
740
741 /* reset it. since we don't want to return
742 * garbage to the ULD */
743 *dtrh = 0;
744 return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
745 }
746
747 /**
748 * xge_hal_ring_dtr_free - Free descriptor.
749 * @channelh: Channel handle.
750 * @dtrh: Descriptor handle.
751 *
752 * Free the reserved descriptor. This operation is "symmetrical" to
753 * xge_hal_ring_dtr_reserve. The "free-ing" completes the descriptor's
754 * lifecycle.
755 *
756 * After free-ing (see xge_hal_ring_dtr_free()) the descriptor again can
757 * be:
758 *
759 * - reserved (xge_hal_ring_dtr_reserve);
760 *
761 * - posted (xge_hal_ring_dtr_post);
762 *
763 * - completed (xge_hal_ring_dtr_next_completed);
764 *
765 * - and recycled again (xge_hal_ring_dtr_free).
766 *
767 * For alternative state transitions and more details please refer to
768 * the design doc.
769 *
770 * See also: xge_hal_ring_dtr_reserve(), xge_hal_fifo_dtr_free().
771 * Usage: See ex_rx_compl{}.
772 */
773 __HAL_STATIC_RING __HAL_INLINE_RING void
xge_hal_ring_dtr_free(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh)774 xge_hal_ring_dtr_free(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
775 {
776 #if defined(XGE_HAL_RX_MULTI_FREE_IRQ)
777 unsigned long flags;
778 #endif
779
780 #if defined(XGE_HAL_RX_MULTI_FREE)
781 xge_os_spin_lock(&((xge_hal_channel_t*)channelh)->free_lock);
782 #elif defined(XGE_HAL_RX_MULTI_FREE_IRQ)
783 xge_os_spin_lock_irq(&((xge_hal_channel_t*)channelh)->free_lock,
784 flags);
785 #endif
786
787 __hal_channel_dtr_free(channelh, dtrh);
788 #if defined(XGE_OS_MEMORY_CHECK)
789 __hal_ring_rxd_priv((xge_hal_ring_t * ) channelh, dtrh)->allocated = 0;
790 #endif
791
792 #if defined(XGE_HAL_RX_MULTI_FREE)
793 xge_os_spin_unlock(&((xge_hal_channel_t*)channelh)->free_lock);
794 #elif defined(XGE_HAL_RX_MULTI_FREE_IRQ)
795 xge_os_spin_unlock_irq(&((xge_hal_channel_t*)channelh)->free_lock,
796 flags);
797 #endif
798 }
799
800 /**
801 * xge_hal_ring_is_next_dtr_completed - Check if the next dtr is completed
802 * @channelh: Channel handle.
803 *
804 * Checks if the _next_ completed descriptor is in host memory
805 *
806 * Returns: XGE_HAL_OK - success.
807 * XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors
808 * are currently available for processing.
809 */
810 __HAL_STATIC_RING __HAL_INLINE_RING xge_hal_status_e
xge_hal_ring_is_next_dtr_completed(xge_hal_channel_h channelh)811 xge_hal_ring_is_next_dtr_completed(xge_hal_channel_h channelh)
812 {
813 xge_hal_ring_rxd_1_t *rxdp; /* doesn't matter 1, 3 or 5... */
814 xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
815 xge_hal_dtr_h dtrh;
816
817 __hal_channel_dtr_try_complete(ring, &dtrh);
818 rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
819 if (rxdp == NULL) {
820 return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
821 }
822
823 /* check whether it is not the end */
824 if (!(rxdp->control_2 & XGE_HAL_RXD_NOT_COMPLETED) &&
825 !(rxdp->control_1 & XGE_HAL_RXD_POSTED_4_XFRAME)) {
826
827 #ifdef XGE_DEBUG_ASSERT
828 #if defined(XGE_HAL_USE_5B_MODE)
829 #if !defined(XGE_OS_PLATFORM_64BIT)
830 if (ring->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
831 xge_assert(((xge_hal_ring_rxd_5_t *)
832 rxdp)->host_control!=0);
833 }
834 #endif
835
836 #else
837 xge_assert(rxdp->host_control!=0);
838 #endif
839 #endif
840 return XGE_HAL_OK;
841 }
842
843 return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
844 }
845