1 /*-
2 * Copyright(c) 2002-2011 Exar Corp.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification are permitted provided the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * 3. Neither the name of the Exar Corporation nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31 /*$FreeBSD: stable/9/sys/dev/vxge/vxgehal/vxgehal-mm.c 221167 2011-04-28 14:33:15Z gnn $*/
32
33 #include <dev/vxge/vxgehal/vxgehal.h>
34
35 /*
36 * __hal_mempool_grow
37 *
38 * Will resize mempool up to %num_allocate value.
39 */
40 static vxge_hal_status_e
__hal_mempool_grow(vxge_hal_mempool_t * mempool,u32 num_allocate,u32 * num_allocated)41 __hal_mempool_grow(
42 vxge_hal_mempool_t *mempool,
43 u32 num_allocate,
44 u32 *num_allocated)
45 {
46 u32 i, j, k, item_index, is_last;
47 u32 first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
48 u32 n_items = mempool->items_per_memblock;
49 u32 start_block_idx = mempool->memblocks_allocated;
50 u32 end_block_idx = mempool->memblocks_allocated + num_allocate;
51 __hal_device_t *hldev;
52
53 vxge_assert(mempool != NULL);
54
55 hldev = (__hal_device_t *) mempool->devh;
56
57 vxge_hal_trace_log_mm("==> %s:%s:%d",
58 __FILE__, __func__, __LINE__);
59
60 vxge_hal_trace_log_mm(
61 "mempool = 0x"VXGE_OS_STXFMT", num_allocate = %d, "
62 "num_allocated = 0x"VXGE_OS_STXFMT, (ptr_t) mempool,
63 num_allocate, (ptr_t) num_allocated);
64
65 *num_allocated = 0;
66
67 if (end_block_idx > mempool->memblocks_max) {
68 vxge_hal_err_log_mm("%s",
69 "__hal_mempool_grow: can grow anymore");
70 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
71 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
72 return (VXGE_HAL_ERR_OUT_OF_MEMORY);
73 }
74
75 for (i = start_block_idx; i < end_block_idx; i++) {
76
77 void *the_memblock;
78 vxge_hal_mempool_dma_t *dma_object;
79
80 is_last = ((end_block_idx - 1) == i);
81 dma_object = mempool->memblocks_dma_arr + i;
82
83 /*
84 * allocate memblock's private part. Each DMA memblock
85 * has a space allocated for item's private usage upon
86 * mempool's user request. Each time mempool grows, it will
87 * allocate new memblock and its private part at once.
88 * This helps to minimize memory usage a lot.
89 */
90 mempool->memblocks_priv_arr[i] = vxge_os_malloc(
91 ((__hal_device_t *) mempool->devh)->header.pdev,
92 mempool->items_priv_size * n_items);
93 if (mempool->memblocks_priv_arr[i] == NULL) {
94
95 vxge_hal_err_log_mm("memblock_priv[%d]: \
96 out of virtual memory, "
97 "requested %d(%d:%d) bytes", i,
98 mempool->items_priv_size * n_items,
99 mempool->items_priv_size, n_items);
100 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
101 __FILE__, __func__, __LINE__,
102 VXGE_HAL_ERR_OUT_OF_MEMORY);
103 return (VXGE_HAL_ERR_OUT_OF_MEMORY);
104
105 }
106
107 vxge_os_memzero(mempool->memblocks_priv_arr[i],
108 mempool->items_priv_size * n_items);
109
110 /* allocate DMA-capable memblock */
111 mempool->memblocks_arr[i] =
112 __hal_blockpool_malloc(mempool->devh,
113 mempool->memblock_size,
114 &dma_object->addr,
115 &dma_object->handle,
116 &dma_object->acc_handle);
117 if (mempool->memblocks_arr[i] == NULL) {
118 vxge_os_free(
119 ((__hal_device_t *) mempool->devh)->header.pdev,
120 mempool->memblocks_priv_arr[i],
121 mempool->items_priv_size * n_items);
122 vxge_hal_err_log_mm("memblock[%d]: \
123 out of DMA memory", i);
124 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
125 __FILE__, __func__, __LINE__,
126 VXGE_HAL_ERR_OUT_OF_MEMORY);
127 return (VXGE_HAL_ERR_OUT_OF_MEMORY);
128 }
129
130 (*num_allocated)++;
131 mempool->memblocks_allocated++;
132
133 vxge_os_memzero(mempool->memblocks_arr[i],
134 mempool->memblock_size);
135
136 the_memblock = mempool->memblocks_arr[i];
137
138 /* fill the items hash array */
139 for (j = 0; j < n_items; j++) {
140 item_index = i * n_items + j;
141
142 if (first_time && (item_index >= mempool->items_initial))
143 break;
144
145 mempool->items_arr[item_index] =
146 ((char *) the_memblock + j *mempool->item_size);
147
148 /* let caller to do more job on each item */
149 if (mempool->item_func_alloc != NULL) {
150 vxge_hal_status_e status;
151
152 if ((status = mempool->item_func_alloc(
153 mempool,
154 the_memblock,
155 i,
156 dma_object,
157 mempool->items_arr[item_index],
158 item_index,
159 is_last,
160 mempool->userdata)) != VXGE_HAL_OK) {
161
162 if (mempool->item_func_free != NULL) {
163
164 for (k = 0; k < j; k++) {
165
166 item_index = i * n_items + k;
167
168 (void) mempool->item_func_free(
169 mempool,
170 the_memblock,
171 i, dma_object,
172 mempool->items_arr[item_index],
173 item_index, is_last,
174 mempool->userdata);
175 }
176 }
177
178 vxge_os_free(((__hal_device_t *)
179 mempool->devh)->header.pdev,
180 mempool->memblocks_priv_arr[i],
181 mempool->items_priv_size *
182 n_items);
183
184 __hal_blockpool_free(mempool->devh,
185 the_memblock,
186 mempool->memblock_size,
187 &dma_object->addr,
188 &dma_object->handle,
189 &dma_object->acc_handle);
190
191 (*num_allocated)--;
192 mempool->memblocks_allocated--;
193 return (status);
194 }
195 }
196
197 mempool->items_current = item_index + 1;
198 }
199
200 vxge_hal_info_log_mm(
201 "memblock%d: allocated %dk, vaddr 0x"VXGE_OS_STXFMT", "
202 "dma_addr 0x"VXGE_OS_STXFMT,
203 i, mempool->memblock_size / 1024,
204 (ptr_t) mempool->memblocks_arr[i], dma_object->addr);
205
206 if (first_time && mempool->items_current ==
207 mempool->items_initial) {
208 break;
209 }
210 }
211
212 vxge_hal_trace_log_mm("<== %s:%s:%d Result: 0",
213 __FILE__, __func__, __LINE__);
214
215 return (VXGE_HAL_OK);
216 }
217
218 /*
219 * vxge_hal_mempool_create
220 * @memblock_size:
221 * @items_initial:
222 * @items_max:
223 * @item_size:
224 * @item_func:
225 *
226 * This function will create memory pool object. Pool may grow but will
227 * never shrink. Pool consists of number of dynamically allocated blocks
228 * with size enough to hold %items_initial number of items. Memory is
229 * DMA-able but client must map/unmap before interoperating with the device.
230 * See also: vxge_os_dma_map(), vxge_hal_dma_unmap(), vxge_hal_status_e {}.
231 */
232 vxge_hal_mempool_t *
vxge_hal_mempool_create(vxge_hal_device_h devh,u32 memblock_size,u32 item_size,u32 items_priv_size,u32 items_initial,u32 items_max,vxge_hal_mempool_item_f item_func_alloc,vxge_hal_mempool_item_f item_func_free,void * userdata)233 vxge_hal_mempool_create(
234 vxge_hal_device_h devh,
235 u32 memblock_size,
236 u32 item_size,
237 u32 items_priv_size,
238 u32 items_initial,
239 u32 items_max,
240 vxge_hal_mempool_item_f item_func_alloc,
241 vxge_hal_mempool_item_f item_func_free,
242 void *userdata)
243 {
244 vxge_hal_status_e status;
245 u32 memblocks_to_allocate;
246 vxge_hal_mempool_t *mempool;
247 __hal_device_t *hldev;
248 u32 allocated;
249
250 vxge_assert(devh != NULL);
251
252 hldev = (__hal_device_t *) devh;
253
254 vxge_hal_trace_log_mm("==> %s:%s:%d",
255 __FILE__, __func__, __LINE__);
256
257 vxge_hal_trace_log_mm(
258 "devh = 0x"VXGE_OS_STXFMT", memblock_size = %d, item_size = %d, "
259 "items_priv_size = %d, items_initial = %d, items_max = %d, "
260 "item_func_alloc = 0x"VXGE_OS_STXFMT", "
261 "item_func_free = 0x"VXGE_OS_STXFMT", "
262 "userdata = 0x"VXGE_OS_STXFMT, (ptr_t) devh,
263 memblock_size, item_size, items_priv_size,
264 items_initial, items_max, (ptr_t) item_func_alloc,
265 (ptr_t) item_func_free, (ptr_t) userdata);
266
267 if (memblock_size < item_size) {
268 vxge_hal_err_log_mm(
269 "memblock_size %d < item_size %d: misconfiguration",
270 memblock_size, item_size);
271 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
272 __FILE__, __func__, __LINE__, VXGE_HAL_FAIL);
273 return (NULL);
274 }
275
276 mempool = (vxge_hal_mempool_t *) vxge_os_malloc(
277 ((__hal_device_t *) devh)->header.pdev, sizeof(vxge_hal_mempool_t));
278 if (mempool == NULL) {
279 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
280 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
281 return (NULL);
282 }
283 vxge_os_memzero(mempool, sizeof(vxge_hal_mempool_t));
284
285 mempool->devh = devh;
286 mempool->memblock_size = memblock_size;
287 mempool->items_max = items_max;
288 mempool->items_initial = items_initial;
289 mempool->item_size = item_size;
290 mempool->items_priv_size = items_priv_size;
291 mempool->item_func_alloc = item_func_alloc;
292 mempool->item_func_free = item_func_free;
293 mempool->userdata = userdata;
294
295 mempool->memblocks_allocated = 0;
296
297 if (memblock_size != VXGE_OS_HOST_PAGE_SIZE)
298 mempool->dma_flags = VXGE_OS_DMA_CACHELINE_ALIGNED;
299
300 #if defined(VXGE_HAL_DMA_CONSISTENT)
301 mempool->dma_flags |= VXGE_OS_DMA_CONSISTENT;
302 #else
303 mempool->dma_flags |= VXGE_OS_DMA_STREAMING;
304 #endif
305
306 mempool->items_per_memblock = memblock_size / item_size;
307
308 mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
309 mempool->items_per_memblock;
310
311 /* allocate array of memblocks */
312 mempool->memblocks_arr = (void **)vxge_os_malloc(
313 ((__hal_device_t *) mempool->devh)->header.pdev,
314 sizeof(void *) * mempool->memblocks_max);
315 if (mempool->memblocks_arr == NULL) {
316 vxge_hal_mempool_destroy(mempool);
317 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
318 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
319 return (NULL);
320 }
321 vxge_os_memzero(mempool->memblocks_arr,
322 sizeof(void *) * mempool->memblocks_max);
323
324 /* allocate array of private parts of items per memblocks */
325 mempool->memblocks_priv_arr = (void **)vxge_os_malloc(
326 ((__hal_device_t *) mempool->devh)->header.pdev,
327 sizeof(void *) * mempool->memblocks_max);
328 if (mempool->memblocks_priv_arr == NULL) {
329 vxge_hal_mempool_destroy(mempool);
330 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
331 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
332 return (NULL);
333 }
334 vxge_os_memzero(mempool->memblocks_priv_arr,
335 sizeof(void *) * mempool->memblocks_max);
336
337 /* allocate array of memblocks DMA objects */
338 mempool->memblocks_dma_arr =
339 (vxge_hal_mempool_dma_t *) vxge_os_malloc(
340 ((__hal_device_t *) mempool->devh)->header.pdev,
341 sizeof(vxge_hal_mempool_dma_t) * mempool->memblocks_max);
342
343 if (mempool->memblocks_dma_arr == NULL) {
344 vxge_hal_mempool_destroy(mempool);
345 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
346 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
347 return (NULL);
348 }
349 vxge_os_memzero(mempool->memblocks_dma_arr,
350 sizeof(vxge_hal_mempool_dma_t) * mempool->memblocks_max);
351
352 /* allocate hash array of items */
353 mempool->items_arr = (void **)vxge_os_malloc(
354 ((__hal_device_t *) mempool->devh)->header.pdev,
355 sizeof(void *) * mempool->items_max);
356 if (mempool->items_arr == NULL) {
357 vxge_hal_mempool_destroy(mempool);
358 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
359 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
360 return (NULL);
361 }
362 vxge_os_memzero(mempool->items_arr,
363 sizeof(void *) * mempool->items_max);
364
365 mempool->shadow_items_arr = (void **)vxge_os_malloc(
366 ((__hal_device_t *) mempool->devh)->header.pdev,
367 sizeof(void *) * mempool->items_max);
368 if (mempool->shadow_items_arr == NULL) {
369 vxge_hal_mempool_destroy(mempool);
370 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
371 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
372 return (NULL);
373 }
374 vxge_os_memzero(mempool->shadow_items_arr,
375 sizeof(void *) * mempool->items_max);
376
377 /* calculate initial number of memblocks */
378 memblocks_to_allocate = (mempool->items_initial +
379 mempool->items_per_memblock - 1) /
380 mempool->items_per_memblock;
381
382 vxge_hal_info_log_mm("allocating %d memblocks, "
383 "%d items per memblock", memblocks_to_allocate,
384 mempool->items_per_memblock);
385
386 /* pre-allocate the mempool */
387 status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated);
388 vxge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr,
389 sizeof(void *) * mempool->items_max);
390 if (status != VXGE_HAL_OK) {
391 vxge_hal_mempool_destroy(mempool);
392 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
393 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
394 return (NULL);
395 }
396
397 vxge_hal_info_log_mm(
398 "total: allocated %dk of DMA-capable memory",
399 mempool->memblock_size * allocated / 1024);
400
401 vxge_hal_trace_log_mm("<== %s:%s:%d Result: 0",
402 __FILE__, __func__, __LINE__);
403
404 return (mempool);
405 }
406
407 /*
408 * vxge_hal_mempool_destroy
409 */
410 void
vxge_hal_mempool_destroy(vxge_hal_mempool_t * mempool)411 vxge_hal_mempool_destroy(
412 vxge_hal_mempool_t *mempool)
413 {
414 u32 i, j, item_index;
415 __hal_device_t *hldev;
416
417 vxge_assert(mempool != NULL);
418
419 hldev = (__hal_device_t *) mempool->devh;
420
421 vxge_hal_trace_log_mm("==> %s:%s:%d",
422 __FILE__, __func__, __LINE__);
423
424 vxge_hal_trace_log_mm("mempool = 0x"VXGE_OS_STXFMT,
425 (ptr_t) mempool);
426
427 for (i = 0; i < mempool->memblocks_allocated; i++) {
428 vxge_hal_mempool_dma_t *dma_object;
429
430 vxge_assert(mempool->memblocks_arr[i]);
431 vxge_assert(mempool->memblocks_dma_arr + i);
432
433 dma_object = mempool->memblocks_dma_arr + i;
434
435 for (j = 0; j < mempool->items_per_memblock; j++) {
436 item_index = i * mempool->items_per_memblock + j;
437
438 /* to skip last partially filled(if any) memblock */
439 if (item_index >= mempool->items_current)
440 break;
441
442 /* let caller to do more job on each item */
443 if (mempool->item_func_free != NULL) {
444
445 mempool->item_func_free(mempool,
446 mempool->memblocks_arr[i],
447 i, dma_object,
448 mempool->shadow_items_arr[item_index],
449 item_index, /* unused */ -1,
450 mempool->userdata);
451 }
452 }
453
454 vxge_os_free(hldev->header.pdev,
455 mempool->memblocks_priv_arr[i],
456 mempool->items_priv_size * mempool->items_per_memblock);
457
458 __hal_blockpool_free(hldev,
459 mempool->memblocks_arr[i],
460 mempool->memblock_size,
461 &dma_object->addr,
462 &dma_object->handle,
463 &dma_object->acc_handle);
464 }
465
466 if (mempool->items_arr) {
467 vxge_os_free(hldev->header.pdev,
468 mempool->items_arr, sizeof(void *) * mempool->items_max);
469 }
470
471 if (mempool->shadow_items_arr) {
472 vxge_os_free(hldev->header.pdev,
473 mempool->shadow_items_arr,
474 sizeof(void *) * mempool->items_max);
475 }
476
477 if (mempool->memblocks_dma_arr) {
478 vxge_os_free(hldev->header.pdev,
479 mempool->memblocks_dma_arr,
480 sizeof(vxge_hal_mempool_dma_t) *
481 mempool->memblocks_max);
482 }
483
484 if (mempool->memblocks_priv_arr) {
485 vxge_os_free(hldev->header.pdev,
486 mempool->memblocks_priv_arr,
487 sizeof(void *) * mempool->memblocks_max);
488 }
489
490 if (mempool->memblocks_arr) {
491 vxge_os_free(hldev->header.pdev,
492 mempool->memblocks_arr,
493 sizeof(void *) * mempool->memblocks_max);
494 }
495
496 vxge_os_free(hldev->header.pdev,
497 mempool, sizeof(vxge_hal_mempool_t));
498
499 vxge_hal_trace_log_mm("<== %s:%s:%d Result: 0",
500 __FILE__, __func__, __LINE__);
501 }
502
503 /*
504 * vxge_hal_check_alignment - Check buffer alignment and calculate the
505 * "misaligned" portion.
506 * @dma_pointer: DMA address of the buffer.
507 * @size: Buffer size, in bytes.
508 * @alignment: Alignment "granularity" (see below), in bytes.
509 * @copy_size: Maximum number of bytes to "extract" from the buffer
510 * (in order to spost it as a separate scatter-gather entry). See below.
511 *
512 * Check buffer alignment and calculate "misaligned" portion, if exists.
513 * The buffer is considered aligned if its address is multiple of
514 * the specified @alignment. If this is the case,
515 * vxge_hal_check_alignment() returns zero.
516 * Otherwise, vxge_hal_check_alignment() uses the last argument,
517 * @copy_size,
518 * to calculate the size to "extract" from the buffer. The @copy_size
519 * may or may not be equal @alignment. The difference between these two
520 * arguments is that the @alignment is used to make the decision: aligned
521 * or not aligned. While the @copy_size is used to calculate the portion
522 * of the buffer to "extract", i.e. to post as a separate entry in the
523 * transmit descriptor. For example, the combination
524 * @alignment = 8 and @copy_size = 64 will work okay on AMD Opteron boxes.
525 *
526 * Note: @copy_size should be a multiple of @alignment. In many practical
527 * cases @copy_size and @alignment will probably be equal.
528 *
529 * See also: vxge_hal_fifo_txdl_buffer_set_aligned().
530 */
531 u32
vxge_hal_check_alignment(dma_addr_t dma_pointer,u32 size,u32 alignment,u32 copy_size)532 vxge_hal_check_alignment(
533 dma_addr_t dma_pointer,
534 u32 size,
535 u32 alignment,
536 u32 copy_size)
537 {
538 u32 misaligned_size;
539
540 misaligned_size = (int)(dma_pointer & (alignment - 1));
541 if (!misaligned_size) {
542 return (0);
543 }
544
545 if (size > copy_size) {
546 misaligned_size = (int)(dma_pointer & (copy_size - 1));
547 misaligned_size = copy_size - misaligned_size;
548 } else {
549 misaligned_size = size;
550 }
551
552 return (misaligned_size);
553 }
554