1 /*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include <linux/mm.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/sched.h>
38 #include <linux/dma-attrs.h>
39 #include <linux/slab.h>
40 #include <linux/module.h>
41 #include <sys/priv.h>
42 #include <sys/resourcevar.h>
43 #include <vm/vm_pageout.h>
44 #include <vm/vm_map.h>
45 #include "uverbs.h"
46
47 #define IB_UMEM_MAX_PAGE_CHUNK (PAGE_SIZE / sizeof (struct page *))
48
49 static int allow_weak_ordering;
50 module_param_named(weak_ordering, allow_weak_ordering, int, 0444);
51 MODULE_PARM_DESC(weak_ordering, "Allow weak ordering for data registered memory");
52
peer_umem_get(struct ib_peer_memory_client * ib_peer_mem,struct ib_umem * umem,unsigned long addr,int dmasync,int invalidation_supported)53 static struct ib_umem *peer_umem_get(struct ib_peer_memory_client *ib_peer_mem,
54 struct ib_umem *umem, unsigned long addr,
55 int dmasync, int invalidation_supported)
56 {
57 int ret;
58 const struct peer_memory_client *peer_mem = ib_peer_mem->peer_mem;
59 struct invalidation_ctx *invalidation_ctx = NULL;
60
61 umem->ib_peer_mem = ib_peer_mem;
62 if (invalidation_supported) {
63 invalidation_ctx = kzalloc(sizeof(*invalidation_ctx), GFP_KERNEL);
64 if (!invalidation_ctx) {
65 ret = -ENOMEM;
66 goto out;
67 }
68 umem->invalidation_ctx = invalidation_ctx;
69 invalidation_ctx->umem = umem;
70 mutex_lock(&ib_peer_mem->lock);
71 invalidation_ctx->context_ticket =
72 ib_peer_insert_context(ib_peer_mem, invalidation_ctx);
73 /* unlock before calling get pages to prevent a dead-lock from the callback */
74 mutex_unlock(&ib_peer_mem->lock);
75 }
76
77 ret = peer_mem->get_pages(addr, umem->length, umem->writable, 1,
78 &umem->sg_head,
79 umem->peer_mem_client_context,
80 invalidation_ctx ?
81 (void *)invalidation_ctx->context_ticket : NULL);
82
83 if (invalidation_ctx) {
84 /* taking the lock back, checking that wasn't invalidated at that time */
85 mutex_lock(&ib_peer_mem->lock);
86 if (invalidation_ctx->peer_invalidated) {
87 printk(KERN_ERR "peer_umem_get: pages were invalidated by peer\n");
88 ret = -EINVAL;
89 }
90 }
91
92 if (ret)
93 goto out;
94
95 umem->page_size = peer_mem->get_page_size
96 (umem->peer_mem_client_context);
97 if (umem->page_size <= 0)
98 goto put_pages;
99
100 umem->offset = addr & ((unsigned long)umem->page_size - 1);
101 ret = peer_mem->dma_map(&umem->sg_head,
102 umem->peer_mem_client_context,
103 umem->context->device->dma_device,
104 dmasync,
105 &umem->nmap);
106 if (ret)
107 goto put_pages;
108
109 ib_peer_mem->stats.num_reg_pages +=
110 umem->nmap * (umem->page_size >> PAGE_SHIFT);
111 ib_peer_mem->stats.num_alloc_mrs += 1;
112 return umem;
113
114 put_pages:
115
116 peer_mem->put_pages(umem->peer_mem_client_context,
117 &umem->sg_head);
118 out:
119 if (invalidation_ctx) {
120 ib_peer_remove_context(ib_peer_mem, invalidation_ctx->context_ticket);
121 mutex_unlock(&umem->ib_peer_mem->lock);
122 kfree(invalidation_ctx);
123 }
124
125 ib_put_peer_client(ib_peer_mem, umem->peer_mem_client_context,
126 umem->peer_mem_srcu_key);
127 kfree(umem);
128 return ERR_PTR(ret);
129 }
130
peer_umem_release(struct ib_umem * umem)131 static void peer_umem_release(struct ib_umem *umem)
132 {
133 struct ib_peer_memory_client *ib_peer_mem = umem->ib_peer_mem;
134 const struct peer_memory_client *peer_mem = ib_peer_mem->peer_mem;
135 struct invalidation_ctx *invalidation_ctx = umem->invalidation_ctx;
136
137 if (invalidation_ctx) {
138
139 int peer_callback;
140 int inflight_invalidation;
141 /* If we are not under peer callback we must take the lock before removing
142 * core ticket from the tree and releasing its umem.
143 * It will let any inflight callbacks to be ended safely.
144 * If we are under peer callback or under error flow of reg_mr so that context
145 * wasn't activated yet lock was already taken.
146 */
147 if (invalidation_ctx->func && !invalidation_ctx->peer_callback)
148 mutex_lock(&ib_peer_mem->lock);
149 ib_peer_remove_context(ib_peer_mem, invalidation_ctx->context_ticket);
150 /* make sure to check inflight flag after took the lock and remove from tree.
151 * in addition, from that point using local variables for peer_callback and
152 * inflight_invalidation as after the complete invalidation_ctx can't be accessed
153 * any more as it may be freed by the callback.
154 */
155 peer_callback = invalidation_ctx->peer_callback;
156 inflight_invalidation = invalidation_ctx->inflight_invalidation;
157 if (inflight_invalidation)
158 complete(&invalidation_ctx->comp);
159 /* On peer callback lock is handled externally */
160 if (!peer_callback)
161 /* unlocking before put_pages */
162 mutex_unlock(&ib_peer_mem->lock);
163 /* in case under callback context or callback is pending let it free the invalidation context */
164 if (!peer_callback && !inflight_invalidation)
165 kfree(invalidation_ctx);
166 }
167
168 peer_mem->dma_unmap(&umem->sg_head,
169 umem->peer_mem_client_context,
170 umem->context->device->dma_device);
171 peer_mem->put_pages(&umem->sg_head,
172 umem->peer_mem_client_context);
173
174 ib_peer_mem->stats.num_dereg_pages +=
175 umem->nmap * (umem->page_size >> PAGE_SHIFT);
176 ib_peer_mem->stats.num_dealloc_mrs += 1;
177 ib_put_peer_client(ib_peer_mem, umem->peer_mem_client_context,
178 umem->peer_mem_srcu_key);
179 kfree(umem);
180
181 return;
182
183 }
184
__ib_umem_release(struct ib_device * dev,struct ib_umem * umem,int dirty)185 static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)
186 {
187
188 vm_object_t object;
189 struct scatterlist *sg;
190 struct page *page;
191 int i;
192
193 object = NULL;
194 if (umem->nmap > 0)
195 ib_dma_unmap_sg(dev, umem->sg_head.sgl,
196 umem->nmap,
197 DMA_BIDIRECTIONAL);
198 for_each_sg(umem->sg_head.sgl, sg, umem->npages, i) {
199 page = sg_page(sg);
200 if (umem->writable && dirty) {
201 if (object && object != page->object)
202 VM_OBJECT_WUNLOCK(object);
203 if (object != page->object) {
204 object = page->object;
205 VM_OBJECT_WLOCK(object);
206 }
207 vm_page_dirty(page);
208 }
209 }
210 sg_free_table(&umem->sg_head);
211 if (object)
212 VM_OBJECT_WUNLOCK(object);
213
214 }
215
ib_umem_activate_invalidation_notifier(struct ib_umem * umem,umem_invalidate_func_t func,void * cookie)216 void ib_umem_activate_invalidation_notifier(struct ib_umem *umem,
217 umem_invalidate_func_t func,
218 void *cookie)
219 {
220 struct invalidation_ctx *invalidation_ctx = umem->invalidation_ctx;
221
222 invalidation_ctx->func = func;
223 invalidation_ctx->cookie = cookie;
224
225 /* from that point any pending invalidations can be called */
226 mutex_unlock(&umem->ib_peer_mem->lock);
227 return;
228 }
229 EXPORT_SYMBOL(ib_umem_activate_invalidation_notifier);
230 /**
231 * ib_umem_get - Pin and DMA map userspace memory.
232 * @context: userspace context to pin memory for
233 * @addr: userspace virtual address to start at
234 * @size: length of region to pin
235 * @access: IB_ACCESS_xxx flags for memory being pinned
236 * @dmasync: flush in-flight DMA when the memory region is written
237 */
ib_umem_get_ex(struct ib_ucontext * context,unsigned long addr,size_t size,int access,int dmasync,int invalidation_supported)238 struct ib_umem *ib_umem_get_ex(struct ib_ucontext *context, unsigned long addr,
239 size_t size, int access, int dmasync,
240 int invalidation_supported)
241 {
242
243 struct ib_umem *umem;
244 struct proc *proc;
245 pmap_t pmap;
246 vm_offset_t end, last, start;
247 vm_size_t npages;
248 int error;
249 int ret;
250 int ents;
251 int i;
252 DEFINE_DMA_ATTRS(attrs);
253 struct scatterlist *sg, *sg_list_start;
254 int need_release = 0;
255
256 error = priv_check(curthread, PRIV_VM_MLOCK);
257 if (error)
258 return ERR_PTR(-error);
259
260 last = addr + size;
261 start = addr & PAGE_MASK; /* Use the linux PAGE_MASK definition. */
262 end = roundup2(last, PAGE_SIZE); /* Use PAGE_MASK safe operation. */
263 if (last < addr || end < addr)
264 return ERR_PTR(-EINVAL);
265 npages = atop(end - start);
266 if (npages > vm_page_max_wired)
267 return ERR_PTR(-ENOMEM);
268 umem = kzalloc(sizeof *umem, GFP_KERNEL);
269 if (!umem)
270 return ERR_PTR(-ENOMEM);
271 proc = curthread->td_proc;
272 PROC_LOCK(proc);
273 if (ptoa(npages +
274 pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map))) >
275 lim_cur_proc(proc, RLIMIT_MEMLOCK)) {
276 PROC_UNLOCK(proc);
277 kfree(umem);
278 return ERR_PTR(-ENOMEM);
279 }
280 PROC_UNLOCK(proc);
281 if (npages + vm_cnt.v_wire_count > vm_page_max_wired) {
282 kfree(umem);
283 return ERR_PTR(-EAGAIN);
284 }
285 error = vm_map_wire(&proc->p_vmspace->vm_map, start, end,
286 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES |
287 (umem->writable ? VM_MAP_WIRE_WRITE : 0));
288 if (error != KERN_SUCCESS) {
289 kfree(umem);
290 return ERR_PTR(-ENOMEM);
291 }
292
293 umem->context = context;
294 umem->length = size;
295 umem->offset = addr & ~PAGE_MASK;
296 umem->page_size = PAGE_SIZE;
297 umem->start = addr;
298 /*
299 * We ask for writable memory if any access flags other than
300 * "remote read" are set. "Local write" and "remote write"
301 * obviously require write access. "Remote atomic" can do
302 * things like fetch and add, which will modify memory, and
303 * "MW bind" can change permissions by binding a window.
304 */
305 umem->writable = !!(access & ~IB_ACCESS_REMOTE_READ);
306
307 if (invalidation_supported || context->peer_mem_private_data) {
308
309 struct ib_peer_memory_client *peer_mem_client;
310
311 peer_mem_client = ib_get_peer_client(context, addr, size,
312 &umem->peer_mem_client_context,
313 &umem->peer_mem_srcu_key);
314 if (peer_mem_client)
315 return peer_umem_get(peer_mem_client, umem, addr,
316 dmasync, invalidation_supported);
317 }
318
319 umem->hugetlb = 0;
320
321 pmap = vm_map_pmap(&proc->p_vmspace->vm_map);
322
323 if (npages == 0) {
324 ret = -EINVAL;
325 goto out;
326 }
327
328 ret = sg_alloc_table(&umem->sg_head, npages, GFP_KERNEL);
329 if (ret)
330 goto out;
331
332 need_release = 1;
333 sg_list_start = umem->sg_head.sgl;
334
335 while (npages) {
336
337 ents = min_t(int, npages, IB_UMEM_MAX_PAGE_CHUNK);
338 umem->npages += ents;
339
340 for_each_sg(sg_list_start, sg, ents, i) {
341 vm_paddr_t pa;
342
343 pa = pmap_extract(pmap, start);
344 if (pa == 0) {
345 ret = -ENOMEM;
346 goto out;
347 }
348 sg_set_page(sg, PHYS_TO_VM_PAGE(pa),
349 PAGE_SIZE, 0);
350 npages--;
351 start += PAGE_SIZE;
352 }
353
354 /* preparing for next loop */
355 sg_list_start = sg;
356 }
357
358 umem->nmap = ib_dma_map_sg_attrs(context->device,
359 umem->sg_head.sgl,
360 umem->npages,
361 DMA_BIDIRECTIONAL,
362 &attrs);
363 if (umem->nmap != umem->npages) {
364 ret = -ENOMEM;
365 goto out;
366 }
367
368 out:
369 if (ret < 0) {
370 if (need_release)
371 __ib_umem_release(context->device, umem, 0);
372 kfree(umem);
373 }
374
375 return ret < 0 ? ERR_PTR(ret) : umem;
376 }
377 EXPORT_SYMBOL(ib_umem_get_ex);
378
ib_umem_get(struct ib_ucontext * context,unsigned long addr,size_t size,int access,int dmasync)379 struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
380 size_t size, int access, int dmasync)
381 {
382 return ib_umem_get_ex(context, addr,
383 size, access, dmasync, 0);
384 }
385 EXPORT_SYMBOL(ib_umem_get);
386
387 /**
388 * ib_umem_release - release memory pinned with ib_umem_get
389 * @umem: umem struct to release
390 */
ib_umem_release(struct ib_umem * umem)391 void ib_umem_release(struct ib_umem *umem)
392 {
393
394 vm_offset_t addr, end, last, start;
395 vm_size_t size;
396 int error;
397
398 if (umem->ib_peer_mem) {
399 peer_umem_release(umem);
400 return;
401 }
402
403 __ib_umem_release(umem->context->device, umem, 1);
404
405 if (umem->context->closing) {
406 kfree(umem);
407 return;
408 }
409
410 error = priv_check(curthread, PRIV_VM_MUNLOCK);
411
412 if (error)
413 return;
414
415 addr = umem->start;
416 size = umem->length;
417 last = addr + size;
418 start = addr & PAGE_MASK; /* Use the linux PAGE_MASK definition. */
419 end = roundup2(last, PAGE_SIZE); /* Use PAGE_MASK safe operation. */
420 vm_map_unwire(&curthread->td_proc->p_vmspace->vm_map, start, end,
421 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
422 kfree(umem);
423
424 }
425 EXPORT_SYMBOL(ib_umem_release);
426
ib_umem_page_count(struct ib_umem * umem)427 int ib_umem_page_count(struct ib_umem *umem)
428 {
429 int shift;
430 int i;
431 int n;
432 struct scatterlist *sg;
433
434 shift = ilog2(umem->page_size);
435
436 n = 0;
437 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i)
438 n += sg_dma_len(sg) >> shift;
439
440 return n;
441 }
442 EXPORT_SYMBOL(ib_umem_page_count);
443