1 /*
2 * Copyright (c) 2019-2020 Jonathan Gray <jsg@openbsd.org>
3 * Copyright (c) 2020 François Tigeot <ftigeot@wolfpond.org>
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include <linux/slab.h>
26 #include <linux/dma-fence.h>
27
28 void
dma_fence_init(struct dma_fence * fence,const struct dma_fence_ops * ops,spinlock_t * lock,u64 context,unsigned seqno)29 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
30 spinlock_t *lock, u64 context, unsigned seqno)
31 {
32 fence->ops = ops;
33 fence->lock = lock;
34 fence->context = context;
35 fence->seqno = seqno;
36 fence->flags = 0;
37 fence->error = 0;
38 kref_init(&fence->refcount);
39 INIT_LIST_HEAD(&fence->cb_list);
40 }
41
42 void
dma_fence_release(struct kref * ref)43 dma_fence_release(struct kref *ref)
44 {
45 struct dma_fence *fence = container_of(ref, struct dma_fence, refcount);
46
47 if (fence->ops && fence->ops->release)
48 fence->ops->release(fence);
49 else
50 kfree(fence);
51 }
52
53 long
dma_fence_wait_timeout(struct dma_fence * fence,bool intr,long timeout)54 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, long timeout)
55 {
56 if (timeout < 0)
57 return -EINVAL;
58
59 if (fence->ops->wait)
60 return fence->ops->wait(fence, intr, timeout);
61 else
62 return dma_fence_default_wait(fence, intr, timeout);
63 }
64
65 static atomic64_t drm_fence_context_count = ATOMIC_INIT(1);
66
67 u64
dma_fence_context_alloc(unsigned num)68 dma_fence_context_alloc(unsigned num)
69 {
70 return atomic64_add_return(num, &drm_fence_context_count) - num;
71 }
72
73 struct default_wait_cb {
74 struct dma_fence_cb base;
75 struct task_struct *task;
76 };
77
78 static void
dma_fence_default_wait_cb(struct dma_fence * fence,struct dma_fence_cb * cb)79 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
80 {
81 struct default_wait_cb *wait =
82 container_of(cb, struct default_wait_cb, base);
83
84 wake_up_process(wait->task);
85 }
86
87 long
dma_fence_default_wait(struct dma_fence * fence,bool intr,signed long timeout)88 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
89 {
90 long ret = timeout ? timeout : 1;
91 unsigned long end;
92 int err;
93 struct default_wait_cb cb;
94 bool was_set;
95
96 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
97 return ret;
98
99 lockmgr(fence->lock, LK_EXCLUSIVE);
100
101 was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
102 &fence->flags);
103
104 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
105 goto out;
106
107 if (!was_set && fence->ops->enable_signaling) {
108 if (!fence->ops->enable_signaling(fence)) {
109 dma_fence_signal_locked(fence);
110 goto out;
111 }
112 }
113
114 if (timeout == 0) {
115 ret = 0;
116 goto out;
117 }
118
119 cb.base.func = dma_fence_default_wait_cb;
120 cb.task = current;
121 list_add(&cb.base.node, &fence->cb_list);
122
123 end = jiffies + timeout;
124 for (ret = timeout; ret > 0; ret = MAX(0, end - jiffies)) {
125 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
126 break;
127 }
128 if (intr) {
129 __set_current_state(TASK_INTERRUPTIBLE);
130 }
131 else {
132 __set_current_state(TASK_UNINTERRUPTIBLE);
133 }
134 /* wake_up_process() directly uses task_struct pointers as sleep identifiers */
135 err = lksleep(current, fence->lock, intr ? PCATCH : 0, "dmafence", ret);
136 if (err == EINTR || err == ERESTART) {
137 ret = -ERESTARTSYS;
138 break;
139 }
140 }
141
142 if (!list_empty(&cb.base.node))
143 list_del(&cb.base.node);
144 __set_current_state(TASK_RUNNING);
145 out:
146 lockmgr(fence->lock, LK_RELEASE);
147 return ret;
148 }
149
150 static bool
dma_fence_test_signaled_any(struct dma_fence ** fences,uint32_t count,uint32_t * idx)151 dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
152 uint32_t *idx)
153 {
154 int i;
155
156 for (i = 0; i < count; ++i) {
157 struct dma_fence *fence = fences[i];
158 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
159 if (idx)
160 *idx = i;
161 return true;
162 }
163 }
164 return false;
165 }
166
167 long
dma_fence_wait_any_timeout(struct dma_fence ** fences,uint32_t count,bool intr,long timeout,uint32_t * idx)168 dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
169 bool intr, long timeout, uint32_t *idx)
170 {
171 struct default_wait_cb *cb;
172 long ret = timeout;
173 unsigned long end;
174 int i, err;
175
176 if (timeout == 0) {
177 for (i = 0; i < count; i++) {
178 if (dma_fence_is_signaled(fences[i])) {
179 if (idx)
180 *idx = i;
181 return 1;
182 }
183 }
184 return 0;
185 }
186
187 cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);
188 if (cb == NULL)
189 return -ENOMEM;
190
191 for (i = 0; i < count; i++) {
192 struct dma_fence *fence = fences[i];
193 cb[i].task = current;
194 if (dma_fence_add_callback(fence, &cb[i].base,
195 dma_fence_default_wait_cb)) {
196 if (idx)
197 *idx = i;
198 goto cb_cleanup;
199 }
200 }
201
202 end = jiffies + timeout;
203 for (ret = timeout; ret > 0; ret = MAX(0, end - jiffies)) {
204 if (dma_fence_test_signaled_any(fences, count, idx))
205 break;
206 err = tsleep(current, intr ? PCATCH : 0, "dfwat", ret);
207 if (err == EINTR || err == ERESTART) {
208 ret = -ERESTARTSYS;
209 break;
210 }
211 }
212
213 cb_cleanup:
214 while (i-- > 0)
215 dma_fence_remove_callback(fences[i], &cb[i].base);
216 kfree(cb);
217 return ret;
218 }
219
220 int
dma_fence_signal_locked(struct dma_fence * fence)221 dma_fence_signal_locked(struct dma_fence *fence)
222 {
223 struct dma_fence_cb *cur, *tmp;
224 struct list_head cb_list;
225
226 if (fence == NULL)
227 return -EINVAL;
228
229 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
230 return -EINVAL;
231
232 list_replace(&fence->cb_list, &cb_list);
233
234 fence->timestamp = ktime_get();
235 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
236
237 list_for_each_entry_safe(cur, tmp, &cb_list, node) {
238 INIT_LIST_HEAD(&cur->node);
239 cur->func(fence, cur);
240 }
241
242 return 0;
243 }
244
245 int
dma_fence_signal(struct dma_fence * fence)246 dma_fence_signal(struct dma_fence *fence)
247 {
248 int r;
249
250 if (fence == NULL)
251 return -EINVAL;
252
253 lockmgr(fence->lock, LK_EXCLUSIVE);
254 r = dma_fence_signal_locked(fence);
255 lockmgr(fence->lock, LK_RELEASE);
256
257 return r;
258 }
259
260 void
dma_fence_enable_sw_signaling(struct dma_fence * fence)261 dma_fence_enable_sw_signaling(struct dma_fence *fence)
262 {
263 if (!test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) &&
264 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) &&
265 fence->ops->enable_signaling) {
266 lockmgr(fence->lock, LK_EXCLUSIVE);
267 if (!fence->ops->enable_signaling(fence))
268 dma_fence_signal_locked(fence);
269 lockmgr(fence->lock, LK_RELEASE);
270 }
271 }
272
273 int
dma_fence_add_callback(struct dma_fence * fence,struct dma_fence_cb * cb,dma_fence_func_t func)274 dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
275 dma_fence_func_t func)
276 {
277 int ret = 0;
278 bool was_set;
279
280 if (WARN_ON(!fence || !func))
281 return -EINVAL;
282
283 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
284 INIT_LIST_HEAD(&cb->node);
285 return -ENOENT;
286 }
287
288 lockmgr(fence->lock, LK_EXCLUSIVE);
289
290 was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags);
291
292 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
293 ret = -ENOENT;
294 else if (!was_set && fence->ops->enable_signaling) {
295 if (!fence->ops->enable_signaling(fence)) {
296 dma_fence_signal_locked(fence);
297 ret = -ENOENT;
298 }
299 }
300
301 if (!ret) {
302 cb->func = func;
303 list_add_tail(&cb->node, &fence->cb_list);
304 } else
305 INIT_LIST_HEAD(&cb->node);
306 lockmgr(fence->lock, LK_RELEASE);
307
308 return ret;
309 }
310
311 bool
dma_fence_remove_callback(struct dma_fence * fence,struct dma_fence_cb * cb)312 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
313 {
314 bool ret;
315
316 lockmgr(fence->lock, LK_EXCLUSIVE);
317
318 ret = !list_empty(&cb->node);
319 if (ret)
320 list_del_init(&cb->node);
321
322 lockmgr(fence->lock, LK_RELEASE);
323
324 return ret;
325 }
326
327 void
dma_fence_free(struct dma_fence * fence)328 dma_fence_free(struct dma_fence *fence)
329 {
330 kfree(fence);
331 }
332