1 /*
2 * Copyright (c) 2015-2020 François Tigeot <ftigeot@wolfpond.org>
3 * Copyright (c) 2020 Matthew Dillon <dillon@backplane.com>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice unmodified, this list of conditions, and the following
11 * disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include <drm/drmP.h>
29 #include <linux/workqueue.h>
30
31 #include <sys/kthread.h>
32
33 /*
34 Running behaviour, from kernel.org docs:
35 - While there are work items on the workqueue the worker executes the functions
36 associated with the work items one after the other.
37 - When there is no work item left on the workqueue the worker becomes idle.
38
39 There are two worker-pools,
40 one for normal work items
41 and the other for high priority ones, for each possible CPU
42 and some extra worker-pools to serve work items queued on unbound workqueues
43 - the number of these backing pools is dynamic.
44 */
45
46 /* XXX: Linux functions often enable/disable irqs on the CPU they run on
47 * this should be investigated */
48
49 struct workqueue_struct *system_wq;
50 struct workqueue_struct *system_highpri_wq;
51 struct workqueue_struct *system_long_wq;
52 struct workqueue_struct *system_unbound_wq;
53 struct workqueue_struct *system_power_efficient_wq;
54
55 /*
56 * Linux now uses these worker pools:
57 * - (per cpu) regular
58 * - (per cpu) regular high priority
59 * - ordered
60 * - ordered high priority
61 * - unbound
62 * - unbound high priority
63 */
64
65 static inline void
process_all_work(struct workqueue_worker * worker)66 process_all_work(struct workqueue_worker *worker)
67 {
68 struct work_struct *work;
69 bool didcan;
70
71 while (STAILQ_FIRST(&worker->ws_list_head)) {
72 work = STAILQ_FIRST(&worker->ws_list_head);
73 STAILQ_REMOVE_HEAD(&worker->ws_list_head, ws_entries);
74 work->on_queue = false;
75
76 /* A work shouldn't be executed concurrently on a single cpu */
77 if (work->running)
78 continue;
79
80 /* Do not run canceled works */
81 if (work->canceled) {
82 /* XXX: should we allow canceled works to be reenabled ? */
83 work->canceled = false;
84 continue;
85 }
86
87 work->running = true;
88 lockmgr(&worker->worker_lock, LK_RELEASE);
89 work->func(work);
90 lwkt_yield();
91 lockmgr(&worker->worker_lock, LK_EXCLUSIVE);
92 if (work->on_queue == false)
93 work->worker = NULL;
94 didcan = work->canceled;
95 cpu_sfence();
96 work->running = false;
97 if (didcan == true)
98 wakeup(work);
99 }
100 }
101
102 static void
wq_worker_thread(void * arg)103 wq_worker_thread(void *arg)
104 {
105 struct workqueue_worker *worker = arg;
106
107 lockmgr(&worker->worker_lock, LK_EXCLUSIVE);
108 while (1) {
109 process_all_work(worker);
110 lksleep(worker, &worker->worker_lock, 0, "wqidle", 0);
111 }
112 lockmgr(&worker->worker_lock, LK_RELEASE);
113 }
114
115 /*
116 * Return false if work was already on a queue
117 * Return true and queue it if this was not the case
118 */
119 int
queue_work(struct workqueue_struct * wq,struct work_struct * work)120 queue_work(struct workqueue_struct *wq, struct work_struct *work)
121 {
122 struct workqueue_worker *worker;
123 int ret = false;
124
125 /* XXX: should we block instead ? */
126 if (wq->is_draining)
127 return false;
128
129 if (wq->num_workers > 1)
130 worker = &(*wq->workers)[mycpuid];
131 else
132 worker = &(*wq->workers)[0];
133
134 lockmgr(&worker->worker_lock, LK_EXCLUSIVE);
135 work->canceled = false;
136 if (work->on_queue == false || work->running == false) {
137 if (work->on_queue == false) {
138 STAILQ_INSERT_TAIL(&worker->ws_list_head, work,
139 ws_entries);
140 work->on_queue = true;
141 work->worker = worker;
142 wakeup_one(worker);
143 }
144 ret = true;
145 }
146 lockmgr(&worker->worker_lock, LK_RELEASE);
147
148 return ret;
149 }
150
151 static inline void
_delayed_work_fn(void * arg)152 _delayed_work_fn(void *arg)
153 {
154 struct delayed_work *dw = arg;
155
156 queue_work(system_wq, &dw->work);
157 }
158
159 int
queue_delayed_work(struct workqueue_struct * wq,struct delayed_work * work,unsigned long delay)160 queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *work,
161 unsigned long delay)
162 {
163 int pending = work->work.on_queue; // XXX: running too ?
164 if (delay != 0) {
165 callout_reset(&work->timer, delay, _delayed_work_fn, work);
166 } else {
167 _delayed_work_fn((void *)work);
168 }
169
170 return (!pending);
171 }
172
173 static int
init_workqueues(void * arg)174 init_workqueues(void *arg)
175 {
176 system_wq = alloc_workqueue("system_wq", 0, 1);
177 system_highpri_wq = alloc_workqueue("system_highpri_wq", WQ_HIGHPRI, 1);
178 system_long_wq = alloc_workqueue("system_long_wq", 0, 1);
179 system_unbound_wq = alloc_workqueue("system_unbound_wq", WQ_UNBOUND, 1);
180 system_power_efficient_wq = alloc_workqueue("system_power_efficient_wq", 0, 1);
181
182 return 0;
183 }
184
destroy_workqueues(void * arg)185 static int destroy_workqueues(void *arg)
186 {
187 destroy_workqueue(system_wq);
188 destroy_workqueue(system_highpri_wq);
189 destroy_workqueue(system_long_wq);
190 destroy_workqueue(system_unbound_wq);
191 destroy_workqueue(system_power_efficient_wq);
192
193 return 0;
194 }
195
196 struct workqueue_struct *
_create_workqueue_common(const char * name,int flags)197 _create_workqueue_common(const char *name, int flags)
198 {
199 struct workqueue_struct *wq;
200 int priority, error;
201
202 wq = kmalloc(sizeof(*wq), M_DRM, M_WAITOK | M_ZERO);
203
204 if (flags & WQ_HIGHPRI)
205 priority = TDPRI_INT_SUPPORT;
206 else
207 priority = TDPRI_KERN_DAEMON;
208
209 if (flags & WQ_UNBOUND) {
210 wq->num_workers = 1;
211 } else {
212 wq->num_workers = ncpus;
213 }
214 wq->workers = kmalloc(sizeof(struct workqueue_worker) * wq->num_workers,
215 M_DRM, M_WAITOK | M_ZERO);
216
217 for (int i = 0;i < wq->num_workers; i++) {
218 struct workqueue_worker *worker = &(*wq->workers)[i];
219
220 lockinit(&worker->worker_lock, "lwq", 0, 0);
221 STAILQ_INIT(&worker->ws_list_head);
222 if (wq->num_workers > 1) {
223 error = lwkt_create(wq_worker_thread, worker,
224 &worker->worker_thread, NULL, TDF_NOSTART, i, "%s/%d", name, i);
225 } else {
226 error = lwkt_create(wq_worker_thread, worker,
227 &worker->worker_thread, NULL, TDF_NOSTART, -1, name);
228 }
229 if (error) {
230 kprintf("%s: lwkt_create(%s/%d): error %d",
231 __func__, name, i, error);
232 /* XXX: destroy kernel threads and free workers[] if applicable */
233 kfree(wq);
234 return NULL;
235 }
236 lwkt_setpri_initial(worker->worker_thread, priority);
237 lwkt_schedule(worker->worker_thread);
238 }
239
240 return wq;
241 }
242
243 void
destroy_workqueue(struct workqueue_struct * wq)244 destroy_workqueue(struct workqueue_struct *wq)
245 {
246 drain_workqueue(wq);
247 // wq->is_draining = true;
248 #if 0 /* XXX TODO */
249 kill_all_threads;
250 kfree(wq->wq_threads);
251 kfree(wq);
252 #endif
253 }
254
255 SYSINIT(linux_workqueue_init, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, init_workqueues, NULL);
256 SYSUNINIT(linux_workqueue_destroy, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, destroy_workqueues, NULL);
257
258 bool
flush_delayed_work(struct delayed_work * dwork)259 flush_delayed_work(struct delayed_work *dwork)
260 {
261 callout_drain(&dwork->timer);
262 return flush_work(&dwork->work);
263 }
264
265 /* Wait until the wq becomes empty */
266 void
drain_workqueue(struct workqueue_struct * wq)267 drain_workqueue(struct workqueue_struct *wq)
268 {
269 struct workqueue_worker *worker;
270
271 wq->is_draining = true;
272
273 for (int i=0;i < wq->num_workers; i++) {
274 worker = &(*wq->workers)[i];
275
276 lockmgr(&worker->worker_lock, LK_EXCLUSIVE);
277 while (!STAILQ_EMPTY(&worker->ws_list_head)) {
278 /* XXX: introduces latency */
279 tsleep(&drain_workqueue, 0, "wkdrain", 1);
280 }
281 lockmgr(&worker->worker_lock, LK_RELEASE);
282 }
283
284 /* XXX: No more work will be queued. is that right ? */
285 // wq->is_draining = false;
286 }
287
288 bool
work_pending(struct work_struct * work)289 work_pending(struct work_struct *work)
290 {
291 /* XXX: is on_queue the only constraint ? */
292 return work->on_queue;
293 }
294
295 unsigned int
work_busy(struct work_struct * work)296 work_busy(struct work_struct *work)
297 {
298 return (work->on_queue || work->running);
299 }
300
301 static inline void
__flush_work_func(struct work_struct * work)302 __flush_work_func(struct work_struct *work)
303 {
304 wakeup_one(work);
305 }
306
307 /* XXX introduces latency ? */
308 void
flush_workqueue(struct workqueue_struct * wq)309 flush_workqueue(struct workqueue_struct *wq)
310 {
311 struct work_struct __flush_work;
312
313 INIT_WORK(&__flush_work, __flush_work_func);
314
315 queue_work(wq, &__flush_work);
316 while (__flush_work.on_queue || __flush_work.running) {
317 tsleep(&__flush_work, 0, "flshwq", 0);
318 }
319 }
320
321 /*
322 * Wait until a work is done (has been executed)
323 * Return true if this function had to wait, and false otherwise
324 */
325 bool
flush_work(struct work_struct * work)326 flush_work(struct work_struct *work)
327 {
328 int ret = false;
329
330 /* XXX: probably unreliable */
331 while (work->on_queue || work->running) {
332 ret = true;
333 /* XXX: use something more intelligent than tsleep() */
334 tsleep(&flush_work, 0, "flshwrk", 1);
335 }
336
337 return ret;
338 }
339
340 static inline bool
_cancel_work(struct work_struct * work,bool sync_wait)341 _cancel_work(struct work_struct *work, bool sync_wait)
342 {
343 struct workqueue_worker *worker;
344 bool ret;
345
346 ret = false;
347
348 for (;;) {
349 if (work->on_queue) {
350 worker = work->worker;
351 if (worker == NULL)
352 continue;
353 lockmgr(&worker->worker_lock, LK_EXCLUSIVE);
354 if (worker != work->worker || work->on_queue == false) {
355 lockmgr(&worker->worker_lock, LK_RELEASE);
356 continue;
357 }
358 STAILQ_REMOVE(&worker->ws_list_head, work,
359 work_struct, ws_entries);
360 work->on_queue = false;
361 ret = true;
362 lockmgr(&worker->worker_lock, LK_RELEASE);
363 }
364 if (work->running == false)
365 break;
366
367 worker = work->worker;
368 if (worker == NULL)
369 continue;
370 lockmgr(&worker->worker_lock, LK_EXCLUSIVE);
371 if (worker != work->worker || work->running == false) {
372 lockmgr(&worker->worker_lock, LK_RELEASE);
373 continue;
374 }
375 work->canceled = true;
376 ret = true;
377 if (sync_wait == false) {
378 lockmgr(&worker->worker_lock, LK_RELEASE);
379 break;
380 }
381 /* XXX this races */
382 lksleep(work, &worker->worker_lock, 0, "wqcan", 1);
383 lockmgr(&worker->worker_lock, LK_RELEASE);
384 /* retest */
385 }
386
387 return ret;
388 }
389
390 /*
391 * If work was queued, remove it from the queue and return true.
392 * If work was not queued, return false.
393 * In any case, wait for work to complete or be removed from the workqueue,
394 * callers may free associated data structures after this call.
395 */
396 bool
cancel_work_sync(struct work_struct * work)397 cancel_work_sync(struct work_struct *work)
398 {
399 return _cancel_work(work, true);
400 }
401
402 /* Return false if work wasn't pending
403 * Return true if work was pending and canceled */
404 bool
cancel_delayed_work(struct delayed_work * dwork)405 cancel_delayed_work(struct delayed_work *dwork)
406 {
407 struct work_struct *work = &dwork->work;
408
409 work->canceled = true;
410 callout_cancel(&dwork->timer);
411
412 return _cancel_work(work, false);
413 }
414
415 bool
cancel_delayed_work_sync(struct delayed_work * dwork)416 cancel_delayed_work_sync(struct delayed_work *dwork)
417 {
418 struct work_struct *work = &dwork->work;
419
420 work->canceled = true;
421 callout_cancel(&dwork->timer);
422
423 return _cancel_work(work, true);
424 }
425
426 bool
delayed_work_pending(struct delayed_work * dw)427 delayed_work_pending(struct delayed_work *dw)
428 {
429 /* XXX: possibly wrong if the timer hasn't yet fired */
430 return work_pending(&dw->work);
431 }
432
433 void
destroy_work_on_stack(struct work_struct * work)434 destroy_work_on_stack(struct work_struct *work)
435 {
436 }
437
438 void
destroy_delayed_work_on_stack(struct delayed_work * work)439 destroy_delayed_work_on_stack(struct delayed_work *work)
440 {
441 }
442