1 /*-
2 * Copyright (c) 2017-2019 Hans Petter Selasky
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD: stable/12/sys/compat/linuxkpi/common/src/linux_work.c 372122 2022-06-10 12:47:05Z gbe $");
29
30 #include <linux/workqueue.h>
31 #include <linux/wait.h>
32 #include <linux/compat.h>
33 #include <linux/spinlock.h>
34 #include <linux/rcupdate.h>
35
36 #include <sys/kernel.h>
37
38 /*
39 * Define all work struct states
40 */
41 enum {
42 WORK_ST_IDLE, /* idle - not started */
43 WORK_ST_TIMER, /* timer is being started */
44 WORK_ST_TASK, /* taskqueue is being queued */
45 WORK_ST_EXEC, /* callback is being called */
46 WORK_ST_CANCEL, /* cancel is being requested */
47 WORK_ST_MAX,
48 };
49
50 /*
51 * Define global workqueues
52 */
53 static struct workqueue_struct *linux_system_short_wq;
54 static struct workqueue_struct *linux_system_long_wq;
55
56 struct workqueue_struct *system_wq;
57 struct workqueue_struct *system_long_wq;
58 struct workqueue_struct *system_unbound_wq;
59 struct workqueue_struct *system_highpri_wq;
60 struct workqueue_struct *system_power_efficient_wq;
61
62 static int linux_default_wq_cpus = 4;
63
64 static void linux_delayed_work_timer_fn(void *);
65
66 /*
67 * This function atomically updates the work state and returns the
68 * previous state at the time of update.
69 */
70 static uint8_t
linux_update_state(atomic_t * v,const uint8_t * pstate)71 linux_update_state(atomic_t *v, const uint8_t *pstate)
72 {
73 int c, old;
74
75 c = v->counter;
76
77 while ((old = atomic_cmpxchg(v, c, pstate[c])) != c)
78 c = old;
79
80 return (c);
81 }
82
83 /*
84 * A LinuxKPI task is allowed to free itself inside the callback function
85 * and cannot safely be referred after the callback function has
86 * completed. This function gives the linux_work_fn() function a hint,
87 * that the task is not going away and can have its state checked
88 * again. Without this extra hint LinuxKPI tasks cannot be serialized
89 * across multiple worker threads.
90 */
91 static bool
linux_work_exec_unblock(struct work_struct * work)92 linux_work_exec_unblock(struct work_struct *work)
93 {
94 struct workqueue_struct *wq;
95 struct work_exec *exec;
96 bool retval = false;
97
98 wq = work->work_queue;
99 if (unlikely(wq == NULL))
100 goto done;
101
102 WQ_EXEC_LOCK(wq);
103 TAILQ_FOREACH(exec, &wq->exec_head, entry) {
104 if (exec->target == work) {
105 exec->target = NULL;
106 retval = true;
107 break;
108 }
109 }
110 WQ_EXEC_UNLOCK(wq);
111 done:
112 return (retval);
113 }
114
115 static void
linux_delayed_work_enqueue(struct delayed_work * dwork)116 linux_delayed_work_enqueue(struct delayed_work *dwork)
117 {
118 struct taskqueue *tq;
119
120 tq = dwork->work.work_queue->taskqueue;
121 taskqueue_enqueue(tq, &dwork->work.work_task);
122 }
123
124 /*
125 * This function queues the given work structure on the given
126 * workqueue. It returns non-zero if the work was successfully
127 * [re-]queued. Else the work is already pending for completion.
128 */
129 bool
linux_queue_work_on(int cpu __unused,struct workqueue_struct * wq,struct work_struct * work)130 linux_queue_work_on(int cpu __unused, struct workqueue_struct *wq,
131 struct work_struct *work)
132 {
133 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
134 [WORK_ST_IDLE] = WORK_ST_TASK, /* start queuing task */
135 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
136 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
137 [WORK_ST_EXEC] = WORK_ST_TASK, /* queue task another time */
138 [WORK_ST_CANCEL] = WORK_ST_TASK, /* start queuing task again */
139 };
140
141 if (atomic_read(&wq->draining) != 0)
142 return (!work_pending(work));
143
144 switch (linux_update_state(&work->state, states)) {
145 case WORK_ST_EXEC:
146 case WORK_ST_CANCEL:
147 if (linux_work_exec_unblock(work) != 0)
148 return (true);
149 /* FALLTHROUGH */
150 case WORK_ST_IDLE:
151 work->work_queue = wq;
152 taskqueue_enqueue(wq->taskqueue, &work->work_task);
153 return (true);
154 default:
155 return (false); /* already on a queue */
156 }
157 }
158
159 /*
160 * Callback func for linux_queue_rcu_work
161 */
162 static void
rcu_work_func(struct rcu_head * rcu)163 rcu_work_func(struct rcu_head *rcu)
164 {
165 struct rcu_work *rwork;
166
167 rwork = container_of(rcu, struct rcu_work, rcu);
168 linux_queue_work_on(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
169 }
170
171 /*
172 * This function queue a work after a grace period
173 * If the work was already pending it returns false,
174 * if not it calls call_rcu and returns true.
175 */
176 bool
linux_queue_rcu_work(struct workqueue_struct * wq,struct rcu_work * rwork)177 linux_queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
178 {
179
180 if (!linux_work_pending(&rwork->work)) {
181 rwork->wq = wq;
182 linux_call_rcu(RCU_TYPE_REGULAR, &rwork->rcu, rcu_work_func);
183 return (true);
184 }
185 return (false);
186 }
187
188 /*
189 * This function waits for the last execution of a work and then
190 * flush the work.
191 * It returns true if the work was pending and we waited, it returns
192 * false otherwise.
193 */
194 bool
linux_flush_rcu_work(struct rcu_work * rwork)195 linux_flush_rcu_work(struct rcu_work *rwork)
196 {
197
198 if (linux_work_pending(&rwork->work)) {
199 linux_rcu_barrier(RCU_TYPE_REGULAR);
200 linux_flush_work(&rwork->work);
201 return (true);
202 }
203 return (linux_flush_work(&rwork->work));
204 }
205
206 /*
207 * This function queues the given work structure on the given
208 * workqueue after a given delay in ticks. It returns non-zero if the
209 * work was successfully [re-]queued. Else the work is already pending
210 * for completion.
211 */
212 bool
linux_queue_delayed_work_on(int cpu,struct workqueue_struct * wq,struct delayed_work * dwork,unsigned delay)213 linux_queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
214 struct delayed_work *dwork, unsigned delay)
215 {
216 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
217 [WORK_ST_IDLE] = WORK_ST_TIMER, /* start timeout */
218 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
219 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
220 [WORK_ST_EXEC] = WORK_ST_TIMER, /* start timeout */
221 [WORK_ST_CANCEL] = WORK_ST_TIMER, /* start timeout */
222 };
223
224 if (atomic_read(&wq->draining) != 0)
225 return (!work_pending(&dwork->work));
226
227 switch (linux_update_state(&dwork->work.state, states)) {
228 case WORK_ST_EXEC:
229 case WORK_ST_CANCEL:
230 if (delay == 0 && linux_work_exec_unblock(&dwork->work) != 0) {
231 dwork->timer.expires = jiffies;
232 return (true);
233 }
234 /* FALLTHROUGH */
235 case WORK_ST_IDLE:
236 dwork->work.work_queue = wq;
237 dwork->timer.expires = jiffies + delay;
238
239 if (delay == 0) {
240 linux_delayed_work_enqueue(dwork);
241 } else if (unlikely(cpu != WORK_CPU_UNBOUND)) {
242 mtx_lock(&dwork->timer.mtx);
243 callout_reset_on(&dwork->timer.callout, delay,
244 &linux_delayed_work_timer_fn, dwork, cpu);
245 mtx_unlock(&dwork->timer.mtx);
246 } else {
247 mtx_lock(&dwork->timer.mtx);
248 callout_reset(&dwork->timer.callout, delay,
249 &linux_delayed_work_timer_fn, dwork);
250 mtx_unlock(&dwork->timer.mtx);
251 }
252 return (true);
253 default:
254 return (false); /* already on a queue */
255 }
256 }
257
258 void
linux_work_fn(void * context,int pending)259 linux_work_fn(void *context, int pending)
260 {
261 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
262 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
263 [WORK_ST_TIMER] = WORK_ST_EXEC, /* delayed work w/o timeout */
264 [WORK_ST_TASK] = WORK_ST_EXEC, /* call callback */
265 [WORK_ST_EXEC] = WORK_ST_IDLE, /* complete callback */
266 [WORK_ST_CANCEL] = WORK_ST_EXEC, /* failed to cancel */
267 };
268 struct work_struct *work;
269 struct workqueue_struct *wq;
270 struct work_exec exec;
271 struct task_struct *task;
272
273 task = current;
274
275 /* setup local variables */
276 work = context;
277 wq = work->work_queue;
278
279 /* store target pointer */
280 exec.target = work;
281
282 /* insert executor into list */
283 WQ_EXEC_LOCK(wq);
284 TAILQ_INSERT_TAIL(&wq->exec_head, &exec, entry);
285 while (1) {
286 switch (linux_update_state(&work->state, states)) {
287 case WORK_ST_TIMER:
288 case WORK_ST_TASK:
289 case WORK_ST_CANCEL:
290 WQ_EXEC_UNLOCK(wq);
291
292 /* set current work structure */
293 task->work = work;
294
295 /* call work function */
296 work->func(work);
297
298 /* set current work structure */
299 task->work = NULL;
300
301 WQ_EXEC_LOCK(wq);
302 /* check if unblocked */
303 if (exec.target != work) {
304 /* reapply block */
305 exec.target = work;
306 break;
307 }
308 /* FALLTHROUGH */
309 default:
310 goto done;
311 }
312 }
313 done:
314 /* remove executor from list */
315 TAILQ_REMOVE(&wq->exec_head, &exec, entry);
316 WQ_EXEC_UNLOCK(wq);
317 }
318
319 void
linux_delayed_work_fn(void * context,int pending)320 linux_delayed_work_fn(void *context, int pending)
321 {
322 struct delayed_work *dwork = context;
323
324 /*
325 * Make sure the timer belonging to the delayed work gets
326 * drained before invoking the work function. Else the timer
327 * mutex may still be in use which can lead to use-after-free
328 * situations, because the work function might free the work
329 * structure before returning.
330 */
331 callout_drain(&dwork->timer.callout);
332
333 linux_work_fn(&dwork->work, pending);
334 }
335
336 static void
linux_delayed_work_timer_fn(void * arg)337 linux_delayed_work_timer_fn(void *arg)
338 {
339 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
340 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
341 [WORK_ST_TIMER] = WORK_ST_TASK, /* start queueing task */
342 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
343 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
344 [WORK_ST_CANCEL] = WORK_ST_TASK, /* failed to cancel */
345 };
346 struct delayed_work *dwork = arg;
347
348 switch (linux_update_state(&dwork->work.state, states)) {
349 case WORK_ST_TIMER:
350 case WORK_ST_CANCEL:
351 linux_delayed_work_enqueue(dwork);
352 break;
353 default:
354 break;
355 }
356 }
357
358 /*
359 * This function cancels the given work structure in a synchronous
360 * fashion. It returns non-zero if the work was successfully
361 * cancelled. Else the work was already cancelled.
362 */
363 bool
linux_cancel_work_sync(struct work_struct * work)364 linux_cancel_work_sync(struct work_struct *work)
365 {
366 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
367 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
368 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */
369 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
370 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
371 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
372 };
373 struct taskqueue *tq;
374 bool retval = false;
375
376 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
377 "linux_cancel_work_sync() might sleep");
378 retry:
379 switch (linux_update_state(&work->state, states)) {
380 case WORK_ST_IDLE:
381 case WORK_ST_TIMER:
382 return (retval);
383 case WORK_ST_EXEC:
384 tq = work->work_queue->taskqueue;
385 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
386 taskqueue_drain(tq, &work->work_task);
387 goto retry; /* work may have restarted itself */
388 default:
389 tq = work->work_queue->taskqueue;
390 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
391 taskqueue_drain(tq, &work->work_task);
392 retval = true;
393 goto retry;
394 }
395 }
396
397 /*
398 * This function atomically stops the timer and callback. The timer
399 * callback will not be called after this function returns. This
400 * functions returns true when the timeout was cancelled. Else the
401 * timeout was not started or has already been called.
402 */
403 static inline bool
linux_cancel_timer(struct delayed_work * dwork,bool drain)404 linux_cancel_timer(struct delayed_work *dwork, bool drain)
405 {
406 bool cancelled;
407
408 mtx_lock(&dwork->timer.mtx);
409 cancelled = (callout_stop(&dwork->timer.callout) == 1);
410 mtx_unlock(&dwork->timer.mtx);
411
412 /* check if we should drain */
413 if (drain)
414 callout_drain(&dwork->timer.callout);
415 return (cancelled);
416 }
417
418 /*
419 * This function cancels the given delayed work structure in a
420 * non-blocking fashion. It returns non-zero if the work was
421 * successfully cancelled. Else the work may still be busy or already
422 * cancelled.
423 */
424 bool
linux_cancel_delayed_work(struct delayed_work * dwork)425 linux_cancel_delayed_work(struct delayed_work *dwork)
426 {
427 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
428 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
429 [WORK_ST_TIMER] = WORK_ST_CANCEL, /* try to cancel */
430 [WORK_ST_TASK] = WORK_ST_CANCEL, /* try to cancel */
431 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
432 [WORK_ST_CANCEL] = WORK_ST_CANCEL, /* NOP */
433 };
434 struct taskqueue *tq;
435
436 switch (linux_update_state(&dwork->work.state, states)) {
437 case WORK_ST_TIMER:
438 case WORK_ST_CANCEL:
439 if (linux_cancel_timer(dwork, 0)) {
440 atomic_cmpxchg(&dwork->work.state,
441 WORK_ST_CANCEL, WORK_ST_IDLE);
442 return (true);
443 }
444 /* FALLTHROUGH */
445 case WORK_ST_TASK:
446 tq = dwork->work.work_queue->taskqueue;
447 if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) == 0) {
448 atomic_cmpxchg(&dwork->work.state,
449 WORK_ST_CANCEL, WORK_ST_IDLE);
450 return (true);
451 }
452 /* FALLTHROUGH */
453 default:
454 return (false);
455 }
456 }
457
458 /*
459 * This function cancels the given work structure in a synchronous
460 * fashion. It returns non-zero if the work was successfully
461 * cancelled. Else the work was already cancelled.
462 */
463 bool
linux_cancel_delayed_work_sync(struct delayed_work * dwork)464 linux_cancel_delayed_work_sync(struct delayed_work *dwork)
465 {
466 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
467 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
468 [WORK_ST_TIMER] = WORK_ST_IDLE, /* cancel and drain */
469 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
470 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
471 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
472 };
473 struct taskqueue *tq;
474 bool retval = false;
475
476 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
477 "linux_cancel_delayed_work_sync() might sleep");
478 retry:
479 switch (linux_update_state(&dwork->work.state, states)) {
480 case WORK_ST_IDLE:
481 return (retval);
482 case WORK_ST_EXEC:
483 tq = dwork->work.work_queue->taskqueue;
484 if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) != 0)
485 taskqueue_drain(tq, &dwork->work.work_task);
486 goto retry; /* work may have restarted itself */
487 case WORK_ST_TIMER:
488 case WORK_ST_CANCEL:
489 if (linux_cancel_timer(dwork, 1)) {
490 /*
491 * Make sure taskqueue is also drained before
492 * returning:
493 */
494 tq = dwork->work.work_queue->taskqueue;
495 taskqueue_drain(tq, &dwork->work.work_task);
496 retval = true;
497 goto retry;
498 }
499 /* FALLTHROUGH */
500 default:
501 tq = dwork->work.work_queue->taskqueue;
502 if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) != 0)
503 taskqueue_drain(tq, &dwork->work.work_task);
504 retval = true;
505 goto retry;
506 }
507 }
508
509 /*
510 * This function waits until the given work structure is completed.
511 * It returns non-zero if the work was successfully
512 * waited for. Else the work was not waited for.
513 */
514 bool
linux_flush_work(struct work_struct * work)515 linux_flush_work(struct work_struct *work)
516 {
517 struct taskqueue *tq;
518 bool retval;
519
520 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
521 "linux_flush_work() might sleep");
522
523 switch (atomic_read(&work->state)) {
524 case WORK_ST_IDLE:
525 return (false);
526 default:
527 tq = work->work_queue->taskqueue;
528 retval = taskqueue_poll_is_busy(tq, &work->work_task);
529 taskqueue_drain(tq, &work->work_task);
530 return (retval);
531 }
532 }
533
534 /*
535 * This function waits until the given delayed work structure is
536 * completed. It returns non-zero if the work was successfully waited
537 * for. Else the work was not waited for.
538 */
539 bool
linux_flush_delayed_work(struct delayed_work * dwork)540 linux_flush_delayed_work(struct delayed_work *dwork)
541 {
542 struct taskqueue *tq;
543 bool retval;
544
545 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
546 "linux_flush_delayed_work() might sleep");
547
548 switch (atomic_read(&dwork->work.state)) {
549 case WORK_ST_IDLE:
550 return (false);
551 case WORK_ST_TIMER:
552 if (linux_cancel_timer(dwork, 1))
553 linux_delayed_work_enqueue(dwork);
554 /* FALLTHROUGH */
555 default:
556 tq = dwork->work.work_queue->taskqueue;
557 retval = taskqueue_poll_is_busy(tq, &dwork->work.work_task);
558 taskqueue_drain(tq, &dwork->work.work_task);
559 return (retval);
560 }
561 }
562
563 /*
564 * This function returns true if the given work is pending, and not
565 * yet executing:
566 */
567 bool
linux_work_pending(struct work_struct * work)568 linux_work_pending(struct work_struct *work)
569 {
570 switch (atomic_read(&work->state)) {
571 case WORK_ST_TIMER:
572 case WORK_ST_TASK:
573 case WORK_ST_CANCEL:
574 return (true);
575 default:
576 return (false);
577 }
578 }
579
580 /*
581 * This function returns true if the given work is busy.
582 */
583 bool
linux_work_busy(struct work_struct * work)584 linux_work_busy(struct work_struct *work)
585 {
586 struct taskqueue *tq;
587
588 switch (atomic_read(&work->state)) {
589 case WORK_ST_IDLE:
590 return (false);
591 case WORK_ST_EXEC:
592 tq = work->work_queue->taskqueue;
593 return (taskqueue_poll_is_busy(tq, &work->work_task));
594 default:
595 return (true);
596 }
597 }
598
599 struct workqueue_struct *
linux_create_workqueue_common(const char * name,int cpus)600 linux_create_workqueue_common(const char *name, int cpus)
601 {
602 struct workqueue_struct *wq;
603
604 /*
605 * If zero CPUs are specified use the default number of CPUs:
606 */
607 if (cpus == 0)
608 cpus = linux_default_wq_cpus;
609
610 wq = kmalloc(sizeof(*wq), M_WAITOK | M_ZERO);
611 wq->taskqueue = taskqueue_create(name, M_WAITOK,
612 taskqueue_thread_enqueue, &wq->taskqueue);
613 atomic_set(&wq->draining, 0);
614 taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name);
615 TAILQ_INIT(&wq->exec_head);
616 mtx_init(&wq->exec_mtx, "linux_wq_exec", NULL, MTX_DEF);
617
618 return (wq);
619 }
620
621 void
linux_destroy_workqueue(struct workqueue_struct * wq)622 linux_destroy_workqueue(struct workqueue_struct *wq)
623 {
624 atomic_inc(&wq->draining);
625 drain_workqueue(wq);
626 taskqueue_free(wq->taskqueue);
627 mtx_destroy(&wq->exec_mtx);
628 kfree(wq);
629 }
630
631 void
linux_init_delayed_work(struct delayed_work * dwork,work_func_t func)632 linux_init_delayed_work(struct delayed_work *dwork, work_func_t func)
633 {
634 memset(dwork, 0, sizeof(*dwork));
635 dwork->work.func = func;
636 TASK_INIT(&dwork->work.work_task, 0, linux_delayed_work_fn, dwork);
637 mtx_init(&dwork->timer.mtx, spin_lock_name("lkpi-dwork"), NULL,
638 MTX_DEF | MTX_NOWITNESS);
639 callout_init_mtx(&dwork->timer.callout, &dwork->timer.mtx, 0);
640 }
641
642 struct work_struct *
linux_current_work(void)643 linux_current_work(void)
644 {
645 return (current->work);
646 }
647
648 static void
linux_work_init(void * arg)649 linux_work_init(void *arg)
650 {
651 int max_wq_cpus = mp_ncpus + 1;
652
653 /* avoid deadlock when there are too few threads */
654 if (max_wq_cpus < 4)
655 max_wq_cpus = 4;
656
657 /* set default number of CPUs */
658 linux_default_wq_cpus = max_wq_cpus;
659
660 linux_system_short_wq = alloc_workqueue("linuxkpi_short_wq", 0, max_wq_cpus);
661 linux_system_long_wq = alloc_workqueue("linuxkpi_long_wq", 0, max_wq_cpus);
662
663 /* populate the workqueue pointers */
664 system_long_wq = linux_system_long_wq;
665 system_wq = linux_system_short_wq;
666 system_power_efficient_wq = linux_system_short_wq;
667 system_unbound_wq = linux_system_short_wq;
668 system_highpri_wq = linux_system_short_wq;
669 }
670 SYSINIT(linux_work_init, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_init, NULL);
671
672 static void
linux_work_uninit(void * arg)673 linux_work_uninit(void *arg)
674 {
675 destroy_workqueue(linux_system_short_wq);
676 destroy_workqueue(linux_system_long_wq);
677
678 /* clear workqueue pointers */
679 system_long_wq = NULL;
680 system_wq = NULL;
681 system_power_efficient_wq = NULL;
682 system_unbound_wq = NULL;
683 system_highpri_wq = NULL;
684 }
685 SYSUNINIT(linux_work_uninit, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_uninit, NULL);
686