1 /*-
2 * Copyright (c) 2017-2019 Hans Petter Selasky
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 #include <linux/workqueue.h>
29 #include <linux/wait.h>
30 #include <linux/compat.h>
31 #include <linux/spinlock.h>
32 #include <linux/rcupdate.h>
33 #include <linux/irq_work.h>
34
35 #include <sys/kernel.h>
36
37 /*
38 * Define all work struct states
39 */
40 enum {
41 WORK_ST_IDLE, /* idle - not started */
42 WORK_ST_TIMER, /* timer is being started */
43 WORK_ST_TASK, /* taskqueue is being queued */
44 WORK_ST_EXEC, /* callback is being called */
45 WORK_ST_CANCEL, /* cancel is being requested */
46 WORK_ST_MAX,
47 };
48
49 /*
50 * Define global workqueues
51 */
52 static struct workqueue_struct *linux_system_short_wq;
53 static struct workqueue_struct *linux_system_long_wq;
54
55 struct workqueue_struct *system_wq;
56 struct workqueue_struct *system_long_wq;
57 struct workqueue_struct *system_unbound_wq;
58 struct workqueue_struct *system_highpri_wq;
59 struct workqueue_struct *system_power_efficient_wq;
60
61 struct taskqueue *linux_irq_work_tq;
62
63 static int linux_default_wq_cpus = 4;
64
65 static void linux_delayed_work_timer_fn(void *);
66
67 /*
68 * This function atomically updates the work state and returns the
69 * previous state at the time of update.
70 */
71 static uint8_t
linux_update_state(atomic_t * v,const uint8_t * pstate)72 linux_update_state(atomic_t *v, const uint8_t *pstate)
73 {
74 int c, old;
75
76 c = v->counter;
77
78 while ((old = atomic_cmpxchg(v, c, pstate[c])) != c)
79 c = old;
80
81 return (c);
82 }
83
84 /*
85 * A LinuxKPI task is allowed to free itself inside the callback function
86 * and cannot safely be referred after the callback function has
87 * completed. This function gives the linux_work_fn() function a hint,
88 * that the task is not going away and can have its state checked
89 * again. Without this extra hint LinuxKPI tasks cannot be serialized
90 * across multiple worker threads.
91 */
92 static bool
linux_work_exec_unblock(struct work_struct * work)93 linux_work_exec_unblock(struct work_struct *work)
94 {
95 struct workqueue_struct *wq;
96 struct work_exec *exec;
97 bool retval = false;
98
99 wq = work->work_queue;
100 if (unlikely(wq == NULL))
101 goto done;
102
103 WQ_EXEC_LOCK(wq);
104 TAILQ_FOREACH(exec, &wq->exec_head, entry) {
105 if (exec->target == work) {
106 exec->target = NULL;
107 retval = true;
108 break;
109 }
110 }
111 WQ_EXEC_UNLOCK(wq);
112 done:
113 return (retval);
114 }
115
116 static void
linux_delayed_work_enqueue(struct delayed_work * dwork)117 linux_delayed_work_enqueue(struct delayed_work *dwork)
118 {
119 struct taskqueue *tq;
120
121 tq = dwork->work.work_queue->taskqueue;
122 taskqueue_enqueue(tq, &dwork->work.work_task);
123 }
124
125 /*
126 * This function queues the given work structure on the given
127 * workqueue. It returns non-zero if the work was successfully
128 * [re-]queued. Else the work is already pending for completion.
129 */
130 bool
linux_queue_work_on(int cpu __unused,struct workqueue_struct * wq,struct work_struct * work)131 linux_queue_work_on(int cpu __unused, struct workqueue_struct *wq,
132 struct work_struct *work)
133 {
134 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
135 [WORK_ST_IDLE] = WORK_ST_TASK, /* start queuing task */
136 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
137 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
138 [WORK_ST_EXEC] = WORK_ST_TASK, /* queue task another time */
139 [WORK_ST_CANCEL] = WORK_ST_TASK, /* start queuing task again */
140 };
141
142 if (atomic_read(&wq->draining) != 0)
143 return (!work_pending(work));
144
145 switch (linux_update_state(&work->state, states)) {
146 case WORK_ST_EXEC:
147 case WORK_ST_CANCEL:
148 if (linux_work_exec_unblock(work) != 0)
149 return (true);
150 /* FALLTHROUGH */
151 case WORK_ST_IDLE:
152 work->work_queue = wq;
153 taskqueue_enqueue(wq->taskqueue, &work->work_task);
154 return (true);
155 default:
156 return (false); /* already on a queue */
157 }
158 }
159
160 /*
161 * Callback func for linux_queue_rcu_work
162 */
163 static void
rcu_work_func(struct rcu_head * rcu)164 rcu_work_func(struct rcu_head *rcu)
165 {
166 struct rcu_work *rwork;
167
168 rwork = container_of(rcu, struct rcu_work, rcu);
169 linux_queue_work_on(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
170 }
171
172 /*
173 * This function queue a work after a grace period
174 * If the work was already pending it returns false,
175 * if not it calls call_rcu and returns true.
176 */
177 bool
linux_queue_rcu_work(struct workqueue_struct * wq,struct rcu_work * rwork)178 linux_queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
179 {
180
181 if (!linux_work_pending(&rwork->work)) {
182 rwork->wq = wq;
183 linux_call_rcu(RCU_TYPE_REGULAR, &rwork->rcu, rcu_work_func);
184 return (true);
185 }
186 return (false);
187 }
188
189 /*
190 * This function waits for the last execution of a work and then
191 * flush the work.
192 * It returns true if the work was pending and we waited, it returns
193 * false otherwise.
194 */
195 bool
linux_flush_rcu_work(struct rcu_work * rwork)196 linux_flush_rcu_work(struct rcu_work *rwork)
197 {
198
199 if (linux_work_pending(&rwork->work)) {
200 linux_rcu_barrier(RCU_TYPE_REGULAR);
201 linux_flush_work(&rwork->work);
202 return (true);
203 }
204 return (linux_flush_work(&rwork->work));
205 }
206
207 /*
208 * This function queues the given work structure on the given
209 * workqueue after a given delay in ticks. It returns true if the
210 * work was successfully [re-]queued. Else the work is already pending
211 * for completion.
212 */
213 bool
linux_queue_delayed_work_on(int cpu,struct workqueue_struct * wq,struct delayed_work * dwork,unsigned delay)214 linux_queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
215 struct delayed_work *dwork, unsigned delay)
216 {
217 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
218 [WORK_ST_IDLE] = WORK_ST_TIMER, /* start timeout */
219 [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */
220 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
221 [WORK_ST_EXEC] = WORK_ST_TIMER, /* start timeout */
222 [WORK_ST_CANCEL] = WORK_ST_TIMER, /* start timeout */
223 };
224 bool res;
225
226 if (atomic_read(&wq->draining) != 0)
227 return (!work_pending(&dwork->work));
228
229 mtx_lock(&dwork->timer.mtx);
230 switch (linux_update_state(&dwork->work.state, states)) {
231 case WORK_ST_EXEC:
232 case WORK_ST_CANCEL:
233 if (delay == 0 && linux_work_exec_unblock(&dwork->work)) {
234 dwork->timer.expires = jiffies;
235 res = true;
236 goto out;
237 }
238 /* FALLTHROUGH */
239 case WORK_ST_IDLE:
240 dwork->work.work_queue = wq;
241 dwork->timer.expires = jiffies + delay;
242
243 if (delay == 0) {
244 linux_delayed_work_enqueue(dwork);
245 } else if (unlikely(cpu != WORK_CPU_UNBOUND)) {
246 callout_reset_on(&dwork->timer.callout, delay,
247 &linux_delayed_work_timer_fn, dwork, cpu);
248 } else {
249 callout_reset(&dwork->timer.callout, delay,
250 &linux_delayed_work_timer_fn, dwork);
251 }
252 res = true;
253 break;
254 default:
255 res = false;
256 break;
257 }
258 out:
259 mtx_unlock(&dwork->timer.mtx);
260 return (res);
261 }
262
263 void
linux_work_fn(void * context,int pending)264 linux_work_fn(void *context, int pending)
265 {
266 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
267 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
268 [WORK_ST_TIMER] = WORK_ST_EXEC, /* delayed work w/o timeout */
269 [WORK_ST_TASK] = WORK_ST_EXEC, /* call callback */
270 [WORK_ST_EXEC] = WORK_ST_IDLE, /* complete callback */
271 [WORK_ST_CANCEL] = WORK_ST_EXEC, /* failed to cancel */
272 };
273 struct work_struct *work;
274 struct workqueue_struct *wq;
275 struct work_exec exec;
276 struct task_struct *task;
277
278 task = current;
279
280 /* setup local variables */
281 work = context;
282 wq = work->work_queue;
283
284 /* store target pointer */
285 exec.target = work;
286
287 /* insert executor into list */
288 WQ_EXEC_LOCK(wq);
289 TAILQ_INSERT_TAIL(&wq->exec_head, &exec, entry);
290 while (1) {
291 switch (linux_update_state(&work->state, states)) {
292 case WORK_ST_TIMER:
293 case WORK_ST_TASK:
294 case WORK_ST_CANCEL:
295 WQ_EXEC_UNLOCK(wq);
296
297 /* set current work structure */
298 task->work = work;
299
300 /* call work function */
301 work->func(work);
302
303 /* set current work structure */
304 task->work = NULL;
305
306 WQ_EXEC_LOCK(wq);
307 /* check if unblocked */
308 if (exec.target != work) {
309 /* reapply block */
310 exec.target = work;
311 break;
312 }
313 /* FALLTHROUGH */
314 default:
315 goto done;
316 }
317 }
318 done:
319 /* remove executor from list */
320 TAILQ_REMOVE(&wq->exec_head, &exec, entry);
321 WQ_EXEC_UNLOCK(wq);
322 }
323
324 void
linux_delayed_work_fn(void * context,int pending)325 linux_delayed_work_fn(void *context, int pending)
326 {
327 struct delayed_work *dwork = context;
328
329 /*
330 * Make sure the timer belonging to the delayed work gets
331 * drained before invoking the work function. Else the timer
332 * mutex may still be in use which can lead to use-after-free
333 * situations, because the work function might free the work
334 * structure before returning.
335 */
336 callout_drain(&dwork->timer.callout);
337
338 linux_work_fn(&dwork->work, pending);
339 }
340
341 static void
linux_delayed_work_timer_fn(void * arg)342 linux_delayed_work_timer_fn(void *arg)
343 {
344 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
345 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
346 [WORK_ST_TIMER] = WORK_ST_TASK, /* start queueing task */
347 [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */
348 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
349 [WORK_ST_CANCEL] = WORK_ST_TASK, /* failed to cancel */
350 };
351 struct delayed_work *dwork = arg;
352
353 switch (linux_update_state(&dwork->work.state, states)) {
354 case WORK_ST_TIMER:
355 case WORK_ST_CANCEL:
356 linux_delayed_work_enqueue(dwork);
357 break;
358 default:
359 break;
360 }
361 }
362
363 /*
364 * This function cancels the given work structure in a synchronous
365 * fashion. It returns non-zero if the work was successfully
366 * cancelled. Else the work was already cancelled.
367 */
368 bool
linux_cancel_work_sync(struct work_struct * work)369 linux_cancel_work_sync(struct work_struct *work)
370 {
371 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
372 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
373 [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */
374 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
375 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
376 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
377 };
378 struct taskqueue *tq;
379 bool retval = false;
380
381 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
382 "linux_cancel_work_sync() might sleep");
383 retry:
384 switch (linux_update_state(&work->state, states)) {
385 case WORK_ST_IDLE:
386 case WORK_ST_TIMER:
387 return (retval);
388 case WORK_ST_EXEC:
389 tq = work->work_queue->taskqueue;
390 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
391 taskqueue_drain(tq, &work->work_task);
392 goto retry; /* work may have restarted itself */
393 default:
394 tq = work->work_queue->taskqueue;
395 if (taskqueue_cancel(tq, &work->work_task, NULL) != 0)
396 taskqueue_drain(tq, &work->work_task);
397 retval = true;
398 goto retry;
399 }
400 }
401
402 /*
403 * This function atomically stops the timer and callback. The timer
404 * callback will not be called after this function returns. This
405 * functions returns true when the timeout was cancelled. Else the
406 * timeout was not started or has already been called.
407 */
408 static inline bool
linux_cancel_timer(struct delayed_work * dwork,bool drain)409 linux_cancel_timer(struct delayed_work *dwork, bool drain)
410 {
411 bool cancelled;
412
413 mtx_lock(&dwork->timer.mtx);
414 cancelled = (callout_stop(&dwork->timer.callout) == 1);
415 mtx_unlock(&dwork->timer.mtx);
416
417 /* check if we should drain */
418 if (drain)
419 callout_drain(&dwork->timer.callout);
420 return (cancelled);
421 }
422
423 /*
424 * This function cancels the given delayed work structure in a
425 * non-blocking fashion. It returns non-zero if the work was
426 * successfully cancelled. Else the work may still be busy or already
427 * cancelled.
428 */
429 bool
linux_cancel_delayed_work(struct delayed_work * dwork)430 linux_cancel_delayed_work(struct delayed_work *dwork)
431 {
432 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
433 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
434 [WORK_ST_TIMER] = WORK_ST_CANCEL, /* try to cancel */
435 [WORK_ST_TASK] = WORK_ST_CANCEL, /* try to cancel */
436 [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */
437 [WORK_ST_CANCEL] = WORK_ST_CANCEL, /* NOP */
438 };
439 struct taskqueue *tq;
440 bool cancelled;
441
442 mtx_lock(&dwork->timer.mtx);
443 switch (linux_update_state(&dwork->work.state, states)) {
444 case WORK_ST_TIMER:
445 case WORK_ST_CANCEL:
446 cancelled = (callout_stop(&dwork->timer.callout) == 1);
447 if (cancelled) {
448 atomic_cmpxchg(&dwork->work.state,
449 WORK_ST_CANCEL, WORK_ST_IDLE);
450 mtx_unlock(&dwork->timer.mtx);
451 return (true);
452 }
453 /* FALLTHROUGH */
454 case WORK_ST_TASK:
455 tq = dwork->work.work_queue->taskqueue;
456 if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) == 0) {
457 atomic_cmpxchg(&dwork->work.state,
458 WORK_ST_CANCEL, WORK_ST_IDLE);
459 mtx_unlock(&dwork->timer.mtx);
460 return (true);
461 }
462 /* FALLTHROUGH */
463 default:
464 mtx_unlock(&dwork->timer.mtx);
465 return (false);
466 }
467 }
468
469 /*
470 * This function cancels the given work structure in a synchronous
471 * fashion. It returns true if the work was successfully
472 * cancelled. Else the work was already cancelled.
473 */
474 static bool
linux_cancel_delayed_work_sync_int(struct delayed_work * dwork)475 linux_cancel_delayed_work_sync_int(struct delayed_work *dwork)
476 {
477 static const uint8_t states[WORK_ST_MAX] __aligned(8) = {
478 [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */
479 [WORK_ST_TIMER] = WORK_ST_IDLE, /* cancel and drain */
480 [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */
481 [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */
482 [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */
483 };
484 struct taskqueue *tq;
485 int ret, state;
486 bool cancelled;
487
488 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
489 "linux_cancel_delayed_work_sync() might sleep");
490 mtx_lock(&dwork->timer.mtx);
491
492 state = linux_update_state(&dwork->work.state, states);
493 switch (state) {
494 case WORK_ST_IDLE:
495 mtx_unlock(&dwork->timer.mtx);
496 return (false);
497 case WORK_ST_TIMER:
498 case WORK_ST_CANCEL:
499 cancelled = (callout_stop(&dwork->timer.callout) == 1);
500
501 tq = dwork->work.work_queue->taskqueue;
502 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL);
503 mtx_unlock(&dwork->timer.mtx);
504
505 callout_drain(&dwork->timer.callout);
506 taskqueue_drain(tq, &dwork->work.work_task);
507 return (cancelled || (ret != 0));
508 default:
509 tq = dwork->work.work_queue->taskqueue;
510 ret = taskqueue_cancel(tq, &dwork->work.work_task, NULL);
511 mtx_unlock(&dwork->timer.mtx);
512 if (ret != 0)
513 taskqueue_drain(tq, &dwork->work.work_task);
514 return (ret != 0);
515 }
516 }
517
518 bool
linux_cancel_delayed_work_sync(struct delayed_work * dwork)519 linux_cancel_delayed_work_sync(struct delayed_work *dwork)
520 {
521 bool res;
522
523 res = false;
524 while (linux_cancel_delayed_work_sync_int(dwork))
525 res = true;
526 return (res);
527 }
528
529 /*
530 * This function waits until the given work structure is completed.
531 * It returns non-zero if the work was successfully
532 * waited for. Else the work was not waited for.
533 */
534 bool
linux_flush_work(struct work_struct * work)535 linux_flush_work(struct work_struct *work)
536 {
537 struct taskqueue *tq;
538 bool retval;
539
540 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
541 "linux_flush_work() might sleep");
542
543 switch (atomic_read(&work->state)) {
544 case WORK_ST_IDLE:
545 return (false);
546 default:
547 tq = work->work_queue->taskqueue;
548 retval = taskqueue_poll_is_busy(tq, &work->work_task);
549 taskqueue_drain(tq, &work->work_task);
550 return (retval);
551 }
552 }
553
554 /*
555 * This function waits until the given delayed work structure is
556 * completed. It returns non-zero if the work was successfully waited
557 * for. Else the work was not waited for.
558 */
559 bool
linux_flush_delayed_work(struct delayed_work * dwork)560 linux_flush_delayed_work(struct delayed_work *dwork)
561 {
562 struct taskqueue *tq;
563 bool retval;
564
565 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
566 "linux_flush_delayed_work() might sleep");
567
568 switch (atomic_read(&dwork->work.state)) {
569 case WORK_ST_IDLE:
570 return (false);
571 case WORK_ST_TIMER:
572 if (linux_cancel_timer(dwork, 1))
573 linux_delayed_work_enqueue(dwork);
574 /* FALLTHROUGH */
575 default:
576 tq = dwork->work.work_queue->taskqueue;
577 retval = taskqueue_poll_is_busy(tq, &dwork->work.work_task);
578 taskqueue_drain(tq, &dwork->work.work_task);
579 return (retval);
580 }
581 }
582
583 /*
584 * This function returns true if the given work is pending, and not
585 * yet executing:
586 */
587 bool
linux_work_pending(struct work_struct * work)588 linux_work_pending(struct work_struct *work)
589 {
590 switch (atomic_read(&work->state)) {
591 case WORK_ST_TIMER:
592 case WORK_ST_TASK:
593 case WORK_ST_CANCEL:
594 return (true);
595 default:
596 return (false);
597 }
598 }
599
600 /*
601 * This function returns true if the given work is busy.
602 */
603 bool
linux_work_busy(struct work_struct * work)604 linux_work_busy(struct work_struct *work)
605 {
606 struct taskqueue *tq;
607
608 switch (atomic_read(&work->state)) {
609 case WORK_ST_IDLE:
610 return (false);
611 case WORK_ST_EXEC:
612 tq = work->work_queue->taskqueue;
613 return (taskqueue_poll_is_busy(tq, &work->work_task));
614 default:
615 return (true);
616 }
617 }
618
619 struct workqueue_struct *
linux_create_workqueue_common(const char * name,int cpus)620 linux_create_workqueue_common(const char *name, int cpus)
621 {
622 struct workqueue_struct *wq;
623
624 /*
625 * If zero CPUs are specified use the default number of CPUs:
626 */
627 if (cpus == 0)
628 cpus = linux_default_wq_cpus;
629
630 wq = kmalloc(sizeof(*wq), M_WAITOK | M_ZERO);
631 wq->taskqueue = taskqueue_create(name, M_WAITOK,
632 taskqueue_thread_enqueue, &wq->taskqueue);
633 atomic_set(&wq->draining, 0);
634 taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name);
635 TAILQ_INIT(&wq->exec_head);
636 mtx_init(&wq->exec_mtx, "linux_wq_exec", NULL, MTX_DEF);
637
638 return (wq);
639 }
640
641 void
linux_destroy_workqueue(struct workqueue_struct * wq)642 linux_destroy_workqueue(struct workqueue_struct *wq)
643 {
644 atomic_inc(&wq->draining);
645 drain_workqueue(wq);
646 taskqueue_free(wq->taskqueue);
647 mtx_destroy(&wq->exec_mtx);
648 kfree(wq);
649 }
650
651 void
linux_init_delayed_work(struct delayed_work * dwork,work_func_t func)652 linux_init_delayed_work(struct delayed_work *dwork, work_func_t func)
653 {
654 memset(dwork, 0, sizeof(*dwork));
655 dwork->work.func = func;
656 TASK_INIT(&dwork->work.work_task, 0, linux_delayed_work_fn, dwork);
657 mtx_init(&dwork->timer.mtx, spin_lock_name("lkpi-dwork"), NULL,
658 MTX_DEF | MTX_NOWITNESS);
659 callout_init_mtx(&dwork->timer.callout, &dwork->timer.mtx, 0);
660 }
661
662 struct work_struct *
linux_current_work(void)663 linux_current_work(void)
664 {
665 return (current->work);
666 }
667
668 static void
linux_work_init(void * arg)669 linux_work_init(void *arg)
670 {
671 int max_wq_cpus = mp_ncpus + 1;
672
673 /* avoid deadlock when there are too few threads */
674 if (max_wq_cpus < 4)
675 max_wq_cpus = 4;
676
677 /* set default number of CPUs */
678 linux_default_wq_cpus = max_wq_cpus;
679
680 linux_system_short_wq = alloc_workqueue("linuxkpi_short_wq", 0, max_wq_cpus);
681 linux_system_long_wq = alloc_workqueue("linuxkpi_long_wq", 0, max_wq_cpus);
682
683 /* populate the workqueue pointers */
684 system_long_wq = linux_system_long_wq;
685 system_wq = linux_system_short_wq;
686 system_power_efficient_wq = linux_system_short_wq;
687 system_unbound_wq = linux_system_short_wq;
688 system_highpri_wq = linux_system_short_wq;
689 }
690 SYSINIT(linux_work_init, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_init, NULL);
691
692 static void
linux_work_uninit(void * arg)693 linux_work_uninit(void *arg)
694 {
695 destroy_workqueue(linux_system_short_wq);
696 destroy_workqueue(linux_system_long_wq);
697
698 /* clear workqueue pointers */
699 system_long_wq = NULL;
700 system_wq = NULL;
701 system_power_efficient_wq = NULL;
702 system_unbound_wq = NULL;
703 system_highpri_wq = NULL;
704 }
705 SYSUNINIT(linux_work_uninit, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_uninit, NULL);
706
707 void
linux_irq_work_fn(void * context,int pending)708 linux_irq_work_fn(void *context, int pending)
709 {
710 struct irq_work *irqw = context;
711
712 irqw->func(irqw);
713 }
714
715 static void
linux_irq_work_init_fn(void * context,int pending)716 linux_irq_work_init_fn(void *context, int pending)
717 {
718 /*
719 * LinuxKPI performs lazy allocation of memory structures required by
720 * current on the first access to it. As some irq_work clients read
721 * it with spinlock taken, we have to preallocate td_lkpi_task before
722 * first call to irq_work_queue(). As irq_work uses a single thread,
723 * it is enough to read current once at SYSINIT stage.
724 */
725 if (current == NULL)
726 panic("irq_work taskqueue is not initialized");
727 }
728 static struct task linux_irq_work_init_task =
729 TASK_INITIALIZER(0, linux_irq_work_init_fn, &linux_irq_work_init_task);
730
731 static void
linux_irq_work_init(void * arg)732 linux_irq_work_init(void *arg)
733 {
734 linux_irq_work_tq = taskqueue_create_fast("linuxkpi_irq_wq",
735 M_WAITOK, taskqueue_thread_enqueue, &linux_irq_work_tq);
736 taskqueue_start_threads(&linux_irq_work_tq, 1, PWAIT,
737 "linuxkpi_irq_wq");
738 taskqueue_enqueue(linux_irq_work_tq, &linux_irq_work_init_task);
739 }
740 SYSINIT(linux_irq_work_init, SI_SUB_TASKQ, SI_ORDER_SECOND,
741 linux_irq_work_init, NULL);
742
743 static void
linux_irq_work_uninit(void * arg)744 linux_irq_work_uninit(void *arg)
745 {
746 taskqueue_drain_all(linux_irq_work_tq);
747 taskqueue_free(linux_irq_work_tq);
748 }
749 SYSUNINIT(linux_irq_work_uninit, SI_SUB_TASKQ, SI_ORDER_SECOND,
750 linux_irq_work_uninit, NULL);
751