1 /*
2 * Copyright (c) 2020 François Tigeot <ftigeot@wolfpond.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <linux/interrupt.h>
28 #include <linux/slab.h>
29
30 #include <sys/kthread.h>
31
32 /*
33 * Linux tasklet constraints:
34 * - tasklets that have the same type cannot be run on multiple processors at
35 * the same time
36 * - tasklets always run on the processor from which they were originally
37 * submitted
38 * - when a tasklet is scheduled, its state is set to TASKLET_STATE_SCHED,
39 * and the tasklet added to a queue
40 * - during the execution of its function, the tasklet state is set to
41 * TASKLET_STATE_RUN and the TASKLET_STATE_SCHED state is removed
42 */
43
44 struct tasklet_entry {
45 struct tasklet_struct *ts;
46 STAILQ_ENTRY(tasklet_entry) tasklet_entries;
47 };
48
49 static struct lock tasklet_lock = LOCK_INITIALIZER("dltll", 0, LK_CANRECURSE);
50
51 static struct thread *tasklet_td = NULL;
52 STAILQ_HEAD(tasklet_list_head, tasklet_entry) tlist = STAILQ_HEAD_INITIALIZER(tlist);
53 STAILQ_HEAD(tasklet_hi_list_head, tasklet_entry) tlist_hi = STAILQ_HEAD_INITIALIZER(tlist_hi);
54
55 static int tasklet_pending = 0;
56
57 /*
58 * Linux does:
59 * 1 - copy list locally
60 * 2 - empty global list
61 * 3 - process local list from head to tail
62 *****
63 * local list processing:
64 * - if element cannot be run, put it at the tail
65 * - last element == null
66 */
67 #define PROCESS_TASKLET_LIST(which_list) do { \
68 STAILQ_FOREACH_MUTABLE(te, &which_list, tasklet_entries, tmp_te) { \
69 struct tasklet_struct *t = te->ts; \
70 \
71 /* \
72 This tasklet is dying, remove it from the list. \
73 We allow to it to run one last time if it has \
74 already been scheduled. \
75 */ \
76 if (test_bit(TASKLET_IS_DYING, &t->state)) { \
77 STAILQ_REMOVE(&which_list, te, tasklet_entry, tasklet_entries); \
78 kfree(te); \
79 } \
80 \
81 /* This tasklet is not enabled, try the next one */ \
82 if (atomic_read(&t->count) != 0) \
83 continue; \
84 \
85 /* This tasklet is not scheduled, try the next one */ \
86 if (!test_bit(TASKLET_STATE_SCHED, &t->state)) \
87 continue; \
88 \
89 clear_bit(TASKLET_STATE_SCHED, &t->state); \
90 set_bit(TASKLET_STATE_RUN, &t->state); \
91 \
92 lockmgr(&tasklet_lock, LK_RELEASE); \
93 if (t->func) \
94 t->func(t->data); \
95 lockmgr(&tasklet_lock, LK_EXCLUSIVE); \
96 \
97 clear_bit(TASKLET_STATE_RUN, &t->state); \
98 } \
99 } while (0)
100
101 /* XXX runners should be CPU-specific */
102 static void
tasklet_runner(void * arg)103 tasklet_runner(void *arg)
104 {
105 struct tasklet_entry *te, *tmp_te;
106
107 lockmgr(&tasklet_lock, LK_EXCLUSIVE);
108 while (1) {
109 /*
110 Only sleep if we haven't been raced by a _schedule()
111 call during an unlock window
112 */
113 if (tasklet_pending == 0) {
114 lksleep(&tasklet_runner, &tasklet_lock, 0, "tkidle", 0);
115 }
116 tasklet_pending = 0;
117
118 /* Process hi tasklets first */
119 PROCESS_TASKLET_LIST(tlist_hi);
120 PROCESS_TASKLET_LIST(tlist);
121 }
122 lockmgr(&tasklet_lock, LK_RELEASE);
123 }
124
125 void
tasklet_init(struct tasklet_struct * t,void (* func)(unsigned long),unsigned long data)126 tasklet_init(struct tasklet_struct *t,
127 void (*func)(unsigned long), unsigned long data)
128 {
129 t->state = 0;
130 t->func = func;
131 t->data = data;
132 atomic_set(&t->count, 0);
133 }
134
135 #define TASKLET_SCHEDULE_COMMON(t, list) do { \
136 struct tasklet_entry *te; \
137 \
138 lockmgr(&tasklet_lock, LK_EXCLUSIVE); \
139 set_bit(TASKLET_STATE_SCHED, &t->state); \
140 \
141 STAILQ_FOREACH(te, &(list), tasklet_entries) { \
142 if (te->ts == t) \
143 goto found_and_done; \
144 } \
145 \
146 te = kzalloc(sizeof(struct tasklet_entry), M_WAITOK); \
147 te->ts = t; \
148 STAILQ_INSERT_TAIL(&(list), te, tasklet_entries); \
149 \
150 found_and_done: \
151 tasklet_pending = 1; \
152 wakeup(&tasklet_runner); \
153 lockmgr(&tasklet_lock, LK_RELEASE); \
154 } while (0)
155
156 void
tasklet_schedule(struct tasklet_struct * t)157 tasklet_schedule(struct tasklet_struct *t)
158 {
159 TASKLET_SCHEDULE_COMMON(t, tlist);
160 }
161
162 void
tasklet_hi_schedule(struct tasklet_struct * t)163 tasklet_hi_schedule(struct tasklet_struct *t)
164 {
165 TASKLET_SCHEDULE_COMMON(t, tlist_hi);
166 }
167
168 void
tasklet_kill(struct tasklet_struct * t)169 tasklet_kill(struct tasklet_struct *t)
170 {
171 set_bit(TASKLET_IS_DYING, &t->state);
172 wakeup(&tasklet_runner);
173 }
174
init_tasklets(void * arg)175 static int init_tasklets(void *arg)
176 {
177 kthread_create(tasklet_runner, NULL, &tasklet_td, "tasklet_runner");
178
179 return 0;
180 }
181
182 SYSINIT(linux_tasklet_init, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, init_tasklets, NULL);
183