1 /*
2 * Copyright (c) 2001 Daniel Eischen <deischen@freebsd.org>
3 * Copyright (c) 2000-2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * $FreeBSD$
28 */
29
30 #include <sys/types.h>
31 #include <sys/mman.h>
32 #include <sys/queue.h>
33 #include <sys/resource.h>
34 #include <sys/sysctl.h>
35 #include <stdlib.h>
36 #include <pthread.h>
37 #include <link.h>
38
39 #include "thr_private.h"
40
41 /* Spare thread stack. */
42 struct stack {
43 LIST_ENTRY(stack) qe; /* Stack queue linkage. */
44 size_t stacksize; /* Stack size (rounded up). */
45 size_t guardsize; /* Guard size. */
46 void *stackaddr; /* Stack address. */
47 };
48
49 /*
50 * Default sized (stack and guard) spare stack queue. Stacks are cached
51 * to avoid additional complexity managing mmap()ed stack regions. Spare
52 * stacks are used in LIFO order to increase cache locality.
53 */
54 static LIST_HEAD(, stack) dstackq = LIST_HEAD_INITIALIZER(dstackq);
55
56 /*
57 * Miscellaneous sized (non-default stack and/or guard) spare stack queue.
58 * Stacks are cached to avoid additional complexity managing mmap()ed
59 * stack regions. This list is unordered, since ordering on both stack
60 * size and guard size would be more trouble than it's worth. Stacks are
61 * allocated from this cache on a first size match basis.
62 */
63 static LIST_HEAD(, stack) mstackq = LIST_HEAD_INITIALIZER(mstackq);
64
65 /**
66 * Base address of the last stack allocated (including its red zone, if
67 * there is one). Stacks are allocated contiguously, starting beyond the
68 * top of the main stack. When a new stack is created, a red zone is
69 * typically created (actually, the red zone is mapped with PROT_NONE) above
70 * the top of the stack, such that the stack will not be able to grow all
71 * the way to the bottom of the next stack. This isn't fool-proof. It is
72 * possible for a stack to grow by a large amount, such that it grows into
73 * the next stack, and as long as the memory within the red zone is never
74 * accessed, nothing will prevent one thread stack from trouncing all over
75 * the next.
76 *
77 * low memory
78 * . . . . . . . . . . . . . . . . . .
79 * | |
80 * | stack 3 | start of 3rd thread stack
81 * +-----------------------------------+
82 * | |
83 * | Red Zone (guard page) | red zone for 2nd thread
84 * | |
85 * +-----------------------------------+
86 * | stack 2 - THR_STACK_DEFAULT | top of 2nd thread stack
87 * | |
88 * | |
89 * | |
90 * | |
91 * | stack 2 |
92 * +-----------------------------------+ <-- start of 2nd thread stack
93 * | |
94 * | Red Zone | red zone for 1st thread
95 * | |
96 * +-----------------------------------+
97 * | stack 1 - THR_STACK_DEFAULT | top of 1st thread stack
98 * | |
99 * | |
100 * | |
101 * | |
102 * | stack 1 |
103 * +-----------------------------------+ <-- start of 1st thread stack
104 * | | (initial value of last_stack)
105 * | Red Zone |
106 * | | red zone for main thread
107 * +-----------------------------------+
108 * | KERN_USRSTACK - THR_STACK_INITIAL | top of main thread stack
109 * | | ^
110 * | | |
111 * | | |
112 * | | | stack growth
113 * | |
114 * +-----------------------------------+ <-- start of main thread stack
115 * (USRSTACK)
116 * high memory
117 *
118 */
119
120 /*
121 * Round size up to the nearest multiple of
122 * _thr_page_size.
123 */
124 static inline size_t
round_up(size_t size)125 round_up(size_t size)
126 {
127 if (size % _thr_page_size != 0)
128 size = ((size / _thr_page_size) + 1) *
129 _thr_page_size;
130 return size;
131 }
132
133 void
_thr_stack_fix_protection(struct pthread * thrd)134 _thr_stack_fix_protection(struct pthread *thrd)
135 {
136
137 mprotect((char *)thrd->attr.stackaddr_attr +
138 round_up(thrd->attr.guardsize_attr),
139 round_up(thrd->attr.stacksize_attr),
140 _rtld_get_stack_prot());
141 }
142
143 static void
singlethread_map_stacks_exec(void)144 singlethread_map_stacks_exec(void)
145 {
146 int mib[2];
147 struct rlimit rlim;
148 u_long usrstack;
149 size_t len;
150
151 mib[0] = CTL_KERN;
152 mib[1] = KERN_USRSTACK;
153 len = sizeof(usrstack);
154 if (sysctl(mib, sizeof(mib) / sizeof(mib[0]), &usrstack, &len, NULL, 0)
155 == -1)
156 return;
157 if (getrlimit(RLIMIT_STACK, &rlim) == -1)
158 return;
159 mprotect((void *)(uintptr_t)(usrstack - rlim.rlim_cur),
160 rlim.rlim_cur, _rtld_get_stack_prot());
161 }
162
163 void __pthread_map_stacks_exec(void);
164 void
__pthread_map_stacks_exec(void)165 __pthread_map_stacks_exec(void)
166 {
167 struct pthread *curthread, *thrd;
168 struct stack *st;
169
170 if (!_thr_is_inited()) {
171 singlethread_map_stacks_exec();
172 return;
173 }
174 curthread = _get_curthread();
175 THREAD_LIST_RDLOCK(curthread);
176 LIST_FOREACH(st, &mstackq, qe)
177 mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
178 _rtld_get_stack_prot());
179 LIST_FOREACH(st, &dstackq, qe)
180 mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
181 _rtld_get_stack_prot());
182 TAILQ_FOREACH(thrd, &_thread_gc_list, gcle)
183 _thr_stack_fix_protection(thrd);
184 TAILQ_FOREACH(thrd, &_thread_list, tle)
185 _thr_stack_fix_protection(thrd);
186 THREAD_LIST_UNLOCK(curthread);
187 }
188
189 int
_thr_stack_alloc(struct pthread_attr * attr)190 _thr_stack_alloc(struct pthread_attr *attr)
191 {
192 struct pthread *curthread = _get_curthread();
193 struct stack *spare_stack;
194 size_t stacksize;
195 size_t guardsize;
196
197 /*
198 * Round up stack size to nearest multiple of _thr_page_size so
199 * that thr_stack() will work. If the stack size is not an even
200 * multiple, we end up initializing things such that there is
201 * unused space above the beginning of the stack, so the stack
202 * sits snugly against its guard.
203 */
204 stacksize = round_up(attr->stacksize_attr);
205 guardsize = round_up(attr->guardsize_attr);
206
207 attr->stackaddr_attr = NULL;
208 attr->flags &= ~THR_STACK_USER;
209
210 /*
211 * Use the garbage collector lock for synchronization of the
212 * spare stack lists.
213 */
214 THREAD_LIST_WRLOCK(curthread);
215 /*
216 * If the stack and guard sizes are default, try to allocate a stack
217 * from the default-size stack cache:
218 */
219 if ((stacksize == THR_STACK_DEFAULT) &&
220 (guardsize == _thr_guard_default)) {
221 if ((spare_stack = LIST_FIRST(&dstackq)) != NULL) {
222 /* Use the spare stack. */
223 LIST_REMOVE(spare_stack, qe);
224 attr->stackaddr_attr = spare_stack->stackaddr;
225 }
226 }
227 /*
228 * The user specified a non-default stack and/or guard size, so try to
229 * allocate a stack from the non-default size stack cache, using the
230 * rounded up stack size (stack_size) in the search:
231 */
232 else {
233 LIST_FOREACH(spare_stack, &mstackq, qe) {
234 if (spare_stack->stacksize == stacksize &&
235 spare_stack->guardsize == guardsize) {
236 LIST_REMOVE(spare_stack, qe);
237 attr->stackaddr_attr = spare_stack->stackaddr;
238 break;
239 }
240 }
241 }
242 if (attr->stackaddr_attr != NULL) {
243 /* A cached stack was found. Release the lock. */
244 THREAD_LIST_UNLOCK(curthread);
245 }
246 else {
247 /* thr_stack() can block so release the lock */
248 THREAD_LIST_UNLOCK(curthread);
249
250 attr->stackaddr_attr = thr_stack(stacksize, guardsize);
251 }
252 if (attr->stackaddr_attr != NULL)
253 return (0);
254 else
255 return (-1);
256 }
257
258 /* This function must be called with _thread_list_lock held. */
259 void
_thr_stack_free(struct pthread_attr * attr)260 _thr_stack_free(struct pthread_attr *attr)
261 {
262 struct stack *spare_stack;
263
264 if ((attr != NULL) && ((attr->flags & THR_STACK_USER) == 0)
265 && (attr->stackaddr_attr != NULL)) {
266 spare_stack = (struct stack *)
267 ((char *)attr->stackaddr_attr +
268 attr->stacksize_attr - sizeof(struct stack));
269 spare_stack->stacksize = round_up(attr->stacksize_attr);
270 spare_stack->guardsize = round_up(attr->guardsize_attr);
271 spare_stack->stackaddr = attr->stackaddr_attr;
272
273 if (spare_stack->stacksize == THR_STACK_DEFAULT &&
274 spare_stack->guardsize == _thr_guard_default) {
275 /* Default stack/guard size. */
276 LIST_INSERT_HEAD(&dstackq, spare_stack, qe);
277 } else {
278 /* Non-default stack/guard size. */
279 LIST_INSERT_HEAD(&mstackq, spare_stack, qe);
280 }
281 attr->stackaddr_attr = NULL;
282 }
283 }
284