1 /*
2 * Copyright (c) 2001 Daniel Eischen <deischen@freebsd.org>
3 * Copyright (c) 2000-2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30
31 #include <sys/types.h>
32 #include <sys/mman.h>
33 #include <sys/queue.h>
34 #include <sys/resource.h>
35 #include <sys/sysctl.h>
36 #include <stdlib.h>
37 #include <pthread.h>
38 #include <link.h>
39
40 #include "thr_private.h"
41
42 /* Spare thread stack. */
43 struct stack {
44 LIST_ENTRY(stack) qe; /* Stack queue linkage. */
45 size_t stacksize; /* Stack size (rounded up). */
46 size_t guardsize; /* Guard size. */
47 void *stackaddr; /* Stack address. */
48 };
49
50 /*
51 * Default sized (stack and guard) spare stack queue. Stacks are cached
52 * to avoid additional complexity managing mmap()ed stack regions. Spare
53 * stacks are used in LIFO order to increase cache locality.
54 */
55 static LIST_HEAD(, stack) dstackq = LIST_HEAD_INITIALIZER(dstackq);
56
57 /*
58 * Miscellaneous sized (non-default stack and/or guard) spare stack queue.
59 * Stacks are cached to avoid additional complexity managing mmap()ed
60 * stack regions. This list is unordered, since ordering on both stack
61 * size and guard size would be more trouble than it's worth. Stacks are
62 * allocated from this cache on a first size match basis.
63 */
64 static LIST_HEAD(, stack) mstackq = LIST_HEAD_INITIALIZER(mstackq);
65
66 /**
67 * Base address of the last stack allocated (including its red zone, if
68 * there is one). Stacks are allocated contiguously, starting beyond the
69 * top of the main stack. When a new stack is created, a red zone is
70 * typically created (actually, the red zone is mapped with PROT_NONE) above
71 * the top of the stack, such that the stack will not be able to grow all
72 * the way to the bottom of the next stack. This isn't fool-proof. It is
73 * possible for a stack to grow by a large amount, such that it grows into
74 * the next stack, and as long as the memory within the red zone is never
75 * accessed, nothing will prevent one thread stack from trouncing all over
76 * the next.
77 *
78 * low memory
79 * . . . . . . . . . . . . . . . . . .
80 * | |
81 * | stack 3 | start of 3rd thread stack
82 * +-----------------------------------+
83 * | |
84 * | Red Zone (guard page) | red zone for 2nd thread
85 * | |
86 * +-----------------------------------+
87 * | stack 2 - _thr_stack_default | top of 2nd thread stack
88 * | |
89 * | |
90 * | |
91 * | |
92 * | stack 2 |
93 * +-----------------------------------+ <-- start of 2nd thread stack
94 * | |
95 * | Red Zone | red zone for 1st thread
96 * | |
97 * +-----------------------------------+
98 * | stack 1 - _thr_stack_default | top of 1st thread stack
99 * | |
100 * | |
101 * | |
102 * | |
103 * | stack 1 |
104 * +-----------------------------------+ <-- start of 1st thread stack
105 * | | (initial value of last_stack)
106 * | Red Zone |
107 * | | red zone for main thread
108 * +-----------------------------------+
109 * | USRSTACK - _thr_stack_initial | top of main thread stack
110 * | | ^
111 * | | |
112 * | | |
113 * | | | stack growth
114 * | |
115 * +-----------------------------------+ <-- start of main thread stack
116 * (USRSTACK)
117 * high memory
118 *
119 */
120 static char *last_stack = NULL;
121
122 /*
123 * Round size up to the nearest multiple of
124 * _thr_page_size.
125 */
126 static inline size_t
round_up(size_t size)127 round_up(size_t size)
128 {
129 if (size % _thr_page_size != 0)
130 size = ((size / _thr_page_size) + 1) *
131 _thr_page_size;
132 return size;
133 }
134
135 void
_thr_stack_fix_protection(struct pthread * thrd)136 _thr_stack_fix_protection(struct pthread *thrd)
137 {
138
139 mprotect((char *)thrd->attr.stackaddr_attr +
140 round_up(thrd->attr.guardsize_attr),
141 round_up(thrd->attr.stacksize_attr),
142 _rtld_get_stack_prot());
143 }
144
145 static void
singlethread_map_stacks_exec(void)146 singlethread_map_stacks_exec(void)
147 {
148 int mib[2];
149 struct rlimit rlim;
150 u_long usrstack;
151 size_t len;
152
153 mib[0] = CTL_KERN;
154 mib[1] = KERN_USRSTACK;
155 len = sizeof(usrstack);
156 if (sysctl(mib, sizeof(mib) / sizeof(mib[0]), &usrstack, &len, NULL, 0)
157 == -1)
158 return;
159 if (getrlimit(RLIMIT_STACK, &rlim) == -1)
160 return;
161 mprotect((void *)(uintptr_t)(usrstack - rlim.rlim_cur),
162 rlim.rlim_cur, _rtld_get_stack_prot());
163 }
164
165 void
__thr_map_stacks_exec(void)166 __thr_map_stacks_exec(void)
167 {
168 struct pthread *curthread, *thrd;
169 struct stack *st;
170
171 if (!_thr_is_inited()) {
172 singlethread_map_stacks_exec();
173 return;
174 }
175 curthread = _get_curthread();
176 THREAD_LIST_RDLOCK(curthread);
177 LIST_FOREACH(st, &mstackq, qe)
178 mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
179 _rtld_get_stack_prot());
180 LIST_FOREACH(st, &dstackq, qe)
181 mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
182 _rtld_get_stack_prot());
183 TAILQ_FOREACH(thrd, &_thread_gc_list, gcle)
184 _thr_stack_fix_protection(thrd);
185 TAILQ_FOREACH(thrd, &_thread_list, tle)
186 _thr_stack_fix_protection(thrd);
187 THREAD_LIST_UNLOCK(curthread);
188 }
189
190 int
_thr_stack_alloc(struct pthread_attr * attr)191 _thr_stack_alloc(struct pthread_attr *attr)
192 {
193 struct pthread *curthread = _get_curthread();
194 struct stack *spare_stack;
195 size_t stacksize;
196 size_t guardsize;
197 char *stackaddr;
198
199 /*
200 * Round up stack size to nearest multiple of _thr_page_size so
201 * that mmap() * will work. If the stack size is not an even
202 * multiple, we end up initializing things such that there is
203 * unused space above the beginning of the stack, so the stack
204 * sits snugly against its guard.
205 */
206 stacksize = round_up(attr->stacksize_attr);
207 guardsize = round_up(attr->guardsize_attr);
208
209 attr->stackaddr_attr = NULL;
210 attr->flags &= ~THR_STACK_USER;
211
212 /*
213 * Use the garbage collector lock for synchronization of the
214 * spare stack lists and allocations from usrstack.
215 */
216 THREAD_LIST_WRLOCK(curthread);
217 /*
218 * If the stack and guard sizes are default, try to allocate a stack
219 * from the default-size stack cache:
220 */
221 if ((stacksize == THR_STACK_DEFAULT) &&
222 (guardsize == _thr_guard_default)) {
223 if ((spare_stack = LIST_FIRST(&dstackq)) != NULL) {
224 /* Use the spare stack. */
225 LIST_REMOVE(spare_stack, qe);
226 attr->stackaddr_attr = spare_stack->stackaddr;
227 }
228 }
229 /*
230 * The user specified a non-default stack and/or guard size, so try to
231 * allocate a stack from the non-default size stack cache, using the
232 * rounded up stack size (stack_size) in the search:
233 */
234 else {
235 LIST_FOREACH(spare_stack, &mstackq, qe) {
236 if (spare_stack->stacksize == stacksize &&
237 spare_stack->guardsize == guardsize) {
238 LIST_REMOVE(spare_stack, qe);
239 attr->stackaddr_attr = spare_stack->stackaddr;
240 break;
241 }
242 }
243 }
244 if (attr->stackaddr_attr != NULL) {
245 /* A cached stack was found. Release the lock. */
246 THREAD_LIST_UNLOCK(curthread);
247 }
248 else {
249 /*
250 * Allocate a stack from or below usrstack, depending
251 * on the LIBPTHREAD_BIGSTACK_MAIN env variable.
252 */
253 if (last_stack == NULL)
254 last_stack = _usrstack - _thr_stack_initial -
255 _thr_guard_default;
256
257 /* Allocate a new stack. */
258 stackaddr = last_stack - stacksize - guardsize;
259
260 /*
261 * Even if stack allocation fails, we don't want to try to
262 * use this location again, so unconditionally decrement
263 * last_stack. Under normal operating conditions, the most
264 * likely reason for an mmap() error is a stack overflow of
265 * the adjacent thread stack.
266 */
267 last_stack -= (stacksize + guardsize);
268
269 /* Release the lock before mmap'ing it. */
270 THREAD_LIST_UNLOCK(curthread);
271
272 /* Map the stack and guard page together, and split guard
273 page from allocated space: */
274 if ((stackaddr = mmap(stackaddr, stacksize + guardsize,
275 _rtld_get_stack_prot(), MAP_STACK,
276 -1, 0)) != MAP_FAILED &&
277 (guardsize == 0 ||
278 mprotect(stackaddr, guardsize, PROT_NONE) == 0)) {
279 stackaddr += guardsize;
280 } else {
281 if (stackaddr != MAP_FAILED)
282 munmap(stackaddr, stacksize + guardsize);
283 stackaddr = NULL;
284 }
285 attr->stackaddr_attr = stackaddr;
286 }
287 if (attr->stackaddr_attr != NULL)
288 return (0);
289 else
290 return (-1);
291 }
292
293 /* This function must be called with _thread_list_lock held. */
294 void
_thr_stack_free(struct pthread_attr * attr)295 _thr_stack_free(struct pthread_attr *attr)
296 {
297 struct stack *spare_stack;
298
299 if ((attr != NULL) && ((attr->flags & THR_STACK_USER) == 0)
300 && (attr->stackaddr_attr != NULL)) {
301 spare_stack = (struct stack *)
302 ((char *)attr->stackaddr_attr +
303 attr->stacksize_attr - sizeof(struct stack));
304 spare_stack->stacksize = round_up(attr->stacksize_attr);
305 spare_stack->guardsize = round_up(attr->guardsize_attr);
306 spare_stack->stackaddr = attr->stackaddr_attr;
307
308 if (spare_stack->stacksize == THR_STACK_DEFAULT &&
309 spare_stack->guardsize == _thr_guard_default) {
310 /* Default stack/guard size. */
311 LIST_INSERT_HEAD(&dstackq, spare_stack, qe);
312 } else {
313 /* Non-default stack/guard size. */
314 LIST_INSERT_HEAD(&mstackq, spare_stack, qe);
315 }
316 attr->stackaddr_attr = NULL;
317 }
318 }
319