1 /* $NetBSD: pthread.c,v 1.187 2025/04/02 14:23:34 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020
5 * The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Nathan J. Williams and Andrew Doran.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __RCSID("$NetBSD: pthread.c,v 1.187 2025/04/02 14:23:34 riastradh Exp $");
35
36 #define __EXPOSE_STACK 1
37
38 /* Need to use libc-private names for atomic operations. */
39 #include "../../common/lib/libc/atomic/atomic_op_namespace.h"
40
41 #include <sys/param.h>
42 #include <sys/exec_elf.h>
43 #include <sys/mman.h>
44 #include <sys/lwp.h>
45 #include <sys/lwpctl.h>
46 #include <sys/resource.h>
47 #include <sys/sysctl.h>
48 #include <sys/tls.h>
49 #include <uvm/uvm_param.h>
50
51 #include <assert.h>
52 #include <dlfcn.h>
53 #include <err.h>
54 #include <errno.h>
55 #include <lwp.h>
56 #include <signal.h>
57 #include <stdatomic.h>
58 #include <stdio.h>
59 #include <stdlib.h>
60 #include <stddef.h>
61 #include <string.h>
62 #include <syslog.h>
63 #include <ucontext.h>
64 #include <unistd.h>
65 #include <sched.h>
66
67 #include "atexit.h"
68 #include "pthread.h"
69 #include "pthread_int.h"
70 #include "pthread_makelwp.h"
71 #include "reentrant.h"
72
73 #define atomic_load_relaxed(p) \
74 atomic_load_explicit(p, memory_order_relaxed)
75
76 #define atomic_store_relaxed(p, v) \
77 atomic_store_explicit(p, v, memory_order_relaxed)
78
79 #define atomic_store_release(p, v) \
80 atomic_store_explicit(p, v, memory_order_release)
81
82 __BEGIN_DECLS
83 void _malloc_thread_cleanup(void) __weak;
84 __END_DECLS
85
86 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER;
87 static rb_tree_t pthread__alltree;
88
89 static signed int pthread__cmp(void *, const void *, const void *);
90
91 static const rb_tree_ops_t pthread__alltree_ops = {
92 .rbto_compare_nodes = pthread__cmp,
93 .rbto_compare_key = pthread__cmp,
94 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree),
95 .rbto_context = NULL
96 };
97
98 static void pthread__create_tramp(void *);
99 static void pthread__initthread(pthread_t);
100 static void pthread__scrubthread(pthread_t, char *, int);
101 static void pthread__initmain(pthread_t *);
102 static void pthread__reap(pthread_t);
103
104 void pthread__init(void);
105
106 int pthread__started;
107 int __uselibcstub = 1;
108 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER;
109 pthread_queue_t pthread__deadqueue;
110 pthread_queue_t pthread__allqueue;
111
112 static pthread_attr_t pthread_default_attr;
113 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE };
114
115 enum {
116 DIAGASSERT_ABORT = 1<<0,
117 DIAGASSERT_STDERR = 1<<1,
118 DIAGASSERT_SYSLOG = 1<<2
119 };
120
121 static int pthread__diagassert;
122
123 int pthread__concurrency;
124 int pthread__nspins;
125 size_t pthread__unpark_max = PTHREAD__UNPARK_MAX;
126 int pthread__dbg; /* set by libpthread_dbg if active */
127
128 /*
129 * We have to initialize the pthread_stack* variables here because
130 * mutexes are used before pthread_init() and thus pthread__initmain()
131 * are called. Since mutexes only save the stack pointer and not a
132 * pointer to the thread data, it is safe to change the mapping from
133 * stack pointer to thread data afterwards.
134 */
135 size_t pthread__stacksize;
136 size_t pthread__guardsize;
137 size_t pthread__pagesize;
138 static struct __pthread_st *pthread__main;
139 static size_t __pthread_st_size;
140
141 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *);
142
143 __strong_alias(__libc_thr_self,pthread_self)
144 __strong_alias(__libc_thr_create,pthread_create)
145 __strong_alias(__libc_thr_exit,pthread_exit)
146 __strong_alias(__libc_thr_errno,pthread__errno)
147 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate)
148 __strong_alias(__libc_thr_equal,pthread_equal)
149 __strong_alias(__libc_thr_init,pthread__init)
150
151 /*
152 * Static library kludge. Place a reference to a symbol any library
153 * file which does not already have a reference here.
154 */
155 extern int pthread__cancel_stub_binder;
156
157 void *pthread__static_lib_binder[] = {
158 &pthread__cancel_stub_binder,
159 pthread_cond_init,
160 pthread_mutex_init,
161 pthread_rwlock_init,
162 pthread_barrier_init,
163 pthread_key_create,
164 pthread_setspecific,
165 };
166
167 #define NHASHLOCK 64
168
169 static union hashlock {
170 pthread_mutex_t mutex;
171 char pad[64];
172 } hashlocks[NHASHLOCK] __aligned(64);
173
174 static void
pthread__prefork(void)175 pthread__prefork(void)
176 {
177 pthread_mutex_lock(&pthread__deadqueue_lock);
178 }
179
180 static void
pthread__fork_parent(void)181 pthread__fork_parent(void)
182 {
183 pthread_mutex_unlock(&pthread__deadqueue_lock);
184 }
185
186 static void
pthread__fork_child(void)187 pthread__fork_child(void)
188 {
189 struct __pthread_st *self = pthread__self();
190
191 pthread_mutex_init(&pthread__deadqueue_lock, NULL);
192
193 /* lwpctl state is not copied across fork. */
194 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
195 err(EXIT_FAILURE, "_lwp_ctl");
196 }
197 self->pt_lid = _lwp_self();
198 }
199
200 /*
201 * This needs to be started by the library loading code, before main()
202 * gets to run, for various things that use the state of the initial thread
203 * to work properly (thread-specific data is an application-visible example;
204 * spinlock counts for mutexes is an internal example).
205 */
206 void
pthread__init(void)207 pthread__init(void)
208 {
209 pthread_t first;
210 char *p;
211 int mib[2];
212 unsigned int value;
213 ssize_t slen;
214 size_t len;
215 extern int __isthreaded;
216
217 /*
218 * Allocate pthread_keys descriptors before
219 * resetting __uselibcstub because otherwise
220 * malloc() will call pthread_keys_create()
221 * while pthread_keys descriptors are not
222 * yet allocated.
223 */
224 pthread__main = pthread_tsd_init(&__pthread_st_size);
225 if (pthread__main == NULL)
226 err(EXIT_FAILURE, "Cannot allocate pthread storage");
227
228 __uselibcstub = 0;
229
230 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE);
231 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF);
232
233 mib[0] = CTL_VM;
234 mib[1] = VM_THREAD_GUARD_SIZE;
235 len = sizeof(value);
236 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
237 pthread__guardsize = value;
238 else
239 pthread__guardsize = pthread__pagesize;
240
241 /* Initialize locks first; they're needed elsewhere. */
242 pthread__lockprim_init();
243 for (int i = 0; i < NHASHLOCK; i++) {
244 pthread_mutex_init(&hashlocks[i].mutex, NULL);
245 }
246
247 /* Fetch parameters. */
248 slen = _lwp_unpark_all(NULL, 0, NULL);
249 if (slen < 0)
250 err(EXIT_FAILURE, "_lwp_unpark_all");
251 if ((size_t)slen < pthread__unpark_max)
252 pthread__unpark_max = slen;
253
254 /* Basic data structure setup */
255 pthread_attr_init(&pthread_default_attr);
256 PTQ_INIT(&pthread__allqueue);
257 PTQ_INIT(&pthread__deadqueue);
258
259 rb_tree_init(&pthread__alltree, &pthread__alltree_ops);
260
261 /* Create the thread structure corresponding to main() */
262 pthread__initmain(&first);
263 pthread__initthread(first);
264 pthread__scrubthread(first, NULL, 0);
265
266 first->pt_lid = _lwp_self();
267 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq);
268 (void)rb_tree_insert_node(&pthread__alltree, first);
269
270 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) {
271 err(EXIT_FAILURE, "_lwp_ctl");
272 }
273
274 /* Start subsystems */
275 PTHREAD_MD_INIT
276
277 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) {
278 switch (*p) {
279 case 'a':
280 pthread__diagassert |= DIAGASSERT_ABORT;
281 break;
282 case 'A':
283 pthread__diagassert &= ~DIAGASSERT_ABORT;
284 break;
285 case 'e':
286 pthread__diagassert |= DIAGASSERT_STDERR;
287 break;
288 case 'E':
289 pthread__diagassert &= ~DIAGASSERT_STDERR;
290 break;
291 case 'l':
292 pthread__diagassert |= DIAGASSERT_SYSLOG;
293 break;
294 case 'L':
295 pthread__diagassert &= ~DIAGASSERT_SYSLOG;
296 break;
297 }
298 }
299
300 /* Tell libc that we're here and it should role-play accordingly. */
301 pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child);
302 __isthreaded = 1;
303 }
304
305 /* General-purpose thread data structure sanitization. */
306 /* ARGSUSED */
307 static void
pthread__initthread(pthread_t t)308 pthread__initthread(pthread_t t)
309 {
310
311 t->pt_self = t;
312 t->pt_magic = PT_MAGIC;
313 t->pt_sleepobj = NULL;
314 t->pt_havespecific = 0;
315 t->pt_lwpctl = &pthread__dummy_lwpctl;
316
317 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops));
318 pthread_mutex_init(&t->pt_lock, NULL);
319 PTQ_INIT(&t->pt_cleanup_stack);
320 }
321
322 static void
pthread__scrubthread(pthread_t t,char * name,int flags)323 pthread__scrubthread(pthread_t t, char *name, int flags)
324 {
325
326 t->pt_state = PT_STATE_RUNNING;
327 t->pt_exitval = NULL;
328 t->pt_flags = flags;
329 t->pt_cancel = 0;
330 t->pt_errno = 0;
331 t->pt_name = name;
332 t->pt_lid = 0;
333 }
334
335 static int
pthread__getstack(pthread_t newthread,const pthread_attr_t * attr)336 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr)
337 {
338 void *stackbase, *stackbase2, *redzone;
339 size_t stacksize, guardsize;
340 bool allocated;
341
342 if (attr != NULL) {
343 pthread_attr_getstack(attr, &stackbase, &stacksize);
344 if (stackbase == NULL)
345 pthread_attr_getguardsize(attr, &guardsize);
346 else
347 guardsize = 0;
348 } else {
349 stackbase = NULL;
350 stacksize = 0;
351 guardsize = pthread__guardsize;
352 }
353 if (stacksize == 0)
354 stacksize = pthread__stacksize;
355
356 if (newthread->pt_stack_allocated) {
357 if (stackbase == NULL &&
358 newthread->pt_stack.ss_size == stacksize &&
359 newthread->pt_guardsize == guardsize)
360 return 0;
361 stackbase2 = newthread->pt_stack.ss_sp;
362 #ifndef __MACHINE_STACK_GROWS_UP
363 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize;
364 #endif
365 munmap(stackbase2,
366 newthread->pt_stack.ss_size + newthread->pt_guardsize);
367 newthread->pt_stack.ss_sp = NULL;
368 newthread->pt_stack.ss_size = 0;
369 newthread->pt_guardsize = 0;
370 newthread->pt_stack_allocated = false;
371 }
372
373 newthread->pt_stack_allocated = false;
374
375 if (stackbase == NULL) {
376 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1;
377 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1;
378 stackbase = mmap(NULL, stacksize + guardsize,
379 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0);
380 if (stackbase == MAP_FAILED)
381 return ENOMEM;
382 allocated = true;
383 } else {
384 allocated = false;
385 }
386 #ifdef __MACHINE_STACK_GROWS_UP
387 redzone = (char *)stackbase + stacksize;
388 stackbase2 = (char *)stackbase;
389 #else
390 redzone = (char *)stackbase;
391 stackbase2 = (char *)stackbase + guardsize;
392 #endif
393 if (allocated && guardsize &&
394 mprotect(redzone, guardsize, PROT_NONE) == -1) {
395 munmap(stackbase, stacksize + guardsize);
396 return EPERM;
397 }
398 newthread->pt_stack.ss_size = stacksize;
399 newthread->pt_stack.ss_sp = stackbase2;
400 newthread->pt_guardsize = guardsize;
401 newthread->pt_stack_allocated = allocated;
402 return 0;
403 }
404
405 int
pthread_create(pthread_t * thread,const pthread_attr_t * attr,void * (* startfunc)(void *),void * arg)406 pthread_create(pthread_t *thread, const pthread_attr_t *attr,
407 void *(*startfunc)(void *), void *arg)
408 {
409 pthread_t newthread;
410 pthread_attr_t nattr;
411 struct pthread_attr_private *p;
412 char * volatile name;
413 unsigned long flag;
414 void *private_area;
415 int ret;
416
417 if (__predict_false(__uselibcstub)) {
418 pthread__errorfunc(__FILE__, __LINE__, __func__,
419 "pthread_create() requires linking with -lpthread");
420 return __libc_thr_create_stub(thread, attr, startfunc, arg);
421 }
422
423 if (attr == NULL)
424 nattr = pthread_default_attr;
425 else if (attr->pta_magic == PT_ATTR_MAGIC)
426 nattr = *attr;
427 else
428 return EINVAL;
429
430 if (!pthread__started) {
431 /*
432 * Force the _lwp_park symbol to be resolved before we
433 * begin any activity that might rely on concurrent
434 * wakeups.
435 *
436 * This is necessary because rtld itself uses _lwp_park
437 * and _lwp_unpark internally for its own locking: If
438 * we wait to resolve _lwp_park until there is an
439 * _lwp_unpark from another thread pending in the
440 * current lwp (for example, pthread_mutex_unlock or
441 * pthread_cond_signal), rtld's internal use of
442 * _lwp_park might consume the pending unpark. The
443 * result is a deadlock where libpthread and rtld have
444 * both correctly used _lwp_park and _lwp_unpark for
445 * themselves, but rtld has consumed the wakeup meant
446 * for libpthread so it is lost to libpthread.
447 *
448 * For the very first thread, before pthread__started
449 * is set to true, pthread__self()->pt_lid should have
450 * been initialized in pthread__init by the time we get
451 * here to the correct lid so we go to sleep and wake
452 * ourselves at the same time as a no-op.
453 */
454 _lwp_park(CLOCK_REALTIME, 0, NULL, pthread__self()->pt_lid,
455 NULL, NULL);
456 }
457
458 pthread__started = 1;
459
460 /* Fetch misc. attributes from the attr structure. */
461 name = NULL;
462 if ((p = nattr.pta_private) != NULL)
463 if (p->ptap_name[0] != '\0')
464 if ((name = strdup(p->ptap_name)) == NULL)
465 return ENOMEM;
466
467 newthread = NULL;
468
469 /*
470 * Try to reclaim a dead thread.
471 */
472 if (!PTQ_EMPTY(&pthread__deadqueue)) {
473 pthread_mutex_lock(&pthread__deadqueue_lock);
474 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) {
475 /* Still running? */
476 if (_lwp_kill(newthread->pt_lid, 0) == -1 &&
477 errno == ESRCH)
478 break;
479 }
480 if (newthread)
481 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq);
482 pthread_mutex_unlock(&pthread__deadqueue_lock);
483 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
484 if (newthread && newthread->pt_tls) {
485 _rtld_tls_free(newthread->pt_tls);
486 newthread->pt_tls = NULL;
487 }
488 #endif
489 }
490
491 /*
492 * If necessary set up a stack, allocate space for a pthread_st,
493 * and initialize it.
494 */
495 if (newthread == NULL) {
496 newthread = calloc(1, __pthread_st_size);
497 if (newthread == NULL) {
498 free(name);
499 return ENOMEM;
500 }
501 newthread->pt_stack_allocated = false;
502
503 if (pthread__getstack(newthread, attr)) {
504 free(newthread);
505 free(name);
506 return ENOMEM;
507 }
508
509 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
510 newthread->pt_tls = NULL;
511 #endif
512
513 /* Add to list of all threads. */
514 pthread_rwlock_wrlock(&pthread__alltree_lock);
515 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq);
516 (void)rb_tree_insert_node(&pthread__alltree, newthread);
517 pthread_rwlock_unlock(&pthread__alltree_lock);
518
519 /* Will be reset by the thread upon exit. */
520 pthread__initthread(newthread);
521 } else {
522 if (pthread__getstack(newthread, attr)) {
523 pthread_mutex_lock(&pthread__deadqueue_lock);
524 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq);
525 pthread_mutex_unlock(&pthread__deadqueue_lock);
526 return ENOMEM;
527 }
528 }
529
530 /*
531 * Create the new LWP.
532 */
533 pthread__scrubthread(newthread, name, nattr.pta_flags);
534 newthread->pt_func = startfunc;
535 newthread->pt_arg = arg;
536 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
537 private_area = newthread->pt_tls = _rtld_tls_allocate();
538 newthread->pt_tls->tcb_pthread = newthread;
539 #else
540 private_area = newthread;
541 #endif
542
543 flag = 0;
544 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 ||
545 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0)
546 flag |= LWP_SUSPENDED;
547 if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0)
548 flag |= LWP_DETACHED;
549
550 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area,
551 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size,
552 flag, &newthread->pt_lid);
553 if (ret != 0) {
554 ret = errno;
555 pthread_mutex_lock(&newthread->pt_lock);
556 /* Will unlock and free name. */
557 pthread__reap(newthread);
558 return ret;
559 }
560
561 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) {
562 if (p != NULL) {
563 (void)pthread_setschedparam(newthread, p->ptap_policy,
564 &p->ptap_sp);
565 }
566 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) {
567 (void)_lwp_continue(newthread->pt_lid);
568 }
569 }
570
571 *thread = newthread;
572
573 return 0;
574 }
575
576
577 __dead static void
pthread__create_tramp(void * cookie)578 pthread__create_tramp(void *cookie)
579 {
580 pthread_t self;
581 void *retval;
582 void *junk __unused;
583
584 self = cookie;
585
586 /*
587 * Throw away some stack in a feeble attempt to reduce cache
588 * thrash. May help for SMT processors. XXX We should not
589 * be allocating stacks on fixed 2MB boundaries. Needs a
590 * thread register or decent thread local storage.
591 */
592 junk = alloca(((unsigned)self->pt_lid & 7) << 8);
593
594 if (self->pt_name != NULL) {
595 pthread_mutex_lock(&self->pt_lock);
596 if (self->pt_name != NULL)
597 (void)_lwp_setname(0, self->pt_name);
598 pthread_mutex_unlock(&self->pt_lock);
599 }
600
601 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
602 err(EXIT_FAILURE, "_lwp_ctl");
603 }
604
605 retval = (*self->pt_func)(self->pt_arg);
606
607 pthread_exit(retval);
608
609 /*NOTREACHED*/
610 pthread__abort();
611 }
612
613 int
pthread_suspend_np(pthread_t thread)614 pthread_suspend_np(pthread_t thread)
615 {
616 pthread_t self;
617
618 pthread__error(EINVAL, "Invalid thread",
619 thread->pt_magic == PT_MAGIC);
620
621 self = pthread__self();
622 if (self == thread) {
623 return EDEADLK;
624 }
625 if (pthread__find(thread) != 0)
626 return ESRCH;
627 if (_lwp_suspend(thread->pt_lid) == 0)
628 return 0;
629 return errno;
630 }
631
632 int
pthread_resume_np(pthread_t thread)633 pthread_resume_np(pthread_t thread)
634 {
635
636 pthread__error(EINVAL, "Invalid thread",
637 thread->pt_magic == PT_MAGIC);
638
639 if (pthread__find(thread) != 0)
640 return ESRCH;
641 if (_lwp_continue(thread->pt_lid) == 0)
642 return 0;
643 return errno;
644 }
645
646 void
pthread_exit(void * retval)647 pthread_exit(void *retval)
648 {
649 pthread_t self;
650 struct pt_clean_t *cleanup;
651
652 if (__predict_false(__uselibcstub)) {
653 __libc_thr_exit_stub(retval);
654 goto out;
655 }
656
657 self = pthread__self();
658
659 /* Disable cancellability. */
660 atomic_store_relaxed(&self->pt_cancel, PT_CANCEL_DISABLED);
661
662 /* Call any cancellation cleanup handlers */
663 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
664 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
665 cleanup = PTQ_FIRST(&self->pt_cleanup_stack);
666 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next);
667 (*cleanup->ptc_cleanup)(cleanup->ptc_arg);
668 }
669 }
670
671 __cxa_thread_run_atexit();
672
673 /* Perform cleanup of thread-specific data */
674 pthread__destroy_tsd(self);
675
676 if (_malloc_thread_cleanup)
677 _malloc_thread_cleanup();
678
679 /*
680 * Signal our exit. Our stack and pthread_t won't be reused until
681 * pthread_create() can see from kernel info that this LWP is gone.
682 */
683 pthread_mutex_lock(&self->pt_lock);
684 self->pt_exitval = retval;
685 if (self->pt_flags & PT_FLAG_DETACHED) {
686 /* pthread__reap() will drop the lock. */
687 pthread__reap(self);
688 _lwp_exit();
689 } else {
690 self->pt_state = PT_STATE_ZOMBIE;
691 pthread_mutex_unlock(&self->pt_lock);
692 /* Note: name will be freed by the joiner. */
693 _lwp_exit();
694 }
695
696 out:
697 /*NOTREACHED*/
698 pthread__abort();
699 exit(1);
700 }
701
702
703 int
pthread_join(pthread_t thread,void ** valptr)704 pthread_join(pthread_t thread, void **valptr)
705 {
706 pthread_t self;
707
708 pthread__error(EINVAL, "Invalid thread",
709 thread->pt_magic == PT_MAGIC);
710
711 self = pthread__self();
712
713 if (pthread__find(thread) != 0)
714 return ESRCH;
715
716 if (thread == self)
717 return EDEADLK;
718
719 /* IEEE Std 1003.1 says pthread_join() never returns EINTR. */
720 for (;;) {
721 pthread__testcancel(self);
722 if (_lwp_wait(thread->pt_lid, NULL) == 0)
723 break;
724 if (errno != EINTR)
725 return errno;
726 }
727
728 /*
729 * Don't test for cancellation again. The spec is that if
730 * cancelled, pthread_join() must not have succeeded.
731 */
732 pthread_mutex_lock(&thread->pt_lock);
733 if (thread->pt_state != PT_STATE_ZOMBIE) {
734 pthread__errorfunc(__FILE__, __LINE__, __func__,
735 "not a zombie");
736 }
737 if (valptr != NULL)
738 *valptr = thread->pt_exitval;
739
740 /* pthread__reap() will drop the lock. */
741 pthread__reap(thread);
742 return 0;
743 }
744
745 static void
pthread__reap(pthread_t thread)746 pthread__reap(pthread_t thread)
747 {
748 char *name;
749
750 name = thread->pt_name;
751 thread->pt_name = NULL;
752 thread->pt_state = PT_STATE_DEAD;
753 pthread_mutex_unlock(&thread->pt_lock);
754
755 pthread_mutex_lock(&pthread__deadqueue_lock);
756 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq);
757 pthread_mutex_unlock(&pthread__deadqueue_lock);
758
759 if (name != NULL)
760 free(name);
761 }
762
763 int
pthread_equal(pthread_t t1,pthread_t t2)764 pthread_equal(pthread_t t1, pthread_t t2)
765 {
766
767 if (__predict_false(__uselibcstub))
768 return __libc_thr_equal_stub(t1, t2);
769
770 pthread__error(0, "Invalid thread",
771 (t1 != NULL) && (t1->pt_magic == PT_MAGIC));
772
773 pthread__error(0, "Invalid thread",
774 (t2 != NULL) && (t2->pt_magic == PT_MAGIC));
775
776 /* Nothing special here. */
777 return (t1 == t2);
778 }
779
780
781 int
pthread_detach(pthread_t thread)782 pthread_detach(pthread_t thread)
783 {
784 int error;
785
786 pthread__error(EINVAL, "Invalid thread",
787 thread->pt_magic == PT_MAGIC);
788
789 if (pthread__find(thread) != 0)
790 return ESRCH;
791
792 pthread_mutex_lock(&thread->pt_lock);
793 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) {
794 error = EINVAL;
795 } else {
796 error = _lwp_detach(thread->pt_lid);
797 if (error == 0)
798 thread->pt_flags |= PT_FLAG_DETACHED;
799 else
800 error = errno;
801 }
802 if (thread->pt_state == PT_STATE_ZOMBIE) {
803 /* pthread__reap() will drop the lock. */
804 pthread__reap(thread);
805 } else
806 pthread_mutex_unlock(&thread->pt_lock);
807 return error;
808 }
809
810
811 int
pthread_getname_np(pthread_t thread,char * name,size_t len)812 pthread_getname_np(pthread_t thread, char *name, size_t len)
813 {
814
815 pthread__error(EINVAL, "Invalid thread",
816 thread->pt_magic == PT_MAGIC);
817
818 if (pthread__find(thread) != 0)
819 return ESRCH;
820
821 pthread_mutex_lock(&thread->pt_lock);
822 if (thread->pt_name == NULL)
823 name[0] = '\0';
824 else
825 strlcpy(name, thread->pt_name, len);
826 pthread_mutex_unlock(&thread->pt_lock);
827
828 return 0;
829 }
830
831
832 int
pthread_setname_np(pthread_t thread,const char * name,void * arg)833 pthread_setname_np(pthread_t thread, const char *name, void *arg)
834 {
835 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP];
836 int namelen;
837
838 pthread__error(EINVAL, "Invalid thread",
839 thread->pt_magic == PT_MAGIC);
840
841 if (pthread__find(thread) != 0)
842 return ESRCH;
843
844 namelen = snprintf(newname, sizeof(newname), name, arg);
845 if (namelen >= PTHREAD_MAX_NAMELEN_NP)
846 return EINVAL;
847
848 cp = strdup(newname);
849 if (cp == NULL)
850 return ENOMEM;
851
852 pthread_mutex_lock(&thread->pt_lock);
853 oldname = thread->pt_name;
854 thread->pt_name = cp;
855 (void)_lwp_setname(thread->pt_lid, cp);
856 pthread_mutex_unlock(&thread->pt_lock);
857
858 if (oldname != NULL)
859 free(oldname);
860
861 return 0;
862 }
863
864
865 pthread_t
pthread_self(void)866 pthread_self(void)
867 {
868 if (__predict_false(__uselibcstub))
869 return (pthread_t)__libc_thr_self_stub();
870
871 return pthread__self();
872 }
873
874
875 int
pthread_cancel(pthread_t thread)876 pthread_cancel(pthread_t thread)
877 {
878 unsigned old, new;
879 bool wake;
880
881 pthread__error(EINVAL, "Invalid thread",
882 thread->pt_magic == PT_MAGIC);
883
884 if (pthread__find(thread) != 0)
885 return ESRCH;
886
887 /*
888 * membar_release matches membar_acquire in
889 * pthread_setcancelstate and pthread__testcancel.
890 */
891 membar_release();
892
893 do {
894 old = atomic_load_relaxed(&thread->pt_cancel);
895 new = old | PT_CANCEL_PENDING;
896 wake = false;
897 if ((old & PT_CANCEL_DISABLED) == 0) {
898 new |= PT_CANCEL_CANCELLED;
899 wake = true;
900 }
901 } while (__predict_false(!atomic_compare_exchange_weak_explicit(
902 &thread->pt_cancel, &old, new,
903 memory_order_relaxed, memory_order_relaxed)));
904
905 if (wake)
906 _lwp_wakeup(thread->pt_lid);
907
908 return 0;
909 }
910
911
912 int
pthread_setcancelstate(int state,int * oldstate)913 pthread_setcancelstate(int state, int *oldstate)
914 {
915 pthread_t self;
916 unsigned flags, old, new;
917 bool cancelled;
918
919 if (__predict_false(__uselibcstub))
920 return __libc_thr_setcancelstate_stub(state, oldstate);
921
922 self = pthread__self();
923
924 switch (state) {
925 case PTHREAD_CANCEL_ENABLE:
926 flags = 0;
927 break;
928 case PTHREAD_CANCEL_DISABLE:
929 flags = PT_CANCEL_DISABLED;
930 break;
931 default:
932 return EINVAL;
933 }
934
935 do {
936 old = atomic_load_relaxed(&self->pt_cancel);
937 new = (old & ~PT_CANCEL_DISABLED) | flags;
938 /*
939 * If we disable while cancelled, switch back to
940 * pending so future cancellation tests will not fire
941 * until enabled again.
942 *
943 * If a cancellation was requested while cancellation
944 * was disabled, note that fact for future
945 * cancellation tests.
946 */
947 cancelled = false;
948 if (__predict_false((flags | (old & PT_CANCEL_CANCELLED)) ==
949 (PT_CANCEL_DISABLED|PT_CANCEL_CANCELLED))) {
950 new &= ~PT_CANCEL_CANCELLED;
951 new |= PT_CANCEL_PENDING;
952 } else if (__predict_false((flags |
953 (old & PT_CANCEL_PENDING)) ==
954 PT_CANCEL_PENDING)) {
955 new |= PT_CANCEL_CANCELLED;
956 /* This is not a deferred cancellation point. */
957 if (__predict_false(old & PT_CANCEL_ASYNC))
958 cancelled = true;
959 }
960 } while (__predict_false(!atomic_compare_exchange_weak_explicit(
961 &self->pt_cancel, &old, new,
962 memory_order_relaxed, memory_order_relaxed)));
963
964 /*
965 * If we transitioned from PTHREAD_CANCEL_DISABLED to
966 * PTHREAD_CANCEL_ENABLED, there was a pending cancel, and we
967 * are configured with asynchronous cancellation, we are now
968 * cancelled -- make it happen.
969 */
970 if (__predict_false(cancelled)) {
971 /*
972 * membar_acquire matches membar_release in
973 * pthread_cancel.
974 */
975 membar_acquire();
976 pthread__cancelled();
977 }
978
979 if (oldstate) {
980 if (old & PT_CANCEL_DISABLED)
981 *oldstate = PTHREAD_CANCEL_DISABLE;
982 else
983 *oldstate = PTHREAD_CANCEL_ENABLE;
984 }
985
986 return 0;
987 }
988
989
990 int
pthread_setcanceltype(int type,int * oldtype)991 pthread_setcanceltype(int type, int *oldtype)
992 {
993 pthread_t self;
994 unsigned flags, old, new;
995 bool cancelled;
996
997 self = pthread__self();
998
999 switch (type) {
1000 case PTHREAD_CANCEL_DEFERRED:
1001 flags = 0;
1002 break;
1003 case PTHREAD_CANCEL_ASYNCHRONOUS:
1004 flags = PT_CANCEL_ASYNC;
1005 break;
1006 default:
1007 return EINVAL;
1008 }
1009
1010 do {
1011 old = atomic_load_relaxed(&self->pt_cancel);
1012 new = (old & ~PT_CANCEL_ASYNC) | flags;
1013 cancelled = false;
1014 if (__predict_false((flags | (old & PT_CANCEL_CANCELLED)) ==
1015 (PT_CANCEL_ASYNC|PT_CANCEL_CANCELLED)))
1016 cancelled = true;
1017 } while (__predict_false(!atomic_compare_exchange_weak_explicit(
1018 &self->pt_cancel, &old, new,
1019 memory_order_relaxed, memory_order_relaxed)));
1020
1021 if (__predict_false(cancelled)) {
1022 membar_acquire();
1023 pthread__cancelled();
1024 }
1025
1026 if (oldtype != NULL) {
1027 if (old & PT_CANCEL_ASYNC)
1028 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS;
1029 else
1030 *oldtype = PTHREAD_CANCEL_DEFERRED;
1031 }
1032
1033 return 0;
1034 }
1035
1036
1037 void
pthread_testcancel(void)1038 pthread_testcancel(void)
1039 {
1040 pthread_t self;
1041
1042 self = pthread__self();
1043
1044 pthread__testcancel(self);
1045 }
1046
1047
1048 /*
1049 * POSIX requires that certain functions return an error rather than
1050 * invoking undefined behavior even when handed completely bogus
1051 * pthread_t values, e.g. stack garbage.
1052 */
1053 int
pthread__find(pthread_t id)1054 pthread__find(pthread_t id)
1055 {
1056 pthread_t target;
1057 int error;
1058
1059 pthread_rwlock_rdlock(&pthread__alltree_lock);
1060 target = rb_tree_find_node(&pthread__alltree, id);
1061 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH;
1062 pthread_rwlock_unlock(&pthread__alltree_lock);
1063
1064 return error;
1065 }
1066
1067
1068 void
pthread__testcancel(pthread_t self)1069 pthread__testcancel(pthread_t self)
1070 {
1071
1072 /*
1073 * We use atomic_load_relaxed and then a conditional
1074 * membar_acquire, rather than atomic_load_acquire, in order to
1075 * avoid incurring the cost of an acquire barrier in the common
1076 * case of not having been cancelled.
1077 *
1078 * membar_acquire matches membar_release in pthread_cancel.
1079 */
1080 if (__predict_false(atomic_load_relaxed(&self->pt_cancel) &
1081 PT_CANCEL_CANCELLED)) {
1082 membar_acquire();
1083 pthread__cancelled();
1084 }
1085 }
1086
1087
1088 void
pthread__cancelled(void)1089 pthread__cancelled(void)
1090 {
1091
1092 pthread_exit(PTHREAD_CANCELED);
1093 }
1094
1095
1096 void
pthread__cleanup_push(void (* cleanup)(void *),void * arg,void * store)1097 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store)
1098 {
1099 pthread_t self;
1100 struct pt_clean_t *entry;
1101
1102 self = pthread__self();
1103 entry = store;
1104 entry->ptc_cleanup = cleanup;
1105 entry->ptc_arg = arg;
1106 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next);
1107 }
1108
1109
1110 void
pthread__cleanup_pop(int ex,void * store)1111 pthread__cleanup_pop(int ex, void *store)
1112 {
1113 pthread_t self;
1114 struct pt_clean_t *entry;
1115
1116 self = pthread__self();
1117 entry = store;
1118
1119 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next);
1120 if (ex)
1121 (*entry->ptc_cleanup)(entry->ptc_arg);
1122 }
1123
1124
1125 int *
pthread__errno(void)1126 pthread__errno(void)
1127 {
1128 pthread_t self;
1129
1130 if (__predict_false(__uselibcstub)) {
1131 pthread__errorfunc(__FILE__, __LINE__, __func__,
1132 "pthread__errno() requires linking with -lpthread");
1133 return __libc_thr_errno_stub();
1134 }
1135
1136 self = pthread__self();
1137
1138 return &(self->pt_errno);
1139 }
1140
1141 ssize_t _sys_write(int, const void *, size_t);
1142
1143 void
pthread__assertfunc(const char * file,int line,const char * function,const char * expr)1144 pthread__assertfunc(const char *file, int line, const char *function,
1145 const char *expr)
1146 {
1147 char buf[1024];
1148 int len;
1149
1150 /*
1151 * snprintf_ss should not acquire any locks, or we could
1152 * end up deadlocked if the assert caller held locks.
1153 */
1154 len = snprintf_ss(buf, 1024,
1155 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n",
1156 expr, file, line,
1157 function ? ", function \"" : "",
1158 function ? function : "",
1159 function ? "\"" : "");
1160
1161 _sys_write(STDERR_FILENO, buf, (size_t)len);
1162 (void)raise(SIGABRT);
1163 _exit(1);
1164 }
1165
1166
1167 void
pthread__errorfunc(const char * file,int line,const char * function,const char * msg,...)1168 pthread__errorfunc(const char *file, int line, const char *function,
1169 const char *msg, ...)
1170 {
1171 char buf[1024];
1172 char buf2[1024];
1173 size_t len;
1174 va_list ap;
1175
1176 if (pthread__diagassert == 0)
1177 return;
1178
1179 va_start(ap, msg);
1180 vsnprintf_ss(buf2, sizeof(buf2), msg, ap);
1181 va_end(ap);
1182
1183 /*
1184 * snprintf_ss should not acquire any locks, or we could
1185 * end up deadlocked if the assert caller held locks.
1186 */
1187 len = snprintf_ss(buf, sizeof(buf),
1188 "%s: Error detected by libpthread: %s.\n"
1189 "Detected by file \"%s\", line %d%s%s%s.\n"
1190 "See pthread(3) for information.\n",
1191 getprogname(), buf2, file, line,
1192 function ? ", function \"" : "",
1193 function ? function : "",
1194 function ? "\"" : "");
1195
1196 if (pthread__diagassert & DIAGASSERT_STDERR)
1197 _sys_write(STDERR_FILENO, buf, len);
1198
1199 if (pthread__diagassert & DIAGASSERT_SYSLOG)
1200 syslog(LOG_DEBUG | LOG_USER, "%s", buf);
1201
1202 if (pthread__diagassert & DIAGASSERT_ABORT) {
1203 (void)raise(SIGABRT);
1204 _exit(1);
1205 }
1206 }
1207
1208 /*
1209 * Thread park/unpark operations. The kernel operations are
1210 * modelled after a brief description from "Multithreading in
1211 * the Solaris Operating Environment":
1212 *
1213 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf
1214 */
1215
1216 int
pthread__park(pthread_t self,pthread_mutex_t * lock,pthread_queue_t * queue,const struct timespec * abstime,int cancelpt)1217 pthread__park(pthread_t self, pthread_mutex_t *lock,
1218 pthread_queue_t *queue, const struct timespec *abstime,
1219 int cancelpt)
1220 {
1221 int rv, error;
1222
1223 pthread_mutex_unlock(lock);
1224
1225 /*
1226 * Wait until we are awoken by a pending unpark operation,
1227 * a signal, an unpark posted after we have gone asleep,
1228 * or an expired timeout.
1229 *
1230 * It is fine to test the value of pt_sleepobj without
1231 * holding any locks, because:
1232 *
1233 * o Only the blocking thread (this thread) ever sets it
1234 * to a non-NULL value.
1235 *
1236 * o Other threads may set it NULL, but if they do so they
1237 * must also make this thread return from _lwp_park.
1238 *
1239 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system
1240 * calls and all make use of spinlocks in the kernel. So
1241 * these system calls act as full memory barriers.
1242 */
1243 rv = 0;
1244 do {
1245 /*
1246 * If we deferred unparking a thread, arrange to
1247 * have _lwp_park() restart it before blocking.
1248 */
1249 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
1250 __UNCONST(abstime), 0, NULL, NULL);
1251 if (error != 0) {
1252 switch (rv = errno) {
1253 case EINTR:
1254 case EALREADY:
1255 rv = 0;
1256 break;
1257 case ETIMEDOUT:
1258 break;
1259 default:
1260 pthread__errorfunc(__FILE__, __LINE__,
1261 __func__, "_lwp_park failed: %d", errno);
1262 break;
1263 }
1264 }
1265 /* Check for cancellation. */
1266 if (cancelpt &&
1267 (atomic_load_relaxed(&self->pt_cancel) &
1268 PT_CANCEL_CANCELLED))
1269 rv = EINTR;
1270 } while (self->pt_sleepobj != NULL && rv == 0);
1271 return rv;
1272 }
1273
1274 void
pthread__unpark(pthread_queue_t * queue,pthread_t self,pthread_mutex_t * interlock)1275 pthread__unpark(pthread_queue_t *queue, pthread_t self,
1276 pthread_mutex_t *interlock)
1277 {
1278 pthread_t target;
1279
1280 target = PTQ_FIRST(queue);
1281 target->pt_sleepobj = NULL;
1282 PTQ_REMOVE(queue, target, pt_sleep);
1283 (void)_lwp_unpark(target->pt_lid, NULL);
1284 }
1285
1286 void
pthread__unpark_all(pthread_queue_t * queue,pthread_t self,pthread_mutex_t * interlock)1287 pthread__unpark_all(pthread_queue_t *queue, pthread_t self,
1288 pthread_mutex_t *interlock)
1289 {
1290 lwpid_t lids[PTHREAD__UNPARK_MAX];
1291 const size_t mlid = pthread__unpark_max;
1292 pthread_t target;
1293 size_t nlid = 0;
1294
1295 PTQ_FOREACH(target, queue, pt_sleep) {
1296 if (nlid == mlid) {
1297 (void)_lwp_unpark_all(lids, nlid, NULL);
1298 nlid = 0;
1299 }
1300 target->pt_sleepobj = NULL;
1301 lids[nlid++] = target->pt_lid;
1302 }
1303 PTQ_INIT(queue);
1304 if (nlid == 1) {
1305 (void)_lwp_unpark(lids[0], NULL);
1306 } else if (nlid > 1) {
1307 (void)_lwp_unpark_all(lids, nlid, NULL);
1308 }
1309 }
1310
1311 #undef OOPS
1312
1313 static void
pthread__initmainstack(void)1314 pthread__initmainstack(void)
1315 {
1316 struct rlimit slimit;
1317 const AuxInfo *aux;
1318 size_t size, len;
1319 int mib[2];
1320 unsigned int value;
1321
1322 _DIAGASSERT(_dlauxinfo() != NULL);
1323
1324 if (getrlimit(RLIMIT_STACK, &slimit) == -1)
1325 err(EXIT_FAILURE,
1326 "Couldn't get stack resource consumption limits");
1327 size = slimit.rlim_cur;
1328 pthread__main->pt_stack.ss_size = size;
1329 pthread__main->pt_guardsize = pthread__pagesize;
1330
1331 mib[0] = CTL_VM;
1332 mib[1] = VM_GUARD_SIZE;
1333 len = sizeof(value);
1334 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
1335 pthread__main->pt_guardsize = value;
1336
1337 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) {
1338 if (aux->a_type == AT_STACKBASE) {
1339 #ifdef __MACHINE_STACK_GROWS_UP
1340 pthread__main->pt_stack.ss_sp = (void *)aux->a_v;
1341 #else
1342 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size;
1343 #endif
1344 break;
1345 }
1346 }
1347 pthread__copy_tsd(pthread__main);
1348 }
1349
1350 /*
1351 * Set up the slightly special stack for the "initial" thread, which
1352 * runs on the normal system stack, and thus gets slightly different
1353 * treatment.
1354 */
1355 static void
pthread__initmain(pthread_t * newt)1356 pthread__initmain(pthread_t *newt)
1357 {
1358 char *value;
1359
1360 pthread__initmainstack();
1361
1362 value = pthread__getenv("PTHREAD_STACKSIZE");
1363 if (value != NULL) {
1364 pthread__stacksize = atoi(value) * 1024;
1365 if (pthread__stacksize > pthread__main->pt_stack.ss_size)
1366 pthread__stacksize = pthread__main->pt_stack.ss_size;
1367 }
1368 if (pthread__stacksize == 0)
1369 pthread__stacksize = pthread__main->pt_stack.ss_size;
1370 pthread__stacksize += pthread__pagesize - 1;
1371 pthread__stacksize &= ~(pthread__pagesize - 1);
1372 if (pthread__stacksize < 4 * pthread__pagesize)
1373 errx(1, "Stacksize limit is too low, minimum %zd kbyte.",
1374 4 * pthread__pagesize / 1024);
1375
1376 *newt = pthread__main;
1377 #if defined(_PTHREAD_GETTCB_EXT)
1378 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT();
1379 #elif defined(__HAVE___LWP_GETTCB_FAST)
1380 pthread__main->pt_tls = __lwp_gettcb_fast();
1381 #else
1382 pthread__main->pt_tls = _lwp_getprivate();
1383 #endif
1384 pthread__main->pt_tls->tcb_pthread = pthread__main;
1385 }
1386
1387 static signed int
1388 /*ARGSUSED*/
pthread__cmp(void * ctx,const void * n1,const void * n2)1389 pthread__cmp(void *ctx, const void *n1, const void *n2)
1390 {
1391 const uintptr_t p1 = (const uintptr_t)n1;
1392 const uintptr_t p2 = (const uintptr_t)n2;
1393
1394 if (p1 < p2)
1395 return -1;
1396 if (p1 > p2)
1397 return 1;
1398 return 0;
1399 }
1400
1401 /* Because getenv() wants to use locks. */
1402 char *
pthread__getenv(const char * name)1403 pthread__getenv(const char *name)
1404 {
1405 extern char **environ;
1406 size_t l_name, offset;
1407
1408 if (issetugid())
1409 return (NULL);
1410
1411 l_name = strlen(name);
1412 for (offset = 0; environ[offset] != NULL; offset++) {
1413 if (strncmp(name, environ[offset], l_name) == 0 &&
1414 environ[offset][l_name] == '=') {
1415 return environ[offset] + l_name + 1;
1416 }
1417 }
1418
1419 return NULL;
1420 }
1421
1422 pthread_mutex_t *
pthread__hashlock(volatile const void * p)1423 pthread__hashlock(volatile const void *p)
1424 {
1425 uintptr_t v;
1426
1427 v = (uintptr_t)p;
1428 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex;
1429 }
1430
1431 int
pthread__checkpri(int pri)1432 pthread__checkpri(int pri)
1433 {
1434 static int havepri;
1435 static long min, max;
1436
1437 if (!havepri) {
1438 min = sysconf(_SC_SCHED_PRI_MIN);
1439 max = sysconf(_SC_SCHED_PRI_MAX);
1440 havepri = 1;
1441 }
1442 return (pri < min || pri > max) ? EINVAL : 0;
1443 }
1444