1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2004 Doug Rabson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD: stable/12/lib/libc/gen/tls.c 370354 2021-08-19 07:37:08Z gbe $
29 */
30
31 /*
32 * Define stubs for TLS internals so that programs and libraries can
33 * link. These functions will be replaced by functional versions at
34 * runtime from ld-elf.so.1.
35 */
36
37 #include <sys/cdefs.h>
38 #include <sys/param.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <elf.h>
42 #include <unistd.h>
43
44 #include "libc_private.h"
45
46 #define tls_assert(cond) ((cond) ? (void) 0 : \
47 (tls_msg(#cond ": assert failed: " __FILE__ ":" \
48 __XSTRING(__LINE__) "\n"), abort()))
49 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s))
50
51 /* Provided by jemalloc to avoid bootstrapping issues. */
52 void *__je_bootstrap_malloc(size_t size);
53 void *__je_bootstrap_calloc(size_t num, size_t size);
54 void __je_bootstrap_free(void *ptr);
55
56 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls);
57 __weak_reference(__libc_free_tls, _rtld_free_tls);
58
59 #ifdef __i386__
60
61 __weak_reference(___libc_tls_get_addr, ___tls_get_addr);
62 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *);
63
64 #endif
65
66 void * __libc_tls_get_addr(void *);
67 __weak_reference(__libc_tls_get_addr, __tls_get_addr);
68
69 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
70 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
71 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
72 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
73
74 #if defined(__amd64__)
75 #define TLS_TCB_ALIGN 16
76 #elif defined(__aarch64__) || defined(__arm__) || defined(__i386__) || \
77 defined(__mips__) || defined(__powerpc__) || defined(__riscv) || \
78 defined(__sparc64__)
79 #define TLS_TCB_ALIGN sizeof(void *)
80 #else
81 #error TLS_TCB_ALIGN undefined for target architecture
82 #endif
83
84 #if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \
85 defined(__powerpc__) || defined(__riscv)
86 #define TLS_VARIANT_I
87 #endif
88 #if defined(__i386__) || defined(__amd64__) || defined(__sparc64__)
89 #define TLS_VARIANT_II
90 #endif
91
92 #if defined(__mips__) || defined(__powerpc__) || defined(__riscv)
93 #define DTV_OFFSET 0x8000
94 #else
95 #define DTV_OFFSET 0
96 #endif
97
98 #ifndef PIC
99
100 static size_t tls_static_space;
101 static size_t tls_init_size;
102 static size_t tls_init_align;
103 static void *tls_init;
104 #endif
105
106 #ifdef __i386__
107
108 /* GNU ABI */
109
110 __attribute__((__regparm__(1)))
111 void *
___libc_tls_get_addr(void * ti __unused)112 ___libc_tls_get_addr(void *ti __unused)
113 {
114 return (0);
115 }
116
117 #endif
118
119 void *
__libc_tls_get_addr(void * ti __unused)120 __libc_tls_get_addr(void *ti __unused)
121 {
122 return (0);
123 }
124
125 #ifndef PIC
126
127 static void *
malloc_aligned(size_t size,size_t align)128 malloc_aligned(size_t size, size_t align)
129 {
130 void *mem, *res;
131
132 if (align < sizeof(void *))
133 align = sizeof(void *);
134
135 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1);
136 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align);
137 *(void **)((uintptr_t)res - sizeof(void *)) = mem;
138 return (res);
139 }
140
141 static void
free_aligned(void * ptr)142 free_aligned(void *ptr)
143 {
144 void *mem;
145 uintptr_t x;
146
147 if (ptr == NULL)
148 return;
149
150 x = (uintptr_t)ptr;
151 x -= sizeof(void *);
152 mem = *(void **)x;
153 __je_bootstrap_free(mem);
154 }
155
156 #ifdef TLS_VARIANT_I
157
158 /*
159 * There are two versions of variant I of TLS
160 *
161 * - ARM and aarch64 uses original variant I as is described in [1] and [2],
162 * where TP points to start of TCB followed by aligned TLS segment.
163 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0]
164 * points to DTV vector and DTV values are real addresses (without bias).
165 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to
166 * TLS variables are computed by linker, so we cannot overalign TLS section.
167 *
168 * - MIPS, PowerPC and RISC-V use modified version of variant I,
169 * described in [3] where TP points (with bias) to TLS and TCB immediately
170 * precedes TLS without any alignment gap[4]. Only TLS should be aligned.
171 * The TCB[0] points to DTV vector and DTV values are biased by constant
172 * value (0x8000) from real addresses[5].
173 *
174 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage
175 * www.akkadia.org/drepper/tls.pdf
176 *
177 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r)
178 * Architecture
179 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf
180 *
181 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification
182 * https://members.openpowerfoundation.org/document/dl/576
183 *
184 * [4] Its unclear if "without any alignment gap" is hard ABI requirement,
185 * but we must follow this rule due to suboptimal _set_tp()
186 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but
187 * TCB as argument.
188 *
189 * [5] I'm not able to validate "values are biased" assertions.
190 */
191
192 #define TLS_TCB_SIZE (2 * sizeof(void *))
193
194 /*
195 * Return pointer to allocated TLS block
196 */
197 static void *
get_tls_block_ptr(void * tcb,size_t tcbsize)198 get_tls_block_ptr(void *tcb, size_t tcbsize)
199 {
200 size_t extra_size, post_size, pre_size, tls_block_size;
201
202 /* Compute fragments sizes. */
203 extra_size = tcbsize - TLS_TCB_SIZE;
204 #if defined(__aarch64__) || defined(__arm__)
205 post_size = roundup2(TLS_TCB_SIZE, tls_init_align) - TLS_TCB_SIZE;
206 #else
207 post_size = 0;
208 #endif
209 tls_block_size = tcbsize + post_size;
210 pre_size = roundup2(tls_block_size, tls_init_align) - tls_block_size;
211
212 return ((char *)tcb - pre_size - extra_size);
213 }
214
215 /*
216 * Free Static TLS using the Variant I method. The tcbsize
217 * and tcbalign parameters must be the same as those used to allocate
218 * the block.
219 */
220 void
__libc_free_tls(void * tcb,size_t tcbsize,size_t tcbalign __unused)221 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused)
222 {
223 Elf_Addr *dtv;
224 Elf_Addr **tls;
225
226 tls = (Elf_Addr **)tcb;
227 dtv = tls[0];
228 __je_bootstrap_free(dtv);
229 free_aligned(get_tls_block_ptr(tcb, tcbsize));
230 }
231
232 /*
233 * Allocate Static TLS using the Variant I method.
234 *
235 * To handle all above requirements, we setup the following layout for
236 * TLS block:
237 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align))
238 *
239 * +----------+--------------+--------------+-----------+------------------+
240 * | pre gap | extended TCB | TCB | post gap | TLS segment |
241 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space |
242 * +----------+--------------+--------------+-----------+------------------+
243 *
244 * where:
245 * extra_size is tcbsize - TLS_TCB_SIZE
246 * post_size is used to adjust TCB to TLS alignment for first version of TLS
247 * layout and is always 0 for second version.
248 * pre_size is used to adjust TCB alignment for first version and to adjust
249 * TLS alignment for second version.
250 *
251 */
252 void *
__libc_allocate_tls(void * oldtcb,size_t tcbsize,size_t tcbalign)253 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign)
254 {
255 Elf_Addr *dtv, **tcb;
256 char *tls_block, *tls;
257 size_t extra_size, maxalign, post_size, pre_size, tls_block_size;
258
259 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE)
260 return (oldtcb);
261
262 tls_assert(tcbalign >= TLS_TCB_ALIGN);
263 maxalign = MAX(tcbalign, tls_init_align);
264
265 /* Compute fragmets sizes. */
266 extra_size = tcbsize - TLS_TCB_SIZE;
267 #if defined(__aarch64__) || defined(__arm__)
268 post_size = roundup2(TLS_TCB_SIZE, tls_init_align) - TLS_TCB_SIZE;
269 #else
270 post_size = 0;
271 #endif
272 tls_block_size = tcbsize + post_size;
273 pre_size = roundup2(tls_block_size, tls_init_align) - tls_block_size;
274 tls_block_size += pre_size + tls_static_space;
275
276 /* Allocate whole TLS block */
277 tls_block = malloc_aligned(tls_block_size, maxalign);
278 if (tls_block == NULL) {
279 tls_msg("__libc_allocate_tls: Out of memory.\n");
280 abort();
281 }
282 memset(tls_block, 0, tls_block_size);
283 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size);
284 tls = (char *)tcb + TLS_TCB_SIZE + post_size;
285
286 if (oldtcb != NULL) {
287 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize),
288 tls_block_size);
289 free_aligned(oldtcb);
290
291 /* Adjust the DTV. */
292 dtv = tcb[0];
293 dtv[2] = (Elf_Addr)(tls + DTV_OFFSET);
294 } else {
295 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
296 if (dtv == NULL) {
297 tls_msg("__libc_allocate_tls: Out of memory.\n");
298 abort();
299 }
300 /* Build the DTV. */
301 tcb[0] = dtv;
302 dtv[0] = 1; /* Generation. */
303 dtv[1] = 1; /* Segments count. */
304 dtv[2] = (Elf_Addr)(tls + DTV_OFFSET);
305
306 if (tls_init_size > 0)
307 memcpy(tls, tls_init, tls_init_size);
308 }
309
310 return (tcb);
311 }
312
313 #endif
314
315 #ifdef TLS_VARIANT_II
316
317 #define TLS_TCB_SIZE (3 * sizeof(Elf_Addr))
318
319 /*
320 * Free Static TLS using the Variant II method.
321 */
322 void
__libc_free_tls(void * tcb,size_t tcbsize __unused,size_t tcbalign)323 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign)
324 {
325 size_t size;
326 Elf_Addr* dtv;
327 Elf_Addr tlsstart, tlsend;
328
329 /*
330 * Figure out the size of the initial TLS block so that we can
331 * find stuff which ___tls_get_addr() allocated dynamically.
332 */
333 tcbalign = MAX(tcbalign, tls_init_align);
334 size = roundup2(tls_static_space, tcbalign);
335
336 dtv = ((Elf_Addr**)tcb)[1];
337 tlsend = (Elf_Addr) tcb;
338 tlsstart = tlsend - size;
339 free_aligned((void*)tlsstart);
340 __je_bootstrap_free(dtv);
341 }
342
343 /*
344 * Allocate Static TLS using the Variant II method.
345 */
346 void *
__libc_allocate_tls(void * oldtls,size_t tcbsize,size_t tcbalign)347 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign)
348 {
349 size_t size;
350 char *tls;
351 Elf_Addr *dtv;
352 Elf_Addr segbase, oldsegbase;
353
354 tcbalign = MAX(tcbalign, tls_init_align);
355 size = roundup2(tls_static_space, tcbalign);
356
357 if (tcbsize < 2 * sizeof(Elf_Addr))
358 tcbsize = 2 * sizeof(Elf_Addr);
359 tls = malloc_aligned(size + tcbsize, tcbalign);
360 if (tls == NULL) {
361 tls_msg("__libc_allocate_tls: Out of memory.\n");
362 abort();
363 }
364 memset(tls, 0, size + tcbsize);
365 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
366 if (dtv == NULL) {
367 tls_msg("__libc_allocate_tls: Out of memory.\n");
368 abort();
369 }
370
371 segbase = (Elf_Addr)(tls + size);
372 ((Elf_Addr*)segbase)[0] = segbase;
373 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv;
374
375 dtv[0] = 1;
376 dtv[1] = 1;
377 dtv[2] = segbase - tls_static_space;
378
379 if (oldtls) {
380 /*
381 * Copy the static TLS block over whole.
382 */
383 oldsegbase = (Elf_Addr) oldtls;
384 memcpy((void *)(segbase - tls_static_space),
385 (const void *)(oldsegbase - tls_static_space),
386 tls_static_space);
387
388 /*
389 * We assume that this block was the one we created with
390 * allocate_initial_tls().
391 */
392 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr));
393 } else {
394 memcpy((void *)(segbase - tls_static_space),
395 tls_init, tls_init_size);
396 memset((void *)(segbase - tls_static_space + tls_init_size),
397 0, tls_static_space - tls_init_size);
398 }
399
400 return (void*) segbase;
401 }
402
403 #endif /* TLS_VARIANT_II */
404
405 #else
406
407 void *
__libc_allocate_tls(void * oldtls __unused,size_t tcbsize __unused,size_t tcbalign __unused)408 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused,
409 size_t tcbalign __unused)
410 {
411 return (0);
412 }
413
414 void
__libc_free_tls(void * tcb __unused,size_t tcbsize __unused,size_t tcbalign __unused)415 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused,
416 size_t tcbalign __unused)
417 {
418 }
419
420 #endif /* PIC */
421
422 extern char **environ;
423
424 void
_init_tls(void)425 _init_tls(void)
426 {
427 #ifndef PIC
428 Elf_Addr *sp;
429 Elf_Auxinfo *aux, *auxp;
430 Elf_Phdr *phdr;
431 size_t phent, phnum;
432 int i;
433 void *tls;
434
435 sp = (Elf_Addr *) environ;
436 while (*sp++ != 0)
437 ;
438 aux = (Elf_Auxinfo *) sp;
439 phdr = NULL;
440 phent = phnum = 0;
441 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) {
442 switch (auxp->a_type) {
443 case AT_PHDR:
444 phdr = auxp->a_un.a_ptr;
445 break;
446
447 case AT_PHENT:
448 phent = auxp->a_un.a_val;
449 break;
450
451 case AT_PHNUM:
452 phnum = auxp->a_un.a_val;
453 break;
454 }
455 }
456 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0)
457 return;
458
459 for (i = 0; (unsigned) i < phnum; i++) {
460 if (phdr[i].p_type == PT_TLS) {
461 tls_static_space = roundup2(phdr[i].p_memsz,
462 phdr[i].p_align);
463 tls_init_size = phdr[i].p_filesz;
464 tls_init_align = phdr[i].p_align;
465 tls_init = (void*) phdr[i].p_vaddr;
466 break;
467 }
468 }
469 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN);
470
471 _set_tp(tls);
472 #endif
473 }
474