1 /*-
2 * Copyright (c) 2004 Doug Rabson
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29 /*
30 * Define stubs for TLS internals so that programs and libraries can
31 * link. These functions will be replaced by functional versions at
32 * runtime from ld-elf.so.1.
33 */
34
35 #include <sys/cdefs.h>
36 #include <sys/param.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #include <elf.h>
40 #include <unistd.h>
41
42 #include "libc_private.h"
43
44 #define tls_assert(cond) ((cond) ? (void) 0 : \
45 (tls_msg(#cond ": assert failed: " __FILE__ ":" \
46 __XSTRING(__LINE__) "\n"), abort()))
47 #define tls_msg(s) write(STDOUT_FILENO, s, strlen(s))
48
49 /* Provided by jemalloc to avoid bootstrapping issues. */
50 void *__je_bootstrap_malloc(size_t size);
51 void *__je_bootstrap_calloc(size_t num, size_t size);
52 void __je_bootstrap_free(void *ptr);
53
54 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls);
55 __weak_reference(__libc_free_tls, _rtld_free_tls);
56
57 #ifdef __i386__
58
59 __weak_reference(___libc_tls_get_addr, ___tls_get_addr);
60 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *);
61
62 #endif
63
64 void * __libc_tls_get_addr(void *);
65 __weak_reference(__libc_tls_get_addr, __tls_get_addr);
66
67 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
68 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
69 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
70 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
71
72 #if defined(__amd64__)
73 #define TLS_TCB_ALIGN 16
74 #elif defined(__aarch64__) || defined(__arm__) || defined(__i386__) || \
75 defined(__mips__) || defined(__powerpc__) || defined(__riscv__) || \
76 defined(__sparc64__)
77 #define TLS_TCB_ALIGN sizeof(void *)
78 #else
79 #error TLS_TCB_ALIGN undefined for target architecture
80 #endif
81
82 #if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \
83 defined(__powerpc__) || defined(__riscv__)
84 #define TLS_VARIANT_I
85 #endif
86 #if defined(__i386__) || defined(__amd64__) || defined(__sparc64__)
87 #define TLS_VARIANT_II
88 #endif
89
90 #if defined(__mips__) || defined(__powerpc__) || defined(__riscv)
91 #define DTV_OFFSET 0x8000
92 #else
93 #define DTV_OFFSET 0
94 #endif
95
96 #ifndef PIC
97
98 static size_t tls_static_space;
99 static size_t tls_init_size;
100 static size_t tls_init_align;
101 static void *tls_init;
102 #endif
103
104 #ifdef __i386__
105
106 /* GNU ABI */
107
108 __attribute__((__regparm__(1)))
109 void *
___libc_tls_get_addr(void * ti __unused)110 ___libc_tls_get_addr(void *ti __unused)
111 {
112 return (0);
113 }
114
115 #endif
116
117 void *
__libc_tls_get_addr(void * ti __unused)118 __libc_tls_get_addr(void *ti __unused)
119 {
120 return (0);
121 }
122
123 #ifndef PIC
124
125 static void *
malloc_aligned(size_t size,size_t align)126 malloc_aligned(size_t size, size_t align)
127 {
128 void *mem, *res;
129
130 if (align < sizeof(void *))
131 align = sizeof(void *);
132
133 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1);
134 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align);
135 *(void **)((uintptr_t)res - sizeof(void *)) = mem;
136 return (res);
137 }
138
139 static void
free_aligned(void * ptr)140 free_aligned(void *ptr)
141 {
142 void *mem;
143 uintptr_t x;
144
145 if (ptr == NULL)
146 return;
147
148 x = (uintptr_t)ptr;
149 x -= sizeof(void *);
150 mem = *(void **)x;
151 __je_bootstrap_free(mem);
152 }
153
154 #ifdef TLS_VARIANT_I
155
156 /*
157 * There are two versions of variant I of TLS
158 *
159 * - ARM and aarch64 uses original variant I as is described in [1] and [2],
160 * where TP points to start of TCB followed by aligned TLS segment.
161 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0]
162 * points to DTV vector and DTV values are real addresses (without bias).
163 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to
164 * TLS variables are computed by linker, so we cannot overalign TLS section.
165 *
166 * - MIPS, PowerPC and RISC-V use modified version of variant I,
167 * described in [3] where TP points (with bias) to TLS and TCB immediately
168 * precedes TLS without any alignment gap[4]. Only TLS should be aligned.
169 * The TCB[0] points to DTV vector and DTV values are biased by constant
170 * value (0x8000) from real addresses[5].
171 *
172 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage
173 * www.akkadia.org/drepper/tls.pdf
174 *
175 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r)
176 * Architecture
177 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf
178 *
179 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification
180 * https://members.openpowerfoundation.org/document/dl/576
181 *
182 * [4] Its unclear if "without any alignment gap" is hard ABI requirement,
183 * but we must follow this rule due to suboptimal _set_tp()
184 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but
185 * TCB as argument.
186 *
187 * [5] I'm not able to validate "values are biased" assertions.
188 */
189
190 #define TLS_TCB_SIZE (2 * sizeof(void *))
191
192 /*
193 * Return pointer to allocated TLS block
194 */
195 static void *
get_tls_block_ptr(void * tcb,size_t tcbsize)196 get_tls_block_ptr(void *tcb, size_t tcbsize)
197 {
198 size_t extra_size, post_size, pre_size, tls_block_size;
199
200 /* Compute fragments sizes. */
201 extra_size = tcbsize - TLS_TCB_SIZE;
202 #if defined(__aarch64__) || defined(__arm__)
203 post_size = roundup2(TLS_TCB_SIZE, tls_init_align) - TLS_TCB_SIZE;
204 #else
205 post_size = 0;
206 #endif
207 tls_block_size = tcbsize + post_size;
208 pre_size = roundup2(tls_block_size, tls_init_align) - tls_block_size;
209
210 return ((char *)tcb - pre_size - extra_size);
211 }
212
213 /*
214 * Free Static TLS using the Variant I method. The tcbsize
215 * and tcbalign parameters must be the same as those used to allocate
216 * the block.
217 */
218 void
__libc_free_tls(void * tcb,size_t tcbsize,size_t tcbalign __unused)219 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused)
220 {
221 Elf_Addr *dtv;
222 Elf_Addr **tls;
223
224 tls = (Elf_Addr **)tcb;
225 dtv = tls[0];
226 __je_bootstrap_free(dtv);
227 free_aligned(get_tls_block_ptr(tcb, tcbsize));
228 }
229
230 /*
231 * Allocate Static TLS using the Variant I method.
232 *
233 * To handle all above requirements, we setup the following layout for
234 * TLS block:
235 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align))
236 *
237 * +----------+--------------+--------------+-----------+------------------+
238 * | pre gap | extended TCB | TCB | post gap | TLS segment |
239 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space |
240 * +----------+--------------+--------------+-----------+------------------+
241 *
242 * where:
243 * extra_size is tcbsize - TLS_TCB_SIZE
244 * post_size is used to adjust TCB to TLS aligment for first version of TLS
245 * layout and is always 0 for second version.
246 * pre_size is used to adjust TCB aligment for first version and to adjust
247 * TLS alignment for second version.
248 *
249 */
250 void *
__libc_allocate_tls(void * oldtcb,size_t tcbsize,size_t tcbalign)251 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign)
252 {
253 Elf_Addr *dtv, **tcb;
254 char *tls_block, *tls;
255 size_t extra_size, maxalign, post_size, pre_size, tls_block_size;
256
257 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE)
258 return (oldtcb);
259
260 tls_assert(tcbalign >= TLS_TCB_ALIGN);
261 maxalign = MAX(tcbalign, tls_init_align);
262
263 /* Compute fragmets sizes. */
264 extra_size = tcbsize - TLS_TCB_SIZE;
265 #if defined(__aarch64__) || defined(__arm__)
266 post_size = roundup2(TLS_TCB_SIZE, tls_init_align) - TLS_TCB_SIZE;
267 #else
268 post_size = 0;
269 #endif
270 tls_block_size = tcbsize + post_size;
271 pre_size = roundup2(tls_block_size, tls_init_align) - tls_block_size;
272 tls_block_size += pre_size + tls_static_space;
273
274 /* Allocate whole TLS block */
275 tls_block = malloc_aligned(tls_block_size, maxalign);
276 if (tls_block == NULL) {
277 tls_msg("__libc_allocate_tls: Out of memory.\n");
278 abort();
279 }
280 memset(tls_block, 0, tls_block_size);
281 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size);
282 tls = (char *)tcb + TLS_TCB_SIZE + post_size;
283
284 if (oldtcb != NULL) {
285 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize),
286 tls_block_size);
287 free_aligned(oldtcb);
288
289 /* Adjust the DTV. */
290 dtv = tcb[0];
291 dtv[2] = (Elf_Addr)(tls + DTV_OFFSET);
292 } else {
293 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
294 if (dtv == NULL) {
295 tls_msg("__libc_allocate_tls: Out of memory.\n");
296 abort();
297 }
298 /* Build the DTV. */
299 tcb[0] = dtv;
300 dtv[0] = 1; /* Generation. */
301 dtv[1] = 1; /* Segments count. */
302 dtv[2] = (Elf_Addr)(tls + DTV_OFFSET);
303
304 if (tls_init_size > 0)
305 memcpy(tls, tls_init, tls_init_size);
306 }
307
308 return (tcb);
309 }
310
311 #endif
312
313 #ifdef TLS_VARIANT_II
314
315 #define TLS_TCB_SIZE (3 * sizeof(Elf_Addr))
316
317 /*
318 * Free Static TLS using the Variant II method.
319 */
320 void
__libc_free_tls(void * tcb,size_t tcbsize __unused,size_t tcbalign)321 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign)
322 {
323 size_t size;
324 Elf_Addr* dtv;
325 Elf_Addr tlsstart, tlsend;
326
327 /*
328 * Figure out the size of the initial TLS block so that we can
329 * find stuff which ___tls_get_addr() allocated dynamically.
330 */
331 tcbalign = MAX(tcbalign, tls_init_align);
332 size = roundup2(tls_static_space, tcbalign);
333
334 dtv = ((Elf_Addr**)tcb)[1];
335 tlsend = (Elf_Addr) tcb;
336 tlsstart = tlsend - size;
337 free_aligned((void*)tlsstart);
338 __je_bootstrap_free(dtv);
339 }
340
341 /*
342 * Allocate Static TLS using the Variant II method.
343 */
344 void *
__libc_allocate_tls(void * oldtls,size_t tcbsize,size_t tcbalign)345 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign)
346 {
347 size_t size;
348 char *tls;
349 Elf_Addr *dtv;
350 Elf_Addr segbase, oldsegbase;
351
352 tcbalign = MAX(tcbalign, tls_init_align);
353 size = roundup2(tls_static_space, tcbalign);
354
355 if (tcbsize < 2 * sizeof(Elf_Addr))
356 tcbsize = 2 * sizeof(Elf_Addr);
357 tls = malloc_aligned(size + tcbsize, tcbalign);
358 if (tls == NULL) {
359 tls_msg("__libc_allocate_tls: Out of memory.\n");
360 abort();
361 }
362 memset(tls, 0, size + tcbsize);
363 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
364 if (dtv == NULL) {
365 tls_msg("__libc_allocate_tls: Out of memory.\n");
366 abort();
367 }
368
369 segbase = (Elf_Addr)(tls + size);
370 ((Elf_Addr*)segbase)[0] = segbase;
371 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv;
372
373 dtv[0] = 1;
374 dtv[1] = 1;
375 dtv[2] = segbase - tls_static_space;
376
377 if (oldtls) {
378 /*
379 * Copy the static TLS block over whole.
380 */
381 oldsegbase = (Elf_Addr) oldtls;
382 memcpy((void *)(segbase - tls_static_space),
383 (const void *)(oldsegbase - tls_static_space),
384 tls_static_space);
385
386 /*
387 * We assume that this block was the one we created with
388 * allocate_initial_tls().
389 */
390 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr));
391 } else {
392 memcpy((void *)(segbase - tls_static_space),
393 tls_init, tls_init_size);
394 memset((void *)(segbase - tls_static_space + tls_init_size),
395 0, tls_static_space - tls_init_size);
396 }
397
398 return (void*) segbase;
399 }
400
401 #endif /* TLS_VARIANT_II */
402
403 #else
404
405 void *
__libc_allocate_tls(void * oldtls __unused,size_t tcbsize __unused,size_t tcbalign __unused)406 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused,
407 size_t tcbalign __unused)
408 {
409 return (0);
410 }
411
412 void
__libc_free_tls(void * tcb __unused,size_t tcbsize __unused,size_t tcbalign __unused)413 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused,
414 size_t tcbalign __unused)
415 {
416 }
417
418 #endif /* PIC */
419
420 extern char **environ;
421
422 void
_init_tls(void)423 _init_tls(void)
424 {
425 #ifndef PIC
426 Elf_Addr *sp;
427 Elf_Auxinfo *aux, *auxp;
428 Elf_Phdr *phdr;
429 size_t phent, phnum;
430 int i;
431 void *tls;
432
433 sp = (Elf_Addr *) environ;
434 while (*sp++ != 0)
435 ;
436 aux = (Elf_Auxinfo *) sp;
437 phdr = NULL;
438 phent = phnum = 0;
439 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) {
440 switch (auxp->a_type) {
441 case AT_PHDR:
442 phdr = auxp->a_un.a_ptr;
443 break;
444
445 case AT_PHENT:
446 phent = auxp->a_un.a_val;
447 break;
448
449 case AT_PHNUM:
450 phnum = auxp->a_un.a_val;
451 break;
452 }
453 }
454 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0)
455 return;
456
457 for (i = 0; (unsigned) i < phnum; i++) {
458 if (phdr[i].p_type == PT_TLS) {
459 tls_static_space = roundup2(phdr[i].p_memsz,
460 phdr[i].p_align);
461 tls_init_size = phdr[i].p_filesz;
462 tls_init_align = phdr[i].p_align;
463 tls_init = (void*) phdr[i].p_vaddr;
464 break;
465 }
466 }
467 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN);
468
469 _set_tp(tls);
470 #endif
471 }
472