1 /*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_APACHE_LICENSE_HEADER_START@
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * @APPLE_APACHE_LICENSE_HEADER_END@
19 */
20
21 /*
22 * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
23 * which are subject to change in future releases of Mac OS X. Any applications
24 * relying on these interfaces WILL break.
25 */
26
27 #ifndef __DISPATCH_VOUCHER_INTERNAL__
28 #define __DISPATCH_VOUCHER_INTERNAL__
29
30 #ifndef __DISPATCH_INDIRECT__
31 #error "Please #include <dispatch/dispatch.h> instead of this file directly."
32 #include <dispatch/base.h> // for HeaderDoc
33 #endif
34
35 #pragma mark -
36 #pragma mark voucher_recipe_t (disabled)
37
38 #if VOUCHER_ENABLE_RECIPE_OBJECTS
39 /*!
40 * @group Voucher Creation SPI
41 * SPI intended for clients that need to create vouchers.
42 */
43
44 #if OS_OBJECT_USE_OBJC
45 OS_OBJECT_DECL(voucher_recipe);
46 #else
47 typedef struct voucher_recipe_s *voucher_recipe_t;
48 #endif
49
50 /*!
51 * @function voucher_create
52 *
53 * @abstract
54 * Creates a new voucher object from a recipe.
55 *
56 * @discussion
57 * Error handling TBD
58 *
59 * @result
60 * The newly created voucher object.
61 */
62 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
63 OS_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW
64 voucher_t
65 voucher_create(voucher_recipe_t recipe);
66 #endif // VOUCHER_ENABLE_RECIPE_OBJECTS
67
68 #if VOUCHER_ENABLE_GET_MACH_VOUCHER
69 /*!
70 * @function voucher_get_mach_voucher
71 *
72 * @abstract
73 * Returns the mach voucher port underlying the specified voucher object.
74 *
75 * @discussion
76 * The caller must either maintain a reference on the voucher object while the
77 * returned mach voucher port is in use to ensure it stays valid for the
78 * duration, or it must retain the mach voucher port with mach_port_mod_refs().
79 *
80 * @param voucher
81 * The voucher object to query.
82 *
83 * @result
84 * A mach voucher port.
85 */
86 __OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0)
87 OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW
88 mach_voucher_t
89 voucher_get_mach_voucher(voucher_t voucher);
90 #endif // VOUCHER_ENABLE_GET_MACH_VOUCHER
91
92 #pragma mark -
93 #pragma mark voucher_t
94
95 #if TARGET_IPHONE_SIMULATOR && \
96 IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000
97 #undef VOUCHER_USE_MACH_VOUCHER
98 #define VOUCHER_USE_MACH_VOUCHER 0
99 #endif
100 #ifndef VOUCHER_USE_MACH_VOUCHER
101 #if __has_include(<mach/mach_voucher.h>)
102 #define VOUCHER_USE_MACH_VOUCHER 1
103 #endif
104 #endif
105
106 #if VOUCHER_USE_MACH_VOUCHER
107 #undef DISPATCH_USE_IMPORTANCE_ASSERTION
108 #define DISPATCH_USE_IMPORTANCE_ASSERTION 0
109 #else
110 #undef MACH_RCV_VOUCHER
111 #define MACH_RCV_VOUCHER 0
112 #endif // VOUCHER_USE_MACH_VOUCHER
113
114 extern voucher_activity_mode_t _voucher_activity_mode;
115
116 void _voucher_init(void);
117 void _voucher_atfork_child(void);
118 void _voucher_activity_heap_pressure_warn(void);
119 void _voucher_activity_heap_pressure_normal(void);
120 void _voucher_xref_dispose(voucher_t voucher);
121 void _voucher_dispose(voucher_t voucher);
122 size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz);
123 void _voucher_thread_cleanup(void *voucher);
124 mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher);
125 voucher_t _voucher_create_without_importance(voucher_t voucher);
126 mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher,
127 pthread_priority_t priority);
128 voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher,
129 pthread_priority_t priority, mach_voucher_t kv);
130 void _voucher_dealloc_mach_voucher(mach_voucher_t kv);
131
132 #if OS_OBJECT_USE_OBJC
133 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher, object)
134 #if VOUCHER_ENABLE_RECIPE_OBJECTS
135 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object)
136 #endif
137 #endif
138
139 #define _TAILQ_IS_ENQUEUED(elm, field) \
140 ((elm)->field.tqe_prev != NULL)
141 #define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \
142 do { (elm)->field.tqe_prev = NULL; } while (0)
143
144 #define VOUCHER_NO_MACH_VOUCHER MACH_PORT_DEAD
145
146 #if VOUCHER_USE_MACH_VOUCHER
147
148 #if DISPATCH_DEBUG
149 #define DISPATCH_VOUCHER_DEBUG 1
150 #define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1
151 #endif
152
153 typedef struct voucher_s {
154 _OS_OBJECT_HEADER(
155 void *os_obj_isa,
156 os_obj_ref_cnt,
157 os_obj_xref_cnt);
158 TAILQ_ENTRY(voucher_s) v_list;
159 mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference
160 voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference
161 _voucher_activity_t v_activity;
162 #if VOUCHER_ENABLE_RECIPE_OBJECTS
163 size_t v_recipe_extra_offset;
164 mach_voucher_attr_recipe_size_t v_recipe_extra_size;
165 #endif
166 unsigned int v_has_priority:1;
167 unsigned int v_activities;
168 mach_voucher_attr_recipe_data_t v_recipes[];
169 } voucher_s;
170
171 #if VOUCHER_ENABLE_RECIPE_OBJECTS
172 typedef struct voucher_recipe_s {
173 _OS_OBJECT_HEADER(
174 const _os_object_class_s *os_obj_isa,
175 os_obj_ref_cnt,
176 os_obj_xref_cnt);
177 size_t vr_allocation_size;
178 mach_voucher_attr_recipe_size_t volatile vr_size;
179 mach_voucher_attr_recipe_t vr_data;
180 } voucher_recipe_s;
181 #endif
182
183 #define _voucher_recipes_base(r) (r[0])
184 #define _voucher_recipes_atm(r) (r[1])
185 #define _voucher_recipes_bits(r) (r[2])
186 #define _voucher_base_recipe(v) (_voucher_recipes_base((v)->v_recipes))
187 #define _voucher_atm_recipe(v) (_voucher_recipes_atm((v)->v_recipes))
188 #define _voucher_bits_recipe(v) (_voucher_recipes_bits((v)->v_recipes))
189 #define _voucher_recipes_size() (3 * sizeof(mach_voucher_attr_recipe_data_t))
190
191 #if TARGET_OS_EMBEDDED
192 #define VL_HASH_SIZE 64u // must be a power of two
193 #else
194 #define VL_HASH_SIZE 256u // must be a power of two
195 #endif
196 #define VL_HASH(kv) (MACH_PORT_INDEX(kv) & (VL_HASH_SIZE - 1))
197
198 typedef uint32_t _voucher_magic_t;
199 const _voucher_magic_t _voucher_magic_v1 = 0x0190cefa; // little-endian FACE9001
200 #define _voucher_recipes_magic(r) ((_voucher_magic_t*) \
201 (_voucher_recipes_bits(r).content))
202 #define _voucher_magic(v) _voucher_recipes_magic((v)->v_recipes)
203 typedef uint32_t _voucher_priority_t;
204 #define _voucher_recipes_priority(r) ((_voucher_priority_t*) \
205 (_voucher_recipes_bits(r).content + sizeof(_voucher_magic_t)))
206 #define _voucher_priority(v) _voucher_recipes_priority((v)->v_recipes)
207 #define _voucher_activity_ids(v) ((voucher_activity_id_t*) \
208 (_voucher_bits_recipe(v).content + sizeof(_voucher_magic_t) + \
209 sizeof(_voucher_priority_t)))
210 #define _voucher_bits_size(activities) \
211 (sizeof(_voucher_magic_t) + sizeof(_voucher_priority_t) + \
212 (activities) * sizeof(voucher_activity_id_t))
213
214 #if VOUCHER_ENABLE_RECIPE_OBJECTS
215 #define _voucher_extra_size(v) ((v)->v_recipe_extra_size)
216 #define _voucher_extra_recipes(v) ((char*)(v) + (v)->v_recipe_extra_offset)
217 #else
218 #define _voucher_extra_size(v) 0
219 #define _voucher_extra_recipes(v) NULL
220 #endif
221
222 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_DEBUG
223 #define _dispatch_voucher_debug(msg, v, ...) \
224 _dispatch_debug("voucher[%p]: " msg, v, ##__VA_ARGS__)
225 #define _dispatch_kvoucher_debug(msg, kv, ...) \
226 _dispatch_debug("kvoucher[0x%08x]: " msg, kv, ##__VA_ARGS__)
227 #define _dispatch_voucher_debug_machport(name) \
228 dispatch_debug_machport((name), __func__)
229 #else
230 #define _dispatch_voucher_debug(msg, v, ...)
231 #define _dispatch_kvoucher_debug(msg, kv, ...)
232 #define _dispatch_voucher_debug_machport(name) ((void)(name))
233 #endif
234
235 #if !(USE_OBJC && __OBJC2__)
236
237 DISPATCH_ALWAYS_INLINE
238 static inline voucher_t
_voucher_retain(voucher_t voucher)239 _voucher_retain(voucher_t voucher)
240 {
241 #if !DISPATCH_VOUCHER_OBJC_DEBUG
242 int xref_cnt = dispatch_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed);
243 _dispatch_voucher_debug("retain -> %d", voucher, xref_cnt + 1);
244 if (slowpath(xref_cnt <= 0)) {
245 _dispatch_voucher_debug("resurrection", voucher);
246 DISPATCH_CRASH("Voucher resurrection");
247 }
248 #else
249 os_retain(voucher);
250 _dispatch_voucher_debug("retain -> %d", voucher,
251 voucher->os_obj_xref_cnt + 1);
252 #endif // DISPATCH_DEBUG
253 return voucher;
254 }
255
256 DISPATCH_ALWAYS_INLINE
257 static inline void
_voucher_release(voucher_t voucher)258 _voucher_release(voucher_t voucher)
259 {
260 #if !DISPATCH_VOUCHER_OBJC_DEBUG
261 int xref_cnt = dispatch_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed);
262 _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1);
263 if (fastpath(xref_cnt >= 0)) {
264 return;
265 }
266 if (slowpath(xref_cnt < -1)) {
267 _dispatch_voucher_debug("overrelease", voucher);
268 DISPATCH_CRASH("Voucher overrelease");
269 }
270 return _os_object_xref_dispose((_os_object_t)voucher);
271 #else
272 _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt);
273 return os_release(voucher);
274 #endif // DISPATCH_DEBUG
275 }
276
277 DISPATCH_ALWAYS_INLINE
278 static inline voucher_t
_voucher_get(void)279 _voucher_get(void)
280 {
281 return _dispatch_thread_getspecific(dispatch_voucher_key);
282 }
283
284 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
285 static inline voucher_t
_voucher_copy(void)286 _voucher_copy(void)
287 {
288 voucher_t voucher = _voucher_get();
289 if (voucher) _voucher_retain(voucher);
290 return voucher;
291 }
292
293 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
294 static inline voucher_t
_voucher_copy_without_importance(void)295 _voucher_copy_without_importance(void)
296 {
297 voucher_t voucher = _voucher_get();
298 if (voucher) voucher = _voucher_create_without_importance(voucher);
299 return voucher;
300 }
301
302 DISPATCH_ALWAYS_INLINE
303 static inline void
_voucher_mach_voucher_set(mach_voucher_t kv)304 _voucher_mach_voucher_set(mach_voucher_t kv)
305 {
306 if (kv == VOUCHER_NO_MACH_VOUCHER) return;
307 _dispatch_set_priority_and_mach_voucher(0, kv);
308 }
309
310 DISPATCH_ALWAYS_INLINE
311 static inline mach_voucher_t
_voucher_swap_and_get_mach_voucher(voucher_t ov,voucher_t voucher)312 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
313 {
314 if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER;
315 _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov);
316 _dispatch_thread_setspecific(dispatch_voucher_key, voucher);
317 mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL;
318 mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL;
319 return (kv != okv) ? kv : VOUCHER_NO_MACH_VOUCHER;
320 }
321
322 DISPATCH_ALWAYS_INLINE
323 static inline void
_voucher_swap(voucher_t ov,voucher_t voucher)324 _voucher_swap(voucher_t ov, voucher_t voucher)
325 {
326 _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher));
327 if (ov) _voucher_release(ov);
328 }
329
330 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
331 static inline voucher_t
_voucher_adopt(voucher_t voucher)332 _voucher_adopt(voucher_t voucher)
333 {
334 voucher_t ov = _voucher_get();
335 _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher));
336 return ov;
337 }
338
339 DISPATCH_ALWAYS_INLINE
340 static inline void
_voucher_replace(voucher_t voucher)341 _voucher_replace(voucher_t voucher)
342 {
343 voucher_t ov = _voucher_get();
344 _voucher_swap(ov, voucher);
345 }
346
347 DISPATCH_ALWAYS_INLINE
348 static inline void
_voucher_clear(void)349 _voucher_clear(void)
350 {
351 _voucher_replace(NULL);
352 }
353
354 DISPATCH_ALWAYS_INLINE
355 static inline pthread_priority_t
_voucher_get_priority(voucher_t voucher)356 _voucher_get_priority(voucher_t voucher)
357 {
358 return voucher && voucher->v_has_priority ?
359 (pthread_priority_t)*_voucher_priority(voucher) : 0;
360 }
361
362 void _voucher_task_mach_voucher_init(void* ctxt);
363 extern dispatch_once_t _voucher_task_mach_voucher_pred;
364 extern mach_voucher_t _voucher_task_mach_voucher;
365
366 DISPATCH_ALWAYS_INLINE
367 static inline mach_voucher_t
_voucher_get_task_mach_voucher(void)368 _voucher_get_task_mach_voucher(void)
369 {
370 dispatch_once_f(&_voucher_task_mach_voucher_pred, NULL,
371 _voucher_task_mach_voucher_init);
372 return _voucher_task_mach_voucher;
373 }
374
375 DISPATCH_ALWAYS_INLINE
376 static inline bool
_voucher_mach_msg_set_mach_voucher(mach_msg_header_t * msg,mach_voucher_t kv,bool move_send)377 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
378 bool move_send)
379 {
380 if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
381 if (!kv) return false;
382 msg->msgh_voucher_port = kv;
383 msg->msgh_bits |= MACH_MSGH_BITS_SET_PORTS(0, 0, move_send ?
384 MACH_MSG_TYPE_MOVE_SEND : MACH_MSG_TYPE_COPY_SEND);
385 _dispatch_kvoucher_debug("msg[%p] set %s", kv, msg, move_send ?
386 "move-send" : "copy-send");
387 _dispatch_voucher_debug_machport(kv);
388 return true;
389 }
390
391 DISPATCH_ALWAYS_INLINE
392 static inline bool
_voucher_mach_msg_set(mach_msg_header_t * msg,voucher_t voucher)393 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
394 {
395 if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false;
396 mach_voucher_t kv;
397 if (voucher) {
398 kv = _voucher_get_mach_voucher(voucher);
399 } else {
400 kv = _voucher_get_task_mach_voucher();
401 }
402 return _voucher_mach_msg_set_mach_voucher(msg, kv, false);
403 }
404
405 DISPATCH_ALWAYS_INLINE
406 static inline mach_voucher_t
_voucher_mach_msg_get(mach_msg_header_t * msg)407 _voucher_mach_msg_get(mach_msg_header_t *msg)
408 {
409 if (!MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return MACH_VOUCHER_NULL;
410 mach_voucher_t kv = msg->msgh_voucher_port;
411 msg->msgh_voucher_port = MACH_VOUCHER_NULL;
412 msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK;
413 return kv;
414 }
415
416 DISPATCH_ALWAYS_INLINE
417 static inline mach_voucher_t
_voucher_mach_msg_clear(mach_msg_header_t * msg,bool move_send)418 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
419 {
420 mach_msg_bits_t kvbits = MACH_MSGH_BITS_VOUCHER(msg->msgh_bits);
421 mach_voucher_t kv = msg->msgh_voucher_port, kvm = MACH_VOUCHER_NULL;
422 if ((kvbits == MACH_MSG_TYPE_COPY_SEND ||
423 kvbits == MACH_MSG_TYPE_MOVE_SEND) && kv) {
424 _dispatch_kvoucher_debug("msg[%p] clear %s", kv, msg, move_send ?
425 "move-send" : "copy-send");
426 _dispatch_voucher_debug_machport(kv);
427 if (kvbits == MACH_MSG_TYPE_MOVE_SEND) {
428 // <rdar://problem/15694142> return/drop received or pseudo-received
429 // voucher reference (e.g. due to send failure).
430 if (move_send) {
431 kvm = kv;
432 } else {
433 _voucher_dealloc_mach_voucher(kv);
434 }
435 }
436 msg->msgh_voucher_port = MACH_VOUCHER_NULL;
437 msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK;
438 }
439 return kvm;
440 }
441
442 #pragma mark -
443 #pragma mark dispatch_continuation_t + voucher_t
444
445 #if DISPATCH_USE_KDEBUG_TRACE
446 DISPATCH_ALWAYS_INLINE
447 static inline void
_dispatch_voucher_ktrace(int code,natural_t voucher,void * container)448 _dispatch_voucher_ktrace(int code, natural_t voucher, void *container)
449 {
450 if (!voucher) return;
451 __kdebug_trace(APPSDBG_CODE(DBG_MACH_CHUD, (0xfac >> 2)) | DBG_FUNC_NONE,
452 code, (int)voucher, (int)(uintptr_t)container,
453 #ifdef __LP64__
454 (int)((uintptr_t)container >> 32)
455 #else
456 0
457 #endif
458 );
459 }
460 #define _dispatch_voucher_ktrace_dc_push(dc) \
461 _dispatch_voucher_ktrace(0x1, (dc)->dc_voucher ? \
462 (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc))
463 #define _dispatch_voucher_ktrace_dc_pop(dc) \
464 _dispatch_voucher_ktrace(0x2, (dc)->dc_voucher ? \
465 (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc))
466 #define _dispatch_voucher_ktrace_dmsg_push(dmsg) \
467 _dispatch_voucher_ktrace(0x3, (dmsg)->dmsg_voucher ? \
468 (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg))
469 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg) \
470 _dispatch_voucher_ktrace(0x4, (dmsg)->dmsg_voucher ? \
471 (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg))
472 #else
473 #define _dispatch_voucher_ktrace_dc_push(dc)
474 #define _dispatch_voucher_ktrace_dc_pop(dc)
475 #define _dispatch_voucher_ktrace_dmsg_push(dmsg)
476 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg)
477 #endif // DISPATCH_USE_KDEBUG_TRACE
478
479 DISPATCH_ALWAYS_INLINE
480 static inline void
_dispatch_continuation_voucher_set(dispatch_continuation_t dc,dispatch_block_flags_t flags)481 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
482 dispatch_block_flags_t flags)
483 {
484 unsigned long bits = (unsigned long)dc->do_vtable;
485 voucher_t v = NULL;
486
487 if (flags & DISPATCH_BLOCK_HAS_VOUCHER) {
488 bits |= DISPATCH_OBJ_HAS_VOUCHER_BIT;
489 } else if (!(flags & DISPATCH_BLOCK_NO_VOUCHER)) {
490 v = _voucher_copy();
491 }
492 dc->do_vtable = (void*)bits;
493 dc->dc_voucher = v;
494 _dispatch_voucher_debug("continuation[%p] set", dc->dc_voucher, dc);
495 _dispatch_voucher_ktrace_dc_push(dc);
496 }
497
498 DISPATCH_ALWAYS_INLINE
499 static inline void
_dispatch_continuation_voucher_adopt(dispatch_continuation_t dc)500 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc)
501 {
502 unsigned long bits = (unsigned long)dc->do_vtable;
503 voucher_t v = DISPATCH_NO_VOUCHER;
504 if (!(bits & DISPATCH_OBJ_HAS_VOUCHER_BIT)) {
505 _dispatch_voucher_ktrace_dc_pop(dc);
506 _dispatch_voucher_debug("continuation[%p] adopt", dc->dc_voucher, dc);
507 v = dc->dc_voucher;
508 dc->dc_voucher = NULL;
509 }
510 _dispatch_adopt_priority_and_replace_voucher(dc->dc_priority, v, 0);
511 }
512
513 #pragma mark -
514 #pragma mark _voucher_activity_heap
515
516 typedef uint32_t _voucher_atm_subid_t;
517 static const size_t _voucher_activity_hash_bits = 6;
518 static const size_t _voucher_activity_hash_size =
519 1 << _voucher_activity_hash_bits;
520 #define VACTID_HASH(x) ((((uint32_t)((x) >> 32) + (uint32_t)(x)) * \
521 2654435761u) >> (32-_voucher_activity_hash_bits))
522 #define VATMID_HASH(x) \
523 (((uint32_t)(x) * 2654435761u) >> (32-_voucher_activity_hash_bits))
524 #define VATMID2ACTID(x) ((uint64_t)(x) << 32)
525 #define VACTID_BASEID(x) ((uint64_t)(x) & (((uint64_t)UINT32_MAX) << 32))
526 #define VACTID_SUBID(x) ((uint32_t)(x))
527 #define VATM_ACTID(vatm, subid) (VATMID2ACTID((vatm)->vatm_id) + (subid))
528 #define VATM_SUBID_BITS2MAX(bits) ((1u << (bits)) - 1)
529 #define VATM_SUBID_MAXBITS (32)
530 #define VATM_SUBID_MAX (ATM_SUBAID32_MAX)
531 #define MAILBOX_OFFSET_UNSET UINT64_MAX
532
533 static const size_t _voucher_activity_buffers_per_heap = 512;
534 typedef unsigned long _voucher_activity_bitmap_base_t;
535 static const size_t _voucher_activity_bits_per_bitmap_base_t =
536 8 * sizeof(_voucher_activity_bitmap_base_t);
537 static const size_t _voucher_activity_bitmaps_per_heap =
538 _voucher_activity_buffers_per_heap /
539 _voucher_activity_bits_per_bitmap_base_t;
540 typedef _voucher_activity_bitmap_base_t
541 _voucher_activity_bitmap_t[_voucher_activity_bitmaps_per_heap];
542
543 typedef struct _voucher_activity_metadata_s {
544 _voucher_activity_buffer_t vam_kernel_metadata;
545 _voucher_activity_buffer_t vam_client_metadata;
546 struct _voucher_activity_self_metadata_s vam_self_metadata;
547 #if __LP64__
548 uintptr_t vam_pad0[7];
549 #else
550 uintptr_t vam_pad0[15];
551 #endif
552 // cacheline
553 _voucher_activity_bitmap_t volatile vam_atm_mbox_bitmap;
554 _voucher_activity_bitmap_t volatile vam_buffer_bitmap;
555 _voucher_activity_bitmap_t volatile vam_pressure_locked_bitmap;
556 // cacheline
557 _voucher_atm_subid_t vam_base_atm_subid;
558 _voucher_atm_subid_t vam_base_atm_subid_max;
559 _voucher_atm_subid_t vam_nested_atm_subid;
560 _voucher_atm_t vam_default_activity_atm;
561 _voucher_atm_t volatile vam_base_atm;
562 voucher_activity_id_t volatile vam_nested_atm_id;
563 #if __LP64__
564 uintptr_t vam_pad2[3];
565 #else
566 uintptr_t vam_pad2[1];
567 #endif
568 _voucher_activity_lock_s vam_base_atm_lock;
569 _voucher_activity_lock_s vam_nested_atm_lock;
570 _voucher_activity_lock_s vam_atms_lock;
571 _voucher_activity_lock_s vam_activities_lock;
572 // cacheline
573 TAILQ_HEAD(, _voucher_atm_s) vam_atms[_voucher_activity_hash_size];
574 TAILQ_HEAD(, _voucher_activity_s)
575 vam_activities[_voucher_activity_hash_size];
576 } *_voucher_activity_metadata_t;
577
578 #pragma mark -
579 #pragma mark _voucher_activity_t
580
581 _voucher_activity_tracepoint_t _voucher_activity_tracepoint_get_slow(
582 unsigned int slots);
583 extern _voucher_activity_t _voucher_activity_default;
584
585 #if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG
586 #define _dispatch_voucher_activity_debug(msg, act, ...) \
587 _dispatch_debug("activity[%p] <0x%x>: atm[%p] <%lld>: " msg, (act), \
588 (act) ? VACTID_SUBID((act)->va_id) : 0, (act) ? (act)->va_atm : NULL, \
589 (act) && (act)->va_atm ? (act)->va_atm->vatm_id : 0, ##__VA_ARGS__)
590 #define _dispatch_voucher_atm_debug(msg, atm, ...) \
591 _dispatch_debug("atm[%p] <%lld> kvoucher[0x%08x]: " msg, (atm), \
592 (atm) ? (atm)->vatm_id : 0, (atm) ? (atm)->vatm_kvoucher : 0, \
593 ##__VA_ARGS__)
594 #else
595 #define _dispatch_voucher_activity_debug(msg, act, ...)
596 #define _dispatch_voucher_atm_debug(msg, atm, ...)
597 #endif
598
599 DISPATCH_ALWAYS_INLINE
600 static inline uint64_t
_voucher_activity_timestamp(void)601 _voucher_activity_timestamp(void)
602 {
603 #if TARGET_IPHONE_SIMULATOR && \
604 IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000
605 return mach_absolute_time();
606 #else
607 return mach_approximate_time();
608 #endif
609 }
610
611 DISPATCH_ALWAYS_INLINE
612 static inline uint64_t
_voucher_activity_thread_id(void)613 _voucher_activity_thread_id(void)
614 {
615 uint64_t thread_id;
616 pthread_threadid_np(NULL, &thread_id); // TODO: 15923074: use TSD thread_id
617 return thread_id;
618 }
619
620 DISPATCH_ALWAYS_INLINE
621 static inline _voucher_activity_tracepoint_t
_voucher_activity_buffer_tracepoint_get(_voucher_activity_buffer_header_t vab,unsigned int slots)622 _voucher_activity_buffer_tracepoint_get(_voucher_activity_buffer_header_t vab,
623 unsigned int slots)
624 {
625 uint32_t idx = dispatch_atomic_add2o(vab, vabh_next_tracepoint_idx,
626 slots, relaxed);
627 if (idx <= _voucher_activity_tracepoints_per_buffer) {
628 return (_voucher_activity_tracepoint_t)vab + (idx - slots);
629 }
630 return NULL;
631 }
632
633 DISPATCH_ALWAYS_INLINE
634 static inline _voucher_activity_tracepoint_t
_voucher_activity_tracepoint_get_from_activity(_voucher_activity_t va,unsigned int slots)635 _voucher_activity_tracepoint_get_from_activity(_voucher_activity_t va,
636 unsigned int slots)
637 {
638 _voucher_activity_buffer_header_t vab = va ? va->va_current_buffer : NULL;
639 return vab ? _voucher_activity_buffer_tracepoint_get(vab, slots) : NULL;
640 }
641
642 DISPATCH_ALWAYS_INLINE
643 static inline _voucher_activity_tracepoint_t
_voucher_activity_tracepoint_get(unsigned int slots)644 _voucher_activity_tracepoint_get(unsigned int slots)
645 {
646 _voucher_activity_t va;
647 voucher_t v = _voucher_get();
648 va = v && v->v_activity ? v->v_activity : _voucher_activity_default;
649 return _voucher_activity_tracepoint_get_from_activity(va, slots);
650 }
651
652 DISPATCH_ALWAYS_INLINE
653 static inline uint64_t
_voucher_activity_tracepoint_init(_voucher_activity_tracepoint_t vat,uint8_t type,uint8_t code_namespace,uint32_t code,uint64_t location)654 _voucher_activity_tracepoint_init(_voucher_activity_tracepoint_t vat,
655 uint8_t type, uint8_t code_namespace, uint32_t code, uint64_t location)
656 {
657 if (!location) location = (uint64_t)__builtin_return_address(0);
658 uint64_t timestamp = _voucher_activity_timestamp();
659 vat->vat_flags = _voucher_activity_trace_flag_tracepoint,
660 vat->vat_type = type,
661 vat->vat_namespace = code_namespace,
662 vat->vat_code = code,
663 vat->vat_timestamp = timestamp,
664 vat->vat_thread = _voucher_activity_thread_id(),
665 vat->vat_location = location;
666 return timestamp;
667 }
668
669 DISPATCH_ALWAYS_INLINE
670 static inline uint64_t
_voucher_activity_tracepoint_init_with_id(_voucher_activity_tracepoint_t vat,voucher_activity_trace_id_t trace_id,uint64_t location)671 _voucher_activity_tracepoint_init_with_id(_voucher_activity_tracepoint_t vat,
672 voucher_activity_trace_id_t trace_id, uint64_t location)
673 {
674 uint8_t type = (uint8_t)(trace_id >> _voucher_activity_trace_id_type_shift);
675 uint8_t cns = (uint8_t)(trace_id >>
676 _voucher_activity_trace_id_code_namespace_shift);
677 uint32_t code = (uint32_t)trace_id;
678 return _voucher_activity_tracepoint_init(vat, type, cns, code, location);
679 }
680
681 DISPATCH_ALWAYS_INLINE
682 static inline bool
_voucher_activity_trace_id_is_subtype(voucher_activity_trace_id_t trace_id,uint8_t type)683 _voucher_activity_trace_id_is_subtype(voucher_activity_trace_id_t trace_id,
684 uint8_t type)
685 {
686 voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0);
687 return (trace_id & type_id) == type_id;
688 }
689 #define _voucher_activity_trace_id_is_subtype(trace_id, name) \
690 _voucher_activity_trace_id_is_subtype(trace_id, \
691 voucher_activity_tracepoint_type_ ## name)
692
693 DISPATCH_ALWAYS_INLINE
694 static inline bool
_voucher_activity_trace_id_enabled(voucher_activity_trace_id_t trace_id)695 _voucher_activity_trace_id_enabled(voucher_activity_trace_id_t trace_id)
696 {
697 switch (_voucher_activity_mode) {
698 case voucher_activity_mode_release:
699 return _voucher_activity_trace_id_is_subtype(trace_id, release);
700 case voucher_activity_mode_stream:
701 case voucher_activity_mode_debug:
702 return _voucher_activity_trace_id_is_subtype(trace_id, debug) ||
703 _voucher_activity_trace_id_is_subtype(trace_id, release);
704 }
705 return false;
706 }
707
708 DISPATCH_ALWAYS_INLINE
709 static inline bool
_voucher_activity_trace_type_enabled(uint8_t type)710 _voucher_activity_trace_type_enabled(uint8_t type)
711 {
712 voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0);
713 return _voucher_activity_trace_id_enabled(type_id);
714 }
715
716 DISPATCH_ALWAYS_INLINE
717 static inline bool
_voucher_activity_disabled(void)718 _voucher_activity_disabled(void)
719 {
720 return slowpath(_voucher_activity_mode == voucher_activity_mode_disable);
721 }
722
723 DISPATCH_ALWAYS_INLINE
724 static inline _voucher_activity_tracepoint_t
_voucher_activity_trace_args_inline(uint8_t type,uint8_t code_namespace,uint32_t code,uintptr_t arg1,uintptr_t arg2,uintptr_t arg3,uintptr_t arg4)725 _voucher_activity_trace_args_inline(uint8_t type, uint8_t code_namespace,
726 uint32_t code, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3,
727 uintptr_t arg4)
728 {
729 if (!_voucher_activity_trace_type_enabled(type)) return NULL;
730 _voucher_activity_tracepoint_t vat;
731 vat = _voucher_activity_tracepoint_get(1);
732 if (!vat) return NULL;
733 _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0);
734 vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args;
735 vat->vat_data[0] = arg1;
736 vat->vat_data[1] = arg2;
737 vat->vat_data[2] = arg3;
738 vat->vat_data[3] = arg4;
739 return vat;
740 }
741
742 DISPATCH_ALWAYS_INLINE
743 static inline _voucher_activity_tracepoint_t
_voucher_activity_trace_with_id_inline(voucher_activity_trace_id_t trace_id)744 _voucher_activity_trace_with_id_inline(voucher_activity_trace_id_t trace_id)
745 {
746 _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1);
747 if (!vat) return NULL;
748 _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0);
749 return vat;
750 }
751
752 DISPATCH_ALWAYS_INLINE
753 static inline _voucher_activity_tracepoint_t
_voucher_activity_trace_with_id(voucher_activity_trace_id_t trace_id)754 _voucher_activity_trace_with_id(voucher_activity_trace_id_t trace_id)
755 {
756 _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1);
757 if (!vat) vat = _voucher_activity_tracepoint_get_slow(1);
758 if (!vat) return NULL;
759 _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0);
760 return vat;
761 }
762
763 DISPATCH_ALWAYS_INLINE
764 static inline void
_voucher_activity_trace_msg(voucher_t v,mach_msg_header_t * msg,uint32_t code)765 _voucher_activity_trace_msg(voucher_t v, mach_msg_header_t *msg, uint32_t code)
766 {
767 if (!v || !v->v_activity) return; // Don't use default activity for IPC
768 const uint8_t type = voucher_activity_tracepoint_type_release;
769 const uint8_t code_namespace = _voucher_activity_tracepoint_namespace_ipc;
770 if (!_voucher_activity_trace_type_enabled(type)) return;
771 _voucher_activity_tracepoint_t vat;
772 vat = _voucher_activity_tracepoint_get_from_activity(v->v_activity, 1);
773 if (!vat) return; // TODO: slowpath ?
774 _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0);
775 vat->vat_flags |= _voucher_activity_trace_flag_libdispatch;
776 #if __has_extension(c_static_assert)
777 _Static_assert(sizeof(mach_msg_header_t) <= sizeof(vat->vat_data),
778 "mach_msg_header_t too large");
779 #endif
780 memcpy(vat->vat_data, msg, sizeof(mach_msg_header_t));
781 }
782 #define _voucher_activity_trace_msg(v, msg, type) \
783 _voucher_activity_trace_msg(v, msg, \
784 _voucher_activity_tracepoint_namespace_ipc_ ## type)
785
786 #endif // !(USE_OBJC && __OBJC2__)
787
788 #else // VOUCHER_USE_MACH_VOUCHER
789
790 #pragma mark -
791 #pragma mark Simulator / vouchers disabled
792
793 #define _dispatch_voucher_debug(msg, v, ...)
794 #define _dispatch_kvoucher_debug(msg, kv, ...)
795
796 DISPATCH_ALWAYS_INLINE
797 static inline voucher_t
798 _voucher_retain(voucher_t voucher)
799 {
800 return voucher;
801 }
802
803 DISPATCH_ALWAYS_INLINE
804 static inline void
805 _voucher_release(voucher_t voucher)
806 {
807 (void)voucher;
808 }
809
810 DISPATCH_ALWAYS_INLINE
811 static inline voucher_t
812 _voucher_get(void)
813 {
814 return NULL;
815 }
816
817 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
818 static inline voucher_t
819 _voucher_copy(void)
820 {
821 return NULL;
822 }
823
824 DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
825 static inline voucher_t
826 _voucher_copy_without_importance(void)
827 {
828 return NULL;
829 }
830
831 DISPATCH_ALWAYS_INLINE
832 static inline mach_voucher_t
833 _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher)
834 {
835 (void)ov; (void)voucher;
836 return MACH_VOUCHER_NULL;
837 }
838
839 DISPATCH_ALWAYS_INLINE
840 static inline voucher_t
841 _voucher_adopt(voucher_t voucher)
842 {
843 return voucher;
844 }
845
846 DISPATCH_ALWAYS_INLINE
847 static inline void
848 _voucher_replace(voucher_t voucher)
849 {
850 (void)voucher;
851 }
852
853 DISPATCH_ALWAYS_INLINE
854 static inline void
855 _voucher_clear(void)
856 {
857 }
858
859 DISPATCH_ALWAYS_INLINE
860 static inline pthread_priority_t
861 _voucher_get_priority(voucher_t voucher)
862 {
863 (void)voucher;
864 return 0;
865 }
866
867 DISPATCH_ALWAYS_INLINE
868 static inline bool
869 _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv,
870 bool move_send)
871 {
872 (void)msg; (void)kv; (void)move_send;
873 return false;
874
875 }
876
877 DISPATCH_ALWAYS_INLINE
878 static inline bool
879 _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher)
880 {
881 (void)msg; (void)voucher;
882 return false;
883 }
884
885 DISPATCH_ALWAYS_INLINE
886 static inline mach_voucher_t
887 _voucher_mach_msg_get(mach_msg_header_t *msg)
888 {
889 (void)msg;
890 return 0;
891 }
892
893 DISPATCH_ALWAYS_INLINE
894 static inline mach_voucher_t
895 _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send)
896 {
897 (void)msg; (void)move_send;
898 return MACH_VOUCHER_NULL;
899 }
900
901 #define _dispatch_voucher_ktrace_dmsg_push(dmsg)
902 #define _dispatch_voucher_ktrace_dmsg_pop(dmsg)
903
904 DISPATCH_ALWAYS_INLINE
905 static inline void
906 _dispatch_continuation_voucher_set(dispatch_continuation_t dc,
907 dispatch_block_flags_t flags)
908 {
909 (void)dc; (void)flags;
910 }
911
912 DISPATCH_ALWAYS_INLINE
913 static inline void
914 _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc)
915 {
916 (void)dc;
917 }
918
919 #define _voucher_activity_trace_msg(v, msg, type)
920
921 DISPATCH_ALWAYS_INLINE
922 static inline bool
923 _voucher_activity_disabled(void)
924 {
925 return true;
926 }
927
928 #endif // VOUCHER_USE_MACH_VOUCHER
929
930 #endif /* __DISPATCH_VOUCHER_INTERNAL__ */
931