1 /*
2 * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_APACHE_LICENSE_HEADER_START@
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * @APPLE_APACHE_LICENSE_HEADER_END@
19 */
20
21 /*
22 * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
23 * which are subject to change in future releases of Mac OS X. Any applications
24 * relying on these interfaces WILL break.
25 */
26
27 #ifndef __DISPATCH_QUEUE_INTERNAL__
28 #define __DISPATCH_QUEUE_INTERNAL__
29
30 #ifndef __DISPATCH_INDIRECT__
31 #error "Please #include <dispatch/dispatch.h> instead of this file directly."
32 #include <dispatch/base.h> // for HeaderDoc
33 #endif
34
35 #if defined(__BLOCKS__) && !defined(DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES)
36 #define DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES 1 // <rdar://problem/10719357>
37 #endif
38
39 /* x86 & cortex-a8 have a 64 byte cacheline */
40 #define DISPATCH_CACHELINE_SIZE 64u
41 #define ROUND_UP_TO_CACHELINE_SIZE(x) \
42 (((x) + (DISPATCH_CACHELINE_SIZE - 1u)) & \
43 ~(DISPATCH_CACHELINE_SIZE - 1u))
44 #define DISPATCH_CACHELINE_ALIGN \
45 __attribute__((__aligned__(DISPATCH_CACHELINE_SIZE)))
46
47
48 #pragma mark -
49 #pragma mark dispatch_queue_t
50
51 #define DISPATCH_QUEUE_HEADER \
52 uint32_t volatile dq_running; \
53 struct dispatch_object_s *volatile dq_items_head; \
54 /* LP64 global queue cacheline boundary */ \
55 struct dispatch_object_s *volatile dq_items_tail; \
56 dispatch_queue_t dq_specific_q; \
57 uint16_t dq_width; \
58 uint16_t dq_is_thread_bound:1; \
59 pthread_priority_t dq_priority; \
60 mach_port_t dq_thread; \
61 mach_port_t volatile dq_tqthread; \
62 uint32_t volatile dq_override; \
63 unsigned long dq_serialnum; \
64 const char *dq_label; \
65 DISPATCH_INTROSPECTION_QUEUE_LIST;
66
67 #define DISPATCH_QUEUE_WIDTH_MAX UINT16_MAX
68
69 #define DISPATCH_QUEUE_CACHELINE_PADDING \
70 char _dq_pad[DISPATCH_QUEUE_CACHELINE_PAD]
71 #ifdef __LP64__
72 #define DISPATCH_QUEUE_CACHELINE_PAD (( \
73 (0*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_LIST_SIZE) \
74 + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE)
75 #else
76 #define DISPATCH_QUEUE_CACHELINE_PAD (( \
77 (13*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_LIST_SIZE) \
78 + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE)
79 #endif
80
81 DISPATCH_CLASS_DECL(queue);
82 struct dispatch_queue_s {
83 DISPATCH_STRUCT_HEADER(queue);
84 DISPATCH_QUEUE_HEADER;
85 DISPATCH_QUEUE_CACHELINE_PADDING; // for static queues only
86 };
87
88 DISPATCH_INTERNAL_SUBCLASS_DECL(queue_root, queue);
89 DISPATCH_INTERNAL_SUBCLASS_DECL(queue_runloop, queue);
90 DISPATCH_INTERNAL_SUBCLASS_DECL(queue_mgr, queue);
91
92 DISPATCH_DECL_INTERNAL_SUBCLASS(dispatch_queue_specific_queue, dispatch_queue);
93 DISPATCH_CLASS_DECL(queue_specific_queue);
94
95 void _dispatch_queue_destroy(dispatch_object_t dou);
96 void _dispatch_queue_dispose(dispatch_queue_t dq);
97 void _dispatch_queue_invoke(dispatch_queue_t dq);
98 void _dispatch_queue_push_list_slow(dispatch_queue_t dq,
99 pthread_priority_t pp, struct dispatch_object_s *obj, unsigned int n,
100 bool retained);
101 void _dispatch_queue_push_slow(dispatch_queue_t dq,
102 pthread_priority_t pp, struct dispatch_object_s *obj, bool retained);
103 unsigned long _dispatch_queue_probe(dispatch_queue_t dq);
104 dispatch_queue_t _dispatch_wakeup(dispatch_object_t dou);
105 dispatch_queue_t _dispatch_queue_wakeup(dispatch_queue_t dq);
106 void _dispatch_queue_wakeup_with_qos(dispatch_queue_t dq,
107 pthread_priority_t pp);
108 void _dispatch_queue_wakeup_with_qos_and_release(dispatch_queue_t dq,
109 pthread_priority_t pp);
110 _dispatch_thread_semaphore_t _dispatch_queue_drain(dispatch_object_t dou);
111 void _dispatch_queue_specific_queue_dispose(dispatch_queue_specific_queue_t
112 dqsq);
113 unsigned long _dispatch_root_queue_probe(dispatch_queue_t dq);
114 void _dispatch_pthread_root_queue_dispose(dispatch_queue_t dq);
115 unsigned long _dispatch_runloop_queue_probe(dispatch_queue_t dq);
116 void _dispatch_runloop_queue_xref_dispose(dispatch_queue_t dq);
117 void _dispatch_runloop_queue_dispose(dispatch_queue_t dq);
118 void _dispatch_mgr_queue_drain(void);
119 unsigned long _dispatch_mgr_queue_probe(dispatch_queue_t dq);
120 #if DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES
121 void _dispatch_mgr_priority_init(void);
122 #else
_dispatch_mgr_priority_init(void)123 static inline void _dispatch_mgr_priority_init(void) {}
124 #endif
125 void _dispatch_after_timer_callback(void *ctxt);
126 void _dispatch_async_redirect_invoke(void *ctxt);
127 void _dispatch_sync_recurse_invoke(void *ctxt);
128 void _dispatch_apply_invoke(void *ctxt);
129 void _dispatch_apply_redirect_invoke(void *ctxt);
130 void _dispatch_barrier_async_detached_f(dispatch_queue_t dq, void *ctxt,
131 dispatch_function_t func);
132 void _dispatch_barrier_trysync_f(dispatch_queue_t dq, void *ctxt,
133 dispatch_function_t func);
134
135 #if DISPATCH_DEBUG
136 void dispatch_debug_queue(dispatch_queue_t dq, const char* str);
137 #else
dispatch_debug_queue(dispatch_queue_t dq DISPATCH_UNUSED,const char * str DISPATCH_UNUSED)138 static inline void dispatch_debug_queue(dispatch_queue_t dq DISPATCH_UNUSED,
139 const char* str DISPATCH_UNUSED) {}
140 #endif
141
142 size_t dispatch_queue_debug(dispatch_queue_t dq, char* buf, size_t bufsiz);
143 size_t _dispatch_queue_debug_attr(dispatch_queue_t dq, char* buf,
144 size_t bufsiz);
145
146 #define DISPATCH_QUEUE_QOS_COUNT 6
147 #define DISPATCH_ROOT_QUEUE_COUNT (DISPATCH_QUEUE_QOS_COUNT * 2)
148
149 // must be in lowest to highest qos order (as encoded in pthread_priority_t)
150 // overcommit qos index values need bit 1 set
151 enum {
152 DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS = 0,
153 DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT,
154 DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS,
155 DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT,
156 DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS,
157 DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT,
158 DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS,
159 DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT,
160 DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS,
161 DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT,
162 DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS,
163 DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT,
164 };
165
166 extern unsigned long volatile _dispatch_queue_serial_numbers;
167 extern struct dispatch_queue_s _dispatch_root_queues[];
168 extern struct dispatch_queue_s _dispatch_mgr_q;
169
170 #if HAVE_PTHREAD_WORKQUEUE_QOS
171 extern pthread_priority_t _dispatch_background_priority;
172 extern pthread_priority_t _dispatch_user_initiated_priority;
173 #endif
174
175 #pragma mark -
176 #pragma mark dispatch_queue_attr_t
177
178 DISPATCH_CLASS_DECL(queue_attr);
179 struct dispatch_queue_attr_s {
180 DISPATCH_STRUCT_HEADER(queue_attr);
181 qos_class_t dqa_qos_class;
182 int dqa_relative_priority;
183 unsigned int dqa_overcommit:1, dqa_concurrent:1;
184 };
185
186 enum {
187 DQA_INDEX_NON_OVERCOMMIT = 0,
188 DQA_INDEX_OVERCOMMIT,
189 };
190
191 enum {
192 DQA_INDEX_CONCURRENT = 0,
193 DQA_INDEX_SERIAL,
194 };
195
196 #define DISPATCH_QUEUE_ATTR_PRIO_COUNT (1 - QOS_MIN_RELATIVE_PRIORITY)
197
198 typedef enum {
199 DQA_INDEX_QOS_CLASS_UNSPECIFIED = 0,
200 DQA_INDEX_QOS_CLASS_MAINTENANCE,
201 DQA_INDEX_QOS_CLASS_BACKGROUND,
202 DQA_INDEX_QOS_CLASS_UTILITY,
203 DQA_INDEX_QOS_CLASS_DEFAULT,
204 DQA_INDEX_QOS_CLASS_USER_INITIATED,
205 DQA_INDEX_QOS_CLASS_USER_INTERACTIVE,
206 } _dispatch_queue_attr_index_qos_class_t;
207
208 extern const struct dispatch_queue_attr_s _dispatch_queue_attrs[]
209 [DISPATCH_QUEUE_ATTR_PRIO_COUNT][2][2];
210
211 #pragma mark -
212 #pragma mark dispatch_continuation_t
213
214 // If dc_vtable is less than 127, then the object is a continuation.
215 // Otherwise, the object has a private layout and memory management rules. The
216 // layout until after 'do_next' must align with normal objects.
217 #if __LP64__
218 #define DISPATCH_CONTINUATION_HEADER(x) \
219 const void *do_vtable; \
220 union { \
221 pthread_priority_t dc_priority; \
222 int dc_cache_cnt; \
223 uintptr_t dc_pad; \
224 }; \
225 struct dispatch_##x##_s *volatile do_next; \
226 struct voucher_s *dc_voucher; \
227 dispatch_function_t dc_func; \
228 void *dc_ctxt; \
229 void *dc_data; \
230 void *dc_other;
231 #define _DISPATCH_SIZEOF_PTR 8
232 #else
233 #define DISPATCH_CONTINUATION_HEADER(x) \
234 const void *do_vtable; \
235 union { \
236 pthread_priority_t dc_priority; \
237 int dc_cache_cnt; \
238 uintptr_t dc_pad; \
239 }; \
240 struct voucher_s *dc_voucher; \
241 struct dispatch_##x##_s *volatile do_next; \
242 dispatch_function_t dc_func; \
243 void *dc_ctxt; \
244 void *dc_data; \
245 void *dc_other;
246 #define _DISPATCH_SIZEOF_PTR 4
247 #endif
248 #define _DISPATCH_CONTINUATION_PTRS 8
249 #if DISPATCH_HW_CONFIG_UP
250 // UP devices don't contend on continuations so we don't need to force them to
251 // occupy a whole cacheline (which is intended to avoid contention)
252 #define DISPATCH_CONTINUATION_SIZE \
253 (_DISPATCH_CONTINUATION_PTRS * _DISPATCH_SIZEOF_PTR)
254 #else
255 #define DISPATCH_CONTINUATION_SIZE ROUND_UP_TO_CACHELINE_SIZE( \
256 (_DISPATCH_CONTINUATION_PTRS * _DISPATCH_SIZEOF_PTR))
257 #endif
258 #define ROUND_UP_TO_CONTINUATION_SIZE(x) \
259 (((x) + (DISPATCH_CONTINUATION_SIZE - 1u)) & \
260 ~(DISPATCH_CONTINUATION_SIZE - 1u))
261
262 #define DISPATCH_OBJ_ASYNC_BIT 0x1
263 #define DISPATCH_OBJ_BARRIER_BIT 0x2
264 #define DISPATCH_OBJ_GROUP_BIT 0x4
265 #define DISPATCH_OBJ_SYNC_SLOW_BIT 0x8
266 #define DISPATCH_OBJ_BLOCK_RELEASE_BIT 0x10
267 #define DISPATCH_OBJ_CTXT_FETCH_BIT 0x20
268 #define DISPATCH_OBJ_HAS_VOUCHER_BIT 0x80
269 // vtables are pointers far away from the low page in memory
270 #define DISPATCH_OBJ_IS_VTABLE(x) ((unsigned long)(x)->do_vtable > 0xfful)
271
272 struct dispatch_continuation_s {
273 DISPATCH_CONTINUATION_HEADER(continuation);
274 };
275 typedef struct dispatch_continuation_s *dispatch_continuation_t;
276
277 #ifndef DISPATCH_CONTINUATION_CACHE_LIMIT
278 #if TARGET_OS_EMBEDDED
279 #define DISPATCH_CONTINUATION_CACHE_LIMIT 112 // one 256k heap for 64 threads
280 #define DISPATCH_CONTINUATION_CACHE_LIMIT_MEMORYSTATUS_PRESSURE_WARN 16
281 #else
282 #define DISPATCH_CONTINUATION_CACHE_LIMIT 65536
283 #define DISPATCH_CONTINUATION_CACHE_LIMIT_MEMORYSTATUS_PRESSURE_WARN 128
284 #endif
285 #endif
286
287 dispatch_continuation_t _dispatch_continuation_alloc_from_heap(void);
288 void _dispatch_continuation_free_to_heap(dispatch_continuation_t c);
289
290 #if DISPATCH_USE_MEMORYSTATUS_SOURCE
291 extern int _dispatch_continuation_cache_limit;
292 void _dispatch_continuation_free_to_cache_limit(dispatch_continuation_t c);
293 #else
294 #define _dispatch_continuation_cache_limit DISPATCH_CONTINUATION_CACHE_LIMIT
295 #define _dispatch_continuation_free_to_cache_limit(c) \
296 _dispatch_continuation_free_to_heap(c)
297 #endif
298
299 #pragma mark -
300 #pragma mark dispatch_apply_t
301
302 struct dispatch_apply_s {
303 size_t volatile da_index, da_todo;
304 size_t da_iterations, da_nested;
305 dispatch_continuation_t da_dc;
306 _dispatch_thread_semaphore_t da_sema;
307 uint32_t da_thr_cnt;
308 };
309 typedef struct dispatch_apply_s *dispatch_apply_t;
310
311 #pragma mark -
312 #pragma mark dispatch_block_t
313
314 #ifdef __BLOCKS__
315
316 #define DISPATCH_BLOCK_API_MASK (0x80u - 1)
317 #define DISPATCH_BLOCK_HAS_VOUCHER (1u << 31)
318 #define DISPATCH_BLOCK_HAS_PRIORITY (1u << 30)
319
320 struct dispatch_block_private_data_s {
321 unsigned long dbpd_magic;
322 dispatch_block_flags_t dbpd_flags;
323 unsigned int volatile dbpd_atomic_flags;
324 int volatile dbpd_performed;
325 pthread_priority_t dbpd_priority;
326 voucher_t dbpd_voucher;
327 dispatch_block_t dbpd_block;
328 struct dispatch_semaphore_s dbpd_group;
329 dispatch_queue_t volatile dbpd_queue;
330 mach_port_t dbpd_thread;
331 };
332 typedef struct dispatch_block_private_data_s *dispatch_block_private_data_t;
333
334 // dbpd_atomic_flags bits
335 #define DBF_CANCELED 1u // block has been cancelled
336 #define DBF_WAITING 2u // dispatch_block_wait has begun
337 #define DBF_WAITED 4u // dispatch_block_wait has finished without timeout
338 #define DBF_PERFORM 8u // dispatch_block_perform: don't group_leave
339
340 #define DISPATCH_BLOCK_PRIVATE_DATA_MAGIC 0xD159B10C // 0xDISPatch_BLOCk
341
342 #define DISPATCH_BLOCK_PRIVATE_DATA_INITIALIZER(flags, voucher, prio, block) \
343 { \
344 .dbpd_magic = DISPATCH_BLOCK_PRIVATE_DATA_MAGIC, \
345 .dbpd_flags = (flags), \
346 .dbpd_priority = (prio), \
347 .dbpd_voucher = (voucher), \
348 .dbpd_block = (block), \
349 .dbpd_group = DISPATCH_GROUP_INITIALIZER(1), \
350 }
351
352 dispatch_block_t _dispatch_block_create(dispatch_block_flags_t flags,
353 voucher_t voucher, pthread_priority_t priority, dispatch_block_t block);
354 void _dispatch_block_invoke(const struct dispatch_block_private_data_s *dbcpd);
355
356 #endif /* __BLOCKS__ */
357
358 #endif
359