Lines Matching refs:gtid
79 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_tas_lock_timed_template() argument
84 if ((curr != 0) && (curr != gtid + 1)) in __kmp_acquire_tas_lock_timed_template()
90 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); in __kmp_acquire_tas_lock_timed_template()
118 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_tas_lock() argument
119 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid); in __kmp_acquire_tas_lock()
124 kmp_int32 gtid) { in __kmp_acquire_tas_lock_with_checks() argument
130 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) { in __kmp_acquire_tas_lock_with_checks()
133 return __kmp_acquire_tas_lock(lck, gtid); in __kmp_acquire_tas_lock_with_checks()
136 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_test_tas_lock() argument
138 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); in __kmp_test_tas_lock()
148 kmp_int32 gtid) { in __kmp_test_tas_lock_with_checks() argument
154 return __kmp_test_tas_lock(lck, gtid); in __kmp_test_tas_lock_with_checks()
157 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_release_tas_lock() argument
169 kmp_int32 gtid) { in __kmp_release_tas_lock_with_checks() argument
179 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) && in __kmp_release_tas_lock_with_checks()
180 (__kmp_get_tas_lock_owner(lck) != gtid)) { in __kmp_release_tas_lock_with_checks()
183 return __kmp_release_tas_lock(lck, gtid); in __kmp_release_tas_lock_with_checks()
206 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_tas_lock() argument
207 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_acquire_nested_tas_lock()
209 if (__kmp_get_tas_lock_owner(lck) == gtid) { in __kmp_acquire_nested_tas_lock()
213 __kmp_acquire_tas_lock_timed_template(lck, gtid); in __kmp_acquire_nested_tas_lock()
220 kmp_int32 gtid) { in __kmp_acquire_nested_tas_lock_with_checks() argument
225 return __kmp_acquire_nested_tas_lock(lck, gtid); in __kmp_acquire_nested_tas_lock_with_checks()
228 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_tas_lock() argument
231 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_test_nested_tas_lock()
233 if (__kmp_get_tas_lock_owner(lck) == gtid) { in __kmp_test_nested_tas_lock()
235 } else if (!__kmp_test_tas_lock(lck, gtid)) { in __kmp_test_nested_tas_lock()
245 kmp_int32 gtid) { in __kmp_test_nested_tas_lock_with_checks() argument
250 return __kmp_test_nested_tas_lock(lck, gtid); in __kmp_test_nested_tas_lock_with_checks()
253 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_tas_lock() argument
254 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_release_nested_tas_lock()
258 __kmp_release_tas_lock(lck, gtid); in __kmp_release_nested_tas_lock()
265 kmp_int32 gtid) { in __kmp_release_nested_tas_lock_with_checks() argument
274 if (__kmp_get_tas_lock_owner(lck) != gtid) { in __kmp_release_nested_tas_lock_with_checks()
277 return __kmp_release_nested_tas_lock(lck, gtid); in __kmp_release_nested_tas_lock_with_checks()
320 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_futex_lock_timed_template() argument
321 kmp_int32 gtid_code = (gtid + 1) << 1; in __kmp_acquire_futex_lock_timed_template()
334 lck, lck->lk.poll, gtid)); in __kmp_acquire_futex_lock_timed_template()
346 lck, gtid, poll_val, cond)); in __kmp_acquire_futex_lock_timed_template()
362 lck, lck->lk.poll, gtid)); in __kmp_acquire_futex_lock_timed_template()
369 lck->lk.poll, gtid)); in __kmp_acquire_futex_lock_timed_template()
375 lck, gtid, poll_val)); in __kmp_acquire_futex_lock_timed_template()
382 lck, gtid, poll_val, rc, errno)); in __kmp_acquire_futex_lock_timed_template()
388 lck, gtid, poll_val)); in __kmp_acquire_futex_lock_timed_template()
397 lck->lk.poll, gtid)); in __kmp_acquire_futex_lock_timed_template()
401 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_futex_lock() argument
402 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid); in __kmp_acquire_futex_lock()
407 kmp_int32 gtid) { in __kmp_acquire_futex_lock_with_checks() argument
413 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) { in __kmp_acquire_futex_lock_with_checks()
416 return __kmp_acquire_futex_lock(lck, gtid); in __kmp_acquire_futex_lock_with_checks()
419 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_test_futex_lock() argument
421 KMP_LOCK_BUSY((gtid + 1) << 1, futex))) { in __kmp_test_futex_lock()
429 kmp_int32 gtid) { in __kmp_test_futex_lock_with_checks() argument
435 return __kmp_test_futex_lock(lck, gtid); in __kmp_test_futex_lock_with_checks()
438 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_release_futex_lock() argument
442 lck, lck->lk.poll, gtid)); in __kmp_release_futex_lock()
450 lck, gtid, poll_val)); in __kmp_release_futex_lock()
455 lck, gtid)); in __kmp_release_futex_lock()
463 lck->lk.poll, gtid)); in __kmp_release_futex_lock()
470 kmp_int32 gtid) { in __kmp_release_futex_lock_with_checks() argument
480 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) && in __kmp_release_futex_lock_with_checks()
481 (__kmp_get_futex_lock_owner(lck) != gtid)) { in __kmp_release_futex_lock_with_checks()
484 return __kmp_release_futex_lock(lck, gtid); in __kmp_release_futex_lock_with_checks()
507 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_futex_lock() argument
508 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_acquire_nested_futex_lock()
510 if (__kmp_get_futex_lock_owner(lck) == gtid) { in __kmp_acquire_nested_futex_lock()
514 __kmp_acquire_futex_lock_timed_template(lck, gtid); in __kmp_acquire_nested_futex_lock()
521 kmp_int32 gtid) { in __kmp_acquire_nested_futex_lock_with_checks() argument
526 return __kmp_acquire_nested_futex_lock(lck, gtid); in __kmp_acquire_nested_futex_lock_with_checks()
529 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_futex_lock() argument
532 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_test_nested_futex_lock()
534 if (__kmp_get_futex_lock_owner(lck) == gtid) { in __kmp_test_nested_futex_lock()
536 } else if (!__kmp_test_futex_lock(lck, gtid)) { in __kmp_test_nested_futex_lock()
546 kmp_int32 gtid) { in __kmp_test_nested_futex_lock_with_checks() argument
551 return __kmp_test_nested_futex_lock(lck, gtid); in __kmp_test_nested_futex_lock_with_checks()
554 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_futex_lock() argument
555 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_release_nested_futex_lock()
559 __kmp_release_futex_lock(lck, gtid); in __kmp_release_nested_futex_lock()
566 kmp_int32 gtid) { in __kmp_release_nested_futex_lock_with_checks() argument
575 if (__kmp_get_futex_lock_owner(lck) != gtid) { in __kmp_release_nested_futex_lock_with_checks()
578 return __kmp_release_nested_futex_lock(lck, gtid); in __kmp_release_nested_futex_lock_with_checks()
625 kmp_int32 gtid) { in __kmp_acquire_ticket_lock_timed_template() argument
644 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_ticket_lock() argument
645 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid); in __kmp_acquire_ticket_lock()
650 kmp_int32 gtid) { in __kmp_acquire_ticket_lock_with_checks() argument
663 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) { in __kmp_acquire_ticket_lock_with_checks()
667 __kmp_acquire_ticket_lock(lck, gtid); in __kmp_acquire_ticket_lock_with_checks()
669 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1, in __kmp_acquire_ticket_lock_with_checks()
674 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_test_ticket_lock() argument
691 kmp_int32 gtid) { in __kmp_test_ticket_lock_with_checks() argument
705 int retval = __kmp_test_ticket_lock(lck, gtid); in __kmp_test_ticket_lock_with_checks()
708 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1, in __kmp_test_ticket_lock_with_checks()
714 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_release_ticket_lock() argument
729 kmp_int32 gtid) { in __kmp_release_ticket_lock_with_checks() argument
745 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) && in __kmp_release_ticket_lock_with_checks()
746 (__kmp_get_ticket_lock_owner(lck) != gtid)) { in __kmp_release_ticket_lock_with_checks()
750 return __kmp_release_ticket_lock(lck, gtid); in __kmp_release_ticket_lock_with_checks()
805 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_ticket_lock() argument
806 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_acquire_nested_ticket_lock()
808 if (__kmp_get_ticket_lock_owner(lck) == gtid) { in __kmp_acquire_nested_ticket_lock()
813 __kmp_acquire_ticket_lock_timed_template(lck, gtid); in __kmp_acquire_nested_ticket_lock()
816 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1, in __kmp_acquire_nested_ticket_lock()
823 kmp_int32 gtid) { in __kmp_acquire_nested_ticket_lock_with_checks() argument
836 return __kmp_acquire_nested_ticket_lock(lck, gtid); in __kmp_acquire_nested_ticket_lock_with_checks()
839 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_ticket_lock() argument
842 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_test_nested_ticket_lock()
844 if (__kmp_get_ticket_lock_owner(lck) == gtid) { in __kmp_test_nested_ticket_lock()
848 } else if (!__kmp_test_ticket_lock(lck, gtid)) { in __kmp_test_nested_ticket_lock()
853 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1, in __kmp_test_nested_ticket_lock()
861 kmp_int32 gtid) { in __kmp_test_nested_ticket_lock_with_checks() argument
874 return __kmp_test_nested_ticket_lock(lck, gtid); in __kmp_test_nested_ticket_lock_with_checks()
877 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_ticket_lock() argument
878 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_release_nested_ticket_lock()
884 __kmp_release_ticket_lock(lck, gtid); in __kmp_release_nested_ticket_lock()
891 kmp_int32 gtid) { in __kmp_release_nested_ticket_lock_with_checks() argument
907 if (__kmp_get_ticket_lock_owner(lck) != gtid) { in __kmp_release_nested_ticket_lock_with_checks()
910 return __kmp_release_nested_ticket_lock(lck, gtid); in __kmp_release_nested_ticket_lock_with_checks()
1035 static void __kmp_dump_queuing_lock(kmp_info_t *this_thr, kmp_int32 gtid, in __kmp_dump_queuing_lock() argument
1053 gtid + 1, this_thr->th.th_spin_here, in __kmp_dump_queuing_lock()
1085 kmp_int32 gtid) { in __kmp_acquire_queuing_lock_timed_template() argument
1086 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid); in __kmp_acquire_queuing_lock_timed_template()
1096 ("__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid)); in __kmp_acquire_queuing_lock_timed_template()
1103 TRACE_LOCK(gtid + 1, "acq ent"); in __kmp_acquire_queuing_lock_timed_template()
1105 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_acquire_queuing_lock_timed_template()
1107 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_acquire_queuing_lock_timed_template()
1133 TRACE_LOCK_HT(gtid + 1, "acq read: ", head, tail); in __kmp_acquire_queuing_lock_timed_template()
1143 KMP_PACK_64(gtid + 1, gtid + 1)); in __kmp_acquire_queuing_lock_timed_template()
1146 TRACE_LOCK(gtid + 1, "acq enq: (-1,0)->(tid,tid)"); in __kmp_acquire_queuing_lock_timed_template()
1152 KMP_DEBUG_ASSERT(tail != gtid + 1); in __kmp_acquire_queuing_lock_timed_template()
1155 TRACE_LOCK_HT(gtid + 1, "acq read: ", head, tail); in __kmp_acquire_queuing_lock_timed_template()
1162 enqueued = KMP_COMPARE_AND_STORE_ACQ32(tail_id_p, tail, gtid + 1); in __kmp_acquire_queuing_lock_timed_template()
1166 TRACE_LOCK(gtid + 1, "acq enq: (h,t)->(h,tid)"); in __kmp_acquire_queuing_lock_timed_template()
1177 TRACE_LOCK_HT(gtid + 1, "acq read: ", head, tail); in __kmp_acquire_queuing_lock_timed_template()
1192 lck, gtid)); in __kmp_acquire_queuing_lock_timed_template()
1194 TRACE_LOCK_HT(gtid + 1, "acq exit: ", head, 0); in __kmp_acquire_queuing_lock_timed_template()
1225 tail_thr->th.th_next_waiting = gtid + 1; in __kmp_acquire_queuing_lock_timed_template()
1230 lck, gtid)); in __kmp_acquire_queuing_lock_timed_template()
1240 TRACE_LOCK(gtid + 1, "acq spin"); in __kmp_acquire_queuing_lock_timed_template()
1243 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_acquire_queuing_lock_timed_template()
1248 lck, gtid)); in __kmp_acquire_queuing_lock_timed_template()
1251 TRACE_LOCK(gtid + 1, "acq exit 2"); in __kmp_acquire_queuing_lock_timed_template()
1270 TRACE_LOCK(gtid + 1, "acq retry"); in __kmp_acquire_queuing_lock_timed_template()
1277 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_queuing_lock() argument
1278 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_acquire_queuing_lock()
1280 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid); in __kmp_acquire_queuing_lock()
1285 kmp_int32 gtid) { in __kmp_acquire_queuing_lock_with_checks() argument
1293 if (__kmp_get_queuing_lock_owner(lck) == gtid) { in __kmp_acquire_queuing_lock_with_checks()
1297 __kmp_acquire_queuing_lock(lck, gtid); in __kmp_acquire_queuing_lock_with_checks()
1299 lck->lk.owner_id = gtid + 1; in __kmp_acquire_queuing_lock_with_checks()
1303 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_test_queuing_lock() argument
1310 KA_TRACE(1000, ("__kmp_test_queuing_lock: T#%d entering\n", gtid)); in __kmp_test_queuing_lock()
1311 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_test_queuing_lock()
1313 this_thr = __kmp_thread_from_gtid(gtid); in __kmp_test_queuing_lock()
1324 ("__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid)); in __kmp_test_queuing_lock()
1331 ("__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid)); in __kmp_test_queuing_lock()
1336 kmp_int32 gtid) { in __kmp_test_queuing_lock_with_checks() argument
1345 int retval = __kmp_test_queuing_lock(lck, gtid); in __kmp_test_queuing_lock_with_checks()
1348 lck->lk.owner_id = gtid + 1; in __kmp_test_queuing_lock_with_checks()
1353 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_release_queuing_lock() argument
1358 ("__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid)); in __kmp_release_queuing_lock()
1359 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_release_queuing_lock()
1361 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid); in __kmp_release_queuing_lock()
1365 TRACE_LOCK(gtid + 1, "rel ent"); in __kmp_release_queuing_lock()
1368 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_release_queuing_lock()
1370 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p); in __kmp_release_queuing_lock()
1386 TRACE_LOCK_HT(gtid + 1, "rel read: ", head, tail); in __kmp_release_queuing_lock()
1388 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail); in __kmp_release_queuing_lock()
1399 lck, gtid)); in __kmp_release_queuing_lock()
1401 TRACE_LOCK_HT(gtid + 1, "rel exit: ", 0, 0); in __kmp_release_queuing_lock()
1416 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail); in __kmp_release_queuing_lock()
1425 TRACE_LOCK(gtid + 1, "rel deq: (h,h)->(-1,0)"); in __kmp_release_queuing_lock()
1437 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail); in __kmp_release_queuing_lock()
1448 TRACE_LOCK(gtid + 1, "rel deq: (h,t)->(h',t)"); in __kmp_release_queuing_lock()
1461 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail); in __kmp_release_queuing_lock()
1469 TRACE_LOCK_T(gtid + 1, "rel nw=0 for t=", head); in __kmp_release_queuing_lock()
1478 lck, gtid)); in __kmp_release_queuing_lock()
1480 TRACE_LOCK(gtid + 1, "rel exit 2"); in __kmp_release_queuing_lock()
1488 TRACE_LOCK(gtid + 1, "rel retry"); in __kmp_release_queuing_lock()
1497 kmp_int32 gtid) { in __kmp_release_queuing_lock_with_checks() argument
1509 if (__kmp_get_queuing_lock_owner(lck) != gtid) { in __kmp_release_queuing_lock_with_checks()
1513 return __kmp_release_queuing_lock(lck, gtid); in __kmp_release_queuing_lock_with_checks()
1556 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_queuing_lock() argument
1557 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_acquire_nested_queuing_lock()
1559 if (__kmp_get_queuing_lock_owner(lck) == gtid) { in __kmp_acquire_nested_queuing_lock()
1563 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid); in __kmp_acquire_nested_queuing_lock()
1567 lck->lk.owner_id = gtid + 1; in __kmp_acquire_nested_queuing_lock()
1574 kmp_int32 gtid) { in __kmp_acquire_nested_queuing_lock_with_checks() argument
1582 return __kmp_acquire_nested_queuing_lock(lck, gtid); in __kmp_acquire_nested_queuing_lock_with_checks()
1585 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_queuing_lock() argument
1588 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_test_nested_queuing_lock()
1590 if (__kmp_get_queuing_lock_owner(lck) == gtid) { in __kmp_test_nested_queuing_lock()
1592 } else if (!__kmp_test_queuing_lock(lck, gtid)) { in __kmp_test_nested_queuing_lock()
1598 lck->lk.owner_id = gtid + 1; in __kmp_test_nested_queuing_lock()
1604 kmp_int32 gtid) { in __kmp_test_nested_queuing_lock_with_checks() argument
1612 return __kmp_test_nested_queuing_lock(lck, gtid); in __kmp_test_nested_queuing_lock_with_checks()
1615 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_queuing_lock() argument
1616 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_release_nested_queuing_lock()
1622 __kmp_release_queuing_lock(lck, gtid); in __kmp_release_nested_queuing_lock()
1630 kmp_int32 gtid) { in __kmp_release_nested_queuing_lock_with_checks() argument
1642 if (__kmp_get_queuing_lock_owner(lck) != gtid) { in __kmp_release_nested_queuing_lock_with_checks()
1645 return __kmp_release_nested_queuing_lock(lck, gtid); in __kmp_release_nested_queuing_lock_with_checks()
1987 kmp_int32 gtid) { in __kmp_should_speculate() argument
1998 kmp_int32 gtid) { in __kmp_test_adaptive_lock_only() argument
2045 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) { in __kmp_test_adaptive_lock() argument
2047 if (__kmp_should_speculate(lck, gtid) && in __kmp_test_adaptive_lock()
2048 __kmp_test_adaptive_lock_only(lck, gtid)) in __kmp_test_adaptive_lock()
2056 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) { in __kmp_test_adaptive_lock()
2065 kmp_int32 gtid) { in __kmp_test_adaptive_lock_with_checks() argument
2071 int retval = __kmp_test_adaptive_lock(lck, gtid); in __kmp_test_adaptive_lock_with_checks()
2074 lck->lk.qlk.owner_id = gtid + 1; in __kmp_test_adaptive_lock_with_checks()
2091 kmp_int32 gtid) { in __kmp_acquire_adaptive_lock() argument
2092 if (__kmp_should_speculate(lck, gtid)) { in __kmp_acquire_adaptive_lock()
2094 if (__kmp_test_adaptive_lock_only(lck, gtid)) in __kmp_acquire_adaptive_lock()
2108 if (__kmp_test_adaptive_lock_only(lck, gtid)) in __kmp_acquire_adaptive_lock()
2117 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid); in __kmp_acquire_adaptive_lock()
2123 kmp_int32 gtid) { in __kmp_acquire_adaptive_lock_with_checks() argument
2128 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) { in __kmp_acquire_adaptive_lock_with_checks()
2132 __kmp_acquire_adaptive_lock(lck, gtid); in __kmp_acquire_adaptive_lock_with_checks()
2134 lck->lk.qlk.owner_id = gtid + 1; in __kmp_acquire_adaptive_lock_with_checks()
2139 kmp_int32 gtid) { in __kmp_release_adaptive_lock() argument
2148 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid); in __kmp_release_adaptive_lock()
2154 kmp_int32 gtid) { in __kmp_release_adaptive_lock_with_checks() argument
2163 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) { in __kmp_release_adaptive_lock_with_checks()
2167 __kmp_release_adaptive_lock(lck, gtid); in __kmp_release_adaptive_lock_with_checks()
2218 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_drdpa_lock_timed_template() argument
2352 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_drdpa_lock() argument
2353 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid); in __kmp_acquire_drdpa_lock()
2358 kmp_int32 gtid) { in __kmp_acquire_drdpa_lock_with_checks() argument
2366 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) { in __kmp_acquire_drdpa_lock_with_checks()
2370 __kmp_acquire_drdpa_lock(lck, gtid); in __kmp_acquire_drdpa_lock_with_checks()
2372 lck->lk.owner_id = gtid + 1; in __kmp_acquire_drdpa_lock_with_checks()
2376 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_test_drdpa_lock() argument
2404 kmp_int32 gtid) { in __kmp_test_drdpa_lock_with_checks() argument
2413 int retval = __kmp_test_drdpa_lock(lck, gtid); in __kmp_test_drdpa_lock_with_checks()
2416 lck->lk.owner_id = gtid + 1; in __kmp_test_drdpa_lock_with_checks()
2421 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_release_drdpa_lock() argument
2435 kmp_int32 gtid) { in __kmp_release_drdpa_lock_with_checks() argument
2447 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) && in __kmp_release_drdpa_lock_with_checks()
2448 (__kmp_get_drdpa_lock_owner(lck) != gtid)) { in __kmp_release_drdpa_lock_with_checks()
2452 return __kmp_release_drdpa_lock(lck, gtid); in __kmp_release_drdpa_lock_with_checks()
2508 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_nested_drdpa_lock() argument
2509 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_acquire_nested_drdpa_lock()
2511 if (__kmp_get_drdpa_lock_owner(lck) == gtid) { in __kmp_acquire_nested_drdpa_lock()
2515 __kmp_acquire_drdpa_lock_timed_template(lck, gtid); in __kmp_acquire_nested_drdpa_lock()
2519 lck->lk.owner_id = gtid + 1; in __kmp_acquire_nested_drdpa_lock()
2525 kmp_int32 gtid) { in __kmp_acquire_nested_drdpa_lock_with_checks() argument
2533 __kmp_acquire_nested_drdpa_lock(lck, gtid); in __kmp_acquire_nested_drdpa_lock_with_checks()
2536 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_test_nested_drdpa_lock() argument
2539 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_test_nested_drdpa_lock()
2541 if (__kmp_get_drdpa_lock_owner(lck) == gtid) { in __kmp_test_nested_drdpa_lock()
2543 } else if (!__kmp_test_drdpa_lock(lck, gtid)) { in __kmp_test_nested_drdpa_lock()
2549 lck->lk.owner_id = gtid + 1; in __kmp_test_nested_drdpa_lock()
2555 kmp_int32 gtid) { in __kmp_test_nested_drdpa_lock_with_checks() argument
2563 return __kmp_test_nested_drdpa_lock(lck, gtid); in __kmp_test_nested_drdpa_lock_with_checks()
2566 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) { in __kmp_release_nested_drdpa_lock() argument
2567 KMP_DEBUG_ASSERT(gtid >= 0); in __kmp_release_nested_drdpa_lock()
2573 __kmp_release_drdpa_lock(lck, gtid); in __kmp_release_nested_drdpa_lock()
2580 kmp_int32 gtid) { in __kmp_release_nested_drdpa_lock_with_checks() argument
2592 if (__kmp_get_drdpa_lock_owner(lck) != gtid) { in __kmp_release_nested_drdpa_lock_with_checks()
2595 return __kmp_release_nested_drdpa_lock(lck, gtid); in __kmp_release_nested_drdpa_lock_with_checks()
2715 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) { in __kmp_acquire_hle_lock() argument
2730 kmp_int32 gtid) { in __kmp_acquire_hle_lock_with_checks() argument
2731 __kmp_acquire_hle_lock(lck, gtid); // TODO: add checks in __kmp_acquire_hle_lock_with_checks()
2734 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) { in __kmp_release_hle_lock() argument
2743 kmp_int32 gtid) { in __kmp_release_hle_lock_with_checks() argument
2744 return __kmp_release_hle_lock(lck, gtid); // TODO: add checks in __kmp_release_hle_lock_with_checks()
2747 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) { in __kmp_test_hle_lock() argument
2752 kmp_int32 gtid) { in __kmp_test_hle_lock_with_checks() argument
2753 return __kmp_test_hle_lock(lck, gtid); // TODO: add checks in __kmp_test_hle_lock_with_checks()
2771 kmp_int32 gtid) { in __kmp_acquire_rtm_queuing_lock() argument
2790 __kmp_acquire_queuing_lock(lck, gtid); in __kmp_acquire_rtm_queuing_lock()
2794 kmp_int32 gtid) { in __kmp_acquire_rtm_queuing_lock_with_checks() argument
2795 __kmp_acquire_rtm_queuing_lock(lck, gtid); in __kmp_acquire_rtm_queuing_lock_with_checks()
2800 kmp_int32 gtid) { in __kmp_release_rtm_queuing_lock() argument
2806 __kmp_release_queuing_lock(lck, gtid); in __kmp_release_rtm_queuing_lock()
2812 kmp_int32 gtid) { in __kmp_release_rtm_queuing_lock_with_checks() argument
2813 return __kmp_release_rtm_queuing_lock(lck, gtid); in __kmp_release_rtm_queuing_lock_with_checks()
2818 kmp_int32 gtid) { in __kmp_test_rtm_queuing_lock() argument
2829 return __kmp_test_queuing_lock(lck, gtid); in __kmp_test_rtm_queuing_lock()
2833 kmp_int32 gtid) { in __kmp_test_rtm_queuing_lock_with_checks() argument
2834 return __kmp_test_rtm_queuing_lock(lck, gtid); in __kmp_test_rtm_queuing_lock_with_checks()
2850 kmp_int32 gtid) { in __kmp_acquire_rtm_spin_lock() argument
2882 kmp_int32 gtid) { in __kmp_acquire_rtm_spin_lock_with_checks() argument
2883 return __kmp_acquire_rtm_spin_lock(lck, gtid); in __kmp_acquire_rtm_spin_lock_with_checks()
2888 kmp_int32 gtid) { in __kmp_release_rtm_spin_lock() argument
2901 kmp_int32 gtid) { in __kmp_release_rtm_spin_lock_with_checks() argument
2902 return __kmp_release_rtm_spin_lock(lck, gtid); in __kmp_release_rtm_spin_lock_with_checks()
2906 static int __kmp_test_rtm_spin_lock(kmp_rtm_spin_lock_t *lck, kmp_int32 gtid) { in __kmp_test_rtm_spin_lock() argument
2929 kmp_int32 gtid) { in __kmp_test_rtm_spin_lock_with_checks() argument
2930 return __kmp_test_rtm_spin_lock(lck, gtid); in __kmp_test_rtm_spin_lock_with_checks()
2958 kmp_int32 gtid) { \
2959 return __kmp_##op##_##lk##_##lock(&lock->lk, gtid); \
3121 kmp_int32 gtid, in __kmp_allocate_indirect_lock() argument
3126 __kmp_acquire_lock(&__kmp_global_lock, gtid); in __kmp_allocate_indirect_lock()
3178 __kmp_release_lock(&__kmp_global_lock, gtid); in __kmp_allocate_indirect_lock()
3242 kmp_uint32 gtid = __kmp_entry_gtid(); in __kmp_destroy_indirect_lock() local
3248 __kmp_acquire_lock(&__kmp_global_lock, gtid); in __kmp_destroy_indirect_lock()
3257 __kmp_release_lock(&__kmp_global_lock, gtid); in __kmp_destroy_indirect_lock()
3260 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) { in __kmp_set_indirect_lock() argument
3262 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid); in __kmp_set_indirect_lock()
3265 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) { in __kmp_unset_indirect_lock() argument
3267 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid); in __kmp_unset_indirect_lock()
3270 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) { in __kmp_test_indirect_lock() argument
3272 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid); in __kmp_test_indirect_lock()
3276 kmp_int32 gtid) { in __kmp_set_indirect_lock_with_checks() argument
3279 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid); in __kmp_set_indirect_lock_with_checks()
3283 kmp_int32 gtid) { in __kmp_unset_indirect_lock_with_checks() argument
3286 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid); in __kmp_unset_indirect_lock_with_checks()
3290 kmp_int32 gtid) { in __kmp_test_indirect_lock_with_checks() argument
3293 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid); in __kmp_test_indirect_lock_with_checks()
3547 kmp_int32 gtid) = NULL;
3550 kmp_int32 gtid) = NULL;
3552 kmp_int32 gtid) = NULL;
3557 kmp_int32 gtid) = NULL;
3560 kmp_int32 gtid) = NULL;
3562 kmp_int32 gtid) = NULL;
3854 kmp_user_lock_p __kmp_user_lock_allocate(void **user_lock, kmp_int32 gtid, in __kmp_user_lock_allocate() argument
3860 __kmp_acquire_lock(&__kmp_global_lock, gtid); in __kmp_user_lock_allocate()
3892 __kmp_release_lock(&__kmp_global_lock, gtid); // AC: TODO move this line upper in __kmp_user_lock_allocate()
3898 void __kmp_user_lock_free(void **user_lock, kmp_int32 gtid, in __kmp_user_lock_free() argument
3903 __kmp_acquire_lock(&__kmp_global_lock, gtid); in __kmp_user_lock_free()
3913 __kmp_release_lock(&__kmp_global_lock, gtid); in __kmp_user_lock_free()