171 #define	cc_exec_curr(cc, dir)		cc->cc_exec_entity[dir].cc_curr  argument
172 #define	cc_exec_drain(cc, dir)		cc->cc_exec_entity[dir].cc_drain  argument
173 #define	cc_exec_next(cc)		cc->cc_next  argument
174 #define	cc_exec_cancel(cc, dir)		cc->cc_exec_entity[dir].cc_cancel  argument
175 #define	cc_exec_waiting(cc, dir)	cc->cc_exec_entity[dir].cc_waiting  argument
177 #define	cc_migration_func(cc, dir)	cc->cc_exec_entity[dir].ce_migration_func  argument
178 #define	cc_migration_arg(cc, dir)	cc->cc_exec_entity[dir].ce_migration_arg  argument
179 #define	cc_migration_cpu(cc, dir)	cc->cc_exec_entity[dir].ce_migration_cpu  argument
180 #define	cc_migration_time(cc, dir)	cc->cc_exec_entity[dir].ce_migration_time  argument
181 #define	cc_migration_prec(cc, dir)	cc->cc_exec_entity[dir].ce_migration_prec  argument
192 #define	CC_LOCK(cc)	mtx_lock_spin(&(cc)->cc_lock)  argument
193 #define	CC_UNLOCK(cc)	mtx_unlock_spin(&(cc)->cc_lock)  argument
194 #define	CC_LOCK_ASSERT(cc)	mtx_assert(&(cc)->cc_lock, MA_OWNED)  argument
198 static void	callout_cpu_init(struct callout_cpu *cc, int cpu);
199 static void	softclock_call_cc(struct callout *c, struct callout_cpu *cc,
228 cc_cce_cleanup(struct callout_cpu *cc, int direct)  in cc_cce_cleanup()  argument
231 	cc_exec_curr(cc, direct) = NULL;  in cc_cce_cleanup()
232 	cc_exec_cancel(cc, direct) = false;  in cc_cce_cleanup()
233 	cc_exec_waiting(cc, direct) = false;  in cc_cce_cleanup()
235 	cc_migration_cpu(cc, direct) = CPUBLOCK;  in cc_cce_cleanup()
236 	cc_migration_time(cc, direct) = 0;  in cc_cce_cleanup()
237 	cc_migration_prec(cc, direct) = 0;  in cc_cce_cleanup()
238 	cc_migration_func(cc, direct) = NULL;  in cc_cce_cleanup()
239 	cc_migration_arg(cc, direct) = NULL;  in cc_cce_cleanup()
247 cc_cce_migrating(struct callout_cpu *cc, int direct)  in cc_cce_migrating()  argument
251 	return (cc_migration_cpu(cc, direct) != CPUBLOCK);  in cc_cce_migrating()
264 	struct callout_cpu *cc;  in callout_callwheel_init()  local
296 	cc = CC_CPU(timeout_cpu);  in callout_callwheel_init()
297 	cc->cc_callout = malloc(ncallout * sizeof(struct callout),  in callout_callwheel_init()
299 	callout_cpu_init(cc, timeout_cpu);  in callout_callwheel_init()
307 callout_cpu_init(struct callout_cpu *cc, int cpu)  in callout_cpu_init()  argument
312 	mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE);  in callout_cpu_init()
313 	SLIST_INIT(&cc->cc_callfree);  in callout_cpu_init()
314 	cc->cc_inited = 1;  in callout_cpu_init()
315 	cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize,  in callout_cpu_init()
318 		LIST_INIT(&cc->cc_callwheel[i]);  in callout_cpu_init()
319 	TAILQ_INIT(&cc->cc_expireq);  in callout_cpu_init()
320 	cc->cc_firstevent = SBT_MAX;  in callout_cpu_init()
322 		cc_cce_cleanup(cc, i);  in callout_cpu_init()
323 	snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name),  in callout_cpu_init()
325 	if (cc->cc_callout == NULL)	/* Only cpu0 handles timeout(9) */  in callout_cpu_init()
328 		c = &cc->cc_callout[i];  in callout_cpu_init()
331 		SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);  in callout_cpu_init()
342 callout_cpu_switch(struct callout *c, struct callout_cpu *cc, int new_cpu)  in callout_cpu_switch()  argument
346 	MPASS(c != NULL && cc != NULL);  in callout_cpu_switch()
347 	CC_LOCK_ASSERT(cc);  in callout_cpu_switch()
356 	CC_UNLOCK(cc);  in callout_cpu_switch()
371 	struct callout_cpu *cc;  in start_softclock()  local
378 	cc = CC_CPU(timeout_cpu);  in start_softclock()
380 	if (swi_add(&clk_intr_event, name, softclock, cc, SWI_CLOCK,  in start_softclock()
381 	    INTR_MPSAFE, &cc->cc_cookie))  in start_softclock()
394 		cc = CC_CPU(cpu);  in start_softclock()
395 		cc->cc_callout = NULL;	/* Only cpu0 handles timeout(9). */  in start_softclock()
396 		callout_cpu_init(cc, cpu);  in start_softclock()
399 		if (swi_add(&ie, name, softclock, cc, SWI_CLOCK,  in start_softclock()
400 		    INTR_MPSAFE, &cc->cc_cookie))  in start_softclock()
433 	struct callout_cpu *cc;  in callout_process()  local
442 	cc = CC_SELF();  in callout_process()
443 	mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET);  in callout_process()
446 	firstb = callout_hash(cc->cc_lastscan);  in callout_process()
447 	cc->cc_lastscan = now;  in callout_process()
476 		sc = &cc->cc_callwheel[firstb & callwheelmask];  in callout_process()
489 					cc_exec_next(cc) =  in callout_process()
491 					cc->cc_bucket = firstb & callwheelmask;  in callout_process()
493 					softclock_call_cc(tmp, cc,  in callout_process()
498 					tmp = cc_exec_next(cc);  in callout_process()
499 					cc_exec_next(cc) = NULL;  in callout_process()
503 					TAILQ_INSERT_TAIL(&cc->cc_expireq,  in callout_process()
538 	cc->cc_firstevent = last;  in callout_process()
547 	mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);  in callout_process()
552 	if (!TAILQ_EMPTY(&cc->cc_expireq))  in callout_process()
553 		swi_sched(cc->cc_cookie, 0);  in callout_process()
559 	struct callout_cpu *cc;  in callout_lock()  local
571 		cc = CC_CPU(cpu);  in callout_lock()
572 		CC_LOCK(cc);  in callout_lock()
575 		CC_UNLOCK(cc);  in callout_lock()
577 	return (cc);  in callout_lock()
581 callout_cc_add(struct callout *c, struct callout_cpu *cc,  in callout_cc_add()  argument
587 	CC_LOCK_ASSERT(cc);  in callout_cc_add()
588 	if (sbt < cc->cc_lastscan)  in callout_cc_add()
589 		sbt = cc->cc_lastscan;  in callout_cc_add()
603 	LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le);  in callout_cc_add()
604 	if (cc->cc_bucket == bucket)  in callout_cc_add()
605 		cc_exec_next(cc) = c;  in callout_cc_add()
614 	if (sbt < cc->cc_firstevent) {  in callout_cc_add()
615 		cc->cc_firstevent = sbt;  in callout_cc_add()
622 callout_cc_del(struct callout *c, struct callout_cpu *cc)  in callout_cc_del()  argument
628 	SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);  in callout_cc_del()
632 softclock_call_cc(struct callout *c, struct callout_cpu *cc,  in softclock_call_cc()  argument
680 	cc_exec_curr(cc, direct) = c;  in softclock_call_cc()
681 	cc_exec_cancel(cc, direct) = false;  in softclock_call_cc()
682 	cc_exec_drain(cc, direct) = NULL;  in softclock_call_cc()
683 	CC_UNLOCK(cc);  in softclock_call_cc()
690 		if (cc_exec_cancel(cc, direct)) {  in softclock_call_cc()
695 		cc_exec_cancel(cc, direct) = true;  in softclock_call_cc()
716 	KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running",  in softclock_call_cc()
740 	KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle");  in softclock_call_cc()
745 	CC_LOCK(cc);  in softclock_call_cc()
746 	KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr"));  in softclock_call_cc()
747 	cc_exec_curr(cc, direct) = NULL;  in softclock_call_cc()
748 	if (cc_exec_drain(cc, direct)) {  in softclock_call_cc()
751 		drain = cc_exec_drain(cc, direct);  in softclock_call_cc()
752 		cc_exec_drain(cc, direct) = NULL;  in softclock_call_cc()
753 		CC_UNLOCK(cc);  in softclock_call_cc()
755 		CC_LOCK(cc);  in softclock_call_cc()
757 	if (cc_exec_waiting(cc, direct)) {  in softclock_call_cc()
764 		if (cc_cce_migrating(cc, direct)) {  in softclock_call_cc()
765 			cc_cce_cleanup(cc, direct);  in softclock_call_cc()
773 		cc_exec_waiting(cc, direct) = false;  in softclock_call_cc()
774 		CC_UNLOCK(cc);  in softclock_call_cc()
775 		wakeup(&cc_exec_waiting(cc, direct));  in softclock_call_cc()
776 		CC_LOCK(cc);  in softclock_call_cc()
777 	} else if (cc_cce_migrating(cc, direct)) {  in softclock_call_cc()
785 		new_cpu = cc_migration_cpu(cc, direct);  in softclock_call_cc()
786 		new_time = cc_migration_time(cc, direct);  in softclock_call_cc()
787 		new_prec = cc_migration_prec(cc, direct);  in softclock_call_cc()
788 		new_func = cc_migration_func(cc, direct);  in softclock_call_cc()
789 		new_arg = cc_migration_arg(cc, direct);  in softclock_call_cc()
790 		cc_cce_cleanup(cc, direct);  in softclock_call_cc()
802 			callout_cc_del(c, cc);  in softclock_call_cc()
807 		new_cc = callout_cpu_switch(c, cc, new_cpu);  in softclock_call_cc()
812 		CC_LOCK(cc);  in softclock_call_cc()
829 		callout_cc_del(c, cc);  in softclock_call_cc()
851 	struct callout_cpu *cc;  in softclock()  local
857 	cc = (struct callout_cpu *)arg;  in softclock()
858 	CC_LOCK(cc);  in softclock()
859 	while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) {  in softclock()
860 		TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);  in softclock()
861 		softclock_call_cc(c, cc,  in softclock()
876 	CC_UNLOCK(cc);  in softclock()
898 	struct callout_cpu *cc;  in timeout()  local
902 	cc = CC_CPU(timeout_cpu);  in timeout()
903 	CC_LOCK(cc);  in timeout()
905 	new = SLIST_FIRST(&cc->cc_callfree);  in timeout()
909 	SLIST_REMOVE_HEAD(&cc->cc_callfree, c_links.sle);  in timeout()
912 	CC_UNLOCK(cc);  in timeout()
920 	struct callout_cpu *cc;  in untimeout()  local
930 	cc = callout_lock(handle.callout);  in untimeout()
933 	CC_UNLOCK(cc);  in untimeout()
963 	struct callout_cpu *cc;  local
1028 	cc = callout_lock(c);
1039 	if (cc_exec_curr(cc, direct) == c) {
1045 		if (c->c_lock != NULL && !cc_exec_cancel(cc, direct))
1046 			cancelled = cc_exec_cancel(cc, direct) = true;
1047 		if (cc_exec_waiting(cc, direct)) {
1055 			CC_UNLOCK(cc);
1067 			cc_migration_cpu(cc, direct) = cpu;
1068 			cc_migration_time(cc, direct) = to_sbt;
1069 			cc_migration_prec(cc, direct) = precision;
1070 			cc_migration_func(cc, direct) = ftn;
1071 			cc_migration_arg(cc, direct) = arg;
1073 			CC_UNLOCK(cc);
1080 			if (cc_exec_next(cc) == c)
1081 				cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1084 			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1098 		if (cc_exec_curr(cc, direct) == c) {
1115 			cc_migration_cpu(cc, direct) = cpu;
1116 			cc_migration_time(cc, direct) = to_sbt;
1117 			cc_migration_prec(cc, direct) = precision;
1118 			cc_migration_func(cc, direct) = ftn;
1119 			cc_migration_arg(cc, direct) = arg;
1126 			CC_UNLOCK(cc);
1129 		cc = callout_cpu_switch(c, cc, cpu);
1133 	callout_cc_add(c, cc, to_sbt, precision, ftn, arg, cpu, flags);
1137 	CC_UNLOCK(cc);
1160 	struct callout_cpu *cc, *old_cc;  local
1191 	cc = callout_lock(c);
1220 	if (sq_locked != 0 && cc != old_cc) {
1222 		CC_UNLOCK(cc);
1247 		if (cc_exec_curr(cc, direct) != c) {
1250 			CC_UNLOCK(cc);
1252 				sleepq_release(&cc_exec_waiting(cc, direct));
1263 			while (cc_exec_curr(cc, direct) == c) {
1282 					CC_UNLOCK(cc);
1284 					    &cc_exec_waiting(cc, direct));
1286 					old_cc = cc;
1296 				cc_exec_waiting(cc, direct) = true;
1298 				CC_UNLOCK(cc);
1300 				    &cc_exec_waiting(cc, direct),
1301 				    &cc->cc_lock.lock_object, "codrain",
1304 				    &cc_exec_waiting(cc, direct),
1311 				CC_LOCK(cc);
1314 			   !cc_exec_cancel(cc, direct) && (drain == NULL)) {
1325 			cc_exec_cancel(cc, direct) = true;
1328 			KASSERT(!cc_cce_migrating(cc, direct),
1333 				cc_migration_cpu(cc, direct) = CPUBLOCK;
1334 				cc_migration_time(cc, direct) = 0;
1335 				cc_migration_prec(cc, direct) = 0;
1336 				cc_migration_func(cc, direct) = NULL;
1337 				cc_migration_arg(cc, direct) = NULL;
1340 			CC_UNLOCK(cc);
1361 			cc_migration_cpu(cc, direct) = CPUBLOCK;
1362 			cc_migration_time(cc, direct) = 0;
1363 			cc_migration_prec(cc, direct) = 0;
1364 			cc_migration_func(cc, direct) = NULL;
1365 			cc_migration_arg(cc, direct) = NULL;
1370 				cc_exec_drain(cc, direct) = drain;
1372 			CC_UNLOCK(cc);
1378 			cc_exec_drain(cc, direct) = drain;
1380 		CC_UNLOCK(cc);
1385 		sleepq_release(&cc_exec_waiting(cc, direct));
1394 			if (cc_exec_next(cc) == c)
1395 				cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1398 			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1401 	callout_cc_del(c, cc);
1402 	CC_UNLOCK(cc);
1484 	CC_LOCK(cc);
1495 	CC_UNLOCK(cc);
1520 	struct callout_cpu *cc;  local
1541 		cc = CC_CPU(cpu);
1543 		cc = CC_CPU(timeout_cpu);
1545 		CC_LOCK(cc);
1547 			sc = &cc->cc_callwheel[i];
1568 		CC_UNLOCK(cc);