1 /*-
2 * Copyright (c) 2003-2009 RMI Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. Neither the name of RMI Corporation, nor the names of its contributors,
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * RMI_BSD */
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 #include <sys/types.h>
33 #include <sys/systm.h>
34 #include <sys/param.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/proc.h>
38 #include <sys/limits.h>
39 #include <sys/bus.h>
40
41 #include <sys/ktr.h>
42 #include <sys/kernel.h>
43 #include <sys/kthread.h>
44 #include <sys/resourcevar.h>
45 #include <sys/sched.h>
46 #include <sys/unistd.h>
47 #include <sys/sysctl.h>
48 #include <sys/malloc.h>
49
50 #include <machine/reg.h>
51 #include <machine/cpu.h>
52 #include <machine/hwfunc.h>
53 #include <machine/mips_opcode.h>
54
55 #include <machine/param.h>
56 #include <machine/intr_machdep.h>
57 #include <mips/rmi/interrupt.h>
58 #include <mips/rmi/msgring.h>
59 #include <mips/rmi/pic.h>
60 #include <mips/rmi/board.h>
61
62 #define MSGRNG_CC_INIT_CPU_DEST(dest, counter) \
63 do { \
64 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][0], 0 ); \
65 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][1], 1 ); \
66 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][2], 2 ); \
67 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][3], 3 ); \
68 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][4], 4 ); \
69 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][5], 5 ); \
70 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][6], 6 ); \
71 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][7], 7 ); \
72 } while(0)
73
74
75 /*
76 * Keep track of our message ring handler threads, each core has a
77 * different message station. Ideally we will need to start a few
78 * message handling threads every core, and wake them up depending on
79 * load
80 */
81 struct msgring_thread {
82 struct {
83 struct thread *thread; /* msgring handler threads */
84 int needed; /* thread needs to wake up */
85 } threads[XLR_NTHREADS];
86 int running; /* number of threads running */
87 int nthreads; /* number of threads started */
88 struct mtx lock; /* for changing running/active */
89 };
90 static struct msgring_thread msgring_threads[XLR_MAX_CORES];
91 static struct proc *msgring_proc; /* all threads are under a proc */
92
93 /*
94 * The maximum number of software message handler threads to be started
95 * per core. Default is 3 per core
96 */
97 static int msgring_maxthreads = 3;
98 TUNABLE_INT("hw.fmn.maxthreads", &msgring_maxthreads);
99
100 /*
101 * The device drivers can register a handler for the messages sent
102 * from a station (corresponding to the device).
103 */
104 struct tx_stn_handler {
105 msgring_handler action;
106 void *arg;
107 };
108 static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS];
109 static struct mtx msgmap_lock;
110
111 /*
112 * Initialize the messaging subsystem.
113 *
114 * Message Stations are shared among all threads in a cpu core, this
115 * has to be called once from every core which is online.
116 */
117 void
xlr_msgring_cpu_init(void)118 xlr_msgring_cpu_init(void)
119 {
120 struct stn_cc *cc_config;
121 struct bucket_size *bucket_sizes;
122 uint32_t flags;
123 int id;
124
125 KASSERT(xlr_thr_id() == 0,
126 ("xlr_msgring_cpu_init from non-zero thread"));
127 id = xlr_core_id();
128 bucket_sizes = xlr_board_info.bucket_sizes;
129 cc_config = xlr_board_info.credit_configs[id];
130
131 flags = msgrng_access_enable();
132
133 /*
134 * FMN messages are received in 8 buckets per core, set up
135 * the bucket sizes for each bucket
136 */
137 msgrng_write_bucksize(0, bucket_sizes->bucket[id * 8 + 0]);
138 msgrng_write_bucksize(1, bucket_sizes->bucket[id * 8 + 1]);
139 msgrng_write_bucksize(2, bucket_sizes->bucket[id * 8 + 2]);
140 msgrng_write_bucksize(3, bucket_sizes->bucket[id * 8 + 3]);
141 msgrng_write_bucksize(4, bucket_sizes->bucket[id * 8 + 4]);
142 msgrng_write_bucksize(5, bucket_sizes->bucket[id * 8 + 5]);
143 msgrng_write_bucksize(6, bucket_sizes->bucket[id * 8 + 6]);
144 msgrng_write_bucksize(7, bucket_sizes->bucket[id * 8 + 7]);
145
146 /*
147 * For sending FMN messages, we need credits on the destination
148 * bucket. Program the credits this core has on the 128 possible
149 * destination buckets.
150 * We cannot use a loop here, because the first argument has
151 * to be a constant integer value.
152 */
153 MSGRNG_CC_INIT_CPU_DEST(0, cc_config->counters);
154 MSGRNG_CC_INIT_CPU_DEST(1, cc_config->counters);
155 MSGRNG_CC_INIT_CPU_DEST(2, cc_config->counters);
156 MSGRNG_CC_INIT_CPU_DEST(3, cc_config->counters);
157 MSGRNG_CC_INIT_CPU_DEST(4, cc_config->counters);
158 MSGRNG_CC_INIT_CPU_DEST(5, cc_config->counters);
159 MSGRNG_CC_INIT_CPU_DEST(6, cc_config->counters);
160 MSGRNG_CC_INIT_CPU_DEST(7, cc_config->counters);
161 MSGRNG_CC_INIT_CPU_DEST(8, cc_config->counters);
162 MSGRNG_CC_INIT_CPU_DEST(9, cc_config->counters);
163 MSGRNG_CC_INIT_CPU_DEST(10, cc_config->counters);
164 MSGRNG_CC_INIT_CPU_DEST(11, cc_config->counters);
165 MSGRNG_CC_INIT_CPU_DEST(12, cc_config->counters);
166 MSGRNG_CC_INIT_CPU_DEST(13, cc_config->counters);
167 MSGRNG_CC_INIT_CPU_DEST(14, cc_config->counters);
168 MSGRNG_CC_INIT_CPU_DEST(15, cc_config->counters);
169 msgrng_restore(flags);
170 }
171
172 /*
173 * Boot time init, called only once
174 */
175 void
xlr_msgring_config(void)176 xlr_msgring_config(void)
177 {
178 mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN);
179
180 /* check value */
181 if (msgring_maxthreads < 0 || msgring_maxthreads > XLR_NTHREADS)
182 msgring_maxthreads = XLR_NTHREADS;
183 }
184
185 /*
186 * Drain out max_messages for the buckets set in the bucket mask.
187 * Use max_messages = 0 to drain out all messages.
188 */
189 uint32_t
xlr_msgring_handler(uint8_t bucket_mask,uint32_t max_messages)190 xlr_msgring_handler(uint8_t bucket_mask, uint32_t max_messages)
191 {
192 int bucket = 0;
193 int size = 0, code = 0, rx_stid = 0;
194 struct msgrng_msg msg;
195 struct tx_stn_handler *he;
196 unsigned int status = 0;
197 unsigned long mflags;
198 uint32_t n_msgs;
199 uint32_t msgbuckets;
200
201 n_msgs = 0;
202 mflags = msgrng_access_enable();
203 for (;;) {
204 msgbuckets = (~msgrng_read_status() >> 24) & bucket_mask;
205
206 /* all buckets empty, break */
207 if (msgbuckets == 0)
208 break;
209
210 for (bucket = 0; bucket < 8; bucket++) {
211 if ((msgbuckets & (1 << bucket)) == 0) /* empty */
212 continue;
213
214 status = message_receive(bucket, &size, &code,
215 &rx_stid, &msg);
216 if (status != 0)
217 continue;
218 n_msgs++;
219 he = &msgmap[rx_stid];
220 if (he->action == NULL) {
221 printf("[%s]: No Handler for message from "
222 "stn_id=%d, bucket=%d, size=%d, msg0=%jx\n",
223 __func__, rx_stid, bucket, size,
224 (uintmax_t)msg.msg0);
225 } else {
226 msgrng_restore(mflags);
227 (*he->action)(bucket, size, code, rx_stid,
228 &msg, he->arg);
229 mflags = msgrng_access_enable();
230 }
231 if (max_messages > 0 && n_msgs >= max_messages)
232 goto done;
233 }
234 }
235
236 done:
237 msgrng_restore(mflags);
238 return (n_msgs);
239 }
240
241 /*
242 * XLR COP2 supports watermark interrupts based on the number of
243 * messages pending in all the buckets in the core. We increase
244 * the watermark until all the possible handler threads in the core
245 * are woken up.
246 */
247 static void
msgrng_setconfig(int running,int nthr)248 msgrng_setconfig(int running, int nthr)
249 {
250 uint32_t config, mflags;
251 int watermark = 1; /* non zero needed */
252 int wm_intr_value;
253
254 KASSERT(nthr >= 0 && nthr <= msgring_maxthreads,
255 ("Bad value of nthr %d", nthr));
256 KASSERT(running <= nthr, ("Bad value of running %d", running));
257
258 if (running == nthr) {
259 wm_intr_value = 0;
260 } else {
261 switch (running) {
262 case 0: break; /* keep default */
263 case 1:
264 watermark = 32; break;
265 case 2:
266 watermark = 48; break;
267 case 3:
268 watermark = 56; break;
269 }
270 wm_intr_value = 0x2; /* set watermark enable interrupt */
271 }
272 mflags = msgrng_access_enable();
273 config = (watermark << 24) | (IRQ_MSGRING << 16) | (1 << 8) |
274 wm_intr_value;
275 /* clear pending interrupts, they will get re-raised if still valid */
276 write_c0_eirr64(1ULL << IRQ_MSGRING);
277 msgrng_write_config(config);
278 msgrng_restore(mflags);
279 }
280
281 /* Debug counters */
282 static int msgring_nintr[XLR_MAX_CORES];
283 static int msgring_badintr[XLR_MAX_CORES];
284 static int msgring_wakeup_sleep[XLR_MAX_CORES * XLR_NTHREADS];
285 static int msgring_wakeup_nosleep[XLR_MAX_CORES * XLR_NTHREADS];
286 static int msgring_nmsgs[XLR_MAX_CORES * XLR_NTHREADS];
287
288 static int
msgring_process_fast_intr(void * arg)289 msgring_process_fast_intr(void *arg)
290 {
291 struct msgring_thread *mthd;
292 struct thread *td;
293 uint32_t mflags;
294 int core, nt;
295
296 core = xlr_core_id();
297 mthd = &msgring_threads[core];
298 msgring_nintr[core]++;
299 mtx_lock_spin(&mthd->lock);
300 nt = mthd->running;
301 if(nt >= mthd->nthreads) {
302 msgring_badintr[core]++;
303 mtx_unlock_spin(&mthd->lock);
304 return (FILTER_HANDLED);
305 }
306
307 td = mthd->threads[nt].thread;
308 mflags = msgrng_access_enable();
309
310 /* default value with interrupts disabled */
311 msgrng_write_config((1 << 24) | (IRQ_MSGRING << 16) | (1 << 8));
312 /* clear pending interrupts */
313 write_c0_eirr64(1ULL << IRQ_MSGRING);
314 msgrng_restore(mflags);
315 mtx_unlock_spin(&mthd->lock);
316
317 /* wake up the target thread */
318 mthd->threads[nt].needed = 1;
319 thread_lock(td);
320 if (TD_AWAITING_INTR(td)) {
321 msgring_wakeup_sleep[core*4+nt]++;
322 TD_CLR_IWAIT(td);
323 sched_add(td, SRQ_INTR);
324 } else
325 msgring_wakeup_nosleep[core*4+nt]++;
326 thread_unlock(td);
327 return (FILTER_HANDLED);
328 }
329
330 static void
msgring_process(void * arg)331 msgring_process(void *arg)
332 {
333 struct msgring_thread *mthd;
334 struct thread *td;
335 int hwtid, tid, core;
336 int nmsgs;
337
338 hwtid = (intptr_t)arg;
339 core = hwtid / 4;
340 tid = hwtid % 4;
341 mthd = &msgring_threads[core];
342 td = mthd->threads[tid].thread;
343 KASSERT(curthread == td,
344 ("Incorrect thread core %d, thread %d", core, hwtid));
345
346 /* First bind this thread to the right CPU */
347 thread_lock(td);
348 sched_bind(td, xlr_hwtid_to_cpuid[hwtid]);
349 thread_unlock(td);
350
351 mtx_lock_spin(&mthd->lock);
352 ++mthd->nthreads; /* Active thread count */
353 mtx_unlock_spin(&mthd->lock);
354
355 /* start processing messages */
356 for(;;) {
357 mtx_lock_spin(&mthd->lock);
358 ++mthd->running;
359 msgrng_setconfig(mthd->running, mthd->nthreads);
360 mtx_unlock_spin(&mthd->lock);
361
362 atomic_store_rel_int(&mthd->threads[tid].needed, 0);
363 nmsgs = xlr_msgring_handler(0xff, 0);
364 msgring_nmsgs[hwtid] += nmsgs;
365
366 mtx_lock_spin(&mthd->lock);
367 --mthd->running;
368 msgrng_setconfig(mthd->running, mthd->nthreads);
369 mtx_unlock_spin(&mthd->lock);
370
371 /* sleep */
372 thread_lock(td);
373 if (mthd->threads[tid].needed) {
374 thread_unlock(td);
375 continue;
376 }
377 sched_class(td, PRI_ITHD);
378 TD_SET_IWAIT(td);
379 mi_switch(SW_VOL, NULL);
380 thread_unlock(td);
381 }
382 }
383
384 static void
create_msgring_thread(int hwtid)385 create_msgring_thread(int hwtid)
386 {
387 struct msgring_thread *mthd;
388 struct thread *td;
389 int tid, core;
390 int error;
391
392 core = hwtid / 4;
393 tid = hwtid % 4;
394 mthd = &msgring_threads[core];
395 if (tid == 0) {
396 mtx_init(&mthd->lock, "msgrngcore", NULL, MTX_SPIN);
397 mthd->running = mthd->nthreads = 0;
398 }
399 error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid,
400 &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc",
401 "msgthr%d", hwtid);
402 if (error)
403 panic("kproc_kthread_add() failed with %d", error);
404 mthd->threads[tid].thread = td;
405
406 thread_lock(td);
407 sched_class(td, PRI_ITHD);
408 sched_add(td, SRQ_INTR);
409 thread_unlock(td);
410 CTR2(KTR_INTR, "%s: created %s", __func__, td->td_name);
411 }
412
413 int
register_msgring_handler(int startb,int endb,msgring_handler action,void * arg)414 register_msgring_handler(int startb, int endb, msgring_handler action,
415 void *arg)
416 {
417 void *cookie;
418 int i;
419 static int msgring_int_enabled = 0;
420
421 KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS,
422 ("Invalid value for for bucket range %d,%d", startb, endb));
423
424 mtx_lock_spin(&msgmap_lock);
425 for (i = startb; i <= endb; i++) {
426 KASSERT(msgmap[i].action == NULL,
427 ("Bucket %d already used [action %p]", i, msgmap[i].action));
428 msgmap[i].action = action;
429 msgmap[i].arg = arg;
430 }
431 mtx_unlock_spin(&msgmap_lock);
432
433 if (xlr_test_and_set(&msgring_int_enabled)) {
434 create_msgring_thread(0);
435 if (msgring_maxthreads > xlr_threads_per_core)
436 msgring_maxthreads = xlr_threads_per_core;
437 cpu_establish_hardintr("msgring", msgring_process_fast_intr,
438 NULL, NULL, IRQ_MSGRING,
439 INTR_TYPE_NET, &cookie);
440 }
441 return (0);
442 }
443
444 /*
445 * Start message ring processing threads on other CPUs, after SMP start
446 */
447 static void
start_msgring_threads(void * arg)448 start_msgring_threads(void *arg)
449 {
450 int hwt, tid;
451
452 for (hwt = 1; hwt < XLR_MAX_CORES * XLR_NTHREADS; hwt++) {
453 if ((xlr_hw_thread_mask & (1 << hwt)) == 0)
454 continue;
455 tid = hwt % XLR_NTHREADS;
456 if (tid >= msgring_maxthreads)
457 continue;
458 create_msgring_thread(hwt);
459 }
460 }
461
462 SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE,
463 start_msgring_threads, NULL);
464
465 /*
466 * DEBUG support, XXX: static buffer, not locked
467 */
468 static int
sys_print_debug(SYSCTL_HANDLER_ARGS)469 sys_print_debug(SYSCTL_HANDLER_ARGS)
470 {
471 int error, nb, i, fs;
472 static char xprintb[4096], *buf;
473
474 buf = xprintb;
475 fs = sizeof(xprintb);
476 nb = snprintf(buf, fs,
477 "\nID INTR ER WU-SLP WU-ERR MSGS\n");
478 buf += nb;
479 fs -= nb;
480 for (i = 0; i < 32; i++) {
481 if ((xlr_hw_thread_mask & (1 << i)) == 0)
482 continue;
483 nb = snprintf(buf, fs,
484 "%2d: %8d %4d %8d %8d %8d\n", i,
485 msgring_nintr[i/4], msgring_badintr[i/4],
486 msgring_wakeup_sleep[i], msgring_wakeup_nosleep[i],
487 msgring_nmsgs[i]);
488 buf += nb;
489 fs -= nb;
490 }
491 error = SYSCTL_OUT(req, xprintb, buf - xprintb);
492 return (error);
493 }
494
495 SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0,
496 sys_print_debug, "A", "msgring debug info");
497