1 /* $NetBSD: cache.c,v 1.69 2022/03/13 17:50:55 andvar Exp $ */
2
3 /*
4 * Copyright 2001, 2002 Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Written by Jason R. Thorpe and Simon Burge for Wasabi Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed for the NetBSD Project by
20 * Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 * or promote products derived from this software without specific prior
23 * written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38 /*
39 * Copyright 2000, 2001
40 * Broadcom Corporation. All rights reserved.
41 *
42 * This software is furnished under license and may be used and copied only
43 * in accordance with the following terms and conditions. Subject to these
44 * conditions, you may download, copy, install, use, modify and distribute
45 * modified or unmodified copies of this software in source and/or binary
46 * form. No title or ownership is transferred hereby.
47 *
48 * 1) Any source code used, modified or distributed must reproduce and
49 * retain this copyright notice and list of conditions as they appear in
50 * the source file.
51 *
52 * 2) No right is granted to use any trade name, trademark, or logo of
53 * Broadcom Corporation. The "Broadcom Corporation" name may not be
54 * used to endorse or promote products derived from this software
55 * without the prior written permission of Broadcom Corporation.
56 *
57 * 3) THIS SOFTWARE IS PROVIDED "AS-IS" AND ANY EXPRESS OR IMPLIED
58 * WARRANTIES, INCLUDING BUT NOT LIMITED TO, ANY IMPLIED WARRANTIES OF
59 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR
60 * NON-INFRINGEMENT ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM BE LIABLE
61 * FOR ANY DAMAGES WHATSOEVER, AND IN PARTICULAR, BROADCOM SHALL NOT BE
62 * LIABLE FOR DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
63 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
64 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
65 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
66 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
67 * OR OTHERWISE), EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
68 */
69
70 #include <sys/cdefs.h>
71 __KERNEL_RCSID(0, "$NetBSD: cache.c,v 1.69 2022/03/13 17:50:55 andvar Exp $");
72
73 #include "opt_cputype.h"
74 #include "opt_mips_cache.h"
75
76 #include <sys/param.h>
77
78 #include <uvm/uvm_extern.h>
79
80 #include <mips/cache.h>
81 #include <mips/locore.h>
82
83 #ifdef MIPS1
84 #include <mips/cache_r3k.h>
85 #endif
86
87 #ifdef MIPS3_PLUS
88 #include <mips/cache_r4k.h>
89 #include <mips/cache_r5k.h>
90 #ifdef ENABLE_MIPS4_CACHE_R10K
91 #include <mips/cache_r10k.h>
92 #endif
93 #ifdef MIPS3_LOONGSON2
94 #include <mips/cache_ls2.h>
95 #endif
96 #endif
97
98 #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
99 #include <mips/mipsNN.h> /* MIPS32/MIPS64 registers */
100 #include <mips/cache_mipsNN.h>
101 #ifdef MIPS64_OCTEON
102 #include <mips/cache_octeon.h>
103 #endif
104 #endif
105
106 #ifdef MIPS1
107 #ifdef ENABLE_MIPS_TX3900
108 #include <mips/cache_tx39.h>
109 void tx3900_get_cache_config(void);
110 void tx3920_get_cache_config(void);
111 void tx39_cache_config_write_through(void);
112 #endif /* ENABLE_MIPS_TX3900 */
113 #endif /* MIPS1 */
114
115 #if defined(MIPS3) || defined(MIPS4)
116 void mips3_get_cache_config(int);
117 #ifdef ENABLE_MIPS4_CACHE_R10K
118 void mips4_get_cache_config(int);
119 #endif /* ENABLE_MIPS4_CACHE_R10K */
120 #endif /* MIPS3 || MIPS4 */
121
122 #if defined(MIPS1) || defined(MIPS3) || defined(MIPS4)
123 static void mips_config_cache_prehistoric(void);
124 static void mips_config_cache_emips(void);
125 #endif
126 #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
127 static void mips_config_cache_modern(uint32_t);
128 #endif
129
130 #if (MIPS1 + MIPS3 + MIPS4 + MIPS32 + MIPS64 + MIPS32R2 + MIPS64R2) > 0
131 /* no-cache definition */
132 static void no_cache_op(void);
133 static void no_cache_op_range(register_t va, vsize_t size);
134 static void no_cache_op_range_index(vaddr_t va, vsize_t size);
135
136 /* no-cache implementation */
no_cache_op(void)137 static void no_cache_op(void) {}
no_cache_op_range(register_t va,vsize_t size)138 static void no_cache_op_range(register_t va, vsize_t size) {}
no_cache_op_range_index(vaddr_t va,vsize_t size)139 static void no_cache_op_range_index(vaddr_t va, vsize_t size) {}
140 #endif
141
142 struct mips_cache_info mips_cache_info;
143 struct mips_cache_ops mips_cache_ops = {
144 .mco_intern_icache_sync_range_index = no_cache_op_range_index,
145 .mco_intern_icache_sync_range = no_cache_op_range,
146 .mco_intern_pdcache_sync_all= no_cache_op,
147 .mco_intern_pdcache_sync_range_index = no_cache_op_range_index,
148 .mco_intern_pdcache_sync_range = no_cache_op_range,
149 .mco_intern_pdcache_wbinv_range_index = no_cache_op_range_index,
150 .mco_intern_sdcache_sync_all= no_cache_op,
151 .mco_intern_sdcache_sync_range_index = no_cache_op_range_index,
152 .mco_intern_sdcache_sync_range = no_cache_op_range,
153 .mco_intern_sdcache_wbinv_range_index = no_cache_op_range_index,
154 };
155
156 /*
157 * mips_dcache_compute_align:
158 *
159 * Compute the D-cache alignment values.
160 */
161 void
mips_dcache_compute_align(void)162 mips_dcache_compute_align(void)
163 {
164 struct mips_cache_info * const mci = &mips_cache_info;
165 u_int align;
166
167 align = mci->mci_pdcache_line_size;
168
169 if (mci->mci_sdcache_line_size > align)
170 align = mci->mci_sdcache_line_size;
171
172 if (mci->mci_tcache_line_size > align)
173 align = mci->mci_tcache_line_size;
174
175 mci->mci_dcache_align = align;
176 mci->mci_dcache_align_mask = align - 1;
177 }
178
179 /*
180 * mips_config_cache:
181 *
182 * Configure the cache for the system.
183 *
184 * XXX DOES NOT HANDLE SPLIT SECONDARY CACHES.
185 */
186 void
mips_config_cache(void)187 mips_config_cache(void)
188 {
189 #ifdef DIAGNOSTIC
190 struct mips_cache_info * const mci = &mips_cache_info;
191 struct mips_cache_ops * const mco = &mips_cache_ops;
192 #endif
193 const mips_prid_t cpu_id = mips_options.mips_cpu_id;
194
195 #if defined(MIPS1) || defined(MIPS3) || defined(MIPS4)
196 if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_PREHISTORIC)
197 mips_config_cache_prehistoric();
198 else if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_MICROSOFT)
199 mips_config_cache_emips();
200 #endif
201 #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
202 if (MIPS_PRID_CID(cpu_id) != MIPS_PRID_CID_PREHISTORIC)
203 mips_config_cache_modern(cpu_id);
204 #endif
205
206 #ifdef DIAGNOSTIC
207 /* Check that all cache ops are set up. */
208 if (mci->mci_picache_size || 1) { /* XXX- must have primary Icache */
209 if (!mco->mco_icache_sync_all)
210 panic("no icache_sync_all cache op");
211 if (!mco->mco_icache_sync_range)
212 panic("no icache_sync_range cache op");
213 if (!mco->mco_icache_sync_range_index)
214 panic("no icache_sync_range_index cache op");
215 }
216 if (mci->mci_pdcache_size || 1) { /* XXX- must have primary Dcache */
217 if (!mco->mco_pdcache_wbinv_all)
218 panic("no pdcache_wbinv_all");
219 if (!mco->mco_pdcache_wbinv_range)
220 panic("no pdcache_wbinv_range");
221 if (!mco->mco_pdcache_wbinv_range_index)
222 panic("no pdcache_wbinv_range_index");
223 if (!mco->mco_pdcache_inv_range)
224 panic("no pdcache_inv_range");
225 if (!mco->mco_pdcache_wb_range)
226 panic("no pdcache_wb_range");
227 }
228 if (mci->mci_sdcache_size) {
229 if (!mco->mco_sdcache_wbinv_all)
230 panic("no sdcache_wbinv_all");
231 if (!mco->mco_sdcache_wbinv_range)
232 panic("no sdcache_wbinv_range");
233 if (!mco->mco_sdcache_wbinv_range_index)
234 panic("no sdcache_wbinv_range_index");
235 if (!mco->mco_sdcache_inv_range)
236 panic("no sdcache_inv_range");
237 if (!mco->mco_sdcache_wb_range)
238 panic("no sdcache_wb_range");
239 }
240 #endif /* DIAGNOSTIC */
241 }
242
243 #if defined(MIPS1) || defined(MIPS3) || defined(MIPS4)
244 /*
245 * XXX DOES NOT HANDLE SPLIT SECONDARY CACHES.
246 */
247 void
mips_config_cache_prehistoric(void)248 mips_config_cache_prehistoric(void)
249 {
250 struct mips_cache_info * const mci = &mips_cache_info;
251 struct mips_cache_ops * const mco = &mips_cache_ops;
252 const mips_prid_t cpu_id = mips_options.mips_cpu_id;
253 #if defined(MIPS3) || defined(MIPS4)
254 int csizebase = MIPS3_CONFIG_C_DEFBASE;
255 #endif
256
257 KASSERT(PAGE_SIZE != 0);
258
259 /*
260 * Configure primary caches.
261 */
262 switch (MIPS_PRID_IMPL(cpu_id)) {
263 #ifdef MIPS1
264 case MIPS_R2000:
265 case MIPS_R3000:
266 mci->mci_picache_size = r3k_picache_size();
267 mci->mci_pdcache_size = r3k_pdcache_size();
268
269 mci->mci_picache_line_size = 4;
270 mci->mci_pdcache_line_size = 4;
271
272 mci->mci_picache_ways = 1;
273 mci->mci_pdcache_ways = 1;
274
275 mci->mci_pdcache_write_through = true;
276
277 mco->mco_icache_sync_all =
278 r3k_icache_sync_all;
279 mco->mco_icache_sync_range =
280 r3k_icache_sync_range;
281 mco->mco_icache_sync_range_index =
282 (void (*)(vaddr_t, vsize_t)) mco->mco_icache_sync_range;
283
284 mco->mco_pdcache_wbinv_all =
285 r3k_pdcache_wbinv_all;
286 mco->mco_pdcache_wbinv_range =
287 r3k_pdcache_inv_range;
288 mco->mco_pdcache_wbinv_range_index =
289 (void (*)(vaddr_t, vsize_t)) mco->mco_pdcache_wbinv_range;
290 mco->mco_pdcache_inv_range =
291 r3k_pdcache_inv_range;
292 mco->mco_pdcache_wb_range =
293 r3k_pdcache_wb_range;
294
295 uvmexp.ncolors = atop(mci->mci_pdcache_size);
296 break;
297
298 #ifdef ENABLE_MIPS_TX3900
299 case MIPS_TX3900:
300 switch (MIPS_PRID_REV_MAJ(cpu_id)) {
301 case 1: /* TX3912 */
302 mci->mci_picache_ways = 1;
303 mci->mci_picache_line_size = 16;
304 mci->mci_pdcache_line_size = 4;
305
306 tx3900_get_cache_config();
307
308 mci->mci_pdcache_write_through = true;
309
310 mco->mco_icache_sync_all =
311 tx3900_icache_sync_all_16;
312 mco->mco_icache_sync_range =
313 tx3900_icache_sync_range_16;
314 mco->mco_icache_sync_range_index =
315 (void (*)(vaddr_t, vsize_t))
316 tx3900_icache_sync_range_16;
317
318 mco->mco_pdcache_wbinv_all =
319 tx3900_pdcache_wbinv_all_4;
320 mco->mco_pdcache_wbinv_range =
321 tx3900_pdcache_inv_range_4;
322 mco->mco_pdcache_wbinv_range_index =
323 (void (*)(vaddr_t, vsize_t))
324 tx3900_pdcache_inv_range_4;
325 mco->mco_pdcache_inv_range =
326 tx3900_pdcache_inv_range_4;
327 mco->mco_pdcache_wb_range =
328 tx3900_pdcache_wb_range_4;
329 break;
330
331 case 3: /* TX3922 */
332 mci->mci_picache_ways = 2;
333 mci->mci_picache_line_size = 16;
334 mci->mci_pdcache_line_size = 16;
335
336 tx3920_get_cache_config();
337
338 mco->mco_icache_sync_all =
339 mci->mci_pdcache_write_through ?
340 tx3900_icache_sync_all_16 :
341 tx3920_icache_sync_all_16wb;
342 mco->mco_icache_sync_range =
343 mci->mci_pdcache_write_through ?
344 tx3920_icache_sync_range_16wt :
345 tx3920_icache_sync_range_16wb;
346 mco->mco_icache_sync_range_index =
347 (void (*)(vaddr_t, vsize_t))
348 mco->mco_icache_sync_range;
349
350 mco->mco_pdcache_wbinv_all =
351 mci->mci_pdcache_write_through ?
352 tx3920_pdcache_wbinv_all_16wt :
353 tx3920_pdcache_wbinv_all_16wb;
354 mco->mco_pdcache_wbinv_range =
355 mci->mci_pdcache_write_through ?
356 tx3920_pdcache_inv_range_16 :
357 tx3920_pdcache_wbinv_range_16wb;
358 mco->mco_pdcache_wbinv_range_index =
359 (void (*)(vaddr_t, vsize_t))
360 mco->mco_pdcache_wbinv_range;
361 mco->mco_pdcache_inv_range =
362 tx3920_pdcache_inv_range_16;
363 mco->mco_pdcache_wb_range =
364 mci->mci_pdcache_write_through ?
365 tx3920_pdcache_wb_range_16wt :
366 tx3920_pdcache_wb_range_16wb;
367 break;
368
369 default:
370 panic("mips_config_cache: unsupported TX3900");
371 }
372
373 mci->mci_pdcache_ways = 2;
374 tx3900_get_cache_config();
375 /* change to write-through mode */
376 tx39_cache_config_write_through();
377
378 uvmexp.ncolors =
379 atop(mci->mci_pdcache_size) / mci->mci_pdcache_ways;
380 break;
381 #endif /* ENABLE_MIPS_TX3900 */
382 #endif /* MIPS1 */
383
384 #if defined(MIPS3) || defined(MIPS4)
385 case MIPS_R4100:
386 if ((mips3_cp0_config_read() & MIPS3_CONFIG_CS) != 0)
387 csizebase = MIPS3_CONFIG_C_4100BASE;
388
389 /*
390 * R4100 (NEC VR series) revision number means:
391 *
392 * MIPS_PRID_REV_MAJ MIPS_PRID_REV_MIN
393 * VR4102 4 ?
394 * VR4111 5 ?
395 * VR4181 5 ?
396 * VR4121 6 ?
397 * VR4122 7 0 or 1
398 * VR4181A 7 3 <
399 * VR4131 8 ?
400 */
401 /* Vr4131 has R4600 style 2-way set-associative cache */
402 if (MIPS_PRID_REV_MAJ(cpu_id) == 8)
403 goto primary_cache_is_2way;
404 /* FALLTHROUGH */
405
406 case MIPS_R4000:
407 case MIPS_R4300:
408 mci->mci_picache_ways = 1;
409 mci->mci_pdcache_ways = 1;
410 mci->mci_sdcache_ways = 1;
411
412 mips3_get_cache_config(csizebase);
413
414 /* no VCE support if there is no L2 cache */
415 if (mci->mci_picache_size > PAGE_SIZE)
416 mci->mci_icache_virtual_alias = true;
417 if (mci->mci_pdcache_size > PAGE_SIZE)
418 mci->mci_cache_virtual_alias = true;
419
420 mco->mco_icache_sync_all = r4k_icache_sync_all_generic;
421 switch (mci->mci_picache_line_size) {
422 case 16:
423 mco->mco_icache_sync_range =
424 cache_r4k_icache_hit_inv_16;
425 mco->mco_icache_sync_range_index =
426 cache_r4k_icache_index_inv_16;
427 break;
428
429 case 32:
430 mco->mco_icache_sync_range =
431 cache_r4k_icache_hit_inv_32;
432 mco->mco_icache_sync_range_index =
433 cache_r4k_icache_index_inv_32;
434 break;
435
436 default:
437 panic("r4k picache line size %d",
438 mci->mci_picache_line_size);
439 }
440
441 mco->mco_pdcache_wbinv_all = r4k_pdcache_wbinv_all_generic;
442 switch (mci->mci_pdcache_line_size) {
443 case 16:
444 mco->mco_pdcache_wbinv_range =
445 cache_r4k_pdcache_hit_wb_inv_16;
446 mco->mco_pdcache_wbinv_range_index =
447 cache_r4k_pdcache_index_wb_inv_16;
448 mco->mco_pdcache_inv_range =
449 cache_r4k_pdcache_hit_inv_16;
450 mco->mco_pdcache_wb_range =
451 cache_r4k_pdcache_hit_wb_16;
452 break;
453
454 case 32:
455 mco->mco_pdcache_wbinv_range =
456 cache_r4k_pdcache_hit_wb_inv_32;
457 mco->mco_pdcache_wbinv_range_index =
458 cache_r4k_pdcache_index_wb_inv_32;
459 mco->mco_pdcache_inv_range =
460 cache_r4k_pdcache_hit_inv_32;
461 mco->mco_pdcache_wb_range =
462 cache_r4k_pdcache_hit_wb_32;
463 break;
464
465 default:
466 panic("r4k pdcache line size %d",
467 mci->mci_pdcache_line_size);
468 }
469 break;
470
471 case MIPS_R4600:
472 #ifdef ENABLE_MIPS_R4700
473 case MIPS_R4700:
474 #endif
475 #ifndef ENABLE_MIPS_R3NKK
476 case MIPS_R5000:
477 #endif
478 case MIPS_RM5200:
479 primary_cache_is_2way:
480 mci->mci_picache_ways = 2;
481 mci->mci_pdcache_ways = 2;
482
483 mips3_get_cache_config(csizebase);
484
485 if (mci->mci_picache_size / mci->mci_picache_ways > PAGE_SIZE)
486 mci->mci_icache_virtual_alias = true;
487 if (mci->mci_pdcache_size / mci->mci_pdcache_ways > PAGE_SIZE)
488 mci->mci_cache_virtual_alias = true;
489
490 mco->mco_icache_sync_all = r5k_picache_sync_all;
491 mco->mco_icache_sync_range = r5k_picache_sync_range;
492 mco->mco_icache_sync_range_index = r5k_picache_sync_range_index;
493
494 switch (mci->mci_picache_line_size) {
495 case 32:
496 /* used internally by mipsNN_picache_sync_range */
497 mco->mco_intern_icache_sync_range =
498 cache_r4k_icache_hit_inv_32;
499
500 /* used internally by mipsNN_picache_sync_range_index */
501 mco->mco_intern_icache_sync_range_index =
502 cache_r4k_icache_index_inv_32;
503 break;
504
505 default:
506 panic("r5k picache line size %u",
507 mci->mci_picache_line_size);
508 }
509
510 mco->mco_pdcache_wbinv_all = r5k_pdcache_wbinv_all;
511 mco->mco_pdcache_wbinv_range_index =
512 r5k_pdcache_wbinv_range_index;
513
514 switch (mci->mci_pdcache_line_size) {
515 case 16:
516 mco->mco_pdcache_wbinv_range =
517 cache_r4k_pdcache_hit_wb_inv_16;
518 mco->mco_pdcache_inv_range =
519 cache_r4k_pdcache_hit_inv_16;
520 mco->mco_pdcache_wb_range =
521 cache_r4k_pdcache_hit_wb_16;
522
523 /* used internally by r5k_pdcache_wbinv_range_index */
524 mco->mco_intern_pdcache_wbinv_range_index =
525 cache_r4k_pdcache_index_wb_inv_16;
526 break;
527
528 case 32:
529 mco->mco_pdcache_wbinv_range =
530 cache_r4k_pdcache_hit_wb_inv_32;
531 mco->mco_pdcache_inv_range =
532 cache_r4k_pdcache_hit_inv_32;
533 mco->mco_pdcache_wb_range =
534 cache_r4k_pdcache_hit_wb_32;
535
536 /* used internally by r5k_pdcache_wbinv_range_index */
537 mco->mco_intern_pdcache_wbinv_range_index =
538 cache_r4k_pdcache_index_wb_inv_32;
539 break;
540
541 default:
542 panic("r5k pdcache line size %d",
543 mci->mci_pdcache_line_size);
544 }
545
546 mco->mco_intern_pdcache_sync_all = mco->mco_pdcache_wbinv_all;
547 mco->mco_intern_pdcache_sync_range_index =
548 mco->mco_intern_pdcache_wbinv_range_index;
549 mco->mco_intern_pdcache_sync_range = mco->mco_pdcache_wb_range;
550
551 /*
552 * Deal with R4600 chip bugs.
553 */
554 if (MIPS_PRID_IMPL(cpu_id) == MIPS_R4600 &&
555 MIPS_PRID_REV_MAJ(cpu_id) == 1) {
556 KASSERT(mci->mci_pdcache_line_size == 32);
557 mco->mco_pdcache_wbinv_range =
558 r4600v1_pdcache_wbinv_range_32;
559 mco->mco_pdcache_inv_range =
560 r4600v1_pdcache_inv_range_32;
561 mco->mco_pdcache_wb_range =
562 r4600v1_pdcache_wb_range_32;
563 } else if (MIPS_PRID_IMPL(cpu_id) == MIPS_R4600 &&
564 MIPS_PRID_REV_MAJ(cpu_id) == 2) {
565 KASSERT(mci->mci_pdcache_line_size == 32);
566 mco->mco_pdcache_wbinv_range =
567 r4600v2_pdcache_wbinv_range_32;
568 mco->mco_pdcache_inv_range =
569 r4600v2_pdcache_inv_range_32;
570 mco->mco_pdcache_wb_range =
571 r4600v2_pdcache_wb_range_32;
572 }
573
574 /*
575 * Deal with VR4131 chip bugs.
576 */
577 if (MIPS_PRID_IMPL(cpu_id) == MIPS_R4100 &&
578 MIPS_PRID_REV_MAJ(cpu_id) == 8) {
579 KASSERT(mci->mci_pdcache_line_size == 16);
580 mco->mco_pdcache_wbinv_range =
581 vr4131v1_pdcache_wbinv_range_16;
582 }
583 break;
584 #ifdef ENABLE_MIPS4_CACHE_R10K
585 case MIPS_R10000:
586 case MIPS_R12000:
587 case MIPS_R14000:
588 mci->mci_picache_ways = 2;
589 mci->mci_pdcache_ways = 2;
590 mci->mci_sdcache_ways = 2;
591
592 mips4_get_cache_config(csizebase);
593
594 /* VCE is handled by hardware */
595
596 mco->mco_icache_sync_all =
597 r10k_icache_sync_all;
598 mco->mco_icache_sync_range =
599 r10k_icache_sync_range;
600 mco->mco_icache_sync_range_index =
601 r10k_icache_sync_range_index;
602 mco->mco_pdcache_wbinv_all =
603 r10k_pdcache_wbinv_all;
604 mco->mco_pdcache_wbinv_range =
605 r10k_pdcache_wbinv_range;
606 mco->mco_pdcache_wbinv_range_index =
607 r10k_pdcache_wbinv_range_index;
608 mco->mco_pdcache_inv_range =
609 r10k_pdcache_inv_range;
610 mco->mco_pdcache_wb_range =
611 r10k_pdcache_wb_range;
612 break;
613 #endif /* ENABLE_MIPS4_CACHE_R10K */
614 #ifdef MIPS3_LOONGSON2
615 case MIPS_LOONGSON2:
616 mci->mci_picache_ways = 4;
617 mci->mci_pdcache_ways = 4;
618
619 mips3_get_cache_config(csizebase);
620
621 mci->mci_sdcache_line_size = 32; /* don't trust config reg */
622
623 if (mci->mci_picache_size / mci->mci_picache_ways > PAGE_SIZE)
624 mci->mci_icache_virtual_alias = true;
625 if (mci->mci_pdcache_size / mci->mci_pdcache_ways > PAGE_SIZE)
626 mci->mci_cache_virtual_alias = true;
627
628 mco->mco_icache_sync_all =
629 ls2_icache_sync_all;
630 mco->mco_icache_sync_range =
631 ls2_icache_sync_range;
632 mco->mco_icache_sync_range_index =
633 ls2_icache_sync_range_index;
634
635 mco->mco_pdcache_wbinv_all =
636 ls2_pdcache_wbinv_all;
637 mco->mco_pdcache_wbinv_range =
638 ls2_pdcache_wbinv_range;
639 mco->mco_pdcache_wbinv_range_index =
640 ls2_pdcache_wbinv_range_index;
641 mco->mco_pdcache_inv_range =
642 ls2_pdcache_inv_range;
643 mco->mco_pdcache_wb_range =
644 ls2_pdcache_wb_range;
645
646 /*
647 * For current version chips, [the] operating system is
648 * obliged to eliminate the potential for virtual aliasing.
649 */
650 uvmexp.ncolors = mci->mci_pdcache_ways;
651 break;
652 #endif
653 #endif /* MIPS3 || MIPS4 */
654 default:
655 panic("can't handle primary cache on impl 0x%x",
656 MIPS_PRID_IMPL(cpu_id));
657 }
658
659 /*
660 * Compute the "way mask" for each cache.
661 */
662 if (mci->mci_picache_size) {
663 KASSERT(mci->mci_picache_ways != 0);
664 mci->mci_picache_way_size =
665 mci->mci_picache_size / mci->mci_picache_ways;
666 mci->mci_picache_way_mask = mci->mci_picache_way_size - 1;
667 #if (MIPS2 + MIPS3 + MIPS4) > 0
668 if (mci->mci_icache_virtual_alias)
669 mci->mci_icache_alias_mask =
670 mci->mci_picache_way_mask & -PAGE_SIZE;
671 #endif
672 }
673 if (mci->mci_pdcache_size) {
674 KASSERT(mci->mci_pdcache_ways != 0);
675 mci->mci_pdcache_way_size =
676 mci->mci_pdcache_size / mci->mci_pdcache_ways;
677 mci->mci_pdcache_way_mask = mci->mci_pdcache_way_size - 1;
678 #if (MIPS2 + MIPS3 + MIPS4) > 0
679 if (mci->mci_cache_virtual_alias)
680 mci->mci_cache_alias_mask =
681 mci->mci_pdcache_way_mask & -PAGE_SIZE;
682 #endif
683 }
684
685 #if (MIPS2 + MIPS3 + MIPS4) > 0
686 if (mci->mci_cache_virtual_alias) {
687 mci->mci_cache_prefer_mask = mci->mci_pdcache_way_mask;
688
689 uvmexp.ncolors = (mci->mci_cache_prefer_mask >> PAGE_SHIFT) + 1;
690 }
691 #endif
692
693 mips_dcache_compute_align();
694
695 if (mci->mci_sdcache_line_size == 0)
696 return;
697
698 /*
699 * Configure the secondary cache.
700 */
701 switch (MIPS_PRID_IMPL(cpu_id)) {
702 #if defined(MIPS3) || defined(MIPS4)
703 case MIPS_R4000:
704 /*
705 * R4000/R4400 detects virtual alias by VCE as if
706 * its primary cache size were 32KB, because it always
707 * compares 3 bits of vaddr[14:12] which causes
708 * primary cache miss and PIdx[2:0] in the secondary
709 * cache tag regardless of its primary cache size.
710 * i.e. VCE could happen even if there is no actual
711 * virtual alias on its 8KB or 16KB primary cache
712 * which has only 1 or 2 bit valid PIdx in 4KB page.
713 * Actual primary cache size is ignored wrt VCE
714 * and virtual aliases are resolved by the VCE handler,
715 * but it's still worth to avoid unnecessary VCE by
716 * setting alias mask and prefer mask to 32K, though
717 * some other possible aliases (maybe caused by KSEG0
718 * accesses which can't be managed by PMAP_PREFER(9))
719 * will still be resolved by the VCED/VCEI handler.
720 */
721 mci->mci_cache_alias_mask =
722 (MIPS3_MAX_PCACHE_SIZE - 1) & ~PAGE_MASK; /* va[14:12] */
723 mci->mci_cache_prefer_mask = MIPS3_MAX_PCACHE_SIZE - 1;
724
725 mci->mci_icache_virtual_alias = false;
726 mci->mci_cache_virtual_alias = false;
727 /* FALLTHROUGH */
728 case MIPS_R4600:
729 #ifdef ENABLE_MIPS_R4700
730 case MIPS_R4700:
731 #endif
732 switch (mci->mci_sdcache_ways) {
733 case 1:
734 mco->mco_sdcache_wbinv_all =
735 r4k_sdcache_wbinv_all_generic;
736 switch (mci->mci_sdcache_line_size) {
737 case 16:
738 mco->mco_sdcache_wbinv_range =
739 cache_r4k_sdcache_hit_wb_inv_16;
740 mco->mco_sdcache_wbinv_range_index =
741 cache_r4k_sdcache_index_wb_inv_16;
742 mco->mco_sdcache_inv_range =
743 cache_r4k_sdcache_hit_inv_16;
744 mco->mco_sdcache_wb_range =
745 cache_r4k_sdcache_hit_wb_16;
746 break;
747
748 case 32:
749 mco->mco_sdcache_wbinv_range =
750 cache_r4k_sdcache_hit_wb_inv_32;
751 mco->mco_sdcache_wbinv_range_index =
752 cache_r4k_sdcache_index_wb_inv_32;
753 mco->mco_sdcache_inv_range =
754 cache_r4k_sdcache_hit_inv_32;
755 mco->mco_sdcache_wb_range =
756 cache_r4k_sdcache_hit_wb_32;
757 break;
758
759 case 64:
760 mco->mco_sdcache_wbinv_range =
761 cache_r4k_sdcache_hit_wb_inv_64;
762 mco->mco_sdcache_wbinv_range_index =
763 cache_r4k_sdcache_index_wb_inv_64;
764 mco->mco_sdcache_inv_range =
765 cache_r4k_sdcache_hit_inv_64;
766 mco->mco_sdcache_wb_range =
767 cache_r4k_sdcache_hit_wb_64;
768 break;
769
770 case 128:
771 mco->mco_sdcache_wbinv_range =
772 cache_r4k_sdcache_hit_wb_inv_128;
773 mco->mco_sdcache_wbinv_range_index =
774 cache_r4k_sdcache_index_wb_inv_128;
775 mco->mco_sdcache_inv_range =
776 cache_r4k_sdcache_hit_inv_128;
777 mco->mco_sdcache_wb_range =
778 cache_r4k_sdcache_hit_wb_128;
779 break;
780
781 default:
782 panic("r4k sdcache %d way line size %d",
783 mci->mci_sdcache_ways,
784 mci->mci_sdcache_line_size);
785 }
786 break;
787
788 default:
789 panic("r4k sdcache %d way line size %d",
790 mci->mci_sdcache_ways, mci->mci_sdcache_line_size);
791 }
792 break;
793 #ifndef ENABLE_MIPS_R3NKK
794 case MIPS_R5000:
795 #endif
796 case MIPS_RM5200:
797 mci->mci_sdcache_write_through = true;
798 mco->mco_sdcache_wbinv_all =
799 r5k_sdcache_wbinv_all;
800 mco->mco_sdcache_wbinv_range =
801 r5k_sdcache_wbinv_range;
802 mco->mco_sdcache_wbinv_range_index =
803 r5k_sdcache_wbinv_range_index;
804 mco->mco_sdcache_inv_range =
805 r5k_sdcache_wbinv_range;
806 mco->mco_sdcache_wb_range = no_cache_op_range;
807 break;
808 #ifdef ENABLE_MIPS4_CACHE_R10K
809 case MIPS_R10000:
810 case MIPS_R12000:
811 case MIPS_R14000:
812 mco->mco_sdcache_wbinv_all =
813 r10k_sdcache_wbinv_all;
814 mco->mco_sdcache_wbinv_range =
815 r10k_sdcache_wbinv_range;
816 mco->mco_sdcache_wbinv_range_index =
817 r10k_sdcache_wbinv_range_index;
818 mco->mco_sdcache_inv_range =
819 r10k_sdcache_inv_range;
820 mco->mco_sdcache_wb_range =
821 r10k_sdcache_wb_range;
822 break;
823 #endif /* ENABLE_MIPS4_CACHE_R10K */
824 #ifdef MIPS3_LOONGSON2
825 case MIPS_LOONGSON2:
826 mci->mci_sdcache_ways = 4;
827 mci->mci_sdcache_size = 512*1024;
828 mci->mci_scache_unified = 1;
829
830 mco->mco_sdcache_wbinv_all =
831 ls2_sdcache_wbinv_all;
832 mco->mco_sdcache_wbinv_range =
833 ls2_sdcache_wbinv_range;
834 mco->mco_sdcache_wbinv_range_index =
835 ls2_sdcache_wbinv_range_index;
836 mco->mco_sdcache_inv_range =
837 ls2_sdcache_inv_range;
838 mco->mco_sdcache_wb_range =
839 ls2_sdcache_wb_range;
840
841 /*
842 * The secondary cache is physically indexed and tagged
843 */
844 break;
845 #endif
846 #endif /* MIPS3 || MIPS4 */
847
848 default:
849 panic("can't handle secondary cache on impl 0x%x",
850 MIPS_PRID_IMPL(cpu_id));
851 }
852
853 /*
854 * Compute the "way mask" for each secondary cache.
855 */
856 if (mci->mci_sdcache_size) {
857 KASSERT(mci->mci_sdcache_ways != 0);
858 mci->mci_sdcache_way_size =
859 mci->mci_sdcache_size / mci->mci_sdcache_ways;
860 mci->mci_sdcache_way_mask = mci->mci_sdcache_way_size - 1;
861 }
862
863 mips_dcache_compute_align();
864 }
865
866 #if defined(MIPS1) || defined(MIPS3) || defined(MIPS4)
867 void
mips_config_cache_emips(void)868 mips_config_cache_emips(void)
869 {
870 struct mips_cache_info * const mci = &mips_cache_info;
871 struct mips_cache_ops * const mco = &mips_cache_ops;
872 const mips_prid_t cpu_id = mips_options.mips_cpu_id;
873 KASSERT(PAGE_SIZE != 0);
874
875 /*
876 * Configure primary caches.
877 */
878 switch (MIPS_PRID_IMPL(cpu_id)) {
879 case MIPS_eMIPS:
880 mci->mci_picache_size = 0;
881 mci->mci_pdcache_size = 0;
882
883 mci->mci_picache_line_size = 4;
884 mci->mci_pdcache_line_size = 4;
885
886 mci->mci_picache_ways = 1;
887 mci->mci_pdcache_ways = 1;
888
889 mci->mci_pdcache_write_through = true;
890
891 mco->mco_icache_sync_all = no_cache_op;
892 mco->mco_icache_sync_range = no_cache_op_range;
893 mco->mco_icache_sync_range_index = no_cache_op_range_index;
894
895 mco->mco_pdcache_wbinv_all = no_cache_op;
896 mco->mco_pdcache_wbinv_range = no_cache_op_range;
897 mco->mco_pdcache_wbinv_range_index = no_cache_op_range_index;
898 mco->mco_pdcache_inv_range = no_cache_op_range;
899 mco->mco_pdcache_wb_range = no_cache_op_range;
900
901 uvmexp.ncolors = 1;
902 break;
903
904 default:
905 panic("%s: unsupported eMIPS", __func__);
906 }
907 }
908 #endif
909
910 #ifdef MIPS1
911 #ifdef ENABLE_MIPS_TX3900
912 /*
913 * tx3900_get_cache_config:
914 *
915 * Fetch cache size information for the TX3900.
916 */
917 void
tx3900_get_cache_config(void)918 tx3900_get_cache_config(void)
919 {
920 struct mips_cache_info * const mci = &mips_cache_info;
921 uint32_t config;
922
923 config = tx3900_cp0_config_read();
924
925 mci->mci_picache_size = R3900_C_SIZE_MIN <<
926 ((config & R3900_CONFIG_ICS_MASK) >> R3900_CONFIG_ICS_SHIFT);
927
928 mci->mci_pdcache_size = R3900_C_SIZE_MIN <<
929 ((config & R3900_CONFIG_DCS_MASK) >> R3900_CONFIG_DCS_SHIFT);
930 }
931
932 /*
933 * tx3920_get_cache_config:
934 *
935 * Fetch cache size information for the TX3920.
936 */
937 void
tx3920_get_cache_config(void)938 tx3920_get_cache_config(void)
939 {
940 struct mips_cache_info * const mci = &mips_cache_info;
941
942 /* Size is the same as TX3900. */
943 tx3900_get_cache_config();
944
945 /* Now determine write-through/write-back mode. */
946 if ((tx3900_cp0_config_read() & R3900_CONFIG_WBON) == 0)
947 mci->mci_pdcache_write_through = true;
948 }
949
950 /*
951 * tx39_cache_config_write_through:
952 *
953 * TX3922 write-through D-cache mode.
954 * for TX3912, no meaning. (no write-back mode)
955 */
956 void
tx39_cache_config_write_through(void)957 tx39_cache_config_write_through(void)
958 {
959 u_int32_t r;
960
961 mips_dcache_wbinv_all();
962
963 __asm volatile("mfc0 %0, $3" : "=r"(r));
964 r &= 0xffffdfff;
965 __asm volatile("mtc0 %0, $3" : : "r"(r));
966 }
967
968 #endif /* ENABLE_MIPS_TX3900 */
969 #endif /* MIPS1 */
970
971 #if defined(MIPS3) || defined(MIPS4)
972 /*
973 * mips3_get_cache_config:
974 *
975 * Fetch the cache config information for a MIPS-3 or MIPS-4
976 * processor (virtually-indexed cache).
977 *
978 * NOTE: Fetching the size of the secondary cache is something
979 * that platform specific code has to do. We'd appreciate it
980 * if they initialized the size before now.
981 *
982 * ALSO NOTE: The number of ways in the cache must already be
983 * initialized.
984 */
985 void
mips3_get_cache_config(int csizebase)986 mips3_get_cache_config(int csizebase)
987 {
988 struct mips_cache_info * const mci = &mips_cache_info;
989 const mips_prid_t cpu_id = mips_options.mips_cpu_id;
990 bool has_sdcache_enable = false;
991 uint32_t config = mips3_cp0_config_read();
992
993 mci->mci_picache_size = MIPS3_CONFIG_CACHE_SIZE(config,
994 MIPS3_CONFIG_IC_MASK, csizebase, MIPS3_CONFIG_IC_SHIFT);
995 mci->mci_picache_line_size = MIPS3_CONFIG_CACHE_L1_LSIZE(config,
996 MIPS3_CONFIG_IB);
997
998 mci->mci_pdcache_size = MIPS3_CONFIG_CACHE_SIZE(config,
999 MIPS3_CONFIG_DC_MASK, csizebase, MIPS3_CONFIG_DC_SHIFT);
1000 mci->mci_pdcache_line_size = MIPS3_CONFIG_CACHE_L1_LSIZE(config,
1001 MIPS3_CONFIG_DB);
1002
1003 switch(MIPS_PRID_IMPL(cpu_id)) {
1004 #ifndef ENABLE_MIPS_R3NKK
1005 case MIPS_R5000:
1006 #endif
1007 case MIPS_RM5200:
1008 has_sdcache_enable = true;
1009 break;
1010 }
1011
1012 /*
1013 * If CPU has a software-enabled L2 cache, check both if it's
1014 * present and if it's enabled before making assumptions the
1015 * L2 is usable. If the L2 is disabled, we treat it the same
1016 * as if there were no L2 cache.
1017 */
1018 if ((config & MIPS3_CONFIG_SC) == 0) {
1019 if (has_sdcache_enable == 0 ||
1020 (has_sdcache_enable && (config & MIPS3_CONFIG_SE))) {
1021 mci->mci_sdcache_line_size =
1022 MIPS3_CONFIG_CACHE_L2_LSIZE(config);
1023 if ((config & MIPS3_CONFIG_SS) == 0)
1024 mci->mci_scache_unified = true;
1025 } else {
1026 #ifdef CACHE_DEBUG
1027 printf("External cache detected, but is disabled -- "
1028 "WILL NOT ENABLE!\n");
1029 #endif /* CACHE_DEBUG */
1030 }
1031 }
1032 }
1033
1034 #ifdef ENABLE_MIPS4_CACHE_R10K
1035 void
mips4_get_cache_config(int csizebase)1036 mips4_get_cache_config(int csizebase)
1037 {
1038 struct mips_cache_info * const mci = &mips_cache_info;
1039 uint32_t config = mips3_cp0_config_read();
1040
1041 mci->mci_picache_size = MIPS4_CONFIG_CACHE_SIZE(config,
1042 MIPS4_CONFIG_IC_MASK, csizebase, MIPS4_CONFIG_IC_SHIFT);
1043 mci->mci_picache_line_size = 64; /* 64 Byte */
1044
1045 mci->mci_pdcache_size = MIPS4_CONFIG_CACHE_SIZE(config,
1046 MIPS4_CONFIG_DC_MASK, csizebase, MIPS4_CONFIG_DC_SHIFT);
1047 mci->mci_pdcache_line_size = 32; /* 32 Byte */
1048
1049 mci->mci_cache_alias_mask =
1050 ((mci->mci_pdcache_size / mci->mci_pdcache_ways) - 1) & ~PAGE_MASK;
1051 mci->mci_cache_prefer_mask =
1052 uimax(mci->mci_pdcache_size, mci->mci_picache_size) - 1;
1053 }
1054 #endif /* ENABLE_MIPS4_CACHE_R10K */
1055 #endif /* MIPS3 || MIPS4 */
1056 #endif /* MIPS1 || MIPS3 || MIPS4 */
1057
1058 #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
1059
1060 static void
mips_config_cache_modern(uint32_t cpu_id)1061 mips_config_cache_modern(uint32_t cpu_id)
1062 {
1063 struct mips_cache_info * const mci = &mips_cache_info;
1064 struct mips_cache_ops * const mco = &mips_cache_ops;
1065 struct mips_options * const opts = &mips_options;
1066 /* MIPS32/MIPS64, use coprocessor 0 config registers */
1067 uint32_t cfg, cfg1;
1068
1069 cfg = mips3_cp0_config_read();
1070 cfg1 = mipsNN_cp0_config1_read();
1071
1072 #ifdef MIPS_DISABLE_L1_CACHE
1073 cfg1 &= ~MIPSNN_CFG1_IL_MASK;
1074 cfg1 &= ~MIPSNN_CFG1_DL_MASK;
1075 mipsNN_cp0_config1_write(cfg1);
1076 #endif
1077
1078 /* figure out Dcache params. */
1079 switch (MIPSNN_GET(CFG1_DL, cfg1)) {
1080 case MIPSNN_CFG1_DL_NONE:
1081 mci->mci_pdcache_line_size = mci->mci_pdcache_way_size =
1082 mci->mci_pdcache_ways = 0;
1083 #ifdef MIPS64_OCTEON
1084 if (MIPS_PRID_CID(cpu_id) == MIPS_PRID_CID_CAVIUM) {
1085 /*
1086 * Set the cache line size here, remaining Octeon
1087 * cache configuration will be done below.
1088 */
1089 mci->mci_pdcache_line_size = OCTEON_CACHELINE_SIZE;
1090 }
1091 #endif /* MIPS64_OCTEON */
1092 break;
1093 case MIPSNN_CFG1_DL_RSVD:
1094 panic("reserved MIPS32/64 Dcache line size");
1095 break;
1096 default:
1097 if (MIPSNN_GET(CFG1_DS, cfg1) == MIPSNN_CFG1_DS_RSVD)
1098 panic("reserved MIPS32/64 Dcache sets per way");
1099 mci->mci_pdcache_line_size = MIPSNN_CFG1_DL(cfg1);
1100 mci->mci_pdcache_way_size =
1101 mci->mci_pdcache_line_size * MIPSNN_CFG1_DS(cfg1);
1102 mci->mci_pdcache_ways = MIPSNN_CFG1_DA(cfg1) + 1;
1103
1104 /*
1105 * Compute the total size and "way mask" for the
1106 * primary Dcache.
1107 */
1108 mci->mci_pdcache_size =
1109 mci->mci_pdcache_way_size * mci->mci_pdcache_ways;
1110 mci->mci_pdcache_way_mask = mci->mci_pdcache_way_size - 1;
1111 uvmexp.ncolors = atop(mci->mci_pdcache_way_size);
1112 break;
1113 }
1114
1115 /* figure out Icache params. */
1116 switch (MIPSNN_GET(CFG1_IL, cfg1)) {
1117 case MIPSNN_CFG1_IL_NONE:
1118 mci->mci_picache_line_size = mci->mci_picache_way_size =
1119 mci->mci_picache_ways = 0;
1120 break;
1121 case MIPSNN_CFG1_IL_RSVD:
1122 panic("reserved MIPS32/64 Icache line size");
1123 break;
1124 default:
1125 if (MIPSNN_GET(CFG1_IS, cfg1) == MIPSNN_CFG1_IS_RSVD)
1126 panic("reserved MIPS32/64 Icache sets per way");
1127 mci->mci_picache_line_size = MIPSNN_CFG1_IL(cfg1);
1128 mci->mci_picache_way_size =
1129 mci->mci_picache_line_size * MIPSNN_CFG1_IS(cfg1);
1130 mci->mci_picache_ways = MIPSNN_CFG1_IA(cfg1) + 1;
1131
1132 /*
1133 * Is this Icache virtually indexed and virtually tagged?
1134 */
1135 mci->mci_picache_vivt = (cfg & MIPSNN_CFG_VI) != 0;
1136
1137 /*
1138 * Compute the total size and "way mask" for the
1139 * primary Icache.
1140 */
1141 mci->mci_picache_size =
1142 mci->mci_picache_way_size * mci->mci_picache_ways;
1143 mci->mci_picache_way_mask = mci->mci_picache_way_size - 1;
1144 break;
1145 }
1146
1147 mco->mco_icache_sync_all = mipsNN_picache_sync_all;
1148 mco->mco_icache_sync_range = mipsNN_picache_sync_range;
1149 mco->mco_icache_sync_range_index = mipsNN_picache_sync_range_index;
1150
1151 switch (mci->mci_picache_line_size) {
1152 #ifdef MIPS_DISABLE_L1_CACHE
1153 case 0:
1154 mco->mco_icache_sync_all = no_cache_op;
1155 mco->mco_icache_sync_range = no_cache_op_range;
1156 mco->mco_icache_sync_range_index = no_cache_op_range_index;
1157 break;
1158 #endif
1159 case 16:
1160 /* used internally by mipsNN_picache_sync_range */
1161 mco->mco_intern_icache_sync_range =
1162 cache_r4k_icache_hit_inv_16;
1163
1164 /* used internally by mipsNN_picache_sync_range_index */
1165 mco->mco_intern_icache_sync_range_index =
1166 cache_r4k_icache_index_inv_16;
1167 break;
1168 case 32:
1169 /* used internally by mipsNN_picache_sync_range */
1170 mco->mco_intern_icache_sync_range =
1171 cache_r4k_icache_hit_inv_32;
1172
1173 /* used internally by mipsNN_picache_sync_range_index */
1174 mco->mco_intern_icache_sync_range_index =
1175 cache_r4k_icache_index_inv_32;
1176 break;
1177 case 64:
1178 /* used internally by mipsNN_picache_sync_range */
1179 mco->mco_intern_icache_sync_range =
1180 cache_r4k_icache_hit_inv_64;
1181
1182 /* used internally by mipsNN_picache_sync_range_index */
1183 mco->mco_intern_icache_sync_range_index =
1184 cache_r4k_icache_index_inv_64;
1185 break;
1186 case 128:
1187 /* used internally by mipsNN_picache_sync_range */
1188 mco->mco_intern_icache_sync_range =
1189 cache_r4k_icache_hit_inv_128;
1190
1191 /* used internally by mipsNN_picache_sync_range_index */
1192 mco->mco_intern_icache_sync_range_index =
1193 cache_r4k_icache_index_inv_128;
1194 break;
1195 default:
1196 panic("no Icache ops for %dB lines",
1197 mci->mci_picache_line_size);
1198 }
1199
1200 mco->mco_pdcache_wbinv_all = mipsNN_pdcache_wbinv_all;
1201 mco->mco_pdcache_wbinv_range_index = mipsNN_pdcache_wbinv_range_index;
1202
1203 switch (mci->mci_pdcache_line_size) {
1204 #ifdef MIPS_DISABLE_L1_CACHE
1205 case 0:
1206 mco->mco_pdcache_wbinv_all = no_cache_op;
1207 mco->mco_pdcache_wbinv_range = no_cache_op_range;
1208 mco->mco_pdcache_wbinv_range_index = no_cache_op_index;
1209 mco->mco_pdcache_inv_range = no_cache_op_range;
1210 mco->mco_pdcache_wb_range = no_cache_op_range;
1211 break;
1212 #endif
1213 case 16:
1214 mco->mco_pdcache_wbinv_range =
1215 cache_r4k_pdcache_hit_wb_inv_16;
1216 mco->mco_pdcache_inv_range =
1217 cache_r4k_pdcache_hit_inv_16;
1218 mco->mco_pdcache_wb_range =
1219 cache_r4k_pdcache_hit_wb_16;
1220
1221 /* used internally by mipsNN_pdcache_wbinv_range_index */
1222 mco->mco_intern_pdcache_wbinv_range_index =
1223 cache_r4k_pdcache_index_wb_inv_16;
1224 break;
1225 case 32:
1226 mco->mco_pdcache_wbinv_range =
1227 cache_r4k_pdcache_hit_wb_inv_32;
1228 mco->mco_pdcache_inv_range =
1229 cache_r4k_pdcache_hit_inv_32;
1230 mco->mco_pdcache_wb_range =
1231 cache_r4k_pdcache_hit_wb_32;
1232
1233 /* used internally by mipsNN_pdcache_wbinv_range_index */
1234 mco->mco_intern_pdcache_wbinv_range_index =
1235 cache_r4k_pdcache_index_wb_inv_32;
1236 break;
1237 case 64:
1238 mco->mco_pdcache_wbinv_range =
1239 cache_r4k_pdcache_hit_wb_inv_64;
1240 mco->mco_pdcache_inv_range =
1241 cache_r4k_pdcache_hit_inv_64;
1242 mco->mco_pdcache_wb_range =
1243 cache_r4k_pdcache_hit_wb_64;
1244
1245 /* used internally by mipsNN_pdcache_wbinv_range_index */
1246 mco->mco_intern_pdcache_wbinv_range_index =
1247 cache_r4k_pdcache_index_wb_inv_64;
1248 case 128:
1249 mco->mco_pdcache_wbinv_range =
1250 cache_r4k_pdcache_hit_wb_inv_128;
1251 mco->mco_pdcache_inv_range =
1252 cache_r4k_pdcache_hit_inv_128;
1253 mco->mco_pdcache_wb_range =
1254 cache_r4k_pdcache_hit_wb_128;
1255
1256 /* used internally by mipsNN_pdcache_wbinv_range_index */
1257 mco->mco_intern_pdcache_wbinv_range_index =
1258 cache_r4k_pdcache_index_wb_inv_128;
1259 break;
1260 default:
1261 panic("no Dcache ops for %dB lines",
1262 mci->mci_pdcache_line_size);
1263 }
1264
1265 mco->mco_intern_pdcache_sync_all = mco->mco_pdcache_wbinv_all;
1266 mco->mco_intern_pdcache_sync_range_index =
1267 mco->mco_intern_pdcache_wbinv_range_index;
1268 mco->mco_intern_pdcache_sync_range = mco->mco_pdcache_wb_range;
1269
1270 if (MIPSNN_CFG1_M & cfg1) {
1271 uint32_t cfg2 = mipsNN_cp0_config2_read();
1272
1273 switch (MIPSNN_GET(CFG2_SL, cfg2)) {
1274 case MIPSNN_CFG2_SL_NONE:
1275 break;
1276 default:
1277 mci->mci_scache_unified = true;
1278
1279 mci->mci_sdcache_line_size = MIPSNN_CFG2_SL(cfg2);
1280 mci->mci_sdcache_way_size =
1281 mci->mci_sdcache_line_size * MIPSNN_CFG2_SS(cfg2);
1282 mci->mci_sdcache_ways = MIPSNN_CFG2_SA(cfg2) + 1;
1283
1284 /*
1285 * Compute the total size and "way mask" for the
1286 * secondary Dcache.
1287 */
1288 mci->mci_sdcache_size =
1289 mci->mci_sdcache_way_size * mci->mci_sdcache_ways;
1290 mci->mci_sdcache_way_mask =
1291 mci->mci_sdcache_way_size - 1;
1292
1293 /*
1294 * cache is unified so copy data info to inst info.
1295 */
1296 mci->mci_sicache_line_size = mci->mci_sdcache_line_size;
1297 mci->mci_sicache_way_size = mci->mci_sdcache_way_size;
1298 mci->mci_sicache_ways = mci->mci_sdcache_ways;
1299 mci->mci_sicache_size = mci->mci_sdcache_size;
1300 mci->mci_sicache_way_mask = mci->mci_sdcache_way_mask;
1301
1302 break;
1303 }
1304
1305 /*
1306 * Note we don't set up any sd cache ops because we expect that
1307 * the coherence checks below will overwrite them with no ops.
1308 */
1309
1310 switch (MIPSNN_GET(CFG2_TL, cfg2)) {
1311 case MIPSNN_CFG2_TL_NONE:
1312 break;
1313 default:
1314 mci->mci_tcache_line_size = MIPSNN_CFG2_TL(cfg2);
1315 mci->mci_tcache_way_size =
1316 mci->mci_tcache_line_size * MIPSNN_CFG2_TS(cfg2);
1317 mci->mci_tcache_ways = MIPSNN_CFG2_TA(cfg2) + 1;
1318
1319 /*
1320 * Compute the total size and "way mask" for the
1321 * secondary Dcache.
1322 */
1323 mci->mci_tcache_size =
1324 mci->mci_tcache_way_size * mci->mci_tcache_ways;
1325 mci->mci_tcache_way_mask =
1326 mci->mci_tcache_way_size - 1;
1327 break;
1328 }
1329 }
1330
1331 /*
1332 * calculate the alias masks and from them set to virtual alias flags.
1333 */
1334 mci->mci_cache_alias_mask = mci->mci_pdcache_way_mask & -PAGE_SIZE;
1335 mci->mci_cache_virtual_alias = (mci->mci_cache_alias_mask != 0);
1336
1337 mci->mci_icache_alias_mask = mci->mci_picache_way_mask & -PAGE_SIZE;
1338 mci->mci_icache_virtual_alias = (mci->mci_icache_alias_mask != 0);
1339
1340 /*
1341 * Core specific overrides
1342 */
1343 switch (MIPS_PRID_CID(cpu_id)) {
1344 case MIPS_PRID_CID_MTI:
1345 /*
1346 * All MTI cores share a (mostly) common config7 definition.
1347 * Use it to determine if the caches have virtual aliases.
1348 * If the core doesn't have a config7 register, its caches
1349 * are too small or have too many ways to have aliases.
1350 */
1351 if (opts->mips_cpu->cpu_cp0flags & MIPS_CP0FL_CONFIG7) {
1352 const uint32_t cfg7 = mipsNN_cp0_config7_read();
1353 if (cfg7 & MIPSNN_MTI_CFG7_AR) {
1354 /* [Data] Alias Removal Present */
1355 mci->mci_cache_virtual_alias = false;
1356 }
1357 if (cfg7 & MIPSNN_MTI_CFG7_IAR) {
1358 /* Instruction Alias Removal Present */
1359 mci->mci_icache_virtual_alias = false;
1360 }
1361 #if 0
1362 } else {
1363 KASSERT(mci->mci_pdcache_way_size <= PAGE_SIZE);
1364 KASSERT(mci->mci_picache_way_size <= PAGE_SIZE);
1365 #endif
1366 }
1367 break;
1368 case MIPS_PRID_CID_RMI:
1369 /*
1370 * RMI (NetLogic/Broadcom) don't support WB (op 6)
1371 * so we have to make do with WBINV (op 5). This is
1372 * merely for correctness since because the caches are
1373 * coherent, these routines will become noops in a bit.
1374 */
1375 mco->mco_pdcache_wb_range = mco->mco_pdcache_wbinv_range;
1376 mco->mco_intern_pdcache_sync_range =
1377 mco->mco_pdcache_wbinv_range;
1378 if (MIPSNN_GET(CFG_AR, cfg) == MIPSNN_CFG_AR_REV2) {
1379 mci->mci_pdcache_write_through = true;
1380 mci->mci_sdcache_write_through = false;
1381 KASSERT(PAGE_SIZE >= mci->mci_picache_way_size
1382 || MIPS_ICACHE_VIRTUAL_ALIAS);
1383 } else {
1384 KASSERT(MIPS_CACHE_VIRTUAL_ALIAS == 0);
1385 KASSERT(MIPS_ICACHE_VIRTUAL_ALIAS == 0);
1386 }
1387 break;
1388 #ifdef MIPS64_OCTEON
1389 case MIPS_PRID_CID_CAVIUM:
1390 /*
1391 * Cavium Octeon cores have cache configurations that aren't
1392 * describable using the standard MIPS configN definitions.
1393 */
1394 switch (MIPS_PRID_IMPL(cpu_id)) {
1395 case MIPS_CN38XX:
1396 case MIPS_CN31XX:
1397 case MIPS_CN30XX:
1398 case MIPS_CN50XX:
1399 case MIPS_CN52XX:
1400 case MIPS_CN58XX:
1401 case MIPS_CN56XX:
1402 /* OCTEON and OCTEON Plus */
1403
1404 /* Dcache on cnMIPS core doesn't follow spec */
1405 mci->mci_pdcache_line_size = OCTEON_CACHELINE_SIZE;
1406 mci->mci_pdcache_ways = OCTEON_I_DCACHE_WAYS;
1407 mci->mci_pdcache_way_size =
1408 OCTEON_I_DCACHE_SETS * OCTEON_CACHELINE_SIZE;
1409 mci->mci_pdcache_write_through = true;
1410
1411 /* Icache on cnMIPS core does follows MIPS spec */
1412
1413 break;
1414
1415 /* XXX cnMIPS II cores not yet tested */
1416 case MIPS_CN61XX:
1417 case MIPS_CN63XX:
1418 case MIPS_CN66XX:
1419 case MIPS_CN68XX:
1420 case MIPS_CNF71XX:
1421 /* OCTEON II */
1422
1423 mci->mci_pdcache_line_size = OCTEON_CACHELINE_SIZE;
1424 mci->mci_pdcache_ways = OCTEON_II_DCACHE_WAYS;
1425 mci->mci_pdcache_way_size =
1426 OCTEON_II_DCACHE_SETS * OCTEON_CACHELINE_SIZE;
1427 mci->mci_pdcache_write_through = true;
1428
1429 mci->mci_picache_line_size = OCTEON_CACHELINE_SIZE;
1430 mci->mci_picache_ways = OCTEON_II_ICACHE_WAYS;
1431 mci->mci_picache_way_size =
1432 OCTEON_II_ICACHE_SETS * OCTEON_CACHELINE_SIZE;
1433 break;
1434
1435 case MIPS_CN70XX:
1436 case MIPS_CN73XX:
1437 case MIPS_CNF75XX:
1438 case MIPS_CN78XX:
1439 /* OCTEON III */
1440
1441 mci->mci_pdcache_line_size = OCTEON_CACHELINE_SIZE;
1442 mci->mci_pdcache_ways = OCTEON_III_DCACHE_WAYS;
1443 mci->mci_pdcache_way_size =
1444 OCTEON_CACHELINE_SIZE * OCTEON_III_DCACHE_SETS;
1445 mci->mci_pdcache_write_through = true;
1446
1447 mci->mci_picache_line_size = OCTEON_CACHELINE_SIZE;
1448 mci->mci_picache_ways = OCTEON_III_ICACHE_WAYS;
1449 mci->mci_picache_way_size =
1450 OCTEON_CACHELINE_SIZE * OCTEON_III_ICACHE_SETS;
1451 break;
1452
1453 default:
1454 panic("Unknown Octeon Cavium core impl %#x",
1455 MIPS_PRID_IMPL(cpu_id));
1456 }
1457
1458 /* recalculate dcache params */
1459 mci->mci_pdcache_size =
1460 mci->mci_pdcache_way_size * mci->mci_pdcache_ways;
1461 mci->mci_pdcache_way_mask = mci->mci_pdcache_way_size - 1;
1462 uvmexp.ncolors =
1463 atop(mci->mci_pdcache_size) / mci->mci_pdcache_ways;
1464
1465 /* recalculate icache params */
1466 mci->mci_picache_size =
1467 mci->mci_picache_way_size * mci->mci_picache_ways;
1468 mci->mci_picache_way_mask = mci->mci_picache_way_size - 1;
1469
1470 /* recalculate the alias masks */
1471 mci->mci_cache_alias_mask =
1472 mci->mci_pdcache_way_mask & -PAGE_SIZE;
1473 mci->mci_cache_virtual_alias =
1474 (mci->mci_cache_alias_mask != 0);
1475
1476 mci->mci_icache_alias_mask =
1477 mci->mci_picache_way_mask & -PAGE_SIZE;
1478 mci->mci_icache_virtual_alias =
1479 (mci->mci_icache_alias_mask != 0);
1480
1481 /* use Octeon-specific cache ops */
1482 mco->mco_icache_sync_all = octeon_icache_sync_all;
1483 mco->mco_icache_sync_range = octeon_icache_sync_range;
1484 mco->mco_icache_sync_range_index =
1485 octeon_icache_sync_range_index;
1486
1487 mco->mco_pdcache_wbinv_all = octeon_pdcache_inv_all;
1488 mco->mco_pdcache_wbinv_range = octeon_pdcache_inv_range;
1489 mco->mco_pdcache_wbinv_range_index =
1490 octeon_pdcache_inv_range_index;
1491 mco->mco_pdcache_inv_range = octeon_pdcache_inv_range;
1492 mco->mco_pdcache_wb_range = no_cache_op_range;
1493 #endif /* MIPS64_OCTEON */
1494 }
1495
1496 #define CACHE_DEBUG
1497 #ifdef CACHE_DEBUG
1498 printf("MIPS32/64 params: cpu arch: %d\n", opts->mips_cpu_arch);
1499 printf("MIPS32/64 params: TLB entries: %d\n", opts->mips_num_tlb_entries);
1500 if (mci->mci_picache_line_size == 0) {
1501 printf("MIPS32/64 params: no Icache\n");
1502 } else {
1503 printf("MIPS32/64 params: %s: line=%d, total=%d, "
1504 "ways=%d, sets=%d, colors=%d\n", "Icache",
1505 mci->mci_picache_line_size,
1506 mci->mci_picache_way_size * mci->mci_picache_ways,
1507 mci->mci_picache_ways,
1508 mci->mci_picache_way_size / mci->mci_picache_line_size,
1509 mci->mci_picache_way_size >> PAGE_SHIFT);
1510 }
1511 if (mci->mci_pdcache_line_size == 0) {
1512 printf("MIPS32/64 params: no Dcache\n");
1513 } else {
1514 printf("MIPS32/64 params: %s: line=%d, total=%d, "
1515 "ways=%d, sets=%d, colors=%d\n", "Dcache",
1516 mci->mci_pdcache_line_size,
1517 mci->mci_pdcache_way_size * mci->mci_pdcache_ways,
1518 mci->mci_pdcache_ways,
1519 mci->mci_pdcache_way_size / mci->mci_pdcache_line_size,
1520 mci->mci_pdcache_way_size >> PAGE_SHIFT);
1521 }
1522 if (mci->mci_sdcache_line_size != 0) {
1523 printf("MIPS32/64 params: %s: line=%d, total=%d, "
1524 "ways=%d, sets=%d, colors=%d\n", "SDcache",
1525 mci->mci_sdcache_line_size,
1526 mci->mci_sdcache_way_size * mci->mci_sdcache_ways,
1527 mci->mci_sdcache_ways,
1528 mci->mci_sdcache_way_size / mci->mci_sdcache_line_size,
1529 mci->mci_sdcache_way_size >> PAGE_SHIFT);
1530 }
1531 #endif /* CACHE_DEBUG */
1532
1533 mipsNN_cache_init(cfg, cfg1);
1534
1535 if (opts->mips_cpu_flags &
1536 (CPU_MIPS_D_CACHE_COHERENT | CPU_MIPS_I_D_CACHE_COHERENT)) {
1537 #ifdef CACHE_DEBUG
1538 printf(" Dcache is coherent\n");
1539 #endif
1540 mco->mco_pdcache_wbinv_all = no_cache_op;
1541 mco->mco_pdcache_wbinv_range = no_cache_op_range;
1542 mco->mco_pdcache_wbinv_range_index = no_cache_op_range_index;
1543 mco->mco_pdcache_inv_range = no_cache_op_range;
1544 mco->mco_pdcache_wb_range = no_cache_op_range;
1545 mco->mco_sdcache_wbinv_all = no_cache_op;
1546 mco->mco_sdcache_wbinv_range = no_cache_op_range;
1547 mco->mco_sdcache_wbinv_range_index = no_cache_op_range_index;
1548 mco->mco_sdcache_inv_range = no_cache_op_range;
1549 mco->mco_sdcache_wb_range = no_cache_op_range;
1550 }
1551 if (opts->mips_cpu_flags & CPU_MIPS_I_D_CACHE_COHERENT) {
1552 #ifdef CACHE_DEBUG
1553 printf(" Icache is coherent against Dcache\n");
1554 #endif
1555 mco->mco_intern_pdcache_sync_all = no_cache_op;
1556 mco->mco_intern_pdcache_sync_range_index =
1557 no_cache_op_range_index;
1558 mco->mco_intern_pdcache_sync_range = no_cache_op_range;
1559 }
1560
1561 mips_dcache_compute_align();
1562 }
1563 #endif /* MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2 > 0 */
1564