1 /* $NetBSD: cache_mipsNN.c,v 1.10 2005/12/24 20:07:19 perry Exp $ */
2
3 /*
4 * SPDX-License-Identifier: BSD-4-Clause
5 *
6 * Copyright 2001 Wasabi Systems, Inc.
7 * All rights reserved.
8 *
9 * Written by Jason R. Thorpe and Simon Burge for Wasabi Systems, Inc.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed for the NetBSD Project by
22 * Wasabi Systems, Inc.
23 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
24 * or promote products derived from this software without specific prior
25 * written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
38 */
39
40 #include <sys/cdefs.h>
41 #include <sys/types.h>
42 #include <sys/systm.h>
43 #include <sys/param.h>
44
45 #include <machine/cache.h>
46 #include <machine/cache_r4k.h>
47 #include <machine/cpuinfo.h>
48
49 #define round_line16(x) (((x) + 15) & ~15)
50 #define trunc_line16(x) ((x) & ~15)
51
52 #define round_line32(x) (((x) + 31) & ~31)
53 #define trunc_line32(x) ((x) & ~31)
54
55 #define round_line64(x) (((x) + 63) & ~63)
56 #define trunc_line64(x) ((x) & ~63)
57
58 #define round_line128(x) (((x) + 127) & ~127)
59 #define trunc_line128(x) ((x) & ~127)
60
61 #if defined(CPU_NLM)
62 static __inline void
xlp_sync(void)63 xlp_sync(void)
64 {
65 __asm __volatile (
66 ".set push \n"
67 ".set noreorder \n"
68 ".set mips64 \n"
69 "dla $8, 1f \n"
70 "/* jr.hb $8 */ \n"
71 ".word 0x1000408 \n"
72 "nop \n"
73 "1: nop \n"
74 ".set pop \n"
75 : : : "$8");
76 }
77 #endif
78
79 #if defined(SB1250_PASS1)
80 #define SYNC __asm volatile("sync; sync")
81 #elif defined(CPU_NLM)
82 #define SYNC xlp_sync()
83 #else
84 #define SYNC __asm volatile("sync")
85 #endif
86
87 #if defined(CPU_CNMIPS)
88 #define SYNCI mips_sync_icache();
89 #elif defined(CPU_NLM)
90 #define SYNCI xlp_sync()
91 #else
92 #define SYNCI
93 #endif
94
95 /*
96 * Exported variables for consumers like bus_dma code
97 */
98 int mips_picache_linesize;
99 int mips_pdcache_linesize;
100 int mips_sdcache_linesize;
101 int mips_dcache_max_linesize;
102
103 static int picache_size;
104 static int picache_stride;
105 static int picache_loopcount;
106 static int picache_way_mask;
107 static int pdcache_size;
108 static int pdcache_stride;
109 static int pdcache_loopcount;
110 static int pdcache_way_mask;
111 static int sdcache_size;
112 static int sdcache_stride;
113 static int sdcache_loopcount;
114 static int sdcache_way_mask;
115
116 void
mipsNN_cache_init(struct mips_cpuinfo * cpuinfo)117 mipsNN_cache_init(struct mips_cpuinfo * cpuinfo)
118 {
119 int flush_multiple_lines_per_way;
120
121 flush_multiple_lines_per_way = cpuinfo->l1.ic_nsets * cpuinfo->l1.ic_linesize * cpuinfo->l1.ic_linesize > PAGE_SIZE;
122 if (cpuinfo->icache_virtual) {
123 /*
124 * With a virtual Icache we don't need to flush
125 * multiples of the page size with index ops; we just
126 * need to flush one pages' worth.
127 */
128 flush_multiple_lines_per_way = 0;
129 }
130
131 if (flush_multiple_lines_per_way) {
132 picache_stride = PAGE_SIZE;
133 picache_loopcount = (cpuinfo->l1.ic_nsets * cpuinfo->l1.ic_linesize / PAGE_SIZE) *
134 cpuinfo->l1.ic_nways;
135 } else {
136 picache_stride = cpuinfo->l1.ic_nsets * cpuinfo->l1.ic_linesize;
137 picache_loopcount = cpuinfo->l1.ic_nways;
138 }
139
140 if (cpuinfo->l1.dc_nsets * cpuinfo->l1.dc_linesize < PAGE_SIZE) {
141 pdcache_stride = cpuinfo->l1.dc_nsets * cpuinfo->l1.dc_linesize;
142 pdcache_loopcount = cpuinfo->l1.dc_nways;
143 } else {
144 pdcache_stride = PAGE_SIZE;
145 pdcache_loopcount = (cpuinfo->l1.dc_nsets * cpuinfo->l1.dc_linesize / PAGE_SIZE) *
146 cpuinfo->l1.dc_nways;
147 }
148
149 mips_picache_linesize = cpuinfo->l1.ic_linesize;
150 mips_pdcache_linesize = cpuinfo->l1.dc_linesize;
151
152 picache_size = cpuinfo->l1.ic_size;
153 picache_way_mask = cpuinfo->l1.ic_nways - 1;
154 pdcache_size = cpuinfo->l1.dc_size;
155 pdcache_way_mask = cpuinfo->l1.dc_nways - 1;
156
157 sdcache_stride = cpuinfo->l2.dc_nsets * cpuinfo->l2.dc_linesize;
158 sdcache_loopcount = cpuinfo->l2.dc_nways;
159 sdcache_size = cpuinfo->l2.dc_size;
160 sdcache_way_mask = cpuinfo->l2.dc_nways - 1;
161
162 mips_sdcache_linesize = cpuinfo->l2.dc_linesize;
163 mips_dcache_max_linesize = MAX(mips_pdcache_linesize,
164 mips_sdcache_linesize);
165
166 #define CACHE_DEBUG
167 #ifdef CACHE_DEBUG
168 printf("Cache info:\n");
169 if (cpuinfo->icache_virtual)
170 printf(" icache is virtual\n");
171 printf(" picache_stride = %d\n", picache_stride);
172 printf(" picache_loopcount = %d\n", picache_loopcount);
173 printf(" pdcache_stride = %d\n", pdcache_stride);
174 printf(" pdcache_loopcount = %d\n", pdcache_loopcount);
175 printf(" max line size = %d\n", mips_dcache_max_linesize);
176 #endif
177 }
178
179 void
mipsNN_icache_sync_all_16(void)180 mipsNN_icache_sync_all_16(void)
181 {
182 vm_offset_t va, eva;
183
184 va = MIPS_PHYS_TO_KSEG0(0);
185 eva = va + picache_size;
186
187 /*
188 * Since we're hitting the whole thing, we don't have to
189 * worry about the N different "ways".
190 */
191
192 mips_intern_dcache_wbinv_all();
193
194 while (va < eva) {
195 cache_r4k_op_32lines_16(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
196 va += (32 * 16);
197 }
198
199 SYNC;
200 }
201
202 void
mipsNN_icache_sync_all_32(void)203 mipsNN_icache_sync_all_32(void)
204 {
205 vm_offset_t va, eva;
206
207 va = MIPS_PHYS_TO_KSEG0(0);
208 eva = va + picache_size;
209
210 /*
211 * Since we're hitting the whole thing, we don't have to
212 * worry about the N different "ways".
213 */
214
215 mips_intern_dcache_wbinv_all();
216
217 while (va < eva) {
218 cache_r4k_op_32lines_32(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
219 va += (32 * 32);
220 }
221
222 SYNC;
223 }
224
225 void
mipsNN_icache_sync_all_64(void)226 mipsNN_icache_sync_all_64(void)
227 {
228 vm_offset_t va, eva;
229
230 va = MIPS_PHYS_TO_KSEG0(0);
231 eva = va + picache_size;
232
233 /*
234 * Since we're hitting the whole thing, we don't have to
235 * worry about the N different "ways".
236 */
237
238 mips_intern_dcache_wbinv_all();
239
240 while (va < eva) {
241 cache_r4k_op_32lines_64(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
242 va += (32 * 64);
243 }
244
245 SYNC;
246 }
247
248 void
mipsNN_icache_sync_range_16(vm_offset_t va,vm_size_t size)249 mipsNN_icache_sync_range_16(vm_offset_t va, vm_size_t size)
250 {
251 vm_offset_t eva;
252
253 eva = round_line16(va + size);
254 va = trunc_line16(va);
255
256 mips_intern_dcache_wb_range(va, (eva - va));
257
258 while ((eva - va) >= (32 * 16)) {
259 cache_r4k_op_32lines_16(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
260 va += (32 * 16);
261 }
262
263 while (va < eva) {
264 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
265 va += 16;
266 }
267
268 SYNC;
269 }
270
271 void
mipsNN_icache_sync_range_32(vm_offset_t va,vm_size_t size)272 mipsNN_icache_sync_range_32(vm_offset_t va, vm_size_t size)
273 {
274 vm_offset_t eva;
275
276 eva = round_line32(va + size);
277 va = trunc_line32(va);
278
279 mips_intern_dcache_wb_range(va, (eva - va));
280
281 while ((eva - va) >= (32 * 32)) {
282 cache_r4k_op_32lines_32(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
283 va += (32 * 32);
284 }
285
286 while (va < eva) {
287 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
288 va += 32;
289 }
290
291 SYNC;
292 }
293
294 void
mipsNN_icache_sync_range_64(vm_offset_t va,vm_size_t size)295 mipsNN_icache_sync_range_64(vm_offset_t va, vm_size_t size)
296 {
297 vm_offset_t eva;
298
299 eva = round_line64(va + size);
300 va = trunc_line64(va);
301
302 mips_intern_dcache_wb_range(va, (eva - va));
303
304 while ((eva - va) >= (32 * 64)) {
305 cache_r4k_op_32lines_64(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
306 va += (32 * 64);
307 }
308
309 while (va < eva) {
310 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
311 va += 64;
312 }
313
314 SYNC;
315 }
316
317 void
mipsNN_icache_sync_range_index_16(vm_offset_t va,vm_size_t size)318 mipsNN_icache_sync_range_index_16(vm_offset_t va, vm_size_t size)
319 {
320 vm_offset_t eva, tmpva;
321 int i, stride, loopcount;
322
323 /*
324 * Since we're doing Index ops, we expect to not be able
325 * to access the address we've been given. So, get the
326 * bits that determine the cache index, and make a KSEG0
327 * address out of them.
328 */
329 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
330
331 eva = round_line16(va + size);
332 va = trunc_line16(va);
333
334 /*
335 * GCC generates better code in the loops if we reference local
336 * copies of these global variables.
337 */
338 stride = picache_stride;
339 loopcount = picache_loopcount;
340
341 mips_intern_dcache_wbinv_range_index(va, (eva - va));
342
343 while ((eva - va) >= (8 * 16)) {
344 tmpva = va;
345 for (i = 0; i < loopcount; i++, tmpva += stride)
346 cache_r4k_op_8lines_16(tmpva,
347 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
348 va += 8 * 16;
349 }
350
351 while (va < eva) {
352 tmpva = va;
353 for (i = 0; i < loopcount; i++, tmpva += stride)
354 cache_op_r4k_line(tmpva,
355 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
356 va += 16;
357 }
358 }
359
360 void
mipsNN_icache_sync_range_index_32(vm_offset_t va,vm_size_t size)361 mipsNN_icache_sync_range_index_32(vm_offset_t va, vm_size_t size)
362 {
363 vm_offset_t eva, tmpva;
364 int i, stride, loopcount;
365
366 /*
367 * Since we're doing Index ops, we expect to not be able
368 * to access the address we've been given. So, get the
369 * bits that determine the cache index, and make a KSEG0
370 * address out of them.
371 */
372 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
373
374 eva = round_line32(va + size);
375 va = trunc_line32(va);
376
377 /*
378 * GCC generates better code in the loops if we reference local
379 * copies of these global variables.
380 */
381 stride = picache_stride;
382 loopcount = picache_loopcount;
383
384 mips_intern_dcache_wbinv_range_index(va, (eva - va));
385
386 while ((eva - va) >= (8 * 32)) {
387 tmpva = va;
388 for (i = 0; i < loopcount; i++, tmpva += stride)
389 cache_r4k_op_8lines_32(tmpva,
390 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
391 va += 8 * 32;
392 }
393
394 while (va < eva) {
395 tmpva = va;
396 for (i = 0; i < loopcount; i++, tmpva += stride)
397 cache_op_r4k_line(tmpva,
398 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
399 va += 32;
400 }
401 }
402
403 void
mipsNN_icache_sync_range_index_64(vm_offset_t va,vm_size_t size)404 mipsNN_icache_sync_range_index_64(vm_offset_t va, vm_size_t size)
405 {
406 vm_offset_t eva, tmpva;
407 int i, stride, loopcount;
408
409 /*
410 * Since we're doing Index ops, we expect to not be able
411 * to access the address we've been given. So, get the
412 * bits that determine the cache index, and make a KSEG0
413 * address out of them.
414 */
415 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
416
417 eva = round_line64(va + size);
418 va = trunc_line64(va);
419
420 /*
421 * GCC generates better code in the loops if we reference local
422 * copies of these global variables.
423 */
424 stride = picache_stride;
425 loopcount = picache_loopcount;
426
427 mips_intern_dcache_wbinv_range_index(va, (eva - va));
428
429 while ((eva - va) >= (8 * 64)) {
430 tmpva = va;
431 for (i = 0; i < loopcount; i++, tmpva += stride)
432 cache_r4k_op_8lines_64(tmpva,
433 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
434 va += 8 * 64;
435 }
436
437 while (va < eva) {
438 tmpva = va;
439 for (i = 0; i < loopcount; i++, tmpva += stride)
440 cache_op_r4k_line(tmpva,
441 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
442 va += 64;
443 }
444 }
445
446 void
mipsNN_pdcache_wbinv_all_16(void)447 mipsNN_pdcache_wbinv_all_16(void)
448 {
449 vm_offset_t va, eva;
450
451 va = MIPS_PHYS_TO_KSEG0(0);
452 eva = va + pdcache_size;
453
454 /*
455 * Since we're hitting the whole thing, we don't have to
456 * worry about the N different "ways".
457 */
458
459 while (va < eva) {
460 cache_r4k_op_32lines_16(va,
461 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
462 va += (32 * 16);
463 }
464
465 SYNC;
466 }
467
468 void
mipsNN_pdcache_wbinv_all_32(void)469 mipsNN_pdcache_wbinv_all_32(void)
470 {
471 vm_offset_t va, eva;
472
473 va = MIPS_PHYS_TO_KSEG0(0);
474 eva = va + pdcache_size;
475
476 /*
477 * Since we're hitting the whole thing, we don't have to
478 * worry about the N different "ways".
479 */
480
481 while (va < eva) {
482 cache_r4k_op_32lines_32(va,
483 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
484 va += (32 * 32);
485 }
486
487 SYNC;
488 }
489
490 void
mipsNN_pdcache_wbinv_all_64(void)491 mipsNN_pdcache_wbinv_all_64(void)
492 {
493 vm_offset_t va, eva;
494
495 va = MIPS_PHYS_TO_KSEG0(0);
496 eva = va + pdcache_size;
497
498 /*
499 * Since we're hitting the whole thing, we don't have to
500 * worry about the N different "ways".
501 */
502
503 while (va < eva) {
504 cache_r4k_op_32lines_64(va,
505 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
506 va += (32 * 64);
507 }
508
509 SYNC;
510 }
511
512 void
mipsNN_pdcache_wbinv_range_16(vm_offset_t va,vm_size_t size)513 mipsNN_pdcache_wbinv_range_16(vm_offset_t va, vm_size_t size)
514 {
515 vm_offset_t eva;
516
517 eva = round_line16(va + size);
518 va = trunc_line16(va);
519
520 while ((eva - va) >= (32 * 16)) {
521 cache_r4k_op_32lines_16(va,
522 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
523 va += (32 * 16);
524 }
525
526 while (va < eva) {
527 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
528 va += 16;
529 }
530
531 SYNC;
532 }
533
534 void
mipsNN_pdcache_wbinv_range_32(vm_offset_t va,vm_size_t size)535 mipsNN_pdcache_wbinv_range_32(vm_offset_t va, vm_size_t size)
536 {
537 vm_offset_t eva;
538
539 eva = round_line32(va + size);
540 va = trunc_line32(va);
541
542 while ((eva - va) >= (32 * 32)) {
543 cache_r4k_op_32lines_32(va,
544 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
545 va += (32 * 32);
546 }
547
548 while (va < eva) {
549 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
550 va += 32;
551 }
552
553 SYNC;
554 }
555
556 void
mipsNN_pdcache_wbinv_range_64(vm_offset_t va,vm_size_t size)557 mipsNN_pdcache_wbinv_range_64(vm_offset_t va, vm_size_t size)
558 {
559 vm_offset_t eva;
560
561 eva = round_line64(va + size);
562 va = trunc_line64(va);
563
564 while ((eva - va) >= (32 * 64)) {
565 cache_r4k_op_32lines_64(va,
566 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
567 va += (32 * 64);
568 }
569
570 while (va < eva) {
571 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
572 va += 64;
573 }
574
575 SYNC;
576 }
577
578 void
mipsNN_pdcache_wbinv_range_index_16(vm_offset_t va,vm_size_t size)579 mipsNN_pdcache_wbinv_range_index_16(vm_offset_t va, vm_size_t size)
580 {
581 vm_offset_t eva, tmpva;
582 int i, stride, loopcount;
583
584 /*
585 * Since we're doing Index ops, we expect to not be able
586 * to access the address we've been given. So, get the
587 * bits that determine the cache index, and make a KSEG0
588 * address out of them.
589 */
590 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
591
592 eva = round_line16(va + size);
593 va = trunc_line16(va);
594
595 /*
596 * GCC generates better code in the loops if we reference local
597 * copies of these global variables.
598 */
599 stride = pdcache_stride;
600 loopcount = pdcache_loopcount;
601
602 while ((eva - va) >= (8 * 16)) {
603 tmpva = va;
604 for (i = 0; i < loopcount; i++, tmpva += stride)
605 cache_r4k_op_8lines_16(tmpva,
606 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
607 va += 8 * 16;
608 }
609
610 while (va < eva) {
611 tmpva = va;
612 for (i = 0; i < loopcount; i++, tmpva += stride)
613 cache_op_r4k_line(tmpva,
614 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
615 va += 16;
616 }
617 }
618
619 void
mipsNN_pdcache_wbinv_range_index_32(vm_offset_t va,vm_size_t size)620 mipsNN_pdcache_wbinv_range_index_32(vm_offset_t va, vm_size_t size)
621 {
622 vm_offset_t eva, tmpva;
623 int i, stride, loopcount;
624
625 /*
626 * Since we're doing Index ops, we expect to not be able
627 * to access the address we've been given. So, get the
628 * bits that determine the cache index, and make a KSEG0
629 * address out of them.
630 */
631 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
632
633 eva = round_line32(va + size);
634 va = trunc_line32(va);
635
636 /*
637 * GCC generates better code in the loops if we reference local
638 * copies of these global variables.
639 */
640 stride = pdcache_stride;
641 loopcount = pdcache_loopcount;
642
643 while ((eva - va) >= (8 * 32)) {
644 tmpva = va;
645 for (i = 0; i < loopcount; i++, tmpva += stride)
646 cache_r4k_op_8lines_32(tmpva,
647 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
648 va += 8 * 32;
649 }
650
651 while (va < eva) {
652 tmpva = va;
653 for (i = 0; i < loopcount; i++, tmpva += stride)
654 cache_op_r4k_line(tmpva,
655 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
656 va += 32;
657 }
658 }
659
660 void
mipsNN_pdcache_wbinv_range_index_64(vm_offset_t va,vm_size_t size)661 mipsNN_pdcache_wbinv_range_index_64(vm_offset_t va, vm_size_t size)
662 {
663 vm_offset_t eva, tmpva;
664 int i, stride, loopcount;
665
666 /*
667 * Since we're doing Index ops, we expect to not be able
668 * to access the address we've been given. So, get the
669 * bits that determine the cache index, and make a KSEG0
670 * address out of them.
671 */
672 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
673
674 eva = round_line64(va + size);
675 va = trunc_line64(va);
676
677 /*
678 * GCC generates better code in the loops if we reference local
679 * copies of these global variables.
680 */
681 stride = pdcache_stride;
682 loopcount = pdcache_loopcount;
683
684 while ((eva - va) >= (8 * 64)) {
685 tmpva = va;
686 for (i = 0; i < loopcount; i++, tmpva += stride)
687 cache_r4k_op_8lines_64(tmpva,
688 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
689 va += 8 * 64;
690 }
691
692 while (va < eva) {
693 tmpva = va;
694 for (i = 0; i < loopcount; i++, tmpva += stride)
695 cache_op_r4k_line(tmpva,
696 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
697 va += 64;
698 }
699 }
700
701 void
mipsNN_pdcache_inv_range_16(vm_offset_t va,vm_size_t size)702 mipsNN_pdcache_inv_range_16(vm_offset_t va, vm_size_t size)
703 {
704 vm_offset_t eva;
705
706 eva = round_line16(va + size);
707 va = trunc_line16(va);
708
709 while ((eva - va) >= (32 * 16)) {
710 cache_r4k_op_32lines_16(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
711 va += (32 * 16);
712 }
713
714 while (va < eva) {
715 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
716 va += 16;
717 }
718
719 SYNC;
720 }
721
722 void
mipsNN_pdcache_inv_range_32(vm_offset_t va,vm_size_t size)723 mipsNN_pdcache_inv_range_32(vm_offset_t va, vm_size_t size)
724 {
725 vm_offset_t eva;
726
727 eva = round_line32(va + size);
728 va = trunc_line32(va);
729
730 while ((eva - va) >= (32 * 32)) {
731 cache_r4k_op_32lines_32(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
732 va += (32 * 32);
733 }
734
735 while (va < eva) {
736 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
737 va += 32;
738 }
739
740 SYNC;
741 }
742
743 void
mipsNN_pdcache_inv_range_64(vm_offset_t va,vm_size_t size)744 mipsNN_pdcache_inv_range_64(vm_offset_t va, vm_size_t size)
745 {
746 vm_offset_t eva;
747
748 eva = round_line64(va + size);
749 va = trunc_line64(va);
750
751 while ((eva - va) >= (32 * 64)) {
752 cache_r4k_op_32lines_64(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
753 va += (32 * 64);
754 }
755
756 while (va < eva) {
757 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
758 va += 64;
759 }
760
761 SYNC;
762 }
763
764 void
mipsNN_pdcache_wb_range_16(vm_offset_t va,vm_size_t size)765 mipsNN_pdcache_wb_range_16(vm_offset_t va, vm_size_t size)
766 {
767 vm_offset_t eva;
768
769 eva = round_line16(va + size);
770 va = trunc_line16(va);
771
772 while ((eva - va) >= (32 * 16)) {
773 cache_r4k_op_32lines_16(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
774 va += (32 * 16);
775 }
776
777 while (va < eva) {
778 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
779 va += 16;
780 }
781
782 SYNC;
783 }
784
785 void
mipsNN_pdcache_wb_range_32(vm_offset_t va,vm_size_t size)786 mipsNN_pdcache_wb_range_32(vm_offset_t va, vm_size_t size)
787 {
788 vm_offset_t eva;
789
790 eva = round_line32(va + size);
791 va = trunc_line32(va);
792
793 while ((eva - va) >= (32 * 32)) {
794 cache_r4k_op_32lines_32(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
795 va += (32 * 32);
796 }
797
798 while (va < eva) {
799 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
800 va += 32;
801 }
802
803 SYNC;
804 }
805
806 void
mipsNN_pdcache_wb_range_64(vm_offset_t va,vm_size_t size)807 mipsNN_pdcache_wb_range_64(vm_offset_t va, vm_size_t size)
808 {
809 vm_offset_t eva;
810
811 eva = round_line64(va + size);
812 va = trunc_line64(va);
813
814 while ((eva - va) >= (32 * 64)) {
815 cache_r4k_op_32lines_64(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
816 va += (32 * 64);
817 }
818
819 while (va < eva) {
820 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
821 va += 64;
822 }
823
824 SYNC;
825 }
826
827 #ifdef CPU_CNMIPS
828
829 void
mipsNN_icache_sync_all_128(void)830 mipsNN_icache_sync_all_128(void)
831 {
832 SYNCI
833 }
834
835 void
mipsNN_icache_sync_range_128(vm_offset_t va,vm_size_t size)836 mipsNN_icache_sync_range_128(vm_offset_t va, vm_size_t size)
837 {
838 SYNC;
839 }
840
841 void
mipsNN_icache_sync_range_index_128(vm_offset_t va,vm_size_t size)842 mipsNN_icache_sync_range_index_128(vm_offset_t va, vm_size_t size)
843 {
844 }
845
846 void
mipsNN_pdcache_wbinv_all_128(void)847 mipsNN_pdcache_wbinv_all_128(void)
848 {
849 }
850
851 void
mipsNN_pdcache_wbinv_range_128(vm_offset_t va,vm_size_t size)852 mipsNN_pdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
853 {
854 SYNC;
855 }
856
857 void
mipsNN_pdcache_wbinv_range_index_128(vm_offset_t va,vm_size_t size)858 mipsNN_pdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
859 {
860 }
861
862 void
mipsNN_pdcache_inv_range_128(vm_offset_t va,vm_size_t size)863 mipsNN_pdcache_inv_range_128(vm_offset_t va, vm_size_t size)
864 {
865 }
866
867 void
mipsNN_pdcache_wb_range_128(vm_offset_t va,vm_size_t size)868 mipsNN_pdcache_wb_range_128(vm_offset_t va, vm_size_t size)
869 {
870 SYNC;
871 }
872
873 #else
874
875 void
mipsNN_icache_sync_all_128(void)876 mipsNN_icache_sync_all_128(void)
877 {
878 vm_offset_t va, eva;
879
880 va = MIPS_PHYS_TO_KSEG0(0);
881 eva = va + picache_size;
882
883 /*
884 * Since we're hitting the whole thing, we don't have to
885 * worry about the N different "ways".
886 */
887
888 mips_intern_dcache_wbinv_all();
889
890 while (va < eva) {
891 cache_r4k_op_32lines_128(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
892 va += (32 * 128);
893 }
894
895 SYNC;
896 }
897
898 void
mipsNN_icache_sync_range_128(vm_offset_t va,vm_size_t size)899 mipsNN_icache_sync_range_128(vm_offset_t va, vm_size_t size)
900 {
901 vm_offset_t eva;
902
903 eva = round_line128(va + size);
904 va = trunc_line128(va);
905
906 mips_intern_dcache_wb_range(va, (eva - va));
907
908 while ((eva - va) >= (32 * 128)) {
909 cache_r4k_op_32lines_128(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
910 va += (32 * 128);
911 }
912
913 while (va < eva) {
914 cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
915 va += 128;
916 }
917
918 SYNC;
919 }
920
921 void
mipsNN_icache_sync_range_index_128(vm_offset_t va,vm_size_t size)922 mipsNN_icache_sync_range_index_128(vm_offset_t va, vm_size_t size)
923 {
924 vm_offset_t eva, tmpva;
925 int i, stride, loopcount;
926
927 /*
928 * Since we're doing Index ops, we expect to not be able
929 * to access the address we've been given. So, get the
930 * bits that determine the cache index, and make a KSEG0
931 * address out of them.
932 */
933 va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
934
935 eva = round_line128(va + size);
936 va = trunc_line128(va);
937
938 /*
939 * GCC generates better code in the loops if we reference local
940 * copies of these global variables.
941 */
942 stride = picache_stride;
943 loopcount = picache_loopcount;
944
945 mips_intern_dcache_wbinv_range_index(va, (eva - va));
946
947 while ((eva - va) >= (32 * 128)) {
948 tmpva = va;
949 for (i = 0; i < loopcount; i++, tmpva += stride)
950 cache_r4k_op_32lines_128(tmpva,
951 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
952 va += 32 * 128;
953 }
954
955 while (va < eva) {
956 tmpva = va;
957 for (i = 0; i < loopcount; i++, tmpva += stride)
958 cache_op_r4k_line(tmpva,
959 CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
960 va += 128;
961 }
962 }
963
964 void
mipsNN_pdcache_wbinv_all_128(void)965 mipsNN_pdcache_wbinv_all_128(void)
966 {
967 vm_offset_t va, eva;
968
969 va = MIPS_PHYS_TO_KSEG0(0);
970 eva = va + pdcache_size;
971
972 /*
973 * Since we're hitting the whole thing, we don't have to
974 * worry about the N different "ways".
975 */
976
977 while (va < eva) {
978 cache_r4k_op_32lines_128(va,
979 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
980 va += (32 * 128);
981 }
982
983 SYNC;
984 }
985
986 void
mipsNN_pdcache_wbinv_range_128(vm_offset_t va,vm_size_t size)987 mipsNN_pdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
988 {
989 vm_offset_t eva;
990
991 eva = round_line128(va + size);
992 va = trunc_line128(va);
993
994 while ((eva - va) >= (32 * 128)) {
995 cache_r4k_op_32lines_128(va,
996 CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
997 va += (32 * 128);
998 }
999
1000 while (va < eva) {
1001 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
1002 va += 128;
1003 }
1004
1005 SYNC;
1006 }
1007
1008 void
mipsNN_pdcache_wbinv_range_index_128(vm_offset_t va,vm_size_t size)1009 mipsNN_pdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
1010 {
1011 vm_offset_t eva, tmpva;
1012 int i, stride, loopcount;
1013
1014 /*
1015 * Since we're doing Index ops, we expect to not be able
1016 * to access the address we've been given. So, get the
1017 * bits that determine the cache index, and make a KSEG0
1018 * address out of them.
1019 */
1020 va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
1021
1022 eva = round_line128(va + size);
1023 va = trunc_line128(va);
1024
1025 /*
1026 * GCC generates better code in the loops if we reference local
1027 * copies of these global variables.
1028 */
1029 stride = pdcache_stride;
1030 loopcount = pdcache_loopcount;
1031
1032 while ((eva - va) >= (32 * 128)) {
1033 tmpva = va;
1034 for (i = 0; i < loopcount; i++, tmpva += stride)
1035 cache_r4k_op_32lines_128(tmpva,
1036 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
1037 va += 32 * 128;
1038 }
1039
1040 while (va < eva) {
1041 tmpva = va;
1042 for (i = 0; i < loopcount; i++, tmpva += stride)
1043 cache_op_r4k_line(tmpva,
1044 CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
1045 va += 128;
1046 }
1047 }
1048
1049 void
mipsNN_pdcache_inv_range_128(vm_offset_t va,vm_size_t size)1050 mipsNN_pdcache_inv_range_128(vm_offset_t va, vm_size_t size)
1051 {
1052 vm_offset_t eva;
1053
1054 eva = round_line128(va + size);
1055 va = trunc_line128(va);
1056
1057 while ((eva - va) >= (32 * 128)) {
1058 cache_r4k_op_32lines_128(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
1059 va += (32 * 128);
1060 }
1061
1062 while (va < eva) {
1063 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
1064 va += 128;
1065 }
1066
1067 SYNC;
1068 }
1069
1070 void
mipsNN_pdcache_wb_range_128(vm_offset_t va,vm_size_t size)1071 mipsNN_pdcache_wb_range_128(vm_offset_t va, vm_size_t size)
1072 {
1073 vm_offset_t eva;
1074
1075 eva = round_line128(va + size);
1076 va = trunc_line128(va);
1077
1078 while ((eva - va) >= (32 * 128)) {
1079 cache_r4k_op_32lines_128(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
1080 va += (32 * 128);
1081 }
1082
1083 while (va < eva) {
1084 cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
1085 va += 128;
1086 }
1087
1088 SYNC;
1089 }
1090
1091 #endif
1092
1093 void
mipsNN_sdcache_wbinv_all_32(void)1094 mipsNN_sdcache_wbinv_all_32(void)
1095 {
1096 vm_offset_t va = MIPS_PHYS_TO_KSEG0(0);
1097 vm_offset_t eva = va + sdcache_size;
1098
1099 while (va < eva) {
1100 cache_r4k_op_32lines_32(va,
1101 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1102 va += (32 * 32);
1103 }
1104 }
1105
1106 void
mipsNN_sdcache_wbinv_all_64(void)1107 mipsNN_sdcache_wbinv_all_64(void)
1108 {
1109 vm_offset_t va = MIPS_PHYS_TO_KSEG0(0);
1110 vm_offset_t eva = va + sdcache_size;
1111
1112 while (va < eva) {
1113 cache_r4k_op_32lines_64(va,
1114 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1115 va += (32 * 64);
1116 }
1117 }
1118
1119 void
mipsNN_sdcache_wbinv_range_32(vm_offset_t va,vm_size_t size)1120 mipsNN_sdcache_wbinv_range_32(vm_offset_t va, vm_size_t size)
1121 {
1122 vm_offset_t eva = round_line32(va + size);
1123
1124 va = trunc_line32(va);
1125
1126 while ((eva - va) >= (32 * 32)) {
1127 cache_r4k_op_32lines_32(va,
1128 CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1129 va += (32 * 32);
1130 }
1131
1132 while (va < eva) {
1133 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1134 va += 32;
1135 }
1136 }
1137
1138 void
mipsNN_sdcache_wbinv_range_64(vm_offset_t va,vm_size_t size)1139 mipsNN_sdcache_wbinv_range_64(vm_offset_t va, vm_size_t size)
1140 {
1141 vm_offset_t eva = round_line64(va + size);
1142
1143 va = trunc_line64(va);
1144
1145 while ((eva - va) >= (32 * 64)) {
1146 cache_r4k_op_32lines_64(va,
1147 CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1148 va += (32 * 64);
1149 }
1150
1151 while (va < eva) {
1152 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1153 va += 64;
1154 }
1155 }
1156
1157 void
mipsNN_sdcache_wbinv_range_index_32(vm_offset_t va,vm_size_t size)1158 mipsNN_sdcache_wbinv_range_index_32(vm_offset_t va, vm_size_t size)
1159 {
1160 vm_offset_t eva;
1161
1162 /*
1163 * Since we're doing Index ops, we expect to not be able
1164 * to access the address we've been given. So, get the
1165 * bits that determine the cache index, and make a KSEG0
1166 * address out of them.
1167 */
1168 va = MIPS_PHYS_TO_KSEG0(va & (sdcache_size - 1));
1169
1170 eva = round_line32(va + size);
1171 va = trunc_line32(va);
1172
1173 while ((eva - va) >= (32 * 32)) {
1174 cache_r4k_op_32lines_32(va,
1175 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1176 va += (32 * 32);
1177 }
1178
1179 while (va < eva) {
1180 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1181 va += 32;
1182 }
1183 }
1184
1185 void
mipsNN_sdcache_wbinv_range_index_64(vm_offset_t va,vm_size_t size)1186 mipsNN_sdcache_wbinv_range_index_64(vm_offset_t va, vm_size_t size)
1187 {
1188 vm_offset_t eva;
1189
1190 /*
1191 * Since we're doing Index ops, we expect to not be able
1192 * to access the address we've been given. So, get the
1193 * bits that determine the cache index, and make a KSEG0
1194 * address out of them.
1195 */
1196 va = MIPS_PHYS_TO_KSEG0(va & (sdcache_size - 1));
1197
1198 eva = round_line64(va + size);
1199 va = trunc_line64(va);
1200
1201 while ((eva - va) >= (32 * 64)) {
1202 cache_r4k_op_32lines_64(va,
1203 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1204 va += (32 * 64);
1205 }
1206
1207 while (va < eva) {
1208 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1209 va += 64;
1210 }
1211 }
1212
1213 void
mipsNN_sdcache_inv_range_32(vm_offset_t va,vm_size_t size)1214 mipsNN_sdcache_inv_range_32(vm_offset_t va, vm_size_t size)
1215 {
1216 vm_offset_t eva = round_line32(va + size);
1217
1218 va = trunc_line32(va);
1219
1220 while ((eva - va) >= (32 * 32)) {
1221 cache_r4k_op_32lines_32(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1222 va += (32 * 32);
1223 }
1224
1225 while (va < eva) {
1226 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1227 va += 32;
1228 }
1229 }
1230
1231 void
mipsNN_sdcache_inv_range_64(vm_offset_t va,vm_size_t size)1232 mipsNN_sdcache_inv_range_64(vm_offset_t va, vm_size_t size)
1233 {
1234 vm_offset_t eva = round_line64(va + size);
1235
1236 va = trunc_line64(va);
1237
1238 while ((eva - va) >= (32 * 64)) {
1239 cache_r4k_op_32lines_64(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1240 va += (32 * 64);
1241 }
1242
1243 while (va < eva) {
1244 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1245 va += 64;
1246 }
1247 }
1248
1249 void
mipsNN_sdcache_wb_range_32(vm_offset_t va,vm_size_t size)1250 mipsNN_sdcache_wb_range_32(vm_offset_t va, vm_size_t size)
1251 {
1252 vm_offset_t eva = round_line32(va + size);
1253
1254 va = trunc_line32(va);
1255
1256 while ((eva - va) >= (32 * 32)) {
1257 cache_r4k_op_32lines_32(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1258 va += (32 * 32);
1259 }
1260
1261 while (va < eva) {
1262 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1263 va += 32;
1264 }
1265 }
1266
1267 void
mipsNN_sdcache_wb_range_64(vm_offset_t va,vm_size_t size)1268 mipsNN_sdcache_wb_range_64(vm_offset_t va, vm_size_t size)
1269 {
1270 vm_offset_t eva = round_line64(va + size);
1271
1272 va = trunc_line64(va);
1273
1274 while ((eva - va) >= (32 * 64)) {
1275 cache_r4k_op_32lines_64(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1276 va += (32 * 64);
1277 }
1278
1279 while (va < eva) {
1280 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1281 va += 64;
1282 }
1283 }
1284
1285 void
mipsNN_sdcache_wbinv_all_128(void)1286 mipsNN_sdcache_wbinv_all_128(void)
1287 {
1288 vm_offset_t va = MIPS_PHYS_TO_KSEG0(0);
1289 vm_offset_t eva = va + sdcache_size;
1290
1291 while (va < eva) {
1292 cache_r4k_op_32lines_128(va,
1293 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1294 va += (32 * 128);
1295 }
1296 }
1297
1298 void
mipsNN_sdcache_wbinv_range_128(vm_offset_t va,vm_size_t size)1299 mipsNN_sdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
1300 {
1301 vm_offset_t eva = round_line128(va + size);
1302
1303 va = trunc_line128(va);
1304
1305 while ((eva - va) >= (32 * 128)) {
1306 cache_r4k_op_32lines_128(va,
1307 CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1308 va += (32 * 128);
1309 }
1310
1311 while (va < eva) {
1312 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB_INV);
1313 va += 128;
1314 }
1315 }
1316
1317 void
mipsNN_sdcache_wbinv_range_index_128(vm_offset_t va,vm_size_t size)1318 mipsNN_sdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
1319 {
1320 vm_offset_t eva;
1321
1322 /*
1323 * Since we're doing Index ops, we expect to not be able
1324 * to access the address we've been given. So, get the
1325 * bits that determine the cache index, and make a KSEG0
1326 * address out of them.
1327 */
1328 va = MIPS_PHYS_TO_KSEG0(va & (sdcache_size - 1));
1329
1330 eva = round_line128(va + size);
1331 va = trunc_line128(va);
1332
1333 while ((eva - va) >= (32 * 128)) {
1334 cache_r4k_op_32lines_128(va,
1335 CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1336 va += (32 * 128);
1337 }
1338
1339 while (va < eva) {
1340 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_INDEX_WB_INV);
1341 va += 128;
1342 }
1343 }
1344
1345 void
mipsNN_sdcache_inv_range_128(vm_offset_t va,vm_size_t size)1346 mipsNN_sdcache_inv_range_128(vm_offset_t va, vm_size_t size)
1347 {
1348 vm_offset_t eva = round_line128(va + size);
1349
1350 va = trunc_line128(va);
1351
1352 while ((eva - va) >= (32 * 128)) {
1353 cache_r4k_op_32lines_128(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1354 va += (32 * 128);
1355 }
1356
1357 while (va < eva) {
1358 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_INV);
1359 va += 128;
1360 }
1361 }
1362
1363 void
mipsNN_sdcache_wb_range_128(vm_offset_t va,vm_size_t size)1364 mipsNN_sdcache_wb_range_128(vm_offset_t va, vm_size_t size)
1365 {
1366 vm_offset_t eva = round_line128(va + size);
1367
1368 va = trunc_line128(va);
1369
1370 while ((eva - va) >= (32 * 128)) {
1371 cache_r4k_op_32lines_128(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1372 va += (32 * 128);
1373 }
1374
1375 while (va < eva) {
1376 cache_op_r4k_line(va, CACHE_R4K_SD|CACHEOP_R4K_HIT_WB);
1377 va += 128;
1378 }
1379 }
1380