1 /*-
2 * Copyright (c) 2004-2010 Juli Mallett <jmallett@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29 #include "opt_ddb.h"
30
31 #include <sys/param.h>
32 #include <sys/kernel.h>
33 #include <sys/systm.h>
34 #include <sys/pcpu.h>
35 #include <sys/smp.h>
36
37 #include <vm/vm.h>
38 #include <vm/pmap.h>
39
40 #include <machine/pte.h>
41 #include <machine/tlb.h>
42
43 #if defined(CPU_CNMIPS)
44 #define MIPS_MAX_TLB_ENTRIES 128
45 #elif defined(CPU_NLM)
46 #define MIPS_MAX_TLB_ENTRIES (2048 + 128)
47 #else
48 #define MIPS_MAX_TLB_ENTRIES 64
49 #endif
50
51 struct tlb_state {
52 unsigned wired;
53 struct tlb_entry {
54 register_t entryhi;
55 register_t entrylo0;
56 register_t entrylo1;
57 register_t pagemask;
58 } entry[MIPS_MAX_TLB_ENTRIES];
59 };
60
61 static struct tlb_state tlb_state[MAXCPU];
62
63 #if 0
64 /*
65 * PageMask must increment in steps of 2 bits.
66 */
67 COMPILE_TIME_ASSERT(POPCNT(TLBMASK_MASK) % 2 == 0);
68 #endif
69
70 static inline void
tlb_probe(void)71 tlb_probe(void)
72 {
73 __asm __volatile ("tlbp" : : : "memory");
74 mips_cp0_sync();
75 }
76
77 static inline void
tlb_read(void)78 tlb_read(void)
79 {
80 __asm __volatile ("tlbr" : : : "memory");
81 mips_cp0_sync();
82 }
83
84 static inline void
tlb_write_indexed(void)85 tlb_write_indexed(void)
86 {
87 __asm __volatile ("tlbwi" : : : "memory");
88 mips_cp0_sync();
89 }
90
91 static inline void
tlb_write_random(void)92 tlb_write_random(void)
93 {
94 __asm __volatile ("tlbwr" : : : "memory");
95 mips_cp0_sync();
96 }
97
98 static void tlb_invalidate_one(unsigned);
99
100 void
tlb_insert_wired(unsigned i,vm_offset_t va,pt_entry_t pte0,pt_entry_t pte1)101 tlb_insert_wired(unsigned i, vm_offset_t va, pt_entry_t pte0, pt_entry_t pte1)
102 {
103 register_t asid;
104 register_t s;
105
106 va &= ~PAGE_MASK;
107
108 s = intr_disable();
109 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
110
111 mips_wr_index(i);
112 mips_wr_pagemask(0);
113 mips_wr_entryhi(TLBHI_ENTRY(va, 0));
114 mips_wr_entrylo0(pte0);
115 mips_wr_entrylo1(pte1);
116 tlb_write_indexed();
117
118 mips_wr_entryhi(asid);
119 intr_restore(s);
120 }
121
122 void
tlb_invalidate_address(struct pmap * pmap,vm_offset_t va)123 tlb_invalidate_address(struct pmap *pmap, vm_offset_t va)
124 {
125 register_t asid;
126 register_t s;
127 int i;
128
129 va &= ~PAGE_MASK;
130
131 s = intr_disable();
132 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
133
134 mips_wr_pagemask(0);
135 mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
136 tlb_probe();
137 i = mips_rd_index();
138 if (i >= 0)
139 tlb_invalidate_one(i);
140
141 mips_wr_entryhi(asid);
142 intr_restore(s);
143 }
144
145 void
tlb_invalidate_all(void)146 tlb_invalidate_all(void)
147 {
148 register_t asid;
149 register_t s;
150 unsigned i;
151
152 s = intr_disable();
153 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
154
155 for (i = mips_rd_wired(); i < num_tlbentries; i++)
156 tlb_invalidate_one(i);
157
158 mips_wr_entryhi(asid);
159 intr_restore(s);
160 }
161
162 void
tlb_invalidate_all_user(struct pmap * pmap)163 tlb_invalidate_all_user(struct pmap *pmap)
164 {
165 register_t asid;
166 register_t s;
167 unsigned i;
168
169 s = intr_disable();
170 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
171
172 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
173 register_t uasid;
174
175 mips_wr_index(i);
176 tlb_read();
177
178 uasid = mips_rd_entryhi() & TLBHI_ASID_MASK;
179 if (pmap == NULL) {
180 /*
181 * Invalidate all non-kernel entries.
182 */
183 if (uasid == 0)
184 continue;
185 } else {
186 /*
187 * Invalidate this pmap's entries.
188 */
189 if (uasid != pmap_asid(pmap))
190 continue;
191 }
192 tlb_invalidate_one(i);
193 }
194
195 mips_wr_entryhi(asid);
196 intr_restore(s);
197 }
198
199 /*
200 * Invalidates any TLB entries that map a virtual page from the specified
201 * address range. If "end" is zero, then every virtual page is considered to
202 * be within the address range's upper bound.
203 */
204 void
tlb_invalidate_range(pmap_t pmap,vm_offset_t start,vm_offset_t end)205 tlb_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
206 {
207 register_t asid, end_hi, hi, hi_pagemask, s, save_asid, start_hi;
208 int i;
209
210 KASSERT(start < end || (end == 0 && start > 0),
211 ("tlb_invalidate_range: invalid range"));
212
213 /*
214 * Truncate the virtual address "start" to an even page frame number,
215 * and round the virtual address "end" to an even page frame number.
216 */
217 start &= ~((1 << TLBMASK_SHIFT) - 1);
218 end = (end + (1 << TLBMASK_SHIFT) - 1) & ~((1 << TLBMASK_SHIFT) - 1);
219
220 s = intr_disable();
221 save_asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
222
223 asid = pmap_asid(pmap);
224 start_hi = TLBHI_ENTRY(start, asid);
225 end_hi = TLBHI_ENTRY(end, asid);
226
227 /*
228 * Select the fastest method for invalidating the TLB entries.
229 */
230 if (end - start < num_tlbentries << TLBMASK_SHIFT || (end == 0 &&
231 start >= -(num_tlbentries << TLBMASK_SHIFT))) {
232 /*
233 * The virtual address range is small compared to the size of
234 * the TLB. Probe the TLB for each even numbered page frame
235 * within the virtual address range.
236 */
237 for (hi = start_hi; hi != end_hi; hi += 1 << TLBMASK_SHIFT) {
238 mips_wr_pagemask(0);
239 mips_wr_entryhi(hi);
240 tlb_probe();
241 i = mips_rd_index();
242 if (i >= 0)
243 tlb_invalidate_one(i);
244 }
245 } else {
246 /*
247 * The virtual address range is large compared to the size of
248 * the TLB. Test every non-wired TLB entry.
249 */
250 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
251 mips_wr_index(i);
252 tlb_read();
253 hi = mips_rd_entryhi();
254 if ((hi & TLBHI_ASID_MASK) == asid && (hi < end_hi ||
255 end == 0)) {
256 /*
257 * If "hi" is a large page that spans
258 * "start_hi", then it must be invalidated.
259 */
260 hi_pagemask = mips_rd_pagemask();
261 if (hi >= (start_hi & ~(hi_pagemask <<
262 TLBMASK_SHIFT)))
263 tlb_invalidate_one(i);
264 }
265 }
266 }
267
268 mips_wr_entryhi(save_asid);
269 intr_restore(s);
270 }
271
272 /* XXX Only if DDB? */
273 void
tlb_save(void)274 tlb_save(void)
275 {
276 unsigned ntlb, i, cpu;
277
278 cpu = PCPU_GET(cpuid);
279 if (num_tlbentries > MIPS_MAX_TLB_ENTRIES)
280 ntlb = MIPS_MAX_TLB_ENTRIES;
281 else
282 ntlb = num_tlbentries;
283 tlb_state[cpu].wired = mips_rd_wired();
284 for (i = 0; i < ntlb; i++) {
285 mips_wr_index(i);
286 tlb_read();
287
288 tlb_state[cpu].entry[i].entryhi = mips_rd_entryhi();
289 tlb_state[cpu].entry[i].pagemask = mips_rd_pagemask();
290 tlb_state[cpu].entry[i].entrylo0 = mips_rd_entrylo0();
291 tlb_state[cpu].entry[i].entrylo1 = mips_rd_entrylo1();
292 }
293 }
294
295 void
tlb_update(struct pmap * pmap,vm_offset_t va,pt_entry_t pte)296 tlb_update(struct pmap *pmap, vm_offset_t va, pt_entry_t pte)
297 {
298 register_t asid;
299 register_t s;
300 int i;
301
302 va &= ~PAGE_MASK;
303 pte &= ~TLBLO_SWBITS_MASK;
304
305 s = intr_disable();
306 asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
307
308 mips_wr_pagemask(0);
309 mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
310 tlb_probe();
311 i = mips_rd_index();
312 if (i >= 0) {
313 tlb_read();
314
315 if ((va & PAGE_SIZE) == 0) {
316 mips_wr_entrylo0(pte);
317 } else {
318 mips_wr_entrylo1(pte);
319 }
320 tlb_write_indexed();
321 }
322
323 mips_wr_entryhi(asid);
324 intr_restore(s);
325 }
326
327 static void
tlb_invalidate_one(unsigned i)328 tlb_invalidate_one(unsigned i)
329 {
330 /* XXX an invalid ASID? */
331 mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0));
332 mips_wr_entrylo0(0);
333 mips_wr_entrylo1(0);
334 mips_wr_pagemask(0);
335 mips_wr_index(i);
336 tlb_write_indexed();
337 }
338
339 #ifdef DDB
340 #include <ddb/ddb.h>
341
DB_SHOW_COMMAND(tlb,ddb_dump_tlb)342 DB_SHOW_COMMAND(tlb, ddb_dump_tlb)
343 {
344 register_t ehi, elo0, elo1, epagemask;
345 unsigned i, cpu, ntlb;
346
347 /*
348 * XXX
349 * The worst conversion from hex to decimal ever.
350 */
351 if (have_addr)
352 cpu = ((addr >> 4) % 16) * 10 + (addr % 16);
353 else
354 cpu = PCPU_GET(cpuid);
355
356 if (cpu < 0 || cpu >= mp_ncpus) {
357 db_printf("Invalid CPU %u\n", cpu);
358 return;
359 }
360 if (num_tlbentries > MIPS_MAX_TLB_ENTRIES) {
361 ntlb = MIPS_MAX_TLB_ENTRIES;
362 db_printf("Warning: Only %d of %d TLB entries saved!\n",
363 ntlb, num_tlbentries);
364 } else
365 ntlb = num_tlbentries;
366
367 if (cpu == PCPU_GET(cpuid))
368 tlb_save();
369
370 db_printf("Beginning TLB dump for CPU %u...\n", cpu);
371 for (i = 0; i < ntlb; i++) {
372 if (i == tlb_state[cpu].wired) {
373 if (i != 0)
374 db_printf("^^^ WIRED ENTRIES ^^^\n");
375 else
376 db_printf("(No wired entries.)\n");
377 }
378
379 /* XXX PageMask. */
380 ehi = tlb_state[cpu].entry[i].entryhi;
381 elo0 = tlb_state[cpu].entry[i].entrylo0;
382 elo1 = tlb_state[cpu].entry[i].entrylo1;
383 epagemask = tlb_state[cpu].entry[i].pagemask;
384
385 if (elo0 == 0 && elo1 == 0)
386 continue;
387
388 db_printf("#%u\t=> %jx (pagemask %jx)\n", i, (intmax_t)ehi, (intmax_t) epagemask);
389 db_printf(" Lo0\t%jx\t(%#jx)\n", (intmax_t)elo0, (intmax_t)TLBLO_PTE_TO_PA(elo0));
390 db_printf(" Lo1\t%jx\t(%#jx)\n", (intmax_t)elo1, (intmax_t)TLBLO_PTE_TO_PA(elo1));
391 }
392 db_printf("Finished.\n");
393 }
394 #endif
395