1 /*
2 * Copyright (C) 2010 Andreas Tobler
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
18 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
20 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
21 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
22 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
23 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD$");
28
29 #include <sys/param.h>
30 #include <sys/kernel.h>
31 #include <sys/ktr.h>
32 #include <sys/lock.h>
33 #include <sys/msgbuf.h>
34 #include <sys/mutex.h>
35 #include <sys/proc.h>
36 #include <sys/sysctl.h>
37 #include <sys/systm.h>
38 #include <sys/vmmeter.h>
39
40 #include <dev/ofw/openfirm.h>
41 #include <machine/ofw_machdep.h>
42
43 #include <vm/vm.h>
44 #include <vm/vm_param.h>
45 #include <vm/vm_kern.h>
46 #include <vm/vm_page.h>
47 #include <vm/vm_map.h>
48 #include <vm/vm_object.h>
49 #include <vm/vm_extern.h>
50 #include <vm/vm_pageout.h>
51 #include <vm/uma.h>
52
53 #include <powerpc/aim/mmu_oea64.h>
54
55 #include "mmu_if.h"
56 #include "moea64_if.h"
57
58 #include "phyp-hvcall.h"
59
60 extern int n_slbs;
61
62 /*
63 * Kernel MMU interface
64 */
65
66 static void mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart,
67 vm_offset_t kernelend);
68 static void mphyp_cpu_bootstrap(mmu_t mmup, int ap);
69 static void mphyp_pte_synch(mmu_t, uintptr_t pt, struct lpte *pvo_pt);
70 static void mphyp_pte_clear(mmu_t, uintptr_t pt, struct lpte *pvo_pt,
71 uint64_t vpn, u_int64_t ptebit);
72 static void mphyp_pte_unset(mmu_t, uintptr_t pt, struct lpte *pvo_pt,
73 uint64_t vpn);
74 static void mphyp_pte_change(mmu_t, uintptr_t pt, struct lpte *pvo_pt,
75 uint64_t vpn);
76 static int mphyp_pte_insert(mmu_t, u_int ptegidx, struct lpte *pvo_pt);
77 static uintptr_t mphyp_pvo_to_pte(mmu_t, const struct pvo_entry *pvo);
78
79 #define VSID_HASH_MASK 0x0000007fffffffffULL
80
81
82 static mmu_method_t mphyp_methods[] = {
83 MMUMETHOD(mmu_bootstrap, mphyp_bootstrap),
84 MMUMETHOD(mmu_cpu_bootstrap, mphyp_cpu_bootstrap),
85
86 MMUMETHOD(moea64_pte_synch, mphyp_pte_synch),
87 MMUMETHOD(moea64_pte_clear, mphyp_pte_clear),
88 MMUMETHOD(moea64_pte_unset, mphyp_pte_unset),
89 MMUMETHOD(moea64_pte_change, mphyp_pte_change),
90 MMUMETHOD(moea64_pte_insert, mphyp_pte_insert),
91 MMUMETHOD(moea64_pvo_to_pte, mphyp_pvo_to_pte),
92
93 { 0, 0 }
94 };
95
96 MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, 0, oea64_mmu);
97
98 static void
mphyp_bootstrap(mmu_t mmup,vm_offset_t kernelstart,vm_offset_t kernelend)99 mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
100 {
101 uint64_t final_pteg_count = 0;
102 char buf[8];
103 uint32_t prop[2];
104 uint32_t nptlp, shift = 0, slb_encoding = 0;
105 phandle_t dev, node, root;
106 int idx, len, res;
107
108 moea64_early_bootstrap(mmup, kernelstart, kernelend);
109
110 root = OF_peer(0);
111
112 dev = OF_child(root);
113 while (dev != 0) {
114 res = OF_getprop(dev, "name", buf, sizeof(buf));
115 if (res > 0 && strcmp(buf, "cpus") == 0)
116 break;
117 dev = OF_peer(dev);
118 }
119
120 node = OF_child(dev);
121
122 while (node != 0) {
123 res = OF_getprop(node, "device_type", buf, sizeof(buf));
124 if (res > 0 && strcmp(buf, "cpu") == 0)
125 break;
126 node = OF_peer(node);
127 }
128
129 res = OF_getprop(node, "ibm,pft-size", prop, sizeof(prop));
130 if (res <= 0)
131 panic("mmu_phyp: unknown PFT size");
132 final_pteg_count = 1 << prop[1];
133 res = OF_getprop(node, "ibm,slb-size", prop, sizeof(prop[0]));
134 if (res > 0)
135 n_slbs = prop[0];
136
137 moea64_pteg_count = final_pteg_count / sizeof(struct lpteg);
138
139 /*
140 * Scan the large page size property for PAPR compatible machines.
141 * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties'
142 * for the encoding of the property.
143 */
144
145 len = OF_getproplen(node, "ibm,segment-page-sizes");
146 if (len > 0) {
147 /*
148 * We have to use a variable length array on the stack
149 * since we have very limited stack space.
150 */
151 cell_t arr[len/sizeof(cell_t)];
152 res = OF_getprop(node, "ibm,segment-page-sizes", &arr,
153 sizeof(arr));
154 len /= 4;
155 idx = 0;
156 while (len > 0) {
157 shift = arr[idx];
158 slb_encoding = arr[idx + 1];
159 nptlp = arr[idx + 2];
160 idx += 3;
161 len -= 3;
162 while (len > 0 && nptlp) {
163 idx += 2;
164 len -= 2;
165 nptlp--;
166 }
167 }
168
169 /* For now we allow shift only to be <= 0x18. */
170 if (shift >= 0x18)
171 shift = 0x18;
172
173 moea64_large_page_shift = shift;
174 moea64_large_page_size = 1 << shift;
175 }
176
177 moea64_mid_bootstrap(mmup, kernelstart, kernelend);
178 moea64_late_bootstrap(mmup, kernelstart, kernelend);
179 }
180
181 static void
mphyp_cpu_bootstrap(mmu_t mmup,int ap)182 mphyp_cpu_bootstrap(mmu_t mmup, int ap)
183 {
184 struct slb *slb = PCPU_GET(slb);
185 register_t seg0;
186 int i;
187
188 /*
189 * Install kernel SLB entries
190 */
191
192 __asm __volatile ("slbia");
193 __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
194 for (i = 0; i < 64; i++) {
195 if (!(slb[i].slbe & SLBE_VALID))
196 continue;
197
198 __asm __volatile ("slbmte %0, %1" ::
199 "r"(slb[i].slbv), "r"(slb[i].slbe));
200 }
201 }
202
203 static void
mphyp_pte_synch(mmu_t mmu,uintptr_t slot,struct lpte * pvo_pt)204 mphyp_pte_synch(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt)
205 {
206 struct lpte pte;
207 uint64_t junk;
208
209 __asm __volatile("ptesync");
210 phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pte.pte_hi, &pte.pte_lo,
211 &junk);
212
213 pvo_pt->pte_lo |= pte.pte_lo & (LPTE_CHG | LPTE_REF);
214 }
215
216 static void
mphyp_pte_clear(mmu_t mmu,uintptr_t slot,struct lpte * pvo_pt,uint64_t vpn,u_int64_t ptebit)217 mphyp_pte_clear(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn,
218 u_int64_t ptebit)
219 {
220
221 if (ptebit & LPTE_CHG)
222 phyp_hcall(H_CLEAR_MOD, 0, slot);
223 if (ptebit & LPTE_REF)
224 phyp_hcall(H_CLEAR_REF, 0, slot);
225 }
226
227 static void
mphyp_pte_unset(mmu_t mmu,uintptr_t slot,struct lpte * pvo_pt,uint64_t vpn)228 mphyp_pte_unset(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn)
229 {
230 struct lpte pte;
231 uint64_t junk;
232 int err;
233
234 err = phyp_pft_hcall(H_REMOVE, 1UL << 31, slot,
235 pvo_pt->pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo,
236 &junk);
237 KASSERT(err == H_SUCCESS, ("Error removing page: %d", err));
238
239 pvo_pt->pte_lo |= pte.pte_lo & (LPTE_CHG | LPTE_REF);
240 }
241
242 static void
mphyp_pte_change(mmu_t mmu,uintptr_t slot,struct lpte * pvo_pt,uint64_t vpn)243 mphyp_pte_change(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn)
244 {
245 struct lpte evicted;
246 uint64_t index, junk;
247 int64_t result;
248
249 /*
250 * NB: this is protected by the global table lock, so this two-step
251 * is safe, except for the scratch-page case. No CPUs on which we run
252 * this code should be using scratch pages.
253 */
254 KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED),
255 ("Locked pages not supported on PHYP"));
256
257 /* XXX: optimization using H_PROTECT for common case? */
258 mphyp_pte_unset(mmu, slot, pvo_pt, vpn);
259 result = phyp_pft_hcall(H_ENTER, H_EXACT, slot, pvo_pt->pte_hi,
260 pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk);
261 if (result != H_SUCCESS)
262 panic("mphyp_pte_change() insertion failure: %ld\n", result);
263 }
264
265 static __inline int
mphyp_pte_spillable_ident(u_int ptegidx,struct lpte * to_evict)266 mphyp_pte_spillable_ident(u_int ptegidx, struct lpte *to_evict)
267 {
268 uint64_t slot, junk, k;
269 struct lpte pt;
270 int i, j;
271
272 /* Start at a random slot */
273 i = mftb() % 8;
274 k = -1;
275 for (j = 0; j < 8; j++) {
276 slot = (ptegidx << 3) + (i + j) % 8;
277 phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, &pt.pte_lo,
278 &junk);
279
280 if (pt.pte_hi & LPTE_SWBITS)
281 continue;
282
283 /* This is a candidate, so remember it */
284 k = slot;
285
286 /* Try to get a page that has not been used lately */
287 if (!(pt.pte_lo & LPTE_REF)) {
288 memcpy(to_evict, &pt, sizeof(struct lpte));
289 return (k);
290 }
291 }
292
293 phyp_pft_hcall(H_READ, 0, slot, 0, 0, &to_evict->pte_hi,
294 &to_evict->pte_lo, &junk);
295 return (k);
296 }
297
298 static int
mphyp_pte_insert(mmu_t mmu,u_int ptegidx,struct lpte * pvo_pt)299 mphyp_pte_insert(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt)
300 {
301 int64_t result;
302 struct lpte evicted;
303 struct pvo_entry *pvo;
304 uint64_t index, junk;
305 u_int pteg_bktidx;
306
307 /* Check for locked pages, which we can't support on this system */
308 KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED),
309 ("Locked pages not supported on PHYP"));
310
311 /* Initialize PTE */
312 pvo_pt->pte_hi |= LPTE_VALID;
313 pvo_pt->pte_hi &= ~LPTE_HID;
314 evicted.pte_hi = 0;
315
316 /*
317 * First try primary hash.
318 */
319 pteg_bktidx = ptegidx;
320 result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3, pvo_pt->pte_hi,
321 pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk);
322 if (result == H_SUCCESS)
323 return (index & 0x07);
324 KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld "
325 "(ptegidx: %#x/%#x, PTE %#lx/%#lx", result, ptegidx,
326 moea64_pteg_count, pvo_pt->pte_hi, pvo_pt->pte_lo));
327
328 /*
329 * Next try secondary hash.
330 */
331 pteg_bktidx ^= moea64_pteg_mask;
332 pvo_pt->pte_hi |= LPTE_HID;
333 result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3,
334 pvo_pt->pte_hi, pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk);
335 if (result == H_SUCCESS)
336 return (index & 0x07);
337 KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld",
338 result));
339
340 /*
341 * Out of luck. Find a PTE to sacrifice.
342 */
343 pteg_bktidx = ptegidx;
344 index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted);
345 if (index == -1L) {
346 pteg_bktidx ^= moea64_pteg_mask;
347 index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted);
348 }
349
350 if (index == -1L) {
351 /* No freeable slots in either PTEG? We're hosed. */
352 panic("mphyp_pte_insert: overflow");
353 return (-1);
354 }
355
356 if (pteg_bktidx == ptegidx)
357 pvo_pt->pte_hi &= ~LPTE_HID;
358 else
359 pvo_pt->pte_hi |= LPTE_HID;
360
361 /*
362 * Synchronize the sacrifice PTE with its PVO, then mark both
363 * invalid. The PVO will be reused when/if the VM system comes
364 * here after a fault.
365 */
366
367 if (evicted.pte_hi & LPTE_HID)
368 pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */
369
370 LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) {
371 if (pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi) {
372 KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID,
373 ("Invalid PVO for valid PTE!"));
374 mphyp_pte_unset(mmu, index, &pvo->pvo_pte.lpte,
375 pvo->pvo_vpn);
376 PVO_PTEGIDX_CLR(pvo);
377 moea64_pte_overflow++;
378 break;
379 }
380 }
381
382 KASSERT(pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi,
383 ("Unable to find PVO for spilled PTE"));
384
385 /*
386 * Set the new PTE.
387 */
388 result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pvo_pt->pte_hi,
389 pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk);
390 if (result == H_SUCCESS)
391 return (index & 0x07);
392
393 panic("Page replacement error: %ld", result);
394 return (-1);
395 }
396
397 static __inline u_int
va_to_pteg(uint64_t vsid,vm_offset_t addr,int large)398 va_to_pteg(uint64_t vsid, vm_offset_t addr, int large)
399 {
400 uint64_t hash;
401 int shift;
402
403 shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT;
404 hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >>
405 shift);
406 return (hash & moea64_pteg_mask);
407 }
408
409 static uintptr_t
mphyp_pvo_to_pte(mmu_t mmu,const struct pvo_entry * pvo)410 mphyp_pvo_to_pte(mmu_t mmu, const struct pvo_entry *pvo)
411 {
412 uint64_t vsid;
413 u_int ptegidx;
414
415 /* If the PTEG index is not set, then there is no page table entry */
416 if (!PVO_PTEGIDX_ISSET(pvo))
417 return (-1);
418
419 vsid = PVO_VSID(pvo);
420 ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE);
421
422 /*
423 * We can find the actual pte entry without searching by grabbing
424 * the PTEG index from 3 unused bits in pvo_vaddr and by
425 * noticing the HID bit.
426 */
427 if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
428 ptegidx ^= moea64_pteg_mask;
429
430 return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo));
431 }
432
433