1 /*-
2 * Copyright (c) 2013 Ian Lepore <ian@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 /* Routines for mapping device memory. */
29
30 #include "opt_ddb.h"
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/devmap.h>
35 #include <vm/vm.h>
36 #include <vm/vm_extern.h>
37 #include <vm/pmap.h>
38 #include <machine/vmparam.h>
39
40 static const struct devmap_entry *devmap_table;
41 static boolean_t devmap_bootstrap_done = false;
42
43 /*
44 * The allocated-kva (akva) devmap table and metadata. Platforms can call
45 * devmap_add_entry() to add static device mappings to this table using
46 * automatically allocated virtual addresses carved out of the top of kva space.
47 * Allocation begins immediately below the max kernel virtual address.
48 */
49 #define AKVA_DEVMAP_MAX_ENTRIES 32
50 static struct devmap_entry akva_devmap_entries[AKVA_DEVMAP_MAX_ENTRIES];
51 static u_int akva_devmap_idx;
52 static vm_offset_t akva_devmap_vaddr = DEVMAP_MAX_VADDR;
53
54 #if defined(__aarch64__) || defined(__riscv)
55 extern int early_boot;
56 #endif
57
58 /*
59 * Print the contents of the static mapping table using the provided printf-like
60 * output function (which will be either printf or db_printf).
61 */
62 static void
devmap_dump_table(int (* prfunc)(const char *,...))63 devmap_dump_table(int (*prfunc)(const char *, ...))
64 {
65 const struct devmap_entry *pd;
66
67 if (devmap_table == NULL || devmap_table[0].pd_size == 0) {
68 prfunc("No static device mappings.\n");
69 return;
70 }
71
72 prfunc("Static device mappings:\n");
73 for (pd = devmap_table; pd->pd_size != 0; ++pd) {
74 prfunc(" 0x%08jx - 0x%08jx mapped at VA 0x%08jx\n",
75 (uintmax_t)pd->pd_pa,
76 (uintmax_t)(pd->pd_pa + pd->pd_size - 1),
77 (uintmax_t)pd->pd_va);
78 }
79 }
80
81 /*
82 * Print the contents of the static mapping table. Used for bootverbose.
83 */
84 void
devmap_print_table(void)85 devmap_print_table(void)
86 {
87 devmap_dump_table(printf);
88 }
89
90 /*
91 * Return the "last" kva address used by the registered devmap table. It's
92 * actually the lowest address used by the static mappings, i.e., the address of
93 * the first unusable byte of KVA.
94 */
95 vm_offset_t
devmap_lastaddr(void)96 devmap_lastaddr(void)
97 {
98 const struct devmap_entry *pd;
99 vm_offset_t lowaddr;
100
101 if (akva_devmap_idx > 0)
102 return (akva_devmap_vaddr);
103
104 lowaddr = DEVMAP_MAX_VADDR;
105 for (pd = devmap_table; pd != NULL && pd->pd_size != 0; ++pd) {
106 if (lowaddr > pd->pd_va)
107 lowaddr = pd->pd_va;
108 }
109
110 return (lowaddr);
111 }
112
113 /*
114 * Add an entry to the internal "akva" static devmap table using the given
115 * physical address and size and a virtual address allocated from the top of
116 * kva. This automatically registers the akva table on the first call, so all a
117 * platform has to do is call this routine to install as many mappings as it
118 * needs and when the platform-specific init function calls devmap_bootstrap()
119 * it will pick up all the entries in the akva table automatically.
120 */
121 void
devmap_add_entry(vm_paddr_t pa,vm_size_t sz)122 devmap_add_entry(vm_paddr_t pa, vm_size_t sz)
123 {
124 struct devmap_entry *m;
125
126 if (devmap_bootstrap_done)
127 panic("devmap_add_entry() after devmap_bootstrap()");
128
129 if (akva_devmap_idx == (AKVA_DEVMAP_MAX_ENTRIES - 1))
130 panic("AKVA_DEVMAP_MAX_ENTRIES is too small");
131
132 if (akva_devmap_idx == 0)
133 devmap_register_table(akva_devmap_entries);
134
135 /* Allocate virtual address space from the top of kva downwards. */
136 #ifdef __arm__
137 /*
138 * If the range being mapped is aligned and sized to 1MB boundaries then
139 * also align the virtual address to the next-lower 1MB boundary so that
140 * we end with a nice efficient section mapping.
141 */
142 if ((pa & 0x000fffff) == 0 && (sz & 0x000fffff) == 0) {
143 akva_devmap_vaddr = trunc_1mpage(akva_devmap_vaddr - sz);
144 } else
145 #endif
146 {
147 akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - sz);
148 }
149 m = &akva_devmap_entries[akva_devmap_idx++];
150 m->pd_va = akva_devmap_vaddr;
151 m->pd_pa = pa;
152 m->pd_size = sz;
153 }
154
155 /*
156 * Register the given table as the one to use in devmap_bootstrap().
157 */
158 void
devmap_register_table(const struct devmap_entry * table)159 devmap_register_table(const struct devmap_entry *table)
160 {
161
162 devmap_table = table;
163 }
164
165 /*
166 * Map all of the static regions in the devmap table, and remember the devmap
167 * table so the mapdev, ptov, and vtop functions can do lookups later.
168 *
169 * If a non-NULL table pointer is given it is used unconditionally, otherwise
170 * the previously-registered table is used. This smooths transition from legacy
171 * code that fills in a local table then calls this function passing that table,
172 * and newer code that uses devmap_register_table() in platform-specific
173 * code, then lets the common platform-specific init function call this function
174 * with a NULL pointer.
175 */
176 void
devmap_bootstrap(vm_offset_t l1pt,const struct devmap_entry * table)177 devmap_bootstrap(vm_offset_t l1pt, const struct devmap_entry *table)
178 {
179 const struct devmap_entry *pd;
180
181 devmap_bootstrap_done = true;
182
183 /*
184 * If given a table pointer, use it. Otherwise, if a table was
185 * previously registered, use it. Otherwise, no work to do.
186 */
187 if (table != NULL)
188 devmap_table = table;
189 else if (devmap_table == NULL)
190 return;
191
192 for (pd = devmap_table; pd->pd_size != 0; ++pd) {
193 #if defined(__arm__)
194 #if __ARM_ARCH >= 6
195 pmap_preboot_map_attr(pd->pd_pa, pd->pd_va, pd->pd_size,
196 VM_PROT_READ | VM_PROT_WRITE, VM_MEMATTR_DEVICE);
197 #else
198 pmap_map_chunk(l1pt, pd->pd_va, pd->pd_pa, pd->pd_size,
199 VM_PROT_READ | VM_PROT_WRITE, PTE_DEVICE);
200 #endif
201 #elif defined(__aarch64__) || defined(__riscv)
202 pmap_kenter_device(pd->pd_va, pd->pd_size, pd->pd_pa);
203 #endif
204 }
205 }
206
207 /*
208 * Look up the given physical address in the static mapping data and return the
209 * corresponding virtual address, or NULL if not found.
210 */
211 void *
devmap_ptov(vm_paddr_t pa,vm_size_t size)212 devmap_ptov(vm_paddr_t pa, vm_size_t size)
213 {
214 const struct devmap_entry *pd;
215
216 if (devmap_table == NULL)
217 return (NULL);
218
219 for (pd = devmap_table; pd->pd_size != 0; ++pd) {
220 if (pa >= pd->pd_pa && pa + size <= pd->pd_pa + pd->pd_size)
221 return ((void *)(pd->pd_va + (pa - pd->pd_pa)));
222 }
223
224 return (NULL);
225 }
226
227 /*
228 * Look up the given virtual address in the static mapping data and return the
229 * corresponding physical address, or DEVMAP_PADDR_NOTFOUND if not found.
230 */
231 vm_paddr_t
devmap_vtop(void * vpva,vm_size_t size)232 devmap_vtop(void * vpva, vm_size_t size)
233 {
234 const struct devmap_entry *pd;
235 vm_offset_t va;
236
237 if (devmap_table == NULL)
238 return (DEVMAP_PADDR_NOTFOUND);
239
240 va = (vm_offset_t)vpva;
241 for (pd = devmap_table; pd->pd_size != 0; ++pd) {
242 if (va >= pd->pd_va && va + size <= pd->pd_va + pd->pd_size)
243 return ((vm_paddr_t)(pd->pd_pa + (va - pd->pd_va)));
244 }
245
246 return (DEVMAP_PADDR_NOTFOUND);
247 }
248
249 /*
250 * Map a set of physical memory pages into the kernel virtual address space.
251 * Return a pointer to where it is mapped.
252 *
253 * This uses a pre-established static mapping if one exists for the requested
254 * range, otherwise it allocates kva space and maps the physical pages into it.
255 *
256 * This routine is intended to be used for mapping device memory, NOT real
257 * memory; the mapping type is inherently VM_MEMATTR_DEVICE in
258 * pmap_kenter_device().
259 */
260 void *
pmap_mapdev(vm_offset_t pa,vm_size_t size)261 pmap_mapdev(vm_offset_t pa, vm_size_t size)
262 {
263 vm_offset_t va, offset;
264 void * rva;
265
266 /* First look in the static mapping table. */
267 if ((rva = devmap_ptov(pa, size)) != NULL)
268 return (rva);
269
270 offset = pa & PAGE_MASK;
271 pa = trunc_page(pa);
272 size = round_page(size + offset);
273
274 #if defined(__aarch64__) || defined(__riscv)
275 if (early_boot) {
276 akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - size);
277 va = akva_devmap_vaddr;
278 KASSERT(va >= VM_MAX_KERNEL_ADDRESS - PMAP_MAPDEV_EARLY_SIZE,
279 ("Too many early devmap mappings"));
280 } else
281 #endif
282 va = kva_alloc(size);
283 if (!va)
284 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
285
286 pmap_kenter_device(va, size, pa);
287
288 return ((void *)(va + offset));
289 }
290
291 #if defined(__aarch64__) || defined(__riscv)
292 void *
pmap_mapdev_attr(vm_offset_t pa,vm_size_t size,vm_memattr_t ma)293 pmap_mapdev_attr(vm_offset_t pa, vm_size_t size, vm_memattr_t ma)
294 {
295 vm_offset_t va, offset;
296 void * rva;
297
298 /* First look in the static mapping table. */
299 if ((rva = devmap_ptov(pa, size)) != NULL)
300 return (rva);
301
302 offset = pa & PAGE_MASK;
303 pa = trunc_page(pa);
304 size = round_page(size + offset);
305
306 if (early_boot) {
307 akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - size);
308 va = akva_devmap_vaddr;
309 KASSERT(va >= (VM_MAX_KERNEL_ADDRESS - (PMAP_MAPDEV_EARLY_SIZE)),
310 ("Too many early devmap mappings 2"));
311 } else
312 va = kva_alloc(size);
313 if (!va)
314 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
315
316 pmap_kenter(va, size, pa, ma);
317
318 return ((void *)(va + offset));
319 }
320 #endif
321
322 /*
323 * Unmap device memory and free the kva space.
324 */
325 void
pmap_unmapdev(vm_offset_t va,vm_size_t size)326 pmap_unmapdev(vm_offset_t va, vm_size_t size)
327 {
328 vm_offset_t offset;
329
330 /* Nothing to do if we find the mapping in the static table. */
331 if (devmap_vtop((void*)va, size) != DEVMAP_PADDR_NOTFOUND)
332 return;
333
334 offset = va & PAGE_MASK;
335 va = trunc_page(va);
336 size = round_page(size + offset);
337
338 pmap_kremove_device(va, size);
339 kva_free(va, size);
340 }
341
342 #ifdef DDB
343 #include <ddb/ddb.h>
344
DB_SHOW_COMMAND(devmap,db_show_devmap)345 DB_SHOW_COMMAND(devmap, db_show_devmap)
346 {
347 devmap_dump_table(db_printf);
348 }
349
350 #endif /* DDB */
351