1 /*-
2 * Copyright (c) 2015 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Semihalf under
6 * the sponsorship of the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bitset.h>
36 #include <sys/bitstring.h>
37 #include <sys/bus.h>
38 #include <sys/endian.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/rman.h>
42 #include <sys/pciio.h>
43 #include <sys/pcpu.h>
44 #include <sys/lock.h>
45 #include <sys/mutex.h>
46 #include <sys/smp.h>
47
48 #include <dev/pci/pcireg.h>
49 #include <dev/pci/pcivar.h>
50
51 #include <vm/vm.h>
52 #include <vm/pmap.h>
53
54 #include <machine/bus.h>
55 #include <machine/cpu.h>
56 #include <machine/cpufunc.h>
57 #include <machine/intr.h>
58
59 #include "gic_v3_reg.h"
60 #include "gic_v3_var.h"
61
62 #define GIC_V3_ITS_QUIRK_THUNDERX_PEM_BUS_OFFSET 144
63
64 #include "pic_if.h"
65
66 /* Device and PIC methods */
67 static int gic_v3_its_attach(device_t);
68
69 static device_method_t gic_v3_its_methods[] = {
70 /* Device interface */
71 DEVMETHOD(device_attach, gic_v3_its_attach),
72 /*
73 * PIC interface
74 */
75 /* MSI-X */
76 DEVMETHOD(pic_alloc_msix, gic_v3_its_alloc_msix),
77 /* MSI */
78 DEVMETHOD(pic_alloc_msi, gic_v3_its_alloc_msi),
79 DEVMETHOD(pic_map_msi, gic_v3_its_map_msi),
80
81 /* End */
82 DEVMETHOD_END
83 };
84
85 DEFINE_CLASS_0(gic_v3_its, gic_v3_its_driver, gic_v3_its_methods,
86 sizeof(struct gic_v3_its_softc));
87
88 MALLOC_DEFINE(M_GIC_V3_ITS, "GICv3 ITS", GIC_V3_ITS_DEVSTR);
89
90 static int its_alloc_tables(struct gic_v3_its_softc *);
91 static void its_free_tables(struct gic_v3_its_softc *);
92 static void its_init_commandq(struct gic_v3_its_softc *);
93 static void its_init_cpu_collection(struct gic_v3_its_softc *);
94 static uint32_t its_get_devid(device_t);
95
96 static int its_cmd_send(struct gic_v3_its_softc *, struct its_cmd_desc *);
97
98 static void its_cmd_mapc(struct gic_v3_its_softc *, struct its_col *, uint8_t);
99 static void its_cmd_mapvi(struct gic_v3_its_softc *, struct its_dev *, uint32_t,
100 uint32_t);
101 static void its_cmd_mapi(struct gic_v3_its_softc *, struct its_dev *, uint32_t);
102 static void its_cmd_inv(struct gic_v3_its_softc *, struct its_dev *, uint32_t);
103 static void its_cmd_invall(struct gic_v3_its_softc *, struct its_col *);
104
105 static uint32_t its_get_devbits(device_t);
106
107 static void lpi_init_conftable(struct gic_v3_its_softc *);
108 static void lpi_bitmap_init(struct gic_v3_its_softc *);
109 static int lpi_config_cpu(struct gic_v3_its_softc *);
110 static void lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *);
111
112 const char *its_ptab_cache[] = {
113 [GITS_BASER_CACHE_NCNB] = "(NC,NB)",
114 [GITS_BASER_CACHE_NC] = "(NC)",
115 [GITS_BASER_CACHE_RAWT] = "(RA,WT)",
116 [GITS_BASER_CACHE_RAWB] = "(RA,WB)",
117 [GITS_BASER_CACHE_WAWT] = "(WA,WT)",
118 [GITS_BASER_CACHE_WAWB] = "(WA,WB)",
119 [GITS_BASER_CACHE_RAWAWT] = "(RAWA,WT)",
120 [GITS_BASER_CACHE_RAWAWB] = "(RAWA,WB)",
121 };
122
123 const char *its_ptab_share[] = {
124 [GITS_BASER_SHARE_NS] = "none",
125 [GITS_BASER_SHARE_IS] = "inner",
126 [GITS_BASER_SHARE_OS] = "outer",
127 [GITS_BASER_SHARE_RES] = "none",
128 };
129
130 const char *its_ptab_type[] = {
131 [GITS_BASER_TYPE_UNIMPL] = "Unimplemented",
132 [GITS_BASER_TYPE_DEV] = "Devices",
133 [GITS_BASER_TYPE_VP] = "Virtual Processors",
134 [GITS_BASER_TYPE_PP] = "Physical Processors",
135 [GITS_BASER_TYPE_IC] = "Interrupt Collections",
136 [GITS_BASER_TYPE_RES5] = "Reserved (5)",
137 [GITS_BASER_TYPE_RES6] = "Reserved (6)",
138 [GITS_BASER_TYPE_RES7] = "Reserved (7)",
139 };
140
141 /*
142 * Vendor specific quirks.
143 * One needs to add appropriate entry to its_quirks[]
144 * table if the imlementation varies from the generic ARM ITS.
145 */
146
147 /* Cavium ThunderX PCI devid acquire function */
148 static uint32_t its_get_devbits_thunder(device_t);
149 static uint32_t its_get_devid_thunder(device_t);
150
151 static const struct its_quirks its_quirks[] = {
152 {
153 /*
154 * Hardware: Cavium ThunderX
155 * Chip revision: Pass 1.0, Pass 1.1
156 */
157 .cpuid = CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDER, 0, 0),
158 .cpuid_mask = CPU_IMPL_MASK | CPU_PART_MASK,
159 .devid_func = its_get_devid_thunder,
160 .devbits_func = its_get_devbits_thunder,
161 },
162 };
163
164 static struct gic_v3_its_softc *its_sc;
165
166 #define gic_its_read(sc, len, reg) \
167 bus_read_##len(&sc->its_res[0], reg)
168
169 #define gic_its_write(sc, len, reg, val) \
170 bus_write_##len(&sc->its_res[0], reg, val)
171
172 static int
gic_v3_its_attach(device_t dev)173 gic_v3_its_attach(device_t dev)
174 {
175 struct gic_v3_its_softc *sc;
176 uint64_t gits_tmp;
177 uint32_t gits_pidr2;
178 int rid;
179 int ret;
180
181 sc = device_get_softc(dev);
182
183 /*
184 * XXX ARM64TODO: Avoid configuration of more than one ITS
185 * device. To be removed when multi-PIC support is added
186 * to FreeBSD (or at least multi-ITS is implemented). Limit
187 * supported ITS sockets to '0' only.
188 */
189 if (device_get_unit(dev) != 0) {
190 device_printf(dev,
191 "Only single instance of ITS is supported, exiting...\n");
192 return (ENXIO);
193 }
194 sc->its_socket = 0;
195
196 /*
197 * Initialize sleep & spin mutex for ITS
198 */
199 /* Protects ITS device list and assigned LPIs bitmaps. */
200 mtx_init(&sc->its_mtx, "ITS sleep lock", NULL, MTX_DEF);
201 /* Protects access to ITS command circular buffer. */
202 mtx_init(&sc->its_spin_mtx, "ITS spin lock", NULL, MTX_SPIN);
203
204 rid = 0;
205 sc->its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
206 RF_ACTIVE);
207 if (sc->its_res == NULL) {
208 device_printf(dev, "Could not allocate memory\n");
209 return (ENXIO);
210 }
211
212 sc->dev = dev;
213
214 gits_pidr2 = gic_its_read(sc, 4, GITS_PIDR2);
215 switch (gits_pidr2 & GITS_PIDR2_ARCH_MASK) {
216 case GITS_PIDR2_ARCH_GICv3: /* fall through */
217 case GITS_PIDR2_ARCH_GICv4:
218 if (bootverbose) {
219 device_printf(dev, "ITS found. Architecture rev. %u\n",
220 (u_int)(gits_pidr2 & GITS_PIDR2_ARCH_MASK) >> 4);
221 }
222 break;
223 default:
224 device_printf(dev, "No ITS found in the system\n");
225 gic_v3_its_detach(dev);
226 return (ENODEV);
227 }
228
229 /* 1. Initialize commands queue */
230 its_init_commandq(sc);
231
232 /* 2. Provide memory for any private ITS tables */
233 ret = its_alloc_tables(sc);
234 if (ret != 0) {
235 gic_v3_its_detach(dev);
236 return (ret);
237 }
238
239 /* 3. Allocate collections. One per-CPU */
240 for (int cpu = 0; cpu < mp_ncpus; cpu++)
241 if (CPU_ISSET(cpu, &all_cpus) != 0)
242 sc->its_cols[cpu] = malloc(sizeof(*sc->its_cols[0]),
243 M_GIC_V3_ITS, (M_WAITOK | M_ZERO));
244 else
245 sc->its_cols[cpu] = NULL;
246
247 /* 4. Enable ITS in GITS_CTLR */
248 gits_tmp = gic_its_read(sc, 4, GITS_CTLR);
249 gic_its_write(sc, 4, GITS_CTLR, gits_tmp | GITS_CTLR_EN);
250
251 /* 5. Initialize LPIs configuration table */
252 lpi_init_conftable(sc);
253
254 /* 6. LPIs bitmap init */
255 lpi_bitmap_init(sc);
256
257 /* 7. Allocate pending tables for all CPUs */
258 lpi_alloc_cpu_pendtables(sc);
259
260 /* 8. CPU init */
261 (void)its_init_cpu(sc);
262
263 /* 9. Init ITS devices list */
264 TAILQ_INIT(&sc->its_dev_list);
265
266 arm_register_msi_pic(dev);
267
268 /*
269 * XXX ARM64TODO: We need to have ITS software context
270 * when being called by the interrupt code (mask/unmask).
271 * This may be used only when one ITS is present in
272 * the system and eventually should be removed.
273 */
274 KASSERT(its_sc == NULL,
275 ("Trying to assign its_sc that is already set"));
276 its_sc = sc;
277
278 return (0);
279 }
280
281 /* Will not detach but use it for convenience */
282 int
gic_v3_its_detach(device_t dev)283 gic_v3_its_detach(device_t dev)
284 {
285 device_t parent;
286 struct gic_v3_softc *gic_sc;
287 struct gic_v3_its_softc *sc;
288 u_int cpuid;
289 int rid = 0;
290
291 sc = device_get_softc(dev);
292 cpuid = PCPU_GET(cpuid);
293
294 /* Release what's possible */
295
296 /* Command queue */
297 if ((void *)sc->its_cmdq_base != NULL) {
298 contigfree((void *)sc->its_cmdq_base,
299 ITS_CMDQ_SIZE, M_GIC_V3_ITS);
300 }
301 /* ITTs */
302 its_free_tables(sc);
303 /* Collections */
304 for (cpuid = 0; cpuid < mp_ncpus; cpuid++)
305 free(sc->its_cols[cpuid], M_GIC_V3_ITS);
306 /* LPI config table */
307 parent = device_get_parent(sc->dev);
308 gic_sc = device_get_softc(parent);
309 if ((void *)gic_sc->gic_redists.lpis.conf_base != NULL) {
310 contigfree((void *)gic_sc->gic_redists.lpis.conf_base,
311 LPI_CONFTAB_SIZE, M_GIC_V3_ITS);
312 }
313 for (cpuid = 0; cpuid < mp_ncpus; cpuid++)
314 if ((void *)gic_sc->gic_redists.lpis.pend_base[cpuid] != NULL) {
315 contigfree(
316 (void *)gic_sc->gic_redists.lpis.pend_base[cpuid],
317 roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K),
318 M_GIC_V3_ITS);
319 }
320
321 /* Resource... */
322 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->its_res);
323
324 /* XXX ARM64TODO: Reset global pointer to ITS software context */
325 its_sc = NULL;
326
327 return (0);
328 }
329
330 static int
its_alloc_tables(struct gic_v3_its_softc * sc)331 its_alloc_tables(struct gic_v3_its_softc *sc)
332 {
333 uint64_t gits_baser, gits_tmp;
334 uint64_t type, esize, cache, share, psz;
335 size_t page_size, npages, nitspages, nidents, tn;
336 size_t its_tbl_size;
337 vm_offset_t ptab_vaddr;
338 vm_paddr_t ptab_paddr;
339 boolean_t first = TRUE;
340
341 page_size = PAGE_SIZE_64K;
342
343 for (tn = 0; tn < GITS_BASER_NUM; tn++) {
344 gits_baser = gic_its_read(sc, 8, GITS_BASER(tn));
345 type = GITS_BASER_TYPE(gits_baser);
346 /* Get the Table Entry size */
347 esize = GITS_BASER_ESIZE(gits_baser);
348
349 switch (type) {
350 case GITS_BASER_TYPE_UNIMPL: /* fall through */
351 case GITS_BASER_TYPE_RES5:
352 case GITS_BASER_TYPE_RES6:
353 case GITS_BASER_TYPE_RES7:
354 continue;
355 case GITS_BASER_TYPE_DEV:
356 nidents = (1 << its_get_devbits(sc->dev));
357 its_tbl_size = esize * nidents;
358 its_tbl_size = roundup2(its_tbl_size, page_size);
359 npages = howmany(its_tbl_size, PAGE_SIZE);
360 break;
361 default:
362 npages = howmany(page_size, PAGE_SIZE);
363 break;
364 }
365
366 /* Allocate required space */
367 ptab_vaddr = (vm_offset_t)contigmalloc(npages * PAGE_SIZE,
368 M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE, 0);
369
370 sc->its_ptabs[tn].ptab_vaddr = ptab_vaddr;
371 sc->its_ptabs[tn].ptab_pgsz = PAGE_SIZE;
372 sc->its_ptabs[tn].ptab_npages = npages;
373
374 ptab_paddr = vtophys(ptab_vaddr);
375 KASSERT((ptab_paddr & GITS_BASER_PA_MASK) == ptab_paddr,
376 ("%s: Unaligned PA for Interrupt Translation Table",
377 device_get_name(sc->dev)));
378
379 /* Set defaults: WAWB, IS */
380 cache = GITS_BASER_CACHE_WAWB;
381 share = GITS_BASER_SHARE_IS;
382
383 for (;;) {
384 nitspages = howmany(its_tbl_size, page_size);
385
386 switch (page_size) {
387 case PAGE_SIZE: /* 4KB */
388 psz = GITS_BASER_PSZ_4K;
389 break;
390 case PAGE_SIZE_16K: /* 16KB */
391 psz = GITS_BASER_PSZ_4K;
392 break;
393 case PAGE_SIZE_64K: /* 64KB */
394 psz = GITS_BASER_PSZ_64K;
395 break;
396 default:
397 device_printf(sc->dev,
398 "Unsupported page size: %zuKB\n",
399 (page_size / 1024));
400 its_free_tables(sc);
401 return (ENXIO);
402 }
403
404 /* Clear fields under modification first */
405 gits_baser &= ~(GITS_BASER_VALID |
406 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
407 GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK |
408 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
409 GITS_BASER_SIZE_MASK);
410 /* Construct register value */
411 gits_baser |=
412 (type << GITS_BASER_TYPE_SHIFT) |
413 ((esize - 1) << GITS_BASER_ESIZE_SHIFT) |
414 (cache << GITS_BASER_CACHE_SHIFT) |
415 (share << GITS_BASER_SHARE_SHIFT) |
416 (psz << GITS_BASER_PSZ_SHIFT) |
417 ptab_paddr | (nitspages - 1) |
418 GITS_BASER_VALID;
419
420 gic_its_write(sc, 8, GITS_BASER(tn), gits_baser);
421 /*
422 * Verify.
423 * Depending on implementation we may encounter
424 * shareability and page size mismatch.
425 */
426 gits_tmp = gic_its_read(sc, 8, GITS_BASER(tn));
427 if (((gits_tmp ^ gits_baser) & GITS_BASER_SHARE_MASK) != 0) {
428 share = gits_tmp & GITS_BASER_SHARE_MASK;
429 share >>= GITS_BASER_SHARE_SHIFT;
430 continue;
431 }
432
433 if (((gits_tmp ^ gits_baser) & GITS_BASER_PSZ_MASK) != 0) {
434 switch (page_size) {
435 case PAGE_SIZE_16K:
436 /* Drop to 4KB page */
437 page_size = PAGE_SIZE;
438 continue;
439 case PAGE_SIZE_64K:
440 /* Drop to 16KB page */
441 page_size = PAGE_SIZE_16K;
442 continue;
443 }
444 }
445 /*
446 * All possible adjustments should
447 * be applied by now so just break the loop.
448 */
449 break;
450 }
451 /*
452 * Do not compare Cacheability field since
453 * it is implementation defined.
454 */
455 gits_tmp &= ~GITS_BASER_CACHE_MASK;
456 gits_baser &= ~GITS_BASER_CACHE_MASK;
457
458 if (gits_tmp != gits_baser) {
459 device_printf(sc->dev,
460 "Could not allocate ITS tables\n");
461 its_free_tables(sc);
462 return (ENXIO);
463 }
464
465 if (bootverbose) {
466 if (first) {
467 device_printf(sc->dev,
468 "Allocated ITS private tables:\n");
469 first = FALSE;
470 }
471 device_printf(sc->dev,
472 "\tPTAB%zu for %s: PA 0x%lx,"
473 " %lu entries,"
474 " cache policy %s, %s shareable,"
475 " page size %zuKB\n",
476 tn, its_ptab_type[type], ptab_paddr,
477 (page_size * nitspages) / esize,
478 its_ptab_cache[cache], its_ptab_share[share],
479 page_size / 1024);
480 }
481 }
482
483 return (0);
484 }
485
486 static void
its_free_tables(struct gic_v3_its_softc * sc)487 its_free_tables(struct gic_v3_its_softc *sc)
488 {
489 vm_offset_t ptab_vaddr;
490 size_t size;
491 size_t tn;
492
493 for (tn = 0; tn < GITS_BASER_NUM; tn++) {
494 ptab_vaddr = sc->its_ptabs[tn].ptab_vaddr;
495 if (ptab_vaddr == 0)
496 continue;
497 size = sc->its_ptabs[tn].ptab_pgsz;
498 size *= sc->its_ptabs[tn].ptab_npages;
499
500 if ((void *)ptab_vaddr != NULL)
501 contigfree((void *)ptab_vaddr, size, M_GIC_V3_ITS);
502
503 /* Clear the table description */
504 memset(&sc->its_ptabs[tn], 0, sizeof(sc->its_ptabs[tn]));
505 }
506 }
507
508 static void
its_init_commandq(struct gic_v3_its_softc * sc)509 its_init_commandq(struct gic_v3_its_softc *sc)
510 {
511 uint64_t gits_cbaser, gits_tmp;
512 uint64_t cache, share;
513 vm_paddr_t cmdq_paddr;
514 device_t dev;
515
516 dev = sc->dev;
517 /* Allocate memory for command queue */
518 sc->its_cmdq_base = contigmalloc(ITS_CMDQ_SIZE, M_GIC_V3_ITS,
519 (M_WAITOK | M_ZERO), 0, ~0UL, ITS_CMDQ_SIZE, 0);
520 /* Set command queue write pointer (command queue empty) */
521 sc->its_cmdq_write = sc->its_cmdq_base;
522
523 /* Save command queue pointer and attributes */
524 cmdq_paddr = vtophys(sc->its_cmdq_base);
525
526 /* Set defaults: Normal Inner WAWB, IS */
527 cache = GITS_CBASER_CACHE_NIWAWB;
528 share = GITS_CBASER_SHARE_IS;
529
530 gits_cbaser = (cmdq_paddr |
531 (cache << GITS_CBASER_CACHE_SHIFT) |
532 (share << GITS_CBASER_SHARE_SHIFT) |
533 /* Number of 4KB pages - 1 */
534 ((ITS_CMDQ_SIZE / PAGE_SIZE) - 1) |
535 /* Valid bit */
536 GITS_CBASER_VALID);
537
538 gic_its_write(sc, 8, GITS_CBASER, gits_cbaser);
539 gits_tmp = gic_its_read(sc, 8, GITS_CBASER);
540
541 if (((gits_tmp ^ gits_cbaser) & GITS_CBASER_SHARE_MASK) != 0) {
542 if (bootverbose) {
543 device_printf(dev,
544 "Will use cache flushing for commands queue\n");
545 }
546 /* Command queue needs cache flushing */
547 sc->its_flags |= ITS_FLAGS_CMDQ_FLUSH;
548 }
549
550 gic_its_write(sc, 8, GITS_CWRITER, 0x0);
551 }
552
553 int
its_init_cpu(struct gic_v3_its_softc * sc)554 its_init_cpu(struct gic_v3_its_softc *sc)
555 {
556 device_t parent;
557 struct gic_v3_softc *gic_sc;
558
559 /*
560 * NULL in place of the softc pointer means that
561 * this function was called during GICv3 secondary initialization.
562 */
563 if (sc == NULL) {
564 if (device_is_attached(its_sc->dev)) {
565 /*
566 * XXX ARM64TODO: This is part of the workaround that
567 * saves ITS software context for further use in
568 * mask/unmask and here. This should be removed as soon
569 * as the upper layer is capable of passing the ITS
570 * context to this function.
571 */
572 sc = its_sc;
573 } else
574 return (ENXIO);
575
576 /* Skip if running secondary init on a wrong socket */
577 if (sc->its_socket != CPU_CURRENT_SOCKET)
578 return (ENXIO);
579 }
580
581 /*
582 * Check for LPIs support on this Re-Distributor.
583 */
584 parent = device_get_parent(sc->dev);
585 gic_sc = device_get_softc(parent);
586 if ((gic_r_read(gic_sc, 4, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) {
587 if (bootverbose) {
588 device_printf(sc->dev,
589 "LPIs not supported on CPU%u\n", PCPU_GET(cpuid));
590 }
591 return (ENXIO);
592 }
593
594 /* Configure LPIs for this CPU */
595 lpi_config_cpu(sc);
596
597 /* Initialize collections */
598 its_init_cpu_collection(sc);
599
600 return (0);
601 }
602
603 static void
its_init_cpu_collection(struct gic_v3_its_softc * sc)604 its_init_cpu_collection(struct gic_v3_its_softc *sc)
605 {
606 device_t parent;
607 struct gic_v3_softc *gic_sc;
608 uint64_t typer;
609 uint64_t target;
610 vm_offset_t redist_base;
611 u_int cpuid;
612
613 cpuid = PCPU_GET(cpuid);
614 parent = device_get_parent(sc->dev);
615 gic_sc = device_get_softc(parent);
616
617 typer = gic_its_read(sc, 8, GITS_TYPER);
618 if ((typer & GITS_TYPER_PTA) != 0) {
619 redist_base =
620 rman_get_bushandle(gic_sc->gic_redists.pcpu[cpuid]);
621 /*
622 * Target Address correspond to the base physical
623 * address of Re-Distributors.
624 */
625 target = vtophys(redist_base);
626 } else {
627 /* Target Address correspond to unique processor numbers */
628 typer = gic_r_read(gic_sc, 8, GICR_TYPER);
629 target = GICR_TYPER_CPUNUM(typer);
630 }
631
632 sc->its_cols[cpuid]->col_target = target;
633 sc->its_cols[cpuid]->col_id = cpuid;
634
635 its_cmd_mapc(sc, sc->its_cols[cpuid], 1);
636 its_cmd_invall(sc, sc->its_cols[cpuid]);
637
638 }
639
640 static void
lpi_init_conftable(struct gic_v3_its_softc * sc)641 lpi_init_conftable(struct gic_v3_its_softc *sc)
642 {
643 device_t parent;
644 struct gic_v3_softc *gic_sc;
645 vm_offset_t conf_base;
646 uint8_t prio_default;
647
648 parent = device_get_parent(sc->dev);
649 gic_sc = device_get_softc(parent);
650 /*
651 * LPI Configuration Table settings.
652 * Notice that Configuration Table is shared among all
653 * Re-Distributors, so this is going to be created just once.
654 */
655 conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE,
656 M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0);
657
658 if (bootverbose) {
659 device_printf(sc->dev,
660 "LPI Configuration Table at PA: 0x%lx\n",
661 vtophys(conf_base));
662 }
663
664 /*
665 * Let the default priority be aligned with all other
666 * interrupts assuming that each interrupt is assigned
667 * MAX priority at startup. MAX priority on the other
668 * hand cannot be higher than 0xFC for LPIs.
669 */
670 prio_default = GIC_PRIORITY_MAX;
671
672 /* Write each settings byte to LPI configuration table */
673 memset((void *)conf_base,
674 (prio_default & LPI_CONF_PRIO_MASK) | LPI_CONF_GROUP1,
675 LPI_CONFTAB_SIZE);
676
677 cpu_dcache_wb_range((vm_offset_t)conf_base, roundup2(LPI_CONFTAB_SIZE,
678 PAGE_SIZE_64K));
679
680 gic_sc->gic_redists.lpis.conf_base = conf_base;
681 }
682
683 static void
lpi_alloc_cpu_pendtables(struct gic_v3_its_softc * sc)684 lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *sc)
685 {
686 device_t parent;
687 struct gic_v3_softc *gic_sc;
688 vm_offset_t pend_base;
689 u_int cpuid;
690
691 parent = device_get_parent(sc->dev);
692 gic_sc = device_get_softc(parent);
693
694 /*
695 * LPI Pending Table settings.
696 * This has to be done for each Re-Distributor, hence for each CPU.
697 */
698 for (cpuid = 0; cpuid < mp_ncpus; cpuid++) {
699
700 /* Limit allocation to active CPUs only */
701 if (CPU_ISSET(cpuid, &all_cpus) == 0)
702 continue;
703
704 pend_base = (vm_offset_t)contigmalloc(
705 roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS,
706 (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0);
707
708 /* Clean D-cache so that ITS can see zeroed pages */
709 cpu_dcache_wb_range((vm_offset_t)pend_base,
710 roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K));
711
712 if (bootverbose) {
713 device_printf(sc->dev,
714 "LPI Pending Table for CPU%u at PA: 0x%lx\n",
715 cpuid, vtophys(pend_base));
716 }
717
718 gic_sc->gic_redists.lpis.pend_base[cpuid] = pend_base;
719 }
720
721 /* Ensure visibility of pend_base addresses on other CPUs */
722 wmb();
723 }
724
725 static int
lpi_config_cpu(struct gic_v3_its_softc * sc)726 lpi_config_cpu(struct gic_v3_its_softc *sc)
727 {
728 device_t parent;
729 struct gic_v3_softc *gic_sc;
730 vm_offset_t conf_base, pend_base;
731 uint64_t gicr_xbaser, gicr_temp;
732 uint64_t cache, share, idbits;
733 uint32_t gicr_ctlr;
734 u_int cpuid;
735
736 parent = device_get_parent(sc->dev);
737 gic_sc = device_get_softc(parent);
738 cpuid = PCPU_GET(cpuid);
739
740 /* Ensure data observability on a current CPU */
741 rmb();
742
743 conf_base = gic_sc->gic_redists.lpis.conf_base;
744 pend_base = gic_sc->gic_redists.lpis.pend_base[cpuid];
745
746 /* Disable LPIs */
747 gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR);
748 gicr_ctlr &= ~GICR_CTLR_LPI_ENABLE;
749 gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr);
750 /* Perform full system barrier */
751 dsb(sy);
752
753 /*
754 * Set GICR_PROPBASER
755 */
756
757 /*
758 * Find out how many bits do we need for LPI identifiers.
759 * Remark 1.: Even though we have (LPI_CONFTAB_SIZE / 8) LPIs
760 * the notified LPI ID still starts from 8192
761 * (GIC_FIRST_LPI).
762 * Remark 2.: This could be done on compilation time but there
763 * seems to be no sufficient macro.
764 */
765 idbits = flsl(LPI_CONFTAB_SIZE + GIC_FIRST_LPI) - 1;
766
767 /* Set defaults: Normal Inner WAWB, IS */
768 cache = GICR_PROPBASER_CACHE_NIWAWB;
769 share = GICR_PROPBASER_SHARE_IS;
770
771 gicr_xbaser = vtophys(conf_base) |
772 ((idbits - 1) & GICR_PROPBASER_IDBITS_MASK) |
773 (cache << GICR_PROPBASER_CACHE_SHIFT) |
774 (share << GICR_PROPBASER_SHARE_SHIFT);
775
776 gic_r_write(gic_sc, 8, GICR_PROPBASER, gicr_xbaser);
777 gicr_temp = gic_r_read(gic_sc, 8, GICR_PROPBASER);
778
779 if (((gicr_xbaser ^ gicr_temp) & GICR_PROPBASER_SHARE_MASK) != 0) {
780 if (bootverbose) {
781 device_printf(sc->dev,
782 "Will use cache flushing for LPI "
783 "Configuration Table\n");
784 }
785 gic_sc->gic_redists.lpis.flags |= LPI_FLAGS_CONF_FLUSH;
786 }
787
788 /*
789 * Set GICR_PENDBASER
790 */
791
792 /* Set defaults: Normal Inner WAWB, IS */
793 cache = GICR_PENDBASER_CACHE_NIWAWB;
794 share = GICR_PENDBASER_SHARE_IS;
795
796 gicr_xbaser = vtophys(pend_base) |
797 (cache << GICR_PENDBASER_CACHE_SHIFT) |
798 (share << GICR_PENDBASER_SHARE_SHIFT);
799
800 gic_r_write(gic_sc, 8, GICR_PENDBASER, gicr_xbaser);
801
802 /* Enable LPIs */
803 gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR);
804 gicr_ctlr |= GICR_CTLR_LPI_ENABLE;
805 gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr);
806
807 dsb(sy);
808
809 return (0);
810 }
811
812 static void
lpi_bitmap_init(struct gic_v3_its_softc * sc)813 lpi_bitmap_init(struct gic_v3_its_softc *sc)
814 {
815 device_t parent;
816 struct gic_v3_softc *gic_sc;
817 uint32_t lpi_id_num;
818 size_t lpi_chunks_num;
819 size_t bits_in_chunk;
820
821 parent = device_get_parent(sc->dev);
822 gic_sc = device_get_softc(parent);
823
824 lpi_id_num = (1 << gic_sc->gic_idbits) - 1;
825 /* Substract IDs dedicated for SGIs, PPIs and SPIs */
826 lpi_id_num -= GIC_FIRST_LPI;
827
828 sc->its_lpi_maxid = lpi_id_num;
829
830 bits_in_chunk = sizeof(*sc->its_lpi_bitmap) * NBBY;
831
832 /*
833 * Round up to the number of bits in chunk.
834 * We will need to take care to avoid using invalid LPI IDs later.
835 */
836 lpi_id_num = roundup2(lpi_id_num, bits_in_chunk);
837 lpi_chunks_num = lpi_id_num / bits_in_chunk;
838
839 sc->its_lpi_bitmap =
840 contigmalloc((lpi_chunks_num * sizeof(*sc->its_lpi_bitmap)),
841 M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL,
842 sizeof(*sc->its_lpi_bitmap), 0);
843 }
844
845 static int
lpi_alloc_chunk(struct gic_v3_its_softc * sc,struct lpi_chunk * lpic,u_int nvecs)846 lpi_alloc_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic,
847 u_int nvecs)
848 {
849 int fclr; /* First cleared bit */
850 uint8_t *bitmap;
851 size_t nb, i;
852
853 bitmap = (uint8_t *)sc->its_lpi_bitmap;
854
855 fclr = 0;
856 retry:
857 /* Check other bits - sloooow */
858 for (i = 0, nb = fclr; i < nvecs; i++, nb++) {
859 if (nb > sc->its_lpi_maxid)
860 return (EINVAL);
861
862 if (isset(bitmap, nb)) {
863 /* To little free bits in this area. Move on. */
864 fclr = nb + 1;
865 goto retry;
866 }
867 }
868 /* This area is free. Take it. */
869 bit_nset(bitmap, fclr, fclr + nvecs - 1);
870 lpic->lpi_base = fclr + GIC_FIRST_LPI;
871 lpic->lpi_num = nvecs;
872 lpic->lpi_free = lpic->lpi_num;
873
874 return (0);
875 }
876
877 static void
lpi_free_chunk(struct gic_v3_its_softc * sc,struct lpi_chunk * lpic)878 lpi_free_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic)
879 {
880 int start, end;
881 uint8_t *bitmap;
882
883 bitmap = (uint8_t *)sc->its_lpi_bitmap;
884
885 KASSERT((lpic->lpi_free == lpic->lpi_num),
886 ("Trying to free LPI chunk that is still in use.\n"));
887
888 /* First bit of this chunk in a global bitmap */
889 start = lpic->lpi_base - GIC_FIRST_LPI;
890 /* and last bit of this chunk... */
891 end = start + lpic->lpi_num - 1;
892
893 /* Finally free this chunk */
894 bit_nclear(bitmap, start, end);
895 }
896
897 static void
lpi_configure(struct gic_v3_its_softc * sc,struct its_dev * its_dev,uint32_t lpinum,boolean_t unmask)898 lpi_configure(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
899 uint32_t lpinum, boolean_t unmask)
900 {
901 device_t parent;
902 struct gic_v3_softc *gic_sc;
903 uint8_t *conf_byte;
904
905 parent = device_get_parent(sc->dev);
906 gic_sc = device_get_softc(parent);
907
908 conf_byte = (uint8_t *)gic_sc->gic_redists.lpis.conf_base;
909 conf_byte += (lpinum - GIC_FIRST_LPI);
910
911 if (unmask)
912 *conf_byte |= LPI_CONF_ENABLE;
913 else
914 *conf_byte &= ~LPI_CONF_ENABLE;
915
916 if ((gic_sc->gic_redists.lpis.flags & LPI_FLAGS_CONF_FLUSH) != 0) {
917 /* Clean D-cache under configuration byte */
918 cpu_dcache_wb_range((vm_offset_t)conf_byte, sizeof(*conf_byte));
919 } else {
920 /* DSB inner shareable, store */
921 dsb(ishst);
922 }
923
924 its_cmd_inv(sc, its_dev, lpinum);
925 }
926
927 static void
lpi_map_to_device(struct gic_v3_its_softc * sc,struct its_dev * its_dev,uint32_t id,uint32_t pid)928 lpi_map_to_device(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
929 uint32_t id, uint32_t pid)
930 {
931
932 if ((pid < its_dev->lpis.lpi_base) ||
933 (pid >= (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num)))
934 panic("Trying to map ivalid LPI %u for the device\n", pid);
935
936 its_cmd_mapvi(sc, its_dev, id, pid);
937 }
938
939 static void
lpi_xmask_irq(device_t parent,uint32_t irq,boolean_t unmask)940 lpi_xmask_irq(device_t parent, uint32_t irq, boolean_t unmask)
941 {
942 struct its_dev *its_dev;
943
944 TAILQ_FOREACH(its_dev, &its_sc->its_dev_list, entry) {
945 if (irq >= its_dev->lpis.lpi_base &&
946 irq < (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num)) {
947 lpi_configure(its_sc, its_dev, irq, unmask);
948 return;
949 }
950 }
951
952 panic("Trying to %s not existing LPI: %u\n",
953 (unmask == TRUE) ? "unmask" : "mask", irq);
954 }
955
956 void
lpi_unmask_irq(device_t parent,uint32_t irq)957 lpi_unmask_irq(device_t parent, uint32_t irq)
958 {
959
960 lpi_xmask_irq(parent, irq, 1);
961 }
962
963 void
lpi_mask_irq(device_t parent,uint32_t irq)964 lpi_mask_irq(device_t parent, uint32_t irq)
965 {
966
967 lpi_xmask_irq(parent, irq, 0);
968 }
969
970 /*
971 * Commands handling.
972 */
973
974 static __inline void
cmd_format_command(struct its_cmd * cmd,uint8_t cmd_type)975 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
976 {
977 /* Command field: DW0 [7:0] */
978 cmd->cmd_dword[0] &= ~CMD_COMMAND_MASK;
979 cmd->cmd_dword[0] |= cmd_type;
980 }
981
982 static __inline void
cmd_format_devid(struct its_cmd * cmd,uint32_t devid)983 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
984 {
985 /* Device ID field: DW0 [63:32] */
986 cmd->cmd_dword[0] &= ~CMD_DEVID_MASK;
987 cmd->cmd_dword[0] |= ((uint64_t)devid << CMD_DEVID_SHIFT);
988 }
989
990 static __inline void
cmd_format_size(struct its_cmd * cmd,uint16_t size)991 cmd_format_size(struct its_cmd *cmd, uint16_t size)
992 {
993 /* Size field: DW1 [4:0] */
994 cmd->cmd_dword[1] &= ~CMD_SIZE_MASK;
995 cmd->cmd_dword[1] |= (size & CMD_SIZE_MASK);
996 }
997
998 static __inline void
cmd_format_id(struct its_cmd * cmd,uint32_t id)999 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1000 {
1001 /* ID field: DW1 [31:0] */
1002 cmd->cmd_dword[1] &= ~CMD_ID_MASK;
1003 cmd->cmd_dword[1] |= id;
1004 }
1005
1006 static __inline void
cmd_format_pid(struct its_cmd * cmd,uint32_t pid)1007 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1008 {
1009 /* Physical ID field: DW1 [63:32] */
1010 cmd->cmd_dword[1] &= ~CMD_PID_MASK;
1011 cmd->cmd_dword[1] |= ((uint64_t)pid << CMD_PID_SHIFT);
1012 }
1013
1014 static __inline void
cmd_format_col(struct its_cmd * cmd,uint16_t col_id)1015 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1016 {
1017 /* Collection field: DW2 [16:0] */
1018 cmd->cmd_dword[2] &= ~CMD_COL_MASK;
1019 cmd->cmd_dword[2] |= col_id;
1020 }
1021
1022 static __inline void
cmd_format_target(struct its_cmd * cmd,uint64_t target)1023 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1024 {
1025 /* Target Address field: DW2 [47:16] */
1026 cmd->cmd_dword[2] &= ~CMD_TARGET_MASK;
1027 cmd->cmd_dword[2] |= (target & CMD_TARGET_MASK);
1028 }
1029
1030 static __inline void
cmd_format_itt(struct its_cmd * cmd,uint64_t itt)1031 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1032 {
1033 /* ITT Address field: DW2 [47:8] */
1034 cmd->cmd_dword[2] &= ~CMD_ITT_MASK;
1035 cmd->cmd_dword[2] |= (itt & CMD_ITT_MASK);
1036 }
1037
1038 static __inline void
cmd_format_valid(struct its_cmd * cmd,uint8_t valid)1039 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1040 {
1041 /* Valid field: DW2 [63] */
1042 cmd->cmd_dword[2] &= ~CMD_VALID_MASK;
1043 cmd->cmd_dword[2] |= ((uint64_t)valid << CMD_VALID_SHIFT);
1044 }
1045
1046 static __inline void
cmd_fix_endian(struct its_cmd * cmd)1047 cmd_fix_endian(struct its_cmd *cmd)
1048 {
1049 size_t i;
1050
1051 for (i = 0; i < nitems(cmd->cmd_dword); i++)
1052 cmd->cmd_dword[i] = htole64(cmd->cmd_dword[i]);
1053 }
1054
1055 static void
its_cmd_mapc(struct gic_v3_its_softc * sc,struct its_col * col,uint8_t valid)1056 its_cmd_mapc(struct gic_v3_its_softc *sc, struct its_col *col, uint8_t valid)
1057 {
1058 struct its_cmd_desc desc;
1059
1060 desc.cmd_type = ITS_CMD_MAPC;
1061 desc.cmd_desc_mapc.col = col;
1062 /*
1063 * Valid bit set - map the collection.
1064 * Valid bit cleared - unmap the collection.
1065 */
1066 desc.cmd_desc_mapc.valid = valid;
1067
1068 its_cmd_send(sc, &desc);
1069 }
1070
1071 static void
its_cmd_mapvi(struct gic_v3_its_softc * sc,struct its_dev * its_dev,uint32_t id,uint32_t pid)1072 its_cmd_mapvi(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
1073 uint32_t id, uint32_t pid)
1074 {
1075 struct its_cmd_desc desc;
1076
1077 desc.cmd_type = ITS_CMD_MAPVI;
1078 desc.cmd_desc_mapvi.its_dev = its_dev;
1079 desc.cmd_desc_mapvi.id = id;
1080 desc.cmd_desc_mapvi.pid = pid;
1081
1082 its_cmd_send(sc, &desc);
1083 }
1084
1085 static void __unused
its_cmd_mapi(struct gic_v3_its_softc * sc,struct its_dev * its_dev,uint32_t lpinum)1086 its_cmd_mapi(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
1087 uint32_t lpinum)
1088 {
1089 struct its_cmd_desc desc;
1090
1091 desc.cmd_type = ITS_CMD_MAPI;
1092 desc.cmd_desc_mapi.its_dev = its_dev;
1093 desc.cmd_desc_mapi.lpinum = lpinum;
1094
1095 its_cmd_send(sc, &desc);
1096 }
1097
1098 static void
its_cmd_mapd(struct gic_v3_its_softc * sc,struct its_dev * its_dev,uint8_t valid)1099 its_cmd_mapd(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
1100 uint8_t valid)
1101 {
1102 struct its_cmd_desc desc;
1103
1104 desc.cmd_type = ITS_CMD_MAPD;
1105 desc.cmd_desc_mapd.its_dev = its_dev;
1106 desc.cmd_desc_mapd.valid = valid;
1107
1108 its_cmd_send(sc, &desc);
1109 }
1110
1111 static void
its_cmd_inv(struct gic_v3_its_softc * sc,struct its_dev * its_dev,uint32_t lpinum)1112 its_cmd_inv(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
1113 uint32_t lpinum)
1114 {
1115 struct its_cmd_desc desc;
1116
1117 desc.cmd_type = ITS_CMD_INV;
1118 desc.cmd_desc_inv.lpinum = lpinum - its_dev->lpis.lpi_base;
1119 desc.cmd_desc_inv.its_dev = its_dev;
1120
1121 its_cmd_send(sc, &desc);
1122 }
1123
1124 static void
its_cmd_invall(struct gic_v3_its_softc * sc,struct its_col * col)1125 its_cmd_invall(struct gic_v3_its_softc *sc, struct its_col *col)
1126 {
1127 struct its_cmd_desc desc;
1128
1129 desc.cmd_type = ITS_CMD_INVALL;
1130 desc.cmd_desc_invall.col = col;
1131
1132 its_cmd_send(sc, &desc);
1133 }
1134
1135 /*
1136 * Helper routines for commands processing.
1137 */
1138 static __inline boolean_t
its_cmd_queue_full(struct gic_v3_its_softc * sc)1139 its_cmd_queue_full(struct gic_v3_its_softc *sc)
1140 {
1141 size_t read_idx, write_idx;
1142
1143 write_idx = (size_t)(sc->its_cmdq_write - sc->its_cmdq_base);
1144 read_idx = gic_its_read(sc, 4, GITS_CREADR) / sizeof(struct its_cmd);
1145
1146 /*
1147 * The queue is full when the write offset points
1148 * at the command before the current read offset.
1149 */
1150 if (((write_idx + 1) % ITS_CMDQ_NENTRIES) == read_idx)
1151 return (TRUE);
1152
1153 return (FALSE);
1154 }
1155
1156 static __inline void
its_cmd_sync(struct gic_v3_its_softc * sc,struct its_cmd * cmd)1157 its_cmd_sync(struct gic_v3_its_softc *sc, struct its_cmd *cmd)
1158 {
1159
1160 if ((sc->its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1161 /* Clean D-cache under command. */
1162 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
1163 } else {
1164 /* DSB inner shareable, store */
1165 dsb(ishst);
1166 }
1167
1168 }
1169
1170 static struct its_cmd *
its_cmd_alloc_locked(struct gic_v3_its_softc * sc)1171 its_cmd_alloc_locked(struct gic_v3_its_softc *sc)
1172 {
1173 struct its_cmd *cmd;
1174 size_t us_left;
1175
1176 /*
1177 * XXX ARM64TODO: This is obviously a significant delay.
1178 * The reason for that is that currently the time frames for
1179 * the command to complete (and therefore free the descriptor)
1180 * are not known.
1181 */
1182 us_left = 1000000;
1183
1184 mtx_assert(&sc->its_spin_mtx, MA_OWNED);
1185 while (its_cmd_queue_full(sc)) {
1186 if (us_left-- == 0) {
1187 /* Timeout while waiting for free command */
1188 device_printf(sc->dev,
1189 "Timeout while waiting for free command\n");
1190 return (NULL);
1191 }
1192 DELAY(1);
1193 }
1194
1195 cmd = sc->its_cmdq_write;
1196 sc->its_cmdq_write++;
1197
1198 if (sc->its_cmdq_write == (sc->its_cmdq_base + ITS_CMDQ_NENTRIES)) {
1199 /* Wrap the queue */
1200 sc->its_cmdq_write = sc->its_cmdq_base;
1201 }
1202
1203 return (cmd);
1204 }
1205
1206 static uint64_t
its_cmd_prepare(struct its_cmd * cmd,struct its_cmd_desc * desc)1207 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1208 {
1209 uint64_t target;
1210 uint8_t cmd_type;
1211 u_int size;
1212 boolean_t error;
1213
1214 error = FALSE;
1215 cmd_type = desc->cmd_type;
1216 target = ITS_TARGET_NONE;
1217
1218 switch (cmd_type) {
1219 case ITS_CMD_SYNC: /* Wait for previous commands completion */
1220 target = desc->cmd_desc_sync.col->col_target;
1221 cmd_format_command(cmd, ITS_CMD_SYNC);
1222 cmd_format_target(cmd, target);
1223 break;
1224 case ITS_CMD_MAPD: /* Assign ITT to device */
1225 target = desc->cmd_desc_mapd.its_dev->col->col_target;
1226 cmd_format_command(cmd, ITS_CMD_MAPD);
1227 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1228 /*
1229 * Size describes number of bits to encode interrupt IDs
1230 * supported by the device minus one.
1231 * When V (valid) bit is zero, this field should be written
1232 * as zero.
1233 */
1234 if (desc->cmd_desc_mapd.valid != 0) {
1235 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1236 size = MAX(1, size) - 1;
1237 } else
1238 size = 0;
1239
1240 cmd_format_size(cmd, size);
1241 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1242 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1243 break;
1244 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */
1245 target = desc->cmd_desc_mapc.col->col_target;
1246 cmd_format_command(cmd, ITS_CMD_MAPC);
1247 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
1248 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
1249 cmd_format_target(cmd, target);
1250 break;
1251 case ITS_CMD_MAPVI:
1252 target = desc->cmd_desc_mapvi.its_dev->col->col_target;
1253 cmd_format_command(cmd, ITS_CMD_MAPVI);
1254 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
1255 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
1256 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
1257 cmd_format_col(cmd, desc->cmd_desc_mapvi.its_dev->col->col_id);
1258 break;
1259 case ITS_CMD_MAPI:
1260 target = desc->cmd_desc_mapi.its_dev->col->col_target;
1261 cmd_format_command(cmd, ITS_CMD_MAPI);
1262 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
1263 cmd_format_id(cmd, desc->cmd_desc_mapi.lpinum);
1264 cmd_format_col(cmd, desc->cmd_desc_mapi.its_dev->col->col_id);
1265 break;
1266 case ITS_CMD_INV:
1267 target = desc->cmd_desc_inv.its_dev->col->col_target;
1268 cmd_format_command(cmd, ITS_CMD_INV);
1269 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
1270 cmd_format_id(cmd, desc->cmd_desc_inv.lpinum);
1271 break;
1272 case ITS_CMD_INVALL:
1273 cmd_format_command(cmd, ITS_CMD_INVALL);
1274 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
1275 break;
1276 default:
1277 error = TRUE;
1278 break;
1279 }
1280
1281 if (!error)
1282 cmd_fix_endian(cmd);
1283
1284 return (target);
1285 }
1286
1287 static __inline uint64_t
its_cmd_cwriter_offset(struct gic_v3_its_softc * sc,struct its_cmd * cmd)1288 its_cmd_cwriter_offset(struct gic_v3_its_softc *sc, struct its_cmd *cmd)
1289 {
1290 uint64_t off;
1291
1292 off = (cmd - sc->its_cmdq_base) * sizeof(*cmd);
1293
1294 return (off);
1295 }
1296
1297 static void
its_cmd_wait_completion(struct gic_v3_its_softc * sc,struct its_cmd * cmd_first,struct its_cmd * cmd_last)1298 its_cmd_wait_completion(struct gic_v3_its_softc *sc, struct its_cmd *cmd_first,
1299 struct its_cmd *cmd_last)
1300 {
1301 uint64_t first, last, read;
1302 size_t us_left;
1303
1304 /*
1305 * XXX ARM64TODO: This is obviously a significant delay.
1306 * The reason for that is that currently the time frames for
1307 * the command to complete are not known.
1308 */
1309 us_left = 1000000;
1310
1311 first = its_cmd_cwriter_offset(sc, cmd_first);
1312 last = its_cmd_cwriter_offset(sc, cmd_last);
1313
1314 for (;;) {
1315 read = gic_its_read(sc, 8, GITS_CREADR);
1316 if (read < first || read >= last)
1317 break;
1318
1319 if (us_left-- == 0) {
1320 /* This means timeout */
1321 device_printf(sc->dev,
1322 "Timeout while waiting for CMD completion.\n");
1323 return;
1324 }
1325 DELAY(1);
1326 }
1327 }
1328
1329 static int
its_cmd_send(struct gic_v3_its_softc * sc,struct its_cmd_desc * desc)1330 its_cmd_send(struct gic_v3_its_softc *sc, struct its_cmd_desc *desc)
1331 {
1332 struct its_cmd *cmd, *cmd_sync, *cmd_write;
1333 struct its_col col_sync;
1334 struct its_cmd_desc desc_sync;
1335 uint64_t target, cwriter;
1336
1337 mtx_lock_spin(&sc->its_spin_mtx);
1338 cmd = its_cmd_alloc_locked(sc);
1339 if (cmd == NULL) {
1340 device_printf(sc->dev, "could not allocate ITS command\n");
1341 mtx_unlock_spin(&sc->its_spin_mtx);
1342 return (EBUSY);
1343 }
1344
1345 target = its_cmd_prepare(cmd, desc);
1346 its_cmd_sync(sc, cmd);
1347
1348 if (target != ITS_TARGET_NONE) {
1349 cmd_sync = its_cmd_alloc_locked(sc);
1350 if (cmd_sync == NULL)
1351 goto end;
1352 desc_sync.cmd_type = ITS_CMD_SYNC;
1353 col_sync.col_target = target;
1354 desc_sync.cmd_desc_sync.col = &col_sync;
1355 its_cmd_prepare(cmd_sync, &desc_sync);
1356 its_cmd_sync(sc, cmd_sync);
1357 }
1358 end:
1359 /* Update GITS_CWRITER */
1360 cwriter = its_cmd_cwriter_offset(sc, sc->its_cmdq_write);
1361 gic_its_write(sc, 8, GITS_CWRITER, cwriter);
1362 cmd_write = sc->its_cmdq_write;
1363 mtx_unlock_spin(&sc->its_spin_mtx);
1364
1365 its_cmd_wait_completion(sc, cmd, cmd_write);
1366
1367 return (0);
1368 }
1369
1370 static struct its_dev *
its_device_find_locked(struct gic_v3_its_softc * sc,device_t pci_dev)1371 its_device_find_locked(struct gic_v3_its_softc *sc, device_t pci_dev)
1372 {
1373 struct its_dev *its_dev;
1374
1375 mtx_assert(&sc->its_mtx, MA_OWNED);
1376 /* Find existing device if any */
1377 TAILQ_FOREACH(its_dev, &sc->its_dev_list, entry) {
1378 if (its_dev->pci_dev == pci_dev)
1379 return (its_dev);
1380 }
1381
1382 return (NULL);
1383 }
1384
1385 static struct its_dev *
its_device_alloc_locked(struct gic_v3_its_softc * sc,device_t pci_dev,u_int nvecs)1386 its_device_alloc_locked(struct gic_v3_its_softc *sc, device_t pci_dev,
1387 u_int nvecs)
1388 {
1389 struct its_dev *newdev;
1390 uint64_t typer;
1391 uint32_t devid;
1392 u_int cpuid;
1393 size_t esize;
1394
1395 mtx_assert(&sc->its_mtx, MA_OWNED);
1396 /* Find existing device if any */
1397 newdev = its_device_find_locked(sc, pci_dev);
1398 if (newdev != NULL)
1399 return (newdev);
1400
1401 devid = its_get_devid(pci_dev);
1402
1403 /* There was no previously created device. Create one now */
1404 newdev = malloc(sizeof(*newdev), M_GIC_V3_ITS, (M_NOWAIT | M_ZERO));
1405 if (newdev == NULL)
1406 return (NULL);
1407
1408 newdev->pci_dev = pci_dev;
1409 newdev->devid = devid;
1410
1411 if (lpi_alloc_chunk(sc, &newdev->lpis, nvecs) != 0) {
1412 free(newdev, M_GIC_V3_ITS);
1413 return (NULL);
1414 }
1415
1416 /* Get ITT entry size */
1417 typer = gic_its_read(sc, 8, GITS_TYPER);
1418 esize = GITS_TYPER_ITTES(typer);
1419 /*
1420 * Allocate ITT for this device.
1421 * PA has to be 256 B aligned. At least two entries for device.
1422 */
1423 newdev->itt = (vm_offset_t)contigmalloc(
1424 roundup2(roundup2(nvecs, 2) * esize, 0x100), M_GIC_V3_ITS,
1425 (M_NOWAIT | M_ZERO), 0, ~0UL, 0x100, 0);
1426 if (newdev->itt == 0) {
1427 lpi_free_chunk(sc, &newdev->lpis);
1428 free(newdev, M_GIC_V3_ITS);
1429 return (NULL);
1430 }
1431
1432 /*
1433 * XXX ARM64TODO: Currently all interrupts are going
1434 * to be bound to the CPU that performs the configuration.
1435 */
1436 cpuid = PCPU_GET(cpuid);
1437 newdev->col = sc->its_cols[cpuid];
1438
1439 TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry);
1440
1441 /* Map device to its ITT */
1442 its_cmd_mapd(sc, newdev, 1);
1443
1444 return (newdev);
1445 }
1446
1447 static __inline void
its_device_asign_lpi_locked(struct gic_v3_its_softc * sc,struct its_dev * its_dev,u_int * irq)1448 its_device_asign_lpi_locked(struct gic_v3_its_softc *sc,
1449 struct its_dev *its_dev, u_int *irq)
1450 {
1451
1452 mtx_assert(&sc->its_mtx, MA_OWNED);
1453 if (its_dev->lpis.lpi_free == 0) {
1454 panic("Requesting more LPIs than allocated for this device. "
1455 "LPI num: %u, free %u", its_dev->lpis.lpi_num,
1456 its_dev->lpis.lpi_free);
1457 }
1458 *irq = its_dev->lpis.lpi_base + (its_dev->lpis.lpi_num -
1459 its_dev->lpis.lpi_free);
1460 its_dev->lpis.lpi_free--;
1461 }
1462
1463 /*
1464 * ITS quirks.
1465 * Add vendor specific PCI devid function here.
1466 */
1467 static uint32_t
its_get_devid_thunder(device_t pci_dev)1468 its_get_devid_thunder(device_t pci_dev)
1469 {
1470 int bsf;
1471 int pem;
1472 uint32_t bus;
1473
1474 bus = pci_get_bus(pci_dev);
1475
1476 bsf = PCI_RID(pci_get_bus(pci_dev), pci_get_slot(pci_dev),
1477 pci_get_function(pci_dev));
1478
1479 /* Check if accessing internal PCIe (low bus numbers) */
1480 if (bus < GIC_V3_ITS_QUIRK_THUNDERX_PEM_BUS_OFFSET) {
1481 return ((pci_get_domain(pci_dev) << PCI_RID_DOMAIN_SHIFT) |
1482 bsf);
1483 /* PEM otherwise */
1484 } else {
1485 /* PEM (PCIe MAC/root complex) number is equal to domain */
1486 pem = pci_get_domain(pci_dev);
1487
1488 /*
1489 * Set appropriate device ID (passed by the HW along with
1490 * the transaction to memory) for different root complex
1491 * numbers using hard-coded domain portion for each group.
1492 */
1493 if (pem < 3)
1494 return ((0x1 << PCI_RID_DOMAIN_SHIFT) | bsf);
1495
1496 if (pem < 6)
1497 return ((0x3 << PCI_RID_DOMAIN_SHIFT) | bsf);
1498
1499 if (pem < 9)
1500 return ((0x9 << PCI_RID_DOMAIN_SHIFT) | bsf);
1501
1502 if (pem < 12)
1503 return ((0xB << PCI_RID_DOMAIN_SHIFT) | bsf);
1504 }
1505
1506 return (0);
1507 }
1508
1509 static uint32_t
its_get_devbits_thunder(device_t dev)1510 its_get_devbits_thunder(device_t dev)
1511 {
1512 uint32_t devid_bits;
1513
1514 /*
1515 * GITS_TYPER[17:13] of ThunderX reports that device IDs
1516 * are to be 21 bits in length.
1517 * The entry size of the ITS table can be read from GITS_BASERn[52:48]
1518 * and on ThunderX is supposed to be 8 bytes in length (for device
1519 * table). Finally the page size that is to be used by ITS to access
1520 * this table will be set to 64KB.
1521 *
1522 * This gives 0x200000 entries of size 0x8 bytes covered by 256 pages
1523 * each of which 64KB in size. The number of pages (minus 1) should
1524 * then be written to GITS_BASERn[7:0]. In that case this value would
1525 * be 0xFF but on ThunderX the maximum value that HW accepts is 0xFD.
1526 *
1527 * Set arbitrary number of device ID bits to 20 in order to limit
1528 * the number of entries in ITS device table to 0x100000 and hence
1529 * the table size to 8MB.
1530 */
1531 devid_bits = 20;
1532 if (bootverbose) {
1533 device_printf(dev,
1534 "Limiting number of Device ID bits implemented to %d\n",
1535 devid_bits);
1536 }
1537
1538 return (devid_bits);
1539 }
1540
1541 static __inline uint32_t
its_get_devbits_default(device_t dev)1542 its_get_devbits_default(device_t dev)
1543 {
1544 uint64_t gits_typer;
1545 struct gic_v3_its_softc *sc;
1546
1547 sc = device_get_softc(dev);
1548
1549 gits_typer = gic_its_read(sc, 8, GITS_TYPER);
1550
1551 return (GITS_TYPER_DEVB(gits_typer));
1552 }
1553
1554 static uint32_t
its_get_devbits(device_t dev)1555 its_get_devbits(device_t dev)
1556 {
1557 const struct its_quirks *quirk;
1558 size_t i;
1559
1560 for (i = 0; i < nitems(its_quirks); i++) {
1561 quirk = &its_quirks[i];
1562 if (CPU_MATCH_RAW(quirk->cpuid_mask, quirk->cpuid)) {
1563 if (quirk->devbits_func != NULL)
1564 return ((*quirk->devbits_func)(dev));
1565 }
1566 }
1567
1568 return (its_get_devbits_default(dev));
1569 }
1570
1571 static __inline uint32_t
its_get_devid_default(device_t pci_dev)1572 its_get_devid_default(device_t pci_dev)
1573 {
1574
1575 return (PCI_DEVID_GENERIC(pci_dev));
1576 }
1577
1578 static uint32_t
its_get_devid(device_t pci_dev)1579 its_get_devid(device_t pci_dev)
1580 {
1581 const struct its_quirks *quirk;
1582 size_t i;
1583
1584 for (i = 0; i < nitems(its_quirks); i++) {
1585 quirk = &its_quirks[i];
1586 if (CPU_MATCH_RAW(quirk->cpuid_mask, quirk->cpuid)) {
1587 if (quirk->devid_func != NULL)
1588 return ((*quirk->devid_func)(pci_dev));
1589 }
1590 }
1591
1592 return (its_get_devid_default(pci_dev));
1593 }
1594
1595 /*
1596 * Message signalled interrupts handling.
1597 */
1598
1599 /*
1600 * XXX ARM64TODO: Watch out for "irq" type.
1601 *
1602 * In theory GIC can handle up to (2^32 - 1) interrupt IDs whereas
1603 * we pass "irq" pointer of type integer. This is obviously wrong but
1604 * is determined by the way as PCI layer wants it to be done.
1605 */
1606 int
gic_v3_its_alloc_msix(device_t dev,device_t pci_dev,int * irq)1607 gic_v3_its_alloc_msix(device_t dev, device_t pci_dev, int *irq)
1608 {
1609 struct gic_v3_its_softc *sc;
1610 struct its_dev *its_dev;
1611 u_int nvecs;
1612
1613 sc = device_get_softc(dev);
1614
1615 mtx_lock(&sc->its_mtx);
1616 nvecs = PCI_MSIX_NUM(pci_dev);
1617
1618 /*
1619 * Allocate device as seen by ITS if not already available.
1620 * Notice that MSI-X interrupts are allocated on one-by-one basis.
1621 */
1622 its_dev = its_device_alloc_locked(sc, pci_dev, nvecs);
1623 if (its_dev == NULL) {
1624 mtx_unlock(&sc->its_mtx);
1625 return (ENOMEM);
1626 }
1627
1628 its_device_asign_lpi_locked(sc, its_dev, irq);
1629 mtx_unlock(&sc->its_mtx);
1630
1631 return (0);
1632 }
1633
1634 int
gic_v3_its_alloc_msi(device_t dev,device_t pci_dev,int count,int * irqs)1635 gic_v3_its_alloc_msi(device_t dev, device_t pci_dev, int count, int *irqs)
1636 {
1637 struct gic_v3_its_softc *sc;
1638 struct its_dev *its_dev;
1639
1640 sc = device_get_softc(dev);
1641
1642 /* Allocate device as seen by ITS if not already available. */
1643 mtx_lock(&sc->its_mtx);
1644 its_dev = its_device_alloc_locked(sc, pci_dev, count);
1645 if (its_dev == NULL) {
1646 mtx_unlock(&sc->its_mtx);
1647 return (ENOMEM);
1648 }
1649
1650 for (; count > 0; count--) {
1651 its_device_asign_lpi_locked(sc, its_dev, irqs);
1652 irqs++;
1653 }
1654 mtx_unlock(&sc->its_mtx);
1655
1656 return (0);
1657 }
1658
1659 int
gic_v3_its_map_msi(device_t dev,device_t pci_dev,int irq,uint64_t * addr,uint32_t * data)1660 gic_v3_its_map_msi(device_t dev, device_t pci_dev, int irq, uint64_t *addr,
1661 uint32_t *data)
1662 {
1663 struct gic_v3_its_softc *sc;
1664 bus_space_handle_t its_bsh;
1665 struct its_dev *its_dev;
1666 uint64_t its_pa;
1667 uint32_t id;
1668
1669 sc = device_get_softc(dev);
1670 /* Verify that this device is allocated and owns this LPI */
1671 mtx_lock(&sc->its_mtx);
1672 its_dev = its_device_find_locked(sc, pci_dev);
1673 mtx_unlock(&sc->its_mtx);
1674 if (its_dev == NULL)
1675 return (EINVAL);
1676
1677 id = irq - its_dev->lpis.lpi_base;
1678 lpi_map_to_device(sc, its_dev, id, irq);
1679
1680 its_bsh = rman_get_bushandle(&sc->its_res[0]);
1681 its_pa = vtophys(its_bsh);
1682
1683 *addr = (its_pa + GITS_TRANSLATER);
1684 *data = id;
1685
1686 return (0);
1687 }
1688