1 /*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28 #include <linux/power_supply.h>
29 #include <linux/kthread.h>
30 #include <linux/console.h>
31 #include <linux/slab.h>
32 #include <drm/drmP.h>
33 #include <drm/drm_crtc_helper.h>
34 #include <drm/drm_atomic_helper.h>
35 #include <drm/amdgpu_drm.h>
36 #include <linux/vgaarb.h>
37 #include <linux/vga_switcheroo.h>
38 #include "amdgpu.h"
39 #include "amdgpu_trace.h"
40 #include "amdgpu_i2c.h"
41 #include "atom.h"
42 #include "amdgpu_atombios.h"
43 #include "amdgpu_atomfirmware.h"
44 #include "amd_pcie.h"
45 #ifdef CONFIG_DRM_AMDGPU_SI
46 #include "si.h"
47 #endif
48 #ifdef CONFIG_DRM_AMDGPU_CIK
49 #include "cik.h"
50 #endif
51 #include "vi.h"
52 #include "soc15.h"
53 #include "bif/bif_4_1_d.h"
54 #include <linux/pci.h>
55 #include <linux/firmware.h>
56 #include "amdgpu_vf_error.h"
57
58 #include "amdgpu_amdkfd.h"
59 #include "amdgpu_pm.h"
60
61 MODULE_FIRMWARE("amdgpufw_vega10_gpu_info");
62 MODULE_FIRMWARE("amdgpufw_vega12_gpu_info");
63 MODULE_FIRMWARE("amdgpufw_raven_gpu_info");
64
65 #define AMDGPU_RESUME_MS 2000
66
67 static const char *amdgpu_asic_name[] = {
68 "TAHITI",
69 "PITCAIRN",
70 "VERDE",
71 "OLAND",
72 "HAINAN",
73 "BONAIRE",
74 "KAVERI",
75 "KABINI",
76 "HAWAII",
77 "MULLINS",
78 "TOPAZ",
79 "TONGA",
80 "FIJI",
81 "CARRIZO",
82 "STONEY",
83 "POLARIS10",
84 "POLARIS11",
85 "POLARIS12",
86 "VEGAM",
87 "VEGA10",
88 "VEGA12",
89 "VEGA20",
90 "RAVEN",
91 "LAST",
92 };
93
94 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
95
96 /**
97 * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control
98 *
99 * @dev: drm_device pointer
100 *
101 * Returns true if the device is a dGPU with HG/PX power control,
102 * otherwise return false.
103 */
amdgpu_device_is_px(struct drm_device * dev)104 bool amdgpu_device_is_px(struct drm_device *dev)
105 {
106 struct amdgpu_device *adev = dev->dev_private;
107
108 if (adev->flags & AMD_IS_PX)
109 return true;
110 return false;
111 }
112
113 /*
114 * MMIO register access helper functions.
115 */
116 /**
117 * amdgpu_mm_rreg - read a memory mapped IO register
118 *
119 * @adev: amdgpu_device pointer
120 * @reg: dword aligned register offset
121 * @acc_flags: access flags which require special behavior
122 *
123 * Returns the 32 bit value from the offset specified.
124 */
amdgpu_mm_rreg(struct amdgpu_device * adev,uint32_t reg,uint32_t acc_flags)125 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
126 uint32_t acc_flags)
127 {
128 uint32_t ret;
129
130 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
131 return amdgpu_virt_kiq_rreg(adev, reg);
132
133 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
134 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
135 else {
136 unsigned long flags;
137
138 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
139 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
140 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
141 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
142 }
143 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
144 return ret;
145 }
146
147 /*
148 * MMIO register read with bytes helper functions
149 * @offset:bytes offset from MMIO start
150 *
151 */
152
153 /**
154 * amdgpu_mm_rreg8 - read a memory mapped IO register
155 *
156 * @adev: amdgpu_device pointer
157 * @offset: byte aligned register offset
158 *
159 * Returns the 8 bit value from the offset specified.
160 */
amdgpu_mm_rreg8(struct amdgpu_device * adev,uint32_t offset)161 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
162 if (offset < adev->rmmio_size)
163 return (readb(adev->rmmio + offset));
164 BUG();
165 }
166
167 /*
168 * MMIO register write with bytes helper functions
169 * @offset:bytes offset from MMIO start
170 * @value: the value want to be written to the register
171 *
172 */
173 /**
174 * amdgpu_mm_wreg8 - read a memory mapped IO register
175 *
176 * @adev: amdgpu_device pointer
177 * @offset: byte aligned register offset
178 * @value: 8 bit value to write
179 *
180 * Writes the value specified to the offset specified.
181 */
amdgpu_mm_wreg8(struct amdgpu_device * adev,uint32_t offset,uint8_t value)182 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
183 if (offset < adev->rmmio_size)
184 writeb(value, adev->rmmio + offset);
185 else
186 BUG();
187 }
188
189 /**
190 * amdgpu_mm_wreg - write to a memory mapped IO register
191 *
192 * @adev: amdgpu_device pointer
193 * @reg: dword aligned register offset
194 * @v: 32 bit value to write to the register
195 * @acc_flags: access flags which require special behavior
196 *
197 * Writes the value specified to the offset specified.
198 */
amdgpu_mm_wreg(struct amdgpu_device * adev,uint32_t reg,uint32_t v,uint32_t acc_flags)199 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
200 uint32_t acc_flags)
201 {
202 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
203
204 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
205 adev->last_mm_index = v;
206 }
207
208 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
209 return amdgpu_virt_kiq_wreg(adev, reg, v);
210
211 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
212 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
213 else {
214 unsigned long flags;
215
216 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
217 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
218 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
219 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
220 }
221
222 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
223 udelay(500);
224 }
225 }
226
227 /**
228 * amdgpu_io_rreg - read an IO register
229 *
230 * @adev: amdgpu_device pointer
231 * @reg: dword aligned register offset
232 *
233 * Returns the 32 bit value from the offset specified.
234 */
amdgpu_io_rreg(struct amdgpu_device * adev,u32 reg)235 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
236 {
237 if ((reg * 4) < adev->rio_mem_size)
238 return bus_read_4(adev->rio_mem, (reg * 4));
239 else {
240 bus_write_4(adev->rio_mem, (mmMM_INDEX * 4), (reg * 4));
241 return bus_read_4(adev->rio_mem, (mmMM_DATA * 4));
242 }
243 }
244
245 /**
246 * amdgpu_io_wreg - write to an IO register
247 *
248 * @adev: amdgpu_device pointer
249 * @reg: dword aligned register offset
250 * @v: 32 bit value to write to the register
251 *
252 * Writes the value specified to the offset specified.
253 */
amdgpu_io_wreg(struct amdgpu_device * adev,u32 reg,u32 v)254 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
255 {
256 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
257 adev->last_mm_index = v;
258 }
259
260 if ((reg * 4) < adev->rio_mem_size)
261 bus_write_4(adev->rio_mem, (reg * 4), v);
262 else {
263 bus_write_4(adev->rio_mem, (mmMM_INDEX * 4), (reg * 4));
264 bus_write_4(adev->rio_mem, (mmMM_DATA * 4), v);
265 }
266
267 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
268 udelay(500);
269 }
270 }
271
272 /**
273 * amdgpu_mm_rdoorbell - read a doorbell dword
274 *
275 * @adev: amdgpu_device pointer
276 * @index: doorbell index
277 *
278 * Returns the value in the doorbell aperture at the
279 * requested doorbell index (CIK).
280 */
amdgpu_mm_rdoorbell(struct amdgpu_device * adev,u32 index)281 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
282 {
283 if (index < adev->doorbell.num_doorbells) {
284 return readl(adev->doorbell.ptr + index);
285 } else {
286 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
287 return 0;
288 }
289 }
290
291 /**
292 * amdgpu_mm_wdoorbell - write a doorbell dword
293 *
294 * @adev: amdgpu_device pointer
295 * @index: doorbell index
296 * @v: value to write
297 *
298 * Writes @v to the doorbell aperture at the
299 * requested doorbell index (CIK).
300 */
amdgpu_mm_wdoorbell(struct amdgpu_device * adev,u32 index,u32 v)301 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
302 {
303 if (index < adev->doorbell.num_doorbells) {
304 writel(v, adev->doorbell.ptr + index);
305 } else {
306 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
307 }
308 }
309
310 /**
311 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
312 *
313 * @adev: amdgpu_device pointer
314 * @index: doorbell index
315 *
316 * Returns the value in the doorbell aperture at the
317 * requested doorbell index (VEGA10+).
318 */
amdgpu_mm_rdoorbell64(struct amdgpu_device * adev,u32 index)319 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
320 {
321 if (index < adev->doorbell.num_doorbells) {
322 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
323 } else {
324 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
325 return 0;
326 }
327 }
328
329 /**
330 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
331 *
332 * @adev: amdgpu_device pointer
333 * @index: doorbell index
334 * @v: value to write
335 *
336 * Writes @v to the doorbell aperture at the
337 * requested doorbell index (VEGA10+).
338 */
amdgpu_mm_wdoorbell64(struct amdgpu_device * adev,u32 index,u64 v)339 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
340 {
341 if (index < adev->doorbell.num_doorbells) {
342 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
343 } else {
344 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
345 }
346 }
347
348 /**
349 * amdgpu_invalid_rreg - dummy reg read function
350 *
351 * @adev: amdgpu device pointer
352 * @reg: offset of register
353 *
354 * Dummy register read function. Used for register blocks
355 * that certain asics don't have (all asics).
356 * Returns the value in the register.
357 */
amdgpu_invalid_rreg(struct amdgpu_device * adev,uint32_t reg)358 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
359 {
360 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
361 BUG();
362 return 0;
363 }
364
365 /**
366 * amdgpu_invalid_wreg - dummy reg write function
367 *
368 * @adev: amdgpu device pointer
369 * @reg: offset of register
370 * @v: value to write to the register
371 *
372 * Dummy register read function. Used for register blocks
373 * that certain asics don't have (all asics).
374 */
amdgpu_invalid_wreg(struct amdgpu_device * adev,uint32_t reg,uint32_t v)375 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
376 {
377 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
378 reg, v);
379 BUG();
380 }
381
382 /**
383 * amdgpu_block_invalid_rreg - dummy reg read function
384 *
385 * @adev: amdgpu device pointer
386 * @block: offset of instance
387 * @reg: offset of register
388 *
389 * Dummy register read function. Used for register blocks
390 * that certain asics don't have (all asics).
391 * Returns the value in the register.
392 */
amdgpu_block_invalid_rreg(struct amdgpu_device * adev,uint32_t block,uint32_t reg)393 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
394 uint32_t block, uint32_t reg)
395 {
396 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
397 reg, block);
398 BUG();
399 return 0;
400 }
401
402 /**
403 * amdgpu_block_invalid_wreg - dummy reg write function
404 *
405 * @adev: amdgpu device pointer
406 * @block: offset of instance
407 * @reg: offset of register
408 * @v: value to write to the register
409 *
410 * Dummy register read function. Used for register blocks
411 * that certain asics don't have (all asics).
412 */
amdgpu_block_invalid_wreg(struct amdgpu_device * adev,uint32_t block,uint32_t reg,uint32_t v)413 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
414 uint32_t block,
415 uint32_t reg, uint32_t v)
416 {
417 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
418 reg, block, v);
419 BUG();
420 }
421
422 /**
423 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
424 *
425 * @adev: amdgpu device pointer
426 *
427 * Allocates a scratch page of VRAM for use by various things in the
428 * driver.
429 */
amdgpu_device_vram_scratch_init(struct amdgpu_device * adev)430 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
431 {
432 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
433 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
434 &adev->vram_scratch.robj,
435 &adev->vram_scratch.gpu_addr,
436 (void **)&adev->vram_scratch.ptr);
437 }
438
439 /**
440 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
441 *
442 * @adev: amdgpu device pointer
443 *
444 * Frees the VRAM scratch page.
445 */
amdgpu_device_vram_scratch_fini(struct amdgpu_device * adev)446 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
447 {
448 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
449 }
450
451 /**
452 * amdgpu_device_program_register_sequence - program an array of registers.
453 *
454 * @adev: amdgpu_device pointer
455 * @registers: pointer to the register array
456 * @array_size: size of the register array
457 *
458 * Programs an array or registers with and and or masks.
459 * This is a helper for setting golden registers.
460 */
amdgpu_device_program_register_sequence(struct amdgpu_device * adev,const u32 * registers,const u32 array_size)461 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
462 const u32 *registers,
463 const u32 array_size)
464 {
465 u32 tmp, reg, and_mask, or_mask;
466 int i;
467
468 if (array_size % 3)
469 return;
470
471 for (i = 0; i < array_size; i +=3) {
472 reg = registers[i + 0];
473 and_mask = registers[i + 1];
474 or_mask = registers[i + 2];
475
476 if (and_mask == 0xffffffff) {
477 tmp = or_mask;
478 } else {
479 tmp = RREG32(reg);
480 tmp &= ~and_mask;
481 tmp |= or_mask;
482 }
483 WREG32(reg, tmp);
484 }
485 }
486
487 /**
488 * amdgpu_device_pci_config_reset - reset the GPU
489 *
490 * @adev: amdgpu_device pointer
491 *
492 * Resets the GPU using the pci config reset sequence.
493 * Only applicable to asics prior to vega10.
494 */
amdgpu_device_pci_config_reset(struct amdgpu_device * adev)495 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
496 {
497 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
498 }
499
500 /*
501 * GPU doorbell aperture helpers function.
502 */
503 /**
504 * amdgpu_device_doorbell_init - Init doorbell driver information.
505 *
506 * @adev: amdgpu_device pointer
507 *
508 * Init doorbell driver information (CIK)
509 * Returns 0 on success, error on failure.
510 */
amdgpu_device_doorbell_init(struct amdgpu_device * adev)511 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
512 {
513 /* No doorbell on SI hardware generation */
514 if (adev->asic_type < CHIP_BONAIRE) {
515 adev->doorbell.base = 0;
516 adev->doorbell.size = 0;
517 adev->doorbell.num_doorbells = 0;
518 adev->doorbell.ptr = NULL;
519 return 0;
520 }
521
522 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
523 return -EINVAL;
524
525 /* doorbell bar mapping */
526 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
527 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
528
529 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
530 AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
531 if (adev->doorbell.num_doorbells == 0)
532 return -EINVAL;
533
534 adev->doorbell.ptr = ioremap(adev->doorbell.base,
535 adev->doorbell.num_doorbells *
536 sizeof(u32));
537 if (adev->doorbell.ptr == NULL)
538 return -ENOMEM;
539
540 return 0;
541 }
542
543 /**
544 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
545 *
546 * @adev: amdgpu_device pointer
547 *
548 * Tear down doorbell driver information (CIK)
549 */
amdgpu_device_doorbell_fini(struct amdgpu_device * adev)550 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
551 {
552 iounmap(adev->doorbell.ptr);
553 adev->doorbell.ptr = NULL;
554 }
555
556
557
558 /*
559 * amdgpu_device_wb_*()
560 * Writeback is the method by which the GPU updates special pages in memory
561 * with the status of certain GPU events (fences, ring pointers,etc.).
562 */
563
564 /**
565 * amdgpu_device_wb_fini - Disable Writeback and free memory
566 *
567 * @adev: amdgpu_device pointer
568 *
569 * Disables Writeback and frees the Writeback memory (all asics).
570 * Used at driver shutdown.
571 */
amdgpu_device_wb_fini(struct amdgpu_device * adev)572 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
573 {
574 if (adev->wb.wb_obj) {
575 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
576 (u64 *)&adev->wb.gpu_addr,
577 (void **)&adev->wb.wb);
578 adev->wb.wb_obj = NULL;
579 }
580 }
581
582 /**
583 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
584 *
585 * @adev: amdgpu_device pointer
586 *
587 * Initializes writeback and allocates writeback memory (all asics).
588 * Used at driver startup.
589 * Returns 0 on success or an -error on failure.
590 */
amdgpu_device_wb_init(struct amdgpu_device * adev)591 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
592 {
593 int r;
594
595 if (adev->wb.wb_obj == NULL) {
596 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
597 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
598 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
599 &adev->wb.wb_obj, (u64 *)&adev->wb.gpu_addr,
600 (void **)&adev->wb.wb);
601 if (r) {
602 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
603 return r;
604 }
605
606 adev->wb.num_wb = AMDGPU_MAX_WB;
607 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
608
609 /* clear wb memory */
610 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
611 }
612
613 return 0;
614 }
615
616 /**
617 * amdgpu_device_wb_get - Allocate a wb entry
618 *
619 * @adev: amdgpu_device pointer
620 * @wb: wb index
621 *
622 * Allocate a wb slot for use by the driver (all asics).
623 * Returns 0 on success or -EINVAL on failure.
624 */
amdgpu_device_wb_get(struct amdgpu_device * adev,u32 * wb)625 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
626 {
627 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
628
629 if (offset < adev->wb.num_wb) {
630 __set_bit(offset, adev->wb.used);
631 *wb = offset << 3; /* convert to dw offset */
632 return 0;
633 } else {
634 return -EINVAL;
635 }
636 }
637
638 /**
639 * amdgpu_device_wb_free - Free a wb entry
640 *
641 * @adev: amdgpu_device pointer
642 * @wb: wb index
643 *
644 * Free a wb slot allocated for use by the driver (all asics)
645 */
amdgpu_device_wb_free(struct amdgpu_device * adev,u32 wb)646 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
647 {
648 wb >>= 3;
649 if (wb < adev->wb.num_wb)
650 __clear_bit(wb, adev->wb.used);
651 }
652
653 /**
654 * amdgpu_device_vram_location - try to find VRAM location
655 *
656 * @adev: amdgpu device structure holding all necessary informations
657 * @mc: memory controller structure holding memory informations
658 * @base: base address at which to put VRAM
659 *
660 * Function will try to place VRAM at base address provided
661 * as parameter.
662 */
amdgpu_device_vram_location(struct amdgpu_device * adev,struct amdgpu_gmc * mc,u64 base)663 void amdgpu_device_vram_location(struct amdgpu_device *adev,
664 struct amdgpu_gmc *mc, u64 base)
665 {
666 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
667
668 mc->vram_start = base;
669 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
670 if (limit && limit < mc->real_vram_size)
671 mc->real_vram_size = limit;
672 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
673 mc->mc_vram_size >> 20, mc->vram_start,
674 mc->vram_end, mc->real_vram_size >> 20);
675 }
676
677 /**
678 * amdgpu_device_gart_location - try to find GART location
679 *
680 * @adev: amdgpu device structure holding all necessary informations
681 * @mc: memory controller structure holding memory informations
682 *
683 * Function will place try to place GART before or after VRAM.
684 *
685 * If GART size is bigger than space left then we ajust GART size.
686 * Thus function will never fails.
687 */
amdgpu_device_gart_location(struct amdgpu_device * adev,struct amdgpu_gmc * mc)688 void amdgpu_device_gart_location(struct amdgpu_device *adev,
689 struct amdgpu_gmc *mc)
690 {
691 u64 size_af, size_bf;
692
693 mc->gart_size += adev->pm.smu_prv_buffer_size;
694
695 size_af = adev->gmc.mc_mask - mc->vram_end;
696 size_bf = mc->vram_start;
697 if (size_bf > size_af) {
698 if (mc->gart_size > size_bf) {
699 dev_warn(adev->dev, "limiting GART\n");
700 mc->gart_size = size_bf;
701 }
702 mc->gart_start = 0;
703 } else {
704 if (mc->gart_size > size_af) {
705 dev_warn(adev->dev, "limiting GART\n");
706 mc->gart_size = size_af;
707 }
708 /* VCE doesn't like it when BOs cross a 4GB segment, so align
709 * the GART base on a 4GB boundary as well.
710 */
711 mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
712 }
713 mc->gart_end = mc->gart_start + mc->gart_size - 1;
714 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n",
715 mc->gart_size >> 20, mc->gart_start, mc->gart_end);
716 }
717
718 /**
719 * amdgpu_device_resize_fb_bar - try to resize FB BAR
720 *
721 * @adev: amdgpu_device pointer
722 *
723 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
724 * to fail, but if any of the BARs is not accessible after the size we abort
725 * driver loading by returning -ENODEV.
726 */
amdgpu_device_resize_fb_bar(struct amdgpu_device * adev)727 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
728 {
729 kprintf("amdgpu_device_resize_fb_bar: not implemented\n");
730 return 0;
731 #if 0
732 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
733 u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
734 struct pci_bus *root;
735 struct resource *res;
736 unsigned i;
737 u16 cmd;
738 int r;
739
740 /* Bypass for VF */
741 if (amdgpu_sriov_vf(adev))
742 return 0;
743
744 /* Check if the root BUS has 64bit memory resources */
745 root = adev->pdev->bus;
746 while (root->parent)
747 root = root->parent;
748
749 pci_bus_for_each_resource(root, res, i) {
750 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
751 res->start > 0x100000000ull)
752 break;
753 }
754
755 /* Trying to resize is pointless without a root hub window above 4GB */
756 if (!res)
757 return 0;
758
759 /* Disable memory decoding while we change the BAR addresses and size */
760 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
761 pci_write_config_word(adev->pdev, PCI_COMMAND,
762 cmd & ~PCI_COMMAND_MEMORY);
763
764 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
765 amdgpu_device_doorbell_fini(adev);
766 if (adev->asic_type >= CHIP_BONAIRE)
767 pci_release_resource(adev->pdev, 2);
768
769 pci_release_resource(adev->pdev, 0);
770
771 r = pci_resize_resource(adev->pdev, 0, rbar_size);
772 if (r == -ENOSPC)
773 DRM_INFO("Not enough PCI address space for a large BAR.");
774 else if (r && r != -ENOTSUPP)
775 DRM_ERROR("Problem resizing BAR0 (%d).", r);
776
777 pci_assign_unassigned_bus_resources(adev->pdev->bus);
778
779 /* When the doorbell or fb BAR isn't available we have no chance of
780 * using the device.
781 */
782 r = amdgpu_device_doorbell_init(adev);
783 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
784 return -ENODEV;
785
786 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
787
788 return 0;
789 #endif
790 }
791
792 /*
793 * GPU helpers function.
794 */
795 /**
796 * amdgpu_device_need_post - check if the hw need post or not
797 *
798 * @adev: amdgpu_device pointer
799 *
800 * Check if the asic has been initialized (all asics) at driver startup
801 * or post is needed if hw reset is performed.
802 * Returns true if need or false if not.
803 */
amdgpu_device_need_post(struct amdgpu_device * adev)804 bool amdgpu_device_need_post(struct amdgpu_device *adev)
805 {
806 uint32_t reg;
807
808 if (amdgpu_sriov_vf(adev))
809 return false;
810
811 if (amdgpu_passthrough(adev)) {
812 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
813 * some old smc fw still need driver do vPost otherwise gpu hang, while
814 * those smc fw version above 22.15 doesn't have this flaw, so we force
815 * vpost executed for smc version below 22.15
816 */
817 if (adev->asic_type == CHIP_FIJI) {
818 int err;
819 uint32_t fw_ver;
820 err = request_firmware(&adev->pm.fw, "amdgpufw_fiji_smc", adev->dev);
821 /* force vPost if error occured */
822 if (err)
823 return true;
824
825 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
826 if (fw_ver < 0x00160e00)
827 return true;
828 }
829 }
830
831 if (adev->has_hw_reset) {
832 adev->has_hw_reset = false;
833 return true;
834 }
835
836 /* bios scratch used on CIK+ */
837 if (adev->asic_type >= CHIP_BONAIRE)
838 return amdgpu_atombios_scratch_need_asic_init(adev);
839
840 /* check MEM_SIZE for older asics */
841 reg = amdgpu_asic_get_config_memsize(adev);
842
843 if ((reg != 0) && (reg != 0xffffffff))
844 return false;
845
846 return true;
847 }
848
849 #if 0
850 /* if we get transitioned to only one device, take VGA back */
851 /**
852 * amdgpu_device_vga_set_decode - enable/disable vga decode
853 *
854 * @cookie: amdgpu_device pointer
855 * @state: enable/disable vga decode
856 *
857 * Enable/disable vga decode (all asics).
858 * Returns VGA resource flags.
859 */
860 static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
861 {
862 struct amdgpu_device *adev = cookie;
863 amdgpu_asic_set_vga_state(adev, state);
864 if (state)
865 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
866 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
867 else
868 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
869 }
870 #endif
871
872 /**
873 * amdgpu_device_check_block_size - validate the vm block size
874 *
875 * @adev: amdgpu_device pointer
876 *
877 * Validates the vm block size specified via module parameter.
878 * The vm block size defines number of bits in page table versus page directory,
879 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
880 * page table and the remaining bits are in the page directory.
881 */
amdgpu_device_check_block_size(struct amdgpu_device * adev)882 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
883 {
884 /* defines number of bits in page table versus page directory,
885 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
886 * page table and the remaining bits are in the page directory */
887 if (amdgpu_vm_block_size == -1)
888 return;
889
890 if (amdgpu_vm_block_size < 9) {
891 dev_warn(adev->dev, "VM page table size (%d) too small\n",
892 amdgpu_vm_block_size);
893 amdgpu_vm_block_size = -1;
894 }
895 }
896
897 /**
898 * amdgpu_device_check_vm_size - validate the vm size
899 *
900 * @adev: amdgpu_device pointer
901 *
902 * Validates the vm size in GB specified via module parameter.
903 * The VM size is the size of the GPU virtual memory space in GB.
904 */
amdgpu_device_check_vm_size(struct amdgpu_device * adev)905 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
906 {
907 /* no need to check the default value */
908 if (amdgpu_vm_size == -1)
909 return;
910
911 if (amdgpu_vm_size < 1) {
912 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
913 amdgpu_vm_size);
914 amdgpu_vm_size = -1;
915 }
916 }
917
amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device * adev)918 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
919 {
920 struct sysinfo si;
921 bool is_os_64 = (sizeof(void *) == 8) ? true : false;
922 uint64_t total_memory;
923 uint64_t dram_size_seven_GB = 0x1B8000000;
924 uint64_t dram_size_three_GB = 0xB8000000;
925
926 if (amdgpu_smu_memory_pool_size == 0)
927 return;
928
929 if (!is_os_64) {
930 DRM_WARN("Not 64-bit OS, feature not supported\n");
931 goto def_value;
932 }
933 si_meminfo(&si);
934 total_memory = (uint64_t)si.totalram * si.mem_unit;
935
936 if ((amdgpu_smu_memory_pool_size == 1) ||
937 (amdgpu_smu_memory_pool_size == 2)) {
938 if (total_memory < dram_size_three_GB)
939 goto def_value1;
940 } else if ((amdgpu_smu_memory_pool_size == 4) ||
941 (amdgpu_smu_memory_pool_size == 8)) {
942 if (total_memory < dram_size_seven_GB)
943 goto def_value1;
944 } else {
945 DRM_WARN("Smu memory pool size not supported\n");
946 goto def_value;
947 }
948 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
949
950 return;
951
952 def_value1:
953 DRM_WARN("No enough system memory\n");
954 def_value:
955 adev->pm.smu_prv_buffer_size = 0;
956 }
957
958 /**
959 * amdgpu_device_check_arguments - validate module params
960 *
961 * @adev: amdgpu_device pointer
962 *
963 * Validates certain module parameters and updates
964 * the associated values used by the driver (all asics).
965 */
amdgpu_device_check_arguments(struct amdgpu_device * adev)966 static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
967 {
968 if (amdgpu_sched_jobs < 4) {
969 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
970 amdgpu_sched_jobs);
971 amdgpu_sched_jobs = 4;
972 } else if (!is_power_of_2(amdgpu_sched_jobs)){
973 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
974 amdgpu_sched_jobs);
975 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
976 }
977
978 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
979 /* gart size must be greater or equal to 32M */
980 dev_warn(adev->dev, "gart size (%d) too small\n",
981 amdgpu_gart_size);
982 amdgpu_gart_size = -1;
983 }
984
985 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
986 /* gtt size must be greater or equal to 32M */
987 dev_warn(adev->dev, "gtt size (%d) too small\n",
988 amdgpu_gtt_size);
989 amdgpu_gtt_size = -1;
990 }
991
992 /* valid range is between 4 and 9 inclusive */
993 if (amdgpu_vm_fragment_size != -1 &&
994 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
995 dev_warn(adev->dev, "valid range is between 4 and 9\n");
996 amdgpu_vm_fragment_size = -1;
997 }
998
999 amdgpu_device_check_smu_prv_buffer_size(adev);
1000
1001 amdgpu_device_check_vm_size(adev);
1002
1003 amdgpu_device_check_block_size(adev);
1004
1005 if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1006 !is_power_of_2(amdgpu_vram_page_split))) {
1007 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1008 amdgpu_vram_page_split);
1009 amdgpu_vram_page_split = 1024;
1010 }
1011
1012 if (amdgpu_lockup_timeout == 0) {
1013 dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n");
1014 amdgpu_lockup_timeout = 10000;
1015 }
1016
1017 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1018 }
1019
1020 /**
1021 * amdgpu_switcheroo_set_state - set switcheroo state
1022 *
1023 * @pdev: pci dev pointer
1024 * @state: vga_switcheroo state
1025 *
1026 * Callback for the switcheroo driver. Suspends or resumes the
1027 * the asics before or after it is powered up using ACPI methods.
1028 */
1029 #if 0
1030 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1031 {
1032 struct drm_device *dev = pci_get_drvdata(pdev);
1033
1034 if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1035 return;
1036
1037 if (state == VGA_SWITCHEROO_ON) {
1038 pr_info("amdgpu: switched on\n");
1039 /* don't suspend or resume card normally */
1040 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1041
1042 amdgpu_device_resume(dev, true, true);
1043
1044 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1045 drm_kms_helper_poll_enable(dev);
1046 } else {
1047 pr_info("amdgpu: switched off\n");
1048 drm_kms_helper_poll_disable(dev);
1049 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1050 amdgpu_device_suspend(dev, true, true);
1051 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1052 }
1053 }
1054
1055 /**
1056 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1057 *
1058 * @pdev: pci dev pointer
1059 *
1060 * Callback for the switcheroo driver. Check of the switcheroo
1061 * state can be changed.
1062 * Returns true if the state can be changed, false if not.
1063 */
1064 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1065 {
1066 struct drm_device *dev = pci_get_drvdata(pdev);
1067
1068 /*
1069 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1070 * locking inversion with the driver load path. And the access here is
1071 * completely racy anyway. So don't bother with locking for now.
1072 */
1073 return dev->open_count == 0;
1074 }
1075
1076 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1077 .set_gpu_state = amdgpu_switcheroo_set_state,
1078 .reprobe = NULL,
1079 .can_switch = amdgpu_switcheroo_can_switch,
1080 };
1081 #endif
1082
1083 /**
1084 * amdgpu_device_ip_set_clockgating_state - set the CG state
1085 *
1086 * @dev: amdgpu_device pointer
1087 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1088 * @state: clockgating state (gate or ungate)
1089 *
1090 * Sets the requested clockgating state for all instances of
1091 * the hardware IP specified.
1092 * Returns the error code from the last instance.
1093 */
amdgpu_device_ip_set_clockgating_state(void * dev,enum amd_ip_block_type block_type,enum amd_clockgating_state state)1094 int amdgpu_device_ip_set_clockgating_state(void *dev,
1095 enum amd_ip_block_type block_type,
1096 enum amd_clockgating_state state)
1097 {
1098 struct amdgpu_device *adev = dev;
1099 int i, r = 0;
1100
1101 for (i = 0; i < adev->num_ip_blocks; i++) {
1102 if (!adev->ip_blocks[i].status.valid)
1103 continue;
1104 if (adev->ip_blocks[i].version->type != block_type)
1105 continue;
1106 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1107 continue;
1108 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1109 (void *)adev, state);
1110 if (r)
1111 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1112 adev->ip_blocks[i].version->funcs->name, r);
1113 }
1114 return r;
1115 }
1116
1117 /**
1118 * amdgpu_device_ip_set_powergating_state - set the PG state
1119 *
1120 * @dev: amdgpu_device pointer
1121 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1122 * @state: powergating state (gate or ungate)
1123 *
1124 * Sets the requested powergating state for all instances of
1125 * the hardware IP specified.
1126 * Returns the error code from the last instance.
1127 */
amdgpu_device_ip_set_powergating_state(void * dev,enum amd_ip_block_type block_type,enum amd_powergating_state state)1128 int amdgpu_device_ip_set_powergating_state(void *dev,
1129 enum amd_ip_block_type block_type,
1130 enum amd_powergating_state state)
1131 {
1132 struct amdgpu_device *adev = dev;
1133 int i, r = 0;
1134
1135 for (i = 0; i < adev->num_ip_blocks; i++) {
1136 if (!adev->ip_blocks[i].status.valid)
1137 continue;
1138 if (adev->ip_blocks[i].version->type != block_type)
1139 continue;
1140 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1141 continue;
1142 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1143 (void *)adev, state);
1144 if (r)
1145 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1146 adev->ip_blocks[i].version->funcs->name, r);
1147 }
1148 return r;
1149 }
1150
1151 /**
1152 * amdgpu_device_ip_get_clockgating_state - get the CG state
1153 *
1154 * @adev: amdgpu_device pointer
1155 * @flags: clockgating feature flags
1156 *
1157 * Walks the list of IPs on the device and updates the clockgating
1158 * flags for each IP.
1159 * Updates @flags with the feature flags for each hardware IP where
1160 * clockgating is enabled.
1161 */
amdgpu_device_ip_get_clockgating_state(struct amdgpu_device * adev,u32 * flags)1162 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1163 u32 *flags)
1164 {
1165 int i;
1166
1167 for (i = 0; i < adev->num_ip_blocks; i++) {
1168 if (!adev->ip_blocks[i].status.valid)
1169 continue;
1170 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1171 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1172 }
1173 }
1174
1175 /**
1176 * amdgpu_device_ip_wait_for_idle - wait for idle
1177 *
1178 * @adev: amdgpu_device pointer
1179 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1180 *
1181 * Waits for the request hardware IP to be idle.
1182 * Returns 0 for success or a negative error code on failure.
1183 */
amdgpu_device_ip_wait_for_idle(struct amdgpu_device * adev,enum amd_ip_block_type block_type)1184 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1185 enum amd_ip_block_type block_type)
1186 {
1187 int i, r;
1188
1189 for (i = 0; i < adev->num_ip_blocks; i++) {
1190 if (!adev->ip_blocks[i].status.valid)
1191 continue;
1192 if (adev->ip_blocks[i].version->type == block_type) {
1193 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1194 if (r)
1195 return r;
1196 break;
1197 }
1198 }
1199 return 0;
1200
1201 }
1202
1203 /**
1204 * amdgpu_device_ip_is_idle - is the hardware IP idle
1205 *
1206 * @adev: amdgpu_device pointer
1207 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1208 *
1209 * Check if the hardware IP is idle or not.
1210 * Returns true if it the IP is idle, false if not.
1211 */
amdgpu_device_ip_is_idle(struct amdgpu_device * adev,enum amd_ip_block_type block_type)1212 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1213 enum amd_ip_block_type block_type)
1214 {
1215 int i;
1216
1217 for (i = 0; i < adev->num_ip_blocks; i++) {
1218 if (!adev->ip_blocks[i].status.valid)
1219 continue;
1220 if (adev->ip_blocks[i].version->type == block_type)
1221 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1222 }
1223 return true;
1224
1225 }
1226
1227 /**
1228 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1229 *
1230 * @adev: amdgpu_device pointer
1231 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1232 *
1233 * Returns a pointer to the hardware IP block structure
1234 * if it exists for the asic, otherwise NULL.
1235 */
1236 struct amdgpu_ip_block *
amdgpu_device_ip_get_ip_block(struct amdgpu_device * adev,enum amd_ip_block_type type)1237 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1238 enum amd_ip_block_type type)
1239 {
1240 int i;
1241
1242 for (i = 0; i < adev->num_ip_blocks; i++)
1243 if (adev->ip_blocks[i].version->type == type)
1244 return &adev->ip_blocks[i];
1245
1246 return NULL;
1247 }
1248
1249 /**
1250 * amdgpu_device_ip_block_version_cmp
1251 *
1252 * @adev: amdgpu_device pointer
1253 * @type: enum amd_ip_block_type
1254 * @major: major version
1255 * @minor: minor version
1256 *
1257 * return 0 if equal or greater
1258 * return 1 if smaller or the ip_block doesn't exist
1259 */
amdgpu_device_ip_block_version_cmp(struct amdgpu_device * adev,enum amd_ip_block_type type,u32 major,u32 minor)1260 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1261 enum amd_ip_block_type type,
1262 u32 major, u32 minor)
1263 {
1264 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1265
1266 if (ip_block && ((ip_block->version->major > major) ||
1267 ((ip_block->version->major == major) &&
1268 (ip_block->version->minor >= minor))))
1269 return 0;
1270
1271 return 1;
1272 }
1273
1274 /**
1275 * amdgpu_device_ip_block_add
1276 *
1277 * @adev: amdgpu_device pointer
1278 * @ip_block_version: pointer to the IP to add
1279 *
1280 * Adds the IP block driver information to the collection of IPs
1281 * on the asic.
1282 */
amdgpu_device_ip_block_add(struct amdgpu_device * adev,const struct amdgpu_ip_block_version * ip_block_version)1283 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1284 const struct amdgpu_ip_block_version *ip_block_version)
1285 {
1286 if (!ip_block_version)
1287 return -EINVAL;
1288
1289 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1290 ip_block_version->funcs->name);
1291
1292 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1293
1294 return 0;
1295 }
1296
1297 /**
1298 * amdgpu_device_enable_virtual_display - enable virtual display feature
1299 *
1300 * @adev: amdgpu_device pointer
1301 *
1302 * Enabled the virtual display feature if the user has enabled it via
1303 * the module parameter virtual_display. This feature provides a virtual
1304 * display hardware on headless boards or in virtualized environments.
1305 * This function parses and validates the configuration string specified by
1306 * the user and configues the virtual display configuration (number of
1307 * virtual connectors, crtcs, etc.) specified.
1308 */
amdgpu_device_enable_virtual_display(struct amdgpu_device * adev)1309 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1310 {
1311 adev->enable_virtual_display = false;
1312
1313 if (amdgpu_virtual_display) {
1314 struct drm_device *ddev = adev->ddev;
1315 const char *pci_address_name = pci_name(ddev->pdev);
1316 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1317
1318 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1319 pciaddstr_tmp = pciaddstr;
1320 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1321 pciaddname = strsep(&pciaddname_tmp, ",");
1322 if (!strcmp("all", pciaddname)
1323 || !strcmp(pci_address_name, pciaddname)) {
1324 long num_crtc;
1325 int res = -1;
1326
1327 adev->enable_virtual_display = true;
1328
1329 if (pciaddname_tmp)
1330 res = kstrtol(pciaddname_tmp, 10,
1331 &num_crtc);
1332
1333 if (!res) {
1334 if (num_crtc < 1)
1335 num_crtc = 1;
1336 if (num_crtc > 6)
1337 num_crtc = 6;
1338 adev->mode_info.num_crtc = num_crtc;
1339 } else {
1340 adev->mode_info.num_crtc = 1;
1341 }
1342 break;
1343 }
1344 }
1345
1346 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1347 amdgpu_virtual_display, pci_address_name,
1348 adev->enable_virtual_display, adev->mode_info.num_crtc);
1349
1350 kfree(pciaddstr);
1351 }
1352 }
1353
1354 /**
1355 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1356 *
1357 * @adev: amdgpu_device pointer
1358 *
1359 * Parses the asic configuration parameters specified in the gpu info
1360 * firmware and makes them availale to the driver for use in configuring
1361 * the asic.
1362 * Returns 0 on success, -EINVAL on failure.
1363 */
amdgpu_device_parse_gpu_info_fw(struct amdgpu_device * adev)1364 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1365 {
1366 const char *chip_name;
1367 char fw_name[30];
1368 int err;
1369 const struct gpu_info_firmware_header_v1_0 *hdr;
1370
1371 adev->firmware.gpu_info_fw = NULL;
1372
1373 switch (adev->asic_type) {
1374 case CHIP_TOPAZ:
1375 case CHIP_TONGA:
1376 case CHIP_FIJI:
1377 case CHIP_POLARIS10:
1378 case CHIP_POLARIS11:
1379 case CHIP_POLARIS12:
1380 case CHIP_VEGAM:
1381 case CHIP_CARRIZO:
1382 case CHIP_STONEY:
1383 #ifdef CONFIG_DRM_AMDGPU_SI
1384 case CHIP_VERDE:
1385 case CHIP_TAHITI:
1386 case CHIP_PITCAIRN:
1387 case CHIP_OLAND:
1388 case CHIP_HAINAN:
1389 #endif
1390 #ifdef CONFIG_DRM_AMDGPU_CIK
1391 case CHIP_BONAIRE:
1392 case CHIP_HAWAII:
1393 case CHIP_KAVERI:
1394 case CHIP_KABINI:
1395 case CHIP_MULLINS:
1396 #endif
1397 case CHIP_VEGA20:
1398 default:
1399 return 0;
1400 case CHIP_VEGA10:
1401 chip_name = "vega10";
1402 break;
1403 case CHIP_VEGA12:
1404 chip_name = "vega12";
1405 break;
1406 case CHIP_RAVEN:
1407 chip_name = "raven";
1408 break;
1409 }
1410
1411 snprintf(fw_name, sizeof(fw_name), "amdgpufw_%s_gpu_info", chip_name);
1412 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1413 if (err) {
1414 dev_err(adev->dev,
1415 "Failed to load gpu_info firmware \"%s\"\n",
1416 fw_name);
1417 goto out;
1418 }
1419 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1420 if (err) {
1421 dev_err(adev->dev,
1422 "Failed to validate gpu_info firmware \"%s\"\n",
1423 fw_name);
1424 goto out;
1425 }
1426
1427 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1428 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1429
1430 switch (hdr->version_major) {
1431 case 1:
1432 {
1433 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1434 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1435 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1436
1437 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1438 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1439 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1440 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1441 adev->gfx.config.max_texture_channel_caches =
1442 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1443 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1444 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1445 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1446 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1447 adev->gfx.config.double_offchip_lds_buf =
1448 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1449 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1450 adev->gfx.cu_info.max_waves_per_simd =
1451 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1452 adev->gfx.cu_info.max_scratch_slots_per_cu =
1453 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1454 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1455 break;
1456 }
1457 default:
1458 dev_err(adev->dev,
1459 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1460 err = -EINVAL;
1461 goto out;
1462 }
1463 out:
1464 return err;
1465 }
1466
1467 /**
1468 * amdgpu_device_ip_early_init - run early init for hardware IPs
1469 *
1470 * @adev: amdgpu_device pointer
1471 *
1472 * Early initialization pass for hardware IPs. The hardware IPs that make
1473 * up each asic are discovered each IP's early_init callback is run. This
1474 * is the first stage in initializing the asic.
1475 * Returns 0 on success, negative error code on failure.
1476 */
amdgpu_device_ip_early_init(struct amdgpu_device * adev)1477 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1478 {
1479 int i, r;
1480
1481 amdgpu_device_enable_virtual_display(adev);
1482
1483 switch (adev->asic_type) {
1484 case CHIP_TOPAZ:
1485 case CHIP_TONGA:
1486 case CHIP_FIJI:
1487 case CHIP_POLARIS10:
1488 case CHIP_POLARIS11:
1489 case CHIP_POLARIS12:
1490 case CHIP_VEGAM:
1491 case CHIP_CARRIZO:
1492 case CHIP_STONEY:
1493 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1494 adev->family = AMDGPU_FAMILY_CZ;
1495 else
1496 adev->family = AMDGPU_FAMILY_VI;
1497
1498 r = vi_set_ip_blocks(adev);
1499 if (r)
1500 return r;
1501 break;
1502 #ifdef CONFIG_DRM_AMDGPU_SI
1503 case CHIP_VERDE:
1504 case CHIP_TAHITI:
1505 case CHIP_PITCAIRN:
1506 case CHIP_OLAND:
1507 case CHIP_HAINAN:
1508 adev->family = AMDGPU_FAMILY_SI;
1509 r = si_set_ip_blocks(adev);
1510 if (r)
1511 return r;
1512 break;
1513 #endif
1514 #ifdef CONFIG_DRM_AMDGPU_CIK
1515 case CHIP_BONAIRE:
1516 case CHIP_HAWAII:
1517 case CHIP_KAVERI:
1518 case CHIP_KABINI:
1519 case CHIP_MULLINS:
1520 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1521 adev->family = AMDGPU_FAMILY_CI;
1522 else
1523 adev->family = AMDGPU_FAMILY_KV;
1524
1525 r = cik_set_ip_blocks(adev);
1526 if (r)
1527 return r;
1528 break;
1529 #endif
1530 case CHIP_VEGA10:
1531 case CHIP_VEGA12:
1532 case CHIP_VEGA20:
1533 case CHIP_RAVEN:
1534 if (adev->asic_type == CHIP_RAVEN)
1535 adev->family = AMDGPU_FAMILY_RV;
1536 else
1537 adev->family = AMDGPU_FAMILY_AI;
1538
1539 r = soc15_set_ip_blocks(adev);
1540 if (r)
1541 return r;
1542 break;
1543 default:
1544 /* FIXME: not supported yet */
1545 return -EINVAL;
1546 }
1547
1548 r = amdgpu_device_parse_gpu_info_fw(adev);
1549 if (r)
1550 return r;
1551
1552 amdgpu_amdkfd_device_probe(adev);
1553
1554 if (amdgpu_sriov_vf(adev)) {
1555 r = amdgpu_virt_request_full_gpu(adev, true);
1556 if (r)
1557 return -EAGAIN;
1558 }
1559
1560 adev->powerplay.pp_feature = amdgpu_pp_feature_mask;
1561
1562 for (i = 0; i < adev->num_ip_blocks; i++) {
1563 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1564 DRM_ERROR("disabled ip block: %d <%s>\n",
1565 i, adev->ip_blocks[i].version->funcs->name);
1566 adev->ip_blocks[i].status.valid = false;
1567 } else {
1568 if (adev->ip_blocks[i].version->funcs->early_init) {
1569 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1570 if (r == -ENOENT) {
1571 adev->ip_blocks[i].status.valid = false;
1572 } else if (r) {
1573 DRM_ERROR("early_init of IP block <%s> failed %d\n",
1574 adev->ip_blocks[i].version->funcs->name, r);
1575 return r;
1576 } else {
1577 adev->ip_blocks[i].status.valid = true;
1578 }
1579 } else {
1580 adev->ip_blocks[i].status.valid = true;
1581 }
1582 }
1583 }
1584
1585 adev->cg_flags &= amdgpu_cg_mask;
1586 adev->pg_flags &= amdgpu_pg_mask;
1587
1588 return 0;
1589 }
1590
1591 /**
1592 * amdgpu_device_ip_init - run init for hardware IPs
1593 *
1594 * @adev: amdgpu_device pointer
1595 *
1596 * Main initialization pass for hardware IPs. The list of all the hardware
1597 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
1598 * are run. sw_init initializes the software state associated with each IP
1599 * and hw_init initializes the hardware associated with each IP.
1600 * Returns 0 on success, negative error code on failure.
1601 */
amdgpu_device_ip_init(struct amdgpu_device * adev)1602 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
1603 {
1604 int i, r;
1605
1606 for (i = 0; i < adev->num_ip_blocks; i++) {
1607 if (!adev->ip_blocks[i].status.valid)
1608 continue;
1609 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1610 if (r) {
1611 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1612 adev->ip_blocks[i].version->funcs->name, r);
1613 return r;
1614 }
1615 adev->ip_blocks[i].status.sw = true;
1616
1617 /* need to do gmc hw init early so we can allocate gpu mem */
1618 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1619 r = amdgpu_device_vram_scratch_init(adev);
1620 if (r) {
1621 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1622 return r;
1623 }
1624 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1625 if (r) {
1626 DRM_ERROR("hw_init %d failed %d\n", i, r);
1627 return r;
1628 }
1629 r = amdgpu_device_wb_init(adev);
1630 if (r) {
1631 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
1632 return r;
1633 }
1634 adev->ip_blocks[i].status.hw = true;
1635
1636 /* right after GMC hw init, we create CSA */
1637 if (amdgpu_sriov_vf(adev)) {
1638 r = amdgpu_allocate_static_csa(adev);
1639 if (r) {
1640 DRM_ERROR("allocate CSA failed %d\n", r);
1641 return r;
1642 }
1643 }
1644 }
1645 }
1646
1647 for (i = 0; i < adev->num_ip_blocks; i++) {
1648 if (!adev->ip_blocks[i].status.sw)
1649 continue;
1650 if (adev->ip_blocks[i].status.hw)
1651 continue;
1652 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1653 if (r) {
1654 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1655 adev->ip_blocks[i].version->funcs->name, r);
1656 return r;
1657 }
1658 adev->ip_blocks[i].status.hw = true;
1659 }
1660
1661 amdgpu_amdkfd_device_init(adev);
1662
1663 if (amdgpu_sriov_vf(adev)) {
1664 amdgpu_virt_init_data_exchange(adev);
1665 amdgpu_virt_release_full_gpu(adev, true);
1666 }
1667
1668 return 0;
1669 }
1670
1671 /**
1672 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
1673 *
1674 * @adev: amdgpu_device pointer
1675 *
1676 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
1677 * this function before a GPU reset. If the value is retained after a
1678 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
1679 */
amdgpu_device_fill_reset_magic(struct amdgpu_device * adev)1680 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
1681 {
1682 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1683 }
1684
1685 /**
1686 * amdgpu_device_check_vram_lost - check if vram is valid
1687 *
1688 * @adev: amdgpu_device pointer
1689 *
1690 * Checks the reset magic value written to the gart pointer in VRAM.
1691 * The driver calls this after a GPU reset to see if the contents of
1692 * VRAM is lost or now.
1693 * returns true if vram is lost, false if not.
1694 */
amdgpu_device_check_vram_lost(struct amdgpu_device * adev)1695 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
1696 {
1697 return !!memcmp(adev->gart.ptr, adev->reset_magic,
1698 AMDGPU_RESET_MAGIC_NUM);
1699 }
1700
1701 /**
1702 * amdgpu_device_ip_late_set_cg_state - late init for clockgating
1703 *
1704 * @adev: amdgpu_device pointer
1705 *
1706 * Late initialization pass enabling clockgating for hardware IPs.
1707 * The list of all the hardware IPs that make up the asic is walked and the
1708 * set_clockgating_state callbacks are run. This stage is run late
1709 * in the init process.
1710 * Returns 0 on success, negative error code on failure.
1711 */
amdgpu_device_ip_late_set_cg_state(struct amdgpu_device * adev)1712 static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
1713 {
1714 int i = 0, r;
1715
1716 if (amdgpu_emu_mode == 1)
1717 return 0;
1718
1719 for (i = 0; i < adev->num_ip_blocks; i++) {
1720 if (!adev->ip_blocks[i].status.valid)
1721 continue;
1722 /* skip CG for VCE/UVD, it's handled specially */
1723 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1724 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1725 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1726 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1727 /* enable clockgating to save power */
1728 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1729 AMD_CG_STATE_GATE);
1730 if (r) {
1731 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1732 adev->ip_blocks[i].version->funcs->name, r);
1733 return r;
1734 }
1735 }
1736 }
1737
1738 return 0;
1739 }
1740
amdgpu_device_ip_late_set_pg_state(struct amdgpu_device * adev)1741 static int amdgpu_device_ip_late_set_pg_state(struct amdgpu_device *adev)
1742 {
1743 int i = 0, r;
1744
1745 if (amdgpu_emu_mode == 1)
1746 return 0;
1747
1748 for (i = 0; i < adev->num_ip_blocks; i++) {
1749 if (!adev->ip_blocks[i].status.valid)
1750 continue;
1751 /* skip CG for VCE/UVD, it's handled specially */
1752 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1753 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1754 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1755 adev->ip_blocks[i].version->funcs->set_powergating_state) {
1756 /* enable powergating to save power */
1757 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
1758 AMD_PG_STATE_GATE);
1759 if (r) {
1760 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
1761 adev->ip_blocks[i].version->funcs->name, r);
1762 return r;
1763 }
1764 }
1765 }
1766 return 0;
1767 }
1768
1769 /**
1770 * amdgpu_device_ip_late_init - run late init for hardware IPs
1771 *
1772 * @adev: amdgpu_device pointer
1773 *
1774 * Late initialization pass for hardware IPs. The list of all the hardware
1775 * IPs that make up the asic is walked and the late_init callbacks are run.
1776 * late_init covers any special initialization that an IP requires
1777 * after all of the have been initialized or something that needs to happen
1778 * late in the init process.
1779 * Returns 0 on success, negative error code on failure.
1780 */
amdgpu_device_ip_late_init(struct amdgpu_device * adev)1781 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
1782 {
1783 int i = 0, r;
1784
1785 for (i = 0; i < adev->num_ip_blocks; i++) {
1786 if (!adev->ip_blocks[i].status.valid)
1787 continue;
1788 if (adev->ip_blocks[i].version->funcs->late_init) {
1789 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1790 if (r) {
1791 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1792 adev->ip_blocks[i].version->funcs->name, r);
1793 return r;
1794 }
1795 adev->ip_blocks[i].status.late_initialized = true;
1796 }
1797 }
1798
1799 amdgpu_device_ip_late_set_cg_state(adev);
1800 amdgpu_device_ip_late_set_pg_state(adev);
1801
1802 queue_delayed_work(system_wq, &adev->late_init_work,
1803 msecs_to_jiffies(AMDGPU_RESUME_MS));
1804
1805 amdgpu_device_fill_reset_magic(adev);
1806
1807 return 0;
1808 }
1809
1810 /**
1811 * amdgpu_device_ip_fini - run fini for hardware IPs
1812 *
1813 * @adev: amdgpu_device pointer
1814 *
1815 * Main teardown pass for hardware IPs. The list of all the hardware
1816 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
1817 * are run. hw_fini tears down the hardware associated with each IP
1818 * and sw_fini tears down any software state associated with each IP.
1819 * Returns 0 on success, negative error code on failure.
1820 */
amdgpu_device_ip_fini(struct amdgpu_device * adev)1821 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
1822 {
1823 int i, r;
1824
1825 kprintf("amdgpu_device_ip_fini: 1\n");
1826 amdgpu_amdkfd_device_fini(adev);
1827 /* need to disable SMC first */
1828 for (i = 0; i < adev->num_ip_blocks; i++) {
1829 if (!adev->ip_blocks[i].status.hw)
1830 continue;
1831 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC &&
1832 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1833 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1834 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1835 AMD_CG_STATE_UNGATE);
1836 if (r) {
1837 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1838 adev->ip_blocks[i].version->funcs->name, r);
1839 return r;
1840 }
1841 if (adev->powerplay.pp_funcs->set_powergating_by_smu)
1842 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false);
1843 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1844 /* XXX handle errors */
1845 if (r) {
1846 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1847 adev->ip_blocks[i].version->funcs->name, r);
1848 }
1849 adev->ip_blocks[i].status.hw = false;
1850 break;
1851 }
1852 }
1853
1854 kprintf("amdgpu_device_ip_fini: 2\n");
1855 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1856 if (!adev->ip_blocks[i].status.hw)
1857 continue;
1858
1859 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1860 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1861 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1862 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1863 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1864 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1865 AMD_CG_STATE_UNGATE);
1866 if (r) {
1867 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1868 adev->ip_blocks[i].version->funcs->name, r);
1869 return r;
1870 }
1871 }
1872
1873 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1874 /* XXX handle errors */
1875 if (r) {
1876 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1877 adev->ip_blocks[i].version->funcs->name, r);
1878 }
1879
1880 adev->ip_blocks[i].status.hw = false;
1881 }
1882
1883 kprintf("amdgpu_device_ip_fini: 3\n");
1884 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1885 if (!adev->ip_blocks[i].status.sw)
1886 continue;
1887
1888 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1889 amdgpu_free_static_csa(adev);
1890 amdgpu_device_wb_fini(adev);
1891 amdgpu_device_vram_scratch_fini(adev);
1892 }
1893
1894 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1895 /* XXX handle errors */
1896 if (r) {
1897 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1898 adev->ip_blocks[i].version->funcs->name, r);
1899 }
1900 adev->ip_blocks[i].status.sw = false;
1901 adev->ip_blocks[i].status.valid = false;
1902 }
1903
1904 kprintf("amdgpu_device_ip_fini: 4\n");
1905 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1906 if (!adev->ip_blocks[i].status.late_initialized)
1907 continue;
1908 if (adev->ip_blocks[i].version->funcs->late_fini)
1909 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1910 adev->ip_blocks[i].status.late_initialized = false;
1911 }
1912
1913 kprintf("amdgpu_device_ip_fini: 5\n");
1914 if (amdgpu_sriov_vf(adev))
1915 if (amdgpu_virt_release_full_gpu(adev, false))
1916 DRM_ERROR("failed to release exclusive mode on fini\n");
1917
1918 return 0;
1919 }
1920
1921 /**
1922 * amdgpu_device_ip_late_init_func_handler - work handler for clockgating
1923 *
1924 * @work: work_struct
1925 *
1926 * Work handler for amdgpu_device_ip_late_set_cg_state. We put the
1927 * clockgating setup into a worker thread to speed up driver init and
1928 * resume from suspend.
1929 */
amdgpu_device_ip_late_init_func_handler(struct work_struct * work)1930 static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
1931 {
1932 struct amdgpu_device *adev =
1933 container_of(work, struct amdgpu_device, late_init_work.work);
1934 int r;
1935
1936 r = amdgpu_ib_ring_tests(adev);
1937 if (r)
1938 DRM_ERROR("ib ring test failed (%d).\n", r);
1939 }
1940
1941 /**
1942 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
1943 *
1944 * @adev: amdgpu_device pointer
1945 *
1946 * Main suspend function for hardware IPs. The list of all the hardware
1947 * IPs that make up the asic is walked, clockgating is disabled and the
1948 * suspend callbacks are run. suspend puts the hardware and software state
1949 * in each IP into a state suitable for suspend.
1950 * Returns 0 on success, negative error code on failure.
1951 */
amdgpu_device_ip_suspend_phase1(struct amdgpu_device * adev)1952 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
1953 {
1954 int i, r;
1955
1956 if (amdgpu_sriov_vf(adev))
1957 amdgpu_virt_request_full_gpu(adev, false);
1958
1959 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1960 if (!adev->ip_blocks[i].status.valid)
1961 continue;
1962 /* displays are handled separately */
1963 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
1964 /* ungate blocks so that suspend can properly shut them down */
1965 if (adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1966 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1967 AMD_CG_STATE_UNGATE);
1968 if (r) {
1969 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1970 adev->ip_blocks[i].version->funcs->name, r);
1971 }
1972 }
1973 /* XXX handle errors */
1974 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1975 /* XXX handle errors */
1976 if (r) {
1977 DRM_ERROR("suspend of IP block <%s> failed %d\n",
1978 adev->ip_blocks[i].version->funcs->name, r);
1979 }
1980 }
1981 }
1982
1983 if (amdgpu_sriov_vf(adev))
1984 amdgpu_virt_release_full_gpu(adev, false);
1985
1986 return 0;
1987 }
1988
1989 /**
1990 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
1991 *
1992 * @adev: amdgpu_device pointer
1993 *
1994 * Main suspend function for hardware IPs. The list of all the hardware
1995 * IPs that make up the asic is walked, clockgating is disabled and the
1996 * suspend callbacks are run. suspend puts the hardware and software state
1997 * in each IP into a state suitable for suspend.
1998 * Returns 0 on success, negative error code on failure.
1999 */
amdgpu_device_ip_suspend_phase2(struct amdgpu_device * adev)2000 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2001 {
2002 int i, r;
2003
2004 if (amdgpu_sriov_vf(adev))
2005 amdgpu_virt_request_full_gpu(adev, false);
2006
2007 /* ungate SMC block first */
2008 r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
2009 AMD_CG_STATE_UNGATE);
2010 if (r) {
2011 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n", r);
2012 }
2013
2014 /* call smu to disable gfx off feature first when suspend */
2015 if (adev->powerplay.pp_funcs->set_powergating_by_smu)
2016 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false);
2017
2018 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2019 if (!adev->ip_blocks[i].status.valid)
2020 continue;
2021 /* displays are handled in phase1 */
2022 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2023 continue;
2024 /* ungate blocks so that suspend can properly shut them down */
2025 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_SMC &&
2026 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2027 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2028 AMD_CG_STATE_UNGATE);
2029 if (r) {
2030 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
2031 adev->ip_blocks[i].version->funcs->name, r);
2032 }
2033 }
2034 /* XXX handle errors */
2035 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2036 /* XXX handle errors */
2037 if (r) {
2038 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2039 adev->ip_blocks[i].version->funcs->name, r);
2040 }
2041 }
2042
2043 if (amdgpu_sriov_vf(adev))
2044 amdgpu_virt_release_full_gpu(adev, false);
2045
2046 return 0;
2047 }
2048
2049 /**
2050 * amdgpu_device_ip_suspend - run suspend for hardware IPs
2051 *
2052 * @adev: amdgpu_device pointer
2053 *
2054 * Main suspend function for hardware IPs. The list of all the hardware
2055 * IPs that make up the asic is walked, clockgating is disabled and the
2056 * suspend callbacks are run. suspend puts the hardware and software state
2057 * in each IP into a state suitable for suspend.
2058 * Returns 0 on success, negative error code on failure.
2059 */
amdgpu_device_ip_suspend(struct amdgpu_device * adev)2060 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2061 {
2062 int r;
2063
2064 r = amdgpu_device_ip_suspend_phase1(adev);
2065 if (r)
2066 return r;
2067 r = amdgpu_device_ip_suspend_phase2(adev);
2068
2069 return r;
2070 }
2071
amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device * adev)2072 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2073 {
2074 int i, r;
2075
2076 static enum amd_ip_block_type ip_order[] = {
2077 AMD_IP_BLOCK_TYPE_GMC,
2078 AMD_IP_BLOCK_TYPE_COMMON,
2079 AMD_IP_BLOCK_TYPE_PSP,
2080 AMD_IP_BLOCK_TYPE_IH,
2081 };
2082
2083 for (i = 0; i < adev->num_ip_blocks; i++) {
2084 int j;
2085 struct amdgpu_ip_block *block;
2086
2087 for (j = 0; j < adev->num_ip_blocks; j++) {
2088 block = &adev->ip_blocks[j];
2089
2090 if (block->version->type != ip_order[i] ||
2091 !block->status.valid)
2092 continue;
2093
2094 r = block->version->funcs->hw_init(adev);
2095 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2096 if (r)
2097 return r;
2098 }
2099 }
2100
2101 return 0;
2102 }
2103
amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device * adev)2104 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2105 {
2106 int i, r;
2107
2108 static enum amd_ip_block_type ip_order[] = {
2109 AMD_IP_BLOCK_TYPE_SMC,
2110 AMD_IP_BLOCK_TYPE_DCE,
2111 AMD_IP_BLOCK_TYPE_GFX,
2112 AMD_IP_BLOCK_TYPE_SDMA,
2113 AMD_IP_BLOCK_TYPE_UVD,
2114 AMD_IP_BLOCK_TYPE_VCE
2115 };
2116
2117 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2118 int j;
2119 struct amdgpu_ip_block *block;
2120
2121 for (j = 0; j < adev->num_ip_blocks; j++) {
2122 block = &adev->ip_blocks[j];
2123
2124 if (block->version->type != ip_order[i] ||
2125 !block->status.valid)
2126 continue;
2127
2128 r = block->version->funcs->hw_init(adev);
2129 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2130 if (r)
2131 return r;
2132 }
2133 }
2134
2135 return 0;
2136 }
2137
2138 /**
2139 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2140 *
2141 * @adev: amdgpu_device pointer
2142 *
2143 * First resume function for hardware IPs. The list of all the hardware
2144 * IPs that make up the asic is walked and the resume callbacks are run for
2145 * COMMON, GMC, and IH. resume puts the hardware into a functional state
2146 * after a suspend and updates the software state as necessary. This
2147 * function is also used for restoring the GPU after a GPU reset.
2148 * Returns 0 on success, negative error code on failure.
2149 */
amdgpu_device_ip_resume_phase1(struct amdgpu_device * adev)2150 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
2151 {
2152 int i, r;
2153
2154 for (i = 0; i < adev->num_ip_blocks; i++) {
2155 if (!adev->ip_blocks[i].status.valid)
2156 continue;
2157 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2158 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2159 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2160 r = adev->ip_blocks[i].version->funcs->resume(adev);
2161 if (r) {
2162 DRM_ERROR("resume of IP block <%s> failed %d\n",
2163 adev->ip_blocks[i].version->funcs->name, r);
2164 return r;
2165 }
2166 }
2167 }
2168
2169 return 0;
2170 }
2171
2172 /**
2173 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2174 *
2175 * @adev: amdgpu_device pointer
2176 *
2177 * First resume function for hardware IPs. The list of all the hardware
2178 * IPs that make up the asic is walked and the resume callbacks are run for
2179 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
2180 * functional state after a suspend and updates the software state as
2181 * necessary. This function is also used for restoring the GPU after a GPU
2182 * reset.
2183 * Returns 0 on success, negative error code on failure.
2184 */
amdgpu_device_ip_resume_phase2(struct amdgpu_device * adev)2185 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
2186 {
2187 int i, r;
2188
2189 for (i = 0; i < adev->num_ip_blocks; i++) {
2190 if (!adev->ip_blocks[i].status.valid)
2191 continue;
2192 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2193 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2194 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)
2195 continue;
2196 r = adev->ip_blocks[i].version->funcs->resume(adev);
2197 if (r) {
2198 DRM_ERROR("resume of IP block <%s> failed %d\n",
2199 adev->ip_blocks[i].version->funcs->name, r);
2200 return r;
2201 }
2202 }
2203
2204 return 0;
2205 }
2206
2207 /**
2208 * amdgpu_device_ip_resume - run resume for hardware IPs
2209 *
2210 * @adev: amdgpu_device pointer
2211 *
2212 * Main resume function for hardware IPs. The hardware IPs
2213 * are split into two resume functions because they are
2214 * are also used in in recovering from a GPU reset and some additional
2215 * steps need to be take between them. In this case (S3/S4) they are
2216 * run sequentially.
2217 * Returns 0 on success, negative error code on failure.
2218 */
amdgpu_device_ip_resume(struct amdgpu_device * adev)2219 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2220 {
2221 int r;
2222
2223 r = amdgpu_device_ip_resume_phase1(adev);
2224 if (r)
2225 return r;
2226 r = amdgpu_device_ip_resume_phase2(adev);
2227
2228 return r;
2229 }
2230
2231 /**
2232 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2233 *
2234 * @adev: amdgpu_device pointer
2235 *
2236 * Query the VBIOS data tables to determine if the board supports SR-IOV.
2237 */
amdgpu_device_detect_sriov_bios(struct amdgpu_device * adev)2238 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2239 {
2240 if (amdgpu_sriov_vf(adev)) {
2241 if (adev->is_atom_fw) {
2242 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2243 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2244 } else {
2245 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2246 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2247 }
2248
2249 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
2250 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2251 }
2252 }
2253
2254 /**
2255 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
2256 *
2257 * @asic_type: AMD asic type
2258 *
2259 * Check if there is DC (new modesetting infrastructre) support for an asic.
2260 * returns true if DC has support, false if not.
2261 */
amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)2262 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
2263 {
2264 switch (asic_type) {
2265 #if defined(CONFIG_DRM_AMD_DC)
2266 case CHIP_BONAIRE:
2267 case CHIP_KAVERI:
2268 case CHIP_KABINI:
2269 case CHIP_MULLINS:
2270 /*
2271 * We have systems in the wild with these ASICs that require
2272 * LVDS and VGA support which is not supported with DC.
2273 *
2274 * Fallback to the non-DC driver here by default so as not to
2275 * cause regressions.
2276 */
2277 return amdgpu_dc > 0;
2278 case CHIP_HAWAII:
2279 case CHIP_CARRIZO:
2280 case CHIP_STONEY:
2281 case CHIP_POLARIS10:
2282 case CHIP_POLARIS11:
2283 case CHIP_POLARIS12:
2284 case CHIP_VEGAM:
2285 case CHIP_TONGA:
2286 case CHIP_FIJI:
2287 case CHIP_VEGA10:
2288 case CHIP_VEGA12:
2289 case CHIP_VEGA20:
2290 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
2291 case CHIP_RAVEN:
2292 #endif
2293 return amdgpu_dc != 0;
2294 #endif
2295 default:
2296 return false;
2297 }
2298 }
2299
2300 /**
2301 * amdgpu_device_has_dc_support - check if dc is supported
2302 *
2303 * @adev: amdgpu_device_pointer
2304 *
2305 * Returns true for supported, false for not supported
2306 */
amdgpu_device_has_dc_support(struct amdgpu_device * adev)2307 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
2308 {
2309 if (amdgpu_sriov_vf(adev))
2310 return false;
2311
2312 return amdgpu_device_asic_has_dc_support(adev->asic_type);
2313 }
2314
2315 /**
2316 * amdgpu_device_init - initialize the driver
2317 *
2318 * @adev: amdgpu_device pointer
2319 * @ddev: drm dev pointer
2320 * @pdev: pci dev pointer
2321 * @flags: driver flags
2322 *
2323 * Initializes the driver info and hw (all asics).
2324 * Returns 0 for success or an error on failure.
2325 * Called at driver startup.
2326 */
amdgpu_device_init(struct amdgpu_device * adev,struct drm_device * ddev,struct pci_dev * pdev,uint32_t flags)2327 int amdgpu_device_init(struct amdgpu_device *adev,
2328 struct drm_device *ddev,
2329 struct pci_dev *pdev,
2330 uint32_t flags)
2331 {
2332 int r, i;
2333 bool runtime = false;
2334 u32 max_MBps;
2335
2336 kprintf("amdgpu_device_init: start\n");
2337 adev->shutdown = false;
2338 adev->dev = &pdev->dev;
2339 adev->ddev = ddev;
2340 adev->pdev = pdev;
2341 adev->flags = flags;
2342 adev->asic_type = flags & AMD_ASIC_MASK;
2343 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2344 if (amdgpu_emu_mode == 1)
2345 adev->usec_timeout *= 2;
2346 adev->gmc.gart_size = 512 * 1024 * 1024;
2347 adev->accel_working = false;
2348 #ifdef __DragonFly__
2349 adev->fictitious_range_registered = false;
2350 #endif
2351 adev->num_rings = 0;
2352 adev->mman.buffer_funcs = NULL;
2353 adev->mman.buffer_funcs_ring = NULL;
2354 adev->vm_manager.vm_pte_funcs = NULL;
2355 adev->vm_manager.vm_pte_num_rings = 0;
2356 adev->gmc.gmc_funcs = NULL;
2357 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2358 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2359
2360 adev->smc_rreg = &amdgpu_invalid_rreg;
2361 adev->smc_wreg = &amdgpu_invalid_wreg;
2362 adev->pcie_rreg = &amdgpu_invalid_rreg;
2363 adev->pcie_wreg = &amdgpu_invalid_wreg;
2364 adev->pciep_rreg = &amdgpu_invalid_rreg;
2365 adev->pciep_wreg = &amdgpu_invalid_wreg;
2366 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2367 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2368 adev->didt_rreg = &amdgpu_invalid_rreg;
2369 adev->didt_wreg = &amdgpu_invalid_wreg;
2370 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2371 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
2372 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2373 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2374
2375 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2376 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2377 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
2378
2379 /* mutex initialization are all done here so we
2380 * can recall function without having locking issues */
2381 atomic_set(&adev->irq.ih.lock, 0);
2382 lockinit(&adev->firmware.mutex, "agfwm", 0, LK_CANRECURSE);
2383 lockinit(&adev->pm.mutex, "agpmm", 0, LK_CANRECURSE);
2384 lockinit(&adev->gfx.gpu_clock_mutex, "agggcm", 0, LK_CANRECURSE);
2385 lockinit(&adev->srbm_mutex, "agsm", 0, LK_CANRECURSE);
2386 lockinit(&adev->gfx.pipe_reserve_mutex, "aggprm", 0, LK_CANRECURSE);
2387 lockinit(&adev->grbm_idx_mutex, "aggim", 0, LK_CANRECURSE);
2388 lockinit(&adev->mn_lock, "agaml", 0, LK_CANRECURSE);
2389 lockinit(&adev->virt.vf_errors.lock, "agvfel", 0, LK_CANRECURSE);
2390 hash_init(adev->mn_hash);
2391 lockinit(&adev->lock_reset, "aglr", 0, LK_CANRECURSE);
2392
2393 amdgpu_device_check_arguments(adev);
2394
2395 lockinit(&adev->mmio_idx_lock, "agmil", 0, LK_CANRECURSE);
2396 lockinit(&adev->smc_idx_lock, "agsil", 0, LK_CANRECURSE);
2397 lockinit(&adev->pcie_idx_lock, "agpil", 0, LK_CANRECURSE);
2398 lockinit(&adev->uvd_ctx_idx_lock, "agucil", 0, LK_CANRECURSE);
2399 lockinit(&adev->didt_idx_lock, "agdil", 0, LK_CANRECURSE);
2400 lockinit(&adev->gc_cac_idx_lock, "aggcil", 0, LK_CANRECURSE);
2401 lockinit(&adev->se_cac_idx_lock, "agscil", 0, LK_CANRECURSE);
2402 lockinit(&adev->audio_endpt_idx_lock, "agaeil", 0, LK_CANRECURSE);
2403 spin_init(&adev->mm_stats.lock, "agammsl");
2404
2405 INIT_LIST_HEAD(&adev->shadow_list);
2406 lockinit(&adev->shadow_list_lock, "agasll", 0, LK_CANRECURSE);
2407
2408 INIT_LIST_HEAD(&adev->ring_lru_list);
2409 spin_init(&adev->ring_lru_list_lock, "agrlll");
2410
2411 INIT_DELAYED_WORK(&adev->late_init_work,
2412 amdgpu_device_ip_late_init_func_handler);
2413
2414 adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false;
2415
2416 kprintf("amdgpu_device_init: 1\n");
2417 /* Registers mapping */
2418 /* TODO: block userspace mapping of io register */
2419 if (adev->asic_type >= CHIP_BONAIRE) {
2420 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2421 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2422 } else {
2423 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2424 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2425 }
2426
2427 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2428 if (adev->rmmio == NULL) {
2429 return -ENOMEM;
2430 }
2431 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2432 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2433
2434 /* doorbell bar mapping */
2435 amdgpu_device_doorbell_init(adev);
2436
2437 kprintf("amdgpu_device_init: 2\n");
2438 /* io port mapping */
2439 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2440 uint32_t data;
2441
2442 kprintf("amdgpu_device_init: for loop %d\n", i);
2443 data = pci_read_config(adev->dev->bsddev, PCIR_BAR(i), 4);
2444 if (PCI_BAR_IO(data)) {
2445 adev->rio_rid = PCIR_BAR(i);
2446 adev->rio_mem = bus_alloc_resource_any(adev->dev->bsddev,
2447 SYS_RES_IOPORT, &adev->rio_rid,
2448 RF_ACTIVE | RF_SHAREABLE);
2449 adev->rio_mem_size = rman_get_size(adev->rio_mem);
2450 kprintf("amdgpu_device_init: rio_rid=%d\n", adev->rio_rid);
2451 kprintf("amdgpu_device_init: mem_size=%ld\n", adev->rio_mem_size);
2452 break;
2453 }
2454 }
2455 if (adev->rio_mem == NULL)
2456 DRM_INFO("PCI I/O BAR is not found.\n");
2457
2458 kprintf("amdgpu_device_init: 3\n");
2459 amdgpu_device_get_pcie_info(adev);
2460
2461 /* early init functions */
2462 r = amdgpu_device_ip_early_init(adev);
2463 if (r)
2464 return r;
2465
2466 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2467 /* this will fail for cards that aren't VGA class devices, just
2468 * ignore it */
2469 #if 0
2470 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
2471 #endif
2472
2473 if (amdgpu_device_is_px(ddev))
2474 runtime = true;
2475 #if 0
2476 if (!pci_is_thunderbolt_attached(adev->pdev))
2477 vga_switcheroo_register_client(adev->pdev,
2478 &amdgpu_switcheroo_ops, runtime);
2479 if (runtime)
2480 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2481 #endif
2482
2483 if (amdgpu_emu_mode == 1) {
2484 /* post the asic on emulation mode */
2485 emu_soc_asic_init(adev);
2486 goto fence_driver_init;
2487 }
2488 kprintf("amdgpu_device_init: 4\n");
2489 /* Read BIOS */
2490 if (!amdgpu_get_bios(adev)) {
2491 r = -EINVAL;
2492 goto failed;
2493 }
2494
2495 kprintf("amdgpu_device_init: 5\n");
2496 r = amdgpu_atombios_init(adev);
2497 if (r) {
2498 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2499 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2500 goto failed;
2501 }
2502
2503 /* detect if we are with an SRIOV vbios */
2504 amdgpu_device_detect_sriov_bios(adev);
2505
2506 /* Post card if necessary */
2507 if (amdgpu_device_need_post(adev)) {
2508 if (!adev->bios) {
2509 dev_err(adev->dev, "no vBIOS found\n");
2510 r = -EINVAL;
2511 goto failed;
2512 }
2513 DRM_INFO("GPU posting now...\n");
2514 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2515 if (r) {
2516 dev_err(adev->dev, "gpu post error!\n");
2517 goto failed;
2518 }
2519 }
2520
2521 kprintf("amdgpu_device_init: 6\n");
2522 if (adev->is_atom_fw) {
2523 kprintf("amdgpu_device_init: 6.1\n");
2524 /* Initialize clocks */
2525 r = amdgpu_atomfirmware_get_clock_info(adev);
2526 if (r) {
2527 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2528 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2529 goto failed;
2530 }
2531 } else {
2532 kprintf("amdgpu_device_init: 6.2\n");
2533 /* Initialize clocks */
2534 r = amdgpu_atombios_get_clock_info(adev);
2535 if (r) {
2536 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2537 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2538 goto failed;
2539 }
2540 kprintf("amdgpu_device_init: 6.3\n");
2541 /* init i2c buses */
2542 if (!amdgpu_device_has_dc_support(adev))
2543 amdgpu_atombios_i2c_init(adev);
2544 }
2545
2546 fence_driver_init:
2547 kprintf("amdgpu_device_init: 7\n");
2548 /* Fence driver */
2549 r = amdgpu_fence_driver_init(adev);
2550 if (r) {
2551 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2552 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2553 goto failed;
2554 }
2555
2556 kprintf("amdgpu_device_init: 8\n");
2557 /* init the mode config */
2558 drm_mode_config_init(adev->ddev);
2559
2560 r = amdgpu_device_ip_init(adev);
2561 if (r) {
2562 /* failed in exclusive mode due to timeout */
2563 if (amdgpu_sriov_vf(adev) &&
2564 !amdgpu_sriov_runtime(adev) &&
2565 amdgpu_virt_mmio_blocked(adev) &&
2566 !amdgpu_virt_wait_reset(adev)) {
2567 dev_err(adev->dev, "VF exclusive mode timeout\n");
2568 /* Don't send request since VF is inactive. */
2569 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2570 adev->virt.ops = NULL;
2571 r = -EAGAIN;
2572 goto failed;
2573 }
2574 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
2575 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2576 goto failed;
2577 }
2578 kprintf("amdgpu_device_init: 9\n");
2579 adev->accel_working = true;
2580
2581 amdgpu_vm_check_compute_bug(adev);
2582
2583 /* Initialize the buffer migration limit. */
2584 if (amdgpu_moverate >= 0)
2585 max_MBps = amdgpu_moverate;
2586 else
2587 max_MBps = 8; /* Allow 8 MB/s. */
2588 /* Get a log2 for easy divisions. */
2589 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2590
2591 r = amdgpu_ib_pool_init(adev);
2592 if (r) {
2593 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2594 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2595 goto failed;
2596 }
2597
2598 #ifdef __DragonFly__
2599 DRM_INFO("%s: Taking over the fictitious range 0x%lx-0x%llx\n", __func__,
2600 (uintmax_t)adev->gmc.aper_base,
2601 (uintmax_t)adev->gmc.aper_base + adev->gmc.visible_vram_size);
2602 r = vm_phys_fictitious_reg_range(
2603 adev->gmc.aper_base,
2604 adev->gmc.aper_base + adev->gmc.visible_vram_size,
2605 VM_MEMATTR_WRITE_COMBINING);
2606 if (r != 0) {
2607 DRM_ERROR("Failed to register fictitious range 0x%lx-0x%llx (%d).\n",
2608 (uintmax_t)adev->gmc.aper_base,
2609 (uintmax_t)adev->gmc.aper_base + adev->gmc.visible_vram_size, r);
2610 return (-r);
2611 }
2612 adev->fictitious_range_registered = true;
2613 #endif
2614
2615 kprintf("amdgpu_device_init: 10\n");
2616 amdgpu_fbdev_init(adev);
2617
2618 r = amdgpu_pm_sysfs_init(adev);
2619 if (r)
2620 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2621
2622 r = amdgpu_debugfs_gem_init(adev);
2623 if (r)
2624 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2625
2626 r = amdgpu_debugfs_regs_init(adev);
2627 if (r)
2628 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2629
2630 r = amdgpu_debugfs_firmware_init(adev);
2631 if (r)
2632 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2633
2634 r = amdgpu_debugfs_init(adev);
2635 if (r)
2636 DRM_ERROR("Creating debugfs files failed (%d).\n", r);
2637
2638 if ((amdgpu_testing & 1)) {
2639 if (adev->accel_working)
2640 amdgpu_test_moves(adev);
2641 else
2642 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2643 }
2644 if (amdgpu_benchmarking) {
2645 if (adev->accel_working)
2646 amdgpu_benchmark(adev, amdgpu_benchmarking);
2647 else
2648 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2649 }
2650
2651 /* enable clockgating, etc. after ib tests, etc. since some blocks require
2652 * explicit gating rather than handling it automatically.
2653 */
2654 r = amdgpu_device_ip_late_init(adev);
2655 if (r) {
2656 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
2657 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2658 goto failed;
2659 }
2660
2661 return 0;
2662
2663 failed:
2664 amdgpu_vf_error_trans_all(adev);
2665 #if 0
2666 if (runtime)
2667 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2668 #endif
2669
2670 return r;
2671 }
2672
2673 /**
2674 * amdgpu_device_fini - tear down the driver
2675 *
2676 * @adev: amdgpu_device pointer
2677 *
2678 * Tear down the driver info (all asics).
2679 * Called at driver shutdown.
2680 */
amdgpu_device_fini(struct amdgpu_device * adev)2681 void amdgpu_device_fini(struct amdgpu_device *adev)
2682 {
2683 int r;
2684
2685 DRM_INFO("amdgpu: finishing device.\n");
2686 adev->shutdown = true;
2687 /* disable all interrupts */
2688 amdgpu_irq_disable_all(adev);
2689 if (adev->mode_info.mode_config_initialized){
2690 if (!amdgpu_device_has_dc_support(adev))
2691 drm_crtc_force_disable_all(adev->ddev);
2692 else
2693 drm_atomic_helper_shutdown(adev->ddev);
2694 }
2695 amdgpu_ib_pool_fini(adev);
2696 amdgpu_fence_driver_fini(adev);
2697 amdgpu_pm_sysfs_fini(adev);
2698 amdgpu_fbdev_fini(adev);
2699 r = amdgpu_device_ip_fini(adev);
2700 if (adev->firmware.gpu_info_fw) {
2701 release_firmware(adev->firmware.gpu_info_fw);
2702 adev->firmware.gpu_info_fw = NULL;
2703 }
2704 adev->accel_working = false;
2705 #ifdef __DragonFly__
2706 adev->fictitious_range_registered = false;
2707 #endif
2708 cancel_delayed_work_sync(&adev->late_init_work);
2709 /* free i2c buses */
2710 if (!amdgpu_device_has_dc_support(adev))
2711 amdgpu_i2c_fini(adev);
2712
2713 if (amdgpu_emu_mode != 1)
2714 amdgpu_atombios_fini(adev);
2715
2716 kfree(adev->bios);
2717 adev->bios = NULL;
2718 #if 0
2719 if (!pci_is_thunderbolt_attached(adev->pdev))
2720 vga_switcheroo_unregister_client(adev->pdev);
2721 if (adev->flags & AMD_IS_PX)
2722 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2723 vga_client_register(adev->pdev, NULL, NULL, NULL);
2724 #endif
2725 if (adev->rio_mem)
2726 bus_release_resource(adev->dev->bsddev, SYS_RES_IOPORT, adev->rio_rid,
2727 adev->rio_mem);
2728 adev->rio_mem = NULL;
2729 iounmap(adev->rmmio);
2730 adev->rmmio = NULL;
2731 amdgpu_device_doorbell_fini(adev);
2732 amdgpu_debugfs_regs_cleanup(adev);
2733 }
2734
2735
2736 /*
2737 * Suspend & resume.
2738 */
2739 /**
2740 * amdgpu_device_suspend - initiate device suspend
2741 *
2742 * @dev: drm dev pointer
2743 * @suspend: suspend state
2744 * @fbcon : notify the fbdev of suspend
2745 *
2746 * Puts the hw in the suspend state (all asics).
2747 * Returns 0 for success or an error on failure.
2748 * Called at driver suspend.
2749 */
amdgpu_device_suspend(struct drm_device * dev,bool suspend,bool fbcon)2750 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2751 {
2752 struct amdgpu_device *adev;
2753 struct drm_crtc *crtc;
2754 struct drm_connector *connector;
2755 int r;
2756
2757 if (dev == NULL || dev->dev_private == NULL) {
2758 return -ENODEV;
2759 }
2760
2761 adev = dev->dev_private;
2762
2763 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2764 return 0;
2765
2766 drm_kms_helper_poll_disable(dev);
2767
2768 if (fbcon)
2769 amdgpu_fbdev_set_suspend(adev, 1);
2770
2771 if (!amdgpu_device_has_dc_support(adev)) {
2772 /* turn off display hw */
2773 drm_modeset_lock_all(dev);
2774 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2775 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2776 }
2777 drm_modeset_unlock_all(dev);
2778 /* unpin the front buffers and cursors */
2779 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2780 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2781 struct drm_framebuffer *fb = crtc->primary->fb;
2782 struct amdgpu_bo *robj;
2783
2784 if (amdgpu_crtc->cursor_bo) {
2785 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2786 r = amdgpu_bo_reserve(aobj, true);
2787 if (r == 0) {
2788 amdgpu_bo_unpin(aobj);
2789 amdgpu_bo_unreserve(aobj);
2790 }
2791 }
2792
2793 if (fb == NULL || fb->obj[0] == NULL) {
2794 continue;
2795 }
2796 robj = gem_to_amdgpu_bo(fb->obj[0]);
2797 /* don't unpin kernel fb objects */
2798 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2799 r = amdgpu_bo_reserve(robj, true);
2800 if (r == 0) {
2801 amdgpu_bo_unpin(robj);
2802 amdgpu_bo_unreserve(robj);
2803 }
2804 }
2805 }
2806 }
2807
2808 amdgpu_amdkfd_suspend(adev);
2809
2810 r = amdgpu_device_ip_suspend_phase1(adev);
2811
2812 /* evict vram memory */
2813 amdgpu_bo_evict_vram(adev);
2814
2815 amdgpu_fence_driver_suspend(adev);
2816
2817 r = amdgpu_device_ip_suspend_phase2(adev);
2818
2819 /* evict remaining vram memory
2820 * This second call to evict vram is to evict the gart page table
2821 * using the CPU.
2822 */
2823 amdgpu_bo_evict_vram(adev);
2824
2825 pci_save_state(device_get_parent(adev->dev->bsddev));
2826 if (suspend) {
2827 /* Shut down the device */
2828 #if 0
2829 pci_disable_device(dev->pdev);
2830 pci_set_power_state(dev->pdev, PCI_D3hot);
2831 #endif
2832 } else {
2833 r = amdgpu_asic_reset(adev);
2834 if (r)
2835 DRM_ERROR("amdgpu asic reset failed\n");
2836 }
2837
2838 return 0;
2839 }
2840
2841 /**
2842 * amdgpu_device_resume - initiate device resume
2843 *
2844 * @dev: drm dev pointer
2845 * @resume: resume state
2846 * @fbcon : notify the fbdev of resume
2847 *
2848 * Bring the hw back to operating state (all asics).
2849 * Returns 0 for success or an error on failure.
2850 * Called at driver resume.
2851 */
amdgpu_device_resume(struct drm_device * dev,bool resume,bool fbcon)2852 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2853 {
2854 struct drm_connector *connector;
2855 struct amdgpu_device *adev = dev->dev_private;
2856 struct drm_crtc *crtc;
2857 int r = 0;
2858
2859 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2860 return 0;
2861
2862 if (resume) {
2863 #if 0
2864 pci_set_power_state(dev->pdev, PCI_D0);
2865 pci_restore_state(dev->pdev);
2866 r = pci_enable_device(dev->pdev);
2867 if (r)
2868 return r;
2869 #endif
2870 }
2871
2872 /* post card */
2873 if (amdgpu_device_need_post(adev)) {
2874 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2875 if (r)
2876 DRM_ERROR("amdgpu asic init failed\n");
2877 }
2878
2879 r = amdgpu_device_ip_resume(adev);
2880 if (r) {
2881 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
2882 return r;
2883 }
2884 amdgpu_fence_driver_resume(adev);
2885
2886
2887 r = amdgpu_device_ip_late_init(adev);
2888 if (r)
2889 return r;
2890
2891 if (!amdgpu_device_has_dc_support(adev)) {
2892 /* pin cursors */
2893 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2894 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2895
2896 if (amdgpu_crtc->cursor_bo) {
2897 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2898 r = amdgpu_bo_reserve(aobj, true);
2899 if (r == 0) {
2900 r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
2901 if (r != 0)
2902 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2903 amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
2904 amdgpu_bo_unreserve(aobj);
2905 }
2906 }
2907 }
2908 }
2909 r = amdgpu_amdkfd_resume(adev);
2910 if (r)
2911 return r;
2912
2913 /* Make sure IB tests flushed */
2914 flush_delayed_work(&adev->late_init_work);
2915
2916 /* blat the mode back in */
2917 if (fbcon) {
2918 if (!amdgpu_device_has_dc_support(adev)) {
2919 /* pre DCE11 */
2920 drm_helper_resume_force_mode(dev);
2921
2922 /* turn on display hw */
2923 drm_modeset_lock_all(dev);
2924 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2925 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2926 }
2927 drm_modeset_unlock_all(dev);
2928 }
2929 amdgpu_fbdev_set_suspend(adev, 0);
2930 }
2931
2932 drm_kms_helper_poll_enable(dev);
2933
2934 /*
2935 * Most of the connector probing functions try to acquire runtime pm
2936 * refs to ensure that the GPU is powered on when connector polling is
2937 * performed. Since we're calling this from a runtime PM callback,
2938 * trying to acquire rpm refs will cause us to deadlock.
2939 *
2940 * Since we're guaranteed to be holding the rpm lock, it's safe to
2941 * temporarily disable the rpm helpers so this doesn't deadlock us.
2942 */
2943 #ifdef CONFIG_PM
2944 dev->dev->power.disable_depth++;
2945 #endif
2946 if (!amdgpu_device_has_dc_support(adev))
2947 drm_helper_hpd_irq_event(dev);
2948 else
2949 drm_kms_helper_hotplug_event(dev);
2950 #ifdef CONFIG_PM
2951 dev->dev->power.disable_depth--;
2952 #endif
2953 return 0;
2954 }
2955
2956 /**
2957 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
2958 *
2959 * @adev: amdgpu_device pointer
2960 *
2961 * The list of all the hardware IPs that make up the asic is walked and
2962 * the check_soft_reset callbacks are run. check_soft_reset determines
2963 * if the asic is still hung or not.
2964 * Returns true if any of the IPs are still in a hung state, false if not.
2965 */
amdgpu_device_ip_check_soft_reset(struct amdgpu_device * adev)2966 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
2967 {
2968 int i;
2969 bool asic_hang = false;
2970
2971 if (amdgpu_sriov_vf(adev))
2972 return true;
2973
2974 if (amdgpu_asic_need_full_reset(adev))
2975 return true;
2976
2977 for (i = 0; i < adev->num_ip_blocks; i++) {
2978 if (!adev->ip_blocks[i].status.valid)
2979 continue;
2980 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2981 adev->ip_blocks[i].status.hang =
2982 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2983 if (adev->ip_blocks[i].status.hang) {
2984 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2985 asic_hang = true;
2986 }
2987 }
2988 return asic_hang;
2989 }
2990
2991 /**
2992 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
2993 *
2994 * @adev: amdgpu_device pointer
2995 *
2996 * The list of all the hardware IPs that make up the asic is walked and the
2997 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
2998 * handles any IP specific hardware or software state changes that are
2999 * necessary for a soft reset to succeed.
3000 * Returns 0 on success, negative error code on failure.
3001 */
amdgpu_device_ip_pre_soft_reset(struct amdgpu_device * adev)3002 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
3003 {
3004 int i, r = 0;
3005
3006 for (i = 0; i < adev->num_ip_blocks; i++) {
3007 if (!adev->ip_blocks[i].status.valid)
3008 continue;
3009 if (adev->ip_blocks[i].status.hang &&
3010 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
3011 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
3012 if (r)
3013 return r;
3014 }
3015 }
3016
3017 return 0;
3018 }
3019
3020 /**
3021 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
3022 *
3023 * @adev: amdgpu_device pointer
3024 *
3025 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
3026 * reset is necessary to recover.
3027 * Returns true if a full asic reset is required, false if not.
3028 */
amdgpu_device_ip_need_full_reset(struct amdgpu_device * adev)3029 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
3030 {
3031 int i;
3032
3033 if (amdgpu_asic_need_full_reset(adev))
3034 return true;
3035
3036 for (i = 0; i < adev->num_ip_blocks; i++) {
3037 if (!adev->ip_blocks[i].status.valid)
3038 continue;
3039 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
3040 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
3041 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
3042 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
3043 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3044 if (adev->ip_blocks[i].status.hang) {
3045 DRM_INFO("Some block need full reset!\n");
3046 return true;
3047 }
3048 }
3049 }
3050 return false;
3051 }
3052
3053 /**
3054 * amdgpu_device_ip_soft_reset - do a soft reset
3055 *
3056 * @adev: amdgpu_device pointer
3057 *
3058 * The list of all the hardware IPs that make up the asic is walked and the
3059 * soft_reset callbacks are run if the block is hung. soft_reset handles any
3060 * IP specific hardware or software state changes that are necessary to soft
3061 * reset the IP.
3062 * Returns 0 on success, negative error code on failure.
3063 */
amdgpu_device_ip_soft_reset(struct amdgpu_device * adev)3064 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
3065 {
3066 int i, r = 0;
3067
3068 for (i = 0; i < adev->num_ip_blocks; i++) {
3069 if (!adev->ip_blocks[i].status.valid)
3070 continue;
3071 if (adev->ip_blocks[i].status.hang &&
3072 adev->ip_blocks[i].version->funcs->soft_reset) {
3073 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
3074 if (r)
3075 return r;
3076 }
3077 }
3078
3079 return 0;
3080 }
3081
3082 /**
3083 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
3084 *
3085 * @adev: amdgpu_device pointer
3086 *
3087 * The list of all the hardware IPs that make up the asic is walked and the
3088 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
3089 * handles any IP specific hardware or software state changes that are
3090 * necessary after the IP has been soft reset.
3091 * Returns 0 on success, negative error code on failure.
3092 */
amdgpu_device_ip_post_soft_reset(struct amdgpu_device * adev)3093 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
3094 {
3095 int i, r = 0;
3096
3097 for (i = 0; i < adev->num_ip_blocks; i++) {
3098 if (!adev->ip_blocks[i].status.valid)
3099 continue;
3100 if (adev->ip_blocks[i].status.hang &&
3101 adev->ip_blocks[i].version->funcs->post_soft_reset)
3102 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
3103 if (r)
3104 return r;
3105 }
3106
3107 return 0;
3108 }
3109
3110 /**
3111 * amdgpu_device_recover_vram_from_shadow - restore shadowed VRAM buffers
3112 *
3113 * @adev: amdgpu_device pointer
3114 * @ring: amdgpu_ring for the engine handling the buffer operations
3115 * @bo: amdgpu_bo buffer whose shadow is being restored
3116 * @fence: dma_fence associated with the operation
3117 *
3118 * Restores the VRAM buffer contents from the shadow in GTT. Used to
3119 * restore things like GPUVM page tables after a GPU reset where
3120 * the contents of VRAM might be lost.
3121 * Returns 0 on success, negative error code on failure.
3122 */
amdgpu_device_recover_vram_from_shadow(struct amdgpu_device * adev,struct amdgpu_ring * ring,struct amdgpu_bo * bo,struct dma_fence ** fence)3123 static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev,
3124 struct amdgpu_ring *ring,
3125 struct amdgpu_bo *bo,
3126 struct dma_fence **fence)
3127 {
3128 uint32_t domain;
3129 int r;
3130
3131 if (!bo->shadow)
3132 return 0;
3133
3134 r = amdgpu_bo_reserve(bo, true);
3135 if (r)
3136 return r;
3137 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
3138 /* if bo has been evicted, then no need to recover */
3139 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
3140 r = amdgpu_bo_validate(bo->shadow);
3141 if (r) {
3142 DRM_ERROR("bo validate failed!\n");
3143 goto err;
3144 }
3145
3146 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
3147 NULL, fence, true);
3148 if (r) {
3149 DRM_ERROR("recover page table failed!\n");
3150 goto err;
3151 }
3152 }
3153 err:
3154 amdgpu_bo_unreserve(bo);
3155 return r;
3156 }
3157
3158 /**
3159 * amdgpu_device_handle_vram_lost - Handle the loss of VRAM contents
3160 *
3161 * @adev: amdgpu_device pointer
3162 *
3163 * Restores the contents of VRAM buffers from the shadows in GTT. Used to
3164 * restore things like GPUVM page tables after a GPU reset where
3165 * the contents of VRAM might be lost.
3166 * Returns 0 on success, 1 on failure.
3167 */
amdgpu_device_handle_vram_lost(struct amdgpu_device * adev)3168 static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev)
3169 {
3170 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
3171 struct amdgpu_bo *bo, *tmp;
3172 struct dma_fence *fence = NULL, *next = NULL;
3173 long r = 1;
3174 int i = 0;
3175 long tmo;
3176
3177 if (amdgpu_sriov_runtime(adev))
3178 tmo = msecs_to_jiffies(8000);
3179 else
3180 tmo = msecs_to_jiffies(100);
3181
3182 DRM_INFO("recover vram bo from shadow start\n");
3183 mutex_lock(&adev->shadow_list_lock);
3184 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
3185 next = NULL;
3186 amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next);
3187 if (fence) {
3188 r = dma_fence_wait_timeout(fence, false, tmo);
3189 if (r == 0)
3190 pr_err("wait fence %p[%d] timeout\n", fence, i);
3191 else if (r < 0)
3192 pr_err("wait fence %p[%d] interrupted\n", fence, i);
3193 if (r < 1) {
3194 dma_fence_put(fence);
3195 fence = next;
3196 break;
3197 }
3198 i++;
3199 }
3200
3201 dma_fence_put(fence);
3202 fence = next;
3203 }
3204 mutex_unlock(&adev->shadow_list_lock);
3205
3206 if (fence) {
3207 r = dma_fence_wait_timeout(fence, false, tmo);
3208 if (r == 0)
3209 pr_err("wait fence %p[%d] timeout\n", fence, i);
3210 else if (r < 0)
3211 pr_err("wait fence %p[%d] interrupted\n", fence, i);
3212
3213 }
3214 dma_fence_put(fence);
3215
3216 if (r > 0)
3217 DRM_INFO("recover vram bo from shadow done\n");
3218 else
3219 DRM_ERROR("recover vram bo from shadow failed\n");
3220
3221 return (r > 0) ? 0 : 1;
3222 }
3223
3224 /**
3225 * amdgpu_device_reset - reset ASIC/GPU for bare-metal or passthrough
3226 *
3227 * @adev: amdgpu device pointer
3228 *
3229 * attempt to do soft-reset or full-reset and reinitialize Asic
3230 * return 0 means succeeded otherwise failed
3231 */
amdgpu_device_reset(struct amdgpu_device * adev)3232 static int amdgpu_device_reset(struct amdgpu_device *adev)
3233 {
3234 bool need_full_reset, vram_lost = 0;
3235 int r;
3236
3237 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
3238
3239 if (!need_full_reset) {
3240 amdgpu_device_ip_pre_soft_reset(adev);
3241 r = amdgpu_device_ip_soft_reset(adev);
3242 amdgpu_device_ip_post_soft_reset(adev);
3243 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
3244 DRM_INFO("soft reset failed, will fallback to full reset!\n");
3245 need_full_reset = true;
3246 }
3247 }
3248
3249 if (need_full_reset) {
3250 r = amdgpu_device_ip_suspend(adev);
3251
3252 retry:
3253 r = amdgpu_asic_reset(adev);
3254 /* post card */
3255 amdgpu_atom_asic_init(adev->mode_info.atom_context);
3256
3257 if (!r) {
3258 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
3259 r = amdgpu_device_ip_resume_phase1(adev);
3260 if (r)
3261 goto out;
3262
3263 vram_lost = amdgpu_device_check_vram_lost(adev);
3264 if (vram_lost) {
3265 DRM_ERROR("VRAM is lost!\n");
3266 atomic_inc(&adev->vram_lost_counter);
3267 }
3268
3269 r = amdgpu_gtt_mgr_recover(
3270 &adev->mman.bdev.man[TTM_PL_TT]);
3271 if (r)
3272 goto out;
3273
3274 r = amdgpu_device_ip_resume_phase2(adev);
3275 if (r)
3276 goto out;
3277
3278 if (vram_lost)
3279 amdgpu_device_fill_reset_magic(adev);
3280 }
3281 }
3282
3283 out:
3284 if (!r) {
3285 amdgpu_irq_gpu_reset_resume_helper(adev);
3286 r = amdgpu_ib_ring_tests(adev);
3287 if (r) {
3288 dev_err(adev->dev, "ib ring test failed (%d).\n", r);
3289 r = amdgpu_device_ip_suspend(adev);
3290 need_full_reset = true;
3291 goto retry;
3292 }
3293 }
3294
3295 if (!r && ((need_full_reset && !(adev->flags & AMD_IS_APU)) || vram_lost))
3296 r = amdgpu_device_handle_vram_lost(adev);
3297
3298 return r;
3299 }
3300
3301 /**
3302 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
3303 *
3304 * @adev: amdgpu device pointer
3305 * @from_hypervisor: request from hypervisor
3306 *
3307 * do VF FLR and reinitialize Asic
3308 * return 0 means succeeded otherwise failed
3309 */
amdgpu_device_reset_sriov(struct amdgpu_device * adev,bool from_hypervisor)3310 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
3311 bool from_hypervisor)
3312 {
3313 int r;
3314
3315 if (from_hypervisor)
3316 r = amdgpu_virt_request_full_gpu(adev, true);
3317 else
3318 r = amdgpu_virt_reset_gpu(adev);
3319 if (r)
3320 return r;
3321
3322 /* Resume IP prior to SMC */
3323 r = amdgpu_device_ip_reinit_early_sriov(adev);
3324 if (r)
3325 goto error;
3326
3327 /* we need recover gart prior to run SMC/CP/SDMA resume */
3328 amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
3329
3330 /* now we are okay to resume SMC/CP/SDMA */
3331 r = amdgpu_device_ip_reinit_late_sriov(adev);
3332 if (r)
3333 goto error;
3334
3335 amdgpu_irq_gpu_reset_resume_helper(adev);
3336 r = amdgpu_ib_ring_tests(adev);
3337
3338 error:
3339 amdgpu_virt_init_data_exchange(adev);
3340 amdgpu_virt_release_full_gpu(adev, true);
3341 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
3342 atomic_inc(&adev->vram_lost_counter);
3343 r = amdgpu_device_handle_vram_lost(adev);
3344 }
3345
3346 return r;
3347 }
3348
3349 /**
3350 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
3351 *
3352 * @adev: amdgpu device pointer
3353 * @job: which job trigger hang
3354 * @force: forces reset regardless of amdgpu_gpu_recovery
3355 *
3356 * Attempt to reset the GPU if it has hung (all asics).
3357 * Returns 0 for success or an error on failure.
3358 */
amdgpu_device_gpu_recover(struct amdgpu_device * adev,struct amdgpu_job * job,bool force)3359 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
3360 struct amdgpu_job *job, bool force)
3361 {
3362 int i, r, resched;
3363
3364 if (!force && !amdgpu_device_ip_check_soft_reset(adev)) {
3365 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
3366 return 0;
3367 }
3368
3369 if (!force && (amdgpu_gpu_recovery == 0 ||
3370 (amdgpu_gpu_recovery == -1 && !amdgpu_sriov_vf(adev)))) {
3371 DRM_INFO("GPU recovery disabled.\n");
3372 return 0;
3373 }
3374
3375 dev_info(adev->dev, "GPU reset begin!\n");
3376
3377 mutex_lock(&adev->lock_reset);
3378 atomic_inc(&adev->gpu_reset_counter);
3379 adev->in_gpu_reset = 1;
3380
3381 /* Block kfd */
3382 amdgpu_amdkfd_pre_reset(adev);
3383
3384 /* block TTM */
3385 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3386
3387 /* block all schedulers and reset given job's ring */
3388 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3389 struct amdgpu_ring *ring = adev->rings[i];
3390
3391 if (!ring || !ring->sched.thread)
3392 continue;
3393
3394 kthread_park(ring->sched.thread);
3395
3396 if (job && job->base.sched == &ring->sched)
3397 continue;
3398
3399 drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
3400
3401 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
3402 amdgpu_fence_driver_force_completion(ring);
3403 }
3404
3405 if (amdgpu_sriov_vf(adev))
3406 r = amdgpu_device_reset_sriov(adev, job ? false : true);
3407 else
3408 r = amdgpu_device_reset(adev);
3409
3410 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3411 struct amdgpu_ring *ring = adev->rings[i];
3412
3413 if (!ring || !ring->sched.thread)
3414 continue;
3415
3416 /* only need recovery sched of the given job's ring
3417 * or all rings (in the case @job is NULL)
3418 * after above amdgpu_reset accomplished
3419 */
3420 if ((!job || job->base.sched == &ring->sched) && !r)
3421 drm_sched_job_recovery(&ring->sched);
3422
3423 kthread_unpark(ring->sched.thread);
3424 }
3425
3426 if (!amdgpu_device_has_dc_support(adev)) {
3427 drm_helper_resume_force_mode(adev->ddev);
3428 }
3429
3430 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
3431
3432 if (r) {
3433 /* bad news, how to tell it to userspace ? */
3434 dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
3435 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
3436 } else {
3437 dev_info(adev->dev, "GPU reset(%d) succeeded!\n",atomic_read(&adev->gpu_reset_counter));
3438 }
3439
3440 /*unlock kfd */
3441 amdgpu_amdkfd_post_reset(adev);
3442 amdgpu_vf_error_trans_all(adev);
3443 adev->in_gpu_reset = 0;
3444 mutex_unlock(&adev->lock_reset);
3445 return r;
3446 }
3447
3448 /**
3449 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
3450 *
3451 * @adev: amdgpu_device pointer
3452 *
3453 * Fetchs and stores in the driver the PCIE capabilities (gen speed
3454 * and lanes) of the slot the device is in. Handles APUs and
3455 * virtualized environments where PCIE config space may not be available.
3456 */
amdgpu_device_get_pcie_info(struct amdgpu_device * adev)3457 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
3458 {
3459 struct pci_dev *pdev;
3460 enum pci_bus_speed speed_cap;
3461 enum pcie_link_width link_width;
3462
3463 if (amdgpu_pcie_gen_cap)
3464 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3465
3466 if (amdgpu_pcie_lane_cap)
3467 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3468
3469 /* covers APUs as well */
3470 #if 0
3471 if (pci_is_root_bus(adev->pdev->bus)) {
3472 #endif
3473 if (adev->pm.pcie_gen_mask == 0)
3474 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3475 if (adev->pm.pcie_mlw_mask == 0)
3476 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3477 return;
3478 #if 0 /* pci_is_root_bus() */
3479 }
3480 #endif
3481
3482 if (adev->pm.pcie_gen_mask == 0) {
3483 /* asic caps */
3484 pdev = adev->pdev;
3485 speed_cap = pcie_get_speed_cap(pdev);
3486 if (speed_cap == PCI_SPEED_UNKNOWN) {
3487 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3488 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3489 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
3490 } else {
3491 if (speed_cap == PCIE_SPEED_16_0GT)
3492 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3493 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3494 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
3495 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
3496 else if (speed_cap == PCIE_SPEED_8_0GT)
3497 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3498 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3499 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
3500 else if (speed_cap == PCIE_SPEED_5_0GT)
3501 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3502 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
3503 else
3504 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
3505 }
3506 /* platform caps */
3507 pdev = adev->ddev->pdev->bus->self;
3508 speed_cap = pcie_get_speed_cap(pdev);
3509 if (speed_cap == PCI_SPEED_UNKNOWN) {
3510 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3511 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
3512 } else {
3513 if (speed_cap == PCIE_SPEED_16_0GT)
3514 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3515 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3516 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
3517 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
3518 else if (speed_cap == PCIE_SPEED_8_0GT)
3519 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3520 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3521 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
3522 else if (speed_cap == PCIE_SPEED_5_0GT)
3523 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3524 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
3525 else
3526 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
3527
3528 }
3529 }
3530 if (adev->pm.pcie_mlw_mask == 0) {
3531 pdev = adev->ddev->pdev->bus->self;
3532 link_width = pcie_get_width_cap(pdev);
3533 if (link_width == PCIE_LNK_WIDTH_UNKNOWN) {
3534 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
3535 } else {
3536 switch (link_width) {
3537 case PCIE_LNK_X32:
3538 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
3539 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3540 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3541 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3542 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3543 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3544 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3545 break;
3546 case PCIE_LNK_X16:
3547 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3548 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3549 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3550 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3551 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3552 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3553 break;
3554 case PCIE_LNK_X12:
3555 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3556 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3557 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3558 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3559 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3560 break;
3561 case PCIE_LNK_X8:
3562 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3563 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3564 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3565 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3566 break;
3567 case PCIE_LNK_X4:
3568 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3569 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3570 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3571 break;
3572 case PCIE_LNK_X2:
3573 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3574 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3575 break;
3576 case PCIE_LNK_X1:
3577 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3578 break;
3579 default:
3580 break;
3581 }
3582 }
3583 }
3584 }
3585
3586