1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <drm/drmP.h>
27 #include "radeon.h"
28 #include "radeon_asic.h"
29 #include "radeon_audio.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
32 #include "atom.h"
33 #include "avivod.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36 #include "radeon_ucode.h"
37
38 #define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
39 #define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
40 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
41
42 /*
43 * Indirect registers accessor
44 */
eg_cg_rreg(struct radeon_device * rdev,u32 reg)45 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
46 {
47 unsigned long flags;
48 u32 r;
49
50 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
51 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
52 r = RREG32(EVERGREEN_CG_IND_DATA);
53 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
54 return r;
55 }
56
eg_cg_wreg(struct radeon_device * rdev,u32 reg,u32 v)57 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
58 {
59 unsigned long flags;
60
61 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
62 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
63 WREG32(EVERGREEN_CG_IND_DATA, (v));
64 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
65 }
66
eg_pif_phy0_rreg(struct radeon_device * rdev,u32 reg)67 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
68 {
69 unsigned long flags;
70 u32 r;
71
72 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
73 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
74 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
75 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
76 return r;
77 }
78
eg_pif_phy0_wreg(struct radeon_device * rdev,u32 reg,u32 v)79 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
80 {
81 unsigned long flags;
82
83 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
84 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
85 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
86 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
87 }
88
eg_pif_phy1_rreg(struct radeon_device * rdev,u32 reg)89 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
90 {
91 unsigned long flags;
92 u32 r;
93
94 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
95 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
96 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
97 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
98 return r;
99 }
100
eg_pif_phy1_wreg(struct radeon_device * rdev,u32 reg,u32 v)101 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
102 {
103 unsigned long flags;
104
105 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
106 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
107 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
108 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
109 }
110
111 static const u32 crtc_offsets[6] =
112 {
113 EVERGREEN_CRTC0_REGISTER_OFFSET,
114 EVERGREEN_CRTC1_REGISTER_OFFSET,
115 EVERGREEN_CRTC2_REGISTER_OFFSET,
116 EVERGREEN_CRTC3_REGISTER_OFFSET,
117 EVERGREEN_CRTC4_REGISTER_OFFSET,
118 EVERGREEN_CRTC5_REGISTER_OFFSET
119 };
120
121 #include "clearstate_evergreen.h"
122
123 static const u32 sumo_rlc_save_restore_register_list[] =
124 {
125 0x98fc,
126 0x9830,
127 0x9834,
128 0x9838,
129 0x9870,
130 0x9874,
131 0x8a14,
132 0x8b24,
133 0x8bcc,
134 0x8b10,
135 0x8d00,
136 0x8d04,
137 0x8c00,
138 0x8c04,
139 0x8c08,
140 0x8c0c,
141 0x8d8c,
142 0x8c20,
143 0x8c24,
144 0x8c28,
145 0x8c18,
146 0x8c1c,
147 0x8cf0,
148 0x8e2c,
149 0x8e38,
150 0x8c30,
151 0x9508,
152 0x9688,
153 0x9608,
154 0x960c,
155 0x9610,
156 0x9614,
157 0x88c4,
158 0x88d4,
159 0xa008,
160 0x900c,
161 0x9100,
162 0x913c,
163 0x98f8,
164 0x98f4,
165 0x9b7c,
166 0x3f8c,
167 0x8950,
168 0x8954,
169 0x8a18,
170 0x8b28,
171 0x9144,
172 0x9148,
173 0x914c,
174 0x3f90,
175 0x3f94,
176 0x915c,
177 0x9160,
178 0x9178,
179 0x917c,
180 0x9180,
181 0x918c,
182 0x9190,
183 0x9194,
184 0x9198,
185 0x919c,
186 0x91a8,
187 0x91ac,
188 0x91b0,
189 0x91b4,
190 0x91b8,
191 0x91c4,
192 0x91c8,
193 0x91cc,
194 0x91d0,
195 0x91d4,
196 0x91e0,
197 0x91e4,
198 0x91ec,
199 0x91f0,
200 0x91f4,
201 0x9200,
202 0x9204,
203 0x929c,
204 0x9150,
205 0x802c,
206 };
207
208 static void evergreen_gpu_init(struct radeon_device *rdev);
209
210 static const u32 evergreen_golden_registers[] =
211 {
212 0x3f90, 0xffff0000, 0xff000000,
213 0x9148, 0xffff0000, 0xff000000,
214 0x3f94, 0xffff0000, 0xff000000,
215 0x914c, 0xffff0000, 0xff000000,
216 0x9b7c, 0xffffffff, 0x00000000,
217 0x8a14, 0xffffffff, 0x00000007,
218 0x8b10, 0xffffffff, 0x00000000,
219 0x960c, 0xffffffff, 0x54763210,
220 0x88c4, 0xffffffff, 0x000000c2,
221 0x88d4, 0xffffffff, 0x00000010,
222 0x8974, 0xffffffff, 0x00000000,
223 0xc78, 0x00000080, 0x00000080,
224 0x5eb4, 0xffffffff, 0x00000002,
225 0x5e78, 0xffffffff, 0x001000f0,
226 0x6104, 0x01000300, 0x00000000,
227 0x5bc0, 0x00300000, 0x00000000,
228 0x7030, 0xffffffff, 0x00000011,
229 0x7c30, 0xffffffff, 0x00000011,
230 0x10830, 0xffffffff, 0x00000011,
231 0x11430, 0xffffffff, 0x00000011,
232 0x12030, 0xffffffff, 0x00000011,
233 0x12c30, 0xffffffff, 0x00000011,
234 0xd02c, 0xffffffff, 0x08421000,
235 0x240c, 0xffffffff, 0x00000380,
236 0x8b24, 0xffffffff, 0x00ff0fff,
237 0x28a4c, 0x06000000, 0x06000000,
238 0x10c, 0x00000001, 0x00000001,
239 0x8d00, 0xffffffff, 0x100e4848,
240 0x8d04, 0xffffffff, 0x00164745,
241 0x8c00, 0xffffffff, 0xe4000003,
242 0x8c04, 0xffffffff, 0x40600060,
243 0x8c08, 0xffffffff, 0x001c001c,
244 0x8cf0, 0xffffffff, 0x08e00620,
245 0x8c20, 0xffffffff, 0x00800080,
246 0x8c24, 0xffffffff, 0x00800080,
247 0x8c18, 0xffffffff, 0x20202078,
248 0x8c1c, 0xffffffff, 0x00001010,
249 0x28350, 0xffffffff, 0x00000000,
250 0xa008, 0xffffffff, 0x00010000,
251 0x5c4, 0xffffffff, 0x00000001,
252 0x9508, 0xffffffff, 0x00000002,
253 0x913c, 0x0000000f, 0x0000000a
254 };
255
256 static const u32 evergreen_golden_registers2[] =
257 {
258 0x2f4c, 0xffffffff, 0x00000000,
259 0x54f4, 0xffffffff, 0x00000000,
260 0x54f0, 0xffffffff, 0x00000000,
261 0x5498, 0xffffffff, 0x00000000,
262 0x549c, 0xffffffff, 0x00000000,
263 0x5494, 0xffffffff, 0x00000000,
264 0x53cc, 0xffffffff, 0x00000000,
265 0x53c8, 0xffffffff, 0x00000000,
266 0x53c4, 0xffffffff, 0x00000000,
267 0x53c0, 0xffffffff, 0x00000000,
268 0x53bc, 0xffffffff, 0x00000000,
269 0x53b8, 0xffffffff, 0x00000000,
270 0x53b4, 0xffffffff, 0x00000000,
271 0x53b0, 0xffffffff, 0x00000000
272 };
273
274 static const u32 cypress_mgcg_init[] =
275 {
276 0x802c, 0xffffffff, 0xc0000000,
277 0x5448, 0xffffffff, 0x00000100,
278 0x55e4, 0xffffffff, 0x00000100,
279 0x160c, 0xffffffff, 0x00000100,
280 0x5644, 0xffffffff, 0x00000100,
281 0xc164, 0xffffffff, 0x00000100,
282 0x8a18, 0xffffffff, 0x00000100,
283 0x897c, 0xffffffff, 0x06000100,
284 0x8b28, 0xffffffff, 0x00000100,
285 0x9144, 0xffffffff, 0x00000100,
286 0x9a60, 0xffffffff, 0x00000100,
287 0x9868, 0xffffffff, 0x00000100,
288 0x8d58, 0xffffffff, 0x00000100,
289 0x9510, 0xffffffff, 0x00000100,
290 0x949c, 0xffffffff, 0x00000100,
291 0x9654, 0xffffffff, 0x00000100,
292 0x9030, 0xffffffff, 0x00000100,
293 0x9034, 0xffffffff, 0x00000100,
294 0x9038, 0xffffffff, 0x00000100,
295 0x903c, 0xffffffff, 0x00000100,
296 0x9040, 0xffffffff, 0x00000100,
297 0xa200, 0xffffffff, 0x00000100,
298 0xa204, 0xffffffff, 0x00000100,
299 0xa208, 0xffffffff, 0x00000100,
300 0xa20c, 0xffffffff, 0x00000100,
301 0x971c, 0xffffffff, 0x00000100,
302 0x977c, 0xffffffff, 0x00000100,
303 0x3f80, 0xffffffff, 0x00000100,
304 0xa210, 0xffffffff, 0x00000100,
305 0xa214, 0xffffffff, 0x00000100,
306 0x4d8, 0xffffffff, 0x00000100,
307 0x9784, 0xffffffff, 0x00000100,
308 0x9698, 0xffffffff, 0x00000100,
309 0x4d4, 0xffffffff, 0x00000200,
310 0x30cc, 0xffffffff, 0x00000100,
311 0xd0c0, 0xffffffff, 0xff000100,
312 0x802c, 0xffffffff, 0x40000000,
313 0x915c, 0xffffffff, 0x00010000,
314 0x9160, 0xffffffff, 0x00030002,
315 0x9178, 0xffffffff, 0x00070000,
316 0x917c, 0xffffffff, 0x00030002,
317 0x9180, 0xffffffff, 0x00050004,
318 0x918c, 0xffffffff, 0x00010006,
319 0x9190, 0xffffffff, 0x00090008,
320 0x9194, 0xffffffff, 0x00070000,
321 0x9198, 0xffffffff, 0x00030002,
322 0x919c, 0xffffffff, 0x00050004,
323 0x91a8, 0xffffffff, 0x00010006,
324 0x91ac, 0xffffffff, 0x00090008,
325 0x91b0, 0xffffffff, 0x00070000,
326 0x91b4, 0xffffffff, 0x00030002,
327 0x91b8, 0xffffffff, 0x00050004,
328 0x91c4, 0xffffffff, 0x00010006,
329 0x91c8, 0xffffffff, 0x00090008,
330 0x91cc, 0xffffffff, 0x00070000,
331 0x91d0, 0xffffffff, 0x00030002,
332 0x91d4, 0xffffffff, 0x00050004,
333 0x91e0, 0xffffffff, 0x00010006,
334 0x91e4, 0xffffffff, 0x00090008,
335 0x91e8, 0xffffffff, 0x00000000,
336 0x91ec, 0xffffffff, 0x00070000,
337 0x91f0, 0xffffffff, 0x00030002,
338 0x91f4, 0xffffffff, 0x00050004,
339 0x9200, 0xffffffff, 0x00010006,
340 0x9204, 0xffffffff, 0x00090008,
341 0x9208, 0xffffffff, 0x00070000,
342 0x920c, 0xffffffff, 0x00030002,
343 0x9210, 0xffffffff, 0x00050004,
344 0x921c, 0xffffffff, 0x00010006,
345 0x9220, 0xffffffff, 0x00090008,
346 0x9224, 0xffffffff, 0x00070000,
347 0x9228, 0xffffffff, 0x00030002,
348 0x922c, 0xffffffff, 0x00050004,
349 0x9238, 0xffffffff, 0x00010006,
350 0x923c, 0xffffffff, 0x00090008,
351 0x9240, 0xffffffff, 0x00070000,
352 0x9244, 0xffffffff, 0x00030002,
353 0x9248, 0xffffffff, 0x00050004,
354 0x9254, 0xffffffff, 0x00010006,
355 0x9258, 0xffffffff, 0x00090008,
356 0x925c, 0xffffffff, 0x00070000,
357 0x9260, 0xffffffff, 0x00030002,
358 0x9264, 0xffffffff, 0x00050004,
359 0x9270, 0xffffffff, 0x00010006,
360 0x9274, 0xffffffff, 0x00090008,
361 0x9278, 0xffffffff, 0x00070000,
362 0x927c, 0xffffffff, 0x00030002,
363 0x9280, 0xffffffff, 0x00050004,
364 0x928c, 0xffffffff, 0x00010006,
365 0x9290, 0xffffffff, 0x00090008,
366 0x9294, 0xffffffff, 0x00000000,
367 0x929c, 0xffffffff, 0x00000001,
368 0x802c, 0xffffffff, 0x40010000,
369 0x915c, 0xffffffff, 0x00010000,
370 0x9160, 0xffffffff, 0x00030002,
371 0x9178, 0xffffffff, 0x00070000,
372 0x917c, 0xffffffff, 0x00030002,
373 0x9180, 0xffffffff, 0x00050004,
374 0x918c, 0xffffffff, 0x00010006,
375 0x9190, 0xffffffff, 0x00090008,
376 0x9194, 0xffffffff, 0x00070000,
377 0x9198, 0xffffffff, 0x00030002,
378 0x919c, 0xffffffff, 0x00050004,
379 0x91a8, 0xffffffff, 0x00010006,
380 0x91ac, 0xffffffff, 0x00090008,
381 0x91b0, 0xffffffff, 0x00070000,
382 0x91b4, 0xffffffff, 0x00030002,
383 0x91b8, 0xffffffff, 0x00050004,
384 0x91c4, 0xffffffff, 0x00010006,
385 0x91c8, 0xffffffff, 0x00090008,
386 0x91cc, 0xffffffff, 0x00070000,
387 0x91d0, 0xffffffff, 0x00030002,
388 0x91d4, 0xffffffff, 0x00050004,
389 0x91e0, 0xffffffff, 0x00010006,
390 0x91e4, 0xffffffff, 0x00090008,
391 0x91e8, 0xffffffff, 0x00000000,
392 0x91ec, 0xffffffff, 0x00070000,
393 0x91f0, 0xffffffff, 0x00030002,
394 0x91f4, 0xffffffff, 0x00050004,
395 0x9200, 0xffffffff, 0x00010006,
396 0x9204, 0xffffffff, 0x00090008,
397 0x9208, 0xffffffff, 0x00070000,
398 0x920c, 0xffffffff, 0x00030002,
399 0x9210, 0xffffffff, 0x00050004,
400 0x921c, 0xffffffff, 0x00010006,
401 0x9220, 0xffffffff, 0x00090008,
402 0x9224, 0xffffffff, 0x00070000,
403 0x9228, 0xffffffff, 0x00030002,
404 0x922c, 0xffffffff, 0x00050004,
405 0x9238, 0xffffffff, 0x00010006,
406 0x923c, 0xffffffff, 0x00090008,
407 0x9240, 0xffffffff, 0x00070000,
408 0x9244, 0xffffffff, 0x00030002,
409 0x9248, 0xffffffff, 0x00050004,
410 0x9254, 0xffffffff, 0x00010006,
411 0x9258, 0xffffffff, 0x00090008,
412 0x925c, 0xffffffff, 0x00070000,
413 0x9260, 0xffffffff, 0x00030002,
414 0x9264, 0xffffffff, 0x00050004,
415 0x9270, 0xffffffff, 0x00010006,
416 0x9274, 0xffffffff, 0x00090008,
417 0x9278, 0xffffffff, 0x00070000,
418 0x927c, 0xffffffff, 0x00030002,
419 0x9280, 0xffffffff, 0x00050004,
420 0x928c, 0xffffffff, 0x00010006,
421 0x9290, 0xffffffff, 0x00090008,
422 0x9294, 0xffffffff, 0x00000000,
423 0x929c, 0xffffffff, 0x00000001,
424 0x802c, 0xffffffff, 0xc0000000
425 };
426
427 static const u32 redwood_mgcg_init[] =
428 {
429 0x802c, 0xffffffff, 0xc0000000,
430 0x5448, 0xffffffff, 0x00000100,
431 0x55e4, 0xffffffff, 0x00000100,
432 0x160c, 0xffffffff, 0x00000100,
433 0x5644, 0xffffffff, 0x00000100,
434 0xc164, 0xffffffff, 0x00000100,
435 0x8a18, 0xffffffff, 0x00000100,
436 0x897c, 0xffffffff, 0x06000100,
437 0x8b28, 0xffffffff, 0x00000100,
438 0x9144, 0xffffffff, 0x00000100,
439 0x9a60, 0xffffffff, 0x00000100,
440 0x9868, 0xffffffff, 0x00000100,
441 0x8d58, 0xffffffff, 0x00000100,
442 0x9510, 0xffffffff, 0x00000100,
443 0x949c, 0xffffffff, 0x00000100,
444 0x9654, 0xffffffff, 0x00000100,
445 0x9030, 0xffffffff, 0x00000100,
446 0x9034, 0xffffffff, 0x00000100,
447 0x9038, 0xffffffff, 0x00000100,
448 0x903c, 0xffffffff, 0x00000100,
449 0x9040, 0xffffffff, 0x00000100,
450 0xa200, 0xffffffff, 0x00000100,
451 0xa204, 0xffffffff, 0x00000100,
452 0xa208, 0xffffffff, 0x00000100,
453 0xa20c, 0xffffffff, 0x00000100,
454 0x971c, 0xffffffff, 0x00000100,
455 0x977c, 0xffffffff, 0x00000100,
456 0x3f80, 0xffffffff, 0x00000100,
457 0xa210, 0xffffffff, 0x00000100,
458 0xa214, 0xffffffff, 0x00000100,
459 0x4d8, 0xffffffff, 0x00000100,
460 0x9784, 0xffffffff, 0x00000100,
461 0x9698, 0xffffffff, 0x00000100,
462 0x4d4, 0xffffffff, 0x00000200,
463 0x30cc, 0xffffffff, 0x00000100,
464 0xd0c0, 0xffffffff, 0xff000100,
465 0x802c, 0xffffffff, 0x40000000,
466 0x915c, 0xffffffff, 0x00010000,
467 0x9160, 0xffffffff, 0x00030002,
468 0x9178, 0xffffffff, 0x00070000,
469 0x917c, 0xffffffff, 0x00030002,
470 0x9180, 0xffffffff, 0x00050004,
471 0x918c, 0xffffffff, 0x00010006,
472 0x9190, 0xffffffff, 0x00090008,
473 0x9194, 0xffffffff, 0x00070000,
474 0x9198, 0xffffffff, 0x00030002,
475 0x919c, 0xffffffff, 0x00050004,
476 0x91a8, 0xffffffff, 0x00010006,
477 0x91ac, 0xffffffff, 0x00090008,
478 0x91b0, 0xffffffff, 0x00070000,
479 0x91b4, 0xffffffff, 0x00030002,
480 0x91b8, 0xffffffff, 0x00050004,
481 0x91c4, 0xffffffff, 0x00010006,
482 0x91c8, 0xffffffff, 0x00090008,
483 0x91cc, 0xffffffff, 0x00070000,
484 0x91d0, 0xffffffff, 0x00030002,
485 0x91d4, 0xffffffff, 0x00050004,
486 0x91e0, 0xffffffff, 0x00010006,
487 0x91e4, 0xffffffff, 0x00090008,
488 0x91e8, 0xffffffff, 0x00000000,
489 0x91ec, 0xffffffff, 0x00070000,
490 0x91f0, 0xffffffff, 0x00030002,
491 0x91f4, 0xffffffff, 0x00050004,
492 0x9200, 0xffffffff, 0x00010006,
493 0x9204, 0xffffffff, 0x00090008,
494 0x9294, 0xffffffff, 0x00000000,
495 0x929c, 0xffffffff, 0x00000001,
496 0x802c, 0xffffffff, 0xc0000000
497 };
498
499 static const u32 cedar_golden_registers[] =
500 {
501 0x3f90, 0xffff0000, 0xff000000,
502 0x9148, 0xffff0000, 0xff000000,
503 0x3f94, 0xffff0000, 0xff000000,
504 0x914c, 0xffff0000, 0xff000000,
505 0x9b7c, 0xffffffff, 0x00000000,
506 0x8a14, 0xffffffff, 0x00000007,
507 0x8b10, 0xffffffff, 0x00000000,
508 0x960c, 0xffffffff, 0x54763210,
509 0x88c4, 0xffffffff, 0x000000c2,
510 0x88d4, 0xffffffff, 0x00000000,
511 0x8974, 0xffffffff, 0x00000000,
512 0xc78, 0x00000080, 0x00000080,
513 0x5eb4, 0xffffffff, 0x00000002,
514 0x5e78, 0xffffffff, 0x001000f0,
515 0x6104, 0x01000300, 0x00000000,
516 0x5bc0, 0x00300000, 0x00000000,
517 0x7030, 0xffffffff, 0x00000011,
518 0x7c30, 0xffffffff, 0x00000011,
519 0x10830, 0xffffffff, 0x00000011,
520 0x11430, 0xffffffff, 0x00000011,
521 0xd02c, 0xffffffff, 0x08421000,
522 0x240c, 0xffffffff, 0x00000380,
523 0x8b24, 0xffffffff, 0x00ff0fff,
524 0x28a4c, 0x06000000, 0x06000000,
525 0x10c, 0x00000001, 0x00000001,
526 0x8d00, 0xffffffff, 0x100e4848,
527 0x8d04, 0xffffffff, 0x00164745,
528 0x8c00, 0xffffffff, 0xe4000003,
529 0x8c04, 0xffffffff, 0x40600060,
530 0x8c08, 0xffffffff, 0x001c001c,
531 0x8cf0, 0xffffffff, 0x08e00410,
532 0x8c20, 0xffffffff, 0x00800080,
533 0x8c24, 0xffffffff, 0x00800080,
534 0x8c18, 0xffffffff, 0x20202078,
535 0x8c1c, 0xffffffff, 0x00001010,
536 0x28350, 0xffffffff, 0x00000000,
537 0xa008, 0xffffffff, 0x00010000,
538 0x5c4, 0xffffffff, 0x00000001,
539 0x9508, 0xffffffff, 0x00000002
540 };
541
542 static const u32 cedar_mgcg_init[] =
543 {
544 0x802c, 0xffffffff, 0xc0000000,
545 0x5448, 0xffffffff, 0x00000100,
546 0x55e4, 0xffffffff, 0x00000100,
547 0x160c, 0xffffffff, 0x00000100,
548 0x5644, 0xffffffff, 0x00000100,
549 0xc164, 0xffffffff, 0x00000100,
550 0x8a18, 0xffffffff, 0x00000100,
551 0x897c, 0xffffffff, 0x06000100,
552 0x8b28, 0xffffffff, 0x00000100,
553 0x9144, 0xffffffff, 0x00000100,
554 0x9a60, 0xffffffff, 0x00000100,
555 0x9868, 0xffffffff, 0x00000100,
556 0x8d58, 0xffffffff, 0x00000100,
557 0x9510, 0xffffffff, 0x00000100,
558 0x949c, 0xffffffff, 0x00000100,
559 0x9654, 0xffffffff, 0x00000100,
560 0x9030, 0xffffffff, 0x00000100,
561 0x9034, 0xffffffff, 0x00000100,
562 0x9038, 0xffffffff, 0x00000100,
563 0x903c, 0xffffffff, 0x00000100,
564 0x9040, 0xffffffff, 0x00000100,
565 0xa200, 0xffffffff, 0x00000100,
566 0xa204, 0xffffffff, 0x00000100,
567 0xa208, 0xffffffff, 0x00000100,
568 0xa20c, 0xffffffff, 0x00000100,
569 0x971c, 0xffffffff, 0x00000100,
570 0x977c, 0xffffffff, 0x00000100,
571 0x3f80, 0xffffffff, 0x00000100,
572 0xa210, 0xffffffff, 0x00000100,
573 0xa214, 0xffffffff, 0x00000100,
574 0x4d8, 0xffffffff, 0x00000100,
575 0x9784, 0xffffffff, 0x00000100,
576 0x9698, 0xffffffff, 0x00000100,
577 0x4d4, 0xffffffff, 0x00000200,
578 0x30cc, 0xffffffff, 0x00000100,
579 0xd0c0, 0xffffffff, 0xff000100,
580 0x802c, 0xffffffff, 0x40000000,
581 0x915c, 0xffffffff, 0x00010000,
582 0x9178, 0xffffffff, 0x00050000,
583 0x917c, 0xffffffff, 0x00030002,
584 0x918c, 0xffffffff, 0x00010004,
585 0x9190, 0xffffffff, 0x00070006,
586 0x9194, 0xffffffff, 0x00050000,
587 0x9198, 0xffffffff, 0x00030002,
588 0x91a8, 0xffffffff, 0x00010004,
589 0x91ac, 0xffffffff, 0x00070006,
590 0x91e8, 0xffffffff, 0x00000000,
591 0x9294, 0xffffffff, 0x00000000,
592 0x929c, 0xffffffff, 0x00000001,
593 0x802c, 0xffffffff, 0xc0000000
594 };
595
596 static const u32 juniper_mgcg_init[] =
597 {
598 0x802c, 0xffffffff, 0xc0000000,
599 0x5448, 0xffffffff, 0x00000100,
600 0x55e4, 0xffffffff, 0x00000100,
601 0x160c, 0xffffffff, 0x00000100,
602 0x5644, 0xffffffff, 0x00000100,
603 0xc164, 0xffffffff, 0x00000100,
604 0x8a18, 0xffffffff, 0x00000100,
605 0x897c, 0xffffffff, 0x06000100,
606 0x8b28, 0xffffffff, 0x00000100,
607 0x9144, 0xffffffff, 0x00000100,
608 0x9a60, 0xffffffff, 0x00000100,
609 0x9868, 0xffffffff, 0x00000100,
610 0x8d58, 0xffffffff, 0x00000100,
611 0x9510, 0xffffffff, 0x00000100,
612 0x949c, 0xffffffff, 0x00000100,
613 0x9654, 0xffffffff, 0x00000100,
614 0x9030, 0xffffffff, 0x00000100,
615 0x9034, 0xffffffff, 0x00000100,
616 0x9038, 0xffffffff, 0x00000100,
617 0x903c, 0xffffffff, 0x00000100,
618 0x9040, 0xffffffff, 0x00000100,
619 0xa200, 0xffffffff, 0x00000100,
620 0xa204, 0xffffffff, 0x00000100,
621 0xa208, 0xffffffff, 0x00000100,
622 0xa20c, 0xffffffff, 0x00000100,
623 0x971c, 0xffffffff, 0x00000100,
624 0xd0c0, 0xffffffff, 0xff000100,
625 0x802c, 0xffffffff, 0x40000000,
626 0x915c, 0xffffffff, 0x00010000,
627 0x9160, 0xffffffff, 0x00030002,
628 0x9178, 0xffffffff, 0x00070000,
629 0x917c, 0xffffffff, 0x00030002,
630 0x9180, 0xffffffff, 0x00050004,
631 0x918c, 0xffffffff, 0x00010006,
632 0x9190, 0xffffffff, 0x00090008,
633 0x9194, 0xffffffff, 0x00070000,
634 0x9198, 0xffffffff, 0x00030002,
635 0x919c, 0xffffffff, 0x00050004,
636 0x91a8, 0xffffffff, 0x00010006,
637 0x91ac, 0xffffffff, 0x00090008,
638 0x91b0, 0xffffffff, 0x00070000,
639 0x91b4, 0xffffffff, 0x00030002,
640 0x91b8, 0xffffffff, 0x00050004,
641 0x91c4, 0xffffffff, 0x00010006,
642 0x91c8, 0xffffffff, 0x00090008,
643 0x91cc, 0xffffffff, 0x00070000,
644 0x91d0, 0xffffffff, 0x00030002,
645 0x91d4, 0xffffffff, 0x00050004,
646 0x91e0, 0xffffffff, 0x00010006,
647 0x91e4, 0xffffffff, 0x00090008,
648 0x91e8, 0xffffffff, 0x00000000,
649 0x91ec, 0xffffffff, 0x00070000,
650 0x91f0, 0xffffffff, 0x00030002,
651 0x91f4, 0xffffffff, 0x00050004,
652 0x9200, 0xffffffff, 0x00010006,
653 0x9204, 0xffffffff, 0x00090008,
654 0x9208, 0xffffffff, 0x00070000,
655 0x920c, 0xffffffff, 0x00030002,
656 0x9210, 0xffffffff, 0x00050004,
657 0x921c, 0xffffffff, 0x00010006,
658 0x9220, 0xffffffff, 0x00090008,
659 0x9224, 0xffffffff, 0x00070000,
660 0x9228, 0xffffffff, 0x00030002,
661 0x922c, 0xffffffff, 0x00050004,
662 0x9238, 0xffffffff, 0x00010006,
663 0x923c, 0xffffffff, 0x00090008,
664 0x9240, 0xffffffff, 0x00070000,
665 0x9244, 0xffffffff, 0x00030002,
666 0x9248, 0xffffffff, 0x00050004,
667 0x9254, 0xffffffff, 0x00010006,
668 0x9258, 0xffffffff, 0x00090008,
669 0x925c, 0xffffffff, 0x00070000,
670 0x9260, 0xffffffff, 0x00030002,
671 0x9264, 0xffffffff, 0x00050004,
672 0x9270, 0xffffffff, 0x00010006,
673 0x9274, 0xffffffff, 0x00090008,
674 0x9278, 0xffffffff, 0x00070000,
675 0x927c, 0xffffffff, 0x00030002,
676 0x9280, 0xffffffff, 0x00050004,
677 0x928c, 0xffffffff, 0x00010006,
678 0x9290, 0xffffffff, 0x00090008,
679 0x9294, 0xffffffff, 0x00000000,
680 0x929c, 0xffffffff, 0x00000001,
681 0x802c, 0xffffffff, 0xc0000000,
682 0x977c, 0xffffffff, 0x00000100,
683 0x3f80, 0xffffffff, 0x00000100,
684 0xa210, 0xffffffff, 0x00000100,
685 0xa214, 0xffffffff, 0x00000100,
686 0x4d8, 0xffffffff, 0x00000100,
687 0x9784, 0xffffffff, 0x00000100,
688 0x9698, 0xffffffff, 0x00000100,
689 0x4d4, 0xffffffff, 0x00000200,
690 0x30cc, 0xffffffff, 0x00000100,
691 0x802c, 0xffffffff, 0xc0000000
692 };
693
694 static const u32 supersumo_golden_registers[] =
695 {
696 0x5eb4, 0xffffffff, 0x00000002,
697 0x5c4, 0xffffffff, 0x00000001,
698 0x7030, 0xffffffff, 0x00000011,
699 0x7c30, 0xffffffff, 0x00000011,
700 0x6104, 0x01000300, 0x00000000,
701 0x5bc0, 0x00300000, 0x00000000,
702 0x8c04, 0xffffffff, 0x40600060,
703 0x8c08, 0xffffffff, 0x001c001c,
704 0x8c20, 0xffffffff, 0x00800080,
705 0x8c24, 0xffffffff, 0x00800080,
706 0x8c18, 0xffffffff, 0x20202078,
707 0x8c1c, 0xffffffff, 0x00001010,
708 0x918c, 0xffffffff, 0x00010006,
709 0x91a8, 0xffffffff, 0x00010006,
710 0x91c4, 0xffffffff, 0x00010006,
711 0x91e0, 0xffffffff, 0x00010006,
712 0x9200, 0xffffffff, 0x00010006,
713 0x9150, 0xffffffff, 0x6e944040,
714 0x917c, 0xffffffff, 0x00030002,
715 0x9180, 0xffffffff, 0x00050004,
716 0x9198, 0xffffffff, 0x00030002,
717 0x919c, 0xffffffff, 0x00050004,
718 0x91b4, 0xffffffff, 0x00030002,
719 0x91b8, 0xffffffff, 0x00050004,
720 0x91d0, 0xffffffff, 0x00030002,
721 0x91d4, 0xffffffff, 0x00050004,
722 0x91f0, 0xffffffff, 0x00030002,
723 0x91f4, 0xffffffff, 0x00050004,
724 0x915c, 0xffffffff, 0x00010000,
725 0x9160, 0xffffffff, 0x00030002,
726 0x3f90, 0xffff0000, 0xff000000,
727 0x9178, 0xffffffff, 0x00070000,
728 0x9194, 0xffffffff, 0x00070000,
729 0x91b0, 0xffffffff, 0x00070000,
730 0x91cc, 0xffffffff, 0x00070000,
731 0x91ec, 0xffffffff, 0x00070000,
732 0x9148, 0xffff0000, 0xff000000,
733 0x9190, 0xffffffff, 0x00090008,
734 0x91ac, 0xffffffff, 0x00090008,
735 0x91c8, 0xffffffff, 0x00090008,
736 0x91e4, 0xffffffff, 0x00090008,
737 0x9204, 0xffffffff, 0x00090008,
738 0x3f94, 0xffff0000, 0xff000000,
739 0x914c, 0xffff0000, 0xff000000,
740 0x929c, 0xffffffff, 0x00000001,
741 0x8a18, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x5644, 0xffffffff, 0x00000100,
745 0x9b7c, 0xffffffff, 0x00000000,
746 0x8030, 0xffffffff, 0x0000100a,
747 0x8a14, 0xffffffff, 0x00000007,
748 0x8b24, 0xffffffff, 0x00ff0fff,
749 0x8b10, 0xffffffff, 0x00000000,
750 0x28a4c, 0x06000000, 0x06000000,
751 0x4d8, 0xffffffff, 0x00000100,
752 0x913c, 0xffff000f, 0x0100000a,
753 0x960c, 0xffffffff, 0x54763210,
754 0x88c4, 0xffffffff, 0x000000c2,
755 0x88d4, 0xffffffff, 0x00000010,
756 0x8974, 0xffffffff, 0x00000000,
757 0xc78, 0x00000080, 0x00000080,
758 0x5e78, 0xffffffff, 0x001000f0,
759 0xd02c, 0xffffffff, 0x08421000,
760 0xa008, 0xffffffff, 0x00010000,
761 0x8d00, 0xffffffff, 0x100e4848,
762 0x8d04, 0xffffffff, 0x00164745,
763 0x8c00, 0xffffffff, 0xe4000003,
764 0x8cf0, 0x1fffffff, 0x08e00620,
765 0x28350, 0xffffffff, 0x00000000,
766 0x9508, 0xffffffff, 0x00000002
767 };
768
769 static const u32 sumo_golden_registers[] =
770 {
771 0x900c, 0x00ffffff, 0x0017071f,
772 0x8c18, 0xffffffff, 0x10101060,
773 0x8c1c, 0xffffffff, 0x00001010,
774 0x8c30, 0x0000000f, 0x00000005,
775 0x9688, 0x0000000f, 0x00000007
776 };
777
778 static const u32 wrestler_golden_registers[] =
779 {
780 0x5eb4, 0xffffffff, 0x00000002,
781 0x5c4, 0xffffffff, 0x00000001,
782 0x7030, 0xffffffff, 0x00000011,
783 0x7c30, 0xffffffff, 0x00000011,
784 0x6104, 0x01000300, 0x00000000,
785 0x5bc0, 0x00300000, 0x00000000,
786 0x918c, 0xffffffff, 0x00010006,
787 0x91a8, 0xffffffff, 0x00010006,
788 0x9150, 0xffffffff, 0x6e944040,
789 0x917c, 0xffffffff, 0x00030002,
790 0x9198, 0xffffffff, 0x00030002,
791 0x915c, 0xffffffff, 0x00010000,
792 0x3f90, 0xffff0000, 0xff000000,
793 0x9178, 0xffffffff, 0x00070000,
794 0x9194, 0xffffffff, 0x00070000,
795 0x9148, 0xffff0000, 0xff000000,
796 0x9190, 0xffffffff, 0x00090008,
797 0x91ac, 0xffffffff, 0x00090008,
798 0x3f94, 0xffff0000, 0xff000000,
799 0x914c, 0xffff0000, 0xff000000,
800 0x929c, 0xffffffff, 0x00000001,
801 0x8a18, 0xffffffff, 0x00000100,
802 0x8b28, 0xffffffff, 0x00000100,
803 0x9144, 0xffffffff, 0x00000100,
804 0x9b7c, 0xffffffff, 0x00000000,
805 0x8030, 0xffffffff, 0x0000100a,
806 0x8a14, 0xffffffff, 0x00000001,
807 0x8b24, 0xffffffff, 0x00ff0fff,
808 0x8b10, 0xffffffff, 0x00000000,
809 0x28a4c, 0x06000000, 0x06000000,
810 0x4d8, 0xffffffff, 0x00000100,
811 0x913c, 0xffff000f, 0x0100000a,
812 0x960c, 0xffffffff, 0x54763210,
813 0x88c4, 0xffffffff, 0x000000c2,
814 0x88d4, 0xffffffff, 0x00000010,
815 0x8974, 0xffffffff, 0x00000000,
816 0xc78, 0x00000080, 0x00000080,
817 0x5e78, 0xffffffff, 0x001000f0,
818 0xd02c, 0xffffffff, 0x08421000,
819 0xa008, 0xffffffff, 0x00010000,
820 0x8d00, 0xffffffff, 0x100e4848,
821 0x8d04, 0xffffffff, 0x00164745,
822 0x8c00, 0xffffffff, 0xe4000003,
823 0x8cf0, 0x1fffffff, 0x08e00410,
824 0x28350, 0xffffffff, 0x00000000,
825 0x9508, 0xffffffff, 0x00000002,
826 0x900c, 0xffffffff, 0x0017071f,
827 0x8c18, 0xffffffff, 0x10101060,
828 0x8c1c, 0xffffffff, 0x00001010
829 };
830
831 static const u32 barts_golden_registers[] =
832 {
833 0x5eb4, 0xffffffff, 0x00000002,
834 0x5e78, 0x8f311ff1, 0x001000f0,
835 0x3f90, 0xffff0000, 0xff000000,
836 0x9148, 0xffff0000, 0xff000000,
837 0x3f94, 0xffff0000, 0xff000000,
838 0x914c, 0xffff0000, 0xff000000,
839 0xc78, 0x00000080, 0x00000080,
840 0xbd4, 0x70073777, 0x00010001,
841 0xd02c, 0xbfffff1f, 0x08421000,
842 0xd0b8, 0x03773777, 0x02011003,
843 0x5bc0, 0x00200000, 0x50100000,
844 0x98f8, 0x33773777, 0x02011003,
845 0x98fc, 0xffffffff, 0x76543210,
846 0x7030, 0x31000311, 0x00000011,
847 0x2f48, 0x00000007, 0x02011003,
848 0x6b28, 0x00000010, 0x00000012,
849 0x7728, 0x00000010, 0x00000012,
850 0x10328, 0x00000010, 0x00000012,
851 0x10f28, 0x00000010, 0x00000012,
852 0x11b28, 0x00000010, 0x00000012,
853 0x12728, 0x00000010, 0x00000012,
854 0x240c, 0x000007ff, 0x00000380,
855 0x8a14, 0xf000001f, 0x00000007,
856 0x8b24, 0x3fff3fff, 0x00ff0fff,
857 0x8b10, 0x0000ff0f, 0x00000000,
858 0x28a4c, 0x07ffffff, 0x06000000,
859 0x10c, 0x00000001, 0x00010003,
860 0xa02c, 0xffffffff, 0x0000009b,
861 0x913c, 0x0000000f, 0x0100000a,
862 0x8d00, 0xffff7f7f, 0x100e4848,
863 0x8d04, 0x00ffffff, 0x00164745,
864 0x8c00, 0xfffc0003, 0xe4000003,
865 0x8c04, 0xf8ff00ff, 0x40600060,
866 0x8c08, 0x00ff00ff, 0x001c001c,
867 0x8cf0, 0x1fff1fff, 0x08e00620,
868 0x8c20, 0x0fff0fff, 0x00800080,
869 0x8c24, 0x0fff0fff, 0x00800080,
870 0x8c18, 0xffffffff, 0x20202078,
871 0x8c1c, 0x0000ffff, 0x00001010,
872 0x28350, 0x00000f01, 0x00000000,
873 0x9508, 0x3700001f, 0x00000002,
874 0x960c, 0xffffffff, 0x54763210,
875 0x88c4, 0x001f3ae3, 0x000000c2,
876 0x88d4, 0x0000001f, 0x00000010,
877 0x8974, 0xffffffff, 0x00000000
878 };
879
880 static const u32 turks_golden_registers[] =
881 {
882 0x5eb4, 0xffffffff, 0x00000002,
883 0x5e78, 0x8f311ff1, 0x001000f0,
884 0x8c8, 0x00003000, 0x00001070,
885 0x8cc, 0x000fffff, 0x00040035,
886 0x3f90, 0xffff0000, 0xfff00000,
887 0x9148, 0xffff0000, 0xfff00000,
888 0x3f94, 0xffff0000, 0xfff00000,
889 0x914c, 0xffff0000, 0xfff00000,
890 0xc78, 0x00000080, 0x00000080,
891 0xbd4, 0x00073007, 0x00010002,
892 0xd02c, 0xbfffff1f, 0x08421000,
893 0xd0b8, 0x03773777, 0x02010002,
894 0x5bc0, 0x00200000, 0x50100000,
895 0x98f8, 0x33773777, 0x00010002,
896 0x98fc, 0xffffffff, 0x33221100,
897 0x7030, 0x31000311, 0x00000011,
898 0x2f48, 0x33773777, 0x00010002,
899 0x6b28, 0x00000010, 0x00000012,
900 0x7728, 0x00000010, 0x00000012,
901 0x10328, 0x00000010, 0x00000012,
902 0x10f28, 0x00000010, 0x00000012,
903 0x11b28, 0x00000010, 0x00000012,
904 0x12728, 0x00000010, 0x00000012,
905 0x240c, 0x000007ff, 0x00000380,
906 0x8a14, 0xf000001f, 0x00000007,
907 0x8b24, 0x3fff3fff, 0x00ff0fff,
908 0x8b10, 0x0000ff0f, 0x00000000,
909 0x28a4c, 0x07ffffff, 0x06000000,
910 0x10c, 0x00000001, 0x00010003,
911 0xa02c, 0xffffffff, 0x0000009b,
912 0x913c, 0x0000000f, 0x0100000a,
913 0x8d00, 0xffff7f7f, 0x100e4848,
914 0x8d04, 0x00ffffff, 0x00164745,
915 0x8c00, 0xfffc0003, 0xe4000003,
916 0x8c04, 0xf8ff00ff, 0x40600060,
917 0x8c08, 0x00ff00ff, 0x001c001c,
918 0x8cf0, 0x1fff1fff, 0x08e00410,
919 0x8c20, 0x0fff0fff, 0x00800080,
920 0x8c24, 0x0fff0fff, 0x00800080,
921 0x8c18, 0xffffffff, 0x20202078,
922 0x8c1c, 0x0000ffff, 0x00001010,
923 0x28350, 0x00000f01, 0x00000000,
924 0x9508, 0x3700001f, 0x00000002,
925 0x960c, 0xffffffff, 0x54763210,
926 0x88c4, 0x001f3ae3, 0x000000c2,
927 0x88d4, 0x0000001f, 0x00000010,
928 0x8974, 0xffffffff, 0x00000000
929 };
930
931 static const u32 caicos_golden_registers[] =
932 {
933 0x5eb4, 0xffffffff, 0x00000002,
934 0x5e78, 0x8f311ff1, 0x001000f0,
935 0x8c8, 0x00003420, 0x00001450,
936 0x8cc, 0x000fffff, 0x00040035,
937 0x3f90, 0xffff0000, 0xfffc0000,
938 0x9148, 0xffff0000, 0xfffc0000,
939 0x3f94, 0xffff0000, 0xfffc0000,
940 0x914c, 0xffff0000, 0xfffc0000,
941 0xc78, 0x00000080, 0x00000080,
942 0xbd4, 0x00073007, 0x00010001,
943 0xd02c, 0xbfffff1f, 0x08421000,
944 0xd0b8, 0x03773777, 0x02010001,
945 0x5bc0, 0x00200000, 0x50100000,
946 0x98f8, 0x33773777, 0x02010001,
947 0x98fc, 0xffffffff, 0x33221100,
948 0x7030, 0x31000311, 0x00000011,
949 0x2f48, 0x33773777, 0x02010001,
950 0x6b28, 0x00000010, 0x00000012,
951 0x7728, 0x00000010, 0x00000012,
952 0x10328, 0x00000010, 0x00000012,
953 0x10f28, 0x00000010, 0x00000012,
954 0x11b28, 0x00000010, 0x00000012,
955 0x12728, 0x00000010, 0x00000012,
956 0x240c, 0x000007ff, 0x00000380,
957 0x8a14, 0xf000001f, 0x00000001,
958 0x8b24, 0x3fff3fff, 0x00ff0fff,
959 0x8b10, 0x0000ff0f, 0x00000000,
960 0x28a4c, 0x07ffffff, 0x06000000,
961 0x10c, 0x00000001, 0x00010003,
962 0xa02c, 0xffffffff, 0x0000009b,
963 0x913c, 0x0000000f, 0x0100000a,
964 0x8d00, 0xffff7f7f, 0x100e4848,
965 0x8d04, 0x00ffffff, 0x00164745,
966 0x8c00, 0xfffc0003, 0xe4000003,
967 0x8c04, 0xf8ff00ff, 0x40600060,
968 0x8c08, 0x00ff00ff, 0x001c001c,
969 0x8cf0, 0x1fff1fff, 0x08e00410,
970 0x8c20, 0x0fff0fff, 0x00800080,
971 0x8c24, 0x0fff0fff, 0x00800080,
972 0x8c18, 0xffffffff, 0x20202078,
973 0x8c1c, 0x0000ffff, 0x00001010,
974 0x28350, 0x00000f01, 0x00000000,
975 0x9508, 0x3700001f, 0x00000002,
976 0x960c, 0xffffffff, 0x54763210,
977 0x88c4, 0x001f3ae3, 0x000000c2,
978 0x88d4, 0x0000001f, 0x00000010,
979 0x8974, 0xffffffff, 0x00000000
980 };
981
evergreen_init_golden_registers(struct radeon_device * rdev)982 static void evergreen_init_golden_registers(struct radeon_device *rdev)
983 {
984 switch (rdev->family) {
985 case CHIP_CYPRESS:
986 case CHIP_HEMLOCK:
987 radeon_program_register_sequence(rdev,
988 evergreen_golden_registers,
989 (const u32)ARRAY_SIZE(evergreen_golden_registers));
990 radeon_program_register_sequence(rdev,
991 evergreen_golden_registers2,
992 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
993 radeon_program_register_sequence(rdev,
994 cypress_mgcg_init,
995 (const u32)ARRAY_SIZE(cypress_mgcg_init));
996 break;
997 case CHIP_JUNIPER:
998 radeon_program_register_sequence(rdev,
999 evergreen_golden_registers,
1000 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1001 radeon_program_register_sequence(rdev,
1002 evergreen_golden_registers2,
1003 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1004 radeon_program_register_sequence(rdev,
1005 juniper_mgcg_init,
1006 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1007 break;
1008 case CHIP_REDWOOD:
1009 radeon_program_register_sequence(rdev,
1010 evergreen_golden_registers,
1011 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1012 radeon_program_register_sequence(rdev,
1013 evergreen_golden_registers2,
1014 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1015 radeon_program_register_sequence(rdev,
1016 redwood_mgcg_init,
1017 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1018 break;
1019 case CHIP_CEDAR:
1020 radeon_program_register_sequence(rdev,
1021 cedar_golden_registers,
1022 (const u32)ARRAY_SIZE(cedar_golden_registers));
1023 radeon_program_register_sequence(rdev,
1024 evergreen_golden_registers2,
1025 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1026 radeon_program_register_sequence(rdev,
1027 cedar_mgcg_init,
1028 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1029 break;
1030 case CHIP_PALM:
1031 radeon_program_register_sequence(rdev,
1032 wrestler_golden_registers,
1033 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1034 break;
1035 case CHIP_SUMO:
1036 radeon_program_register_sequence(rdev,
1037 supersumo_golden_registers,
1038 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1039 break;
1040 case CHIP_SUMO2:
1041 radeon_program_register_sequence(rdev,
1042 supersumo_golden_registers,
1043 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1044 radeon_program_register_sequence(rdev,
1045 sumo_golden_registers,
1046 (const u32)ARRAY_SIZE(sumo_golden_registers));
1047 break;
1048 case CHIP_BARTS:
1049 radeon_program_register_sequence(rdev,
1050 barts_golden_registers,
1051 (const u32)ARRAY_SIZE(barts_golden_registers));
1052 break;
1053 case CHIP_TURKS:
1054 radeon_program_register_sequence(rdev,
1055 turks_golden_registers,
1056 (const u32)ARRAY_SIZE(turks_golden_registers));
1057 break;
1058 case CHIP_CAICOS:
1059 radeon_program_register_sequence(rdev,
1060 caicos_golden_registers,
1061 (const u32)ARRAY_SIZE(caicos_golden_registers));
1062 break;
1063 default:
1064 break;
1065 }
1066 }
1067
1068 /**
1069 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1070 *
1071 * @rdev: radeon_device pointer
1072 * @reg: register offset in bytes
1073 * @val: register value
1074 *
1075 * Returns 0 for success or -EINVAL for an invalid register
1076 *
1077 */
evergreen_get_allowed_info_register(struct radeon_device * rdev,u32 reg,u32 * val)1078 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1079 u32 reg, u32 *val)
1080 {
1081 switch (reg) {
1082 case GRBM_STATUS:
1083 case GRBM_STATUS_SE0:
1084 case GRBM_STATUS_SE1:
1085 case SRBM_STATUS:
1086 case SRBM_STATUS2:
1087 case DMA_STATUS_REG:
1088 case UVD_STATUS:
1089 *val = RREG32(reg);
1090 return 0;
1091 default:
1092 return -EINVAL;
1093 }
1094 }
1095
evergreen_tiling_fields(unsigned tiling_flags,unsigned * bankw,unsigned * bankh,unsigned * mtaspect,unsigned * tile_split)1096 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1097 unsigned *bankh, unsigned *mtaspect,
1098 unsigned *tile_split)
1099 {
1100 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1101 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1102 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1103 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1104 switch (*bankw) {
1105 default:
1106 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1107 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1108 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1109 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1110 }
1111 switch (*bankh) {
1112 default:
1113 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1114 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1115 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1116 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1117 }
1118 switch (*mtaspect) {
1119 default:
1120 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1121 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1122 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1123 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1124 }
1125 }
1126
sumo_set_uvd_clock(struct radeon_device * rdev,u32 clock,u32 cntl_reg,u32 status_reg)1127 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1128 u32 cntl_reg, u32 status_reg)
1129 {
1130 int r, i;
1131 struct atom_clock_dividers dividers;
1132
1133 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1134 clock, false, ÷rs);
1135 if (r)
1136 return r;
1137
1138 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1139
1140 for (i = 0; i < 100; i++) {
1141 if (RREG32(status_reg) & DCLK_STATUS)
1142 break;
1143 mdelay(10);
1144 }
1145 if (i == 100)
1146 return -ETIMEDOUT;
1147
1148 return 0;
1149 }
1150
sumo_set_uvd_clocks(struct radeon_device * rdev,u32 vclk,u32 dclk)1151 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1152 {
1153 int r = 0;
1154 u32 cg_scratch = RREG32(CG_SCRATCH1);
1155
1156 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1157 if (r)
1158 goto done;
1159 cg_scratch &= 0xffff0000;
1160 cg_scratch |= vclk / 100; /* Mhz */
1161
1162 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1163 if (r)
1164 goto done;
1165 cg_scratch &= 0x0000ffff;
1166 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1167
1168 done:
1169 WREG32(CG_SCRATCH1, cg_scratch);
1170
1171 return r;
1172 }
1173
evergreen_set_uvd_clocks(struct radeon_device * rdev,u32 vclk,u32 dclk)1174 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1175 {
1176 /* start off with something large */
1177 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1178 int r;
1179
1180 /* bypass vclk and dclk with bclk */
1181 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1182 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1183 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1184
1185 /* put PLL in bypass mode */
1186 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1187
1188 if (!vclk || !dclk) {
1189 /* keep the Bypass mode, put PLL to sleep */
1190 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1191 return 0;
1192 }
1193
1194 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1195 16384, 0x03FFFFFF, 0, 128, 5,
1196 &fb_div, &vclk_div, &dclk_div);
1197 if (r)
1198 return r;
1199
1200 /* set VCO_MODE to 1 */
1201 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1202
1203 /* toggle UPLL_SLEEP to 1 then back to 0 */
1204 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1205 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1206
1207 /* deassert UPLL_RESET */
1208 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1209
1210 mdelay(1);
1211
1212 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1213 if (r)
1214 return r;
1215
1216 /* assert UPLL_RESET again */
1217 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1218
1219 /* disable spread spectrum. */
1220 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1221
1222 /* set feedback divider */
1223 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1224
1225 /* set ref divider to 0 */
1226 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1227
1228 if (fb_div < 307200)
1229 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1230 else
1231 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1232
1233 /* set PDIV_A and PDIV_B */
1234 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1235 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1236 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1237
1238 /* give the PLL some time to settle */
1239 mdelay(15);
1240
1241 /* deassert PLL_RESET */
1242 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1243
1244 mdelay(15);
1245
1246 /* switch from bypass mode to normal mode */
1247 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1248
1249 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1250 if (r)
1251 return r;
1252
1253 /* switch VCLK and DCLK selection */
1254 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1255 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1256 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1257
1258 mdelay(100);
1259
1260 return 0;
1261 }
1262
evergreen_fix_pci_max_read_req_size(struct radeon_device * rdev)1263 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1264 {
1265 int readrq;
1266 u16 v;
1267
1268 readrq = pcie_get_readrq(rdev->pdev);
1269 v = ffs(readrq) - 8;
1270 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1271 * to avoid hangs or perfomance issues
1272 */
1273 if ((v == 0) || (v == 6) || (v == 7))
1274 pcie_set_readrq(rdev->pdev, 512);
1275 }
1276
dce4_program_fmt(struct drm_encoder * encoder)1277 void dce4_program_fmt(struct drm_encoder *encoder)
1278 {
1279 struct drm_device *dev = encoder->dev;
1280 struct radeon_device *rdev = dev->dev_private;
1281 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1282 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1283 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1284 int bpc = 0;
1285 u32 tmp = 0;
1286 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1287
1288 if (connector) {
1289 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1290 bpc = radeon_get_monitor_bpc(connector);
1291 dither = radeon_connector->dither;
1292 }
1293
1294 /* LVDS/eDP FMT is set up by atom */
1295 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1296 return;
1297
1298 /* not needed for analog */
1299 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1300 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1301 return;
1302
1303 if (bpc == 0)
1304 return;
1305
1306 switch (bpc) {
1307 case 6:
1308 if (dither == RADEON_FMT_DITHER_ENABLE)
1309 /* XXX sort out optimal dither settings */
1310 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1311 FMT_SPATIAL_DITHER_EN);
1312 else
1313 tmp |= FMT_TRUNCATE_EN;
1314 break;
1315 case 8:
1316 if (dither == RADEON_FMT_DITHER_ENABLE)
1317 /* XXX sort out optimal dither settings */
1318 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1319 FMT_RGB_RANDOM_ENABLE |
1320 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1321 else
1322 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1323 break;
1324 case 10:
1325 default:
1326 /* not needed */
1327 break;
1328 }
1329
1330 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1331 }
1332
dce4_is_in_vblank(struct radeon_device * rdev,int crtc)1333 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1334 {
1335 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1336 return true;
1337 else
1338 return false;
1339 }
1340
dce4_is_counter_moving(struct radeon_device * rdev,int crtc)1341 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1342 {
1343 u32 pos1, pos2;
1344
1345 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1346 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1347
1348 if (pos1 != pos2)
1349 return true;
1350 else
1351 return false;
1352 }
1353
1354 /**
1355 * dce4_wait_for_vblank - vblank wait asic callback.
1356 *
1357 * @rdev: radeon_device pointer
1358 * @crtc: crtc to wait for vblank on
1359 *
1360 * Wait for vblank on the requested crtc (evergreen+).
1361 */
dce4_wait_for_vblank(struct radeon_device * rdev,int crtc)1362 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1363 {
1364 unsigned i = 0;
1365
1366 if (crtc >= rdev->num_crtc)
1367 return;
1368
1369 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1370 return;
1371
1372 /* depending on when we hit vblank, we may be close to active; if so,
1373 * wait for another frame.
1374 */
1375 while (dce4_is_in_vblank(rdev, crtc)) {
1376 if (i++ % 100 == 0) {
1377 if (!dce4_is_counter_moving(rdev, crtc))
1378 break;
1379 }
1380 }
1381
1382 while (!dce4_is_in_vblank(rdev, crtc)) {
1383 if (i++ % 100 == 0) {
1384 if (!dce4_is_counter_moving(rdev, crtc))
1385 break;
1386 }
1387 }
1388 }
1389
1390 /**
1391 * evergreen_page_flip - pageflip callback.
1392 *
1393 * @rdev: radeon_device pointer
1394 * @crtc_id: crtc to cleanup pageflip on
1395 * @crtc_base: new address of the crtc (GPU MC address)
1396 *
1397 * Triggers the actual pageflip by updating the primary
1398 * surface base address (evergreen+).
1399 */
evergreen_page_flip(struct radeon_device * rdev,int crtc_id,u64 crtc_base,bool async)1400 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1401 bool async)
1402 {
1403 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1404
1405 /* update the scanout addresses */
1406 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1407 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1408 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1409 upper_32_bits(crtc_base));
1410 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1411 (u32)crtc_base);
1412 /* post the write */
1413 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1414 }
1415
1416 /**
1417 * evergreen_page_flip_pending - check if page flip is still pending
1418 *
1419 * @rdev: radeon_device pointer
1420 * @crtc_id: crtc to check
1421 *
1422 * Returns the current update pending status.
1423 */
evergreen_page_flip_pending(struct radeon_device * rdev,int crtc_id)1424 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1425 {
1426 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1427
1428 /* Return current update_pending status: */
1429 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1430 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1431 }
1432
1433 /* get temperature in millidegrees */
evergreen_get_temp(struct radeon_device * rdev)1434 int evergreen_get_temp(struct radeon_device *rdev)
1435 {
1436 u32 temp, toffset;
1437 int actual_temp = 0;
1438
1439 if (rdev->family == CHIP_JUNIPER) {
1440 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1441 TOFFSET_SHIFT;
1442 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1443 TS0_ADC_DOUT_SHIFT;
1444
1445 if (toffset & 0x100)
1446 actual_temp = temp / 2 - (0x200 - toffset);
1447 else
1448 actual_temp = temp / 2 + toffset;
1449
1450 actual_temp = actual_temp * 1000;
1451
1452 } else {
1453 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1454 ASIC_T_SHIFT;
1455
1456 if (temp & 0x400)
1457 actual_temp = -256;
1458 else if (temp & 0x200)
1459 actual_temp = 255;
1460 else if (temp & 0x100) {
1461 actual_temp = temp & 0x1ff;
1462 actual_temp |= ~0x1ff;
1463 } else
1464 actual_temp = temp & 0xff;
1465
1466 actual_temp = (actual_temp * 1000) / 2;
1467 }
1468
1469 return actual_temp;
1470 }
1471
sumo_get_temp(struct radeon_device * rdev)1472 int sumo_get_temp(struct radeon_device *rdev)
1473 {
1474 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1475 int actual_temp = temp - 49;
1476
1477 return actual_temp * 1000;
1478 }
1479
1480 /**
1481 * sumo_pm_init_profile - Initialize power profiles callback.
1482 *
1483 * @rdev: radeon_device pointer
1484 *
1485 * Initialize the power states used in profile mode
1486 * (sumo, trinity, SI).
1487 * Used for profile mode only.
1488 */
sumo_pm_init_profile(struct radeon_device * rdev)1489 void sumo_pm_init_profile(struct radeon_device *rdev)
1490 {
1491 int idx;
1492
1493 /* default */
1494 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1495 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1496 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1497 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1498
1499 /* low,mid sh/mh */
1500 if (rdev->flags & RADEON_IS_MOBILITY)
1501 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1502 else
1503 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1504
1505 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1506 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1507 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1508 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1509
1510 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1511 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1512 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1513 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1514
1515 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1516 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1517 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1518 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1519
1520 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1521 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1523 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1524
1525 /* high sh/mh */
1526 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1527 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1529 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1530 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1531 rdev->pm.power_state[idx].num_clock_modes - 1;
1532
1533 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1534 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1536 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1537 rdev->pm.power_state[idx].num_clock_modes - 1;
1538 }
1539
1540 /**
1541 * btc_pm_init_profile - Initialize power profiles callback.
1542 *
1543 * @rdev: radeon_device pointer
1544 *
1545 * Initialize the power states used in profile mode
1546 * (BTC, cayman).
1547 * Used for profile mode only.
1548 */
btc_pm_init_profile(struct radeon_device * rdev)1549 void btc_pm_init_profile(struct radeon_device *rdev)
1550 {
1551 int idx;
1552
1553 /* default */
1554 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1555 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1556 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1557 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1558 /* starting with BTC, there is one state that is used for both
1559 * MH and SH. Difference is that we always use the high clock index for
1560 * mclk.
1561 */
1562 if (rdev->flags & RADEON_IS_MOBILITY)
1563 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1564 else
1565 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1566 /* low sh */
1567 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1568 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1569 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1570 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1571 /* mid sh */
1572 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1573 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1574 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1575 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1576 /* high sh */
1577 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1578 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1579 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1580 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1581 /* low mh */
1582 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1583 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1584 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1585 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1586 /* mid mh */
1587 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1588 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1589 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1590 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1591 /* high mh */
1592 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1593 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1594 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1595 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1596 }
1597
1598 /**
1599 * evergreen_pm_misc - set additional pm hw parameters callback.
1600 *
1601 * @rdev: radeon_device pointer
1602 *
1603 * Set non-clock parameters associated with a power state
1604 * (voltage, etc.) (evergreen+).
1605 */
evergreen_pm_misc(struct radeon_device * rdev)1606 void evergreen_pm_misc(struct radeon_device *rdev)
1607 {
1608 int req_ps_idx = rdev->pm.requested_power_state_index;
1609 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1610 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1611 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1612
1613 if (voltage->type == VOLTAGE_SW) {
1614 /* 0xff0x are flags rather then an actual voltage */
1615 if ((voltage->voltage & 0xff00) == 0xff00)
1616 return;
1617 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1618 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1619 rdev->pm.current_vddc = voltage->voltage;
1620 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1621 }
1622
1623 /* starting with BTC, there is one state that is used for both
1624 * MH and SH. Difference is that we always use the high clock index for
1625 * mclk and vddci.
1626 */
1627 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1628 (rdev->family >= CHIP_BARTS) &&
1629 rdev->pm.active_crtc_count &&
1630 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1631 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1632 voltage = &rdev->pm.power_state[req_ps_idx].
1633 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1634
1635 /* 0xff0x are flags rather then an actual voltage */
1636 if ((voltage->vddci & 0xff00) == 0xff00)
1637 return;
1638 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1639 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1640 rdev->pm.current_vddci = voltage->vddci;
1641 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1642 }
1643 }
1644 }
1645
1646 /**
1647 * evergreen_pm_prepare - pre-power state change callback.
1648 *
1649 * @rdev: radeon_device pointer
1650 *
1651 * Prepare for a power state change (evergreen+).
1652 */
evergreen_pm_prepare(struct radeon_device * rdev)1653 void evergreen_pm_prepare(struct radeon_device *rdev)
1654 {
1655 struct drm_device *ddev = rdev->ddev;
1656 struct drm_crtc *crtc;
1657 struct radeon_crtc *radeon_crtc;
1658 u32 tmp;
1659
1660 /* disable any active CRTCs */
1661 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1662 radeon_crtc = to_radeon_crtc(crtc);
1663 if (radeon_crtc->enabled) {
1664 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1665 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1666 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1667 }
1668 }
1669 }
1670
1671 /**
1672 * evergreen_pm_finish - post-power state change callback.
1673 *
1674 * @rdev: radeon_device pointer
1675 *
1676 * Clean up after a power state change (evergreen+).
1677 */
evergreen_pm_finish(struct radeon_device * rdev)1678 void evergreen_pm_finish(struct radeon_device *rdev)
1679 {
1680 struct drm_device *ddev = rdev->ddev;
1681 struct drm_crtc *crtc;
1682 struct radeon_crtc *radeon_crtc;
1683 u32 tmp;
1684
1685 /* enable any active CRTCs */
1686 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1687 radeon_crtc = to_radeon_crtc(crtc);
1688 if (radeon_crtc->enabled) {
1689 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1690 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1691 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1692 }
1693 }
1694 }
1695
1696 /**
1697 * evergreen_hpd_sense - hpd sense callback.
1698 *
1699 * @rdev: radeon_device pointer
1700 * @hpd: hpd (hotplug detect) pin
1701 *
1702 * Checks if a digital monitor is connected (evergreen+).
1703 * Returns true if connected, false if not connected.
1704 */
evergreen_hpd_sense(struct radeon_device * rdev,enum radeon_hpd_id hpd)1705 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1706 {
1707 if (hpd == RADEON_HPD_NONE)
1708 return false;
1709
1710 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1711 }
1712
1713 /**
1714 * evergreen_hpd_set_polarity - hpd set polarity callback.
1715 *
1716 * @rdev: radeon_device pointer
1717 * @hpd: hpd (hotplug detect) pin
1718 *
1719 * Set the polarity of the hpd pin (evergreen+).
1720 */
evergreen_hpd_set_polarity(struct radeon_device * rdev,enum radeon_hpd_id hpd)1721 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1722 enum radeon_hpd_id hpd)
1723 {
1724 bool connected = evergreen_hpd_sense(rdev, hpd);
1725
1726 if (hpd == RADEON_HPD_NONE)
1727 return;
1728
1729 if (connected)
1730 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1731 else
1732 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1733 }
1734
1735 /**
1736 * evergreen_hpd_init - hpd setup callback.
1737 *
1738 * @rdev: radeon_device pointer
1739 *
1740 * Setup the hpd pins used by the card (evergreen+).
1741 * Enable the pin, set the polarity, and enable the hpd interrupts.
1742 */
evergreen_hpd_init(struct radeon_device * rdev)1743 void evergreen_hpd_init(struct radeon_device *rdev)
1744 {
1745 struct drm_device *dev = rdev->ddev;
1746 struct drm_connector *connector;
1747 unsigned enabled = 0;
1748 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1749 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1750
1751 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1752 enum radeon_hpd_id hpd =
1753 to_radeon_connector(connector)->hpd.hpd;
1754
1755 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1756 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1757 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1758 * aux dp channel on imac and help (but not completely fix)
1759 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1760 * also avoid interrupt storms during dpms.
1761 */
1762 continue;
1763 }
1764
1765 if (hpd == RADEON_HPD_NONE)
1766 continue;
1767
1768 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1769 enabled |= 1 << hpd;
1770
1771 radeon_hpd_set_polarity(rdev, hpd);
1772 }
1773 radeon_irq_kms_enable_hpd(rdev, enabled);
1774 }
1775
1776 /**
1777 * evergreen_hpd_fini - hpd tear down callback.
1778 *
1779 * @rdev: radeon_device pointer
1780 *
1781 * Tear down the hpd pins used by the card (evergreen+).
1782 * Disable the hpd interrupts.
1783 */
evergreen_hpd_fini(struct radeon_device * rdev)1784 void evergreen_hpd_fini(struct radeon_device *rdev)
1785 {
1786 struct drm_device *dev = rdev->ddev;
1787 struct drm_connector *connector;
1788 unsigned disabled = 0;
1789
1790 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1791 enum radeon_hpd_id hpd =
1792 to_radeon_connector(connector)->hpd.hpd;
1793
1794 if (hpd == RADEON_HPD_NONE)
1795 continue;
1796
1797 WREG32(DC_HPDx_CONTROL(hpd), 0);
1798 disabled |= 1 << hpd;
1799 }
1800 radeon_irq_kms_disable_hpd(rdev, disabled);
1801 }
1802
1803 /* watermark setup */
1804
evergreen_line_buffer_adjust(struct radeon_device * rdev,struct radeon_crtc * radeon_crtc,struct drm_display_mode * mode,struct drm_display_mode * other_mode)1805 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1806 struct radeon_crtc *radeon_crtc,
1807 struct drm_display_mode *mode,
1808 struct drm_display_mode *other_mode)
1809 {
1810 u32 tmp, buffer_alloc, i;
1811 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1812 /*
1813 * Line Buffer Setup
1814 * There are 3 line buffers, each one shared by 2 display controllers.
1815 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1816 * the display controllers. The paritioning is done via one of four
1817 * preset allocations specified in bits 2:0:
1818 * first display controller
1819 * 0 - first half of lb (3840 * 2)
1820 * 1 - first 3/4 of lb (5760 * 2)
1821 * 2 - whole lb (7680 * 2), other crtc must be disabled
1822 * 3 - first 1/4 of lb (1920 * 2)
1823 * second display controller
1824 * 4 - second half of lb (3840 * 2)
1825 * 5 - second 3/4 of lb (5760 * 2)
1826 * 6 - whole lb (7680 * 2), other crtc must be disabled
1827 * 7 - last 1/4 of lb (1920 * 2)
1828 */
1829 /* this can get tricky if we have two large displays on a paired group
1830 * of crtcs. Ideally for multiple large displays we'd assign them to
1831 * non-linked crtcs for maximum line buffer allocation.
1832 */
1833 if (radeon_crtc->base.enabled && mode) {
1834 if (other_mode) {
1835 tmp = 0; /* 1/2 */
1836 buffer_alloc = 1;
1837 } else {
1838 tmp = 2; /* whole */
1839 buffer_alloc = 2;
1840 }
1841 } else {
1842 tmp = 0;
1843 buffer_alloc = 0;
1844 }
1845
1846 /* second controller of the pair uses second half of the lb */
1847 if (radeon_crtc->crtc_id % 2)
1848 tmp += 4;
1849 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1850
1851 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1852 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1853 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1854 for (i = 0; i < rdev->usec_timeout; i++) {
1855 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1856 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1857 break;
1858 udelay(1);
1859 }
1860 }
1861
1862 if (radeon_crtc->base.enabled && mode) {
1863 switch (tmp) {
1864 case 0:
1865 case 4:
1866 default:
1867 if (ASIC_IS_DCE5(rdev))
1868 return 4096 * 2;
1869 else
1870 return 3840 * 2;
1871 case 1:
1872 case 5:
1873 if (ASIC_IS_DCE5(rdev))
1874 return 6144 * 2;
1875 else
1876 return 5760 * 2;
1877 case 2:
1878 case 6:
1879 if (ASIC_IS_DCE5(rdev))
1880 return 8192 * 2;
1881 else
1882 return 7680 * 2;
1883 case 3:
1884 case 7:
1885 if (ASIC_IS_DCE5(rdev))
1886 return 2048 * 2;
1887 else
1888 return 1920 * 2;
1889 }
1890 }
1891
1892 /* controller not enabled, so no lb used */
1893 return 0;
1894 }
1895
evergreen_get_number_of_dram_channels(struct radeon_device * rdev)1896 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1897 {
1898 u32 tmp = RREG32(MC_SHARED_CHMAP);
1899
1900 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1901 case 0:
1902 default:
1903 return 1;
1904 case 1:
1905 return 2;
1906 case 2:
1907 return 4;
1908 case 3:
1909 return 8;
1910 }
1911 }
1912
1913 struct evergreen_wm_params {
1914 u32 dram_channels; /* number of dram channels */
1915 u32 yclk; /* bandwidth per dram data pin in kHz */
1916 u32 sclk; /* engine clock in kHz */
1917 u32 disp_clk; /* display clock in kHz */
1918 u32 src_width; /* viewport width */
1919 u32 active_time; /* active display time in ns */
1920 u32 blank_time; /* blank time in ns */
1921 bool interlaced; /* mode is interlaced */
1922 fixed20_12 vsc; /* vertical scale ratio */
1923 u32 num_heads; /* number of active crtcs */
1924 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1925 u32 lb_size; /* line buffer allocated to pipe */
1926 u32 vtaps; /* vertical scaler taps */
1927 };
1928
evergreen_dram_bandwidth(struct evergreen_wm_params * wm)1929 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1930 {
1931 /* Calculate DRAM Bandwidth and the part allocated to display. */
1932 fixed20_12 dram_efficiency; /* 0.7 */
1933 fixed20_12 yclk, dram_channels, bandwidth;
1934 fixed20_12 a;
1935
1936 a.full = dfixed_const(1000);
1937 yclk.full = dfixed_const(wm->yclk);
1938 yclk.full = dfixed_div(yclk, a);
1939 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1940 a.full = dfixed_const(10);
1941 dram_efficiency.full = dfixed_const(7);
1942 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1943 bandwidth.full = dfixed_mul(dram_channels, yclk);
1944 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1945
1946 return dfixed_trunc(bandwidth);
1947 }
1948
evergreen_dram_bandwidth_for_display(struct evergreen_wm_params * wm)1949 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1950 {
1951 /* Calculate DRAM Bandwidth and the part allocated to display. */
1952 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1953 fixed20_12 yclk, dram_channels, bandwidth;
1954 fixed20_12 a;
1955
1956 a.full = dfixed_const(1000);
1957 yclk.full = dfixed_const(wm->yclk);
1958 yclk.full = dfixed_div(yclk, a);
1959 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1960 a.full = dfixed_const(10);
1961 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1962 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1963 bandwidth.full = dfixed_mul(dram_channels, yclk);
1964 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1965
1966 return dfixed_trunc(bandwidth);
1967 }
1968
evergreen_data_return_bandwidth(struct evergreen_wm_params * wm)1969 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1970 {
1971 /* Calculate the display Data return Bandwidth */
1972 fixed20_12 return_efficiency; /* 0.8 */
1973 fixed20_12 sclk, bandwidth;
1974 fixed20_12 a;
1975
1976 a.full = dfixed_const(1000);
1977 sclk.full = dfixed_const(wm->sclk);
1978 sclk.full = dfixed_div(sclk, a);
1979 a.full = dfixed_const(10);
1980 return_efficiency.full = dfixed_const(8);
1981 return_efficiency.full = dfixed_div(return_efficiency, a);
1982 a.full = dfixed_const(32);
1983 bandwidth.full = dfixed_mul(a, sclk);
1984 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1985
1986 return dfixed_trunc(bandwidth);
1987 }
1988
evergreen_dmif_request_bandwidth(struct evergreen_wm_params * wm)1989 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1990 {
1991 /* Calculate the DMIF Request Bandwidth */
1992 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1993 fixed20_12 disp_clk, bandwidth;
1994 fixed20_12 a;
1995
1996 a.full = dfixed_const(1000);
1997 disp_clk.full = dfixed_const(wm->disp_clk);
1998 disp_clk.full = dfixed_div(disp_clk, a);
1999 a.full = dfixed_const(10);
2000 disp_clk_request_efficiency.full = dfixed_const(8);
2001 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2002 a.full = dfixed_const(32);
2003 bandwidth.full = dfixed_mul(a, disp_clk);
2004 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2005
2006 return dfixed_trunc(bandwidth);
2007 }
2008
evergreen_available_bandwidth(struct evergreen_wm_params * wm)2009 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2010 {
2011 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2012 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2013 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2014 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2015
2016 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2017 }
2018
evergreen_average_bandwidth(struct evergreen_wm_params * wm)2019 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2020 {
2021 /* Calculate the display mode Average Bandwidth
2022 * DisplayMode should contain the source and destination dimensions,
2023 * timing, etc.
2024 */
2025 fixed20_12 bpp;
2026 fixed20_12 line_time;
2027 fixed20_12 src_width;
2028 fixed20_12 bandwidth;
2029 fixed20_12 a;
2030
2031 a.full = dfixed_const(1000);
2032 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2033 line_time.full = dfixed_div(line_time, a);
2034 bpp.full = dfixed_const(wm->bytes_per_pixel);
2035 src_width.full = dfixed_const(wm->src_width);
2036 bandwidth.full = dfixed_mul(src_width, bpp);
2037 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2038 bandwidth.full = dfixed_div(bandwidth, line_time);
2039
2040 return dfixed_trunc(bandwidth);
2041 }
2042
evergreen_latency_watermark(struct evergreen_wm_params * wm)2043 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2044 {
2045 /* First calcualte the latency in ns */
2046 u32 mc_latency = 2000; /* 2000 ns. */
2047 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2048 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2049 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2050 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2051 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2052 (wm->num_heads * cursor_line_pair_return_time);
2053 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2054 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2055 fixed20_12 a, b, c;
2056
2057 if (wm->num_heads == 0)
2058 return 0;
2059
2060 a.full = dfixed_const(2);
2061 b.full = dfixed_const(1);
2062 if ((wm->vsc.full > a.full) ||
2063 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2064 (wm->vtaps >= 5) ||
2065 ((wm->vsc.full >= a.full) && wm->interlaced))
2066 max_src_lines_per_dst_line = 4;
2067 else
2068 max_src_lines_per_dst_line = 2;
2069
2070 a.full = dfixed_const(available_bandwidth);
2071 b.full = dfixed_const(wm->num_heads);
2072 a.full = dfixed_div(a, b);
2073
2074 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2075
2076 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2077 b.full = dfixed_const(1000);
2078 c.full = dfixed_const(lb_fill_bw);
2079 b.full = dfixed_div(c, b);
2080 a.full = dfixed_div(a, b);
2081 line_fill_time = dfixed_trunc(a);
2082
2083 if (line_fill_time < wm->active_time)
2084 return latency;
2085 else
2086 return latency + (line_fill_time - wm->active_time);
2087
2088 }
2089
evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params * wm)2090 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2091 {
2092 if (evergreen_average_bandwidth(wm) <=
2093 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2094 return true;
2095 else
2096 return false;
2097 };
2098
evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params * wm)2099 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2100 {
2101 if (evergreen_average_bandwidth(wm) <=
2102 (evergreen_available_bandwidth(wm) / wm->num_heads))
2103 return true;
2104 else
2105 return false;
2106 };
2107
evergreen_check_latency_hiding(struct evergreen_wm_params * wm)2108 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2109 {
2110 u32 lb_partitions = wm->lb_size / wm->src_width;
2111 u32 line_time = wm->active_time + wm->blank_time;
2112 u32 latency_tolerant_lines;
2113 u32 latency_hiding;
2114 fixed20_12 a;
2115
2116 a.full = dfixed_const(1);
2117 if (wm->vsc.full > a.full)
2118 latency_tolerant_lines = 1;
2119 else {
2120 if (lb_partitions <= (wm->vtaps + 1))
2121 latency_tolerant_lines = 1;
2122 else
2123 latency_tolerant_lines = 2;
2124 }
2125
2126 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2127
2128 if (evergreen_latency_watermark(wm) <= latency_hiding)
2129 return true;
2130 else
2131 return false;
2132 }
2133
evergreen_program_watermarks(struct radeon_device * rdev,struct radeon_crtc * radeon_crtc,u32 lb_size,u32 num_heads)2134 static void evergreen_program_watermarks(struct radeon_device *rdev,
2135 struct radeon_crtc *radeon_crtc,
2136 u32 lb_size, u32 num_heads)
2137 {
2138 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2139 struct evergreen_wm_params wm_low, wm_high;
2140 u32 dram_channels;
2141 u32 active_time;
2142 u32 line_time = 0;
2143 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2144 u32 priority_a_mark = 0, priority_b_mark = 0;
2145 u32 priority_a_cnt = PRIORITY_OFF;
2146 u32 priority_b_cnt = PRIORITY_OFF;
2147 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2148 u32 tmp, arb_control3;
2149 fixed20_12 a, b, c;
2150
2151 if (radeon_crtc->base.enabled && num_heads && mode) {
2152 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2153 (u32)mode->clock);
2154 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2155 (u32)mode->clock);
2156 line_time = min(line_time, (u32)65535);
2157 priority_a_cnt = 0;
2158 priority_b_cnt = 0;
2159 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2160
2161 /* watermark for high clocks */
2162 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2163 wm_high.yclk =
2164 radeon_dpm_get_mclk(rdev, false) * 10;
2165 wm_high.sclk =
2166 radeon_dpm_get_sclk(rdev, false) * 10;
2167 } else {
2168 wm_high.yclk = rdev->pm.current_mclk * 10;
2169 wm_high.sclk = rdev->pm.current_sclk * 10;
2170 }
2171
2172 wm_high.disp_clk = mode->clock;
2173 wm_high.src_width = mode->crtc_hdisplay;
2174 wm_high.active_time = active_time;
2175 wm_high.blank_time = line_time - wm_high.active_time;
2176 wm_high.interlaced = false;
2177 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2178 wm_high.interlaced = true;
2179 wm_high.vsc = radeon_crtc->vsc;
2180 wm_high.vtaps = 1;
2181 if (radeon_crtc->rmx_type != RMX_OFF)
2182 wm_high.vtaps = 2;
2183 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2184 wm_high.lb_size = lb_size;
2185 wm_high.dram_channels = dram_channels;
2186 wm_high.num_heads = num_heads;
2187
2188 /* watermark for low clocks */
2189 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2190 wm_low.yclk =
2191 radeon_dpm_get_mclk(rdev, true) * 10;
2192 wm_low.sclk =
2193 radeon_dpm_get_sclk(rdev, true) * 10;
2194 } else {
2195 wm_low.yclk = rdev->pm.current_mclk * 10;
2196 wm_low.sclk = rdev->pm.current_sclk * 10;
2197 }
2198
2199 wm_low.disp_clk = mode->clock;
2200 wm_low.src_width = mode->crtc_hdisplay;
2201 wm_low.active_time = active_time;
2202 wm_low.blank_time = line_time - wm_low.active_time;
2203 wm_low.interlaced = false;
2204 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2205 wm_low.interlaced = true;
2206 wm_low.vsc = radeon_crtc->vsc;
2207 wm_low.vtaps = 1;
2208 if (radeon_crtc->rmx_type != RMX_OFF)
2209 wm_low.vtaps = 2;
2210 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2211 wm_low.lb_size = lb_size;
2212 wm_low.dram_channels = dram_channels;
2213 wm_low.num_heads = num_heads;
2214
2215 /* set for high clocks */
2216 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2217 /* set for low clocks */
2218 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2219
2220 /* possibly force display priority to high */
2221 /* should really do this at mode validation time... */
2222 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2223 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2224 !evergreen_check_latency_hiding(&wm_high) ||
2225 (rdev->disp_priority == 2)) {
2226 DRM_DEBUG_KMS("force priority a to high\n");
2227 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2228 }
2229 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2230 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2231 !evergreen_check_latency_hiding(&wm_low) ||
2232 (rdev->disp_priority == 2)) {
2233 DRM_DEBUG_KMS("force priority b to high\n");
2234 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2235 }
2236
2237 a.full = dfixed_const(1000);
2238 b.full = dfixed_const(mode->clock);
2239 b.full = dfixed_div(b, a);
2240 c.full = dfixed_const(latency_watermark_a);
2241 c.full = dfixed_mul(c, b);
2242 c.full = dfixed_mul(c, radeon_crtc->hsc);
2243 c.full = dfixed_div(c, a);
2244 a.full = dfixed_const(16);
2245 c.full = dfixed_div(c, a);
2246 priority_a_mark = dfixed_trunc(c);
2247 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2248
2249 a.full = dfixed_const(1000);
2250 b.full = dfixed_const(mode->clock);
2251 b.full = dfixed_div(b, a);
2252 c.full = dfixed_const(latency_watermark_b);
2253 c.full = dfixed_mul(c, b);
2254 c.full = dfixed_mul(c, radeon_crtc->hsc);
2255 c.full = dfixed_div(c, a);
2256 a.full = dfixed_const(16);
2257 c.full = dfixed_div(c, a);
2258 priority_b_mark = dfixed_trunc(c);
2259 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2260
2261 /* Save number of lines the linebuffer leads before the scanout */
2262 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2263 }
2264
2265 /* select wm A */
2266 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2267 tmp = arb_control3;
2268 tmp &= ~LATENCY_WATERMARK_MASK(3);
2269 tmp |= LATENCY_WATERMARK_MASK(1);
2270 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2271 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2272 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2273 LATENCY_HIGH_WATERMARK(line_time)));
2274 /* select wm B */
2275 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2276 tmp &= ~LATENCY_WATERMARK_MASK(3);
2277 tmp |= LATENCY_WATERMARK_MASK(2);
2278 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2279 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2280 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2281 LATENCY_HIGH_WATERMARK(line_time)));
2282 /* restore original selection */
2283 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2284
2285 /* write the priority marks */
2286 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2287 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2288
2289 /* save values for DPM */
2290 radeon_crtc->line_time = line_time;
2291 radeon_crtc->wm_high = latency_watermark_a;
2292 radeon_crtc->wm_low = latency_watermark_b;
2293 }
2294
2295 /**
2296 * evergreen_bandwidth_update - update display watermarks callback.
2297 *
2298 * @rdev: radeon_device pointer
2299 *
2300 * Update the display watermarks based on the requested mode(s)
2301 * (evergreen+).
2302 */
evergreen_bandwidth_update(struct radeon_device * rdev)2303 void evergreen_bandwidth_update(struct radeon_device *rdev)
2304 {
2305 struct drm_display_mode *mode0 = NULL;
2306 struct drm_display_mode *mode1 = NULL;
2307 u32 num_heads = 0, lb_size;
2308 int i;
2309
2310 if (!rdev->mode_info.mode_config_initialized)
2311 return;
2312
2313 radeon_update_display_priority(rdev);
2314
2315 for (i = 0; i < rdev->num_crtc; i++) {
2316 if (rdev->mode_info.crtcs[i]->base.enabled)
2317 num_heads++;
2318 }
2319 for (i = 0; i < rdev->num_crtc; i += 2) {
2320 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2321 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2322 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2323 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2324 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2325 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2326 }
2327 }
2328
2329 /**
2330 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2331 *
2332 * @rdev: radeon_device pointer
2333 *
2334 * Wait for the MC (memory controller) to be idle.
2335 * (evergreen+).
2336 * Returns 0 if the MC is idle, -1 if not.
2337 */
evergreen_mc_wait_for_idle(struct radeon_device * rdev)2338 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2339 {
2340 unsigned i;
2341 u32 tmp;
2342
2343 for (i = 0; i < rdev->usec_timeout; i++) {
2344 /* read MC_STATUS */
2345 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2346 if (!tmp)
2347 return 0;
2348 udelay(1);
2349 }
2350 return -1;
2351 }
2352
2353 /*
2354 * GART
2355 */
evergreen_pcie_gart_tlb_flush(struct radeon_device * rdev)2356 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2357 {
2358 unsigned i;
2359 u32 tmp;
2360
2361 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2362
2363 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2364 for (i = 0; i < rdev->usec_timeout; i++) {
2365 /* read MC_STATUS */
2366 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2367 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2368 if (tmp == 2) {
2369 pr_warn("[drm] r600 flush TLB failed\n");
2370 return;
2371 }
2372 if (tmp) {
2373 return;
2374 }
2375 udelay(1);
2376 }
2377 }
2378
evergreen_pcie_gart_enable(struct radeon_device * rdev)2379 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2380 {
2381 u32 tmp;
2382 int r;
2383
2384 if (rdev->gart.robj == NULL) {
2385 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2386 return -EINVAL;
2387 }
2388 r = radeon_gart_table_vram_pin(rdev);
2389 if (r)
2390 return r;
2391 /* Setup L2 cache */
2392 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2393 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2394 EFFECTIVE_L2_QUEUE_SIZE(7));
2395 WREG32(VM_L2_CNTL2, 0);
2396 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2397 /* Setup TLB control */
2398 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2399 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2400 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2401 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2402 if (rdev->flags & RADEON_IS_IGP) {
2403 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2404 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2405 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2406 } else {
2407 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2408 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2409 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2410 if ((rdev->family == CHIP_JUNIPER) ||
2411 (rdev->family == CHIP_CYPRESS) ||
2412 (rdev->family == CHIP_HEMLOCK) ||
2413 (rdev->family == CHIP_BARTS))
2414 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2415 }
2416 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2417 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2418 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2419 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2420 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2421 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2422 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2423 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2424 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2425 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2426 (u32)(rdev->dummy_page.addr >> 12));
2427 WREG32(VM_CONTEXT1_CNTL, 0);
2428
2429 evergreen_pcie_gart_tlb_flush(rdev);
2430 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2431 (unsigned)(rdev->mc.gtt_size >> 20),
2432 (unsigned long long)rdev->gart.table_addr);
2433 rdev->gart.ready = true;
2434 return 0;
2435 }
2436
evergreen_pcie_gart_disable(struct radeon_device * rdev)2437 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2438 {
2439 u32 tmp;
2440
2441 /* Disable all tables */
2442 WREG32(VM_CONTEXT0_CNTL, 0);
2443 WREG32(VM_CONTEXT1_CNTL, 0);
2444
2445 /* Setup L2 cache */
2446 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2447 EFFECTIVE_L2_QUEUE_SIZE(7));
2448 WREG32(VM_L2_CNTL2, 0);
2449 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2450 /* Setup TLB control */
2451 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2452 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2453 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2454 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2455 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2456 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2457 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2458 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2459 radeon_gart_table_vram_unpin(rdev);
2460 }
2461
evergreen_pcie_gart_fini(struct radeon_device * rdev)2462 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2463 {
2464 evergreen_pcie_gart_disable(rdev);
2465 radeon_gart_table_vram_free(rdev);
2466 radeon_gart_fini(rdev);
2467 }
2468
2469
evergreen_agp_enable(struct radeon_device * rdev)2470 static void evergreen_agp_enable(struct radeon_device *rdev)
2471 {
2472 u32 tmp;
2473
2474 /* Setup L2 cache */
2475 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2476 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2477 EFFECTIVE_L2_QUEUE_SIZE(7));
2478 WREG32(VM_L2_CNTL2, 0);
2479 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2480 /* Setup TLB control */
2481 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2482 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2483 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2484 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2485 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2486 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2487 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2488 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2489 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2490 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2491 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2492 WREG32(VM_CONTEXT0_CNTL, 0);
2493 WREG32(VM_CONTEXT1_CNTL, 0);
2494 }
2495
2496 static const unsigned ni_dig_offsets[] =
2497 {
2498 NI_DIG0_REGISTER_OFFSET,
2499 NI_DIG1_REGISTER_OFFSET,
2500 NI_DIG2_REGISTER_OFFSET,
2501 NI_DIG3_REGISTER_OFFSET,
2502 NI_DIG4_REGISTER_OFFSET,
2503 NI_DIG5_REGISTER_OFFSET
2504 };
2505
2506 static const unsigned ni_tx_offsets[] =
2507 {
2508 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2509 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2510 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2511 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2512 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2513 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2514 };
2515
2516 static const unsigned evergreen_dp_offsets[] =
2517 {
2518 EVERGREEN_DP0_REGISTER_OFFSET,
2519 EVERGREEN_DP1_REGISTER_OFFSET,
2520 EVERGREEN_DP2_REGISTER_OFFSET,
2521 EVERGREEN_DP3_REGISTER_OFFSET,
2522 EVERGREEN_DP4_REGISTER_OFFSET,
2523 EVERGREEN_DP5_REGISTER_OFFSET
2524 };
2525
2526 static const unsigned evergreen_disp_int_status[] =
2527 {
2528 DISP_INTERRUPT_STATUS,
2529 DISP_INTERRUPT_STATUS_CONTINUE,
2530 DISP_INTERRUPT_STATUS_CONTINUE2,
2531 DISP_INTERRUPT_STATUS_CONTINUE3,
2532 DISP_INTERRUPT_STATUS_CONTINUE4,
2533 DISP_INTERRUPT_STATUS_CONTINUE5
2534 };
2535
2536 /*
2537 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2538 * We go from crtc to connector and it is not relible since it
2539 * should be an opposite direction .If crtc is enable then
2540 * find the dig_fe which selects this crtc and insure that it enable.
2541 * if such dig_fe is found then find dig_be which selects found dig_be and
2542 * insure that it enable and in DP_SST mode.
2543 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2544 * from dp symbols clocks .
2545 */
evergreen_is_dp_sst_stream_enabled(struct radeon_device * rdev,unsigned crtc_id,unsigned * ret_dig_fe)2546 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2547 unsigned crtc_id, unsigned *ret_dig_fe)
2548 {
2549 unsigned i;
2550 unsigned dig_fe;
2551 unsigned dig_be;
2552 unsigned dig_en_be;
2553 unsigned uniphy_pll;
2554 unsigned digs_fe_selected;
2555 unsigned dig_be_mode;
2556 unsigned dig_fe_mask;
2557 bool is_enabled = false;
2558 bool found_crtc = false;
2559
2560 /* loop through all running dig_fe to find selected crtc */
2561 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2562 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2563 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2564 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2565 /* found running pipe */
2566 found_crtc = true;
2567 dig_fe_mask = 1 << i;
2568 dig_fe = i;
2569 break;
2570 }
2571 }
2572
2573 if (found_crtc) {
2574 /* loop through all running dig_be to find selected dig_fe */
2575 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2576 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2577 /* if dig_fe_selected by dig_be? */
2578 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2579 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2580 if (dig_fe_mask & digs_fe_selected &&
2581 /* if dig_be in sst mode? */
2582 dig_be_mode == NI_DIG_BE_DPSST) {
2583 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2584 ni_dig_offsets[i]);
2585 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2586 ni_tx_offsets[i]);
2587 /* dig_be enable and tx is running */
2588 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2589 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2590 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2591 is_enabled = true;
2592 *ret_dig_fe = dig_fe;
2593 break;
2594 }
2595 }
2596 }
2597 }
2598
2599 return is_enabled;
2600 }
2601
2602 /*
2603 * Blank dig when in dp sst mode
2604 * Dig ignores crtc timing
2605 */
evergreen_blank_dp_output(struct radeon_device * rdev,unsigned dig_fe)2606 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2607 unsigned dig_fe)
2608 {
2609 unsigned stream_ctrl;
2610 unsigned fifo_ctrl;
2611 unsigned counter = 0;
2612
2613 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2614 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2615 return;
2616 }
2617
2618 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2619 evergreen_dp_offsets[dig_fe]);
2620 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2621 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2622 return;
2623 }
2624
2625 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2626 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2627 evergreen_dp_offsets[dig_fe], stream_ctrl);
2628
2629 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2630 evergreen_dp_offsets[dig_fe]);
2631 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2632 msleep(1);
2633 counter++;
2634 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2635 evergreen_dp_offsets[dig_fe]);
2636 }
2637 if (counter >= 32 )
2638 DRM_ERROR("counter exceeds %d\n", counter);
2639
2640 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2641 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2642 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2643
2644 }
2645
evergreen_mc_stop(struct radeon_device * rdev,struct evergreen_mc_save * save)2646 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2647 {
2648 u32 crtc_enabled, tmp, frame_count, blackout;
2649 int i, j;
2650 unsigned dig_fe;
2651
2652 if (!ASIC_IS_NODCE(rdev)) {
2653 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2654 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2655
2656 /* disable VGA render */
2657 WREG32(VGA_RENDER_CONTROL, 0);
2658 }
2659 /* blank the display controllers */
2660 for (i = 0; i < rdev->num_crtc; i++) {
2661 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2662 if (crtc_enabled) {
2663 save->crtc_enabled[i] = true;
2664 if (ASIC_IS_DCE6(rdev)) {
2665 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2666 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2667 radeon_wait_for_vblank(rdev, i);
2668 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2669 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2670 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2671 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2672 }
2673 } else {
2674 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2675 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2676 radeon_wait_for_vblank(rdev, i);
2677 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2678 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2679 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2680 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2681 }
2682 }
2683 /* wait for the next frame */
2684 frame_count = radeon_get_vblank_counter(rdev, i);
2685 for (j = 0; j < rdev->usec_timeout; j++) {
2686 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2687 break;
2688 udelay(1);
2689 }
2690 /*we should disable dig if it drives dp sst*/
2691 /*but we are in radeon_device_init and the topology is unknown*/
2692 /*and it is available after radeon_modeset_init*/
2693 /*the following method radeon_atom_encoder_dpms_dig*/
2694 /*does the job if we initialize it properly*/
2695 /*for now we do it this manually*/
2696 /**/
2697 if (ASIC_IS_DCE5(rdev) &&
2698 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2699 evergreen_blank_dp_output(rdev, dig_fe);
2700 /*we could remove 6 lines below*/
2701 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2702 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2703 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2704 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2705 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2706 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2707 save->crtc_enabled[i] = false;
2708 /* ***** */
2709 } else {
2710 save->crtc_enabled[i] = false;
2711 }
2712 }
2713
2714 radeon_mc_wait_for_idle(rdev);
2715
2716 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2717 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2718 /* Block CPU access */
2719 WREG32(BIF_FB_EN, 0);
2720 /* blackout the MC */
2721 blackout &= ~BLACKOUT_MODE_MASK;
2722 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2723 }
2724 /* wait for the MC to settle */
2725 udelay(100);
2726
2727 /* lock double buffered regs */
2728 for (i = 0; i < rdev->num_crtc; i++) {
2729 if (save->crtc_enabled[i]) {
2730 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2731 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2732 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2733 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2734 }
2735 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2736 if (!(tmp & 1)) {
2737 tmp |= 1;
2738 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2739 }
2740 }
2741 }
2742 }
2743
evergreen_mc_resume(struct radeon_device * rdev,struct evergreen_mc_save * save)2744 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2745 {
2746 u32 tmp, frame_count;
2747 int i, j;
2748
2749 /* update crtc base addresses */
2750 for (i = 0; i < rdev->num_crtc; i++) {
2751 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2752 upper_32_bits(rdev->mc.vram_start));
2753 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2754 upper_32_bits(rdev->mc.vram_start));
2755 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2756 (u32)rdev->mc.vram_start);
2757 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2758 (u32)rdev->mc.vram_start);
2759 }
2760
2761 if (!ASIC_IS_NODCE(rdev)) {
2762 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2763 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2764 }
2765
2766 /* unlock regs and wait for update */
2767 for (i = 0; i < rdev->num_crtc; i++) {
2768 if (save->crtc_enabled[i]) {
2769 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2770 if ((tmp & 0x7) != 0) {
2771 tmp &= ~0x7;
2772 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2773 }
2774 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2775 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2776 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2777 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2778 }
2779 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2780 if (tmp & 1) {
2781 tmp &= ~1;
2782 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2783 }
2784 for (j = 0; j < rdev->usec_timeout; j++) {
2785 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2786 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2787 break;
2788 udelay(1);
2789 }
2790 }
2791 }
2792
2793 /* unblackout the MC */
2794 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2795 tmp &= ~BLACKOUT_MODE_MASK;
2796 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2797 /* allow CPU access */
2798 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2799
2800 for (i = 0; i < rdev->num_crtc; i++) {
2801 if (save->crtc_enabled[i]) {
2802 if (ASIC_IS_DCE6(rdev)) {
2803 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2804 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2805 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2806 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2807 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2808 } else {
2809 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2810 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2811 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2812 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2813 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2814 }
2815 /* wait for the next frame */
2816 frame_count = radeon_get_vblank_counter(rdev, i);
2817 for (j = 0; j < rdev->usec_timeout; j++) {
2818 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2819 break;
2820 udelay(1);
2821 }
2822 }
2823 }
2824 if (!ASIC_IS_NODCE(rdev)) {
2825 /* Unlock vga access */
2826 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2827 mdelay(1);
2828 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2829 }
2830 }
2831
evergreen_mc_program(struct radeon_device * rdev)2832 void evergreen_mc_program(struct radeon_device *rdev)
2833 {
2834 struct evergreen_mc_save save;
2835 u32 tmp;
2836 int i, j;
2837
2838 /* Initialize HDP */
2839 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2840 WREG32((0x2c14 + j), 0x00000000);
2841 WREG32((0x2c18 + j), 0x00000000);
2842 WREG32((0x2c1c + j), 0x00000000);
2843 WREG32((0x2c20 + j), 0x00000000);
2844 WREG32((0x2c24 + j), 0x00000000);
2845 }
2846 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2847
2848 evergreen_mc_stop(rdev, &save);
2849 if (evergreen_mc_wait_for_idle(rdev)) {
2850 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2851 }
2852 /* Lockout access through VGA aperture*/
2853 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2854 /* Update configuration */
2855 if (rdev->flags & RADEON_IS_AGP) {
2856 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2857 /* VRAM before AGP */
2858 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2859 rdev->mc.vram_start >> 12);
2860 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2861 rdev->mc.gtt_end >> 12);
2862 } else {
2863 /* VRAM after AGP */
2864 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2865 rdev->mc.gtt_start >> 12);
2866 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2867 rdev->mc.vram_end >> 12);
2868 }
2869 } else {
2870 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2871 rdev->mc.vram_start >> 12);
2872 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2873 rdev->mc.vram_end >> 12);
2874 }
2875 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2876 /* llano/ontario only */
2877 if ((rdev->family == CHIP_PALM) ||
2878 (rdev->family == CHIP_SUMO) ||
2879 (rdev->family == CHIP_SUMO2)) {
2880 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2881 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2882 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2883 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2884 }
2885 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2886 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2887 WREG32(MC_VM_FB_LOCATION, tmp);
2888 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2889 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2890 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2891 if (rdev->flags & RADEON_IS_AGP) {
2892 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2893 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2894 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2895 } else {
2896 WREG32(MC_VM_AGP_BASE, 0);
2897 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2898 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2899 }
2900 if (evergreen_mc_wait_for_idle(rdev)) {
2901 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2902 }
2903 evergreen_mc_resume(rdev, &save);
2904 /* we need to own VRAM, so turn off the VGA renderer here
2905 * to stop it overwriting our objects */
2906 rv515_vga_render_disable(rdev);
2907 }
2908
2909 /*
2910 * CP.
2911 */
evergreen_ring_ib_execute(struct radeon_device * rdev,struct radeon_ib * ib)2912 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2913 {
2914 struct radeon_ring *ring = &rdev->ring[ib->ring];
2915 u32 next_rptr;
2916
2917 /* set to DX10/11 mode */
2918 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2919 radeon_ring_write(ring, 1);
2920
2921 if (ring->rptr_save_reg) {
2922 next_rptr = ring->wptr + 3 + 4;
2923 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2924 radeon_ring_write(ring, ((ring->rptr_save_reg -
2925 PACKET3_SET_CONFIG_REG_START) >> 2));
2926 radeon_ring_write(ring, next_rptr);
2927 } else if (rdev->wb.enabled) {
2928 next_rptr = ring->wptr + 5 + 4;
2929 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2930 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2931 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2932 radeon_ring_write(ring, next_rptr);
2933 radeon_ring_write(ring, 0);
2934 }
2935
2936 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2937 radeon_ring_write(ring,
2938 #ifdef __BIG_ENDIAN
2939 (2 << 0) |
2940 #endif
2941 (ib->gpu_addr & 0xFFFFFFFC));
2942 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2943 radeon_ring_write(ring, ib->length_dw);
2944 }
2945
2946
evergreen_cp_load_microcode(struct radeon_device * rdev)2947 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2948 {
2949 const __be32 *fw_data;
2950 int i;
2951
2952 if (!rdev->me_fw || !rdev->pfp_fw)
2953 return -EINVAL;
2954
2955 r700_cp_stop(rdev);
2956 WREG32(CP_RB_CNTL,
2957 #ifdef __BIG_ENDIAN
2958 BUF_SWAP_32BIT |
2959 #endif
2960 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2961
2962 fw_data = (const __be32 *)rdev->pfp_fw->data;
2963 WREG32(CP_PFP_UCODE_ADDR, 0);
2964 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2965 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2966 WREG32(CP_PFP_UCODE_ADDR, 0);
2967
2968 fw_data = (const __be32 *)rdev->me_fw->data;
2969 WREG32(CP_ME_RAM_WADDR, 0);
2970 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2971 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2972
2973 WREG32(CP_PFP_UCODE_ADDR, 0);
2974 WREG32(CP_ME_RAM_WADDR, 0);
2975 WREG32(CP_ME_RAM_RADDR, 0);
2976 return 0;
2977 }
2978
evergreen_cp_start(struct radeon_device * rdev)2979 static int evergreen_cp_start(struct radeon_device *rdev)
2980 {
2981 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2982 int r, i;
2983 uint32_t cp_me;
2984
2985 r = radeon_ring_lock(rdev, ring, 7);
2986 if (r) {
2987 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2988 return r;
2989 }
2990 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2991 radeon_ring_write(ring, 0x1);
2992 radeon_ring_write(ring, 0x0);
2993 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2994 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2995 radeon_ring_write(ring, 0);
2996 radeon_ring_write(ring, 0);
2997 radeon_ring_unlock_commit(rdev, ring, false);
2998
2999 cp_me = 0xff;
3000 WREG32(CP_ME_CNTL, cp_me);
3001
3002 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3003 if (r) {
3004 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3005 return r;
3006 }
3007
3008 /* setup clear context state */
3009 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3010 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3011
3012 for (i = 0; i < evergreen_default_size; i++)
3013 radeon_ring_write(ring, evergreen_default_state[i]);
3014
3015 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3016 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3017
3018 /* set clear context state */
3019 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3020 radeon_ring_write(ring, 0);
3021
3022 /* SQ_VTX_BASE_VTX_LOC */
3023 radeon_ring_write(ring, 0xc0026f00);
3024 radeon_ring_write(ring, 0x00000000);
3025 radeon_ring_write(ring, 0x00000000);
3026 radeon_ring_write(ring, 0x00000000);
3027
3028 /* Clear consts */
3029 radeon_ring_write(ring, 0xc0036f00);
3030 radeon_ring_write(ring, 0x00000bc4);
3031 radeon_ring_write(ring, 0xffffffff);
3032 radeon_ring_write(ring, 0xffffffff);
3033 radeon_ring_write(ring, 0xffffffff);
3034
3035 radeon_ring_write(ring, 0xc0026900);
3036 radeon_ring_write(ring, 0x00000316);
3037 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3038 radeon_ring_write(ring, 0x00000010); /* */
3039
3040 radeon_ring_unlock_commit(rdev, ring, false);
3041
3042 return 0;
3043 }
3044
evergreen_cp_resume(struct radeon_device * rdev)3045 static int evergreen_cp_resume(struct radeon_device *rdev)
3046 {
3047 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3048 u32 tmp;
3049 u32 rb_bufsz;
3050 int r;
3051
3052 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3053 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3054 SOFT_RESET_PA |
3055 SOFT_RESET_SH |
3056 SOFT_RESET_VGT |
3057 SOFT_RESET_SPI |
3058 SOFT_RESET_SX));
3059 RREG32(GRBM_SOFT_RESET);
3060 mdelay(15);
3061 WREG32(GRBM_SOFT_RESET, 0);
3062 RREG32(GRBM_SOFT_RESET);
3063
3064 /* Set ring buffer size */
3065 rb_bufsz = order_base_2(ring->ring_size / 8);
3066 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3067 #ifdef __BIG_ENDIAN
3068 tmp |= BUF_SWAP_32BIT;
3069 #endif
3070 WREG32(CP_RB_CNTL, tmp);
3071 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3072 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3073
3074 /* Set the write pointer delay */
3075 WREG32(CP_RB_WPTR_DELAY, 0);
3076
3077 /* Initialize the ring buffer's read and write pointers */
3078 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3079 WREG32(CP_RB_RPTR_WR, 0);
3080 ring->wptr = 0;
3081 WREG32(CP_RB_WPTR, ring->wptr);
3082
3083 /* set the wb address whether it's enabled or not */
3084 WREG32(CP_RB_RPTR_ADDR,
3085 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3086 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3087 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3088
3089 if (rdev->wb.enabled)
3090 WREG32(SCRATCH_UMSK, 0xff);
3091 else {
3092 tmp |= RB_NO_UPDATE;
3093 WREG32(SCRATCH_UMSK, 0);
3094 }
3095
3096 mdelay(1);
3097 WREG32(CP_RB_CNTL, tmp);
3098
3099 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3100 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3101
3102 evergreen_cp_start(rdev);
3103 ring->ready = true;
3104 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3105 if (r) {
3106 ring->ready = false;
3107 return r;
3108 }
3109 return 0;
3110 }
3111
3112 /*
3113 * Core functions
3114 */
evergreen_gpu_init(struct radeon_device * rdev)3115 static void evergreen_gpu_init(struct radeon_device *rdev)
3116 {
3117 u32 gb_addr_config;
3118 u32 mc_shared_chmap, mc_arb_ramcfg;
3119 u32 sx_debug_1;
3120 u32 smx_dc_ctl0;
3121 u32 sq_config;
3122 u32 sq_lds_resource_mgmt;
3123 u32 sq_gpr_resource_mgmt_1;
3124 u32 sq_gpr_resource_mgmt_2;
3125 u32 sq_gpr_resource_mgmt_3;
3126 u32 sq_thread_resource_mgmt;
3127 u32 sq_thread_resource_mgmt_2;
3128 u32 sq_stack_resource_mgmt_1;
3129 u32 sq_stack_resource_mgmt_2;
3130 u32 sq_stack_resource_mgmt_3;
3131 u32 vgt_cache_invalidation;
3132 u32 hdp_host_path_cntl, tmp;
3133 u32 disabled_rb_mask;
3134 int i, j, ps_thread_count;
3135
3136 switch (rdev->family) {
3137 case CHIP_CYPRESS:
3138 case CHIP_HEMLOCK:
3139 rdev->config.evergreen.num_ses = 2;
3140 rdev->config.evergreen.max_pipes = 4;
3141 rdev->config.evergreen.max_tile_pipes = 8;
3142 rdev->config.evergreen.max_simds = 10;
3143 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3144 rdev->config.evergreen.max_gprs = 256;
3145 rdev->config.evergreen.max_threads = 248;
3146 rdev->config.evergreen.max_gs_threads = 32;
3147 rdev->config.evergreen.max_stack_entries = 512;
3148 rdev->config.evergreen.sx_num_of_sets = 4;
3149 rdev->config.evergreen.sx_max_export_size = 256;
3150 rdev->config.evergreen.sx_max_export_pos_size = 64;
3151 rdev->config.evergreen.sx_max_export_smx_size = 192;
3152 rdev->config.evergreen.max_hw_contexts = 8;
3153 rdev->config.evergreen.sq_num_cf_insts = 2;
3154
3155 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3156 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3157 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3158 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3159 break;
3160 case CHIP_JUNIPER:
3161 rdev->config.evergreen.num_ses = 1;
3162 rdev->config.evergreen.max_pipes = 4;
3163 rdev->config.evergreen.max_tile_pipes = 4;
3164 rdev->config.evergreen.max_simds = 10;
3165 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3166 rdev->config.evergreen.max_gprs = 256;
3167 rdev->config.evergreen.max_threads = 248;
3168 rdev->config.evergreen.max_gs_threads = 32;
3169 rdev->config.evergreen.max_stack_entries = 512;
3170 rdev->config.evergreen.sx_num_of_sets = 4;
3171 rdev->config.evergreen.sx_max_export_size = 256;
3172 rdev->config.evergreen.sx_max_export_pos_size = 64;
3173 rdev->config.evergreen.sx_max_export_smx_size = 192;
3174 rdev->config.evergreen.max_hw_contexts = 8;
3175 rdev->config.evergreen.sq_num_cf_insts = 2;
3176
3177 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3178 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3179 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3180 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3181 break;
3182 case CHIP_REDWOOD:
3183 rdev->config.evergreen.num_ses = 1;
3184 rdev->config.evergreen.max_pipes = 4;
3185 rdev->config.evergreen.max_tile_pipes = 4;
3186 rdev->config.evergreen.max_simds = 5;
3187 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3188 rdev->config.evergreen.max_gprs = 256;
3189 rdev->config.evergreen.max_threads = 248;
3190 rdev->config.evergreen.max_gs_threads = 32;
3191 rdev->config.evergreen.max_stack_entries = 256;
3192 rdev->config.evergreen.sx_num_of_sets = 4;
3193 rdev->config.evergreen.sx_max_export_size = 256;
3194 rdev->config.evergreen.sx_max_export_pos_size = 64;
3195 rdev->config.evergreen.sx_max_export_smx_size = 192;
3196 rdev->config.evergreen.max_hw_contexts = 8;
3197 rdev->config.evergreen.sq_num_cf_insts = 2;
3198
3199 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3200 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3201 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3202 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3203 break;
3204 case CHIP_CEDAR:
3205 default:
3206 rdev->config.evergreen.num_ses = 1;
3207 rdev->config.evergreen.max_pipes = 2;
3208 rdev->config.evergreen.max_tile_pipes = 2;
3209 rdev->config.evergreen.max_simds = 2;
3210 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3211 rdev->config.evergreen.max_gprs = 256;
3212 rdev->config.evergreen.max_threads = 192;
3213 rdev->config.evergreen.max_gs_threads = 16;
3214 rdev->config.evergreen.max_stack_entries = 256;
3215 rdev->config.evergreen.sx_num_of_sets = 4;
3216 rdev->config.evergreen.sx_max_export_size = 128;
3217 rdev->config.evergreen.sx_max_export_pos_size = 32;
3218 rdev->config.evergreen.sx_max_export_smx_size = 96;
3219 rdev->config.evergreen.max_hw_contexts = 4;
3220 rdev->config.evergreen.sq_num_cf_insts = 1;
3221
3222 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3223 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3224 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3225 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3226 break;
3227 case CHIP_PALM:
3228 rdev->config.evergreen.num_ses = 1;
3229 rdev->config.evergreen.max_pipes = 2;
3230 rdev->config.evergreen.max_tile_pipes = 2;
3231 rdev->config.evergreen.max_simds = 2;
3232 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3233 rdev->config.evergreen.max_gprs = 256;
3234 rdev->config.evergreen.max_threads = 192;
3235 rdev->config.evergreen.max_gs_threads = 16;
3236 rdev->config.evergreen.max_stack_entries = 256;
3237 rdev->config.evergreen.sx_num_of_sets = 4;
3238 rdev->config.evergreen.sx_max_export_size = 128;
3239 rdev->config.evergreen.sx_max_export_pos_size = 32;
3240 rdev->config.evergreen.sx_max_export_smx_size = 96;
3241 rdev->config.evergreen.max_hw_contexts = 4;
3242 rdev->config.evergreen.sq_num_cf_insts = 1;
3243
3244 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3245 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3246 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3247 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3248 break;
3249 case CHIP_SUMO:
3250 rdev->config.evergreen.num_ses = 1;
3251 rdev->config.evergreen.max_pipes = 4;
3252 rdev->config.evergreen.max_tile_pipes = 4;
3253 if (rdev->pdev->device == 0x9648)
3254 rdev->config.evergreen.max_simds = 3;
3255 else if ((rdev->pdev->device == 0x9647) ||
3256 (rdev->pdev->device == 0x964a))
3257 rdev->config.evergreen.max_simds = 4;
3258 else
3259 rdev->config.evergreen.max_simds = 5;
3260 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3261 rdev->config.evergreen.max_gprs = 256;
3262 rdev->config.evergreen.max_threads = 248;
3263 rdev->config.evergreen.max_gs_threads = 32;
3264 rdev->config.evergreen.max_stack_entries = 256;
3265 rdev->config.evergreen.sx_num_of_sets = 4;
3266 rdev->config.evergreen.sx_max_export_size = 256;
3267 rdev->config.evergreen.sx_max_export_pos_size = 64;
3268 rdev->config.evergreen.sx_max_export_smx_size = 192;
3269 rdev->config.evergreen.max_hw_contexts = 8;
3270 rdev->config.evergreen.sq_num_cf_insts = 2;
3271
3272 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3273 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3274 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3275 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3276 break;
3277 case CHIP_SUMO2:
3278 rdev->config.evergreen.num_ses = 1;
3279 rdev->config.evergreen.max_pipes = 4;
3280 rdev->config.evergreen.max_tile_pipes = 4;
3281 rdev->config.evergreen.max_simds = 2;
3282 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3283 rdev->config.evergreen.max_gprs = 256;
3284 rdev->config.evergreen.max_threads = 248;
3285 rdev->config.evergreen.max_gs_threads = 32;
3286 rdev->config.evergreen.max_stack_entries = 512;
3287 rdev->config.evergreen.sx_num_of_sets = 4;
3288 rdev->config.evergreen.sx_max_export_size = 256;
3289 rdev->config.evergreen.sx_max_export_pos_size = 64;
3290 rdev->config.evergreen.sx_max_export_smx_size = 192;
3291 rdev->config.evergreen.max_hw_contexts = 4;
3292 rdev->config.evergreen.sq_num_cf_insts = 2;
3293
3294 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3295 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3296 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3297 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3298 break;
3299 case CHIP_BARTS:
3300 rdev->config.evergreen.num_ses = 2;
3301 rdev->config.evergreen.max_pipes = 4;
3302 rdev->config.evergreen.max_tile_pipes = 8;
3303 rdev->config.evergreen.max_simds = 7;
3304 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3305 rdev->config.evergreen.max_gprs = 256;
3306 rdev->config.evergreen.max_threads = 248;
3307 rdev->config.evergreen.max_gs_threads = 32;
3308 rdev->config.evergreen.max_stack_entries = 512;
3309 rdev->config.evergreen.sx_num_of_sets = 4;
3310 rdev->config.evergreen.sx_max_export_size = 256;
3311 rdev->config.evergreen.sx_max_export_pos_size = 64;
3312 rdev->config.evergreen.sx_max_export_smx_size = 192;
3313 rdev->config.evergreen.max_hw_contexts = 8;
3314 rdev->config.evergreen.sq_num_cf_insts = 2;
3315
3316 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3317 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3318 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3319 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3320 break;
3321 case CHIP_TURKS:
3322 rdev->config.evergreen.num_ses = 1;
3323 rdev->config.evergreen.max_pipes = 4;
3324 rdev->config.evergreen.max_tile_pipes = 4;
3325 rdev->config.evergreen.max_simds = 6;
3326 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3327 rdev->config.evergreen.max_gprs = 256;
3328 rdev->config.evergreen.max_threads = 248;
3329 rdev->config.evergreen.max_gs_threads = 32;
3330 rdev->config.evergreen.max_stack_entries = 256;
3331 rdev->config.evergreen.sx_num_of_sets = 4;
3332 rdev->config.evergreen.sx_max_export_size = 256;
3333 rdev->config.evergreen.sx_max_export_pos_size = 64;
3334 rdev->config.evergreen.sx_max_export_smx_size = 192;
3335 rdev->config.evergreen.max_hw_contexts = 8;
3336 rdev->config.evergreen.sq_num_cf_insts = 2;
3337
3338 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3339 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3340 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3341 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3342 break;
3343 case CHIP_CAICOS:
3344 rdev->config.evergreen.num_ses = 1;
3345 rdev->config.evergreen.max_pipes = 2;
3346 rdev->config.evergreen.max_tile_pipes = 2;
3347 rdev->config.evergreen.max_simds = 2;
3348 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3349 rdev->config.evergreen.max_gprs = 256;
3350 rdev->config.evergreen.max_threads = 192;
3351 rdev->config.evergreen.max_gs_threads = 16;
3352 rdev->config.evergreen.max_stack_entries = 256;
3353 rdev->config.evergreen.sx_num_of_sets = 4;
3354 rdev->config.evergreen.sx_max_export_size = 128;
3355 rdev->config.evergreen.sx_max_export_pos_size = 32;
3356 rdev->config.evergreen.sx_max_export_smx_size = 96;
3357 rdev->config.evergreen.max_hw_contexts = 4;
3358 rdev->config.evergreen.sq_num_cf_insts = 1;
3359
3360 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3361 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3362 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3363 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3364 break;
3365 }
3366
3367 /* Initialize HDP */
3368 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3369 WREG32((0x2c14 + j), 0x00000000);
3370 WREG32((0x2c18 + j), 0x00000000);
3371 WREG32((0x2c1c + j), 0x00000000);
3372 WREG32((0x2c20 + j), 0x00000000);
3373 WREG32((0x2c24 + j), 0x00000000);
3374 }
3375
3376 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3377 WREG32(SRBM_INT_CNTL, 0x1);
3378 WREG32(SRBM_INT_ACK, 0x1);
3379
3380 evergreen_fix_pci_max_read_req_size(rdev);
3381
3382 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3383 if ((rdev->family == CHIP_PALM) ||
3384 (rdev->family == CHIP_SUMO) ||
3385 (rdev->family == CHIP_SUMO2))
3386 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3387 else
3388 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3389
3390 /* setup tiling info dword. gb_addr_config is not adequate since it does
3391 * not have bank info, so create a custom tiling dword.
3392 * bits 3:0 num_pipes
3393 * bits 7:4 num_banks
3394 * bits 11:8 group_size
3395 * bits 15:12 row_size
3396 */
3397 rdev->config.evergreen.tile_config = 0;
3398 switch (rdev->config.evergreen.max_tile_pipes) {
3399 case 1:
3400 default:
3401 rdev->config.evergreen.tile_config |= (0 << 0);
3402 break;
3403 case 2:
3404 rdev->config.evergreen.tile_config |= (1 << 0);
3405 break;
3406 case 4:
3407 rdev->config.evergreen.tile_config |= (2 << 0);
3408 break;
3409 case 8:
3410 rdev->config.evergreen.tile_config |= (3 << 0);
3411 break;
3412 }
3413 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3414 if (rdev->flags & RADEON_IS_IGP)
3415 rdev->config.evergreen.tile_config |= 1 << 4;
3416 else {
3417 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3418 case 0: /* four banks */
3419 rdev->config.evergreen.tile_config |= 0 << 4;
3420 break;
3421 case 1: /* eight banks */
3422 rdev->config.evergreen.tile_config |= 1 << 4;
3423 break;
3424 case 2: /* sixteen banks */
3425 default:
3426 rdev->config.evergreen.tile_config |= 2 << 4;
3427 break;
3428 }
3429 }
3430 rdev->config.evergreen.tile_config |= 0 << 8;
3431 rdev->config.evergreen.tile_config |=
3432 ((gb_addr_config & 0x30000000) >> 28) << 12;
3433
3434 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3435 u32 efuse_straps_4;
3436 u32 efuse_straps_3;
3437
3438 efuse_straps_4 = RREG32_RCU(0x204);
3439 efuse_straps_3 = RREG32_RCU(0x203);
3440 tmp = (((efuse_straps_4 & 0xf) << 4) |
3441 ((efuse_straps_3 & 0xf0000000) >> 28));
3442 } else {
3443 tmp = 0;
3444 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3445 u32 rb_disable_bitmap;
3446
3447 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3448 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3449 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3450 tmp <<= 4;
3451 tmp |= rb_disable_bitmap;
3452 }
3453 }
3454 /* enabled rb are just the one not disabled :) */
3455 disabled_rb_mask = tmp;
3456 tmp = 0;
3457 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3458 tmp |= (1 << i);
3459 /* if all the backends are disabled, fix it up here */
3460 if ((disabled_rb_mask & tmp) == tmp) {
3461 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3462 disabled_rb_mask &= ~(1 << i);
3463 }
3464
3465 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3466 u32 simd_disable_bitmap;
3467
3468 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3469 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3470 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3471 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3472 tmp <<= 16;
3473 tmp |= simd_disable_bitmap;
3474 }
3475 rdev->config.evergreen.active_simds = hweight32(~tmp);
3476
3477 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3478 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3479
3480 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3481 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3482 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3483 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3484 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3485 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3486 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3487
3488 if ((rdev->config.evergreen.max_backends == 1) &&
3489 (rdev->flags & RADEON_IS_IGP)) {
3490 if ((disabled_rb_mask & 3) == 1) {
3491 /* RB0 disabled, RB1 enabled */
3492 tmp = 0x11111111;
3493 } else {
3494 /* RB1 disabled, RB0 enabled */
3495 tmp = 0x00000000;
3496 }
3497 } else {
3498 tmp = gb_addr_config & NUM_PIPES_MASK;
3499 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3500 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3501 }
3502 WREG32(GB_BACKEND_MAP, tmp);
3503
3504 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3505 WREG32(CGTS_TCC_DISABLE, 0);
3506 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3507 WREG32(CGTS_USER_TCC_DISABLE, 0);
3508
3509 /* set HW defaults for 3D engine */
3510 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3511 ROQ_IB2_START(0x2b)));
3512
3513 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3514
3515 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3516 SYNC_GRADIENT |
3517 SYNC_WALKER |
3518 SYNC_ALIGNER));
3519
3520 sx_debug_1 = RREG32(SX_DEBUG_1);
3521 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3522 WREG32(SX_DEBUG_1, sx_debug_1);
3523
3524
3525 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3526 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3527 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3528 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3529
3530 if (rdev->family <= CHIP_SUMO2)
3531 WREG32(SMX_SAR_CTL0, 0x00010000);
3532
3533 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3534 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3535 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3536
3537 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3538 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3539 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3540
3541 WREG32(VGT_NUM_INSTANCES, 1);
3542 WREG32(SPI_CONFIG_CNTL, 0);
3543 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3544 WREG32(CP_PERFMON_CNTL, 0);
3545
3546 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3547 FETCH_FIFO_HIWATER(0x4) |
3548 DONE_FIFO_HIWATER(0xe0) |
3549 ALU_UPDATE_FIFO_HIWATER(0x8)));
3550
3551 sq_config = RREG32(SQ_CONFIG);
3552 sq_config &= ~(PS_PRIO(3) |
3553 VS_PRIO(3) |
3554 GS_PRIO(3) |
3555 ES_PRIO(3));
3556 sq_config |= (VC_ENABLE |
3557 EXPORT_SRC_C |
3558 PS_PRIO(0) |
3559 VS_PRIO(1) |
3560 GS_PRIO(2) |
3561 ES_PRIO(3));
3562
3563 switch (rdev->family) {
3564 case CHIP_CEDAR:
3565 case CHIP_PALM:
3566 case CHIP_SUMO:
3567 case CHIP_SUMO2:
3568 case CHIP_CAICOS:
3569 /* no vertex cache */
3570 sq_config &= ~VC_ENABLE;
3571 break;
3572 default:
3573 break;
3574 }
3575
3576 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3577
3578 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3579 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3580 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3581 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3582 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3583 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3584 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3585
3586 switch (rdev->family) {
3587 case CHIP_CEDAR:
3588 case CHIP_PALM:
3589 case CHIP_SUMO:
3590 case CHIP_SUMO2:
3591 ps_thread_count = 96;
3592 break;
3593 default:
3594 ps_thread_count = 128;
3595 break;
3596 }
3597
3598 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3599 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3600 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3601 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3602 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3603 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3604
3605 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3606 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3607 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3608 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3609 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3610 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3611
3612 WREG32(SQ_CONFIG, sq_config);
3613 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3614 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3615 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3616 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3617 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3618 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3619 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3620 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3621 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3622 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3623
3624 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3625 FORCE_EOV_MAX_REZ_CNT(255)));
3626
3627 switch (rdev->family) {
3628 case CHIP_CEDAR:
3629 case CHIP_PALM:
3630 case CHIP_SUMO:
3631 case CHIP_SUMO2:
3632 case CHIP_CAICOS:
3633 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3634 break;
3635 default:
3636 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3637 break;
3638 }
3639 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3640 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3641
3642 WREG32(VGT_GS_VERTEX_REUSE, 16);
3643 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3644 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3645
3646 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3647 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3648
3649 WREG32(CB_PERF_CTR0_SEL_0, 0);
3650 WREG32(CB_PERF_CTR0_SEL_1, 0);
3651 WREG32(CB_PERF_CTR1_SEL_0, 0);
3652 WREG32(CB_PERF_CTR1_SEL_1, 0);
3653 WREG32(CB_PERF_CTR2_SEL_0, 0);
3654 WREG32(CB_PERF_CTR2_SEL_1, 0);
3655 WREG32(CB_PERF_CTR3_SEL_0, 0);
3656 WREG32(CB_PERF_CTR3_SEL_1, 0);
3657
3658 /* clear render buffer base addresses */
3659 WREG32(CB_COLOR0_BASE, 0);
3660 WREG32(CB_COLOR1_BASE, 0);
3661 WREG32(CB_COLOR2_BASE, 0);
3662 WREG32(CB_COLOR3_BASE, 0);
3663 WREG32(CB_COLOR4_BASE, 0);
3664 WREG32(CB_COLOR5_BASE, 0);
3665 WREG32(CB_COLOR6_BASE, 0);
3666 WREG32(CB_COLOR7_BASE, 0);
3667 WREG32(CB_COLOR8_BASE, 0);
3668 WREG32(CB_COLOR9_BASE, 0);
3669 WREG32(CB_COLOR10_BASE, 0);
3670 WREG32(CB_COLOR11_BASE, 0);
3671
3672 /* set the shader const cache sizes to 0 */
3673 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3674 WREG32(i, 0);
3675 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3676 WREG32(i, 0);
3677
3678 tmp = RREG32(HDP_MISC_CNTL);
3679 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3680 WREG32(HDP_MISC_CNTL, tmp);
3681
3682 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3683 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3684
3685 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3686
3687 udelay(50);
3688
3689 }
3690
evergreen_mc_init(struct radeon_device * rdev)3691 int evergreen_mc_init(struct radeon_device *rdev)
3692 {
3693 u32 tmp;
3694 int chansize, numchan;
3695
3696 /* Get VRAM informations */
3697 rdev->mc.vram_is_ddr = true;
3698 if ((rdev->family == CHIP_PALM) ||
3699 (rdev->family == CHIP_SUMO) ||
3700 (rdev->family == CHIP_SUMO2))
3701 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3702 else
3703 tmp = RREG32(MC_ARB_RAMCFG);
3704 if (tmp & CHANSIZE_OVERRIDE) {
3705 chansize = 16;
3706 } else if (tmp & CHANSIZE_MASK) {
3707 chansize = 64;
3708 } else {
3709 chansize = 32;
3710 }
3711 tmp = RREG32(MC_SHARED_CHMAP);
3712 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3713 case 0:
3714 default:
3715 numchan = 1;
3716 break;
3717 case 1:
3718 numchan = 2;
3719 break;
3720 case 2:
3721 numchan = 4;
3722 break;
3723 case 3:
3724 numchan = 8;
3725 break;
3726 }
3727 rdev->mc.vram_width = numchan * chansize;
3728 /* Could aper size report 0 ? */
3729 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3730 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3731 /* Setup GPU memory space */
3732 if ((rdev->family == CHIP_PALM) ||
3733 (rdev->family == CHIP_SUMO) ||
3734 (rdev->family == CHIP_SUMO2)) {
3735 /* size in bytes on fusion */
3736 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3737 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3738 } else {
3739 /* size in MB on evergreen/cayman/tn */
3740 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3741 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3742 }
3743 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3744 r700_vram_gtt_location(rdev, &rdev->mc);
3745 radeon_update_bandwidth_info(rdev);
3746
3747 return 0;
3748 }
3749
evergreen_print_gpu_status_regs(struct radeon_device * rdev)3750 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3751 {
3752 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3753 RREG32(GRBM_STATUS));
3754 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3755 RREG32(GRBM_STATUS_SE0));
3756 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3757 RREG32(GRBM_STATUS_SE1));
3758 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3759 RREG32(SRBM_STATUS));
3760 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3761 RREG32(SRBM_STATUS2));
3762 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3763 RREG32(CP_STALLED_STAT1));
3764 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3765 RREG32(CP_STALLED_STAT2));
3766 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3767 RREG32(CP_BUSY_STAT));
3768 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3769 RREG32(CP_STAT));
3770 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3771 RREG32(DMA_STATUS_REG));
3772 if (rdev->family >= CHIP_CAYMAN) {
3773 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3774 RREG32(DMA_STATUS_REG + 0x800));
3775 }
3776 }
3777
evergreen_is_display_hung(struct radeon_device * rdev)3778 bool evergreen_is_display_hung(struct radeon_device *rdev)
3779 {
3780 u32 crtc_hung = 0;
3781 u32 crtc_status[6];
3782 u32 i, j, tmp;
3783
3784 for (i = 0; i < rdev->num_crtc; i++) {
3785 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3786 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3787 crtc_hung |= (1 << i);
3788 }
3789 }
3790
3791 for (j = 0; j < 10; j++) {
3792 for (i = 0; i < rdev->num_crtc; i++) {
3793 if (crtc_hung & (1 << i)) {
3794 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3795 if (tmp != crtc_status[i])
3796 crtc_hung &= ~(1 << i);
3797 }
3798 }
3799 if (crtc_hung == 0)
3800 return false;
3801 udelay(100);
3802 }
3803
3804 return true;
3805 }
3806
evergreen_gpu_check_soft_reset(struct radeon_device * rdev)3807 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3808 {
3809 u32 reset_mask = 0;
3810 u32 tmp;
3811
3812 /* GRBM_STATUS */
3813 tmp = RREG32(GRBM_STATUS);
3814 if (tmp & (PA_BUSY | SC_BUSY |
3815 SH_BUSY | SX_BUSY |
3816 TA_BUSY | VGT_BUSY |
3817 DB_BUSY | CB_BUSY |
3818 SPI_BUSY | VGT_BUSY_NO_DMA))
3819 reset_mask |= RADEON_RESET_GFX;
3820
3821 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3822 CP_BUSY | CP_COHERENCY_BUSY))
3823 reset_mask |= RADEON_RESET_CP;
3824
3825 if (tmp & GRBM_EE_BUSY)
3826 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3827
3828 /* DMA_STATUS_REG */
3829 tmp = RREG32(DMA_STATUS_REG);
3830 if (!(tmp & DMA_IDLE))
3831 reset_mask |= RADEON_RESET_DMA;
3832
3833 /* SRBM_STATUS2 */
3834 tmp = RREG32(SRBM_STATUS2);
3835 if (tmp & DMA_BUSY)
3836 reset_mask |= RADEON_RESET_DMA;
3837
3838 /* SRBM_STATUS */
3839 tmp = RREG32(SRBM_STATUS);
3840 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3841 reset_mask |= RADEON_RESET_RLC;
3842
3843 if (tmp & IH_BUSY)
3844 reset_mask |= RADEON_RESET_IH;
3845
3846 if (tmp & SEM_BUSY)
3847 reset_mask |= RADEON_RESET_SEM;
3848
3849 if (tmp & GRBM_RQ_PENDING)
3850 reset_mask |= RADEON_RESET_GRBM;
3851
3852 if (tmp & VMC_BUSY)
3853 reset_mask |= RADEON_RESET_VMC;
3854
3855 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3856 MCC_BUSY | MCD_BUSY))
3857 reset_mask |= RADEON_RESET_MC;
3858
3859 if (evergreen_is_display_hung(rdev))
3860 reset_mask |= RADEON_RESET_DISPLAY;
3861
3862 /* VM_L2_STATUS */
3863 tmp = RREG32(VM_L2_STATUS);
3864 if (tmp & L2_BUSY)
3865 reset_mask |= RADEON_RESET_VMC;
3866
3867 /* Skip MC reset as it's mostly likely not hung, just busy */
3868 if (reset_mask & RADEON_RESET_MC) {
3869 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3870 reset_mask &= ~RADEON_RESET_MC;
3871 }
3872
3873 return reset_mask;
3874 }
3875
evergreen_gpu_soft_reset(struct radeon_device * rdev,u32 reset_mask)3876 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3877 {
3878 struct evergreen_mc_save save;
3879 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3880 u32 tmp;
3881
3882 if (reset_mask == 0)
3883 return;
3884
3885 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3886
3887 evergreen_print_gpu_status_regs(rdev);
3888
3889 /* Disable CP parsing/prefetching */
3890 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3891
3892 if (reset_mask & RADEON_RESET_DMA) {
3893 /* Disable DMA */
3894 tmp = RREG32(DMA_RB_CNTL);
3895 tmp &= ~DMA_RB_ENABLE;
3896 WREG32(DMA_RB_CNTL, tmp);
3897 }
3898
3899 udelay(50);
3900
3901 evergreen_mc_stop(rdev, &save);
3902 if (evergreen_mc_wait_for_idle(rdev)) {
3903 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3904 }
3905
3906 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3907 grbm_soft_reset |= SOFT_RESET_DB |
3908 SOFT_RESET_CB |
3909 SOFT_RESET_PA |
3910 SOFT_RESET_SC |
3911 SOFT_RESET_SPI |
3912 SOFT_RESET_SX |
3913 SOFT_RESET_SH |
3914 SOFT_RESET_TC |
3915 SOFT_RESET_TA |
3916 SOFT_RESET_VC |
3917 SOFT_RESET_VGT;
3918 }
3919
3920 if (reset_mask & RADEON_RESET_CP) {
3921 grbm_soft_reset |= SOFT_RESET_CP |
3922 SOFT_RESET_VGT;
3923
3924 srbm_soft_reset |= SOFT_RESET_GRBM;
3925 }
3926
3927 if (reset_mask & RADEON_RESET_DMA)
3928 srbm_soft_reset |= SOFT_RESET_DMA;
3929
3930 if (reset_mask & RADEON_RESET_DISPLAY)
3931 srbm_soft_reset |= SOFT_RESET_DC;
3932
3933 if (reset_mask & RADEON_RESET_RLC)
3934 srbm_soft_reset |= SOFT_RESET_RLC;
3935
3936 if (reset_mask & RADEON_RESET_SEM)
3937 srbm_soft_reset |= SOFT_RESET_SEM;
3938
3939 if (reset_mask & RADEON_RESET_IH)
3940 srbm_soft_reset |= SOFT_RESET_IH;
3941
3942 if (reset_mask & RADEON_RESET_GRBM)
3943 srbm_soft_reset |= SOFT_RESET_GRBM;
3944
3945 if (reset_mask & RADEON_RESET_VMC)
3946 srbm_soft_reset |= SOFT_RESET_VMC;
3947
3948 if (!(rdev->flags & RADEON_IS_IGP)) {
3949 if (reset_mask & RADEON_RESET_MC)
3950 srbm_soft_reset |= SOFT_RESET_MC;
3951 }
3952
3953 if (grbm_soft_reset) {
3954 tmp = RREG32(GRBM_SOFT_RESET);
3955 tmp |= grbm_soft_reset;
3956 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3957 WREG32(GRBM_SOFT_RESET, tmp);
3958 tmp = RREG32(GRBM_SOFT_RESET);
3959
3960 udelay(50);
3961
3962 tmp &= ~grbm_soft_reset;
3963 WREG32(GRBM_SOFT_RESET, tmp);
3964 tmp = RREG32(GRBM_SOFT_RESET);
3965 }
3966
3967 if (srbm_soft_reset) {
3968 tmp = RREG32(SRBM_SOFT_RESET);
3969 tmp |= srbm_soft_reset;
3970 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3971 WREG32(SRBM_SOFT_RESET, tmp);
3972 tmp = RREG32(SRBM_SOFT_RESET);
3973
3974 udelay(50);
3975
3976 tmp &= ~srbm_soft_reset;
3977 WREG32(SRBM_SOFT_RESET, tmp);
3978 tmp = RREG32(SRBM_SOFT_RESET);
3979 }
3980
3981 /* Wait a little for things to settle down */
3982 udelay(50);
3983
3984 evergreen_mc_resume(rdev, &save);
3985 udelay(50);
3986
3987 evergreen_print_gpu_status_regs(rdev);
3988 }
3989
evergreen_gpu_pci_config_reset(struct radeon_device * rdev)3990 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3991 {
3992 struct evergreen_mc_save save;
3993 u32 tmp, i;
3994
3995 dev_info(rdev->dev, "GPU pci config reset\n");
3996
3997 /* disable dpm? */
3998
3999 /* Disable CP parsing/prefetching */
4000 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4001 udelay(50);
4002 /* Disable DMA */
4003 tmp = RREG32(DMA_RB_CNTL);
4004 tmp &= ~DMA_RB_ENABLE;
4005 WREG32(DMA_RB_CNTL, tmp);
4006 /* XXX other engines? */
4007
4008 /* halt the rlc */
4009 r600_rlc_stop(rdev);
4010
4011 udelay(50);
4012
4013 /* set mclk/sclk to bypass */
4014 rv770_set_clk_bypass_mode(rdev);
4015 /* disable BM */
4016 pci_clear_master(rdev->pdev);
4017 /* disable mem access */
4018 evergreen_mc_stop(rdev, &save);
4019 if (evergreen_mc_wait_for_idle(rdev)) {
4020 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4021 }
4022 /* reset */
4023 radeon_pci_config_reset(rdev);
4024 /* wait for asic to come out of reset */
4025 for (i = 0; i < rdev->usec_timeout; i++) {
4026 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4027 break;
4028 udelay(1);
4029 }
4030 }
4031
evergreen_asic_reset(struct radeon_device * rdev,bool hard)4032 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4033 {
4034 u32 reset_mask;
4035
4036 if (hard) {
4037 evergreen_gpu_pci_config_reset(rdev);
4038 return 0;
4039 }
4040
4041 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4042
4043 if (reset_mask)
4044 r600_set_bios_scratch_engine_hung(rdev, true);
4045
4046 /* try soft reset */
4047 evergreen_gpu_soft_reset(rdev, reset_mask);
4048
4049 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4050
4051 /* try pci config reset */
4052 if (reset_mask && radeon_hard_reset)
4053 evergreen_gpu_pci_config_reset(rdev);
4054
4055 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4056
4057 if (!reset_mask)
4058 r600_set_bios_scratch_engine_hung(rdev, false);
4059
4060 return 0;
4061 }
4062
4063 /**
4064 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4065 *
4066 * @rdev: radeon_device pointer
4067 * @ring: radeon_ring structure holding ring information
4068 *
4069 * Check if the GFX engine is locked up.
4070 * Returns true if the engine appears to be locked up, false if not.
4071 */
evergreen_gfx_is_lockup(struct radeon_device * rdev,struct radeon_ring * ring)4072 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4073 {
4074 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4075
4076 if (!(reset_mask & (RADEON_RESET_GFX |
4077 RADEON_RESET_COMPUTE |
4078 RADEON_RESET_CP))) {
4079 radeon_ring_lockup_update(rdev, ring);
4080 return false;
4081 }
4082 return radeon_ring_test_lockup(rdev, ring);
4083 }
4084
4085 /*
4086 * RLC
4087 */
4088 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4089 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
4090
sumo_rlc_fini(struct radeon_device * rdev)4091 void sumo_rlc_fini(struct radeon_device *rdev)
4092 {
4093 int r;
4094
4095 /* save restore block */
4096 if (rdev->rlc.save_restore_obj) {
4097 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4098 if (unlikely(r != 0))
4099 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4100 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4101 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4102
4103 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4104 rdev->rlc.save_restore_obj = NULL;
4105 }
4106
4107 /* clear state block */
4108 if (rdev->rlc.clear_state_obj) {
4109 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4110 if (unlikely(r != 0))
4111 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4112 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4113 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4114
4115 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4116 rdev->rlc.clear_state_obj = NULL;
4117 }
4118
4119 /* clear state block */
4120 if (rdev->rlc.cp_table_obj) {
4121 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4122 if (unlikely(r != 0))
4123 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4124 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4125 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4126
4127 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4128 rdev->rlc.cp_table_obj = NULL;
4129 }
4130 }
4131
4132 #define CP_ME_TABLE_SIZE 96
4133
sumo_rlc_init(struct radeon_device * rdev)4134 int sumo_rlc_init(struct radeon_device *rdev)
4135 {
4136 const u32 *src_ptr;
4137 volatile u32 *dst_ptr;
4138 u32 dws, data, i, j, k, reg_num;
4139 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4140 u64 reg_list_mc_addr;
4141 const struct cs_section_def *cs_data;
4142 int r;
4143
4144 src_ptr = rdev->rlc.reg_list;
4145 dws = rdev->rlc.reg_list_size;
4146 if (rdev->family >= CHIP_BONAIRE) {
4147 dws += (5 * 16) + 48 + 48 + 64;
4148 }
4149 cs_data = rdev->rlc.cs_data;
4150
4151 if (src_ptr) {
4152 /* save restore block */
4153 if (rdev->rlc.save_restore_obj == NULL) {
4154 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4155 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4156 NULL, &rdev->rlc.save_restore_obj);
4157 if (r) {
4158 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4159 return r;
4160 }
4161 }
4162
4163 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4164 if (unlikely(r != 0)) {
4165 sumo_rlc_fini(rdev);
4166 return r;
4167 }
4168 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4169 &rdev->rlc.save_restore_gpu_addr);
4170 if (r) {
4171 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4172 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4173 sumo_rlc_fini(rdev);
4174 return r;
4175 }
4176
4177 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4178 if (r) {
4179 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4180 sumo_rlc_fini(rdev);
4181 return r;
4182 }
4183 /* write the sr buffer */
4184 dst_ptr = rdev->rlc.sr_ptr;
4185 if (rdev->family >= CHIP_TAHITI) {
4186 /* SI */
4187 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4188 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4189 } else {
4190 /* ON/LN/TN */
4191 /* format:
4192 * dw0: (reg2 << 16) | reg1
4193 * dw1: reg1 save space
4194 * dw2: reg2 save space
4195 */
4196 for (i = 0; i < dws; i++) {
4197 data = src_ptr[i] >> 2;
4198 i++;
4199 if (i < dws)
4200 data |= (src_ptr[i] >> 2) << 16;
4201 j = (((i - 1) * 3) / 2);
4202 dst_ptr[j] = cpu_to_le32(data);
4203 }
4204 j = ((i * 3) / 2);
4205 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4206 }
4207 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4208 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4209 }
4210
4211 if (cs_data) {
4212 /* clear state block */
4213 if (rdev->family >= CHIP_BONAIRE) {
4214 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4215 } else if (rdev->family >= CHIP_TAHITI) {
4216 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4217 dws = rdev->rlc.clear_state_size + (256 / 4);
4218 } else {
4219 reg_list_num = 0;
4220 dws = 0;
4221 for (i = 0; cs_data[i].section != NULL; i++) {
4222 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4223 reg_list_num++;
4224 dws += cs_data[i].section[j].reg_count;
4225 }
4226 }
4227 reg_list_blk_index = (3 * reg_list_num + 2);
4228 dws += reg_list_blk_index;
4229 rdev->rlc.clear_state_size = dws;
4230 }
4231
4232 if (rdev->rlc.clear_state_obj == NULL) {
4233 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4234 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4235 NULL, &rdev->rlc.clear_state_obj);
4236 if (r) {
4237 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4238 sumo_rlc_fini(rdev);
4239 return r;
4240 }
4241 }
4242 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4243 if (unlikely(r != 0)) {
4244 sumo_rlc_fini(rdev);
4245 return r;
4246 }
4247 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4248 &rdev->rlc.clear_state_gpu_addr);
4249 if (r) {
4250 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4251 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4252 sumo_rlc_fini(rdev);
4253 return r;
4254 }
4255
4256 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4257 if (r) {
4258 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4259 sumo_rlc_fini(rdev);
4260 return r;
4261 }
4262 /* set up the cs buffer */
4263 dst_ptr = rdev->rlc.cs_ptr;
4264 if (rdev->family >= CHIP_BONAIRE) {
4265 cik_get_csb_buffer(rdev, dst_ptr);
4266 } else if (rdev->family >= CHIP_TAHITI) {
4267 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4268 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4269 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4270 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4271 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4272 } else {
4273 reg_list_hdr_blk_index = 0;
4274 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4275 data = upper_32_bits(reg_list_mc_addr);
4276 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4277 reg_list_hdr_blk_index++;
4278 for (i = 0; cs_data[i].section != NULL; i++) {
4279 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4280 reg_num = cs_data[i].section[j].reg_count;
4281 data = reg_list_mc_addr & 0xffffffff;
4282 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4283 reg_list_hdr_blk_index++;
4284
4285 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4286 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4287 reg_list_hdr_blk_index++;
4288
4289 data = 0x08000000 | (reg_num * 4);
4290 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4291 reg_list_hdr_blk_index++;
4292
4293 for (k = 0; k < reg_num; k++) {
4294 data = cs_data[i].section[j].extent[k];
4295 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4296 }
4297 reg_list_mc_addr += reg_num * 4;
4298 reg_list_blk_index += reg_num;
4299 }
4300 }
4301 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4302 }
4303 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4304 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4305 }
4306
4307 if (rdev->rlc.cp_table_size) {
4308 if (rdev->rlc.cp_table_obj == NULL) {
4309 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4310 PAGE_SIZE, true,
4311 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4312 NULL, &rdev->rlc.cp_table_obj);
4313 if (r) {
4314 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4315 sumo_rlc_fini(rdev);
4316 return r;
4317 }
4318 }
4319
4320 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4321 if (unlikely(r != 0)) {
4322 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4323 sumo_rlc_fini(rdev);
4324 return r;
4325 }
4326 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4327 &rdev->rlc.cp_table_gpu_addr);
4328 if (r) {
4329 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4330 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4331 sumo_rlc_fini(rdev);
4332 return r;
4333 }
4334 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4335 if (r) {
4336 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4337 sumo_rlc_fini(rdev);
4338 return r;
4339 }
4340
4341 cik_init_cp_pg_table(rdev);
4342
4343 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4344 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4345
4346 }
4347
4348 return 0;
4349 }
4350
evergreen_rlc_start(struct radeon_device * rdev)4351 static void evergreen_rlc_start(struct radeon_device *rdev)
4352 {
4353 u32 mask = RLC_ENABLE;
4354
4355 if (rdev->flags & RADEON_IS_IGP) {
4356 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4357 }
4358
4359 WREG32(RLC_CNTL, mask);
4360 }
4361
evergreen_rlc_resume(struct radeon_device * rdev)4362 int evergreen_rlc_resume(struct radeon_device *rdev)
4363 {
4364 u32 i;
4365 const __be32 *fw_data;
4366
4367 if (!rdev->rlc_fw)
4368 return -EINVAL;
4369
4370 r600_rlc_stop(rdev);
4371
4372 WREG32(RLC_HB_CNTL, 0);
4373
4374 if (rdev->flags & RADEON_IS_IGP) {
4375 if (rdev->family == CHIP_ARUBA) {
4376 u32 always_on_bitmap =
4377 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4378 /* find out the number of active simds */
4379 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4380 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4381 tmp = hweight32(~tmp);
4382 if (tmp == rdev->config.cayman.max_simds_per_se) {
4383 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4384 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4385 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4386 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4387 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4388 }
4389 } else {
4390 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4391 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4392 }
4393 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4394 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4395 } else {
4396 WREG32(RLC_HB_BASE, 0);
4397 WREG32(RLC_HB_RPTR, 0);
4398 WREG32(RLC_HB_WPTR, 0);
4399 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4400 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4401 }
4402 WREG32(RLC_MC_CNTL, 0);
4403 WREG32(RLC_UCODE_CNTL, 0);
4404
4405 fw_data = (const __be32 *)rdev->rlc_fw->data;
4406 if (rdev->family >= CHIP_ARUBA) {
4407 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4408 WREG32(RLC_UCODE_ADDR, i);
4409 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4410 }
4411 } else if (rdev->family >= CHIP_CAYMAN) {
4412 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4413 WREG32(RLC_UCODE_ADDR, i);
4414 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4415 }
4416 } else {
4417 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4418 WREG32(RLC_UCODE_ADDR, i);
4419 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4420 }
4421 }
4422 WREG32(RLC_UCODE_ADDR, 0);
4423
4424 evergreen_rlc_start(rdev);
4425
4426 return 0;
4427 }
4428
4429 /* Interrupts */
4430
evergreen_get_vblank_counter(struct radeon_device * rdev,int crtc)4431 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4432 {
4433 if (crtc >= rdev->num_crtc)
4434 return 0;
4435 else
4436 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4437 }
4438
evergreen_disable_interrupt_state(struct radeon_device * rdev)4439 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4440 {
4441 int i;
4442 u32 tmp;
4443
4444 if (rdev->family >= CHIP_CAYMAN) {
4445 cayman_cp_int_cntl_setup(rdev, 0,
4446 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4447 cayman_cp_int_cntl_setup(rdev, 1, 0);
4448 cayman_cp_int_cntl_setup(rdev, 2, 0);
4449 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4450 WREG32(CAYMAN_DMA1_CNTL, tmp);
4451 } else
4452 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4453 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4454 WREG32(DMA_CNTL, tmp);
4455 WREG32(GRBM_INT_CNTL, 0);
4456 WREG32(SRBM_INT_CNTL, 0);
4457 for (i = 0; i < rdev->num_crtc; i++)
4458 WREG32(INT_MASK + crtc_offsets[i], 0);
4459 for (i = 0; i < rdev->num_crtc; i++)
4460 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4461
4462 /* only one DAC on DCE5 */
4463 if (!ASIC_IS_DCE5(rdev))
4464 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4465 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4466
4467 for (i = 0; i < 6; i++)
4468 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4469 }
4470
4471 /* Note that the order we write back regs here is important */
evergreen_irq_set(struct radeon_device * rdev)4472 int evergreen_irq_set(struct radeon_device *rdev)
4473 {
4474 int i;
4475 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4476 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4477 u32 grbm_int_cntl = 0;
4478 u32 dma_cntl, dma_cntl1 = 0;
4479 u32 thermal_int = 0;
4480
4481 if (!rdev->irq.installed) {
4482 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4483 return -EINVAL;
4484 }
4485 /* don't enable anything if the ih is disabled */
4486 if (!rdev->ih.enabled) {
4487 r600_disable_interrupts(rdev);
4488 /* force the active interrupt state to all disabled */
4489 evergreen_disable_interrupt_state(rdev);
4490 return 0;
4491 }
4492
4493 if (rdev->family == CHIP_ARUBA)
4494 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4495 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4496 else
4497 thermal_int = RREG32(CG_THERMAL_INT) &
4498 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4499
4500 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4501
4502 if (rdev->family >= CHIP_CAYMAN) {
4503 /* enable CP interrupts on all rings */
4504 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4505 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4506 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4507 }
4508 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4509 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4510 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4511 }
4512 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4513 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4514 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4515 }
4516 } else {
4517 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4518 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4519 cp_int_cntl |= RB_INT_ENABLE;
4520 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4521 }
4522 }
4523
4524 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4525 DRM_DEBUG("r600_irq_set: sw int dma\n");
4526 dma_cntl |= TRAP_ENABLE;
4527 }
4528
4529 if (rdev->family >= CHIP_CAYMAN) {
4530 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4531 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4532 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4533 dma_cntl1 |= TRAP_ENABLE;
4534 }
4535 }
4536
4537 if (rdev->irq.dpm_thermal) {
4538 DRM_DEBUG("dpm thermal\n");
4539 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4540 }
4541
4542 if (rdev->family >= CHIP_CAYMAN) {
4543 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4544 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4545 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4546 } else
4547 WREG32(CP_INT_CNTL, cp_int_cntl);
4548
4549 WREG32(DMA_CNTL, dma_cntl);
4550
4551 if (rdev->family >= CHIP_CAYMAN)
4552 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4553
4554 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4555
4556 for (i = 0; i < rdev->num_crtc; i++) {
4557 radeon_irq_kms_set_irq_n_enabled(
4558 rdev, INT_MASK + crtc_offsets[i],
4559 VBLANK_INT_MASK,
4560 rdev->irq.crtc_vblank_int[i] ||
4561 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4562 }
4563
4564 for (i = 0; i < rdev->num_crtc; i++)
4565 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4566
4567 for (i = 0; i < 6; i++) {
4568 radeon_irq_kms_set_irq_n_enabled(
4569 rdev, DC_HPDx_INT_CONTROL(i),
4570 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4571 rdev->irq.hpd[i], "HPD", i);
4572 }
4573
4574 if (rdev->family == CHIP_ARUBA)
4575 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4576 else
4577 WREG32(CG_THERMAL_INT, thermal_int);
4578
4579 for (i = 0; i < 6; i++) {
4580 radeon_irq_kms_set_irq_n_enabled(
4581 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4582 AFMT_AZ_FORMAT_WTRIG_MASK,
4583 rdev->irq.afmt[i], "HDMI", i);
4584 }
4585
4586 /* posting read */
4587 RREG32(SRBM_STATUS);
4588
4589 return 0;
4590 }
4591
4592 /* Note that the order we write back regs here is important */
evergreen_irq_ack(struct radeon_device * rdev)4593 static void evergreen_irq_ack(struct radeon_device *rdev)
4594 {
4595 int i, j;
4596 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4597 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4598 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4599
4600 for (i = 0; i < 6; i++) {
4601 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4602 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4603 if (i < rdev->num_crtc)
4604 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4605 }
4606
4607 /* We write back each interrupt register in pairs of two */
4608 for (i = 0; i < rdev->num_crtc; i += 2) {
4609 for (j = i; j < (i + 2); j++) {
4610 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4611 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4612 GRPH_PFLIP_INT_CLEAR);
4613 }
4614
4615 for (j = i; j < (i + 2); j++) {
4616 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4617 WREG32(VBLANK_STATUS + crtc_offsets[j],
4618 VBLANK_ACK);
4619 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4620 WREG32(VLINE_STATUS + crtc_offsets[j],
4621 VLINE_ACK);
4622 }
4623 }
4624
4625 for (i = 0; i < 6; i++) {
4626 if (disp_int[i] & DC_HPD1_INTERRUPT)
4627 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4628 }
4629
4630 for (i = 0; i < 6; i++) {
4631 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4632 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4633 }
4634
4635 for (i = 0; i < 6; i++) {
4636 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4637 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4638 AFMT_AZ_FORMAT_WTRIG_ACK);
4639 }
4640 }
4641
evergreen_irq_disable(struct radeon_device * rdev)4642 static void evergreen_irq_disable(struct radeon_device *rdev)
4643 {
4644 r600_disable_interrupts(rdev);
4645 /* Wait and acknowledge irq */
4646 mdelay(1);
4647 evergreen_irq_ack(rdev);
4648 evergreen_disable_interrupt_state(rdev);
4649 }
4650
evergreen_irq_suspend(struct radeon_device * rdev)4651 void evergreen_irq_suspend(struct radeon_device *rdev)
4652 {
4653 evergreen_irq_disable(rdev);
4654 r600_rlc_stop(rdev);
4655 }
4656
evergreen_get_ih_wptr(struct radeon_device * rdev)4657 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4658 {
4659 u32 wptr, tmp;
4660
4661 if (rdev->wb.enabled)
4662 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4663 else
4664 wptr = RREG32(IH_RB_WPTR);
4665
4666 if (wptr & RB_OVERFLOW) {
4667 wptr &= ~RB_OVERFLOW;
4668 /* When a ring buffer overflow happen start parsing interrupt
4669 * from the last not overwritten vector (wptr + 16). Hopefully
4670 * this should allow us to catchup.
4671 */
4672 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4673 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4674 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4675 tmp = RREG32(IH_RB_CNTL);
4676 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4677 WREG32(IH_RB_CNTL, tmp);
4678 }
4679 return (wptr & rdev->ih.ptr_mask);
4680 }
4681
evergreen_irq_process(struct radeon_device * rdev)4682 irqreturn_t evergreen_irq_process(struct radeon_device *rdev)
4683 {
4684 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4685 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4686 u32 crtc_idx, hpd_idx, afmt_idx;
4687 u32 mask;
4688 u32 wptr;
4689 u32 rptr;
4690 u32 src_id, src_data;
4691 u32 ring_index;
4692 bool queue_hotplug = false;
4693 bool queue_hdmi = false;
4694 bool queue_dp = false;
4695 bool queue_thermal = false;
4696 u32 status, addr;
4697 const char *event_name;
4698
4699 if (!rdev->ih.enabled || rdev->shutdown)
4700 return IRQ_NONE;
4701
4702 wptr = evergreen_get_ih_wptr(rdev);
4703
4704 restart_ih:
4705 /* is somebody else already processing irqs? */
4706 if (atomic_xchg(&rdev->ih.lock, 1))
4707 return IRQ_NONE;
4708
4709 rptr = rdev->ih.rptr;
4710 DRM_DEBUG_VBLANK("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4711
4712 /* Order reading of wptr vs. reading of IH ring data */
4713 rmb();
4714
4715 /* display interrupts */
4716 evergreen_irq_ack(rdev);
4717
4718 while (rptr != wptr) {
4719 /* wptr/rptr are in bytes! */
4720 ring_index = rptr / 4;
4721 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4722 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4723
4724 switch (src_id) {
4725 case 1: /* D1 vblank/vline */
4726 case 2: /* D2 vblank/vline */
4727 case 3: /* D3 vblank/vline */
4728 case 4: /* D4 vblank/vline */
4729 case 5: /* D5 vblank/vline */
4730 case 6: /* D6 vblank/vline */
4731 crtc_idx = src_id - 1;
4732
4733 if (src_data == 0) { /* vblank */
4734 mask = LB_D1_VBLANK_INTERRUPT;
4735 event_name = "vblank";
4736
4737 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4738 drm_handle_vblank(rdev->ddev, crtc_idx);
4739 rdev->pm.vblank_sync = true;
4740 wake_up(&rdev->irq.vblank_queue);
4741 }
4742 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4743 radeon_crtc_handle_vblank(rdev,
4744 crtc_idx);
4745 }
4746
4747 } else if (src_data == 1) { /* vline */
4748 mask = LB_D1_VLINE_INTERRUPT;
4749 event_name = "vline";
4750 } else {
4751 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4752 src_id, src_data);
4753 break;
4754 }
4755
4756 if (!(disp_int[crtc_idx] & mask)) {
4757 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4758 crtc_idx + 1, event_name);
4759 }
4760
4761 disp_int[crtc_idx] &= ~mask;
4762 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4763
4764 break;
4765 case 8: /* D1 page flip */
4766 case 10: /* D2 page flip */
4767 case 12: /* D3 page flip */
4768 case 14: /* D4 page flip */
4769 case 16: /* D5 page flip */
4770 case 18: /* D6 page flip */
4771 DRM_DEBUG_VBLANK("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4772 if (radeon_use_pflipirq > 0)
4773 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4774 break;
4775 case 42: /* HPD hotplug */
4776 if (src_data <= 5) {
4777 hpd_idx = src_data;
4778 mask = DC_HPD1_INTERRUPT;
4779 queue_hotplug = true;
4780 event_name = "HPD";
4781
4782 } else if (src_data <= 11) {
4783 hpd_idx = src_data - 6;
4784 mask = DC_HPD1_RX_INTERRUPT;
4785 queue_dp = true;
4786 event_name = "HPD_RX";
4787
4788 } else {
4789 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4790 src_id, src_data);
4791 break;
4792 }
4793
4794 if (!(disp_int[hpd_idx] & mask))
4795 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4796
4797 disp_int[hpd_idx] &= ~mask;
4798 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4799
4800 break;
4801 case 44: /* hdmi */
4802 afmt_idx = src_data;
4803 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4804 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4805
4806 if (afmt_idx > 5) {
4807 DRM_ERROR("Unhandled interrupt: %d %d\n",
4808 src_id, src_data);
4809 break;
4810 }
4811 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4812 queue_hdmi = true;
4813 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4814 break;
4815 case 96:
4816 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4817 WREG32(SRBM_INT_ACK, 0x1);
4818 break;
4819 case 124: /* UVD */
4820 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4821 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4822 break;
4823 case 146:
4824 case 147:
4825 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4826 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4827 /* reset addr and status */
4828 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4829 if (addr == 0x0 && status == 0x0)
4830 break;
4831 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4832 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4833 addr);
4834 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4835 status);
4836 cayman_vm_decode_fault(rdev, status, addr);
4837 break;
4838 case 176: /* CP_INT in ring buffer */
4839 case 177: /* CP_INT in IB1 */
4840 case 178: /* CP_INT in IB2 */
4841 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4842 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4843 break;
4844 case 181: /* CP EOP event */
4845 DRM_DEBUG("IH: CP EOP\n");
4846 if (rdev->family >= CHIP_CAYMAN) {
4847 switch (src_data) {
4848 case 0:
4849 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4850 break;
4851 case 1:
4852 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4853 break;
4854 case 2:
4855 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4856 break;
4857 }
4858 } else
4859 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4860 break;
4861 case 224: /* DMA trap event */
4862 DRM_DEBUG("IH: DMA trap\n");
4863 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4864 break;
4865 case 230: /* thermal low to high */
4866 DRM_DEBUG("IH: thermal low to high\n");
4867 rdev->pm.dpm.thermal.high_to_low = false;
4868 queue_thermal = true;
4869 break;
4870 case 231: /* thermal high to low */
4871 DRM_DEBUG("IH: thermal high to low\n");
4872 rdev->pm.dpm.thermal.high_to_low = true;
4873 queue_thermal = true;
4874 break;
4875 case 233: /* GUI IDLE */
4876 DRM_DEBUG("IH: GUI idle\n");
4877 break;
4878 case 244: /* DMA trap event */
4879 if (rdev->family >= CHIP_CAYMAN) {
4880 DRM_DEBUG("IH: DMA1 trap\n");
4881 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4882 }
4883 break;
4884 default:
4885 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4886 break;
4887 }
4888
4889 /* wptr/rptr are in bytes! */
4890 rptr += 16;
4891 rptr &= rdev->ih.ptr_mask;
4892 WREG32(IH_RB_RPTR, rptr);
4893 }
4894 if (queue_dp)
4895 schedule_work(&rdev->dp_work);
4896 if (queue_hotplug)
4897 schedule_delayed_work(&rdev->hotplug_work, 0);
4898 if (queue_hdmi)
4899 schedule_work(&rdev->audio_work);
4900 if (queue_thermal && rdev->pm.dpm_enabled)
4901 schedule_work(&rdev->pm.dpm.thermal.work);
4902 rdev->ih.rptr = rptr;
4903 atomic_set(&rdev->ih.lock, 0);
4904
4905 /* make sure wptr hasn't changed while processing */
4906 wptr = evergreen_get_ih_wptr(rdev);
4907 if (wptr != rptr)
4908 goto restart_ih;
4909
4910 return IRQ_HANDLED;
4911 }
4912
evergreen_uvd_init(struct radeon_device * rdev)4913 static void evergreen_uvd_init(struct radeon_device *rdev)
4914 {
4915 int r;
4916
4917 if (!rdev->has_uvd)
4918 return;
4919
4920 r = radeon_uvd_init(rdev);
4921 if (r) {
4922 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4923 /*
4924 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4925 * to early fails uvd_v2_2_resume() and thus nothing happens
4926 * there. So it is pointless to try to go through that code
4927 * hence why we disable uvd here.
4928 */
4929 rdev->has_uvd = 0;
4930 return;
4931 }
4932 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4933 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4934 }
4935
evergreen_uvd_start(struct radeon_device * rdev)4936 static void evergreen_uvd_start(struct radeon_device *rdev)
4937 {
4938 int r;
4939
4940 if (!rdev->has_uvd)
4941 return;
4942
4943 r = uvd_v2_2_resume(rdev);
4944 if (r) {
4945 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4946 goto error;
4947 }
4948 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4949 if (r) {
4950 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4951 goto error;
4952 }
4953 return;
4954
4955 error:
4956 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4957 }
4958
evergreen_uvd_resume(struct radeon_device * rdev)4959 static void evergreen_uvd_resume(struct radeon_device *rdev)
4960 {
4961 struct radeon_ring *ring;
4962 int r;
4963
4964 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4965 return;
4966
4967 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4968 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4969 if (r) {
4970 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4971 return;
4972 }
4973 r = uvd_v1_0_init(rdev);
4974 if (r) {
4975 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4976 return;
4977 }
4978 }
4979
evergreen_startup(struct radeon_device * rdev)4980 static int evergreen_startup(struct radeon_device *rdev)
4981 {
4982 struct radeon_ring *ring;
4983 int r;
4984
4985 /* enable pcie gen2 link */
4986 evergreen_pcie_gen2_enable(rdev);
4987 /* enable aspm */
4988 evergreen_program_aspm(rdev);
4989
4990 /* scratch needs to be initialized before MC */
4991 r = r600_vram_scratch_init(rdev);
4992 if (r)
4993 return r;
4994
4995 evergreen_mc_program(rdev);
4996
4997 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
4998 r = ni_mc_load_microcode(rdev);
4999 if (r) {
5000 DRM_ERROR("Failed to load MC firmware!\n");
5001 return r;
5002 }
5003 }
5004
5005 if (rdev->flags & RADEON_IS_AGP) {
5006 evergreen_agp_enable(rdev);
5007 } else {
5008 r = evergreen_pcie_gart_enable(rdev);
5009 if (r)
5010 return r;
5011 }
5012 evergreen_gpu_init(rdev);
5013
5014 /* allocate rlc buffers */
5015 if (rdev->flags & RADEON_IS_IGP) {
5016 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5017 rdev->rlc.reg_list_size =
5018 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5019 rdev->rlc.cs_data = evergreen_cs_data;
5020 r = sumo_rlc_init(rdev);
5021 if (r) {
5022 DRM_ERROR("Failed to init rlc BOs!\n");
5023 return r;
5024 }
5025 }
5026
5027 /* allocate wb buffer */
5028 r = radeon_wb_init(rdev);
5029 if (r)
5030 return r;
5031
5032 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5033 if (r) {
5034 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5035 return r;
5036 }
5037
5038 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5039 if (r) {
5040 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5041 return r;
5042 }
5043
5044 evergreen_uvd_start(rdev);
5045
5046 /* Enable IRQ */
5047 if (!rdev->irq.installed) {
5048 r = radeon_irq_kms_init(rdev);
5049 if (r)
5050 return r;
5051 }
5052
5053 r = r600_irq_init(rdev);
5054 if (r) {
5055 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5056 radeon_irq_kms_fini(rdev);
5057 return r;
5058 }
5059 evergreen_irq_set(rdev);
5060
5061 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5062 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5063 RADEON_CP_PACKET2);
5064 if (r)
5065 return r;
5066
5067 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5068 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5069 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5070 if (r)
5071 return r;
5072
5073 r = evergreen_cp_load_microcode(rdev);
5074 if (r)
5075 return r;
5076 r = evergreen_cp_resume(rdev);
5077 if (r)
5078 return r;
5079 r = r600_dma_resume(rdev);
5080 if (r)
5081 return r;
5082
5083 evergreen_uvd_resume(rdev);
5084
5085 r = radeon_ib_pool_init(rdev);
5086 if (r) {
5087 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5088 return r;
5089 }
5090
5091 r = radeon_audio_init(rdev);
5092 if (r) {
5093 DRM_ERROR("radeon: audio init failed\n");
5094 return r;
5095 }
5096
5097 return 0;
5098 }
5099
evergreen_resume(struct radeon_device * rdev)5100 int evergreen_resume(struct radeon_device *rdev)
5101 {
5102 int r;
5103
5104 /* reset the asic, the gfx blocks are often in a bad state
5105 * after the driver is unloaded or after a resume
5106 */
5107 if (radeon_asic_reset(rdev))
5108 dev_warn(rdev->dev, "GPU reset failed !\n");
5109 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5110 * posting will perform necessary task to bring back GPU into good
5111 * shape.
5112 */
5113 /* post card */
5114 atom_asic_init(rdev->mode_info.atom_context);
5115
5116 /* init golden registers */
5117 evergreen_init_golden_registers(rdev);
5118
5119 if (rdev->pm.pm_method == PM_METHOD_DPM)
5120 radeon_pm_resume(rdev);
5121
5122 rdev->accel_working = true;
5123 r = evergreen_startup(rdev);
5124 if (r) {
5125 DRM_ERROR("evergreen startup failed on resume\n");
5126 rdev->accel_working = false;
5127 return r;
5128 }
5129
5130 return r;
5131
5132 }
5133
evergreen_suspend(struct radeon_device * rdev)5134 int evergreen_suspend(struct radeon_device *rdev)
5135 {
5136 radeon_pm_suspend(rdev);
5137 radeon_audio_fini(rdev);
5138 if (rdev->has_uvd) {
5139 uvd_v1_0_fini(rdev);
5140 radeon_uvd_suspend(rdev);
5141 }
5142 r700_cp_stop(rdev);
5143 r600_dma_stop(rdev);
5144 evergreen_irq_suspend(rdev);
5145 radeon_wb_disable(rdev);
5146 evergreen_pcie_gart_disable(rdev);
5147
5148 return 0;
5149 }
5150
5151 /* Plan is to move initialization in that function and use
5152 * helper function so that radeon_device_init pretty much
5153 * do nothing more than calling asic specific function. This
5154 * should also allow to remove a bunch of callback function
5155 * like vram_info.
5156 */
evergreen_init(struct radeon_device * rdev)5157 int evergreen_init(struct radeon_device *rdev)
5158 {
5159 int r;
5160
5161 /* Read BIOS */
5162 if (!radeon_get_bios(rdev)) {
5163 if (ASIC_IS_AVIVO(rdev))
5164 return -EINVAL;
5165 }
5166 /* Must be an ATOMBIOS */
5167 if (!rdev->is_atom_bios) {
5168 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5169 return -EINVAL;
5170 }
5171 r = radeon_atombios_init(rdev);
5172 if (r)
5173 return r;
5174 /* reset the asic, the gfx blocks are often in a bad state
5175 * after the driver is unloaded or after a resume
5176 */
5177 if (radeon_asic_reset(rdev))
5178 dev_warn(rdev->dev, "GPU reset failed !\n");
5179 /* Post card if necessary */
5180 if (!radeon_card_posted(rdev)) {
5181 if (!rdev->bios) {
5182 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5183 return -EINVAL;
5184 }
5185 DRM_INFO("GPU not posted. posting now...\n");
5186 atom_asic_init(rdev->mode_info.atom_context);
5187 }
5188 /* init golden registers */
5189 evergreen_init_golden_registers(rdev);
5190 /* Initialize scratch registers */
5191 r600_scratch_init(rdev);
5192 /* Initialize surface registers */
5193 radeon_surface_init(rdev);
5194 /* Initialize clocks */
5195 radeon_get_clock_info(rdev->ddev);
5196 /* Fence driver */
5197 r = radeon_fence_driver_init(rdev);
5198 if (r)
5199 return r;
5200 /* initialize AGP */
5201 if (rdev->flags & RADEON_IS_AGP) {
5202 r = radeon_agp_init(rdev);
5203 if (r)
5204 radeon_agp_disable(rdev);
5205 }
5206 /* initialize memory controller */
5207 r = evergreen_mc_init(rdev);
5208 if (r)
5209 return r;
5210 /* Memory manager */
5211 r = radeon_bo_init(rdev);
5212 if (r)
5213 return r;
5214
5215 if (ASIC_IS_DCE5(rdev)) {
5216 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5217 r = ni_init_microcode(rdev);
5218 if (r) {
5219 DRM_ERROR("Failed to load firmware!\n");
5220 return r;
5221 }
5222 }
5223 } else {
5224 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5225 r = r600_init_microcode(rdev);
5226 if (r) {
5227 DRM_ERROR("Failed to load firmware!\n");
5228 return r;
5229 }
5230 }
5231 }
5232
5233 /* Initialize power management */
5234 radeon_pm_init(rdev);
5235
5236 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5237 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5238
5239 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5240 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5241
5242 evergreen_uvd_init(rdev);
5243
5244 rdev->ih.ring_obj = NULL;
5245 r600_ih_ring_init(rdev, 64 * 1024);
5246
5247 r = r600_pcie_gart_init(rdev);
5248 if (r)
5249 return r;
5250
5251 #ifdef __DragonFly__
5252 /*
5253 There are unresolved crashes on evergreen hardware,
5254 tell userland acceleration is not working properly
5255 Bug report: https://bugs.dragonflybsd.org/issues/3198
5256 */
5257 rdev->accel_working = false;
5258 DRM_ERROR("GPU acceleration disabled for now on DragonFly\n");
5259 #else
5260 rdev->accel_working = true;
5261 #endif
5262 r = evergreen_startup(rdev);
5263 if (r) {
5264 dev_err(rdev->dev, "disabling GPU acceleration\n");
5265 r700_cp_fini(rdev);
5266 r600_dma_fini(rdev);
5267 r600_irq_fini(rdev);
5268 if (rdev->flags & RADEON_IS_IGP)
5269 sumo_rlc_fini(rdev);
5270 radeon_wb_fini(rdev);
5271 radeon_ib_pool_fini(rdev);
5272 radeon_irq_kms_fini(rdev);
5273 evergreen_pcie_gart_fini(rdev);
5274 rdev->accel_working = false;
5275 }
5276
5277 /* Don't start up if the MC ucode is missing on BTC parts.
5278 * The default clocks and voltages before the MC ucode
5279 * is loaded are not suffient for advanced operations.
5280 */
5281 if (ASIC_IS_DCE5(rdev)) {
5282 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5283 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5284 return -EINVAL;
5285 }
5286 }
5287
5288 return 0;
5289 }
5290
evergreen_fini(struct radeon_device * rdev)5291 void evergreen_fini(struct radeon_device *rdev)
5292 {
5293 radeon_pm_fini(rdev);
5294 radeon_audio_fini(rdev);
5295 r700_cp_fini(rdev);
5296 r600_dma_fini(rdev);
5297 r600_irq_fini(rdev);
5298 if (rdev->flags & RADEON_IS_IGP)
5299 sumo_rlc_fini(rdev);
5300 radeon_wb_fini(rdev);
5301 radeon_ib_pool_fini(rdev);
5302 radeon_irq_kms_fini(rdev);
5303 uvd_v1_0_fini(rdev);
5304 radeon_uvd_fini(rdev);
5305 evergreen_pcie_gart_fini(rdev);
5306 r600_vram_scratch_fini(rdev);
5307 radeon_gem_fini(rdev);
5308 radeon_fence_driver_fini(rdev);
5309 radeon_agp_fini(rdev);
5310 radeon_bo_fini(rdev);
5311 radeon_atombios_fini(rdev);
5312 kfree(rdev->bios);
5313 rdev->bios = NULL;
5314 }
5315
evergreen_pcie_gen2_enable(struct radeon_device * rdev)5316 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5317 {
5318 u32 link_width_cntl, speed_cntl;
5319 u32 mask;
5320
5321 if (radeon_pcie_gen2 == 0)
5322 return;
5323
5324 if (rdev->flags & RADEON_IS_IGP)
5325 return;
5326
5327 if (!(rdev->flags & RADEON_IS_PCIE))
5328 return;
5329
5330 /* x2 cards have a special sequence */
5331 if (ASIC_IS_X2(rdev))
5332 return;
5333
5334 #ifdef __DragonFly__
5335 if (drm_pcie_get_speed_cap_mask(rdev->ddev, &mask) != 0)
5336 return;
5337
5338 rdev->pdev->bus->max_bus_speed = (mask &0xff);
5339 #endif
5340
5341 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5342 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5343 return;
5344
5345 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5346 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5347 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5348 return;
5349 }
5350
5351 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5352
5353 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5354 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5355
5356 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5357 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5358 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5359
5360 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5361 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5362 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5363
5364 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5365 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5366 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5367
5368 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5369 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5370 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5371
5372 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5373 speed_cntl |= LC_GEN2_EN_STRAP;
5374 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5375
5376 } else {
5377 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5378 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5379 if (1)
5380 link_width_cntl |= LC_UPCONFIGURE_DIS;
5381 else
5382 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5383 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5384 }
5385 }
5386
evergreen_program_aspm(struct radeon_device * rdev)5387 void evergreen_program_aspm(struct radeon_device *rdev)
5388 {
5389 u32 data, orig;
5390 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5391 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5392 /* fusion_platform = true
5393 * if the system is a fusion system
5394 * (APU or DGPU in a fusion system).
5395 * todo: check if the system is a fusion platform.
5396 */
5397 bool fusion_platform = false;
5398
5399 if (radeon_aspm == 0)
5400 return;
5401
5402 if (!(rdev->flags & RADEON_IS_PCIE))
5403 return;
5404
5405 switch (rdev->family) {
5406 case CHIP_CYPRESS:
5407 case CHIP_HEMLOCK:
5408 case CHIP_JUNIPER:
5409 case CHIP_REDWOOD:
5410 case CHIP_CEDAR:
5411 case CHIP_SUMO:
5412 case CHIP_SUMO2:
5413 case CHIP_PALM:
5414 case CHIP_ARUBA:
5415 disable_l0s = true;
5416 break;
5417 default:
5418 disable_l0s = false;
5419 break;
5420 }
5421
5422 if (rdev->flags & RADEON_IS_IGP)
5423 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5424
5425 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5426 if (fusion_platform)
5427 data &= ~MULTI_PIF;
5428 else
5429 data |= MULTI_PIF;
5430 if (data != orig)
5431 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5432
5433 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5434 if (fusion_platform)
5435 data &= ~MULTI_PIF;
5436 else
5437 data |= MULTI_PIF;
5438 if (data != orig)
5439 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5440
5441 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5442 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5443 if (!disable_l0s) {
5444 if (rdev->family >= CHIP_BARTS)
5445 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5446 else
5447 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5448 }
5449
5450 if (!disable_l1) {
5451 if (rdev->family >= CHIP_BARTS)
5452 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5453 else
5454 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5455
5456 if (!disable_plloff_in_l1) {
5457 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5458 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5459 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5460 if (data != orig)
5461 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5462
5463 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5464 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5465 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5466 if (data != orig)
5467 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5468
5469 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5470 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5471 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5472 if (data != orig)
5473 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5474
5475 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5476 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5477 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5478 if (data != orig)
5479 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5480
5481 if (rdev->family >= CHIP_BARTS) {
5482 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5483 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5484 data |= PLL_RAMP_UP_TIME_0(4);
5485 if (data != orig)
5486 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5487
5488 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5489 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5490 data |= PLL_RAMP_UP_TIME_1(4);
5491 if (data != orig)
5492 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5493
5494 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5495 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5496 data |= PLL_RAMP_UP_TIME_0(4);
5497 if (data != orig)
5498 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5499
5500 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5501 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5502 data |= PLL_RAMP_UP_TIME_1(4);
5503 if (data != orig)
5504 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5505 }
5506
5507 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5508 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5509 data |= LC_DYN_LANES_PWR_STATE(3);
5510 if (data != orig)
5511 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5512
5513 if (rdev->family >= CHIP_BARTS) {
5514 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5515 data &= ~LS2_EXIT_TIME_MASK;
5516 data |= LS2_EXIT_TIME(1);
5517 if (data != orig)
5518 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5519
5520 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5521 data &= ~LS2_EXIT_TIME_MASK;
5522 data |= LS2_EXIT_TIME(1);
5523 if (data != orig)
5524 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5525 }
5526 }
5527 }
5528
5529 /* evergreen parts only */
5530 if (rdev->family < CHIP_BARTS)
5531 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5532
5533 if (pcie_lc_cntl != pcie_lc_cntl_old)
5534 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5535 }
5536