1 /* SPDX-License-Identifier: MIT */
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include <linux/string_helpers.h>
7
8 #include "i915_drv.h"
9 #include "i915_irq.h"
10 #include "i915_reg.h"
11 #include "intel_backlight_regs.h"
12 #include "intel_cdclk.h"
13 #include "intel_clock_gating.h"
14 #include "intel_combo_phy.h"
15 #include "intel_de.h"
16 #include "intel_display_power.h"
17 #include "intel_display_power_map.h"
18 #include "intel_display_power_well.h"
19 #include "intel_display_types.h"
20 #include "intel_dmc.h"
21 #include "intel_mchbar_regs.h"
22 #include "intel_pch_refclk.h"
23 #include "intel_pcode.h"
24 #include "intel_pmdemand.h"
25 #include "intel_pps_regs.h"
26 #include "intel_snps_phy.h"
27 #include "skl_watermark.h"
28 #include "skl_watermark_regs.h"
29 #include "vlv_sideband.h"
30
31 #define for_each_power_domain_well(__dev_priv, __power_well, __domain) \
32 for_each_power_well(__dev_priv, __power_well) \
33 for_each_if(test_bit((__domain), (__power_well)->domains.bits))
34
35 #define for_each_power_domain_well_reverse(__dev_priv, __power_well, __domain) \
36 for_each_power_well_reverse(__dev_priv, __power_well) \
37 for_each_if(test_bit((__domain), (__power_well)->domains.bits))
38
39 static const char *
intel_display_power_domain_str(enum intel_display_power_domain domain)40 intel_display_power_domain_str(enum intel_display_power_domain domain)
41 {
42 switch (domain) {
43 case POWER_DOMAIN_DISPLAY_CORE:
44 return "DISPLAY_CORE";
45 case POWER_DOMAIN_PIPE_A:
46 return "PIPE_A";
47 case POWER_DOMAIN_PIPE_B:
48 return "PIPE_B";
49 case POWER_DOMAIN_PIPE_C:
50 return "PIPE_C";
51 case POWER_DOMAIN_PIPE_D:
52 return "PIPE_D";
53 case POWER_DOMAIN_PIPE_PANEL_FITTER_A:
54 return "PIPE_PANEL_FITTER_A";
55 case POWER_DOMAIN_PIPE_PANEL_FITTER_B:
56 return "PIPE_PANEL_FITTER_B";
57 case POWER_DOMAIN_PIPE_PANEL_FITTER_C:
58 return "PIPE_PANEL_FITTER_C";
59 case POWER_DOMAIN_PIPE_PANEL_FITTER_D:
60 return "PIPE_PANEL_FITTER_D";
61 case POWER_DOMAIN_TRANSCODER_A:
62 return "TRANSCODER_A";
63 case POWER_DOMAIN_TRANSCODER_B:
64 return "TRANSCODER_B";
65 case POWER_DOMAIN_TRANSCODER_C:
66 return "TRANSCODER_C";
67 case POWER_DOMAIN_TRANSCODER_D:
68 return "TRANSCODER_D";
69 case POWER_DOMAIN_TRANSCODER_EDP:
70 return "TRANSCODER_EDP";
71 case POWER_DOMAIN_TRANSCODER_DSI_A:
72 return "TRANSCODER_DSI_A";
73 case POWER_DOMAIN_TRANSCODER_DSI_C:
74 return "TRANSCODER_DSI_C";
75 case POWER_DOMAIN_TRANSCODER_VDSC_PW2:
76 return "TRANSCODER_VDSC_PW2";
77 case POWER_DOMAIN_PORT_DDI_LANES_A:
78 return "PORT_DDI_LANES_A";
79 case POWER_DOMAIN_PORT_DDI_LANES_B:
80 return "PORT_DDI_LANES_B";
81 case POWER_DOMAIN_PORT_DDI_LANES_C:
82 return "PORT_DDI_LANES_C";
83 case POWER_DOMAIN_PORT_DDI_LANES_D:
84 return "PORT_DDI_LANES_D";
85 case POWER_DOMAIN_PORT_DDI_LANES_E:
86 return "PORT_DDI_LANES_E";
87 case POWER_DOMAIN_PORT_DDI_LANES_F:
88 return "PORT_DDI_LANES_F";
89 case POWER_DOMAIN_PORT_DDI_LANES_TC1:
90 return "PORT_DDI_LANES_TC1";
91 case POWER_DOMAIN_PORT_DDI_LANES_TC2:
92 return "PORT_DDI_LANES_TC2";
93 case POWER_DOMAIN_PORT_DDI_LANES_TC3:
94 return "PORT_DDI_LANES_TC3";
95 case POWER_DOMAIN_PORT_DDI_LANES_TC4:
96 return "PORT_DDI_LANES_TC4";
97 case POWER_DOMAIN_PORT_DDI_LANES_TC5:
98 return "PORT_DDI_LANES_TC5";
99 case POWER_DOMAIN_PORT_DDI_LANES_TC6:
100 return "PORT_DDI_LANES_TC6";
101 case POWER_DOMAIN_PORT_DDI_IO_A:
102 return "PORT_DDI_IO_A";
103 case POWER_DOMAIN_PORT_DDI_IO_B:
104 return "PORT_DDI_IO_B";
105 case POWER_DOMAIN_PORT_DDI_IO_C:
106 return "PORT_DDI_IO_C";
107 case POWER_DOMAIN_PORT_DDI_IO_D:
108 return "PORT_DDI_IO_D";
109 case POWER_DOMAIN_PORT_DDI_IO_E:
110 return "PORT_DDI_IO_E";
111 case POWER_DOMAIN_PORT_DDI_IO_F:
112 return "PORT_DDI_IO_F";
113 case POWER_DOMAIN_PORT_DDI_IO_TC1:
114 return "PORT_DDI_IO_TC1";
115 case POWER_DOMAIN_PORT_DDI_IO_TC2:
116 return "PORT_DDI_IO_TC2";
117 case POWER_DOMAIN_PORT_DDI_IO_TC3:
118 return "PORT_DDI_IO_TC3";
119 case POWER_DOMAIN_PORT_DDI_IO_TC4:
120 return "PORT_DDI_IO_TC4";
121 case POWER_DOMAIN_PORT_DDI_IO_TC5:
122 return "PORT_DDI_IO_TC5";
123 case POWER_DOMAIN_PORT_DDI_IO_TC6:
124 return "PORT_DDI_IO_TC6";
125 case POWER_DOMAIN_PORT_DSI:
126 return "PORT_DSI";
127 case POWER_DOMAIN_PORT_CRT:
128 return "PORT_CRT";
129 case POWER_DOMAIN_PORT_OTHER:
130 return "PORT_OTHER";
131 case POWER_DOMAIN_VGA:
132 return "VGA";
133 case POWER_DOMAIN_AUDIO_MMIO:
134 return "AUDIO_MMIO";
135 case POWER_DOMAIN_AUDIO_PLAYBACK:
136 return "AUDIO_PLAYBACK";
137 case POWER_DOMAIN_AUX_IO_A:
138 return "AUX_IO_A";
139 case POWER_DOMAIN_AUX_IO_B:
140 return "AUX_IO_B";
141 case POWER_DOMAIN_AUX_IO_C:
142 return "AUX_IO_C";
143 case POWER_DOMAIN_AUX_IO_D:
144 return "AUX_IO_D";
145 case POWER_DOMAIN_AUX_IO_E:
146 return "AUX_IO_E";
147 case POWER_DOMAIN_AUX_IO_F:
148 return "AUX_IO_F";
149 case POWER_DOMAIN_AUX_A:
150 return "AUX_A";
151 case POWER_DOMAIN_AUX_B:
152 return "AUX_B";
153 case POWER_DOMAIN_AUX_C:
154 return "AUX_C";
155 case POWER_DOMAIN_AUX_D:
156 return "AUX_D";
157 case POWER_DOMAIN_AUX_E:
158 return "AUX_E";
159 case POWER_DOMAIN_AUX_F:
160 return "AUX_F";
161 case POWER_DOMAIN_AUX_USBC1:
162 return "AUX_USBC1";
163 case POWER_DOMAIN_AUX_USBC2:
164 return "AUX_USBC2";
165 case POWER_DOMAIN_AUX_USBC3:
166 return "AUX_USBC3";
167 case POWER_DOMAIN_AUX_USBC4:
168 return "AUX_USBC4";
169 case POWER_DOMAIN_AUX_USBC5:
170 return "AUX_USBC5";
171 case POWER_DOMAIN_AUX_USBC6:
172 return "AUX_USBC6";
173 case POWER_DOMAIN_AUX_TBT1:
174 return "AUX_TBT1";
175 case POWER_DOMAIN_AUX_TBT2:
176 return "AUX_TBT2";
177 case POWER_DOMAIN_AUX_TBT3:
178 return "AUX_TBT3";
179 case POWER_DOMAIN_AUX_TBT4:
180 return "AUX_TBT4";
181 case POWER_DOMAIN_AUX_TBT5:
182 return "AUX_TBT5";
183 case POWER_DOMAIN_AUX_TBT6:
184 return "AUX_TBT6";
185 case POWER_DOMAIN_GMBUS:
186 return "GMBUS";
187 case POWER_DOMAIN_INIT:
188 return "INIT";
189 case POWER_DOMAIN_GT_IRQ:
190 return "GT_IRQ";
191 case POWER_DOMAIN_DC_OFF:
192 return "DC_OFF";
193 case POWER_DOMAIN_TC_COLD_OFF:
194 return "TC_COLD_OFF";
195 default:
196 MISSING_CASE(domain);
197 return "?";
198 }
199 }
200
__intel_display_power_is_enabled(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain)201 static bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
202 enum intel_display_power_domain domain)
203 {
204 struct i915_power_well *power_well;
205 bool is_enabled;
206
207 if (pm_runtime_suspended(dev_priv->drm.dev))
208 return false;
209
210 is_enabled = true;
211
212 for_each_power_domain_well_reverse(dev_priv, power_well, domain) {
213 if (intel_power_well_is_always_on(power_well))
214 continue;
215
216 if (!intel_power_well_is_enabled_cached(power_well)) {
217 is_enabled = false;
218 break;
219 }
220 }
221
222 return is_enabled;
223 }
224
225 /**
226 * intel_display_power_is_enabled - check for a power domain
227 * @dev_priv: i915 device instance
228 * @domain: power domain to check
229 *
230 * This function can be used to check the hw power domain state. It is mostly
231 * used in hardware state readout functions. Everywhere else code should rely
232 * upon explicit power domain reference counting to ensure that the hardware
233 * block is powered up before accessing it.
234 *
235 * Callers must hold the relevant modesetting locks to ensure that concurrent
236 * threads can't disable the power well while the caller tries to read a few
237 * registers.
238 *
239 * Returns:
240 * True when the power domain is enabled, false otherwise.
241 */
intel_display_power_is_enabled(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain)242 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
243 enum intel_display_power_domain domain)
244 {
245 struct i915_power_domains *power_domains;
246 bool ret;
247
248 power_domains = &dev_priv->display.power.domains;
249
250 mutex_lock(&power_domains->lock);
251 ret = __intel_display_power_is_enabled(dev_priv, domain);
252 mutex_unlock(&power_domains->lock);
253
254 return ret;
255 }
256
257 static u32
sanitize_target_dc_state(struct drm_i915_private * i915,u32 target_dc_state)258 sanitize_target_dc_state(struct drm_i915_private *i915,
259 u32 target_dc_state)
260 {
261 struct i915_power_domains *power_domains = &i915->display.power.domains;
262 static const u32 states[] = {
263 DC_STATE_EN_UPTO_DC6,
264 DC_STATE_EN_UPTO_DC5,
265 DC_STATE_EN_DC3CO,
266 DC_STATE_DISABLE,
267 };
268 int i;
269
270 for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
271 if (target_dc_state != states[i])
272 continue;
273
274 if (power_domains->allowed_dc_mask & target_dc_state)
275 break;
276
277 target_dc_state = states[i + 1];
278 }
279
280 return target_dc_state;
281 }
282
283 /**
284 * intel_display_power_set_target_dc_state - Set target dc state.
285 * @dev_priv: i915 device
286 * @state: state which needs to be set as target_dc_state.
287 *
288 * This function set the "DC off" power well target_dc_state,
289 * based upon this target_dc_stste, "DC off" power well will
290 * enable desired DC state.
291 */
intel_display_power_set_target_dc_state(struct drm_i915_private * dev_priv,u32 state)292 void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
293 u32 state)
294 {
295 struct i915_power_well *power_well;
296 bool dc_off_enabled;
297 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
298
299 mutex_lock(&power_domains->lock);
300 power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF);
301
302 if (drm_WARN_ON(&dev_priv->drm, !power_well))
303 goto unlock;
304
305 state = sanitize_target_dc_state(dev_priv, state);
306
307 if (state == power_domains->target_dc_state)
308 goto unlock;
309
310 dc_off_enabled = intel_power_well_is_enabled(dev_priv, power_well);
311 /*
312 * If DC off power well is disabled, need to enable and disable the
313 * DC off power well to effect target DC state.
314 */
315 if (!dc_off_enabled)
316 intel_power_well_enable(dev_priv, power_well);
317
318 power_domains->target_dc_state = state;
319
320 if (!dc_off_enabled)
321 intel_power_well_disable(dev_priv, power_well);
322
323 unlock:
324 mutex_unlock(&power_domains->lock);
325 }
326
__async_put_domains_mask(struct i915_power_domains * power_domains,struct intel_power_domain_mask * mask)327 static void __async_put_domains_mask(struct i915_power_domains *power_domains,
328 struct intel_power_domain_mask *mask)
329 {
330 bitmap_or(mask->bits,
331 power_domains->async_put_domains[0].bits,
332 power_domains->async_put_domains[1].bits,
333 POWER_DOMAIN_NUM);
334 }
335
336 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
337
338 static bool
assert_async_put_domain_masks_disjoint(struct i915_power_domains * power_domains)339 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
340 {
341 struct drm_i915_private *i915 = container_of(power_domains,
342 struct drm_i915_private,
343 display.power.domains);
344
345 return !drm_WARN_ON(&i915->drm,
346 bitmap_intersects(power_domains->async_put_domains[0].bits,
347 power_domains->async_put_domains[1].bits,
348 POWER_DOMAIN_NUM));
349 }
350
351 static bool
__async_put_domains_state_ok(struct i915_power_domains * power_domains)352 __async_put_domains_state_ok(struct i915_power_domains *power_domains)
353 {
354 struct drm_i915_private *i915 = container_of(power_domains,
355 struct drm_i915_private,
356 display.power.domains);
357 struct intel_power_domain_mask async_put_mask;
358 enum intel_display_power_domain domain;
359 bool err = false;
360
361 err |= !assert_async_put_domain_masks_disjoint(power_domains);
362 __async_put_domains_mask(power_domains, &async_put_mask);
363 err |= drm_WARN_ON(&i915->drm,
364 !!power_domains->async_put_wakeref !=
365 !bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM));
366
367 for_each_power_domain(domain, &async_put_mask)
368 err |= drm_WARN_ON(&i915->drm,
369 power_domains->domain_use_count[domain] != 1);
370
371 return !err;
372 }
373
print_power_domains(struct i915_power_domains * power_domains,const char * prefix,struct intel_power_domain_mask * mask)374 static void print_power_domains(struct i915_power_domains *power_domains,
375 const char *prefix, struct intel_power_domain_mask *mask)
376 {
377 struct drm_i915_private *i915 = container_of(power_domains,
378 struct drm_i915_private,
379 display.power.domains);
380 enum intel_display_power_domain domain;
381
382 drm_dbg(&i915->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM));
383 for_each_power_domain(domain, mask)
384 drm_dbg(&i915->drm, "%s use_count %d\n",
385 intel_display_power_domain_str(domain),
386 power_domains->domain_use_count[domain]);
387 }
388
389 static void
print_async_put_domains_state(struct i915_power_domains * power_domains)390 print_async_put_domains_state(struct i915_power_domains *power_domains)
391 {
392 struct drm_i915_private *i915 = container_of(power_domains,
393 struct drm_i915_private,
394 display.power.domains);
395
396 drm_dbg(&i915->drm, "async_put_wakeref: %s\n",
397 str_yes_no(power_domains->async_put_wakeref));
398
399 print_power_domains(power_domains, "async_put_domains[0]",
400 &power_domains->async_put_domains[0]);
401 print_power_domains(power_domains, "async_put_domains[1]",
402 &power_domains->async_put_domains[1]);
403 }
404
405 static void
verify_async_put_domains_state(struct i915_power_domains * power_domains)406 verify_async_put_domains_state(struct i915_power_domains *power_domains)
407 {
408 if (!__async_put_domains_state_ok(power_domains))
409 print_async_put_domains_state(power_domains);
410 }
411
412 #else
413
414 static void
assert_async_put_domain_masks_disjoint(struct i915_power_domains * power_domains)415 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
416 {
417 }
418
419 static void
verify_async_put_domains_state(struct i915_power_domains * power_domains)420 verify_async_put_domains_state(struct i915_power_domains *power_domains)
421 {
422 }
423
424 #endif /* CONFIG_DRM_I915_DEBUG_RUNTIME_PM */
425
async_put_domains_mask(struct i915_power_domains * power_domains,struct intel_power_domain_mask * mask)426 static void async_put_domains_mask(struct i915_power_domains *power_domains,
427 struct intel_power_domain_mask *mask)
428
429 {
430 assert_async_put_domain_masks_disjoint(power_domains);
431
432 __async_put_domains_mask(power_domains, mask);
433 }
434
435 static void
async_put_domains_clear_domain(struct i915_power_domains * power_domains,enum intel_display_power_domain domain)436 async_put_domains_clear_domain(struct i915_power_domains *power_domains,
437 enum intel_display_power_domain domain)
438 {
439 assert_async_put_domain_masks_disjoint(power_domains);
440
441 clear_bit(domain, power_domains->async_put_domains[0].bits);
442 clear_bit(domain, power_domains->async_put_domains[1].bits);
443 }
444
445 static void
cancel_async_put_work(struct i915_power_domains * power_domains,bool sync)446 cancel_async_put_work(struct i915_power_domains *power_domains, bool sync)
447 {
448 if (sync)
449 cancel_delayed_work_sync(&power_domains->async_put_work);
450 else
451 cancel_delayed_work(&power_domains->async_put_work);
452
453 power_domains->async_put_next_delay = 0;
454 }
455
456 static bool
intel_display_power_grab_async_put_ref(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain)457 intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv,
458 enum intel_display_power_domain domain)
459 {
460 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
461 struct intel_power_domain_mask async_put_mask;
462 bool ret = false;
463
464 async_put_domains_mask(power_domains, &async_put_mask);
465 if (!test_bit(domain, async_put_mask.bits))
466 goto out_verify;
467
468 async_put_domains_clear_domain(power_domains, domain);
469
470 ret = true;
471
472 async_put_domains_mask(power_domains, &async_put_mask);
473 if (!bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM))
474 goto out_verify;
475
476 cancel_async_put_work(power_domains, false);
477 intel_runtime_pm_put_raw(&dev_priv->runtime_pm,
478 fetch_and_zero(&power_domains->async_put_wakeref));
479 out_verify:
480 verify_async_put_domains_state(power_domains);
481
482 return ret;
483 }
484
485 static void
__intel_display_power_get_domain(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain)486 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
487 enum intel_display_power_domain domain)
488 {
489 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
490 struct i915_power_well *power_well;
491
492 if (intel_display_power_grab_async_put_ref(dev_priv, domain))
493 return;
494
495 for_each_power_domain_well(dev_priv, power_well, domain)
496 intel_power_well_get(dev_priv, power_well);
497
498 power_domains->domain_use_count[domain]++;
499 }
500
501 /**
502 * intel_display_power_get - grab a power domain reference
503 * @dev_priv: i915 device instance
504 * @domain: power domain to reference
505 *
506 * This function grabs a power domain reference for @domain and ensures that the
507 * power domain and all its parents are powered up. Therefore users should only
508 * grab a reference to the innermost power domain they need.
509 *
510 * Any power domain reference obtained by this function must have a symmetric
511 * call to intel_display_power_put() to release the reference again.
512 */
intel_display_power_get(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain)513 intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv,
514 enum intel_display_power_domain domain)
515 {
516 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
517 intel_wakeref_t wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
518
519 mutex_lock(&power_domains->lock);
520 __intel_display_power_get_domain(dev_priv, domain);
521 mutex_unlock(&power_domains->lock);
522
523 return wakeref;
524 }
525
526 /**
527 * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
528 * @dev_priv: i915 device instance
529 * @domain: power domain to reference
530 *
531 * This function grabs a power domain reference for @domain and ensures that the
532 * power domain and all its parents are powered up. Therefore users should only
533 * grab a reference to the innermost power domain they need.
534 *
535 * Any power domain reference obtained by this function must have a symmetric
536 * call to intel_display_power_put() to release the reference again.
537 */
538 intel_wakeref_t
intel_display_power_get_if_enabled(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain)539 intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
540 enum intel_display_power_domain domain)
541 {
542 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
543 intel_wakeref_t wakeref;
544 bool is_enabled;
545
546 wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
547 if (!wakeref)
548 return false;
549
550 mutex_lock(&power_domains->lock);
551
552 if (__intel_display_power_is_enabled(dev_priv, domain)) {
553 __intel_display_power_get_domain(dev_priv, domain);
554 is_enabled = true;
555 } else {
556 is_enabled = false;
557 }
558
559 mutex_unlock(&power_domains->lock);
560
561 if (!is_enabled) {
562 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
563 wakeref = 0;
564 }
565
566 return wakeref;
567 }
568
569 static void
__intel_display_power_put_domain(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain)570 __intel_display_power_put_domain(struct drm_i915_private *dev_priv,
571 enum intel_display_power_domain domain)
572 {
573 struct i915_power_domains *power_domains;
574 struct i915_power_well *power_well;
575 const char *name = intel_display_power_domain_str(domain);
576 struct intel_power_domain_mask async_put_mask;
577
578 power_domains = &dev_priv->display.power.domains;
579
580 drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain],
581 "Use count on domain %s is already zero\n",
582 name);
583 async_put_domains_mask(power_domains, &async_put_mask);
584 drm_WARN(&dev_priv->drm,
585 test_bit(domain, async_put_mask.bits),
586 "Async disabling of domain %s is pending\n",
587 name);
588
589 power_domains->domain_use_count[domain]--;
590
591 for_each_power_domain_well_reverse(dev_priv, power_well, domain)
592 intel_power_well_put(dev_priv, power_well);
593 }
594
__intel_display_power_put(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain)595 static void __intel_display_power_put(struct drm_i915_private *dev_priv,
596 enum intel_display_power_domain domain)
597 {
598 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
599
600 mutex_lock(&power_domains->lock);
601 __intel_display_power_put_domain(dev_priv, domain);
602 mutex_unlock(&power_domains->lock);
603 }
604
605 static void
queue_async_put_domains_work(struct i915_power_domains * power_domains,intel_wakeref_t wakeref,int delay_ms)606 queue_async_put_domains_work(struct i915_power_domains *power_domains,
607 intel_wakeref_t wakeref,
608 int delay_ms)
609 {
610 struct drm_i915_private *i915 = container_of(power_domains,
611 struct drm_i915_private,
612 display.power.domains);
613 drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
614 power_domains->async_put_wakeref = wakeref;
615 drm_WARN_ON(&i915->drm, !queue_delayed_work(system_unbound_wq,
616 &power_domains->async_put_work,
617 msecs_to_jiffies(delay_ms)));
618 }
619
620 static void
release_async_put_domains(struct i915_power_domains * power_domains,struct intel_power_domain_mask * mask)621 release_async_put_domains(struct i915_power_domains *power_domains,
622 struct intel_power_domain_mask *mask)
623 {
624 struct drm_i915_private *dev_priv =
625 container_of(power_domains, struct drm_i915_private,
626 display.power.domains);
627 struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
628 enum intel_display_power_domain domain;
629 intel_wakeref_t wakeref;
630
631 wakeref = intel_runtime_pm_get_noresume(rpm);
632
633 for_each_power_domain(domain, mask) {
634 /* Clear before put, so put's sanity check is happy. */
635 async_put_domains_clear_domain(power_domains, domain);
636 __intel_display_power_put_domain(dev_priv, domain);
637 }
638
639 intel_runtime_pm_put(rpm, wakeref);
640 }
641
642 static void
intel_display_power_put_async_work(struct work_struct * work)643 intel_display_power_put_async_work(struct work_struct *work)
644 {
645 struct drm_i915_private *dev_priv =
646 container_of(work, struct drm_i915_private,
647 display.power.domains.async_put_work.work);
648 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
649 struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
650 intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm);
651 intel_wakeref_t old_work_wakeref = 0;
652
653 mutex_lock(&power_domains->lock);
654
655 /*
656 * Bail out if all the domain refs pending to be released were grabbed
657 * by subsequent gets or a flush_work.
658 */
659 old_work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
660 if (!old_work_wakeref)
661 goto out_verify;
662
663 release_async_put_domains(power_domains,
664 &power_domains->async_put_domains[0]);
665
666 /*
667 * Cancel the work that got queued after this one got dequeued,
668 * since here we released the corresponding async-put reference.
669 */
670 cancel_async_put_work(power_domains, false);
671
672 /* Requeue the work if more domains were async put meanwhile. */
673 if (!bitmap_empty(power_domains->async_put_domains[1].bits, POWER_DOMAIN_NUM)) {
674 bitmap_copy(power_domains->async_put_domains[0].bits,
675 power_domains->async_put_domains[1].bits,
676 POWER_DOMAIN_NUM);
677 bitmap_zero(power_domains->async_put_domains[1].bits,
678 POWER_DOMAIN_NUM);
679 queue_async_put_domains_work(power_domains,
680 fetch_and_zero(&new_work_wakeref),
681 power_domains->async_put_next_delay);
682 power_domains->async_put_next_delay = 0;
683 }
684
685 out_verify:
686 verify_async_put_domains_state(power_domains);
687
688 mutex_unlock(&power_domains->lock);
689
690 if (old_work_wakeref)
691 intel_runtime_pm_put_raw(rpm, old_work_wakeref);
692 if (new_work_wakeref)
693 intel_runtime_pm_put_raw(rpm, new_work_wakeref);
694 }
695
696 /**
697 * __intel_display_power_put_async - release a power domain reference asynchronously
698 * @i915: i915 device instance
699 * @domain: power domain to reference
700 * @wakeref: wakeref acquired for the reference that is being released
701 * @delay_ms: delay of powering down the power domain
702 *
703 * This function drops the power domain reference obtained by
704 * intel_display_power_get*() and schedules a work to power down the
705 * corresponding hardware block if this is the last reference.
706 * The power down is delayed by @delay_ms if this is >= 0, or by a default
707 * 100 ms otherwise.
708 */
__intel_display_power_put_async(struct drm_i915_private * i915,enum intel_display_power_domain domain,intel_wakeref_t wakeref,int delay_ms)709 void __intel_display_power_put_async(struct drm_i915_private *i915,
710 enum intel_display_power_domain domain,
711 intel_wakeref_t wakeref,
712 int delay_ms)
713 {
714 struct i915_power_domains *power_domains = &i915->display.power.domains;
715 struct intel_runtime_pm *rpm = &i915->runtime_pm;
716 intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm);
717
718 delay_ms = delay_ms >= 0 ? delay_ms : 100;
719
720 mutex_lock(&power_domains->lock);
721
722 if (power_domains->domain_use_count[domain] > 1) {
723 __intel_display_power_put_domain(i915, domain);
724
725 goto out_verify;
726 }
727
728 drm_WARN_ON(&i915->drm, power_domains->domain_use_count[domain] != 1);
729
730 /* Let a pending work requeue itself or queue a new one. */
731 if (power_domains->async_put_wakeref) {
732 set_bit(domain, power_domains->async_put_domains[1].bits);
733 power_domains->async_put_next_delay = max(power_domains->async_put_next_delay,
734 delay_ms);
735 } else {
736 set_bit(domain, power_domains->async_put_domains[0].bits);
737 queue_async_put_domains_work(power_domains,
738 fetch_and_zero(&work_wakeref),
739 delay_ms);
740 }
741
742 out_verify:
743 verify_async_put_domains_state(power_domains);
744
745 mutex_unlock(&power_domains->lock);
746
747 if (work_wakeref)
748 intel_runtime_pm_put_raw(rpm, work_wakeref);
749
750 intel_runtime_pm_put(rpm, wakeref);
751 }
752
753 /**
754 * intel_display_power_flush_work - flushes the async display power disabling work
755 * @i915: i915 device instance
756 *
757 * Flushes any pending work that was scheduled by a preceding
758 * intel_display_power_put_async() call, completing the disabling of the
759 * corresponding power domains.
760 *
761 * Note that the work handler function may still be running after this
762 * function returns; to ensure that the work handler isn't running use
763 * intel_display_power_flush_work_sync() instead.
764 */
intel_display_power_flush_work(struct drm_i915_private * i915)765 void intel_display_power_flush_work(struct drm_i915_private *i915)
766 {
767 struct i915_power_domains *power_domains = &i915->display.power.domains;
768 struct intel_power_domain_mask async_put_mask;
769 intel_wakeref_t work_wakeref;
770
771 mutex_lock(&power_domains->lock);
772
773 work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
774 if (!work_wakeref)
775 goto out_verify;
776
777 async_put_domains_mask(power_domains, &async_put_mask);
778 release_async_put_domains(power_domains, &async_put_mask);
779 cancel_async_put_work(power_domains, false);
780
781 out_verify:
782 verify_async_put_domains_state(power_domains);
783
784 mutex_unlock(&power_domains->lock);
785
786 if (work_wakeref)
787 intel_runtime_pm_put_raw(&i915->runtime_pm, work_wakeref);
788 }
789
790 /**
791 * intel_display_power_flush_work_sync - flushes and syncs the async display power disabling work
792 * @i915: i915 device instance
793 *
794 * Like intel_display_power_flush_work(), but also ensure that the work
795 * handler function is not running any more when this function returns.
796 */
797 static void
intel_display_power_flush_work_sync(struct drm_i915_private * i915)798 intel_display_power_flush_work_sync(struct drm_i915_private *i915)
799 {
800 struct i915_power_domains *power_domains = &i915->display.power.domains;
801
802 intel_display_power_flush_work(i915);
803 cancel_async_put_work(power_domains, true);
804
805 verify_async_put_domains_state(power_domains);
806
807 drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
808 }
809
810 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
811 /**
812 * intel_display_power_put - release a power domain reference
813 * @dev_priv: i915 device instance
814 * @domain: power domain to reference
815 * @wakeref: wakeref acquired for the reference that is being released
816 *
817 * This function drops the power domain reference obtained by
818 * intel_display_power_get() and might power down the corresponding hardware
819 * block right away if this is the last reference.
820 */
intel_display_power_put(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain,intel_wakeref_t wakeref)821 void intel_display_power_put(struct drm_i915_private *dev_priv,
822 enum intel_display_power_domain domain,
823 intel_wakeref_t wakeref)
824 {
825 __intel_display_power_put(dev_priv, domain);
826 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
827 }
828 #else
829 /**
830 * intel_display_power_put_unchecked - release an unchecked power domain reference
831 * @dev_priv: i915 device instance
832 * @domain: power domain to reference
833 *
834 * This function drops the power domain reference obtained by
835 * intel_display_power_get() and might power down the corresponding hardware
836 * block right away if this is the last reference.
837 *
838 * This function is only for the power domain code's internal use to suppress wakeref
839 * tracking when the correspondig debug kconfig option is disabled, should not
840 * be used otherwise.
841 */
intel_display_power_put_unchecked(struct drm_i915_private * dev_priv,enum intel_display_power_domain domain)842 void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv,
843 enum intel_display_power_domain domain)
844 {
845 __intel_display_power_put(dev_priv, domain);
846 intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
847 }
848 #endif
849
850 void
intel_display_power_get_in_set(struct drm_i915_private * i915,struct intel_display_power_domain_set * power_domain_set,enum intel_display_power_domain domain)851 intel_display_power_get_in_set(struct drm_i915_private *i915,
852 struct intel_display_power_domain_set *power_domain_set,
853 enum intel_display_power_domain domain)
854 {
855 intel_wakeref_t __maybe_unused wf;
856
857 drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
858
859 wf = intel_display_power_get(i915, domain);
860 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
861 power_domain_set->wakerefs[domain] = wf;
862 #endif
863 set_bit(domain, power_domain_set->mask.bits);
864 }
865
866 bool
intel_display_power_get_in_set_if_enabled(struct drm_i915_private * i915,struct intel_display_power_domain_set * power_domain_set,enum intel_display_power_domain domain)867 intel_display_power_get_in_set_if_enabled(struct drm_i915_private *i915,
868 struct intel_display_power_domain_set *power_domain_set,
869 enum intel_display_power_domain domain)
870 {
871 intel_wakeref_t wf;
872
873 drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
874
875 wf = intel_display_power_get_if_enabled(i915, domain);
876 if (!wf)
877 return false;
878
879 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
880 power_domain_set->wakerefs[domain] = wf;
881 #endif
882 set_bit(domain, power_domain_set->mask.bits);
883
884 return true;
885 }
886
887 void
intel_display_power_put_mask_in_set(struct drm_i915_private * i915,struct intel_display_power_domain_set * power_domain_set,struct intel_power_domain_mask * mask)888 intel_display_power_put_mask_in_set(struct drm_i915_private *i915,
889 struct intel_display_power_domain_set *power_domain_set,
890 struct intel_power_domain_mask *mask)
891 {
892 enum intel_display_power_domain domain;
893
894 #ifdef notyet
895 drm_WARN_ON(&i915->drm,
896 !bitmap_subset(mask->bits, power_domain_set->mask.bits, POWER_DOMAIN_NUM));
897 #endif
898
899 for_each_power_domain(domain, mask) {
900 intel_wakeref_t __maybe_unused wf = -1;
901
902 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
903 wf = fetch_and_zero(&power_domain_set->wakerefs[domain]);
904 #endif
905 intel_display_power_put(i915, domain, wf);
906 clear_bit(domain, power_domain_set->mask.bits);
907 }
908 }
909
910 static int
sanitize_disable_power_well_option(const struct drm_i915_private * dev_priv,int disable_power_well)911 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
912 int disable_power_well)
913 {
914 if (disable_power_well >= 0)
915 return !!disable_power_well;
916
917 return 1;
918 }
919
get_allowed_dc_mask(const struct drm_i915_private * dev_priv,int enable_dc)920 static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
921 int enable_dc)
922 {
923 u32 mask;
924 int requested_dc;
925 int max_dc;
926
927 if (!HAS_DISPLAY(dev_priv))
928 return 0;
929
930 if (DISPLAY_VER(dev_priv) >= 20)
931 max_dc = 2;
932 else if (IS_DG2(dev_priv))
933 max_dc = 1;
934 else if (IS_DG1(dev_priv))
935 max_dc = 3;
936 else if (DISPLAY_VER(dev_priv) >= 12)
937 max_dc = 4;
938 else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
939 max_dc = 1;
940 else if (DISPLAY_VER(dev_priv) >= 9)
941 max_dc = 2;
942 else
943 max_dc = 0;
944
945 /*
946 * DC9 has a separate HW flow from the rest of the DC states,
947 * not depending on the DMC firmware. It's needed by system
948 * suspend/resume, so allow it unconditionally.
949 */
950 mask = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ||
951 DISPLAY_VER(dev_priv) >= 11 ?
952 DC_STATE_EN_DC9 : 0;
953
954 if (!dev_priv->display.params.disable_power_well)
955 max_dc = 0;
956
957 if (enable_dc >= 0 && enable_dc <= max_dc) {
958 requested_dc = enable_dc;
959 } else if (enable_dc == -1) {
960 requested_dc = max_dc;
961 } else if (enable_dc > max_dc && enable_dc <= 4) {
962 drm_dbg_kms(&dev_priv->drm,
963 "Adjusting requested max DC state (%d->%d)\n",
964 enable_dc, max_dc);
965 requested_dc = max_dc;
966 } else {
967 drm_err(&dev_priv->drm,
968 "Unexpected value for enable_dc (%d)\n", enable_dc);
969 requested_dc = max_dc;
970 }
971
972 switch (requested_dc) {
973 case 4:
974 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
975 break;
976 case 3:
977 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
978 break;
979 case 2:
980 mask |= DC_STATE_EN_UPTO_DC6;
981 break;
982 case 1:
983 mask |= DC_STATE_EN_UPTO_DC5;
984 break;
985 }
986
987 drm_dbg_kms(&dev_priv->drm, "Allowed DC state mask %02x\n", mask);
988
989 return mask;
990 }
991
992 /**
993 * intel_power_domains_init - initializes the power domain structures
994 * @dev_priv: i915 device instance
995 *
996 * Initializes the power domain structures for @dev_priv depending upon the
997 * supported platform.
998 */
intel_power_domains_init(struct drm_i915_private * dev_priv)999 int intel_power_domains_init(struct drm_i915_private *dev_priv)
1000 {
1001 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1002
1003 dev_priv->display.params.disable_power_well =
1004 sanitize_disable_power_well_option(dev_priv,
1005 dev_priv->display.params.disable_power_well);
1006 power_domains->allowed_dc_mask =
1007 get_allowed_dc_mask(dev_priv, dev_priv->display.params.enable_dc);
1008
1009 power_domains->target_dc_state =
1010 sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
1011
1012 rw_init(&power_domains->lock, "ipdl");
1013
1014 INIT_DELAYED_WORK(&power_domains->async_put_work,
1015 intel_display_power_put_async_work);
1016
1017 return intel_display_power_map_init(power_domains);
1018 }
1019
1020 /**
1021 * intel_power_domains_cleanup - clean up power domains resources
1022 * @dev_priv: i915 device instance
1023 *
1024 * Release any resources acquired by intel_power_domains_init()
1025 */
intel_power_domains_cleanup(struct drm_i915_private * dev_priv)1026 void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
1027 {
1028 intel_display_power_map_cleanup(&dev_priv->display.power.domains);
1029 }
1030
intel_power_domains_sync_hw(struct drm_i915_private * dev_priv)1031 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
1032 {
1033 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1034 struct i915_power_well *power_well;
1035
1036 mutex_lock(&power_domains->lock);
1037 for_each_power_well(dev_priv, power_well)
1038 intel_power_well_sync_hw(dev_priv, power_well);
1039 mutex_unlock(&power_domains->lock);
1040 }
1041
gen9_dbuf_slice_set(struct drm_i915_private * dev_priv,enum dbuf_slice slice,bool enable)1042 static void gen9_dbuf_slice_set(struct drm_i915_private *dev_priv,
1043 enum dbuf_slice slice, bool enable)
1044 {
1045 i915_reg_t reg = DBUF_CTL_S(slice);
1046 bool state;
1047
1048 intel_de_rmw(dev_priv, reg, DBUF_POWER_REQUEST,
1049 enable ? DBUF_POWER_REQUEST : 0);
1050 intel_de_posting_read(dev_priv, reg);
1051 udelay(10);
1052
1053 state = intel_de_read(dev_priv, reg) & DBUF_POWER_STATE;
1054 drm_WARN(&dev_priv->drm, enable != state,
1055 "DBuf slice %d power %s timeout!\n",
1056 slice, str_enable_disable(enable));
1057 }
1058
gen9_dbuf_slices_update(struct drm_i915_private * dev_priv,u8 req_slices)1059 void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv,
1060 u8 req_slices)
1061 {
1062 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1063 u8 slice_mask = DISPLAY_INFO(dev_priv)->dbuf.slice_mask;
1064 enum dbuf_slice slice;
1065
1066 drm_WARN(&dev_priv->drm, req_slices & ~slice_mask,
1067 "Invalid set of dbuf slices (0x%x) requested (total dbuf slices 0x%x)\n",
1068 req_slices, slice_mask);
1069
1070 drm_dbg_kms(&dev_priv->drm, "Updating dbuf slices to 0x%x\n",
1071 req_slices);
1072
1073 /*
1074 * Might be running this in parallel to gen9_dc_off_power_well_enable
1075 * being called from intel_dp_detect for instance,
1076 * which causes assertion triggered by race condition,
1077 * as gen9_assert_dbuf_enabled might preempt this when registers
1078 * were already updated, while dev_priv was not.
1079 */
1080 mutex_lock(&power_domains->lock);
1081
1082 for_each_dbuf_slice(dev_priv, slice)
1083 gen9_dbuf_slice_set(dev_priv, slice, req_slices & BIT(slice));
1084
1085 dev_priv->display.dbuf.enabled_slices = req_slices;
1086
1087 mutex_unlock(&power_domains->lock);
1088 }
1089
gen9_dbuf_enable(struct drm_i915_private * dev_priv)1090 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
1091 {
1092 u8 slices_mask;
1093
1094 dev_priv->display.dbuf.enabled_slices =
1095 intel_enabled_dbuf_slices_mask(dev_priv);
1096
1097 slices_mask = BIT(DBUF_S1) | dev_priv->display.dbuf.enabled_slices;
1098
1099 if (DISPLAY_VER(dev_priv) >= 14)
1100 intel_pmdemand_program_dbuf(dev_priv, slices_mask);
1101
1102 /*
1103 * Just power up at least 1 slice, we will
1104 * figure out later which slices we have and what we need.
1105 */
1106 gen9_dbuf_slices_update(dev_priv, slices_mask);
1107 }
1108
gen9_dbuf_disable(struct drm_i915_private * dev_priv)1109 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
1110 {
1111 gen9_dbuf_slices_update(dev_priv, 0);
1112
1113 if (DISPLAY_VER(dev_priv) >= 14)
1114 intel_pmdemand_program_dbuf(dev_priv, 0);
1115 }
1116
gen12_dbuf_slices_config(struct drm_i915_private * dev_priv)1117 static void gen12_dbuf_slices_config(struct drm_i915_private *dev_priv)
1118 {
1119 enum dbuf_slice slice;
1120
1121 if (IS_ALDERLAKE_P(dev_priv))
1122 return;
1123
1124 for_each_dbuf_slice(dev_priv, slice)
1125 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
1126 DBUF_TRACKER_STATE_SERVICE_MASK,
1127 DBUF_TRACKER_STATE_SERVICE(8));
1128 }
1129
icl_mbus_init(struct drm_i915_private * dev_priv)1130 static void icl_mbus_init(struct drm_i915_private *dev_priv)
1131 {
1132 unsigned long abox_regs = DISPLAY_INFO(dev_priv)->abox_mask;
1133 u32 mask, val, i;
1134
1135 if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
1136 return;
1137
1138 mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
1139 MBUS_ABOX_BT_CREDIT_POOL2_MASK |
1140 MBUS_ABOX_B_CREDIT_MASK |
1141 MBUS_ABOX_BW_CREDIT_MASK;
1142 val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
1143 MBUS_ABOX_BT_CREDIT_POOL2(16) |
1144 MBUS_ABOX_B_CREDIT(1) |
1145 MBUS_ABOX_BW_CREDIT(1);
1146
1147 /*
1148 * gen12 platforms that use abox1 and abox2 for pixel data reads still
1149 * expect us to program the abox_ctl0 register as well, even though
1150 * we don't have to program other instance-0 registers like BW_BUDDY.
1151 */
1152 if (DISPLAY_VER(dev_priv) == 12)
1153 abox_regs |= BIT(0);
1154
1155 for_each_set_bit(i, &abox_regs, sizeof(abox_regs))
1156 intel_de_rmw(dev_priv, MBUS_ABOX_CTL(i), mask, val);
1157 }
1158
hsw_assert_cdclk(struct drm_i915_private * dev_priv)1159 static void hsw_assert_cdclk(struct drm_i915_private *dev_priv)
1160 {
1161 u32 val = intel_de_read(dev_priv, LCPLL_CTL);
1162
1163 /*
1164 * The LCPLL register should be turned on by the BIOS. For now
1165 * let's just check its state and print errors in case
1166 * something is wrong. Don't even try to turn it on.
1167 */
1168
1169 if (val & LCPLL_CD_SOURCE_FCLK)
1170 drm_err(&dev_priv->drm, "CDCLK source is not LCPLL\n");
1171
1172 if (val & LCPLL_PLL_DISABLE)
1173 drm_err(&dev_priv->drm, "LCPLL is disabled\n");
1174
1175 if ((val & LCPLL_REF_MASK) != LCPLL_REF_NON_SSC)
1176 drm_err(&dev_priv->drm, "LCPLL not using non-SSC reference\n");
1177 }
1178
assert_can_disable_lcpll(struct drm_i915_private * dev_priv)1179 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
1180 {
1181 struct intel_crtc *crtc;
1182
1183 for_each_intel_crtc(&dev_priv->drm, crtc)
1184 I915_STATE_WARN(dev_priv, crtc->active,
1185 "CRTC for pipe %c enabled\n",
1186 pipe_name(crtc->pipe));
1187
1188 I915_STATE_WARN(dev_priv, intel_de_read(dev_priv, HSW_PWR_WELL_CTL2),
1189 "Display power well on\n");
1190 I915_STATE_WARN(dev_priv,
1191 intel_de_read(dev_priv, SPLL_CTL) & SPLL_PLL_ENABLE,
1192 "SPLL enabled\n");
1193 I915_STATE_WARN(dev_priv,
1194 intel_de_read(dev_priv, WRPLL_CTL(0)) & WRPLL_PLL_ENABLE,
1195 "WRPLL1 enabled\n");
1196 I915_STATE_WARN(dev_priv,
1197 intel_de_read(dev_priv, WRPLL_CTL(1)) & WRPLL_PLL_ENABLE,
1198 "WRPLL2 enabled\n");
1199 I915_STATE_WARN(dev_priv,
1200 intel_de_read(dev_priv, PP_STATUS(dev_priv, 0)) & PP_ON,
1201 "Panel power on\n");
1202 I915_STATE_WARN(dev_priv,
1203 intel_de_read(dev_priv, BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
1204 "CPU PWM1 enabled\n");
1205 if (IS_HASWELL(dev_priv))
1206 I915_STATE_WARN(dev_priv,
1207 intel_de_read(dev_priv, HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
1208 "CPU PWM2 enabled\n");
1209 I915_STATE_WARN(dev_priv,
1210 intel_de_read(dev_priv, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
1211 "PCH PWM1 enabled\n");
1212 I915_STATE_WARN(dev_priv,
1213 (intel_de_read(dev_priv, UTIL_PIN_CTL) & (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) == (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
1214 "Utility pin enabled in PWM mode\n");
1215 I915_STATE_WARN(dev_priv,
1216 intel_de_read(dev_priv, PCH_GTC_CTL) & PCH_GTC_ENABLE,
1217 "PCH GTC enabled\n");
1218
1219 /*
1220 * In theory we can still leave IRQs enabled, as long as only the HPD
1221 * interrupts remain enabled. We used to check for that, but since it's
1222 * gen-specific and since we only disable LCPLL after we fully disable
1223 * the interrupts, the check below should be enough.
1224 */
1225 I915_STATE_WARN(dev_priv, intel_irqs_enabled(dev_priv),
1226 "IRQs enabled\n");
1227 }
1228
hsw_read_dcomp(struct drm_i915_private * dev_priv)1229 static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv)
1230 {
1231 if (IS_HASWELL(dev_priv))
1232 return intel_de_read(dev_priv, D_COMP_HSW);
1233 else
1234 return intel_de_read(dev_priv, D_COMP_BDW);
1235 }
1236
hsw_write_dcomp(struct drm_i915_private * dev_priv,u32 val)1237 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val)
1238 {
1239 if (IS_HASWELL(dev_priv)) {
1240 if (snb_pcode_write(&dev_priv->uncore, GEN6_PCODE_WRITE_D_COMP, val))
1241 drm_dbg_kms(&dev_priv->drm,
1242 "Failed to write to D_COMP\n");
1243 } else {
1244 intel_de_write(dev_priv, D_COMP_BDW, val);
1245 intel_de_posting_read(dev_priv, D_COMP_BDW);
1246 }
1247 }
1248
1249 /*
1250 * This function implements pieces of two sequences from BSpec:
1251 * - Sequence for display software to disable LCPLL
1252 * - Sequence for display software to allow package C8+
1253 * The steps implemented here are just the steps that actually touch the LCPLL
1254 * register. Callers should take care of disabling all the display engine
1255 * functions, doing the mode unset, fixing interrupts, etc.
1256 */
hsw_disable_lcpll(struct drm_i915_private * dev_priv,bool switch_to_fclk,bool allow_power_down)1257 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
1258 bool switch_to_fclk, bool allow_power_down)
1259 {
1260 u32 val;
1261
1262 assert_can_disable_lcpll(dev_priv);
1263
1264 val = intel_de_read(dev_priv, LCPLL_CTL);
1265
1266 if (switch_to_fclk) {
1267 val |= LCPLL_CD_SOURCE_FCLK;
1268 intel_de_write(dev_priv, LCPLL_CTL, val);
1269
1270 if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
1271 LCPLL_CD_SOURCE_FCLK_DONE, 1))
1272 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
1273
1274 val = intel_de_read(dev_priv, LCPLL_CTL);
1275 }
1276
1277 val |= LCPLL_PLL_DISABLE;
1278 intel_de_write(dev_priv, LCPLL_CTL, val);
1279 intel_de_posting_read(dev_priv, LCPLL_CTL);
1280
1281 if (intel_de_wait_for_clear(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 1))
1282 drm_err(&dev_priv->drm, "LCPLL still locked\n");
1283
1284 val = hsw_read_dcomp(dev_priv);
1285 val |= D_COMP_COMP_DISABLE;
1286 hsw_write_dcomp(dev_priv, val);
1287 ndelay(100);
1288
1289 if (wait_for((hsw_read_dcomp(dev_priv) &
1290 D_COMP_RCOMP_IN_PROGRESS) == 0, 1))
1291 drm_err(&dev_priv->drm, "D_COMP RCOMP still in progress\n");
1292
1293 if (allow_power_down) {
1294 intel_de_rmw(dev_priv, LCPLL_CTL, 0, LCPLL_POWER_DOWN_ALLOW);
1295 intel_de_posting_read(dev_priv, LCPLL_CTL);
1296 }
1297 }
1298
1299 /*
1300 * Fully restores LCPLL, disallowing power down and switching back to LCPLL
1301 * source.
1302 */
hsw_restore_lcpll(struct drm_i915_private * dev_priv)1303 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
1304 {
1305 u32 val;
1306
1307 val = intel_de_read(dev_priv, LCPLL_CTL);
1308
1309 if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
1310 LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
1311 return;
1312
1313 /*
1314 * Make sure we're not on PC8 state before disabling PC8, otherwise
1315 * we'll hang the machine. To prevent PC8 state, just enable force_wake.
1316 */
1317 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
1318
1319 if (val & LCPLL_POWER_DOWN_ALLOW) {
1320 val &= ~LCPLL_POWER_DOWN_ALLOW;
1321 intel_de_write(dev_priv, LCPLL_CTL, val);
1322 intel_de_posting_read(dev_priv, LCPLL_CTL);
1323 }
1324
1325 val = hsw_read_dcomp(dev_priv);
1326 val |= D_COMP_COMP_FORCE;
1327 val &= ~D_COMP_COMP_DISABLE;
1328 hsw_write_dcomp(dev_priv, val);
1329
1330 val = intel_de_read(dev_priv, LCPLL_CTL);
1331 val &= ~LCPLL_PLL_DISABLE;
1332 intel_de_write(dev_priv, LCPLL_CTL, val);
1333
1334 if (intel_de_wait_for_set(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 5))
1335 drm_err(&dev_priv->drm, "LCPLL not locked yet\n");
1336
1337 if (val & LCPLL_CD_SOURCE_FCLK) {
1338 intel_de_rmw(dev_priv, LCPLL_CTL, LCPLL_CD_SOURCE_FCLK, 0);
1339
1340 if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
1341 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
1342 drm_err(&dev_priv->drm,
1343 "Switching back to LCPLL failed\n");
1344 }
1345
1346 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
1347
1348 intel_update_cdclk(dev_priv);
1349 intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1350 }
1351
1352 /*
1353 * Package states C8 and deeper are really deep PC states that can only be
1354 * reached when all the devices on the system allow it, so even if the graphics
1355 * device allows PC8+, it doesn't mean the system will actually get to these
1356 * states. Our driver only allows PC8+ when going into runtime PM.
1357 *
1358 * The requirements for PC8+ are that all the outputs are disabled, the power
1359 * well is disabled and most interrupts are disabled, and these are also
1360 * requirements for runtime PM. When these conditions are met, we manually do
1361 * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
1362 * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
1363 * hang the machine.
1364 *
1365 * When we really reach PC8 or deeper states (not just when we allow it) we lose
1366 * the state of some registers, so when we come back from PC8+ we need to
1367 * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
1368 * need to take care of the registers kept by RC6. Notice that this happens even
1369 * if we don't put the device in PCI D3 state (which is what currently happens
1370 * because of the runtime PM support).
1371 *
1372 * For more, read "Display Sequences for Package C8" on the hardware
1373 * documentation.
1374 */
hsw_enable_pc8(struct drm_i915_private * dev_priv)1375 static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
1376 {
1377 drm_dbg_kms(&dev_priv->drm, "Enabling package C8+\n");
1378
1379 if (HAS_PCH_LPT_LP(dev_priv))
1380 intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
1381 PCH_LP_PARTITION_LEVEL_DISABLE, 0);
1382
1383 lpt_disable_clkout_dp(dev_priv);
1384 hsw_disable_lcpll(dev_priv, true, true);
1385 }
1386
hsw_disable_pc8(struct drm_i915_private * dev_priv)1387 static void hsw_disable_pc8(struct drm_i915_private *dev_priv)
1388 {
1389 drm_dbg_kms(&dev_priv->drm, "Disabling package C8+\n");
1390
1391 hsw_restore_lcpll(dev_priv);
1392 intel_init_pch_refclk(dev_priv);
1393
1394 /* Many display registers don't survive PC8+ */
1395 intel_clock_gating_init(dev_priv);
1396 }
1397
intel_pch_reset_handshake(struct drm_i915_private * dev_priv,bool enable)1398 static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
1399 bool enable)
1400 {
1401 i915_reg_t reg;
1402 u32 reset_bits;
1403
1404 if (IS_IVYBRIDGE(dev_priv)) {
1405 reg = GEN7_MSG_CTL;
1406 reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
1407 } else {
1408 reg = HSW_NDE_RSTWRN_OPT;
1409 reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
1410 }
1411
1412 if (DISPLAY_VER(dev_priv) >= 14)
1413 reset_bits |= MTL_RESET_PICA_HANDSHAKE_EN;
1414
1415 intel_de_rmw(dev_priv, reg, reset_bits, enable ? reset_bits : 0);
1416 }
1417
skl_display_core_init(struct drm_i915_private * dev_priv,bool resume)1418 static void skl_display_core_init(struct drm_i915_private *dev_priv,
1419 bool resume)
1420 {
1421 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1422 struct i915_power_well *well;
1423
1424 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1425
1426 /* enable PCH reset handshake */
1427 intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
1428
1429 if (!HAS_DISPLAY(dev_priv))
1430 return;
1431
1432 /* enable PG1 and Misc I/O */
1433 mutex_lock(&power_domains->lock);
1434
1435 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1436 intel_power_well_enable(dev_priv, well);
1437
1438 well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
1439 intel_power_well_enable(dev_priv, well);
1440
1441 mutex_unlock(&power_domains->lock);
1442
1443 intel_cdclk_init_hw(dev_priv);
1444
1445 gen9_dbuf_enable(dev_priv);
1446
1447 if (resume)
1448 intel_dmc_load_program(dev_priv);
1449 }
1450
skl_display_core_uninit(struct drm_i915_private * dev_priv)1451 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
1452 {
1453 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1454 struct i915_power_well *well;
1455
1456 if (!HAS_DISPLAY(dev_priv))
1457 return;
1458
1459 gen9_disable_dc_states(dev_priv);
1460 /* TODO: disable DMC program */
1461
1462 gen9_dbuf_disable(dev_priv);
1463
1464 intel_cdclk_uninit_hw(dev_priv);
1465
1466 /* The spec doesn't call for removing the reset handshake flag */
1467 /* disable PG1 and Misc I/O */
1468
1469 mutex_lock(&power_domains->lock);
1470
1471 /*
1472 * BSpec says to keep the MISC IO power well enabled here, only
1473 * remove our request for power well 1.
1474 * Note that even though the driver's request is removed power well 1
1475 * may stay enabled after this due to DMC's own request on it.
1476 */
1477 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1478 intel_power_well_disable(dev_priv, well);
1479
1480 mutex_unlock(&power_domains->lock);
1481
1482 usleep_range(10, 30); /* 10 us delay per Bspec */
1483 }
1484
bxt_display_core_init(struct drm_i915_private * dev_priv,bool resume)1485 static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume)
1486 {
1487 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1488 struct i915_power_well *well;
1489
1490 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1491
1492 /*
1493 * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
1494 * or else the reset will hang because there is no PCH to respond.
1495 * Move the handshake programming to initialization sequence.
1496 * Previously was left up to BIOS.
1497 */
1498 intel_pch_reset_handshake(dev_priv, false);
1499
1500 if (!HAS_DISPLAY(dev_priv))
1501 return;
1502
1503 /* Enable PG1 */
1504 mutex_lock(&power_domains->lock);
1505
1506 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1507 intel_power_well_enable(dev_priv, well);
1508
1509 mutex_unlock(&power_domains->lock);
1510
1511 intel_cdclk_init_hw(dev_priv);
1512
1513 gen9_dbuf_enable(dev_priv);
1514
1515 if (resume)
1516 intel_dmc_load_program(dev_priv);
1517 }
1518
bxt_display_core_uninit(struct drm_i915_private * dev_priv)1519 static void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
1520 {
1521 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1522 struct i915_power_well *well;
1523
1524 if (!HAS_DISPLAY(dev_priv))
1525 return;
1526
1527 gen9_disable_dc_states(dev_priv);
1528 /* TODO: disable DMC program */
1529
1530 gen9_dbuf_disable(dev_priv);
1531
1532 intel_cdclk_uninit_hw(dev_priv);
1533
1534 /* The spec doesn't call for removing the reset handshake flag */
1535
1536 /*
1537 * Disable PW1 (PG1).
1538 * Note that even though the driver's request is removed power well 1
1539 * may stay enabled after this due to DMC's own request on it.
1540 */
1541 mutex_lock(&power_domains->lock);
1542
1543 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1544 intel_power_well_disable(dev_priv, well);
1545
1546 mutex_unlock(&power_domains->lock);
1547
1548 usleep_range(10, 30); /* 10 us delay per Bspec */
1549 }
1550
1551 struct buddy_page_mask {
1552 u32 page_mask;
1553 u8 type;
1554 u8 num_channels;
1555 };
1556
1557 static const struct buddy_page_mask tgl_buddy_page_masks[] = {
1558 { .num_channels = 1, .type = INTEL_DRAM_DDR4, .page_mask = 0xF },
1559 { .num_channels = 1, .type = INTEL_DRAM_DDR5, .page_mask = 0xF },
1560 { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C },
1561 { .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1C },
1562 { .num_channels = 2, .type = INTEL_DRAM_DDR4, .page_mask = 0x1F },
1563 { .num_channels = 2, .type = INTEL_DRAM_DDR5, .page_mask = 0x1E },
1564 { .num_channels = 4, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x38 },
1565 { .num_channels = 4, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x38 },
1566 {}
1567 };
1568
1569 static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
1570 { .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1 },
1571 { .num_channels = 1, .type = INTEL_DRAM_DDR4, .page_mask = 0x1 },
1572 { .num_channels = 1, .type = INTEL_DRAM_DDR5, .page_mask = 0x1 },
1573 { .num_channels = 1, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1 },
1574 { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x3 },
1575 { .num_channels = 2, .type = INTEL_DRAM_DDR4, .page_mask = 0x3 },
1576 { .num_channels = 2, .type = INTEL_DRAM_DDR5, .page_mask = 0x3 },
1577 { .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x3 },
1578 {}
1579 };
1580
tgl_bw_buddy_init(struct drm_i915_private * dev_priv)1581 static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
1582 {
1583 enum intel_dram_type type = dev_priv->dram_info.type;
1584 u8 num_channels = dev_priv->dram_info.num_channels;
1585 const struct buddy_page_mask *table;
1586 unsigned long abox_mask = DISPLAY_INFO(dev_priv)->abox_mask;
1587 int config, i;
1588
1589 /* BW_BUDDY registers are not used on dgpu's beyond DG1 */
1590 if (IS_DGFX(dev_priv) && !IS_DG1(dev_priv))
1591 return;
1592
1593 if (IS_ALDERLAKE_S(dev_priv) ||
1594 (IS_ROCKETLAKE(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)))
1595 /* Wa_1409767108 */
1596 table = wa_1409767108_buddy_page_masks;
1597 else
1598 table = tgl_buddy_page_masks;
1599
1600 for (config = 0; table[config].page_mask != 0; config++)
1601 if (table[config].num_channels == num_channels &&
1602 table[config].type == type)
1603 break;
1604
1605 if (table[config].page_mask == 0) {
1606 drm_dbg(&dev_priv->drm,
1607 "Unknown memory configuration; disabling address buddy logic.\n");
1608 for_each_set_bit(i, &abox_mask, sizeof(abox_mask))
1609 intel_de_write(dev_priv, BW_BUDDY_CTL(i),
1610 BW_BUDDY_DISABLE);
1611 } else {
1612 for_each_set_bit(i, &abox_mask, sizeof(abox_mask)) {
1613 intel_de_write(dev_priv, BW_BUDDY_PAGE_MASK(i),
1614 table[config].page_mask);
1615
1616 /* Wa_22010178259:tgl,dg1,rkl,adl-s */
1617 if (DISPLAY_VER(dev_priv) == 12)
1618 intel_de_rmw(dev_priv, BW_BUDDY_CTL(i),
1619 BW_BUDDY_TLB_REQ_TIMER_MASK,
1620 BW_BUDDY_TLB_REQ_TIMER(0x8));
1621 }
1622 }
1623 }
1624
icl_display_core_init(struct drm_i915_private * dev_priv,bool resume)1625 static void icl_display_core_init(struct drm_i915_private *dev_priv,
1626 bool resume)
1627 {
1628 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1629 struct i915_power_well *well;
1630
1631 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1632
1633 /* Wa_14011294188:ehl,jsl,tgl,rkl,adl-s */
1634 if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP &&
1635 INTEL_PCH_TYPE(dev_priv) < PCH_DG1)
1636 intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D, 0,
1637 PCH_DPMGUNIT_CLOCK_GATE_DISABLE);
1638
1639 /* 1. Enable PCH reset handshake. */
1640 intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
1641
1642 if (!HAS_DISPLAY(dev_priv))
1643 return;
1644
1645 /* 2. Initialize all combo phys */
1646 intel_combo_phy_init(dev_priv);
1647
1648 /*
1649 * 3. Enable Power Well 1 (PG1).
1650 * The AUX IO power wells will be enabled on demand.
1651 */
1652 mutex_lock(&power_domains->lock);
1653 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1654 intel_power_well_enable(dev_priv, well);
1655 mutex_unlock(&power_domains->lock);
1656
1657 if (DISPLAY_VER(dev_priv) == 14)
1658 intel_de_rmw(dev_priv, DC_STATE_EN,
1659 HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH, 0);
1660
1661 /* 4. Enable CDCLK. */
1662 intel_cdclk_init_hw(dev_priv);
1663
1664 if (DISPLAY_VER(dev_priv) >= 12)
1665 gen12_dbuf_slices_config(dev_priv);
1666
1667 /* 5. Enable DBUF. */
1668 gen9_dbuf_enable(dev_priv);
1669
1670 /* 6. Setup MBUS. */
1671 icl_mbus_init(dev_priv);
1672
1673 /* 7. Program arbiter BW_BUDDY registers */
1674 if (DISPLAY_VER(dev_priv) >= 12)
1675 tgl_bw_buddy_init(dev_priv);
1676
1677 /* 8. Ensure PHYs have completed calibration and adaptation */
1678 if (IS_DG2(dev_priv))
1679 intel_snps_phy_wait_for_calibration(dev_priv);
1680
1681 /* 9. XE2_HPD: Program CHICKEN_MISC_2 before any cursor or planes are enabled */
1682 if (DISPLAY_VER_FULL(dev_priv) == IP_VER(14, 1))
1683 intel_de_rmw(dev_priv, CHICKEN_MISC_2, BMG_DARB_HALF_BLK_END_BURST, 1);
1684
1685 if (resume)
1686 intel_dmc_load_program(dev_priv);
1687
1688 /* Wa_14011508470:tgl,dg1,rkl,adl-s,adl-p,dg2 */
1689 if (IS_DISPLAY_VER_FULL(dev_priv, IP_VER(12, 0), IP_VER(13, 0)))
1690 intel_de_rmw(dev_priv, GEN11_CHICKEN_DCPR_2, 0,
1691 DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
1692 DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR);
1693
1694 /* Wa_14011503030:xelpd */
1695 if (DISPLAY_VER(dev_priv) == 13)
1696 intel_de_write(dev_priv, XELPD_DISPLAY_ERR_FATAL_MASK, ~0);
1697
1698 /* Wa_15013987218 */
1699 if (DISPLAY_VER(dev_priv) == 20) {
1700 intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
1701 0, PCH_GMBUSUNIT_CLOCK_GATE_DISABLE);
1702 intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
1703 PCH_GMBUSUNIT_CLOCK_GATE_DISABLE, 0);
1704 }
1705 }
1706
icl_display_core_uninit(struct drm_i915_private * dev_priv)1707 static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
1708 {
1709 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1710 struct i915_power_well *well;
1711
1712 if (!HAS_DISPLAY(dev_priv))
1713 return;
1714
1715 gen9_disable_dc_states(dev_priv);
1716 intel_dmc_disable_program(dev_priv);
1717
1718 /* 1. Disable all display engine functions -> aready done */
1719
1720 /* 2. Disable DBUF */
1721 gen9_dbuf_disable(dev_priv);
1722
1723 /* 3. Disable CD clock */
1724 intel_cdclk_uninit_hw(dev_priv);
1725
1726 if (DISPLAY_VER(dev_priv) == 14)
1727 intel_de_rmw(dev_priv, DC_STATE_EN, 0,
1728 HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH);
1729
1730 /*
1731 * 4. Disable Power Well 1 (PG1).
1732 * The AUX IO power wells are toggled on demand, so they are already
1733 * disabled at this point.
1734 */
1735 mutex_lock(&power_domains->lock);
1736 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1737 intel_power_well_disable(dev_priv, well);
1738 mutex_unlock(&power_domains->lock);
1739
1740 /* 5. */
1741 intel_combo_phy_uninit(dev_priv);
1742 }
1743
chv_phy_control_init(struct drm_i915_private * dev_priv)1744 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
1745 {
1746 struct i915_power_well *cmn_bc =
1747 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1748 struct i915_power_well *cmn_d =
1749 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1750
1751 /*
1752 * DISPLAY_PHY_CONTROL can get corrupted if read. As a
1753 * workaround never ever read DISPLAY_PHY_CONTROL, and
1754 * instead maintain a shadow copy ourselves. Use the actual
1755 * power well state and lane status to reconstruct the
1756 * expected initial value.
1757 */
1758 dev_priv->display.power.chv_phy_control =
1759 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
1760 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
1761 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
1762 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
1763 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
1764
1765 /*
1766 * If all lanes are disabled we leave the override disabled
1767 * with all power down bits cleared to match the state we
1768 * would use after disabling the port. Otherwise enable the
1769 * override and set the lane powerdown bits accding to the
1770 * current lane status.
1771 */
1772 if (intel_power_well_is_enabled(dev_priv, cmn_bc)) {
1773 u32 status = intel_de_read(dev_priv, DPLL(dev_priv, PIPE_A));
1774 unsigned int mask;
1775
1776 mask = status & DPLL_PORTB_READY_MASK;
1777 if (mask == 0xf)
1778 mask = 0x0;
1779 else
1780 dev_priv->display.power.chv_phy_control |=
1781 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
1782
1783 dev_priv->display.power.chv_phy_control |=
1784 PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
1785
1786 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
1787 if (mask == 0xf)
1788 mask = 0x0;
1789 else
1790 dev_priv->display.power.chv_phy_control |=
1791 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
1792
1793 dev_priv->display.power.chv_phy_control |=
1794 PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
1795
1796 dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
1797
1798 dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = false;
1799 } else {
1800 dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = true;
1801 }
1802
1803 if (intel_power_well_is_enabled(dev_priv, cmn_d)) {
1804 u32 status = intel_de_read(dev_priv, DPIO_PHY_STATUS);
1805 unsigned int mask;
1806
1807 mask = status & DPLL_PORTD_READY_MASK;
1808
1809 if (mask == 0xf)
1810 mask = 0x0;
1811 else
1812 dev_priv->display.power.chv_phy_control |=
1813 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
1814
1815 dev_priv->display.power.chv_phy_control |=
1816 PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
1817
1818 dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
1819
1820 dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = false;
1821 } else {
1822 dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = true;
1823 }
1824
1825 drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n",
1826 dev_priv->display.power.chv_phy_control);
1827
1828 /* Defer application of initial phy_control to enabling the powerwell */
1829 }
1830
vlv_cmnlane_wa(struct drm_i915_private * dev_priv)1831 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
1832 {
1833 struct i915_power_well *cmn =
1834 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1835 struct i915_power_well *disp2d =
1836 lookup_power_well(dev_priv, VLV_DISP_PW_DISP2D);
1837
1838 /* If the display might be already active skip this */
1839 if (intel_power_well_is_enabled(dev_priv, cmn) &&
1840 intel_power_well_is_enabled(dev_priv, disp2d) &&
1841 intel_de_read(dev_priv, DPIO_CTL) & DPIO_CMNRST)
1842 return;
1843
1844 drm_dbg_kms(&dev_priv->drm, "toggling display PHY side reset\n");
1845
1846 /* cmnlane needs DPLL registers */
1847 intel_power_well_enable(dev_priv, disp2d);
1848
1849 /*
1850 * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
1851 * Need to assert and de-assert PHY SB reset by gating the
1852 * common lane power, then un-gating it.
1853 * Simply ungating isn't enough to reset the PHY enough to get
1854 * ports and lanes running.
1855 */
1856 intel_power_well_disable(dev_priv, cmn);
1857 }
1858
vlv_punit_is_power_gated(struct drm_i915_private * dev_priv,u32 reg0)1859 static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0)
1860 {
1861 bool ret;
1862
1863 vlv_punit_get(dev_priv);
1864 ret = (vlv_punit_read(dev_priv, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
1865 vlv_punit_put(dev_priv);
1866
1867 return ret;
1868 }
1869
assert_ved_power_gated(struct drm_i915_private * dev_priv)1870 static void assert_ved_power_gated(struct drm_i915_private *dev_priv)
1871 {
1872 drm_WARN(&dev_priv->drm,
1873 !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0),
1874 "VED not power gated\n");
1875 }
1876
assert_isp_power_gated(struct drm_i915_private * dev_priv)1877 static void assert_isp_power_gated(struct drm_i915_private *dev_priv)
1878 {
1879 #ifdef notyet
1880 static const struct pci_device_id isp_ids[] = {
1881 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
1882 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
1883 {}
1884 };
1885
1886 drm_WARN(&dev_priv->drm, !pci_dev_present(isp_ids) &&
1887 !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0),
1888 "ISP not power gated\n");
1889 #endif
1890 }
1891
1892 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
1893
1894 /**
1895 * intel_power_domains_init_hw - initialize hardware power domain state
1896 * @i915: i915 device instance
1897 * @resume: Called from resume code paths or not
1898 *
1899 * This function initializes the hardware power domain state and enables all
1900 * power wells belonging to the INIT power domain. Power wells in other
1901 * domains (and not in the INIT domain) are referenced or disabled by
1902 * intel_modeset_readout_hw_state(). After that the reference count of each
1903 * power well must match its HW enabled state, see
1904 * intel_power_domains_verify_state().
1905 *
1906 * It will return with power domains disabled (to be enabled later by
1907 * intel_power_domains_enable()) and must be paired with
1908 * intel_power_domains_driver_remove().
1909 */
intel_power_domains_init_hw(struct drm_i915_private * i915,bool resume)1910 void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
1911 {
1912 struct i915_power_domains *power_domains = &i915->display.power.domains;
1913
1914 power_domains->initializing = true;
1915
1916 if (DISPLAY_VER(i915) >= 11) {
1917 icl_display_core_init(i915, resume);
1918 } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
1919 bxt_display_core_init(i915, resume);
1920 } else if (DISPLAY_VER(i915) == 9) {
1921 skl_display_core_init(i915, resume);
1922 } else if (IS_CHERRYVIEW(i915)) {
1923 mutex_lock(&power_domains->lock);
1924 chv_phy_control_init(i915);
1925 mutex_unlock(&power_domains->lock);
1926 assert_isp_power_gated(i915);
1927 } else if (IS_VALLEYVIEW(i915)) {
1928 mutex_lock(&power_domains->lock);
1929 vlv_cmnlane_wa(i915);
1930 mutex_unlock(&power_domains->lock);
1931 assert_ved_power_gated(i915);
1932 assert_isp_power_gated(i915);
1933 } else if (IS_BROADWELL(i915) || IS_HASWELL(i915)) {
1934 hsw_assert_cdclk(i915);
1935 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
1936 } else if (IS_IVYBRIDGE(i915)) {
1937 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
1938 }
1939
1940 /*
1941 * Keep all power wells enabled for any dependent HW access during
1942 * initialization and to make sure we keep BIOS enabled display HW
1943 * resources powered until display HW readout is complete. We drop
1944 * this reference in intel_power_domains_enable().
1945 */
1946 drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
1947 power_domains->init_wakeref =
1948 intel_display_power_get(i915, POWER_DOMAIN_INIT);
1949
1950 /* Disable power support if the user asked so. */
1951 if (!i915->display.params.disable_power_well) {
1952 drm_WARN_ON(&i915->drm, power_domains->disable_wakeref);
1953 i915->display.power.domains.disable_wakeref = intel_display_power_get(i915,
1954 POWER_DOMAIN_INIT);
1955 }
1956 intel_power_domains_sync_hw(i915);
1957
1958 power_domains->initializing = false;
1959 }
1960
1961 /**
1962 * intel_power_domains_driver_remove - deinitialize hw power domain state
1963 * @i915: i915 device instance
1964 *
1965 * De-initializes the display power domain HW state. It also ensures that the
1966 * device stays powered up so that the driver can be reloaded.
1967 *
1968 * It must be called with power domains already disabled (after a call to
1969 * intel_power_domains_disable()) and must be paired with
1970 * intel_power_domains_init_hw().
1971 */
intel_power_domains_driver_remove(struct drm_i915_private * i915)1972 void intel_power_domains_driver_remove(struct drm_i915_private *i915)
1973 {
1974 intel_wakeref_t wakeref __maybe_unused =
1975 fetch_and_zero(&i915->display.power.domains.init_wakeref);
1976
1977 /* Remove the refcount we took to keep power well support disabled. */
1978 if (!i915->display.params.disable_power_well)
1979 intel_display_power_put(i915, POWER_DOMAIN_INIT,
1980 fetch_and_zero(&i915->display.power.domains.disable_wakeref));
1981
1982 intel_display_power_flush_work_sync(i915);
1983
1984 intel_power_domains_verify_state(i915);
1985
1986 /* Keep the power well enabled, but cancel its rpm wakeref. */
1987 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1988 }
1989
1990 /**
1991 * intel_power_domains_sanitize_state - sanitize power domains state
1992 * @i915: i915 device instance
1993 *
1994 * Sanitize the power domains state during driver loading and system resume.
1995 * The function will disable all display power wells that BIOS has enabled
1996 * without a user for it (any user for a power well has taken a reference
1997 * on it by the time this function is called, after the state of all the
1998 * pipe, encoder, etc. HW resources have been sanitized).
1999 */
intel_power_domains_sanitize_state(struct drm_i915_private * i915)2000 void intel_power_domains_sanitize_state(struct drm_i915_private *i915)
2001 {
2002 struct i915_power_domains *power_domains = &i915->display.power.domains;
2003 struct i915_power_well *power_well;
2004
2005 mutex_lock(&power_domains->lock);
2006
2007 for_each_power_well_reverse(i915, power_well) {
2008 if (power_well->desc->always_on || power_well->count ||
2009 !intel_power_well_is_enabled(i915, power_well))
2010 continue;
2011
2012 drm_dbg_kms(&i915->drm,
2013 "BIOS left unused %s power well enabled, disabling it\n",
2014 intel_power_well_name(power_well));
2015 intel_power_well_disable(i915, power_well);
2016 }
2017
2018 mutex_unlock(&power_domains->lock);
2019 }
2020
2021 /**
2022 * intel_power_domains_enable - enable toggling of display power wells
2023 * @i915: i915 device instance
2024 *
2025 * Enable the ondemand enabling/disabling of the display power wells. Note that
2026 * power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
2027 * only at specific points of the display modeset sequence, thus they are not
2028 * affected by the intel_power_domains_enable()/disable() calls. The purpose
2029 * of these function is to keep the rest of power wells enabled until the end
2030 * of display HW readout (which will acquire the power references reflecting
2031 * the current HW state).
2032 */
intel_power_domains_enable(struct drm_i915_private * i915)2033 void intel_power_domains_enable(struct drm_i915_private *i915)
2034 {
2035 intel_wakeref_t wakeref __maybe_unused =
2036 fetch_and_zero(&i915->display.power.domains.init_wakeref);
2037
2038 intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
2039 intel_power_domains_verify_state(i915);
2040 }
2041
2042 /**
2043 * intel_power_domains_disable - disable toggling of display power wells
2044 * @i915: i915 device instance
2045 *
2046 * Disable the ondemand enabling/disabling of the display power wells. See
2047 * intel_power_domains_enable() for which power wells this call controls.
2048 */
intel_power_domains_disable(struct drm_i915_private * i915)2049 void intel_power_domains_disable(struct drm_i915_private *i915)
2050 {
2051 struct i915_power_domains *power_domains = &i915->display.power.domains;
2052
2053 drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2054 power_domains->init_wakeref =
2055 intel_display_power_get(i915, POWER_DOMAIN_INIT);
2056
2057 intel_power_domains_verify_state(i915);
2058 }
2059
2060 /**
2061 * intel_power_domains_suspend - suspend power domain state
2062 * @i915: i915 device instance
2063 * @s2idle: specifies whether we go to idle, or deeper sleep
2064 *
2065 * This function prepares the hardware power domain state before entering
2066 * system suspend.
2067 *
2068 * It must be called with power domains already disabled (after a call to
2069 * intel_power_domains_disable()) and paired with intel_power_domains_resume().
2070 */
intel_power_domains_suspend(struct drm_i915_private * i915,bool s2idle)2071 void intel_power_domains_suspend(struct drm_i915_private *i915, bool s2idle)
2072 {
2073 struct i915_power_domains *power_domains = &i915->display.power.domains;
2074 intel_wakeref_t wakeref __maybe_unused =
2075 fetch_and_zero(&power_domains->init_wakeref);
2076
2077 intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
2078
2079 /*
2080 * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
2081 * support don't manually deinit the power domains. This also means the
2082 * DMC firmware will stay active, it will power down any HW
2083 * resources as required and also enable deeper system power states
2084 * that would be blocked if the firmware was inactive.
2085 */
2086 if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC9) && s2idle &&
2087 intel_dmc_has_payload(i915)) {
2088 intel_display_power_flush_work(i915);
2089 intel_power_domains_verify_state(i915);
2090 return;
2091 }
2092
2093 /*
2094 * Even if power well support was disabled we still want to disable
2095 * power wells if power domains must be deinitialized for suspend.
2096 */
2097 if (!i915->display.params.disable_power_well)
2098 intel_display_power_put(i915, POWER_DOMAIN_INIT,
2099 fetch_and_zero(&i915->display.power.domains.disable_wakeref));
2100
2101 intel_display_power_flush_work(i915);
2102 intel_power_domains_verify_state(i915);
2103
2104 if (DISPLAY_VER(i915) >= 11)
2105 icl_display_core_uninit(i915);
2106 else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915))
2107 bxt_display_core_uninit(i915);
2108 else if (DISPLAY_VER(i915) == 9)
2109 skl_display_core_uninit(i915);
2110
2111 power_domains->display_core_suspended = true;
2112 }
2113
2114 /**
2115 * intel_power_domains_resume - resume power domain state
2116 * @i915: i915 device instance
2117 *
2118 * This function resume the hardware power domain state during system resume.
2119 *
2120 * It will return with power domain support disabled (to be enabled later by
2121 * intel_power_domains_enable()) and must be paired with
2122 * intel_power_domains_suspend().
2123 */
intel_power_domains_resume(struct drm_i915_private * i915)2124 void intel_power_domains_resume(struct drm_i915_private *i915)
2125 {
2126 struct i915_power_domains *power_domains = &i915->display.power.domains;
2127
2128 if (power_domains->display_core_suspended) {
2129 intel_power_domains_init_hw(i915, true);
2130 power_domains->display_core_suspended = false;
2131 } else {
2132 drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2133 power_domains->init_wakeref =
2134 intel_display_power_get(i915, POWER_DOMAIN_INIT);
2135 }
2136
2137 intel_power_domains_verify_state(i915);
2138 }
2139
2140 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2141
intel_power_domains_dump_info(struct drm_i915_private * i915)2142 static void intel_power_domains_dump_info(struct drm_i915_private *i915)
2143 {
2144 struct i915_power_domains *power_domains = &i915->display.power.domains;
2145 struct i915_power_well *power_well;
2146
2147 for_each_power_well(i915, power_well) {
2148 enum intel_display_power_domain domain;
2149
2150 drm_dbg(&i915->drm, "%-25s %d\n",
2151 intel_power_well_name(power_well), intel_power_well_refcount(power_well));
2152
2153 for_each_power_domain(domain, intel_power_well_domains(power_well))
2154 drm_dbg(&i915->drm, " %-23s %d\n",
2155 intel_display_power_domain_str(domain),
2156 power_domains->domain_use_count[domain]);
2157 }
2158 }
2159
2160 /**
2161 * intel_power_domains_verify_state - verify the HW/SW state for all power wells
2162 * @i915: i915 device instance
2163 *
2164 * Verify if the reference count of each power well matches its HW enabled
2165 * state and the total refcount of the domains it belongs to. This must be
2166 * called after modeset HW state sanitization, which is responsible for
2167 * acquiring reference counts for any power wells in use and disabling the
2168 * ones left on by BIOS but not required by any active output.
2169 */
intel_power_domains_verify_state(struct drm_i915_private * i915)2170 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2171 {
2172 struct i915_power_domains *power_domains = &i915->display.power.domains;
2173 struct i915_power_well *power_well;
2174 bool dump_domain_info;
2175
2176 mutex_lock(&power_domains->lock);
2177
2178 verify_async_put_domains_state(power_domains);
2179
2180 dump_domain_info = false;
2181 for_each_power_well(i915, power_well) {
2182 enum intel_display_power_domain domain;
2183 int domains_count;
2184 bool enabled;
2185
2186 enabled = intel_power_well_is_enabled(i915, power_well);
2187 if ((intel_power_well_refcount(power_well) ||
2188 intel_power_well_is_always_on(power_well)) !=
2189 enabled)
2190 drm_err(&i915->drm,
2191 "power well %s state mismatch (refcount %d/enabled %d)",
2192 intel_power_well_name(power_well),
2193 intel_power_well_refcount(power_well), enabled);
2194
2195 domains_count = 0;
2196 for_each_power_domain(domain, intel_power_well_domains(power_well))
2197 domains_count += power_domains->domain_use_count[domain];
2198
2199 if (intel_power_well_refcount(power_well) != domains_count) {
2200 drm_err(&i915->drm,
2201 "power well %s refcount/domain refcount mismatch "
2202 "(refcount %d/domains refcount %d)\n",
2203 intel_power_well_name(power_well),
2204 intel_power_well_refcount(power_well),
2205 domains_count);
2206 dump_domain_info = true;
2207 }
2208 }
2209
2210 if (dump_domain_info) {
2211 static bool dumped;
2212
2213 if (!dumped) {
2214 intel_power_domains_dump_info(i915);
2215 dumped = true;
2216 }
2217 }
2218
2219 mutex_unlock(&power_domains->lock);
2220 }
2221
2222 #else
2223
intel_power_domains_verify_state(struct drm_i915_private * i915)2224 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2225 {
2226 }
2227
2228 #endif
2229
intel_display_power_suspend_late(struct drm_i915_private * i915)2230 void intel_display_power_suspend_late(struct drm_i915_private *i915)
2231 {
2232 if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2233 IS_BROXTON(i915)) {
2234 bxt_enable_dc9(i915);
2235 } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2236 hsw_enable_pc8(i915);
2237 }
2238
2239 /* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2240 if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2241 intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
2242 }
2243
intel_display_power_resume_early(struct drm_i915_private * i915)2244 void intel_display_power_resume_early(struct drm_i915_private *i915)
2245 {
2246 if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2247 IS_BROXTON(i915)) {
2248 gen9_sanitize_dc_state(i915);
2249 bxt_disable_dc9(i915);
2250 } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2251 hsw_disable_pc8(i915);
2252 }
2253
2254 /* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2255 if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2256 intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0);
2257 }
2258
intel_display_power_suspend(struct drm_i915_private * i915)2259 void intel_display_power_suspend(struct drm_i915_private *i915)
2260 {
2261 if (DISPLAY_VER(i915) >= 11) {
2262 icl_display_core_uninit(i915);
2263 bxt_enable_dc9(i915);
2264 } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2265 bxt_display_core_uninit(i915);
2266 bxt_enable_dc9(i915);
2267 } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2268 hsw_enable_pc8(i915);
2269 }
2270 }
2271
intel_display_power_resume(struct drm_i915_private * i915)2272 void intel_display_power_resume(struct drm_i915_private *i915)
2273 {
2274 struct i915_power_domains *power_domains = &i915->display.power.domains;
2275
2276 if (DISPLAY_VER(i915) >= 11) {
2277 bxt_disable_dc9(i915);
2278 icl_display_core_init(i915, true);
2279 if (intel_dmc_has_payload(i915)) {
2280 if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
2281 skl_enable_dc6(i915);
2282 else if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
2283 gen9_enable_dc5(i915);
2284 }
2285 } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2286 bxt_disable_dc9(i915);
2287 bxt_display_core_init(i915, true);
2288 if (intel_dmc_has_payload(i915) &&
2289 (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2290 gen9_enable_dc5(i915);
2291 } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2292 hsw_disable_pc8(i915);
2293 }
2294 }
2295
intel_display_power_debug(struct drm_i915_private * i915,struct seq_file * m)2296 void intel_display_power_debug(struct drm_i915_private *i915, struct seq_file *m)
2297 {
2298 struct i915_power_domains *power_domains = &i915->display.power.domains;
2299 int i;
2300
2301 mutex_lock(&power_domains->lock);
2302
2303 seq_printf(m, "%-25s %s\n", "Power well/domain", "Use count");
2304 for (i = 0; i < power_domains->power_well_count; i++) {
2305 struct i915_power_well *power_well;
2306 enum intel_display_power_domain power_domain;
2307
2308 power_well = &power_domains->power_wells[i];
2309 seq_printf(m, "%-25s %d\n", intel_power_well_name(power_well),
2310 intel_power_well_refcount(power_well));
2311
2312 for_each_power_domain(power_domain, intel_power_well_domains(power_well))
2313 seq_printf(m, " %-23s %d\n",
2314 intel_display_power_domain_str(power_domain),
2315 power_domains->domain_use_count[power_domain]);
2316 }
2317
2318 mutex_unlock(&power_domains->lock);
2319 }
2320
2321 struct intel_ddi_port_domains {
2322 enum port port_start;
2323 enum port port_end;
2324 enum aux_ch aux_ch_start;
2325 enum aux_ch aux_ch_end;
2326
2327 enum intel_display_power_domain ddi_lanes;
2328 enum intel_display_power_domain ddi_io;
2329 enum intel_display_power_domain aux_io;
2330 enum intel_display_power_domain aux_legacy_usbc;
2331 enum intel_display_power_domain aux_tbt;
2332 };
2333
2334 static const struct intel_ddi_port_domains
2335 i9xx_port_domains[] = {
2336 {
2337 .port_start = PORT_A,
2338 .port_end = PORT_F,
2339 .aux_ch_start = AUX_CH_A,
2340 .aux_ch_end = AUX_CH_F,
2341
2342 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2343 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2344 .aux_io = POWER_DOMAIN_AUX_IO_A,
2345 .aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2346 .aux_tbt = POWER_DOMAIN_INVALID,
2347 },
2348 };
2349
2350 static const struct intel_ddi_port_domains
2351 d11_port_domains[] = {
2352 {
2353 .port_start = PORT_A,
2354 .port_end = PORT_B,
2355 .aux_ch_start = AUX_CH_A,
2356 .aux_ch_end = AUX_CH_B,
2357
2358 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2359 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2360 .aux_io = POWER_DOMAIN_AUX_IO_A,
2361 .aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2362 .aux_tbt = POWER_DOMAIN_INVALID,
2363 }, {
2364 .port_start = PORT_C,
2365 .port_end = PORT_F,
2366 .aux_ch_start = AUX_CH_C,
2367 .aux_ch_end = AUX_CH_F,
2368
2369 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_C,
2370 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_C,
2371 .aux_io = POWER_DOMAIN_AUX_IO_C,
2372 .aux_legacy_usbc = POWER_DOMAIN_AUX_C,
2373 .aux_tbt = POWER_DOMAIN_AUX_TBT1,
2374 },
2375 };
2376
2377 static const struct intel_ddi_port_domains
2378 d12_port_domains[] = {
2379 {
2380 .port_start = PORT_A,
2381 .port_end = PORT_C,
2382 .aux_ch_start = AUX_CH_A,
2383 .aux_ch_end = AUX_CH_C,
2384
2385 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2386 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2387 .aux_io = POWER_DOMAIN_AUX_IO_A,
2388 .aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2389 .aux_tbt = POWER_DOMAIN_INVALID,
2390 }, {
2391 .port_start = PORT_TC1,
2392 .port_end = PORT_TC6,
2393 .aux_ch_start = AUX_CH_USBC1,
2394 .aux_ch_end = AUX_CH_USBC6,
2395
2396 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2397 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2398 .aux_io = POWER_DOMAIN_INVALID,
2399 .aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2400 .aux_tbt = POWER_DOMAIN_AUX_TBT1,
2401 },
2402 };
2403
2404 static const struct intel_ddi_port_domains
2405 d13_port_domains[] = {
2406 {
2407 .port_start = PORT_A,
2408 .port_end = PORT_C,
2409 .aux_ch_start = AUX_CH_A,
2410 .aux_ch_end = AUX_CH_C,
2411
2412 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2413 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2414 .aux_io = POWER_DOMAIN_AUX_IO_A,
2415 .aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2416 .aux_tbt = POWER_DOMAIN_INVALID,
2417 }, {
2418 .port_start = PORT_TC1,
2419 .port_end = PORT_TC4,
2420 .aux_ch_start = AUX_CH_USBC1,
2421 .aux_ch_end = AUX_CH_USBC4,
2422
2423 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2424 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2425 .aux_io = POWER_DOMAIN_INVALID,
2426 .aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2427 .aux_tbt = POWER_DOMAIN_AUX_TBT1,
2428 }, {
2429 .port_start = PORT_D_XELPD,
2430 .port_end = PORT_E_XELPD,
2431 .aux_ch_start = AUX_CH_D_XELPD,
2432 .aux_ch_end = AUX_CH_E_XELPD,
2433
2434 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_D,
2435 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_D,
2436 .aux_io = POWER_DOMAIN_AUX_IO_D,
2437 .aux_legacy_usbc = POWER_DOMAIN_AUX_D,
2438 .aux_tbt = POWER_DOMAIN_INVALID,
2439 },
2440 };
2441
2442 static void
intel_port_domains_for_platform(struct drm_i915_private * i915,const struct intel_ddi_port_domains ** domains,int * domains_size)2443 intel_port_domains_for_platform(struct drm_i915_private *i915,
2444 const struct intel_ddi_port_domains **domains,
2445 int *domains_size)
2446 {
2447 if (DISPLAY_VER(i915) >= 13) {
2448 *domains = d13_port_domains;
2449 *domains_size = ARRAY_SIZE(d13_port_domains);
2450 } else if (DISPLAY_VER(i915) >= 12) {
2451 *domains = d12_port_domains;
2452 *domains_size = ARRAY_SIZE(d12_port_domains);
2453 } else if (DISPLAY_VER(i915) >= 11) {
2454 *domains = d11_port_domains;
2455 *domains_size = ARRAY_SIZE(d11_port_domains);
2456 } else {
2457 *domains = i9xx_port_domains;
2458 *domains_size = ARRAY_SIZE(i9xx_port_domains);
2459 }
2460 }
2461
2462 static const struct intel_ddi_port_domains *
intel_port_domains_for_port(struct drm_i915_private * i915,enum port port)2463 intel_port_domains_for_port(struct drm_i915_private *i915, enum port port)
2464 {
2465 const struct intel_ddi_port_domains *domains;
2466 int domains_size;
2467 int i;
2468
2469 intel_port_domains_for_platform(i915, &domains, &domains_size);
2470 for (i = 0; i < domains_size; i++)
2471 if (port >= domains[i].port_start && port <= domains[i].port_end)
2472 return &domains[i];
2473
2474 return NULL;
2475 }
2476
2477 enum intel_display_power_domain
intel_display_power_ddi_io_domain(struct drm_i915_private * i915,enum port port)2478 intel_display_power_ddi_io_domain(struct drm_i915_private *i915, enum port port)
2479 {
2480 const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2481
2482 if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_io == POWER_DOMAIN_INVALID))
2483 return POWER_DOMAIN_PORT_DDI_IO_A;
2484
2485 return domains->ddi_io + (int)(port - domains->port_start);
2486 }
2487
2488 enum intel_display_power_domain
intel_display_power_ddi_lanes_domain(struct drm_i915_private * i915,enum port port)2489 intel_display_power_ddi_lanes_domain(struct drm_i915_private *i915, enum port port)
2490 {
2491 const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2492
2493 if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_lanes == POWER_DOMAIN_INVALID))
2494 return POWER_DOMAIN_PORT_DDI_LANES_A;
2495
2496 return domains->ddi_lanes + (int)(port - domains->port_start);
2497 }
2498
2499 static const struct intel_ddi_port_domains *
intel_port_domains_for_aux_ch(struct drm_i915_private * i915,enum aux_ch aux_ch)2500 intel_port_domains_for_aux_ch(struct drm_i915_private *i915, enum aux_ch aux_ch)
2501 {
2502 const struct intel_ddi_port_domains *domains;
2503 int domains_size;
2504 int i;
2505
2506 intel_port_domains_for_platform(i915, &domains, &domains_size);
2507 for (i = 0; i < domains_size; i++)
2508 if (aux_ch >= domains[i].aux_ch_start && aux_ch <= domains[i].aux_ch_end)
2509 return &domains[i];
2510
2511 return NULL;
2512 }
2513
2514 enum intel_display_power_domain
intel_display_power_aux_io_domain(struct drm_i915_private * i915,enum aux_ch aux_ch)2515 intel_display_power_aux_io_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2516 {
2517 const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2518
2519 if (drm_WARN_ON(&i915->drm, !domains || domains->aux_io == POWER_DOMAIN_INVALID))
2520 return POWER_DOMAIN_AUX_IO_A;
2521
2522 return domains->aux_io + (int)(aux_ch - domains->aux_ch_start);
2523 }
2524
2525 enum intel_display_power_domain
intel_display_power_legacy_aux_domain(struct drm_i915_private * i915,enum aux_ch aux_ch)2526 intel_display_power_legacy_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2527 {
2528 const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2529
2530 if (drm_WARN_ON(&i915->drm, !domains || domains->aux_legacy_usbc == POWER_DOMAIN_INVALID))
2531 return POWER_DOMAIN_AUX_A;
2532
2533 return domains->aux_legacy_usbc + (int)(aux_ch - domains->aux_ch_start);
2534 }
2535
2536 enum intel_display_power_domain
intel_display_power_tbt_aux_domain(struct drm_i915_private * i915,enum aux_ch aux_ch)2537 intel_display_power_tbt_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2538 {
2539 const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2540
2541 if (drm_WARN_ON(&i915->drm, !domains || domains->aux_tbt == POWER_DOMAIN_INVALID))
2542 return POWER_DOMAIN_AUX_TBT1;
2543
2544 return domains->aux_tbt + (int)(aux_ch - domains->aux_ch_start);
2545 }
2546