2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
46 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
47 MODULE_FIRMWARE("radeon/bonaire_me.bin");
48 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
49 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
50 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
51 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
52 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
53 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
55 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
56 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
57 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
58 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
59 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
60 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
63 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
65 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
66 MODULE_FIRMWARE("radeon/hawaii_me.bin");
67 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
68 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
69 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
70 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
71 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
72 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
74 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
75 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
76 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
77 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
78 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
79 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
81 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
82 MODULE_FIRMWARE("radeon/kaveri_me.bin");
83 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
84 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
85 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
86 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
87 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
89 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
90 MODULE_FIRMWARE("radeon/KABINI_me.bin");
91 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
92 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
93 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
94 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
96 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
97 MODULE_FIRMWARE("radeon/kabini_me.bin");
98 MODULE_FIRMWARE("radeon/kabini_ce.bin");
99 MODULE_FIRMWARE("radeon/kabini_mec.bin");
100 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
101 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
103 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
104 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
105 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
106 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
107 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
108 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
110 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
111 MODULE_FIRMWARE("radeon/mullins_me.bin");
112 MODULE_FIRMWARE("radeon/mullins_ce.bin");
113 MODULE_FIRMWARE("radeon/mullins_mec.bin");
114 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
115 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
117 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
118 extern void r600_ih_ring_fini(struct radeon_device *rdev);
119 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
120 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
121 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
122 extern void sumo_rlc_fini(struct radeon_device *rdev);
123 extern int sumo_rlc_init(struct radeon_device *rdev);
124 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
125 extern void si_rlc_reset(struct radeon_device *rdev);
126 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
127 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
128 extern int cik_sdma_resume(struct radeon_device *rdev);
129 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
130 extern void cik_sdma_fini(struct radeon_device *rdev);
131 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
132 static void cik_rlc_stop(struct radeon_device *rdev);
133 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
134 static void cik_program_aspm(struct radeon_device *rdev);
135 static void cik_init_pg(struct radeon_device *rdev);
136 static void cik_init_cg(struct radeon_device *rdev);
137 static void cik_fini_pg(struct radeon_device *rdev);
138 static void cik_fini_cg(struct radeon_device *rdev);
139 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
142 /* get temperature in millidegrees */
143 int ci_get_temp(struct radeon_device *rdev)
148 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
154 actual_temp = temp & 0x1ff;
156 actual_temp = actual_temp * 1000;
161 /* get temperature in millidegrees */
162 int kv_get_temp(struct radeon_device *rdev)
167 temp = RREG32_SMC(0xC0300E0C);
170 actual_temp = (temp / 8) - 49;
174 actual_temp = actual_temp * 1000;
180 * Indirect registers accessor
182 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
187 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
188 WREG32(PCIE_INDEX, reg);
189 (void)RREG32(PCIE_INDEX);
190 r = RREG32(PCIE_DATA);
191 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
195 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
199 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
200 WREG32(PCIE_INDEX, reg);
201 (void)RREG32(PCIE_INDEX);
202 WREG32(PCIE_DATA, v);
203 (void)RREG32(PCIE_DATA);
204 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
207 static const u32 spectre_rlc_save_restore_register_list[] =
209 (0x0e00 << 16) | (0xc12c >> 2),
211 (0x0e00 << 16) | (0xc140 >> 2),
213 (0x0e00 << 16) | (0xc150 >> 2),
215 (0x0e00 << 16) | (0xc15c >> 2),
217 (0x0e00 << 16) | (0xc168 >> 2),
219 (0x0e00 << 16) | (0xc170 >> 2),
221 (0x0e00 << 16) | (0xc178 >> 2),
223 (0x0e00 << 16) | (0xc204 >> 2),
225 (0x0e00 << 16) | (0xc2b4 >> 2),
227 (0x0e00 << 16) | (0xc2b8 >> 2),
229 (0x0e00 << 16) | (0xc2bc >> 2),
231 (0x0e00 << 16) | (0xc2c0 >> 2),
233 (0x0e00 << 16) | (0x8228 >> 2),
235 (0x0e00 << 16) | (0x829c >> 2),
237 (0x0e00 << 16) | (0x869c >> 2),
239 (0x0600 << 16) | (0x98f4 >> 2),
241 (0x0e00 << 16) | (0x98f8 >> 2),
243 (0x0e00 << 16) | (0x9900 >> 2),
245 (0x0e00 << 16) | (0xc260 >> 2),
247 (0x0e00 << 16) | (0x90e8 >> 2),
249 (0x0e00 << 16) | (0x3c000 >> 2),
251 (0x0e00 << 16) | (0x3c00c >> 2),
253 (0x0e00 << 16) | (0x8c1c >> 2),
255 (0x0e00 << 16) | (0x9700 >> 2),
257 (0x0e00 << 16) | (0xcd20 >> 2),
259 (0x4e00 << 16) | (0xcd20 >> 2),
261 (0x5e00 << 16) | (0xcd20 >> 2),
263 (0x6e00 << 16) | (0xcd20 >> 2),
265 (0x7e00 << 16) | (0xcd20 >> 2),
267 (0x8e00 << 16) | (0xcd20 >> 2),
269 (0x9e00 << 16) | (0xcd20 >> 2),
271 (0xae00 << 16) | (0xcd20 >> 2),
273 (0xbe00 << 16) | (0xcd20 >> 2),
275 (0x0e00 << 16) | (0x89bc >> 2),
277 (0x0e00 << 16) | (0x8900 >> 2),
280 (0x0e00 << 16) | (0xc130 >> 2),
282 (0x0e00 << 16) | (0xc134 >> 2),
284 (0x0e00 << 16) | (0xc1fc >> 2),
286 (0x0e00 << 16) | (0xc208 >> 2),
288 (0x0e00 << 16) | (0xc264 >> 2),
290 (0x0e00 << 16) | (0xc268 >> 2),
292 (0x0e00 << 16) | (0xc26c >> 2),
294 (0x0e00 << 16) | (0xc270 >> 2),
296 (0x0e00 << 16) | (0xc274 >> 2),
298 (0x0e00 << 16) | (0xc278 >> 2),
300 (0x0e00 << 16) | (0xc27c >> 2),
302 (0x0e00 << 16) | (0xc280 >> 2),
304 (0x0e00 << 16) | (0xc284 >> 2),
306 (0x0e00 << 16) | (0xc288 >> 2),
308 (0x0e00 << 16) | (0xc28c >> 2),
310 (0x0e00 << 16) | (0xc290 >> 2),
312 (0x0e00 << 16) | (0xc294 >> 2),
314 (0x0e00 << 16) | (0xc298 >> 2),
316 (0x0e00 << 16) | (0xc29c >> 2),
318 (0x0e00 << 16) | (0xc2a0 >> 2),
320 (0x0e00 << 16) | (0xc2a4 >> 2),
322 (0x0e00 << 16) | (0xc2a8 >> 2),
324 (0x0e00 << 16) | (0xc2ac >> 2),
326 (0x0e00 << 16) | (0xc2b0 >> 2),
328 (0x0e00 << 16) | (0x301d0 >> 2),
330 (0x0e00 << 16) | (0x30238 >> 2),
332 (0x0e00 << 16) | (0x30250 >> 2),
334 (0x0e00 << 16) | (0x30254 >> 2),
336 (0x0e00 << 16) | (0x30258 >> 2),
338 (0x0e00 << 16) | (0x3025c >> 2),
340 (0x4e00 << 16) | (0xc900 >> 2),
342 (0x5e00 << 16) | (0xc900 >> 2),
344 (0x6e00 << 16) | (0xc900 >> 2),
346 (0x7e00 << 16) | (0xc900 >> 2),
348 (0x8e00 << 16) | (0xc900 >> 2),
350 (0x9e00 << 16) | (0xc900 >> 2),
352 (0xae00 << 16) | (0xc900 >> 2),
354 (0xbe00 << 16) | (0xc900 >> 2),
356 (0x4e00 << 16) | (0xc904 >> 2),
358 (0x5e00 << 16) | (0xc904 >> 2),
360 (0x6e00 << 16) | (0xc904 >> 2),
362 (0x7e00 << 16) | (0xc904 >> 2),
364 (0x8e00 << 16) | (0xc904 >> 2),
366 (0x9e00 << 16) | (0xc904 >> 2),
368 (0xae00 << 16) | (0xc904 >> 2),
370 (0xbe00 << 16) | (0xc904 >> 2),
372 (0x4e00 << 16) | (0xc908 >> 2),
374 (0x5e00 << 16) | (0xc908 >> 2),
376 (0x6e00 << 16) | (0xc908 >> 2),
378 (0x7e00 << 16) | (0xc908 >> 2),
380 (0x8e00 << 16) | (0xc908 >> 2),
382 (0x9e00 << 16) | (0xc908 >> 2),
384 (0xae00 << 16) | (0xc908 >> 2),
386 (0xbe00 << 16) | (0xc908 >> 2),
388 (0x4e00 << 16) | (0xc90c >> 2),
390 (0x5e00 << 16) | (0xc90c >> 2),
392 (0x6e00 << 16) | (0xc90c >> 2),
394 (0x7e00 << 16) | (0xc90c >> 2),
396 (0x8e00 << 16) | (0xc90c >> 2),
398 (0x9e00 << 16) | (0xc90c >> 2),
400 (0xae00 << 16) | (0xc90c >> 2),
402 (0xbe00 << 16) | (0xc90c >> 2),
404 (0x4e00 << 16) | (0xc910 >> 2),
406 (0x5e00 << 16) | (0xc910 >> 2),
408 (0x6e00 << 16) | (0xc910 >> 2),
410 (0x7e00 << 16) | (0xc910 >> 2),
412 (0x8e00 << 16) | (0xc910 >> 2),
414 (0x9e00 << 16) | (0xc910 >> 2),
416 (0xae00 << 16) | (0xc910 >> 2),
418 (0xbe00 << 16) | (0xc910 >> 2),
420 (0x0e00 << 16) | (0xc99c >> 2),
422 (0x0e00 << 16) | (0x9834 >> 2),
424 (0x0000 << 16) | (0x30f00 >> 2),
426 (0x0001 << 16) | (0x30f00 >> 2),
428 (0x0000 << 16) | (0x30f04 >> 2),
430 (0x0001 << 16) | (0x30f04 >> 2),
432 (0x0000 << 16) | (0x30f08 >> 2),
434 (0x0001 << 16) | (0x30f08 >> 2),
436 (0x0000 << 16) | (0x30f0c >> 2),
438 (0x0001 << 16) | (0x30f0c >> 2),
440 (0x0600 << 16) | (0x9b7c >> 2),
442 (0x0e00 << 16) | (0x8a14 >> 2),
444 (0x0e00 << 16) | (0x8a18 >> 2),
446 (0x0600 << 16) | (0x30a00 >> 2),
448 (0x0e00 << 16) | (0x8bf0 >> 2),
450 (0x0e00 << 16) | (0x8bcc >> 2),
452 (0x0e00 << 16) | (0x8b24 >> 2),
454 (0x0e00 << 16) | (0x30a04 >> 2),
456 (0x0600 << 16) | (0x30a10 >> 2),
458 (0x0600 << 16) | (0x30a14 >> 2),
460 (0x0600 << 16) | (0x30a18 >> 2),
462 (0x0600 << 16) | (0x30a2c >> 2),
464 (0x0e00 << 16) | (0xc700 >> 2),
466 (0x0e00 << 16) | (0xc704 >> 2),
468 (0x0e00 << 16) | (0xc708 >> 2),
470 (0x0e00 << 16) | (0xc768 >> 2),
472 (0x0400 << 16) | (0xc770 >> 2),
474 (0x0400 << 16) | (0xc774 >> 2),
476 (0x0400 << 16) | (0xc778 >> 2),
478 (0x0400 << 16) | (0xc77c >> 2),
480 (0x0400 << 16) | (0xc780 >> 2),
482 (0x0400 << 16) | (0xc784 >> 2),
484 (0x0400 << 16) | (0xc788 >> 2),
486 (0x0400 << 16) | (0xc78c >> 2),
488 (0x0400 << 16) | (0xc798 >> 2),
490 (0x0400 << 16) | (0xc79c >> 2),
492 (0x0400 << 16) | (0xc7a0 >> 2),
494 (0x0400 << 16) | (0xc7a4 >> 2),
496 (0x0400 << 16) | (0xc7a8 >> 2),
498 (0x0400 << 16) | (0xc7ac >> 2),
500 (0x0400 << 16) | (0xc7b0 >> 2),
502 (0x0400 << 16) | (0xc7b4 >> 2),
504 (0x0e00 << 16) | (0x9100 >> 2),
506 (0x0e00 << 16) | (0x3c010 >> 2),
508 (0x0e00 << 16) | (0x92a8 >> 2),
510 (0x0e00 << 16) | (0x92ac >> 2),
512 (0x0e00 << 16) | (0x92b4 >> 2),
514 (0x0e00 << 16) | (0x92b8 >> 2),
516 (0x0e00 << 16) | (0x92bc >> 2),
518 (0x0e00 << 16) | (0x92c0 >> 2),
520 (0x0e00 << 16) | (0x92c4 >> 2),
522 (0x0e00 << 16) | (0x92c8 >> 2),
524 (0x0e00 << 16) | (0x92cc >> 2),
526 (0x0e00 << 16) | (0x92d0 >> 2),
528 (0x0e00 << 16) | (0x8c00 >> 2),
530 (0x0e00 << 16) | (0x8c04 >> 2),
532 (0x0e00 << 16) | (0x8c20 >> 2),
534 (0x0e00 << 16) | (0x8c38 >> 2),
536 (0x0e00 << 16) | (0x8c3c >> 2),
538 (0x0e00 << 16) | (0xae00 >> 2),
540 (0x0e00 << 16) | (0x9604 >> 2),
542 (0x0e00 << 16) | (0xac08 >> 2),
544 (0x0e00 << 16) | (0xac0c >> 2),
546 (0x0e00 << 16) | (0xac10 >> 2),
548 (0x0e00 << 16) | (0xac14 >> 2),
550 (0x0e00 << 16) | (0xac58 >> 2),
552 (0x0e00 << 16) | (0xac68 >> 2),
554 (0x0e00 << 16) | (0xac6c >> 2),
556 (0x0e00 << 16) | (0xac70 >> 2),
558 (0x0e00 << 16) | (0xac74 >> 2),
560 (0x0e00 << 16) | (0xac78 >> 2),
562 (0x0e00 << 16) | (0xac7c >> 2),
564 (0x0e00 << 16) | (0xac80 >> 2),
566 (0x0e00 << 16) | (0xac84 >> 2),
568 (0x0e00 << 16) | (0xac88 >> 2),
570 (0x0e00 << 16) | (0xac8c >> 2),
572 (0x0e00 << 16) | (0x970c >> 2),
574 (0x0e00 << 16) | (0x9714 >> 2),
576 (0x0e00 << 16) | (0x9718 >> 2),
578 (0x0e00 << 16) | (0x971c >> 2),
580 (0x0e00 << 16) | (0x31068 >> 2),
582 (0x4e00 << 16) | (0x31068 >> 2),
584 (0x5e00 << 16) | (0x31068 >> 2),
586 (0x6e00 << 16) | (0x31068 >> 2),
588 (0x7e00 << 16) | (0x31068 >> 2),
590 (0x8e00 << 16) | (0x31068 >> 2),
592 (0x9e00 << 16) | (0x31068 >> 2),
594 (0xae00 << 16) | (0x31068 >> 2),
596 (0xbe00 << 16) | (0x31068 >> 2),
598 (0x0e00 << 16) | (0xcd10 >> 2),
600 (0x0e00 << 16) | (0xcd14 >> 2),
602 (0x0e00 << 16) | (0x88b0 >> 2),
604 (0x0e00 << 16) | (0x88b4 >> 2),
606 (0x0e00 << 16) | (0x88b8 >> 2),
608 (0x0e00 << 16) | (0x88bc >> 2),
610 (0x0400 << 16) | (0x89c0 >> 2),
612 (0x0e00 << 16) | (0x88c4 >> 2),
614 (0x0e00 << 16) | (0x88c8 >> 2),
616 (0x0e00 << 16) | (0x88d0 >> 2),
618 (0x0e00 << 16) | (0x88d4 >> 2),
620 (0x0e00 << 16) | (0x88d8 >> 2),
622 (0x0e00 << 16) | (0x8980 >> 2),
624 (0x0e00 << 16) | (0x30938 >> 2),
626 (0x0e00 << 16) | (0x3093c >> 2),
628 (0x0e00 << 16) | (0x30940 >> 2),
630 (0x0e00 << 16) | (0x89a0 >> 2),
632 (0x0e00 << 16) | (0x30900 >> 2),
634 (0x0e00 << 16) | (0x30904 >> 2),
636 (0x0e00 << 16) | (0x89b4 >> 2),
638 (0x0e00 << 16) | (0x3c210 >> 2),
640 (0x0e00 << 16) | (0x3c214 >> 2),
642 (0x0e00 << 16) | (0x3c218 >> 2),
644 (0x0e00 << 16) | (0x8904 >> 2),
647 (0x0e00 << 16) | (0x8c28 >> 2),
648 (0x0e00 << 16) | (0x8c2c >> 2),
649 (0x0e00 << 16) | (0x8c30 >> 2),
650 (0x0e00 << 16) | (0x8c34 >> 2),
651 (0x0e00 << 16) | (0x9600 >> 2),
654 static const u32 kalindi_rlc_save_restore_register_list[] =
656 (0x0e00 << 16) | (0xc12c >> 2),
658 (0x0e00 << 16) | (0xc140 >> 2),
660 (0x0e00 << 16) | (0xc150 >> 2),
662 (0x0e00 << 16) | (0xc15c >> 2),
664 (0x0e00 << 16) | (0xc168 >> 2),
666 (0x0e00 << 16) | (0xc170 >> 2),
668 (0x0e00 << 16) | (0xc204 >> 2),
670 (0x0e00 << 16) | (0xc2b4 >> 2),
672 (0x0e00 << 16) | (0xc2b8 >> 2),
674 (0x0e00 << 16) | (0xc2bc >> 2),
676 (0x0e00 << 16) | (0xc2c0 >> 2),
678 (0x0e00 << 16) | (0x8228 >> 2),
680 (0x0e00 << 16) | (0x829c >> 2),
682 (0x0e00 << 16) | (0x869c >> 2),
684 (0x0600 << 16) | (0x98f4 >> 2),
686 (0x0e00 << 16) | (0x98f8 >> 2),
688 (0x0e00 << 16) | (0x9900 >> 2),
690 (0x0e00 << 16) | (0xc260 >> 2),
692 (0x0e00 << 16) | (0x90e8 >> 2),
694 (0x0e00 << 16) | (0x3c000 >> 2),
696 (0x0e00 << 16) | (0x3c00c >> 2),
698 (0x0e00 << 16) | (0x8c1c >> 2),
700 (0x0e00 << 16) | (0x9700 >> 2),
702 (0x0e00 << 16) | (0xcd20 >> 2),
704 (0x4e00 << 16) | (0xcd20 >> 2),
706 (0x5e00 << 16) | (0xcd20 >> 2),
708 (0x6e00 << 16) | (0xcd20 >> 2),
710 (0x7e00 << 16) | (0xcd20 >> 2),
712 (0x0e00 << 16) | (0x89bc >> 2),
714 (0x0e00 << 16) | (0x8900 >> 2),
717 (0x0e00 << 16) | (0xc130 >> 2),
719 (0x0e00 << 16) | (0xc134 >> 2),
721 (0x0e00 << 16) | (0xc1fc >> 2),
723 (0x0e00 << 16) | (0xc208 >> 2),
725 (0x0e00 << 16) | (0xc264 >> 2),
727 (0x0e00 << 16) | (0xc268 >> 2),
729 (0x0e00 << 16) | (0xc26c >> 2),
731 (0x0e00 << 16) | (0xc270 >> 2),
733 (0x0e00 << 16) | (0xc274 >> 2),
735 (0x0e00 << 16) | (0xc28c >> 2),
737 (0x0e00 << 16) | (0xc290 >> 2),
739 (0x0e00 << 16) | (0xc294 >> 2),
741 (0x0e00 << 16) | (0xc298 >> 2),
743 (0x0e00 << 16) | (0xc2a0 >> 2),
745 (0x0e00 << 16) | (0xc2a4 >> 2),
747 (0x0e00 << 16) | (0xc2a8 >> 2),
749 (0x0e00 << 16) | (0xc2ac >> 2),
751 (0x0e00 << 16) | (0x301d0 >> 2),
753 (0x0e00 << 16) | (0x30238 >> 2),
755 (0x0e00 << 16) | (0x30250 >> 2),
757 (0x0e00 << 16) | (0x30254 >> 2),
759 (0x0e00 << 16) | (0x30258 >> 2),
761 (0x0e00 << 16) | (0x3025c >> 2),
763 (0x4e00 << 16) | (0xc900 >> 2),
765 (0x5e00 << 16) | (0xc900 >> 2),
767 (0x6e00 << 16) | (0xc900 >> 2),
769 (0x7e00 << 16) | (0xc900 >> 2),
771 (0x4e00 << 16) | (0xc904 >> 2),
773 (0x5e00 << 16) | (0xc904 >> 2),
775 (0x6e00 << 16) | (0xc904 >> 2),
777 (0x7e00 << 16) | (0xc904 >> 2),
779 (0x4e00 << 16) | (0xc908 >> 2),
781 (0x5e00 << 16) | (0xc908 >> 2),
783 (0x6e00 << 16) | (0xc908 >> 2),
785 (0x7e00 << 16) | (0xc908 >> 2),
787 (0x4e00 << 16) | (0xc90c >> 2),
789 (0x5e00 << 16) | (0xc90c >> 2),
791 (0x6e00 << 16) | (0xc90c >> 2),
793 (0x7e00 << 16) | (0xc90c >> 2),
795 (0x4e00 << 16) | (0xc910 >> 2),
797 (0x5e00 << 16) | (0xc910 >> 2),
799 (0x6e00 << 16) | (0xc910 >> 2),
801 (0x7e00 << 16) | (0xc910 >> 2),
803 (0x0e00 << 16) | (0xc99c >> 2),
805 (0x0e00 << 16) | (0x9834 >> 2),
807 (0x0000 << 16) | (0x30f00 >> 2),
809 (0x0000 << 16) | (0x30f04 >> 2),
811 (0x0000 << 16) | (0x30f08 >> 2),
813 (0x0000 << 16) | (0x30f0c >> 2),
815 (0x0600 << 16) | (0x9b7c >> 2),
817 (0x0e00 << 16) | (0x8a14 >> 2),
819 (0x0e00 << 16) | (0x8a18 >> 2),
821 (0x0600 << 16) | (0x30a00 >> 2),
823 (0x0e00 << 16) | (0x8bf0 >> 2),
825 (0x0e00 << 16) | (0x8bcc >> 2),
827 (0x0e00 << 16) | (0x8b24 >> 2),
829 (0x0e00 << 16) | (0x30a04 >> 2),
831 (0x0600 << 16) | (0x30a10 >> 2),
833 (0x0600 << 16) | (0x30a14 >> 2),
835 (0x0600 << 16) | (0x30a18 >> 2),
837 (0x0600 << 16) | (0x30a2c >> 2),
839 (0x0e00 << 16) | (0xc700 >> 2),
841 (0x0e00 << 16) | (0xc704 >> 2),
843 (0x0e00 << 16) | (0xc708 >> 2),
845 (0x0e00 << 16) | (0xc768 >> 2),
847 (0x0400 << 16) | (0xc770 >> 2),
849 (0x0400 << 16) | (0xc774 >> 2),
851 (0x0400 << 16) | (0xc798 >> 2),
853 (0x0400 << 16) | (0xc79c >> 2),
855 (0x0e00 << 16) | (0x9100 >> 2),
857 (0x0e00 << 16) | (0x3c010 >> 2),
859 (0x0e00 << 16) | (0x8c00 >> 2),
861 (0x0e00 << 16) | (0x8c04 >> 2),
863 (0x0e00 << 16) | (0x8c20 >> 2),
865 (0x0e00 << 16) | (0x8c38 >> 2),
867 (0x0e00 << 16) | (0x8c3c >> 2),
869 (0x0e00 << 16) | (0xae00 >> 2),
871 (0x0e00 << 16) | (0x9604 >> 2),
873 (0x0e00 << 16) | (0xac08 >> 2),
875 (0x0e00 << 16) | (0xac0c >> 2),
877 (0x0e00 << 16) | (0xac10 >> 2),
879 (0x0e00 << 16) | (0xac14 >> 2),
881 (0x0e00 << 16) | (0xac58 >> 2),
883 (0x0e00 << 16) | (0xac68 >> 2),
885 (0x0e00 << 16) | (0xac6c >> 2),
887 (0x0e00 << 16) | (0xac70 >> 2),
889 (0x0e00 << 16) | (0xac74 >> 2),
891 (0x0e00 << 16) | (0xac78 >> 2),
893 (0x0e00 << 16) | (0xac7c >> 2),
895 (0x0e00 << 16) | (0xac80 >> 2),
897 (0x0e00 << 16) | (0xac84 >> 2),
899 (0x0e00 << 16) | (0xac88 >> 2),
901 (0x0e00 << 16) | (0xac8c >> 2),
903 (0x0e00 << 16) | (0x970c >> 2),
905 (0x0e00 << 16) | (0x9714 >> 2),
907 (0x0e00 << 16) | (0x9718 >> 2),
909 (0x0e00 << 16) | (0x971c >> 2),
911 (0x0e00 << 16) | (0x31068 >> 2),
913 (0x4e00 << 16) | (0x31068 >> 2),
915 (0x5e00 << 16) | (0x31068 >> 2),
917 (0x6e00 << 16) | (0x31068 >> 2),
919 (0x7e00 << 16) | (0x31068 >> 2),
921 (0x0e00 << 16) | (0xcd10 >> 2),
923 (0x0e00 << 16) | (0xcd14 >> 2),
925 (0x0e00 << 16) | (0x88b0 >> 2),
927 (0x0e00 << 16) | (0x88b4 >> 2),
929 (0x0e00 << 16) | (0x88b8 >> 2),
931 (0x0e00 << 16) | (0x88bc >> 2),
933 (0x0400 << 16) | (0x89c0 >> 2),
935 (0x0e00 << 16) | (0x88c4 >> 2),
937 (0x0e00 << 16) | (0x88c8 >> 2),
939 (0x0e00 << 16) | (0x88d0 >> 2),
941 (0x0e00 << 16) | (0x88d4 >> 2),
943 (0x0e00 << 16) | (0x88d8 >> 2),
945 (0x0e00 << 16) | (0x8980 >> 2),
947 (0x0e00 << 16) | (0x30938 >> 2),
949 (0x0e00 << 16) | (0x3093c >> 2),
951 (0x0e00 << 16) | (0x30940 >> 2),
953 (0x0e00 << 16) | (0x89a0 >> 2),
955 (0x0e00 << 16) | (0x30900 >> 2),
957 (0x0e00 << 16) | (0x30904 >> 2),
959 (0x0e00 << 16) | (0x89b4 >> 2),
961 (0x0e00 << 16) | (0x3e1fc >> 2),
963 (0x0e00 << 16) | (0x3c210 >> 2),
965 (0x0e00 << 16) | (0x3c214 >> 2),
967 (0x0e00 << 16) | (0x3c218 >> 2),
969 (0x0e00 << 16) | (0x8904 >> 2),
972 (0x0e00 << 16) | (0x8c28 >> 2),
973 (0x0e00 << 16) | (0x8c2c >> 2),
974 (0x0e00 << 16) | (0x8c30 >> 2),
975 (0x0e00 << 16) | (0x8c34 >> 2),
976 (0x0e00 << 16) | (0x9600 >> 2),
979 static const u32 bonaire_golden_spm_registers[] =
981 0x30800, 0xe0ffffff, 0xe0000000
984 static const u32 bonaire_golden_common_registers[] =
986 0xc770, 0xffffffff, 0x00000800,
987 0xc774, 0xffffffff, 0x00000800,
988 0xc798, 0xffffffff, 0x00007fbf,
989 0xc79c, 0xffffffff, 0x00007faf
992 static const u32 bonaire_golden_registers[] =
994 0x3354, 0x00000333, 0x00000333,
995 0x3350, 0x000c0fc0, 0x00040200,
996 0x9a10, 0x00010000, 0x00058208,
997 0x3c000, 0xffff1fff, 0x00140000,
998 0x3c200, 0xfdfc0fff, 0x00000100,
999 0x3c234, 0x40000000, 0x40000200,
1000 0x9830, 0xffffffff, 0x00000000,
1001 0x9834, 0xf00fffff, 0x00000400,
1002 0x9838, 0x0002021c, 0x00020200,
1003 0xc78, 0x00000080, 0x00000000,
1004 0x5bb0, 0x000000f0, 0x00000070,
1005 0x5bc0, 0xf0311fff, 0x80300000,
1006 0x98f8, 0x73773777, 0x12010001,
1007 0x350c, 0x00810000, 0x408af000,
1008 0x7030, 0x31000111, 0x00000011,
1009 0x2f48, 0x73773777, 0x12010001,
1010 0x220c, 0x00007fb6, 0x0021a1b1,
1011 0x2210, 0x00007fb6, 0x002021b1,
1012 0x2180, 0x00007fb6, 0x00002191,
1013 0x2218, 0x00007fb6, 0x002121b1,
1014 0x221c, 0x00007fb6, 0x002021b1,
1015 0x21dc, 0x00007fb6, 0x00002191,
1016 0x21e0, 0x00007fb6, 0x00002191,
1017 0x3628, 0x0000003f, 0x0000000a,
1018 0x362c, 0x0000003f, 0x0000000a,
1019 0x2ae4, 0x00073ffe, 0x000022a2,
1020 0x240c, 0x000007ff, 0x00000000,
1021 0x8a14, 0xf000003f, 0x00000007,
1022 0x8bf0, 0x00002001, 0x00000001,
1023 0x8b24, 0xffffffff, 0x00ffffff,
1024 0x30a04, 0x0000ff0f, 0x00000000,
1025 0x28a4c, 0x07ffffff, 0x06000000,
1026 0x4d8, 0x00000fff, 0x00000100,
1027 0x3e78, 0x00000001, 0x00000002,
1028 0x9100, 0x03000000, 0x0362c688,
1029 0x8c00, 0x000000ff, 0x00000001,
1030 0xe40, 0x00001fff, 0x00001fff,
1031 0x9060, 0x0000007f, 0x00000020,
1032 0x9508, 0x00010000, 0x00010000,
1033 0xac14, 0x000003ff, 0x000000f3,
1034 0xac0c, 0xffffffff, 0x00001032
1037 static const u32 bonaire_mgcg_cgcg_init[] =
1039 0xc420, 0xffffffff, 0xfffffffc,
1040 0x30800, 0xffffffff, 0xe0000000,
1041 0x3c2a0, 0xffffffff, 0x00000100,
1042 0x3c208, 0xffffffff, 0x00000100,
1043 0x3c2c0, 0xffffffff, 0xc0000100,
1044 0x3c2c8, 0xffffffff, 0xc0000100,
1045 0x3c2c4, 0xffffffff, 0xc0000100,
1046 0x55e4, 0xffffffff, 0x00600100,
1047 0x3c280, 0xffffffff, 0x00000100,
1048 0x3c214, 0xffffffff, 0x06000100,
1049 0x3c220, 0xffffffff, 0x00000100,
1050 0x3c218, 0xffffffff, 0x06000100,
1051 0x3c204, 0xffffffff, 0x00000100,
1052 0x3c2e0, 0xffffffff, 0x00000100,
1053 0x3c224, 0xffffffff, 0x00000100,
1054 0x3c200, 0xffffffff, 0x00000100,
1055 0x3c230, 0xffffffff, 0x00000100,
1056 0x3c234, 0xffffffff, 0x00000100,
1057 0x3c250, 0xffffffff, 0x00000100,
1058 0x3c254, 0xffffffff, 0x00000100,
1059 0x3c258, 0xffffffff, 0x00000100,
1060 0x3c25c, 0xffffffff, 0x00000100,
1061 0x3c260, 0xffffffff, 0x00000100,
1062 0x3c27c, 0xffffffff, 0x00000100,
1063 0x3c278, 0xffffffff, 0x00000100,
1064 0x3c210, 0xffffffff, 0x06000100,
1065 0x3c290, 0xffffffff, 0x00000100,
1066 0x3c274, 0xffffffff, 0x00000100,
1067 0x3c2b4, 0xffffffff, 0x00000100,
1068 0x3c2b0, 0xffffffff, 0x00000100,
1069 0x3c270, 0xffffffff, 0x00000100,
1070 0x30800, 0xffffffff, 0xe0000000,
1071 0x3c020, 0xffffffff, 0x00010000,
1072 0x3c024, 0xffffffff, 0x00030002,
1073 0x3c028, 0xffffffff, 0x00040007,
1074 0x3c02c, 0xffffffff, 0x00060005,
1075 0x3c030, 0xffffffff, 0x00090008,
1076 0x3c034, 0xffffffff, 0x00010000,
1077 0x3c038, 0xffffffff, 0x00030002,
1078 0x3c03c, 0xffffffff, 0x00040007,
1079 0x3c040, 0xffffffff, 0x00060005,
1080 0x3c044, 0xffffffff, 0x00090008,
1081 0x3c048, 0xffffffff, 0x00010000,
1082 0x3c04c, 0xffffffff, 0x00030002,
1083 0x3c050, 0xffffffff, 0x00040007,
1084 0x3c054, 0xffffffff, 0x00060005,
1085 0x3c058, 0xffffffff, 0x00090008,
1086 0x3c05c, 0xffffffff, 0x00010000,
1087 0x3c060, 0xffffffff, 0x00030002,
1088 0x3c064, 0xffffffff, 0x00040007,
1089 0x3c068, 0xffffffff, 0x00060005,
1090 0x3c06c, 0xffffffff, 0x00090008,
1091 0x3c070, 0xffffffff, 0x00010000,
1092 0x3c074, 0xffffffff, 0x00030002,
1093 0x3c078, 0xffffffff, 0x00040007,
1094 0x3c07c, 0xffffffff, 0x00060005,
1095 0x3c080, 0xffffffff, 0x00090008,
1096 0x3c084, 0xffffffff, 0x00010000,
1097 0x3c088, 0xffffffff, 0x00030002,
1098 0x3c08c, 0xffffffff, 0x00040007,
1099 0x3c090, 0xffffffff, 0x00060005,
1100 0x3c094, 0xffffffff, 0x00090008,
1101 0x3c098, 0xffffffff, 0x00010000,
1102 0x3c09c, 0xffffffff, 0x00030002,
1103 0x3c0a0, 0xffffffff, 0x00040007,
1104 0x3c0a4, 0xffffffff, 0x00060005,
1105 0x3c0a8, 0xffffffff, 0x00090008,
1106 0x3c000, 0xffffffff, 0x96e00200,
1107 0x8708, 0xffffffff, 0x00900100,
1108 0xc424, 0xffffffff, 0x0020003f,
1109 0x38, 0xffffffff, 0x0140001c,
1110 0x3c, 0x000f0000, 0x000f0000,
1111 0x220, 0xffffffff, 0xC060000C,
1112 0x224, 0xc0000fff, 0x00000100,
1113 0xf90, 0xffffffff, 0x00000100,
1114 0xf98, 0x00000101, 0x00000000,
1115 0x20a8, 0xffffffff, 0x00000104,
1116 0x55e4, 0xff000fff, 0x00000100,
1117 0x30cc, 0xc0000fff, 0x00000104,
1118 0xc1e4, 0x00000001, 0x00000001,
1119 0xd00c, 0xff000ff0, 0x00000100,
1120 0xd80c, 0xff000ff0, 0x00000100
1123 static const u32 spectre_golden_spm_registers[] =
1125 0x30800, 0xe0ffffff, 0xe0000000
1128 static const u32 spectre_golden_common_registers[] =
1130 0xc770, 0xffffffff, 0x00000800,
1131 0xc774, 0xffffffff, 0x00000800,
1132 0xc798, 0xffffffff, 0x00007fbf,
1133 0xc79c, 0xffffffff, 0x00007faf
1136 static const u32 spectre_golden_registers[] =
1138 0x3c000, 0xffff1fff, 0x96940200,
1139 0x3c00c, 0xffff0001, 0xff000000,
1140 0x3c200, 0xfffc0fff, 0x00000100,
1141 0x6ed8, 0x00010101, 0x00010000,
1142 0x9834, 0xf00fffff, 0x00000400,
1143 0x9838, 0xfffffffc, 0x00020200,
1144 0x5bb0, 0x000000f0, 0x00000070,
1145 0x5bc0, 0xf0311fff, 0x80300000,
1146 0x98f8, 0x73773777, 0x12010001,
1147 0x9b7c, 0x00ff0000, 0x00fc0000,
1148 0x2f48, 0x73773777, 0x12010001,
1149 0x8a14, 0xf000003f, 0x00000007,
1150 0x8b24, 0xffffffff, 0x00ffffff,
1151 0x28350, 0x3f3f3fff, 0x00000082,
1152 0x28354, 0x0000003f, 0x00000000,
1153 0x3e78, 0x00000001, 0x00000002,
1154 0x913c, 0xffff03df, 0x00000004,
1155 0xc768, 0x00000008, 0x00000008,
1156 0x8c00, 0x000008ff, 0x00000800,
1157 0x9508, 0x00010000, 0x00010000,
1158 0xac0c, 0xffffffff, 0x54763210,
1159 0x214f8, 0x01ff01ff, 0x00000002,
1160 0x21498, 0x007ff800, 0x00200000,
1161 0x2015c, 0xffffffff, 0x00000f40,
1162 0x30934, 0xffffffff, 0x00000001
1165 static const u32 spectre_mgcg_cgcg_init[] =
1167 0xc420, 0xffffffff, 0xfffffffc,
1168 0x30800, 0xffffffff, 0xe0000000,
1169 0x3c2a0, 0xffffffff, 0x00000100,
1170 0x3c208, 0xffffffff, 0x00000100,
1171 0x3c2c0, 0xffffffff, 0x00000100,
1172 0x3c2c8, 0xffffffff, 0x00000100,
1173 0x3c2c4, 0xffffffff, 0x00000100,
1174 0x55e4, 0xffffffff, 0x00600100,
1175 0x3c280, 0xffffffff, 0x00000100,
1176 0x3c214, 0xffffffff, 0x06000100,
1177 0x3c220, 0xffffffff, 0x00000100,
1178 0x3c218, 0xffffffff, 0x06000100,
1179 0x3c204, 0xffffffff, 0x00000100,
1180 0x3c2e0, 0xffffffff, 0x00000100,
1181 0x3c224, 0xffffffff, 0x00000100,
1182 0x3c200, 0xffffffff, 0x00000100,
1183 0x3c230, 0xffffffff, 0x00000100,
1184 0x3c234, 0xffffffff, 0x00000100,
1185 0x3c250, 0xffffffff, 0x00000100,
1186 0x3c254, 0xffffffff, 0x00000100,
1187 0x3c258, 0xffffffff, 0x00000100,
1188 0x3c25c, 0xffffffff, 0x00000100,
1189 0x3c260, 0xffffffff, 0x00000100,
1190 0x3c27c, 0xffffffff, 0x00000100,
1191 0x3c278, 0xffffffff, 0x00000100,
1192 0x3c210, 0xffffffff, 0x06000100,
1193 0x3c290, 0xffffffff, 0x00000100,
1194 0x3c274, 0xffffffff, 0x00000100,
1195 0x3c2b4, 0xffffffff, 0x00000100,
1196 0x3c2b0, 0xffffffff, 0x00000100,
1197 0x3c270, 0xffffffff, 0x00000100,
1198 0x30800, 0xffffffff, 0xe0000000,
1199 0x3c020, 0xffffffff, 0x00010000,
1200 0x3c024, 0xffffffff, 0x00030002,
1201 0x3c028, 0xffffffff, 0x00040007,
1202 0x3c02c, 0xffffffff, 0x00060005,
1203 0x3c030, 0xffffffff, 0x00090008,
1204 0x3c034, 0xffffffff, 0x00010000,
1205 0x3c038, 0xffffffff, 0x00030002,
1206 0x3c03c, 0xffffffff, 0x00040007,
1207 0x3c040, 0xffffffff, 0x00060005,
1208 0x3c044, 0xffffffff, 0x00090008,
1209 0x3c048, 0xffffffff, 0x00010000,
1210 0x3c04c, 0xffffffff, 0x00030002,
1211 0x3c050, 0xffffffff, 0x00040007,
1212 0x3c054, 0xffffffff, 0x00060005,
1213 0x3c058, 0xffffffff, 0x00090008,
1214 0x3c05c, 0xffffffff, 0x00010000,
1215 0x3c060, 0xffffffff, 0x00030002,
1216 0x3c064, 0xffffffff, 0x00040007,
1217 0x3c068, 0xffffffff, 0x00060005,
1218 0x3c06c, 0xffffffff, 0x00090008,
1219 0x3c070, 0xffffffff, 0x00010000,
1220 0x3c074, 0xffffffff, 0x00030002,
1221 0x3c078, 0xffffffff, 0x00040007,
1222 0x3c07c, 0xffffffff, 0x00060005,
1223 0x3c080, 0xffffffff, 0x00090008,
1224 0x3c084, 0xffffffff, 0x00010000,
1225 0x3c088, 0xffffffff, 0x00030002,
1226 0x3c08c, 0xffffffff, 0x00040007,
1227 0x3c090, 0xffffffff, 0x00060005,
1228 0x3c094, 0xffffffff, 0x00090008,
1229 0x3c098, 0xffffffff, 0x00010000,
1230 0x3c09c, 0xffffffff, 0x00030002,
1231 0x3c0a0, 0xffffffff, 0x00040007,
1232 0x3c0a4, 0xffffffff, 0x00060005,
1233 0x3c0a8, 0xffffffff, 0x00090008,
1234 0x3c0ac, 0xffffffff, 0x00010000,
1235 0x3c0b0, 0xffffffff, 0x00030002,
1236 0x3c0b4, 0xffffffff, 0x00040007,
1237 0x3c0b8, 0xffffffff, 0x00060005,
1238 0x3c0bc, 0xffffffff, 0x00090008,
1239 0x3c000, 0xffffffff, 0x96e00200,
1240 0x8708, 0xffffffff, 0x00900100,
1241 0xc424, 0xffffffff, 0x0020003f,
1242 0x38, 0xffffffff, 0x0140001c,
1243 0x3c, 0x000f0000, 0x000f0000,
1244 0x220, 0xffffffff, 0xC060000C,
1245 0x224, 0xc0000fff, 0x00000100,
1246 0xf90, 0xffffffff, 0x00000100,
1247 0xf98, 0x00000101, 0x00000000,
1248 0x20a8, 0xffffffff, 0x00000104,
1249 0x55e4, 0xff000fff, 0x00000100,
1250 0x30cc, 0xc0000fff, 0x00000104,
1251 0xc1e4, 0x00000001, 0x00000001,
1252 0xd00c, 0xff000ff0, 0x00000100,
1253 0xd80c, 0xff000ff0, 0x00000100
1256 static const u32 kalindi_golden_spm_registers[] =
1258 0x30800, 0xe0ffffff, 0xe0000000
1261 static const u32 kalindi_golden_common_registers[] =
1263 0xc770, 0xffffffff, 0x00000800,
1264 0xc774, 0xffffffff, 0x00000800,
1265 0xc798, 0xffffffff, 0x00007fbf,
1266 0xc79c, 0xffffffff, 0x00007faf
1269 static const u32 kalindi_golden_registers[] =
1271 0x3c000, 0xffffdfff, 0x6e944040,
1272 0x55e4, 0xff607fff, 0xfc000100,
1273 0x3c220, 0xff000fff, 0x00000100,
1274 0x3c224, 0xff000fff, 0x00000100,
1275 0x3c200, 0xfffc0fff, 0x00000100,
1276 0x6ed8, 0x00010101, 0x00010000,
1277 0x9830, 0xffffffff, 0x00000000,
1278 0x9834, 0xf00fffff, 0x00000400,
1279 0x5bb0, 0x000000f0, 0x00000070,
1280 0x5bc0, 0xf0311fff, 0x80300000,
1281 0x98f8, 0x73773777, 0x12010001,
1282 0x98fc, 0xffffffff, 0x00000010,
1283 0x9b7c, 0x00ff0000, 0x00fc0000,
1284 0x8030, 0x00001f0f, 0x0000100a,
1285 0x2f48, 0x73773777, 0x12010001,
1286 0x2408, 0x000fffff, 0x000c007f,
1287 0x8a14, 0xf000003f, 0x00000007,
1288 0x8b24, 0x3fff3fff, 0x00ffcfff,
1289 0x30a04, 0x0000ff0f, 0x00000000,
1290 0x28a4c, 0x07ffffff, 0x06000000,
1291 0x4d8, 0x00000fff, 0x00000100,
1292 0x3e78, 0x00000001, 0x00000002,
1293 0xc768, 0x00000008, 0x00000008,
1294 0x8c00, 0x000000ff, 0x00000003,
1295 0x214f8, 0x01ff01ff, 0x00000002,
1296 0x21498, 0x007ff800, 0x00200000,
1297 0x2015c, 0xffffffff, 0x00000f40,
1298 0x88c4, 0x001f3ae3, 0x00000082,
1299 0x88d4, 0x0000001f, 0x00000010,
1300 0x30934, 0xffffffff, 0x00000000
1303 static const u32 kalindi_mgcg_cgcg_init[] =
1305 0xc420, 0xffffffff, 0xfffffffc,
1306 0x30800, 0xffffffff, 0xe0000000,
1307 0x3c2a0, 0xffffffff, 0x00000100,
1308 0x3c208, 0xffffffff, 0x00000100,
1309 0x3c2c0, 0xffffffff, 0x00000100,
1310 0x3c2c8, 0xffffffff, 0x00000100,
1311 0x3c2c4, 0xffffffff, 0x00000100,
1312 0x55e4, 0xffffffff, 0x00600100,
1313 0x3c280, 0xffffffff, 0x00000100,
1314 0x3c214, 0xffffffff, 0x06000100,
1315 0x3c220, 0xffffffff, 0x00000100,
1316 0x3c218, 0xffffffff, 0x06000100,
1317 0x3c204, 0xffffffff, 0x00000100,
1318 0x3c2e0, 0xffffffff, 0x00000100,
1319 0x3c224, 0xffffffff, 0x00000100,
1320 0x3c200, 0xffffffff, 0x00000100,
1321 0x3c230, 0xffffffff, 0x00000100,
1322 0x3c234, 0xffffffff, 0x00000100,
1323 0x3c250, 0xffffffff, 0x00000100,
1324 0x3c254, 0xffffffff, 0x00000100,
1325 0x3c258, 0xffffffff, 0x00000100,
1326 0x3c25c, 0xffffffff, 0x00000100,
1327 0x3c260, 0xffffffff, 0x00000100,
1328 0x3c27c, 0xffffffff, 0x00000100,
1329 0x3c278, 0xffffffff, 0x00000100,
1330 0x3c210, 0xffffffff, 0x06000100,
1331 0x3c290, 0xffffffff, 0x00000100,
1332 0x3c274, 0xffffffff, 0x00000100,
1333 0x3c2b4, 0xffffffff, 0x00000100,
1334 0x3c2b0, 0xffffffff, 0x00000100,
1335 0x3c270, 0xffffffff, 0x00000100,
1336 0x30800, 0xffffffff, 0xe0000000,
1337 0x3c020, 0xffffffff, 0x00010000,
1338 0x3c024, 0xffffffff, 0x00030002,
1339 0x3c028, 0xffffffff, 0x00040007,
1340 0x3c02c, 0xffffffff, 0x00060005,
1341 0x3c030, 0xffffffff, 0x00090008,
1342 0x3c034, 0xffffffff, 0x00010000,
1343 0x3c038, 0xffffffff, 0x00030002,
1344 0x3c03c, 0xffffffff, 0x00040007,
1345 0x3c040, 0xffffffff, 0x00060005,
1346 0x3c044, 0xffffffff, 0x00090008,
1347 0x3c000, 0xffffffff, 0x96e00200,
1348 0x8708, 0xffffffff, 0x00900100,
1349 0xc424, 0xffffffff, 0x0020003f,
1350 0x38, 0xffffffff, 0x0140001c,
1351 0x3c, 0x000f0000, 0x000f0000,
1352 0x220, 0xffffffff, 0xC060000C,
1353 0x224, 0xc0000fff, 0x00000100,
1354 0x20a8, 0xffffffff, 0x00000104,
1355 0x55e4, 0xff000fff, 0x00000100,
1356 0x30cc, 0xc0000fff, 0x00000104,
1357 0xc1e4, 0x00000001, 0x00000001,
1358 0xd00c, 0xff000ff0, 0x00000100,
1359 0xd80c, 0xff000ff0, 0x00000100
1362 static const u32 hawaii_golden_spm_registers[] =
1364 0x30800, 0xe0ffffff, 0xe0000000
1367 static const u32 hawaii_golden_common_registers[] =
1369 0x30800, 0xffffffff, 0xe0000000,
1370 0x28350, 0xffffffff, 0x3a00161a,
1371 0x28354, 0xffffffff, 0x0000002e,
1372 0x9a10, 0xffffffff, 0x00018208,
1373 0x98f8, 0xffffffff, 0x12011003
1376 static const u32 hawaii_golden_registers[] =
1378 0x3354, 0x00000333, 0x00000333,
1379 0x9a10, 0x00010000, 0x00058208,
1380 0x9830, 0xffffffff, 0x00000000,
1381 0x9834, 0xf00fffff, 0x00000400,
1382 0x9838, 0x0002021c, 0x00020200,
1383 0xc78, 0x00000080, 0x00000000,
1384 0x5bb0, 0x000000f0, 0x00000070,
1385 0x5bc0, 0xf0311fff, 0x80300000,
1386 0x350c, 0x00810000, 0x408af000,
1387 0x7030, 0x31000111, 0x00000011,
1388 0x2f48, 0x73773777, 0x12010001,
1389 0x2120, 0x0000007f, 0x0000001b,
1390 0x21dc, 0x00007fb6, 0x00002191,
1391 0x3628, 0x0000003f, 0x0000000a,
1392 0x362c, 0x0000003f, 0x0000000a,
1393 0x2ae4, 0x00073ffe, 0x000022a2,
1394 0x240c, 0x000007ff, 0x00000000,
1395 0x8bf0, 0x00002001, 0x00000001,
1396 0x8b24, 0xffffffff, 0x00ffffff,
1397 0x30a04, 0x0000ff0f, 0x00000000,
1398 0x28a4c, 0x07ffffff, 0x06000000,
1399 0x3e78, 0x00000001, 0x00000002,
1400 0xc768, 0x00000008, 0x00000008,
1401 0xc770, 0x00000f00, 0x00000800,
1402 0xc774, 0x00000f00, 0x00000800,
1403 0xc798, 0x00ffffff, 0x00ff7fbf,
1404 0xc79c, 0x00ffffff, 0x00ff7faf,
1405 0x8c00, 0x000000ff, 0x00000800,
1406 0xe40, 0x00001fff, 0x00001fff,
1407 0x9060, 0x0000007f, 0x00000020,
1408 0x9508, 0x00010000, 0x00010000,
1409 0xae00, 0x00100000, 0x000ff07c,
1410 0xac14, 0x000003ff, 0x0000000f,
1411 0xac10, 0xffffffff, 0x7564fdec,
1412 0xac0c, 0xffffffff, 0x3120b9a8,
1413 0xac08, 0x20000000, 0x0f9c0000
1416 static const u32 hawaii_mgcg_cgcg_init[] =
1418 0xc420, 0xffffffff, 0xfffffffd,
1419 0x30800, 0xffffffff, 0xe0000000,
1420 0x3c2a0, 0xffffffff, 0x00000100,
1421 0x3c208, 0xffffffff, 0x00000100,
1422 0x3c2c0, 0xffffffff, 0x00000100,
1423 0x3c2c8, 0xffffffff, 0x00000100,
1424 0x3c2c4, 0xffffffff, 0x00000100,
1425 0x55e4, 0xffffffff, 0x00200100,
1426 0x3c280, 0xffffffff, 0x00000100,
1427 0x3c214, 0xffffffff, 0x06000100,
1428 0x3c220, 0xffffffff, 0x00000100,
1429 0x3c218, 0xffffffff, 0x06000100,
1430 0x3c204, 0xffffffff, 0x00000100,
1431 0x3c2e0, 0xffffffff, 0x00000100,
1432 0x3c224, 0xffffffff, 0x00000100,
1433 0x3c200, 0xffffffff, 0x00000100,
1434 0x3c230, 0xffffffff, 0x00000100,
1435 0x3c234, 0xffffffff, 0x00000100,
1436 0x3c250, 0xffffffff, 0x00000100,
1437 0x3c254, 0xffffffff, 0x00000100,
1438 0x3c258, 0xffffffff, 0x00000100,
1439 0x3c25c, 0xffffffff, 0x00000100,
1440 0x3c260, 0xffffffff, 0x00000100,
1441 0x3c27c, 0xffffffff, 0x00000100,
1442 0x3c278, 0xffffffff, 0x00000100,
1443 0x3c210, 0xffffffff, 0x06000100,
1444 0x3c290, 0xffffffff, 0x00000100,
1445 0x3c274, 0xffffffff, 0x00000100,
1446 0x3c2b4, 0xffffffff, 0x00000100,
1447 0x3c2b0, 0xffffffff, 0x00000100,
1448 0x3c270, 0xffffffff, 0x00000100,
1449 0x30800, 0xffffffff, 0xe0000000,
1450 0x3c020, 0xffffffff, 0x00010000,
1451 0x3c024, 0xffffffff, 0x00030002,
1452 0x3c028, 0xffffffff, 0x00040007,
1453 0x3c02c, 0xffffffff, 0x00060005,
1454 0x3c030, 0xffffffff, 0x00090008,
1455 0x3c034, 0xffffffff, 0x00010000,
1456 0x3c038, 0xffffffff, 0x00030002,
1457 0x3c03c, 0xffffffff, 0x00040007,
1458 0x3c040, 0xffffffff, 0x00060005,
1459 0x3c044, 0xffffffff, 0x00090008,
1460 0x3c048, 0xffffffff, 0x00010000,
1461 0x3c04c, 0xffffffff, 0x00030002,
1462 0x3c050, 0xffffffff, 0x00040007,
1463 0x3c054, 0xffffffff, 0x00060005,
1464 0x3c058, 0xffffffff, 0x00090008,
1465 0x3c05c, 0xffffffff, 0x00010000,
1466 0x3c060, 0xffffffff, 0x00030002,
1467 0x3c064, 0xffffffff, 0x00040007,
1468 0x3c068, 0xffffffff, 0x00060005,
1469 0x3c06c, 0xffffffff, 0x00090008,
1470 0x3c070, 0xffffffff, 0x00010000,
1471 0x3c074, 0xffffffff, 0x00030002,
1472 0x3c078, 0xffffffff, 0x00040007,
1473 0x3c07c, 0xffffffff, 0x00060005,
1474 0x3c080, 0xffffffff, 0x00090008,
1475 0x3c084, 0xffffffff, 0x00010000,
1476 0x3c088, 0xffffffff, 0x00030002,
1477 0x3c08c, 0xffffffff, 0x00040007,
1478 0x3c090, 0xffffffff, 0x00060005,
1479 0x3c094, 0xffffffff, 0x00090008,
1480 0x3c098, 0xffffffff, 0x00010000,
1481 0x3c09c, 0xffffffff, 0x00030002,
1482 0x3c0a0, 0xffffffff, 0x00040007,
1483 0x3c0a4, 0xffffffff, 0x00060005,
1484 0x3c0a8, 0xffffffff, 0x00090008,
1485 0x3c0ac, 0xffffffff, 0x00010000,
1486 0x3c0b0, 0xffffffff, 0x00030002,
1487 0x3c0b4, 0xffffffff, 0x00040007,
1488 0x3c0b8, 0xffffffff, 0x00060005,
1489 0x3c0bc, 0xffffffff, 0x00090008,
1490 0x3c0c0, 0xffffffff, 0x00010000,
1491 0x3c0c4, 0xffffffff, 0x00030002,
1492 0x3c0c8, 0xffffffff, 0x00040007,
1493 0x3c0cc, 0xffffffff, 0x00060005,
1494 0x3c0d0, 0xffffffff, 0x00090008,
1495 0x3c0d4, 0xffffffff, 0x00010000,
1496 0x3c0d8, 0xffffffff, 0x00030002,
1497 0x3c0dc, 0xffffffff, 0x00040007,
1498 0x3c0e0, 0xffffffff, 0x00060005,
1499 0x3c0e4, 0xffffffff, 0x00090008,
1500 0x3c0e8, 0xffffffff, 0x00010000,
1501 0x3c0ec, 0xffffffff, 0x00030002,
1502 0x3c0f0, 0xffffffff, 0x00040007,
1503 0x3c0f4, 0xffffffff, 0x00060005,
1504 0x3c0f8, 0xffffffff, 0x00090008,
1505 0xc318, 0xffffffff, 0x00020200,
1506 0x3350, 0xffffffff, 0x00000200,
1507 0x15c0, 0xffffffff, 0x00000400,
1508 0x55e8, 0xffffffff, 0x00000000,
1509 0x2f50, 0xffffffff, 0x00000902,
1510 0x3c000, 0xffffffff, 0x96940200,
1511 0x8708, 0xffffffff, 0x00900100,
1512 0xc424, 0xffffffff, 0x0020003f,
1513 0x38, 0xffffffff, 0x0140001c,
1514 0x3c, 0x000f0000, 0x000f0000,
1515 0x220, 0xffffffff, 0xc060000c,
1516 0x224, 0xc0000fff, 0x00000100,
1517 0xf90, 0xffffffff, 0x00000100,
1518 0xf98, 0x00000101, 0x00000000,
1519 0x20a8, 0xffffffff, 0x00000104,
1520 0x55e4, 0xff000fff, 0x00000100,
1521 0x30cc, 0xc0000fff, 0x00000104,
1522 0xc1e4, 0x00000001, 0x00000001,
1523 0xd00c, 0xff000ff0, 0x00000100,
1524 0xd80c, 0xff000ff0, 0x00000100
1527 static const u32 godavari_golden_registers[] =
1529 0x55e4, 0xff607fff, 0xfc000100,
1530 0x6ed8, 0x00010101, 0x00010000,
1531 0x9830, 0xffffffff, 0x00000000,
1532 0x98302, 0xf00fffff, 0x00000400,
1533 0x6130, 0xffffffff, 0x00010000,
1534 0x5bb0, 0x000000f0, 0x00000070,
1535 0x5bc0, 0xf0311fff, 0x80300000,
1536 0x98f8, 0x73773777, 0x12010001,
1537 0x98fc, 0xffffffff, 0x00000010,
1538 0x8030, 0x00001f0f, 0x0000100a,
1539 0x2f48, 0x73773777, 0x12010001,
1540 0x2408, 0x000fffff, 0x000c007f,
1541 0x8a14, 0xf000003f, 0x00000007,
1542 0x8b24, 0xffffffff, 0x00ff0fff,
1543 0x30a04, 0x0000ff0f, 0x00000000,
1544 0x28a4c, 0x07ffffff, 0x06000000,
1545 0x4d8, 0x00000fff, 0x00000100,
1546 0xd014, 0x00010000, 0x00810001,
1547 0xd814, 0x00010000, 0x00810001,
1548 0x3e78, 0x00000001, 0x00000002,
1549 0xc768, 0x00000008, 0x00000008,
1550 0xc770, 0x00000f00, 0x00000800,
1551 0xc774, 0x00000f00, 0x00000800,
1552 0xc798, 0x00ffffff, 0x00ff7fbf,
1553 0xc79c, 0x00ffffff, 0x00ff7faf,
1554 0x8c00, 0x000000ff, 0x00000001,
1555 0x214f8, 0x01ff01ff, 0x00000002,
1556 0x21498, 0x007ff800, 0x00200000,
1557 0x2015c, 0xffffffff, 0x00000f40,
1558 0x88c4, 0x001f3ae3, 0x00000082,
1559 0x88d4, 0x0000001f, 0x00000010,
1560 0x30934, 0xffffffff, 0x00000000
1564 static void cik_init_golden_registers(struct radeon_device *rdev)
1566 switch (rdev->family) {
1568 radeon_program_register_sequence(rdev,
1569 bonaire_mgcg_cgcg_init,
1570 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1571 radeon_program_register_sequence(rdev,
1572 bonaire_golden_registers,
1573 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1574 radeon_program_register_sequence(rdev,
1575 bonaire_golden_common_registers,
1576 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1577 radeon_program_register_sequence(rdev,
1578 bonaire_golden_spm_registers,
1579 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1582 radeon_program_register_sequence(rdev,
1583 kalindi_mgcg_cgcg_init,
1584 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1585 radeon_program_register_sequence(rdev,
1586 kalindi_golden_registers,
1587 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1588 radeon_program_register_sequence(rdev,
1589 kalindi_golden_common_registers,
1590 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1591 radeon_program_register_sequence(rdev,
1592 kalindi_golden_spm_registers,
1593 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1596 radeon_program_register_sequence(rdev,
1597 kalindi_mgcg_cgcg_init,
1598 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1599 radeon_program_register_sequence(rdev,
1600 godavari_golden_registers,
1601 (const u32)ARRAY_SIZE(godavari_golden_registers));
1602 radeon_program_register_sequence(rdev,
1603 kalindi_golden_common_registers,
1604 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1605 radeon_program_register_sequence(rdev,
1606 kalindi_golden_spm_registers,
1607 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1610 radeon_program_register_sequence(rdev,
1611 spectre_mgcg_cgcg_init,
1612 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1613 radeon_program_register_sequence(rdev,
1614 spectre_golden_registers,
1615 (const u32)ARRAY_SIZE(spectre_golden_registers));
1616 radeon_program_register_sequence(rdev,
1617 spectre_golden_common_registers,
1618 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1619 radeon_program_register_sequence(rdev,
1620 spectre_golden_spm_registers,
1621 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1624 radeon_program_register_sequence(rdev,
1625 hawaii_mgcg_cgcg_init,
1626 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1627 radeon_program_register_sequence(rdev,
1628 hawaii_golden_registers,
1629 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1630 radeon_program_register_sequence(rdev,
1631 hawaii_golden_common_registers,
1632 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1633 radeon_program_register_sequence(rdev,
1634 hawaii_golden_spm_registers,
1635 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1643 * cik_get_xclk - get the xclk
1645 * @rdev: radeon_device pointer
1647 * Returns the reference clock used by the gfx engine
1650 u32 cik_get_xclk(struct radeon_device *rdev)
1652 u32 reference_clock = rdev->clock.spll.reference_freq;
1654 if (rdev->flags & RADEON_IS_IGP) {
1655 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1656 return reference_clock / 2;
1658 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1659 return reference_clock / 4;
1661 return reference_clock;
1665 * cik_mm_rdoorbell - read a doorbell dword
1667 * @rdev: radeon_device pointer
1668 * @index: doorbell index
1670 * Returns the value in the doorbell aperture at the
1671 * requested doorbell index (CIK).
1673 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1675 if (index < rdev->doorbell.num_doorbells) {
1676 return readl(rdev->doorbell.ptr + index);
1678 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1684 * cik_mm_wdoorbell - write a doorbell dword
1686 * @rdev: radeon_device pointer
1687 * @index: doorbell index
1688 * @v: value to write
1690 * Writes @v to the doorbell aperture at the
1691 * requested doorbell index (CIK).
1693 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1695 if (index < rdev->doorbell.num_doorbells) {
1696 writel(v, rdev->doorbell.ptr + index);
1698 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1702 #define BONAIRE_IO_MC_REGS_SIZE 36
1704 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1706 {0x00000070, 0x04400000},
1707 {0x00000071, 0x80c01803},
1708 {0x00000072, 0x00004004},
1709 {0x00000073, 0x00000100},
1710 {0x00000074, 0x00ff0000},
1711 {0x00000075, 0x34000000},
1712 {0x00000076, 0x08000014},
1713 {0x00000077, 0x00cc08ec},
1714 {0x00000078, 0x00000400},
1715 {0x00000079, 0x00000000},
1716 {0x0000007a, 0x04090000},
1717 {0x0000007c, 0x00000000},
1718 {0x0000007e, 0x4408a8e8},
1719 {0x0000007f, 0x00000304},
1720 {0x00000080, 0x00000000},
1721 {0x00000082, 0x00000001},
1722 {0x00000083, 0x00000002},
1723 {0x00000084, 0xf3e4f400},
1724 {0x00000085, 0x052024e3},
1725 {0x00000087, 0x00000000},
1726 {0x00000088, 0x01000000},
1727 {0x0000008a, 0x1c0a0000},
1728 {0x0000008b, 0xff010000},
1729 {0x0000008d, 0xffffefff},
1730 {0x0000008e, 0xfff3efff},
1731 {0x0000008f, 0xfff3efbf},
1732 {0x00000092, 0xf7ffffff},
1733 {0x00000093, 0xffffff7f},
1734 {0x00000095, 0x00101101},
1735 {0x00000096, 0x00000fff},
1736 {0x00000097, 0x00116fff},
1737 {0x00000098, 0x60010000},
1738 {0x00000099, 0x10010000},
1739 {0x0000009a, 0x00006000},
1740 {0x0000009b, 0x00001000},
1741 {0x0000009f, 0x00b48000}
1744 #define HAWAII_IO_MC_REGS_SIZE 22
1746 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1748 {0x0000007d, 0x40000000},
1749 {0x0000007e, 0x40180304},
1750 {0x0000007f, 0x0000ff00},
1751 {0x00000081, 0x00000000},
1752 {0x00000083, 0x00000800},
1753 {0x00000086, 0x00000000},
1754 {0x00000087, 0x00000100},
1755 {0x00000088, 0x00020100},
1756 {0x00000089, 0x00000000},
1757 {0x0000008b, 0x00040000},
1758 {0x0000008c, 0x00000100},
1759 {0x0000008e, 0xff010000},
1760 {0x00000090, 0xffffefff},
1761 {0x00000091, 0xfff3efff},
1762 {0x00000092, 0xfff3efbf},
1763 {0x00000093, 0xf7ffffff},
1764 {0x00000094, 0xffffff7f},
1765 {0x00000095, 0x00000fff},
1766 {0x00000096, 0x00116fff},
1767 {0x00000097, 0x60010000},
1768 {0x00000098, 0x10010000},
1769 {0x0000009f, 0x00c79000}
1774 * cik_srbm_select - select specific register instances
1776 * @rdev: radeon_device pointer
1777 * @me: selected ME (micro engine)
1782 * Switches the currently active registers instances. Some
1783 * registers are instanced per VMID, others are instanced per
1784 * me/pipe/queue combination.
1786 static void cik_srbm_select(struct radeon_device *rdev,
1787 u32 me, u32 pipe, u32 queue, u32 vmid)
1789 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1792 QUEUEID(queue & 0x7));
1793 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1798 * ci_mc_load_microcode - load MC ucode into the hw
1800 * @rdev: radeon_device pointer
1802 * Load the GDDR MC ucode into the hw (CIK).
1803 * Returns 0 on success, error on failure.
1805 int ci_mc_load_microcode(struct radeon_device *rdev)
1807 const __be32 *fw_data = NULL;
1808 const __le32 *new_fw_data = NULL;
1809 u32 running, blackout = 0;
1810 u32 *io_mc_regs = NULL;
1811 const __le32 *new_io_mc_regs = NULL;
1812 int i, regs_size, ucode_size;
1818 const struct mc_firmware_header_v1_0 *hdr =
1819 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1821 radeon_ucode_print_mc_hdr(&hdr->header);
1823 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1824 new_io_mc_regs = (const __le32 *)
1825 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1826 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1827 new_fw_data = (const __le32 *)
1828 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1830 ucode_size = rdev->mc_fw->size / 4;
1832 switch (rdev->family) {
1834 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1835 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1838 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1839 regs_size = HAWAII_IO_MC_REGS_SIZE;
1844 fw_data = (const __be32 *)rdev->mc_fw->data;
1847 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1851 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1852 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1855 /* reset the engine and set to writable */
1856 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1857 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1859 /* load mc io regs */
1860 for (i = 0; i < regs_size; i++) {
1862 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1863 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1865 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1866 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1869 /* load the MC ucode */
1870 for (i = 0; i < ucode_size; i++) {
1872 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1874 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1877 /* put the engine back into the active state */
1878 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1879 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1880 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1882 /* wait for training to complete */
1883 for (i = 0; i < rdev->usec_timeout; i++) {
1884 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1888 for (i = 0; i < rdev->usec_timeout; i++) {
1889 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1895 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1902 * cik_init_microcode - load ucode images from disk
1904 * @rdev: radeon_device pointer
1906 * Use the firmware interface to load the ucode images into
1907 * the driver (not loaded into hw).
1908 * Returns 0 on success, error on failure.
1910 static int cik_init_microcode(struct radeon_device *rdev)
1912 const char *chip_name;
1913 const char *new_chip_name;
1914 size_t pfp_req_size, me_req_size, ce_req_size,
1915 mec_req_size, rlc_req_size, mc_req_size = 0,
1916 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1924 switch (rdev->family) {
1926 chip_name = "BONAIRE";
1927 new_chip_name = "bonaire";
1928 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1929 me_req_size = CIK_ME_UCODE_SIZE * 4;
1930 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1931 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1932 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1933 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1934 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1935 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1936 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1940 chip_name = "HAWAII";
1941 new_chip_name = "hawaii";
1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1948 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1954 chip_name = "KAVERI";
1955 new_chip_name = "kaveri";
1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1961 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1965 chip_name = "KABINI";
1966 new_chip_name = "kabini";
1967 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1968 me_req_size = CIK_ME_UCODE_SIZE * 4;
1969 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1970 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1971 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1972 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1976 chip_name = "MULLINS";
1977 new_chip_name = "mullins";
1978 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1979 me_req_size = CIK_ME_UCODE_SIZE * 4;
1980 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1981 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1982 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1983 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1989 DRM_INFO("Loading %s Microcode\n", new_chip_name);
1991 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1992 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1994 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1995 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1998 if (rdev->pfp_fw->size != pfp_req_size) {
2000 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2001 rdev->pfp_fw->size, fw_name);
2006 err = radeon_ucode_validate(rdev->pfp_fw);
2009 "cik_fw: validation failed for firmware \"%s\"\n",
2017 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2018 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2020 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2021 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2024 if (rdev->me_fw->size != me_req_size) {
2026 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2027 rdev->me_fw->size, fw_name);
2031 err = radeon_ucode_validate(rdev->me_fw);
2034 "cik_fw: validation failed for firmware \"%s\"\n",
2042 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2043 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2045 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2046 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2049 if (rdev->ce_fw->size != ce_req_size) {
2051 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2052 rdev->ce_fw->size, fw_name);
2056 err = radeon_ucode_validate(rdev->ce_fw);
2059 "cik_fw: validation failed for firmware \"%s\"\n",
2067 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2068 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2070 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2071 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2074 if (rdev->mec_fw->size != mec_req_size) {
2076 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2077 rdev->mec_fw->size, fw_name);
2081 err = radeon_ucode_validate(rdev->mec_fw);
2084 "cik_fw: validation failed for firmware \"%s\"\n",
2092 if (rdev->family == CHIP_KAVERI) {
2093 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2094 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2098 err = radeon_ucode_validate(rdev->mec2_fw);
2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2108 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2110 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2111 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2114 if (rdev->rlc_fw->size != rlc_req_size) {
2116 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2117 rdev->rlc_fw->size, fw_name);
2121 err = radeon_ucode_validate(rdev->rlc_fw);
2124 "cik_fw: validation failed for firmware \"%s\"\n",
2132 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2133 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2135 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2136 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2139 if (rdev->sdma_fw->size != sdma_req_size) {
2141 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2142 rdev->sdma_fw->size, fw_name);
2146 err = radeon_ucode_validate(rdev->sdma_fw);
2149 "cik_fw: validation failed for firmware \"%s\"\n",
2157 /* No SMC, MC ucode on APUs */
2158 if (!(rdev->flags & RADEON_IS_IGP)) {
2159 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2160 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2162 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2163 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2165 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2166 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2170 if ((rdev->mc_fw->size != mc_req_size) &&
2171 (rdev->mc_fw->size != mc2_req_size)){
2173 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2174 rdev->mc_fw->size, fw_name);
2177 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2179 err = radeon_ucode_validate(rdev->mc_fw);
2182 "cik_fw: validation failed for firmware \"%s\"\n",
2190 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2191 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2193 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2194 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2197 "smc: error loading firmware \"%s\"\n",
2199 release_firmware(rdev->smc_fw);
2200 rdev->smc_fw = NULL;
2202 } else if (rdev->smc_fw->size != smc_req_size) {
2204 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2205 rdev->smc_fw->size, fw_name);
2209 err = radeon_ucode_validate(rdev->smc_fw);
2212 "cik_fw: validation failed for firmware \"%s\"\n",
2222 rdev->new_fw = false;
2223 } else if (new_fw < num_fw) {
2224 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2227 rdev->new_fw = true;
2234 "cik_cp: Failed to load firmware \"%s\"\n",
2236 release_firmware(rdev->pfp_fw);
2237 rdev->pfp_fw = NULL;
2238 release_firmware(rdev->me_fw);
2240 release_firmware(rdev->ce_fw);
2242 release_firmware(rdev->mec_fw);
2243 rdev->mec_fw = NULL;
2244 release_firmware(rdev->mec2_fw);
2245 rdev->mec2_fw = NULL;
2246 release_firmware(rdev->rlc_fw);
2247 rdev->rlc_fw = NULL;
2248 release_firmware(rdev->sdma_fw);
2249 rdev->sdma_fw = NULL;
2250 release_firmware(rdev->mc_fw);
2252 release_firmware(rdev->smc_fw);
2253 rdev->smc_fw = NULL;
2262 * cik_tiling_mode_table_init - init the hw tiling table
2264 * @rdev: radeon_device pointer
2266 * Starting with SI, the tiling setup is done globally in a
2267 * set of 32 tiling modes. Rather than selecting each set of
2268 * parameters per surface as on older asics, we just select
2269 * which index in the tiling table we want to use, and the
2270 * surface uses those parameters (CIK).
2272 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2274 const u32 num_tile_mode_states = 32;
2275 const u32 num_secondary_tile_mode_states = 16;
2276 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2277 u32 num_pipe_configs;
2278 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2279 rdev->config.cik.max_shader_engines;
2281 switch (rdev->config.cik.mem_row_size_in_kb) {
2283 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2287 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2290 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2294 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2295 if (num_pipe_configs > 8)
2296 num_pipe_configs = 16;
2298 if (num_pipe_configs == 16) {
2299 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2300 switch (reg_offset) {
2302 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2303 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2304 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2305 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2310 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2311 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2314 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2316 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2322 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2326 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2327 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2328 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2329 TILE_SPLIT(split_equal_to_row_size));
2332 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2333 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2334 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2337 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2338 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2339 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2340 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2343 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2344 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2345 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346 TILE_SPLIT(split_equal_to_row_size));
2349 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2350 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2353 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2354 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2355 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2361 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2364 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2365 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2367 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2370 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2376 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2377 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2378 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2387 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2388 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2389 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2390 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2406 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2410 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2411 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2413 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2416 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2425 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2426 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2428 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2429 switch (reg_offset) {
2431 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2432 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2433 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2434 NUM_BANKS(ADDR_SURF_16_BANK));
2437 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2439 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2440 NUM_BANKS(ADDR_SURF_16_BANK));
2443 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2446 NUM_BANKS(ADDR_SURF_16_BANK));
2449 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2452 NUM_BANKS(ADDR_SURF_16_BANK));
2455 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2458 NUM_BANKS(ADDR_SURF_8_BANK));
2461 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464 NUM_BANKS(ADDR_SURF_4_BANK));
2467 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2470 NUM_BANKS(ADDR_SURF_2_BANK));
2473 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476 NUM_BANKS(ADDR_SURF_16_BANK));
2479 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2481 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2482 NUM_BANKS(ADDR_SURF_16_BANK));
2485 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488 NUM_BANKS(ADDR_SURF_16_BANK));
2491 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2494 NUM_BANKS(ADDR_SURF_8_BANK));
2497 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500 NUM_BANKS(ADDR_SURF_4_BANK));
2503 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506 NUM_BANKS(ADDR_SURF_2_BANK));
2509 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2511 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2512 NUM_BANKS(ADDR_SURF_2_BANK));
2518 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2519 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2521 } else if (num_pipe_configs == 8) {
2522 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2523 switch (reg_offset) {
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552 TILE_SPLIT(split_equal_to_row_size));
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2556 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2557 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2560 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2561 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2566 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2567 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2568 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569 TILE_SPLIT(split_equal_to_row_size));
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2600 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2601 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2607 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2611 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2613 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2616 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2623 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2624 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2627 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2628 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2629 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2630 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2633 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2635 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2648 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2649 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2651 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2652 switch (reg_offset) {
2654 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2657 NUM_BANKS(ADDR_SURF_16_BANK));
2660 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2663 NUM_BANKS(ADDR_SURF_16_BANK));
2666 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2669 NUM_BANKS(ADDR_SURF_16_BANK));
2672 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2675 NUM_BANKS(ADDR_SURF_16_BANK));
2678 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2681 NUM_BANKS(ADDR_SURF_8_BANK));
2684 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2685 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2686 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2687 NUM_BANKS(ADDR_SURF_4_BANK));
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2693 NUM_BANKS(ADDR_SURF_2_BANK));
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2735 NUM_BANKS(ADDR_SURF_2_BANK));
2741 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2742 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2744 } else if (num_pipe_configs == 4) {
2746 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2747 switch (reg_offset) {
2749 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2752 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2755 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2757 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2761 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2762 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2763 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2767 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2773 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2775 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2776 TILE_SPLIT(split_equal_to_row_size));
2779 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2780 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2781 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2793 TILE_SPLIT(split_equal_to_row_size));
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2797 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2800 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2801 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2805 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2808 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2811 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2812 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2817 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2818 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2819 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2823 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2824 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2828 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2830 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2831 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2834 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2836 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2840 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2841 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2842 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2846 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2847 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2848 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2851 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2852 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2853 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2854 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2857 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2859 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2865 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2872 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2873 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2875 } else if (num_rbs < 4) {
2876 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2877 switch (reg_offset) {
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2882 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2897 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2900 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2903 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2906 TILE_SPLIT(split_equal_to_row_size));
2909 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2910 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2911 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2914 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2915 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2916 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2917 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2920 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2923 TILE_SPLIT(split_equal_to_row_size));
2926 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2927 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2930 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2931 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2932 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2935 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2941 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2942 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2943 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2944 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2947 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2953 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2954 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2955 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2958 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2959 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2961 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2964 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2965 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2966 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2967 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2970 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2973 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2976 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2977 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2978 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2981 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2982 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2983 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2984 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2987 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2988 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2990 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2993 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2996 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3002 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3003 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3006 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3007 switch (reg_offset) {
3009 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3010 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3011 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3012 NUM_BANKS(ADDR_SURF_16_BANK));
3015 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018 NUM_BANKS(ADDR_SURF_16_BANK));
3021 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3023 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3024 NUM_BANKS(ADDR_SURF_16_BANK));
3027 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3030 NUM_BANKS(ADDR_SURF_16_BANK));
3033 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3034 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3035 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3036 NUM_BANKS(ADDR_SURF_16_BANK));
3039 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3040 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3041 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3042 NUM_BANKS(ADDR_SURF_8_BANK));
3045 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3048 NUM_BANKS(ADDR_SURF_4_BANK));
3051 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3052 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3053 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3054 NUM_BANKS(ADDR_SURF_16_BANK));
3057 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3058 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3059 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3060 NUM_BANKS(ADDR_SURF_16_BANK));
3063 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3064 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3065 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3066 NUM_BANKS(ADDR_SURF_16_BANK));
3069 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3070 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3071 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3072 NUM_BANKS(ADDR_SURF_16_BANK));
3075 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3076 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3077 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3078 NUM_BANKS(ADDR_SURF_16_BANK));
3081 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3083 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3084 NUM_BANKS(ADDR_SURF_8_BANK));
3087 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3089 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3090 NUM_BANKS(ADDR_SURF_4_BANK));
3096 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3097 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3099 } else if (num_pipe_configs == 2) {
3100 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3101 switch (reg_offset) {
3103 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3104 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3105 PIPE_CONFIG(ADDR_SURF_P2) |
3106 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3109 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3111 PIPE_CONFIG(ADDR_SURF_P2) |
3112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3115 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3116 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3117 PIPE_CONFIG(ADDR_SURF_P2) |
3118 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3121 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3122 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3123 PIPE_CONFIG(ADDR_SURF_P2) |
3124 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3127 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3128 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3129 PIPE_CONFIG(ADDR_SURF_P2) |
3130 TILE_SPLIT(split_equal_to_row_size));
3133 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3134 PIPE_CONFIG(ADDR_SURF_P2) |
3135 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3138 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3139 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3140 PIPE_CONFIG(ADDR_SURF_P2) |
3141 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3144 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3145 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3146 PIPE_CONFIG(ADDR_SURF_P2) |
3147 TILE_SPLIT(split_equal_to_row_size));
3150 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3151 PIPE_CONFIG(ADDR_SURF_P2);
3154 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3155 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3156 PIPE_CONFIG(ADDR_SURF_P2));
3159 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3160 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3161 PIPE_CONFIG(ADDR_SURF_P2) |
3162 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3165 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3166 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3167 PIPE_CONFIG(ADDR_SURF_P2) |
3168 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3171 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3172 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3173 PIPE_CONFIG(ADDR_SURF_P2) |
3174 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3177 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3178 PIPE_CONFIG(ADDR_SURF_P2) |
3179 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3182 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3183 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3184 PIPE_CONFIG(ADDR_SURF_P2) |
3185 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3188 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3189 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3190 PIPE_CONFIG(ADDR_SURF_P2) |
3191 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3194 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3195 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3196 PIPE_CONFIG(ADDR_SURF_P2) |
3197 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3200 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3201 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3202 PIPE_CONFIG(ADDR_SURF_P2));
3205 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3206 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3207 PIPE_CONFIG(ADDR_SURF_P2) |
3208 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3211 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3212 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3213 PIPE_CONFIG(ADDR_SURF_P2) |
3214 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3217 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3218 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3219 PIPE_CONFIG(ADDR_SURF_P2) |
3220 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3226 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3227 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3229 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3230 switch (reg_offset) {
3232 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3233 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3234 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3235 NUM_BANKS(ADDR_SURF_16_BANK));
3238 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3239 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3240 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3241 NUM_BANKS(ADDR_SURF_16_BANK));
3244 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3247 NUM_BANKS(ADDR_SURF_16_BANK));
3250 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3251 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3252 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3253 NUM_BANKS(ADDR_SURF_16_BANK));
3256 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3259 NUM_BANKS(ADDR_SURF_16_BANK));
3262 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3263 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3264 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3265 NUM_BANKS(ADDR_SURF_16_BANK));
3268 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3271 NUM_BANKS(ADDR_SURF_8_BANK));
3274 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3275 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3276 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3277 NUM_BANKS(ADDR_SURF_16_BANK));
3280 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3281 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3282 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3283 NUM_BANKS(ADDR_SURF_16_BANK));
3286 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3289 NUM_BANKS(ADDR_SURF_16_BANK));
3292 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3293 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3294 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3295 NUM_BANKS(ADDR_SURF_16_BANK));
3298 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3299 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3300 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3301 NUM_BANKS(ADDR_SURF_16_BANK));
3304 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3305 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3306 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307 NUM_BANKS(ADDR_SURF_16_BANK));
3310 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3312 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3313 NUM_BANKS(ADDR_SURF_8_BANK));
3319 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3320 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3323 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3327 * cik_select_se_sh - select which SE, SH to address
3329 * @rdev: radeon_device pointer
3330 * @se_num: shader engine to address
3331 * @sh_num: sh block to address
3333 * Select which SE, SH combinations to address. Certain
3334 * registers are instanced per SE or SH. 0xffffffff means
3335 * broadcast to all SEs or SHs (CIK).
3337 static void cik_select_se_sh(struct radeon_device *rdev,
3338 u32 se_num, u32 sh_num)
3340 u32 data = INSTANCE_BROADCAST_WRITES;
3342 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3343 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3344 else if (se_num == 0xffffffff)
3345 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3346 else if (sh_num == 0xffffffff)
3347 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3349 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3350 WREG32(GRBM_GFX_INDEX, data);
3354 * cik_create_bitmask - create a bitmask
3356 * @bit_width: length of the mask
3358 * create a variable length bit mask (CIK).
3359 * Returns the bitmask.
3361 static u32 cik_create_bitmask(u32 bit_width)
3365 for (i = 0; i < bit_width; i++) {
3373 * cik_get_rb_disabled - computes the mask of disabled RBs
3375 * @rdev: radeon_device pointer
3376 * @max_rb_num: max RBs (render backends) for the asic
3377 * @se_num: number of SEs (shader engines) for the asic
3378 * @sh_per_se: number of SH blocks per SE for the asic
3380 * Calculates the bitmask of disabled RBs (CIK).
3381 * Returns the disabled RB bitmask.
3383 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3384 u32 max_rb_num_per_se,
3389 data = RREG32(CC_RB_BACKEND_DISABLE);
3391 data &= BACKEND_DISABLE_MASK;
3394 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3396 data >>= BACKEND_DISABLE_SHIFT;
3398 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3404 * cik_setup_rb - setup the RBs on the asic
3406 * @rdev: radeon_device pointer
3407 * @se_num: number of SEs (shader engines) for the asic
3408 * @sh_per_se: number of SH blocks per SE for the asic
3409 * @max_rb_num: max RBs (render backends) for the asic
3411 * Configures per-SE/SH RB registers (CIK).
3413 static void cik_setup_rb(struct radeon_device *rdev,
3414 u32 se_num, u32 sh_per_se,
3415 u32 max_rb_num_per_se)
3419 u32 disabled_rbs = 0;
3420 u32 enabled_rbs = 0;
3422 for (i = 0; i < se_num; i++) {
3423 for (j = 0; j < sh_per_se; j++) {
3424 cik_select_se_sh(rdev, i, j);
3425 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3426 if (rdev->family == CHIP_HAWAII)
3427 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3429 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3432 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3435 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3436 if (!(disabled_rbs & mask))
3437 enabled_rbs |= mask;
3441 rdev->config.cik.backend_enable_mask = enabled_rbs;
3443 for (i = 0; i < se_num; i++) {
3444 cik_select_se_sh(rdev, i, 0xffffffff);
3446 for (j = 0; j < sh_per_se; j++) {
3447 switch (enabled_rbs & 3) {
3450 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3452 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3455 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3458 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3462 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3467 WREG32(PA_SC_RASTER_CONFIG, data);
3469 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3473 * cik_gpu_init - setup the 3D engine
3475 * @rdev: radeon_device pointer
3477 * Configures the 3D engine and tiling configuration
3478 * registers so that the 3D engine is usable.
3480 static void cik_gpu_init(struct radeon_device *rdev)
3482 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3483 u32 mc_shared_chmap, mc_arb_ramcfg;
3484 u32 hdp_host_path_cntl;
3488 switch (rdev->family) {
3490 rdev->config.cik.max_shader_engines = 2;
3491 rdev->config.cik.max_tile_pipes = 4;
3492 rdev->config.cik.max_cu_per_sh = 7;
3493 rdev->config.cik.max_sh_per_se = 1;
3494 rdev->config.cik.max_backends_per_se = 2;
3495 rdev->config.cik.max_texture_channel_caches = 4;
3496 rdev->config.cik.max_gprs = 256;
3497 rdev->config.cik.max_gs_threads = 32;
3498 rdev->config.cik.max_hw_contexts = 8;
3500 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3501 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3502 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3503 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3504 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3507 rdev->config.cik.max_shader_engines = 4;
3508 rdev->config.cik.max_tile_pipes = 16;
3509 rdev->config.cik.max_cu_per_sh = 11;
3510 rdev->config.cik.max_sh_per_se = 1;
3511 rdev->config.cik.max_backends_per_se = 4;
3512 rdev->config.cik.max_texture_channel_caches = 16;
3513 rdev->config.cik.max_gprs = 256;
3514 rdev->config.cik.max_gs_threads = 32;
3515 rdev->config.cik.max_hw_contexts = 8;
3517 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3518 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3519 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3520 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3521 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3524 rdev->config.cik.max_shader_engines = 1;
3525 rdev->config.cik.max_tile_pipes = 4;
3526 if ((rdev->pdev->device == 0x1304) ||
3527 (rdev->pdev->device == 0x1305) ||
3528 (rdev->pdev->device == 0x130C) ||
3529 (rdev->pdev->device == 0x130F) ||
3530 (rdev->pdev->device == 0x1310) ||
3531 (rdev->pdev->device == 0x1311) ||
3532 (rdev->pdev->device == 0x131C)) {
3533 rdev->config.cik.max_cu_per_sh = 8;
3534 rdev->config.cik.max_backends_per_se = 2;
3535 } else if ((rdev->pdev->device == 0x1309) ||
3536 (rdev->pdev->device == 0x130A) ||
3537 (rdev->pdev->device == 0x130D) ||
3538 (rdev->pdev->device == 0x1313) ||
3539 (rdev->pdev->device == 0x131D)) {
3540 rdev->config.cik.max_cu_per_sh = 6;
3541 rdev->config.cik.max_backends_per_se = 2;
3542 } else if ((rdev->pdev->device == 0x1306) ||
3543 (rdev->pdev->device == 0x1307) ||
3544 (rdev->pdev->device == 0x130B) ||
3545 (rdev->pdev->device == 0x130E) ||
3546 (rdev->pdev->device == 0x1315) ||
3547 (rdev->pdev->device == 0x131B)) {
3548 rdev->config.cik.max_cu_per_sh = 4;
3549 rdev->config.cik.max_backends_per_se = 1;
3551 rdev->config.cik.max_cu_per_sh = 3;
3552 rdev->config.cik.max_backends_per_se = 1;
3554 rdev->config.cik.max_sh_per_se = 1;
3555 rdev->config.cik.max_texture_channel_caches = 4;
3556 rdev->config.cik.max_gprs = 256;
3557 rdev->config.cik.max_gs_threads = 16;
3558 rdev->config.cik.max_hw_contexts = 8;
3560 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3561 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3562 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3563 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3564 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3569 rdev->config.cik.max_shader_engines = 1;
3570 rdev->config.cik.max_tile_pipes = 2;
3571 rdev->config.cik.max_cu_per_sh = 2;
3572 rdev->config.cik.max_sh_per_se = 1;
3573 rdev->config.cik.max_backends_per_se = 1;
3574 rdev->config.cik.max_texture_channel_caches = 2;
3575 rdev->config.cik.max_gprs = 256;
3576 rdev->config.cik.max_gs_threads = 16;
3577 rdev->config.cik.max_hw_contexts = 8;
3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3587 /* Initialize HDP */
3588 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3589 WREG32((0x2c14 + j), 0x00000000);
3590 WREG32((0x2c18 + j), 0x00000000);
3591 WREG32((0x2c1c + j), 0x00000000);
3592 WREG32((0x2c20 + j), 0x00000000);
3593 WREG32((0x2c24 + j), 0x00000000);
3596 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3598 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3600 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3601 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3603 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3604 rdev->config.cik.mem_max_burst_length_bytes = 256;
3605 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3606 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3607 if (rdev->config.cik.mem_row_size_in_kb > 4)
3608 rdev->config.cik.mem_row_size_in_kb = 4;
3609 /* XXX use MC settings? */
3610 rdev->config.cik.shader_engine_tile_size = 32;
3611 rdev->config.cik.num_gpus = 1;
3612 rdev->config.cik.multi_gpu_tile_size = 64;
3614 /* fix up row size */
3615 gb_addr_config &= ~ROW_SIZE_MASK;
3616 switch (rdev->config.cik.mem_row_size_in_kb) {
3619 gb_addr_config |= ROW_SIZE(0);
3622 gb_addr_config |= ROW_SIZE(1);
3625 gb_addr_config |= ROW_SIZE(2);
3629 /* setup tiling info dword. gb_addr_config is not adequate since it does
3630 * not have bank info, so create a custom tiling dword.
3631 * bits 3:0 num_pipes
3632 * bits 7:4 num_banks
3633 * bits 11:8 group_size
3634 * bits 15:12 row_size
3636 rdev->config.cik.tile_config = 0;
3637 switch (rdev->config.cik.num_tile_pipes) {
3639 rdev->config.cik.tile_config |= (0 << 0);
3642 rdev->config.cik.tile_config |= (1 << 0);
3645 rdev->config.cik.tile_config |= (2 << 0);
3649 /* XXX what about 12? */
3650 rdev->config.cik.tile_config |= (3 << 0);
3653 rdev->config.cik.tile_config |=
3654 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3655 rdev->config.cik.tile_config |=
3656 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3657 rdev->config.cik.tile_config |=
3658 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3660 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3661 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3662 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3663 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3664 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3665 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3666 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3667 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3669 cik_tiling_mode_table_init(rdev);
3671 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3672 rdev->config.cik.max_sh_per_se,
3673 rdev->config.cik.max_backends_per_se);
3675 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3676 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3677 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k++) {
3678 rdev->config.cik.active_cus +=
3679 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3684 /* set HW defaults for 3D engine */
3685 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3687 WREG32(SX_DEBUG_1, 0x20);
3689 WREG32(TA_CNTL_AUX, 0x00010000);
3691 tmp = RREG32(SPI_CONFIG_CNTL);
3693 WREG32(SPI_CONFIG_CNTL, tmp);
3695 WREG32(SQ_CONFIG, 1);
3697 WREG32(DB_DEBUG, 0);
3699 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3701 WREG32(DB_DEBUG2, tmp);
3703 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3705 WREG32(DB_DEBUG3, tmp);
3707 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3709 WREG32(CB_HW_CONTROL, tmp);
3711 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3713 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3714 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3715 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3716 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3718 WREG32(VGT_NUM_INSTANCES, 1);
3720 WREG32(CP_PERFMON_CNTL, 0);
3722 WREG32(SQ_CONFIG, 0);
3724 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3725 FORCE_EOV_MAX_REZ_CNT(255)));
3727 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3728 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3730 WREG32(VGT_GS_VERTEX_REUSE, 16);
3731 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3733 tmp = RREG32(HDP_MISC_CNTL);
3734 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3735 WREG32(HDP_MISC_CNTL, tmp);
3737 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3738 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3740 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3741 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3747 * GPU scratch registers helpers function.
3750 * cik_scratch_init - setup driver info for CP scratch regs
3752 * @rdev: radeon_device pointer
3754 * Set up the number and offset of the CP scratch registers.
3755 * NOTE: use of CP scratch registers is a legacy inferface and
3756 * is not used by default on newer asics (r6xx+). On newer asics,
3757 * memory buffers are used for fences rather than scratch regs.
3759 static void cik_scratch_init(struct radeon_device *rdev)
3763 rdev->scratch.num_reg = 7;
3764 rdev->scratch.reg_base = SCRATCH_REG0;
3765 for (i = 0; i < rdev->scratch.num_reg; i++) {
3766 rdev->scratch.free[i] = true;
3767 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3772 * cik_ring_test - basic gfx ring test
3774 * @rdev: radeon_device pointer
3775 * @ring: radeon_ring structure holding ring information
3777 * Allocate a scratch register and write to it using the gfx ring (CIK).
3778 * Provides a basic gfx ring test to verify that the ring is working.
3779 * Used by cik_cp_gfx_resume();
3780 * Returns 0 on success, error on failure.
3782 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3789 r = radeon_scratch_get(rdev, &scratch);
3791 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3794 WREG32(scratch, 0xCAFEDEAD);
3795 r = radeon_ring_lock(rdev, ring, 3);
3797 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3798 radeon_scratch_free(rdev, scratch);
3801 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3802 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3803 radeon_ring_write(ring, 0xDEADBEEF);
3804 radeon_ring_unlock_commit(rdev, ring);
3806 for (i = 0; i < rdev->usec_timeout; i++) {
3807 tmp = RREG32(scratch);
3808 if (tmp == 0xDEADBEEF)
3812 if (i < rdev->usec_timeout) {
3813 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3815 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3816 ring->idx, scratch, tmp);
3819 radeon_scratch_free(rdev, scratch);
3824 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3826 * @rdev: radeon_device pointer
3827 * @ridx: radeon ring index
3829 * Emits an hdp flush on the cp.
3831 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3834 struct radeon_ring *ring = &rdev->ring[ridx];
3837 switch (ring->idx) {
3838 case CAYMAN_RING_TYPE_CP1_INDEX:
3839 case CAYMAN_RING_TYPE_CP2_INDEX:
3843 ref_and_mask = CP2 << ring->pipe;
3846 ref_and_mask = CP6 << ring->pipe;
3852 case RADEON_RING_TYPE_GFX_INDEX:
3857 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3858 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3859 WAIT_REG_MEM_FUNCTION(3) | /* == */
3860 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3861 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3862 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3863 radeon_ring_write(ring, ref_and_mask);
3864 radeon_ring_write(ring, ref_and_mask);
3865 radeon_ring_write(ring, 0x20); /* poll interval */
3869 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3871 * @rdev: radeon_device pointer
3872 * @fence: radeon fence object
3874 * Emits a fence sequnce number on the gfx ring and flushes
3877 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3878 struct radeon_fence *fence)
3880 struct radeon_ring *ring = &rdev->ring[fence->ring];
3881 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3883 /* EVENT_WRITE_EOP - flush caches, send int */
3884 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3885 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3887 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3889 radeon_ring_write(ring, addr & 0xfffffffc);
3890 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3891 radeon_ring_write(ring, fence->seq);
3892 radeon_ring_write(ring, 0);
3896 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3898 * @rdev: radeon_device pointer
3899 * @fence: radeon fence object
3901 * Emits a fence sequnce number on the compute ring and flushes
3904 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3905 struct radeon_fence *fence)
3907 struct radeon_ring *ring = &rdev->ring[fence->ring];
3908 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3910 /* RELEASE_MEM - flush caches, send int */
3911 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3912 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3914 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3916 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3917 radeon_ring_write(ring, addr & 0xfffffffc);
3918 radeon_ring_write(ring, upper_32_bits(addr));
3919 radeon_ring_write(ring, fence->seq);
3920 radeon_ring_write(ring, 0);
3923 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3924 struct radeon_ring *ring,
3925 struct radeon_semaphore *semaphore,
3928 uint64_t addr = semaphore->gpu_addr;
3929 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3931 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3932 radeon_ring_write(ring, lower_32_bits(addr));
3933 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3939 * cik_copy_cpdma - copy pages using the CP DMA engine
3941 * @rdev: radeon_device pointer
3942 * @src_offset: src GPU address
3943 * @dst_offset: dst GPU address
3944 * @num_gpu_pages: number of GPU pages to xfer
3945 * @fence: radeon fence object
3947 * Copy GPU paging using the CP DMA engine (CIK+).
3948 * Used by the radeon ttm implementation to move pages if
3949 * registered as the asic copy callback.
3951 int cik_copy_cpdma(struct radeon_device *rdev,
3952 uint64_t src_offset, uint64_t dst_offset,
3953 unsigned num_gpu_pages,
3954 struct radeon_fence **fence)
3956 struct radeon_semaphore *sem = NULL;
3957 int ring_index = rdev->asic->copy.blit_ring_index;
3958 struct radeon_ring *ring = &rdev->ring[ring_index];
3959 u32 size_in_bytes, cur_size_in_bytes, control;
3963 r = radeon_semaphore_create(rdev, &sem);
3965 DRM_ERROR("radeon: moving bo (%d).\n", r);
3969 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3970 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3971 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3973 DRM_ERROR("radeon: moving bo (%d).\n", r);
3974 radeon_semaphore_free(rdev, &sem, NULL);
3978 radeon_semaphore_sync_to(sem, *fence);
3979 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3981 for (i = 0; i < num_loops; i++) {
3982 cur_size_in_bytes = size_in_bytes;
3983 if (cur_size_in_bytes > 0x1fffff)
3984 cur_size_in_bytes = 0x1fffff;
3985 size_in_bytes -= cur_size_in_bytes;
3987 if (size_in_bytes == 0)
3988 control |= PACKET3_DMA_DATA_CP_SYNC;
3989 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3990 radeon_ring_write(ring, control);
3991 radeon_ring_write(ring, lower_32_bits(src_offset));
3992 radeon_ring_write(ring, upper_32_bits(src_offset));
3993 radeon_ring_write(ring, lower_32_bits(dst_offset));
3994 radeon_ring_write(ring, upper_32_bits(dst_offset));
3995 radeon_ring_write(ring, cur_size_in_bytes);
3996 src_offset += cur_size_in_bytes;
3997 dst_offset += cur_size_in_bytes;
4000 r = radeon_fence_emit(rdev, fence, ring->idx);
4002 radeon_ring_unlock_undo(rdev, ring);
4003 radeon_semaphore_free(rdev, &sem, NULL);
4007 radeon_ring_unlock_commit(rdev, ring);
4008 radeon_semaphore_free(rdev, &sem, *fence);
4017 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4019 * @rdev: radeon_device pointer
4020 * @ib: radeon indirect buffer object
4022 * Emits an DE (drawing engine) or CE (constant engine) IB
4023 * on the gfx ring. IBs are usually generated by userspace
4024 * acceleration drivers and submitted to the kernel for
4025 * sheduling on the ring. This function schedules the IB
4026 * on the gfx ring for execution by the GPU.
4028 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4030 struct radeon_ring *ring = &rdev->ring[ib->ring];
4031 u32 header, control = INDIRECT_BUFFER_VALID;
4033 if (ib->is_const_ib) {
4034 /* set switch buffer packet before const IB */
4035 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4036 radeon_ring_write(ring, 0);
4038 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4041 if (ring->rptr_save_reg) {
4042 next_rptr = ring->wptr + 3 + 4;
4043 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4044 radeon_ring_write(ring, ((ring->rptr_save_reg -
4045 PACKET3_SET_UCONFIG_REG_START) >> 2));
4046 radeon_ring_write(ring, next_rptr);
4047 } else if (rdev->wb.enabled) {
4048 next_rptr = ring->wptr + 5 + 4;
4049 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4050 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4051 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4052 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4053 radeon_ring_write(ring, next_rptr);
4056 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4059 control |= ib->length_dw |
4060 (ib->vm ? (ib->vm->id << 24) : 0);
4062 radeon_ring_write(ring, header);
4063 radeon_ring_write(ring,
4067 (ib->gpu_addr & 0xFFFFFFFC));
4068 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4069 radeon_ring_write(ring, control);
4073 * cik_ib_test - basic gfx ring IB test
4075 * @rdev: radeon_device pointer
4076 * @ring: radeon_ring structure holding ring information
4078 * Allocate an IB and execute it on the gfx ring (CIK).
4079 * Provides a basic gfx ring test to verify that IBs are working.
4080 * Returns 0 on success, error on failure.
4082 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4084 struct radeon_ib ib;
4090 r = radeon_scratch_get(rdev, &scratch);
4092 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4095 WREG32(scratch, 0xCAFEDEAD);
4096 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4098 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4099 radeon_scratch_free(rdev, scratch);
4102 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4103 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4104 ib.ptr[2] = 0xDEADBEEF;
4106 r = radeon_ib_schedule(rdev, &ib, NULL);
4108 radeon_scratch_free(rdev, scratch);
4109 radeon_ib_free(rdev, &ib);
4110 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4113 r = radeon_fence_wait(ib.fence, false);
4115 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4116 radeon_scratch_free(rdev, scratch);
4117 radeon_ib_free(rdev, &ib);
4120 for (i = 0; i < rdev->usec_timeout; i++) {
4121 tmp = RREG32(scratch);
4122 if (tmp == 0xDEADBEEF)
4126 if (i < rdev->usec_timeout) {
4127 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4129 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4133 radeon_scratch_free(rdev, scratch);
4134 radeon_ib_free(rdev, &ib);
4140 * On CIK, gfx and compute now have independant command processors.
4143 * Gfx consists of a single ring and can process both gfx jobs and
4144 * compute jobs. The gfx CP consists of three microengines (ME):
4145 * PFP - Pre-Fetch Parser
4147 * CE - Constant Engine
4148 * The PFP and ME make up what is considered the Drawing Engine (DE).
4149 * The CE is an asynchronous engine used for updating buffer desciptors
4150 * used by the DE so that they can be loaded into cache in parallel
4151 * while the DE is processing state update packets.
4154 * The compute CP consists of two microengines (ME):
4155 * MEC1 - Compute MicroEngine 1
4156 * MEC2 - Compute MicroEngine 2
4157 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4158 * The queues are exposed to userspace and are programmed directly
4159 * by the compute runtime.
4162 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4164 * @rdev: radeon_device pointer
4165 * @enable: enable or disable the MEs
4167 * Halts or unhalts the gfx MEs.
4169 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4172 WREG32(CP_ME_CNTL, 0);
4174 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4175 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4176 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4177 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4183 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4185 * @rdev: radeon_device pointer
4187 * Loads the gfx PFP, ME, and CE ucode.
4188 * Returns 0 for success, -EINVAL if the ucode is not available.
4190 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4194 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4197 cik_cp_gfx_enable(rdev, false);
4200 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4201 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4202 const struct gfx_firmware_header_v1_0 *ce_hdr =
4203 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4204 const struct gfx_firmware_header_v1_0 *me_hdr =
4205 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4206 const __le32 *fw_data;
4209 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4210 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4211 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4214 fw_data = (const __le32 *)
4215 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4216 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4217 WREG32(CP_PFP_UCODE_ADDR, 0);
4218 for (i = 0; i < fw_size; i++)
4219 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4220 WREG32(CP_PFP_UCODE_ADDR, 0);
4223 fw_data = (const __le32 *)
4224 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4225 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4226 WREG32(CP_CE_UCODE_ADDR, 0);
4227 for (i = 0; i < fw_size; i++)
4228 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4229 WREG32(CP_CE_UCODE_ADDR, 0);
4232 fw_data = (const __be32 *)
4233 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4234 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4235 WREG32(CP_ME_RAM_WADDR, 0);
4236 for (i = 0; i < fw_size; i++)
4237 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4238 WREG32(CP_ME_RAM_WADDR, 0);
4240 const __be32 *fw_data;
4243 fw_data = (const __be32 *)rdev->pfp_fw->data;
4244 WREG32(CP_PFP_UCODE_ADDR, 0);
4245 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4246 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4247 WREG32(CP_PFP_UCODE_ADDR, 0);
4250 fw_data = (const __be32 *)rdev->ce_fw->data;
4251 WREG32(CP_CE_UCODE_ADDR, 0);
4252 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4253 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4254 WREG32(CP_CE_UCODE_ADDR, 0);
4257 fw_data = (const __be32 *)rdev->me_fw->data;
4258 WREG32(CP_ME_RAM_WADDR, 0);
4259 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4260 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4261 WREG32(CP_ME_RAM_WADDR, 0);
4264 WREG32(CP_PFP_UCODE_ADDR, 0);
4265 WREG32(CP_CE_UCODE_ADDR, 0);
4266 WREG32(CP_ME_RAM_WADDR, 0);
4267 WREG32(CP_ME_RAM_RADDR, 0);
4272 * cik_cp_gfx_start - start the gfx ring
4274 * @rdev: radeon_device pointer
4276 * Enables the ring and loads the clear state context and other
4277 * packets required to init the ring.
4278 * Returns 0 for success, error for failure.
4280 static int cik_cp_gfx_start(struct radeon_device *rdev)
4282 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4286 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4287 WREG32(CP_ENDIAN_SWAP, 0);
4288 WREG32(CP_DEVICE_ID, 1);
4290 cik_cp_gfx_enable(rdev, true);
4292 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4294 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4298 /* init the CE partitions. CE only used for gfx on CIK */
4299 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4300 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4301 radeon_ring_write(ring, 0xc000);
4302 radeon_ring_write(ring, 0xc000);
4304 /* setup clear context state */
4305 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4306 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4308 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4309 radeon_ring_write(ring, 0x80000000);
4310 radeon_ring_write(ring, 0x80000000);
4312 for (i = 0; i < cik_default_size; i++)
4313 radeon_ring_write(ring, cik_default_state[i]);
4315 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4316 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4318 /* set clear context state */
4319 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4320 radeon_ring_write(ring, 0);
4322 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4323 radeon_ring_write(ring, 0x00000316);
4324 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4325 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4327 radeon_ring_unlock_commit(rdev, ring);
4333 * cik_cp_gfx_fini - stop the gfx ring
4335 * @rdev: radeon_device pointer
4337 * Stop the gfx ring and tear down the driver ring
4340 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4342 cik_cp_gfx_enable(rdev, false);
4343 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4347 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4349 * @rdev: radeon_device pointer
4351 * Program the location and size of the gfx ring buffer
4352 * and test it to make sure it's working.
4353 * Returns 0 for success, error for failure.
4355 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4357 struct radeon_ring *ring;
4363 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4364 if (rdev->family != CHIP_HAWAII)
4365 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4367 /* Set the write pointer delay */
4368 WREG32(CP_RB_WPTR_DELAY, 0);
4370 /* set the RB to use vmid 0 */
4371 WREG32(CP_RB_VMID, 0);
4373 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4375 /* ring 0 - compute and gfx */
4376 /* Set ring buffer size */
4377 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4378 rb_bufsz = order_base_2(ring->ring_size / 8);
4379 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4381 tmp |= BUF_SWAP_32BIT;
4383 WREG32(CP_RB0_CNTL, tmp);
4385 /* Initialize the ring buffer's read and write pointers */
4386 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4388 WREG32(CP_RB0_WPTR, ring->wptr);
4390 /* set the wb address wether it's enabled or not */
4391 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4392 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4394 /* scratch register shadowing is no longer supported */
4395 WREG32(SCRATCH_UMSK, 0);
4397 if (!rdev->wb.enabled)
4398 tmp |= RB_NO_UPDATE;
4401 WREG32(CP_RB0_CNTL, tmp);
4403 rb_addr = ring->gpu_addr >> 8;
4404 WREG32(CP_RB0_BASE, rb_addr);
4405 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4407 /* start the ring */
4408 cik_cp_gfx_start(rdev);
4409 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4410 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4412 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4416 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4417 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4422 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4423 struct radeon_ring *ring)
4427 if (rdev->wb.enabled)
4428 rptr = rdev->wb.wb[ring->rptr_offs/4];
4430 rptr = RREG32(CP_RB0_RPTR);
4435 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4436 struct radeon_ring *ring)
4440 wptr = RREG32(CP_RB0_WPTR);
4445 void cik_gfx_set_wptr(struct radeon_device *rdev,
4446 struct radeon_ring *ring)
4448 WREG32(CP_RB0_WPTR, ring->wptr);
4449 (void)RREG32(CP_RB0_WPTR);
4452 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4453 struct radeon_ring *ring)
4457 if (rdev->wb.enabled) {
4458 rptr = rdev->wb.wb[ring->rptr_offs/4];
4460 mutex_lock(&rdev->srbm_mutex);
4461 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4462 rptr = RREG32(CP_HQD_PQ_RPTR);
4463 cik_srbm_select(rdev, 0, 0, 0, 0);
4464 mutex_unlock(&rdev->srbm_mutex);
4470 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4471 struct radeon_ring *ring)
4475 if (rdev->wb.enabled) {
4476 /* XXX check if swapping is necessary on BE */
4477 wptr = rdev->wb.wb[ring->wptr_offs/4];
4479 mutex_lock(&rdev->srbm_mutex);
4480 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4481 wptr = RREG32(CP_HQD_PQ_WPTR);
4482 cik_srbm_select(rdev, 0, 0, 0, 0);
4483 mutex_unlock(&rdev->srbm_mutex);
4489 void cik_compute_set_wptr(struct radeon_device *rdev,
4490 struct radeon_ring *ring)
4492 /* XXX check if swapping is necessary on BE */
4493 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4494 WDOORBELL32(ring->doorbell_index, ring->wptr);
4498 * cik_cp_compute_enable - enable/disable the compute CP MEs
4500 * @rdev: radeon_device pointer
4501 * @enable: enable or disable the MEs
4503 * Halts or unhalts the compute MEs.
4505 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4508 WREG32(CP_MEC_CNTL, 0);
4510 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4511 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4512 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4518 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4520 * @rdev: radeon_device pointer
4522 * Loads the compute MEC1&2 ucode.
4523 * Returns 0 for success, -EINVAL if the ucode is not available.
4525 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4532 cik_cp_compute_enable(rdev, false);
4535 const struct gfx_firmware_header_v1_0 *mec_hdr =
4536 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4537 const __le32 *fw_data;
4540 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4543 fw_data = (const __le32 *)
4544 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4545 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4546 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4547 for (i = 0; i < fw_size; i++)
4548 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4549 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4552 if (rdev->family == CHIP_KAVERI) {
4553 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4554 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4556 fw_data = (const __le32 *)
4557 (rdev->mec2_fw->data +
4558 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4559 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4560 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4561 for (i = 0; i < fw_size; i++)
4562 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4563 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4566 const __be32 *fw_data;
4569 fw_data = (const __be32 *)rdev->mec_fw->data;
4570 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4571 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4572 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4573 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4575 if (rdev->family == CHIP_KAVERI) {
4577 fw_data = (const __be32 *)rdev->mec_fw->data;
4578 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4579 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4580 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4581 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4589 * cik_cp_compute_start - start the compute queues
4591 * @rdev: radeon_device pointer
4593 * Enable the compute queues.
4594 * Returns 0 for success, error for failure.
4596 static int cik_cp_compute_start(struct radeon_device *rdev)
4598 cik_cp_compute_enable(rdev, true);
4604 * cik_cp_compute_fini - stop the compute queues
4606 * @rdev: radeon_device pointer
4608 * Stop the compute queues and tear down the driver queue
4611 static void cik_cp_compute_fini(struct radeon_device *rdev)
4615 cik_cp_compute_enable(rdev, false);
4617 for (i = 0; i < 2; i++) {
4619 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4621 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4623 if (rdev->ring[idx].mqd_obj) {
4624 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4625 if (unlikely(r != 0))
4626 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4628 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4629 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4631 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4632 rdev->ring[idx].mqd_obj = NULL;
4637 static void cik_mec_fini(struct radeon_device *rdev)
4641 if (rdev->mec.hpd_eop_obj) {
4642 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4643 if (unlikely(r != 0))
4644 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4645 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4646 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4648 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4649 rdev->mec.hpd_eop_obj = NULL;
4653 #define MEC_HPD_SIZE 2048
4655 static int cik_mec_init(struct radeon_device *rdev)
4661 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4662 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4664 if (rdev->family == CHIP_KAVERI)
4665 rdev->mec.num_mec = 2;
4667 rdev->mec.num_mec = 1;
4668 rdev->mec.num_pipe = 4;
4669 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4671 if (rdev->mec.hpd_eop_obj == NULL) {
4672 r = radeon_bo_create(rdev,
4673 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4675 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4676 &rdev->mec.hpd_eop_obj);
4678 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4683 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4684 if (unlikely(r != 0)) {
4688 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4689 &rdev->mec.hpd_eop_gpu_addr);
4691 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4695 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4697 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4702 /* clear memory. Not sure if this is required or not */
4703 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4705 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4706 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4711 struct hqd_registers
4713 u32 cp_mqd_base_addr;
4714 u32 cp_mqd_base_addr_hi;
4717 u32 cp_hqd_persistent_state;
4718 u32 cp_hqd_pipe_priority;
4719 u32 cp_hqd_queue_priority;
4722 u32 cp_hqd_pq_base_hi;
4724 u32 cp_hqd_pq_rptr_report_addr;
4725 u32 cp_hqd_pq_rptr_report_addr_hi;
4726 u32 cp_hqd_pq_wptr_poll_addr;
4727 u32 cp_hqd_pq_wptr_poll_addr_hi;
4728 u32 cp_hqd_pq_doorbell_control;
4730 u32 cp_hqd_pq_control;
4731 u32 cp_hqd_ib_base_addr;
4732 u32 cp_hqd_ib_base_addr_hi;
4734 u32 cp_hqd_ib_control;
4735 u32 cp_hqd_iq_timer;
4737 u32 cp_hqd_dequeue_request;
4738 u32 cp_hqd_dma_offload;
4739 u32 cp_hqd_sema_cmd;
4740 u32 cp_hqd_msg_type;
4741 u32 cp_hqd_atomic0_preop_lo;
4742 u32 cp_hqd_atomic0_preop_hi;
4743 u32 cp_hqd_atomic1_preop_lo;
4744 u32 cp_hqd_atomic1_preop_hi;
4745 u32 cp_hqd_hq_scheduler0;
4746 u32 cp_hqd_hq_scheduler1;
4753 u32 dispatch_initiator;
4757 u32 pipeline_stat_enable;
4758 u32 perf_counter_enable;
4764 u32 resource_limits;
4765 u32 static_thread_mgmt01[2];
4767 u32 static_thread_mgmt23[2];
4769 u32 thread_trace_enable;
4772 u32 vgtcs_invoke_count[2];
4773 struct hqd_registers queue_state;
4775 u32 interrupt_queue[64];
4779 * cik_cp_compute_resume - setup the compute queue registers
4781 * @rdev: radeon_device pointer
4783 * Program the compute queues and test them to make sure they
4785 * Returns 0 for success, error for failure.
4787 static int cik_cp_compute_resume(struct radeon_device *rdev)
4791 bool use_doorbell = true;
4797 struct bonaire_mqd *mqd;
4799 r = cik_cp_compute_start(rdev);
4803 /* fix up chicken bits */
4804 tmp = RREG32(CP_CPF_DEBUG);
4806 WREG32(CP_CPF_DEBUG, tmp);
4808 /* init the pipes */
4809 mutex_lock(&rdev->srbm_mutex);
4810 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4811 int me = (i < 4) ? 1 : 2;
4812 int pipe = (i < 4) ? i : (i - 4);
4814 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4816 cik_srbm_select(rdev, me, pipe, 0, 0);
4818 /* write the EOP addr */
4819 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4820 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4822 /* set the VMID assigned */
4823 WREG32(CP_HPD_EOP_VMID, 0);
4825 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4826 tmp = RREG32(CP_HPD_EOP_CONTROL);
4827 tmp &= ~EOP_SIZE_MASK;
4828 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4829 WREG32(CP_HPD_EOP_CONTROL, tmp);
4831 cik_srbm_select(rdev, 0, 0, 0, 0);
4832 mutex_unlock(&rdev->srbm_mutex);
4834 /* init the queues. Just two for now. */
4835 for (i = 0; i < 2; i++) {
4837 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4839 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4841 if (rdev->ring[idx].mqd_obj == NULL) {
4842 r = radeon_bo_create(rdev,
4843 sizeof(struct bonaire_mqd),
4845 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4846 &rdev->ring[idx].mqd_obj);
4848 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4853 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4854 if (unlikely(r != 0)) {
4855 cik_cp_compute_fini(rdev);
4858 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4861 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4862 cik_cp_compute_fini(rdev);
4865 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4867 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4868 cik_cp_compute_fini(rdev);
4872 /* init the mqd struct */
4873 memset(buf, 0, sizeof(struct bonaire_mqd));
4875 mqd = (struct bonaire_mqd *)buf;
4876 mqd->header = 0xC0310800;
4877 mqd->static_thread_mgmt01[0] = 0xffffffff;
4878 mqd->static_thread_mgmt01[1] = 0xffffffff;
4879 mqd->static_thread_mgmt23[0] = 0xffffffff;
4880 mqd->static_thread_mgmt23[1] = 0xffffffff;
4882 mutex_lock(&rdev->srbm_mutex);
4883 cik_srbm_select(rdev, rdev->ring[idx].me,
4884 rdev->ring[idx].pipe,
4885 rdev->ring[idx].queue, 0);
4887 /* disable wptr polling */
4888 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4889 tmp &= ~WPTR_POLL_EN;
4890 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4892 /* enable doorbell? */
4893 mqd->queue_state.cp_hqd_pq_doorbell_control =
4894 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4896 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4898 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4899 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4900 mqd->queue_state.cp_hqd_pq_doorbell_control);
4902 /* disable the queue if it's active */
4903 mqd->queue_state.cp_hqd_dequeue_request = 0;
4904 mqd->queue_state.cp_hqd_pq_rptr = 0;
4905 mqd->queue_state.cp_hqd_pq_wptr= 0;
4906 if (RREG32(CP_HQD_ACTIVE) & 1) {
4907 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4908 for (i = 0; i < rdev->usec_timeout; i++) {
4909 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4913 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4914 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4915 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4918 /* set the pointer to the MQD */
4919 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4920 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4921 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4922 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4923 /* set MQD vmid to 0 */
4924 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4925 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4926 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4928 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4929 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4930 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4931 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4932 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4933 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4935 /* set up the HQD, this is similar to CP_RB0_CNTL */
4936 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4937 mqd->queue_state.cp_hqd_pq_control &=
4938 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4940 mqd->queue_state.cp_hqd_pq_control |=
4941 order_base_2(rdev->ring[idx].ring_size / 8);
4942 mqd->queue_state.cp_hqd_pq_control |=
4943 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4945 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4947 mqd->queue_state.cp_hqd_pq_control &=
4948 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4949 mqd->queue_state.cp_hqd_pq_control |=
4950 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4951 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4953 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4955 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4957 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4958 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4959 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4960 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4961 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4962 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4964 /* set the wb address wether it's enabled or not */
4966 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4968 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4969 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4970 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4971 upper_32_bits(wb_gpu_addr) & 0xffff;
4972 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4973 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4974 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4975 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4977 /* enable the doorbell if requested */
4979 mqd->queue_state.cp_hqd_pq_doorbell_control =
4980 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4981 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4982 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4983 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4984 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4985 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4986 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4989 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4991 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4992 mqd->queue_state.cp_hqd_pq_doorbell_control);
4994 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4995 rdev->ring[idx].wptr = 0;
4996 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4997 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4998 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5000 /* set the vmid for the queue */
5001 mqd->queue_state.cp_hqd_vmid = 0;
5002 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5004 /* activate the queue */
5005 mqd->queue_state.cp_hqd_active = 1;
5006 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5008 cik_srbm_select(rdev, 0, 0, 0, 0);
5009 mutex_unlock(&rdev->srbm_mutex);
5011 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5012 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5014 rdev->ring[idx].ready = true;
5015 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5017 rdev->ring[idx].ready = false;
5023 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5025 cik_cp_gfx_enable(rdev, enable);
5026 cik_cp_compute_enable(rdev, enable);
5029 static int cik_cp_load_microcode(struct radeon_device *rdev)
5033 r = cik_cp_gfx_load_microcode(rdev);
5036 r = cik_cp_compute_load_microcode(rdev);
5043 static void cik_cp_fini(struct radeon_device *rdev)
5045 cik_cp_gfx_fini(rdev);
5046 cik_cp_compute_fini(rdev);
5049 static int cik_cp_resume(struct radeon_device *rdev)
5053 cik_enable_gui_idle_interrupt(rdev, false);
5055 r = cik_cp_load_microcode(rdev);
5059 r = cik_cp_gfx_resume(rdev);
5062 r = cik_cp_compute_resume(rdev);
5066 cik_enable_gui_idle_interrupt(rdev, true);
5071 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5073 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5074 RREG32(GRBM_STATUS));
5075 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5076 RREG32(GRBM_STATUS2));
5077 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5078 RREG32(GRBM_STATUS_SE0));
5079 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5080 RREG32(GRBM_STATUS_SE1));
5081 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5082 RREG32(GRBM_STATUS_SE2));
5083 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5084 RREG32(GRBM_STATUS_SE3));
5085 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5086 RREG32(SRBM_STATUS));
5087 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5088 RREG32(SRBM_STATUS2));
5089 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5090 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5091 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5092 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5093 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5094 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5095 RREG32(CP_STALLED_STAT1));
5096 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5097 RREG32(CP_STALLED_STAT2));
5098 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5099 RREG32(CP_STALLED_STAT3));
5100 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5101 RREG32(CP_CPF_BUSY_STAT));
5102 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5103 RREG32(CP_CPF_STALLED_STAT1));
5104 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5105 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5106 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5107 RREG32(CP_CPC_STALLED_STAT1));
5108 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5112 * cik_gpu_check_soft_reset - check which blocks are busy
5114 * @rdev: radeon_device pointer
5116 * Check which blocks are busy and return the relevant reset
5117 * mask to be used by cik_gpu_soft_reset().
5118 * Returns a mask of the blocks to be reset.
5120 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5126 tmp = RREG32(GRBM_STATUS);
5127 if (tmp & (PA_BUSY | SC_BUSY |
5128 BCI_BUSY | SX_BUSY |
5129 TA_BUSY | VGT_BUSY |
5131 GDS_BUSY | SPI_BUSY |
5132 IA_BUSY | IA_BUSY_NO_DMA))
5133 reset_mask |= RADEON_RESET_GFX;
5135 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5136 reset_mask |= RADEON_RESET_CP;
5139 tmp = RREG32(GRBM_STATUS2);
5141 reset_mask |= RADEON_RESET_RLC;
5143 /* SDMA0_STATUS_REG */
5144 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5145 if (!(tmp & SDMA_IDLE))
5146 reset_mask |= RADEON_RESET_DMA;
5148 /* SDMA1_STATUS_REG */
5149 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5150 if (!(tmp & SDMA_IDLE))
5151 reset_mask |= RADEON_RESET_DMA1;
5154 tmp = RREG32(SRBM_STATUS2);
5155 if (tmp & SDMA_BUSY)
5156 reset_mask |= RADEON_RESET_DMA;
5158 if (tmp & SDMA1_BUSY)
5159 reset_mask |= RADEON_RESET_DMA1;
5162 tmp = RREG32(SRBM_STATUS);
5165 reset_mask |= RADEON_RESET_IH;
5168 reset_mask |= RADEON_RESET_SEM;
5170 if (tmp & GRBM_RQ_PENDING)
5171 reset_mask |= RADEON_RESET_GRBM;
5174 reset_mask |= RADEON_RESET_VMC;
5176 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5177 MCC_BUSY | MCD_BUSY))
5178 reset_mask |= RADEON_RESET_MC;
5180 if (evergreen_is_display_hung(rdev))
5181 reset_mask |= RADEON_RESET_DISPLAY;
5183 /* Skip MC reset as it's mostly likely not hung, just busy */
5184 if (reset_mask & RADEON_RESET_MC) {
5185 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5186 reset_mask &= ~RADEON_RESET_MC;
5193 * cik_gpu_soft_reset - soft reset GPU
5195 * @rdev: radeon_device pointer
5196 * @reset_mask: mask of which blocks to reset
5198 * Soft reset the blocks specified in @reset_mask.
5200 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5202 struct evergreen_mc_save save;
5203 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5206 if (reset_mask == 0)
5209 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5211 cik_print_gpu_status_regs(rdev);
5212 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5213 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5214 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5215 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5224 /* Disable GFX parsing/prefetching */
5225 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5227 /* Disable MEC parsing/prefetching */
5228 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5230 if (reset_mask & RADEON_RESET_DMA) {
5232 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5234 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5236 if (reset_mask & RADEON_RESET_DMA1) {
5238 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5240 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5243 evergreen_mc_stop(rdev, &save);
5244 if (evergreen_mc_wait_for_idle(rdev)) {
5245 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5248 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5249 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5251 if (reset_mask & RADEON_RESET_CP) {
5252 grbm_soft_reset |= SOFT_RESET_CP;
5254 srbm_soft_reset |= SOFT_RESET_GRBM;
5257 if (reset_mask & RADEON_RESET_DMA)
5258 srbm_soft_reset |= SOFT_RESET_SDMA;
5260 if (reset_mask & RADEON_RESET_DMA1)
5261 srbm_soft_reset |= SOFT_RESET_SDMA1;
5263 if (reset_mask & RADEON_RESET_DISPLAY)
5264 srbm_soft_reset |= SOFT_RESET_DC;
5266 if (reset_mask & RADEON_RESET_RLC)
5267 grbm_soft_reset |= SOFT_RESET_RLC;
5269 if (reset_mask & RADEON_RESET_SEM)
5270 srbm_soft_reset |= SOFT_RESET_SEM;
5272 if (reset_mask & RADEON_RESET_IH)
5273 srbm_soft_reset |= SOFT_RESET_IH;
5275 if (reset_mask & RADEON_RESET_GRBM)
5276 srbm_soft_reset |= SOFT_RESET_GRBM;
5278 if (reset_mask & RADEON_RESET_VMC)
5279 srbm_soft_reset |= SOFT_RESET_VMC;
5281 if (!(rdev->flags & RADEON_IS_IGP)) {
5282 if (reset_mask & RADEON_RESET_MC)
5283 srbm_soft_reset |= SOFT_RESET_MC;
5286 if (grbm_soft_reset) {
5287 tmp = RREG32(GRBM_SOFT_RESET);
5288 tmp |= grbm_soft_reset;
5289 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5290 WREG32(GRBM_SOFT_RESET, tmp);
5291 tmp = RREG32(GRBM_SOFT_RESET);
5295 tmp &= ~grbm_soft_reset;
5296 WREG32(GRBM_SOFT_RESET, tmp);
5297 tmp = RREG32(GRBM_SOFT_RESET);
5300 if (srbm_soft_reset) {
5301 tmp = RREG32(SRBM_SOFT_RESET);
5302 tmp |= srbm_soft_reset;
5303 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5304 WREG32(SRBM_SOFT_RESET, tmp);
5305 tmp = RREG32(SRBM_SOFT_RESET);
5309 tmp &= ~srbm_soft_reset;
5310 WREG32(SRBM_SOFT_RESET, tmp);
5311 tmp = RREG32(SRBM_SOFT_RESET);
5314 /* Wait a little for things to settle down */
5317 evergreen_mc_resume(rdev, &save);
5320 cik_print_gpu_status_regs(rdev);
5323 struct kv_reset_save_regs {
5324 u32 gmcon_reng_execute;
5329 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5330 struct kv_reset_save_regs *save)
5332 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5333 save->gmcon_misc = RREG32(GMCON_MISC);
5334 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5336 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5337 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5338 STCTRL_STUTTER_EN));
5341 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5342 struct kv_reset_save_regs *save)
5346 WREG32(GMCON_PGFSM_WRITE, 0);
5347 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5349 for (i = 0; i < 5; i++)
5350 WREG32(GMCON_PGFSM_WRITE, 0);
5352 WREG32(GMCON_PGFSM_WRITE, 0);
5353 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5355 for (i = 0; i < 5; i++)
5356 WREG32(GMCON_PGFSM_WRITE, 0);
5358 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5359 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5361 for (i = 0; i < 5; i++)
5362 WREG32(GMCON_PGFSM_WRITE, 0);
5364 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5365 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5367 for (i = 0; i < 5; i++)
5368 WREG32(GMCON_PGFSM_WRITE, 0);
5370 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5371 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5373 for (i = 0; i < 5; i++)
5374 WREG32(GMCON_PGFSM_WRITE, 0);
5376 WREG32(GMCON_PGFSM_WRITE, 0);
5377 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5379 for (i = 0; i < 5; i++)
5380 WREG32(GMCON_PGFSM_WRITE, 0);
5382 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5383 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5385 for (i = 0; i < 5; i++)
5386 WREG32(GMCON_PGFSM_WRITE, 0);
5388 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5389 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5391 for (i = 0; i < 5; i++)
5392 WREG32(GMCON_PGFSM_WRITE, 0);
5394 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5395 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5397 for (i = 0; i < 5; i++)
5398 WREG32(GMCON_PGFSM_WRITE, 0);
5400 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5401 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5403 for (i = 0; i < 5; i++)
5404 WREG32(GMCON_PGFSM_WRITE, 0);
5406 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5407 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5409 WREG32(GMCON_MISC3, save->gmcon_misc3);
5410 WREG32(GMCON_MISC, save->gmcon_misc);
5411 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5414 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5416 struct evergreen_mc_save save;
5417 struct kv_reset_save_regs kv_save = { 0 };
5420 dev_info(rdev->dev, "GPU pci config reset\n");
5428 /* Disable GFX parsing/prefetching */
5429 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5431 /* Disable MEC parsing/prefetching */
5432 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5435 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5437 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5439 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5441 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5442 /* XXX other engines? */
5444 /* halt the rlc, disable cp internal ints */
5449 /* disable mem access */
5450 evergreen_mc_stop(rdev, &save);
5451 if (evergreen_mc_wait_for_idle(rdev)) {
5452 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5455 if (rdev->flags & RADEON_IS_IGP)
5456 kv_save_regs_for_reset(rdev, &kv_save);
5459 pci_clear_master(rdev->pdev);
5461 radeon_pci_config_reset(rdev);
5465 /* wait for asic to come out of reset */
5466 for (i = 0; i < rdev->usec_timeout; i++) {
5467 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5472 /* does asic init need to be run first??? */
5473 if (rdev->flags & RADEON_IS_IGP)
5474 kv_restore_regs_for_reset(rdev, &kv_save);
5478 * cik_asic_reset - soft reset GPU
5480 * @rdev: radeon_device pointer
5482 * Look up which blocks are hung and attempt
5484 * Returns 0 for success.
5486 int cik_asic_reset(struct radeon_device *rdev)
5490 reset_mask = cik_gpu_check_soft_reset(rdev);
5493 r600_set_bios_scratch_engine_hung(rdev, true);
5495 /* try soft reset */
5496 cik_gpu_soft_reset(rdev, reset_mask);
5498 reset_mask = cik_gpu_check_soft_reset(rdev);
5500 /* try pci config reset */
5501 if (reset_mask && radeon_hard_reset)
5502 cik_gpu_pci_config_reset(rdev);
5504 reset_mask = cik_gpu_check_soft_reset(rdev);
5507 r600_set_bios_scratch_engine_hung(rdev, false);
5513 * cik_gfx_is_lockup - check if the 3D engine is locked up
5515 * @rdev: radeon_device pointer
5516 * @ring: radeon_ring structure holding ring information
5518 * Check if the 3D engine is locked up (CIK).
5519 * Returns true if the engine is locked, false if not.
5521 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5523 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5525 if (!(reset_mask & (RADEON_RESET_GFX |
5526 RADEON_RESET_COMPUTE |
5527 RADEON_RESET_CP))) {
5528 radeon_ring_lockup_update(rdev, ring);
5531 return radeon_ring_test_lockup(rdev, ring);
5536 * cik_mc_program - program the GPU memory controller
5538 * @rdev: radeon_device pointer
5540 * Set the location of vram, gart, and AGP in the GPU's
5541 * physical address space (CIK).
5543 static void cik_mc_program(struct radeon_device *rdev)
5545 struct evergreen_mc_save save;
5549 /* Initialize HDP */
5550 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5551 WREG32((0x2c14 + j), 0x00000000);
5552 WREG32((0x2c18 + j), 0x00000000);
5553 WREG32((0x2c1c + j), 0x00000000);
5554 WREG32((0x2c20 + j), 0x00000000);
5555 WREG32((0x2c24 + j), 0x00000000);
5557 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5559 evergreen_mc_stop(rdev, &save);
5560 if (radeon_mc_wait_for_idle(rdev)) {
5561 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5563 /* Lockout access through VGA aperture*/
5564 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5565 /* Update configuration */
5566 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5567 rdev->mc.vram_start >> 12);
5568 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5569 rdev->mc.vram_end >> 12);
5570 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5571 rdev->vram_scratch.gpu_addr >> 12);
5572 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5573 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5574 WREG32(MC_VM_FB_LOCATION, tmp);
5575 /* XXX double check these! */
5576 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5577 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5578 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5579 WREG32(MC_VM_AGP_BASE, 0);
5580 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5581 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5582 if (radeon_mc_wait_for_idle(rdev)) {
5583 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5585 evergreen_mc_resume(rdev, &save);
5586 /* we need to own VRAM, so turn off the VGA renderer here
5587 * to stop it overwriting our objects */
5588 rv515_vga_render_disable(rdev);
5592 * cik_mc_init - initialize the memory controller driver params
5594 * @rdev: radeon_device pointer
5596 * Look up the amount of vram, vram width, and decide how to place
5597 * vram and gart within the GPU's physical address space (CIK).
5598 * Returns 0 for success.
5600 static int cik_mc_init(struct radeon_device *rdev)
5603 int chansize, numchan;
5605 /* Get VRAM informations */
5606 rdev->mc.vram_is_ddr = true;
5607 tmp = RREG32(MC_ARB_RAMCFG);
5608 if (tmp & CHANSIZE_MASK) {
5613 tmp = RREG32(MC_SHARED_CHMAP);
5614 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5644 rdev->mc.vram_width = numchan * chansize;
5645 /* Could aper size report 0 ? */
5646 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5647 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5648 /* size in MB on si */
5649 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5650 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5651 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5652 si_vram_gtt_location(rdev, &rdev->mc);
5653 radeon_update_bandwidth_info(rdev);
5660 * VMID 0 is the physical GPU addresses as used by the kernel.
5661 * VMIDs 1-15 are used for userspace clients and are handled
5662 * by the radeon vm/hsa code.
5665 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5667 * @rdev: radeon_device pointer
5669 * Flush the TLB for the VMID 0 page table (CIK).
5671 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5673 /* flush hdp cache */
5674 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5676 /* bits 0-15 are the VM contexts0-15 */
5677 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5681 * cik_pcie_gart_enable - gart enable
5683 * @rdev: radeon_device pointer
5685 * This sets up the TLBs, programs the page tables for VMID0,
5686 * sets up the hw for VMIDs 1-15 which are allocated on
5687 * demand, and sets up the global locations for the LDS, GDS,
5688 * and GPUVM for FSA64 clients (CIK).
5689 * Returns 0 for success, errors for failure.
5691 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5695 if (rdev->gart.robj == NULL) {
5696 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5699 r = radeon_gart_table_vram_pin(rdev);
5702 /* Setup TLB control */
5703 WREG32(MC_VM_MX_L1_TLB_CNTL,
5706 ENABLE_L1_FRAGMENT_PROCESSING |
5707 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5708 ENABLE_ADVANCED_DRIVER_MODEL |
5709 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5710 /* Setup L2 cache */
5711 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5712 ENABLE_L2_FRAGMENT_PROCESSING |
5713 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5714 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5715 EFFECTIVE_L2_QUEUE_SIZE(7) |
5716 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5717 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5718 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5720 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5721 /* setup context0 */
5722 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5723 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5724 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5725 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5726 (u32)(rdev->dummy_page.addr >> 12));
5727 WREG32(VM_CONTEXT0_CNTL2, 0);
5728 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5729 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5735 /* empty context1-15 */
5736 /* FIXME start with 4G, once using 2 level pt switch to full
5739 /* set vm size, must be a multiple of 4 */
5740 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5741 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5742 for (i = 1; i < 16; i++) {
5744 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5745 rdev->gart.table_addr >> 12);
5747 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5748 rdev->gart.table_addr >> 12);
5751 /* enable context1-15 */
5752 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5753 (u32)(rdev->dummy_page.addr >> 12));
5754 WREG32(VM_CONTEXT1_CNTL2, 4);
5755 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5756 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5757 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5758 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5759 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5760 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5761 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5762 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5763 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5764 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5765 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5766 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5767 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5768 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5770 if (rdev->family == CHIP_KAVERI) {
5771 u32 tmp = RREG32(CHUB_CONTROL);
5773 WREG32(CHUB_CONTROL, tmp);
5776 /* XXX SH_MEM regs */
5777 /* where to put LDS, scratch, GPUVM in FSA64 space */
5778 mutex_lock(&rdev->srbm_mutex);
5779 for (i = 0; i < 16; i++) {
5780 cik_srbm_select(rdev, 0, 0, 0, i);
5781 /* CP and shaders */
5782 WREG32(SH_MEM_CONFIG, 0);
5783 WREG32(SH_MEM_APE1_BASE, 1);
5784 WREG32(SH_MEM_APE1_LIMIT, 0);
5785 WREG32(SH_MEM_BASES, 0);
5787 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5788 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5789 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5790 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5791 /* XXX SDMA RLC - todo */
5793 cik_srbm_select(rdev, 0, 0, 0, 0);
5794 mutex_unlock(&rdev->srbm_mutex);
5796 cik_pcie_gart_tlb_flush(rdev);
5797 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5798 (unsigned)(rdev->mc.gtt_size >> 20),
5799 (unsigned long long)rdev->gart.table_addr);
5800 rdev->gart.ready = true;
5805 * cik_pcie_gart_disable - gart disable
5807 * @rdev: radeon_device pointer
5809 * This disables all VM page table (CIK).
5811 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5813 /* Disable all tables */
5814 WREG32(VM_CONTEXT0_CNTL, 0);
5815 WREG32(VM_CONTEXT1_CNTL, 0);
5816 /* Setup TLB control */
5817 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5818 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5819 /* Setup L2 cache */
5821 ENABLE_L2_FRAGMENT_PROCESSING |
5822 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5823 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5824 EFFECTIVE_L2_QUEUE_SIZE(7) |
5825 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5826 WREG32(VM_L2_CNTL2, 0);
5827 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5828 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5829 radeon_gart_table_vram_unpin(rdev);
5833 * cik_pcie_gart_fini - vm fini callback
5835 * @rdev: radeon_device pointer
5837 * Tears down the driver GART/VM setup (CIK).
5839 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5841 cik_pcie_gart_disable(rdev);
5842 radeon_gart_table_vram_free(rdev);
5843 radeon_gart_fini(rdev);
5848 * cik_ib_parse - vm ib_parse callback
5850 * @rdev: radeon_device pointer
5851 * @ib: indirect buffer pointer
5853 * CIK uses hw IB checking so this is a nop (CIK).
5855 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5862 * VMID 0 is the physical GPU addresses as used by the kernel.
5863 * VMIDs 1-15 are used for userspace clients and are handled
5864 * by the radeon vm/hsa code.
5867 * cik_vm_init - cik vm init callback
5869 * @rdev: radeon_device pointer
5871 * Inits cik specific vm parameters (number of VMs, base of vram for
5872 * VMIDs 1-15) (CIK).
5873 * Returns 0 for success.
5875 int cik_vm_init(struct radeon_device *rdev)
5878 rdev->vm_manager.nvm = 16;
5879 /* base offset of vram pages */
5880 if (rdev->flags & RADEON_IS_IGP) {
5881 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5883 rdev->vm_manager.vram_base_offset = tmp;
5885 rdev->vm_manager.vram_base_offset = 0;
5891 * cik_vm_fini - cik vm fini callback
5893 * @rdev: radeon_device pointer
5895 * Tear down any asic specific VM setup (CIK).
5897 void cik_vm_fini(struct radeon_device *rdev)
5902 * cik_vm_decode_fault - print human readable fault info
5904 * @rdev: radeon_device pointer
5905 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5906 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5908 * Print human readable fault information (CIK).
5910 static void cik_vm_decode_fault(struct radeon_device *rdev,
5911 u32 status, u32 addr, u32 mc_client)
5914 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5915 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5916 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5917 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5919 if (rdev->family == CHIP_HAWAII)
5920 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5922 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5924 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5925 protections, vmid, addr,
5926 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5927 block, mc_client, mc_id);
5931 * cik_vm_flush - cik vm flush using the CP
5933 * @rdev: radeon_device pointer
5935 * Update the page table base and flush the VM TLB
5936 * using the CP (CIK).
5938 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5940 struct radeon_ring *ring = &rdev->ring[ridx];
5941 int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
5946 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5947 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5948 WRITE_DATA_DST_SEL(0)));
5950 radeon_ring_write(ring,
5951 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5953 radeon_ring_write(ring,
5954 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5956 radeon_ring_write(ring, 0);
5957 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5959 /* update SH_MEM_* regs */
5960 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5961 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5962 WRITE_DATA_DST_SEL(0)));
5963 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5964 radeon_ring_write(ring, 0);
5965 radeon_ring_write(ring, VMID(vm->id));
5967 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5968 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5969 WRITE_DATA_DST_SEL(0)));
5970 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5971 radeon_ring_write(ring, 0);
5973 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5974 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5975 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5976 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5978 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5979 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5980 WRITE_DATA_DST_SEL(0)));
5981 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5982 radeon_ring_write(ring, 0);
5983 radeon_ring_write(ring, VMID(0));
5986 cik_hdp_flush_cp_ring_emit(rdev, ridx);
5988 /* bits 0-15 are the VM contexts0-15 */
5989 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5990 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5991 WRITE_DATA_DST_SEL(0)));
5992 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5993 radeon_ring_write(ring, 0);
5994 radeon_ring_write(ring, 1 << vm->id);
5996 /* compute doesn't have PFP */
5998 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5999 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6000 radeon_ring_write(ring, 0x0);
6006 * The RLC is a multi-purpose microengine that handles a
6007 * variety of functions, the most important of which is
6008 * the interrupt controller.
6010 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6013 u32 tmp = RREG32(CP_INT_CNTL_RING0);
6016 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6018 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6019 WREG32(CP_INT_CNTL_RING0, tmp);
6022 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6026 tmp = RREG32(RLC_LB_CNTL);
6028 tmp |= LOAD_BALANCE_ENABLE;
6030 tmp &= ~LOAD_BALANCE_ENABLE;
6031 WREG32(RLC_LB_CNTL, tmp);
6034 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6039 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6040 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6041 cik_select_se_sh(rdev, i, j);
6042 for (k = 0; k < rdev->usec_timeout; k++) {
6043 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6049 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6051 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6052 for (k = 0; k < rdev->usec_timeout; k++) {
6053 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6059 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6063 tmp = RREG32(RLC_CNTL);
6065 WREG32(RLC_CNTL, rlc);
6068 static u32 cik_halt_rlc(struct radeon_device *rdev)
6072 orig = data = RREG32(RLC_CNTL);
6074 if (data & RLC_ENABLE) {
6077 data &= ~RLC_ENABLE;
6078 WREG32(RLC_CNTL, data);
6080 for (i = 0; i < rdev->usec_timeout; i++) {
6081 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6086 cik_wait_for_rlc_serdes(rdev);
6092 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6096 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6097 WREG32(RLC_GPR_REG2, tmp);
6099 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6100 for (i = 0; i < rdev->usec_timeout; i++) {
6101 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6106 for (i = 0; i < rdev->usec_timeout; i++) {
6107 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6113 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6117 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6118 WREG32(RLC_GPR_REG2, tmp);
6122 * cik_rlc_stop - stop the RLC ME
6124 * @rdev: radeon_device pointer
6126 * Halt the RLC ME (MicroEngine) (CIK).
6128 static void cik_rlc_stop(struct radeon_device *rdev)
6130 WREG32(RLC_CNTL, 0);
6132 cik_enable_gui_idle_interrupt(rdev, false);
6134 cik_wait_for_rlc_serdes(rdev);
6138 * cik_rlc_start - start the RLC ME
6140 * @rdev: radeon_device pointer
6142 * Unhalt the RLC ME (MicroEngine) (CIK).
6144 static void cik_rlc_start(struct radeon_device *rdev)
6146 WREG32(RLC_CNTL, RLC_ENABLE);
6148 cik_enable_gui_idle_interrupt(rdev, true);
6154 * cik_rlc_resume - setup the RLC hw
6156 * @rdev: radeon_device pointer
6158 * Initialize the RLC registers, load the ucode,
6159 * and start the RLC (CIK).
6160 * Returns 0 for success, -EINVAL if the ucode is not available.
6162 static int cik_rlc_resume(struct radeon_device *rdev)
6172 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6173 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6181 WREG32(RLC_LB_CNTR_INIT, 0);
6182 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6184 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6185 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6186 WREG32(RLC_LB_PARAMS, 0x00600408);
6187 WREG32(RLC_LB_CNTL, 0x80000004);
6189 WREG32(RLC_MC_CNTL, 0);
6190 WREG32(RLC_UCODE_CNTL, 0);
6193 const struct rlc_firmware_header_v1_0 *hdr =
6194 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6195 const __le32 *fw_data = (const __le32 *)
6196 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6198 radeon_ucode_print_rlc_hdr(&hdr->header);
6200 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6201 WREG32(RLC_GPM_UCODE_ADDR, 0);
6202 for (i = 0; i < size; i++)
6203 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6204 WREG32(RLC_GPM_UCODE_ADDR, 0);
6206 const __be32 *fw_data;
6208 switch (rdev->family) {
6212 size = BONAIRE_RLC_UCODE_SIZE;
6215 size = KV_RLC_UCODE_SIZE;
6218 size = KB_RLC_UCODE_SIZE;
6221 size = ML_RLC_UCODE_SIZE;
6225 fw_data = (const __be32 *)rdev->rlc_fw->data;
6226 WREG32(RLC_GPM_UCODE_ADDR, 0);
6227 for (i = 0; i < size; i++)
6228 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6229 WREG32(RLC_GPM_UCODE_ADDR, 0);
6232 /* XXX - find out what chips support lbpw */
6233 cik_enable_lbpw(rdev, false);
6235 if (rdev->family == CHIP_BONAIRE)
6236 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6238 cik_rlc_start(rdev);
6243 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6245 u32 data, orig, tmp, tmp2;
6247 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6249 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6250 cik_enable_gui_idle_interrupt(rdev, true);
6252 tmp = cik_halt_rlc(rdev);
6254 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6255 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6256 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6257 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6258 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6260 cik_update_rlc(rdev, tmp);
6262 data |= CGCG_EN | CGLS_EN;
6264 cik_enable_gui_idle_interrupt(rdev, false);
6266 RREG32(CB_CGTT_SCLK_CTRL);
6267 RREG32(CB_CGTT_SCLK_CTRL);
6268 RREG32(CB_CGTT_SCLK_CTRL);
6269 RREG32(CB_CGTT_SCLK_CTRL);
6271 data &= ~(CGCG_EN | CGLS_EN);
6275 WREG32(RLC_CGCG_CGLS_CTRL, data);
6279 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6281 u32 data, orig, tmp = 0;
6283 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6284 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6285 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6286 orig = data = RREG32(CP_MEM_SLP_CNTL);
6287 data |= CP_MEM_LS_EN;
6289 WREG32(CP_MEM_SLP_CNTL, data);
6293 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6296 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6298 tmp = cik_halt_rlc(rdev);
6300 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6301 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6302 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6303 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6304 WREG32(RLC_SERDES_WR_CTRL, data);
6306 cik_update_rlc(rdev, tmp);
6308 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6309 orig = data = RREG32(CGTS_SM_CTRL_REG);
6310 data &= ~SM_MODE_MASK;
6311 data |= SM_MODE(0x2);
6312 data |= SM_MODE_ENABLE;
6313 data &= ~CGTS_OVERRIDE;
6314 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6315 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6316 data &= ~CGTS_LS_OVERRIDE;
6317 data &= ~ON_MONITOR_ADD_MASK;
6318 data |= ON_MONITOR_ADD_EN;
6319 data |= ON_MONITOR_ADD(0x96);
6321 WREG32(CGTS_SM_CTRL_REG, data);
6324 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6327 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6329 data = RREG32(RLC_MEM_SLP_CNTL);
6330 if (data & RLC_MEM_LS_EN) {
6331 data &= ~RLC_MEM_LS_EN;
6332 WREG32(RLC_MEM_SLP_CNTL, data);
6335 data = RREG32(CP_MEM_SLP_CNTL);
6336 if (data & CP_MEM_LS_EN) {
6337 data &= ~CP_MEM_LS_EN;
6338 WREG32(CP_MEM_SLP_CNTL, data);
6341 orig = data = RREG32(CGTS_SM_CTRL_REG);
6342 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6344 WREG32(CGTS_SM_CTRL_REG, data);
6346 tmp = cik_halt_rlc(rdev);
6348 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6349 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6350 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6351 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6352 WREG32(RLC_SERDES_WR_CTRL, data);
6354 cik_update_rlc(rdev, tmp);
6358 static const u32 mc_cg_registers[] =
6371 static void cik_enable_mc_ls(struct radeon_device *rdev,
6377 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6378 orig = data = RREG32(mc_cg_registers[i]);
6379 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6380 data |= MC_LS_ENABLE;
6382 data &= ~MC_LS_ENABLE;
6384 WREG32(mc_cg_registers[i], data);
6388 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6394 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6395 orig = data = RREG32(mc_cg_registers[i]);
6396 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6397 data |= MC_CG_ENABLE;
6399 data &= ~MC_CG_ENABLE;
6401 WREG32(mc_cg_registers[i], data);
6405 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6410 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6411 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6412 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6414 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6417 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6419 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6422 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6426 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6431 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6432 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6435 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6437 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6440 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6442 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6445 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6447 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6450 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6454 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6459 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6460 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6462 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6464 orig = data = RREG32(UVD_CGC_CTRL);
6467 WREG32(UVD_CGC_CTRL, data);
6469 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6471 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6473 orig = data = RREG32(UVD_CGC_CTRL);
6476 WREG32(UVD_CGC_CTRL, data);
6480 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6485 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6487 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6488 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6489 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6491 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6492 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6495 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6498 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6503 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6505 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6506 data &= ~CLOCK_GATING_DIS;
6508 data |= CLOCK_GATING_DIS;
6511 WREG32(HDP_HOST_PATH_CNTL, data);
6514 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6519 orig = data = RREG32(HDP_MEM_POWER_LS);
6521 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6522 data |= HDP_LS_ENABLE;
6524 data &= ~HDP_LS_ENABLE;
6527 WREG32(HDP_MEM_POWER_LS, data);
6530 void cik_update_cg(struct radeon_device *rdev,
6531 u32 block, bool enable)
6534 if (block & RADEON_CG_BLOCK_GFX) {
6535 cik_enable_gui_idle_interrupt(rdev, false);
6536 /* order matters! */
6538 cik_enable_mgcg(rdev, true);
6539 cik_enable_cgcg(rdev, true);
6541 cik_enable_cgcg(rdev, false);
6542 cik_enable_mgcg(rdev, false);
6544 cik_enable_gui_idle_interrupt(rdev, true);
6547 if (block & RADEON_CG_BLOCK_MC) {
6548 if (!(rdev->flags & RADEON_IS_IGP)) {
6549 cik_enable_mc_mgcg(rdev, enable);
6550 cik_enable_mc_ls(rdev, enable);
6554 if (block & RADEON_CG_BLOCK_SDMA) {
6555 cik_enable_sdma_mgcg(rdev, enable);
6556 cik_enable_sdma_mgls(rdev, enable);
6559 if (block & RADEON_CG_BLOCK_BIF) {
6560 cik_enable_bif_mgls(rdev, enable);
6563 if (block & RADEON_CG_BLOCK_UVD) {
6565 cik_enable_uvd_mgcg(rdev, enable);
6568 if (block & RADEON_CG_BLOCK_HDP) {
6569 cik_enable_hdp_mgcg(rdev, enable);
6570 cik_enable_hdp_ls(rdev, enable);
6573 if (block & RADEON_CG_BLOCK_VCE) {
6574 vce_v2_0_enable_mgcg(rdev, enable);
6578 static void cik_init_cg(struct radeon_device *rdev)
6581 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6584 si_init_uvd_internal_cg(rdev);
6586 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6587 RADEON_CG_BLOCK_SDMA |
6588 RADEON_CG_BLOCK_BIF |
6589 RADEON_CG_BLOCK_UVD |
6590 RADEON_CG_BLOCK_HDP), true);
6593 static void cik_fini_cg(struct radeon_device *rdev)
6595 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6596 RADEON_CG_BLOCK_SDMA |
6597 RADEON_CG_BLOCK_BIF |
6598 RADEON_CG_BLOCK_UVD |
6599 RADEON_CG_BLOCK_HDP), false);
6601 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6604 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6609 orig = data = RREG32(RLC_PG_CNTL);
6610 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6611 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6613 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6615 WREG32(RLC_PG_CNTL, data);
6618 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6623 orig = data = RREG32(RLC_PG_CNTL);
6624 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6625 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6627 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6629 WREG32(RLC_PG_CNTL, data);
6632 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6636 orig = data = RREG32(RLC_PG_CNTL);
6637 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6638 data &= ~DISABLE_CP_PG;
6640 data |= DISABLE_CP_PG;
6642 WREG32(RLC_PG_CNTL, data);
6645 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6649 orig = data = RREG32(RLC_PG_CNTL);
6650 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6651 data &= ~DISABLE_GDS_PG;
6653 data |= DISABLE_GDS_PG;
6655 WREG32(RLC_PG_CNTL, data);
6658 #define CP_ME_TABLE_SIZE 96
6659 #define CP_ME_TABLE_OFFSET 2048
6660 #define CP_MEC_TABLE_OFFSET 4096
6662 void cik_init_cp_pg_table(struct radeon_device *rdev)
6664 volatile u32 *dst_ptr;
6665 int me, i, max_me = 4;
6667 u32 table_offset, table_size;
6669 if (rdev->family == CHIP_KAVERI)
6672 if (rdev->rlc.cp_table_ptr == NULL)
6675 /* write the cp table buffer */
6676 dst_ptr = rdev->rlc.cp_table_ptr;
6677 for (me = 0; me < max_me; me++) {
6679 const __le32 *fw_data;
6680 const struct gfx_firmware_header_v1_0 *hdr;
6683 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6684 fw_data = (const __le32 *)
6685 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6686 table_offset = le32_to_cpu(hdr->jt_offset);
6687 table_size = le32_to_cpu(hdr->jt_size);
6688 } else if (me == 1) {
6689 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6690 fw_data = (const __le32 *)
6691 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6692 table_offset = le32_to_cpu(hdr->jt_offset);
6693 table_size = le32_to_cpu(hdr->jt_size);
6694 } else if (me == 2) {
6695 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6696 fw_data = (const __le32 *)
6697 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6698 table_offset = le32_to_cpu(hdr->jt_offset);
6699 table_size = le32_to_cpu(hdr->jt_size);
6700 } else if (me == 3) {
6701 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6702 fw_data = (const __le32 *)
6703 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6704 table_offset = le32_to_cpu(hdr->jt_offset);
6705 table_size = le32_to_cpu(hdr->jt_size);
6707 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6708 fw_data = (const __le32 *)
6709 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6710 table_offset = le32_to_cpu(hdr->jt_offset);
6711 table_size = le32_to_cpu(hdr->jt_size);
6714 for (i = 0; i < table_size; i ++) {
6715 dst_ptr[bo_offset + i] =
6716 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6718 bo_offset += table_size;
6720 const __be32 *fw_data;
6721 table_size = CP_ME_TABLE_SIZE;
6724 fw_data = (const __be32 *)rdev->ce_fw->data;
6725 table_offset = CP_ME_TABLE_OFFSET;
6726 } else if (me == 1) {
6727 fw_data = (const __be32 *)rdev->pfp_fw->data;
6728 table_offset = CP_ME_TABLE_OFFSET;
6729 } else if (me == 2) {
6730 fw_data = (const __be32 *)rdev->me_fw->data;
6731 table_offset = CP_ME_TABLE_OFFSET;
6733 fw_data = (const __be32 *)rdev->mec_fw->data;
6734 table_offset = CP_MEC_TABLE_OFFSET;
6737 for (i = 0; i < table_size; i ++) {
6738 dst_ptr[bo_offset + i] =
6739 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6741 bo_offset += table_size;
6746 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6751 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6752 orig = data = RREG32(RLC_PG_CNTL);
6753 data |= GFX_PG_ENABLE;
6755 WREG32(RLC_PG_CNTL, data);
6757 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6760 WREG32(RLC_AUTO_PG_CTRL, data);
6762 orig = data = RREG32(RLC_PG_CNTL);
6763 data &= ~GFX_PG_ENABLE;
6765 WREG32(RLC_PG_CNTL, data);
6767 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6768 data &= ~AUTO_PG_EN;
6770 WREG32(RLC_AUTO_PG_CTRL, data);
6772 data = RREG32(DB_RENDER_CONTROL);
6776 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6778 u32 mask = 0, tmp, tmp1;
6781 cik_select_se_sh(rdev, se, sh);
6782 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6783 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6784 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6791 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6796 return (~tmp) & mask;
6799 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6801 u32 i, j, k, active_cu_number = 0;
6802 u32 mask, counter, cu_bitmap;
6805 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6806 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6810 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6811 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6819 active_cu_number += counter;
6820 tmp |= (cu_bitmap << (i * 16 + j * 8));
6824 WREG32(RLC_PG_AO_CU_MASK, tmp);
6826 tmp = RREG32(RLC_MAX_PG_CU);
6827 tmp &= ~MAX_PU_CU_MASK;
6828 tmp |= MAX_PU_CU(active_cu_number);
6829 WREG32(RLC_MAX_PG_CU, tmp);
6832 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6837 orig = data = RREG32(RLC_PG_CNTL);
6838 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6839 data |= STATIC_PER_CU_PG_ENABLE;
6841 data &= ~STATIC_PER_CU_PG_ENABLE;
6843 WREG32(RLC_PG_CNTL, data);
6846 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6851 orig = data = RREG32(RLC_PG_CNTL);
6852 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6853 data |= DYN_PER_CU_PG_ENABLE;
6855 data &= ~DYN_PER_CU_PG_ENABLE;
6857 WREG32(RLC_PG_CNTL, data);
6860 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6861 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6863 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6868 if (rdev->rlc.cs_data) {
6869 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6870 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6871 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6872 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6874 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6875 for (i = 0; i < 3; i++)
6876 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6878 if (rdev->rlc.reg_list) {
6879 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6880 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6881 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6884 orig = data = RREG32(RLC_PG_CNTL);
6887 WREG32(RLC_PG_CNTL, data);
6889 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6890 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6892 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6893 data &= ~IDLE_POLL_COUNT_MASK;
6894 data |= IDLE_POLL_COUNT(0x60);
6895 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6898 WREG32(RLC_PG_DELAY, data);
6900 data = RREG32(RLC_PG_DELAY_2);
6903 WREG32(RLC_PG_DELAY_2, data);
6905 data = RREG32(RLC_AUTO_PG_CTRL);
6906 data &= ~GRBM_REG_SGIT_MASK;
6907 data |= GRBM_REG_SGIT(0x700);
6908 WREG32(RLC_AUTO_PG_CTRL, data);
6912 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6914 cik_enable_gfx_cgpg(rdev, enable);
6915 cik_enable_gfx_static_mgpg(rdev, enable);
6916 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6919 u32 cik_get_csb_size(struct radeon_device *rdev)
6922 const struct cs_section_def *sect = NULL;
6923 const struct cs_extent_def *ext = NULL;
6925 if (rdev->rlc.cs_data == NULL)
6928 /* begin clear state */
6930 /* context control state */
6933 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6934 for (ext = sect->section; ext->extent != NULL; ++ext) {
6935 if (sect->id == SECT_CONTEXT)
6936 count += 2 + ext->reg_count;
6941 /* pa_sc_raster_config/pa_sc_raster_config1 */
6943 /* end clear state */
6951 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6954 const struct cs_section_def *sect = NULL;
6955 const struct cs_extent_def *ext = NULL;
6957 if (rdev->rlc.cs_data == NULL)
6962 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6963 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6965 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6966 buffer[count++] = cpu_to_le32(0x80000000);
6967 buffer[count++] = cpu_to_le32(0x80000000);
6969 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6970 for (ext = sect->section; ext->extent != NULL; ++ext) {
6971 if (sect->id == SECT_CONTEXT) {
6973 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6974 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6975 for (i = 0; i < ext->reg_count; i++)
6976 buffer[count++] = cpu_to_le32(ext->extent[i]);
6983 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6984 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6985 switch (rdev->family) {
6987 buffer[count++] = cpu_to_le32(0x16000012);
6988 buffer[count++] = cpu_to_le32(0x00000000);
6991 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6992 buffer[count++] = cpu_to_le32(0x00000000);
6996 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6997 buffer[count++] = cpu_to_le32(0x00000000);
7000 buffer[count++] = cpu_to_le32(0x3a00161a);
7001 buffer[count++] = cpu_to_le32(0x0000002e);
7004 buffer[count++] = cpu_to_le32(0x00000000);
7005 buffer[count++] = cpu_to_le32(0x00000000);
7009 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7010 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7012 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7013 buffer[count++] = cpu_to_le32(0);
7016 static void cik_init_pg(struct radeon_device *rdev)
7018 if (rdev->pg_flags) {
7019 cik_enable_sck_slowdown_on_pu(rdev, true);
7020 cik_enable_sck_slowdown_on_pd(rdev, true);
7021 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7022 cik_init_gfx_cgpg(rdev);
7023 cik_enable_cp_pg(rdev, true);
7024 cik_enable_gds_pg(rdev, true);
7026 cik_init_ao_cu_mask(rdev);
7027 cik_update_gfx_pg(rdev, true);
7031 static void cik_fini_pg(struct radeon_device *rdev)
7033 if (rdev->pg_flags) {
7034 cik_update_gfx_pg(rdev, false);
7035 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7036 cik_enable_cp_pg(rdev, false);
7037 cik_enable_gds_pg(rdev, false);
7044 * Starting with r6xx, interrupts are handled via a ring buffer.
7045 * Ring buffers are areas of GPU accessible memory that the GPU
7046 * writes interrupt vectors into and the host reads vectors out of.
7047 * There is a rptr (read pointer) that determines where the
7048 * host is currently reading, and a wptr (write pointer)
7049 * which determines where the GPU has written. When the
7050 * pointers are equal, the ring is idle. When the GPU
7051 * writes vectors to the ring buffer, it increments the
7052 * wptr. When there is an interrupt, the host then starts
7053 * fetching commands and processing them until the pointers are
7054 * equal again at which point it updates the rptr.
7058 * cik_enable_interrupts - Enable the interrupt ring buffer
7060 * @rdev: radeon_device pointer
7062 * Enable the interrupt ring buffer (CIK).
7064 static void cik_enable_interrupts(struct radeon_device *rdev)
7066 u32 ih_cntl = RREG32(IH_CNTL);
7067 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7069 ih_cntl |= ENABLE_INTR;
7070 ih_rb_cntl |= IH_RB_ENABLE;
7071 WREG32(IH_CNTL, ih_cntl);
7072 WREG32(IH_RB_CNTL, ih_rb_cntl);
7073 rdev->ih.enabled = true;
7077 * cik_disable_interrupts - Disable the interrupt ring buffer
7079 * @rdev: radeon_device pointer
7081 * Disable the interrupt ring buffer (CIK).
7083 static void cik_disable_interrupts(struct radeon_device *rdev)
7085 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7086 u32 ih_cntl = RREG32(IH_CNTL);
7088 ih_rb_cntl &= ~IH_RB_ENABLE;
7089 ih_cntl &= ~ENABLE_INTR;
7090 WREG32(IH_RB_CNTL, ih_rb_cntl);
7091 WREG32(IH_CNTL, ih_cntl);
7092 /* set rptr, wptr to 0 */
7093 WREG32(IH_RB_RPTR, 0);
7094 WREG32(IH_RB_WPTR, 0);
7095 rdev->ih.enabled = false;
7100 * cik_disable_interrupt_state - Disable all interrupt sources
7102 * @rdev: radeon_device pointer
7104 * Clear all interrupt enable bits used by the driver (CIK).
7106 static void cik_disable_interrupt_state(struct radeon_device *rdev)
7111 tmp = RREG32(CP_INT_CNTL_RING0) &
7112 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7113 WREG32(CP_INT_CNTL_RING0, tmp);
7115 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7116 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7117 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7118 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7119 /* compute queues */
7120 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7121 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7122 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7123 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7124 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7125 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7126 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7127 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7129 WREG32(GRBM_INT_CNTL, 0);
7130 /* vline/vblank, etc. */
7131 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7132 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7133 if (rdev->num_crtc >= 4) {
7134 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7135 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7137 if (rdev->num_crtc >= 6) {
7138 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7139 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7142 if (rdev->num_crtc >= 2) {
7143 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7144 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7146 if (rdev->num_crtc >= 4) {
7147 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7148 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7150 if (rdev->num_crtc >= 6) {
7151 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7152 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7156 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7158 /* digital hotplug */
7159 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7160 WREG32(DC_HPD1_INT_CONTROL, tmp);
7161 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7162 WREG32(DC_HPD2_INT_CONTROL, tmp);
7163 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7164 WREG32(DC_HPD3_INT_CONTROL, tmp);
7165 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7166 WREG32(DC_HPD4_INT_CONTROL, tmp);
7167 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7168 WREG32(DC_HPD5_INT_CONTROL, tmp);
7169 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7170 WREG32(DC_HPD6_INT_CONTROL, tmp);
7175 * cik_irq_init - init and enable the interrupt ring
7177 * @rdev: radeon_device pointer
7179 * Allocate a ring buffer for the interrupt controller,
7180 * enable the RLC, disable interrupts, enable the IH
7181 * ring buffer and enable it (CIK).
7182 * Called at device load and reume.
7183 * Returns 0 for success, errors for failure.
7185 static int cik_irq_init(struct radeon_device *rdev)
7189 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7192 ret = r600_ih_ring_alloc(rdev);
7197 cik_disable_interrupts(rdev);
7200 ret = cik_rlc_resume(rdev);
7202 r600_ih_ring_fini(rdev);
7206 /* setup interrupt control */
7207 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7208 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7209 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7210 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7211 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7213 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7214 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7215 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7216 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7218 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7219 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7221 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7222 IH_WPTR_OVERFLOW_CLEAR |
7225 if (rdev->wb.enabled)
7226 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7228 /* set the writeback address whether it's enabled or not */
7229 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7230 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7232 WREG32(IH_RB_CNTL, ih_rb_cntl);
7234 /* set rptr, wptr to 0 */
7235 WREG32(IH_RB_RPTR, 0);
7236 WREG32(IH_RB_WPTR, 0);
7238 /* Default settings for IH_CNTL (disabled at first) */
7239 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7240 /* RPTR_REARM only works if msi's are enabled */
7241 if (rdev->msi_enabled)
7242 ih_cntl |= RPTR_REARM;
7243 WREG32(IH_CNTL, ih_cntl);
7245 /* force the active interrupt state to all disabled */
7246 cik_disable_interrupt_state(rdev);
7248 pci_set_master(rdev->pdev);
7251 cik_enable_interrupts(rdev);
7257 * cik_irq_set - enable/disable interrupt sources
7259 * @rdev: radeon_device pointer
7261 * Enable interrupt sources on the GPU (vblanks, hpd,
7263 * Returns 0 for success, errors for failure.
7265 int cik_irq_set(struct radeon_device *rdev)
7268 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7269 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7270 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7271 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7272 u32 grbm_int_cntl = 0;
7273 u32 dma_cntl, dma_cntl1;
7276 if (!rdev->irq.installed) {
7277 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7280 /* don't enable anything if the ih is disabled */
7281 if (!rdev->ih.enabled) {
7282 cik_disable_interrupts(rdev);
7283 /* force the active interrupt state to all disabled */
7284 cik_disable_interrupt_state(rdev);
7288 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7289 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7290 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7292 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7293 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7294 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7295 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7296 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7297 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7299 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7300 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7302 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7303 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7304 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7305 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7306 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7307 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7308 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7309 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7311 if (rdev->flags & RADEON_IS_IGP)
7312 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7313 ~(THERM_INTH_MASK | THERM_INTL_MASK);
7315 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7316 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
7318 /* enable CP interrupts on all rings */
7319 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7320 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7321 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7323 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7324 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7325 DRM_DEBUG("si_irq_set: sw int cp1\n");
7326 if (ring->me == 1) {
7327 switch (ring->pipe) {
7329 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7332 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7335 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7338 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7341 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7344 } else if (ring->me == 2) {
7345 switch (ring->pipe) {
7347 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7350 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7353 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7356 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7359 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7363 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7366 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7367 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7368 DRM_DEBUG("si_irq_set: sw int cp2\n");
7369 if (ring->me == 1) {
7370 switch (ring->pipe) {
7372 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7375 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7378 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7381 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7384 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7387 } else if (ring->me == 2) {
7388 switch (ring->pipe) {
7390 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7393 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7396 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7399 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7402 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7406 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7410 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7411 DRM_DEBUG("cik_irq_set: sw int dma\n");
7412 dma_cntl |= TRAP_ENABLE;
7415 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7416 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7417 dma_cntl1 |= TRAP_ENABLE;
7420 if (rdev->irq.crtc_vblank_int[0] ||
7421 atomic_read(&rdev->irq.pflip[0])) {
7422 DRM_DEBUG("cik_irq_set: vblank 0\n");
7423 crtc1 |= VBLANK_INTERRUPT_MASK;
7425 if (rdev->irq.crtc_vblank_int[1] ||
7426 atomic_read(&rdev->irq.pflip[1])) {
7427 DRM_DEBUG("cik_irq_set: vblank 1\n");
7428 crtc2 |= VBLANK_INTERRUPT_MASK;
7430 if (rdev->irq.crtc_vblank_int[2] ||
7431 atomic_read(&rdev->irq.pflip[2])) {
7432 DRM_DEBUG("cik_irq_set: vblank 2\n");
7433 crtc3 |= VBLANK_INTERRUPT_MASK;
7435 if (rdev->irq.crtc_vblank_int[3] ||
7436 atomic_read(&rdev->irq.pflip[3])) {
7437 DRM_DEBUG("cik_irq_set: vblank 3\n");
7438 crtc4 |= VBLANK_INTERRUPT_MASK;
7440 if (rdev->irq.crtc_vblank_int[4] ||
7441 atomic_read(&rdev->irq.pflip[4])) {
7442 DRM_DEBUG("cik_irq_set: vblank 4\n");
7443 crtc5 |= VBLANK_INTERRUPT_MASK;
7445 if (rdev->irq.crtc_vblank_int[5] ||
7446 atomic_read(&rdev->irq.pflip[5])) {
7447 DRM_DEBUG("cik_irq_set: vblank 5\n");
7448 crtc6 |= VBLANK_INTERRUPT_MASK;
7450 if (rdev->irq.hpd[0]) {
7451 DRM_DEBUG("cik_irq_set: hpd 1\n");
7452 hpd1 |= DC_HPDx_INT_EN;
7454 if (rdev->irq.hpd[1]) {
7455 DRM_DEBUG("cik_irq_set: hpd 2\n");
7456 hpd2 |= DC_HPDx_INT_EN;
7458 if (rdev->irq.hpd[2]) {
7459 DRM_DEBUG("cik_irq_set: hpd 3\n");
7460 hpd3 |= DC_HPDx_INT_EN;
7462 if (rdev->irq.hpd[3]) {
7463 DRM_DEBUG("cik_irq_set: hpd 4\n");
7464 hpd4 |= DC_HPDx_INT_EN;
7466 if (rdev->irq.hpd[4]) {
7467 DRM_DEBUG("cik_irq_set: hpd 5\n");
7468 hpd5 |= DC_HPDx_INT_EN;
7470 if (rdev->irq.hpd[5]) {
7471 DRM_DEBUG("cik_irq_set: hpd 6\n");
7472 hpd6 |= DC_HPDx_INT_EN;
7475 if (rdev->irq.dpm_thermal) {
7476 DRM_DEBUG("dpm thermal\n");
7477 if (rdev->flags & RADEON_IS_IGP)
7478 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7480 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7483 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7485 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7486 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7488 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7489 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7490 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7491 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7492 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7493 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7494 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7495 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7497 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7499 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7500 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7501 if (rdev->num_crtc >= 4) {
7502 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7503 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7505 if (rdev->num_crtc >= 6) {
7506 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7507 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7510 if (rdev->num_crtc >= 2) {
7511 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7512 GRPH_PFLIP_INT_MASK);
7513 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7514 GRPH_PFLIP_INT_MASK);
7516 if (rdev->num_crtc >= 4) {
7517 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7518 GRPH_PFLIP_INT_MASK);
7519 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7520 GRPH_PFLIP_INT_MASK);
7522 if (rdev->num_crtc >= 6) {
7523 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7524 GRPH_PFLIP_INT_MASK);
7525 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7526 GRPH_PFLIP_INT_MASK);
7529 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7530 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7531 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7532 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7533 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7534 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7536 if (rdev->flags & RADEON_IS_IGP)
7537 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7539 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7545 * cik_irq_ack - ack interrupt sources
7547 * @rdev: radeon_device pointer
7549 * Ack interrupt sources on the GPU (vblanks, hpd,
7550 * etc.) (CIK). Certain interrupts sources are sw
7551 * generated and do not require an explicit ack.
7553 static inline void cik_irq_ack(struct radeon_device *rdev)
7557 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7558 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7559 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7560 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7561 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7562 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7563 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7565 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7566 EVERGREEN_CRTC0_REGISTER_OFFSET);
7567 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7568 EVERGREEN_CRTC1_REGISTER_OFFSET);
7569 if (rdev->num_crtc >= 4) {
7570 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7571 EVERGREEN_CRTC2_REGISTER_OFFSET);
7572 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7573 EVERGREEN_CRTC3_REGISTER_OFFSET);
7575 if (rdev->num_crtc >= 6) {
7576 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7577 EVERGREEN_CRTC4_REGISTER_OFFSET);
7578 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7579 EVERGREEN_CRTC5_REGISTER_OFFSET);
7582 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7583 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7584 GRPH_PFLIP_INT_CLEAR);
7585 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7586 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7587 GRPH_PFLIP_INT_CLEAR);
7588 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7589 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7590 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7591 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7592 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7593 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7594 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7595 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7597 if (rdev->num_crtc >= 4) {
7598 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7599 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7600 GRPH_PFLIP_INT_CLEAR);
7601 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7602 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7603 GRPH_PFLIP_INT_CLEAR);
7604 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7605 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7606 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7607 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7608 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7609 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7610 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7611 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7614 if (rdev->num_crtc >= 6) {
7615 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7616 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7617 GRPH_PFLIP_INT_CLEAR);
7618 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7619 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7620 GRPH_PFLIP_INT_CLEAR);
7621 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7622 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7623 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7624 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7625 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7626 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7627 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7628 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7631 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7632 tmp = RREG32(DC_HPD1_INT_CONTROL);
7633 tmp |= DC_HPDx_INT_ACK;
7634 WREG32(DC_HPD1_INT_CONTROL, tmp);
7636 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7637 tmp = RREG32(DC_HPD2_INT_CONTROL);
7638 tmp |= DC_HPDx_INT_ACK;
7639 WREG32(DC_HPD2_INT_CONTROL, tmp);
7641 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7642 tmp = RREG32(DC_HPD3_INT_CONTROL);
7643 tmp |= DC_HPDx_INT_ACK;
7644 WREG32(DC_HPD3_INT_CONTROL, tmp);
7646 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7647 tmp = RREG32(DC_HPD4_INT_CONTROL);
7648 tmp |= DC_HPDx_INT_ACK;
7649 WREG32(DC_HPD4_INT_CONTROL, tmp);
7651 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7652 tmp = RREG32(DC_HPD5_INT_CONTROL);
7653 tmp |= DC_HPDx_INT_ACK;
7654 WREG32(DC_HPD5_INT_CONTROL, tmp);
7656 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7657 tmp = RREG32(DC_HPD5_INT_CONTROL);
7658 tmp |= DC_HPDx_INT_ACK;
7659 WREG32(DC_HPD6_INT_CONTROL, tmp);
7664 * cik_irq_disable - disable interrupts
7666 * @rdev: radeon_device pointer
7668 * Disable interrupts on the hw (CIK).
7670 static void cik_irq_disable(struct radeon_device *rdev)
7672 cik_disable_interrupts(rdev);
7673 /* Wait and acknowledge irq */
7676 cik_disable_interrupt_state(rdev);
7680 * cik_irq_disable - disable interrupts for suspend
7682 * @rdev: radeon_device pointer
7684 * Disable interrupts and stop the RLC (CIK).
7687 static void cik_irq_suspend(struct radeon_device *rdev)
7689 cik_irq_disable(rdev);
7694 * cik_irq_fini - tear down interrupt support
7696 * @rdev: radeon_device pointer
7698 * Disable interrupts on the hw and free the IH ring
7700 * Used for driver unload.
7702 static void cik_irq_fini(struct radeon_device *rdev)
7704 cik_irq_suspend(rdev);
7705 r600_ih_ring_fini(rdev);
7709 * cik_get_ih_wptr - get the IH ring buffer wptr
7711 * @rdev: radeon_device pointer
7713 * Get the IH ring buffer wptr from either the register
7714 * or the writeback memory buffer (CIK). Also check for
7715 * ring buffer overflow and deal with it.
7716 * Used by cik_irq_process().
7717 * Returns the value of the wptr.
7719 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7723 if (rdev->wb.enabled)
7724 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7726 wptr = RREG32(IH_RB_WPTR);
7728 if (wptr & RB_OVERFLOW) {
7729 /* When a ring buffer overflow happen start parsing interrupt
7730 * from the last not overwritten vector (wptr + 16). Hopefully
7731 * this should allow us to catchup.
7733 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7734 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7735 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7736 tmp = RREG32(IH_RB_CNTL);
7737 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7738 WREG32(IH_RB_CNTL, tmp);
7739 wptr &= ~RB_OVERFLOW;
7741 return (wptr & rdev->ih.ptr_mask);
7745 * Each IV ring entry is 128 bits:
7746 * [7:0] - interrupt source id
7748 * [59:32] - interrupt source data
7749 * [63:60] - reserved
7752 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7753 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7754 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7755 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7756 * PIPE_ID - ME0 0=3D
7757 * - ME1&2 compute dispatcher (4 pipes each)
7759 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7760 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7761 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7764 * [127:96] - reserved
7767 * cik_irq_process - interrupt handler
7769 * @rdev: radeon_device pointer
7771 * Interrupt hander (CIK). Walk the IH ring,
7772 * ack interrupts and schedule work to handle
7774 * Returns irq process return code.
7776 int cik_irq_process(struct radeon_device *rdev)
7778 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7779 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7782 u32 src_id, src_data, ring_id;
7783 u8 me_id, pipe_id, queue_id;
7785 bool queue_hotplug = false;
7786 bool queue_reset = false;
7787 u32 addr, status, mc_client;
7788 bool queue_thermal = false;
7790 if (!rdev->ih.enabled || rdev->shutdown)
7793 wptr = cik_get_ih_wptr(rdev);
7796 /* is somebody else already processing irqs? */
7797 if (atomic_xchg(&rdev->ih.lock, 1))
7800 rptr = rdev->ih.rptr;
7801 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7803 /* Order reading of wptr vs. reading of IH ring data */
7806 /* display interrupts */
7809 while (rptr != wptr) {
7810 /* wptr/rptr are in bytes! */
7811 ring_index = rptr / 4;
7812 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7813 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7814 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7817 case 1: /* D1 vblank/vline */
7819 case 0: /* D1 vblank */
7820 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7821 if (rdev->irq.crtc_vblank_int[0]) {
7822 drm_handle_vblank(rdev->ddev, 0);
7823 rdev->pm.vblank_sync = true;
7824 wake_up(&rdev->irq.vblank_queue);
7826 if (atomic_read(&rdev->irq.pflip[0]))
7827 radeon_crtc_handle_vblank(rdev, 0);
7828 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7829 DRM_DEBUG("IH: D1 vblank\n");
7832 case 1: /* D1 vline */
7833 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7834 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7835 DRM_DEBUG("IH: D1 vline\n");
7839 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7843 case 2: /* D2 vblank/vline */
7845 case 0: /* D2 vblank */
7846 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7847 if (rdev->irq.crtc_vblank_int[1]) {
7848 drm_handle_vblank(rdev->ddev, 1);
7849 rdev->pm.vblank_sync = true;
7850 wake_up(&rdev->irq.vblank_queue);
7852 if (atomic_read(&rdev->irq.pflip[1]))
7853 radeon_crtc_handle_vblank(rdev, 1);
7854 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7855 DRM_DEBUG("IH: D2 vblank\n");
7858 case 1: /* D2 vline */
7859 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7860 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7861 DRM_DEBUG("IH: D2 vline\n");
7865 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7869 case 3: /* D3 vblank/vline */
7871 case 0: /* D3 vblank */
7872 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7873 if (rdev->irq.crtc_vblank_int[2]) {
7874 drm_handle_vblank(rdev->ddev, 2);
7875 rdev->pm.vblank_sync = true;
7876 wake_up(&rdev->irq.vblank_queue);
7878 if (atomic_read(&rdev->irq.pflip[2]))
7879 radeon_crtc_handle_vblank(rdev, 2);
7880 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7881 DRM_DEBUG("IH: D3 vblank\n");
7884 case 1: /* D3 vline */
7885 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7886 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7887 DRM_DEBUG("IH: D3 vline\n");
7891 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7895 case 4: /* D4 vblank/vline */
7897 case 0: /* D4 vblank */
7898 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7899 if (rdev->irq.crtc_vblank_int[3]) {
7900 drm_handle_vblank(rdev->ddev, 3);
7901 rdev->pm.vblank_sync = true;
7902 wake_up(&rdev->irq.vblank_queue);
7904 if (atomic_read(&rdev->irq.pflip[3]))
7905 radeon_crtc_handle_vblank(rdev, 3);
7906 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7907 DRM_DEBUG("IH: D4 vblank\n");
7910 case 1: /* D4 vline */
7911 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7912 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7913 DRM_DEBUG("IH: D4 vline\n");
7917 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7921 case 5: /* D5 vblank/vline */
7923 case 0: /* D5 vblank */
7924 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7925 if (rdev->irq.crtc_vblank_int[4]) {
7926 drm_handle_vblank(rdev->ddev, 4);
7927 rdev->pm.vblank_sync = true;
7928 wake_up(&rdev->irq.vblank_queue);
7930 if (atomic_read(&rdev->irq.pflip[4]))
7931 radeon_crtc_handle_vblank(rdev, 4);
7932 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7933 DRM_DEBUG("IH: D5 vblank\n");
7936 case 1: /* D5 vline */
7937 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7938 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7939 DRM_DEBUG("IH: D5 vline\n");
7943 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7947 case 6: /* D6 vblank/vline */
7949 case 0: /* D6 vblank */
7950 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7951 if (rdev->irq.crtc_vblank_int[5]) {
7952 drm_handle_vblank(rdev->ddev, 5);
7953 rdev->pm.vblank_sync = true;
7954 wake_up(&rdev->irq.vblank_queue);
7956 if (atomic_read(&rdev->irq.pflip[5]))
7957 radeon_crtc_handle_vblank(rdev, 5);
7958 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7959 DRM_DEBUG("IH: D6 vblank\n");
7962 case 1: /* D6 vline */
7963 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7964 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7965 DRM_DEBUG("IH: D6 vline\n");
7969 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7973 case 8: /* D1 page flip */
7974 case 10: /* D2 page flip */
7975 case 12: /* D3 page flip */
7976 case 14: /* D4 page flip */
7977 case 16: /* D5 page flip */
7978 case 18: /* D6 page flip */
7979 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7980 if (radeon_use_pflipirq > 0)
7981 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7983 case 42: /* HPD hotplug */
7986 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7987 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7988 queue_hotplug = true;
7989 DRM_DEBUG("IH: HPD1\n");
7993 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7994 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7995 queue_hotplug = true;
7996 DRM_DEBUG("IH: HPD2\n");
8000 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8001 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8002 queue_hotplug = true;
8003 DRM_DEBUG("IH: HPD3\n");
8007 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8008 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8009 queue_hotplug = true;
8010 DRM_DEBUG("IH: HPD4\n");
8014 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8015 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8016 queue_hotplug = true;
8017 DRM_DEBUG("IH: HPD5\n");
8021 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8022 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8023 queue_hotplug = true;
8024 DRM_DEBUG("IH: HPD6\n");
8028 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8033 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8034 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8038 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8039 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8040 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8041 /* reset addr and status */
8042 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8043 if (addr == 0x0 && status == 0x0)
8045 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8046 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
8048 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8050 cik_vm_decode_fault(rdev, status, addr, mc_client);
8053 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8056 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8059 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8062 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8066 case 176: /* GFX RB CP_INT */
8067 case 177: /* GFX IB CP_INT */
8068 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8070 case 181: /* CP EOP event */
8071 DRM_DEBUG("IH: CP EOP\n");
8072 /* XXX check the bitfield order! */
8073 me_id = (ring_id & 0x60) >> 5;
8074 pipe_id = (ring_id & 0x18) >> 3;
8075 queue_id = (ring_id & 0x7) >> 0;
8078 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8082 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8083 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8084 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8085 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8089 case 184: /* CP Privileged reg access */
8090 DRM_ERROR("Illegal register access in command stream\n");
8091 /* XXX check the bitfield order! */
8092 me_id = (ring_id & 0x60) >> 5;
8093 pipe_id = (ring_id & 0x18) >> 3;
8094 queue_id = (ring_id & 0x7) >> 0;
8097 /* This results in a full GPU reset, but all we need to do is soft
8098 * reset the CP for gfx
8112 case 185: /* CP Privileged inst */
8113 DRM_ERROR("Illegal instruction in command stream\n");
8114 /* XXX check the bitfield order! */
8115 me_id = (ring_id & 0x60) >> 5;
8116 pipe_id = (ring_id & 0x18) >> 3;
8117 queue_id = (ring_id & 0x7) >> 0;
8120 /* This results in a full GPU reset, but all we need to do is soft
8121 * reset the CP for gfx
8135 case 224: /* SDMA trap event */
8136 /* XXX check the bitfield order! */
8137 me_id = (ring_id & 0x3) >> 0;
8138 queue_id = (ring_id & 0xc) >> 2;
8139 DRM_DEBUG("IH: SDMA trap\n");
8144 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8157 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8169 case 230: /* thermal low to high */
8170 DRM_DEBUG("IH: thermal low to high\n");
8171 rdev->pm.dpm.thermal.high_to_low = false;
8172 queue_thermal = true;
8174 case 231: /* thermal high to low */
8175 DRM_DEBUG("IH: thermal high to low\n");
8176 rdev->pm.dpm.thermal.high_to_low = true;
8177 queue_thermal = true;
8179 case 233: /* GUI IDLE */
8180 DRM_DEBUG("IH: GUI idle\n");
8182 case 241: /* SDMA Privileged inst */
8183 case 247: /* SDMA Privileged inst */
8184 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8185 /* XXX check the bitfield order! */
8186 me_id = (ring_id & 0x3) >> 0;
8187 queue_id = (ring_id & 0xc) >> 2;
8222 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8226 /* wptr/rptr are in bytes! */
8228 rptr &= rdev->ih.ptr_mask;
8231 schedule_work(&rdev->hotplug_work);
8233 schedule_work(&rdev->reset_work);
8235 schedule_work(&rdev->pm.dpm.thermal.work);
8236 rdev->ih.rptr = rptr;
8237 WREG32(IH_RB_RPTR, rdev->ih.rptr);
8238 atomic_set(&rdev->ih.lock, 0);
8240 /* make sure wptr hasn't changed while processing */
8241 wptr = cik_get_ih_wptr(rdev);
8249 * startup/shutdown callbacks
8252 * cik_startup - program the asic to a functional state
8254 * @rdev: radeon_device pointer
8256 * Programs the asic to a functional state (CIK).
8257 * Called by cik_init() and cik_resume().
8258 * Returns 0 for success, error for failure.
8260 static int cik_startup(struct radeon_device *rdev)
8262 struct radeon_ring *ring;
8266 /* enable pcie gen2/3 link */
8267 cik_pcie_gen3_enable(rdev);
8269 cik_program_aspm(rdev);
8271 /* scratch needs to be initialized before MC */
8272 r = r600_vram_scratch_init(rdev);
8276 cik_mc_program(rdev);
8278 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8279 r = ci_mc_load_microcode(rdev);
8281 DRM_ERROR("Failed to load MC firmware!\n");
8286 r = cik_pcie_gart_enable(rdev);
8291 /* allocate rlc buffers */
8292 if (rdev->flags & RADEON_IS_IGP) {
8293 if (rdev->family == CHIP_KAVERI) {
8294 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8295 rdev->rlc.reg_list_size =
8296 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8298 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8299 rdev->rlc.reg_list_size =
8300 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8303 rdev->rlc.cs_data = ci_cs_data;
8304 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8305 r = sumo_rlc_init(rdev);
8307 DRM_ERROR("Failed to init rlc BOs!\n");
8311 /* allocate wb buffer */
8312 r = radeon_wb_init(rdev);
8316 /* allocate mec buffers */
8317 r = cik_mec_init(rdev);
8319 DRM_ERROR("Failed to init MEC BOs!\n");
8323 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8325 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8329 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8331 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8335 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8337 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8341 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8343 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8347 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8349 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8353 r = radeon_uvd_resume(rdev);
8355 r = uvd_v4_2_resume(rdev);
8357 r = radeon_fence_driver_start_ring(rdev,
8358 R600_RING_TYPE_UVD_INDEX);
8360 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8364 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8366 r = radeon_vce_resume(rdev);
8368 r = vce_v2_0_resume(rdev);
8370 r = radeon_fence_driver_start_ring(rdev,
8371 TN_RING_TYPE_VCE1_INDEX);
8373 r = radeon_fence_driver_start_ring(rdev,
8374 TN_RING_TYPE_VCE2_INDEX);
8377 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8378 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8379 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8383 if (!rdev->irq.installed) {
8384 r = radeon_irq_kms_init(rdev);
8389 r = cik_irq_init(rdev);
8391 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8392 radeon_irq_kms_fini(rdev);
8397 if (rdev->family == CHIP_HAWAII) {
8399 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8401 nop = RADEON_CP_PACKET2;
8403 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8406 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8407 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8412 /* set up the compute queues */
8413 /* type-2 packets are deprecated on MEC, use type-3 instead */
8414 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8415 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8419 ring->me = 1; /* first MEC */
8420 ring->pipe = 0; /* first pipe */
8421 ring->queue = 0; /* first queue */
8422 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8424 /* type-2 packets are deprecated on MEC, use type-3 instead */
8425 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8426 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8430 /* dGPU only have 1 MEC */
8431 ring->me = 1; /* first MEC */
8432 ring->pipe = 0; /* first pipe */
8433 ring->queue = 1; /* second queue */
8434 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8436 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8437 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8438 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8442 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8443 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8444 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8448 r = cik_cp_resume(rdev);
8452 r = cik_sdma_resume(rdev);
8456 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8457 if (ring->ring_size) {
8458 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8461 r = uvd_v1_0_init(rdev);
8463 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8468 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8469 if (ring->ring_size)
8470 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8473 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8474 if (ring->ring_size)
8475 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8479 r = vce_v1_0_init(rdev);
8480 else if (r != -ENOENT)
8481 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8483 r = radeon_ib_pool_init(rdev);
8485 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8489 r = radeon_vm_manager_init(rdev);
8491 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8495 r = dce6_audio_init(rdev);
8503 * cik_resume - resume the asic to a functional state
8505 * @rdev: radeon_device pointer
8507 * Programs the asic to a functional state (CIK).
8509 * Returns 0 for success, error for failure.
8511 int cik_resume(struct radeon_device *rdev)
8516 atom_asic_init(rdev->mode_info.atom_context);
8518 /* init golden registers */
8519 cik_init_golden_registers(rdev);
8521 if (rdev->pm.pm_method == PM_METHOD_DPM)
8522 radeon_pm_resume(rdev);
8524 rdev->accel_working = true;
8525 r = cik_startup(rdev);
8527 DRM_ERROR("cik startup failed on resume\n");
8528 rdev->accel_working = false;
8537 * cik_suspend - suspend the asic
8539 * @rdev: radeon_device pointer
8541 * Bring the chip into a state suitable for suspend (CIK).
8542 * Called at suspend.
8543 * Returns 0 for success.
8545 int cik_suspend(struct radeon_device *rdev)
8547 radeon_pm_suspend(rdev);
8548 dce6_audio_fini(rdev);
8549 radeon_vm_manager_fini(rdev);
8550 cik_cp_enable(rdev, false);
8551 cik_sdma_enable(rdev, false);
8552 uvd_v1_0_fini(rdev);
8553 radeon_uvd_suspend(rdev);
8554 radeon_vce_suspend(rdev);
8557 cik_irq_suspend(rdev);
8558 radeon_wb_disable(rdev);
8559 cik_pcie_gart_disable(rdev);
8563 /* Plan is to move initialization in that function and use
8564 * helper function so that radeon_device_init pretty much
8565 * do nothing more than calling asic specific function. This
8566 * should also allow to remove a bunch of callback function
8570 * cik_init - asic specific driver and hw init
8572 * @rdev: radeon_device pointer
8574 * Setup asic specific driver variables and program the hw
8575 * to a functional state (CIK).
8576 * Called at driver startup.
8577 * Returns 0 for success, errors for failure.
8579 int cik_init(struct radeon_device *rdev)
8581 struct radeon_ring *ring;
8585 if (!radeon_get_bios(rdev)) {
8586 if (ASIC_IS_AVIVO(rdev))
8589 /* Must be an ATOMBIOS */
8590 if (!rdev->is_atom_bios) {
8591 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8594 r = radeon_atombios_init(rdev);
8598 /* Post card if necessary */
8599 if (!radeon_card_posted(rdev)) {
8601 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8604 DRM_INFO("GPU not posted. posting now...\n");
8605 atom_asic_init(rdev->mode_info.atom_context);
8607 /* init golden registers */
8608 cik_init_golden_registers(rdev);
8609 /* Initialize scratch registers */
8610 cik_scratch_init(rdev);
8611 /* Initialize surface registers */
8612 radeon_surface_init(rdev);
8613 /* Initialize clocks */
8614 radeon_get_clock_info(rdev->ddev);
8617 r = radeon_fence_driver_init(rdev);
8621 /* initialize memory controller */
8622 r = cik_mc_init(rdev);
8625 /* Memory manager */
8626 r = radeon_bo_init(rdev);
8630 if (rdev->flags & RADEON_IS_IGP) {
8631 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8632 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8633 r = cik_init_microcode(rdev);
8635 DRM_ERROR("Failed to load firmware!\n");
8640 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8641 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8643 r = cik_init_microcode(rdev);
8645 DRM_ERROR("Failed to load firmware!\n");
8651 /* Initialize power management */
8652 radeon_pm_init(rdev);
8654 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8655 ring->ring_obj = NULL;
8656 r600_ring_init(rdev, ring, 1024 * 1024);
8658 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8659 ring->ring_obj = NULL;
8660 r600_ring_init(rdev, ring, 1024 * 1024);
8661 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8665 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8666 ring->ring_obj = NULL;
8667 r600_ring_init(rdev, ring, 1024 * 1024);
8668 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8672 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8673 ring->ring_obj = NULL;
8674 r600_ring_init(rdev, ring, 256 * 1024);
8676 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8677 ring->ring_obj = NULL;
8678 r600_ring_init(rdev, ring, 256 * 1024);
8680 r = radeon_uvd_init(rdev);
8682 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8683 ring->ring_obj = NULL;
8684 r600_ring_init(rdev, ring, 4096);
8687 r = radeon_vce_init(rdev);
8689 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8690 ring->ring_obj = NULL;
8691 r600_ring_init(rdev, ring, 4096);
8693 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8694 ring->ring_obj = NULL;
8695 r600_ring_init(rdev, ring, 4096);
8698 rdev->ih.ring_obj = NULL;
8699 r600_ih_ring_init(rdev, 64 * 1024);
8701 r = r600_pcie_gart_init(rdev);
8705 rdev->accel_working = true;
8706 r = cik_startup(rdev);
8708 dev_err(rdev->dev, "disabling GPU acceleration\n");
8710 cik_sdma_fini(rdev);
8712 sumo_rlc_fini(rdev);
8714 radeon_wb_fini(rdev);
8715 radeon_ib_pool_fini(rdev);
8716 radeon_vm_manager_fini(rdev);
8717 radeon_irq_kms_fini(rdev);
8718 cik_pcie_gart_fini(rdev);
8719 rdev->accel_working = false;
8722 /* Don't start up if the MC ucode is missing.
8723 * The default clocks and voltages before the MC ucode
8724 * is loaded are not suffient for advanced operations.
8726 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8727 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8735 * cik_fini - asic specific driver and hw fini
8737 * @rdev: radeon_device pointer
8739 * Tear down the asic specific driver variables and program the hw
8740 * to an idle state (CIK).
8741 * Called at driver unload.
8743 void cik_fini(struct radeon_device *rdev)
8745 radeon_pm_fini(rdev);
8747 cik_sdma_fini(rdev);
8751 sumo_rlc_fini(rdev);
8753 radeon_wb_fini(rdev);
8754 radeon_vm_manager_fini(rdev);
8755 radeon_ib_pool_fini(rdev);
8756 radeon_irq_kms_fini(rdev);
8757 uvd_v1_0_fini(rdev);
8758 radeon_uvd_fini(rdev);
8759 radeon_vce_fini(rdev);
8760 cik_pcie_gart_fini(rdev);
8761 r600_vram_scratch_fini(rdev);
8762 radeon_gem_fini(rdev);
8763 radeon_fence_driver_fini(rdev);
8764 radeon_bo_fini(rdev);
8765 radeon_atombios_fini(rdev);
8770 void dce8_program_fmt(struct drm_encoder *encoder)
8772 struct drm_device *dev = encoder->dev;
8773 struct radeon_device *rdev = dev->dev_private;
8774 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8775 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8776 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8779 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8782 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8783 bpc = radeon_get_monitor_bpc(connector);
8784 dither = radeon_connector->dither;
8787 /* LVDS/eDP FMT is set up by atom */
8788 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8791 /* not needed for analog */
8792 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8793 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8801 if (dither == RADEON_FMT_DITHER_ENABLE)
8802 /* XXX sort out optimal dither settings */
8803 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8804 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8806 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8809 if (dither == RADEON_FMT_DITHER_ENABLE)
8810 /* XXX sort out optimal dither settings */
8811 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8812 FMT_RGB_RANDOM_ENABLE |
8813 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8815 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8818 if (dither == RADEON_FMT_DITHER_ENABLE)
8819 /* XXX sort out optimal dither settings */
8820 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8821 FMT_RGB_RANDOM_ENABLE |
8822 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8824 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8831 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8834 /* display watermark setup */
8836 * dce8_line_buffer_adjust - Set up the line buffer
8838 * @rdev: radeon_device pointer
8839 * @radeon_crtc: the selected display controller
8840 * @mode: the current display mode on the selected display
8843 * Setup up the line buffer allocation for
8844 * the selected display controller (CIK).
8845 * Returns the line buffer size in pixels.
8847 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8848 struct radeon_crtc *radeon_crtc,
8849 struct drm_display_mode *mode)
8851 u32 tmp, buffer_alloc, i;
8852 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8855 * There are 6 line buffers, one for each display controllers.
8856 * There are 3 partitions per LB. Select the number of partitions
8857 * to enable based on the display width. For display widths larger
8858 * than 4096, you need use to use 2 display controllers and combine
8859 * them using the stereo blender.
8861 if (radeon_crtc->base.enabled && mode) {
8862 if (mode->crtc_hdisplay < 1920) {
8865 } else if (mode->crtc_hdisplay < 2560) {
8868 } else if (mode->crtc_hdisplay < 4096) {
8870 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8872 DRM_DEBUG_KMS("Mode too big for LB!\n");
8874 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8881 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8882 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8884 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8885 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8886 for (i = 0; i < rdev->usec_timeout; i++) {
8887 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8888 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8893 if (radeon_crtc->base.enabled && mode) {
8905 /* controller not enabled, so no lb used */
8910 * cik_get_number_of_dram_channels - get the number of dram channels
8912 * @rdev: radeon_device pointer
8914 * Look up the number of video ram channels (CIK).
8915 * Used for display watermark bandwidth calculations
8916 * Returns the number of dram channels
8918 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8920 u32 tmp = RREG32(MC_SHARED_CHMAP);
8922 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8945 struct dce8_wm_params {
8946 u32 dram_channels; /* number of dram channels */
8947 u32 yclk; /* bandwidth per dram data pin in kHz */
8948 u32 sclk; /* engine clock in kHz */
8949 u32 disp_clk; /* display clock in kHz */
8950 u32 src_width; /* viewport width */
8951 u32 active_time; /* active display time in ns */
8952 u32 blank_time; /* blank time in ns */
8953 bool interlaced; /* mode is interlaced */
8954 fixed20_12 vsc; /* vertical scale ratio */
8955 u32 num_heads; /* number of active crtcs */
8956 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8957 u32 lb_size; /* line buffer allocated to pipe */
8958 u32 vtaps; /* vertical scaler taps */
8962 * dce8_dram_bandwidth - get the dram bandwidth
8964 * @wm: watermark calculation data
8966 * Calculate the raw dram bandwidth (CIK).
8967 * Used for display watermark bandwidth calculations
8968 * Returns the dram bandwidth in MBytes/s
8970 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8972 /* Calculate raw DRAM Bandwidth */
8973 fixed20_12 dram_efficiency; /* 0.7 */
8974 fixed20_12 yclk, dram_channels, bandwidth;
8977 a.full = dfixed_const(1000);
8978 yclk.full = dfixed_const(wm->yclk);
8979 yclk.full = dfixed_div(yclk, a);
8980 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8981 a.full = dfixed_const(10);
8982 dram_efficiency.full = dfixed_const(7);
8983 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8984 bandwidth.full = dfixed_mul(dram_channels, yclk);
8985 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8987 return dfixed_trunc(bandwidth);
8991 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8993 * @wm: watermark calculation data
8995 * Calculate the dram bandwidth used for display (CIK).
8996 * Used for display watermark bandwidth calculations
8997 * Returns the dram bandwidth for display in MBytes/s
8999 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9001 /* Calculate DRAM Bandwidth and the part allocated to display. */
9002 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9003 fixed20_12 yclk, dram_channels, bandwidth;
9006 a.full = dfixed_const(1000);
9007 yclk.full = dfixed_const(wm->yclk);
9008 yclk.full = dfixed_div(yclk, a);
9009 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9010 a.full = dfixed_const(10);
9011 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9012 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9013 bandwidth.full = dfixed_mul(dram_channels, yclk);
9014 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9016 return dfixed_trunc(bandwidth);
9020 * dce8_data_return_bandwidth - get the data return bandwidth
9022 * @wm: watermark calculation data
9024 * Calculate the data return bandwidth used for display (CIK).
9025 * Used for display watermark bandwidth calculations
9026 * Returns the data return bandwidth in MBytes/s
9028 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9030 /* Calculate the display Data return Bandwidth */
9031 fixed20_12 return_efficiency; /* 0.8 */
9032 fixed20_12 sclk, bandwidth;
9035 a.full = dfixed_const(1000);
9036 sclk.full = dfixed_const(wm->sclk);
9037 sclk.full = dfixed_div(sclk, a);
9038 a.full = dfixed_const(10);
9039 return_efficiency.full = dfixed_const(8);
9040 return_efficiency.full = dfixed_div(return_efficiency, a);
9041 a.full = dfixed_const(32);
9042 bandwidth.full = dfixed_mul(a, sclk);
9043 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9045 return dfixed_trunc(bandwidth);
9049 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9051 * @wm: watermark calculation data
9053 * Calculate the dmif bandwidth used for display (CIK).
9054 * Used for display watermark bandwidth calculations
9055 * Returns the dmif bandwidth in MBytes/s
9057 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9059 /* Calculate the DMIF Request Bandwidth */
9060 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9061 fixed20_12 disp_clk, bandwidth;
9064 a.full = dfixed_const(1000);
9065 disp_clk.full = dfixed_const(wm->disp_clk);
9066 disp_clk.full = dfixed_div(disp_clk, a);
9067 a.full = dfixed_const(32);
9068 b.full = dfixed_mul(a, disp_clk);
9070 a.full = dfixed_const(10);
9071 disp_clk_request_efficiency.full = dfixed_const(8);
9072 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9074 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9076 return dfixed_trunc(bandwidth);
9080 * dce8_available_bandwidth - get the min available bandwidth
9082 * @wm: watermark calculation data
9084 * Calculate the min available bandwidth used for display (CIK).
9085 * Used for display watermark bandwidth calculations
9086 * Returns the min available bandwidth in MBytes/s
9088 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9090 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9091 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9092 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9093 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9095 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9099 * dce8_average_bandwidth - get the average available bandwidth
9101 * @wm: watermark calculation data
9103 * Calculate the average available bandwidth used for display (CIK).
9104 * Used for display watermark bandwidth calculations
9105 * Returns the average available bandwidth in MBytes/s
9107 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9109 /* Calculate the display mode Average Bandwidth
9110 * DisplayMode should contain the source and destination dimensions,
9114 fixed20_12 line_time;
9115 fixed20_12 src_width;
9116 fixed20_12 bandwidth;
9119 a.full = dfixed_const(1000);
9120 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9121 line_time.full = dfixed_div(line_time, a);
9122 bpp.full = dfixed_const(wm->bytes_per_pixel);
9123 src_width.full = dfixed_const(wm->src_width);
9124 bandwidth.full = dfixed_mul(src_width, bpp);
9125 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9126 bandwidth.full = dfixed_div(bandwidth, line_time);
9128 return dfixed_trunc(bandwidth);
9132 * dce8_latency_watermark - get the latency watermark
9134 * @wm: watermark calculation data
9136 * Calculate the latency watermark (CIK).
9137 * Used for display watermark bandwidth calculations
9138 * Returns the latency watermark in ns
9140 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9142 /* First calculate the latency in ns */
9143 u32 mc_latency = 2000; /* 2000 ns. */
9144 u32 available_bandwidth = dce8_available_bandwidth(wm);
9145 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9146 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9147 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9148 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9149 (wm->num_heads * cursor_line_pair_return_time);
9150 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9151 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9152 u32 tmp, dmif_size = 12288;
9155 if (wm->num_heads == 0)
9158 a.full = dfixed_const(2);
9159 b.full = dfixed_const(1);
9160 if ((wm->vsc.full > a.full) ||
9161 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9163 ((wm->vsc.full >= a.full) && wm->interlaced))
9164 max_src_lines_per_dst_line = 4;
9166 max_src_lines_per_dst_line = 2;
9168 a.full = dfixed_const(available_bandwidth);
9169 b.full = dfixed_const(wm->num_heads);
9170 a.full = dfixed_div(a, b);
9172 b.full = dfixed_const(mc_latency + 512);
9173 c.full = dfixed_const(wm->disp_clk);
9174 b.full = dfixed_div(b, c);
9176 c.full = dfixed_const(dmif_size);
9177 b.full = dfixed_div(c, b);
9179 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9181 b.full = dfixed_const(1000);
9182 c.full = dfixed_const(wm->disp_clk);
9183 b.full = dfixed_div(c, b);
9184 c.full = dfixed_const(wm->bytes_per_pixel);
9185 b.full = dfixed_mul(b, c);
9187 lb_fill_bw = min(tmp, dfixed_trunc(b));
9189 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9190 b.full = dfixed_const(1000);
9191 c.full = dfixed_const(lb_fill_bw);
9192 b.full = dfixed_div(c, b);
9193 a.full = dfixed_div(a, b);
9194 line_fill_time = dfixed_trunc(a);
9196 if (line_fill_time < wm->active_time)
9199 return latency + (line_fill_time - wm->active_time);
9204 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9205 * average and available dram bandwidth
9207 * @wm: watermark calculation data
9209 * Check if the display average bandwidth fits in the display
9210 * dram bandwidth (CIK).
9211 * Used for display watermark bandwidth calculations
9212 * Returns true if the display fits, false if not.
9214 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9216 if (dce8_average_bandwidth(wm) <=
9217 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9224 * dce8_average_bandwidth_vs_available_bandwidth - check
9225 * average and available bandwidth
9227 * @wm: watermark calculation data
9229 * Check if the display average bandwidth fits in the display
9230 * available bandwidth (CIK).
9231 * Used for display watermark bandwidth calculations
9232 * Returns true if the display fits, false if not.
9234 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9236 if (dce8_average_bandwidth(wm) <=
9237 (dce8_available_bandwidth(wm) / wm->num_heads))
9244 * dce8_check_latency_hiding - check latency hiding
9246 * @wm: watermark calculation data
9248 * Check latency hiding (CIK).
9249 * Used for display watermark bandwidth calculations
9250 * Returns true if the display fits, false if not.
9252 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9254 u32 lb_partitions = wm->lb_size / wm->src_width;
9255 u32 line_time = wm->active_time + wm->blank_time;
9256 u32 latency_tolerant_lines;
9260 a.full = dfixed_const(1);
9261 if (wm->vsc.full > a.full)
9262 latency_tolerant_lines = 1;
9264 if (lb_partitions <= (wm->vtaps + 1))
9265 latency_tolerant_lines = 1;
9267 latency_tolerant_lines = 2;
9270 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9272 if (dce8_latency_watermark(wm) <= latency_hiding)
9279 * dce8_program_watermarks - program display watermarks
9281 * @rdev: radeon_device pointer
9282 * @radeon_crtc: the selected display controller
9283 * @lb_size: line buffer size
9284 * @num_heads: number of display controllers in use
9286 * Calculate and program the display watermarks for the
9287 * selected display controller (CIK).
9289 static void dce8_program_watermarks(struct radeon_device *rdev,
9290 struct radeon_crtc *radeon_crtc,
9291 u32 lb_size, u32 num_heads)
9293 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9294 struct dce8_wm_params wm_low, wm_high;
9297 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9300 if (radeon_crtc->base.enabled && num_heads && mode) {
9301 pixel_period = 1000000 / (u32)mode->clock;
9302 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9304 /* watermark for high clocks */
9305 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9306 rdev->pm.dpm_enabled) {
9308 radeon_dpm_get_mclk(rdev, false) * 10;
9310 radeon_dpm_get_sclk(rdev, false) * 10;
9312 wm_high.yclk = rdev->pm.current_mclk * 10;
9313 wm_high.sclk = rdev->pm.current_sclk * 10;
9316 wm_high.disp_clk = mode->clock;
9317 wm_high.src_width = mode->crtc_hdisplay;
9318 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9319 wm_high.blank_time = line_time - wm_high.active_time;
9320 wm_high.interlaced = false;
9321 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9322 wm_high.interlaced = true;
9323 wm_high.vsc = radeon_crtc->vsc;
9325 if (radeon_crtc->rmx_type != RMX_OFF)
9327 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9328 wm_high.lb_size = lb_size;
9329 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9330 wm_high.num_heads = num_heads;
9332 /* set for high clocks */
9333 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9335 /* possibly force display priority to high */
9336 /* should really do this at mode validation time... */
9337 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9338 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9339 !dce8_check_latency_hiding(&wm_high) ||
9340 (rdev->disp_priority == 2)) {
9341 DRM_DEBUG_KMS("force priority to high\n");
9344 /* watermark for low clocks */
9345 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9346 rdev->pm.dpm_enabled) {
9348 radeon_dpm_get_mclk(rdev, true) * 10;
9350 radeon_dpm_get_sclk(rdev, true) * 10;
9352 wm_low.yclk = rdev->pm.current_mclk * 10;
9353 wm_low.sclk = rdev->pm.current_sclk * 10;
9356 wm_low.disp_clk = mode->clock;
9357 wm_low.src_width = mode->crtc_hdisplay;
9358 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9359 wm_low.blank_time = line_time - wm_low.active_time;
9360 wm_low.interlaced = false;
9361 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9362 wm_low.interlaced = true;
9363 wm_low.vsc = radeon_crtc->vsc;
9365 if (radeon_crtc->rmx_type != RMX_OFF)
9367 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9368 wm_low.lb_size = lb_size;
9369 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9370 wm_low.num_heads = num_heads;
9372 /* set for low clocks */
9373 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9375 /* possibly force display priority to high */
9376 /* should really do this at mode validation time... */
9377 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9378 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9379 !dce8_check_latency_hiding(&wm_low) ||
9380 (rdev->disp_priority == 2)) {
9381 DRM_DEBUG_KMS("force priority to high\n");
9386 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9388 tmp &= ~LATENCY_WATERMARK_MASK(3);
9389 tmp |= LATENCY_WATERMARK_MASK(1);
9390 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9391 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9392 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9393 LATENCY_HIGH_WATERMARK(line_time)));
9395 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9396 tmp &= ~LATENCY_WATERMARK_MASK(3);
9397 tmp |= LATENCY_WATERMARK_MASK(2);
9398 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9399 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9400 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9401 LATENCY_HIGH_WATERMARK(line_time)));
9402 /* restore original selection */
9403 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9405 /* save values for DPM */
9406 radeon_crtc->line_time = line_time;
9407 radeon_crtc->wm_high = latency_watermark_a;
9408 radeon_crtc->wm_low = latency_watermark_b;
9412 * dce8_bandwidth_update - program display watermarks
9414 * @rdev: radeon_device pointer
9416 * Calculate and program the display watermarks and line
9417 * buffer allocation (CIK).
9419 void dce8_bandwidth_update(struct radeon_device *rdev)
9421 struct drm_display_mode *mode = NULL;
9422 u32 num_heads = 0, lb_size;
9425 radeon_update_display_priority(rdev);
9427 for (i = 0; i < rdev->num_crtc; i++) {
9428 if (rdev->mode_info.crtcs[i]->base.enabled)
9431 for (i = 0; i < rdev->num_crtc; i++) {
9432 mode = &rdev->mode_info.crtcs[i]->base.mode;
9433 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9434 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9439 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9441 * @rdev: radeon_device pointer
9443 * Fetches a GPU clock counter snapshot (SI).
9444 * Returns the 64 bit clock counter snapshot.
9446 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9450 mutex_lock(&rdev->gpu_clock_mutex);
9451 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9452 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9453 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9454 mutex_unlock(&rdev->gpu_clock_mutex);
9458 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9459 u32 cntl_reg, u32 status_reg)
9462 struct atom_clock_dividers dividers;
9465 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9466 clock, false, ÷rs);
9470 tmp = RREG32_SMC(cntl_reg);
9471 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9472 tmp |= dividers.post_divider;
9473 WREG32_SMC(cntl_reg, tmp);
9475 for (i = 0; i < 100; i++) {
9476 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9486 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9490 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9494 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9498 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9501 struct atom_clock_dividers dividers;
9504 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9505 ecclk, false, ÷rs);
9509 for (i = 0; i < 100; i++) {
9510 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9517 tmp = RREG32_SMC(CG_ECLK_CNTL);
9518 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9519 tmp |= dividers.post_divider;
9520 WREG32_SMC(CG_ECLK_CNTL, tmp);
9522 for (i = 0; i < 100; i++) {
9523 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9533 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9535 struct pci_dev *root = rdev->pdev->bus->self;
9536 int bridge_pos, gpu_pos;
9537 u32 speed_cntl, mask, current_data_rate;
9541 if (radeon_pcie_gen2 == 0)
9544 if (rdev->flags & RADEON_IS_IGP)
9547 if (!(rdev->flags & RADEON_IS_PCIE))
9550 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9554 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9557 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9558 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9559 LC_CURRENT_DATA_RATE_SHIFT;
9560 if (mask & DRM_PCIE_SPEED_80) {
9561 if (current_data_rate == 2) {
9562 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9565 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9566 } else if (mask & DRM_PCIE_SPEED_50) {
9567 if (current_data_rate == 1) {
9568 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9571 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9574 bridge_pos = pci_pcie_cap(root);
9578 gpu_pos = pci_pcie_cap(rdev->pdev);
9582 if (mask & DRM_PCIE_SPEED_80) {
9583 /* re-try equalization if gen3 is not already enabled */
9584 if (current_data_rate != 2) {
9585 u16 bridge_cfg, gpu_cfg;
9586 u16 bridge_cfg2, gpu_cfg2;
9587 u32 max_lw, current_lw, tmp;
9589 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9590 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9592 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9593 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9595 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9596 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9598 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9599 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9600 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9602 if (current_lw < max_lw) {
9603 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9604 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9605 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9606 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9607 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9608 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9612 for (i = 0; i < 10; i++) {
9614 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9615 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9618 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9619 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9621 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9622 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9624 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9625 tmp |= LC_SET_QUIESCE;
9626 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9628 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9630 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9635 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9636 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9637 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9638 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9640 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9641 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9642 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9643 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9646 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9647 tmp16 &= ~((1 << 4) | (7 << 9));
9648 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9649 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9651 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9652 tmp16 &= ~((1 << 4) | (7 << 9));
9653 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9654 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9656 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9657 tmp &= ~LC_SET_QUIESCE;
9658 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9663 /* set the link speed */
9664 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9665 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9666 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9668 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9670 if (mask & DRM_PCIE_SPEED_80)
9671 tmp16 |= 3; /* gen3 */
9672 else if (mask & DRM_PCIE_SPEED_50)
9673 tmp16 |= 2; /* gen2 */
9675 tmp16 |= 1; /* gen1 */
9676 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9678 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9679 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9680 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9682 for (i = 0; i < rdev->usec_timeout; i++) {
9683 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9684 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9690 static void cik_program_aspm(struct radeon_device *rdev)
9693 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9694 bool disable_clkreq = false;
9696 if (radeon_aspm == 0)
9699 /* XXX double check IGPs */
9700 if (rdev->flags & RADEON_IS_IGP)
9703 if (!(rdev->flags & RADEON_IS_PCIE))
9706 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9707 data &= ~LC_XMIT_N_FTS_MASK;
9708 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9710 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9712 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9713 data |= LC_GO_TO_RECOVERY;
9715 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9717 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9718 data |= P_IGNORE_EDB_ERR;
9720 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9722 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9723 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9724 data |= LC_PMI_TO_L1_DIS;
9726 data |= LC_L0S_INACTIVITY(7);
9729 data |= LC_L1_INACTIVITY(7);
9730 data &= ~LC_PMI_TO_L1_DIS;
9732 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9734 if (!disable_plloff_in_l1) {
9735 bool clk_req_support;
9737 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9738 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9739 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9741 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9743 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9744 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9745 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9747 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9749 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9750 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9751 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9753 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9755 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9756 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9757 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9759 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9761 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9762 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9763 data |= LC_DYN_LANES_PWR_STATE(3);
9765 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9767 if (!disable_clkreq) {
9768 struct pci_dev *root = rdev->pdev->bus->self;
9771 clk_req_support = false;
9772 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9773 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9774 clk_req_support = true;
9776 clk_req_support = false;
9779 if (clk_req_support) {
9780 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9781 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9783 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9785 orig = data = RREG32_SMC(THM_CLK_CNTL);
9786 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9787 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9789 WREG32_SMC(THM_CLK_CNTL, data);
9791 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9792 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9793 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9795 WREG32_SMC(MISC_CLK_CTRL, data);
9797 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9798 data &= ~BCLK_AS_XCLK;
9800 WREG32_SMC(CG_CLKPIN_CNTL, data);
9802 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9803 data &= ~FORCE_BIF_REFCLK_EN;
9805 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9807 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9808 data &= ~MPLL_CLKOUT_SEL_MASK;
9809 data |= MPLL_CLKOUT_SEL(4);
9811 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9816 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9819 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9820 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9822 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9825 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9826 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9827 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9828 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9829 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9830 data &= ~LC_L0S_INACTIVITY_MASK;
9832 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);