2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
33 #include "cik_blit_shaders.h"
34 #include "radeon_ucode.h"
35 #include "clearstate_ci.h"
36 #include "radeon_kfd.h"
38 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
45 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
46 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
48 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
49 MODULE_FIRMWARE("radeon/bonaire_me.bin");
50 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
51 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
52 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
53 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
54 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
55 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
57 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
58 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
59 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
60 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
63 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
64 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
65 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
67 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
68 MODULE_FIRMWARE("radeon/hawaii_me.bin");
69 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
70 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
71 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
72 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
73 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
74 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
76 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
77 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
78 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
79 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
80 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
81 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
83 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
84 MODULE_FIRMWARE("radeon/kaveri_me.bin");
85 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
86 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
87 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
88 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
89 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
91 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
92 MODULE_FIRMWARE("radeon/KABINI_me.bin");
93 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
94 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
95 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
96 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
98 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
99 MODULE_FIRMWARE("radeon/kabini_me.bin");
100 MODULE_FIRMWARE("radeon/kabini_ce.bin");
101 MODULE_FIRMWARE("radeon/kabini_mec.bin");
102 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
103 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
105 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
106 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
107 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
108 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
109 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
110 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
112 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
113 MODULE_FIRMWARE("radeon/mullins_me.bin");
114 MODULE_FIRMWARE("radeon/mullins_ce.bin");
115 MODULE_FIRMWARE("radeon/mullins_mec.bin");
116 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
117 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
119 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
120 extern void r600_ih_ring_fini(struct radeon_device *rdev);
121 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
123 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
124 extern void sumo_rlc_fini(struct radeon_device *rdev);
125 extern int sumo_rlc_init(struct radeon_device *rdev);
126 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
127 extern void si_rlc_reset(struct radeon_device *rdev);
128 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
129 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
130 extern int cik_sdma_resume(struct radeon_device *rdev);
131 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
132 extern void cik_sdma_fini(struct radeon_device *rdev);
133 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
134 static void cik_rlc_stop(struct radeon_device *rdev);
135 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
136 static void cik_program_aspm(struct radeon_device *rdev);
137 static void cik_init_pg(struct radeon_device *rdev);
138 static void cik_init_cg(struct radeon_device *rdev);
139 static void cik_fini_pg(struct radeon_device *rdev);
140 static void cik_fini_cg(struct radeon_device *rdev);
141 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
144 /* get temperature in millidegrees */
145 int ci_get_temp(struct radeon_device *rdev)
150 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
156 actual_temp = temp & 0x1ff;
158 actual_temp = actual_temp * 1000;
163 /* get temperature in millidegrees */
164 int kv_get_temp(struct radeon_device *rdev)
169 temp = RREG32_SMC(0xC0300E0C);
172 actual_temp = (temp / 8) - 49;
176 actual_temp = actual_temp * 1000;
182 * Indirect registers accessor
184 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
189 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
190 WREG32(PCIE_INDEX, reg);
191 (void)RREG32(PCIE_INDEX);
192 r = RREG32(PCIE_DATA);
193 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
197 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
201 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
202 WREG32(PCIE_INDEX, reg);
203 (void)RREG32(PCIE_INDEX);
204 WREG32(PCIE_DATA, v);
205 (void)RREG32(PCIE_DATA);
206 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
209 static const u32 spectre_rlc_save_restore_register_list[] =
211 (0x0e00 << 16) | (0xc12c >> 2),
213 (0x0e00 << 16) | (0xc140 >> 2),
215 (0x0e00 << 16) | (0xc150 >> 2),
217 (0x0e00 << 16) | (0xc15c >> 2),
219 (0x0e00 << 16) | (0xc168 >> 2),
221 (0x0e00 << 16) | (0xc170 >> 2),
223 (0x0e00 << 16) | (0xc178 >> 2),
225 (0x0e00 << 16) | (0xc204 >> 2),
227 (0x0e00 << 16) | (0xc2b4 >> 2),
229 (0x0e00 << 16) | (0xc2b8 >> 2),
231 (0x0e00 << 16) | (0xc2bc >> 2),
233 (0x0e00 << 16) | (0xc2c0 >> 2),
235 (0x0e00 << 16) | (0x8228 >> 2),
237 (0x0e00 << 16) | (0x829c >> 2),
239 (0x0e00 << 16) | (0x869c >> 2),
241 (0x0600 << 16) | (0x98f4 >> 2),
243 (0x0e00 << 16) | (0x98f8 >> 2),
245 (0x0e00 << 16) | (0x9900 >> 2),
247 (0x0e00 << 16) | (0xc260 >> 2),
249 (0x0e00 << 16) | (0x90e8 >> 2),
251 (0x0e00 << 16) | (0x3c000 >> 2),
253 (0x0e00 << 16) | (0x3c00c >> 2),
255 (0x0e00 << 16) | (0x8c1c >> 2),
257 (0x0e00 << 16) | (0x9700 >> 2),
259 (0x0e00 << 16) | (0xcd20 >> 2),
261 (0x4e00 << 16) | (0xcd20 >> 2),
263 (0x5e00 << 16) | (0xcd20 >> 2),
265 (0x6e00 << 16) | (0xcd20 >> 2),
267 (0x7e00 << 16) | (0xcd20 >> 2),
269 (0x8e00 << 16) | (0xcd20 >> 2),
271 (0x9e00 << 16) | (0xcd20 >> 2),
273 (0xae00 << 16) | (0xcd20 >> 2),
275 (0xbe00 << 16) | (0xcd20 >> 2),
277 (0x0e00 << 16) | (0x89bc >> 2),
279 (0x0e00 << 16) | (0x8900 >> 2),
282 (0x0e00 << 16) | (0xc130 >> 2),
284 (0x0e00 << 16) | (0xc134 >> 2),
286 (0x0e00 << 16) | (0xc1fc >> 2),
288 (0x0e00 << 16) | (0xc208 >> 2),
290 (0x0e00 << 16) | (0xc264 >> 2),
292 (0x0e00 << 16) | (0xc268 >> 2),
294 (0x0e00 << 16) | (0xc26c >> 2),
296 (0x0e00 << 16) | (0xc270 >> 2),
298 (0x0e00 << 16) | (0xc274 >> 2),
300 (0x0e00 << 16) | (0xc278 >> 2),
302 (0x0e00 << 16) | (0xc27c >> 2),
304 (0x0e00 << 16) | (0xc280 >> 2),
306 (0x0e00 << 16) | (0xc284 >> 2),
308 (0x0e00 << 16) | (0xc288 >> 2),
310 (0x0e00 << 16) | (0xc28c >> 2),
312 (0x0e00 << 16) | (0xc290 >> 2),
314 (0x0e00 << 16) | (0xc294 >> 2),
316 (0x0e00 << 16) | (0xc298 >> 2),
318 (0x0e00 << 16) | (0xc29c >> 2),
320 (0x0e00 << 16) | (0xc2a0 >> 2),
322 (0x0e00 << 16) | (0xc2a4 >> 2),
324 (0x0e00 << 16) | (0xc2a8 >> 2),
326 (0x0e00 << 16) | (0xc2ac >> 2),
328 (0x0e00 << 16) | (0xc2b0 >> 2),
330 (0x0e00 << 16) | (0x301d0 >> 2),
332 (0x0e00 << 16) | (0x30238 >> 2),
334 (0x0e00 << 16) | (0x30250 >> 2),
336 (0x0e00 << 16) | (0x30254 >> 2),
338 (0x0e00 << 16) | (0x30258 >> 2),
340 (0x0e00 << 16) | (0x3025c >> 2),
342 (0x4e00 << 16) | (0xc900 >> 2),
344 (0x5e00 << 16) | (0xc900 >> 2),
346 (0x6e00 << 16) | (0xc900 >> 2),
348 (0x7e00 << 16) | (0xc900 >> 2),
350 (0x8e00 << 16) | (0xc900 >> 2),
352 (0x9e00 << 16) | (0xc900 >> 2),
354 (0xae00 << 16) | (0xc900 >> 2),
356 (0xbe00 << 16) | (0xc900 >> 2),
358 (0x4e00 << 16) | (0xc904 >> 2),
360 (0x5e00 << 16) | (0xc904 >> 2),
362 (0x6e00 << 16) | (0xc904 >> 2),
364 (0x7e00 << 16) | (0xc904 >> 2),
366 (0x8e00 << 16) | (0xc904 >> 2),
368 (0x9e00 << 16) | (0xc904 >> 2),
370 (0xae00 << 16) | (0xc904 >> 2),
372 (0xbe00 << 16) | (0xc904 >> 2),
374 (0x4e00 << 16) | (0xc908 >> 2),
376 (0x5e00 << 16) | (0xc908 >> 2),
378 (0x6e00 << 16) | (0xc908 >> 2),
380 (0x7e00 << 16) | (0xc908 >> 2),
382 (0x8e00 << 16) | (0xc908 >> 2),
384 (0x9e00 << 16) | (0xc908 >> 2),
386 (0xae00 << 16) | (0xc908 >> 2),
388 (0xbe00 << 16) | (0xc908 >> 2),
390 (0x4e00 << 16) | (0xc90c >> 2),
392 (0x5e00 << 16) | (0xc90c >> 2),
394 (0x6e00 << 16) | (0xc90c >> 2),
396 (0x7e00 << 16) | (0xc90c >> 2),
398 (0x8e00 << 16) | (0xc90c >> 2),
400 (0x9e00 << 16) | (0xc90c >> 2),
402 (0xae00 << 16) | (0xc90c >> 2),
404 (0xbe00 << 16) | (0xc90c >> 2),
406 (0x4e00 << 16) | (0xc910 >> 2),
408 (0x5e00 << 16) | (0xc910 >> 2),
410 (0x6e00 << 16) | (0xc910 >> 2),
412 (0x7e00 << 16) | (0xc910 >> 2),
414 (0x8e00 << 16) | (0xc910 >> 2),
416 (0x9e00 << 16) | (0xc910 >> 2),
418 (0xae00 << 16) | (0xc910 >> 2),
420 (0xbe00 << 16) | (0xc910 >> 2),
422 (0x0e00 << 16) | (0xc99c >> 2),
424 (0x0e00 << 16) | (0x9834 >> 2),
426 (0x0000 << 16) | (0x30f00 >> 2),
428 (0x0001 << 16) | (0x30f00 >> 2),
430 (0x0000 << 16) | (0x30f04 >> 2),
432 (0x0001 << 16) | (0x30f04 >> 2),
434 (0x0000 << 16) | (0x30f08 >> 2),
436 (0x0001 << 16) | (0x30f08 >> 2),
438 (0x0000 << 16) | (0x30f0c >> 2),
440 (0x0001 << 16) | (0x30f0c >> 2),
442 (0x0600 << 16) | (0x9b7c >> 2),
444 (0x0e00 << 16) | (0x8a14 >> 2),
446 (0x0e00 << 16) | (0x8a18 >> 2),
448 (0x0600 << 16) | (0x30a00 >> 2),
450 (0x0e00 << 16) | (0x8bf0 >> 2),
452 (0x0e00 << 16) | (0x8bcc >> 2),
454 (0x0e00 << 16) | (0x8b24 >> 2),
456 (0x0e00 << 16) | (0x30a04 >> 2),
458 (0x0600 << 16) | (0x30a10 >> 2),
460 (0x0600 << 16) | (0x30a14 >> 2),
462 (0x0600 << 16) | (0x30a18 >> 2),
464 (0x0600 << 16) | (0x30a2c >> 2),
466 (0x0e00 << 16) | (0xc700 >> 2),
468 (0x0e00 << 16) | (0xc704 >> 2),
470 (0x0e00 << 16) | (0xc708 >> 2),
472 (0x0e00 << 16) | (0xc768 >> 2),
474 (0x0400 << 16) | (0xc770 >> 2),
476 (0x0400 << 16) | (0xc774 >> 2),
478 (0x0400 << 16) | (0xc778 >> 2),
480 (0x0400 << 16) | (0xc77c >> 2),
482 (0x0400 << 16) | (0xc780 >> 2),
484 (0x0400 << 16) | (0xc784 >> 2),
486 (0x0400 << 16) | (0xc788 >> 2),
488 (0x0400 << 16) | (0xc78c >> 2),
490 (0x0400 << 16) | (0xc798 >> 2),
492 (0x0400 << 16) | (0xc79c >> 2),
494 (0x0400 << 16) | (0xc7a0 >> 2),
496 (0x0400 << 16) | (0xc7a4 >> 2),
498 (0x0400 << 16) | (0xc7a8 >> 2),
500 (0x0400 << 16) | (0xc7ac >> 2),
502 (0x0400 << 16) | (0xc7b0 >> 2),
504 (0x0400 << 16) | (0xc7b4 >> 2),
506 (0x0e00 << 16) | (0x9100 >> 2),
508 (0x0e00 << 16) | (0x3c010 >> 2),
510 (0x0e00 << 16) | (0x92a8 >> 2),
512 (0x0e00 << 16) | (0x92ac >> 2),
514 (0x0e00 << 16) | (0x92b4 >> 2),
516 (0x0e00 << 16) | (0x92b8 >> 2),
518 (0x0e00 << 16) | (0x92bc >> 2),
520 (0x0e00 << 16) | (0x92c0 >> 2),
522 (0x0e00 << 16) | (0x92c4 >> 2),
524 (0x0e00 << 16) | (0x92c8 >> 2),
526 (0x0e00 << 16) | (0x92cc >> 2),
528 (0x0e00 << 16) | (0x92d0 >> 2),
530 (0x0e00 << 16) | (0x8c00 >> 2),
532 (0x0e00 << 16) | (0x8c04 >> 2),
534 (0x0e00 << 16) | (0x8c20 >> 2),
536 (0x0e00 << 16) | (0x8c38 >> 2),
538 (0x0e00 << 16) | (0x8c3c >> 2),
540 (0x0e00 << 16) | (0xae00 >> 2),
542 (0x0e00 << 16) | (0x9604 >> 2),
544 (0x0e00 << 16) | (0xac08 >> 2),
546 (0x0e00 << 16) | (0xac0c >> 2),
548 (0x0e00 << 16) | (0xac10 >> 2),
550 (0x0e00 << 16) | (0xac14 >> 2),
552 (0x0e00 << 16) | (0xac58 >> 2),
554 (0x0e00 << 16) | (0xac68 >> 2),
556 (0x0e00 << 16) | (0xac6c >> 2),
558 (0x0e00 << 16) | (0xac70 >> 2),
560 (0x0e00 << 16) | (0xac74 >> 2),
562 (0x0e00 << 16) | (0xac78 >> 2),
564 (0x0e00 << 16) | (0xac7c >> 2),
566 (0x0e00 << 16) | (0xac80 >> 2),
568 (0x0e00 << 16) | (0xac84 >> 2),
570 (0x0e00 << 16) | (0xac88 >> 2),
572 (0x0e00 << 16) | (0xac8c >> 2),
574 (0x0e00 << 16) | (0x970c >> 2),
576 (0x0e00 << 16) | (0x9714 >> 2),
578 (0x0e00 << 16) | (0x9718 >> 2),
580 (0x0e00 << 16) | (0x971c >> 2),
582 (0x0e00 << 16) | (0x31068 >> 2),
584 (0x4e00 << 16) | (0x31068 >> 2),
586 (0x5e00 << 16) | (0x31068 >> 2),
588 (0x6e00 << 16) | (0x31068 >> 2),
590 (0x7e00 << 16) | (0x31068 >> 2),
592 (0x8e00 << 16) | (0x31068 >> 2),
594 (0x9e00 << 16) | (0x31068 >> 2),
596 (0xae00 << 16) | (0x31068 >> 2),
598 (0xbe00 << 16) | (0x31068 >> 2),
600 (0x0e00 << 16) | (0xcd10 >> 2),
602 (0x0e00 << 16) | (0xcd14 >> 2),
604 (0x0e00 << 16) | (0x88b0 >> 2),
606 (0x0e00 << 16) | (0x88b4 >> 2),
608 (0x0e00 << 16) | (0x88b8 >> 2),
610 (0x0e00 << 16) | (0x88bc >> 2),
612 (0x0400 << 16) | (0x89c0 >> 2),
614 (0x0e00 << 16) | (0x88c4 >> 2),
616 (0x0e00 << 16) | (0x88c8 >> 2),
618 (0x0e00 << 16) | (0x88d0 >> 2),
620 (0x0e00 << 16) | (0x88d4 >> 2),
622 (0x0e00 << 16) | (0x88d8 >> 2),
624 (0x0e00 << 16) | (0x8980 >> 2),
626 (0x0e00 << 16) | (0x30938 >> 2),
628 (0x0e00 << 16) | (0x3093c >> 2),
630 (0x0e00 << 16) | (0x30940 >> 2),
632 (0x0e00 << 16) | (0x89a0 >> 2),
634 (0x0e00 << 16) | (0x30900 >> 2),
636 (0x0e00 << 16) | (0x30904 >> 2),
638 (0x0e00 << 16) | (0x89b4 >> 2),
640 (0x0e00 << 16) | (0x3c210 >> 2),
642 (0x0e00 << 16) | (0x3c214 >> 2),
644 (0x0e00 << 16) | (0x3c218 >> 2),
646 (0x0e00 << 16) | (0x8904 >> 2),
649 (0x0e00 << 16) | (0x8c28 >> 2),
650 (0x0e00 << 16) | (0x8c2c >> 2),
651 (0x0e00 << 16) | (0x8c30 >> 2),
652 (0x0e00 << 16) | (0x8c34 >> 2),
653 (0x0e00 << 16) | (0x9600 >> 2),
656 static const u32 kalindi_rlc_save_restore_register_list[] =
658 (0x0e00 << 16) | (0xc12c >> 2),
660 (0x0e00 << 16) | (0xc140 >> 2),
662 (0x0e00 << 16) | (0xc150 >> 2),
664 (0x0e00 << 16) | (0xc15c >> 2),
666 (0x0e00 << 16) | (0xc168 >> 2),
668 (0x0e00 << 16) | (0xc170 >> 2),
670 (0x0e00 << 16) | (0xc204 >> 2),
672 (0x0e00 << 16) | (0xc2b4 >> 2),
674 (0x0e00 << 16) | (0xc2b8 >> 2),
676 (0x0e00 << 16) | (0xc2bc >> 2),
678 (0x0e00 << 16) | (0xc2c0 >> 2),
680 (0x0e00 << 16) | (0x8228 >> 2),
682 (0x0e00 << 16) | (0x829c >> 2),
684 (0x0e00 << 16) | (0x869c >> 2),
686 (0x0600 << 16) | (0x98f4 >> 2),
688 (0x0e00 << 16) | (0x98f8 >> 2),
690 (0x0e00 << 16) | (0x9900 >> 2),
692 (0x0e00 << 16) | (0xc260 >> 2),
694 (0x0e00 << 16) | (0x90e8 >> 2),
696 (0x0e00 << 16) | (0x3c000 >> 2),
698 (0x0e00 << 16) | (0x3c00c >> 2),
700 (0x0e00 << 16) | (0x8c1c >> 2),
702 (0x0e00 << 16) | (0x9700 >> 2),
704 (0x0e00 << 16) | (0xcd20 >> 2),
706 (0x4e00 << 16) | (0xcd20 >> 2),
708 (0x5e00 << 16) | (0xcd20 >> 2),
710 (0x6e00 << 16) | (0xcd20 >> 2),
712 (0x7e00 << 16) | (0xcd20 >> 2),
714 (0x0e00 << 16) | (0x89bc >> 2),
716 (0x0e00 << 16) | (0x8900 >> 2),
719 (0x0e00 << 16) | (0xc130 >> 2),
721 (0x0e00 << 16) | (0xc134 >> 2),
723 (0x0e00 << 16) | (0xc1fc >> 2),
725 (0x0e00 << 16) | (0xc208 >> 2),
727 (0x0e00 << 16) | (0xc264 >> 2),
729 (0x0e00 << 16) | (0xc268 >> 2),
731 (0x0e00 << 16) | (0xc26c >> 2),
733 (0x0e00 << 16) | (0xc270 >> 2),
735 (0x0e00 << 16) | (0xc274 >> 2),
737 (0x0e00 << 16) | (0xc28c >> 2),
739 (0x0e00 << 16) | (0xc290 >> 2),
741 (0x0e00 << 16) | (0xc294 >> 2),
743 (0x0e00 << 16) | (0xc298 >> 2),
745 (0x0e00 << 16) | (0xc2a0 >> 2),
747 (0x0e00 << 16) | (0xc2a4 >> 2),
749 (0x0e00 << 16) | (0xc2a8 >> 2),
751 (0x0e00 << 16) | (0xc2ac >> 2),
753 (0x0e00 << 16) | (0x301d0 >> 2),
755 (0x0e00 << 16) | (0x30238 >> 2),
757 (0x0e00 << 16) | (0x30250 >> 2),
759 (0x0e00 << 16) | (0x30254 >> 2),
761 (0x0e00 << 16) | (0x30258 >> 2),
763 (0x0e00 << 16) | (0x3025c >> 2),
765 (0x4e00 << 16) | (0xc900 >> 2),
767 (0x5e00 << 16) | (0xc900 >> 2),
769 (0x6e00 << 16) | (0xc900 >> 2),
771 (0x7e00 << 16) | (0xc900 >> 2),
773 (0x4e00 << 16) | (0xc904 >> 2),
775 (0x5e00 << 16) | (0xc904 >> 2),
777 (0x6e00 << 16) | (0xc904 >> 2),
779 (0x7e00 << 16) | (0xc904 >> 2),
781 (0x4e00 << 16) | (0xc908 >> 2),
783 (0x5e00 << 16) | (0xc908 >> 2),
785 (0x6e00 << 16) | (0xc908 >> 2),
787 (0x7e00 << 16) | (0xc908 >> 2),
789 (0x4e00 << 16) | (0xc90c >> 2),
791 (0x5e00 << 16) | (0xc90c >> 2),
793 (0x6e00 << 16) | (0xc90c >> 2),
795 (0x7e00 << 16) | (0xc90c >> 2),
797 (0x4e00 << 16) | (0xc910 >> 2),
799 (0x5e00 << 16) | (0xc910 >> 2),
801 (0x6e00 << 16) | (0xc910 >> 2),
803 (0x7e00 << 16) | (0xc910 >> 2),
805 (0x0e00 << 16) | (0xc99c >> 2),
807 (0x0e00 << 16) | (0x9834 >> 2),
809 (0x0000 << 16) | (0x30f00 >> 2),
811 (0x0000 << 16) | (0x30f04 >> 2),
813 (0x0000 << 16) | (0x30f08 >> 2),
815 (0x0000 << 16) | (0x30f0c >> 2),
817 (0x0600 << 16) | (0x9b7c >> 2),
819 (0x0e00 << 16) | (0x8a14 >> 2),
821 (0x0e00 << 16) | (0x8a18 >> 2),
823 (0x0600 << 16) | (0x30a00 >> 2),
825 (0x0e00 << 16) | (0x8bf0 >> 2),
827 (0x0e00 << 16) | (0x8bcc >> 2),
829 (0x0e00 << 16) | (0x8b24 >> 2),
831 (0x0e00 << 16) | (0x30a04 >> 2),
833 (0x0600 << 16) | (0x30a10 >> 2),
835 (0x0600 << 16) | (0x30a14 >> 2),
837 (0x0600 << 16) | (0x30a18 >> 2),
839 (0x0600 << 16) | (0x30a2c >> 2),
841 (0x0e00 << 16) | (0xc700 >> 2),
843 (0x0e00 << 16) | (0xc704 >> 2),
845 (0x0e00 << 16) | (0xc708 >> 2),
847 (0x0e00 << 16) | (0xc768 >> 2),
849 (0x0400 << 16) | (0xc770 >> 2),
851 (0x0400 << 16) | (0xc774 >> 2),
853 (0x0400 << 16) | (0xc798 >> 2),
855 (0x0400 << 16) | (0xc79c >> 2),
857 (0x0e00 << 16) | (0x9100 >> 2),
859 (0x0e00 << 16) | (0x3c010 >> 2),
861 (0x0e00 << 16) | (0x8c00 >> 2),
863 (0x0e00 << 16) | (0x8c04 >> 2),
865 (0x0e00 << 16) | (0x8c20 >> 2),
867 (0x0e00 << 16) | (0x8c38 >> 2),
869 (0x0e00 << 16) | (0x8c3c >> 2),
871 (0x0e00 << 16) | (0xae00 >> 2),
873 (0x0e00 << 16) | (0x9604 >> 2),
875 (0x0e00 << 16) | (0xac08 >> 2),
877 (0x0e00 << 16) | (0xac0c >> 2),
879 (0x0e00 << 16) | (0xac10 >> 2),
881 (0x0e00 << 16) | (0xac14 >> 2),
883 (0x0e00 << 16) | (0xac58 >> 2),
885 (0x0e00 << 16) | (0xac68 >> 2),
887 (0x0e00 << 16) | (0xac6c >> 2),
889 (0x0e00 << 16) | (0xac70 >> 2),
891 (0x0e00 << 16) | (0xac74 >> 2),
893 (0x0e00 << 16) | (0xac78 >> 2),
895 (0x0e00 << 16) | (0xac7c >> 2),
897 (0x0e00 << 16) | (0xac80 >> 2),
899 (0x0e00 << 16) | (0xac84 >> 2),
901 (0x0e00 << 16) | (0xac88 >> 2),
903 (0x0e00 << 16) | (0xac8c >> 2),
905 (0x0e00 << 16) | (0x970c >> 2),
907 (0x0e00 << 16) | (0x9714 >> 2),
909 (0x0e00 << 16) | (0x9718 >> 2),
911 (0x0e00 << 16) | (0x971c >> 2),
913 (0x0e00 << 16) | (0x31068 >> 2),
915 (0x4e00 << 16) | (0x31068 >> 2),
917 (0x5e00 << 16) | (0x31068 >> 2),
919 (0x6e00 << 16) | (0x31068 >> 2),
921 (0x7e00 << 16) | (0x31068 >> 2),
923 (0x0e00 << 16) | (0xcd10 >> 2),
925 (0x0e00 << 16) | (0xcd14 >> 2),
927 (0x0e00 << 16) | (0x88b0 >> 2),
929 (0x0e00 << 16) | (0x88b4 >> 2),
931 (0x0e00 << 16) | (0x88b8 >> 2),
933 (0x0e00 << 16) | (0x88bc >> 2),
935 (0x0400 << 16) | (0x89c0 >> 2),
937 (0x0e00 << 16) | (0x88c4 >> 2),
939 (0x0e00 << 16) | (0x88c8 >> 2),
941 (0x0e00 << 16) | (0x88d0 >> 2),
943 (0x0e00 << 16) | (0x88d4 >> 2),
945 (0x0e00 << 16) | (0x88d8 >> 2),
947 (0x0e00 << 16) | (0x8980 >> 2),
949 (0x0e00 << 16) | (0x30938 >> 2),
951 (0x0e00 << 16) | (0x3093c >> 2),
953 (0x0e00 << 16) | (0x30940 >> 2),
955 (0x0e00 << 16) | (0x89a0 >> 2),
957 (0x0e00 << 16) | (0x30900 >> 2),
959 (0x0e00 << 16) | (0x30904 >> 2),
961 (0x0e00 << 16) | (0x89b4 >> 2),
963 (0x0e00 << 16) | (0x3e1fc >> 2),
965 (0x0e00 << 16) | (0x3c210 >> 2),
967 (0x0e00 << 16) | (0x3c214 >> 2),
969 (0x0e00 << 16) | (0x3c218 >> 2),
971 (0x0e00 << 16) | (0x8904 >> 2),
974 (0x0e00 << 16) | (0x8c28 >> 2),
975 (0x0e00 << 16) | (0x8c2c >> 2),
976 (0x0e00 << 16) | (0x8c30 >> 2),
977 (0x0e00 << 16) | (0x8c34 >> 2),
978 (0x0e00 << 16) | (0x9600 >> 2),
981 static const u32 bonaire_golden_spm_registers[] =
983 0x30800, 0xe0ffffff, 0xe0000000
986 static const u32 bonaire_golden_common_registers[] =
988 0xc770, 0xffffffff, 0x00000800,
989 0xc774, 0xffffffff, 0x00000800,
990 0xc798, 0xffffffff, 0x00007fbf,
991 0xc79c, 0xffffffff, 0x00007faf
994 static const u32 bonaire_golden_registers[] =
996 0x3354, 0x00000333, 0x00000333,
997 0x3350, 0x000c0fc0, 0x00040200,
998 0x9a10, 0x00010000, 0x00058208,
999 0x3c000, 0xffff1fff, 0x00140000,
1000 0x3c200, 0xfdfc0fff, 0x00000100,
1001 0x3c234, 0x40000000, 0x40000200,
1002 0x9830, 0xffffffff, 0x00000000,
1003 0x9834, 0xf00fffff, 0x00000400,
1004 0x9838, 0x0002021c, 0x00020200,
1005 0xc78, 0x00000080, 0x00000000,
1006 0x5bb0, 0x000000f0, 0x00000070,
1007 0x5bc0, 0xf0311fff, 0x80300000,
1008 0x98f8, 0x73773777, 0x12010001,
1009 0x350c, 0x00810000, 0x408af000,
1010 0x7030, 0x31000111, 0x00000011,
1011 0x2f48, 0x73773777, 0x12010001,
1012 0x220c, 0x00007fb6, 0x0021a1b1,
1013 0x2210, 0x00007fb6, 0x002021b1,
1014 0x2180, 0x00007fb6, 0x00002191,
1015 0x2218, 0x00007fb6, 0x002121b1,
1016 0x221c, 0x00007fb6, 0x002021b1,
1017 0x21dc, 0x00007fb6, 0x00002191,
1018 0x21e0, 0x00007fb6, 0x00002191,
1019 0x3628, 0x0000003f, 0x0000000a,
1020 0x362c, 0x0000003f, 0x0000000a,
1021 0x2ae4, 0x00073ffe, 0x000022a2,
1022 0x240c, 0x000007ff, 0x00000000,
1023 0x8a14, 0xf000003f, 0x00000007,
1024 0x8bf0, 0x00002001, 0x00000001,
1025 0x8b24, 0xffffffff, 0x00ffffff,
1026 0x30a04, 0x0000ff0f, 0x00000000,
1027 0x28a4c, 0x07ffffff, 0x06000000,
1028 0x4d8, 0x00000fff, 0x00000100,
1029 0x3e78, 0x00000001, 0x00000002,
1030 0x9100, 0x03000000, 0x0362c688,
1031 0x8c00, 0x000000ff, 0x00000001,
1032 0xe40, 0x00001fff, 0x00001fff,
1033 0x9060, 0x0000007f, 0x00000020,
1034 0x9508, 0x00010000, 0x00010000,
1035 0xac14, 0x000003ff, 0x000000f3,
1036 0xac0c, 0xffffffff, 0x00001032
1039 static const u32 bonaire_mgcg_cgcg_init[] =
1041 0xc420, 0xffffffff, 0xfffffffc,
1042 0x30800, 0xffffffff, 0xe0000000,
1043 0x3c2a0, 0xffffffff, 0x00000100,
1044 0x3c208, 0xffffffff, 0x00000100,
1045 0x3c2c0, 0xffffffff, 0xc0000100,
1046 0x3c2c8, 0xffffffff, 0xc0000100,
1047 0x3c2c4, 0xffffffff, 0xc0000100,
1048 0x55e4, 0xffffffff, 0x00600100,
1049 0x3c280, 0xffffffff, 0x00000100,
1050 0x3c214, 0xffffffff, 0x06000100,
1051 0x3c220, 0xffffffff, 0x00000100,
1052 0x3c218, 0xffffffff, 0x06000100,
1053 0x3c204, 0xffffffff, 0x00000100,
1054 0x3c2e0, 0xffffffff, 0x00000100,
1055 0x3c224, 0xffffffff, 0x00000100,
1056 0x3c200, 0xffffffff, 0x00000100,
1057 0x3c230, 0xffffffff, 0x00000100,
1058 0x3c234, 0xffffffff, 0x00000100,
1059 0x3c250, 0xffffffff, 0x00000100,
1060 0x3c254, 0xffffffff, 0x00000100,
1061 0x3c258, 0xffffffff, 0x00000100,
1062 0x3c25c, 0xffffffff, 0x00000100,
1063 0x3c260, 0xffffffff, 0x00000100,
1064 0x3c27c, 0xffffffff, 0x00000100,
1065 0x3c278, 0xffffffff, 0x00000100,
1066 0x3c210, 0xffffffff, 0x06000100,
1067 0x3c290, 0xffffffff, 0x00000100,
1068 0x3c274, 0xffffffff, 0x00000100,
1069 0x3c2b4, 0xffffffff, 0x00000100,
1070 0x3c2b0, 0xffffffff, 0x00000100,
1071 0x3c270, 0xffffffff, 0x00000100,
1072 0x30800, 0xffffffff, 0xe0000000,
1073 0x3c020, 0xffffffff, 0x00010000,
1074 0x3c024, 0xffffffff, 0x00030002,
1075 0x3c028, 0xffffffff, 0x00040007,
1076 0x3c02c, 0xffffffff, 0x00060005,
1077 0x3c030, 0xffffffff, 0x00090008,
1078 0x3c034, 0xffffffff, 0x00010000,
1079 0x3c038, 0xffffffff, 0x00030002,
1080 0x3c03c, 0xffffffff, 0x00040007,
1081 0x3c040, 0xffffffff, 0x00060005,
1082 0x3c044, 0xffffffff, 0x00090008,
1083 0x3c048, 0xffffffff, 0x00010000,
1084 0x3c04c, 0xffffffff, 0x00030002,
1085 0x3c050, 0xffffffff, 0x00040007,
1086 0x3c054, 0xffffffff, 0x00060005,
1087 0x3c058, 0xffffffff, 0x00090008,
1088 0x3c05c, 0xffffffff, 0x00010000,
1089 0x3c060, 0xffffffff, 0x00030002,
1090 0x3c064, 0xffffffff, 0x00040007,
1091 0x3c068, 0xffffffff, 0x00060005,
1092 0x3c06c, 0xffffffff, 0x00090008,
1093 0x3c070, 0xffffffff, 0x00010000,
1094 0x3c074, 0xffffffff, 0x00030002,
1095 0x3c078, 0xffffffff, 0x00040007,
1096 0x3c07c, 0xffffffff, 0x00060005,
1097 0x3c080, 0xffffffff, 0x00090008,
1098 0x3c084, 0xffffffff, 0x00010000,
1099 0x3c088, 0xffffffff, 0x00030002,
1100 0x3c08c, 0xffffffff, 0x00040007,
1101 0x3c090, 0xffffffff, 0x00060005,
1102 0x3c094, 0xffffffff, 0x00090008,
1103 0x3c098, 0xffffffff, 0x00010000,
1104 0x3c09c, 0xffffffff, 0x00030002,
1105 0x3c0a0, 0xffffffff, 0x00040007,
1106 0x3c0a4, 0xffffffff, 0x00060005,
1107 0x3c0a8, 0xffffffff, 0x00090008,
1108 0x3c000, 0xffffffff, 0x96e00200,
1109 0x8708, 0xffffffff, 0x00900100,
1110 0xc424, 0xffffffff, 0x0020003f,
1111 0x38, 0xffffffff, 0x0140001c,
1112 0x3c, 0x000f0000, 0x000f0000,
1113 0x220, 0xffffffff, 0xC060000C,
1114 0x224, 0xc0000fff, 0x00000100,
1115 0xf90, 0xffffffff, 0x00000100,
1116 0xf98, 0x00000101, 0x00000000,
1117 0x20a8, 0xffffffff, 0x00000104,
1118 0x55e4, 0xff000fff, 0x00000100,
1119 0x30cc, 0xc0000fff, 0x00000104,
1120 0xc1e4, 0x00000001, 0x00000001,
1121 0xd00c, 0xff000ff0, 0x00000100,
1122 0xd80c, 0xff000ff0, 0x00000100
1125 static const u32 spectre_golden_spm_registers[] =
1127 0x30800, 0xe0ffffff, 0xe0000000
1130 static const u32 spectre_golden_common_registers[] =
1132 0xc770, 0xffffffff, 0x00000800,
1133 0xc774, 0xffffffff, 0x00000800,
1134 0xc798, 0xffffffff, 0x00007fbf,
1135 0xc79c, 0xffffffff, 0x00007faf
1138 static const u32 spectre_golden_registers[] =
1140 0x3c000, 0xffff1fff, 0x96940200,
1141 0x3c00c, 0xffff0001, 0xff000000,
1142 0x3c200, 0xfffc0fff, 0x00000100,
1143 0x6ed8, 0x00010101, 0x00010000,
1144 0x9834, 0xf00fffff, 0x00000400,
1145 0x9838, 0xfffffffc, 0x00020200,
1146 0x5bb0, 0x000000f0, 0x00000070,
1147 0x5bc0, 0xf0311fff, 0x80300000,
1148 0x98f8, 0x73773777, 0x12010001,
1149 0x9b7c, 0x00ff0000, 0x00fc0000,
1150 0x2f48, 0x73773777, 0x12010001,
1151 0x8a14, 0xf000003f, 0x00000007,
1152 0x8b24, 0xffffffff, 0x00ffffff,
1153 0x28350, 0x3f3f3fff, 0x00000082,
1154 0x28354, 0x0000003f, 0x00000000,
1155 0x3e78, 0x00000001, 0x00000002,
1156 0x913c, 0xffff03df, 0x00000004,
1157 0xc768, 0x00000008, 0x00000008,
1158 0x8c00, 0x000008ff, 0x00000800,
1159 0x9508, 0x00010000, 0x00010000,
1160 0xac0c, 0xffffffff, 0x54763210,
1161 0x214f8, 0x01ff01ff, 0x00000002,
1162 0x21498, 0x007ff800, 0x00200000,
1163 0x2015c, 0xffffffff, 0x00000f40,
1164 0x30934, 0xffffffff, 0x00000001
1167 static const u32 spectre_mgcg_cgcg_init[] =
1169 0xc420, 0xffffffff, 0xfffffffc,
1170 0x30800, 0xffffffff, 0xe0000000,
1171 0x3c2a0, 0xffffffff, 0x00000100,
1172 0x3c208, 0xffffffff, 0x00000100,
1173 0x3c2c0, 0xffffffff, 0x00000100,
1174 0x3c2c8, 0xffffffff, 0x00000100,
1175 0x3c2c4, 0xffffffff, 0x00000100,
1176 0x55e4, 0xffffffff, 0x00600100,
1177 0x3c280, 0xffffffff, 0x00000100,
1178 0x3c214, 0xffffffff, 0x06000100,
1179 0x3c220, 0xffffffff, 0x00000100,
1180 0x3c218, 0xffffffff, 0x06000100,
1181 0x3c204, 0xffffffff, 0x00000100,
1182 0x3c2e0, 0xffffffff, 0x00000100,
1183 0x3c224, 0xffffffff, 0x00000100,
1184 0x3c200, 0xffffffff, 0x00000100,
1185 0x3c230, 0xffffffff, 0x00000100,
1186 0x3c234, 0xffffffff, 0x00000100,
1187 0x3c250, 0xffffffff, 0x00000100,
1188 0x3c254, 0xffffffff, 0x00000100,
1189 0x3c258, 0xffffffff, 0x00000100,
1190 0x3c25c, 0xffffffff, 0x00000100,
1191 0x3c260, 0xffffffff, 0x00000100,
1192 0x3c27c, 0xffffffff, 0x00000100,
1193 0x3c278, 0xffffffff, 0x00000100,
1194 0x3c210, 0xffffffff, 0x06000100,
1195 0x3c290, 0xffffffff, 0x00000100,
1196 0x3c274, 0xffffffff, 0x00000100,
1197 0x3c2b4, 0xffffffff, 0x00000100,
1198 0x3c2b0, 0xffffffff, 0x00000100,
1199 0x3c270, 0xffffffff, 0x00000100,
1200 0x30800, 0xffffffff, 0xe0000000,
1201 0x3c020, 0xffffffff, 0x00010000,
1202 0x3c024, 0xffffffff, 0x00030002,
1203 0x3c028, 0xffffffff, 0x00040007,
1204 0x3c02c, 0xffffffff, 0x00060005,
1205 0x3c030, 0xffffffff, 0x00090008,
1206 0x3c034, 0xffffffff, 0x00010000,
1207 0x3c038, 0xffffffff, 0x00030002,
1208 0x3c03c, 0xffffffff, 0x00040007,
1209 0x3c040, 0xffffffff, 0x00060005,
1210 0x3c044, 0xffffffff, 0x00090008,
1211 0x3c048, 0xffffffff, 0x00010000,
1212 0x3c04c, 0xffffffff, 0x00030002,
1213 0x3c050, 0xffffffff, 0x00040007,
1214 0x3c054, 0xffffffff, 0x00060005,
1215 0x3c058, 0xffffffff, 0x00090008,
1216 0x3c05c, 0xffffffff, 0x00010000,
1217 0x3c060, 0xffffffff, 0x00030002,
1218 0x3c064, 0xffffffff, 0x00040007,
1219 0x3c068, 0xffffffff, 0x00060005,
1220 0x3c06c, 0xffffffff, 0x00090008,
1221 0x3c070, 0xffffffff, 0x00010000,
1222 0x3c074, 0xffffffff, 0x00030002,
1223 0x3c078, 0xffffffff, 0x00040007,
1224 0x3c07c, 0xffffffff, 0x00060005,
1225 0x3c080, 0xffffffff, 0x00090008,
1226 0x3c084, 0xffffffff, 0x00010000,
1227 0x3c088, 0xffffffff, 0x00030002,
1228 0x3c08c, 0xffffffff, 0x00040007,
1229 0x3c090, 0xffffffff, 0x00060005,
1230 0x3c094, 0xffffffff, 0x00090008,
1231 0x3c098, 0xffffffff, 0x00010000,
1232 0x3c09c, 0xffffffff, 0x00030002,
1233 0x3c0a0, 0xffffffff, 0x00040007,
1234 0x3c0a4, 0xffffffff, 0x00060005,
1235 0x3c0a8, 0xffffffff, 0x00090008,
1236 0x3c0ac, 0xffffffff, 0x00010000,
1237 0x3c0b0, 0xffffffff, 0x00030002,
1238 0x3c0b4, 0xffffffff, 0x00040007,
1239 0x3c0b8, 0xffffffff, 0x00060005,
1240 0x3c0bc, 0xffffffff, 0x00090008,
1241 0x3c000, 0xffffffff, 0x96e00200,
1242 0x8708, 0xffffffff, 0x00900100,
1243 0xc424, 0xffffffff, 0x0020003f,
1244 0x38, 0xffffffff, 0x0140001c,
1245 0x3c, 0x000f0000, 0x000f0000,
1246 0x220, 0xffffffff, 0xC060000C,
1247 0x224, 0xc0000fff, 0x00000100,
1248 0xf90, 0xffffffff, 0x00000100,
1249 0xf98, 0x00000101, 0x00000000,
1250 0x20a8, 0xffffffff, 0x00000104,
1251 0x55e4, 0xff000fff, 0x00000100,
1252 0x30cc, 0xc0000fff, 0x00000104,
1253 0xc1e4, 0x00000001, 0x00000001,
1254 0xd00c, 0xff000ff0, 0x00000100,
1255 0xd80c, 0xff000ff0, 0x00000100
1258 static const u32 kalindi_golden_spm_registers[] =
1260 0x30800, 0xe0ffffff, 0xe0000000
1263 static const u32 kalindi_golden_common_registers[] =
1265 0xc770, 0xffffffff, 0x00000800,
1266 0xc774, 0xffffffff, 0x00000800,
1267 0xc798, 0xffffffff, 0x00007fbf,
1268 0xc79c, 0xffffffff, 0x00007faf
1271 static const u32 kalindi_golden_registers[] =
1273 0x3c000, 0xffffdfff, 0x6e944040,
1274 0x55e4, 0xff607fff, 0xfc000100,
1275 0x3c220, 0xff000fff, 0x00000100,
1276 0x3c224, 0xff000fff, 0x00000100,
1277 0x3c200, 0xfffc0fff, 0x00000100,
1278 0x6ed8, 0x00010101, 0x00010000,
1279 0x9830, 0xffffffff, 0x00000000,
1280 0x9834, 0xf00fffff, 0x00000400,
1281 0x5bb0, 0x000000f0, 0x00000070,
1282 0x5bc0, 0xf0311fff, 0x80300000,
1283 0x98f8, 0x73773777, 0x12010001,
1284 0x98fc, 0xffffffff, 0x00000010,
1285 0x9b7c, 0x00ff0000, 0x00fc0000,
1286 0x8030, 0x00001f0f, 0x0000100a,
1287 0x2f48, 0x73773777, 0x12010001,
1288 0x2408, 0x000fffff, 0x000c007f,
1289 0x8a14, 0xf000003f, 0x00000007,
1290 0x8b24, 0x3fff3fff, 0x00ffcfff,
1291 0x30a04, 0x0000ff0f, 0x00000000,
1292 0x28a4c, 0x07ffffff, 0x06000000,
1293 0x4d8, 0x00000fff, 0x00000100,
1294 0x3e78, 0x00000001, 0x00000002,
1295 0xc768, 0x00000008, 0x00000008,
1296 0x8c00, 0x000000ff, 0x00000003,
1297 0x214f8, 0x01ff01ff, 0x00000002,
1298 0x21498, 0x007ff800, 0x00200000,
1299 0x2015c, 0xffffffff, 0x00000f40,
1300 0x88c4, 0x001f3ae3, 0x00000082,
1301 0x88d4, 0x0000001f, 0x00000010,
1302 0x30934, 0xffffffff, 0x00000000
1305 static const u32 kalindi_mgcg_cgcg_init[] =
1307 0xc420, 0xffffffff, 0xfffffffc,
1308 0x30800, 0xffffffff, 0xe0000000,
1309 0x3c2a0, 0xffffffff, 0x00000100,
1310 0x3c208, 0xffffffff, 0x00000100,
1311 0x3c2c0, 0xffffffff, 0x00000100,
1312 0x3c2c8, 0xffffffff, 0x00000100,
1313 0x3c2c4, 0xffffffff, 0x00000100,
1314 0x55e4, 0xffffffff, 0x00600100,
1315 0x3c280, 0xffffffff, 0x00000100,
1316 0x3c214, 0xffffffff, 0x06000100,
1317 0x3c220, 0xffffffff, 0x00000100,
1318 0x3c218, 0xffffffff, 0x06000100,
1319 0x3c204, 0xffffffff, 0x00000100,
1320 0x3c2e0, 0xffffffff, 0x00000100,
1321 0x3c224, 0xffffffff, 0x00000100,
1322 0x3c200, 0xffffffff, 0x00000100,
1323 0x3c230, 0xffffffff, 0x00000100,
1324 0x3c234, 0xffffffff, 0x00000100,
1325 0x3c250, 0xffffffff, 0x00000100,
1326 0x3c254, 0xffffffff, 0x00000100,
1327 0x3c258, 0xffffffff, 0x00000100,
1328 0x3c25c, 0xffffffff, 0x00000100,
1329 0x3c260, 0xffffffff, 0x00000100,
1330 0x3c27c, 0xffffffff, 0x00000100,
1331 0x3c278, 0xffffffff, 0x00000100,
1332 0x3c210, 0xffffffff, 0x06000100,
1333 0x3c290, 0xffffffff, 0x00000100,
1334 0x3c274, 0xffffffff, 0x00000100,
1335 0x3c2b4, 0xffffffff, 0x00000100,
1336 0x3c2b0, 0xffffffff, 0x00000100,
1337 0x3c270, 0xffffffff, 0x00000100,
1338 0x30800, 0xffffffff, 0xe0000000,
1339 0x3c020, 0xffffffff, 0x00010000,
1340 0x3c024, 0xffffffff, 0x00030002,
1341 0x3c028, 0xffffffff, 0x00040007,
1342 0x3c02c, 0xffffffff, 0x00060005,
1343 0x3c030, 0xffffffff, 0x00090008,
1344 0x3c034, 0xffffffff, 0x00010000,
1345 0x3c038, 0xffffffff, 0x00030002,
1346 0x3c03c, 0xffffffff, 0x00040007,
1347 0x3c040, 0xffffffff, 0x00060005,
1348 0x3c044, 0xffffffff, 0x00090008,
1349 0x3c000, 0xffffffff, 0x96e00200,
1350 0x8708, 0xffffffff, 0x00900100,
1351 0xc424, 0xffffffff, 0x0020003f,
1352 0x38, 0xffffffff, 0x0140001c,
1353 0x3c, 0x000f0000, 0x000f0000,
1354 0x220, 0xffffffff, 0xC060000C,
1355 0x224, 0xc0000fff, 0x00000100,
1356 0x20a8, 0xffffffff, 0x00000104,
1357 0x55e4, 0xff000fff, 0x00000100,
1358 0x30cc, 0xc0000fff, 0x00000104,
1359 0xc1e4, 0x00000001, 0x00000001,
1360 0xd00c, 0xff000ff0, 0x00000100,
1361 0xd80c, 0xff000ff0, 0x00000100
1364 static const u32 hawaii_golden_spm_registers[] =
1366 0x30800, 0xe0ffffff, 0xe0000000
1369 static const u32 hawaii_golden_common_registers[] =
1371 0x30800, 0xffffffff, 0xe0000000,
1372 0x28350, 0xffffffff, 0x3a00161a,
1373 0x28354, 0xffffffff, 0x0000002e,
1374 0x9a10, 0xffffffff, 0x00018208,
1375 0x98f8, 0xffffffff, 0x12011003
1378 static const u32 hawaii_golden_registers[] =
1380 0x3354, 0x00000333, 0x00000333,
1381 0x9a10, 0x00010000, 0x00058208,
1382 0x9830, 0xffffffff, 0x00000000,
1383 0x9834, 0xf00fffff, 0x00000400,
1384 0x9838, 0x0002021c, 0x00020200,
1385 0xc78, 0x00000080, 0x00000000,
1386 0x5bb0, 0x000000f0, 0x00000070,
1387 0x5bc0, 0xf0311fff, 0x80300000,
1388 0x350c, 0x00810000, 0x408af000,
1389 0x7030, 0x31000111, 0x00000011,
1390 0x2f48, 0x73773777, 0x12010001,
1391 0x2120, 0x0000007f, 0x0000001b,
1392 0x21dc, 0x00007fb6, 0x00002191,
1393 0x3628, 0x0000003f, 0x0000000a,
1394 0x362c, 0x0000003f, 0x0000000a,
1395 0x2ae4, 0x00073ffe, 0x000022a2,
1396 0x240c, 0x000007ff, 0x00000000,
1397 0x8bf0, 0x00002001, 0x00000001,
1398 0x8b24, 0xffffffff, 0x00ffffff,
1399 0x30a04, 0x0000ff0f, 0x00000000,
1400 0x28a4c, 0x07ffffff, 0x06000000,
1401 0x3e78, 0x00000001, 0x00000002,
1402 0xc768, 0x00000008, 0x00000008,
1403 0xc770, 0x00000f00, 0x00000800,
1404 0xc774, 0x00000f00, 0x00000800,
1405 0xc798, 0x00ffffff, 0x00ff7fbf,
1406 0xc79c, 0x00ffffff, 0x00ff7faf,
1407 0x8c00, 0x000000ff, 0x00000800,
1408 0xe40, 0x00001fff, 0x00001fff,
1409 0x9060, 0x0000007f, 0x00000020,
1410 0x9508, 0x00010000, 0x00010000,
1411 0xae00, 0x00100000, 0x000ff07c,
1412 0xac14, 0x000003ff, 0x0000000f,
1413 0xac10, 0xffffffff, 0x7564fdec,
1414 0xac0c, 0xffffffff, 0x3120b9a8,
1415 0xac08, 0x20000000, 0x0f9c0000
1418 static const u32 hawaii_mgcg_cgcg_init[] =
1420 0xc420, 0xffffffff, 0xfffffffd,
1421 0x30800, 0xffffffff, 0xe0000000,
1422 0x3c2a0, 0xffffffff, 0x00000100,
1423 0x3c208, 0xffffffff, 0x00000100,
1424 0x3c2c0, 0xffffffff, 0x00000100,
1425 0x3c2c8, 0xffffffff, 0x00000100,
1426 0x3c2c4, 0xffffffff, 0x00000100,
1427 0x55e4, 0xffffffff, 0x00200100,
1428 0x3c280, 0xffffffff, 0x00000100,
1429 0x3c214, 0xffffffff, 0x06000100,
1430 0x3c220, 0xffffffff, 0x00000100,
1431 0x3c218, 0xffffffff, 0x06000100,
1432 0x3c204, 0xffffffff, 0x00000100,
1433 0x3c2e0, 0xffffffff, 0x00000100,
1434 0x3c224, 0xffffffff, 0x00000100,
1435 0x3c200, 0xffffffff, 0x00000100,
1436 0x3c230, 0xffffffff, 0x00000100,
1437 0x3c234, 0xffffffff, 0x00000100,
1438 0x3c250, 0xffffffff, 0x00000100,
1439 0x3c254, 0xffffffff, 0x00000100,
1440 0x3c258, 0xffffffff, 0x00000100,
1441 0x3c25c, 0xffffffff, 0x00000100,
1442 0x3c260, 0xffffffff, 0x00000100,
1443 0x3c27c, 0xffffffff, 0x00000100,
1444 0x3c278, 0xffffffff, 0x00000100,
1445 0x3c210, 0xffffffff, 0x06000100,
1446 0x3c290, 0xffffffff, 0x00000100,
1447 0x3c274, 0xffffffff, 0x00000100,
1448 0x3c2b4, 0xffffffff, 0x00000100,
1449 0x3c2b0, 0xffffffff, 0x00000100,
1450 0x3c270, 0xffffffff, 0x00000100,
1451 0x30800, 0xffffffff, 0xe0000000,
1452 0x3c020, 0xffffffff, 0x00010000,
1453 0x3c024, 0xffffffff, 0x00030002,
1454 0x3c028, 0xffffffff, 0x00040007,
1455 0x3c02c, 0xffffffff, 0x00060005,
1456 0x3c030, 0xffffffff, 0x00090008,
1457 0x3c034, 0xffffffff, 0x00010000,
1458 0x3c038, 0xffffffff, 0x00030002,
1459 0x3c03c, 0xffffffff, 0x00040007,
1460 0x3c040, 0xffffffff, 0x00060005,
1461 0x3c044, 0xffffffff, 0x00090008,
1462 0x3c048, 0xffffffff, 0x00010000,
1463 0x3c04c, 0xffffffff, 0x00030002,
1464 0x3c050, 0xffffffff, 0x00040007,
1465 0x3c054, 0xffffffff, 0x00060005,
1466 0x3c058, 0xffffffff, 0x00090008,
1467 0x3c05c, 0xffffffff, 0x00010000,
1468 0x3c060, 0xffffffff, 0x00030002,
1469 0x3c064, 0xffffffff, 0x00040007,
1470 0x3c068, 0xffffffff, 0x00060005,
1471 0x3c06c, 0xffffffff, 0x00090008,
1472 0x3c070, 0xffffffff, 0x00010000,
1473 0x3c074, 0xffffffff, 0x00030002,
1474 0x3c078, 0xffffffff, 0x00040007,
1475 0x3c07c, 0xffffffff, 0x00060005,
1476 0x3c080, 0xffffffff, 0x00090008,
1477 0x3c084, 0xffffffff, 0x00010000,
1478 0x3c088, 0xffffffff, 0x00030002,
1479 0x3c08c, 0xffffffff, 0x00040007,
1480 0x3c090, 0xffffffff, 0x00060005,
1481 0x3c094, 0xffffffff, 0x00090008,
1482 0x3c098, 0xffffffff, 0x00010000,
1483 0x3c09c, 0xffffffff, 0x00030002,
1484 0x3c0a0, 0xffffffff, 0x00040007,
1485 0x3c0a4, 0xffffffff, 0x00060005,
1486 0x3c0a8, 0xffffffff, 0x00090008,
1487 0x3c0ac, 0xffffffff, 0x00010000,
1488 0x3c0b0, 0xffffffff, 0x00030002,
1489 0x3c0b4, 0xffffffff, 0x00040007,
1490 0x3c0b8, 0xffffffff, 0x00060005,
1491 0x3c0bc, 0xffffffff, 0x00090008,
1492 0x3c0c0, 0xffffffff, 0x00010000,
1493 0x3c0c4, 0xffffffff, 0x00030002,
1494 0x3c0c8, 0xffffffff, 0x00040007,
1495 0x3c0cc, 0xffffffff, 0x00060005,
1496 0x3c0d0, 0xffffffff, 0x00090008,
1497 0x3c0d4, 0xffffffff, 0x00010000,
1498 0x3c0d8, 0xffffffff, 0x00030002,
1499 0x3c0dc, 0xffffffff, 0x00040007,
1500 0x3c0e0, 0xffffffff, 0x00060005,
1501 0x3c0e4, 0xffffffff, 0x00090008,
1502 0x3c0e8, 0xffffffff, 0x00010000,
1503 0x3c0ec, 0xffffffff, 0x00030002,
1504 0x3c0f0, 0xffffffff, 0x00040007,
1505 0x3c0f4, 0xffffffff, 0x00060005,
1506 0x3c0f8, 0xffffffff, 0x00090008,
1507 0xc318, 0xffffffff, 0x00020200,
1508 0x3350, 0xffffffff, 0x00000200,
1509 0x15c0, 0xffffffff, 0x00000400,
1510 0x55e8, 0xffffffff, 0x00000000,
1511 0x2f50, 0xffffffff, 0x00000902,
1512 0x3c000, 0xffffffff, 0x96940200,
1513 0x8708, 0xffffffff, 0x00900100,
1514 0xc424, 0xffffffff, 0x0020003f,
1515 0x38, 0xffffffff, 0x0140001c,
1516 0x3c, 0x000f0000, 0x000f0000,
1517 0x220, 0xffffffff, 0xc060000c,
1518 0x224, 0xc0000fff, 0x00000100,
1519 0xf90, 0xffffffff, 0x00000100,
1520 0xf98, 0x00000101, 0x00000000,
1521 0x20a8, 0xffffffff, 0x00000104,
1522 0x55e4, 0xff000fff, 0x00000100,
1523 0x30cc, 0xc0000fff, 0x00000104,
1524 0xc1e4, 0x00000001, 0x00000001,
1525 0xd00c, 0xff000ff0, 0x00000100,
1526 0xd80c, 0xff000ff0, 0x00000100
1529 static const u32 godavari_golden_registers[] =
1531 0x55e4, 0xff607fff, 0xfc000100,
1532 0x6ed8, 0x00010101, 0x00010000,
1533 0x9830, 0xffffffff, 0x00000000,
1534 0x98302, 0xf00fffff, 0x00000400,
1535 0x6130, 0xffffffff, 0x00010000,
1536 0x5bb0, 0x000000f0, 0x00000070,
1537 0x5bc0, 0xf0311fff, 0x80300000,
1538 0x98f8, 0x73773777, 0x12010001,
1539 0x98fc, 0xffffffff, 0x00000010,
1540 0x8030, 0x00001f0f, 0x0000100a,
1541 0x2f48, 0x73773777, 0x12010001,
1542 0x2408, 0x000fffff, 0x000c007f,
1543 0x8a14, 0xf000003f, 0x00000007,
1544 0x8b24, 0xffffffff, 0x00ff0fff,
1545 0x30a04, 0x0000ff0f, 0x00000000,
1546 0x28a4c, 0x07ffffff, 0x06000000,
1547 0x4d8, 0x00000fff, 0x00000100,
1548 0xd014, 0x00010000, 0x00810001,
1549 0xd814, 0x00010000, 0x00810001,
1550 0x3e78, 0x00000001, 0x00000002,
1551 0xc768, 0x00000008, 0x00000008,
1552 0xc770, 0x00000f00, 0x00000800,
1553 0xc774, 0x00000f00, 0x00000800,
1554 0xc798, 0x00ffffff, 0x00ff7fbf,
1555 0xc79c, 0x00ffffff, 0x00ff7faf,
1556 0x8c00, 0x000000ff, 0x00000001,
1557 0x214f8, 0x01ff01ff, 0x00000002,
1558 0x21498, 0x007ff800, 0x00200000,
1559 0x2015c, 0xffffffff, 0x00000f40,
1560 0x88c4, 0x001f3ae3, 0x00000082,
1561 0x88d4, 0x0000001f, 0x00000010,
1562 0x30934, 0xffffffff, 0x00000000
1566 static void cik_init_golden_registers(struct radeon_device *rdev)
1568 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1569 mutex_lock(&rdev->grbm_idx_mutex);
1570 switch (rdev->family) {
1572 radeon_program_register_sequence(rdev,
1573 bonaire_mgcg_cgcg_init,
1574 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1575 radeon_program_register_sequence(rdev,
1576 bonaire_golden_registers,
1577 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1578 radeon_program_register_sequence(rdev,
1579 bonaire_golden_common_registers,
1580 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1581 radeon_program_register_sequence(rdev,
1582 bonaire_golden_spm_registers,
1583 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1586 radeon_program_register_sequence(rdev,
1587 kalindi_mgcg_cgcg_init,
1588 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1589 radeon_program_register_sequence(rdev,
1590 kalindi_golden_registers,
1591 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1592 radeon_program_register_sequence(rdev,
1593 kalindi_golden_common_registers,
1594 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1595 radeon_program_register_sequence(rdev,
1596 kalindi_golden_spm_registers,
1597 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1600 radeon_program_register_sequence(rdev,
1601 kalindi_mgcg_cgcg_init,
1602 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1603 radeon_program_register_sequence(rdev,
1604 godavari_golden_registers,
1605 (const u32)ARRAY_SIZE(godavari_golden_registers));
1606 radeon_program_register_sequence(rdev,
1607 kalindi_golden_common_registers,
1608 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1609 radeon_program_register_sequence(rdev,
1610 kalindi_golden_spm_registers,
1611 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1614 radeon_program_register_sequence(rdev,
1615 spectre_mgcg_cgcg_init,
1616 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1617 radeon_program_register_sequence(rdev,
1618 spectre_golden_registers,
1619 (const u32)ARRAY_SIZE(spectre_golden_registers));
1620 radeon_program_register_sequence(rdev,
1621 spectre_golden_common_registers,
1622 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1623 radeon_program_register_sequence(rdev,
1624 spectre_golden_spm_registers,
1625 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1628 radeon_program_register_sequence(rdev,
1629 hawaii_mgcg_cgcg_init,
1630 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1631 radeon_program_register_sequence(rdev,
1632 hawaii_golden_registers,
1633 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1634 radeon_program_register_sequence(rdev,
1635 hawaii_golden_common_registers,
1636 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1637 radeon_program_register_sequence(rdev,
1638 hawaii_golden_spm_registers,
1639 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1644 mutex_unlock(&rdev->grbm_idx_mutex);
1648 * cik_get_xclk - get the xclk
1650 * @rdev: radeon_device pointer
1652 * Returns the reference clock used by the gfx engine
1655 u32 cik_get_xclk(struct radeon_device *rdev)
1657 u32 reference_clock = rdev->clock.spll.reference_freq;
1659 if (rdev->flags & RADEON_IS_IGP) {
1660 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1661 return reference_clock / 2;
1663 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1664 return reference_clock / 4;
1666 return reference_clock;
1670 * cik_mm_rdoorbell - read a doorbell dword
1672 * @rdev: radeon_device pointer
1673 * @index: doorbell index
1675 * Returns the value in the doorbell aperture at the
1676 * requested doorbell index (CIK).
1678 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1680 if (index < rdev->doorbell.num_doorbells) {
1681 return readl(rdev->doorbell.ptr + index);
1683 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1689 * cik_mm_wdoorbell - write a doorbell dword
1691 * @rdev: radeon_device pointer
1692 * @index: doorbell index
1693 * @v: value to write
1695 * Writes @v to the doorbell aperture at the
1696 * requested doorbell index (CIK).
1698 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1700 if (index < rdev->doorbell.num_doorbells) {
1701 writel(v, rdev->doorbell.ptr + index);
1703 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1707 #define BONAIRE_IO_MC_REGS_SIZE 36
1709 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1711 {0x00000070, 0x04400000},
1712 {0x00000071, 0x80c01803},
1713 {0x00000072, 0x00004004},
1714 {0x00000073, 0x00000100},
1715 {0x00000074, 0x00ff0000},
1716 {0x00000075, 0x34000000},
1717 {0x00000076, 0x08000014},
1718 {0x00000077, 0x00cc08ec},
1719 {0x00000078, 0x00000400},
1720 {0x00000079, 0x00000000},
1721 {0x0000007a, 0x04090000},
1722 {0x0000007c, 0x00000000},
1723 {0x0000007e, 0x4408a8e8},
1724 {0x0000007f, 0x00000304},
1725 {0x00000080, 0x00000000},
1726 {0x00000082, 0x00000001},
1727 {0x00000083, 0x00000002},
1728 {0x00000084, 0xf3e4f400},
1729 {0x00000085, 0x052024e3},
1730 {0x00000087, 0x00000000},
1731 {0x00000088, 0x01000000},
1732 {0x0000008a, 0x1c0a0000},
1733 {0x0000008b, 0xff010000},
1734 {0x0000008d, 0xffffefff},
1735 {0x0000008e, 0xfff3efff},
1736 {0x0000008f, 0xfff3efbf},
1737 {0x00000092, 0xf7ffffff},
1738 {0x00000093, 0xffffff7f},
1739 {0x00000095, 0x00101101},
1740 {0x00000096, 0x00000fff},
1741 {0x00000097, 0x00116fff},
1742 {0x00000098, 0x60010000},
1743 {0x00000099, 0x10010000},
1744 {0x0000009a, 0x00006000},
1745 {0x0000009b, 0x00001000},
1746 {0x0000009f, 0x00b48000}
1749 #define HAWAII_IO_MC_REGS_SIZE 22
1751 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1753 {0x0000007d, 0x40000000},
1754 {0x0000007e, 0x40180304},
1755 {0x0000007f, 0x0000ff00},
1756 {0x00000081, 0x00000000},
1757 {0x00000083, 0x00000800},
1758 {0x00000086, 0x00000000},
1759 {0x00000087, 0x00000100},
1760 {0x00000088, 0x00020100},
1761 {0x00000089, 0x00000000},
1762 {0x0000008b, 0x00040000},
1763 {0x0000008c, 0x00000100},
1764 {0x0000008e, 0xff010000},
1765 {0x00000090, 0xffffefff},
1766 {0x00000091, 0xfff3efff},
1767 {0x00000092, 0xfff3efbf},
1768 {0x00000093, 0xf7ffffff},
1769 {0x00000094, 0xffffff7f},
1770 {0x00000095, 0x00000fff},
1771 {0x00000096, 0x00116fff},
1772 {0x00000097, 0x60010000},
1773 {0x00000098, 0x10010000},
1774 {0x0000009f, 0x00c79000}
1779 * cik_srbm_select - select specific register instances
1781 * @rdev: radeon_device pointer
1782 * @me: selected ME (micro engine)
1787 * Switches the currently active registers instances. Some
1788 * registers are instanced per VMID, others are instanced per
1789 * me/pipe/queue combination.
1791 static void cik_srbm_select(struct radeon_device *rdev,
1792 u32 me, u32 pipe, u32 queue, u32 vmid)
1794 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1797 QUEUEID(queue & 0x7));
1798 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1803 * ci_mc_load_microcode - load MC ucode into the hw
1805 * @rdev: radeon_device pointer
1807 * Load the GDDR MC ucode into the hw (CIK).
1808 * Returns 0 on success, error on failure.
1810 int ci_mc_load_microcode(struct radeon_device *rdev)
1812 const __be32 *fw_data = NULL;
1813 const __le32 *new_fw_data = NULL;
1814 u32 running, blackout = 0, tmp;
1815 u32 *io_mc_regs = NULL;
1816 const __le32 *new_io_mc_regs = NULL;
1817 int i, regs_size, ucode_size;
1823 const struct mc_firmware_header_v1_0 *hdr =
1824 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1826 radeon_ucode_print_mc_hdr(&hdr->header);
1828 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1829 new_io_mc_regs = (const __le32 *)
1830 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1831 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1832 new_fw_data = (const __le32 *)
1833 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1835 ucode_size = rdev->mc_fw->size / 4;
1837 switch (rdev->family) {
1839 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1840 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1843 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1844 regs_size = HAWAII_IO_MC_REGS_SIZE;
1849 fw_data = (const __be32 *)rdev->mc_fw->data;
1852 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1856 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1857 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1860 /* reset the engine and set to writable */
1861 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1862 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1864 /* load mc io regs */
1865 for (i = 0; i < regs_size; i++) {
1867 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1868 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1870 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1871 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1875 tmp = RREG32(MC_SEQ_MISC0);
1876 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1877 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1878 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1879 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1880 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1883 /* load the MC ucode */
1884 for (i = 0; i < ucode_size; i++) {
1886 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1888 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1891 /* put the engine back into the active state */
1892 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1893 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1894 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1896 /* wait for training to complete */
1897 for (i = 0; i < rdev->usec_timeout; i++) {
1898 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1902 for (i = 0; i < rdev->usec_timeout; i++) {
1903 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1909 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1916 * cik_init_microcode - load ucode images from disk
1918 * @rdev: radeon_device pointer
1920 * Use the firmware interface to load the ucode images into
1921 * the driver (not loaded into hw).
1922 * Returns 0 on success, error on failure.
1924 static int cik_init_microcode(struct radeon_device *rdev)
1926 const char *chip_name;
1927 const char *new_chip_name;
1928 size_t pfp_req_size, me_req_size, ce_req_size,
1929 mec_req_size, rlc_req_size, mc_req_size = 0,
1930 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1938 switch (rdev->family) {
1940 chip_name = "BONAIRE";
1941 new_chip_name = "bonaire";
1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1948 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1954 chip_name = "HAWAII";
1955 new_chip_name = "hawaii";
1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1961 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1962 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1963 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1964 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1968 chip_name = "KAVERI";
1969 new_chip_name = "kaveri";
1970 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1971 me_req_size = CIK_ME_UCODE_SIZE * 4;
1972 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1973 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1974 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1975 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1979 chip_name = "KABINI";
1980 new_chip_name = "kabini";
1981 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1982 me_req_size = CIK_ME_UCODE_SIZE * 4;
1983 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1984 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1985 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1986 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1990 chip_name = "MULLINS";
1991 new_chip_name = "mullins";
1992 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1993 me_req_size = CIK_ME_UCODE_SIZE * 4;
1994 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1995 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1996 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1997 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2003 DRM_INFO("Loading %s Microcode\n", new_chip_name);
2005 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2006 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2008 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2009 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2012 if (rdev->pfp_fw->size != pfp_req_size) {
2014 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2015 rdev->pfp_fw->size, fw_name);
2020 err = radeon_ucode_validate(rdev->pfp_fw);
2023 "cik_fw: validation failed for firmware \"%s\"\n",
2031 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2032 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2034 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2035 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2038 if (rdev->me_fw->size != me_req_size) {
2040 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2041 rdev->me_fw->size, fw_name);
2045 err = radeon_ucode_validate(rdev->me_fw);
2048 "cik_fw: validation failed for firmware \"%s\"\n",
2056 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2057 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2059 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2060 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2063 if (rdev->ce_fw->size != ce_req_size) {
2065 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2066 rdev->ce_fw->size, fw_name);
2070 err = radeon_ucode_validate(rdev->ce_fw);
2073 "cik_fw: validation failed for firmware \"%s\"\n",
2081 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2082 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2084 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2085 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2088 if (rdev->mec_fw->size != mec_req_size) {
2090 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2091 rdev->mec_fw->size, fw_name);
2095 err = radeon_ucode_validate(rdev->mec_fw);
2098 "cik_fw: validation failed for firmware \"%s\"\n",
2106 if (rdev->family == CHIP_KAVERI) {
2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2108 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2112 err = radeon_ucode_validate(rdev->mec2_fw);
2121 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2122 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2124 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2125 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2128 if (rdev->rlc_fw->size != rlc_req_size) {
2130 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2131 rdev->rlc_fw->size, fw_name);
2135 err = radeon_ucode_validate(rdev->rlc_fw);
2138 "cik_fw: validation failed for firmware \"%s\"\n",
2146 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2147 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2149 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2150 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2153 if (rdev->sdma_fw->size != sdma_req_size) {
2155 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2156 rdev->sdma_fw->size, fw_name);
2160 err = radeon_ucode_validate(rdev->sdma_fw);
2163 "cik_fw: validation failed for firmware \"%s\"\n",
2171 /* No SMC, MC ucode on APUs */
2172 if (!(rdev->flags & RADEON_IS_IGP)) {
2173 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2174 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2176 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2177 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2179 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2180 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2184 if ((rdev->mc_fw->size != mc_req_size) &&
2185 (rdev->mc_fw->size != mc2_req_size)){
2187 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2188 rdev->mc_fw->size, fw_name);
2191 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2193 err = radeon_ucode_validate(rdev->mc_fw);
2196 "cik_fw: validation failed for firmware \"%s\"\n",
2204 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2205 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2207 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2208 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2211 "smc: error loading firmware \"%s\"\n",
2213 release_firmware(rdev->smc_fw);
2214 rdev->smc_fw = NULL;
2216 } else if (rdev->smc_fw->size != smc_req_size) {
2218 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2219 rdev->smc_fw->size, fw_name);
2223 err = radeon_ucode_validate(rdev->smc_fw);
2226 "cik_fw: validation failed for firmware \"%s\"\n",
2236 rdev->new_fw = false;
2237 } else if (new_fw < num_fw) {
2238 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2241 rdev->new_fw = true;
2248 "cik_cp: Failed to load firmware \"%s\"\n",
2250 release_firmware(rdev->pfp_fw);
2251 rdev->pfp_fw = NULL;
2252 release_firmware(rdev->me_fw);
2254 release_firmware(rdev->ce_fw);
2256 release_firmware(rdev->mec_fw);
2257 rdev->mec_fw = NULL;
2258 release_firmware(rdev->mec2_fw);
2259 rdev->mec2_fw = NULL;
2260 release_firmware(rdev->rlc_fw);
2261 rdev->rlc_fw = NULL;
2262 release_firmware(rdev->sdma_fw);
2263 rdev->sdma_fw = NULL;
2264 release_firmware(rdev->mc_fw);
2266 release_firmware(rdev->smc_fw);
2267 rdev->smc_fw = NULL;
2276 * cik_tiling_mode_table_init - init the hw tiling table
2278 * @rdev: radeon_device pointer
2280 * Starting with SI, the tiling setup is done globally in a
2281 * set of 32 tiling modes. Rather than selecting each set of
2282 * parameters per surface as on older asics, we just select
2283 * which index in the tiling table we want to use, and the
2284 * surface uses those parameters (CIK).
2286 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2288 const u32 num_tile_mode_states = 32;
2289 const u32 num_secondary_tile_mode_states = 16;
2290 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2291 u32 num_pipe_configs;
2292 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2293 rdev->config.cik.max_shader_engines;
2295 switch (rdev->config.cik.mem_row_size_in_kb) {
2297 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2301 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2304 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2308 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2309 if (num_pipe_configs > 8)
2310 num_pipe_configs = 16;
2312 if (num_pipe_configs == 16) {
2313 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2314 switch (reg_offset) {
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2319 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2325 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2334 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2335 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2336 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2337 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2341 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2342 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2343 TILE_SPLIT(split_equal_to_row_size));
2346 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2347 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2348 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2357 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2359 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2360 TILE_SPLIT(split_equal_to_row_size));
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2364 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2369 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2372 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2373 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2374 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2375 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2378 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2379 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2381 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2384 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2385 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2386 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2387 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2390 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2391 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2392 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2395 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2396 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2397 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2398 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2402 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2404 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2407 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2408 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2409 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2410 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2414 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2419 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2420 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2421 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2424 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2425 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2427 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2439 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2440 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2442 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2443 switch (reg_offset) {
2445 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2448 NUM_BANKS(ADDR_SURF_16_BANK));
2451 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2452 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2453 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2454 NUM_BANKS(ADDR_SURF_16_BANK));
2457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2460 NUM_BANKS(ADDR_SURF_16_BANK));
2463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2466 NUM_BANKS(ADDR_SURF_16_BANK));
2469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2472 NUM_BANKS(ADDR_SURF_8_BANK));
2475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2478 NUM_BANKS(ADDR_SURF_4_BANK));
2481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2484 NUM_BANKS(ADDR_SURF_2_BANK));
2487 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2490 NUM_BANKS(ADDR_SURF_16_BANK));
2493 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2496 NUM_BANKS(ADDR_SURF_16_BANK));
2499 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2502 NUM_BANKS(ADDR_SURF_16_BANK));
2505 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2508 NUM_BANKS(ADDR_SURF_8_BANK));
2511 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2514 NUM_BANKS(ADDR_SURF_4_BANK));
2517 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2520 NUM_BANKS(ADDR_SURF_2_BANK));
2523 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2526 NUM_BANKS(ADDR_SURF_2_BANK));
2532 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2533 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2535 } else if (num_pipe_configs == 8) {
2536 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2537 switch (reg_offset) {
2539 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2545 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2548 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2551 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2552 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2553 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2554 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2557 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2558 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2559 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2560 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2566 TILE_SPLIT(split_equal_to_row_size));
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2574 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2575 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2576 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2577 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2580 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2581 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2583 TILE_SPLIT(split_equal_to_row_size));
2586 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2587 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2590 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2592 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2595 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2598 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2601 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2602 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2603 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2604 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2607 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2608 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2610 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2615 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2638 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2643 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2647 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2648 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2649 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2662 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2663 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2665 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2666 switch (reg_offset) {
2668 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2671 NUM_BANKS(ADDR_SURF_16_BANK));
2674 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2675 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2676 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2677 NUM_BANKS(ADDR_SURF_16_BANK));
2680 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2683 NUM_BANKS(ADDR_SURF_16_BANK));
2686 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2689 NUM_BANKS(ADDR_SURF_16_BANK));
2692 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2693 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2694 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2695 NUM_BANKS(ADDR_SURF_8_BANK));
2698 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2701 NUM_BANKS(ADDR_SURF_4_BANK));
2704 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2705 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2706 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2707 NUM_BANKS(ADDR_SURF_2_BANK));
2710 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2713 NUM_BANKS(ADDR_SURF_16_BANK));
2716 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2719 NUM_BANKS(ADDR_SURF_16_BANK));
2722 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2723 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2724 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2725 NUM_BANKS(ADDR_SURF_16_BANK));
2728 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2731 NUM_BANKS(ADDR_SURF_16_BANK));
2734 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2737 NUM_BANKS(ADDR_SURF_8_BANK));
2740 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2741 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2742 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2743 NUM_BANKS(ADDR_SURF_4_BANK));
2746 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2749 NUM_BANKS(ADDR_SURF_2_BANK));
2755 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2756 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2758 } else if (num_pipe_configs == 4) {
2760 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2761 switch (reg_offset) {
2763 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2765 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2766 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2771 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2775 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2777 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2778 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2783 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2790 TILE_SPLIT(split_equal_to_row_size));
2793 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2794 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2798 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2799 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2800 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2801 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2804 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2805 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2806 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2807 TILE_SPLIT(split_equal_to_row_size));
2810 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2811 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2815 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2822 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2825 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2827 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2828 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2834 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2838 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2839 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2842 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2843 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2844 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2845 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2848 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2849 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2850 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2851 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2854 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2855 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2856 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2857 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2860 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2861 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2862 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2879 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2886 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2887 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2889 } else if (num_rbs < 4) {
2890 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2891 switch (reg_offset) {
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2896 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2899 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2900 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2901 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2902 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2905 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2906 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2907 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2908 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2911 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2912 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2913 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2914 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2917 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2919 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2920 TILE_SPLIT(split_equal_to_row_size));
2923 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2924 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2925 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2928 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2929 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2930 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2931 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2934 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2935 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2936 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2937 TILE_SPLIT(split_equal_to_row_size));
2940 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2941 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2944 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2945 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2946 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2949 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2950 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2951 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2952 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2955 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2956 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2957 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2958 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2961 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2962 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2963 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2964 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2967 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2968 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2969 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2972 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2973 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2974 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2975 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2978 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2979 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2980 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2981 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2984 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2985 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2986 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2987 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2990 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2991 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2992 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2995 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2996 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2997 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2998 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3001 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3002 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3003 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3004 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3007 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3008 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3009 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3010 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3016 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3017 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3020 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3021 switch (reg_offset) {
3023 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3026 NUM_BANKS(ADDR_SURF_16_BANK));
3029 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3030 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3031 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3032 NUM_BANKS(ADDR_SURF_16_BANK));
3035 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3038 NUM_BANKS(ADDR_SURF_16_BANK));
3041 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3042 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3043 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3044 NUM_BANKS(ADDR_SURF_16_BANK));
3047 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3048 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3049 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3050 NUM_BANKS(ADDR_SURF_16_BANK));
3053 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3054 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3055 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3056 NUM_BANKS(ADDR_SURF_8_BANK));
3059 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3060 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3061 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3062 NUM_BANKS(ADDR_SURF_4_BANK));
3065 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3066 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3067 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3068 NUM_BANKS(ADDR_SURF_16_BANK));
3071 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3072 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3073 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3074 NUM_BANKS(ADDR_SURF_16_BANK));
3077 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3078 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3079 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3080 NUM_BANKS(ADDR_SURF_16_BANK));
3083 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3084 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3085 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3086 NUM_BANKS(ADDR_SURF_16_BANK));
3089 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3090 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3091 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3092 NUM_BANKS(ADDR_SURF_16_BANK));
3095 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3096 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3097 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3098 NUM_BANKS(ADDR_SURF_8_BANK));
3101 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3102 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3103 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3104 NUM_BANKS(ADDR_SURF_4_BANK));
3110 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3111 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3113 } else if (num_pipe_configs == 2) {
3114 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3115 switch (reg_offset) {
3117 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3118 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3119 PIPE_CONFIG(ADDR_SURF_P2) |
3120 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3123 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3124 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3125 PIPE_CONFIG(ADDR_SURF_P2) |
3126 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3129 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3130 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3131 PIPE_CONFIG(ADDR_SURF_P2) |
3132 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3135 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3136 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3137 PIPE_CONFIG(ADDR_SURF_P2) |
3138 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3141 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3142 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3143 PIPE_CONFIG(ADDR_SURF_P2) |
3144 TILE_SPLIT(split_equal_to_row_size));
3147 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3148 PIPE_CONFIG(ADDR_SURF_P2) |
3149 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3152 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3153 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3154 PIPE_CONFIG(ADDR_SURF_P2) |
3155 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3158 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3159 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3160 PIPE_CONFIG(ADDR_SURF_P2) |
3161 TILE_SPLIT(split_equal_to_row_size));
3164 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3165 PIPE_CONFIG(ADDR_SURF_P2);
3168 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3170 PIPE_CONFIG(ADDR_SURF_P2));
3173 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3174 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3175 PIPE_CONFIG(ADDR_SURF_P2) |
3176 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3179 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3180 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3181 PIPE_CONFIG(ADDR_SURF_P2) |
3182 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3185 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3186 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3187 PIPE_CONFIG(ADDR_SURF_P2) |
3188 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3191 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3192 PIPE_CONFIG(ADDR_SURF_P2) |
3193 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3196 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3197 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3198 PIPE_CONFIG(ADDR_SURF_P2) |
3199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3202 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3203 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3204 PIPE_CONFIG(ADDR_SURF_P2) |
3205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3208 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3209 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3210 PIPE_CONFIG(ADDR_SURF_P2) |
3211 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3214 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3215 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3216 PIPE_CONFIG(ADDR_SURF_P2));
3219 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3220 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3221 PIPE_CONFIG(ADDR_SURF_P2) |
3222 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3225 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3226 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3227 PIPE_CONFIG(ADDR_SURF_P2) |
3228 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3231 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3232 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3233 PIPE_CONFIG(ADDR_SURF_P2) |
3234 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3240 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3241 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3243 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3244 switch (reg_offset) {
3246 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3249 NUM_BANKS(ADDR_SURF_16_BANK));
3252 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3255 NUM_BANKS(ADDR_SURF_16_BANK));
3258 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3261 NUM_BANKS(ADDR_SURF_16_BANK));
3264 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3267 NUM_BANKS(ADDR_SURF_16_BANK));
3270 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3271 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3272 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3273 NUM_BANKS(ADDR_SURF_16_BANK));
3276 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3279 NUM_BANKS(ADDR_SURF_16_BANK));
3282 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3285 NUM_BANKS(ADDR_SURF_8_BANK));
3288 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3291 NUM_BANKS(ADDR_SURF_16_BANK));
3294 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3297 NUM_BANKS(ADDR_SURF_16_BANK));
3300 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3303 NUM_BANKS(ADDR_SURF_16_BANK));
3306 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3309 NUM_BANKS(ADDR_SURF_16_BANK));
3312 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3315 NUM_BANKS(ADDR_SURF_16_BANK));
3318 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3321 NUM_BANKS(ADDR_SURF_16_BANK));
3324 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3325 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3326 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3327 NUM_BANKS(ADDR_SURF_8_BANK));
3333 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3334 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3337 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3341 * cik_select_se_sh - select which SE, SH to address
3343 * @rdev: radeon_device pointer
3344 * @se_num: shader engine to address
3345 * @sh_num: sh block to address
3347 * Select which SE, SH combinations to address. Certain
3348 * registers are instanced per SE or SH. 0xffffffff means
3349 * broadcast to all SEs or SHs (CIK).
3351 static void cik_select_se_sh(struct radeon_device *rdev,
3352 u32 se_num, u32 sh_num)
3354 u32 data = INSTANCE_BROADCAST_WRITES;
3356 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3357 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3358 else if (se_num == 0xffffffff)
3359 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3360 else if (sh_num == 0xffffffff)
3361 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3363 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3364 WREG32(GRBM_GFX_INDEX, data);
3368 * cik_create_bitmask - create a bitmask
3370 * @bit_width: length of the mask
3372 * create a variable length bit mask (CIK).
3373 * Returns the bitmask.
3375 static u32 cik_create_bitmask(u32 bit_width)
3379 for (i = 0; i < bit_width; i++) {
3387 * cik_get_rb_disabled - computes the mask of disabled RBs
3389 * @rdev: radeon_device pointer
3390 * @max_rb_num: max RBs (render backends) for the asic
3391 * @se_num: number of SEs (shader engines) for the asic
3392 * @sh_per_se: number of SH blocks per SE for the asic
3394 * Calculates the bitmask of disabled RBs (CIK).
3395 * Returns the disabled RB bitmask.
3397 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3398 u32 max_rb_num_per_se,
3403 data = RREG32(CC_RB_BACKEND_DISABLE);
3405 data &= BACKEND_DISABLE_MASK;
3408 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3410 data >>= BACKEND_DISABLE_SHIFT;
3412 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3418 * cik_setup_rb - setup the RBs on the asic
3420 * @rdev: radeon_device pointer
3421 * @se_num: number of SEs (shader engines) for the asic
3422 * @sh_per_se: number of SH blocks per SE for the asic
3423 * @max_rb_num: max RBs (render backends) for the asic
3425 * Configures per-SE/SH RB registers (CIK).
3427 static void cik_setup_rb(struct radeon_device *rdev,
3428 u32 se_num, u32 sh_per_se,
3429 u32 max_rb_num_per_se)
3433 u32 disabled_rbs = 0;
3434 u32 enabled_rbs = 0;
3436 mutex_lock(&rdev->grbm_idx_mutex);
3437 for (i = 0; i < se_num; i++) {
3438 for (j = 0; j < sh_per_se; j++) {
3439 cik_select_se_sh(rdev, i, j);
3440 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3441 if (rdev->family == CHIP_HAWAII)
3442 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3444 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3447 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3448 mutex_unlock(&rdev->grbm_idx_mutex);
3451 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3452 if (!(disabled_rbs & mask))
3453 enabled_rbs |= mask;
3457 rdev->config.cik.backend_enable_mask = enabled_rbs;
3459 mutex_lock(&rdev->grbm_idx_mutex);
3460 for (i = 0; i < se_num; i++) {
3461 cik_select_se_sh(rdev, i, 0xffffffff);
3463 for (j = 0; j < sh_per_se; j++) {
3464 switch (enabled_rbs & 3) {
3467 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3469 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3472 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3475 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3479 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3484 WREG32(PA_SC_RASTER_CONFIG, data);
3486 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3487 mutex_unlock(&rdev->grbm_idx_mutex);
3491 * cik_gpu_init - setup the 3D engine
3493 * @rdev: radeon_device pointer
3495 * Configures the 3D engine and tiling configuration
3496 * registers so that the 3D engine is usable.
3498 static void cik_gpu_init(struct radeon_device *rdev)
3500 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3501 u32 mc_shared_chmap, mc_arb_ramcfg;
3502 u32 hdp_host_path_cntl;
3506 switch (rdev->family) {
3508 rdev->config.cik.max_shader_engines = 2;
3509 rdev->config.cik.max_tile_pipes = 4;
3510 rdev->config.cik.max_cu_per_sh = 7;
3511 rdev->config.cik.max_sh_per_se = 1;
3512 rdev->config.cik.max_backends_per_se = 2;
3513 rdev->config.cik.max_texture_channel_caches = 4;
3514 rdev->config.cik.max_gprs = 256;
3515 rdev->config.cik.max_gs_threads = 32;
3516 rdev->config.cik.max_hw_contexts = 8;
3518 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3519 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3520 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3521 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3522 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3525 rdev->config.cik.max_shader_engines = 4;
3526 rdev->config.cik.max_tile_pipes = 16;
3527 rdev->config.cik.max_cu_per_sh = 11;
3528 rdev->config.cik.max_sh_per_se = 1;
3529 rdev->config.cik.max_backends_per_se = 4;
3530 rdev->config.cik.max_texture_channel_caches = 16;
3531 rdev->config.cik.max_gprs = 256;
3532 rdev->config.cik.max_gs_threads = 32;
3533 rdev->config.cik.max_hw_contexts = 8;
3535 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3536 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3537 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3538 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3539 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3542 rdev->config.cik.max_shader_engines = 1;
3543 rdev->config.cik.max_tile_pipes = 4;
3544 if ((rdev->pdev->device == 0x1304) ||
3545 (rdev->pdev->device == 0x1305) ||
3546 (rdev->pdev->device == 0x130C) ||
3547 (rdev->pdev->device == 0x130F) ||
3548 (rdev->pdev->device == 0x1310) ||
3549 (rdev->pdev->device == 0x1311) ||
3550 (rdev->pdev->device == 0x131C)) {
3551 rdev->config.cik.max_cu_per_sh = 8;
3552 rdev->config.cik.max_backends_per_se = 2;
3553 } else if ((rdev->pdev->device == 0x1309) ||
3554 (rdev->pdev->device == 0x130A) ||
3555 (rdev->pdev->device == 0x130D) ||
3556 (rdev->pdev->device == 0x1313) ||
3557 (rdev->pdev->device == 0x131D)) {
3558 rdev->config.cik.max_cu_per_sh = 6;
3559 rdev->config.cik.max_backends_per_se = 2;
3560 } else if ((rdev->pdev->device == 0x1306) ||
3561 (rdev->pdev->device == 0x1307) ||
3562 (rdev->pdev->device == 0x130B) ||
3563 (rdev->pdev->device == 0x130E) ||
3564 (rdev->pdev->device == 0x1315) ||
3565 (rdev->pdev->device == 0x1318) ||
3566 (rdev->pdev->device == 0x131B)) {
3567 rdev->config.cik.max_cu_per_sh = 4;
3568 rdev->config.cik.max_backends_per_se = 1;
3570 rdev->config.cik.max_cu_per_sh = 3;
3571 rdev->config.cik.max_backends_per_se = 1;
3573 rdev->config.cik.max_sh_per_se = 1;
3574 rdev->config.cik.max_texture_channel_caches = 4;
3575 rdev->config.cik.max_gprs = 256;
3576 rdev->config.cik.max_gs_threads = 16;
3577 rdev->config.cik.max_hw_contexts = 8;
3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3588 rdev->config.cik.max_shader_engines = 1;
3589 rdev->config.cik.max_tile_pipes = 2;
3590 rdev->config.cik.max_cu_per_sh = 2;
3591 rdev->config.cik.max_sh_per_se = 1;
3592 rdev->config.cik.max_backends_per_se = 1;
3593 rdev->config.cik.max_texture_channel_caches = 2;
3594 rdev->config.cik.max_gprs = 256;
3595 rdev->config.cik.max_gs_threads = 16;
3596 rdev->config.cik.max_hw_contexts = 8;
3598 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3599 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3600 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3601 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3602 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3606 /* Initialize HDP */
3607 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3608 WREG32((0x2c14 + j), 0x00000000);
3609 WREG32((0x2c18 + j), 0x00000000);
3610 WREG32((0x2c1c + j), 0x00000000);
3611 WREG32((0x2c20 + j), 0x00000000);
3612 WREG32((0x2c24 + j), 0x00000000);
3615 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3617 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3619 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3620 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3622 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3623 rdev->config.cik.mem_max_burst_length_bytes = 256;
3624 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3625 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3626 if (rdev->config.cik.mem_row_size_in_kb > 4)
3627 rdev->config.cik.mem_row_size_in_kb = 4;
3628 /* XXX use MC settings? */
3629 rdev->config.cik.shader_engine_tile_size = 32;
3630 rdev->config.cik.num_gpus = 1;
3631 rdev->config.cik.multi_gpu_tile_size = 64;
3633 /* fix up row size */
3634 gb_addr_config &= ~ROW_SIZE_MASK;
3635 switch (rdev->config.cik.mem_row_size_in_kb) {
3638 gb_addr_config |= ROW_SIZE(0);
3641 gb_addr_config |= ROW_SIZE(1);
3644 gb_addr_config |= ROW_SIZE(2);
3648 /* setup tiling info dword. gb_addr_config is not adequate since it does
3649 * not have bank info, so create a custom tiling dword.
3650 * bits 3:0 num_pipes
3651 * bits 7:4 num_banks
3652 * bits 11:8 group_size
3653 * bits 15:12 row_size
3655 rdev->config.cik.tile_config = 0;
3656 switch (rdev->config.cik.num_tile_pipes) {
3658 rdev->config.cik.tile_config |= (0 << 0);
3661 rdev->config.cik.tile_config |= (1 << 0);
3664 rdev->config.cik.tile_config |= (2 << 0);
3668 /* XXX what about 12? */
3669 rdev->config.cik.tile_config |= (3 << 0);
3672 rdev->config.cik.tile_config |=
3673 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3674 rdev->config.cik.tile_config |=
3675 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3676 rdev->config.cik.tile_config |=
3677 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3679 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3680 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3681 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3682 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3683 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3684 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3685 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3686 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3688 cik_tiling_mode_table_init(rdev);
3690 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3691 rdev->config.cik.max_sh_per_se,
3692 rdev->config.cik.max_backends_per_se);
3694 rdev->config.cik.active_cus = 0;
3695 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3696 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3697 rdev->config.cik.active_cus +=
3698 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3702 /* set HW defaults for 3D engine */
3703 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3705 mutex_lock(&rdev->grbm_idx_mutex);
3707 * making sure that the following register writes will be broadcasted
3708 * to all the shaders
3710 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3711 WREG32(SX_DEBUG_1, 0x20);
3713 WREG32(TA_CNTL_AUX, 0x00010000);
3715 tmp = RREG32(SPI_CONFIG_CNTL);
3717 WREG32(SPI_CONFIG_CNTL, tmp);
3719 WREG32(SQ_CONFIG, 1);
3721 WREG32(DB_DEBUG, 0);
3723 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3725 WREG32(DB_DEBUG2, tmp);
3727 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3729 WREG32(DB_DEBUG3, tmp);
3731 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3733 WREG32(CB_HW_CONTROL, tmp);
3735 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3737 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3738 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3739 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3740 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3742 WREG32(VGT_NUM_INSTANCES, 1);
3744 WREG32(CP_PERFMON_CNTL, 0);
3746 WREG32(SQ_CONFIG, 0);
3748 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3749 FORCE_EOV_MAX_REZ_CNT(255)));
3751 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3752 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3754 WREG32(VGT_GS_VERTEX_REUSE, 16);
3755 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3757 tmp = RREG32(HDP_MISC_CNTL);
3758 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3759 WREG32(HDP_MISC_CNTL, tmp);
3761 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3762 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3764 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3765 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3766 mutex_unlock(&rdev->grbm_idx_mutex);
3772 * GPU scratch registers helpers function.
3775 * cik_scratch_init - setup driver info for CP scratch regs
3777 * @rdev: radeon_device pointer
3779 * Set up the number and offset of the CP scratch registers.
3780 * NOTE: use of CP scratch registers is a legacy inferface and
3781 * is not used by default on newer asics (r6xx+). On newer asics,
3782 * memory buffers are used for fences rather than scratch regs.
3784 static void cik_scratch_init(struct radeon_device *rdev)
3788 rdev->scratch.num_reg = 7;
3789 rdev->scratch.reg_base = SCRATCH_REG0;
3790 for (i = 0; i < rdev->scratch.num_reg; i++) {
3791 rdev->scratch.free[i] = true;
3792 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3797 * cik_ring_test - basic gfx ring test
3799 * @rdev: radeon_device pointer
3800 * @ring: radeon_ring structure holding ring information
3802 * Allocate a scratch register and write to it using the gfx ring (CIK).
3803 * Provides a basic gfx ring test to verify that the ring is working.
3804 * Used by cik_cp_gfx_resume();
3805 * Returns 0 on success, error on failure.
3807 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3814 r = radeon_scratch_get(rdev, &scratch);
3816 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3819 WREG32(scratch, 0xCAFEDEAD);
3820 r = radeon_ring_lock(rdev, ring, 3);
3822 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3823 radeon_scratch_free(rdev, scratch);
3826 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3827 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3828 radeon_ring_write(ring, 0xDEADBEEF);
3829 radeon_ring_unlock_commit(rdev, ring, false);
3831 for (i = 0; i < rdev->usec_timeout; i++) {
3832 tmp = RREG32(scratch);
3833 if (tmp == 0xDEADBEEF)
3837 if (i < rdev->usec_timeout) {
3838 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3840 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3841 ring->idx, scratch, tmp);
3844 radeon_scratch_free(rdev, scratch);
3849 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3851 * @rdev: radeon_device pointer
3852 * @ridx: radeon ring index
3854 * Emits an hdp flush on the cp.
3856 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3859 struct radeon_ring *ring = &rdev->ring[ridx];
3862 switch (ring->idx) {
3863 case CAYMAN_RING_TYPE_CP1_INDEX:
3864 case CAYMAN_RING_TYPE_CP2_INDEX:
3868 ref_and_mask = CP2 << ring->pipe;
3871 ref_and_mask = CP6 << ring->pipe;
3877 case RADEON_RING_TYPE_GFX_INDEX:
3882 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3883 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3884 WAIT_REG_MEM_FUNCTION(3) | /* == */
3885 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3886 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3887 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3888 radeon_ring_write(ring, ref_and_mask);
3889 radeon_ring_write(ring, ref_and_mask);
3890 radeon_ring_write(ring, 0x20); /* poll interval */
3894 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3896 * @rdev: radeon_device pointer
3897 * @fence: radeon fence object
3899 * Emits a fence sequnce number on the gfx ring and flushes
3902 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3903 struct radeon_fence *fence)
3905 struct radeon_ring *ring = &rdev->ring[fence->ring];
3906 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3908 /* Workaround for cache flush problems. First send a dummy EOP
3909 * event down the pipe with seq one below.
3911 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3912 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3914 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3916 radeon_ring_write(ring, addr & 0xfffffffc);
3917 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3918 DATA_SEL(1) | INT_SEL(0));
3919 radeon_ring_write(ring, fence->seq - 1);
3920 radeon_ring_write(ring, 0);
3922 /* Then send the real EOP event down the pipe. */
3923 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3924 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3926 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3928 radeon_ring_write(ring, addr & 0xfffffffc);
3929 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3930 radeon_ring_write(ring, fence->seq);
3931 radeon_ring_write(ring, 0);
3935 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3937 * @rdev: radeon_device pointer
3938 * @fence: radeon fence object
3940 * Emits a fence sequnce number on the compute ring and flushes
3943 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3944 struct radeon_fence *fence)
3946 struct radeon_ring *ring = &rdev->ring[fence->ring];
3947 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3949 /* RELEASE_MEM - flush caches, send int */
3950 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3951 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3953 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3955 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3956 radeon_ring_write(ring, addr & 0xfffffffc);
3957 radeon_ring_write(ring, upper_32_bits(addr));
3958 radeon_ring_write(ring, fence->seq);
3959 radeon_ring_write(ring, 0);
3963 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3965 * @rdev: radeon_device pointer
3966 * @ring: radeon ring buffer object
3967 * @semaphore: radeon semaphore object
3968 * @emit_wait: Is this a sempahore wait?
3970 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3971 * from running ahead of semaphore waits.
3973 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3974 struct radeon_ring *ring,
3975 struct radeon_semaphore *semaphore,
3978 uint64_t addr = semaphore->gpu_addr;
3979 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3981 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3982 radeon_ring_write(ring, lower_32_bits(addr));
3983 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3985 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3986 /* Prevent the PFP from running ahead of the semaphore wait */
3987 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3988 radeon_ring_write(ring, 0x0);
3995 * cik_copy_cpdma - copy pages using the CP DMA engine
3997 * @rdev: radeon_device pointer
3998 * @src_offset: src GPU address
3999 * @dst_offset: dst GPU address
4000 * @num_gpu_pages: number of GPU pages to xfer
4001 * @resv: reservation object to sync to
4003 * Copy GPU paging using the CP DMA engine (CIK+).
4004 * Used by the radeon ttm implementation to move pages if
4005 * registered as the asic copy callback.
4007 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
4008 uint64_t src_offset, uint64_t dst_offset,
4009 unsigned num_gpu_pages,
4010 struct reservation_object *resv)
4012 struct radeon_fence *fence;
4013 struct radeon_sync sync;
4014 int ring_index = rdev->asic->copy.blit_ring_index;
4015 struct radeon_ring *ring = &rdev->ring[ring_index];
4016 u32 size_in_bytes, cur_size_in_bytes, control;
4020 radeon_sync_create(&sync);
4022 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
4023 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
4024 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
4026 DRM_ERROR("radeon: moving bo (%d).\n", r);
4027 radeon_sync_free(rdev, &sync, NULL);
4031 radeon_sync_resv(rdev, &sync, resv, false);
4032 radeon_sync_rings(rdev, &sync, ring->idx);
4034 for (i = 0; i < num_loops; i++) {
4035 cur_size_in_bytes = size_in_bytes;
4036 if (cur_size_in_bytes > 0x1fffff)
4037 cur_size_in_bytes = 0x1fffff;
4038 size_in_bytes -= cur_size_in_bytes;
4040 if (size_in_bytes == 0)
4041 control |= PACKET3_DMA_DATA_CP_SYNC;
4042 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4043 radeon_ring_write(ring, control);
4044 radeon_ring_write(ring, lower_32_bits(src_offset));
4045 radeon_ring_write(ring, upper_32_bits(src_offset));
4046 radeon_ring_write(ring, lower_32_bits(dst_offset));
4047 radeon_ring_write(ring, upper_32_bits(dst_offset));
4048 radeon_ring_write(ring, cur_size_in_bytes);
4049 src_offset += cur_size_in_bytes;
4050 dst_offset += cur_size_in_bytes;
4053 r = radeon_fence_emit(rdev, &fence, ring->idx);
4055 radeon_ring_unlock_undo(rdev, ring);
4056 radeon_sync_free(rdev, &sync, NULL);
4060 radeon_ring_unlock_commit(rdev, ring, false);
4061 radeon_sync_free(rdev, &sync, fence);
4070 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4072 * @rdev: radeon_device pointer
4073 * @ib: radeon indirect buffer object
4075 * Emits an DE (drawing engine) or CE (constant engine) IB
4076 * on the gfx ring. IBs are usually generated by userspace
4077 * acceleration drivers and submitted to the kernel for
4078 * sheduling on the ring. This function schedules the IB
4079 * on the gfx ring for execution by the GPU.
4081 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4083 struct radeon_ring *ring = &rdev->ring[ib->ring];
4084 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
4085 u32 header, control = INDIRECT_BUFFER_VALID;
4087 if (ib->is_const_ib) {
4088 /* set switch buffer packet before const IB */
4089 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4090 radeon_ring_write(ring, 0);
4092 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4095 if (ring->rptr_save_reg) {
4096 next_rptr = ring->wptr + 3 + 4;
4097 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4098 radeon_ring_write(ring, ((ring->rptr_save_reg -
4099 PACKET3_SET_UCONFIG_REG_START) >> 2));
4100 radeon_ring_write(ring, next_rptr);
4101 } else if (rdev->wb.enabled) {
4102 next_rptr = ring->wptr + 5 + 4;
4103 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4104 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4105 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4106 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4107 radeon_ring_write(ring, next_rptr);
4110 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4113 control |= ib->length_dw | (vm_id << 24);
4115 radeon_ring_write(ring, header);
4116 radeon_ring_write(ring,
4120 (ib->gpu_addr & 0xFFFFFFFC));
4121 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4122 radeon_ring_write(ring, control);
4126 * cik_ib_test - basic gfx ring IB test
4128 * @rdev: radeon_device pointer
4129 * @ring: radeon_ring structure holding ring information
4131 * Allocate an IB and execute it on the gfx ring (CIK).
4132 * Provides a basic gfx ring test to verify that IBs are working.
4133 * Returns 0 on success, error on failure.
4135 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4137 struct radeon_ib ib;
4143 r = radeon_scratch_get(rdev, &scratch);
4145 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4148 WREG32(scratch, 0xCAFEDEAD);
4149 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4151 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4152 radeon_scratch_free(rdev, scratch);
4155 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4156 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4157 ib.ptr[2] = 0xDEADBEEF;
4159 r = radeon_ib_schedule(rdev, &ib, NULL, false);
4161 radeon_scratch_free(rdev, scratch);
4162 radeon_ib_free(rdev, &ib);
4163 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4166 r = radeon_fence_wait(ib.fence, false);
4168 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4169 radeon_scratch_free(rdev, scratch);
4170 radeon_ib_free(rdev, &ib);
4173 for (i = 0; i < rdev->usec_timeout; i++) {
4174 tmp = RREG32(scratch);
4175 if (tmp == 0xDEADBEEF)
4179 if (i < rdev->usec_timeout) {
4180 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4182 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4186 radeon_scratch_free(rdev, scratch);
4187 radeon_ib_free(rdev, &ib);
4193 * On CIK, gfx and compute now have independant command processors.
4196 * Gfx consists of a single ring and can process both gfx jobs and
4197 * compute jobs. The gfx CP consists of three microengines (ME):
4198 * PFP - Pre-Fetch Parser
4200 * CE - Constant Engine
4201 * The PFP and ME make up what is considered the Drawing Engine (DE).
4202 * The CE is an asynchronous engine used for updating buffer desciptors
4203 * used by the DE so that they can be loaded into cache in parallel
4204 * while the DE is processing state update packets.
4207 * The compute CP consists of two microengines (ME):
4208 * MEC1 - Compute MicroEngine 1
4209 * MEC2 - Compute MicroEngine 2
4210 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4211 * The queues are exposed to userspace and are programmed directly
4212 * by the compute runtime.
4215 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4217 * @rdev: radeon_device pointer
4218 * @enable: enable or disable the MEs
4220 * Halts or unhalts the gfx MEs.
4222 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4225 WREG32(CP_ME_CNTL, 0);
4227 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4228 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4229 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4230 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4236 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4238 * @rdev: radeon_device pointer
4240 * Loads the gfx PFP, ME, and CE ucode.
4241 * Returns 0 for success, -EINVAL if the ucode is not available.
4243 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4247 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4250 cik_cp_gfx_enable(rdev, false);
4253 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4254 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4255 const struct gfx_firmware_header_v1_0 *ce_hdr =
4256 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4257 const struct gfx_firmware_header_v1_0 *me_hdr =
4258 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4259 const __le32 *fw_data;
4262 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4263 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4264 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4267 fw_data = (const __le32 *)
4268 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4269 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4270 WREG32(CP_PFP_UCODE_ADDR, 0);
4271 for (i = 0; i < fw_size; i++)
4272 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4273 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
4276 fw_data = (const __le32 *)
4277 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4278 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4279 WREG32(CP_CE_UCODE_ADDR, 0);
4280 for (i = 0; i < fw_size; i++)
4281 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4282 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
4285 fw_data = (const __be32 *)
4286 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4287 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4288 WREG32(CP_ME_RAM_WADDR, 0);
4289 for (i = 0; i < fw_size; i++)
4290 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4291 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4292 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
4294 const __be32 *fw_data;
4297 fw_data = (const __be32 *)rdev->pfp_fw->data;
4298 WREG32(CP_PFP_UCODE_ADDR, 0);
4299 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4300 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4301 WREG32(CP_PFP_UCODE_ADDR, 0);
4304 fw_data = (const __be32 *)rdev->ce_fw->data;
4305 WREG32(CP_CE_UCODE_ADDR, 0);
4306 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4307 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4308 WREG32(CP_CE_UCODE_ADDR, 0);
4311 fw_data = (const __be32 *)rdev->me_fw->data;
4312 WREG32(CP_ME_RAM_WADDR, 0);
4313 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4314 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4315 WREG32(CP_ME_RAM_WADDR, 0);
4322 * cik_cp_gfx_start - start the gfx ring
4324 * @rdev: radeon_device pointer
4326 * Enables the ring and loads the clear state context and other
4327 * packets required to init the ring.
4328 * Returns 0 for success, error for failure.
4330 static int cik_cp_gfx_start(struct radeon_device *rdev)
4332 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4336 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4337 WREG32(CP_ENDIAN_SWAP, 0);
4338 WREG32(CP_DEVICE_ID, 1);
4340 cik_cp_gfx_enable(rdev, true);
4342 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4344 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4348 /* init the CE partitions. CE only used for gfx on CIK */
4349 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4350 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4351 radeon_ring_write(ring, 0x8000);
4352 radeon_ring_write(ring, 0x8000);
4354 /* setup clear context state */
4355 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4356 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4358 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4359 radeon_ring_write(ring, 0x80000000);
4360 radeon_ring_write(ring, 0x80000000);
4362 for (i = 0; i < cik_default_size; i++)
4363 radeon_ring_write(ring, cik_default_state[i]);
4365 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4366 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4368 /* set clear context state */
4369 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4370 radeon_ring_write(ring, 0);
4372 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4373 radeon_ring_write(ring, 0x00000316);
4374 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4375 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4377 radeon_ring_unlock_commit(rdev, ring, false);
4383 * cik_cp_gfx_fini - stop the gfx ring
4385 * @rdev: radeon_device pointer
4387 * Stop the gfx ring and tear down the driver ring
4390 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4392 cik_cp_gfx_enable(rdev, false);
4393 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4397 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4399 * @rdev: radeon_device pointer
4401 * Program the location and size of the gfx ring buffer
4402 * and test it to make sure it's working.
4403 * Returns 0 for success, error for failure.
4405 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4407 struct radeon_ring *ring;
4413 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4414 if (rdev->family != CHIP_HAWAII)
4415 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4417 /* Set the write pointer delay */
4418 WREG32(CP_RB_WPTR_DELAY, 0);
4420 /* set the RB to use vmid 0 */
4421 WREG32(CP_RB_VMID, 0);
4423 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4425 /* ring 0 - compute and gfx */
4426 /* Set ring buffer size */
4427 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4428 rb_bufsz = order_base_2(ring->ring_size / 8);
4429 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4431 tmp |= BUF_SWAP_32BIT;
4433 WREG32(CP_RB0_CNTL, tmp);
4435 /* Initialize the ring buffer's read and write pointers */
4436 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4438 WREG32(CP_RB0_WPTR, ring->wptr);
4440 /* set the wb address wether it's enabled or not */
4441 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4442 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4444 /* scratch register shadowing is no longer supported */
4445 WREG32(SCRATCH_UMSK, 0);
4447 if (!rdev->wb.enabled)
4448 tmp |= RB_NO_UPDATE;
4451 WREG32(CP_RB0_CNTL, tmp);
4453 rb_addr = ring->gpu_addr >> 8;
4454 WREG32(CP_RB0_BASE, rb_addr);
4455 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4457 /* start the ring */
4458 cik_cp_gfx_start(rdev);
4459 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4460 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4462 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4466 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4467 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4472 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4473 struct radeon_ring *ring)
4477 if (rdev->wb.enabled)
4478 rptr = rdev->wb.wb[ring->rptr_offs/4];
4480 rptr = RREG32(CP_RB0_RPTR);
4485 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4486 struct radeon_ring *ring)
4490 wptr = RREG32(CP_RB0_WPTR);
4495 void cik_gfx_set_wptr(struct radeon_device *rdev,
4496 struct radeon_ring *ring)
4498 WREG32(CP_RB0_WPTR, ring->wptr);
4499 (void)RREG32(CP_RB0_WPTR);
4502 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4503 struct radeon_ring *ring)
4507 if (rdev->wb.enabled) {
4508 rptr = rdev->wb.wb[ring->rptr_offs/4];
4510 mutex_lock(&rdev->srbm_mutex);
4511 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4512 rptr = RREG32(CP_HQD_PQ_RPTR);
4513 cik_srbm_select(rdev, 0, 0, 0, 0);
4514 mutex_unlock(&rdev->srbm_mutex);
4520 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4521 struct radeon_ring *ring)
4525 if (rdev->wb.enabled) {
4526 /* XXX check if swapping is necessary on BE */
4527 wptr = rdev->wb.wb[ring->wptr_offs/4];
4529 mutex_lock(&rdev->srbm_mutex);
4530 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4531 wptr = RREG32(CP_HQD_PQ_WPTR);
4532 cik_srbm_select(rdev, 0, 0, 0, 0);
4533 mutex_unlock(&rdev->srbm_mutex);
4539 void cik_compute_set_wptr(struct radeon_device *rdev,
4540 struct radeon_ring *ring)
4542 /* XXX check if swapping is necessary on BE */
4543 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4544 WDOORBELL32(ring->doorbell_index, ring->wptr);
4548 * cik_cp_compute_enable - enable/disable the compute CP MEs
4550 * @rdev: radeon_device pointer
4551 * @enable: enable or disable the MEs
4553 * Halts or unhalts the compute MEs.
4555 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4558 WREG32(CP_MEC_CNTL, 0);
4560 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4561 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4562 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4568 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4570 * @rdev: radeon_device pointer
4572 * Loads the compute MEC1&2 ucode.
4573 * Returns 0 for success, -EINVAL if the ucode is not available.
4575 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4582 cik_cp_compute_enable(rdev, false);
4585 const struct gfx_firmware_header_v1_0 *mec_hdr =
4586 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4587 const __le32 *fw_data;
4590 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4593 fw_data = (const __le32 *)
4594 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4595 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4596 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4597 for (i = 0; i < fw_size; i++)
4598 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4599 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4602 if (rdev->family == CHIP_KAVERI) {
4603 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4604 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4606 fw_data = (const __le32 *)
4607 (rdev->mec2_fw->data +
4608 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4609 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4610 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4611 for (i = 0; i < fw_size; i++)
4612 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4613 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4616 const __be32 *fw_data;
4619 fw_data = (const __be32 *)rdev->mec_fw->data;
4620 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4621 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4622 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4623 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4625 if (rdev->family == CHIP_KAVERI) {
4627 fw_data = (const __be32 *)rdev->mec_fw->data;
4628 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4629 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4630 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4631 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4639 * cik_cp_compute_start - start the compute queues
4641 * @rdev: radeon_device pointer
4643 * Enable the compute queues.
4644 * Returns 0 for success, error for failure.
4646 static int cik_cp_compute_start(struct radeon_device *rdev)
4648 cik_cp_compute_enable(rdev, true);
4654 * cik_cp_compute_fini - stop the compute queues
4656 * @rdev: radeon_device pointer
4658 * Stop the compute queues and tear down the driver queue
4661 static void cik_cp_compute_fini(struct radeon_device *rdev)
4665 cik_cp_compute_enable(rdev, false);
4667 for (i = 0; i < 2; i++) {
4669 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4671 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4673 if (rdev->ring[idx].mqd_obj) {
4674 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4675 if (unlikely(r != 0))
4676 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4678 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4679 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4681 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4682 rdev->ring[idx].mqd_obj = NULL;
4687 static void cik_mec_fini(struct radeon_device *rdev)
4691 if (rdev->mec.hpd_eop_obj) {
4692 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4693 if (unlikely(r != 0))
4694 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4695 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4696 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4698 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4699 rdev->mec.hpd_eop_obj = NULL;
4703 #define MEC_HPD_SIZE 2048
4705 static int cik_mec_init(struct radeon_device *rdev)
4711 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4712 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4713 * Nonetheless, we assign only 1 pipe because all other pipes will
4716 rdev->mec.num_mec = 1;
4717 rdev->mec.num_pipe = 1;
4718 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4720 if (rdev->mec.hpd_eop_obj == NULL) {
4721 r = radeon_bo_create(rdev,
4722 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4724 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4725 &rdev->mec.hpd_eop_obj);
4727 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4732 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4733 if (unlikely(r != 0)) {
4737 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4738 &rdev->mec.hpd_eop_gpu_addr);
4740 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4744 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4746 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4751 /* clear memory. Not sure if this is required or not */
4752 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4754 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4755 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4760 struct hqd_registers
4762 u32 cp_mqd_base_addr;
4763 u32 cp_mqd_base_addr_hi;
4766 u32 cp_hqd_persistent_state;
4767 u32 cp_hqd_pipe_priority;
4768 u32 cp_hqd_queue_priority;
4771 u32 cp_hqd_pq_base_hi;
4773 u32 cp_hqd_pq_rptr_report_addr;
4774 u32 cp_hqd_pq_rptr_report_addr_hi;
4775 u32 cp_hqd_pq_wptr_poll_addr;
4776 u32 cp_hqd_pq_wptr_poll_addr_hi;
4777 u32 cp_hqd_pq_doorbell_control;
4779 u32 cp_hqd_pq_control;
4780 u32 cp_hqd_ib_base_addr;
4781 u32 cp_hqd_ib_base_addr_hi;
4783 u32 cp_hqd_ib_control;
4784 u32 cp_hqd_iq_timer;
4786 u32 cp_hqd_dequeue_request;
4787 u32 cp_hqd_dma_offload;
4788 u32 cp_hqd_sema_cmd;
4789 u32 cp_hqd_msg_type;
4790 u32 cp_hqd_atomic0_preop_lo;
4791 u32 cp_hqd_atomic0_preop_hi;
4792 u32 cp_hqd_atomic1_preop_lo;
4793 u32 cp_hqd_atomic1_preop_hi;
4794 u32 cp_hqd_hq_scheduler0;
4795 u32 cp_hqd_hq_scheduler1;
4802 u32 dispatch_initiator;
4806 u32 pipeline_stat_enable;
4807 u32 perf_counter_enable;
4813 u32 resource_limits;
4814 u32 static_thread_mgmt01[2];
4816 u32 static_thread_mgmt23[2];
4818 u32 thread_trace_enable;
4821 u32 vgtcs_invoke_count[2];
4822 struct hqd_registers queue_state;
4824 u32 interrupt_queue[64];
4828 * cik_cp_compute_resume - setup the compute queue registers
4830 * @rdev: radeon_device pointer
4832 * Program the compute queues and test them to make sure they
4834 * Returns 0 for success, error for failure.
4836 static int cik_cp_compute_resume(struct radeon_device *rdev)
4840 bool use_doorbell = true;
4846 struct bonaire_mqd *mqd;
4848 r = cik_cp_compute_start(rdev);
4852 /* fix up chicken bits */
4853 tmp = RREG32(CP_CPF_DEBUG);
4855 WREG32(CP_CPF_DEBUG, tmp);
4857 /* init the pipes */
4858 mutex_lock(&rdev->srbm_mutex);
4860 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
4862 cik_srbm_select(rdev, 0, 0, 0, 0);
4864 /* write the EOP addr */
4865 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4866 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4868 /* set the VMID assigned */
4869 WREG32(CP_HPD_EOP_VMID, 0);
4871 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4872 tmp = RREG32(CP_HPD_EOP_CONTROL);
4873 tmp &= ~EOP_SIZE_MASK;
4874 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4875 WREG32(CP_HPD_EOP_CONTROL, tmp);
4877 mutex_unlock(&rdev->srbm_mutex);
4879 /* init the queues. Just two for now. */
4880 for (i = 0; i < 2; i++) {
4882 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4884 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4886 if (rdev->ring[idx].mqd_obj == NULL) {
4887 r = radeon_bo_create(rdev,
4888 sizeof(struct bonaire_mqd),
4890 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4891 NULL, &rdev->ring[idx].mqd_obj);
4893 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4898 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4899 if (unlikely(r != 0)) {
4900 cik_cp_compute_fini(rdev);
4903 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4906 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4907 cik_cp_compute_fini(rdev);
4910 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4912 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4913 cik_cp_compute_fini(rdev);
4917 /* init the mqd struct */
4918 memset(buf, 0, sizeof(struct bonaire_mqd));
4920 mqd = (struct bonaire_mqd *)buf;
4921 mqd->header = 0xC0310800;
4922 mqd->static_thread_mgmt01[0] = 0xffffffff;
4923 mqd->static_thread_mgmt01[1] = 0xffffffff;
4924 mqd->static_thread_mgmt23[0] = 0xffffffff;
4925 mqd->static_thread_mgmt23[1] = 0xffffffff;
4927 mutex_lock(&rdev->srbm_mutex);
4928 cik_srbm_select(rdev, rdev->ring[idx].me,
4929 rdev->ring[idx].pipe,
4930 rdev->ring[idx].queue, 0);
4932 /* disable wptr polling */
4933 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4934 tmp &= ~WPTR_POLL_EN;
4935 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4937 /* enable doorbell? */
4938 mqd->queue_state.cp_hqd_pq_doorbell_control =
4939 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4941 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4943 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4944 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4945 mqd->queue_state.cp_hqd_pq_doorbell_control);
4947 /* disable the queue if it's active */
4948 mqd->queue_state.cp_hqd_dequeue_request = 0;
4949 mqd->queue_state.cp_hqd_pq_rptr = 0;
4950 mqd->queue_state.cp_hqd_pq_wptr= 0;
4951 if (RREG32(CP_HQD_ACTIVE) & 1) {
4952 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4953 for (j = 0; j < rdev->usec_timeout; j++) {
4954 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4958 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4959 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4960 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4963 /* set the pointer to the MQD */
4964 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4965 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4966 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4967 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4968 /* set MQD vmid to 0 */
4969 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4970 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4971 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4973 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4974 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4975 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4976 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4977 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4978 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4980 /* set up the HQD, this is similar to CP_RB0_CNTL */
4981 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4982 mqd->queue_state.cp_hqd_pq_control &=
4983 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4985 mqd->queue_state.cp_hqd_pq_control |=
4986 order_base_2(rdev->ring[idx].ring_size / 8);
4987 mqd->queue_state.cp_hqd_pq_control |=
4988 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4990 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4992 mqd->queue_state.cp_hqd_pq_control &=
4993 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4994 mqd->queue_state.cp_hqd_pq_control |=
4995 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4996 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4998 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
5000 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
5002 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
5003 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
5004 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
5005 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
5006 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
5007 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
5009 /* set the wb address wether it's enabled or not */
5011 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
5013 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
5014 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
5015 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
5016 upper_32_bits(wb_gpu_addr) & 0xffff;
5017 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
5018 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
5019 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
5020 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
5022 /* enable the doorbell if requested */
5024 mqd->queue_state.cp_hqd_pq_doorbell_control =
5025 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5026 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5027 mqd->queue_state.cp_hqd_pq_doorbell_control |=
5028 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
5029 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5030 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5031 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5034 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5036 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5037 mqd->queue_state.cp_hqd_pq_doorbell_control);
5039 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5040 rdev->ring[idx].wptr = 0;
5041 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5042 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5043 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5045 /* set the vmid for the queue */
5046 mqd->queue_state.cp_hqd_vmid = 0;
5047 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5049 /* activate the queue */
5050 mqd->queue_state.cp_hqd_active = 1;
5051 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5053 cik_srbm_select(rdev, 0, 0, 0, 0);
5054 mutex_unlock(&rdev->srbm_mutex);
5056 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5057 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5059 rdev->ring[idx].ready = true;
5060 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5062 rdev->ring[idx].ready = false;
5068 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5070 cik_cp_gfx_enable(rdev, enable);
5071 cik_cp_compute_enable(rdev, enable);
5074 static int cik_cp_load_microcode(struct radeon_device *rdev)
5078 r = cik_cp_gfx_load_microcode(rdev);
5081 r = cik_cp_compute_load_microcode(rdev);
5088 static void cik_cp_fini(struct radeon_device *rdev)
5090 cik_cp_gfx_fini(rdev);
5091 cik_cp_compute_fini(rdev);
5094 static int cik_cp_resume(struct radeon_device *rdev)
5098 cik_enable_gui_idle_interrupt(rdev, false);
5100 r = cik_cp_load_microcode(rdev);
5104 r = cik_cp_gfx_resume(rdev);
5107 r = cik_cp_compute_resume(rdev);
5111 cik_enable_gui_idle_interrupt(rdev, true);
5116 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5118 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5119 RREG32(GRBM_STATUS));
5120 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5121 RREG32(GRBM_STATUS2));
5122 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5123 RREG32(GRBM_STATUS_SE0));
5124 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5125 RREG32(GRBM_STATUS_SE1));
5126 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5127 RREG32(GRBM_STATUS_SE2));
5128 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5129 RREG32(GRBM_STATUS_SE3));
5130 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5131 RREG32(SRBM_STATUS));
5132 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5133 RREG32(SRBM_STATUS2));
5134 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5135 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5136 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5137 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5138 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5139 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5140 RREG32(CP_STALLED_STAT1));
5141 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5142 RREG32(CP_STALLED_STAT2));
5143 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5144 RREG32(CP_STALLED_STAT3));
5145 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5146 RREG32(CP_CPF_BUSY_STAT));
5147 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5148 RREG32(CP_CPF_STALLED_STAT1));
5149 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5150 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5151 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5152 RREG32(CP_CPC_STALLED_STAT1));
5153 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5157 * cik_gpu_check_soft_reset - check which blocks are busy
5159 * @rdev: radeon_device pointer
5161 * Check which blocks are busy and return the relevant reset
5162 * mask to be used by cik_gpu_soft_reset().
5163 * Returns a mask of the blocks to be reset.
5165 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5171 tmp = RREG32(GRBM_STATUS);
5172 if (tmp & (PA_BUSY | SC_BUSY |
5173 BCI_BUSY | SX_BUSY |
5174 TA_BUSY | VGT_BUSY |
5176 GDS_BUSY | SPI_BUSY |
5177 IA_BUSY | IA_BUSY_NO_DMA))
5178 reset_mask |= RADEON_RESET_GFX;
5180 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5181 reset_mask |= RADEON_RESET_CP;
5184 tmp = RREG32(GRBM_STATUS2);
5186 reset_mask |= RADEON_RESET_RLC;
5188 /* SDMA0_STATUS_REG */
5189 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5190 if (!(tmp & SDMA_IDLE))
5191 reset_mask |= RADEON_RESET_DMA;
5193 /* SDMA1_STATUS_REG */
5194 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5195 if (!(tmp & SDMA_IDLE))
5196 reset_mask |= RADEON_RESET_DMA1;
5199 tmp = RREG32(SRBM_STATUS2);
5200 if (tmp & SDMA_BUSY)
5201 reset_mask |= RADEON_RESET_DMA;
5203 if (tmp & SDMA1_BUSY)
5204 reset_mask |= RADEON_RESET_DMA1;
5207 tmp = RREG32(SRBM_STATUS);
5210 reset_mask |= RADEON_RESET_IH;
5213 reset_mask |= RADEON_RESET_SEM;
5215 if (tmp & GRBM_RQ_PENDING)
5216 reset_mask |= RADEON_RESET_GRBM;
5219 reset_mask |= RADEON_RESET_VMC;
5221 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5222 MCC_BUSY | MCD_BUSY))
5223 reset_mask |= RADEON_RESET_MC;
5225 if (evergreen_is_display_hung(rdev))
5226 reset_mask |= RADEON_RESET_DISPLAY;
5228 /* Skip MC reset as it's mostly likely not hung, just busy */
5229 if (reset_mask & RADEON_RESET_MC) {
5230 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5231 reset_mask &= ~RADEON_RESET_MC;
5238 * cik_gpu_soft_reset - soft reset GPU
5240 * @rdev: radeon_device pointer
5241 * @reset_mask: mask of which blocks to reset
5243 * Soft reset the blocks specified in @reset_mask.
5245 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5247 struct evergreen_mc_save save;
5248 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5251 if (reset_mask == 0)
5254 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5256 cik_print_gpu_status_regs(rdev);
5257 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5258 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5259 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5260 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5269 /* Disable GFX parsing/prefetching */
5270 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5272 /* Disable MEC parsing/prefetching */
5273 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5275 if (reset_mask & RADEON_RESET_DMA) {
5277 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5279 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5281 if (reset_mask & RADEON_RESET_DMA1) {
5283 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5285 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5288 evergreen_mc_stop(rdev, &save);
5289 if (evergreen_mc_wait_for_idle(rdev)) {
5290 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5293 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5294 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5296 if (reset_mask & RADEON_RESET_CP) {
5297 grbm_soft_reset |= SOFT_RESET_CP;
5299 srbm_soft_reset |= SOFT_RESET_GRBM;
5302 if (reset_mask & RADEON_RESET_DMA)
5303 srbm_soft_reset |= SOFT_RESET_SDMA;
5305 if (reset_mask & RADEON_RESET_DMA1)
5306 srbm_soft_reset |= SOFT_RESET_SDMA1;
5308 if (reset_mask & RADEON_RESET_DISPLAY)
5309 srbm_soft_reset |= SOFT_RESET_DC;
5311 if (reset_mask & RADEON_RESET_RLC)
5312 grbm_soft_reset |= SOFT_RESET_RLC;
5314 if (reset_mask & RADEON_RESET_SEM)
5315 srbm_soft_reset |= SOFT_RESET_SEM;
5317 if (reset_mask & RADEON_RESET_IH)
5318 srbm_soft_reset |= SOFT_RESET_IH;
5320 if (reset_mask & RADEON_RESET_GRBM)
5321 srbm_soft_reset |= SOFT_RESET_GRBM;
5323 if (reset_mask & RADEON_RESET_VMC)
5324 srbm_soft_reset |= SOFT_RESET_VMC;
5326 if (!(rdev->flags & RADEON_IS_IGP)) {
5327 if (reset_mask & RADEON_RESET_MC)
5328 srbm_soft_reset |= SOFT_RESET_MC;
5331 if (grbm_soft_reset) {
5332 tmp = RREG32(GRBM_SOFT_RESET);
5333 tmp |= grbm_soft_reset;
5334 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5335 WREG32(GRBM_SOFT_RESET, tmp);
5336 tmp = RREG32(GRBM_SOFT_RESET);
5340 tmp &= ~grbm_soft_reset;
5341 WREG32(GRBM_SOFT_RESET, tmp);
5342 tmp = RREG32(GRBM_SOFT_RESET);
5345 if (srbm_soft_reset) {
5346 tmp = RREG32(SRBM_SOFT_RESET);
5347 tmp |= srbm_soft_reset;
5348 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5349 WREG32(SRBM_SOFT_RESET, tmp);
5350 tmp = RREG32(SRBM_SOFT_RESET);
5354 tmp &= ~srbm_soft_reset;
5355 WREG32(SRBM_SOFT_RESET, tmp);
5356 tmp = RREG32(SRBM_SOFT_RESET);
5359 /* Wait a little for things to settle down */
5362 evergreen_mc_resume(rdev, &save);
5365 cik_print_gpu_status_regs(rdev);
5368 struct kv_reset_save_regs {
5369 u32 gmcon_reng_execute;
5374 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5375 struct kv_reset_save_regs *save)
5377 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5378 save->gmcon_misc = RREG32(GMCON_MISC);
5379 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5381 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5382 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5383 STCTRL_STUTTER_EN));
5386 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5387 struct kv_reset_save_regs *save)
5391 WREG32(GMCON_PGFSM_WRITE, 0);
5392 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5394 for (i = 0; i < 5; i++)
5395 WREG32(GMCON_PGFSM_WRITE, 0);
5397 WREG32(GMCON_PGFSM_WRITE, 0);
5398 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5400 for (i = 0; i < 5; i++)
5401 WREG32(GMCON_PGFSM_WRITE, 0);
5403 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5404 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5406 for (i = 0; i < 5; i++)
5407 WREG32(GMCON_PGFSM_WRITE, 0);
5409 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5410 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5412 for (i = 0; i < 5; i++)
5413 WREG32(GMCON_PGFSM_WRITE, 0);
5415 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5416 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5418 for (i = 0; i < 5; i++)
5419 WREG32(GMCON_PGFSM_WRITE, 0);
5421 WREG32(GMCON_PGFSM_WRITE, 0);
5422 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5424 for (i = 0; i < 5; i++)
5425 WREG32(GMCON_PGFSM_WRITE, 0);
5427 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5428 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5430 for (i = 0; i < 5; i++)
5431 WREG32(GMCON_PGFSM_WRITE, 0);
5433 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5434 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5436 for (i = 0; i < 5; i++)
5437 WREG32(GMCON_PGFSM_WRITE, 0);
5439 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5440 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5442 for (i = 0; i < 5; i++)
5443 WREG32(GMCON_PGFSM_WRITE, 0);
5445 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5446 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5448 for (i = 0; i < 5; i++)
5449 WREG32(GMCON_PGFSM_WRITE, 0);
5451 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5452 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5454 WREG32(GMCON_MISC3, save->gmcon_misc3);
5455 WREG32(GMCON_MISC, save->gmcon_misc);
5456 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5459 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5461 struct evergreen_mc_save save;
5462 struct kv_reset_save_regs kv_save = { 0 };
5465 dev_info(rdev->dev, "GPU pci config reset\n");
5473 /* Disable GFX parsing/prefetching */
5474 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5476 /* Disable MEC parsing/prefetching */
5477 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5480 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5482 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5484 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5486 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5487 /* XXX other engines? */
5489 /* halt the rlc, disable cp internal ints */
5494 /* disable mem access */
5495 evergreen_mc_stop(rdev, &save);
5496 if (evergreen_mc_wait_for_idle(rdev)) {
5497 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5500 if (rdev->flags & RADEON_IS_IGP)
5501 kv_save_regs_for_reset(rdev, &kv_save);
5504 pci_clear_master(rdev->pdev);
5506 radeon_pci_config_reset(rdev);
5510 /* wait for asic to come out of reset */
5511 for (i = 0; i < rdev->usec_timeout; i++) {
5512 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5517 /* does asic init need to be run first??? */
5518 if (rdev->flags & RADEON_IS_IGP)
5519 kv_restore_regs_for_reset(rdev, &kv_save);
5523 * cik_asic_reset - soft reset GPU
5525 * @rdev: radeon_device pointer
5527 * Look up which blocks are hung and attempt
5529 * Returns 0 for success.
5531 int cik_asic_reset(struct radeon_device *rdev)
5535 reset_mask = cik_gpu_check_soft_reset(rdev);
5538 r600_set_bios_scratch_engine_hung(rdev, true);
5540 /* try soft reset */
5541 cik_gpu_soft_reset(rdev, reset_mask);
5543 reset_mask = cik_gpu_check_soft_reset(rdev);
5545 /* try pci config reset */
5546 if (reset_mask && radeon_hard_reset)
5547 cik_gpu_pci_config_reset(rdev);
5549 reset_mask = cik_gpu_check_soft_reset(rdev);
5552 r600_set_bios_scratch_engine_hung(rdev, false);
5558 * cik_gfx_is_lockup - check if the 3D engine is locked up
5560 * @rdev: radeon_device pointer
5561 * @ring: radeon_ring structure holding ring information
5563 * Check if the 3D engine is locked up (CIK).
5564 * Returns true if the engine is locked, false if not.
5566 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5568 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5570 if (!(reset_mask & (RADEON_RESET_GFX |
5571 RADEON_RESET_COMPUTE |
5572 RADEON_RESET_CP))) {
5573 radeon_ring_lockup_update(rdev, ring);
5576 return radeon_ring_test_lockup(rdev, ring);
5581 * cik_mc_program - program the GPU memory controller
5583 * @rdev: radeon_device pointer
5585 * Set the location of vram, gart, and AGP in the GPU's
5586 * physical address space (CIK).
5588 static void cik_mc_program(struct radeon_device *rdev)
5590 struct evergreen_mc_save save;
5594 /* Initialize HDP */
5595 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5596 WREG32((0x2c14 + j), 0x00000000);
5597 WREG32((0x2c18 + j), 0x00000000);
5598 WREG32((0x2c1c + j), 0x00000000);
5599 WREG32((0x2c20 + j), 0x00000000);
5600 WREG32((0x2c24 + j), 0x00000000);
5602 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5604 evergreen_mc_stop(rdev, &save);
5605 if (radeon_mc_wait_for_idle(rdev)) {
5606 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5608 /* Lockout access through VGA aperture*/
5609 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5610 /* Update configuration */
5611 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5612 rdev->mc.vram_start >> 12);
5613 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5614 rdev->mc.vram_end >> 12);
5615 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5616 rdev->vram_scratch.gpu_addr >> 12);
5617 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5618 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5619 WREG32(MC_VM_FB_LOCATION, tmp);
5620 /* XXX double check these! */
5621 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5622 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5623 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5624 WREG32(MC_VM_AGP_BASE, 0);
5625 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5626 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5627 if (radeon_mc_wait_for_idle(rdev)) {
5628 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5630 evergreen_mc_resume(rdev, &save);
5631 /* we need to own VRAM, so turn off the VGA renderer here
5632 * to stop it overwriting our objects */
5633 rv515_vga_render_disable(rdev);
5637 * cik_mc_init - initialize the memory controller driver params
5639 * @rdev: radeon_device pointer
5641 * Look up the amount of vram, vram width, and decide how to place
5642 * vram and gart within the GPU's physical address space (CIK).
5643 * Returns 0 for success.
5645 static int cik_mc_init(struct radeon_device *rdev)
5648 int chansize, numchan;
5650 /* Get VRAM informations */
5651 rdev->mc.vram_is_ddr = true;
5652 tmp = RREG32(MC_ARB_RAMCFG);
5653 if (tmp & CHANSIZE_MASK) {
5658 tmp = RREG32(MC_SHARED_CHMAP);
5659 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5689 rdev->mc.vram_width = numchan * chansize;
5690 /* Could aper size report 0 ? */
5691 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5692 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5693 /* size in MB on si */
5694 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5695 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5696 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5697 si_vram_gtt_location(rdev, &rdev->mc);
5698 radeon_update_bandwidth_info(rdev);
5705 * VMID 0 is the physical GPU addresses as used by the kernel.
5706 * VMIDs 1-15 are used for userspace clients and are handled
5707 * by the radeon vm/hsa code.
5710 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5712 * @rdev: radeon_device pointer
5714 * Flush the TLB for the VMID 0 page table (CIK).
5716 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5718 /* flush hdp cache */
5719 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5721 /* bits 0-15 are the VM contexts0-15 */
5722 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5725 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5728 uint32_t sh_mem_bases, sh_mem_config;
5730 sh_mem_bases = 0x6000 | 0x6000 << 16;
5731 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5732 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5734 mutex_lock(&rdev->srbm_mutex);
5735 for (i = 8; i < 16; i++) {
5736 cik_srbm_select(rdev, 0, 0, 0, i);
5737 /* CP and shaders */
5738 WREG32(SH_MEM_CONFIG, sh_mem_config);
5739 WREG32(SH_MEM_APE1_BASE, 1);
5740 WREG32(SH_MEM_APE1_LIMIT, 0);
5741 WREG32(SH_MEM_BASES, sh_mem_bases);
5743 cik_srbm_select(rdev, 0, 0, 0, 0);
5744 mutex_unlock(&rdev->srbm_mutex);
5748 * cik_pcie_gart_enable - gart enable
5750 * @rdev: radeon_device pointer
5752 * This sets up the TLBs, programs the page tables for VMID0,
5753 * sets up the hw for VMIDs 1-15 which are allocated on
5754 * demand, and sets up the global locations for the LDS, GDS,
5755 * and GPUVM for FSA64 clients (CIK).
5756 * Returns 0 for success, errors for failure.
5758 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5762 if (rdev->gart.robj == NULL) {
5763 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5766 r = radeon_gart_table_vram_pin(rdev);
5769 /* Setup TLB control */
5770 WREG32(MC_VM_MX_L1_TLB_CNTL,
5773 ENABLE_L1_FRAGMENT_PROCESSING |
5774 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5775 ENABLE_ADVANCED_DRIVER_MODEL |
5776 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5777 /* Setup L2 cache */
5778 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5779 ENABLE_L2_FRAGMENT_PROCESSING |
5780 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5781 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5782 EFFECTIVE_L2_QUEUE_SIZE(7) |
5783 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5784 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5785 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5787 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5788 /* setup context0 */
5789 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5790 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5791 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5792 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5793 (u32)(rdev->dummy_page.addr >> 12));
5794 WREG32(VM_CONTEXT0_CNTL2, 0);
5795 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5796 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5802 /* restore context1-15 */
5803 /* set vm size, must be a multiple of 4 */
5804 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5805 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5806 for (i = 1; i < 16; i++) {
5808 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5809 rdev->vm_manager.saved_table_addr[i]);
5811 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5812 rdev->vm_manager.saved_table_addr[i]);
5815 /* enable context1-15 */
5816 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5817 (u32)(rdev->dummy_page.addr >> 12));
5818 WREG32(VM_CONTEXT1_CNTL2, 4);
5819 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5820 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5821 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5822 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5823 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5824 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5825 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5826 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5827 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5828 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5829 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5830 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5831 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5832 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5834 if (rdev->family == CHIP_KAVERI) {
5835 u32 tmp = RREG32(CHUB_CONTROL);
5837 WREG32(CHUB_CONTROL, tmp);
5840 /* XXX SH_MEM regs */
5841 /* where to put LDS, scratch, GPUVM in FSA64 space */
5842 mutex_lock(&rdev->srbm_mutex);
5843 for (i = 0; i < 16; i++) {
5844 cik_srbm_select(rdev, 0, 0, 0, i);
5845 /* CP and shaders */
5846 WREG32(SH_MEM_CONFIG, 0);
5847 WREG32(SH_MEM_APE1_BASE, 1);
5848 WREG32(SH_MEM_APE1_LIMIT, 0);
5849 WREG32(SH_MEM_BASES, 0);
5851 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5852 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5853 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5854 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5855 /* XXX SDMA RLC - todo */
5857 cik_srbm_select(rdev, 0, 0, 0, 0);
5858 mutex_unlock(&rdev->srbm_mutex);
5860 cik_pcie_init_compute_vmid(rdev);
5862 cik_pcie_gart_tlb_flush(rdev);
5863 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5864 (unsigned)(rdev->mc.gtt_size >> 20),
5865 (unsigned long long)rdev->gart.table_addr);
5866 rdev->gart.ready = true;
5871 * cik_pcie_gart_disable - gart disable
5873 * @rdev: radeon_device pointer
5875 * This disables all VM page table (CIK).
5877 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5881 for (i = 1; i < 16; ++i) {
5884 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5886 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5887 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5890 /* Disable all tables */
5891 WREG32(VM_CONTEXT0_CNTL, 0);
5892 WREG32(VM_CONTEXT1_CNTL, 0);
5893 /* Setup TLB control */
5894 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5895 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5896 /* Setup L2 cache */
5898 ENABLE_L2_FRAGMENT_PROCESSING |
5899 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5900 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5901 EFFECTIVE_L2_QUEUE_SIZE(7) |
5902 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5903 WREG32(VM_L2_CNTL2, 0);
5904 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5905 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5906 radeon_gart_table_vram_unpin(rdev);
5910 * cik_pcie_gart_fini - vm fini callback
5912 * @rdev: radeon_device pointer
5914 * Tears down the driver GART/VM setup (CIK).
5916 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5918 cik_pcie_gart_disable(rdev);
5919 radeon_gart_table_vram_free(rdev);
5920 radeon_gart_fini(rdev);
5925 * cik_ib_parse - vm ib_parse callback
5927 * @rdev: radeon_device pointer
5928 * @ib: indirect buffer pointer
5930 * CIK uses hw IB checking so this is a nop (CIK).
5932 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5939 * VMID 0 is the physical GPU addresses as used by the kernel.
5940 * VMIDs 1-15 are used for userspace clients and are handled
5941 * by the radeon vm/hsa code.
5944 * cik_vm_init - cik vm init callback
5946 * @rdev: radeon_device pointer
5948 * Inits cik specific vm parameters (number of VMs, base of vram for
5949 * VMIDs 1-15) (CIK).
5950 * Returns 0 for success.
5952 int cik_vm_init(struct radeon_device *rdev)
5956 * VMID 0 is reserved for System
5957 * radeon graphics/compute will use VMIDs 1-7
5958 * amdkfd will use VMIDs 8-15
5960 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
5961 /* base offset of vram pages */
5962 if (rdev->flags & RADEON_IS_IGP) {
5963 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5965 rdev->vm_manager.vram_base_offset = tmp;
5967 rdev->vm_manager.vram_base_offset = 0;
5973 * cik_vm_fini - cik vm fini callback
5975 * @rdev: radeon_device pointer
5977 * Tear down any asic specific VM setup (CIK).
5979 void cik_vm_fini(struct radeon_device *rdev)
5984 * cik_vm_decode_fault - print human readable fault info
5986 * @rdev: radeon_device pointer
5987 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5988 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5990 * Print human readable fault information (CIK).
5992 static void cik_vm_decode_fault(struct radeon_device *rdev,
5993 u32 status, u32 addr, u32 mc_client)
5996 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5997 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5998 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5999 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
6001 if (rdev->family == CHIP_HAWAII)
6002 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6004 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6006 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
6007 protections, vmid, addr,
6008 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
6009 block, mc_client, mc_id);
6013 * cik_vm_flush - cik vm flush using the CP
6015 * @rdev: radeon_device pointer
6017 * Update the page table base and flush the VM TLB
6018 * using the CP (CIK).
6020 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
6021 unsigned vm_id, uint64_t pd_addr)
6023 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
6025 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6026 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6027 WRITE_DATA_DST_SEL(0)));
6029 radeon_ring_write(ring,
6030 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
6032 radeon_ring_write(ring,
6033 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
6035 radeon_ring_write(ring, 0);
6036 radeon_ring_write(ring, pd_addr >> 12);
6038 /* update SH_MEM_* regs */
6039 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6040 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6041 WRITE_DATA_DST_SEL(0)));
6042 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6043 radeon_ring_write(ring, 0);
6044 radeon_ring_write(ring, VMID(vm_id));
6046 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
6047 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6048 WRITE_DATA_DST_SEL(0)));
6049 radeon_ring_write(ring, SH_MEM_BASES >> 2);
6050 radeon_ring_write(ring, 0);
6052 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
6053 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6054 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6055 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
6057 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6058 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6059 WRITE_DATA_DST_SEL(0)));
6060 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6061 radeon_ring_write(ring, 0);
6062 radeon_ring_write(ring, VMID(0));
6065 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
6067 /* bits 0-15 are the VM contexts0-15 */
6068 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6069 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6070 WRITE_DATA_DST_SEL(0)));
6071 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6072 radeon_ring_write(ring, 0);
6073 radeon_ring_write(ring, 1 << vm_id);
6075 /* wait for the invalidate to complete */
6076 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6077 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6078 WAIT_REG_MEM_FUNCTION(0) | /* always */
6079 WAIT_REG_MEM_ENGINE(0))); /* me */
6080 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6081 radeon_ring_write(ring, 0);
6082 radeon_ring_write(ring, 0); /* ref */
6083 radeon_ring_write(ring, 0); /* mask */
6084 radeon_ring_write(ring, 0x20); /* poll interval */
6086 /* compute doesn't have PFP */
6088 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6089 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6090 radeon_ring_write(ring, 0x0);
6096 * The RLC is a multi-purpose microengine that handles a
6097 * variety of functions, the most important of which is
6098 * the interrupt controller.
6100 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6103 u32 tmp = RREG32(CP_INT_CNTL_RING0);
6106 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6108 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6109 WREG32(CP_INT_CNTL_RING0, tmp);
6112 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6116 tmp = RREG32(RLC_LB_CNTL);
6118 tmp |= LOAD_BALANCE_ENABLE;
6120 tmp &= ~LOAD_BALANCE_ENABLE;
6121 WREG32(RLC_LB_CNTL, tmp);
6124 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6129 mutex_lock(&rdev->grbm_idx_mutex);
6130 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6131 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6132 cik_select_se_sh(rdev, i, j);
6133 for (k = 0; k < rdev->usec_timeout; k++) {
6134 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6140 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6141 mutex_unlock(&rdev->grbm_idx_mutex);
6143 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6144 for (k = 0; k < rdev->usec_timeout; k++) {
6145 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6151 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6155 tmp = RREG32(RLC_CNTL);
6157 WREG32(RLC_CNTL, rlc);
6160 static u32 cik_halt_rlc(struct radeon_device *rdev)
6164 orig = data = RREG32(RLC_CNTL);
6166 if (data & RLC_ENABLE) {
6169 data &= ~RLC_ENABLE;
6170 WREG32(RLC_CNTL, data);
6172 for (i = 0; i < rdev->usec_timeout; i++) {
6173 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6178 cik_wait_for_rlc_serdes(rdev);
6184 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6188 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6189 WREG32(RLC_GPR_REG2, tmp);
6191 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6192 for (i = 0; i < rdev->usec_timeout; i++) {
6193 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6198 for (i = 0; i < rdev->usec_timeout; i++) {
6199 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6205 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6209 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6210 WREG32(RLC_GPR_REG2, tmp);
6214 * cik_rlc_stop - stop the RLC ME
6216 * @rdev: radeon_device pointer
6218 * Halt the RLC ME (MicroEngine) (CIK).
6220 static void cik_rlc_stop(struct radeon_device *rdev)
6222 WREG32(RLC_CNTL, 0);
6224 cik_enable_gui_idle_interrupt(rdev, false);
6226 cik_wait_for_rlc_serdes(rdev);
6230 * cik_rlc_start - start the RLC ME
6232 * @rdev: radeon_device pointer
6234 * Unhalt the RLC ME (MicroEngine) (CIK).
6236 static void cik_rlc_start(struct radeon_device *rdev)
6238 WREG32(RLC_CNTL, RLC_ENABLE);
6240 cik_enable_gui_idle_interrupt(rdev, true);
6246 * cik_rlc_resume - setup the RLC hw
6248 * @rdev: radeon_device pointer
6250 * Initialize the RLC registers, load the ucode,
6251 * and start the RLC (CIK).
6252 * Returns 0 for success, -EINVAL if the ucode is not available.
6254 static int cik_rlc_resume(struct radeon_device *rdev)
6264 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6265 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6273 WREG32(RLC_LB_CNTR_INIT, 0);
6274 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6276 mutex_lock(&rdev->grbm_idx_mutex);
6277 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6278 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6279 WREG32(RLC_LB_PARAMS, 0x00600408);
6280 WREG32(RLC_LB_CNTL, 0x80000004);
6281 mutex_unlock(&rdev->grbm_idx_mutex);
6283 WREG32(RLC_MC_CNTL, 0);
6284 WREG32(RLC_UCODE_CNTL, 0);
6287 const struct rlc_firmware_header_v1_0 *hdr =
6288 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6289 const __le32 *fw_data = (const __le32 *)
6290 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6292 radeon_ucode_print_rlc_hdr(&hdr->header);
6294 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6295 WREG32(RLC_GPM_UCODE_ADDR, 0);
6296 for (i = 0; i < size; i++)
6297 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6298 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
6300 const __be32 *fw_data;
6302 switch (rdev->family) {
6306 size = BONAIRE_RLC_UCODE_SIZE;
6309 size = KV_RLC_UCODE_SIZE;
6312 size = KB_RLC_UCODE_SIZE;
6315 size = ML_RLC_UCODE_SIZE;
6319 fw_data = (const __be32 *)rdev->rlc_fw->data;
6320 WREG32(RLC_GPM_UCODE_ADDR, 0);
6321 for (i = 0; i < size; i++)
6322 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6323 WREG32(RLC_GPM_UCODE_ADDR, 0);
6326 /* XXX - find out what chips support lbpw */
6327 cik_enable_lbpw(rdev, false);
6329 if (rdev->family == CHIP_BONAIRE)
6330 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6332 cik_rlc_start(rdev);
6337 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6339 u32 data, orig, tmp, tmp2;
6341 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6343 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6344 cik_enable_gui_idle_interrupt(rdev, true);
6346 tmp = cik_halt_rlc(rdev);
6348 mutex_lock(&rdev->grbm_idx_mutex);
6349 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6350 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6351 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6352 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6353 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6354 mutex_unlock(&rdev->grbm_idx_mutex);
6356 cik_update_rlc(rdev, tmp);
6358 data |= CGCG_EN | CGLS_EN;
6360 cik_enable_gui_idle_interrupt(rdev, false);
6362 RREG32(CB_CGTT_SCLK_CTRL);
6363 RREG32(CB_CGTT_SCLK_CTRL);
6364 RREG32(CB_CGTT_SCLK_CTRL);
6365 RREG32(CB_CGTT_SCLK_CTRL);
6367 data &= ~(CGCG_EN | CGLS_EN);
6371 WREG32(RLC_CGCG_CGLS_CTRL, data);
6375 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6377 u32 data, orig, tmp = 0;
6379 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6380 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6381 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6382 orig = data = RREG32(CP_MEM_SLP_CNTL);
6383 data |= CP_MEM_LS_EN;
6385 WREG32(CP_MEM_SLP_CNTL, data);
6389 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6393 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6395 tmp = cik_halt_rlc(rdev);
6397 mutex_lock(&rdev->grbm_idx_mutex);
6398 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6399 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6400 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6401 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6402 WREG32(RLC_SERDES_WR_CTRL, data);
6403 mutex_unlock(&rdev->grbm_idx_mutex);
6405 cik_update_rlc(rdev, tmp);
6407 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6408 orig = data = RREG32(CGTS_SM_CTRL_REG);
6409 data &= ~SM_MODE_MASK;
6410 data |= SM_MODE(0x2);
6411 data |= SM_MODE_ENABLE;
6412 data &= ~CGTS_OVERRIDE;
6413 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6414 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6415 data &= ~CGTS_LS_OVERRIDE;
6416 data &= ~ON_MONITOR_ADD_MASK;
6417 data |= ON_MONITOR_ADD_EN;
6418 data |= ON_MONITOR_ADD(0x96);
6420 WREG32(CGTS_SM_CTRL_REG, data);
6423 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6426 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6428 data = RREG32(RLC_MEM_SLP_CNTL);
6429 if (data & RLC_MEM_LS_EN) {
6430 data &= ~RLC_MEM_LS_EN;
6431 WREG32(RLC_MEM_SLP_CNTL, data);
6434 data = RREG32(CP_MEM_SLP_CNTL);
6435 if (data & CP_MEM_LS_EN) {
6436 data &= ~CP_MEM_LS_EN;
6437 WREG32(CP_MEM_SLP_CNTL, data);
6440 orig = data = RREG32(CGTS_SM_CTRL_REG);
6441 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6443 WREG32(CGTS_SM_CTRL_REG, data);
6445 tmp = cik_halt_rlc(rdev);
6447 mutex_lock(&rdev->grbm_idx_mutex);
6448 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6449 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6450 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6451 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6452 WREG32(RLC_SERDES_WR_CTRL, data);
6453 mutex_unlock(&rdev->grbm_idx_mutex);
6455 cik_update_rlc(rdev, tmp);
6459 static const u32 mc_cg_registers[] =
6472 static void cik_enable_mc_ls(struct radeon_device *rdev,
6478 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6479 orig = data = RREG32(mc_cg_registers[i]);
6480 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6481 data |= MC_LS_ENABLE;
6483 data &= ~MC_LS_ENABLE;
6485 WREG32(mc_cg_registers[i], data);
6489 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6495 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6496 orig = data = RREG32(mc_cg_registers[i]);
6497 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6498 data |= MC_CG_ENABLE;
6500 data &= ~MC_CG_ENABLE;
6502 WREG32(mc_cg_registers[i], data);
6506 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6511 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6512 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6513 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6515 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6518 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6520 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6523 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6527 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6532 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6533 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6536 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6538 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6541 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6543 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6546 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6548 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6551 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6555 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6560 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6561 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6563 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6565 orig = data = RREG32(UVD_CGC_CTRL);
6568 WREG32(UVD_CGC_CTRL, data);
6570 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6572 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6574 orig = data = RREG32(UVD_CGC_CTRL);
6577 WREG32(UVD_CGC_CTRL, data);
6581 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6586 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6588 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6589 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6590 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6592 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6593 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6596 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6599 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6604 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6606 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6607 data &= ~CLOCK_GATING_DIS;
6609 data |= CLOCK_GATING_DIS;
6612 WREG32(HDP_HOST_PATH_CNTL, data);
6615 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6620 orig = data = RREG32(HDP_MEM_POWER_LS);
6622 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6623 data |= HDP_LS_ENABLE;
6625 data &= ~HDP_LS_ENABLE;
6628 WREG32(HDP_MEM_POWER_LS, data);
6631 void cik_update_cg(struct radeon_device *rdev,
6632 u32 block, bool enable)
6635 if (block & RADEON_CG_BLOCK_GFX) {
6636 cik_enable_gui_idle_interrupt(rdev, false);
6637 /* order matters! */
6639 cik_enable_mgcg(rdev, true);
6640 cik_enable_cgcg(rdev, true);
6642 cik_enable_cgcg(rdev, false);
6643 cik_enable_mgcg(rdev, false);
6645 cik_enable_gui_idle_interrupt(rdev, true);
6648 if (block & RADEON_CG_BLOCK_MC) {
6649 if (!(rdev->flags & RADEON_IS_IGP)) {
6650 cik_enable_mc_mgcg(rdev, enable);
6651 cik_enable_mc_ls(rdev, enable);
6655 if (block & RADEON_CG_BLOCK_SDMA) {
6656 cik_enable_sdma_mgcg(rdev, enable);
6657 cik_enable_sdma_mgls(rdev, enable);
6660 if (block & RADEON_CG_BLOCK_BIF) {
6661 cik_enable_bif_mgls(rdev, enable);
6664 if (block & RADEON_CG_BLOCK_UVD) {
6666 cik_enable_uvd_mgcg(rdev, enable);
6669 if (block & RADEON_CG_BLOCK_HDP) {
6670 cik_enable_hdp_mgcg(rdev, enable);
6671 cik_enable_hdp_ls(rdev, enable);
6674 if (block & RADEON_CG_BLOCK_VCE) {
6675 vce_v2_0_enable_mgcg(rdev, enable);
6679 static void cik_init_cg(struct radeon_device *rdev)
6682 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6685 si_init_uvd_internal_cg(rdev);
6687 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6688 RADEON_CG_BLOCK_SDMA |
6689 RADEON_CG_BLOCK_BIF |
6690 RADEON_CG_BLOCK_UVD |
6691 RADEON_CG_BLOCK_HDP), true);
6694 static void cik_fini_cg(struct radeon_device *rdev)
6696 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6697 RADEON_CG_BLOCK_SDMA |
6698 RADEON_CG_BLOCK_BIF |
6699 RADEON_CG_BLOCK_UVD |
6700 RADEON_CG_BLOCK_HDP), false);
6702 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6705 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6710 orig = data = RREG32(RLC_PG_CNTL);
6711 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6712 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6714 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6716 WREG32(RLC_PG_CNTL, data);
6719 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6724 orig = data = RREG32(RLC_PG_CNTL);
6725 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6726 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6728 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6730 WREG32(RLC_PG_CNTL, data);
6733 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6737 orig = data = RREG32(RLC_PG_CNTL);
6738 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6739 data &= ~DISABLE_CP_PG;
6741 data |= DISABLE_CP_PG;
6743 WREG32(RLC_PG_CNTL, data);
6746 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6750 orig = data = RREG32(RLC_PG_CNTL);
6751 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6752 data &= ~DISABLE_GDS_PG;
6754 data |= DISABLE_GDS_PG;
6756 WREG32(RLC_PG_CNTL, data);
6759 #define CP_ME_TABLE_SIZE 96
6760 #define CP_ME_TABLE_OFFSET 2048
6761 #define CP_MEC_TABLE_OFFSET 4096
6763 void cik_init_cp_pg_table(struct radeon_device *rdev)
6765 volatile u32 *dst_ptr;
6766 int me, i, max_me = 4;
6768 u32 table_offset, table_size;
6770 if (rdev->family == CHIP_KAVERI)
6773 if (rdev->rlc.cp_table_ptr == NULL)
6776 /* write the cp table buffer */
6777 dst_ptr = rdev->rlc.cp_table_ptr;
6778 for (me = 0; me < max_me; me++) {
6780 const __le32 *fw_data;
6781 const struct gfx_firmware_header_v1_0 *hdr;
6784 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6785 fw_data = (const __le32 *)
6786 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6787 table_offset = le32_to_cpu(hdr->jt_offset);
6788 table_size = le32_to_cpu(hdr->jt_size);
6789 } else if (me == 1) {
6790 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6791 fw_data = (const __le32 *)
6792 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6793 table_offset = le32_to_cpu(hdr->jt_offset);
6794 table_size = le32_to_cpu(hdr->jt_size);
6795 } else if (me == 2) {
6796 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6797 fw_data = (const __le32 *)
6798 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6799 table_offset = le32_to_cpu(hdr->jt_offset);
6800 table_size = le32_to_cpu(hdr->jt_size);
6801 } else if (me == 3) {
6802 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6803 fw_data = (const __le32 *)
6804 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6805 table_offset = le32_to_cpu(hdr->jt_offset);
6806 table_size = le32_to_cpu(hdr->jt_size);
6808 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6809 fw_data = (const __le32 *)
6810 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6811 table_offset = le32_to_cpu(hdr->jt_offset);
6812 table_size = le32_to_cpu(hdr->jt_size);
6815 for (i = 0; i < table_size; i ++) {
6816 dst_ptr[bo_offset + i] =
6817 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6819 bo_offset += table_size;
6821 const __be32 *fw_data;
6822 table_size = CP_ME_TABLE_SIZE;
6825 fw_data = (const __be32 *)rdev->ce_fw->data;
6826 table_offset = CP_ME_TABLE_OFFSET;
6827 } else if (me == 1) {
6828 fw_data = (const __be32 *)rdev->pfp_fw->data;
6829 table_offset = CP_ME_TABLE_OFFSET;
6830 } else if (me == 2) {
6831 fw_data = (const __be32 *)rdev->me_fw->data;
6832 table_offset = CP_ME_TABLE_OFFSET;
6834 fw_data = (const __be32 *)rdev->mec_fw->data;
6835 table_offset = CP_MEC_TABLE_OFFSET;
6838 for (i = 0; i < table_size; i ++) {
6839 dst_ptr[bo_offset + i] =
6840 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6842 bo_offset += table_size;
6847 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6852 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6853 orig = data = RREG32(RLC_PG_CNTL);
6854 data |= GFX_PG_ENABLE;
6856 WREG32(RLC_PG_CNTL, data);
6858 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6861 WREG32(RLC_AUTO_PG_CTRL, data);
6863 orig = data = RREG32(RLC_PG_CNTL);
6864 data &= ~GFX_PG_ENABLE;
6866 WREG32(RLC_PG_CNTL, data);
6868 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6869 data &= ~AUTO_PG_EN;
6871 WREG32(RLC_AUTO_PG_CTRL, data);
6873 data = RREG32(DB_RENDER_CONTROL);
6877 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6879 u32 mask = 0, tmp, tmp1;
6882 mutex_lock(&rdev->grbm_idx_mutex);
6883 cik_select_se_sh(rdev, se, sh);
6884 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6885 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6886 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6887 mutex_unlock(&rdev->grbm_idx_mutex);
6894 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6899 return (~tmp) & mask;
6902 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6904 u32 i, j, k, active_cu_number = 0;
6905 u32 mask, counter, cu_bitmap;
6908 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6909 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6913 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6914 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6922 active_cu_number += counter;
6923 tmp |= (cu_bitmap << (i * 16 + j * 8));
6927 WREG32(RLC_PG_AO_CU_MASK, tmp);
6929 tmp = RREG32(RLC_MAX_PG_CU);
6930 tmp &= ~MAX_PU_CU_MASK;
6931 tmp |= MAX_PU_CU(active_cu_number);
6932 WREG32(RLC_MAX_PG_CU, tmp);
6935 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6940 orig = data = RREG32(RLC_PG_CNTL);
6941 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6942 data |= STATIC_PER_CU_PG_ENABLE;
6944 data &= ~STATIC_PER_CU_PG_ENABLE;
6946 WREG32(RLC_PG_CNTL, data);
6949 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6954 orig = data = RREG32(RLC_PG_CNTL);
6955 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6956 data |= DYN_PER_CU_PG_ENABLE;
6958 data &= ~DYN_PER_CU_PG_ENABLE;
6960 WREG32(RLC_PG_CNTL, data);
6963 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6964 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6966 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6971 if (rdev->rlc.cs_data) {
6972 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6973 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6974 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6975 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6977 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6978 for (i = 0; i < 3; i++)
6979 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6981 if (rdev->rlc.reg_list) {
6982 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6983 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6984 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6987 orig = data = RREG32(RLC_PG_CNTL);
6990 WREG32(RLC_PG_CNTL, data);
6992 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6993 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6995 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6996 data &= ~IDLE_POLL_COUNT_MASK;
6997 data |= IDLE_POLL_COUNT(0x60);
6998 WREG32(CP_RB_WPTR_POLL_CNTL, data);
7001 WREG32(RLC_PG_DELAY, data);
7003 data = RREG32(RLC_PG_DELAY_2);
7006 WREG32(RLC_PG_DELAY_2, data);
7008 data = RREG32(RLC_AUTO_PG_CTRL);
7009 data &= ~GRBM_REG_SGIT_MASK;
7010 data |= GRBM_REG_SGIT(0x700);
7011 WREG32(RLC_AUTO_PG_CTRL, data);
7015 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
7017 cik_enable_gfx_cgpg(rdev, enable);
7018 cik_enable_gfx_static_mgpg(rdev, enable);
7019 cik_enable_gfx_dynamic_mgpg(rdev, enable);
7022 u32 cik_get_csb_size(struct radeon_device *rdev)
7025 const struct cs_section_def *sect = NULL;
7026 const struct cs_extent_def *ext = NULL;
7028 if (rdev->rlc.cs_data == NULL)
7031 /* begin clear state */
7033 /* context control state */
7036 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7037 for (ext = sect->section; ext->extent != NULL; ++ext) {
7038 if (sect->id == SECT_CONTEXT)
7039 count += 2 + ext->reg_count;
7044 /* pa_sc_raster_config/pa_sc_raster_config1 */
7046 /* end clear state */
7054 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
7057 const struct cs_section_def *sect = NULL;
7058 const struct cs_extent_def *ext = NULL;
7060 if (rdev->rlc.cs_data == NULL)
7065 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7066 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
7068 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
7069 buffer[count++] = cpu_to_le32(0x80000000);
7070 buffer[count++] = cpu_to_le32(0x80000000);
7072 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7073 for (ext = sect->section; ext->extent != NULL; ++ext) {
7074 if (sect->id == SECT_CONTEXT) {
7076 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
7077 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
7078 for (i = 0; i < ext->reg_count; i++)
7079 buffer[count++] = cpu_to_le32(ext->extent[i]);
7086 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7087 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
7088 switch (rdev->family) {
7090 buffer[count++] = cpu_to_le32(0x16000012);
7091 buffer[count++] = cpu_to_le32(0x00000000);
7094 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7095 buffer[count++] = cpu_to_le32(0x00000000);
7099 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7100 buffer[count++] = cpu_to_le32(0x00000000);
7103 buffer[count++] = cpu_to_le32(0x3a00161a);
7104 buffer[count++] = cpu_to_le32(0x0000002e);
7107 buffer[count++] = cpu_to_le32(0x00000000);
7108 buffer[count++] = cpu_to_le32(0x00000000);
7112 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7113 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7115 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7116 buffer[count++] = cpu_to_le32(0);
7119 static void cik_init_pg(struct radeon_device *rdev)
7121 if (rdev->pg_flags) {
7122 cik_enable_sck_slowdown_on_pu(rdev, true);
7123 cik_enable_sck_slowdown_on_pd(rdev, true);
7124 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7125 cik_init_gfx_cgpg(rdev);
7126 cik_enable_cp_pg(rdev, true);
7127 cik_enable_gds_pg(rdev, true);
7129 cik_init_ao_cu_mask(rdev);
7130 cik_update_gfx_pg(rdev, true);
7134 static void cik_fini_pg(struct radeon_device *rdev)
7136 if (rdev->pg_flags) {
7137 cik_update_gfx_pg(rdev, false);
7138 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7139 cik_enable_cp_pg(rdev, false);
7140 cik_enable_gds_pg(rdev, false);
7147 * Starting with r6xx, interrupts are handled via a ring buffer.
7148 * Ring buffers are areas of GPU accessible memory that the GPU
7149 * writes interrupt vectors into and the host reads vectors out of.
7150 * There is a rptr (read pointer) that determines where the
7151 * host is currently reading, and a wptr (write pointer)
7152 * which determines where the GPU has written. When the
7153 * pointers are equal, the ring is idle. When the GPU
7154 * writes vectors to the ring buffer, it increments the
7155 * wptr. When there is an interrupt, the host then starts
7156 * fetching commands and processing them until the pointers are
7157 * equal again at which point it updates the rptr.
7161 * cik_enable_interrupts - Enable the interrupt ring buffer
7163 * @rdev: radeon_device pointer
7165 * Enable the interrupt ring buffer (CIK).
7167 static void cik_enable_interrupts(struct radeon_device *rdev)
7169 u32 ih_cntl = RREG32(IH_CNTL);
7170 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7172 ih_cntl |= ENABLE_INTR;
7173 ih_rb_cntl |= IH_RB_ENABLE;
7174 WREG32(IH_CNTL, ih_cntl);
7175 WREG32(IH_RB_CNTL, ih_rb_cntl);
7176 rdev->ih.enabled = true;
7180 * cik_disable_interrupts - Disable the interrupt ring buffer
7182 * @rdev: radeon_device pointer
7184 * Disable the interrupt ring buffer (CIK).
7186 static void cik_disable_interrupts(struct radeon_device *rdev)
7188 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7189 u32 ih_cntl = RREG32(IH_CNTL);
7191 ih_rb_cntl &= ~IH_RB_ENABLE;
7192 ih_cntl &= ~ENABLE_INTR;
7193 WREG32(IH_RB_CNTL, ih_rb_cntl);
7194 WREG32(IH_CNTL, ih_cntl);
7195 /* set rptr, wptr to 0 */
7196 WREG32(IH_RB_RPTR, 0);
7197 WREG32(IH_RB_WPTR, 0);
7198 rdev->ih.enabled = false;
7203 * cik_disable_interrupt_state - Disable all interrupt sources
7205 * @rdev: radeon_device pointer
7207 * Clear all interrupt enable bits used by the driver (CIK).
7209 static void cik_disable_interrupt_state(struct radeon_device *rdev)
7214 tmp = RREG32(CP_INT_CNTL_RING0) &
7215 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7216 WREG32(CP_INT_CNTL_RING0, tmp);
7218 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7219 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7220 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7221 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7222 /* compute queues */
7223 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7224 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7225 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7226 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7227 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7228 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7229 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7230 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7232 WREG32(GRBM_INT_CNTL, 0);
7233 /* vline/vblank, etc. */
7234 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7235 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7236 if (rdev->num_crtc >= 4) {
7237 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7238 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7240 if (rdev->num_crtc >= 6) {
7241 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7242 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7245 if (rdev->num_crtc >= 2) {
7246 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7247 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7249 if (rdev->num_crtc >= 4) {
7250 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7251 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7253 if (rdev->num_crtc >= 6) {
7254 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7255 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7259 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7261 /* digital hotplug */
7262 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7263 WREG32(DC_HPD1_INT_CONTROL, tmp);
7264 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7265 WREG32(DC_HPD2_INT_CONTROL, tmp);
7266 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7267 WREG32(DC_HPD3_INT_CONTROL, tmp);
7268 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7269 WREG32(DC_HPD4_INT_CONTROL, tmp);
7270 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7271 WREG32(DC_HPD5_INT_CONTROL, tmp);
7272 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7273 WREG32(DC_HPD6_INT_CONTROL, tmp);
7278 * cik_irq_init - init and enable the interrupt ring
7280 * @rdev: radeon_device pointer
7282 * Allocate a ring buffer for the interrupt controller,
7283 * enable the RLC, disable interrupts, enable the IH
7284 * ring buffer and enable it (CIK).
7285 * Called at device load and reume.
7286 * Returns 0 for success, errors for failure.
7288 static int cik_irq_init(struct radeon_device *rdev)
7292 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7295 ret = r600_ih_ring_alloc(rdev);
7300 cik_disable_interrupts(rdev);
7303 ret = cik_rlc_resume(rdev);
7305 r600_ih_ring_fini(rdev);
7309 /* setup interrupt control */
7310 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7311 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7312 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7313 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7314 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7316 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7317 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7318 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7319 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7321 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7322 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7324 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7325 IH_WPTR_OVERFLOW_CLEAR |
7328 if (rdev->wb.enabled)
7329 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7331 /* set the writeback address whether it's enabled or not */
7332 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7333 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7335 WREG32(IH_RB_CNTL, ih_rb_cntl);
7337 /* set rptr, wptr to 0 */
7338 WREG32(IH_RB_RPTR, 0);
7339 WREG32(IH_RB_WPTR, 0);
7341 /* Default settings for IH_CNTL (disabled at first) */
7342 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7343 /* RPTR_REARM only works if msi's are enabled */
7344 if (rdev->msi_enabled)
7345 ih_cntl |= RPTR_REARM;
7346 WREG32(IH_CNTL, ih_cntl);
7348 /* force the active interrupt state to all disabled */
7349 cik_disable_interrupt_state(rdev);
7351 pci_set_master(rdev->pdev);
7354 cik_enable_interrupts(rdev);
7360 * cik_irq_set - enable/disable interrupt sources
7362 * @rdev: radeon_device pointer
7364 * Enable interrupt sources on the GPU (vblanks, hpd,
7366 * Returns 0 for success, errors for failure.
7368 int cik_irq_set(struct radeon_device *rdev)
7372 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7373 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7374 u32 grbm_int_cntl = 0;
7375 u32 dma_cntl, dma_cntl1;
7377 if (!rdev->irq.installed) {
7378 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7381 /* don't enable anything if the ih is disabled */
7382 if (!rdev->ih.enabled) {
7383 cik_disable_interrupts(rdev);
7384 /* force the active interrupt state to all disabled */
7385 cik_disable_interrupt_state(rdev);
7389 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7390 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7391 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7393 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7394 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7395 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7396 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7397 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7398 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7400 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7401 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7403 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7405 /* enable CP interrupts on all rings */
7406 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7407 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7408 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7410 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7411 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7412 DRM_DEBUG("si_irq_set: sw int cp1\n");
7413 if (ring->me == 1) {
7414 switch (ring->pipe) {
7416 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7419 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7423 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7426 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7427 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7428 DRM_DEBUG("si_irq_set: sw int cp2\n");
7429 if (ring->me == 1) {
7430 switch (ring->pipe) {
7432 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7435 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7439 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7443 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7444 DRM_DEBUG("cik_irq_set: sw int dma\n");
7445 dma_cntl |= TRAP_ENABLE;
7448 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7449 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7450 dma_cntl1 |= TRAP_ENABLE;
7453 if (rdev->irq.crtc_vblank_int[0] ||
7454 atomic_read(&rdev->irq.pflip[0])) {
7455 DRM_DEBUG("cik_irq_set: vblank 0\n");
7456 crtc1 |= VBLANK_INTERRUPT_MASK;
7458 if (rdev->irq.crtc_vblank_int[1] ||
7459 atomic_read(&rdev->irq.pflip[1])) {
7460 DRM_DEBUG("cik_irq_set: vblank 1\n");
7461 crtc2 |= VBLANK_INTERRUPT_MASK;
7463 if (rdev->irq.crtc_vblank_int[2] ||
7464 atomic_read(&rdev->irq.pflip[2])) {
7465 DRM_DEBUG("cik_irq_set: vblank 2\n");
7466 crtc3 |= VBLANK_INTERRUPT_MASK;
7468 if (rdev->irq.crtc_vblank_int[3] ||
7469 atomic_read(&rdev->irq.pflip[3])) {
7470 DRM_DEBUG("cik_irq_set: vblank 3\n");
7471 crtc4 |= VBLANK_INTERRUPT_MASK;
7473 if (rdev->irq.crtc_vblank_int[4] ||
7474 atomic_read(&rdev->irq.pflip[4])) {
7475 DRM_DEBUG("cik_irq_set: vblank 4\n");
7476 crtc5 |= VBLANK_INTERRUPT_MASK;
7478 if (rdev->irq.crtc_vblank_int[5] ||
7479 atomic_read(&rdev->irq.pflip[5])) {
7480 DRM_DEBUG("cik_irq_set: vblank 5\n");
7481 crtc6 |= VBLANK_INTERRUPT_MASK;
7483 if (rdev->irq.hpd[0]) {
7484 DRM_DEBUG("cik_irq_set: hpd 1\n");
7485 hpd1 |= DC_HPDx_INT_EN;
7487 if (rdev->irq.hpd[1]) {
7488 DRM_DEBUG("cik_irq_set: hpd 2\n");
7489 hpd2 |= DC_HPDx_INT_EN;
7491 if (rdev->irq.hpd[2]) {
7492 DRM_DEBUG("cik_irq_set: hpd 3\n");
7493 hpd3 |= DC_HPDx_INT_EN;
7495 if (rdev->irq.hpd[3]) {
7496 DRM_DEBUG("cik_irq_set: hpd 4\n");
7497 hpd4 |= DC_HPDx_INT_EN;
7499 if (rdev->irq.hpd[4]) {
7500 DRM_DEBUG("cik_irq_set: hpd 5\n");
7501 hpd5 |= DC_HPDx_INT_EN;
7503 if (rdev->irq.hpd[5]) {
7504 DRM_DEBUG("cik_irq_set: hpd 6\n");
7505 hpd6 |= DC_HPDx_INT_EN;
7508 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7510 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7511 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7513 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7515 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7517 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7518 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7519 if (rdev->num_crtc >= 4) {
7520 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7521 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7523 if (rdev->num_crtc >= 6) {
7524 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7525 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7528 if (rdev->num_crtc >= 2) {
7529 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7530 GRPH_PFLIP_INT_MASK);
7531 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7532 GRPH_PFLIP_INT_MASK);
7534 if (rdev->num_crtc >= 4) {
7535 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7536 GRPH_PFLIP_INT_MASK);
7537 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7538 GRPH_PFLIP_INT_MASK);
7540 if (rdev->num_crtc >= 6) {
7541 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7542 GRPH_PFLIP_INT_MASK);
7543 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7544 GRPH_PFLIP_INT_MASK);
7547 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7548 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7549 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7550 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7551 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7552 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7558 * cik_irq_ack - ack interrupt sources
7560 * @rdev: radeon_device pointer
7562 * Ack interrupt sources on the GPU (vblanks, hpd,
7563 * etc.) (CIK). Certain interrupts sources are sw
7564 * generated and do not require an explicit ack.
7566 static inline void cik_irq_ack(struct radeon_device *rdev)
7570 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7571 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7572 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7573 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7574 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7575 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7576 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7578 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7579 EVERGREEN_CRTC0_REGISTER_OFFSET);
7580 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7581 EVERGREEN_CRTC1_REGISTER_OFFSET);
7582 if (rdev->num_crtc >= 4) {
7583 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7584 EVERGREEN_CRTC2_REGISTER_OFFSET);
7585 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7586 EVERGREEN_CRTC3_REGISTER_OFFSET);
7588 if (rdev->num_crtc >= 6) {
7589 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7590 EVERGREEN_CRTC4_REGISTER_OFFSET);
7591 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7592 EVERGREEN_CRTC5_REGISTER_OFFSET);
7595 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7596 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7597 GRPH_PFLIP_INT_CLEAR);
7598 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7599 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7600 GRPH_PFLIP_INT_CLEAR);
7601 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7602 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7603 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7604 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7605 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7606 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7607 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7608 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7610 if (rdev->num_crtc >= 4) {
7611 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7612 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7613 GRPH_PFLIP_INT_CLEAR);
7614 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7615 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7616 GRPH_PFLIP_INT_CLEAR);
7617 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7618 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7619 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7620 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7621 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7622 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7623 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7624 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7627 if (rdev->num_crtc >= 6) {
7628 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7629 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7630 GRPH_PFLIP_INT_CLEAR);
7631 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7632 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7633 GRPH_PFLIP_INT_CLEAR);
7634 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7635 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7636 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7637 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7638 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7639 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7640 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7641 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7644 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7645 tmp = RREG32(DC_HPD1_INT_CONTROL);
7646 tmp |= DC_HPDx_INT_ACK;
7647 WREG32(DC_HPD1_INT_CONTROL, tmp);
7649 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7650 tmp = RREG32(DC_HPD2_INT_CONTROL);
7651 tmp |= DC_HPDx_INT_ACK;
7652 WREG32(DC_HPD2_INT_CONTROL, tmp);
7654 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7655 tmp = RREG32(DC_HPD3_INT_CONTROL);
7656 tmp |= DC_HPDx_INT_ACK;
7657 WREG32(DC_HPD3_INT_CONTROL, tmp);
7659 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7660 tmp = RREG32(DC_HPD4_INT_CONTROL);
7661 tmp |= DC_HPDx_INT_ACK;
7662 WREG32(DC_HPD4_INT_CONTROL, tmp);
7664 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7665 tmp = RREG32(DC_HPD5_INT_CONTROL);
7666 tmp |= DC_HPDx_INT_ACK;
7667 WREG32(DC_HPD5_INT_CONTROL, tmp);
7669 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7670 tmp = RREG32(DC_HPD5_INT_CONTROL);
7671 tmp |= DC_HPDx_INT_ACK;
7672 WREG32(DC_HPD6_INT_CONTROL, tmp);
7677 * cik_irq_disable - disable interrupts
7679 * @rdev: radeon_device pointer
7681 * Disable interrupts on the hw (CIK).
7683 static void cik_irq_disable(struct radeon_device *rdev)
7685 cik_disable_interrupts(rdev);
7686 /* Wait and acknowledge irq */
7689 cik_disable_interrupt_state(rdev);
7693 * cik_irq_disable - disable interrupts for suspend
7695 * @rdev: radeon_device pointer
7697 * Disable interrupts and stop the RLC (CIK).
7700 static void cik_irq_suspend(struct radeon_device *rdev)
7702 cik_irq_disable(rdev);
7707 * cik_irq_fini - tear down interrupt support
7709 * @rdev: radeon_device pointer
7711 * Disable interrupts on the hw and free the IH ring
7713 * Used for driver unload.
7715 static void cik_irq_fini(struct radeon_device *rdev)
7717 cik_irq_suspend(rdev);
7718 r600_ih_ring_fini(rdev);
7722 * cik_get_ih_wptr - get the IH ring buffer wptr
7724 * @rdev: radeon_device pointer
7726 * Get the IH ring buffer wptr from either the register
7727 * or the writeback memory buffer (CIK). Also check for
7728 * ring buffer overflow and deal with it.
7729 * Used by cik_irq_process().
7730 * Returns the value of the wptr.
7732 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7736 if (rdev->wb.enabled)
7737 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7739 wptr = RREG32(IH_RB_WPTR);
7741 if (wptr & RB_OVERFLOW) {
7742 wptr &= ~RB_OVERFLOW;
7743 /* When a ring buffer overflow happen start parsing interrupt
7744 * from the last not overwritten vector (wptr + 16). Hopefully
7745 * this should allow us to catchup.
7747 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7748 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7749 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7750 tmp = RREG32(IH_RB_CNTL);
7751 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7752 WREG32(IH_RB_CNTL, tmp);
7754 return (wptr & rdev->ih.ptr_mask);
7758 * Each IV ring entry is 128 bits:
7759 * [7:0] - interrupt source id
7761 * [59:32] - interrupt source data
7762 * [63:60] - reserved
7765 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7766 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7767 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7768 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7769 * PIPE_ID - ME0 0=3D
7770 * - ME1&2 compute dispatcher (4 pipes each)
7772 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7773 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7774 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7777 * [127:96] - reserved
7780 * cik_irq_process - interrupt handler
7782 * @rdev: radeon_device pointer
7784 * Interrupt hander (CIK). Walk the IH ring,
7785 * ack interrupts and schedule work to handle
7787 * Returns irq process return code.
7789 int cik_irq_process(struct radeon_device *rdev)
7791 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7792 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7795 u32 src_id, src_data, ring_id;
7796 u8 me_id, pipe_id, queue_id;
7798 bool queue_hotplug = false;
7799 bool queue_reset = false;
7800 u32 addr, status, mc_client;
7801 bool queue_thermal = false;
7803 if (!rdev->ih.enabled || rdev->shutdown)
7806 wptr = cik_get_ih_wptr(rdev);
7809 /* is somebody else already processing irqs? */
7810 if (atomic_xchg(&rdev->ih.lock, 1))
7813 rptr = rdev->ih.rptr;
7814 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7816 /* Order reading of wptr vs. reading of IH ring data */
7819 /* display interrupts */
7822 while (rptr != wptr) {
7823 /* wptr/rptr are in bytes! */
7824 ring_index = rptr / 4;
7826 radeon_kfd_interrupt(rdev,
7827 (const void *) &rdev->ih.ring[ring_index]);
7829 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7830 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7831 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7834 case 1: /* D1 vblank/vline */
7836 case 0: /* D1 vblank */
7837 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7838 if (rdev->irq.crtc_vblank_int[0]) {
7839 drm_handle_vblank(rdev->ddev, 0);
7840 rdev->pm.vblank_sync = true;
7841 wake_up(&rdev->irq.vblank_queue);
7843 if (atomic_read(&rdev->irq.pflip[0]))
7844 radeon_crtc_handle_vblank(rdev, 0);
7845 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7846 DRM_DEBUG("IH: D1 vblank\n");
7849 case 1: /* D1 vline */
7850 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7851 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7852 DRM_DEBUG("IH: D1 vline\n");
7856 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7860 case 2: /* D2 vblank/vline */
7862 case 0: /* D2 vblank */
7863 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7864 if (rdev->irq.crtc_vblank_int[1]) {
7865 drm_handle_vblank(rdev->ddev, 1);
7866 rdev->pm.vblank_sync = true;
7867 wake_up(&rdev->irq.vblank_queue);
7869 if (atomic_read(&rdev->irq.pflip[1]))
7870 radeon_crtc_handle_vblank(rdev, 1);
7871 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7872 DRM_DEBUG("IH: D2 vblank\n");
7875 case 1: /* D2 vline */
7876 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7877 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7878 DRM_DEBUG("IH: D2 vline\n");
7882 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7886 case 3: /* D3 vblank/vline */
7888 case 0: /* D3 vblank */
7889 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7890 if (rdev->irq.crtc_vblank_int[2]) {
7891 drm_handle_vblank(rdev->ddev, 2);
7892 rdev->pm.vblank_sync = true;
7893 wake_up(&rdev->irq.vblank_queue);
7895 if (atomic_read(&rdev->irq.pflip[2]))
7896 radeon_crtc_handle_vblank(rdev, 2);
7897 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7898 DRM_DEBUG("IH: D3 vblank\n");
7901 case 1: /* D3 vline */
7902 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7903 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7904 DRM_DEBUG("IH: D3 vline\n");
7908 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7912 case 4: /* D4 vblank/vline */
7914 case 0: /* D4 vblank */
7915 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7916 if (rdev->irq.crtc_vblank_int[3]) {
7917 drm_handle_vblank(rdev->ddev, 3);
7918 rdev->pm.vblank_sync = true;
7919 wake_up(&rdev->irq.vblank_queue);
7921 if (atomic_read(&rdev->irq.pflip[3]))
7922 radeon_crtc_handle_vblank(rdev, 3);
7923 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7924 DRM_DEBUG("IH: D4 vblank\n");
7927 case 1: /* D4 vline */
7928 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7929 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7930 DRM_DEBUG("IH: D4 vline\n");
7934 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7938 case 5: /* D5 vblank/vline */
7940 case 0: /* D5 vblank */
7941 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7942 if (rdev->irq.crtc_vblank_int[4]) {
7943 drm_handle_vblank(rdev->ddev, 4);
7944 rdev->pm.vblank_sync = true;
7945 wake_up(&rdev->irq.vblank_queue);
7947 if (atomic_read(&rdev->irq.pflip[4]))
7948 radeon_crtc_handle_vblank(rdev, 4);
7949 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7950 DRM_DEBUG("IH: D5 vblank\n");
7953 case 1: /* D5 vline */
7954 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7955 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7956 DRM_DEBUG("IH: D5 vline\n");
7960 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7964 case 6: /* D6 vblank/vline */
7966 case 0: /* D6 vblank */
7967 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7968 if (rdev->irq.crtc_vblank_int[5]) {
7969 drm_handle_vblank(rdev->ddev, 5);
7970 rdev->pm.vblank_sync = true;
7971 wake_up(&rdev->irq.vblank_queue);
7973 if (atomic_read(&rdev->irq.pflip[5]))
7974 radeon_crtc_handle_vblank(rdev, 5);
7975 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7976 DRM_DEBUG("IH: D6 vblank\n");
7979 case 1: /* D6 vline */
7980 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7981 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7982 DRM_DEBUG("IH: D6 vline\n");
7986 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7990 case 8: /* D1 page flip */
7991 case 10: /* D2 page flip */
7992 case 12: /* D3 page flip */
7993 case 14: /* D4 page flip */
7994 case 16: /* D5 page flip */
7995 case 18: /* D6 page flip */
7996 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7997 if (radeon_use_pflipirq > 0)
7998 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
8000 case 42: /* HPD hotplug */
8003 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8004 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8005 queue_hotplug = true;
8006 DRM_DEBUG("IH: HPD1\n");
8010 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8011 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8012 queue_hotplug = true;
8013 DRM_DEBUG("IH: HPD2\n");
8017 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8018 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8019 queue_hotplug = true;
8020 DRM_DEBUG("IH: HPD3\n");
8024 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8025 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8026 queue_hotplug = true;
8027 DRM_DEBUG("IH: HPD4\n");
8031 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8032 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8033 queue_hotplug = true;
8034 DRM_DEBUG("IH: HPD5\n");
8038 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8039 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8040 queue_hotplug = true;
8041 DRM_DEBUG("IH: HPD6\n");
8045 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8050 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8051 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8055 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8056 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8057 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8058 /* reset addr and status */
8059 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8060 if (addr == 0x0 && status == 0x0)
8062 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8063 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
8065 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8067 cik_vm_decode_fault(rdev, status, addr, mc_client);
8070 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8073 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8076 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8079 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8083 case 176: /* GFX RB CP_INT */
8084 case 177: /* GFX IB CP_INT */
8085 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8087 case 181: /* CP EOP event */
8088 DRM_DEBUG("IH: CP EOP\n");
8089 /* XXX check the bitfield order! */
8090 me_id = (ring_id & 0x60) >> 5;
8091 pipe_id = (ring_id & 0x18) >> 3;
8092 queue_id = (ring_id & 0x7) >> 0;
8095 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8099 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8100 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8101 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8102 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8106 case 184: /* CP Privileged reg access */
8107 DRM_ERROR("Illegal register access in command stream\n");
8108 /* XXX check the bitfield order! */
8109 me_id = (ring_id & 0x60) >> 5;
8110 pipe_id = (ring_id & 0x18) >> 3;
8111 queue_id = (ring_id & 0x7) >> 0;
8114 /* This results in a full GPU reset, but all we need to do is soft
8115 * reset the CP for gfx
8129 case 185: /* CP Privileged inst */
8130 DRM_ERROR("Illegal instruction in command stream\n");
8131 /* XXX check the bitfield order! */
8132 me_id = (ring_id & 0x60) >> 5;
8133 pipe_id = (ring_id & 0x18) >> 3;
8134 queue_id = (ring_id & 0x7) >> 0;
8137 /* This results in a full GPU reset, but all we need to do is soft
8138 * reset the CP for gfx
8152 case 224: /* SDMA trap event */
8153 /* XXX check the bitfield order! */
8154 me_id = (ring_id & 0x3) >> 0;
8155 queue_id = (ring_id & 0xc) >> 2;
8156 DRM_DEBUG("IH: SDMA trap\n");
8161 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8174 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8186 case 230: /* thermal low to high */
8187 DRM_DEBUG("IH: thermal low to high\n");
8188 rdev->pm.dpm.thermal.high_to_low = false;
8189 queue_thermal = true;
8191 case 231: /* thermal high to low */
8192 DRM_DEBUG("IH: thermal high to low\n");
8193 rdev->pm.dpm.thermal.high_to_low = true;
8194 queue_thermal = true;
8196 case 233: /* GUI IDLE */
8197 DRM_DEBUG("IH: GUI idle\n");
8199 case 241: /* SDMA Privileged inst */
8200 case 247: /* SDMA Privileged inst */
8201 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8202 /* XXX check the bitfield order! */
8203 me_id = (ring_id & 0x3) >> 0;
8204 queue_id = (ring_id & 0xc) >> 2;
8239 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8243 /* wptr/rptr are in bytes! */
8245 rptr &= rdev->ih.ptr_mask;
8246 WREG32(IH_RB_RPTR, rptr);
8249 schedule_work(&rdev->hotplug_work);
8251 rdev->needs_reset = true;
8252 wake_up_all(&rdev->fence_queue);
8255 schedule_work(&rdev->pm.dpm.thermal.work);
8256 rdev->ih.rptr = rptr;
8257 atomic_set(&rdev->ih.lock, 0);
8259 /* make sure wptr hasn't changed while processing */
8260 wptr = cik_get_ih_wptr(rdev);
8268 * startup/shutdown callbacks
8271 * cik_startup - program the asic to a functional state
8273 * @rdev: radeon_device pointer
8275 * Programs the asic to a functional state (CIK).
8276 * Called by cik_init() and cik_resume().
8277 * Returns 0 for success, error for failure.
8279 static int cik_startup(struct radeon_device *rdev)
8281 struct radeon_ring *ring;
8285 /* enable pcie gen2/3 link */
8286 cik_pcie_gen3_enable(rdev);
8288 cik_program_aspm(rdev);
8290 /* scratch needs to be initialized before MC */
8291 r = r600_vram_scratch_init(rdev);
8295 cik_mc_program(rdev);
8297 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8298 r = ci_mc_load_microcode(rdev);
8300 DRM_ERROR("Failed to load MC firmware!\n");
8305 r = cik_pcie_gart_enable(rdev);
8310 /* allocate rlc buffers */
8311 if (rdev->flags & RADEON_IS_IGP) {
8312 if (rdev->family == CHIP_KAVERI) {
8313 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8314 rdev->rlc.reg_list_size =
8315 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8317 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8318 rdev->rlc.reg_list_size =
8319 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8322 rdev->rlc.cs_data = ci_cs_data;
8323 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8324 r = sumo_rlc_init(rdev);
8326 DRM_ERROR("Failed to init rlc BOs!\n");
8330 /* allocate wb buffer */
8331 r = radeon_wb_init(rdev);
8335 /* allocate mec buffers */
8336 r = cik_mec_init(rdev);
8338 DRM_ERROR("Failed to init MEC BOs!\n");
8342 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8344 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8348 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8350 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8354 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8356 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8360 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8362 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8366 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8368 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8372 r = radeon_uvd_resume(rdev);
8374 r = uvd_v4_2_resume(rdev);
8376 r = radeon_fence_driver_start_ring(rdev,
8377 R600_RING_TYPE_UVD_INDEX);
8379 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8383 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8385 r = radeon_vce_resume(rdev);
8387 r = vce_v2_0_resume(rdev);
8389 r = radeon_fence_driver_start_ring(rdev,
8390 TN_RING_TYPE_VCE1_INDEX);
8392 r = radeon_fence_driver_start_ring(rdev,
8393 TN_RING_TYPE_VCE2_INDEX);
8396 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8397 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8398 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8402 if (!rdev->irq.installed) {
8403 r = radeon_irq_kms_init(rdev);
8408 r = cik_irq_init(rdev);
8410 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8411 radeon_irq_kms_fini(rdev);
8416 if (rdev->family == CHIP_HAWAII) {
8418 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8420 nop = RADEON_CP_PACKET2;
8422 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8425 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8426 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8431 /* set up the compute queues */
8432 /* type-2 packets are deprecated on MEC, use type-3 instead */
8433 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8434 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8438 ring->me = 1; /* first MEC */
8439 ring->pipe = 0; /* first pipe */
8440 ring->queue = 0; /* first queue */
8441 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8443 /* type-2 packets are deprecated on MEC, use type-3 instead */
8444 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8445 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8449 /* dGPU only have 1 MEC */
8450 ring->me = 1; /* first MEC */
8451 ring->pipe = 0; /* first pipe */
8452 ring->queue = 1; /* second queue */
8453 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8455 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8456 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8457 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8461 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8462 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8463 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8467 r = cik_cp_resume(rdev);
8471 r = cik_sdma_resume(rdev);
8475 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8476 if (ring->ring_size) {
8477 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8480 r = uvd_v1_0_init(rdev);
8482 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8487 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8488 if (ring->ring_size)
8489 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8492 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8493 if (ring->ring_size)
8494 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8498 r = vce_v1_0_init(rdev);
8499 else if (r != -ENOENT)
8500 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8502 r = radeon_ib_pool_init(rdev);
8504 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8508 r = radeon_vm_manager_init(rdev);
8510 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8514 r = radeon_audio_init(rdev);
8518 r = radeon_kfd_resume(rdev);
8526 * cik_resume - resume the asic to a functional state
8528 * @rdev: radeon_device pointer
8530 * Programs the asic to a functional state (CIK).
8532 * Returns 0 for success, error for failure.
8534 int cik_resume(struct radeon_device *rdev)
8539 atom_asic_init(rdev->mode_info.atom_context);
8541 /* init golden registers */
8542 cik_init_golden_registers(rdev);
8544 if (rdev->pm.pm_method == PM_METHOD_DPM)
8545 radeon_pm_resume(rdev);
8547 rdev->accel_working = true;
8548 r = cik_startup(rdev);
8550 DRM_ERROR("cik startup failed on resume\n");
8551 rdev->accel_working = false;
8560 * cik_suspend - suspend the asic
8562 * @rdev: radeon_device pointer
8564 * Bring the chip into a state suitable for suspend (CIK).
8565 * Called at suspend.
8566 * Returns 0 for success.
8568 int cik_suspend(struct radeon_device *rdev)
8570 radeon_kfd_suspend(rdev);
8571 radeon_pm_suspend(rdev);
8572 radeon_audio_fini(rdev);
8573 radeon_vm_manager_fini(rdev);
8574 cik_cp_enable(rdev, false);
8575 cik_sdma_enable(rdev, false);
8576 uvd_v1_0_fini(rdev);
8577 radeon_uvd_suspend(rdev);
8578 radeon_vce_suspend(rdev);
8581 cik_irq_suspend(rdev);
8582 radeon_wb_disable(rdev);
8583 cik_pcie_gart_disable(rdev);
8587 /* Plan is to move initialization in that function and use
8588 * helper function so that radeon_device_init pretty much
8589 * do nothing more than calling asic specific function. This
8590 * should also allow to remove a bunch of callback function
8594 * cik_init - asic specific driver and hw init
8596 * @rdev: radeon_device pointer
8598 * Setup asic specific driver variables and program the hw
8599 * to a functional state (CIK).
8600 * Called at driver startup.
8601 * Returns 0 for success, errors for failure.
8603 int cik_init(struct radeon_device *rdev)
8605 struct radeon_ring *ring;
8609 if (!radeon_get_bios(rdev)) {
8610 if (ASIC_IS_AVIVO(rdev))
8613 /* Must be an ATOMBIOS */
8614 if (!rdev->is_atom_bios) {
8615 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8618 r = radeon_atombios_init(rdev);
8622 /* Post card if necessary */
8623 if (!radeon_card_posted(rdev)) {
8625 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8628 DRM_INFO("GPU not posted. posting now...\n");
8629 atom_asic_init(rdev->mode_info.atom_context);
8631 /* init golden registers */
8632 cik_init_golden_registers(rdev);
8633 /* Initialize scratch registers */
8634 cik_scratch_init(rdev);
8635 /* Initialize surface registers */
8636 radeon_surface_init(rdev);
8637 /* Initialize clocks */
8638 radeon_get_clock_info(rdev->ddev);
8641 r = radeon_fence_driver_init(rdev);
8645 /* initialize memory controller */
8646 r = cik_mc_init(rdev);
8649 /* Memory manager */
8650 r = radeon_bo_init(rdev);
8654 if (rdev->flags & RADEON_IS_IGP) {
8655 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8656 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8657 r = cik_init_microcode(rdev);
8659 DRM_ERROR("Failed to load firmware!\n");
8664 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8665 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8667 r = cik_init_microcode(rdev);
8669 DRM_ERROR("Failed to load firmware!\n");
8675 /* Initialize power management */
8676 radeon_pm_init(rdev);
8678 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8679 ring->ring_obj = NULL;
8680 r600_ring_init(rdev, ring, 1024 * 1024);
8682 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8683 ring->ring_obj = NULL;
8684 r600_ring_init(rdev, ring, 1024 * 1024);
8685 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8689 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8690 ring->ring_obj = NULL;
8691 r600_ring_init(rdev, ring, 1024 * 1024);
8692 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8696 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8697 ring->ring_obj = NULL;
8698 r600_ring_init(rdev, ring, 256 * 1024);
8700 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8701 ring->ring_obj = NULL;
8702 r600_ring_init(rdev, ring, 256 * 1024);
8704 r = radeon_uvd_init(rdev);
8706 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8707 ring->ring_obj = NULL;
8708 r600_ring_init(rdev, ring, 4096);
8711 r = radeon_vce_init(rdev);
8713 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8714 ring->ring_obj = NULL;
8715 r600_ring_init(rdev, ring, 4096);
8717 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8718 ring->ring_obj = NULL;
8719 r600_ring_init(rdev, ring, 4096);
8722 rdev->ih.ring_obj = NULL;
8723 r600_ih_ring_init(rdev, 64 * 1024);
8725 r = r600_pcie_gart_init(rdev);
8729 rdev->accel_working = true;
8730 r = cik_startup(rdev);
8732 dev_err(rdev->dev, "disabling GPU acceleration\n");
8734 cik_sdma_fini(rdev);
8736 sumo_rlc_fini(rdev);
8738 radeon_wb_fini(rdev);
8739 radeon_ib_pool_fini(rdev);
8740 radeon_vm_manager_fini(rdev);
8741 radeon_irq_kms_fini(rdev);
8742 cik_pcie_gart_fini(rdev);
8743 rdev->accel_working = false;
8746 /* Don't start up if the MC ucode is missing.
8747 * The default clocks and voltages before the MC ucode
8748 * is loaded are not suffient for advanced operations.
8750 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8751 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8759 * cik_fini - asic specific driver and hw fini
8761 * @rdev: radeon_device pointer
8763 * Tear down the asic specific driver variables and program the hw
8764 * to an idle state (CIK).
8765 * Called at driver unload.
8767 void cik_fini(struct radeon_device *rdev)
8769 radeon_pm_fini(rdev);
8771 cik_sdma_fini(rdev);
8775 sumo_rlc_fini(rdev);
8777 radeon_wb_fini(rdev);
8778 radeon_vm_manager_fini(rdev);
8779 radeon_ib_pool_fini(rdev);
8780 radeon_irq_kms_fini(rdev);
8781 uvd_v1_0_fini(rdev);
8782 radeon_uvd_fini(rdev);
8783 radeon_vce_fini(rdev);
8784 cik_pcie_gart_fini(rdev);
8785 r600_vram_scratch_fini(rdev);
8786 radeon_gem_fini(rdev);
8787 radeon_fence_driver_fini(rdev);
8788 radeon_bo_fini(rdev);
8789 radeon_atombios_fini(rdev);
8794 void dce8_program_fmt(struct drm_encoder *encoder)
8796 struct drm_device *dev = encoder->dev;
8797 struct radeon_device *rdev = dev->dev_private;
8798 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8799 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8800 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8803 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8806 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8807 bpc = radeon_get_monitor_bpc(connector);
8808 dither = radeon_connector->dither;
8811 /* LVDS/eDP FMT is set up by atom */
8812 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8815 /* not needed for analog */
8816 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8817 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8825 if (dither == RADEON_FMT_DITHER_ENABLE)
8826 /* XXX sort out optimal dither settings */
8827 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8828 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8830 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8833 if (dither == RADEON_FMT_DITHER_ENABLE)
8834 /* XXX sort out optimal dither settings */
8835 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8836 FMT_RGB_RANDOM_ENABLE |
8837 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8839 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8842 if (dither == RADEON_FMT_DITHER_ENABLE)
8843 /* XXX sort out optimal dither settings */
8844 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8845 FMT_RGB_RANDOM_ENABLE |
8846 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8848 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8855 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8858 /* display watermark setup */
8860 * dce8_line_buffer_adjust - Set up the line buffer
8862 * @rdev: radeon_device pointer
8863 * @radeon_crtc: the selected display controller
8864 * @mode: the current display mode on the selected display
8867 * Setup up the line buffer allocation for
8868 * the selected display controller (CIK).
8869 * Returns the line buffer size in pixels.
8871 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8872 struct radeon_crtc *radeon_crtc,
8873 struct drm_display_mode *mode)
8875 u32 tmp, buffer_alloc, i;
8876 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8879 * There are 6 line buffers, one for each display controllers.
8880 * There are 3 partitions per LB. Select the number of partitions
8881 * to enable based on the display width. For display widths larger
8882 * than 4096, you need use to use 2 display controllers and combine
8883 * them using the stereo blender.
8885 if (radeon_crtc->base.enabled && mode) {
8886 if (mode->crtc_hdisplay < 1920) {
8889 } else if (mode->crtc_hdisplay < 2560) {
8892 } else if (mode->crtc_hdisplay < 4096) {
8894 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8896 DRM_DEBUG_KMS("Mode too big for LB!\n");
8898 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8905 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8906 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8908 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8909 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8910 for (i = 0; i < rdev->usec_timeout; i++) {
8911 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8912 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8917 if (radeon_crtc->base.enabled && mode) {
8929 /* controller not enabled, so no lb used */
8934 * cik_get_number_of_dram_channels - get the number of dram channels
8936 * @rdev: radeon_device pointer
8938 * Look up the number of video ram channels (CIK).
8939 * Used for display watermark bandwidth calculations
8940 * Returns the number of dram channels
8942 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8944 u32 tmp = RREG32(MC_SHARED_CHMAP);
8946 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8969 struct dce8_wm_params {
8970 u32 dram_channels; /* number of dram channels */
8971 u32 yclk; /* bandwidth per dram data pin in kHz */
8972 u32 sclk; /* engine clock in kHz */
8973 u32 disp_clk; /* display clock in kHz */
8974 u32 src_width; /* viewport width */
8975 u32 active_time; /* active display time in ns */
8976 u32 blank_time; /* blank time in ns */
8977 bool interlaced; /* mode is interlaced */
8978 fixed20_12 vsc; /* vertical scale ratio */
8979 u32 num_heads; /* number of active crtcs */
8980 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8981 u32 lb_size; /* line buffer allocated to pipe */
8982 u32 vtaps; /* vertical scaler taps */
8986 * dce8_dram_bandwidth - get the dram bandwidth
8988 * @wm: watermark calculation data
8990 * Calculate the raw dram bandwidth (CIK).
8991 * Used for display watermark bandwidth calculations
8992 * Returns the dram bandwidth in MBytes/s
8994 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8996 /* Calculate raw DRAM Bandwidth */
8997 fixed20_12 dram_efficiency; /* 0.7 */
8998 fixed20_12 yclk, dram_channels, bandwidth;
9001 a.full = dfixed_const(1000);
9002 yclk.full = dfixed_const(wm->yclk);
9003 yclk.full = dfixed_div(yclk, a);
9004 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9005 a.full = dfixed_const(10);
9006 dram_efficiency.full = dfixed_const(7);
9007 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9008 bandwidth.full = dfixed_mul(dram_channels, yclk);
9009 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9011 return dfixed_trunc(bandwidth);
9015 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9017 * @wm: watermark calculation data
9019 * Calculate the dram bandwidth used for display (CIK).
9020 * Used for display watermark bandwidth calculations
9021 * Returns the dram bandwidth for display in MBytes/s
9023 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9025 /* Calculate DRAM Bandwidth and the part allocated to display. */
9026 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9027 fixed20_12 yclk, dram_channels, bandwidth;
9030 a.full = dfixed_const(1000);
9031 yclk.full = dfixed_const(wm->yclk);
9032 yclk.full = dfixed_div(yclk, a);
9033 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9034 a.full = dfixed_const(10);
9035 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9036 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9037 bandwidth.full = dfixed_mul(dram_channels, yclk);
9038 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9040 return dfixed_trunc(bandwidth);
9044 * dce8_data_return_bandwidth - get the data return bandwidth
9046 * @wm: watermark calculation data
9048 * Calculate the data return bandwidth used for display (CIK).
9049 * Used for display watermark bandwidth calculations
9050 * Returns the data return bandwidth in MBytes/s
9052 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9054 /* Calculate the display Data return Bandwidth */
9055 fixed20_12 return_efficiency; /* 0.8 */
9056 fixed20_12 sclk, bandwidth;
9059 a.full = dfixed_const(1000);
9060 sclk.full = dfixed_const(wm->sclk);
9061 sclk.full = dfixed_div(sclk, a);
9062 a.full = dfixed_const(10);
9063 return_efficiency.full = dfixed_const(8);
9064 return_efficiency.full = dfixed_div(return_efficiency, a);
9065 a.full = dfixed_const(32);
9066 bandwidth.full = dfixed_mul(a, sclk);
9067 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9069 return dfixed_trunc(bandwidth);
9073 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9075 * @wm: watermark calculation data
9077 * Calculate the dmif bandwidth used for display (CIK).
9078 * Used for display watermark bandwidth calculations
9079 * Returns the dmif bandwidth in MBytes/s
9081 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9083 /* Calculate the DMIF Request Bandwidth */
9084 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9085 fixed20_12 disp_clk, bandwidth;
9088 a.full = dfixed_const(1000);
9089 disp_clk.full = dfixed_const(wm->disp_clk);
9090 disp_clk.full = dfixed_div(disp_clk, a);
9091 a.full = dfixed_const(32);
9092 b.full = dfixed_mul(a, disp_clk);
9094 a.full = dfixed_const(10);
9095 disp_clk_request_efficiency.full = dfixed_const(8);
9096 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9098 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9100 return dfixed_trunc(bandwidth);
9104 * dce8_available_bandwidth - get the min available bandwidth
9106 * @wm: watermark calculation data
9108 * Calculate the min available bandwidth used for display (CIK).
9109 * Used for display watermark bandwidth calculations
9110 * Returns the min available bandwidth in MBytes/s
9112 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9114 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9115 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9116 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9117 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9119 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9123 * dce8_average_bandwidth - get the average available bandwidth
9125 * @wm: watermark calculation data
9127 * Calculate the average available bandwidth used for display (CIK).
9128 * Used for display watermark bandwidth calculations
9129 * Returns the average available bandwidth in MBytes/s
9131 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9133 /* Calculate the display mode Average Bandwidth
9134 * DisplayMode should contain the source and destination dimensions,
9138 fixed20_12 line_time;
9139 fixed20_12 src_width;
9140 fixed20_12 bandwidth;
9143 a.full = dfixed_const(1000);
9144 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9145 line_time.full = dfixed_div(line_time, a);
9146 bpp.full = dfixed_const(wm->bytes_per_pixel);
9147 src_width.full = dfixed_const(wm->src_width);
9148 bandwidth.full = dfixed_mul(src_width, bpp);
9149 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9150 bandwidth.full = dfixed_div(bandwidth, line_time);
9152 return dfixed_trunc(bandwidth);
9156 * dce8_latency_watermark - get the latency watermark
9158 * @wm: watermark calculation data
9160 * Calculate the latency watermark (CIK).
9161 * Used for display watermark bandwidth calculations
9162 * Returns the latency watermark in ns
9164 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9166 /* First calculate the latency in ns */
9167 u32 mc_latency = 2000; /* 2000 ns. */
9168 u32 available_bandwidth = dce8_available_bandwidth(wm);
9169 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9170 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9171 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9172 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9173 (wm->num_heads * cursor_line_pair_return_time);
9174 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9175 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9176 u32 tmp, dmif_size = 12288;
9179 if (wm->num_heads == 0)
9182 a.full = dfixed_const(2);
9183 b.full = dfixed_const(1);
9184 if ((wm->vsc.full > a.full) ||
9185 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9187 ((wm->vsc.full >= a.full) && wm->interlaced))
9188 max_src_lines_per_dst_line = 4;
9190 max_src_lines_per_dst_line = 2;
9192 a.full = dfixed_const(available_bandwidth);
9193 b.full = dfixed_const(wm->num_heads);
9194 a.full = dfixed_div(a, b);
9196 b.full = dfixed_const(mc_latency + 512);
9197 c.full = dfixed_const(wm->disp_clk);
9198 b.full = dfixed_div(b, c);
9200 c.full = dfixed_const(dmif_size);
9201 b.full = dfixed_div(c, b);
9203 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9205 b.full = dfixed_const(1000);
9206 c.full = dfixed_const(wm->disp_clk);
9207 b.full = dfixed_div(c, b);
9208 c.full = dfixed_const(wm->bytes_per_pixel);
9209 b.full = dfixed_mul(b, c);
9211 lb_fill_bw = min(tmp, dfixed_trunc(b));
9213 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9214 b.full = dfixed_const(1000);
9215 c.full = dfixed_const(lb_fill_bw);
9216 b.full = dfixed_div(c, b);
9217 a.full = dfixed_div(a, b);
9218 line_fill_time = dfixed_trunc(a);
9220 if (line_fill_time < wm->active_time)
9223 return latency + (line_fill_time - wm->active_time);
9228 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9229 * average and available dram bandwidth
9231 * @wm: watermark calculation data
9233 * Check if the display average bandwidth fits in the display
9234 * dram bandwidth (CIK).
9235 * Used for display watermark bandwidth calculations
9236 * Returns true if the display fits, false if not.
9238 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9240 if (dce8_average_bandwidth(wm) <=
9241 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9248 * dce8_average_bandwidth_vs_available_bandwidth - check
9249 * average and available bandwidth
9251 * @wm: watermark calculation data
9253 * Check if the display average bandwidth fits in the display
9254 * available bandwidth (CIK).
9255 * Used for display watermark bandwidth calculations
9256 * Returns true if the display fits, false if not.
9258 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9260 if (dce8_average_bandwidth(wm) <=
9261 (dce8_available_bandwidth(wm) / wm->num_heads))
9268 * dce8_check_latency_hiding - check latency hiding
9270 * @wm: watermark calculation data
9272 * Check latency hiding (CIK).
9273 * Used for display watermark bandwidth calculations
9274 * Returns true if the display fits, false if not.
9276 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9278 u32 lb_partitions = wm->lb_size / wm->src_width;
9279 u32 line_time = wm->active_time + wm->blank_time;
9280 u32 latency_tolerant_lines;
9284 a.full = dfixed_const(1);
9285 if (wm->vsc.full > a.full)
9286 latency_tolerant_lines = 1;
9288 if (lb_partitions <= (wm->vtaps + 1))
9289 latency_tolerant_lines = 1;
9291 latency_tolerant_lines = 2;
9294 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9296 if (dce8_latency_watermark(wm) <= latency_hiding)
9303 * dce8_program_watermarks - program display watermarks
9305 * @rdev: radeon_device pointer
9306 * @radeon_crtc: the selected display controller
9307 * @lb_size: line buffer size
9308 * @num_heads: number of display controllers in use
9310 * Calculate and program the display watermarks for the
9311 * selected display controller (CIK).
9313 static void dce8_program_watermarks(struct radeon_device *rdev,
9314 struct radeon_crtc *radeon_crtc,
9315 u32 lb_size, u32 num_heads)
9317 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9318 struct dce8_wm_params wm_low, wm_high;
9321 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9324 if (radeon_crtc->base.enabled && num_heads && mode) {
9325 pixel_period = 1000000 / (u32)mode->clock;
9326 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9328 /* watermark for high clocks */
9329 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9330 rdev->pm.dpm_enabled) {
9332 radeon_dpm_get_mclk(rdev, false) * 10;
9334 radeon_dpm_get_sclk(rdev, false) * 10;
9336 wm_high.yclk = rdev->pm.current_mclk * 10;
9337 wm_high.sclk = rdev->pm.current_sclk * 10;
9340 wm_high.disp_clk = mode->clock;
9341 wm_high.src_width = mode->crtc_hdisplay;
9342 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9343 wm_high.blank_time = line_time - wm_high.active_time;
9344 wm_high.interlaced = false;
9345 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9346 wm_high.interlaced = true;
9347 wm_high.vsc = radeon_crtc->vsc;
9349 if (radeon_crtc->rmx_type != RMX_OFF)
9351 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9352 wm_high.lb_size = lb_size;
9353 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9354 wm_high.num_heads = num_heads;
9356 /* set for high clocks */
9357 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9359 /* possibly force display priority to high */
9360 /* should really do this at mode validation time... */
9361 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9362 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9363 !dce8_check_latency_hiding(&wm_high) ||
9364 (rdev->disp_priority == 2)) {
9365 DRM_DEBUG_KMS("force priority to high\n");
9368 /* watermark for low clocks */
9369 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9370 rdev->pm.dpm_enabled) {
9372 radeon_dpm_get_mclk(rdev, true) * 10;
9374 radeon_dpm_get_sclk(rdev, true) * 10;
9376 wm_low.yclk = rdev->pm.current_mclk * 10;
9377 wm_low.sclk = rdev->pm.current_sclk * 10;
9380 wm_low.disp_clk = mode->clock;
9381 wm_low.src_width = mode->crtc_hdisplay;
9382 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9383 wm_low.blank_time = line_time - wm_low.active_time;
9384 wm_low.interlaced = false;
9385 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9386 wm_low.interlaced = true;
9387 wm_low.vsc = radeon_crtc->vsc;
9389 if (radeon_crtc->rmx_type != RMX_OFF)
9391 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9392 wm_low.lb_size = lb_size;
9393 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9394 wm_low.num_heads = num_heads;
9396 /* set for low clocks */
9397 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9399 /* possibly force display priority to high */
9400 /* should really do this at mode validation time... */
9401 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9402 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9403 !dce8_check_latency_hiding(&wm_low) ||
9404 (rdev->disp_priority == 2)) {
9405 DRM_DEBUG_KMS("force priority to high\n");
9410 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9412 tmp &= ~LATENCY_WATERMARK_MASK(3);
9413 tmp |= LATENCY_WATERMARK_MASK(1);
9414 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9415 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9416 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9417 LATENCY_HIGH_WATERMARK(line_time)));
9419 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9420 tmp &= ~LATENCY_WATERMARK_MASK(3);
9421 tmp |= LATENCY_WATERMARK_MASK(2);
9422 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9423 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9424 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9425 LATENCY_HIGH_WATERMARK(line_time)));
9426 /* restore original selection */
9427 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9429 /* save values for DPM */
9430 radeon_crtc->line_time = line_time;
9431 radeon_crtc->wm_high = latency_watermark_a;
9432 radeon_crtc->wm_low = latency_watermark_b;
9436 * dce8_bandwidth_update - program display watermarks
9438 * @rdev: radeon_device pointer
9440 * Calculate and program the display watermarks and line
9441 * buffer allocation (CIK).
9443 void dce8_bandwidth_update(struct radeon_device *rdev)
9445 struct drm_display_mode *mode = NULL;
9446 u32 num_heads = 0, lb_size;
9449 if (!rdev->mode_info.mode_config_initialized)
9452 radeon_update_display_priority(rdev);
9454 for (i = 0; i < rdev->num_crtc; i++) {
9455 if (rdev->mode_info.crtcs[i]->base.enabled)
9458 for (i = 0; i < rdev->num_crtc; i++) {
9459 mode = &rdev->mode_info.crtcs[i]->base.mode;
9460 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9461 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9466 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9468 * @rdev: radeon_device pointer
9470 * Fetches a GPU clock counter snapshot (SI).
9471 * Returns the 64 bit clock counter snapshot.
9473 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9477 mutex_lock(&rdev->gpu_clock_mutex);
9478 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9479 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9480 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9481 mutex_unlock(&rdev->gpu_clock_mutex);
9485 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9486 u32 cntl_reg, u32 status_reg)
9489 struct atom_clock_dividers dividers;
9492 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9493 clock, false, ÷rs);
9497 tmp = RREG32_SMC(cntl_reg);
9498 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9499 tmp |= dividers.post_divider;
9500 WREG32_SMC(cntl_reg, tmp);
9502 for (i = 0; i < 100; i++) {
9503 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9513 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9517 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9521 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9525 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9528 struct atom_clock_dividers dividers;
9531 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9532 ecclk, false, ÷rs);
9536 for (i = 0; i < 100; i++) {
9537 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9544 tmp = RREG32_SMC(CG_ECLK_CNTL);
9545 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9546 tmp |= dividers.post_divider;
9547 WREG32_SMC(CG_ECLK_CNTL, tmp);
9549 for (i = 0; i < 100; i++) {
9550 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9560 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9562 struct pci_dev *root = rdev->pdev->bus->self;
9563 int bridge_pos, gpu_pos;
9564 u32 speed_cntl, mask, current_data_rate;
9568 if (pci_is_root_bus(rdev->pdev->bus))
9571 if (radeon_pcie_gen2 == 0)
9574 if (rdev->flags & RADEON_IS_IGP)
9577 if (!(rdev->flags & RADEON_IS_PCIE))
9580 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9584 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9587 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9588 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9589 LC_CURRENT_DATA_RATE_SHIFT;
9590 if (mask & DRM_PCIE_SPEED_80) {
9591 if (current_data_rate == 2) {
9592 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9595 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9596 } else if (mask & DRM_PCIE_SPEED_50) {
9597 if (current_data_rate == 1) {
9598 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9601 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9604 bridge_pos = pci_pcie_cap(root);
9608 gpu_pos = pci_pcie_cap(rdev->pdev);
9612 if (mask & DRM_PCIE_SPEED_80) {
9613 /* re-try equalization if gen3 is not already enabled */
9614 if (current_data_rate != 2) {
9615 u16 bridge_cfg, gpu_cfg;
9616 u16 bridge_cfg2, gpu_cfg2;
9617 u32 max_lw, current_lw, tmp;
9619 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9620 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9622 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9623 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9625 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9626 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9628 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9629 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9630 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9632 if (current_lw < max_lw) {
9633 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9634 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9635 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9636 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9637 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9638 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9642 for (i = 0; i < 10; i++) {
9644 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9645 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9648 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9649 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9651 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9652 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9654 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9655 tmp |= LC_SET_QUIESCE;
9656 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9658 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9660 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9665 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9666 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9667 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9668 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9670 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9671 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9672 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9673 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9676 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9677 tmp16 &= ~((1 << 4) | (7 << 9));
9678 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9679 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9681 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9682 tmp16 &= ~((1 << 4) | (7 << 9));
9683 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9684 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9686 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9687 tmp &= ~LC_SET_QUIESCE;
9688 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9693 /* set the link speed */
9694 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9695 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9696 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9698 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9700 if (mask & DRM_PCIE_SPEED_80)
9701 tmp16 |= 3; /* gen3 */
9702 else if (mask & DRM_PCIE_SPEED_50)
9703 tmp16 |= 2; /* gen2 */
9705 tmp16 |= 1; /* gen1 */
9706 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9708 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9709 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9710 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9712 for (i = 0; i < rdev->usec_timeout; i++) {
9713 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9714 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9720 static void cik_program_aspm(struct radeon_device *rdev)
9723 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9724 bool disable_clkreq = false;
9726 if (radeon_aspm == 0)
9729 /* XXX double check IGPs */
9730 if (rdev->flags & RADEON_IS_IGP)
9733 if (!(rdev->flags & RADEON_IS_PCIE))
9736 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9737 data &= ~LC_XMIT_N_FTS_MASK;
9738 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9740 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9742 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9743 data |= LC_GO_TO_RECOVERY;
9745 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9747 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9748 data |= P_IGNORE_EDB_ERR;
9750 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9752 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9753 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9754 data |= LC_PMI_TO_L1_DIS;
9756 data |= LC_L0S_INACTIVITY(7);
9759 data |= LC_L1_INACTIVITY(7);
9760 data &= ~LC_PMI_TO_L1_DIS;
9762 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9764 if (!disable_plloff_in_l1) {
9765 bool clk_req_support;
9767 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9768 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9769 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9771 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9773 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9774 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9775 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9777 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9779 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9780 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9781 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9783 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9785 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9786 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9787 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9789 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9791 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9792 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9793 data |= LC_DYN_LANES_PWR_STATE(3);
9795 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9797 if (!disable_clkreq &&
9798 !pci_is_root_bus(rdev->pdev->bus)) {
9799 struct pci_dev *root = rdev->pdev->bus->self;
9802 clk_req_support = false;
9803 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9804 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9805 clk_req_support = true;
9807 clk_req_support = false;
9810 if (clk_req_support) {
9811 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9812 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9814 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9816 orig = data = RREG32_SMC(THM_CLK_CNTL);
9817 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9818 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9820 WREG32_SMC(THM_CLK_CNTL, data);
9822 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9823 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9824 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9826 WREG32_SMC(MISC_CLK_CTRL, data);
9828 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9829 data &= ~BCLK_AS_XCLK;
9831 WREG32_SMC(CG_CLKPIN_CNTL, data);
9833 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9834 data &= ~FORCE_BIF_REFCLK_EN;
9836 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9838 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9839 data &= ~MPLL_CLKOUT_SEL_MASK;
9840 data |= MPLL_CLKOUT_SEL(4);
9842 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9847 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9850 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9851 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9853 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9856 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9857 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9858 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9859 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9860 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9861 data &= ~LC_L0S_INACTIVITY_MASK;
9863 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);