2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
33 #include "cik_blit_shaders.h"
34 #include "radeon_ucode.h"
35 #include "clearstate_ci.h"
36 #include "radeon_kfd.h"
38 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
45 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
46 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
48 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
49 MODULE_FIRMWARE("radeon/bonaire_me.bin");
50 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
51 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
52 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
53 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
54 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
55 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
57 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
58 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
59 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
60 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
63 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
64 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
65 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
67 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
68 MODULE_FIRMWARE("radeon/hawaii_me.bin");
69 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
70 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
71 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
72 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
73 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
74 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
76 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
77 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
78 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
79 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
80 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
81 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
83 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
84 MODULE_FIRMWARE("radeon/kaveri_me.bin");
85 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
86 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
87 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
88 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
89 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
91 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
92 MODULE_FIRMWARE("radeon/KABINI_me.bin");
93 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
94 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
95 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
96 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
98 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
99 MODULE_FIRMWARE("radeon/kabini_me.bin");
100 MODULE_FIRMWARE("radeon/kabini_ce.bin");
101 MODULE_FIRMWARE("radeon/kabini_mec.bin");
102 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
103 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
105 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
106 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
107 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
108 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
109 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
110 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
112 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
113 MODULE_FIRMWARE("radeon/mullins_me.bin");
114 MODULE_FIRMWARE("radeon/mullins_ce.bin");
115 MODULE_FIRMWARE("radeon/mullins_mec.bin");
116 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
117 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
119 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
120 extern void r600_ih_ring_fini(struct radeon_device *rdev);
121 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
123 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
124 extern void sumo_rlc_fini(struct radeon_device *rdev);
125 extern int sumo_rlc_init(struct radeon_device *rdev);
126 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
127 extern void si_rlc_reset(struct radeon_device *rdev);
128 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
129 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
130 extern int cik_sdma_resume(struct radeon_device *rdev);
131 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
132 extern void cik_sdma_fini(struct radeon_device *rdev);
133 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
134 static void cik_rlc_stop(struct radeon_device *rdev);
135 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
136 static void cik_program_aspm(struct radeon_device *rdev);
137 static void cik_init_pg(struct radeon_device *rdev);
138 static void cik_init_cg(struct radeon_device *rdev);
139 static void cik_fini_pg(struct radeon_device *rdev);
140 static void cik_fini_cg(struct radeon_device *rdev);
141 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
144 /* get temperature in millidegrees */
145 int ci_get_temp(struct radeon_device *rdev)
150 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
156 actual_temp = temp & 0x1ff;
158 actual_temp = actual_temp * 1000;
163 /* get temperature in millidegrees */
164 int kv_get_temp(struct radeon_device *rdev)
169 temp = RREG32_SMC(0xC0300E0C);
172 actual_temp = (temp / 8) - 49;
176 actual_temp = actual_temp * 1000;
182 * Indirect registers accessor
184 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
189 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
190 WREG32(PCIE_INDEX, reg);
191 (void)RREG32(PCIE_INDEX);
192 r = RREG32(PCIE_DATA);
193 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
197 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
201 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
202 WREG32(PCIE_INDEX, reg);
203 (void)RREG32(PCIE_INDEX);
204 WREG32(PCIE_DATA, v);
205 (void)RREG32(PCIE_DATA);
206 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
209 static const u32 spectre_rlc_save_restore_register_list[] =
211 (0x0e00 << 16) | (0xc12c >> 2),
213 (0x0e00 << 16) | (0xc140 >> 2),
215 (0x0e00 << 16) | (0xc150 >> 2),
217 (0x0e00 << 16) | (0xc15c >> 2),
219 (0x0e00 << 16) | (0xc168 >> 2),
221 (0x0e00 << 16) | (0xc170 >> 2),
223 (0x0e00 << 16) | (0xc178 >> 2),
225 (0x0e00 << 16) | (0xc204 >> 2),
227 (0x0e00 << 16) | (0xc2b4 >> 2),
229 (0x0e00 << 16) | (0xc2b8 >> 2),
231 (0x0e00 << 16) | (0xc2bc >> 2),
233 (0x0e00 << 16) | (0xc2c0 >> 2),
235 (0x0e00 << 16) | (0x8228 >> 2),
237 (0x0e00 << 16) | (0x829c >> 2),
239 (0x0e00 << 16) | (0x869c >> 2),
241 (0x0600 << 16) | (0x98f4 >> 2),
243 (0x0e00 << 16) | (0x98f8 >> 2),
245 (0x0e00 << 16) | (0x9900 >> 2),
247 (0x0e00 << 16) | (0xc260 >> 2),
249 (0x0e00 << 16) | (0x90e8 >> 2),
251 (0x0e00 << 16) | (0x3c000 >> 2),
253 (0x0e00 << 16) | (0x3c00c >> 2),
255 (0x0e00 << 16) | (0x8c1c >> 2),
257 (0x0e00 << 16) | (0x9700 >> 2),
259 (0x0e00 << 16) | (0xcd20 >> 2),
261 (0x4e00 << 16) | (0xcd20 >> 2),
263 (0x5e00 << 16) | (0xcd20 >> 2),
265 (0x6e00 << 16) | (0xcd20 >> 2),
267 (0x7e00 << 16) | (0xcd20 >> 2),
269 (0x8e00 << 16) | (0xcd20 >> 2),
271 (0x9e00 << 16) | (0xcd20 >> 2),
273 (0xae00 << 16) | (0xcd20 >> 2),
275 (0xbe00 << 16) | (0xcd20 >> 2),
277 (0x0e00 << 16) | (0x89bc >> 2),
279 (0x0e00 << 16) | (0x8900 >> 2),
282 (0x0e00 << 16) | (0xc130 >> 2),
284 (0x0e00 << 16) | (0xc134 >> 2),
286 (0x0e00 << 16) | (0xc1fc >> 2),
288 (0x0e00 << 16) | (0xc208 >> 2),
290 (0x0e00 << 16) | (0xc264 >> 2),
292 (0x0e00 << 16) | (0xc268 >> 2),
294 (0x0e00 << 16) | (0xc26c >> 2),
296 (0x0e00 << 16) | (0xc270 >> 2),
298 (0x0e00 << 16) | (0xc274 >> 2),
300 (0x0e00 << 16) | (0xc278 >> 2),
302 (0x0e00 << 16) | (0xc27c >> 2),
304 (0x0e00 << 16) | (0xc280 >> 2),
306 (0x0e00 << 16) | (0xc284 >> 2),
308 (0x0e00 << 16) | (0xc288 >> 2),
310 (0x0e00 << 16) | (0xc28c >> 2),
312 (0x0e00 << 16) | (0xc290 >> 2),
314 (0x0e00 << 16) | (0xc294 >> 2),
316 (0x0e00 << 16) | (0xc298 >> 2),
318 (0x0e00 << 16) | (0xc29c >> 2),
320 (0x0e00 << 16) | (0xc2a0 >> 2),
322 (0x0e00 << 16) | (0xc2a4 >> 2),
324 (0x0e00 << 16) | (0xc2a8 >> 2),
326 (0x0e00 << 16) | (0xc2ac >> 2),
328 (0x0e00 << 16) | (0xc2b0 >> 2),
330 (0x0e00 << 16) | (0x301d0 >> 2),
332 (0x0e00 << 16) | (0x30238 >> 2),
334 (0x0e00 << 16) | (0x30250 >> 2),
336 (0x0e00 << 16) | (0x30254 >> 2),
338 (0x0e00 << 16) | (0x30258 >> 2),
340 (0x0e00 << 16) | (0x3025c >> 2),
342 (0x4e00 << 16) | (0xc900 >> 2),
344 (0x5e00 << 16) | (0xc900 >> 2),
346 (0x6e00 << 16) | (0xc900 >> 2),
348 (0x7e00 << 16) | (0xc900 >> 2),
350 (0x8e00 << 16) | (0xc900 >> 2),
352 (0x9e00 << 16) | (0xc900 >> 2),
354 (0xae00 << 16) | (0xc900 >> 2),
356 (0xbe00 << 16) | (0xc900 >> 2),
358 (0x4e00 << 16) | (0xc904 >> 2),
360 (0x5e00 << 16) | (0xc904 >> 2),
362 (0x6e00 << 16) | (0xc904 >> 2),
364 (0x7e00 << 16) | (0xc904 >> 2),
366 (0x8e00 << 16) | (0xc904 >> 2),
368 (0x9e00 << 16) | (0xc904 >> 2),
370 (0xae00 << 16) | (0xc904 >> 2),
372 (0xbe00 << 16) | (0xc904 >> 2),
374 (0x4e00 << 16) | (0xc908 >> 2),
376 (0x5e00 << 16) | (0xc908 >> 2),
378 (0x6e00 << 16) | (0xc908 >> 2),
380 (0x7e00 << 16) | (0xc908 >> 2),
382 (0x8e00 << 16) | (0xc908 >> 2),
384 (0x9e00 << 16) | (0xc908 >> 2),
386 (0xae00 << 16) | (0xc908 >> 2),
388 (0xbe00 << 16) | (0xc908 >> 2),
390 (0x4e00 << 16) | (0xc90c >> 2),
392 (0x5e00 << 16) | (0xc90c >> 2),
394 (0x6e00 << 16) | (0xc90c >> 2),
396 (0x7e00 << 16) | (0xc90c >> 2),
398 (0x8e00 << 16) | (0xc90c >> 2),
400 (0x9e00 << 16) | (0xc90c >> 2),
402 (0xae00 << 16) | (0xc90c >> 2),
404 (0xbe00 << 16) | (0xc90c >> 2),
406 (0x4e00 << 16) | (0xc910 >> 2),
408 (0x5e00 << 16) | (0xc910 >> 2),
410 (0x6e00 << 16) | (0xc910 >> 2),
412 (0x7e00 << 16) | (0xc910 >> 2),
414 (0x8e00 << 16) | (0xc910 >> 2),
416 (0x9e00 << 16) | (0xc910 >> 2),
418 (0xae00 << 16) | (0xc910 >> 2),
420 (0xbe00 << 16) | (0xc910 >> 2),
422 (0x0e00 << 16) | (0xc99c >> 2),
424 (0x0e00 << 16) | (0x9834 >> 2),
426 (0x0000 << 16) | (0x30f00 >> 2),
428 (0x0001 << 16) | (0x30f00 >> 2),
430 (0x0000 << 16) | (0x30f04 >> 2),
432 (0x0001 << 16) | (0x30f04 >> 2),
434 (0x0000 << 16) | (0x30f08 >> 2),
436 (0x0001 << 16) | (0x30f08 >> 2),
438 (0x0000 << 16) | (0x30f0c >> 2),
440 (0x0001 << 16) | (0x30f0c >> 2),
442 (0x0600 << 16) | (0x9b7c >> 2),
444 (0x0e00 << 16) | (0x8a14 >> 2),
446 (0x0e00 << 16) | (0x8a18 >> 2),
448 (0x0600 << 16) | (0x30a00 >> 2),
450 (0x0e00 << 16) | (0x8bf0 >> 2),
452 (0x0e00 << 16) | (0x8bcc >> 2),
454 (0x0e00 << 16) | (0x8b24 >> 2),
456 (0x0e00 << 16) | (0x30a04 >> 2),
458 (0x0600 << 16) | (0x30a10 >> 2),
460 (0x0600 << 16) | (0x30a14 >> 2),
462 (0x0600 << 16) | (0x30a18 >> 2),
464 (0x0600 << 16) | (0x30a2c >> 2),
466 (0x0e00 << 16) | (0xc700 >> 2),
468 (0x0e00 << 16) | (0xc704 >> 2),
470 (0x0e00 << 16) | (0xc708 >> 2),
472 (0x0e00 << 16) | (0xc768 >> 2),
474 (0x0400 << 16) | (0xc770 >> 2),
476 (0x0400 << 16) | (0xc774 >> 2),
478 (0x0400 << 16) | (0xc778 >> 2),
480 (0x0400 << 16) | (0xc77c >> 2),
482 (0x0400 << 16) | (0xc780 >> 2),
484 (0x0400 << 16) | (0xc784 >> 2),
486 (0x0400 << 16) | (0xc788 >> 2),
488 (0x0400 << 16) | (0xc78c >> 2),
490 (0x0400 << 16) | (0xc798 >> 2),
492 (0x0400 << 16) | (0xc79c >> 2),
494 (0x0400 << 16) | (0xc7a0 >> 2),
496 (0x0400 << 16) | (0xc7a4 >> 2),
498 (0x0400 << 16) | (0xc7a8 >> 2),
500 (0x0400 << 16) | (0xc7ac >> 2),
502 (0x0400 << 16) | (0xc7b0 >> 2),
504 (0x0400 << 16) | (0xc7b4 >> 2),
506 (0x0e00 << 16) | (0x9100 >> 2),
508 (0x0e00 << 16) | (0x3c010 >> 2),
510 (0x0e00 << 16) | (0x92a8 >> 2),
512 (0x0e00 << 16) | (0x92ac >> 2),
514 (0x0e00 << 16) | (0x92b4 >> 2),
516 (0x0e00 << 16) | (0x92b8 >> 2),
518 (0x0e00 << 16) | (0x92bc >> 2),
520 (0x0e00 << 16) | (0x92c0 >> 2),
522 (0x0e00 << 16) | (0x92c4 >> 2),
524 (0x0e00 << 16) | (0x92c8 >> 2),
526 (0x0e00 << 16) | (0x92cc >> 2),
528 (0x0e00 << 16) | (0x92d0 >> 2),
530 (0x0e00 << 16) | (0x8c00 >> 2),
532 (0x0e00 << 16) | (0x8c04 >> 2),
534 (0x0e00 << 16) | (0x8c20 >> 2),
536 (0x0e00 << 16) | (0x8c38 >> 2),
538 (0x0e00 << 16) | (0x8c3c >> 2),
540 (0x0e00 << 16) | (0xae00 >> 2),
542 (0x0e00 << 16) | (0x9604 >> 2),
544 (0x0e00 << 16) | (0xac08 >> 2),
546 (0x0e00 << 16) | (0xac0c >> 2),
548 (0x0e00 << 16) | (0xac10 >> 2),
550 (0x0e00 << 16) | (0xac14 >> 2),
552 (0x0e00 << 16) | (0xac58 >> 2),
554 (0x0e00 << 16) | (0xac68 >> 2),
556 (0x0e00 << 16) | (0xac6c >> 2),
558 (0x0e00 << 16) | (0xac70 >> 2),
560 (0x0e00 << 16) | (0xac74 >> 2),
562 (0x0e00 << 16) | (0xac78 >> 2),
564 (0x0e00 << 16) | (0xac7c >> 2),
566 (0x0e00 << 16) | (0xac80 >> 2),
568 (0x0e00 << 16) | (0xac84 >> 2),
570 (0x0e00 << 16) | (0xac88 >> 2),
572 (0x0e00 << 16) | (0xac8c >> 2),
574 (0x0e00 << 16) | (0x970c >> 2),
576 (0x0e00 << 16) | (0x9714 >> 2),
578 (0x0e00 << 16) | (0x9718 >> 2),
580 (0x0e00 << 16) | (0x971c >> 2),
582 (0x0e00 << 16) | (0x31068 >> 2),
584 (0x4e00 << 16) | (0x31068 >> 2),
586 (0x5e00 << 16) | (0x31068 >> 2),
588 (0x6e00 << 16) | (0x31068 >> 2),
590 (0x7e00 << 16) | (0x31068 >> 2),
592 (0x8e00 << 16) | (0x31068 >> 2),
594 (0x9e00 << 16) | (0x31068 >> 2),
596 (0xae00 << 16) | (0x31068 >> 2),
598 (0xbe00 << 16) | (0x31068 >> 2),
600 (0x0e00 << 16) | (0xcd10 >> 2),
602 (0x0e00 << 16) | (0xcd14 >> 2),
604 (0x0e00 << 16) | (0x88b0 >> 2),
606 (0x0e00 << 16) | (0x88b4 >> 2),
608 (0x0e00 << 16) | (0x88b8 >> 2),
610 (0x0e00 << 16) | (0x88bc >> 2),
612 (0x0400 << 16) | (0x89c0 >> 2),
614 (0x0e00 << 16) | (0x88c4 >> 2),
616 (0x0e00 << 16) | (0x88c8 >> 2),
618 (0x0e00 << 16) | (0x88d0 >> 2),
620 (0x0e00 << 16) | (0x88d4 >> 2),
622 (0x0e00 << 16) | (0x88d8 >> 2),
624 (0x0e00 << 16) | (0x8980 >> 2),
626 (0x0e00 << 16) | (0x30938 >> 2),
628 (0x0e00 << 16) | (0x3093c >> 2),
630 (0x0e00 << 16) | (0x30940 >> 2),
632 (0x0e00 << 16) | (0x89a0 >> 2),
634 (0x0e00 << 16) | (0x30900 >> 2),
636 (0x0e00 << 16) | (0x30904 >> 2),
638 (0x0e00 << 16) | (0x89b4 >> 2),
640 (0x0e00 << 16) | (0x3c210 >> 2),
642 (0x0e00 << 16) | (0x3c214 >> 2),
644 (0x0e00 << 16) | (0x3c218 >> 2),
646 (0x0e00 << 16) | (0x8904 >> 2),
649 (0x0e00 << 16) | (0x8c28 >> 2),
650 (0x0e00 << 16) | (0x8c2c >> 2),
651 (0x0e00 << 16) | (0x8c30 >> 2),
652 (0x0e00 << 16) | (0x8c34 >> 2),
653 (0x0e00 << 16) | (0x9600 >> 2),
656 static const u32 kalindi_rlc_save_restore_register_list[] =
658 (0x0e00 << 16) | (0xc12c >> 2),
660 (0x0e00 << 16) | (0xc140 >> 2),
662 (0x0e00 << 16) | (0xc150 >> 2),
664 (0x0e00 << 16) | (0xc15c >> 2),
666 (0x0e00 << 16) | (0xc168 >> 2),
668 (0x0e00 << 16) | (0xc170 >> 2),
670 (0x0e00 << 16) | (0xc204 >> 2),
672 (0x0e00 << 16) | (0xc2b4 >> 2),
674 (0x0e00 << 16) | (0xc2b8 >> 2),
676 (0x0e00 << 16) | (0xc2bc >> 2),
678 (0x0e00 << 16) | (0xc2c0 >> 2),
680 (0x0e00 << 16) | (0x8228 >> 2),
682 (0x0e00 << 16) | (0x829c >> 2),
684 (0x0e00 << 16) | (0x869c >> 2),
686 (0x0600 << 16) | (0x98f4 >> 2),
688 (0x0e00 << 16) | (0x98f8 >> 2),
690 (0x0e00 << 16) | (0x9900 >> 2),
692 (0x0e00 << 16) | (0xc260 >> 2),
694 (0x0e00 << 16) | (0x90e8 >> 2),
696 (0x0e00 << 16) | (0x3c000 >> 2),
698 (0x0e00 << 16) | (0x3c00c >> 2),
700 (0x0e00 << 16) | (0x8c1c >> 2),
702 (0x0e00 << 16) | (0x9700 >> 2),
704 (0x0e00 << 16) | (0xcd20 >> 2),
706 (0x4e00 << 16) | (0xcd20 >> 2),
708 (0x5e00 << 16) | (0xcd20 >> 2),
710 (0x6e00 << 16) | (0xcd20 >> 2),
712 (0x7e00 << 16) | (0xcd20 >> 2),
714 (0x0e00 << 16) | (0x89bc >> 2),
716 (0x0e00 << 16) | (0x8900 >> 2),
719 (0x0e00 << 16) | (0xc130 >> 2),
721 (0x0e00 << 16) | (0xc134 >> 2),
723 (0x0e00 << 16) | (0xc1fc >> 2),
725 (0x0e00 << 16) | (0xc208 >> 2),
727 (0x0e00 << 16) | (0xc264 >> 2),
729 (0x0e00 << 16) | (0xc268 >> 2),
731 (0x0e00 << 16) | (0xc26c >> 2),
733 (0x0e00 << 16) | (0xc270 >> 2),
735 (0x0e00 << 16) | (0xc274 >> 2),
737 (0x0e00 << 16) | (0xc28c >> 2),
739 (0x0e00 << 16) | (0xc290 >> 2),
741 (0x0e00 << 16) | (0xc294 >> 2),
743 (0x0e00 << 16) | (0xc298 >> 2),
745 (0x0e00 << 16) | (0xc2a0 >> 2),
747 (0x0e00 << 16) | (0xc2a4 >> 2),
749 (0x0e00 << 16) | (0xc2a8 >> 2),
751 (0x0e00 << 16) | (0xc2ac >> 2),
753 (0x0e00 << 16) | (0x301d0 >> 2),
755 (0x0e00 << 16) | (0x30238 >> 2),
757 (0x0e00 << 16) | (0x30250 >> 2),
759 (0x0e00 << 16) | (0x30254 >> 2),
761 (0x0e00 << 16) | (0x30258 >> 2),
763 (0x0e00 << 16) | (0x3025c >> 2),
765 (0x4e00 << 16) | (0xc900 >> 2),
767 (0x5e00 << 16) | (0xc900 >> 2),
769 (0x6e00 << 16) | (0xc900 >> 2),
771 (0x7e00 << 16) | (0xc900 >> 2),
773 (0x4e00 << 16) | (0xc904 >> 2),
775 (0x5e00 << 16) | (0xc904 >> 2),
777 (0x6e00 << 16) | (0xc904 >> 2),
779 (0x7e00 << 16) | (0xc904 >> 2),
781 (0x4e00 << 16) | (0xc908 >> 2),
783 (0x5e00 << 16) | (0xc908 >> 2),
785 (0x6e00 << 16) | (0xc908 >> 2),
787 (0x7e00 << 16) | (0xc908 >> 2),
789 (0x4e00 << 16) | (0xc90c >> 2),
791 (0x5e00 << 16) | (0xc90c >> 2),
793 (0x6e00 << 16) | (0xc90c >> 2),
795 (0x7e00 << 16) | (0xc90c >> 2),
797 (0x4e00 << 16) | (0xc910 >> 2),
799 (0x5e00 << 16) | (0xc910 >> 2),
801 (0x6e00 << 16) | (0xc910 >> 2),
803 (0x7e00 << 16) | (0xc910 >> 2),
805 (0x0e00 << 16) | (0xc99c >> 2),
807 (0x0e00 << 16) | (0x9834 >> 2),
809 (0x0000 << 16) | (0x30f00 >> 2),
811 (0x0000 << 16) | (0x30f04 >> 2),
813 (0x0000 << 16) | (0x30f08 >> 2),
815 (0x0000 << 16) | (0x30f0c >> 2),
817 (0x0600 << 16) | (0x9b7c >> 2),
819 (0x0e00 << 16) | (0x8a14 >> 2),
821 (0x0e00 << 16) | (0x8a18 >> 2),
823 (0x0600 << 16) | (0x30a00 >> 2),
825 (0x0e00 << 16) | (0x8bf0 >> 2),
827 (0x0e00 << 16) | (0x8bcc >> 2),
829 (0x0e00 << 16) | (0x8b24 >> 2),
831 (0x0e00 << 16) | (0x30a04 >> 2),
833 (0x0600 << 16) | (0x30a10 >> 2),
835 (0x0600 << 16) | (0x30a14 >> 2),
837 (0x0600 << 16) | (0x30a18 >> 2),
839 (0x0600 << 16) | (0x30a2c >> 2),
841 (0x0e00 << 16) | (0xc700 >> 2),
843 (0x0e00 << 16) | (0xc704 >> 2),
845 (0x0e00 << 16) | (0xc708 >> 2),
847 (0x0e00 << 16) | (0xc768 >> 2),
849 (0x0400 << 16) | (0xc770 >> 2),
851 (0x0400 << 16) | (0xc774 >> 2),
853 (0x0400 << 16) | (0xc798 >> 2),
855 (0x0400 << 16) | (0xc79c >> 2),
857 (0x0e00 << 16) | (0x9100 >> 2),
859 (0x0e00 << 16) | (0x3c010 >> 2),
861 (0x0e00 << 16) | (0x8c00 >> 2),
863 (0x0e00 << 16) | (0x8c04 >> 2),
865 (0x0e00 << 16) | (0x8c20 >> 2),
867 (0x0e00 << 16) | (0x8c38 >> 2),
869 (0x0e00 << 16) | (0x8c3c >> 2),
871 (0x0e00 << 16) | (0xae00 >> 2),
873 (0x0e00 << 16) | (0x9604 >> 2),
875 (0x0e00 << 16) | (0xac08 >> 2),
877 (0x0e00 << 16) | (0xac0c >> 2),
879 (0x0e00 << 16) | (0xac10 >> 2),
881 (0x0e00 << 16) | (0xac14 >> 2),
883 (0x0e00 << 16) | (0xac58 >> 2),
885 (0x0e00 << 16) | (0xac68 >> 2),
887 (0x0e00 << 16) | (0xac6c >> 2),
889 (0x0e00 << 16) | (0xac70 >> 2),
891 (0x0e00 << 16) | (0xac74 >> 2),
893 (0x0e00 << 16) | (0xac78 >> 2),
895 (0x0e00 << 16) | (0xac7c >> 2),
897 (0x0e00 << 16) | (0xac80 >> 2),
899 (0x0e00 << 16) | (0xac84 >> 2),
901 (0x0e00 << 16) | (0xac88 >> 2),
903 (0x0e00 << 16) | (0xac8c >> 2),
905 (0x0e00 << 16) | (0x970c >> 2),
907 (0x0e00 << 16) | (0x9714 >> 2),
909 (0x0e00 << 16) | (0x9718 >> 2),
911 (0x0e00 << 16) | (0x971c >> 2),
913 (0x0e00 << 16) | (0x31068 >> 2),
915 (0x4e00 << 16) | (0x31068 >> 2),
917 (0x5e00 << 16) | (0x31068 >> 2),
919 (0x6e00 << 16) | (0x31068 >> 2),
921 (0x7e00 << 16) | (0x31068 >> 2),
923 (0x0e00 << 16) | (0xcd10 >> 2),
925 (0x0e00 << 16) | (0xcd14 >> 2),
927 (0x0e00 << 16) | (0x88b0 >> 2),
929 (0x0e00 << 16) | (0x88b4 >> 2),
931 (0x0e00 << 16) | (0x88b8 >> 2),
933 (0x0e00 << 16) | (0x88bc >> 2),
935 (0x0400 << 16) | (0x89c0 >> 2),
937 (0x0e00 << 16) | (0x88c4 >> 2),
939 (0x0e00 << 16) | (0x88c8 >> 2),
941 (0x0e00 << 16) | (0x88d0 >> 2),
943 (0x0e00 << 16) | (0x88d4 >> 2),
945 (0x0e00 << 16) | (0x88d8 >> 2),
947 (0x0e00 << 16) | (0x8980 >> 2),
949 (0x0e00 << 16) | (0x30938 >> 2),
951 (0x0e00 << 16) | (0x3093c >> 2),
953 (0x0e00 << 16) | (0x30940 >> 2),
955 (0x0e00 << 16) | (0x89a0 >> 2),
957 (0x0e00 << 16) | (0x30900 >> 2),
959 (0x0e00 << 16) | (0x30904 >> 2),
961 (0x0e00 << 16) | (0x89b4 >> 2),
963 (0x0e00 << 16) | (0x3e1fc >> 2),
965 (0x0e00 << 16) | (0x3c210 >> 2),
967 (0x0e00 << 16) | (0x3c214 >> 2),
969 (0x0e00 << 16) | (0x3c218 >> 2),
971 (0x0e00 << 16) | (0x8904 >> 2),
974 (0x0e00 << 16) | (0x8c28 >> 2),
975 (0x0e00 << 16) | (0x8c2c >> 2),
976 (0x0e00 << 16) | (0x8c30 >> 2),
977 (0x0e00 << 16) | (0x8c34 >> 2),
978 (0x0e00 << 16) | (0x9600 >> 2),
981 static const u32 bonaire_golden_spm_registers[] =
983 0x30800, 0xe0ffffff, 0xe0000000
986 static const u32 bonaire_golden_common_registers[] =
988 0xc770, 0xffffffff, 0x00000800,
989 0xc774, 0xffffffff, 0x00000800,
990 0xc798, 0xffffffff, 0x00007fbf,
991 0xc79c, 0xffffffff, 0x00007faf
994 static const u32 bonaire_golden_registers[] =
996 0x3354, 0x00000333, 0x00000333,
997 0x3350, 0x000c0fc0, 0x00040200,
998 0x9a10, 0x00010000, 0x00058208,
999 0x3c000, 0xffff1fff, 0x00140000,
1000 0x3c200, 0xfdfc0fff, 0x00000100,
1001 0x3c234, 0x40000000, 0x40000200,
1002 0x9830, 0xffffffff, 0x00000000,
1003 0x9834, 0xf00fffff, 0x00000400,
1004 0x9838, 0x0002021c, 0x00020200,
1005 0xc78, 0x00000080, 0x00000000,
1006 0x5bb0, 0x000000f0, 0x00000070,
1007 0x5bc0, 0xf0311fff, 0x80300000,
1008 0x98f8, 0x73773777, 0x12010001,
1009 0x350c, 0x00810000, 0x408af000,
1010 0x7030, 0x31000111, 0x00000011,
1011 0x2f48, 0x73773777, 0x12010001,
1012 0x220c, 0x00007fb6, 0x0021a1b1,
1013 0x2210, 0x00007fb6, 0x002021b1,
1014 0x2180, 0x00007fb6, 0x00002191,
1015 0x2218, 0x00007fb6, 0x002121b1,
1016 0x221c, 0x00007fb6, 0x002021b1,
1017 0x21dc, 0x00007fb6, 0x00002191,
1018 0x21e0, 0x00007fb6, 0x00002191,
1019 0x3628, 0x0000003f, 0x0000000a,
1020 0x362c, 0x0000003f, 0x0000000a,
1021 0x2ae4, 0x00073ffe, 0x000022a2,
1022 0x240c, 0x000007ff, 0x00000000,
1023 0x8a14, 0xf000003f, 0x00000007,
1024 0x8bf0, 0x00002001, 0x00000001,
1025 0x8b24, 0xffffffff, 0x00ffffff,
1026 0x30a04, 0x0000ff0f, 0x00000000,
1027 0x28a4c, 0x07ffffff, 0x06000000,
1028 0x4d8, 0x00000fff, 0x00000100,
1029 0x3e78, 0x00000001, 0x00000002,
1030 0x9100, 0x03000000, 0x0362c688,
1031 0x8c00, 0x000000ff, 0x00000001,
1032 0xe40, 0x00001fff, 0x00001fff,
1033 0x9060, 0x0000007f, 0x00000020,
1034 0x9508, 0x00010000, 0x00010000,
1035 0xac14, 0x000003ff, 0x000000f3,
1036 0xac0c, 0xffffffff, 0x00001032
1039 static const u32 bonaire_mgcg_cgcg_init[] =
1041 0xc420, 0xffffffff, 0xfffffffc,
1042 0x30800, 0xffffffff, 0xe0000000,
1043 0x3c2a0, 0xffffffff, 0x00000100,
1044 0x3c208, 0xffffffff, 0x00000100,
1045 0x3c2c0, 0xffffffff, 0xc0000100,
1046 0x3c2c8, 0xffffffff, 0xc0000100,
1047 0x3c2c4, 0xffffffff, 0xc0000100,
1048 0x55e4, 0xffffffff, 0x00600100,
1049 0x3c280, 0xffffffff, 0x00000100,
1050 0x3c214, 0xffffffff, 0x06000100,
1051 0x3c220, 0xffffffff, 0x00000100,
1052 0x3c218, 0xffffffff, 0x06000100,
1053 0x3c204, 0xffffffff, 0x00000100,
1054 0x3c2e0, 0xffffffff, 0x00000100,
1055 0x3c224, 0xffffffff, 0x00000100,
1056 0x3c200, 0xffffffff, 0x00000100,
1057 0x3c230, 0xffffffff, 0x00000100,
1058 0x3c234, 0xffffffff, 0x00000100,
1059 0x3c250, 0xffffffff, 0x00000100,
1060 0x3c254, 0xffffffff, 0x00000100,
1061 0x3c258, 0xffffffff, 0x00000100,
1062 0x3c25c, 0xffffffff, 0x00000100,
1063 0x3c260, 0xffffffff, 0x00000100,
1064 0x3c27c, 0xffffffff, 0x00000100,
1065 0x3c278, 0xffffffff, 0x00000100,
1066 0x3c210, 0xffffffff, 0x06000100,
1067 0x3c290, 0xffffffff, 0x00000100,
1068 0x3c274, 0xffffffff, 0x00000100,
1069 0x3c2b4, 0xffffffff, 0x00000100,
1070 0x3c2b0, 0xffffffff, 0x00000100,
1071 0x3c270, 0xffffffff, 0x00000100,
1072 0x30800, 0xffffffff, 0xe0000000,
1073 0x3c020, 0xffffffff, 0x00010000,
1074 0x3c024, 0xffffffff, 0x00030002,
1075 0x3c028, 0xffffffff, 0x00040007,
1076 0x3c02c, 0xffffffff, 0x00060005,
1077 0x3c030, 0xffffffff, 0x00090008,
1078 0x3c034, 0xffffffff, 0x00010000,
1079 0x3c038, 0xffffffff, 0x00030002,
1080 0x3c03c, 0xffffffff, 0x00040007,
1081 0x3c040, 0xffffffff, 0x00060005,
1082 0x3c044, 0xffffffff, 0x00090008,
1083 0x3c048, 0xffffffff, 0x00010000,
1084 0x3c04c, 0xffffffff, 0x00030002,
1085 0x3c050, 0xffffffff, 0x00040007,
1086 0x3c054, 0xffffffff, 0x00060005,
1087 0x3c058, 0xffffffff, 0x00090008,
1088 0x3c05c, 0xffffffff, 0x00010000,
1089 0x3c060, 0xffffffff, 0x00030002,
1090 0x3c064, 0xffffffff, 0x00040007,
1091 0x3c068, 0xffffffff, 0x00060005,
1092 0x3c06c, 0xffffffff, 0x00090008,
1093 0x3c070, 0xffffffff, 0x00010000,
1094 0x3c074, 0xffffffff, 0x00030002,
1095 0x3c078, 0xffffffff, 0x00040007,
1096 0x3c07c, 0xffffffff, 0x00060005,
1097 0x3c080, 0xffffffff, 0x00090008,
1098 0x3c084, 0xffffffff, 0x00010000,
1099 0x3c088, 0xffffffff, 0x00030002,
1100 0x3c08c, 0xffffffff, 0x00040007,
1101 0x3c090, 0xffffffff, 0x00060005,
1102 0x3c094, 0xffffffff, 0x00090008,
1103 0x3c098, 0xffffffff, 0x00010000,
1104 0x3c09c, 0xffffffff, 0x00030002,
1105 0x3c0a0, 0xffffffff, 0x00040007,
1106 0x3c0a4, 0xffffffff, 0x00060005,
1107 0x3c0a8, 0xffffffff, 0x00090008,
1108 0x3c000, 0xffffffff, 0x96e00200,
1109 0x8708, 0xffffffff, 0x00900100,
1110 0xc424, 0xffffffff, 0x0020003f,
1111 0x38, 0xffffffff, 0x0140001c,
1112 0x3c, 0x000f0000, 0x000f0000,
1113 0x220, 0xffffffff, 0xC060000C,
1114 0x224, 0xc0000fff, 0x00000100,
1115 0xf90, 0xffffffff, 0x00000100,
1116 0xf98, 0x00000101, 0x00000000,
1117 0x20a8, 0xffffffff, 0x00000104,
1118 0x55e4, 0xff000fff, 0x00000100,
1119 0x30cc, 0xc0000fff, 0x00000104,
1120 0xc1e4, 0x00000001, 0x00000001,
1121 0xd00c, 0xff000ff0, 0x00000100,
1122 0xd80c, 0xff000ff0, 0x00000100
1125 static const u32 spectre_golden_spm_registers[] =
1127 0x30800, 0xe0ffffff, 0xe0000000
1130 static const u32 spectre_golden_common_registers[] =
1132 0xc770, 0xffffffff, 0x00000800,
1133 0xc774, 0xffffffff, 0x00000800,
1134 0xc798, 0xffffffff, 0x00007fbf,
1135 0xc79c, 0xffffffff, 0x00007faf
1138 static const u32 spectre_golden_registers[] =
1140 0x3c000, 0xffff1fff, 0x96940200,
1141 0x3c00c, 0xffff0001, 0xff000000,
1142 0x3c200, 0xfffc0fff, 0x00000100,
1143 0x6ed8, 0x00010101, 0x00010000,
1144 0x9834, 0xf00fffff, 0x00000400,
1145 0x9838, 0xfffffffc, 0x00020200,
1146 0x5bb0, 0x000000f0, 0x00000070,
1147 0x5bc0, 0xf0311fff, 0x80300000,
1148 0x98f8, 0x73773777, 0x12010001,
1149 0x9b7c, 0x00ff0000, 0x00fc0000,
1150 0x2f48, 0x73773777, 0x12010001,
1151 0x8a14, 0xf000003f, 0x00000007,
1152 0x8b24, 0xffffffff, 0x00ffffff,
1153 0x28350, 0x3f3f3fff, 0x00000082,
1154 0x28354, 0x0000003f, 0x00000000,
1155 0x3e78, 0x00000001, 0x00000002,
1156 0x913c, 0xffff03df, 0x00000004,
1157 0xc768, 0x00000008, 0x00000008,
1158 0x8c00, 0x000008ff, 0x00000800,
1159 0x9508, 0x00010000, 0x00010000,
1160 0xac0c, 0xffffffff, 0x54763210,
1161 0x214f8, 0x01ff01ff, 0x00000002,
1162 0x21498, 0x007ff800, 0x00200000,
1163 0x2015c, 0xffffffff, 0x00000f40,
1164 0x30934, 0xffffffff, 0x00000001
1167 static const u32 spectre_mgcg_cgcg_init[] =
1169 0xc420, 0xffffffff, 0xfffffffc,
1170 0x30800, 0xffffffff, 0xe0000000,
1171 0x3c2a0, 0xffffffff, 0x00000100,
1172 0x3c208, 0xffffffff, 0x00000100,
1173 0x3c2c0, 0xffffffff, 0x00000100,
1174 0x3c2c8, 0xffffffff, 0x00000100,
1175 0x3c2c4, 0xffffffff, 0x00000100,
1176 0x55e4, 0xffffffff, 0x00600100,
1177 0x3c280, 0xffffffff, 0x00000100,
1178 0x3c214, 0xffffffff, 0x06000100,
1179 0x3c220, 0xffffffff, 0x00000100,
1180 0x3c218, 0xffffffff, 0x06000100,
1181 0x3c204, 0xffffffff, 0x00000100,
1182 0x3c2e0, 0xffffffff, 0x00000100,
1183 0x3c224, 0xffffffff, 0x00000100,
1184 0x3c200, 0xffffffff, 0x00000100,
1185 0x3c230, 0xffffffff, 0x00000100,
1186 0x3c234, 0xffffffff, 0x00000100,
1187 0x3c250, 0xffffffff, 0x00000100,
1188 0x3c254, 0xffffffff, 0x00000100,
1189 0x3c258, 0xffffffff, 0x00000100,
1190 0x3c25c, 0xffffffff, 0x00000100,
1191 0x3c260, 0xffffffff, 0x00000100,
1192 0x3c27c, 0xffffffff, 0x00000100,
1193 0x3c278, 0xffffffff, 0x00000100,
1194 0x3c210, 0xffffffff, 0x06000100,
1195 0x3c290, 0xffffffff, 0x00000100,
1196 0x3c274, 0xffffffff, 0x00000100,
1197 0x3c2b4, 0xffffffff, 0x00000100,
1198 0x3c2b0, 0xffffffff, 0x00000100,
1199 0x3c270, 0xffffffff, 0x00000100,
1200 0x30800, 0xffffffff, 0xe0000000,
1201 0x3c020, 0xffffffff, 0x00010000,
1202 0x3c024, 0xffffffff, 0x00030002,
1203 0x3c028, 0xffffffff, 0x00040007,
1204 0x3c02c, 0xffffffff, 0x00060005,
1205 0x3c030, 0xffffffff, 0x00090008,
1206 0x3c034, 0xffffffff, 0x00010000,
1207 0x3c038, 0xffffffff, 0x00030002,
1208 0x3c03c, 0xffffffff, 0x00040007,
1209 0x3c040, 0xffffffff, 0x00060005,
1210 0x3c044, 0xffffffff, 0x00090008,
1211 0x3c048, 0xffffffff, 0x00010000,
1212 0x3c04c, 0xffffffff, 0x00030002,
1213 0x3c050, 0xffffffff, 0x00040007,
1214 0x3c054, 0xffffffff, 0x00060005,
1215 0x3c058, 0xffffffff, 0x00090008,
1216 0x3c05c, 0xffffffff, 0x00010000,
1217 0x3c060, 0xffffffff, 0x00030002,
1218 0x3c064, 0xffffffff, 0x00040007,
1219 0x3c068, 0xffffffff, 0x00060005,
1220 0x3c06c, 0xffffffff, 0x00090008,
1221 0x3c070, 0xffffffff, 0x00010000,
1222 0x3c074, 0xffffffff, 0x00030002,
1223 0x3c078, 0xffffffff, 0x00040007,
1224 0x3c07c, 0xffffffff, 0x00060005,
1225 0x3c080, 0xffffffff, 0x00090008,
1226 0x3c084, 0xffffffff, 0x00010000,
1227 0x3c088, 0xffffffff, 0x00030002,
1228 0x3c08c, 0xffffffff, 0x00040007,
1229 0x3c090, 0xffffffff, 0x00060005,
1230 0x3c094, 0xffffffff, 0x00090008,
1231 0x3c098, 0xffffffff, 0x00010000,
1232 0x3c09c, 0xffffffff, 0x00030002,
1233 0x3c0a0, 0xffffffff, 0x00040007,
1234 0x3c0a4, 0xffffffff, 0x00060005,
1235 0x3c0a8, 0xffffffff, 0x00090008,
1236 0x3c0ac, 0xffffffff, 0x00010000,
1237 0x3c0b0, 0xffffffff, 0x00030002,
1238 0x3c0b4, 0xffffffff, 0x00040007,
1239 0x3c0b8, 0xffffffff, 0x00060005,
1240 0x3c0bc, 0xffffffff, 0x00090008,
1241 0x3c000, 0xffffffff, 0x96e00200,
1242 0x8708, 0xffffffff, 0x00900100,
1243 0xc424, 0xffffffff, 0x0020003f,
1244 0x38, 0xffffffff, 0x0140001c,
1245 0x3c, 0x000f0000, 0x000f0000,
1246 0x220, 0xffffffff, 0xC060000C,
1247 0x224, 0xc0000fff, 0x00000100,
1248 0xf90, 0xffffffff, 0x00000100,
1249 0xf98, 0x00000101, 0x00000000,
1250 0x20a8, 0xffffffff, 0x00000104,
1251 0x55e4, 0xff000fff, 0x00000100,
1252 0x30cc, 0xc0000fff, 0x00000104,
1253 0xc1e4, 0x00000001, 0x00000001,
1254 0xd00c, 0xff000ff0, 0x00000100,
1255 0xd80c, 0xff000ff0, 0x00000100
1258 static const u32 kalindi_golden_spm_registers[] =
1260 0x30800, 0xe0ffffff, 0xe0000000
1263 static const u32 kalindi_golden_common_registers[] =
1265 0xc770, 0xffffffff, 0x00000800,
1266 0xc774, 0xffffffff, 0x00000800,
1267 0xc798, 0xffffffff, 0x00007fbf,
1268 0xc79c, 0xffffffff, 0x00007faf
1271 static const u32 kalindi_golden_registers[] =
1273 0x3c000, 0xffffdfff, 0x6e944040,
1274 0x55e4, 0xff607fff, 0xfc000100,
1275 0x3c220, 0xff000fff, 0x00000100,
1276 0x3c224, 0xff000fff, 0x00000100,
1277 0x3c200, 0xfffc0fff, 0x00000100,
1278 0x6ed8, 0x00010101, 0x00010000,
1279 0x9830, 0xffffffff, 0x00000000,
1280 0x9834, 0xf00fffff, 0x00000400,
1281 0x5bb0, 0x000000f0, 0x00000070,
1282 0x5bc0, 0xf0311fff, 0x80300000,
1283 0x98f8, 0x73773777, 0x12010001,
1284 0x98fc, 0xffffffff, 0x00000010,
1285 0x9b7c, 0x00ff0000, 0x00fc0000,
1286 0x8030, 0x00001f0f, 0x0000100a,
1287 0x2f48, 0x73773777, 0x12010001,
1288 0x2408, 0x000fffff, 0x000c007f,
1289 0x8a14, 0xf000003f, 0x00000007,
1290 0x8b24, 0x3fff3fff, 0x00ffcfff,
1291 0x30a04, 0x0000ff0f, 0x00000000,
1292 0x28a4c, 0x07ffffff, 0x06000000,
1293 0x4d8, 0x00000fff, 0x00000100,
1294 0x3e78, 0x00000001, 0x00000002,
1295 0xc768, 0x00000008, 0x00000008,
1296 0x8c00, 0x000000ff, 0x00000003,
1297 0x214f8, 0x01ff01ff, 0x00000002,
1298 0x21498, 0x007ff800, 0x00200000,
1299 0x2015c, 0xffffffff, 0x00000f40,
1300 0x88c4, 0x001f3ae3, 0x00000082,
1301 0x88d4, 0x0000001f, 0x00000010,
1302 0x30934, 0xffffffff, 0x00000000
1305 static const u32 kalindi_mgcg_cgcg_init[] =
1307 0xc420, 0xffffffff, 0xfffffffc,
1308 0x30800, 0xffffffff, 0xe0000000,
1309 0x3c2a0, 0xffffffff, 0x00000100,
1310 0x3c208, 0xffffffff, 0x00000100,
1311 0x3c2c0, 0xffffffff, 0x00000100,
1312 0x3c2c8, 0xffffffff, 0x00000100,
1313 0x3c2c4, 0xffffffff, 0x00000100,
1314 0x55e4, 0xffffffff, 0x00600100,
1315 0x3c280, 0xffffffff, 0x00000100,
1316 0x3c214, 0xffffffff, 0x06000100,
1317 0x3c220, 0xffffffff, 0x00000100,
1318 0x3c218, 0xffffffff, 0x06000100,
1319 0x3c204, 0xffffffff, 0x00000100,
1320 0x3c2e0, 0xffffffff, 0x00000100,
1321 0x3c224, 0xffffffff, 0x00000100,
1322 0x3c200, 0xffffffff, 0x00000100,
1323 0x3c230, 0xffffffff, 0x00000100,
1324 0x3c234, 0xffffffff, 0x00000100,
1325 0x3c250, 0xffffffff, 0x00000100,
1326 0x3c254, 0xffffffff, 0x00000100,
1327 0x3c258, 0xffffffff, 0x00000100,
1328 0x3c25c, 0xffffffff, 0x00000100,
1329 0x3c260, 0xffffffff, 0x00000100,
1330 0x3c27c, 0xffffffff, 0x00000100,
1331 0x3c278, 0xffffffff, 0x00000100,
1332 0x3c210, 0xffffffff, 0x06000100,
1333 0x3c290, 0xffffffff, 0x00000100,
1334 0x3c274, 0xffffffff, 0x00000100,
1335 0x3c2b4, 0xffffffff, 0x00000100,
1336 0x3c2b0, 0xffffffff, 0x00000100,
1337 0x3c270, 0xffffffff, 0x00000100,
1338 0x30800, 0xffffffff, 0xe0000000,
1339 0x3c020, 0xffffffff, 0x00010000,
1340 0x3c024, 0xffffffff, 0x00030002,
1341 0x3c028, 0xffffffff, 0x00040007,
1342 0x3c02c, 0xffffffff, 0x00060005,
1343 0x3c030, 0xffffffff, 0x00090008,
1344 0x3c034, 0xffffffff, 0x00010000,
1345 0x3c038, 0xffffffff, 0x00030002,
1346 0x3c03c, 0xffffffff, 0x00040007,
1347 0x3c040, 0xffffffff, 0x00060005,
1348 0x3c044, 0xffffffff, 0x00090008,
1349 0x3c000, 0xffffffff, 0x96e00200,
1350 0x8708, 0xffffffff, 0x00900100,
1351 0xc424, 0xffffffff, 0x0020003f,
1352 0x38, 0xffffffff, 0x0140001c,
1353 0x3c, 0x000f0000, 0x000f0000,
1354 0x220, 0xffffffff, 0xC060000C,
1355 0x224, 0xc0000fff, 0x00000100,
1356 0x20a8, 0xffffffff, 0x00000104,
1357 0x55e4, 0xff000fff, 0x00000100,
1358 0x30cc, 0xc0000fff, 0x00000104,
1359 0xc1e4, 0x00000001, 0x00000001,
1360 0xd00c, 0xff000ff0, 0x00000100,
1361 0xd80c, 0xff000ff0, 0x00000100
1364 static const u32 hawaii_golden_spm_registers[] =
1366 0x30800, 0xe0ffffff, 0xe0000000
1369 static const u32 hawaii_golden_common_registers[] =
1371 0x30800, 0xffffffff, 0xe0000000,
1372 0x28350, 0xffffffff, 0x3a00161a,
1373 0x28354, 0xffffffff, 0x0000002e,
1374 0x9a10, 0xffffffff, 0x00018208,
1375 0x98f8, 0xffffffff, 0x12011003
1378 static const u32 hawaii_golden_registers[] =
1380 0x3354, 0x00000333, 0x00000333,
1381 0x9a10, 0x00010000, 0x00058208,
1382 0x9830, 0xffffffff, 0x00000000,
1383 0x9834, 0xf00fffff, 0x00000400,
1384 0x9838, 0x0002021c, 0x00020200,
1385 0xc78, 0x00000080, 0x00000000,
1386 0x5bb0, 0x000000f0, 0x00000070,
1387 0x5bc0, 0xf0311fff, 0x80300000,
1388 0x350c, 0x00810000, 0x408af000,
1389 0x7030, 0x31000111, 0x00000011,
1390 0x2f48, 0x73773777, 0x12010001,
1391 0x2120, 0x0000007f, 0x0000001b,
1392 0x21dc, 0x00007fb6, 0x00002191,
1393 0x3628, 0x0000003f, 0x0000000a,
1394 0x362c, 0x0000003f, 0x0000000a,
1395 0x2ae4, 0x00073ffe, 0x000022a2,
1396 0x240c, 0x000007ff, 0x00000000,
1397 0x8bf0, 0x00002001, 0x00000001,
1398 0x8b24, 0xffffffff, 0x00ffffff,
1399 0x30a04, 0x0000ff0f, 0x00000000,
1400 0x28a4c, 0x07ffffff, 0x06000000,
1401 0x3e78, 0x00000001, 0x00000002,
1402 0xc768, 0x00000008, 0x00000008,
1403 0xc770, 0x00000f00, 0x00000800,
1404 0xc774, 0x00000f00, 0x00000800,
1405 0xc798, 0x00ffffff, 0x00ff7fbf,
1406 0xc79c, 0x00ffffff, 0x00ff7faf,
1407 0x8c00, 0x000000ff, 0x00000800,
1408 0xe40, 0x00001fff, 0x00001fff,
1409 0x9060, 0x0000007f, 0x00000020,
1410 0x9508, 0x00010000, 0x00010000,
1411 0xae00, 0x00100000, 0x000ff07c,
1412 0xac14, 0x000003ff, 0x0000000f,
1413 0xac10, 0xffffffff, 0x7564fdec,
1414 0xac0c, 0xffffffff, 0x3120b9a8,
1415 0xac08, 0x20000000, 0x0f9c0000
1418 static const u32 hawaii_mgcg_cgcg_init[] =
1420 0xc420, 0xffffffff, 0xfffffffd,
1421 0x30800, 0xffffffff, 0xe0000000,
1422 0x3c2a0, 0xffffffff, 0x00000100,
1423 0x3c208, 0xffffffff, 0x00000100,
1424 0x3c2c0, 0xffffffff, 0x00000100,
1425 0x3c2c8, 0xffffffff, 0x00000100,
1426 0x3c2c4, 0xffffffff, 0x00000100,
1427 0x55e4, 0xffffffff, 0x00200100,
1428 0x3c280, 0xffffffff, 0x00000100,
1429 0x3c214, 0xffffffff, 0x06000100,
1430 0x3c220, 0xffffffff, 0x00000100,
1431 0x3c218, 0xffffffff, 0x06000100,
1432 0x3c204, 0xffffffff, 0x00000100,
1433 0x3c2e0, 0xffffffff, 0x00000100,
1434 0x3c224, 0xffffffff, 0x00000100,
1435 0x3c200, 0xffffffff, 0x00000100,
1436 0x3c230, 0xffffffff, 0x00000100,
1437 0x3c234, 0xffffffff, 0x00000100,
1438 0x3c250, 0xffffffff, 0x00000100,
1439 0x3c254, 0xffffffff, 0x00000100,
1440 0x3c258, 0xffffffff, 0x00000100,
1441 0x3c25c, 0xffffffff, 0x00000100,
1442 0x3c260, 0xffffffff, 0x00000100,
1443 0x3c27c, 0xffffffff, 0x00000100,
1444 0x3c278, 0xffffffff, 0x00000100,
1445 0x3c210, 0xffffffff, 0x06000100,
1446 0x3c290, 0xffffffff, 0x00000100,
1447 0x3c274, 0xffffffff, 0x00000100,
1448 0x3c2b4, 0xffffffff, 0x00000100,
1449 0x3c2b0, 0xffffffff, 0x00000100,
1450 0x3c270, 0xffffffff, 0x00000100,
1451 0x30800, 0xffffffff, 0xe0000000,
1452 0x3c020, 0xffffffff, 0x00010000,
1453 0x3c024, 0xffffffff, 0x00030002,
1454 0x3c028, 0xffffffff, 0x00040007,
1455 0x3c02c, 0xffffffff, 0x00060005,
1456 0x3c030, 0xffffffff, 0x00090008,
1457 0x3c034, 0xffffffff, 0x00010000,
1458 0x3c038, 0xffffffff, 0x00030002,
1459 0x3c03c, 0xffffffff, 0x00040007,
1460 0x3c040, 0xffffffff, 0x00060005,
1461 0x3c044, 0xffffffff, 0x00090008,
1462 0x3c048, 0xffffffff, 0x00010000,
1463 0x3c04c, 0xffffffff, 0x00030002,
1464 0x3c050, 0xffffffff, 0x00040007,
1465 0x3c054, 0xffffffff, 0x00060005,
1466 0x3c058, 0xffffffff, 0x00090008,
1467 0x3c05c, 0xffffffff, 0x00010000,
1468 0x3c060, 0xffffffff, 0x00030002,
1469 0x3c064, 0xffffffff, 0x00040007,
1470 0x3c068, 0xffffffff, 0x00060005,
1471 0x3c06c, 0xffffffff, 0x00090008,
1472 0x3c070, 0xffffffff, 0x00010000,
1473 0x3c074, 0xffffffff, 0x00030002,
1474 0x3c078, 0xffffffff, 0x00040007,
1475 0x3c07c, 0xffffffff, 0x00060005,
1476 0x3c080, 0xffffffff, 0x00090008,
1477 0x3c084, 0xffffffff, 0x00010000,
1478 0x3c088, 0xffffffff, 0x00030002,
1479 0x3c08c, 0xffffffff, 0x00040007,
1480 0x3c090, 0xffffffff, 0x00060005,
1481 0x3c094, 0xffffffff, 0x00090008,
1482 0x3c098, 0xffffffff, 0x00010000,
1483 0x3c09c, 0xffffffff, 0x00030002,
1484 0x3c0a0, 0xffffffff, 0x00040007,
1485 0x3c0a4, 0xffffffff, 0x00060005,
1486 0x3c0a8, 0xffffffff, 0x00090008,
1487 0x3c0ac, 0xffffffff, 0x00010000,
1488 0x3c0b0, 0xffffffff, 0x00030002,
1489 0x3c0b4, 0xffffffff, 0x00040007,
1490 0x3c0b8, 0xffffffff, 0x00060005,
1491 0x3c0bc, 0xffffffff, 0x00090008,
1492 0x3c0c0, 0xffffffff, 0x00010000,
1493 0x3c0c4, 0xffffffff, 0x00030002,
1494 0x3c0c8, 0xffffffff, 0x00040007,
1495 0x3c0cc, 0xffffffff, 0x00060005,
1496 0x3c0d0, 0xffffffff, 0x00090008,
1497 0x3c0d4, 0xffffffff, 0x00010000,
1498 0x3c0d8, 0xffffffff, 0x00030002,
1499 0x3c0dc, 0xffffffff, 0x00040007,
1500 0x3c0e0, 0xffffffff, 0x00060005,
1501 0x3c0e4, 0xffffffff, 0x00090008,
1502 0x3c0e8, 0xffffffff, 0x00010000,
1503 0x3c0ec, 0xffffffff, 0x00030002,
1504 0x3c0f0, 0xffffffff, 0x00040007,
1505 0x3c0f4, 0xffffffff, 0x00060005,
1506 0x3c0f8, 0xffffffff, 0x00090008,
1507 0xc318, 0xffffffff, 0x00020200,
1508 0x3350, 0xffffffff, 0x00000200,
1509 0x15c0, 0xffffffff, 0x00000400,
1510 0x55e8, 0xffffffff, 0x00000000,
1511 0x2f50, 0xffffffff, 0x00000902,
1512 0x3c000, 0xffffffff, 0x96940200,
1513 0x8708, 0xffffffff, 0x00900100,
1514 0xc424, 0xffffffff, 0x0020003f,
1515 0x38, 0xffffffff, 0x0140001c,
1516 0x3c, 0x000f0000, 0x000f0000,
1517 0x220, 0xffffffff, 0xc060000c,
1518 0x224, 0xc0000fff, 0x00000100,
1519 0xf90, 0xffffffff, 0x00000100,
1520 0xf98, 0x00000101, 0x00000000,
1521 0x20a8, 0xffffffff, 0x00000104,
1522 0x55e4, 0xff000fff, 0x00000100,
1523 0x30cc, 0xc0000fff, 0x00000104,
1524 0xc1e4, 0x00000001, 0x00000001,
1525 0xd00c, 0xff000ff0, 0x00000100,
1526 0xd80c, 0xff000ff0, 0x00000100
1529 static const u32 godavari_golden_registers[] =
1531 0x55e4, 0xff607fff, 0xfc000100,
1532 0x6ed8, 0x00010101, 0x00010000,
1533 0x9830, 0xffffffff, 0x00000000,
1534 0x98302, 0xf00fffff, 0x00000400,
1535 0x6130, 0xffffffff, 0x00010000,
1536 0x5bb0, 0x000000f0, 0x00000070,
1537 0x5bc0, 0xf0311fff, 0x80300000,
1538 0x98f8, 0x73773777, 0x12010001,
1539 0x98fc, 0xffffffff, 0x00000010,
1540 0x8030, 0x00001f0f, 0x0000100a,
1541 0x2f48, 0x73773777, 0x12010001,
1542 0x2408, 0x000fffff, 0x000c007f,
1543 0x8a14, 0xf000003f, 0x00000007,
1544 0x8b24, 0xffffffff, 0x00ff0fff,
1545 0x30a04, 0x0000ff0f, 0x00000000,
1546 0x28a4c, 0x07ffffff, 0x06000000,
1547 0x4d8, 0x00000fff, 0x00000100,
1548 0xd014, 0x00010000, 0x00810001,
1549 0xd814, 0x00010000, 0x00810001,
1550 0x3e78, 0x00000001, 0x00000002,
1551 0xc768, 0x00000008, 0x00000008,
1552 0xc770, 0x00000f00, 0x00000800,
1553 0xc774, 0x00000f00, 0x00000800,
1554 0xc798, 0x00ffffff, 0x00ff7fbf,
1555 0xc79c, 0x00ffffff, 0x00ff7faf,
1556 0x8c00, 0x000000ff, 0x00000001,
1557 0x214f8, 0x01ff01ff, 0x00000002,
1558 0x21498, 0x007ff800, 0x00200000,
1559 0x2015c, 0xffffffff, 0x00000f40,
1560 0x88c4, 0x001f3ae3, 0x00000082,
1561 0x88d4, 0x0000001f, 0x00000010,
1562 0x30934, 0xffffffff, 0x00000000
1566 static void cik_init_golden_registers(struct radeon_device *rdev)
1568 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1569 mutex_lock(&rdev->grbm_idx_mutex);
1570 switch (rdev->family) {
1572 radeon_program_register_sequence(rdev,
1573 bonaire_mgcg_cgcg_init,
1574 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1575 radeon_program_register_sequence(rdev,
1576 bonaire_golden_registers,
1577 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1578 radeon_program_register_sequence(rdev,
1579 bonaire_golden_common_registers,
1580 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1581 radeon_program_register_sequence(rdev,
1582 bonaire_golden_spm_registers,
1583 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1586 radeon_program_register_sequence(rdev,
1587 kalindi_mgcg_cgcg_init,
1588 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1589 radeon_program_register_sequence(rdev,
1590 kalindi_golden_registers,
1591 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1592 radeon_program_register_sequence(rdev,
1593 kalindi_golden_common_registers,
1594 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1595 radeon_program_register_sequence(rdev,
1596 kalindi_golden_spm_registers,
1597 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1600 radeon_program_register_sequence(rdev,
1601 kalindi_mgcg_cgcg_init,
1602 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1603 radeon_program_register_sequence(rdev,
1604 godavari_golden_registers,
1605 (const u32)ARRAY_SIZE(godavari_golden_registers));
1606 radeon_program_register_sequence(rdev,
1607 kalindi_golden_common_registers,
1608 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1609 radeon_program_register_sequence(rdev,
1610 kalindi_golden_spm_registers,
1611 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1614 radeon_program_register_sequence(rdev,
1615 spectre_mgcg_cgcg_init,
1616 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1617 radeon_program_register_sequence(rdev,
1618 spectre_golden_registers,
1619 (const u32)ARRAY_SIZE(spectre_golden_registers));
1620 radeon_program_register_sequence(rdev,
1621 spectre_golden_common_registers,
1622 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1623 radeon_program_register_sequence(rdev,
1624 spectre_golden_spm_registers,
1625 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1628 radeon_program_register_sequence(rdev,
1629 hawaii_mgcg_cgcg_init,
1630 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1631 radeon_program_register_sequence(rdev,
1632 hawaii_golden_registers,
1633 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1634 radeon_program_register_sequence(rdev,
1635 hawaii_golden_common_registers,
1636 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1637 radeon_program_register_sequence(rdev,
1638 hawaii_golden_spm_registers,
1639 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1644 mutex_unlock(&rdev->grbm_idx_mutex);
1648 * cik_get_xclk - get the xclk
1650 * @rdev: radeon_device pointer
1652 * Returns the reference clock used by the gfx engine
1655 u32 cik_get_xclk(struct radeon_device *rdev)
1657 u32 reference_clock = rdev->clock.spll.reference_freq;
1659 if (rdev->flags & RADEON_IS_IGP) {
1660 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1661 return reference_clock / 2;
1663 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1664 return reference_clock / 4;
1666 return reference_clock;
1670 * cik_mm_rdoorbell - read a doorbell dword
1672 * @rdev: radeon_device pointer
1673 * @index: doorbell index
1675 * Returns the value in the doorbell aperture at the
1676 * requested doorbell index (CIK).
1678 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1680 if (index < rdev->doorbell.num_doorbells) {
1681 return readl(rdev->doorbell.ptr + index);
1683 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1689 * cik_mm_wdoorbell - write a doorbell dword
1691 * @rdev: radeon_device pointer
1692 * @index: doorbell index
1693 * @v: value to write
1695 * Writes @v to the doorbell aperture at the
1696 * requested doorbell index (CIK).
1698 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1700 if (index < rdev->doorbell.num_doorbells) {
1701 writel(v, rdev->doorbell.ptr + index);
1703 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1707 #define BONAIRE_IO_MC_REGS_SIZE 36
1709 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1711 {0x00000070, 0x04400000},
1712 {0x00000071, 0x80c01803},
1713 {0x00000072, 0x00004004},
1714 {0x00000073, 0x00000100},
1715 {0x00000074, 0x00ff0000},
1716 {0x00000075, 0x34000000},
1717 {0x00000076, 0x08000014},
1718 {0x00000077, 0x00cc08ec},
1719 {0x00000078, 0x00000400},
1720 {0x00000079, 0x00000000},
1721 {0x0000007a, 0x04090000},
1722 {0x0000007c, 0x00000000},
1723 {0x0000007e, 0x4408a8e8},
1724 {0x0000007f, 0x00000304},
1725 {0x00000080, 0x00000000},
1726 {0x00000082, 0x00000001},
1727 {0x00000083, 0x00000002},
1728 {0x00000084, 0xf3e4f400},
1729 {0x00000085, 0x052024e3},
1730 {0x00000087, 0x00000000},
1731 {0x00000088, 0x01000000},
1732 {0x0000008a, 0x1c0a0000},
1733 {0x0000008b, 0xff010000},
1734 {0x0000008d, 0xffffefff},
1735 {0x0000008e, 0xfff3efff},
1736 {0x0000008f, 0xfff3efbf},
1737 {0x00000092, 0xf7ffffff},
1738 {0x00000093, 0xffffff7f},
1739 {0x00000095, 0x00101101},
1740 {0x00000096, 0x00000fff},
1741 {0x00000097, 0x00116fff},
1742 {0x00000098, 0x60010000},
1743 {0x00000099, 0x10010000},
1744 {0x0000009a, 0x00006000},
1745 {0x0000009b, 0x00001000},
1746 {0x0000009f, 0x00b48000}
1749 #define HAWAII_IO_MC_REGS_SIZE 22
1751 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1753 {0x0000007d, 0x40000000},
1754 {0x0000007e, 0x40180304},
1755 {0x0000007f, 0x0000ff00},
1756 {0x00000081, 0x00000000},
1757 {0x00000083, 0x00000800},
1758 {0x00000086, 0x00000000},
1759 {0x00000087, 0x00000100},
1760 {0x00000088, 0x00020100},
1761 {0x00000089, 0x00000000},
1762 {0x0000008b, 0x00040000},
1763 {0x0000008c, 0x00000100},
1764 {0x0000008e, 0xff010000},
1765 {0x00000090, 0xffffefff},
1766 {0x00000091, 0xfff3efff},
1767 {0x00000092, 0xfff3efbf},
1768 {0x00000093, 0xf7ffffff},
1769 {0x00000094, 0xffffff7f},
1770 {0x00000095, 0x00000fff},
1771 {0x00000096, 0x00116fff},
1772 {0x00000097, 0x60010000},
1773 {0x00000098, 0x10010000},
1774 {0x0000009f, 0x00c79000}
1779 * cik_srbm_select - select specific register instances
1781 * @rdev: radeon_device pointer
1782 * @me: selected ME (micro engine)
1787 * Switches the currently active registers instances. Some
1788 * registers are instanced per VMID, others are instanced per
1789 * me/pipe/queue combination.
1791 static void cik_srbm_select(struct radeon_device *rdev,
1792 u32 me, u32 pipe, u32 queue, u32 vmid)
1794 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1797 QUEUEID(queue & 0x7));
1798 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1803 * ci_mc_load_microcode - load MC ucode into the hw
1805 * @rdev: radeon_device pointer
1807 * Load the GDDR MC ucode into the hw (CIK).
1808 * Returns 0 on success, error on failure.
1810 int ci_mc_load_microcode(struct radeon_device *rdev)
1812 const __be32 *fw_data = NULL;
1813 const __le32 *new_fw_data = NULL;
1814 u32 running, blackout = 0, tmp;
1815 u32 *io_mc_regs = NULL;
1816 const __le32 *new_io_mc_regs = NULL;
1817 int i, regs_size, ucode_size;
1823 const struct mc_firmware_header_v1_0 *hdr =
1824 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1826 radeon_ucode_print_mc_hdr(&hdr->header);
1828 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1829 new_io_mc_regs = (const __le32 *)
1830 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1831 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1832 new_fw_data = (const __le32 *)
1833 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1835 ucode_size = rdev->mc_fw->size / 4;
1837 switch (rdev->family) {
1839 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1840 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1843 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1844 regs_size = HAWAII_IO_MC_REGS_SIZE;
1849 fw_data = (const __be32 *)rdev->mc_fw->data;
1852 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1856 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1857 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1860 /* reset the engine and set to writable */
1861 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1862 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1864 /* load mc io regs */
1865 for (i = 0; i < regs_size; i++) {
1867 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1868 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1870 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1871 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1875 tmp = RREG32(MC_SEQ_MISC0);
1876 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1877 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1878 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1879 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1880 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1883 /* load the MC ucode */
1884 for (i = 0; i < ucode_size; i++) {
1886 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1888 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1891 /* put the engine back into the active state */
1892 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1893 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1894 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1896 /* wait for training to complete */
1897 for (i = 0; i < rdev->usec_timeout; i++) {
1898 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1902 for (i = 0; i < rdev->usec_timeout; i++) {
1903 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1909 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1916 * cik_init_microcode - load ucode images from disk
1918 * @rdev: radeon_device pointer
1920 * Use the firmware interface to load the ucode images into
1921 * the driver (not loaded into hw).
1922 * Returns 0 on success, error on failure.
1924 static int cik_init_microcode(struct radeon_device *rdev)
1926 const char *chip_name;
1927 const char *new_chip_name;
1928 size_t pfp_req_size, me_req_size, ce_req_size,
1929 mec_req_size, rlc_req_size, mc_req_size = 0,
1930 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1938 switch (rdev->family) {
1940 chip_name = "BONAIRE";
1941 new_chip_name = "bonaire";
1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1948 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1954 chip_name = "HAWAII";
1955 new_chip_name = "hawaii";
1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1961 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1962 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1963 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1964 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1968 chip_name = "KAVERI";
1969 new_chip_name = "kaveri";
1970 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1971 me_req_size = CIK_ME_UCODE_SIZE * 4;
1972 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1973 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1974 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1975 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1979 chip_name = "KABINI";
1980 new_chip_name = "kabini";
1981 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1982 me_req_size = CIK_ME_UCODE_SIZE * 4;
1983 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1984 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1985 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1986 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1990 chip_name = "MULLINS";
1991 new_chip_name = "mullins";
1992 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1993 me_req_size = CIK_ME_UCODE_SIZE * 4;
1994 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1995 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1996 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1997 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2003 DRM_INFO("Loading %s Microcode\n", new_chip_name);
2005 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2006 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2008 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2009 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2012 if (rdev->pfp_fw->size != pfp_req_size) {
2014 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2015 rdev->pfp_fw->size, fw_name);
2020 err = radeon_ucode_validate(rdev->pfp_fw);
2023 "cik_fw: validation failed for firmware \"%s\"\n",
2031 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2032 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2034 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2035 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2038 if (rdev->me_fw->size != me_req_size) {
2040 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2041 rdev->me_fw->size, fw_name);
2045 err = radeon_ucode_validate(rdev->me_fw);
2048 "cik_fw: validation failed for firmware \"%s\"\n",
2056 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2057 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2059 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2060 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2063 if (rdev->ce_fw->size != ce_req_size) {
2065 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2066 rdev->ce_fw->size, fw_name);
2070 err = radeon_ucode_validate(rdev->ce_fw);
2073 "cik_fw: validation failed for firmware \"%s\"\n",
2081 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2082 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2084 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2085 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2088 if (rdev->mec_fw->size != mec_req_size) {
2090 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2091 rdev->mec_fw->size, fw_name);
2095 err = radeon_ucode_validate(rdev->mec_fw);
2098 "cik_fw: validation failed for firmware \"%s\"\n",
2106 if (rdev->family == CHIP_KAVERI) {
2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2108 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2112 err = radeon_ucode_validate(rdev->mec2_fw);
2121 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2122 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2124 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2125 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2128 if (rdev->rlc_fw->size != rlc_req_size) {
2130 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2131 rdev->rlc_fw->size, fw_name);
2135 err = radeon_ucode_validate(rdev->rlc_fw);
2138 "cik_fw: validation failed for firmware \"%s\"\n",
2146 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2147 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2149 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2150 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2153 if (rdev->sdma_fw->size != sdma_req_size) {
2155 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2156 rdev->sdma_fw->size, fw_name);
2160 err = radeon_ucode_validate(rdev->sdma_fw);
2163 "cik_fw: validation failed for firmware \"%s\"\n",
2171 /* No SMC, MC ucode on APUs */
2172 if (!(rdev->flags & RADEON_IS_IGP)) {
2173 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2174 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2176 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2177 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2179 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2180 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2184 if ((rdev->mc_fw->size != mc_req_size) &&
2185 (rdev->mc_fw->size != mc2_req_size)){
2187 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2188 rdev->mc_fw->size, fw_name);
2191 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2193 err = radeon_ucode_validate(rdev->mc_fw);
2196 "cik_fw: validation failed for firmware \"%s\"\n",
2204 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2205 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2207 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2208 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2211 "smc: error loading firmware \"%s\"\n",
2213 release_firmware(rdev->smc_fw);
2214 rdev->smc_fw = NULL;
2216 } else if (rdev->smc_fw->size != smc_req_size) {
2218 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2219 rdev->smc_fw->size, fw_name);
2223 err = radeon_ucode_validate(rdev->smc_fw);
2226 "cik_fw: validation failed for firmware \"%s\"\n",
2236 rdev->new_fw = false;
2237 } else if (new_fw < num_fw) {
2238 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2241 rdev->new_fw = true;
2248 "cik_cp: Failed to load firmware \"%s\"\n",
2250 release_firmware(rdev->pfp_fw);
2251 rdev->pfp_fw = NULL;
2252 release_firmware(rdev->me_fw);
2254 release_firmware(rdev->ce_fw);
2256 release_firmware(rdev->mec_fw);
2257 rdev->mec_fw = NULL;
2258 release_firmware(rdev->mec2_fw);
2259 rdev->mec2_fw = NULL;
2260 release_firmware(rdev->rlc_fw);
2261 rdev->rlc_fw = NULL;
2262 release_firmware(rdev->sdma_fw);
2263 rdev->sdma_fw = NULL;
2264 release_firmware(rdev->mc_fw);
2266 release_firmware(rdev->smc_fw);
2267 rdev->smc_fw = NULL;
2276 * cik_tiling_mode_table_init - init the hw tiling table
2278 * @rdev: radeon_device pointer
2280 * Starting with SI, the tiling setup is done globally in a
2281 * set of 32 tiling modes. Rather than selecting each set of
2282 * parameters per surface as on older asics, we just select
2283 * which index in the tiling table we want to use, and the
2284 * surface uses those parameters (CIK).
2286 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2288 const u32 num_tile_mode_states = 32;
2289 const u32 num_secondary_tile_mode_states = 16;
2290 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2291 u32 num_pipe_configs;
2292 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2293 rdev->config.cik.max_shader_engines;
2295 switch (rdev->config.cik.mem_row_size_in_kb) {
2297 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2301 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2304 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2308 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2309 if (num_pipe_configs > 8)
2310 num_pipe_configs = 16;
2312 if (num_pipe_configs == 16) {
2313 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2314 switch (reg_offset) {
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2319 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2325 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2334 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2335 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2336 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2337 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2341 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2342 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2343 TILE_SPLIT(split_equal_to_row_size));
2346 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2347 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2348 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2357 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2359 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2360 TILE_SPLIT(split_equal_to_row_size));
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2364 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2369 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2372 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2373 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2374 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2375 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2378 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2379 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2381 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2384 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2385 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2386 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2387 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2390 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2391 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2392 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2395 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2396 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2397 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2398 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2402 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2404 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2407 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2408 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2409 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2410 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2414 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2419 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2420 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2421 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2424 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2425 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2427 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2439 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2440 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2442 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2443 switch (reg_offset) {
2445 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2448 NUM_BANKS(ADDR_SURF_16_BANK));
2451 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2452 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2453 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2454 NUM_BANKS(ADDR_SURF_16_BANK));
2457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2460 NUM_BANKS(ADDR_SURF_16_BANK));
2463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2466 NUM_BANKS(ADDR_SURF_16_BANK));
2469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2472 NUM_BANKS(ADDR_SURF_8_BANK));
2475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2478 NUM_BANKS(ADDR_SURF_4_BANK));
2481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2484 NUM_BANKS(ADDR_SURF_2_BANK));
2487 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2490 NUM_BANKS(ADDR_SURF_16_BANK));
2493 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2496 NUM_BANKS(ADDR_SURF_16_BANK));
2499 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2502 NUM_BANKS(ADDR_SURF_16_BANK));
2505 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2508 NUM_BANKS(ADDR_SURF_8_BANK));
2511 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2514 NUM_BANKS(ADDR_SURF_4_BANK));
2517 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2520 NUM_BANKS(ADDR_SURF_2_BANK));
2523 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2526 NUM_BANKS(ADDR_SURF_2_BANK));
2532 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2533 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2535 } else if (num_pipe_configs == 8) {
2536 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2537 switch (reg_offset) {
2539 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2545 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2548 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2551 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2552 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2553 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2554 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2557 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2558 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2559 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2560 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2566 TILE_SPLIT(split_equal_to_row_size));
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2574 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2575 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2576 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2577 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2580 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2581 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2583 TILE_SPLIT(split_equal_to_row_size));
2586 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2587 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2590 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2592 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2595 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2598 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2601 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2602 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2603 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2604 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2607 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2608 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2610 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2615 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2638 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2643 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2647 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2648 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2649 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2662 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2663 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2665 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2666 switch (reg_offset) {
2668 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2671 NUM_BANKS(ADDR_SURF_16_BANK));
2674 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2675 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2676 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2677 NUM_BANKS(ADDR_SURF_16_BANK));
2680 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2683 NUM_BANKS(ADDR_SURF_16_BANK));
2686 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2689 NUM_BANKS(ADDR_SURF_16_BANK));
2692 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2693 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2694 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2695 NUM_BANKS(ADDR_SURF_8_BANK));
2698 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2701 NUM_BANKS(ADDR_SURF_4_BANK));
2704 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2705 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2706 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2707 NUM_BANKS(ADDR_SURF_2_BANK));
2710 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2713 NUM_BANKS(ADDR_SURF_16_BANK));
2716 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2719 NUM_BANKS(ADDR_SURF_16_BANK));
2722 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2723 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2724 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2725 NUM_BANKS(ADDR_SURF_16_BANK));
2728 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2731 NUM_BANKS(ADDR_SURF_16_BANK));
2734 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2737 NUM_BANKS(ADDR_SURF_8_BANK));
2740 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2741 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2742 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2743 NUM_BANKS(ADDR_SURF_4_BANK));
2746 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2749 NUM_BANKS(ADDR_SURF_2_BANK));
2755 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2756 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2758 } else if (num_pipe_configs == 4) {
2760 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2761 switch (reg_offset) {
2763 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2765 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2766 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2771 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2775 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2777 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2778 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2783 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2790 TILE_SPLIT(split_equal_to_row_size));
2793 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2794 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2798 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2799 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2800 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2801 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2804 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2805 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2806 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2807 TILE_SPLIT(split_equal_to_row_size));
2810 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2811 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2815 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2822 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2825 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2827 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2828 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2834 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2838 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2839 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2842 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2843 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2844 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2845 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2848 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2849 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2850 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2851 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2854 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2855 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2856 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2857 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2860 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2861 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2862 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2879 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2886 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2887 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2889 } else if (num_rbs < 4) {
2890 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2891 switch (reg_offset) {
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2896 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2899 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2900 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2901 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2902 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2905 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2906 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2907 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2908 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2911 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2912 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2913 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2914 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2917 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2919 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2920 TILE_SPLIT(split_equal_to_row_size));
2923 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2924 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2925 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2928 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2929 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2930 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2931 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2934 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2935 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2936 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2937 TILE_SPLIT(split_equal_to_row_size));
2940 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2941 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2944 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2945 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2946 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2949 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2950 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2951 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2952 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2955 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2956 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2957 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2958 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2961 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2962 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2963 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2964 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2967 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2968 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2969 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2972 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2973 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2974 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2975 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2978 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2979 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2980 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2981 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2984 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2985 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2986 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2987 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2990 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2991 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2992 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2995 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2996 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2997 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2998 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3001 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3002 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3003 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3004 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3007 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3008 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3009 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3010 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3016 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3017 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3020 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3021 switch (reg_offset) {
3023 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3026 NUM_BANKS(ADDR_SURF_16_BANK));
3029 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3030 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3031 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3032 NUM_BANKS(ADDR_SURF_16_BANK));
3035 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3038 NUM_BANKS(ADDR_SURF_16_BANK));
3041 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3042 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3043 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3044 NUM_BANKS(ADDR_SURF_16_BANK));
3047 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3048 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3049 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3050 NUM_BANKS(ADDR_SURF_16_BANK));
3053 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3054 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3055 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3056 NUM_BANKS(ADDR_SURF_8_BANK));
3059 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3060 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3061 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3062 NUM_BANKS(ADDR_SURF_4_BANK));
3065 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3066 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3067 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3068 NUM_BANKS(ADDR_SURF_16_BANK));
3071 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3072 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3073 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3074 NUM_BANKS(ADDR_SURF_16_BANK));
3077 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3078 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3079 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3080 NUM_BANKS(ADDR_SURF_16_BANK));
3083 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3084 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3085 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3086 NUM_BANKS(ADDR_SURF_16_BANK));
3089 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3090 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3091 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3092 NUM_BANKS(ADDR_SURF_16_BANK));
3095 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3096 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3097 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3098 NUM_BANKS(ADDR_SURF_8_BANK));
3101 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3102 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3103 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3104 NUM_BANKS(ADDR_SURF_4_BANK));
3110 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3111 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3113 } else if (num_pipe_configs == 2) {
3114 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3115 switch (reg_offset) {
3117 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3118 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3119 PIPE_CONFIG(ADDR_SURF_P2) |
3120 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3123 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3124 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3125 PIPE_CONFIG(ADDR_SURF_P2) |
3126 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3129 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3130 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3131 PIPE_CONFIG(ADDR_SURF_P2) |
3132 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3135 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3136 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3137 PIPE_CONFIG(ADDR_SURF_P2) |
3138 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3141 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3142 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3143 PIPE_CONFIG(ADDR_SURF_P2) |
3144 TILE_SPLIT(split_equal_to_row_size));
3147 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3148 PIPE_CONFIG(ADDR_SURF_P2) |
3149 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3152 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3153 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3154 PIPE_CONFIG(ADDR_SURF_P2) |
3155 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3158 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3159 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3160 PIPE_CONFIG(ADDR_SURF_P2) |
3161 TILE_SPLIT(split_equal_to_row_size));
3164 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3165 PIPE_CONFIG(ADDR_SURF_P2);
3168 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3170 PIPE_CONFIG(ADDR_SURF_P2));
3173 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3174 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3175 PIPE_CONFIG(ADDR_SURF_P2) |
3176 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3179 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3180 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3181 PIPE_CONFIG(ADDR_SURF_P2) |
3182 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3185 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3186 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3187 PIPE_CONFIG(ADDR_SURF_P2) |
3188 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3191 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3192 PIPE_CONFIG(ADDR_SURF_P2) |
3193 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3196 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3197 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3198 PIPE_CONFIG(ADDR_SURF_P2) |
3199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3202 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3203 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3204 PIPE_CONFIG(ADDR_SURF_P2) |
3205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3208 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3209 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3210 PIPE_CONFIG(ADDR_SURF_P2) |
3211 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3214 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3215 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3216 PIPE_CONFIG(ADDR_SURF_P2));
3219 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3220 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3221 PIPE_CONFIG(ADDR_SURF_P2) |
3222 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3225 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3226 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3227 PIPE_CONFIG(ADDR_SURF_P2) |
3228 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3231 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3232 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3233 PIPE_CONFIG(ADDR_SURF_P2) |
3234 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3240 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3241 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3243 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3244 switch (reg_offset) {
3246 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3249 NUM_BANKS(ADDR_SURF_16_BANK));
3252 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3255 NUM_BANKS(ADDR_SURF_16_BANK));
3258 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3261 NUM_BANKS(ADDR_SURF_16_BANK));
3264 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3267 NUM_BANKS(ADDR_SURF_16_BANK));
3270 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3271 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3272 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3273 NUM_BANKS(ADDR_SURF_16_BANK));
3276 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3279 NUM_BANKS(ADDR_SURF_16_BANK));
3282 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3285 NUM_BANKS(ADDR_SURF_8_BANK));
3288 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3291 NUM_BANKS(ADDR_SURF_16_BANK));
3294 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3297 NUM_BANKS(ADDR_SURF_16_BANK));
3300 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3303 NUM_BANKS(ADDR_SURF_16_BANK));
3306 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3309 NUM_BANKS(ADDR_SURF_16_BANK));
3312 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3315 NUM_BANKS(ADDR_SURF_16_BANK));
3318 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3321 NUM_BANKS(ADDR_SURF_16_BANK));
3324 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3325 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3326 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3327 NUM_BANKS(ADDR_SURF_8_BANK));
3333 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3334 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3337 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3341 * cik_select_se_sh - select which SE, SH to address
3343 * @rdev: radeon_device pointer
3344 * @se_num: shader engine to address
3345 * @sh_num: sh block to address
3347 * Select which SE, SH combinations to address. Certain
3348 * registers are instanced per SE or SH. 0xffffffff means
3349 * broadcast to all SEs or SHs (CIK).
3351 static void cik_select_se_sh(struct radeon_device *rdev,
3352 u32 se_num, u32 sh_num)
3354 u32 data = INSTANCE_BROADCAST_WRITES;
3356 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3357 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3358 else if (se_num == 0xffffffff)
3359 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3360 else if (sh_num == 0xffffffff)
3361 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3363 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3364 WREG32(GRBM_GFX_INDEX, data);
3368 * cik_create_bitmask - create a bitmask
3370 * @bit_width: length of the mask
3372 * create a variable length bit mask (CIK).
3373 * Returns the bitmask.
3375 static u32 cik_create_bitmask(u32 bit_width)
3379 for (i = 0; i < bit_width; i++) {
3387 * cik_get_rb_disabled - computes the mask of disabled RBs
3389 * @rdev: radeon_device pointer
3390 * @max_rb_num: max RBs (render backends) for the asic
3391 * @se_num: number of SEs (shader engines) for the asic
3392 * @sh_per_se: number of SH blocks per SE for the asic
3394 * Calculates the bitmask of disabled RBs (CIK).
3395 * Returns the disabled RB bitmask.
3397 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3398 u32 max_rb_num_per_se,
3403 data = RREG32(CC_RB_BACKEND_DISABLE);
3405 data &= BACKEND_DISABLE_MASK;
3408 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3410 data >>= BACKEND_DISABLE_SHIFT;
3412 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3418 * cik_setup_rb - setup the RBs on the asic
3420 * @rdev: radeon_device pointer
3421 * @se_num: number of SEs (shader engines) for the asic
3422 * @sh_per_se: number of SH blocks per SE for the asic
3423 * @max_rb_num: max RBs (render backends) for the asic
3425 * Configures per-SE/SH RB registers (CIK).
3427 static void cik_setup_rb(struct radeon_device *rdev,
3428 u32 se_num, u32 sh_per_se,
3429 u32 max_rb_num_per_se)
3433 u32 disabled_rbs = 0;
3434 u32 enabled_rbs = 0;
3436 mutex_lock(&rdev->grbm_idx_mutex);
3437 for (i = 0; i < se_num; i++) {
3438 for (j = 0; j < sh_per_se; j++) {
3439 cik_select_se_sh(rdev, i, j);
3440 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3441 if (rdev->family == CHIP_HAWAII)
3442 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3444 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3447 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3448 mutex_unlock(&rdev->grbm_idx_mutex);
3451 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3452 if (!(disabled_rbs & mask))
3453 enabled_rbs |= mask;
3457 rdev->config.cik.backend_enable_mask = enabled_rbs;
3459 mutex_lock(&rdev->grbm_idx_mutex);
3460 for (i = 0; i < se_num; i++) {
3461 cik_select_se_sh(rdev, i, 0xffffffff);
3463 for (j = 0; j < sh_per_se; j++) {
3464 switch (enabled_rbs & 3) {
3467 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3469 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3472 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3475 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3479 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3484 WREG32(PA_SC_RASTER_CONFIG, data);
3486 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3487 mutex_unlock(&rdev->grbm_idx_mutex);
3491 * cik_gpu_init - setup the 3D engine
3493 * @rdev: radeon_device pointer
3495 * Configures the 3D engine and tiling configuration
3496 * registers so that the 3D engine is usable.
3498 static void cik_gpu_init(struct radeon_device *rdev)
3500 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3501 u32 mc_shared_chmap, mc_arb_ramcfg;
3502 u32 hdp_host_path_cntl;
3506 switch (rdev->family) {
3508 rdev->config.cik.max_shader_engines = 2;
3509 rdev->config.cik.max_tile_pipes = 4;
3510 rdev->config.cik.max_cu_per_sh = 7;
3511 rdev->config.cik.max_sh_per_se = 1;
3512 rdev->config.cik.max_backends_per_se = 2;
3513 rdev->config.cik.max_texture_channel_caches = 4;
3514 rdev->config.cik.max_gprs = 256;
3515 rdev->config.cik.max_gs_threads = 32;
3516 rdev->config.cik.max_hw_contexts = 8;
3518 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3519 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3520 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3521 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3522 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3525 rdev->config.cik.max_shader_engines = 4;
3526 rdev->config.cik.max_tile_pipes = 16;
3527 rdev->config.cik.max_cu_per_sh = 11;
3528 rdev->config.cik.max_sh_per_se = 1;
3529 rdev->config.cik.max_backends_per_se = 4;
3530 rdev->config.cik.max_texture_channel_caches = 16;
3531 rdev->config.cik.max_gprs = 256;
3532 rdev->config.cik.max_gs_threads = 32;
3533 rdev->config.cik.max_hw_contexts = 8;
3535 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3536 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3537 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3538 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3539 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3542 rdev->config.cik.max_shader_engines = 1;
3543 rdev->config.cik.max_tile_pipes = 4;
3544 if ((rdev->pdev->device == 0x1304) ||
3545 (rdev->pdev->device == 0x1305) ||
3546 (rdev->pdev->device == 0x130C) ||
3547 (rdev->pdev->device == 0x130F) ||
3548 (rdev->pdev->device == 0x1310) ||
3549 (rdev->pdev->device == 0x1311) ||
3550 (rdev->pdev->device == 0x131C)) {
3551 rdev->config.cik.max_cu_per_sh = 8;
3552 rdev->config.cik.max_backends_per_se = 2;
3553 } else if ((rdev->pdev->device == 0x1309) ||
3554 (rdev->pdev->device == 0x130A) ||
3555 (rdev->pdev->device == 0x130D) ||
3556 (rdev->pdev->device == 0x1313) ||
3557 (rdev->pdev->device == 0x131D)) {
3558 rdev->config.cik.max_cu_per_sh = 6;
3559 rdev->config.cik.max_backends_per_se = 2;
3560 } else if ((rdev->pdev->device == 0x1306) ||
3561 (rdev->pdev->device == 0x1307) ||
3562 (rdev->pdev->device == 0x130B) ||
3563 (rdev->pdev->device == 0x130E) ||
3564 (rdev->pdev->device == 0x1315) ||
3565 (rdev->pdev->device == 0x1318) ||
3566 (rdev->pdev->device == 0x131B)) {
3567 rdev->config.cik.max_cu_per_sh = 4;
3568 rdev->config.cik.max_backends_per_se = 1;
3570 rdev->config.cik.max_cu_per_sh = 3;
3571 rdev->config.cik.max_backends_per_se = 1;
3573 rdev->config.cik.max_sh_per_se = 1;
3574 rdev->config.cik.max_texture_channel_caches = 4;
3575 rdev->config.cik.max_gprs = 256;
3576 rdev->config.cik.max_gs_threads = 16;
3577 rdev->config.cik.max_hw_contexts = 8;
3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3588 rdev->config.cik.max_shader_engines = 1;
3589 rdev->config.cik.max_tile_pipes = 2;
3590 rdev->config.cik.max_cu_per_sh = 2;
3591 rdev->config.cik.max_sh_per_se = 1;
3592 rdev->config.cik.max_backends_per_se = 1;
3593 rdev->config.cik.max_texture_channel_caches = 2;
3594 rdev->config.cik.max_gprs = 256;
3595 rdev->config.cik.max_gs_threads = 16;
3596 rdev->config.cik.max_hw_contexts = 8;
3598 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3599 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3600 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3601 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3602 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3606 /* Initialize HDP */
3607 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3608 WREG32((0x2c14 + j), 0x00000000);
3609 WREG32((0x2c18 + j), 0x00000000);
3610 WREG32((0x2c1c + j), 0x00000000);
3611 WREG32((0x2c20 + j), 0x00000000);
3612 WREG32((0x2c24 + j), 0x00000000);
3615 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3616 WREG32(SRBM_INT_CNTL, 0x1);
3617 WREG32(SRBM_INT_ACK, 0x1);
3619 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3621 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3622 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3624 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3625 rdev->config.cik.mem_max_burst_length_bytes = 256;
3626 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3627 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3628 if (rdev->config.cik.mem_row_size_in_kb > 4)
3629 rdev->config.cik.mem_row_size_in_kb = 4;
3630 /* XXX use MC settings? */
3631 rdev->config.cik.shader_engine_tile_size = 32;
3632 rdev->config.cik.num_gpus = 1;
3633 rdev->config.cik.multi_gpu_tile_size = 64;
3635 /* fix up row size */
3636 gb_addr_config &= ~ROW_SIZE_MASK;
3637 switch (rdev->config.cik.mem_row_size_in_kb) {
3640 gb_addr_config |= ROW_SIZE(0);
3643 gb_addr_config |= ROW_SIZE(1);
3646 gb_addr_config |= ROW_SIZE(2);
3650 /* setup tiling info dword. gb_addr_config is not adequate since it does
3651 * not have bank info, so create a custom tiling dword.
3652 * bits 3:0 num_pipes
3653 * bits 7:4 num_banks
3654 * bits 11:8 group_size
3655 * bits 15:12 row_size
3657 rdev->config.cik.tile_config = 0;
3658 switch (rdev->config.cik.num_tile_pipes) {
3660 rdev->config.cik.tile_config |= (0 << 0);
3663 rdev->config.cik.tile_config |= (1 << 0);
3666 rdev->config.cik.tile_config |= (2 << 0);
3670 /* XXX what about 12? */
3671 rdev->config.cik.tile_config |= (3 << 0);
3674 rdev->config.cik.tile_config |=
3675 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3676 rdev->config.cik.tile_config |=
3677 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3678 rdev->config.cik.tile_config |=
3679 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3681 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3682 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3683 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3684 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3685 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3686 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3687 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3688 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3690 cik_tiling_mode_table_init(rdev);
3692 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3693 rdev->config.cik.max_sh_per_se,
3694 rdev->config.cik.max_backends_per_se);
3696 rdev->config.cik.active_cus = 0;
3697 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3698 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3699 rdev->config.cik.active_cus +=
3700 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3704 /* set HW defaults for 3D engine */
3705 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3707 mutex_lock(&rdev->grbm_idx_mutex);
3709 * making sure that the following register writes will be broadcasted
3710 * to all the shaders
3712 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3713 WREG32(SX_DEBUG_1, 0x20);
3715 WREG32(TA_CNTL_AUX, 0x00010000);
3717 tmp = RREG32(SPI_CONFIG_CNTL);
3719 WREG32(SPI_CONFIG_CNTL, tmp);
3721 WREG32(SQ_CONFIG, 1);
3723 WREG32(DB_DEBUG, 0);
3725 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3727 WREG32(DB_DEBUG2, tmp);
3729 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3731 WREG32(DB_DEBUG3, tmp);
3733 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3735 WREG32(CB_HW_CONTROL, tmp);
3737 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3739 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3740 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3741 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3742 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3744 WREG32(VGT_NUM_INSTANCES, 1);
3746 WREG32(CP_PERFMON_CNTL, 0);
3748 WREG32(SQ_CONFIG, 0);
3750 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3751 FORCE_EOV_MAX_REZ_CNT(255)));
3753 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3754 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3756 WREG32(VGT_GS_VERTEX_REUSE, 16);
3757 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3759 tmp = RREG32(HDP_MISC_CNTL);
3760 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3761 WREG32(HDP_MISC_CNTL, tmp);
3763 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3764 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3766 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3767 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3768 mutex_unlock(&rdev->grbm_idx_mutex);
3774 * GPU scratch registers helpers function.
3777 * cik_scratch_init - setup driver info for CP scratch regs
3779 * @rdev: radeon_device pointer
3781 * Set up the number and offset of the CP scratch registers.
3782 * NOTE: use of CP scratch registers is a legacy inferface and
3783 * is not used by default on newer asics (r6xx+). On newer asics,
3784 * memory buffers are used for fences rather than scratch regs.
3786 static void cik_scratch_init(struct radeon_device *rdev)
3790 rdev->scratch.num_reg = 7;
3791 rdev->scratch.reg_base = SCRATCH_REG0;
3792 for (i = 0; i < rdev->scratch.num_reg; i++) {
3793 rdev->scratch.free[i] = true;
3794 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3799 * cik_ring_test - basic gfx ring test
3801 * @rdev: radeon_device pointer
3802 * @ring: radeon_ring structure holding ring information
3804 * Allocate a scratch register and write to it using the gfx ring (CIK).
3805 * Provides a basic gfx ring test to verify that the ring is working.
3806 * Used by cik_cp_gfx_resume();
3807 * Returns 0 on success, error on failure.
3809 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3816 r = radeon_scratch_get(rdev, &scratch);
3818 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3821 WREG32(scratch, 0xCAFEDEAD);
3822 r = radeon_ring_lock(rdev, ring, 3);
3824 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3825 radeon_scratch_free(rdev, scratch);
3828 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3829 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3830 radeon_ring_write(ring, 0xDEADBEEF);
3831 radeon_ring_unlock_commit(rdev, ring, false);
3833 for (i = 0; i < rdev->usec_timeout; i++) {
3834 tmp = RREG32(scratch);
3835 if (tmp == 0xDEADBEEF)
3839 if (i < rdev->usec_timeout) {
3840 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3842 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3843 ring->idx, scratch, tmp);
3846 radeon_scratch_free(rdev, scratch);
3851 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3853 * @rdev: radeon_device pointer
3854 * @ridx: radeon ring index
3856 * Emits an hdp flush on the cp.
3858 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3861 struct radeon_ring *ring = &rdev->ring[ridx];
3864 switch (ring->idx) {
3865 case CAYMAN_RING_TYPE_CP1_INDEX:
3866 case CAYMAN_RING_TYPE_CP2_INDEX:
3870 ref_and_mask = CP2 << ring->pipe;
3873 ref_and_mask = CP6 << ring->pipe;
3879 case RADEON_RING_TYPE_GFX_INDEX:
3884 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3885 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3886 WAIT_REG_MEM_FUNCTION(3) | /* == */
3887 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3888 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3889 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3890 radeon_ring_write(ring, ref_and_mask);
3891 radeon_ring_write(ring, ref_and_mask);
3892 radeon_ring_write(ring, 0x20); /* poll interval */
3896 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3898 * @rdev: radeon_device pointer
3899 * @fence: radeon fence object
3901 * Emits a fence sequnce number on the gfx ring and flushes
3904 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3905 struct radeon_fence *fence)
3907 struct radeon_ring *ring = &rdev->ring[fence->ring];
3908 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3910 /* Workaround for cache flush problems. First send a dummy EOP
3911 * event down the pipe with seq one below.
3913 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3914 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3916 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3918 radeon_ring_write(ring, addr & 0xfffffffc);
3919 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3920 DATA_SEL(1) | INT_SEL(0));
3921 radeon_ring_write(ring, fence->seq - 1);
3922 radeon_ring_write(ring, 0);
3924 /* Then send the real EOP event down the pipe. */
3925 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3926 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3928 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3930 radeon_ring_write(ring, addr & 0xfffffffc);
3931 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3932 radeon_ring_write(ring, fence->seq);
3933 radeon_ring_write(ring, 0);
3937 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3939 * @rdev: radeon_device pointer
3940 * @fence: radeon fence object
3942 * Emits a fence sequnce number on the compute ring and flushes
3945 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3946 struct radeon_fence *fence)
3948 struct radeon_ring *ring = &rdev->ring[fence->ring];
3949 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3951 /* RELEASE_MEM - flush caches, send int */
3952 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3953 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3955 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3957 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3958 radeon_ring_write(ring, addr & 0xfffffffc);
3959 radeon_ring_write(ring, upper_32_bits(addr));
3960 radeon_ring_write(ring, fence->seq);
3961 radeon_ring_write(ring, 0);
3965 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3967 * @rdev: radeon_device pointer
3968 * @ring: radeon ring buffer object
3969 * @semaphore: radeon semaphore object
3970 * @emit_wait: Is this a sempahore wait?
3972 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3973 * from running ahead of semaphore waits.
3975 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3976 struct radeon_ring *ring,
3977 struct radeon_semaphore *semaphore,
3980 uint64_t addr = semaphore->gpu_addr;
3981 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3983 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3984 radeon_ring_write(ring, lower_32_bits(addr));
3985 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3987 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3988 /* Prevent the PFP from running ahead of the semaphore wait */
3989 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3990 radeon_ring_write(ring, 0x0);
3997 * cik_copy_cpdma - copy pages using the CP DMA engine
3999 * @rdev: radeon_device pointer
4000 * @src_offset: src GPU address
4001 * @dst_offset: dst GPU address
4002 * @num_gpu_pages: number of GPU pages to xfer
4003 * @resv: reservation object to sync to
4005 * Copy GPU paging using the CP DMA engine (CIK+).
4006 * Used by the radeon ttm implementation to move pages if
4007 * registered as the asic copy callback.
4009 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
4010 uint64_t src_offset, uint64_t dst_offset,
4011 unsigned num_gpu_pages,
4012 struct reservation_object *resv)
4014 struct radeon_fence *fence;
4015 struct radeon_sync sync;
4016 int ring_index = rdev->asic->copy.blit_ring_index;
4017 struct radeon_ring *ring = &rdev->ring[ring_index];
4018 u32 size_in_bytes, cur_size_in_bytes, control;
4022 radeon_sync_create(&sync);
4024 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
4025 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
4026 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
4028 DRM_ERROR("radeon: moving bo (%d).\n", r);
4029 radeon_sync_free(rdev, &sync, NULL);
4033 radeon_sync_resv(rdev, &sync, resv, false);
4034 radeon_sync_rings(rdev, &sync, ring->idx);
4036 for (i = 0; i < num_loops; i++) {
4037 cur_size_in_bytes = size_in_bytes;
4038 if (cur_size_in_bytes > 0x1fffff)
4039 cur_size_in_bytes = 0x1fffff;
4040 size_in_bytes -= cur_size_in_bytes;
4042 if (size_in_bytes == 0)
4043 control |= PACKET3_DMA_DATA_CP_SYNC;
4044 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4045 radeon_ring_write(ring, control);
4046 radeon_ring_write(ring, lower_32_bits(src_offset));
4047 radeon_ring_write(ring, upper_32_bits(src_offset));
4048 radeon_ring_write(ring, lower_32_bits(dst_offset));
4049 radeon_ring_write(ring, upper_32_bits(dst_offset));
4050 radeon_ring_write(ring, cur_size_in_bytes);
4051 src_offset += cur_size_in_bytes;
4052 dst_offset += cur_size_in_bytes;
4055 r = radeon_fence_emit(rdev, &fence, ring->idx);
4057 radeon_ring_unlock_undo(rdev, ring);
4058 radeon_sync_free(rdev, &sync, NULL);
4062 radeon_ring_unlock_commit(rdev, ring, false);
4063 radeon_sync_free(rdev, &sync, fence);
4072 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4074 * @rdev: radeon_device pointer
4075 * @ib: radeon indirect buffer object
4077 * Emits an DE (drawing engine) or CE (constant engine) IB
4078 * on the gfx ring. IBs are usually generated by userspace
4079 * acceleration drivers and submitted to the kernel for
4080 * sheduling on the ring. This function schedules the IB
4081 * on the gfx ring for execution by the GPU.
4083 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4085 struct radeon_ring *ring = &rdev->ring[ib->ring];
4086 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
4087 u32 header, control = INDIRECT_BUFFER_VALID;
4089 if (ib->is_const_ib) {
4090 /* set switch buffer packet before const IB */
4091 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4092 radeon_ring_write(ring, 0);
4094 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4097 if (ring->rptr_save_reg) {
4098 next_rptr = ring->wptr + 3 + 4;
4099 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4100 radeon_ring_write(ring, ((ring->rptr_save_reg -
4101 PACKET3_SET_UCONFIG_REG_START) >> 2));
4102 radeon_ring_write(ring, next_rptr);
4103 } else if (rdev->wb.enabled) {
4104 next_rptr = ring->wptr + 5 + 4;
4105 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4106 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4107 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4108 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4109 radeon_ring_write(ring, next_rptr);
4112 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4115 control |= ib->length_dw | (vm_id << 24);
4117 radeon_ring_write(ring, header);
4118 radeon_ring_write(ring,
4122 (ib->gpu_addr & 0xFFFFFFFC));
4123 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4124 radeon_ring_write(ring, control);
4128 * cik_ib_test - basic gfx ring IB test
4130 * @rdev: radeon_device pointer
4131 * @ring: radeon_ring structure holding ring information
4133 * Allocate an IB and execute it on the gfx ring (CIK).
4134 * Provides a basic gfx ring test to verify that IBs are working.
4135 * Returns 0 on success, error on failure.
4137 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4139 struct radeon_ib ib;
4145 r = radeon_scratch_get(rdev, &scratch);
4147 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4150 WREG32(scratch, 0xCAFEDEAD);
4151 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4153 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4154 radeon_scratch_free(rdev, scratch);
4157 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4158 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4159 ib.ptr[2] = 0xDEADBEEF;
4161 r = radeon_ib_schedule(rdev, &ib, NULL, false);
4163 radeon_scratch_free(rdev, scratch);
4164 radeon_ib_free(rdev, &ib);
4165 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4168 r = radeon_fence_wait(ib.fence, false);
4170 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4171 radeon_scratch_free(rdev, scratch);
4172 radeon_ib_free(rdev, &ib);
4175 for (i = 0; i < rdev->usec_timeout; i++) {
4176 tmp = RREG32(scratch);
4177 if (tmp == 0xDEADBEEF)
4181 if (i < rdev->usec_timeout) {
4182 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4184 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4188 radeon_scratch_free(rdev, scratch);
4189 radeon_ib_free(rdev, &ib);
4195 * On CIK, gfx and compute now have independant command processors.
4198 * Gfx consists of a single ring and can process both gfx jobs and
4199 * compute jobs. The gfx CP consists of three microengines (ME):
4200 * PFP - Pre-Fetch Parser
4202 * CE - Constant Engine
4203 * The PFP and ME make up what is considered the Drawing Engine (DE).
4204 * The CE is an asynchronous engine used for updating buffer desciptors
4205 * used by the DE so that they can be loaded into cache in parallel
4206 * while the DE is processing state update packets.
4209 * The compute CP consists of two microengines (ME):
4210 * MEC1 - Compute MicroEngine 1
4211 * MEC2 - Compute MicroEngine 2
4212 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4213 * The queues are exposed to userspace and are programmed directly
4214 * by the compute runtime.
4217 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4219 * @rdev: radeon_device pointer
4220 * @enable: enable or disable the MEs
4222 * Halts or unhalts the gfx MEs.
4224 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4227 WREG32(CP_ME_CNTL, 0);
4229 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4230 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4231 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4232 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4238 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4240 * @rdev: radeon_device pointer
4242 * Loads the gfx PFP, ME, and CE ucode.
4243 * Returns 0 for success, -EINVAL if the ucode is not available.
4245 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4249 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4252 cik_cp_gfx_enable(rdev, false);
4255 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4256 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4257 const struct gfx_firmware_header_v1_0 *ce_hdr =
4258 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4259 const struct gfx_firmware_header_v1_0 *me_hdr =
4260 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4261 const __le32 *fw_data;
4264 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4265 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4266 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4269 fw_data = (const __le32 *)
4270 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4271 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4272 WREG32(CP_PFP_UCODE_ADDR, 0);
4273 for (i = 0; i < fw_size; i++)
4274 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4275 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
4278 fw_data = (const __le32 *)
4279 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4280 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4281 WREG32(CP_CE_UCODE_ADDR, 0);
4282 for (i = 0; i < fw_size; i++)
4283 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4284 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
4287 fw_data = (const __be32 *)
4288 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4289 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4290 WREG32(CP_ME_RAM_WADDR, 0);
4291 for (i = 0; i < fw_size; i++)
4292 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4293 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4294 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
4296 const __be32 *fw_data;
4299 fw_data = (const __be32 *)rdev->pfp_fw->data;
4300 WREG32(CP_PFP_UCODE_ADDR, 0);
4301 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4302 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4303 WREG32(CP_PFP_UCODE_ADDR, 0);
4306 fw_data = (const __be32 *)rdev->ce_fw->data;
4307 WREG32(CP_CE_UCODE_ADDR, 0);
4308 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4309 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4310 WREG32(CP_CE_UCODE_ADDR, 0);
4313 fw_data = (const __be32 *)rdev->me_fw->data;
4314 WREG32(CP_ME_RAM_WADDR, 0);
4315 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4316 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4317 WREG32(CP_ME_RAM_WADDR, 0);
4324 * cik_cp_gfx_start - start the gfx ring
4326 * @rdev: radeon_device pointer
4328 * Enables the ring and loads the clear state context and other
4329 * packets required to init the ring.
4330 * Returns 0 for success, error for failure.
4332 static int cik_cp_gfx_start(struct radeon_device *rdev)
4334 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4338 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4339 WREG32(CP_ENDIAN_SWAP, 0);
4340 WREG32(CP_DEVICE_ID, 1);
4342 cik_cp_gfx_enable(rdev, true);
4344 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4346 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4350 /* init the CE partitions. CE only used for gfx on CIK */
4351 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4352 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4353 radeon_ring_write(ring, 0x8000);
4354 radeon_ring_write(ring, 0x8000);
4356 /* setup clear context state */
4357 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4358 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4360 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4361 radeon_ring_write(ring, 0x80000000);
4362 radeon_ring_write(ring, 0x80000000);
4364 for (i = 0; i < cik_default_size; i++)
4365 radeon_ring_write(ring, cik_default_state[i]);
4367 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4368 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4370 /* set clear context state */
4371 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4372 radeon_ring_write(ring, 0);
4374 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4375 radeon_ring_write(ring, 0x00000316);
4376 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4377 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4379 radeon_ring_unlock_commit(rdev, ring, false);
4385 * cik_cp_gfx_fini - stop the gfx ring
4387 * @rdev: radeon_device pointer
4389 * Stop the gfx ring and tear down the driver ring
4392 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4394 cik_cp_gfx_enable(rdev, false);
4395 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4399 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4401 * @rdev: radeon_device pointer
4403 * Program the location and size of the gfx ring buffer
4404 * and test it to make sure it's working.
4405 * Returns 0 for success, error for failure.
4407 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4409 struct radeon_ring *ring;
4415 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4416 if (rdev->family != CHIP_HAWAII)
4417 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4419 /* Set the write pointer delay */
4420 WREG32(CP_RB_WPTR_DELAY, 0);
4422 /* set the RB to use vmid 0 */
4423 WREG32(CP_RB_VMID, 0);
4425 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4427 /* ring 0 - compute and gfx */
4428 /* Set ring buffer size */
4429 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4430 rb_bufsz = order_base_2(ring->ring_size / 8);
4431 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4433 tmp |= BUF_SWAP_32BIT;
4435 WREG32(CP_RB0_CNTL, tmp);
4437 /* Initialize the ring buffer's read and write pointers */
4438 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4440 WREG32(CP_RB0_WPTR, ring->wptr);
4442 /* set the wb address wether it's enabled or not */
4443 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4444 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4446 /* scratch register shadowing is no longer supported */
4447 WREG32(SCRATCH_UMSK, 0);
4449 if (!rdev->wb.enabled)
4450 tmp |= RB_NO_UPDATE;
4453 WREG32(CP_RB0_CNTL, tmp);
4455 rb_addr = ring->gpu_addr >> 8;
4456 WREG32(CP_RB0_BASE, rb_addr);
4457 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4459 /* start the ring */
4460 cik_cp_gfx_start(rdev);
4461 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4462 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4464 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4468 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4469 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4474 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4475 struct radeon_ring *ring)
4479 if (rdev->wb.enabled)
4480 rptr = rdev->wb.wb[ring->rptr_offs/4];
4482 rptr = RREG32(CP_RB0_RPTR);
4487 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4488 struct radeon_ring *ring)
4492 wptr = RREG32(CP_RB0_WPTR);
4497 void cik_gfx_set_wptr(struct radeon_device *rdev,
4498 struct radeon_ring *ring)
4500 WREG32(CP_RB0_WPTR, ring->wptr);
4501 (void)RREG32(CP_RB0_WPTR);
4504 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4505 struct radeon_ring *ring)
4509 if (rdev->wb.enabled) {
4510 rptr = rdev->wb.wb[ring->rptr_offs/4];
4512 mutex_lock(&rdev->srbm_mutex);
4513 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4514 rptr = RREG32(CP_HQD_PQ_RPTR);
4515 cik_srbm_select(rdev, 0, 0, 0, 0);
4516 mutex_unlock(&rdev->srbm_mutex);
4522 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4523 struct radeon_ring *ring)
4527 if (rdev->wb.enabled) {
4528 /* XXX check if swapping is necessary on BE */
4529 wptr = rdev->wb.wb[ring->wptr_offs/4];
4531 mutex_lock(&rdev->srbm_mutex);
4532 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4533 wptr = RREG32(CP_HQD_PQ_WPTR);
4534 cik_srbm_select(rdev, 0, 0, 0, 0);
4535 mutex_unlock(&rdev->srbm_mutex);
4541 void cik_compute_set_wptr(struct radeon_device *rdev,
4542 struct radeon_ring *ring)
4544 /* XXX check if swapping is necessary on BE */
4545 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4546 WDOORBELL32(ring->doorbell_index, ring->wptr);
4550 * cik_cp_compute_enable - enable/disable the compute CP MEs
4552 * @rdev: radeon_device pointer
4553 * @enable: enable or disable the MEs
4555 * Halts or unhalts the compute MEs.
4557 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4560 WREG32(CP_MEC_CNTL, 0);
4562 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4563 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4564 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4570 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4572 * @rdev: radeon_device pointer
4574 * Loads the compute MEC1&2 ucode.
4575 * Returns 0 for success, -EINVAL if the ucode is not available.
4577 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4584 cik_cp_compute_enable(rdev, false);
4587 const struct gfx_firmware_header_v1_0 *mec_hdr =
4588 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4589 const __le32 *fw_data;
4592 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4595 fw_data = (const __le32 *)
4596 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4597 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4598 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4599 for (i = 0; i < fw_size; i++)
4600 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4601 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4604 if (rdev->family == CHIP_KAVERI) {
4605 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4606 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4608 fw_data = (const __le32 *)
4609 (rdev->mec2_fw->data +
4610 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4611 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4612 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4613 for (i = 0; i < fw_size; i++)
4614 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4615 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4618 const __be32 *fw_data;
4621 fw_data = (const __be32 *)rdev->mec_fw->data;
4622 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4623 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4624 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4625 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4627 if (rdev->family == CHIP_KAVERI) {
4629 fw_data = (const __be32 *)rdev->mec_fw->data;
4630 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4631 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4632 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4633 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4641 * cik_cp_compute_start - start the compute queues
4643 * @rdev: radeon_device pointer
4645 * Enable the compute queues.
4646 * Returns 0 for success, error for failure.
4648 static int cik_cp_compute_start(struct radeon_device *rdev)
4650 cik_cp_compute_enable(rdev, true);
4656 * cik_cp_compute_fini - stop the compute queues
4658 * @rdev: radeon_device pointer
4660 * Stop the compute queues and tear down the driver queue
4663 static void cik_cp_compute_fini(struct radeon_device *rdev)
4667 cik_cp_compute_enable(rdev, false);
4669 for (i = 0; i < 2; i++) {
4671 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4673 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4675 if (rdev->ring[idx].mqd_obj) {
4676 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4677 if (unlikely(r != 0))
4678 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4680 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4681 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4683 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4684 rdev->ring[idx].mqd_obj = NULL;
4689 static void cik_mec_fini(struct radeon_device *rdev)
4693 if (rdev->mec.hpd_eop_obj) {
4694 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4695 if (unlikely(r != 0))
4696 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4697 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4698 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4700 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4701 rdev->mec.hpd_eop_obj = NULL;
4705 #define MEC_HPD_SIZE 2048
4707 static int cik_mec_init(struct radeon_device *rdev)
4713 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4714 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4715 * Nonetheless, we assign only 1 pipe because all other pipes will
4718 rdev->mec.num_mec = 1;
4719 rdev->mec.num_pipe = 1;
4720 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4722 if (rdev->mec.hpd_eop_obj == NULL) {
4723 r = radeon_bo_create(rdev,
4724 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4726 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4727 &rdev->mec.hpd_eop_obj);
4729 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4734 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4735 if (unlikely(r != 0)) {
4739 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4740 &rdev->mec.hpd_eop_gpu_addr);
4742 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4746 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4748 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4753 /* clear memory. Not sure if this is required or not */
4754 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4756 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4757 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4762 struct hqd_registers
4764 u32 cp_mqd_base_addr;
4765 u32 cp_mqd_base_addr_hi;
4768 u32 cp_hqd_persistent_state;
4769 u32 cp_hqd_pipe_priority;
4770 u32 cp_hqd_queue_priority;
4773 u32 cp_hqd_pq_base_hi;
4775 u32 cp_hqd_pq_rptr_report_addr;
4776 u32 cp_hqd_pq_rptr_report_addr_hi;
4777 u32 cp_hqd_pq_wptr_poll_addr;
4778 u32 cp_hqd_pq_wptr_poll_addr_hi;
4779 u32 cp_hqd_pq_doorbell_control;
4781 u32 cp_hqd_pq_control;
4782 u32 cp_hqd_ib_base_addr;
4783 u32 cp_hqd_ib_base_addr_hi;
4785 u32 cp_hqd_ib_control;
4786 u32 cp_hqd_iq_timer;
4788 u32 cp_hqd_dequeue_request;
4789 u32 cp_hqd_dma_offload;
4790 u32 cp_hqd_sema_cmd;
4791 u32 cp_hqd_msg_type;
4792 u32 cp_hqd_atomic0_preop_lo;
4793 u32 cp_hqd_atomic0_preop_hi;
4794 u32 cp_hqd_atomic1_preop_lo;
4795 u32 cp_hqd_atomic1_preop_hi;
4796 u32 cp_hqd_hq_scheduler0;
4797 u32 cp_hqd_hq_scheduler1;
4804 u32 dispatch_initiator;
4808 u32 pipeline_stat_enable;
4809 u32 perf_counter_enable;
4815 u32 resource_limits;
4816 u32 static_thread_mgmt01[2];
4818 u32 static_thread_mgmt23[2];
4820 u32 thread_trace_enable;
4823 u32 vgtcs_invoke_count[2];
4824 struct hqd_registers queue_state;
4826 u32 interrupt_queue[64];
4830 * cik_cp_compute_resume - setup the compute queue registers
4832 * @rdev: radeon_device pointer
4834 * Program the compute queues and test them to make sure they
4836 * Returns 0 for success, error for failure.
4838 static int cik_cp_compute_resume(struct radeon_device *rdev)
4842 bool use_doorbell = true;
4848 struct bonaire_mqd *mqd;
4850 r = cik_cp_compute_start(rdev);
4854 /* fix up chicken bits */
4855 tmp = RREG32(CP_CPF_DEBUG);
4857 WREG32(CP_CPF_DEBUG, tmp);
4859 /* init the pipes */
4860 mutex_lock(&rdev->srbm_mutex);
4862 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
4864 cik_srbm_select(rdev, 0, 0, 0, 0);
4866 /* write the EOP addr */
4867 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4868 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4870 /* set the VMID assigned */
4871 WREG32(CP_HPD_EOP_VMID, 0);
4873 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4874 tmp = RREG32(CP_HPD_EOP_CONTROL);
4875 tmp &= ~EOP_SIZE_MASK;
4876 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4877 WREG32(CP_HPD_EOP_CONTROL, tmp);
4879 mutex_unlock(&rdev->srbm_mutex);
4881 /* init the queues. Just two for now. */
4882 for (i = 0; i < 2; i++) {
4884 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4886 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4888 if (rdev->ring[idx].mqd_obj == NULL) {
4889 r = radeon_bo_create(rdev,
4890 sizeof(struct bonaire_mqd),
4892 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4893 NULL, &rdev->ring[idx].mqd_obj);
4895 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4900 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4901 if (unlikely(r != 0)) {
4902 cik_cp_compute_fini(rdev);
4905 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4908 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4909 cik_cp_compute_fini(rdev);
4912 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4914 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4915 cik_cp_compute_fini(rdev);
4919 /* init the mqd struct */
4920 memset(buf, 0, sizeof(struct bonaire_mqd));
4922 mqd = (struct bonaire_mqd *)buf;
4923 mqd->header = 0xC0310800;
4924 mqd->static_thread_mgmt01[0] = 0xffffffff;
4925 mqd->static_thread_mgmt01[1] = 0xffffffff;
4926 mqd->static_thread_mgmt23[0] = 0xffffffff;
4927 mqd->static_thread_mgmt23[1] = 0xffffffff;
4929 mutex_lock(&rdev->srbm_mutex);
4930 cik_srbm_select(rdev, rdev->ring[idx].me,
4931 rdev->ring[idx].pipe,
4932 rdev->ring[idx].queue, 0);
4934 /* disable wptr polling */
4935 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4936 tmp &= ~WPTR_POLL_EN;
4937 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4939 /* enable doorbell? */
4940 mqd->queue_state.cp_hqd_pq_doorbell_control =
4941 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4943 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4945 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4946 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4947 mqd->queue_state.cp_hqd_pq_doorbell_control);
4949 /* disable the queue if it's active */
4950 mqd->queue_state.cp_hqd_dequeue_request = 0;
4951 mqd->queue_state.cp_hqd_pq_rptr = 0;
4952 mqd->queue_state.cp_hqd_pq_wptr= 0;
4953 if (RREG32(CP_HQD_ACTIVE) & 1) {
4954 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4955 for (j = 0; j < rdev->usec_timeout; j++) {
4956 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4960 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4961 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4962 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4965 /* set the pointer to the MQD */
4966 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4967 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4968 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4969 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4970 /* set MQD vmid to 0 */
4971 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4972 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4973 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4975 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4976 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4977 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4978 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4979 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4980 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4982 /* set up the HQD, this is similar to CP_RB0_CNTL */
4983 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4984 mqd->queue_state.cp_hqd_pq_control &=
4985 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4987 mqd->queue_state.cp_hqd_pq_control |=
4988 order_base_2(rdev->ring[idx].ring_size / 8);
4989 mqd->queue_state.cp_hqd_pq_control |=
4990 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4992 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4994 mqd->queue_state.cp_hqd_pq_control &=
4995 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4996 mqd->queue_state.cp_hqd_pq_control |=
4997 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4998 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
5000 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
5002 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
5004 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
5005 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
5006 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
5007 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
5008 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
5009 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
5011 /* set the wb address wether it's enabled or not */
5013 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
5015 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
5016 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
5017 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
5018 upper_32_bits(wb_gpu_addr) & 0xffff;
5019 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
5020 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
5021 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
5022 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
5024 /* enable the doorbell if requested */
5026 mqd->queue_state.cp_hqd_pq_doorbell_control =
5027 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5028 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5029 mqd->queue_state.cp_hqd_pq_doorbell_control |=
5030 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
5031 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5032 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5033 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5036 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5038 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5039 mqd->queue_state.cp_hqd_pq_doorbell_control);
5041 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5042 rdev->ring[idx].wptr = 0;
5043 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5044 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5045 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5047 /* set the vmid for the queue */
5048 mqd->queue_state.cp_hqd_vmid = 0;
5049 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5051 /* activate the queue */
5052 mqd->queue_state.cp_hqd_active = 1;
5053 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5055 cik_srbm_select(rdev, 0, 0, 0, 0);
5056 mutex_unlock(&rdev->srbm_mutex);
5058 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5059 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5061 rdev->ring[idx].ready = true;
5062 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5064 rdev->ring[idx].ready = false;
5070 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5072 cik_cp_gfx_enable(rdev, enable);
5073 cik_cp_compute_enable(rdev, enable);
5076 static int cik_cp_load_microcode(struct radeon_device *rdev)
5080 r = cik_cp_gfx_load_microcode(rdev);
5083 r = cik_cp_compute_load_microcode(rdev);
5090 static void cik_cp_fini(struct radeon_device *rdev)
5092 cik_cp_gfx_fini(rdev);
5093 cik_cp_compute_fini(rdev);
5096 static int cik_cp_resume(struct radeon_device *rdev)
5100 cik_enable_gui_idle_interrupt(rdev, false);
5102 r = cik_cp_load_microcode(rdev);
5106 r = cik_cp_gfx_resume(rdev);
5109 r = cik_cp_compute_resume(rdev);
5113 cik_enable_gui_idle_interrupt(rdev, true);
5118 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5120 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5121 RREG32(GRBM_STATUS));
5122 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5123 RREG32(GRBM_STATUS2));
5124 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5125 RREG32(GRBM_STATUS_SE0));
5126 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5127 RREG32(GRBM_STATUS_SE1));
5128 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5129 RREG32(GRBM_STATUS_SE2));
5130 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5131 RREG32(GRBM_STATUS_SE3));
5132 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5133 RREG32(SRBM_STATUS));
5134 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5135 RREG32(SRBM_STATUS2));
5136 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5137 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5138 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5139 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5140 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5141 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5142 RREG32(CP_STALLED_STAT1));
5143 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5144 RREG32(CP_STALLED_STAT2));
5145 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5146 RREG32(CP_STALLED_STAT3));
5147 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5148 RREG32(CP_CPF_BUSY_STAT));
5149 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5150 RREG32(CP_CPF_STALLED_STAT1));
5151 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5152 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5153 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5154 RREG32(CP_CPC_STALLED_STAT1));
5155 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5159 * cik_gpu_check_soft_reset - check which blocks are busy
5161 * @rdev: radeon_device pointer
5163 * Check which blocks are busy and return the relevant reset
5164 * mask to be used by cik_gpu_soft_reset().
5165 * Returns a mask of the blocks to be reset.
5167 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5173 tmp = RREG32(GRBM_STATUS);
5174 if (tmp & (PA_BUSY | SC_BUSY |
5175 BCI_BUSY | SX_BUSY |
5176 TA_BUSY | VGT_BUSY |
5178 GDS_BUSY | SPI_BUSY |
5179 IA_BUSY | IA_BUSY_NO_DMA))
5180 reset_mask |= RADEON_RESET_GFX;
5182 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5183 reset_mask |= RADEON_RESET_CP;
5186 tmp = RREG32(GRBM_STATUS2);
5188 reset_mask |= RADEON_RESET_RLC;
5190 /* SDMA0_STATUS_REG */
5191 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5192 if (!(tmp & SDMA_IDLE))
5193 reset_mask |= RADEON_RESET_DMA;
5195 /* SDMA1_STATUS_REG */
5196 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5197 if (!(tmp & SDMA_IDLE))
5198 reset_mask |= RADEON_RESET_DMA1;
5201 tmp = RREG32(SRBM_STATUS2);
5202 if (tmp & SDMA_BUSY)
5203 reset_mask |= RADEON_RESET_DMA;
5205 if (tmp & SDMA1_BUSY)
5206 reset_mask |= RADEON_RESET_DMA1;
5209 tmp = RREG32(SRBM_STATUS);
5212 reset_mask |= RADEON_RESET_IH;
5215 reset_mask |= RADEON_RESET_SEM;
5217 if (tmp & GRBM_RQ_PENDING)
5218 reset_mask |= RADEON_RESET_GRBM;
5221 reset_mask |= RADEON_RESET_VMC;
5223 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5224 MCC_BUSY | MCD_BUSY))
5225 reset_mask |= RADEON_RESET_MC;
5227 if (evergreen_is_display_hung(rdev))
5228 reset_mask |= RADEON_RESET_DISPLAY;
5230 /* Skip MC reset as it's mostly likely not hung, just busy */
5231 if (reset_mask & RADEON_RESET_MC) {
5232 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5233 reset_mask &= ~RADEON_RESET_MC;
5240 * cik_gpu_soft_reset - soft reset GPU
5242 * @rdev: radeon_device pointer
5243 * @reset_mask: mask of which blocks to reset
5245 * Soft reset the blocks specified in @reset_mask.
5247 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5249 struct evergreen_mc_save save;
5250 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5253 if (reset_mask == 0)
5256 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5258 cik_print_gpu_status_regs(rdev);
5259 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5260 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5261 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5262 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5271 /* Disable GFX parsing/prefetching */
5272 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5274 /* Disable MEC parsing/prefetching */
5275 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5277 if (reset_mask & RADEON_RESET_DMA) {
5279 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5281 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5283 if (reset_mask & RADEON_RESET_DMA1) {
5285 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5287 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5290 evergreen_mc_stop(rdev, &save);
5291 if (evergreen_mc_wait_for_idle(rdev)) {
5292 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5295 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5296 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5298 if (reset_mask & RADEON_RESET_CP) {
5299 grbm_soft_reset |= SOFT_RESET_CP;
5301 srbm_soft_reset |= SOFT_RESET_GRBM;
5304 if (reset_mask & RADEON_RESET_DMA)
5305 srbm_soft_reset |= SOFT_RESET_SDMA;
5307 if (reset_mask & RADEON_RESET_DMA1)
5308 srbm_soft_reset |= SOFT_RESET_SDMA1;
5310 if (reset_mask & RADEON_RESET_DISPLAY)
5311 srbm_soft_reset |= SOFT_RESET_DC;
5313 if (reset_mask & RADEON_RESET_RLC)
5314 grbm_soft_reset |= SOFT_RESET_RLC;
5316 if (reset_mask & RADEON_RESET_SEM)
5317 srbm_soft_reset |= SOFT_RESET_SEM;
5319 if (reset_mask & RADEON_RESET_IH)
5320 srbm_soft_reset |= SOFT_RESET_IH;
5322 if (reset_mask & RADEON_RESET_GRBM)
5323 srbm_soft_reset |= SOFT_RESET_GRBM;
5325 if (reset_mask & RADEON_RESET_VMC)
5326 srbm_soft_reset |= SOFT_RESET_VMC;
5328 if (!(rdev->flags & RADEON_IS_IGP)) {
5329 if (reset_mask & RADEON_RESET_MC)
5330 srbm_soft_reset |= SOFT_RESET_MC;
5333 if (grbm_soft_reset) {
5334 tmp = RREG32(GRBM_SOFT_RESET);
5335 tmp |= grbm_soft_reset;
5336 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5337 WREG32(GRBM_SOFT_RESET, tmp);
5338 tmp = RREG32(GRBM_SOFT_RESET);
5342 tmp &= ~grbm_soft_reset;
5343 WREG32(GRBM_SOFT_RESET, tmp);
5344 tmp = RREG32(GRBM_SOFT_RESET);
5347 if (srbm_soft_reset) {
5348 tmp = RREG32(SRBM_SOFT_RESET);
5349 tmp |= srbm_soft_reset;
5350 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5351 WREG32(SRBM_SOFT_RESET, tmp);
5352 tmp = RREG32(SRBM_SOFT_RESET);
5356 tmp &= ~srbm_soft_reset;
5357 WREG32(SRBM_SOFT_RESET, tmp);
5358 tmp = RREG32(SRBM_SOFT_RESET);
5361 /* Wait a little for things to settle down */
5364 evergreen_mc_resume(rdev, &save);
5367 cik_print_gpu_status_regs(rdev);
5370 struct kv_reset_save_regs {
5371 u32 gmcon_reng_execute;
5376 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5377 struct kv_reset_save_regs *save)
5379 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5380 save->gmcon_misc = RREG32(GMCON_MISC);
5381 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5383 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5384 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5385 STCTRL_STUTTER_EN));
5388 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5389 struct kv_reset_save_regs *save)
5393 WREG32(GMCON_PGFSM_WRITE, 0);
5394 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5396 for (i = 0; i < 5; i++)
5397 WREG32(GMCON_PGFSM_WRITE, 0);
5399 WREG32(GMCON_PGFSM_WRITE, 0);
5400 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5402 for (i = 0; i < 5; i++)
5403 WREG32(GMCON_PGFSM_WRITE, 0);
5405 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5406 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5408 for (i = 0; i < 5; i++)
5409 WREG32(GMCON_PGFSM_WRITE, 0);
5411 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5412 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5414 for (i = 0; i < 5; i++)
5415 WREG32(GMCON_PGFSM_WRITE, 0);
5417 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5418 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5420 for (i = 0; i < 5; i++)
5421 WREG32(GMCON_PGFSM_WRITE, 0);
5423 WREG32(GMCON_PGFSM_WRITE, 0);
5424 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5426 for (i = 0; i < 5; i++)
5427 WREG32(GMCON_PGFSM_WRITE, 0);
5429 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5430 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5432 for (i = 0; i < 5; i++)
5433 WREG32(GMCON_PGFSM_WRITE, 0);
5435 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5436 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5438 for (i = 0; i < 5; i++)
5439 WREG32(GMCON_PGFSM_WRITE, 0);
5441 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5442 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5444 for (i = 0; i < 5; i++)
5445 WREG32(GMCON_PGFSM_WRITE, 0);
5447 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5448 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5450 for (i = 0; i < 5; i++)
5451 WREG32(GMCON_PGFSM_WRITE, 0);
5453 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5454 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5456 WREG32(GMCON_MISC3, save->gmcon_misc3);
5457 WREG32(GMCON_MISC, save->gmcon_misc);
5458 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5461 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5463 struct evergreen_mc_save save;
5464 struct kv_reset_save_regs kv_save = { 0 };
5467 dev_info(rdev->dev, "GPU pci config reset\n");
5475 /* Disable GFX parsing/prefetching */
5476 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5478 /* Disable MEC parsing/prefetching */
5479 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5482 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5484 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5486 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5488 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5489 /* XXX other engines? */
5491 /* halt the rlc, disable cp internal ints */
5496 /* disable mem access */
5497 evergreen_mc_stop(rdev, &save);
5498 if (evergreen_mc_wait_for_idle(rdev)) {
5499 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5502 if (rdev->flags & RADEON_IS_IGP)
5503 kv_save_regs_for_reset(rdev, &kv_save);
5506 pci_clear_master(rdev->pdev);
5508 radeon_pci_config_reset(rdev);
5512 /* wait for asic to come out of reset */
5513 for (i = 0; i < rdev->usec_timeout; i++) {
5514 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5519 /* does asic init need to be run first??? */
5520 if (rdev->flags & RADEON_IS_IGP)
5521 kv_restore_regs_for_reset(rdev, &kv_save);
5525 * cik_asic_reset - soft reset GPU
5527 * @rdev: radeon_device pointer
5529 * Look up which blocks are hung and attempt
5531 * Returns 0 for success.
5533 int cik_asic_reset(struct radeon_device *rdev)
5537 reset_mask = cik_gpu_check_soft_reset(rdev);
5540 r600_set_bios_scratch_engine_hung(rdev, true);
5542 /* try soft reset */
5543 cik_gpu_soft_reset(rdev, reset_mask);
5545 reset_mask = cik_gpu_check_soft_reset(rdev);
5547 /* try pci config reset */
5548 if (reset_mask && radeon_hard_reset)
5549 cik_gpu_pci_config_reset(rdev);
5551 reset_mask = cik_gpu_check_soft_reset(rdev);
5554 r600_set_bios_scratch_engine_hung(rdev, false);
5560 * cik_gfx_is_lockup - check if the 3D engine is locked up
5562 * @rdev: radeon_device pointer
5563 * @ring: radeon_ring structure holding ring information
5565 * Check if the 3D engine is locked up (CIK).
5566 * Returns true if the engine is locked, false if not.
5568 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5570 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5572 if (!(reset_mask & (RADEON_RESET_GFX |
5573 RADEON_RESET_COMPUTE |
5574 RADEON_RESET_CP))) {
5575 radeon_ring_lockup_update(rdev, ring);
5578 return radeon_ring_test_lockup(rdev, ring);
5583 * cik_mc_program - program the GPU memory controller
5585 * @rdev: radeon_device pointer
5587 * Set the location of vram, gart, and AGP in the GPU's
5588 * physical address space (CIK).
5590 static void cik_mc_program(struct radeon_device *rdev)
5592 struct evergreen_mc_save save;
5596 /* Initialize HDP */
5597 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5598 WREG32((0x2c14 + j), 0x00000000);
5599 WREG32((0x2c18 + j), 0x00000000);
5600 WREG32((0x2c1c + j), 0x00000000);
5601 WREG32((0x2c20 + j), 0x00000000);
5602 WREG32((0x2c24 + j), 0x00000000);
5604 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5606 evergreen_mc_stop(rdev, &save);
5607 if (radeon_mc_wait_for_idle(rdev)) {
5608 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5610 /* Lockout access through VGA aperture*/
5611 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5612 /* Update configuration */
5613 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5614 rdev->mc.vram_start >> 12);
5615 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5616 rdev->mc.vram_end >> 12);
5617 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5618 rdev->vram_scratch.gpu_addr >> 12);
5619 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5620 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5621 WREG32(MC_VM_FB_LOCATION, tmp);
5622 /* XXX double check these! */
5623 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5624 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5625 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5626 WREG32(MC_VM_AGP_BASE, 0);
5627 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5628 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5629 if (radeon_mc_wait_for_idle(rdev)) {
5630 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5632 evergreen_mc_resume(rdev, &save);
5633 /* we need to own VRAM, so turn off the VGA renderer here
5634 * to stop it overwriting our objects */
5635 rv515_vga_render_disable(rdev);
5639 * cik_mc_init - initialize the memory controller driver params
5641 * @rdev: radeon_device pointer
5643 * Look up the amount of vram, vram width, and decide how to place
5644 * vram and gart within the GPU's physical address space (CIK).
5645 * Returns 0 for success.
5647 static int cik_mc_init(struct radeon_device *rdev)
5650 int chansize, numchan;
5652 /* Get VRAM informations */
5653 rdev->mc.vram_is_ddr = true;
5654 tmp = RREG32(MC_ARB_RAMCFG);
5655 if (tmp & CHANSIZE_MASK) {
5660 tmp = RREG32(MC_SHARED_CHMAP);
5661 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5691 rdev->mc.vram_width = numchan * chansize;
5692 /* Could aper size report 0 ? */
5693 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5694 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5695 /* size in MB on si */
5696 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5697 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5698 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5699 si_vram_gtt_location(rdev, &rdev->mc);
5700 radeon_update_bandwidth_info(rdev);
5707 * VMID 0 is the physical GPU addresses as used by the kernel.
5708 * VMIDs 1-15 are used for userspace clients and are handled
5709 * by the radeon vm/hsa code.
5712 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5714 * @rdev: radeon_device pointer
5716 * Flush the TLB for the VMID 0 page table (CIK).
5718 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5720 /* flush hdp cache */
5721 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5723 /* bits 0-15 are the VM contexts0-15 */
5724 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5727 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5730 uint32_t sh_mem_bases, sh_mem_config;
5732 sh_mem_bases = 0x6000 | 0x6000 << 16;
5733 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5734 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5736 mutex_lock(&rdev->srbm_mutex);
5737 for (i = 8; i < 16; i++) {
5738 cik_srbm_select(rdev, 0, 0, 0, i);
5739 /* CP and shaders */
5740 WREG32(SH_MEM_CONFIG, sh_mem_config);
5741 WREG32(SH_MEM_APE1_BASE, 1);
5742 WREG32(SH_MEM_APE1_LIMIT, 0);
5743 WREG32(SH_MEM_BASES, sh_mem_bases);
5745 cik_srbm_select(rdev, 0, 0, 0, 0);
5746 mutex_unlock(&rdev->srbm_mutex);
5750 * cik_pcie_gart_enable - gart enable
5752 * @rdev: radeon_device pointer
5754 * This sets up the TLBs, programs the page tables for VMID0,
5755 * sets up the hw for VMIDs 1-15 which are allocated on
5756 * demand, and sets up the global locations for the LDS, GDS,
5757 * and GPUVM for FSA64 clients (CIK).
5758 * Returns 0 for success, errors for failure.
5760 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5764 if (rdev->gart.robj == NULL) {
5765 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5768 r = radeon_gart_table_vram_pin(rdev);
5771 /* Setup TLB control */
5772 WREG32(MC_VM_MX_L1_TLB_CNTL,
5775 ENABLE_L1_FRAGMENT_PROCESSING |
5776 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5777 ENABLE_ADVANCED_DRIVER_MODEL |
5778 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5779 /* Setup L2 cache */
5780 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5781 ENABLE_L2_FRAGMENT_PROCESSING |
5782 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5783 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5784 EFFECTIVE_L2_QUEUE_SIZE(7) |
5785 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5786 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5787 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5789 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5790 /* setup context0 */
5791 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5792 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5793 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5794 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5795 (u32)(rdev->dummy_page.addr >> 12));
5796 WREG32(VM_CONTEXT0_CNTL2, 0);
5797 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5798 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5804 /* restore context1-15 */
5805 /* set vm size, must be a multiple of 4 */
5806 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5807 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5808 for (i = 1; i < 16; i++) {
5810 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5811 rdev->vm_manager.saved_table_addr[i]);
5813 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5814 rdev->vm_manager.saved_table_addr[i]);
5817 /* enable context1-15 */
5818 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5819 (u32)(rdev->dummy_page.addr >> 12));
5820 WREG32(VM_CONTEXT1_CNTL2, 4);
5821 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5822 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5823 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5824 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5825 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5826 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5827 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5828 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5829 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5830 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5831 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5832 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5833 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5834 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5836 if (rdev->family == CHIP_KAVERI) {
5837 u32 tmp = RREG32(CHUB_CONTROL);
5839 WREG32(CHUB_CONTROL, tmp);
5842 /* XXX SH_MEM regs */
5843 /* where to put LDS, scratch, GPUVM in FSA64 space */
5844 mutex_lock(&rdev->srbm_mutex);
5845 for (i = 0; i < 16; i++) {
5846 cik_srbm_select(rdev, 0, 0, 0, i);
5847 /* CP and shaders */
5848 WREG32(SH_MEM_CONFIG, 0);
5849 WREG32(SH_MEM_APE1_BASE, 1);
5850 WREG32(SH_MEM_APE1_LIMIT, 0);
5851 WREG32(SH_MEM_BASES, 0);
5853 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5854 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5855 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5856 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5857 /* XXX SDMA RLC - todo */
5859 cik_srbm_select(rdev, 0, 0, 0, 0);
5860 mutex_unlock(&rdev->srbm_mutex);
5862 cik_pcie_init_compute_vmid(rdev);
5864 cik_pcie_gart_tlb_flush(rdev);
5865 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5866 (unsigned)(rdev->mc.gtt_size >> 20),
5867 (unsigned long long)rdev->gart.table_addr);
5868 rdev->gart.ready = true;
5873 * cik_pcie_gart_disable - gart disable
5875 * @rdev: radeon_device pointer
5877 * This disables all VM page table (CIK).
5879 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5883 for (i = 1; i < 16; ++i) {
5886 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5888 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5889 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5892 /* Disable all tables */
5893 WREG32(VM_CONTEXT0_CNTL, 0);
5894 WREG32(VM_CONTEXT1_CNTL, 0);
5895 /* Setup TLB control */
5896 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5897 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5898 /* Setup L2 cache */
5900 ENABLE_L2_FRAGMENT_PROCESSING |
5901 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5902 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5903 EFFECTIVE_L2_QUEUE_SIZE(7) |
5904 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5905 WREG32(VM_L2_CNTL2, 0);
5906 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5907 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5908 radeon_gart_table_vram_unpin(rdev);
5912 * cik_pcie_gart_fini - vm fini callback
5914 * @rdev: radeon_device pointer
5916 * Tears down the driver GART/VM setup (CIK).
5918 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5920 cik_pcie_gart_disable(rdev);
5921 radeon_gart_table_vram_free(rdev);
5922 radeon_gart_fini(rdev);
5927 * cik_ib_parse - vm ib_parse callback
5929 * @rdev: radeon_device pointer
5930 * @ib: indirect buffer pointer
5932 * CIK uses hw IB checking so this is a nop (CIK).
5934 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5941 * VMID 0 is the physical GPU addresses as used by the kernel.
5942 * VMIDs 1-15 are used for userspace clients and are handled
5943 * by the radeon vm/hsa code.
5946 * cik_vm_init - cik vm init callback
5948 * @rdev: radeon_device pointer
5950 * Inits cik specific vm parameters (number of VMs, base of vram for
5951 * VMIDs 1-15) (CIK).
5952 * Returns 0 for success.
5954 int cik_vm_init(struct radeon_device *rdev)
5958 * VMID 0 is reserved for System
5959 * radeon graphics/compute will use VMIDs 1-7
5960 * amdkfd will use VMIDs 8-15
5962 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
5963 /* base offset of vram pages */
5964 if (rdev->flags & RADEON_IS_IGP) {
5965 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5967 rdev->vm_manager.vram_base_offset = tmp;
5969 rdev->vm_manager.vram_base_offset = 0;
5975 * cik_vm_fini - cik vm fini callback
5977 * @rdev: radeon_device pointer
5979 * Tear down any asic specific VM setup (CIK).
5981 void cik_vm_fini(struct radeon_device *rdev)
5986 * cik_vm_decode_fault - print human readable fault info
5988 * @rdev: radeon_device pointer
5989 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5990 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5992 * Print human readable fault information (CIK).
5994 static void cik_vm_decode_fault(struct radeon_device *rdev,
5995 u32 status, u32 addr, u32 mc_client)
5998 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5999 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
6000 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
6001 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
6003 if (rdev->family == CHIP_HAWAII)
6004 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6006 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
6008 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
6009 protections, vmid, addr,
6010 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
6011 block, mc_client, mc_id);
6015 * cik_vm_flush - cik vm flush using the CP
6017 * @rdev: radeon_device pointer
6019 * Update the page table base and flush the VM TLB
6020 * using the CP (CIK).
6022 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
6023 unsigned vm_id, uint64_t pd_addr)
6025 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
6027 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6028 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6029 WRITE_DATA_DST_SEL(0)));
6031 radeon_ring_write(ring,
6032 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
6034 radeon_ring_write(ring,
6035 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
6037 radeon_ring_write(ring, 0);
6038 radeon_ring_write(ring, pd_addr >> 12);
6040 /* update SH_MEM_* regs */
6041 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6042 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6043 WRITE_DATA_DST_SEL(0)));
6044 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6045 radeon_ring_write(ring, 0);
6046 radeon_ring_write(ring, VMID(vm_id));
6048 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
6049 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6050 WRITE_DATA_DST_SEL(0)));
6051 radeon_ring_write(ring, SH_MEM_BASES >> 2);
6052 radeon_ring_write(ring, 0);
6054 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
6055 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6056 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6057 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
6059 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6060 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6061 WRITE_DATA_DST_SEL(0)));
6062 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6063 radeon_ring_write(ring, 0);
6064 radeon_ring_write(ring, VMID(0));
6067 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
6069 /* bits 0-15 are the VM contexts0-15 */
6070 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6071 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6072 WRITE_DATA_DST_SEL(0)));
6073 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6074 radeon_ring_write(ring, 0);
6075 radeon_ring_write(ring, 1 << vm_id);
6077 /* wait for the invalidate to complete */
6078 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6079 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6080 WAIT_REG_MEM_FUNCTION(0) | /* always */
6081 WAIT_REG_MEM_ENGINE(0))); /* me */
6082 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6083 radeon_ring_write(ring, 0);
6084 radeon_ring_write(ring, 0); /* ref */
6085 radeon_ring_write(ring, 0); /* mask */
6086 radeon_ring_write(ring, 0x20); /* poll interval */
6088 /* compute doesn't have PFP */
6090 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6091 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6092 radeon_ring_write(ring, 0x0);
6098 * The RLC is a multi-purpose microengine that handles a
6099 * variety of functions, the most important of which is
6100 * the interrupt controller.
6102 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6105 u32 tmp = RREG32(CP_INT_CNTL_RING0);
6108 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6110 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6111 WREG32(CP_INT_CNTL_RING0, tmp);
6114 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6118 tmp = RREG32(RLC_LB_CNTL);
6120 tmp |= LOAD_BALANCE_ENABLE;
6122 tmp &= ~LOAD_BALANCE_ENABLE;
6123 WREG32(RLC_LB_CNTL, tmp);
6126 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6131 mutex_lock(&rdev->grbm_idx_mutex);
6132 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6133 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6134 cik_select_se_sh(rdev, i, j);
6135 for (k = 0; k < rdev->usec_timeout; k++) {
6136 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6142 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6143 mutex_unlock(&rdev->grbm_idx_mutex);
6145 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6146 for (k = 0; k < rdev->usec_timeout; k++) {
6147 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6153 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6157 tmp = RREG32(RLC_CNTL);
6159 WREG32(RLC_CNTL, rlc);
6162 static u32 cik_halt_rlc(struct radeon_device *rdev)
6166 orig = data = RREG32(RLC_CNTL);
6168 if (data & RLC_ENABLE) {
6171 data &= ~RLC_ENABLE;
6172 WREG32(RLC_CNTL, data);
6174 for (i = 0; i < rdev->usec_timeout; i++) {
6175 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6180 cik_wait_for_rlc_serdes(rdev);
6186 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6190 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6191 WREG32(RLC_GPR_REG2, tmp);
6193 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6194 for (i = 0; i < rdev->usec_timeout; i++) {
6195 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6200 for (i = 0; i < rdev->usec_timeout; i++) {
6201 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6207 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6211 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6212 WREG32(RLC_GPR_REG2, tmp);
6216 * cik_rlc_stop - stop the RLC ME
6218 * @rdev: radeon_device pointer
6220 * Halt the RLC ME (MicroEngine) (CIK).
6222 static void cik_rlc_stop(struct radeon_device *rdev)
6224 WREG32(RLC_CNTL, 0);
6226 cik_enable_gui_idle_interrupt(rdev, false);
6228 cik_wait_for_rlc_serdes(rdev);
6232 * cik_rlc_start - start the RLC ME
6234 * @rdev: radeon_device pointer
6236 * Unhalt the RLC ME (MicroEngine) (CIK).
6238 static void cik_rlc_start(struct radeon_device *rdev)
6240 WREG32(RLC_CNTL, RLC_ENABLE);
6242 cik_enable_gui_idle_interrupt(rdev, true);
6248 * cik_rlc_resume - setup the RLC hw
6250 * @rdev: radeon_device pointer
6252 * Initialize the RLC registers, load the ucode,
6253 * and start the RLC (CIK).
6254 * Returns 0 for success, -EINVAL if the ucode is not available.
6256 static int cik_rlc_resume(struct radeon_device *rdev)
6266 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6267 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6275 WREG32(RLC_LB_CNTR_INIT, 0);
6276 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6278 mutex_lock(&rdev->grbm_idx_mutex);
6279 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6280 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6281 WREG32(RLC_LB_PARAMS, 0x00600408);
6282 WREG32(RLC_LB_CNTL, 0x80000004);
6283 mutex_unlock(&rdev->grbm_idx_mutex);
6285 WREG32(RLC_MC_CNTL, 0);
6286 WREG32(RLC_UCODE_CNTL, 0);
6289 const struct rlc_firmware_header_v1_0 *hdr =
6290 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6291 const __le32 *fw_data = (const __le32 *)
6292 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6294 radeon_ucode_print_rlc_hdr(&hdr->header);
6296 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6297 WREG32(RLC_GPM_UCODE_ADDR, 0);
6298 for (i = 0; i < size; i++)
6299 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6300 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
6302 const __be32 *fw_data;
6304 switch (rdev->family) {
6308 size = BONAIRE_RLC_UCODE_SIZE;
6311 size = KV_RLC_UCODE_SIZE;
6314 size = KB_RLC_UCODE_SIZE;
6317 size = ML_RLC_UCODE_SIZE;
6321 fw_data = (const __be32 *)rdev->rlc_fw->data;
6322 WREG32(RLC_GPM_UCODE_ADDR, 0);
6323 for (i = 0; i < size; i++)
6324 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6325 WREG32(RLC_GPM_UCODE_ADDR, 0);
6328 /* XXX - find out what chips support lbpw */
6329 cik_enable_lbpw(rdev, false);
6331 if (rdev->family == CHIP_BONAIRE)
6332 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6334 cik_rlc_start(rdev);
6339 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6341 u32 data, orig, tmp, tmp2;
6343 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6345 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6346 cik_enable_gui_idle_interrupt(rdev, true);
6348 tmp = cik_halt_rlc(rdev);
6350 mutex_lock(&rdev->grbm_idx_mutex);
6351 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6352 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6353 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6354 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6355 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6356 mutex_unlock(&rdev->grbm_idx_mutex);
6358 cik_update_rlc(rdev, tmp);
6360 data |= CGCG_EN | CGLS_EN;
6362 cik_enable_gui_idle_interrupt(rdev, false);
6364 RREG32(CB_CGTT_SCLK_CTRL);
6365 RREG32(CB_CGTT_SCLK_CTRL);
6366 RREG32(CB_CGTT_SCLK_CTRL);
6367 RREG32(CB_CGTT_SCLK_CTRL);
6369 data &= ~(CGCG_EN | CGLS_EN);
6373 WREG32(RLC_CGCG_CGLS_CTRL, data);
6377 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6379 u32 data, orig, tmp = 0;
6381 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6382 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6383 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6384 orig = data = RREG32(CP_MEM_SLP_CNTL);
6385 data |= CP_MEM_LS_EN;
6387 WREG32(CP_MEM_SLP_CNTL, data);
6391 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6395 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6397 tmp = cik_halt_rlc(rdev);
6399 mutex_lock(&rdev->grbm_idx_mutex);
6400 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6401 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6402 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6403 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6404 WREG32(RLC_SERDES_WR_CTRL, data);
6405 mutex_unlock(&rdev->grbm_idx_mutex);
6407 cik_update_rlc(rdev, tmp);
6409 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6410 orig = data = RREG32(CGTS_SM_CTRL_REG);
6411 data &= ~SM_MODE_MASK;
6412 data |= SM_MODE(0x2);
6413 data |= SM_MODE_ENABLE;
6414 data &= ~CGTS_OVERRIDE;
6415 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6416 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6417 data &= ~CGTS_LS_OVERRIDE;
6418 data &= ~ON_MONITOR_ADD_MASK;
6419 data |= ON_MONITOR_ADD_EN;
6420 data |= ON_MONITOR_ADD(0x96);
6422 WREG32(CGTS_SM_CTRL_REG, data);
6425 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6428 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6430 data = RREG32(RLC_MEM_SLP_CNTL);
6431 if (data & RLC_MEM_LS_EN) {
6432 data &= ~RLC_MEM_LS_EN;
6433 WREG32(RLC_MEM_SLP_CNTL, data);
6436 data = RREG32(CP_MEM_SLP_CNTL);
6437 if (data & CP_MEM_LS_EN) {
6438 data &= ~CP_MEM_LS_EN;
6439 WREG32(CP_MEM_SLP_CNTL, data);
6442 orig = data = RREG32(CGTS_SM_CTRL_REG);
6443 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6445 WREG32(CGTS_SM_CTRL_REG, data);
6447 tmp = cik_halt_rlc(rdev);
6449 mutex_lock(&rdev->grbm_idx_mutex);
6450 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6451 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6452 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6453 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6454 WREG32(RLC_SERDES_WR_CTRL, data);
6455 mutex_unlock(&rdev->grbm_idx_mutex);
6457 cik_update_rlc(rdev, tmp);
6461 static const u32 mc_cg_registers[] =
6474 static void cik_enable_mc_ls(struct radeon_device *rdev,
6480 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6481 orig = data = RREG32(mc_cg_registers[i]);
6482 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6483 data |= MC_LS_ENABLE;
6485 data &= ~MC_LS_ENABLE;
6487 WREG32(mc_cg_registers[i], data);
6491 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6497 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6498 orig = data = RREG32(mc_cg_registers[i]);
6499 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6500 data |= MC_CG_ENABLE;
6502 data &= ~MC_CG_ENABLE;
6504 WREG32(mc_cg_registers[i], data);
6508 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6513 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6514 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6515 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6517 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6520 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6522 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6525 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6529 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6534 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6535 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6538 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6540 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6543 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6545 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6548 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6550 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6553 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6557 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6562 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6563 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6565 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6567 orig = data = RREG32(UVD_CGC_CTRL);
6570 WREG32(UVD_CGC_CTRL, data);
6572 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6574 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6576 orig = data = RREG32(UVD_CGC_CTRL);
6579 WREG32(UVD_CGC_CTRL, data);
6583 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6588 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6590 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6591 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6592 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6594 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6595 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6598 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6601 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6606 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6608 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6609 data &= ~CLOCK_GATING_DIS;
6611 data |= CLOCK_GATING_DIS;
6614 WREG32(HDP_HOST_PATH_CNTL, data);
6617 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6622 orig = data = RREG32(HDP_MEM_POWER_LS);
6624 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6625 data |= HDP_LS_ENABLE;
6627 data &= ~HDP_LS_ENABLE;
6630 WREG32(HDP_MEM_POWER_LS, data);
6633 void cik_update_cg(struct radeon_device *rdev,
6634 u32 block, bool enable)
6637 if (block & RADEON_CG_BLOCK_GFX) {
6638 cik_enable_gui_idle_interrupt(rdev, false);
6639 /* order matters! */
6641 cik_enable_mgcg(rdev, true);
6642 cik_enable_cgcg(rdev, true);
6644 cik_enable_cgcg(rdev, false);
6645 cik_enable_mgcg(rdev, false);
6647 cik_enable_gui_idle_interrupt(rdev, true);
6650 if (block & RADEON_CG_BLOCK_MC) {
6651 if (!(rdev->flags & RADEON_IS_IGP)) {
6652 cik_enable_mc_mgcg(rdev, enable);
6653 cik_enable_mc_ls(rdev, enable);
6657 if (block & RADEON_CG_BLOCK_SDMA) {
6658 cik_enable_sdma_mgcg(rdev, enable);
6659 cik_enable_sdma_mgls(rdev, enable);
6662 if (block & RADEON_CG_BLOCK_BIF) {
6663 cik_enable_bif_mgls(rdev, enable);
6666 if (block & RADEON_CG_BLOCK_UVD) {
6668 cik_enable_uvd_mgcg(rdev, enable);
6671 if (block & RADEON_CG_BLOCK_HDP) {
6672 cik_enable_hdp_mgcg(rdev, enable);
6673 cik_enable_hdp_ls(rdev, enable);
6676 if (block & RADEON_CG_BLOCK_VCE) {
6677 vce_v2_0_enable_mgcg(rdev, enable);
6681 static void cik_init_cg(struct radeon_device *rdev)
6684 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6687 si_init_uvd_internal_cg(rdev);
6689 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6690 RADEON_CG_BLOCK_SDMA |
6691 RADEON_CG_BLOCK_BIF |
6692 RADEON_CG_BLOCK_UVD |
6693 RADEON_CG_BLOCK_HDP), true);
6696 static void cik_fini_cg(struct radeon_device *rdev)
6698 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6699 RADEON_CG_BLOCK_SDMA |
6700 RADEON_CG_BLOCK_BIF |
6701 RADEON_CG_BLOCK_UVD |
6702 RADEON_CG_BLOCK_HDP), false);
6704 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6707 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6712 orig = data = RREG32(RLC_PG_CNTL);
6713 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6714 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6716 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6718 WREG32(RLC_PG_CNTL, data);
6721 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6726 orig = data = RREG32(RLC_PG_CNTL);
6727 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6728 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6730 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6732 WREG32(RLC_PG_CNTL, data);
6735 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6739 orig = data = RREG32(RLC_PG_CNTL);
6740 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6741 data &= ~DISABLE_CP_PG;
6743 data |= DISABLE_CP_PG;
6745 WREG32(RLC_PG_CNTL, data);
6748 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6752 orig = data = RREG32(RLC_PG_CNTL);
6753 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6754 data &= ~DISABLE_GDS_PG;
6756 data |= DISABLE_GDS_PG;
6758 WREG32(RLC_PG_CNTL, data);
6761 #define CP_ME_TABLE_SIZE 96
6762 #define CP_ME_TABLE_OFFSET 2048
6763 #define CP_MEC_TABLE_OFFSET 4096
6765 void cik_init_cp_pg_table(struct radeon_device *rdev)
6767 volatile u32 *dst_ptr;
6768 int me, i, max_me = 4;
6770 u32 table_offset, table_size;
6772 if (rdev->family == CHIP_KAVERI)
6775 if (rdev->rlc.cp_table_ptr == NULL)
6778 /* write the cp table buffer */
6779 dst_ptr = rdev->rlc.cp_table_ptr;
6780 for (me = 0; me < max_me; me++) {
6782 const __le32 *fw_data;
6783 const struct gfx_firmware_header_v1_0 *hdr;
6786 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6787 fw_data = (const __le32 *)
6788 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6789 table_offset = le32_to_cpu(hdr->jt_offset);
6790 table_size = le32_to_cpu(hdr->jt_size);
6791 } else if (me == 1) {
6792 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6793 fw_data = (const __le32 *)
6794 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6795 table_offset = le32_to_cpu(hdr->jt_offset);
6796 table_size = le32_to_cpu(hdr->jt_size);
6797 } else if (me == 2) {
6798 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6799 fw_data = (const __le32 *)
6800 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6801 table_offset = le32_to_cpu(hdr->jt_offset);
6802 table_size = le32_to_cpu(hdr->jt_size);
6803 } else if (me == 3) {
6804 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6805 fw_data = (const __le32 *)
6806 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6807 table_offset = le32_to_cpu(hdr->jt_offset);
6808 table_size = le32_to_cpu(hdr->jt_size);
6810 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6811 fw_data = (const __le32 *)
6812 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6813 table_offset = le32_to_cpu(hdr->jt_offset);
6814 table_size = le32_to_cpu(hdr->jt_size);
6817 for (i = 0; i < table_size; i ++) {
6818 dst_ptr[bo_offset + i] =
6819 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6821 bo_offset += table_size;
6823 const __be32 *fw_data;
6824 table_size = CP_ME_TABLE_SIZE;
6827 fw_data = (const __be32 *)rdev->ce_fw->data;
6828 table_offset = CP_ME_TABLE_OFFSET;
6829 } else if (me == 1) {
6830 fw_data = (const __be32 *)rdev->pfp_fw->data;
6831 table_offset = CP_ME_TABLE_OFFSET;
6832 } else if (me == 2) {
6833 fw_data = (const __be32 *)rdev->me_fw->data;
6834 table_offset = CP_ME_TABLE_OFFSET;
6836 fw_data = (const __be32 *)rdev->mec_fw->data;
6837 table_offset = CP_MEC_TABLE_OFFSET;
6840 for (i = 0; i < table_size; i ++) {
6841 dst_ptr[bo_offset + i] =
6842 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6844 bo_offset += table_size;
6849 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6854 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6855 orig = data = RREG32(RLC_PG_CNTL);
6856 data |= GFX_PG_ENABLE;
6858 WREG32(RLC_PG_CNTL, data);
6860 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6863 WREG32(RLC_AUTO_PG_CTRL, data);
6865 orig = data = RREG32(RLC_PG_CNTL);
6866 data &= ~GFX_PG_ENABLE;
6868 WREG32(RLC_PG_CNTL, data);
6870 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6871 data &= ~AUTO_PG_EN;
6873 WREG32(RLC_AUTO_PG_CTRL, data);
6875 data = RREG32(DB_RENDER_CONTROL);
6879 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6881 u32 mask = 0, tmp, tmp1;
6884 mutex_lock(&rdev->grbm_idx_mutex);
6885 cik_select_se_sh(rdev, se, sh);
6886 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6887 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6888 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6889 mutex_unlock(&rdev->grbm_idx_mutex);
6896 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6901 return (~tmp) & mask;
6904 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6906 u32 i, j, k, active_cu_number = 0;
6907 u32 mask, counter, cu_bitmap;
6910 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6911 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6915 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6916 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6924 active_cu_number += counter;
6925 tmp |= (cu_bitmap << (i * 16 + j * 8));
6929 WREG32(RLC_PG_AO_CU_MASK, tmp);
6931 tmp = RREG32(RLC_MAX_PG_CU);
6932 tmp &= ~MAX_PU_CU_MASK;
6933 tmp |= MAX_PU_CU(active_cu_number);
6934 WREG32(RLC_MAX_PG_CU, tmp);
6937 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6942 orig = data = RREG32(RLC_PG_CNTL);
6943 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6944 data |= STATIC_PER_CU_PG_ENABLE;
6946 data &= ~STATIC_PER_CU_PG_ENABLE;
6948 WREG32(RLC_PG_CNTL, data);
6951 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6956 orig = data = RREG32(RLC_PG_CNTL);
6957 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6958 data |= DYN_PER_CU_PG_ENABLE;
6960 data &= ~DYN_PER_CU_PG_ENABLE;
6962 WREG32(RLC_PG_CNTL, data);
6965 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6966 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6968 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6973 if (rdev->rlc.cs_data) {
6974 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6975 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6976 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6977 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6979 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6980 for (i = 0; i < 3; i++)
6981 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6983 if (rdev->rlc.reg_list) {
6984 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6985 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6986 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6989 orig = data = RREG32(RLC_PG_CNTL);
6992 WREG32(RLC_PG_CNTL, data);
6994 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6995 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6997 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6998 data &= ~IDLE_POLL_COUNT_MASK;
6999 data |= IDLE_POLL_COUNT(0x60);
7000 WREG32(CP_RB_WPTR_POLL_CNTL, data);
7003 WREG32(RLC_PG_DELAY, data);
7005 data = RREG32(RLC_PG_DELAY_2);
7008 WREG32(RLC_PG_DELAY_2, data);
7010 data = RREG32(RLC_AUTO_PG_CTRL);
7011 data &= ~GRBM_REG_SGIT_MASK;
7012 data |= GRBM_REG_SGIT(0x700);
7013 WREG32(RLC_AUTO_PG_CTRL, data);
7017 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
7019 cik_enable_gfx_cgpg(rdev, enable);
7020 cik_enable_gfx_static_mgpg(rdev, enable);
7021 cik_enable_gfx_dynamic_mgpg(rdev, enable);
7024 u32 cik_get_csb_size(struct radeon_device *rdev)
7027 const struct cs_section_def *sect = NULL;
7028 const struct cs_extent_def *ext = NULL;
7030 if (rdev->rlc.cs_data == NULL)
7033 /* begin clear state */
7035 /* context control state */
7038 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7039 for (ext = sect->section; ext->extent != NULL; ++ext) {
7040 if (sect->id == SECT_CONTEXT)
7041 count += 2 + ext->reg_count;
7046 /* pa_sc_raster_config/pa_sc_raster_config1 */
7048 /* end clear state */
7056 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
7059 const struct cs_section_def *sect = NULL;
7060 const struct cs_extent_def *ext = NULL;
7062 if (rdev->rlc.cs_data == NULL)
7067 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7068 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
7070 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
7071 buffer[count++] = cpu_to_le32(0x80000000);
7072 buffer[count++] = cpu_to_le32(0x80000000);
7074 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7075 for (ext = sect->section; ext->extent != NULL; ++ext) {
7076 if (sect->id == SECT_CONTEXT) {
7078 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
7079 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
7080 for (i = 0; i < ext->reg_count; i++)
7081 buffer[count++] = cpu_to_le32(ext->extent[i]);
7088 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7089 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
7090 switch (rdev->family) {
7092 buffer[count++] = cpu_to_le32(0x16000012);
7093 buffer[count++] = cpu_to_le32(0x00000000);
7096 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7097 buffer[count++] = cpu_to_le32(0x00000000);
7101 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7102 buffer[count++] = cpu_to_le32(0x00000000);
7105 buffer[count++] = cpu_to_le32(0x3a00161a);
7106 buffer[count++] = cpu_to_le32(0x0000002e);
7109 buffer[count++] = cpu_to_le32(0x00000000);
7110 buffer[count++] = cpu_to_le32(0x00000000);
7114 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7115 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7117 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7118 buffer[count++] = cpu_to_le32(0);
7121 static void cik_init_pg(struct radeon_device *rdev)
7123 if (rdev->pg_flags) {
7124 cik_enable_sck_slowdown_on_pu(rdev, true);
7125 cik_enable_sck_slowdown_on_pd(rdev, true);
7126 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7127 cik_init_gfx_cgpg(rdev);
7128 cik_enable_cp_pg(rdev, true);
7129 cik_enable_gds_pg(rdev, true);
7131 cik_init_ao_cu_mask(rdev);
7132 cik_update_gfx_pg(rdev, true);
7136 static void cik_fini_pg(struct radeon_device *rdev)
7138 if (rdev->pg_flags) {
7139 cik_update_gfx_pg(rdev, false);
7140 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7141 cik_enable_cp_pg(rdev, false);
7142 cik_enable_gds_pg(rdev, false);
7149 * Starting with r6xx, interrupts are handled via a ring buffer.
7150 * Ring buffers are areas of GPU accessible memory that the GPU
7151 * writes interrupt vectors into and the host reads vectors out of.
7152 * There is a rptr (read pointer) that determines where the
7153 * host is currently reading, and a wptr (write pointer)
7154 * which determines where the GPU has written. When the
7155 * pointers are equal, the ring is idle. When the GPU
7156 * writes vectors to the ring buffer, it increments the
7157 * wptr. When there is an interrupt, the host then starts
7158 * fetching commands and processing them until the pointers are
7159 * equal again at which point it updates the rptr.
7163 * cik_enable_interrupts - Enable the interrupt ring buffer
7165 * @rdev: radeon_device pointer
7167 * Enable the interrupt ring buffer (CIK).
7169 static void cik_enable_interrupts(struct radeon_device *rdev)
7171 u32 ih_cntl = RREG32(IH_CNTL);
7172 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7174 ih_cntl |= ENABLE_INTR;
7175 ih_rb_cntl |= IH_RB_ENABLE;
7176 WREG32(IH_CNTL, ih_cntl);
7177 WREG32(IH_RB_CNTL, ih_rb_cntl);
7178 rdev->ih.enabled = true;
7182 * cik_disable_interrupts - Disable the interrupt ring buffer
7184 * @rdev: radeon_device pointer
7186 * Disable the interrupt ring buffer (CIK).
7188 static void cik_disable_interrupts(struct radeon_device *rdev)
7190 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7191 u32 ih_cntl = RREG32(IH_CNTL);
7193 ih_rb_cntl &= ~IH_RB_ENABLE;
7194 ih_cntl &= ~ENABLE_INTR;
7195 WREG32(IH_RB_CNTL, ih_rb_cntl);
7196 WREG32(IH_CNTL, ih_cntl);
7197 /* set rptr, wptr to 0 */
7198 WREG32(IH_RB_RPTR, 0);
7199 WREG32(IH_RB_WPTR, 0);
7200 rdev->ih.enabled = false;
7205 * cik_disable_interrupt_state - Disable all interrupt sources
7207 * @rdev: radeon_device pointer
7209 * Clear all interrupt enable bits used by the driver (CIK).
7211 static void cik_disable_interrupt_state(struct radeon_device *rdev)
7216 tmp = RREG32(CP_INT_CNTL_RING0) &
7217 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7218 WREG32(CP_INT_CNTL_RING0, tmp);
7220 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7221 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7222 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7223 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7224 /* compute queues */
7225 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7226 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7227 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7228 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7229 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7230 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7231 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7232 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7234 WREG32(GRBM_INT_CNTL, 0);
7236 WREG32(SRBM_INT_CNTL, 0);
7237 /* vline/vblank, etc. */
7238 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7239 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7240 if (rdev->num_crtc >= 4) {
7241 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7242 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7244 if (rdev->num_crtc >= 6) {
7245 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7246 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7249 if (rdev->num_crtc >= 2) {
7250 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7251 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7253 if (rdev->num_crtc >= 4) {
7254 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7255 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7257 if (rdev->num_crtc >= 6) {
7258 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7259 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7263 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7265 /* digital hotplug */
7266 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7267 WREG32(DC_HPD1_INT_CONTROL, tmp);
7268 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7269 WREG32(DC_HPD2_INT_CONTROL, tmp);
7270 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7271 WREG32(DC_HPD3_INT_CONTROL, tmp);
7272 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7273 WREG32(DC_HPD4_INT_CONTROL, tmp);
7274 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7275 WREG32(DC_HPD5_INT_CONTROL, tmp);
7276 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7277 WREG32(DC_HPD6_INT_CONTROL, tmp);
7282 * cik_irq_init - init and enable the interrupt ring
7284 * @rdev: radeon_device pointer
7286 * Allocate a ring buffer for the interrupt controller,
7287 * enable the RLC, disable interrupts, enable the IH
7288 * ring buffer and enable it (CIK).
7289 * Called at device load and reume.
7290 * Returns 0 for success, errors for failure.
7292 static int cik_irq_init(struct radeon_device *rdev)
7296 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7299 ret = r600_ih_ring_alloc(rdev);
7304 cik_disable_interrupts(rdev);
7307 ret = cik_rlc_resume(rdev);
7309 r600_ih_ring_fini(rdev);
7313 /* setup interrupt control */
7314 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7315 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7316 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7317 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7318 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7320 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7321 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7322 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7323 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7325 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7326 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7328 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7329 IH_WPTR_OVERFLOW_CLEAR |
7332 if (rdev->wb.enabled)
7333 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7335 /* set the writeback address whether it's enabled or not */
7336 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7337 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7339 WREG32(IH_RB_CNTL, ih_rb_cntl);
7341 /* set rptr, wptr to 0 */
7342 WREG32(IH_RB_RPTR, 0);
7343 WREG32(IH_RB_WPTR, 0);
7345 /* Default settings for IH_CNTL (disabled at first) */
7346 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7347 /* RPTR_REARM only works if msi's are enabled */
7348 if (rdev->msi_enabled)
7349 ih_cntl |= RPTR_REARM;
7350 WREG32(IH_CNTL, ih_cntl);
7352 /* force the active interrupt state to all disabled */
7353 cik_disable_interrupt_state(rdev);
7355 pci_set_master(rdev->pdev);
7358 cik_enable_interrupts(rdev);
7364 * cik_irq_set - enable/disable interrupt sources
7366 * @rdev: radeon_device pointer
7368 * Enable interrupt sources on the GPU (vblanks, hpd,
7370 * Returns 0 for success, errors for failure.
7372 int cik_irq_set(struct radeon_device *rdev)
7376 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7377 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7378 u32 grbm_int_cntl = 0;
7379 u32 dma_cntl, dma_cntl1;
7381 if (!rdev->irq.installed) {
7382 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7385 /* don't enable anything if the ih is disabled */
7386 if (!rdev->ih.enabled) {
7387 cik_disable_interrupts(rdev);
7388 /* force the active interrupt state to all disabled */
7389 cik_disable_interrupt_state(rdev);
7393 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7394 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7395 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7397 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7398 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7399 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7400 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7401 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7402 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7404 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7405 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7407 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7409 /* enable CP interrupts on all rings */
7410 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7411 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7412 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7414 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7415 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7416 DRM_DEBUG("si_irq_set: sw int cp1\n");
7417 if (ring->me == 1) {
7418 switch (ring->pipe) {
7420 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7423 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7427 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7430 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7431 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7432 DRM_DEBUG("si_irq_set: sw int cp2\n");
7433 if (ring->me == 1) {
7434 switch (ring->pipe) {
7436 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7439 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7443 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7447 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7448 DRM_DEBUG("cik_irq_set: sw int dma\n");
7449 dma_cntl |= TRAP_ENABLE;
7452 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7453 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7454 dma_cntl1 |= TRAP_ENABLE;
7457 if (rdev->irq.crtc_vblank_int[0] ||
7458 atomic_read(&rdev->irq.pflip[0])) {
7459 DRM_DEBUG("cik_irq_set: vblank 0\n");
7460 crtc1 |= VBLANK_INTERRUPT_MASK;
7462 if (rdev->irq.crtc_vblank_int[1] ||
7463 atomic_read(&rdev->irq.pflip[1])) {
7464 DRM_DEBUG("cik_irq_set: vblank 1\n");
7465 crtc2 |= VBLANK_INTERRUPT_MASK;
7467 if (rdev->irq.crtc_vblank_int[2] ||
7468 atomic_read(&rdev->irq.pflip[2])) {
7469 DRM_DEBUG("cik_irq_set: vblank 2\n");
7470 crtc3 |= VBLANK_INTERRUPT_MASK;
7472 if (rdev->irq.crtc_vblank_int[3] ||
7473 atomic_read(&rdev->irq.pflip[3])) {
7474 DRM_DEBUG("cik_irq_set: vblank 3\n");
7475 crtc4 |= VBLANK_INTERRUPT_MASK;
7477 if (rdev->irq.crtc_vblank_int[4] ||
7478 atomic_read(&rdev->irq.pflip[4])) {
7479 DRM_DEBUG("cik_irq_set: vblank 4\n");
7480 crtc5 |= VBLANK_INTERRUPT_MASK;
7482 if (rdev->irq.crtc_vblank_int[5] ||
7483 atomic_read(&rdev->irq.pflip[5])) {
7484 DRM_DEBUG("cik_irq_set: vblank 5\n");
7485 crtc6 |= VBLANK_INTERRUPT_MASK;
7487 if (rdev->irq.hpd[0]) {
7488 DRM_DEBUG("cik_irq_set: hpd 1\n");
7489 hpd1 |= DC_HPDx_INT_EN;
7491 if (rdev->irq.hpd[1]) {
7492 DRM_DEBUG("cik_irq_set: hpd 2\n");
7493 hpd2 |= DC_HPDx_INT_EN;
7495 if (rdev->irq.hpd[2]) {
7496 DRM_DEBUG("cik_irq_set: hpd 3\n");
7497 hpd3 |= DC_HPDx_INT_EN;
7499 if (rdev->irq.hpd[3]) {
7500 DRM_DEBUG("cik_irq_set: hpd 4\n");
7501 hpd4 |= DC_HPDx_INT_EN;
7503 if (rdev->irq.hpd[4]) {
7504 DRM_DEBUG("cik_irq_set: hpd 5\n");
7505 hpd5 |= DC_HPDx_INT_EN;
7507 if (rdev->irq.hpd[5]) {
7508 DRM_DEBUG("cik_irq_set: hpd 6\n");
7509 hpd6 |= DC_HPDx_INT_EN;
7512 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7514 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7515 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7517 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7519 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7521 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7522 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7523 if (rdev->num_crtc >= 4) {
7524 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7525 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7527 if (rdev->num_crtc >= 6) {
7528 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7529 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7532 if (rdev->num_crtc >= 2) {
7533 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7534 GRPH_PFLIP_INT_MASK);
7535 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7536 GRPH_PFLIP_INT_MASK);
7538 if (rdev->num_crtc >= 4) {
7539 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7540 GRPH_PFLIP_INT_MASK);
7541 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7542 GRPH_PFLIP_INT_MASK);
7544 if (rdev->num_crtc >= 6) {
7545 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7546 GRPH_PFLIP_INT_MASK);
7547 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7548 GRPH_PFLIP_INT_MASK);
7551 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7552 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7553 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7554 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7555 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7556 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7562 * cik_irq_ack - ack interrupt sources
7564 * @rdev: radeon_device pointer
7566 * Ack interrupt sources on the GPU (vblanks, hpd,
7567 * etc.) (CIK). Certain interrupts sources are sw
7568 * generated and do not require an explicit ack.
7570 static inline void cik_irq_ack(struct radeon_device *rdev)
7574 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7575 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7576 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7577 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7578 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7579 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7580 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7582 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7583 EVERGREEN_CRTC0_REGISTER_OFFSET);
7584 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7585 EVERGREEN_CRTC1_REGISTER_OFFSET);
7586 if (rdev->num_crtc >= 4) {
7587 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7588 EVERGREEN_CRTC2_REGISTER_OFFSET);
7589 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7590 EVERGREEN_CRTC3_REGISTER_OFFSET);
7592 if (rdev->num_crtc >= 6) {
7593 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7594 EVERGREEN_CRTC4_REGISTER_OFFSET);
7595 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7596 EVERGREEN_CRTC5_REGISTER_OFFSET);
7599 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7600 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7601 GRPH_PFLIP_INT_CLEAR);
7602 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7603 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7604 GRPH_PFLIP_INT_CLEAR);
7605 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7606 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7607 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7608 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7609 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7610 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7611 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7612 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7614 if (rdev->num_crtc >= 4) {
7615 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7616 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7617 GRPH_PFLIP_INT_CLEAR);
7618 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7619 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7620 GRPH_PFLIP_INT_CLEAR);
7621 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7622 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7623 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7624 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7625 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7626 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7627 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7628 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7631 if (rdev->num_crtc >= 6) {
7632 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7633 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7634 GRPH_PFLIP_INT_CLEAR);
7635 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7637 GRPH_PFLIP_INT_CLEAR);
7638 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7639 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7640 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7641 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7642 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7643 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7644 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7645 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7648 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7649 tmp = RREG32(DC_HPD1_INT_CONTROL);
7650 tmp |= DC_HPDx_INT_ACK;
7651 WREG32(DC_HPD1_INT_CONTROL, tmp);
7653 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7654 tmp = RREG32(DC_HPD2_INT_CONTROL);
7655 tmp |= DC_HPDx_INT_ACK;
7656 WREG32(DC_HPD2_INT_CONTROL, tmp);
7658 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7659 tmp = RREG32(DC_HPD3_INT_CONTROL);
7660 tmp |= DC_HPDx_INT_ACK;
7661 WREG32(DC_HPD3_INT_CONTROL, tmp);
7663 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7664 tmp = RREG32(DC_HPD4_INT_CONTROL);
7665 tmp |= DC_HPDx_INT_ACK;
7666 WREG32(DC_HPD4_INT_CONTROL, tmp);
7668 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7669 tmp = RREG32(DC_HPD5_INT_CONTROL);
7670 tmp |= DC_HPDx_INT_ACK;
7671 WREG32(DC_HPD5_INT_CONTROL, tmp);
7673 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7674 tmp = RREG32(DC_HPD5_INT_CONTROL);
7675 tmp |= DC_HPDx_INT_ACK;
7676 WREG32(DC_HPD6_INT_CONTROL, tmp);
7681 * cik_irq_disable - disable interrupts
7683 * @rdev: radeon_device pointer
7685 * Disable interrupts on the hw (CIK).
7687 static void cik_irq_disable(struct radeon_device *rdev)
7689 cik_disable_interrupts(rdev);
7690 /* Wait and acknowledge irq */
7693 cik_disable_interrupt_state(rdev);
7697 * cik_irq_disable - disable interrupts for suspend
7699 * @rdev: radeon_device pointer
7701 * Disable interrupts and stop the RLC (CIK).
7704 static void cik_irq_suspend(struct radeon_device *rdev)
7706 cik_irq_disable(rdev);
7711 * cik_irq_fini - tear down interrupt support
7713 * @rdev: radeon_device pointer
7715 * Disable interrupts on the hw and free the IH ring
7717 * Used for driver unload.
7719 static void cik_irq_fini(struct radeon_device *rdev)
7721 cik_irq_suspend(rdev);
7722 r600_ih_ring_fini(rdev);
7726 * cik_get_ih_wptr - get the IH ring buffer wptr
7728 * @rdev: radeon_device pointer
7730 * Get the IH ring buffer wptr from either the register
7731 * or the writeback memory buffer (CIK). Also check for
7732 * ring buffer overflow and deal with it.
7733 * Used by cik_irq_process().
7734 * Returns the value of the wptr.
7736 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7740 if (rdev->wb.enabled)
7741 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7743 wptr = RREG32(IH_RB_WPTR);
7745 if (wptr & RB_OVERFLOW) {
7746 wptr &= ~RB_OVERFLOW;
7747 /* When a ring buffer overflow happen start parsing interrupt
7748 * from the last not overwritten vector (wptr + 16). Hopefully
7749 * this should allow us to catchup.
7751 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7752 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7753 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7754 tmp = RREG32(IH_RB_CNTL);
7755 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7756 WREG32(IH_RB_CNTL, tmp);
7758 return (wptr & rdev->ih.ptr_mask);
7762 * Each IV ring entry is 128 bits:
7763 * [7:0] - interrupt source id
7765 * [59:32] - interrupt source data
7766 * [63:60] - reserved
7769 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7770 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7771 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7772 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7773 * PIPE_ID - ME0 0=3D
7774 * - ME1&2 compute dispatcher (4 pipes each)
7776 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7777 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7778 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7781 * [127:96] - reserved
7784 * cik_irq_process - interrupt handler
7786 * @rdev: radeon_device pointer
7788 * Interrupt hander (CIK). Walk the IH ring,
7789 * ack interrupts and schedule work to handle
7791 * Returns irq process return code.
7793 int cik_irq_process(struct radeon_device *rdev)
7795 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7796 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7799 u32 src_id, src_data, ring_id;
7800 u8 me_id, pipe_id, queue_id;
7802 bool queue_hotplug = false;
7803 bool queue_reset = false;
7804 u32 addr, status, mc_client;
7805 bool queue_thermal = false;
7807 if (!rdev->ih.enabled || rdev->shutdown)
7810 wptr = cik_get_ih_wptr(rdev);
7813 /* is somebody else already processing irqs? */
7814 if (atomic_xchg(&rdev->ih.lock, 1))
7817 rptr = rdev->ih.rptr;
7818 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7820 /* Order reading of wptr vs. reading of IH ring data */
7823 /* display interrupts */
7826 while (rptr != wptr) {
7827 /* wptr/rptr are in bytes! */
7828 ring_index = rptr / 4;
7830 radeon_kfd_interrupt(rdev,
7831 (const void *) &rdev->ih.ring[ring_index]);
7833 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7834 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7835 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7838 case 1: /* D1 vblank/vline */
7840 case 0: /* D1 vblank */
7841 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7842 if (rdev->irq.crtc_vblank_int[0]) {
7843 drm_handle_vblank(rdev->ddev, 0);
7844 rdev->pm.vblank_sync = true;
7845 wake_up(&rdev->irq.vblank_queue);
7847 if (atomic_read(&rdev->irq.pflip[0]))
7848 radeon_crtc_handle_vblank(rdev, 0);
7849 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7850 DRM_DEBUG("IH: D1 vblank\n");
7853 case 1: /* D1 vline */
7854 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7855 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7856 DRM_DEBUG("IH: D1 vline\n");
7860 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7864 case 2: /* D2 vblank/vline */
7866 case 0: /* D2 vblank */
7867 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7868 if (rdev->irq.crtc_vblank_int[1]) {
7869 drm_handle_vblank(rdev->ddev, 1);
7870 rdev->pm.vblank_sync = true;
7871 wake_up(&rdev->irq.vblank_queue);
7873 if (atomic_read(&rdev->irq.pflip[1]))
7874 radeon_crtc_handle_vblank(rdev, 1);
7875 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7876 DRM_DEBUG("IH: D2 vblank\n");
7879 case 1: /* D2 vline */
7880 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7881 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7882 DRM_DEBUG("IH: D2 vline\n");
7886 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7890 case 3: /* D3 vblank/vline */
7892 case 0: /* D3 vblank */
7893 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7894 if (rdev->irq.crtc_vblank_int[2]) {
7895 drm_handle_vblank(rdev->ddev, 2);
7896 rdev->pm.vblank_sync = true;
7897 wake_up(&rdev->irq.vblank_queue);
7899 if (atomic_read(&rdev->irq.pflip[2]))
7900 radeon_crtc_handle_vblank(rdev, 2);
7901 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7902 DRM_DEBUG("IH: D3 vblank\n");
7905 case 1: /* D3 vline */
7906 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7907 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7908 DRM_DEBUG("IH: D3 vline\n");
7912 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7916 case 4: /* D4 vblank/vline */
7918 case 0: /* D4 vblank */
7919 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7920 if (rdev->irq.crtc_vblank_int[3]) {
7921 drm_handle_vblank(rdev->ddev, 3);
7922 rdev->pm.vblank_sync = true;
7923 wake_up(&rdev->irq.vblank_queue);
7925 if (atomic_read(&rdev->irq.pflip[3]))
7926 radeon_crtc_handle_vblank(rdev, 3);
7927 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7928 DRM_DEBUG("IH: D4 vblank\n");
7931 case 1: /* D4 vline */
7932 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7933 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7934 DRM_DEBUG("IH: D4 vline\n");
7938 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7942 case 5: /* D5 vblank/vline */
7944 case 0: /* D5 vblank */
7945 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7946 if (rdev->irq.crtc_vblank_int[4]) {
7947 drm_handle_vblank(rdev->ddev, 4);
7948 rdev->pm.vblank_sync = true;
7949 wake_up(&rdev->irq.vblank_queue);
7951 if (atomic_read(&rdev->irq.pflip[4]))
7952 radeon_crtc_handle_vblank(rdev, 4);
7953 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7954 DRM_DEBUG("IH: D5 vblank\n");
7957 case 1: /* D5 vline */
7958 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7959 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7960 DRM_DEBUG("IH: D5 vline\n");
7964 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7968 case 6: /* D6 vblank/vline */
7970 case 0: /* D6 vblank */
7971 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7972 if (rdev->irq.crtc_vblank_int[5]) {
7973 drm_handle_vblank(rdev->ddev, 5);
7974 rdev->pm.vblank_sync = true;
7975 wake_up(&rdev->irq.vblank_queue);
7977 if (atomic_read(&rdev->irq.pflip[5]))
7978 radeon_crtc_handle_vblank(rdev, 5);
7979 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7980 DRM_DEBUG("IH: D6 vblank\n");
7983 case 1: /* D6 vline */
7984 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7985 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7986 DRM_DEBUG("IH: D6 vline\n");
7990 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7994 case 8: /* D1 page flip */
7995 case 10: /* D2 page flip */
7996 case 12: /* D3 page flip */
7997 case 14: /* D4 page flip */
7998 case 16: /* D5 page flip */
7999 case 18: /* D6 page flip */
8000 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
8001 if (radeon_use_pflipirq > 0)
8002 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
8004 case 42: /* HPD hotplug */
8007 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8008 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8009 queue_hotplug = true;
8010 DRM_DEBUG("IH: HPD1\n");
8014 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8015 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8016 queue_hotplug = true;
8017 DRM_DEBUG("IH: HPD2\n");
8021 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8022 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8023 queue_hotplug = true;
8024 DRM_DEBUG("IH: HPD3\n");
8028 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8029 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8030 queue_hotplug = true;
8031 DRM_DEBUG("IH: HPD4\n");
8035 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8036 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8037 queue_hotplug = true;
8038 DRM_DEBUG("IH: HPD5\n");
8042 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8043 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8044 queue_hotplug = true;
8045 DRM_DEBUG("IH: HPD6\n");
8049 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8054 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
8055 WREG32(SRBM_INT_ACK, 0x1);
8058 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8059 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8063 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8064 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8065 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8066 /* reset addr and status */
8067 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8068 if (addr == 0x0 && status == 0x0)
8070 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8071 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
8073 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8075 cik_vm_decode_fault(rdev, status, addr, mc_client);
8078 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8081 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8084 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8087 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8091 case 176: /* GFX RB CP_INT */
8092 case 177: /* GFX IB CP_INT */
8093 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8095 case 181: /* CP EOP event */
8096 DRM_DEBUG("IH: CP EOP\n");
8097 /* XXX check the bitfield order! */
8098 me_id = (ring_id & 0x60) >> 5;
8099 pipe_id = (ring_id & 0x18) >> 3;
8100 queue_id = (ring_id & 0x7) >> 0;
8103 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8107 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8108 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8109 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8110 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8114 case 184: /* CP Privileged reg access */
8115 DRM_ERROR("Illegal register access in command stream\n");
8116 /* XXX check the bitfield order! */
8117 me_id = (ring_id & 0x60) >> 5;
8118 pipe_id = (ring_id & 0x18) >> 3;
8119 queue_id = (ring_id & 0x7) >> 0;
8122 /* This results in a full GPU reset, but all we need to do is soft
8123 * reset the CP for gfx
8137 case 185: /* CP Privileged inst */
8138 DRM_ERROR("Illegal instruction in command stream\n");
8139 /* XXX check the bitfield order! */
8140 me_id = (ring_id & 0x60) >> 5;
8141 pipe_id = (ring_id & 0x18) >> 3;
8142 queue_id = (ring_id & 0x7) >> 0;
8145 /* This results in a full GPU reset, but all we need to do is soft
8146 * reset the CP for gfx
8160 case 224: /* SDMA trap event */
8161 /* XXX check the bitfield order! */
8162 me_id = (ring_id & 0x3) >> 0;
8163 queue_id = (ring_id & 0xc) >> 2;
8164 DRM_DEBUG("IH: SDMA trap\n");
8169 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8182 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8194 case 230: /* thermal low to high */
8195 DRM_DEBUG("IH: thermal low to high\n");
8196 rdev->pm.dpm.thermal.high_to_low = false;
8197 queue_thermal = true;
8199 case 231: /* thermal high to low */
8200 DRM_DEBUG("IH: thermal high to low\n");
8201 rdev->pm.dpm.thermal.high_to_low = true;
8202 queue_thermal = true;
8204 case 233: /* GUI IDLE */
8205 DRM_DEBUG("IH: GUI idle\n");
8207 case 241: /* SDMA Privileged inst */
8208 case 247: /* SDMA Privileged inst */
8209 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8210 /* XXX check the bitfield order! */
8211 me_id = (ring_id & 0x3) >> 0;
8212 queue_id = (ring_id & 0xc) >> 2;
8247 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8251 /* wptr/rptr are in bytes! */
8253 rptr &= rdev->ih.ptr_mask;
8254 WREG32(IH_RB_RPTR, rptr);
8257 schedule_work(&rdev->hotplug_work);
8259 rdev->needs_reset = true;
8260 wake_up_all(&rdev->fence_queue);
8263 schedule_work(&rdev->pm.dpm.thermal.work);
8264 rdev->ih.rptr = rptr;
8265 atomic_set(&rdev->ih.lock, 0);
8267 /* make sure wptr hasn't changed while processing */
8268 wptr = cik_get_ih_wptr(rdev);
8276 * startup/shutdown callbacks
8279 * cik_startup - program the asic to a functional state
8281 * @rdev: radeon_device pointer
8283 * Programs the asic to a functional state (CIK).
8284 * Called by cik_init() and cik_resume().
8285 * Returns 0 for success, error for failure.
8287 static int cik_startup(struct radeon_device *rdev)
8289 struct radeon_ring *ring;
8293 /* enable pcie gen2/3 link */
8294 cik_pcie_gen3_enable(rdev);
8296 cik_program_aspm(rdev);
8298 /* scratch needs to be initialized before MC */
8299 r = r600_vram_scratch_init(rdev);
8303 cik_mc_program(rdev);
8305 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8306 r = ci_mc_load_microcode(rdev);
8308 DRM_ERROR("Failed to load MC firmware!\n");
8313 r = cik_pcie_gart_enable(rdev);
8318 /* allocate rlc buffers */
8319 if (rdev->flags & RADEON_IS_IGP) {
8320 if (rdev->family == CHIP_KAVERI) {
8321 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8322 rdev->rlc.reg_list_size =
8323 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8325 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8326 rdev->rlc.reg_list_size =
8327 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8330 rdev->rlc.cs_data = ci_cs_data;
8331 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8332 r = sumo_rlc_init(rdev);
8334 DRM_ERROR("Failed to init rlc BOs!\n");
8338 /* allocate wb buffer */
8339 r = radeon_wb_init(rdev);
8343 /* allocate mec buffers */
8344 r = cik_mec_init(rdev);
8346 DRM_ERROR("Failed to init MEC BOs!\n");
8350 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8352 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8356 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8358 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8362 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8364 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8368 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8370 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8374 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8376 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8380 r = radeon_uvd_resume(rdev);
8382 r = uvd_v4_2_resume(rdev);
8384 r = radeon_fence_driver_start_ring(rdev,
8385 R600_RING_TYPE_UVD_INDEX);
8387 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8391 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8393 r = radeon_vce_resume(rdev);
8395 r = vce_v2_0_resume(rdev);
8397 r = radeon_fence_driver_start_ring(rdev,
8398 TN_RING_TYPE_VCE1_INDEX);
8400 r = radeon_fence_driver_start_ring(rdev,
8401 TN_RING_TYPE_VCE2_INDEX);
8404 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8405 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8406 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8410 if (!rdev->irq.installed) {
8411 r = radeon_irq_kms_init(rdev);
8416 r = cik_irq_init(rdev);
8418 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8419 radeon_irq_kms_fini(rdev);
8424 if (rdev->family == CHIP_HAWAII) {
8426 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8428 nop = RADEON_CP_PACKET2;
8430 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8433 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8434 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8439 /* set up the compute queues */
8440 /* type-2 packets are deprecated on MEC, use type-3 instead */
8441 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8442 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8446 ring->me = 1; /* first MEC */
8447 ring->pipe = 0; /* first pipe */
8448 ring->queue = 0; /* first queue */
8449 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8451 /* type-2 packets are deprecated on MEC, use type-3 instead */
8452 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8453 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8457 /* dGPU only have 1 MEC */
8458 ring->me = 1; /* first MEC */
8459 ring->pipe = 0; /* first pipe */
8460 ring->queue = 1; /* second queue */
8461 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8463 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8464 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8465 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8469 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8470 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8471 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8475 r = cik_cp_resume(rdev);
8479 r = cik_sdma_resume(rdev);
8483 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8484 if (ring->ring_size) {
8485 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8488 r = uvd_v1_0_init(rdev);
8490 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8495 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8496 if (ring->ring_size)
8497 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8500 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8501 if (ring->ring_size)
8502 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8506 r = vce_v1_0_init(rdev);
8507 else if (r != -ENOENT)
8508 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8510 r = radeon_ib_pool_init(rdev);
8512 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8516 r = radeon_vm_manager_init(rdev);
8518 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8522 r = radeon_audio_init(rdev);
8526 r = radeon_kfd_resume(rdev);
8534 * cik_resume - resume the asic to a functional state
8536 * @rdev: radeon_device pointer
8538 * Programs the asic to a functional state (CIK).
8540 * Returns 0 for success, error for failure.
8542 int cik_resume(struct radeon_device *rdev)
8547 atom_asic_init(rdev->mode_info.atom_context);
8549 /* init golden registers */
8550 cik_init_golden_registers(rdev);
8552 if (rdev->pm.pm_method == PM_METHOD_DPM)
8553 radeon_pm_resume(rdev);
8555 rdev->accel_working = true;
8556 r = cik_startup(rdev);
8558 DRM_ERROR("cik startup failed on resume\n");
8559 rdev->accel_working = false;
8568 * cik_suspend - suspend the asic
8570 * @rdev: radeon_device pointer
8572 * Bring the chip into a state suitable for suspend (CIK).
8573 * Called at suspend.
8574 * Returns 0 for success.
8576 int cik_suspend(struct radeon_device *rdev)
8578 radeon_kfd_suspend(rdev);
8579 radeon_pm_suspend(rdev);
8580 radeon_audio_fini(rdev);
8581 radeon_vm_manager_fini(rdev);
8582 cik_cp_enable(rdev, false);
8583 cik_sdma_enable(rdev, false);
8584 uvd_v1_0_fini(rdev);
8585 radeon_uvd_suspend(rdev);
8586 radeon_vce_suspend(rdev);
8589 cik_irq_suspend(rdev);
8590 radeon_wb_disable(rdev);
8591 cik_pcie_gart_disable(rdev);
8595 /* Plan is to move initialization in that function and use
8596 * helper function so that radeon_device_init pretty much
8597 * do nothing more than calling asic specific function. This
8598 * should also allow to remove a bunch of callback function
8602 * cik_init - asic specific driver and hw init
8604 * @rdev: radeon_device pointer
8606 * Setup asic specific driver variables and program the hw
8607 * to a functional state (CIK).
8608 * Called at driver startup.
8609 * Returns 0 for success, errors for failure.
8611 int cik_init(struct radeon_device *rdev)
8613 struct radeon_ring *ring;
8617 if (!radeon_get_bios(rdev)) {
8618 if (ASIC_IS_AVIVO(rdev))
8621 /* Must be an ATOMBIOS */
8622 if (!rdev->is_atom_bios) {
8623 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8626 r = radeon_atombios_init(rdev);
8630 /* Post card if necessary */
8631 if (!radeon_card_posted(rdev)) {
8633 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8636 DRM_INFO("GPU not posted. posting now...\n");
8637 atom_asic_init(rdev->mode_info.atom_context);
8639 /* init golden registers */
8640 cik_init_golden_registers(rdev);
8641 /* Initialize scratch registers */
8642 cik_scratch_init(rdev);
8643 /* Initialize surface registers */
8644 radeon_surface_init(rdev);
8645 /* Initialize clocks */
8646 radeon_get_clock_info(rdev->ddev);
8649 r = radeon_fence_driver_init(rdev);
8653 /* initialize memory controller */
8654 r = cik_mc_init(rdev);
8657 /* Memory manager */
8658 r = radeon_bo_init(rdev);
8662 if (rdev->flags & RADEON_IS_IGP) {
8663 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8664 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8665 r = cik_init_microcode(rdev);
8667 DRM_ERROR("Failed to load firmware!\n");
8672 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8673 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8675 r = cik_init_microcode(rdev);
8677 DRM_ERROR("Failed to load firmware!\n");
8683 /* Initialize power management */
8684 radeon_pm_init(rdev);
8686 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8687 ring->ring_obj = NULL;
8688 r600_ring_init(rdev, ring, 1024 * 1024);
8690 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8691 ring->ring_obj = NULL;
8692 r600_ring_init(rdev, ring, 1024 * 1024);
8693 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8697 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8698 ring->ring_obj = NULL;
8699 r600_ring_init(rdev, ring, 1024 * 1024);
8700 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8704 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8705 ring->ring_obj = NULL;
8706 r600_ring_init(rdev, ring, 256 * 1024);
8708 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8709 ring->ring_obj = NULL;
8710 r600_ring_init(rdev, ring, 256 * 1024);
8712 r = radeon_uvd_init(rdev);
8714 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8715 ring->ring_obj = NULL;
8716 r600_ring_init(rdev, ring, 4096);
8719 r = radeon_vce_init(rdev);
8721 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8722 ring->ring_obj = NULL;
8723 r600_ring_init(rdev, ring, 4096);
8725 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8726 ring->ring_obj = NULL;
8727 r600_ring_init(rdev, ring, 4096);
8730 rdev->ih.ring_obj = NULL;
8731 r600_ih_ring_init(rdev, 64 * 1024);
8733 r = r600_pcie_gart_init(rdev);
8737 rdev->accel_working = true;
8738 r = cik_startup(rdev);
8740 dev_err(rdev->dev, "disabling GPU acceleration\n");
8742 cik_sdma_fini(rdev);
8744 sumo_rlc_fini(rdev);
8746 radeon_wb_fini(rdev);
8747 radeon_ib_pool_fini(rdev);
8748 radeon_vm_manager_fini(rdev);
8749 radeon_irq_kms_fini(rdev);
8750 cik_pcie_gart_fini(rdev);
8751 rdev->accel_working = false;
8754 /* Don't start up if the MC ucode is missing.
8755 * The default clocks and voltages before the MC ucode
8756 * is loaded are not suffient for advanced operations.
8758 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8759 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8767 * cik_fini - asic specific driver and hw fini
8769 * @rdev: radeon_device pointer
8771 * Tear down the asic specific driver variables and program the hw
8772 * to an idle state (CIK).
8773 * Called at driver unload.
8775 void cik_fini(struct radeon_device *rdev)
8777 radeon_pm_fini(rdev);
8779 cik_sdma_fini(rdev);
8783 sumo_rlc_fini(rdev);
8785 radeon_wb_fini(rdev);
8786 radeon_vm_manager_fini(rdev);
8787 radeon_ib_pool_fini(rdev);
8788 radeon_irq_kms_fini(rdev);
8789 uvd_v1_0_fini(rdev);
8790 radeon_uvd_fini(rdev);
8791 radeon_vce_fini(rdev);
8792 cik_pcie_gart_fini(rdev);
8793 r600_vram_scratch_fini(rdev);
8794 radeon_gem_fini(rdev);
8795 radeon_fence_driver_fini(rdev);
8796 radeon_bo_fini(rdev);
8797 radeon_atombios_fini(rdev);
8802 void dce8_program_fmt(struct drm_encoder *encoder)
8804 struct drm_device *dev = encoder->dev;
8805 struct radeon_device *rdev = dev->dev_private;
8806 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8807 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8808 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8811 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8814 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8815 bpc = radeon_get_monitor_bpc(connector);
8816 dither = radeon_connector->dither;
8819 /* LVDS/eDP FMT is set up by atom */
8820 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8823 /* not needed for analog */
8824 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8825 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8833 if (dither == RADEON_FMT_DITHER_ENABLE)
8834 /* XXX sort out optimal dither settings */
8835 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8836 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8838 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8841 if (dither == RADEON_FMT_DITHER_ENABLE)
8842 /* XXX sort out optimal dither settings */
8843 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8844 FMT_RGB_RANDOM_ENABLE |
8845 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8847 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8850 if (dither == RADEON_FMT_DITHER_ENABLE)
8851 /* XXX sort out optimal dither settings */
8852 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8853 FMT_RGB_RANDOM_ENABLE |
8854 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8856 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8863 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8866 /* display watermark setup */
8868 * dce8_line_buffer_adjust - Set up the line buffer
8870 * @rdev: radeon_device pointer
8871 * @radeon_crtc: the selected display controller
8872 * @mode: the current display mode on the selected display
8875 * Setup up the line buffer allocation for
8876 * the selected display controller (CIK).
8877 * Returns the line buffer size in pixels.
8879 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8880 struct radeon_crtc *radeon_crtc,
8881 struct drm_display_mode *mode)
8883 u32 tmp, buffer_alloc, i;
8884 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8887 * There are 6 line buffers, one for each display controllers.
8888 * There are 3 partitions per LB. Select the number of partitions
8889 * to enable based on the display width. For display widths larger
8890 * than 4096, you need use to use 2 display controllers and combine
8891 * them using the stereo blender.
8893 if (radeon_crtc->base.enabled && mode) {
8894 if (mode->crtc_hdisplay < 1920) {
8897 } else if (mode->crtc_hdisplay < 2560) {
8900 } else if (mode->crtc_hdisplay < 4096) {
8902 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8904 DRM_DEBUG_KMS("Mode too big for LB!\n");
8906 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8913 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8914 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8916 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8917 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8918 for (i = 0; i < rdev->usec_timeout; i++) {
8919 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8920 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8925 if (radeon_crtc->base.enabled && mode) {
8937 /* controller not enabled, so no lb used */
8942 * cik_get_number_of_dram_channels - get the number of dram channels
8944 * @rdev: radeon_device pointer
8946 * Look up the number of video ram channels (CIK).
8947 * Used for display watermark bandwidth calculations
8948 * Returns the number of dram channels
8950 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8952 u32 tmp = RREG32(MC_SHARED_CHMAP);
8954 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8977 struct dce8_wm_params {
8978 u32 dram_channels; /* number of dram channels */
8979 u32 yclk; /* bandwidth per dram data pin in kHz */
8980 u32 sclk; /* engine clock in kHz */
8981 u32 disp_clk; /* display clock in kHz */
8982 u32 src_width; /* viewport width */
8983 u32 active_time; /* active display time in ns */
8984 u32 blank_time; /* blank time in ns */
8985 bool interlaced; /* mode is interlaced */
8986 fixed20_12 vsc; /* vertical scale ratio */
8987 u32 num_heads; /* number of active crtcs */
8988 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8989 u32 lb_size; /* line buffer allocated to pipe */
8990 u32 vtaps; /* vertical scaler taps */
8994 * dce8_dram_bandwidth - get the dram bandwidth
8996 * @wm: watermark calculation data
8998 * Calculate the raw dram bandwidth (CIK).
8999 * Used for display watermark bandwidth calculations
9000 * Returns the dram bandwidth in MBytes/s
9002 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
9004 /* Calculate raw DRAM Bandwidth */
9005 fixed20_12 dram_efficiency; /* 0.7 */
9006 fixed20_12 yclk, dram_channels, bandwidth;
9009 a.full = dfixed_const(1000);
9010 yclk.full = dfixed_const(wm->yclk);
9011 yclk.full = dfixed_div(yclk, a);
9012 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9013 a.full = dfixed_const(10);
9014 dram_efficiency.full = dfixed_const(7);
9015 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9016 bandwidth.full = dfixed_mul(dram_channels, yclk);
9017 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9019 return dfixed_trunc(bandwidth);
9023 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9025 * @wm: watermark calculation data
9027 * Calculate the dram bandwidth used for display (CIK).
9028 * Used for display watermark bandwidth calculations
9029 * Returns the dram bandwidth for display in MBytes/s
9031 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9033 /* Calculate DRAM Bandwidth and the part allocated to display. */
9034 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9035 fixed20_12 yclk, dram_channels, bandwidth;
9038 a.full = dfixed_const(1000);
9039 yclk.full = dfixed_const(wm->yclk);
9040 yclk.full = dfixed_div(yclk, a);
9041 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9042 a.full = dfixed_const(10);
9043 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9044 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9045 bandwidth.full = dfixed_mul(dram_channels, yclk);
9046 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9048 return dfixed_trunc(bandwidth);
9052 * dce8_data_return_bandwidth - get the data return bandwidth
9054 * @wm: watermark calculation data
9056 * Calculate the data return bandwidth used for display (CIK).
9057 * Used for display watermark bandwidth calculations
9058 * Returns the data return bandwidth in MBytes/s
9060 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9062 /* Calculate the display Data return Bandwidth */
9063 fixed20_12 return_efficiency; /* 0.8 */
9064 fixed20_12 sclk, bandwidth;
9067 a.full = dfixed_const(1000);
9068 sclk.full = dfixed_const(wm->sclk);
9069 sclk.full = dfixed_div(sclk, a);
9070 a.full = dfixed_const(10);
9071 return_efficiency.full = dfixed_const(8);
9072 return_efficiency.full = dfixed_div(return_efficiency, a);
9073 a.full = dfixed_const(32);
9074 bandwidth.full = dfixed_mul(a, sclk);
9075 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9077 return dfixed_trunc(bandwidth);
9081 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9083 * @wm: watermark calculation data
9085 * Calculate the dmif bandwidth used for display (CIK).
9086 * Used for display watermark bandwidth calculations
9087 * Returns the dmif bandwidth in MBytes/s
9089 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9091 /* Calculate the DMIF Request Bandwidth */
9092 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9093 fixed20_12 disp_clk, bandwidth;
9096 a.full = dfixed_const(1000);
9097 disp_clk.full = dfixed_const(wm->disp_clk);
9098 disp_clk.full = dfixed_div(disp_clk, a);
9099 a.full = dfixed_const(32);
9100 b.full = dfixed_mul(a, disp_clk);
9102 a.full = dfixed_const(10);
9103 disp_clk_request_efficiency.full = dfixed_const(8);
9104 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9106 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9108 return dfixed_trunc(bandwidth);
9112 * dce8_available_bandwidth - get the min available bandwidth
9114 * @wm: watermark calculation data
9116 * Calculate the min available bandwidth used for display (CIK).
9117 * Used for display watermark bandwidth calculations
9118 * Returns the min available bandwidth in MBytes/s
9120 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9122 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9123 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9124 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9125 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9127 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9131 * dce8_average_bandwidth - get the average available bandwidth
9133 * @wm: watermark calculation data
9135 * Calculate the average available bandwidth used for display (CIK).
9136 * Used for display watermark bandwidth calculations
9137 * Returns the average available bandwidth in MBytes/s
9139 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9141 /* Calculate the display mode Average Bandwidth
9142 * DisplayMode should contain the source and destination dimensions,
9146 fixed20_12 line_time;
9147 fixed20_12 src_width;
9148 fixed20_12 bandwidth;
9151 a.full = dfixed_const(1000);
9152 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9153 line_time.full = dfixed_div(line_time, a);
9154 bpp.full = dfixed_const(wm->bytes_per_pixel);
9155 src_width.full = dfixed_const(wm->src_width);
9156 bandwidth.full = dfixed_mul(src_width, bpp);
9157 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9158 bandwidth.full = dfixed_div(bandwidth, line_time);
9160 return dfixed_trunc(bandwidth);
9164 * dce8_latency_watermark - get the latency watermark
9166 * @wm: watermark calculation data
9168 * Calculate the latency watermark (CIK).
9169 * Used for display watermark bandwidth calculations
9170 * Returns the latency watermark in ns
9172 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9174 /* First calculate the latency in ns */
9175 u32 mc_latency = 2000; /* 2000 ns. */
9176 u32 available_bandwidth = dce8_available_bandwidth(wm);
9177 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9178 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9179 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9180 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9181 (wm->num_heads * cursor_line_pair_return_time);
9182 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9183 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9184 u32 tmp, dmif_size = 12288;
9187 if (wm->num_heads == 0)
9190 a.full = dfixed_const(2);
9191 b.full = dfixed_const(1);
9192 if ((wm->vsc.full > a.full) ||
9193 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9195 ((wm->vsc.full >= a.full) && wm->interlaced))
9196 max_src_lines_per_dst_line = 4;
9198 max_src_lines_per_dst_line = 2;
9200 a.full = dfixed_const(available_bandwidth);
9201 b.full = dfixed_const(wm->num_heads);
9202 a.full = dfixed_div(a, b);
9204 b.full = dfixed_const(mc_latency + 512);
9205 c.full = dfixed_const(wm->disp_clk);
9206 b.full = dfixed_div(b, c);
9208 c.full = dfixed_const(dmif_size);
9209 b.full = dfixed_div(c, b);
9211 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9213 b.full = dfixed_const(1000);
9214 c.full = dfixed_const(wm->disp_clk);
9215 b.full = dfixed_div(c, b);
9216 c.full = dfixed_const(wm->bytes_per_pixel);
9217 b.full = dfixed_mul(b, c);
9219 lb_fill_bw = min(tmp, dfixed_trunc(b));
9221 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9222 b.full = dfixed_const(1000);
9223 c.full = dfixed_const(lb_fill_bw);
9224 b.full = dfixed_div(c, b);
9225 a.full = dfixed_div(a, b);
9226 line_fill_time = dfixed_trunc(a);
9228 if (line_fill_time < wm->active_time)
9231 return latency + (line_fill_time - wm->active_time);
9236 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9237 * average and available dram bandwidth
9239 * @wm: watermark calculation data
9241 * Check if the display average bandwidth fits in the display
9242 * dram bandwidth (CIK).
9243 * Used for display watermark bandwidth calculations
9244 * Returns true if the display fits, false if not.
9246 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9248 if (dce8_average_bandwidth(wm) <=
9249 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9256 * dce8_average_bandwidth_vs_available_bandwidth - check
9257 * average and available bandwidth
9259 * @wm: watermark calculation data
9261 * Check if the display average bandwidth fits in the display
9262 * available bandwidth (CIK).
9263 * Used for display watermark bandwidth calculations
9264 * Returns true if the display fits, false if not.
9266 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9268 if (dce8_average_bandwidth(wm) <=
9269 (dce8_available_bandwidth(wm) / wm->num_heads))
9276 * dce8_check_latency_hiding - check latency hiding
9278 * @wm: watermark calculation data
9280 * Check latency hiding (CIK).
9281 * Used for display watermark bandwidth calculations
9282 * Returns true if the display fits, false if not.
9284 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9286 u32 lb_partitions = wm->lb_size / wm->src_width;
9287 u32 line_time = wm->active_time + wm->blank_time;
9288 u32 latency_tolerant_lines;
9292 a.full = dfixed_const(1);
9293 if (wm->vsc.full > a.full)
9294 latency_tolerant_lines = 1;
9296 if (lb_partitions <= (wm->vtaps + 1))
9297 latency_tolerant_lines = 1;
9299 latency_tolerant_lines = 2;
9302 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9304 if (dce8_latency_watermark(wm) <= latency_hiding)
9311 * dce8_program_watermarks - program display watermarks
9313 * @rdev: radeon_device pointer
9314 * @radeon_crtc: the selected display controller
9315 * @lb_size: line buffer size
9316 * @num_heads: number of display controllers in use
9318 * Calculate and program the display watermarks for the
9319 * selected display controller (CIK).
9321 static void dce8_program_watermarks(struct radeon_device *rdev,
9322 struct radeon_crtc *radeon_crtc,
9323 u32 lb_size, u32 num_heads)
9325 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9326 struct dce8_wm_params wm_low, wm_high;
9329 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9332 if (radeon_crtc->base.enabled && num_heads && mode) {
9333 pixel_period = 1000000 / (u32)mode->clock;
9334 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9336 /* watermark for high clocks */
9337 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9338 rdev->pm.dpm_enabled) {
9340 radeon_dpm_get_mclk(rdev, false) * 10;
9342 radeon_dpm_get_sclk(rdev, false) * 10;
9344 wm_high.yclk = rdev->pm.current_mclk * 10;
9345 wm_high.sclk = rdev->pm.current_sclk * 10;
9348 wm_high.disp_clk = mode->clock;
9349 wm_high.src_width = mode->crtc_hdisplay;
9350 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9351 wm_high.blank_time = line_time - wm_high.active_time;
9352 wm_high.interlaced = false;
9353 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9354 wm_high.interlaced = true;
9355 wm_high.vsc = radeon_crtc->vsc;
9357 if (radeon_crtc->rmx_type != RMX_OFF)
9359 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9360 wm_high.lb_size = lb_size;
9361 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9362 wm_high.num_heads = num_heads;
9364 /* set for high clocks */
9365 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9367 /* possibly force display priority to high */
9368 /* should really do this at mode validation time... */
9369 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9370 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9371 !dce8_check_latency_hiding(&wm_high) ||
9372 (rdev->disp_priority == 2)) {
9373 DRM_DEBUG_KMS("force priority to high\n");
9376 /* watermark for low clocks */
9377 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9378 rdev->pm.dpm_enabled) {
9380 radeon_dpm_get_mclk(rdev, true) * 10;
9382 radeon_dpm_get_sclk(rdev, true) * 10;
9384 wm_low.yclk = rdev->pm.current_mclk * 10;
9385 wm_low.sclk = rdev->pm.current_sclk * 10;
9388 wm_low.disp_clk = mode->clock;
9389 wm_low.src_width = mode->crtc_hdisplay;
9390 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9391 wm_low.blank_time = line_time - wm_low.active_time;
9392 wm_low.interlaced = false;
9393 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9394 wm_low.interlaced = true;
9395 wm_low.vsc = radeon_crtc->vsc;
9397 if (radeon_crtc->rmx_type != RMX_OFF)
9399 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9400 wm_low.lb_size = lb_size;
9401 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9402 wm_low.num_heads = num_heads;
9404 /* set for low clocks */
9405 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9407 /* possibly force display priority to high */
9408 /* should really do this at mode validation time... */
9409 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9410 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9411 !dce8_check_latency_hiding(&wm_low) ||
9412 (rdev->disp_priority == 2)) {
9413 DRM_DEBUG_KMS("force priority to high\n");
9418 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9420 tmp &= ~LATENCY_WATERMARK_MASK(3);
9421 tmp |= LATENCY_WATERMARK_MASK(1);
9422 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9423 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9424 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9425 LATENCY_HIGH_WATERMARK(line_time)));
9427 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9428 tmp &= ~LATENCY_WATERMARK_MASK(3);
9429 tmp |= LATENCY_WATERMARK_MASK(2);
9430 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9431 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9432 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9433 LATENCY_HIGH_WATERMARK(line_time)));
9434 /* restore original selection */
9435 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9437 /* save values for DPM */
9438 radeon_crtc->line_time = line_time;
9439 radeon_crtc->wm_high = latency_watermark_a;
9440 radeon_crtc->wm_low = latency_watermark_b;
9444 * dce8_bandwidth_update - program display watermarks
9446 * @rdev: radeon_device pointer
9448 * Calculate and program the display watermarks and line
9449 * buffer allocation (CIK).
9451 void dce8_bandwidth_update(struct radeon_device *rdev)
9453 struct drm_display_mode *mode = NULL;
9454 u32 num_heads = 0, lb_size;
9457 if (!rdev->mode_info.mode_config_initialized)
9460 radeon_update_display_priority(rdev);
9462 for (i = 0; i < rdev->num_crtc; i++) {
9463 if (rdev->mode_info.crtcs[i]->base.enabled)
9466 for (i = 0; i < rdev->num_crtc; i++) {
9467 mode = &rdev->mode_info.crtcs[i]->base.mode;
9468 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9469 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9474 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9476 * @rdev: radeon_device pointer
9478 * Fetches a GPU clock counter snapshot (SI).
9479 * Returns the 64 bit clock counter snapshot.
9481 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9485 mutex_lock(&rdev->gpu_clock_mutex);
9486 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9487 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9488 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9489 mutex_unlock(&rdev->gpu_clock_mutex);
9493 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9494 u32 cntl_reg, u32 status_reg)
9497 struct atom_clock_dividers dividers;
9500 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9501 clock, false, ÷rs);
9505 tmp = RREG32_SMC(cntl_reg);
9506 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9507 tmp |= dividers.post_divider;
9508 WREG32_SMC(cntl_reg, tmp);
9510 for (i = 0; i < 100; i++) {
9511 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9521 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9525 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9529 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9533 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9536 struct atom_clock_dividers dividers;
9539 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9540 ecclk, false, ÷rs);
9544 for (i = 0; i < 100; i++) {
9545 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9552 tmp = RREG32_SMC(CG_ECLK_CNTL);
9553 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9554 tmp |= dividers.post_divider;
9555 WREG32_SMC(CG_ECLK_CNTL, tmp);
9557 for (i = 0; i < 100; i++) {
9558 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9568 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9570 struct pci_dev *root = rdev->pdev->bus->self;
9571 int bridge_pos, gpu_pos;
9572 u32 speed_cntl, mask, current_data_rate;
9576 if (pci_is_root_bus(rdev->pdev->bus))
9579 if (radeon_pcie_gen2 == 0)
9582 if (rdev->flags & RADEON_IS_IGP)
9585 if (!(rdev->flags & RADEON_IS_PCIE))
9588 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9592 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9595 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9596 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9597 LC_CURRENT_DATA_RATE_SHIFT;
9598 if (mask & DRM_PCIE_SPEED_80) {
9599 if (current_data_rate == 2) {
9600 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9603 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9604 } else if (mask & DRM_PCIE_SPEED_50) {
9605 if (current_data_rate == 1) {
9606 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9609 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9612 bridge_pos = pci_pcie_cap(root);
9616 gpu_pos = pci_pcie_cap(rdev->pdev);
9620 if (mask & DRM_PCIE_SPEED_80) {
9621 /* re-try equalization if gen3 is not already enabled */
9622 if (current_data_rate != 2) {
9623 u16 bridge_cfg, gpu_cfg;
9624 u16 bridge_cfg2, gpu_cfg2;
9625 u32 max_lw, current_lw, tmp;
9627 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9628 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9630 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9631 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9633 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9634 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9636 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9637 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9638 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9640 if (current_lw < max_lw) {
9641 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9642 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9643 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9644 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9645 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9646 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9650 for (i = 0; i < 10; i++) {
9652 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9653 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9656 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9657 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9659 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9660 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9662 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9663 tmp |= LC_SET_QUIESCE;
9664 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9666 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9668 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9673 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9674 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9675 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9676 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9678 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9679 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9680 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9681 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9684 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9685 tmp16 &= ~((1 << 4) | (7 << 9));
9686 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9687 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9689 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9690 tmp16 &= ~((1 << 4) | (7 << 9));
9691 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9692 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9694 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9695 tmp &= ~LC_SET_QUIESCE;
9696 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9701 /* set the link speed */
9702 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9703 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9704 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9706 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9708 if (mask & DRM_PCIE_SPEED_80)
9709 tmp16 |= 3; /* gen3 */
9710 else if (mask & DRM_PCIE_SPEED_50)
9711 tmp16 |= 2; /* gen2 */
9713 tmp16 |= 1; /* gen1 */
9714 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9716 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9717 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9718 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9720 for (i = 0; i < rdev->usec_timeout; i++) {
9721 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9722 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9728 static void cik_program_aspm(struct radeon_device *rdev)
9731 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9732 bool disable_clkreq = false;
9734 if (radeon_aspm == 0)
9737 /* XXX double check IGPs */
9738 if (rdev->flags & RADEON_IS_IGP)
9741 if (!(rdev->flags & RADEON_IS_PCIE))
9744 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9745 data &= ~LC_XMIT_N_FTS_MASK;
9746 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9748 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9750 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9751 data |= LC_GO_TO_RECOVERY;
9753 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9755 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9756 data |= P_IGNORE_EDB_ERR;
9758 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9760 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9761 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9762 data |= LC_PMI_TO_L1_DIS;
9764 data |= LC_L0S_INACTIVITY(7);
9767 data |= LC_L1_INACTIVITY(7);
9768 data &= ~LC_PMI_TO_L1_DIS;
9770 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9772 if (!disable_plloff_in_l1) {
9773 bool clk_req_support;
9775 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9776 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9777 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9779 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9781 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9782 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9783 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9785 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9787 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9788 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9789 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9791 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9793 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9794 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9795 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9797 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9799 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9800 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9801 data |= LC_DYN_LANES_PWR_STATE(3);
9803 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9805 if (!disable_clkreq &&
9806 !pci_is_root_bus(rdev->pdev->bus)) {
9807 struct pci_dev *root = rdev->pdev->bus->self;
9810 clk_req_support = false;
9811 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9812 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9813 clk_req_support = true;
9815 clk_req_support = false;
9818 if (clk_req_support) {
9819 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9820 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9822 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9824 orig = data = RREG32_SMC(THM_CLK_CNTL);
9825 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9826 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9828 WREG32_SMC(THM_CLK_CNTL, data);
9830 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9831 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9832 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9834 WREG32_SMC(MISC_CLK_CTRL, data);
9836 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9837 data &= ~BCLK_AS_XCLK;
9839 WREG32_SMC(CG_CLKPIN_CNTL, data);
9841 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9842 data &= ~FORCE_BIF_REFCLK_EN;
9844 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9846 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9847 data &= ~MPLL_CLKOUT_SEL_MASK;
9848 data |= MPLL_CLKOUT_SEL(4);
9850 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9855 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9858 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9859 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9861 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9864 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9865 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9866 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9867 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9868 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9869 data &= ~LC_L0S_INACTIVITY_MASK;
9871 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);