2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
30 #include "radeon_asic.h"
31 #include <drm/radeon_drm.h>
34 #include "si_blit_shaders.h"
36 #define SI_PFP_UCODE_SIZE 2144
37 #define SI_PM4_UCODE_SIZE 2144
38 #define SI_CE_UCODE_SIZE 2144
39 #define SI_RLC_UCODE_SIZE 2048
40 #define SI_MC_UCODE_SIZE 7769
41 #define OLAND_MC_UCODE_SIZE 7863
43 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
44 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
45 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
46 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
47 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
50 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
51 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
52 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
53 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
54 MODULE_FIRMWARE("radeon/VERDE_me.bin");
55 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
56 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
57 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
58 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59 MODULE_FIRMWARE("radeon/OLAND_me.bin");
60 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
64 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
65 extern void r600_ih_ring_fini(struct radeon_device *rdev);
66 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
67 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
69 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
70 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
71 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
73 static const u32 tahiti_golden_rlc_registers[] =
75 0xc424, 0xffffffff, 0x00601005,
76 0xc47c, 0xffffffff, 0x10104040,
77 0xc488, 0xffffffff, 0x0100000a,
78 0xc314, 0xffffffff, 0x00000800,
79 0xc30c, 0xffffffff, 0x800000f4,
80 0xf4a8, 0xffffffff, 0x00000000
83 static const u32 tahiti_golden_registers[] =
85 0x9a10, 0x00010000, 0x00018208,
86 0x9830, 0xffffffff, 0x00000000,
87 0x9834, 0xf00fffff, 0x00000400,
88 0x9838, 0x0002021c, 0x00020200,
89 0xc78, 0x00000080, 0x00000000,
90 0xd030, 0x000300c0, 0x00800040,
91 0xd830, 0x000300c0, 0x00800040,
92 0x5bb0, 0x000000f0, 0x00000070,
93 0x5bc0, 0x00200000, 0x50100000,
94 0x7030, 0x31000311, 0x00000011,
95 0x277c, 0x00000003, 0x000007ff,
96 0x240c, 0x000007ff, 0x00000000,
97 0x8a14, 0xf000001f, 0x00000007,
98 0x8b24, 0xffffffff, 0x00ffffff,
99 0x8b10, 0x0000ff0f, 0x00000000,
100 0x28a4c, 0x07ffffff, 0x4e000000,
101 0x28350, 0x3f3f3fff, 0x2a00126a,
102 0x30, 0x000000ff, 0x0040,
103 0x34, 0x00000040, 0x00004040,
104 0x9100, 0x07ffffff, 0x03000000,
105 0x8e88, 0x01ff1f3f, 0x00000000,
106 0x8e84, 0x01ff1f3f, 0x00000000,
107 0x9060, 0x0000007f, 0x00000020,
108 0x9508, 0x00010000, 0x00010000,
109 0xac14, 0x00000200, 0x000002fb,
110 0xac10, 0xffffffff, 0x0000543b,
111 0xac0c, 0xffffffff, 0xa9210876,
112 0x88d0, 0xffffffff, 0x000fff40,
113 0x88d4, 0x0000001f, 0x00000010,
114 0x1410, 0x20000000, 0x20fffed8,
115 0x15c0, 0x000c0fc0, 0x000c0400
118 static const u32 tahiti_golden_registers2[] =
120 0xc64, 0x00000001, 0x00000001
123 static const u32 pitcairn_golden_rlc_registers[] =
125 0xc424, 0xffffffff, 0x00601004,
126 0xc47c, 0xffffffff, 0x10102020,
127 0xc488, 0xffffffff, 0x01000020,
128 0xc314, 0xffffffff, 0x00000800,
129 0xc30c, 0xffffffff, 0x800000a4
132 static const u32 pitcairn_golden_registers[] =
134 0x9a10, 0x00010000, 0x00018208,
135 0x9830, 0xffffffff, 0x00000000,
136 0x9834, 0xf00fffff, 0x00000400,
137 0x9838, 0x0002021c, 0x00020200,
138 0xc78, 0x00000080, 0x00000000,
139 0xd030, 0x000300c0, 0x00800040,
140 0xd830, 0x000300c0, 0x00800040,
141 0x5bb0, 0x000000f0, 0x00000070,
142 0x5bc0, 0x00200000, 0x50100000,
143 0x7030, 0x31000311, 0x00000011,
144 0x2ae4, 0x00073ffe, 0x000022a2,
145 0x240c, 0x000007ff, 0x00000000,
146 0x8a14, 0xf000001f, 0x00000007,
147 0x8b24, 0xffffffff, 0x00ffffff,
148 0x8b10, 0x0000ff0f, 0x00000000,
149 0x28a4c, 0x07ffffff, 0x4e000000,
150 0x28350, 0x3f3f3fff, 0x2a00126a,
151 0x30, 0x000000ff, 0x0040,
152 0x34, 0x00000040, 0x00004040,
153 0x9100, 0x07ffffff, 0x03000000,
154 0x9060, 0x0000007f, 0x00000020,
155 0x9508, 0x00010000, 0x00010000,
156 0xac14, 0x000003ff, 0x000000f7,
157 0xac10, 0xffffffff, 0x00000000,
158 0xac0c, 0xffffffff, 0x32761054,
159 0x88d4, 0x0000001f, 0x00000010,
160 0x15c0, 0x000c0fc0, 0x000c0400
163 static const u32 verde_golden_rlc_registers[] =
165 0xc424, 0xffffffff, 0x033f1005,
166 0xc47c, 0xffffffff, 0x10808020,
167 0xc488, 0xffffffff, 0x00800008,
168 0xc314, 0xffffffff, 0x00001000,
169 0xc30c, 0xffffffff, 0x80010014
172 static const u32 verde_golden_registers[] =
174 0x9a10, 0x00010000, 0x00018208,
175 0x9830, 0xffffffff, 0x00000000,
176 0x9834, 0xf00fffff, 0x00000400,
177 0x9838, 0x0002021c, 0x00020200,
178 0xc78, 0x00000080, 0x00000000,
179 0xd030, 0x000300c0, 0x00800040,
180 0xd030, 0x000300c0, 0x00800040,
181 0xd830, 0x000300c0, 0x00800040,
182 0xd830, 0x000300c0, 0x00800040,
183 0x5bb0, 0x000000f0, 0x00000070,
184 0x5bc0, 0x00200000, 0x50100000,
185 0x7030, 0x31000311, 0x00000011,
186 0x2ae4, 0x00073ffe, 0x000022a2,
187 0x2ae4, 0x00073ffe, 0x000022a2,
188 0x2ae4, 0x00073ffe, 0x000022a2,
189 0x240c, 0x000007ff, 0x00000000,
190 0x240c, 0x000007ff, 0x00000000,
191 0x240c, 0x000007ff, 0x00000000,
192 0x8a14, 0xf000001f, 0x00000007,
193 0x8a14, 0xf000001f, 0x00000007,
194 0x8a14, 0xf000001f, 0x00000007,
195 0x8b24, 0xffffffff, 0x00ffffff,
196 0x8b10, 0x0000ff0f, 0x00000000,
197 0x28a4c, 0x07ffffff, 0x4e000000,
198 0x28350, 0x3f3f3fff, 0x0000124a,
199 0x28350, 0x3f3f3fff, 0x0000124a,
200 0x28350, 0x3f3f3fff, 0x0000124a,
201 0x30, 0x000000ff, 0x0040,
202 0x34, 0x00000040, 0x00004040,
203 0x9100, 0x07ffffff, 0x03000000,
204 0x9100, 0x07ffffff, 0x03000000,
205 0x8e88, 0x01ff1f3f, 0x00000000,
206 0x8e88, 0x01ff1f3f, 0x00000000,
207 0x8e88, 0x01ff1f3f, 0x00000000,
208 0x8e84, 0x01ff1f3f, 0x00000000,
209 0x8e84, 0x01ff1f3f, 0x00000000,
210 0x8e84, 0x01ff1f3f, 0x00000000,
211 0x9060, 0x0000007f, 0x00000020,
212 0x9508, 0x00010000, 0x00010000,
213 0xac14, 0x000003ff, 0x00000003,
214 0xac14, 0x000003ff, 0x00000003,
215 0xac14, 0x000003ff, 0x00000003,
216 0xac10, 0xffffffff, 0x00000000,
217 0xac10, 0xffffffff, 0x00000000,
218 0xac10, 0xffffffff, 0x00000000,
219 0xac0c, 0xffffffff, 0x00001032,
220 0xac0c, 0xffffffff, 0x00001032,
221 0xac0c, 0xffffffff, 0x00001032,
222 0x88d4, 0x0000001f, 0x00000010,
223 0x88d4, 0x0000001f, 0x00000010,
224 0x88d4, 0x0000001f, 0x00000010,
225 0x15c0, 0x000c0fc0, 0x000c0400
228 static const u32 oland_golden_rlc_registers[] =
230 0xc424, 0xffffffff, 0x00601005,
231 0xc47c, 0xffffffff, 0x10104040,
232 0xc488, 0xffffffff, 0x0100000a,
233 0xc314, 0xffffffff, 0x00000800,
234 0xc30c, 0xffffffff, 0x800000f4
237 static const u32 oland_golden_registers[] =
239 0x9a10, 0x00010000, 0x00018208,
240 0x9830, 0xffffffff, 0x00000000,
241 0x9834, 0xf00fffff, 0x00000400,
242 0x9838, 0x0002021c, 0x00020200,
243 0xc78, 0x00000080, 0x00000000,
244 0xd030, 0x000300c0, 0x00800040,
245 0xd830, 0x000300c0, 0x00800040,
246 0x5bb0, 0x000000f0, 0x00000070,
247 0x5bc0, 0x00200000, 0x50100000,
248 0x7030, 0x31000311, 0x00000011,
249 0x2ae4, 0x00073ffe, 0x000022a2,
250 0x240c, 0x000007ff, 0x00000000,
251 0x8a14, 0xf000001f, 0x00000007,
252 0x8b24, 0xffffffff, 0x00ffffff,
253 0x8b10, 0x0000ff0f, 0x00000000,
254 0x28a4c, 0x07ffffff, 0x4e000000,
255 0x28350, 0x3f3f3fff, 0x00000082,
256 0x30, 0x000000ff, 0x0040,
257 0x34, 0x00000040, 0x00004040,
258 0x9100, 0x07ffffff, 0x03000000,
259 0x9060, 0x0000007f, 0x00000020,
260 0x9508, 0x00010000, 0x00010000,
261 0xac14, 0x000003ff, 0x000000f3,
262 0xac10, 0xffffffff, 0x00000000,
263 0xac0c, 0xffffffff, 0x00003210,
264 0x88d4, 0x0000001f, 0x00000010,
265 0x15c0, 0x000c0fc0, 0x000c0400
268 static const u32 tahiti_mgcg_cgcg_init[] =
270 0xc400, 0xffffffff, 0xfffffffc,
271 0x802c, 0xffffffff, 0xe0000000,
272 0x9a60, 0xffffffff, 0x00000100,
273 0x92a4, 0xffffffff, 0x00000100,
274 0xc164, 0xffffffff, 0x00000100,
275 0x9774, 0xffffffff, 0x00000100,
276 0x8984, 0xffffffff, 0x06000100,
277 0x8a18, 0xffffffff, 0x00000100,
278 0x92a0, 0xffffffff, 0x00000100,
279 0xc380, 0xffffffff, 0x00000100,
280 0x8b28, 0xffffffff, 0x00000100,
281 0x9144, 0xffffffff, 0x00000100,
282 0x8d88, 0xffffffff, 0x00000100,
283 0x8d8c, 0xffffffff, 0x00000100,
284 0x9030, 0xffffffff, 0x00000100,
285 0x9034, 0xffffffff, 0x00000100,
286 0x9038, 0xffffffff, 0x00000100,
287 0x903c, 0xffffffff, 0x00000100,
288 0xad80, 0xffffffff, 0x00000100,
289 0xac54, 0xffffffff, 0x00000100,
290 0x897c, 0xffffffff, 0x06000100,
291 0x9868, 0xffffffff, 0x00000100,
292 0x9510, 0xffffffff, 0x00000100,
293 0xaf04, 0xffffffff, 0x00000100,
294 0xae04, 0xffffffff, 0x00000100,
295 0x949c, 0xffffffff, 0x00000100,
296 0x802c, 0xffffffff, 0xe0000000,
297 0x9160, 0xffffffff, 0x00010000,
298 0x9164, 0xffffffff, 0x00030002,
299 0x9168, 0xffffffff, 0x00040007,
300 0x916c, 0xffffffff, 0x00060005,
301 0x9170, 0xffffffff, 0x00090008,
302 0x9174, 0xffffffff, 0x00020001,
303 0x9178, 0xffffffff, 0x00040003,
304 0x917c, 0xffffffff, 0x00000007,
305 0x9180, 0xffffffff, 0x00060005,
306 0x9184, 0xffffffff, 0x00090008,
307 0x9188, 0xffffffff, 0x00030002,
308 0x918c, 0xffffffff, 0x00050004,
309 0x9190, 0xffffffff, 0x00000008,
310 0x9194, 0xffffffff, 0x00070006,
311 0x9198, 0xffffffff, 0x000a0009,
312 0x919c, 0xffffffff, 0x00040003,
313 0x91a0, 0xffffffff, 0x00060005,
314 0x91a4, 0xffffffff, 0x00000009,
315 0x91a8, 0xffffffff, 0x00080007,
316 0x91ac, 0xffffffff, 0x000b000a,
317 0x91b0, 0xffffffff, 0x00050004,
318 0x91b4, 0xffffffff, 0x00070006,
319 0x91b8, 0xffffffff, 0x0008000b,
320 0x91bc, 0xffffffff, 0x000a0009,
321 0x91c0, 0xffffffff, 0x000d000c,
322 0x91c4, 0xffffffff, 0x00060005,
323 0x91c8, 0xffffffff, 0x00080007,
324 0x91cc, 0xffffffff, 0x0000000b,
325 0x91d0, 0xffffffff, 0x000a0009,
326 0x91d4, 0xffffffff, 0x000d000c,
327 0x91d8, 0xffffffff, 0x00070006,
328 0x91dc, 0xffffffff, 0x00090008,
329 0x91e0, 0xffffffff, 0x0000000c,
330 0x91e4, 0xffffffff, 0x000b000a,
331 0x91e8, 0xffffffff, 0x000e000d,
332 0x91ec, 0xffffffff, 0x00080007,
333 0x91f0, 0xffffffff, 0x000a0009,
334 0x91f4, 0xffffffff, 0x0000000d,
335 0x91f8, 0xffffffff, 0x000c000b,
336 0x91fc, 0xffffffff, 0x000f000e,
337 0x9200, 0xffffffff, 0x00090008,
338 0x9204, 0xffffffff, 0x000b000a,
339 0x9208, 0xffffffff, 0x000c000f,
340 0x920c, 0xffffffff, 0x000e000d,
341 0x9210, 0xffffffff, 0x00110010,
342 0x9214, 0xffffffff, 0x000a0009,
343 0x9218, 0xffffffff, 0x000c000b,
344 0x921c, 0xffffffff, 0x0000000f,
345 0x9220, 0xffffffff, 0x000e000d,
346 0x9224, 0xffffffff, 0x00110010,
347 0x9228, 0xffffffff, 0x000b000a,
348 0x922c, 0xffffffff, 0x000d000c,
349 0x9230, 0xffffffff, 0x00000010,
350 0x9234, 0xffffffff, 0x000f000e,
351 0x9238, 0xffffffff, 0x00120011,
352 0x923c, 0xffffffff, 0x000c000b,
353 0x9240, 0xffffffff, 0x000e000d,
354 0x9244, 0xffffffff, 0x00000011,
355 0x9248, 0xffffffff, 0x0010000f,
356 0x924c, 0xffffffff, 0x00130012,
357 0x9250, 0xffffffff, 0x000d000c,
358 0x9254, 0xffffffff, 0x000f000e,
359 0x9258, 0xffffffff, 0x00100013,
360 0x925c, 0xffffffff, 0x00120011,
361 0x9260, 0xffffffff, 0x00150014,
362 0x9264, 0xffffffff, 0x000e000d,
363 0x9268, 0xffffffff, 0x0010000f,
364 0x926c, 0xffffffff, 0x00000013,
365 0x9270, 0xffffffff, 0x00120011,
366 0x9274, 0xffffffff, 0x00150014,
367 0x9278, 0xffffffff, 0x000f000e,
368 0x927c, 0xffffffff, 0x00110010,
369 0x9280, 0xffffffff, 0x00000014,
370 0x9284, 0xffffffff, 0x00130012,
371 0x9288, 0xffffffff, 0x00160015,
372 0x928c, 0xffffffff, 0x0010000f,
373 0x9290, 0xffffffff, 0x00120011,
374 0x9294, 0xffffffff, 0x00000015,
375 0x9298, 0xffffffff, 0x00140013,
376 0x929c, 0xffffffff, 0x00170016,
377 0x9150, 0xffffffff, 0x96940200,
378 0x8708, 0xffffffff, 0x00900100,
379 0xc478, 0xffffffff, 0x00000080,
380 0xc404, 0xffffffff, 0x0020003f,
381 0x30, 0xffffffff, 0x0000001c,
382 0x34, 0x000f0000, 0x000f0000,
383 0x160c, 0xffffffff, 0x00000100,
384 0x1024, 0xffffffff, 0x00000100,
385 0x102c, 0x00000101, 0x00000000,
386 0x20a8, 0xffffffff, 0x00000104,
387 0x264c, 0x000c0000, 0x000c0000,
388 0x2648, 0x000c0000, 0x000c0000,
389 0x55e4, 0xff000fff, 0x00000100,
390 0x55e8, 0x00000001, 0x00000001,
391 0x2f50, 0x00000001, 0x00000001,
392 0x30cc, 0xc0000fff, 0x00000104,
393 0xc1e4, 0x00000001, 0x00000001,
394 0xd0c0, 0xfffffff0, 0x00000100,
395 0xd8c0, 0xfffffff0, 0x00000100
398 static const u32 pitcairn_mgcg_cgcg_init[] =
400 0xc400, 0xffffffff, 0xfffffffc,
401 0x802c, 0xffffffff, 0xe0000000,
402 0x9a60, 0xffffffff, 0x00000100,
403 0x92a4, 0xffffffff, 0x00000100,
404 0xc164, 0xffffffff, 0x00000100,
405 0x9774, 0xffffffff, 0x00000100,
406 0x8984, 0xffffffff, 0x06000100,
407 0x8a18, 0xffffffff, 0x00000100,
408 0x92a0, 0xffffffff, 0x00000100,
409 0xc380, 0xffffffff, 0x00000100,
410 0x8b28, 0xffffffff, 0x00000100,
411 0x9144, 0xffffffff, 0x00000100,
412 0x8d88, 0xffffffff, 0x00000100,
413 0x8d8c, 0xffffffff, 0x00000100,
414 0x9030, 0xffffffff, 0x00000100,
415 0x9034, 0xffffffff, 0x00000100,
416 0x9038, 0xffffffff, 0x00000100,
417 0x903c, 0xffffffff, 0x00000100,
418 0xad80, 0xffffffff, 0x00000100,
419 0xac54, 0xffffffff, 0x00000100,
420 0x897c, 0xffffffff, 0x06000100,
421 0x9868, 0xffffffff, 0x00000100,
422 0x9510, 0xffffffff, 0x00000100,
423 0xaf04, 0xffffffff, 0x00000100,
424 0xae04, 0xffffffff, 0x00000100,
425 0x949c, 0xffffffff, 0x00000100,
426 0x802c, 0xffffffff, 0xe0000000,
427 0x9160, 0xffffffff, 0x00010000,
428 0x9164, 0xffffffff, 0x00030002,
429 0x9168, 0xffffffff, 0x00040007,
430 0x916c, 0xffffffff, 0x00060005,
431 0x9170, 0xffffffff, 0x00090008,
432 0x9174, 0xffffffff, 0x00020001,
433 0x9178, 0xffffffff, 0x00040003,
434 0x917c, 0xffffffff, 0x00000007,
435 0x9180, 0xffffffff, 0x00060005,
436 0x9184, 0xffffffff, 0x00090008,
437 0x9188, 0xffffffff, 0x00030002,
438 0x918c, 0xffffffff, 0x00050004,
439 0x9190, 0xffffffff, 0x00000008,
440 0x9194, 0xffffffff, 0x00070006,
441 0x9198, 0xffffffff, 0x000a0009,
442 0x919c, 0xffffffff, 0x00040003,
443 0x91a0, 0xffffffff, 0x00060005,
444 0x91a4, 0xffffffff, 0x00000009,
445 0x91a8, 0xffffffff, 0x00080007,
446 0x91ac, 0xffffffff, 0x000b000a,
447 0x91b0, 0xffffffff, 0x00050004,
448 0x91b4, 0xffffffff, 0x00070006,
449 0x91b8, 0xffffffff, 0x0008000b,
450 0x91bc, 0xffffffff, 0x000a0009,
451 0x91c0, 0xffffffff, 0x000d000c,
452 0x9200, 0xffffffff, 0x00090008,
453 0x9204, 0xffffffff, 0x000b000a,
454 0x9208, 0xffffffff, 0x000c000f,
455 0x920c, 0xffffffff, 0x000e000d,
456 0x9210, 0xffffffff, 0x00110010,
457 0x9214, 0xffffffff, 0x000a0009,
458 0x9218, 0xffffffff, 0x000c000b,
459 0x921c, 0xffffffff, 0x0000000f,
460 0x9220, 0xffffffff, 0x000e000d,
461 0x9224, 0xffffffff, 0x00110010,
462 0x9228, 0xffffffff, 0x000b000a,
463 0x922c, 0xffffffff, 0x000d000c,
464 0x9230, 0xffffffff, 0x00000010,
465 0x9234, 0xffffffff, 0x000f000e,
466 0x9238, 0xffffffff, 0x00120011,
467 0x923c, 0xffffffff, 0x000c000b,
468 0x9240, 0xffffffff, 0x000e000d,
469 0x9244, 0xffffffff, 0x00000011,
470 0x9248, 0xffffffff, 0x0010000f,
471 0x924c, 0xffffffff, 0x00130012,
472 0x9250, 0xffffffff, 0x000d000c,
473 0x9254, 0xffffffff, 0x000f000e,
474 0x9258, 0xffffffff, 0x00100013,
475 0x925c, 0xffffffff, 0x00120011,
476 0x9260, 0xffffffff, 0x00150014,
477 0x9150, 0xffffffff, 0x96940200,
478 0x8708, 0xffffffff, 0x00900100,
479 0xc478, 0xffffffff, 0x00000080,
480 0xc404, 0xffffffff, 0x0020003f,
481 0x30, 0xffffffff, 0x0000001c,
482 0x34, 0x000f0000, 0x000f0000,
483 0x160c, 0xffffffff, 0x00000100,
484 0x1024, 0xffffffff, 0x00000100,
485 0x102c, 0x00000101, 0x00000000,
486 0x20a8, 0xffffffff, 0x00000104,
487 0x55e4, 0xff000fff, 0x00000100,
488 0x55e8, 0x00000001, 0x00000001,
489 0x2f50, 0x00000001, 0x00000001,
490 0x30cc, 0xc0000fff, 0x00000104,
491 0xc1e4, 0x00000001, 0x00000001,
492 0xd0c0, 0xfffffff0, 0x00000100,
493 0xd8c0, 0xfffffff0, 0x00000100
496 static const u32 verde_mgcg_cgcg_init[] =
498 0xc400, 0xffffffff, 0xfffffffc,
499 0x802c, 0xffffffff, 0xe0000000,
500 0x9a60, 0xffffffff, 0x00000100,
501 0x92a4, 0xffffffff, 0x00000100,
502 0xc164, 0xffffffff, 0x00000100,
503 0x9774, 0xffffffff, 0x00000100,
504 0x8984, 0xffffffff, 0x06000100,
505 0x8a18, 0xffffffff, 0x00000100,
506 0x92a0, 0xffffffff, 0x00000100,
507 0xc380, 0xffffffff, 0x00000100,
508 0x8b28, 0xffffffff, 0x00000100,
509 0x9144, 0xffffffff, 0x00000100,
510 0x8d88, 0xffffffff, 0x00000100,
511 0x8d8c, 0xffffffff, 0x00000100,
512 0x9030, 0xffffffff, 0x00000100,
513 0x9034, 0xffffffff, 0x00000100,
514 0x9038, 0xffffffff, 0x00000100,
515 0x903c, 0xffffffff, 0x00000100,
516 0xad80, 0xffffffff, 0x00000100,
517 0xac54, 0xffffffff, 0x00000100,
518 0x897c, 0xffffffff, 0x06000100,
519 0x9868, 0xffffffff, 0x00000100,
520 0x9510, 0xffffffff, 0x00000100,
521 0xaf04, 0xffffffff, 0x00000100,
522 0xae04, 0xffffffff, 0x00000100,
523 0x949c, 0xffffffff, 0x00000100,
524 0x802c, 0xffffffff, 0xe0000000,
525 0x9160, 0xffffffff, 0x00010000,
526 0x9164, 0xffffffff, 0x00030002,
527 0x9168, 0xffffffff, 0x00040007,
528 0x916c, 0xffffffff, 0x00060005,
529 0x9170, 0xffffffff, 0x00090008,
530 0x9174, 0xffffffff, 0x00020001,
531 0x9178, 0xffffffff, 0x00040003,
532 0x917c, 0xffffffff, 0x00000007,
533 0x9180, 0xffffffff, 0x00060005,
534 0x9184, 0xffffffff, 0x00090008,
535 0x9188, 0xffffffff, 0x00030002,
536 0x918c, 0xffffffff, 0x00050004,
537 0x9190, 0xffffffff, 0x00000008,
538 0x9194, 0xffffffff, 0x00070006,
539 0x9198, 0xffffffff, 0x000a0009,
540 0x919c, 0xffffffff, 0x00040003,
541 0x91a0, 0xffffffff, 0x00060005,
542 0x91a4, 0xffffffff, 0x00000009,
543 0x91a8, 0xffffffff, 0x00080007,
544 0x91ac, 0xffffffff, 0x000b000a,
545 0x91b0, 0xffffffff, 0x00050004,
546 0x91b4, 0xffffffff, 0x00070006,
547 0x91b8, 0xffffffff, 0x0008000b,
548 0x91bc, 0xffffffff, 0x000a0009,
549 0x91c0, 0xffffffff, 0x000d000c,
550 0x9200, 0xffffffff, 0x00090008,
551 0x9204, 0xffffffff, 0x000b000a,
552 0x9208, 0xffffffff, 0x000c000f,
553 0x920c, 0xffffffff, 0x000e000d,
554 0x9210, 0xffffffff, 0x00110010,
555 0x9214, 0xffffffff, 0x000a0009,
556 0x9218, 0xffffffff, 0x000c000b,
557 0x921c, 0xffffffff, 0x0000000f,
558 0x9220, 0xffffffff, 0x000e000d,
559 0x9224, 0xffffffff, 0x00110010,
560 0x9228, 0xffffffff, 0x000b000a,
561 0x922c, 0xffffffff, 0x000d000c,
562 0x9230, 0xffffffff, 0x00000010,
563 0x9234, 0xffffffff, 0x000f000e,
564 0x9238, 0xffffffff, 0x00120011,
565 0x923c, 0xffffffff, 0x000c000b,
566 0x9240, 0xffffffff, 0x000e000d,
567 0x9244, 0xffffffff, 0x00000011,
568 0x9248, 0xffffffff, 0x0010000f,
569 0x924c, 0xffffffff, 0x00130012,
570 0x9250, 0xffffffff, 0x000d000c,
571 0x9254, 0xffffffff, 0x000f000e,
572 0x9258, 0xffffffff, 0x00100013,
573 0x925c, 0xffffffff, 0x00120011,
574 0x9260, 0xffffffff, 0x00150014,
575 0x9150, 0xffffffff, 0x96940200,
576 0x8708, 0xffffffff, 0x00900100,
577 0xc478, 0xffffffff, 0x00000080,
578 0xc404, 0xffffffff, 0x0020003f,
579 0x30, 0xffffffff, 0x0000001c,
580 0x34, 0x000f0000, 0x000f0000,
581 0x160c, 0xffffffff, 0x00000100,
582 0x1024, 0xffffffff, 0x00000100,
583 0x102c, 0x00000101, 0x00000000,
584 0x20a8, 0xffffffff, 0x00000104,
585 0x264c, 0x000c0000, 0x000c0000,
586 0x2648, 0x000c0000, 0x000c0000,
587 0x55e4, 0xff000fff, 0x00000100,
588 0x55e8, 0x00000001, 0x00000001,
589 0x2f50, 0x00000001, 0x00000001,
590 0x30cc, 0xc0000fff, 0x00000104,
591 0xc1e4, 0x00000001, 0x00000001,
592 0xd0c0, 0xfffffff0, 0x00000100,
593 0xd8c0, 0xfffffff0, 0x00000100
596 static const u32 oland_mgcg_cgcg_init[] =
598 0xc400, 0xffffffff, 0xfffffffc,
599 0x802c, 0xffffffff, 0xe0000000,
600 0x9a60, 0xffffffff, 0x00000100,
601 0x92a4, 0xffffffff, 0x00000100,
602 0xc164, 0xffffffff, 0x00000100,
603 0x9774, 0xffffffff, 0x00000100,
604 0x8984, 0xffffffff, 0x06000100,
605 0x8a18, 0xffffffff, 0x00000100,
606 0x92a0, 0xffffffff, 0x00000100,
607 0xc380, 0xffffffff, 0x00000100,
608 0x8b28, 0xffffffff, 0x00000100,
609 0x9144, 0xffffffff, 0x00000100,
610 0x8d88, 0xffffffff, 0x00000100,
611 0x8d8c, 0xffffffff, 0x00000100,
612 0x9030, 0xffffffff, 0x00000100,
613 0x9034, 0xffffffff, 0x00000100,
614 0x9038, 0xffffffff, 0x00000100,
615 0x903c, 0xffffffff, 0x00000100,
616 0xad80, 0xffffffff, 0x00000100,
617 0xac54, 0xffffffff, 0x00000100,
618 0x897c, 0xffffffff, 0x06000100,
619 0x9868, 0xffffffff, 0x00000100,
620 0x9510, 0xffffffff, 0x00000100,
621 0xaf04, 0xffffffff, 0x00000100,
622 0xae04, 0xffffffff, 0x00000100,
623 0x949c, 0xffffffff, 0x00000100,
624 0x802c, 0xffffffff, 0xe0000000,
625 0x9160, 0xffffffff, 0x00010000,
626 0x9164, 0xffffffff, 0x00030002,
627 0x9168, 0xffffffff, 0x00040007,
628 0x916c, 0xffffffff, 0x00060005,
629 0x9170, 0xffffffff, 0x00090008,
630 0x9174, 0xffffffff, 0x00020001,
631 0x9178, 0xffffffff, 0x00040003,
632 0x917c, 0xffffffff, 0x00000007,
633 0x9180, 0xffffffff, 0x00060005,
634 0x9184, 0xffffffff, 0x00090008,
635 0x9188, 0xffffffff, 0x00030002,
636 0x918c, 0xffffffff, 0x00050004,
637 0x9190, 0xffffffff, 0x00000008,
638 0x9194, 0xffffffff, 0x00070006,
639 0x9198, 0xffffffff, 0x000a0009,
640 0x919c, 0xffffffff, 0x00040003,
641 0x91a0, 0xffffffff, 0x00060005,
642 0x91a4, 0xffffffff, 0x00000009,
643 0x91a8, 0xffffffff, 0x00080007,
644 0x91ac, 0xffffffff, 0x000b000a,
645 0x91b0, 0xffffffff, 0x00050004,
646 0x91b4, 0xffffffff, 0x00070006,
647 0x91b8, 0xffffffff, 0x0008000b,
648 0x91bc, 0xffffffff, 0x000a0009,
649 0x91c0, 0xffffffff, 0x000d000c,
650 0x91c4, 0xffffffff, 0x00060005,
651 0x91c8, 0xffffffff, 0x00080007,
652 0x91cc, 0xffffffff, 0x0000000b,
653 0x91d0, 0xffffffff, 0x000a0009,
654 0x91d4, 0xffffffff, 0x000d000c,
655 0x9150, 0xffffffff, 0x96940200,
656 0x8708, 0xffffffff, 0x00900100,
657 0xc478, 0xffffffff, 0x00000080,
658 0xc404, 0xffffffff, 0x0020003f,
659 0x30, 0xffffffff, 0x0000001c,
660 0x34, 0x000f0000, 0x000f0000,
661 0x160c, 0xffffffff, 0x00000100,
662 0x1024, 0xffffffff, 0x00000100,
663 0x102c, 0x00000101, 0x00000000,
664 0x20a8, 0xffffffff, 0x00000104,
665 0x264c, 0x000c0000, 0x000c0000,
666 0x2648, 0x000c0000, 0x000c0000,
667 0x55e4, 0xff000fff, 0x00000100,
668 0x55e8, 0x00000001, 0x00000001,
669 0x2f50, 0x00000001, 0x00000001,
670 0x30cc, 0xc0000fff, 0x00000104,
671 0xc1e4, 0x00000001, 0x00000001,
672 0xd0c0, 0xfffffff0, 0x00000100,
673 0xd8c0, 0xfffffff0, 0x00000100
676 static u32 verde_pg_init[] =
678 0x353c, 0xffffffff, 0x40000,
679 0x3538, 0xffffffff, 0x200010ff,
680 0x353c, 0xffffffff, 0x0,
681 0x353c, 0xffffffff, 0x0,
682 0x353c, 0xffffffff, 0x0,
683 0x353c, 0xffffffff, 0x0,
684 0x353c, 0xffffffff, 0x0,
685 0x353c, 0xffffffff, 0x7007,
686 0x3538, 0xffffffff, 0x300010ff,
687 0x353c, 0xffffffff, 0x0,
688 0x353c, 0xffffffff, 0x0,
689 0x353c, 0xffffffff, 0x0,
690 0x353c, 0xffffffff, 0x0,
691 0x353c, 0xffffffff, 0x0,
692 0x353c, 0xffffffff, 0x400000,
693 0x3538, 0xffffffff, 0x100010ff,
694 0x353c, 0xffffffff, 0x0,
695 0x353c, 0xffffffff, 0x0,
696 0x353c, 0xffffffff, 0x0,
697 0x353c, 0xffffffff, 0x0,
698 0x353c, 0xffffffff, 0x0,
699 0x353c, 0xffffffff, 0x120200,
700 0x3538, 0xffffffff, 0x500010ff,
701 0x353c, 0xffffffff, 0x0,
702 0x353c, 0xffffffff, 0x0,
703 0x353c, 0xffffffff, 0x0,
704 0x353c, 0xffffffff, 0x0,
705 0x353c, 0xffffffff, 0x0,
706 0x353c, 0xffffffff, 0x1e1e16,
707 0x3538, 0xffffffff, 0x600010ff,
708 0x353c, 0xffffffff, 0x0,
709 0x353c, 0xffffffff, 0x0,
710 0x353c, 0xffffffff, 0x0,
711 0x353c, 0xffffffff, 0x0,
712 0x353c, 0xffffffff, 0x0,
713 0x353c, 0xffffffff, 0x171f1e,
714 0x3538, 0xffffffff, 0x700010ff,
715 0x353c, 0xffffffff, 0x0,
716 0x353c, 0xffffffff, 0x0,
717 0x353c, 0xffffffff, 0x0,
718 0x353c, 0xffffffff, 0x0,
719 0x353c, 0xffffffff, 0x0,
720 0x353c, 0xffffffff, 0x0,
721 0x3538, 0xffffffff, 0x9ff,
722 0x3500, 0xffffffff, 0x0,
723 0x3504, 0xffffffff, 0x10000800,
724 0x3504, 0xffffffff, 0xf,
725 0x3504, 0xffffffff, 0xf,
726 0x3500, 0xffffffff, 0x4,
727 0x3504, 0xffffffff, 0x1000051e,
728 0x3504, 0xffffffff, 0xffff,
729 0x3504, 0xffffffff, 0xffff,
730 0x3500, 0xffffffff, 0x8,
731 0x3504, 0xffffffff, 0x80500,
732 0x3500, 0xffffffff, 0x12,
733 0x3504, 0xffffffff, 0x9050c,
734 0x3500, 0xffffffff, 0x1d,
735 0x3504, 0xffffffff, 0xb052c,
736 0x3500, 0xffffffff, 0x2a,
737 0x3504, 0xffffffff, 0x1053e,
738 0x3500, 0xffffffff, 0x2d,
739 0x3504, 0xffffffff, 0x10546,
740 0x3500, 0xffffffff, 0x30,
741 0x3504, 0xffffffff, 0xa054e,
742 0x3500, 0xffffffff, 0x3c,
743 0x3504, 0xffffffff, 0x1055f,
744 0x3500, 0xffffffff, 0x3f,
745 0x3504, 0xffffffff, 0x10567,
746 0x3500, 0xffffffff, 0x42,
747 0x3504, 0xffffffff, 0x1056f,
748 0x3500, 0xffffffff, 0x45,
749 0x3504, 0xffffffff, 0x10572,
750 0x3500, 0xffffffff, 0x48,
751 0x3504, 0xffffffff, 0x20575,
752 0x3500, 0xffffffff, 0x4c,
753 0x3504, 0xffffffff, 0x190801,
754 0x3500, 0xffffffff, 0x67,
755 0x3504, 0xffffffff, 0x1082a,
756 0x3500, 0xffffffff, 0x6a,
757 0x3504, 0xffffffff, 0x1b082d,
758 0x3500, 0xffffffff, 0x87,
759 0x3504, 0xffffffff, 0x310851,
760 0x3500, 0xffffffff, 0xba,
761 0x3504, 0xffffffff, 0x891,
762 0x3500, 0xffffffff, 0xbc,
763 0x3504, 0xffffffff, 0x893,
764 0x3500, 0xffffffff, 0xbe,
765 0x3504, 0xffffffff, 0x20895,
766 0x3500, 0xffffffff, 0xc2,
767 0x3504, 0xffffffff, 0x20899,
768 0x3500, 0xffffffff, 0xc6,
769 0x3504, 0xffffffff, 0x2089d,
770 0x3500, 0xffffffff, 0xca,
771 0x3504, 0xffffffff, 0x8a1,
772 0x3500, 0xffffffff, 0xcc,
773 0x3504, 0xffffffff, 0x8a3,
774 0x3500, 0xffffffff, 0xce,
775 0x3504, 0xffffffff, 0x308a5,
776 0x3500, 0xffffffff, 0xd3,
777 0x3504, 0xffffffff, 0x6d08cd,
778 0x3500, 0xffffffff, 0x142,
779 0x3504, 0xffffffff, 0x2000095a,
780 0x3504, 0xffffffff, 0x1,
781 0x3500, 0xffffffff, 0x144,
782 0x3504, 0xffffffff, 0x301f095b,
783 0x3500, 0xffffffff, 0x165,
784 0x3504, 0xffffffff, 0xc094d,
785 0x3500, 0xffffffff, 0x173,
786 0x3504, 0xffffffff, 0xf096d,
787 0x3500, 0xffffffff, 0x184,
788 0x3504, 0xffffffff, 0x15097f,
789 0x3500, 0xffffffff, 0x19b,
790 0x3504, 0xffffffff, 0xc0998,
791 0x3500, 0xffffffff, 0x1a9,
792 0x3504, 0xffffffff, 0x409a7,
793 0x3500, 0xffffffff, 0x1af,
794 0x3504, 0xffffffff, 0xcdc,
795 0x3500, 0xffffffff, 0x1b1,
796 0x3504, 0xffffffff, 0x800,
797 0x3508, 0xffffffff, 0x6c9b2000,
798 0x3510, 0xfc00, 0x2000,
799 0x3544, 0xffffffff, 0xfc0,
800 0x28d4, 0x00000100, 0x100
803 static void si_init_golden_registers(struct radeon_device *rdev)
805 switch (rdev->family) {
807 radeon_program_register_sequence(rdev,
808 tahiti_golden_registers,
809 (const u32)ARRAY_SIZE(tahiti_golden_registers));
810 radeon_program_register_sequence(rdev,
811 tahiti_golden_rlc_registers,
812 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
813 radeon_program_register_sequence(rdev,
814 tahiti_mgcg_cgcg_init,
815 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
816 radeon_program_register_sequence(rdev,
817 tahiti_golden_registers2,
818 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
821 radeon_program_register_sequence(rdev,
822 pitcairn_golden_registers,
823 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
824 radeon_program_register_sequence(rdev,
825 pitcairn_golden_rlc_registers,
826 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
827 radeon_program_register_sequence(rdev,
828 pitcairn_mgcg_cgcg_init,
829 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
832 radeon_program_register_sequence(rdev,
833 verde_golden_registers,
834 (const u32)ARRAY_SIZE(verde_golden_registers));
835 radeon_program_register_sequence(rdev,
836 verde_golden_rlc_registers,
837 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
838 radeon_program_register_sequence(rdev,
839 verde_mgcg_cgcg_init,
840 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
841 radeon_program_register_sequence(rdev,
843 (const u32)ARRAY_SIZE(verde_pg_init));
846 radeon_program_register_sequence(rdev,
847 oland_golden_registers,
848 (const u32)ARRAY_SIZE(oland_golden_registers));
849 radeon_program_register_sequence(rdev,
850 oland_golden_rlc_registers,
851 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
852 radeon_program_register_sequence(rdev,
853 oland_mgcg_cgcg_init,
854 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
861 #define PCIE_BUS_CLK 10000
862 #define TCLK (PCIE_BUS_CLK / 10)
865 * si_get_xclk - get the xclk
867 * @rdev: radeon_device pointer
869 * Returns the reference clock used by the gfx engine
872 u32 si_get_xclk(struct radeon_device *rdev)
874 u32 reference_clock = rdev->clock.spll.reference_freq;
877 tmp = RREG32(CG_CLKPIN_CNTL_2);
878 if (tmp & MUX_TCLK_TO_XCLK)
881 tmp = RREG32(CG_CLKPIN_CNTL);
882 if (tmp & XTALIN_DIVIDE)
883 return reference_clock / 4;
885 return reference_clock;
888 /* get temperature in millidegrees */
889 int si_get_temp(struct radeon_device *rdev)
894 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
900 actual_temp = temp & 0x1ff;
902 actual_temp = (actual_temp * 1000);
907 #define TAHITI_IO_MC_REGS_SIZE 36
909 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
910 {0x0000006f, 0x03044000},
911 {0x00000070, 0x0480c018},
912 {0x00000071, 0x00000040},
913 {0x00000072, 0x01000000},
914 {0x00000074, 0x000000ff},
915 {0x00000075, 0x00143400},
916 {0x00000076, 0x08ec0800},
917 {0x00000077, 0x040000cc},
918 {0x00000079, 0x00000000},
919 {0x0000007a, 0x21000409},
920 {0x0000007c, 0x00000000},
921 {0x0000007d, 0xe8000000},
922 {0x0000007e, 0x044408a8},
923 {0x0000007f, 0x00000003},
924 {0x00000080, 0x00000000},
925 {0x00000081, 0x01000000},
926 {0x00000082, 0x02000000},
927 {0x00000083, 0x00000000},
928 {0x00000084, 0xe3f3e4f4},
929 {0x00000085, 0x00052024},
930 {0x00000087, 0x00000000},
931 {0x00000088, 0x66036603},
932 {0x00000089, 0x01000000},
933 {0x0000008b, 0x1c0a0000},
934 {0x0000008c, 0xff010000},
935 {0x0000008e, 0xffffefff},
936 {0x0000008f, 0xfff3efff},
937 {0x00000090, 0xfff3efbf},
938 {0x00000094, 0x00101101},
939 {0x00000095, 0x00000fff},
940 {0x00000096, 0x00116fff},
941 {0x00000097, 0x60010000},
942 {0x00000098, 0x10010000},
943 {0x00000099, 0x00006000},
944 {0x0000009a, 0x00001000},
945 {0x0000009f, 0x00a77400}
948 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
949 {0x0000006f, 0x03044000},
950 {0x00000070, 0x0480c018},
951 {0x00000071, 0x00000040},
952 {0x00000072, 0x01000000},
953 {0x00000074, 0x000000ff},
954 {0x00000075, 0x00143400},
955 {0x00000076, 0x08ec0800},
956 {0x00000077, 0x040000cc},
957 {0x00000079, 0x00000000},
958 {0x0000007a, 0x21000409},
959 {0x0000007c, 0x00000000},
960 {0x0000007d, 0xe8000000},
961 {0x0000007e, 0x044408a8},
962 {0x0000007f, 0x00000003},
963 {0x00000080, 0x00000000},
964 {0x00000081, 0x01000000},
965 {0x00000082, 0x02000000},
966 {0x00000083, 0x00000000},
967 {0x00000084, 0xe3f3e4f4},
968 {0x00000085, 0x00052024},
969 {0x00000087, 0x00000000},
970 {0x00000088, 0x66036603},
971 {0x00000089, 0x01000000},
972 {0x0000008b, 0x1c0a0000},
973 {0x0000008c, 0xff010000},
974 {0x0000008e, 0xffffefff},
975 {0x0000008f, 0xfff3efff},
976 {0x00000090, 0xfff3efbf},
977 {0x00000094, 0x00101101},
978 {0x00000095, 0x00000fff},
979 {0x00000096, 0x00116fff},
980 {0x00000097, 0x60010000},
981 {0x00000098, 0x10010000},
982 {0x00000099, 0x00006000},
983 {0x0000009a, 0x00001000},
984 {0x0000009f, 0x00a47400}
987 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
988 {0x0000006f, 0x03044000},
989 {0x00000070, 0x0480c018},
990 {0x00000071, 0x00000040},
991 {0x00000072, 0x01000000},
992 {0x00000074, 0x000000ff},
993 {0x00000075, 0x00143400},
994 {0x00000076, 0x08ec0800},
995 {0x00000077, 0x040000cc},
996 {0x00000079, 0x00000000},
997 {0x0000007a, 0x21000409},
998 {0x0000007c, 0x00000000},
999 {0x0000007d, 0xe8000000},
1000 {0x0000007e, 0x044408a8},
1001 {0x0000007f, 0x00000003},
1002 {0x00000080, 0x00000000},
1003 {0x00000081, 0x01000000},
1004 {0x00000082, 0x02000000},
1005 {0x00000083, 0x00000000},
1006 {0x00000084, 0xe3f3e4f4},
1007 {0x00000085, 0x00052024},
1008 {0x00000087, 0x00000000},
1009 {0x00000088, 0x66036603},
1010 {0x00000089, 0x01000000},
1011 {0x0000008b, 0x1c0a0000},
1012 {0x0000008c, 0xff010000},
1013 {0x0000008e, 0xffffefff},
1014 {0x0000008f, 0xfff3efff},
1015 {0x00000090, 0xfff3efbf},
1016 {0x00000094, 0x00101101},
1017 {0x00000095, 0x00000fff},
1018 {0x00000096, 0x00116fff},
1019 {0x00000097, 0x60010000},
1020 {0x00000098, 0x10010000},
1021 {0x00000099, 0x00006000},
1022 {0x0000009a, 0x00001000},
1023 {0x0000009f, 0x00a37400}
1026 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1027 {0x0000006f, 0x03044000},
1028 {0x00000070, 0x0480c018},
1029 {0x00000071, 0x00000040},
1030 {0x00000072, 0x01000000},
1031 {0x00000074, 0x000000ff},
1032 {0x00000075, 0x00143400},
1033 {0x00000076, 0x08ec0800},
1034 {0x00000077, 0x040000cc},
1035 {0x00000079, 0x00000000},
1036 {0x0000007a, 0x21000409},
1037 {0x0000007c, 0x00000000},
1038 {0x0000007d, 0xe8000000},
1039 {0x0000007e, 0x044408a8},
1040 {0x0000007f, 0x00000003},
1041 {0x00000080, 0x00000000},
1042 {0x00000081, 0x01000000},
1043 {0x00000082, 0x02000000},
1044 {0x00000083, 0x00000000},
1045 {0x00000084, 0xe3f3e4f4},
1046 {0x00000085, 0x00052024},
1047 {0x00000087, 0x00000000},
1048 {0x00000088, 0x66036603},
1049 {0x00000089, 0x01000000},
1050 {0x0000008b, 0x1c0a0000},
1051 {0x0000008c, 0xff010000},
1052 {0x0000008e, 0xffffefff},
1053 {0x0000008f, 0xfff3efff},
1054 {0x00000090, 0xfff3efbf},
1055 {0x00000094, 0x00101101},
1056 {0x00000095, 0x00000fff},
1057 {0x00000096, 0x00116fff},
1058 {0x00000097, 0x60010000},
1059 {0x00000098, 0x10010000},
1060 {0x00000099, 0x00006000},
1061 {0x0000009a, 0x00001000},
1062 {0x0000009f, 0x00a17730}
1066 static int si_mc_load_microcode(struct radeon_device *rdev)
1068 const __be32 *fw_data;
1069 u32 running, blackout = 0;
1071 int i, ucode_size, regs_size;
1076 switch (rdev->family) {
1078 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1079 ucode_size = SI_MC_UCODE_SIZE;
1080 regs_size = TAHITI_IO_MC_REGS_SIZE;
1083 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1084 ucode_size = SI_MC_UCODE_SIZE;
1085 regs_size = TAHITI_IO_MC_REGS_SIZE;
1089 io_mc_regs = (u32 *)&verde_io_mc_regs;
1090 ucode_size = SI_MC_UCODE_SIZE;
1091 regs_size = TAHITI_IO_MC_REGS_SIZE;
1094 io_mc_regs = (u32 *)&oland_io_mc_regs;
1095 ucode_size = OLAND_MC_UCODE_SIZE;
1096 regs_size = TAHITI_IO_MC_REGS_SIZE;
1100 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1104 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1105 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1108 /* reset the engine and set to writable */
1109 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1110 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1112 /* load mc io regs */
1113 for (i = 0; i < regs_size; i++) {
1114 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1115 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1117 /* load the MC ucode */
1118 fw_data = (const __be32 *)rdev->mc_fw->data;
1119 for (i = 0; i < ucode_size; i++)
1120 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1122 /* put the engine back into the active state */
1123 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1124 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1125 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1127 /* wait for training to complete */
1128 for (i = 0; i < rdev->usec_timeout; i++) {
1129 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1133 for (i = 0; i < rdev->usec_timeout; i++) {
1134 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1140 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1146 static int si_init_microcode(struct radeon_device *rdev)
1148 struct platform_device *pdev;
1149 const char *chip_name;
1150 const char *rlc_chip_name;
1151 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1157 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1160 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1164 switch (rdev->family) {
1166 chip_name = "TAHITI";
1167 rlc_chip_name = "TAHITI";
1168 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1169 me_req_size = SI_PM4_UCODE_SIZE * 4;
1170 ce_req_size = SI_CE_UCODE_SIZE * 4;
1171 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1172 mc_req_size = SI_MC_UCODE_SIZE * 4;
1175 chip_name = "PITCAIRN";
1176 rlc_chip_name = "PITCAIRN";
1177 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1178 me_req_size = SI_PM4_UCODE_SIZE * 4;
1179 ce_req_size = SI_CE_UCODE_SIZE * 4;
1180 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1181 mc_req_size = SI_MC_UCODE_SIZE * 4;
1184 chip_name = "VERDE";
1185 rlc_chip_name = "VERDE";
1186 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1187 me_req_size = SI_PM4_UCODE_SIZE * 4;
1188 ce_req_size = SI_CE_UCODE_SIZE * 4;
1189 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1190 mc_req_size = SI_MC_UCODE_SIZE * 4;
1193 chip_name = "OLAND";
1194 rlc_chip_name = "OLAND";
1195 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1196 me_req_size = SI_PM4_UCODE_SIZE * 4;
1197 ce_req_size = SI_CE_UCODE_SIZE * 4;
1198 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1199 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1204 DRM_INFO("Loading %s Microcode\n", chip_name);
1206 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1207 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1210 if (rdev->pfp_fw->size != pfp_req_size) {
1212 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1213 rdev->pfp_fw->size, fw_name);
1218 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1219 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1222 if (rdev->me_fw->size != me_req_size) {
1224 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1225 rdev->me_fw->size, fw_name);
1229 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1230 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1233 if (rdev->ce_fw->size != ce_req_size) {
1235 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1236 rdev->ce_fw->size, fw_name);
1240 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1241 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1244 if (rdev->rlc_fw->size != rlc_req_size) {
1246 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1247 rdev->rlc_fw->size, fw_name);
1251 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1252 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1255 if (rdev->mc_fw->size != mc_req_size) {
1257 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1258 rdev->mc_fw->size, fw_name);
1263 platform_device_unregister(pdev);
1268 "si_cp: Failed to load firmware \"%s\"\n",
1270 release_firmware(rdev->pfp_fw);
1271 rdev->pfp_fw = NULL;
1272 release_firmware(rdev->me_fw);
1274 release_firmware(rdev->ce_fw);
1276 release_firmware(rdev->rlc_fw);
1277 rdev->rlc_fw = NULL;
1278 release_firmware(rdev->mc_fw);
1284 /* watermark setup */
1285 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1286 struct radeon_crtc *radeon_crtc,
1287 struct drm_display_mode *mode,
1288 struct drm_display_mode *other_mode)
1293 * There are 3 line buffers, each one shared by 2 display controllers.
1294 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1295 * the display controllers. The paritioning is done via one of four
1296 * preset allocations specified in bits 21:20:
1298 * 2 - whole lb, other crtc must be disabled
1300 /* this can get tricky if we have two large displays on a paired group
1301 * of crtcs. Ideally for multiple large displays we'd assign them to
1302 * non-linked crtcs for maximum line buffer allocation.
1304 if (radeon_crtc->base.enabled && mode) {
1308 tmp = 2; /* whole */
1312 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1313 DC_LB_MEMORY_CONFIG(tmp));
1315 if (radeon_crtc->base.enabled && mode) {
1325 /* controller not enabled, so no lb used */
1329 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1331 u32 tmp = RREG32(MC_SHARED_CHMAP);
1333 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1356 struct dce6_wm_params {
1357 u32 dram_channels; /* number of dram channels */
1358 u32 yclk; /* bandwidth per dram data pin in kHz */
1359 u32 sclk; /* engine clock in kHz */
1360 u32 disp_clk; /* display clock in kHz */
1361 u32 src_width; /* viewport width */
1362 u32 active_time; /* active display time in ns */
1363 u32 blank_time; /* blank time in ns */
1364 bool interlaced; /* mode is interlaced */
1365 fixed20_12 vsc; /* vertical scale ratio */
1366 u32 num_heads; /* number of active crtcs */
1367 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1368 u32 lb_size; /* line buffer allocated to pipe */
1369 u32 vtaps; /* vertical scaler taps */
1372 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1374 /* Calculate raw DRAM Bandwidth */
1375 fixed20_12 dram_efficiency; /* 0.7 */
1376 fixed20_12 yclk, dram_channels, bandwidth;
1379 a.full = dfixed_const(1000);
1380 yclk.full = dfixed_const(wm->yclk);
1381 yclk.full = dfixed_div(yclk, a);
1382 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1383 a.full = dfixed_const(10);
1384 dram_efficiency.full = dfixed_const(7);
1385 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1386 bandwidth.full = dfixed_mul(dram_channels, yclk);
1387 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1389 return dfixed_trunc(bandwidth);
1392 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1394 /* Calculate DRAM Bandwidth and the part allocated to display. */
1395 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1396 fixed20_12 yclk, dram_channels, bandwidth;
1399 a.full = dfixed_const(1000);
1400 yclk.full = dfixed_const(wm->yclk);
1401 yclk.full = dfixed_div(yclk, a);
1402 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1403 a.full = dfixed_const(10);
1404 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1405 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1406 bandwidth.full = dfixed_mul(dram_channels, yclk);
1407 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1409 return dfixed_trunc(bandwidth);
1412 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1414 /* Calculate the display Data return Bandwidth */
1415 fixed20_12 return_efficiency; /* 0.8 */
1416 fixed20_12 sclk, bandwidth;
1419 a.full = dfixed_const(1000);
1420 sclk.full = dfixed_const(wm->sclk);
1421 sclk.full = dfixed_div(sclk, a);
1422 a.full = dfixed_const(10);
1423 return_efficiency.full = dfixed_const(8);
1424 return_efficiency.full = dfixed_div(return_efficiency, a);
1425 a.full = dfixed_const(32);
1426 bandwidth.full = dfixed_mul(a, sclk);
1427 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1429 return dfixed_trunc(bandwidth);
1432 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1437 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1439 /* Calculate the DMIF Request Bandwidth */
1440 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1441 fixed20_12 disp_clk, sclk, bandwidth;
1442 fixed20_12 a, b1, b2;
1445 a.full = dfixed_const(1000);
1446 disp_clk.full = dfixed_const(wm->disp_clk);
1447 disp_clk.full = dfixed_div(disp_clk, a);
1448 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1449 b1.full = dfixed_mul(a, disp_clk);
1451 a.full = dfixed_const(1000);
1452 sclk.full = dfixed_const(wm->sclk);
1453 sclk.full = dfixed_div(sclk, a);
1454 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1455 b2.full = dfixed_mul(a, sclk);
1457 a.full = dfixed_const(10);
1458 disp_clk_request_efficiency.full = dfixed_const(8);
1459 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1461 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1463 a.full = dfixed_const(min_bandwidth);
1464 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1466 return dfixed_trunc(bandwidth);
1469 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1471 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1472 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1473 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1474 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1476 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1479 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1481 /* Calculate the display mode Average Bandwidth
1482 * DisplayMode should contain the source and destination dimensions,
1486 fixed20_12 line_time;
1487 fixed20_12 src_width;
1488 fixed20_12 bandwidth;
1491 a.full = dfixed_const(1000);
1492 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1493 line_time.full = dfixed_div(line_time, a);
1494 bpp.full = dfixed_const(wm->bytes_per_pixel);
1495 src_width.full = dfixed_const(wm->src_width);
1496 bandwidth.full = dfixed_mul(src_width, bpp);
1497 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1498 bandwidth.full = dfixed_div(bandwidth, line_time);
1500 return dfixed_trunc(bandwidth);
1503 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1505 /* First calcualte the latency in ns */
1506 u32 mc_latency = 2000; /* 2000 ns. */
1507 u32 available_bandwidth = dce6_available_bandwidth(wm);
1508 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1509 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1510 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1511 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1512 (wm->num_heads * cursor_line_pair_return_time);
1513 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1514 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1515 u32 tmp, dmif_size = 12288;
1518 if (wm->num_heads == 0)
1521 a.full = dfixed_const(2);
1522 b.full = dfixed_const(1);
1523 if ((wm->vsc.full > a.full) ||
1524 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1526 ((wm->vsc.full >= a.full) && wm->interlaced))
1527 max_src_lines_per_dst_line = 4;
1529 max_src_lines_per_dst_line = 2;
1531 a.full = dfixed_const(available_bandwidth);
1532 b.full = dfixed_const(wm->num_heads);
1533 a.full = dfixed_div(a, b);
1535 b.full = dfixed_const(mc_latency + 512);
1536 c.full = dfixed_const(wm->disp_clk);
1537 b.full = dfixed_div(b, c);
1539 c.full = dfixed_const(dmif_size);
1540 b.full = dfixed_div(c, b);
1542 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1544 b.full = dfixed_const(1000);
1545 c.full = dfixed_const(wm->disp_clk);
1546 b.full = dfixed_div(c, b);
1547 c.full = dfixed_const(wm->bytes_per_pixel);
1548 b.full = dfixed_mul(b, c);
1550 lb_fill_bw = min(tmp, dfixed_trunc(b));
1552 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1553 b.full = dfixed_const(1000);
1554 c.full = dfixed_const(lb_fill_bw);
1555 b.full = dfixed_div(c, b);
1556 a.full = dfixed_div(a, b);
1557 line_fill_time = dfixed_trunc(a);
1559 if (line_fill_time < wm->active_time)
1562 return latency + (line_fill_time - wm->active_time);
1566 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1568 if (dce6_average_bandwidth(wm) <=
1569 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1575 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1577 if (dce6_average_bandwidth(wm) <=
1578 (dce6_available_bandwidth(wm) / wm->num_heads))
1584 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1586 u32 lb_partitions = wm->lb_size / wm->src_width;
1587 u32 line_time = wm->active_time + wm->blank_time;
1588 u32 latency_tolerant_lines;
1592 a.full = dfixed_const(1);
1593 if (wm->vsc.full > a.full)
1594 latency_tolerant_lines = 1;
1596 if (lb_partitions <= (wm->vtaps + 1))
1597 latency_tolerant_lines = 1;
1599 latency_tolerant_lines = 2;
1602 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
1604 if (dce6_latency_watermark(wm) <= latency_hiding)
1610 static void dce6_program_watermarks(struct radeon_device *rdev,
1611 struct radeon_crtc *radeon_crtc,
1612 u32 lb_size, u32 num_heads)
1614 struct drm_display_mode *mode = &radeon_crtc->base.mode;
1615 struct dce6_wm_params wm;
1618 u32 latency_watermark_a = 0, latency_watermark_b = 0;
1619 u32 priority_a_mark = 0, priority_b_mark = 0;
1620 u32 priority_a_cnt = PRIORITY_OFF;
1621 u32 priority_b_cnt = PRIORITY_OFF;
1622 u32 tmp, arb_control3;
1625 if (radeon_crtc->base.enabled && num_heads && mode) {
1626 pixel_period = 1000000 / (u32)mode->clock;
1627 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1631 wm.yclk = rdev->pm.current_mclk * 10;
1632 wm.sclk = rdev->pm.current_sclk * 10;
1633 wm.disp_clk = mode->clock;
1634 wm.src_width = mode->crtc_hdisplay;
1635 wm.active_time = mode->crtc_hdisplay * pixel_period;
1636 wm.blank_time = line_time - wm.active_time;
1637 wm.interlaced = false;
1638 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1639 wm.interlaced = true;
1640 wm.vsc = radeon_crtc->vsc;
1642 if (radeon_crtc->rmx_type != RMX_OFF)
1644 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
1645 wm.lb_size = lb_size;
1646 if (rdev->family == CHIP_ARUBA)
1647 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
1649 wm.dram_channels = si_get_number_of_dram_channels(rdev);
1650 wm.num_heads = num_heads;
1652 /* set for high clocks */
1653 latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535);
1654 /* set for low clocks */
1655 /* wm.yclk = low clk; wm.sclk = low clk */
1656 latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535);
1658 /* possibly force display priority to high */
1659 /* should really do this at mode validation time... */
1660 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
1661 !dce6_average_bandwidth_vs_available_bandwidth(&wm) ||
1662 !dce6_check_latency_hiding(&wm) ||
1663 (rdev->disp_priority == 2)) {
1664 DRM_DEBUG_KMS("force priority to high\n");
1665 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1666 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1669 a.full = dfixed_const(1000);
1670 b.full = dfixed_const(mode->clock);
1671 b.full = dfixed_div(b, a);
1672 c.full = dfixed_const(latency_watermark_a);
1673 c.full = dfixed_mul(c, b);
1674 c.full = dfixed_mul(c, radeon_crtc->hsc);
1675 c.full = dfixed_div(c, a);
1676 a.full = dfixed_const(16);
1677 c.full = dfixed_div(c, a);
1678 priority_a_mark = dfixed_trunc(c);
1679 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1681 a.full = dfixed_const(1000);
1682 b.full = dfixed_const(mode->clock);
1683 b.full = dfixed_div(b, a);
1684 c.full = dfixed_const(latency_watermark_b);
1685 c.full = dfixed_mul(c, b);
1686 c.full = dfixed_mul(c, radeon_crtc->hsc);
1687 c.full = dfixed_div(c, a);
1688 a.full = dfixed_const(16);
1689 c.full = dfixed_div(c, a);
1690 priority_b_mark = dfixed_trunc(c);
1691 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1695 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1697 tmp &= ~LATENCY_WATERMARK_MASK(3);
1698 tmp |= LATENCY_WATERMARK_MASK(1);
1699 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1700 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1701 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1702 LATENCY_HIGH_WATERMARK(line_time)));
1704 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1705 tmp &= ~LATENCY_WATERMARK_MASK(3);
1706 tmp |= LATENCY_WATERMARK_MASK(2);
1707 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1708 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1709 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1710 LATENCY_HIGH_WATERMARK(line_time)));
1711 /* restore original selection */
1712 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
1714 /* write the priority marks */
1715 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1716 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1720 void dce6_bandwidth_update(struct radeon_device *rdev)
1722 struct drm_display_mode *mode0 = NULL;
1723 struct drm_display_mode *mode1 = NULL;
1724 u32 num_heads = 0, lb_size;
1727 radeon_update_display_priority(rdev);
1729 for (i = 0; i < rdev->num_crtc; i++) {
1730 if (rdev->mode_info.crtcs[i]->base.enabled)
1733 for (i = 0; i < rdev->num_crtc; i += 2) {
1734 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1735 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1736 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1737 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1738 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1739 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1746 static void si_tiling_mode_table_init(struct radeon_device *rdev)
1748 const u32 num_tile_mode_states = 32;
1749 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1751 switch (rdev->config.si.mem_row_size_in_kb) {
1753 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1757 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1760 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1764 if ((rdev->family == CHIP_TAHITI) ||
1765 (rdev->family == CHIP_PITCAIRN)) {
1766 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1767 switch (reg_offset) {
1768 case 0: /* non-AA compressed depth or any compressed stencil */
1769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1770 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1771 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1773 NUM_BANKS(ADDR_SURF_16_BANK) |
1774 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1775 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1776 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1778 case 1: /* 2xAA/4xAA compressed depth only */
1779 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1780 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1781 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1782 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1783 NUM_BANKS(ADDR_SURF_16_BANK) |
1784 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1785 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1786 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1788 case 2: /* 8xAA compressed depth only */
1789 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1790 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1791 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1792 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1793 NUM_BANKS(ADDR_SURF_16_BANK) |
1794 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1795 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1796 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1798 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
1799 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1800 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1801 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1802 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1803 NUM_BANKS(ADDR_SURF_16_BANK) |
1804 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1805 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1806 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1808 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
1809 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1810 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1811 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1812 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1813 NUM_BANKS(ADDR_SURF_16_BANK) |
1814 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1815 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1816 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1818 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
1819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1820 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1821 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1822 TILE_SPLIT(split_equal_to_row_size) |
1823 NUM_BANKS(ADDR_SURF_16_BANK) |
1824 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1825 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1826 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1828 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
1829 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1830 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1831 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1832 TILE_SPLIT(split_equal_to_row_size) |
1833 NUM_BANKS(ADDR_SURF_16_BANK) |
1834 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1835 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1836 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1838 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
1839 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1840 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1841 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1842 TILE_SPLIT(split_equal_to_row_size) |
1843 NUM_BANKS(ADDR_SURF_16_BANK) |
1844 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1845 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1846 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1848 case 8: /* 1D and 1D Array Surfaces */
1849 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1850 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1851 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1852 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1853 NUM_BANKS(ADDR_SURF_16_BANK) |
1854 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1855 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1856 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1858 case 9: /* Displayable maps. */
1859 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1860 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1861 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1862 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1863 NUM_BANKS(ADDR_SURF_16_BANK) |
1864 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1865 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1866 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1868 case 10: /* Display 8bpp. */
1869 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1870 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1871 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1872 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1873 NUM_BANKS(ADDR_SURF_16_BANK) |
1874 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1875 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1876 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1878 case 11: /* Display 16bpp. */
1879 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1880 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1881 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1882 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1883 NUM_BANKS(ADDR_SURF_16_BANK) |
1884 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1885 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1886 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1888 case 12: /* Display 32bpp. */
1889 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1890 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1891 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1892 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1893 NUM_BANKS(ADDR_SURF_16_BANK) |
1894 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1895 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1896 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1898 case 13: /* Thin. */
1899 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1900 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1901 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1902 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1903 NUM_BANKS(ADDR_SURF_16_BANK) |
1904 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1905 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1906 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1908 case 14: /* Thin 8 bpp. */
1909 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1910 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1911 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1912 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1913 NUM_BANKS(ADDR_SURF_16_BANK) |
1914 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1918 case 15: /* Thin 16 bpp. */
1919 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1920 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1921 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1922 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1923 NUM_BANKS(ADDR_SURF_16_BANK) |
1924 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1928 case 16: /* Thin 32 bpp. */
1929 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1930 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1931 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1932 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1933 NUM_BANKS(ADDR_SURF_16_BANK) |
1934 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1935 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1936 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1938 case 17: /* Thin 64 bpp. */
1939 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1940 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1941 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1942 TILE_SPLIT(split_equal_to_row_size) |
1943 NUM_BANKS(ADDR_SURF_16_BANK) |
1944 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1948 case 21: /* 8 bpp PRT. */
1949 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1950 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1951 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1952 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1953 NUM_BANKS(ADDR_SURF_16_BANK) |
1954 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1958 case 22: /* 16 bpp PRT */
1959 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1960 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1961 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1962 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1963 NUM_BANKS(ADDR_SURF_16_BANK) |
1964 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1965 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1966 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1968 case 23: /* 32 bpp PRT */
1969 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1970 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1971 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1972 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1973 NUM_BANKS(ADDR_SURF_16_BANK) |
1974 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1978 case 24: /* 64 bpp PRT */
1979 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1980 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1981 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1982 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1983 NUM_BANKS(ADDR_SURF_16_BANK) |
1984 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1988 case 25: /* 128 bpp PRT */
1989 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1990 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1991 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1992 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1993 NUM_BANKS(ADDR_SURF_8_BANK) |
1994 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1995 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1996 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2002 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2003 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2005 } else if ((rdev->family == CHIP_VERDE) ||
2006 (rdev->family == CHIP_OLAND)) {
2007 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2008 switch (reg_offset) {
2009 case 0: /* non-AA compressed depth or any compressed stencil */
2010 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2011 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2012 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2013 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2014 NUM_BANKS(ADDR_SURF_16_BANK) |
2015 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2019 case 1: /* 2xAA/4xAA compressed depth only */
2020 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2021 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2022 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2023 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2024 NUM_BANKS(ADDR_SURF_16_BANK) |
2025 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2026 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2027 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2029 case 2: /* 8xAA compressed depth only */
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2031 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2032 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2033 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2034 NUM_BANKS(ADDR_SURF_16_BANK) |
2035 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2039 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2043 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2044 NUM_BANKS(ADDR_SURF_16_BANK) |
2045 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2046 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2047 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2049 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2052 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2053 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2054 NUM_BANKS(ADDR_SURF_16_BANK) |
2055 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2056 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2057 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2059 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2061 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2063 TILE_SPLIT(split_equal_to_row_size) |
2064 NUM_BANKS(ADDR_SURF_16_BANK) |
2065 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2066 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2067 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2069 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2070 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2071 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2072 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2073 TILE_SPLIT(split_equal_to_row_size) |
2074 NUM_BANKS(ADDR_SURF_16_BANK) |
2075 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2076 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2077 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2079 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2080 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2081 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2082 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2083 TILE_SPLIT(split_equal_to_row_size) |
2084 NUM_BANKS(ADDR_SURF_16_BANK) |
2085 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2086 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2087 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2089 case 8: /* 1D and 1D Array Surfaces */
2090 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2091 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2092 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2093 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2094 NUM_BANKS(ADDR_SURF_16_BANK) |
2095 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2096 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2097 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2099 case 9: /* Displayable maps. */
2100 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2101 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2102 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2103 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2104 NUM_BANKS(ADDR_SURF_16_BANK) |
2105 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2106 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2107 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2109 case 10: /* Display 8bpp. */
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2113 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2114 NUM_BANKS(ADDR_SURF_16_BANK) |
2115 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2116 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2117 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2119 case 11: /* Display 16bpp. */
2120 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2121 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2122 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2123 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2124 NUM_BANKS(ADDR_SURF_16_BANK) |
2125 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2129 case 12: /* Display 32bpp. */
2130 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2131 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2132 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2133 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2134 NUM_BANKS(ADDR_SURF_16_BANK) |
2135 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2136 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2137 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2139 case 13: /* Thin. */
2140 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2141 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2142 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2143 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2144 NUM_BANKS(ADDR_SURF_16_BANK) |
2145 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2146 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2147 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2149 case 14: /* Thin 8 bpp. */
2150 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2151 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2152 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2153 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2154 NUM_BANKS(ADDR_SURF_16_BANK) |
2155 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2159 case 15: /* Thin 16 bpp. */
2160 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2161 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2162 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2163 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2164 NUM_BANKS(ADDR_SURF_16_BANK) |
2165 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2166 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2167 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2169 case 16: /* Thin 32 bpp. */
2170 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2171 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2172 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2173 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2174 NUM_BANKS(ADDR_SURF_16_BANK) |
2175 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2176 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2177 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2179 case 17: /* Thin 64 bpp. */
2180 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2181 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2182 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2183 TILE_SPLIT(split_equal_to_row_size) |
2184 NUM_BANKS(ADDR_SURF_16_BANK) |
2185 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2189 case 21: /* 8 bpp PRT. */
2190 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2191 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2192 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2193 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2194 NUM_BANKS(ADDR_SURF_16_BANK) |
2195 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2196 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2197 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2199 case 22: /* 16 bpp PRT */
2200 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2201 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2202 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2203 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2204 NUM_BANKS(ADDR_SURF_16_BANK) |
2205 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2206 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2207 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2209 case 23: /* 32 bpp PRT */
2210 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2211 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2212 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2213 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2214 NUM_BANKS(ADDR_SURF_16_BANK) |
2215 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2216 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2217 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2219 case 24: /* 64 bpp PRT */
2220 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2221 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2222 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2223 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2224 NUM_BANKS(ADDR_SURF_16_BANK) |
2225 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2226 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2227 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2229 case 25: /* 128 bpp PRT */
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2234 NUM_BANKS(ADDR_SURF_8_BANK) |
2235 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2236 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2237 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2243 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2244 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2247 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2250 static void si_select_se_sh(struct radeon_device *rdev,
2251 u32 se_num, u32 sh_num)
2253 u32 data = INSTANCE_BROADCAST_WRITES;
2255 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2256 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2257 else if (se_num == 0xffffffff)
2258 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2259 else if (sh_num == 0xffffffff)
2260 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2262 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2263 WREG32(GRBM_GFX_INDEX, data);
2266 static u32 si_create_bitmask(u32 bit_width)
2270 for (i = 0; i < bit_width; i++) {
2277 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2281 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2283 data &= INACTIVE_CUS_MASK;
2286 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2288 data >>= INACTIVE_CUS_SHIFT;
2290 mask = si_create_bitmask(cu_per_sh);
2292 return ~data & mask;
2295 static void si_setup_spi(struct radeon_device *rdev,
2296 u32 se_num, u32 sh_per_se,
2300 u32 data, mask, active_cu;
2302 for (i = 0; i < se_num; i++) {
2303 for (j = 0; j < sh_per_se; j++) {
2304 si_select_se_sh(rdev, i, j);
2305 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2306 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2309 for (k = 0; k < 16; k++) {
2311 if (active_cu & mask) {
2313 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2319 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2322 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2323 u32 max_rb_num, u32 se_num,
2328 data = RREG32(CC_RB_BACKEND_DISABLE);
2330 data &= BACKEND_DISABLE_MASK;
2333 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2335 data >>= BACKEND_DISABLE_SHIFT;
2337 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2342 static void si_setup_rb(struct radeon_device *rdev,
2343 u32 se_num, u32 sh_per_se,
2348 u32 disabled_rbs = 0;
2349 u32 enabled_rbs = 0;
2351 for (i = 0; i < se_num; i++) {
2352 for (j = 0; j < sh_per_se; j++) {
2353 si_select_se_sh(rdev, i, j);
2354 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2355 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2358 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2361 for (i = 0; i < max_rb_num; i++) {
2362 if (!(disabled_rbs & mask))
2363 enabled_rbs |= mask;
2367 for (i = 0; i < se_num; i++) {
2368 si_select_se_sh(rdev, i, 0xffffffff);
2370 for (j = 0; j < sh_per_se; j++) {
2371 switch (enabled_rbs & 3) {
2373 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2376 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2380 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2385 WREG32(PA_SC_RASTER_CONFIG, data);
2387 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2390 static void si_gpu_init(struct radeon_device *rdev)
2392 u32 gb_addr_config = 0;
2393 u32 mc_shared_chmap, mc_arb_ramcfg;
2395 u32 hdp_host_path_cntl;
2399 switch (rdev->family) {
2401 rdev->config.si.max_shader_engines = 2;
2402 rdev->config.si.max_tile_pipes = 12;
2403 rdev->config.si.max_cu_per_sh = 8;
2404 rdev->config.si.max_sh_per_se = 2;
2405 rdev->config.si.max_backends_per_se = 4;
2406 rdev->config.si.max_texture_channel_caches = 12;
2407 rdev->config.si.max_gprs = 256;
2408 rdev->config.si.max_gs_threads = 32;
2409 rdev->config.si.max_hw_contexts = 8;
2411 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2412 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2413 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2414 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2415 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2418 rdev->config.si.max_shader_engines = 2;
2419 rdev->config.si.max_tile_pipes = 8;
2420 rdev->config.si.max_cu_per_sh = 5;
2421 rdev->config.si.max_sh_per_se = 2;
2422 rdev->config.si.max_backends_per_se = 4;
2423 rdev->config.si.max_texture_channel_caches = 8;
2424 rdev->config.si.max_gprs = 256;
2425 rdev->config.si.max_gs_threads = 32;
2426 rdev->config.si.max_hw_contexts = 8;
2428 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2429 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2430 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2431 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2432 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2436 rdev->config.si.max_shader_engines = 1;
2437 rdev->config.si.max_tile_pipes = 4;
2438 rdev->config.si.max_cu_per_sh = 2;
2439 rdev->config.si.max_sh_per_se = 2;
2440 rdev->config.si.max_backends_per_se = 4;
2441 rdev->config.si.max_texture_channel_caches = 4;
2442 rdev->config.si.max_gprs = 256;
2443 rdev->config.si.max_gs_threads = 32;
2444 rdev->config.si.max_hw_contexts = 8;
2446 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2447 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2448 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2449 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2450 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2453 rdev->config.si.max_shader_engines = 1;
2454 rdev->config.si.max_tile_pipes = 4;
2455 rdev->config.si.max_cu_per_sh = 6;
2456 rdev->config.si.max_sh_per_se = 1;
2457 rdev->config.si.max_backends_per_se = 2;
2458 rdev->config.si.max_texture_channel_caches = 4;
2459 rdev->config.si.max_gprs = 256;
2460 rdev->config.si.max_gs_threads = 16;
2461 rdev->config.si.max_hw_contexts = 8;
2463 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2464 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2465 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2466 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2467 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2471 /* Initialize HDP */
2472 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2473 WREG32((0x2c14 + j), 0x00000000);
2474 WREG32((0x2c18 + j), 0x00000000);
2475 WREG32((0x2c1c + j), 0x00000000);
2476 WREG32((0x2c20 + j), 0x00000000);
2477 WREG32((0x2c24 + j), 0x00000000);
2480 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2482 evergreen_fix_pci_max_read_req_size(rdev);
2484 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2486 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2487 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2489 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
2490 rdev->config.si.mem_max_burst_length_bytes = 256;
2491 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2492 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2493 if (rdev->config.si.mem_row_size_in_kb > 4)
2494 rdev->config.si.mem_row_size_in_kb = 4;
2495 /* XXX use MC settings? */
2496 rdev->config.si.shader_engine_tile_size = 32;
2497 rdev->config.si.num_gpus = 1;
2498 rdev->config.si.multi_gpu_tile_size = 64;
2500 /* fix up row size */
2501 gb_addr_config &= ~ROW_SIZE_MASK;
2502 switch (rdev->config.si.mem_row_size_in_kb) {
2505 gb_addr_config |= ROW_SIZE(0);
2508 gb_addr_config |= ROW_SIZE(1);
2511 gb_addr_config |= ROW_SIZE(2);
2515 /* setup tiling info dword. gb_addr_config is not adequate since it does
2516 * not have bank info, so create a custom tiling dword.
2517 * bits 3:0 num_pipes
2518 * bits 7:4 num_banks
2519 * bits 11:8 group_size
2520 * bits 15:12 row_size
2522 rdev->config.si.tile_config = 0;
2523 switch (rdev->config.si.num_tile_pipes) {
2525 rdev->config.si.tile_config |= (0 << 0);
2528 rdev->config.si.tile_config |= (1 << 0);
2531 rdev->config.si.tile_config |= (2 << 0);
2535 /* XXX what about 12? */
2536 rdev->config.si.tile_config |= (3 << 0);
2539 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2540 case 0: /* four banks */
2541 rdev->config.si.tile_config |= 0 << 4;
2543 case 1: /* eight banks */
2544 rdev->config.si.tile_config |= 1 << 4;
2546 case 2: /* sixteen banks */
2548 rdev->config.si.tile_config |= 2 << 4;
2551 rdev->config.si.tile_config |=
2552 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2553 rdev->config.si.tile_config |=
2554 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2556 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2557 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2558 WREG32(DMIF_ADDR_CALC, gb_addr_config);
2559 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2560 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
2561 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
2562 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2563 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2564 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2566 si_tiling_mode_table_init(rdev);
2568 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
2569 rdev->config.si.max_sh_per_se,
2570 rdev->config.si.max_backends_per_se);
2572 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
2573 rdev->config.si.max_sh_per_se,
2574 rdev->config.si.max_cu_per_sh);
2577 /* set HW defaults for 3D engine */
2578 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2579 ROQ_IB2_START(0x2b)));
2580 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2582 sx_debug_1 = RREG32(SX_DEBUG_1);
2583 WREG32(SX_DEBUG_1, sx_debug_1);
2585 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2587 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
2588 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
2589 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
2590 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
2592 WREG32(VGT_NUM_INSTANCES, 1);
2594 WREG32(CP_PERFMON_CNTL, 0);
2596 WREG32(SQ_CONFIG, 0);
2598 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2599 FORCE_EOV_MAX_REZ_CNT(255)));
2601 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2602 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2604 WREG32(VGT_GS_VERTEX_REUSE, 16);
2605 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2607 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
2608 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
2609 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
2610 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
2611 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
2612 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
2613 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
2614 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
2616 tmp = RREG32(HDP_MISC_CNTL);
2617 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2618 WREG32(HDP_MISC_CNTL, tmp);
2620 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2621 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2623 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2629 * GPU scratch registers helpers function.
2631 static void si_scratch_init(struct radeon_device *rdev)
2635 rdev->scratch.num_reg = 7;
2636 rdev->scratch.reg_base = SCRATCH_REG0;
2637 for (i = 0; i < rdev->scratch.num_reg; i++) {
2638 rdev->scratch.free[i] = true;
2639 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2643 void si_fence_ring_emit(struct radeon_device *rdev,
2644 struct radeon_fence *fence)
2646 struct radeon_ring *ring = &rdev->ring[fence->ring];
2647 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2649 /* flush read cache over gart */
2650 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2651 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2652 radeon_ring_write(ring, 0);
2653 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2654 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2655 PACKET3_TC_ACTION_ENA |
2656 PACKET3_SH_KCACHE_ACTION_ENA |
2657 PACKET3_SH_ICACHE_ACTION_ENA);
2658 radeon_ring_write(ring, 0xFFFFFFFF);
2659 radeon_ring_write(ring, 0);
2660 radeon_ring_write(ring, 10); /* poll interval */
2661 /* EVENT_WRITE_EOP - flush caches, send int */
2662 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2663 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
2664 radeon_ring_write(ring, addr & 0xffffffff);
2665 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2666 radeon_ring_write(ring, fence->seq);
2667 radeon_ring_write(ring, 0);
2673 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2675 struct radeon_ring *ring = &rdev->ring[ib->ring];
2678 if (ib->is_const_ib) {
2679 /* set switch buffer packet before const IB */
2680 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2681 radeon_ring_write(ring, 0);
2683 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
2686 if (ring->rptr_save_reg) {
2687 next_rptr = ring->wptr + 3 + 4 + 8;
2688 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2689 radeon_ring_write(ring, ((ring->rptr_save_reg -
2690 PACKET3_SET_CONFIG_REG_START) >> 2));
2691 radeon_ring_write(ring, next_rptr);
2692 } else if (rdev->wb.enabled) {
2693 next_rptr = ring->wptr + 5 + 4 + 8;
2694 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2695 radeon_ring_write(ring, (1 << 8));
2696 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2697 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
2698 radeon_ring_write(ring, next_rptr);
2701 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
2704 radeon_ring_write(ring, header);
2705 radeon_ring_write(ring,
2709 (ib->gpu_addr & 0xFFFFFFFC));
2710 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2711 radeon_ring_write(ring, ib->length_dw |
2712 (ib->vm ? (ib->vm->id << 24) : 0));
2714 if (!ib->is_const_ib) {
2715 /* flush read cache over gart for this vmid */
2716 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2717 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2718 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
2719 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2720 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2721 PACKET3_TC_ACTION_ENA |
2722 PACKET3_SH_KCACHE_ACTION_ENA |
2723 PACKET3_SH_ICACHE_ACTION_ENA);
2724 radeon_ring_write(ring, 0xFFFFFFFF);
2725 radeon_ring_write(ring, 0);
2726 radeon_ring_write(ring, 10); /* poll interval */
2733 static void si_cp_enable(struct radeon_device *rdev, bool enable)
2736 WREG32(CP_ME_CNTL, 0);
2738 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2739 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
2740 WREG32(SCRATCH_UMSK, 0);
2741 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2742 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2743 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2748 static int si_cp_load_microcode(struct radeon_device *rdev)
2750 const __be32 *fw_data;
2753 if (!rdev->me_fw || !rdev->pfp_fw)
2756 si_cp_enable(rdev, false);
2759 fw_data = (const __be32 *)rdev->pfp_fw->data;
2760 WREG32(CP_PFP_UCODE_ADDR, 0);
2761 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
2762 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2763 WREG32(CP_PFP_UCODE_ADDR, 0);
2766 fw_data = (const __be32 *)rdev->ce_fw->data;
2767 WREG32(CP_CE_UCODE_ADDR, 0);
2768 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
2769 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
2770 WREG32(CP_CE_UCODE_ADDR, 0);
2773 fw_data = (const __be32 *)rdev->me_fw->data;
2774 WREG32(CP_ME_RAM_WADDR, 0);
2775 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
2776 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2777 WREG32(CP_ME_RAM_WADDR, 0);
2779 WREG32(CP_PFP_UCODE_ADDR, 0);
2780 WREG32(CP_CE_UCODE_ADDR, 0);
2781 WREG32(CP_ME_RAM_WADDR, 0);
2782 WREG32(CP_ME_RAM_RADDR, 0);
2786 static int si_cp_start(struct radeon_device *rdev)
2788 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2791 r = radeon_ring_lock(rdev, ring, 7 + 4);
2793 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2797 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2798 radeon_ring_write(ring, 0x1);
2799 radeon_ring_write(ring, 0x0);
2800 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
2801 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2802 radeon_ring_write(ring, 0);
2803 radeon_ring_write(ring, 0);
2805 /* init the CE partitions */
2806 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2807 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2808 radeon_ring_write(ring, 0xc000);
2809 radeon_ring_write(ring, 0xe000);
2810 radeon_ring_unlock_commit(rdev, ring);
2812 si_cp_enable(rdev, true);
2814 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
2816 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2820 /* setup clear context state */
2821 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2822 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2824 for (i = 0; i < si_default_size; i++)
2825 radeon_ring_write(ring, si_default_state[i]);
2827 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2828 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2830 /* set clear context state */
2831 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2832 radeon_ring_write(ring, 0);
2834 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2835 radeon_ring_write(ring, 0x00000316);
2836 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2837 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2839 radeon_ring_unlock_commit(rdev, ring);
2841 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
2842 ring = &rdev->ring[i];
2843 r = radeon_ring_lock(rdev, ring, 2);
2845 /* clear the compute context state */
2846 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
2847 radeon_ring_write(ring, 0);
2849 radeon_ring_unlock_commit(rdev, ring);
2855 static void si_cp_fini(struct radeon_device *rdev)
2857 struct radeon_ring *ring;
2858 si_cp_enable(rdev, false);
2860 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2861 radeon_ring_fini(rdev, ring);
2862 radeon_scratch_free(rdev, ring->rptr_save_reg);
2864 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
2865 radeon_ring_fini(rdev, ring);
2866 radeon_scratch_free(rdev, ring->rptr_save_reg);
2868 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
2869 radeon_ring_fini(rdev, ring);
2870 radeon_scratch_free(rdev, ring->rptr_save_reg);
2873 static int si_cp_resume(struct radeon_device *rdev)
2875 struct radeon_ring *ring;
2880 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2881 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2886 RREG32(GRBM_SOFT_RESET);
2888 WREG32(GRBM_SOFT_RESET, 0);
2889 RREG32(GRBM_SOFT_RESET);
2891 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2892 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2894 /* Set the write pointer delay */
2895 WREG32(CP_RB_WPTR_DELAY, 0);
2897 WREG32(CP_DEBUG, 0);
2898 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2900 /* ring 0 - compute and gfx */
2901 /* Set ring buffer size */
2902 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2903 rb_bufsz = drm_order(ring->ring_size / 8);
2904 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2906 tmp |= BUF_SWAP_32BIT;
2908 WREG32(CP_RB0_CNTL, tmp);
2910 /* Initialize the ring buffer's read and write pointers */
2911 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
2913 WREG32(CP_RB0_WPTR, ring->wptr);
2915 /* set the wb address whether it's enabled or not */
2916 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
2917 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2919 if (rdev->wb.enabled)
2920 WREG32(SCRATCH_UMSK, 0xff);
2922 tmp |= RB_NO_UPDATE;
2923 WREG32(SCRATCH_UMSK, 0);
2927 WREG32(CP_RB0_CNTL, tmp);
2929 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
2931 ring->rptr = RREG32(CP_RB0_RPTR);
2933 /* ring1 - compute only */
2934 /* Set ring buffer size */
2935 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
2936 rb_bufsz = drm_order(ring->ring_size / 8);
2937 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2939 tmp |= BUF_SWAP_32BIT;
2941 WREG32(CP_RB1_CNTL, tmp);
2943 /* Initialize the ring buffer's read and write pointers */
2944 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
2946 WREG32(CP_RB1_WPTR, ring->wptr);
2948 /* set the wb address whether it's enabled or not */
2949 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
2950 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
2953 WREG32(CP_RB1_CNTL, tmp);
2955 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
2957 ring->rptr = RREG32(CP_RB1_RPTR);
2959 /* ring2 - compute only */
2960 /* Set ring buffer size */
2961 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
2962 rb_bufsz = drm_order(ring->ring_size / 8);
2963 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2965 tmp |= BUF_SWAP_32BIT;
2967 WREG32(CP_RB2_CNTL, tmp);
2969 /* Initialize the ring buffer's read and write pointers */
2970 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
2972 WREG32(CP_RB2_WPTR, ring->wptr);
2974 /* set the wb address whether it's enabled or not */
2975 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
2976 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
2979 WREG32(CP_RB2_CNTL, tmp);
2981 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
2983 ring->rptr = RREG32(CP_RB2_RPTR);
2985 /* start the rings */
2987 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
2988 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
2989 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
2990 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
2992 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2993 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2994 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2997 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
2999 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3001 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3003 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3009 static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3015 tmp = RREG32(GRBM_STATUS);
3016 if (tmp & (PA_BUSY | SC_BUSY |
3017 BCI_BUSY | SX_BUSY |
3018 TA_BUSY | VGT_BUSY |
3020 GDS_BUSY | SPI_BUSY |
3021 IA_BUSY | IA_BUSY_NO_DMA))
3022 reset_mask |= RADEON_RESET_GFX;
3024 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3025 CP_BUSY | CP_COHERENCY_BUSY))
3026 reset_mask |= RADEON_RESET_CP;
3028 if (tmp & GRBM_EE_BUSY)
3029 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3032 tmp = RREG32(GRBM_STATUS2);
3033 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3034 reset_mask |= RADEON_RESET_RLC;
3036 /* DMA_STATUS_REG 0 */
3037 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3038 if (!(tmp & DMA_IDLE))
3039 reset_mask |= RADEON_RESET_DMA;
3041 /* DMA_STATUS_REG 1 */
3042 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3043 if (!(tmp & DMA_IDLE))
3044 reset_mask |= RADEON_RESET_DMA1;
3047 tmp = RREG32(SRBM_STATUS2);
3049 reset_mask |= RADEON_RESET_DMA;
3051 if (tmp & DMA1_BUSY)
3052 reset_mask |= RADEON_RESET_DMA1;
3055 tmp = RREG32(SRBM_STATUS);
3058 reset_mask |= RADEON_RESET_IH;
3061 reset_mask |= RADEON_RESET_SEM;
3063 if (tmp & GRBM_RQ_PENDING)
3064 reset_mask |= RADEON_RESET_GRBM;
3067 reset_mask |= RADEON_RESET_VMC;
3069 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3070 MCC_BUSY | MCD_BUSY))
3071 reset_mask |= RADEON_RESET_MC;
3073 if (evergreen_is_display_hung(rdev))
3074 reset_mask |= RADEON_RESET_DISPLAY;
3077 tmp = RREG32(VM_L2_STATUS);
3079 reset_mask |= RADEON_RESET_VMC;
3081 /* Skip MC reset as it's mostly likely not hung, just busy */
3082 if (reset_mask & RADEON_RESET_MC) {
3083 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3084 reset_mask &= ~RADEON_RESET_MC;
3090 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3092 struct evergreen_mc_save save;
3093 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3096 if (reset_mask == 0)
3099 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3101 evergreen_print_gpu_status_regs(rdev);
3102 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3103 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3104 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3105 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3107 /* Disable CP parsing/prefetching */
3108 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3110 if (reset_mask & RADEON_RESET_DMA) {
3112 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3113 tmp &= ~DMA_RB_ENABLE;
3114 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3116 if (reset_mask & RADEON_RESET_DMA1) {
3118 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3119 tmp &= ~DMA_RB_ENABLE;
3120 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3125 evergreen_mc_stop(rdev, &save);
3126 if (evergreen_mc_wait_for_idle(rdev)) {
3127 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3130 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3131 grbm_soft_reset = SOFT_RESET_CB |
3145 if (reset_mask & RADEON_RESET_CP) {
3146 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3148 srbm_soft_reset |= SOFT_RESET_GRBM;
3151 if (reset_mask & RADEON_RESET_DMA)
3152 srbm_soft_reset |= SOFT_RESET_DMA;
3154 if (reset_mask & RADEON_RESET_DMA1)
3155 srbm_soft_reset |= SOFT_RESET_DMA1;
3157 if (reset_mask & RADEON_RESET_DISPLAY)
3158 srbm_soft_reset |= SOFT_RESET_DC;
3160 if (reset_mask & RADEON_RESET_RLC)
3161 grbm_soft_reset |= SOFT_RESET_RLC;
3163 if (reset_mask & RADEON_RESET_SEM)
3164 srbm_soft_reset |= SOFT_RESET_SEM;
3166 if (reset_mask & RADEON_RESET_IH)
3167 srbm_soft_reset |= SOFT_RESET_IH;
3169 if (reset_mask & RADEON_RESET_GRBM)
3170 srbm_soft_reset |= SOFT_RESET_GRBM;
3172 if (reset_mask & RADEON_RESET_VMC)
3173 srbm_soft_reset |= SOFT_RESET_VMC;
3175 if (reset_mask & RADEON_RESET_MC)
3176 srbm_soft_reset |= SOFT_RESET_MC;
3178 if (grbm_soft_reset) {
3179 tmp = RREG32(GRBM_SOFT_RESET);
3180 tmp |= grbm_soft_reset;
3181 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3182 WREG32(GRBM_SOFT_RESET, tmp);
3183 tmp = RREG32(GRBM_SOFT_RESET);
3187 tmp &= ~grbm_soft_reset;
3188 WREG32(GRBM_SOFT_RESET, tmp);
3189 tmp = RREG32(GRBM_SOFT_RESET);
3192 if (srbm_soft_reset) {
3193 tmp = RREG32(SRBM_SOFT_RESET);
3194 tmp |= srbm_soft_reset;
3195 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3196 WREG32(SRBM_SOFT_RESET, tmp);
3197 tmp = RREG32(SRBM_SOFT_RESET);
3201 tmp &= ~srbm_soft_reset;
3202 WREG32(SRBM_SOFT_RESET, tmp);
3203 tmp = RREG32(SRBM_SOFT_RESET);
3206 /* Wait a little for things to settle down */
3209 evergreen_mc_resume(rdev, &save);
3212 evergreen_print_gpu_status_regs(rdev);
3215 int si_asic_reset(struct radeon_device *rdev)
3219 reset_mask = si_gpu_check_soft_reset(rdev);
3222 r600_set_bios_scratch_engine_hung(rdev, true);
3224 si_gpu_soft_reset(rdev, reset_mask);
3226 reset_mask = si_gpu_check_soft_reset(rdev);
3229 r600_set_bios_scratch_engine_hung(rdev, false);
3235 * si_gfx_is_lockup - Check if the GFX engine is locked up
3237 * @rdev: radeon_device pointer
3238 * @ring: radeon_ring structure holding ring information
3240 * Check if the GFX engine is locked up.
3241 * Returns true if the engine appears to be locked up, false if not.
3243 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3245 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3247 if (!(reset_mask & (RADEON_RESET_GFX |
3248 RADEON_RESET_COMPUTE |
3249 RADEON_RESET_CP))) {
3250 radeon_ring_lockup_update(ring);
3253 /* force CP activities */
3254 radeon_ring_force_activity(rdev, ring);
3255 return radeon_ring_test_lockup(rdev, ring);
3259 * si_dma_is_lockup - Check if the DMA engine is locked up
3261 * @rdev: radeon_device pointer
3262 * @ring: radeon_ring structure holding ring information
3264 * Check if the async DMA engine is locked up.
3265 * Returns true if the engine appears to be locked up, false if not.
3267 bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3269 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3272 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3273 mask = RADEON_RESET_DMA;
3275 mask = RADEON_RESET_DMA1;
3277 if (!(reset_mask & mask)) {
3278 radeon_ring_lockup_update(ring);
3281 /* force ring activities */
3282 radeon_ring_force_activity(rdev, ring);
3283 return radeon_ring_test_lockup(rdev, ring);
3287 static void si_mc_program(struct radeon_device *rdev)
3289 struct evergreen_mc_save save;
3293 /* Initialize HDP */
3294 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3295 WREG32((0x2c14 + j), 0x00000000);
3296 WREG32((0x2c18 + j), 0x00000000);
3297 WREG32((0x2c1c + j), 0x00000000);
3298 WREG32((0x2c20 + j), 0x00000000);
3299 WREG32((0x2c24 + j), 0x00000000);
3301 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3303 evergreen_mc_stop(rdev, &save);
3304 if (radeon_mc_wait_for_idle(rdev)) {
3305 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3307 /* Lockout access through VGA aperture*/
3308 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3309 /* Update configuration */
3310 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3311 rdev->mc.vram_start >> 12);
3312 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3313 rdev->mc.vram_end >> 12);
3314 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3315 rdev->vram_scratch.gpu_addr >> 12);
3316 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3317 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3318 WREG32(MC_VM_FB_LOCATION, tmp);
3319 /* XXX double check these! */
3320 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3321 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3322 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3323 WREG32(MC_VM_AGP_BASE, 0);
3324 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3325 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3326 if (radeon_mc_wait_for_idle(rdev)) {
3327 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3329 evergreen_mc_resume(rdev, &save);
3330 /* we need to own VRAM, so turn off the VGA renderer here
3331 * to stop it overwriting our objects */
3332 rv515_vga_render_disable(rdev);
3335 static void si_vram_gtt_location(struct radeon_device *rdev,
3336 struct radeon_mc *mc)
3338 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3339 /* leave room for at least 1024M GTT */
3340 dev_warn(rdev->dev, "limiting VRAM\n");
3341 mc->real_vram_size = 0xFFC0000000ULL;
3342 mc->mc_vram_size = 0xFFC0000000ULL;
3344 radeon_vram_location(rdev, &rdev->mc, 0);
3345 rdev->mc.gtt_base_align = 0;
3346 radeon_gtt_location(rdev, mc);
3349 static int si_mc_init(struct radeon_device *rdev)
3352 int chansize, numchan;
3354 /* Get VRAM informations */
3355 rdev->mc.vram_is_ddr = true;
3356 tmp = RREG32(MC_ARB_RAMCFG);
3357 if (tmp & CHANSIZE_OVERRIDE) {
3359 } else if (tmp & CHANSIZE_MASK) {
3364 tmp = RREG32(MC_SHARED_CHMAP);
3365 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3395 rdev->mc.vram_width = numchan * chansize;
3396 /* Could aper size report 0 ? */
3397 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3398 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3399 /* size in MB on si */
3400 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3401 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3402 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3403 si_vram_gtt_location(rdev, &rdev->mc);
3404 radeon_update_bandwidth_info(rdev);
3412 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3414 /* flush hdp cache */
3415 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3417 /* bits 0-15 are the VM contexts0-15 */
3418 WREG32(VM_INVALIDATE_REQUEST, 1);
3421 static int si_pcie_gart_enable(struct radeon_device *rdev)
3425 if (rdev->gart.robj == NULL) {
3426 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3429 r = radeon_gart_table_vram_pin(rdev);
3432 radeon_gart_restore(rdev);
3433 /* Setup TLB control */
3434 WREG32(MC_VM_MX_L1_TLB_CNTL,
3437 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3438 ENABLE_ADVANCED_DRIVER_MODEL |
3439 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3440 /* Setup L2 cache */
3441 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3442 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3443 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3444 EFFECTIVE_L2_QUEUE_SIZE(7) |
3445 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3446 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3447 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3448 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3449 /* setup context0 */
3450 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3451 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3452 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3453 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3454 (u32)(rdev->dummy_page.addr >> 12));
3455 WREG32(VM_CONTEXT0_CNTL2, 0);
3456 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3457 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3463 /* empty context1-15 */
3464 /* set vm size, must be a multiple of 4 */
3465 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3466 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3467 /* Assign the pt base to something valid for now; the pts used for
3468 * the VMs are determined by the application and setup and assigned
3469 * on the fly in the vm part of radeon_gart.c
3471 for (i = 1; i < 16; i++) {
3473 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3474 rdev->gart.table_addr >> 12);
3476 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3477 rdev->gart.table_addr >> 12);
3480 /* enable context1-15 */
3481 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3482 (u32)(rdev->dummy_page.addr >> 12));
3483 WREG32(VM_CONTEXT1_CNTL2, 4);
3484 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3485 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3486 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3487 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3488 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3489 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3490 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3491 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3492 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3493 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3494 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3495 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3496 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
3498 si_pcie_gart_tlb_flush(rdev);
3499 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3500 (unsigned)(rdev->mc.gtt_size >> 20),
3501 (unsigned long long)rdev->gart.table_addr);
3502 rdev->gart.ready = true;
3506 static void si_pcie_gart_disable(struct radeon_device *rdev)
3508 /* Disable all tables */
3509 WREG32(VM_CONTEXT0_CNTL, 0);
3510 WREG32(VM_CONTEXT1_CNTL, 0);
3511 /* Setup TLB control */
3512 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3513 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3514 /* Setup L2 cache */
3515 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3516 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3517 EFFECTIVE_L2_QUEUE_SIZE(7) |
3518 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3519 WREG32(VM_L2_CNTL2, 0);
3520 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3521 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3522 radeon_gart_table_vram_unpin(rdev);
3525 static void si_pcie_gart_fini(struct radeon_device *rdev)
3527 si_pcie_gart_disable(rdev);
3528 radeon_gart_table_vram_free(rdev);
3529 radeon_gart_fini(rdev);
3533 static bool si_vm_reg_valid(u32 reg)
3535 /* context regs are fine */
3539 /* check config regs */
3541 case GRBM_GFX_INDEX:
3542 case CP_STRMOUT_CNTL:
3543 case VGT_VTX_VECT_EJECT_REG:
3544 case VGT_CACHE_INVALIDATION:
3545 case VGT_ESGS_RING_SIZE:
3546 case VGT_GSVS_RING_SIZE:
3547 case VGT_GS_VERTEX_REUSE:
3548 case VGT_PRIMITIVE_TYPE:
3549 case VGT_INDEX_TYPE:
3550 case VGT_NUM_INDICES:
3551 case VGT_NUM_INSTANCES:
3552 case VGT_TF_RING_SIZE:
3553 case VGT_HS_OFFCHIP_PARAM:
3554 case VGT_TF_MEMORY_BASE:
3556 case PA_SU_LINE_STIPPLE_VALUE:
3557 case PA_SC_LINE_STIPPLE_STATE:
3560 case SPI_STATIC_THREAD_MGMT_1:
3561 case SPI_STATIC_THREAD_MGMT_2:
3562 case SPI_STATIC_THREAD_MGMT_3:
3563 case SPI_PS_MAX_WAVE_ID:
3564 case SPI_CONFIG_CNTL:
3565 case SPI_CONFIG_CNTL_1:
3569 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
3574 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
3575 u32 *ib, struct radeon_cs_packet *pkt)
3577 switch (pkt->opcode) {
3579 case PACKET3_SET_BASE:
3580 case PACKET3_SET_CE_DE_COUNTERS:
3581 case PACKET3_LOAD_CONST_RAM:
3582 case PACKET3_WRITE_CONST_RAM:
3583 case PACKET3_WRITE_CONST_RAM_OFFSET:
3584 case PACKET3_DUMP_CONST_RAM:
3585 case PACKET3_INCREMENT_CE_COUNTER:
3586 case PACKET3_WAIT_ON_DE_COUNTER:
3587 case PACKET3_CE_WRITE:
3590 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
3596 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
3597 u32 *ib, struct radeon_cs_packet *pkt)
3599 u32 idx = pkt->idx + 1;
3600 u32 idx_value = ib[idx];
3601 u32 start_reg, end_reg, reg, i;
3604 switch (pkt->opcode) {
3606 case PACKET3_SET_BASE:
3607 case PACKET3_CLEAR_STATE:
3608 case PACKET3_INDEX_BUFFER_SIZE:
3609 case PACKET3_DISPATCH_DIRECT:
3610 case PACKET3_DISPATCH_INDIRECT:
3611 case PACKET3_ALLOC_GDS:
3612 case PACKET3_WRITE_GDS_RAM:
3613 case PACKET3_ATOMIC_GDS:
3614 case PACKET3_ATOMIC:
3615 case PACKET3_OCCLUSION_QUERY:
3616 case PACKET3_SET_PREDICATION:
3617 case PACKET3_COND_EXEC:
3618 case PACKET3_PRED_EXEC:
3619 case PACKET3_DRAW_INDIRECT:
3620 case PACKET3_DRAW_INDEX_INDIRECT:
3621 case PACKET3_INDEX_BASE:
3622 case PACKET3_DRAW_INDEX_2:
3623 case PACKET3_CONTEXT_CONTROL:
3624 case PACKET3_INDEX_TYPE:
3625 case PACKET3_DRAW_INDIRECT_MULTI:
3626 case PACKET3_DRAW_INDEX_AUTO:
3627 case PACKET3_DRAW_INDEX_IMMD:
3628 case PACKET3_NUM_INSTANCES:
3629 case PACKET3_DRAW_INDEX_MULTI_AUTO:
3630 case PACKET3_STRMOUT_BUFFER_UPDATE:
3631 case PACKET3_DRAW_INDEX_OFFSET_2:
3632 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
3633 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
3634 case PACKET3_MPEG_INDEX:
3635 case PACKET3_WAIT_REG_MEM:
3636 case PACKET3_MEM_WRITE:
3637 case PACKET3_PFP_SYNC_ME:
3638 case PACKET3_SURFACE_SYNC:
3639 case PACKET3_EVENT_WRITE:
3640 case PACKET3_EVENT_WRITE_EOP:
3641 case PACKET3_EVENT_WRITE_EOS:
3642 case PACKET3_SET_CONTEXT_REG:
3643 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3644 case PACKET3_SET_SH_REG:
3645 case PACKET3_SET_SH_REG_OFFSET:
3646 case PACKET3_INCREMENT_DE_COUNTER:
3647 case PACKET3_WAIT_ON_CE_COUNTER:
3648 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3649 case PACKET3_ME_WRITE:
3651 case PACKET3_COPY_DATA:
3652 if ((idx_value & 0xf00) == 0) {
3653 reg = ib[idx + 3] * 4;
3654 if (!si_vm_reg_valid(reg))
3658 case PACKET3_WRITE_DATA:
3659 if ((idx_value & 0xf00) == 0) {
3660 start_reg = ib[idx + 1] * 4;
3661 if (idx_value & 0x10000) {
3662 if (!si_vm_reg_valid(start_reg))
3665 for (i = 0; i < (pkt->count - 2); i++) {
3666 reg = start_reg + (4 * i);
3667 if (!si_vm_reg_valid(reg))
3673 case PACKET3_COND_WRITE:
3674 if (idx_value & 0x100) {
3675 reg = ib[idx + 5] * 4;
3676 if (!si_vm_reg_valid(reg))
3680 case PACKET3_COPY_DW:
3681 if (idx_value & 0x2) {
3682 reg = ib[idx + 3] * 4;
3683 if (!si_vm_reg_valid(reg))
3687 case PACKET3_SET_CONFIG_REG:
3688 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
3689 end_reg = 4 * pkt->count + start_reg - 4;
3690 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
3691 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
3692 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
3693 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3696 for (i = 0; i < pkt->count; i++) {
3697 reg = start_reg + (4 * i);
3698 if (!si_vm_reg_valid(reg))
3702 case PACKET3_CP_DMA:
3703 command = ib[idx + 4];
3705 if (command & PACKET3_CP_DMA_CMD_SAS) {
3706 /* src address space is register */
3707 if (((info & 0x60000000) >> 29) == 0) {
3708 start_reg = idx_value << 2;
3709 if (command & PACKET3_CP_DMA_CMD_SAIC) {
3711 if (!si_vm_reg_valid(reg)) {
3712 DRM_ERROR("CP DMA Bad SRC register\n");
3716 for (i = 0; i < (command & 0x1fffff); i++) {
3717 reg = start_reg + (4 * i);
3718 if (!si_vm_reg_valid(reg)) {
3719 DRM_ERROR("CP DMA Bad SRC register\n");
3726 if (command & PACKET3_CP_DMA_CMD_DAS) {
3727 /* dst address space is register */
3728 if (((info & 0x00300000) >> 20) == 0) {
3729 start_reg = ib[idx + 2];
3730 if (command & PACKET3_CP_DMA_CMD_DAIC) {
3732 if (!si_vm_reg_valid(reg)) {
3733 DRM_ERROR("CP DMA Bad DST register\n");
3737 for (i = 0; i < (command & 0x1fffff); i++) {
3738 reg = start_reg + (4 * i);
3739 if (!si_vm_reg_valid(reg)) {
3740 DRM_ERROR("CP DMA Bad DST register\n");
3749 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
3755 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
3756 u32 *ib, struct radeon_cs_packet *pkt)
3758 u32 idx = pkt->idx + 1;
3759 u32 idx_value = ib[idx];
3760 u32 start_reg, reg, i;
3762 switch (pkt->opcode) {
3764 case PACKET3_SET_BASE:
3765 case PACKET3_CLEAR_STATE:
3766 case PACKET3_DISPATCH_DIRECT:
3767 case PACKET3_DISPATCH_INDIRECT:
3768 case PACKET3_ALLOC_GDS:
3769 case PACKET3_WRITE_GDS_RAM:
3770 case PACKET3_ATOMIC_GDS:
3771 case PACKET3_ATOMIC:
3772 case PACKET3_OCCLUSION_QUERY:
3773 case PACKET3_SET_PREDICATION:
3774 case PACKET3_COND_EXEC:
3775 case PACKET3_PRED_EXEC:
3776 case PACKET3_CONTEXT_CONTROL:
3777 case PACKET3_STRMOUT_BUFFER_UPDATE:
3778 case PACKET3_WAIT_REG_MEM:
3779 case PACKET3_MEM_WRITE:
3780 case PACKET3_PFP_SYNC_ME:
3781 case PACKET3_SURFACE_SYNC:
3782 case PACKET3_EVENT_WRITE:
3783 case PACKET3_EVENT_WRITE_EOP:
3784 case PACKET3_EVENT_WRITE_EOS:
3785 case PACKET3_SET_CONTEXT_REG:
3786 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3787 case PACKET3_SET_SH_REG:
3788 case PACKET3_SET_SH_REG_OFFSET:
3789 case PACKET3_INCREMENT_DE_COUNTER:
3790 case PACKET3_WAIT_ON_CE_COUNTER:
3791 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3792 case PACKET3_ME_WRITE:
3794 case PACKET3_COPY_DATA:
3795 if ((idx_value & 0xf00) == 0) {
3796 reg = ib[idx + 3] * 4;
3797 if (!si_vm_reg_valid(reg))
3801 case PACKET3_WRITE_DATA:
3802 if ((idx_value & 0xf00) == 0) {
3803 start_reg = ib[idx + 1] * 4;
3804 if (idx_value & 0x10000) {
3805 if (!si_vm_reg_valid(start_reg))
3808 for (i = 0; i < (pkt->count - 2); i++) {
3809 reg = start_reg + (4 * i);
3810 if (!si_vm_reg_valid(reg))
3816 case PACKET3_COND_WRITE:
3817 if (idx_value & 0x100) {
3818 reg = ib[idx + 5] * 4;
3819 if (!si_vm_reg_valid(reg))
3823 case PACKET3_COPY_DW:
3824 if (idx_value & 0x2) {
3825 reg = ib[idx + 3] * 4;
3826 if (!si_vm_reg_valid(reg))
3831 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
3837 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
3841 struct radeon_cs_packet pkt;
3845 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
3846 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
3849 case RADEON_PACKET_TYPE0:
3850 dev_err(rdev->dev, "Packet0 not allowed!\n");
3853 case RADEON_PACKET_TYPE2:
3856 case RADEON_PACKET_TYPE3:
3857 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
3858 if (ib->is_const_ib)
3859 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
3862 case RADEON_RING_TYPE_GFX_INDEX:
3863 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
3865 case CAYMAN_RING_TYPE_CP1_INDEX:
3866 case CAYMAN_RING_TYPE_CP2_INDEX:
3867 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
3870 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
3875 idx += pkt.count + 2;
3878 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
3884 } while (idx < ib->length_dw);
3892 int si_vm_init(struct radeon_device *rdev)
3895 rdev->vm_manager.nvm = 16;
3896 /* base offset of vram pages */
3897 rdev->vm_manager.vram_base_offset = 0;
3902 void si_vm_fini(struct radeon_device *rdev)
3907 * si_vm_set_page - update the page tables using the CP
3909 * @rdev: radeon_device pointer
3910 * @ib: indirect buffer to fill with commands
3911 * @pe: addr of the page entry
3912 * @addr: dst addr to write into pe
3913 * @count: number of page entries to update
3914 * @incr: increase next addr by incr bytes
3915 * @flags: access flags
3917 * Update the page tables using the CP (SI).
3919 void si_vm_set_page(struct radeon_device *rdev,
3920 struct radeon_ib *ib,
3922 uint64_t addr, unsigned count,
3923 uint32_t incr, uint32_t flags)
3925 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
3929 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
3931 ndw = 2 + count * 2;
3935 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
3936 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
3937 WRITE_DATA_DST_SEL(1));
3938 ib->ptr[ib->length_dw++] = pe;
3939 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
3940 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
3941 if (flags & RADEON_VM_PAGE_SYSTEM) {
3942 value = radeon_vm_map_gart(rdev, addr);
3943 value &= 0xFFFFFFFFFFFFF000ULL;
3944 } else if (flags & RADEON_VM_PAGE_VALID) {
3950 value |= r600_flags;
3951 ib->ptr[ib->length_dw++] = value;
3952 ib->ptr[ib->length_dw++] = upper_32_bits(value);
3957 if (flags & RADEON_VM_PAGE_SYSTEM) {
3963 /* for non-physically contiguous pages (system) */
3964 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
3965 ib->ptr[ib->length_dw++] = pe;
3966 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
3967 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
3968 if (flags & RADEON_VM_PAGE_SYSTEM) {
3969 value = radeon_vm_map_gart(rdev, addr);
3970 value &= 0xFFFFFFFFFFFFF000ULL;
3971 } else if (flags & RADEON_VM_PAGE_VALID) {
3977 value |= r600_flags;
3978 ib->ptr[ib->length_dw++] = value;
3979 ib->ptr[ib->length_dw++] = upper_32_bits(value);
3988 if (flags & RADEON_VM_PAGE_VALID)
3992 /* for physically contiguous pages (vram) */
3993 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
3994 ib->ptr[ib->length_dw++] = pe; /* dst addr */
3995 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
3996 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
3997 ib->ptr[ib->length_dw++] = 0;
3998 ib->ptr[ib->length_dw++] = value; /* value */
3999 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4000 ib->ptr[ib->length_dw++] = incr; /* increment size */
4001 ib->ptr[ib->length_dw++] = 0;
4003 addr += (ndw / 2) * incr;
4007 while (ib->length_dw & 0x7)
4008 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
4012 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4014 struct radeon_ring *ring = &rdev->ring[ridx];
4019 /* write new base address */
4020 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4021 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4022 WRITE_DATA_DST_SEL(0)));
4025 radeon_ring_write(ring,
4026 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4028 radeon_ring_write(ring,
4029 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4031 radeon_ring_write(ring, 0);
4032 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4034 /* flush hdp cache */
4035 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4036 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4037 WRITE_DATA_DST_SEL(0)));
4038 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4039 radeon_ring_write(ring, 0);
4040 radeon_ring_write(ring, 0x1);
4042 /* bits 0-15 are the VM contexts0-15 */
4043 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4044 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4045 WRITE_DATA_DST_SEL(0)));
4046 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4047 radeon_ring_write(ring, 0);
4048 radeon_ring_write(ring, 1 << vm->id);
4050 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4051 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4052 radeon_ring_write(ring, 0x0);
4055 void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4057 struct radeon_ring *ring = &rdev->ring[ridx];
4062 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4064 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4066 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4068 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4070 /* flush hdp cache */
4071 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4072 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4073 radeon_ring_write(ring, 1);
4075 /* bits 0-7 are the VM contexts0-7 */
4076 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4077 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4078 radeon_ring_write(ring, 1 << vm->id);
4084 void si_rlc_fini(struct radeon_device *rdev)
4088 /* save restore block */
4089 if (rdev->rlc.save_restore_obj) {
4090 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4091 if (unlikely(r != 0))
4092 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4093 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4094 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4096 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4097 rdev->rlc.save_restore_obj = NULL;
4100 /* clear state block */
4101 if (rdev->rlc.clear_state_obj) {
4102 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4103 if (unlikely(r != 0))
4104 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4105 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4106 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4108 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4109 rdev->rlc.clear_state_obj = NULL;
4113 int si_rlc_init(struct radeon_device *rdev)
4117 /* save restore block */
4118 if (rdev->rlc.save_restore_obj == NULL) {
4119 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
4120 RADEON_GEM_DOMAIN_VRAM, NULL,
4121 &rdev->rlc.save_restore_obj);
4123 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4128 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4129 if (unlikely(r != 0)) {
4133 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4134 &rdev->rlc.save_restore_gpu_addr);
4135 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4137 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4142 /* clear state block */
4143 if (rdev->rlc.clear_state_obj == NULL) {
4144 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
4145 RADEON_GEM_DOMAIN_VRAM, NULL,
4146 &rdev->rlc.clear_state_obj);
4148 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4153 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4154 if (unlikely(r != 0)) {
4158 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4159 &rdev->rlc.clear_state_gpu_addr);
4160 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4162 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4170 static void si_rlc_stop(struct radeon_device *rdev)
4172 WREG32(RLC_CNTL, 0);
4175 static void si_rlc_start(struct radeon_device *rdev)
4177 WREG32(RLC_CNTL, RLC_ENABLE);
4180 static int si_rlc_resume(struct radeon_device *rdev)
4183 const __be32 *fw_data;
4190 WREG32(RLC_RL_BASE, 0);
4191 WREG32(RLC_RL_SIZE, 0);
4192 WREG32(RLC_LB_CNTL, 0);
4193 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
4194 WREG32(RLC_LB_CNTR_INIT, 0);
4196 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4197 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4199 WREG32(RLC_MC_CNTL, 0);
4200 WREG32(RLC_UCODE_CNTL, 0);
4202 fw_data = (const __be32 *)rdev->rlc_fw->data;
4203 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
4204 WREG32(RLC_UCODE_ADDR, i);
4205 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4207 WREG32(RLC_UCODE_ADDR, 0);
4214 static void si_enable_interrupts(struct radeon_device *rdev)
4216 u32 ih_cntl = RREG32(IH_CNTL);
4217 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4219 ih_cntl |= ENABLE_INTR;
4220 ih_rb_cntl |= IH_RB_ENABLE;
4221 WREG32(IH_CNTL, ih_cntl);
4222 WREG32(IH_RB_CNTL, ih_rb_cntl);
4223 rdev->ih.enabled = true;
4226 static void si_disable_interrupts(struct radeon_device *rdev)
4228 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4229 u32 ih_cntl = RREG32(IH_CNTL);
4231 ih_rb_cntl &= ~IH_RB_ENABLE;
4232 ih_cntl &= ~ENABLE_INTR;
4233 WREG32(IH_RB_CNTL, ih_rb_cntl);
4234 WREG32(IH_CNTL, ih_cntl);
4235 /* set rptr, wptr to 0 */
4236 WREG32(IH_RB_RPTR, 0);
4237 WREG32(IH_RB_WPTR, 0);
4238 rdev->ih.enabled = false;
4242 static void si_disable_interrupt_state(struct radeon_device *rdev)
4246 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4247 WREG32(CP_INT_CNTL_RING1, 0);
4248 WREG32(CP_INT_CNTL_RING2, 0);
4249 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4250 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
4251 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4252 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
4253 WREG32(GRBM_INT_CNTL, 0);
4254 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4255 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4256 if (rdev->num_crtc >= 4) {
4257 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4258 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4260 if (rdev->num_crtc >= 6) {
4261 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4262 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4265 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4266 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4267 if (rdev->num_crtc >= 4) {
4268 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4269 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4271 if (rdev->num_crtc >= 6) {
4272 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4273 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4276 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4278 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4279 WREG32(DC_HPD1_INT_CONTROL, tmp);
4280 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4281 WREG32(DC_HPD2_INT_CONTROL, tmp);
4282 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4283 WREG32(DC_HPD3_INT_CONTROL, tmp);
4284 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4285 WREG32(DC_HPD4_INT_CONTROL, tmp);
4286 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4287 WREG32(DC_HPD5_INT_CONTROL, tmp);
4288 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4289 WREG32(DC_HPD6_INT_CONTROL, tmp);
4293 static int si_irq_init(struct radeon_device *rdev)
4297 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4300 ret = r600_ih_ring_alloc(rdev);
4305 si_disable_interrupts(rdev);
4308 ret = si_rlc_resume(rdev);
4310 r600_ih_ring_fini(rdev);
4314 /* setup interrupt control */
4315 /* set dummy read address to ring address */
4316 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4317 interrupt_cntl = RREG32(INTERRUPT_CNTL);
4318 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4319 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4321 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4322 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4323 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4324 WREG32(INTERRUPT_CNTL, interrupt_cntl);
4326 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4327 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4329 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4330 IH_WPTR_OVERFLOW_CLEAR |
4333 if (rdev->wb.enabled)
4334 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4336 /* set the writeback address whether it's enabled or not */
4337 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4338 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
4340 WREG32(IH_RB_CNTL, ih_rb_cntl);
4342 /* set rptr, wptr to 0 */
4343 WREG32(IH_RB_RPTR, 0);
4344 WREG32(IH_RB_WPTR, 0);
4346 /* Default settings for IH_CNTL (disabled at first) */
4347 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
4348 /* RPTR_REARM only works if msi's are enabled */
4349 if (rdev->msi_enabled)
4350 ih_cntl |= RPTR_REARM;
4351 WREG32(IH_CNTL, ih_cntl);
4353 /* force the active interrupt state to all disabled */
4354 si_disable_interrupt_state(rdev);
4356 pci_set_master(rdev->pdev);
4359 si_enable_interrupts(rdev);
4364 int si_irq_set(struct radeon_device *rdev)
4366 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4367 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4368 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4369 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4370 u32 grbm_int_cntl = 0;
4371 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4372 u32 dma_cntl, dma_cntl1;
4374 if (!rdev->irq.installed) {
4375 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4378 /* don't enable anything if the ih is disabled */
4379 if (!rdev->ih.enabled) {
4380 si_disable_interrupts(rdev);
4381 /* force the active interrupt state to all disabled */
4382 si_disable_interrupt_state(rdev);
4386 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4387 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4388 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4389 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4390 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4391 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4393 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4394 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4396 /* enable CP interrupts on all rings */
4397 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4398 DRM_DEBUG("si_irq_set: sw int gfx\n");
4399 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4401 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4402 DRM_DEBUG("si_irq_set: sw int cp1\n");
4403 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4405 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4406 DRM_DEBUG("si_irq_set: sw int cp2\n");
4407 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4409 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4410 DRM_DEBUG("si_irq_set: sw int dma\n");
4411 dma_cntl |= TRAP_ENABLE;
4414 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4415 DRM_DEBUG("si_irq_set: sw int dma1\n");
4416 dma_cntl1 |= TRAP_ENABLE;
4418 if (rdev->irq.crtc_vblank_int[0] ||
4419 atomic_read(&rdev->irq.pflip[0])) {
4420 DRM_DEBUG("si_irq_set: vblank 0\n");
4421 crtc1 |= VBLANK_INT_MASK;
4423 if (rdev->irq.crtc_vblank_int[1] ||
4424 atomic_read(&rdev->irq.pflip[1])) {
4425 DRM_DEBUG("si_irq_set: vblank 1\n");
4426 crtc2 |= VBLANK_INT_MASK;
4428 if (rdev->irq.crtc_vblank_int[2] ||
4429 atomic_read(&rdev->irq.pflip[2])) {
4430 DRM_DEBUG("si_irq_set: vblank 2\n");
4431 crtc3 |= VBLANK_INT_MASK;
4433 if (rdev->irq.crtc_vblank_int[3] ||
4434 atomic_read(&rdev->irq.pflip[3])) {
4435 DRM_DEBUG("si_irq_set: vblank 3\n");
4436 crtc4 |= VBLANK_INT_MASK;
4438 if (rdev->irq.crtc_vblank_int[4] ||
4439 atomic_read(&rdev->irq.pflip[4])) {
4440 DRM_DEBUG("si_irq_set: vblank 4\n");
4441 crtc5 |= VBLANK_INT_MASK;
4443 if (rdev->irq.crtc_vblank_int[5] ||
4444 atomic_read(&rdev->irq.pflip[5])) {
4445 DRM_DEBUG("si_irq_set: vblank 5\n");
4446 crtc6 |= VBLANK_INT_MASK;
4448 if (rdev->irq.hpd[0]) {
4449 DRM_DEBUG("si_irq_set: hpd 1\n");
4450 hpd1 |= DC_HPDx_INT_EN;
4452 if (rdev->irq.hpd[1]) {
4453 DRM_DEBUG("si_irq_set: hpd 2\n");
4454 hpd2 |= DC_HPDx_INT_EN;
4456 if (rdev->irq.hpd[2]) {
4457 DRM_DEBUG("si_irq_set: hpd 3\n");
4458 hpd3 |= DC_HPDx_INT_EN;
4460 if (rdev->irq.hpd[3]) {
4461 DRM_DEBUG("si_irq_set: hpd 4\n");
4462 hpd4 |= DC_HPDx_INT_EN;
4464 if (rdev->irq.hpd[4]) {
4465 DRM_DEBUG("si_irq_set: hpd 5\n");
4466 hpd5 |= DC_HPDx_INT_EN;
4468 if (rdev->irq.hpd[5]) {
4469 DRM_DEBUG("si_irq_set: hpd 6\n");
4470 hpd6 |= DC_HPDx_INT_EN;
4473 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
4474 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
4475 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
4477 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
4478 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
4480 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4482 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4483 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4484 if (rdev->num_crtc >= 4) {
4485 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4486 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4488 if (rdev->num_crtc >= 6) {
4489 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4490 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4493 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4494 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4495 if (rdev->num_crtc >= 4) {
4496 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4497 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4499 if (rdev->num_crtc >= 6) {
4500 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4501 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4504 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4505 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4506 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4507 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4508 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4509 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4514 static inline void si_irq_ack(struct radeon_device *rdev)
4518 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4519 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4520 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4521 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4522 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4523 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4524 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4525 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4526 if (rdev->num_crtc >= 4) {
4527 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4528 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4530 if (rdev->num_crtc >= 6) {
4531 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4532 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4535 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4536 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4537 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4538 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4539 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4540 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4541 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4542 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4543 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4544 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4545 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4546 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4548 if (rdev->num_crtc >= 4) {
4549 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4550 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4551 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4552 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4553 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4554 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4555 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4556 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4557 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4558 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4559 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4560 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4563 if (rdev->num_crtc >= 6) {
4564 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4565 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4566 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4567 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4568 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4569 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4570 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4571 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4572 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4573 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4574 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4575 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4578 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4579 tmp = RREG32(DC_HPD1_INT_CONTROL);
4580 tmp |= DC_HPDx_INT_ACK;
4581 WREG32(DC_HPD1_INT_CONTROL, tmp);
4583 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4584 tmp = RREG32(DC_HPD2_INT_CONTROL);
4585 tmp |= DC_HPDx_INT_ACK;
4586 WREG32(DC_HPD2_INT_CONTROL, tmp);
4588 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4589 tmp = RREG32(DC_HPD3_INT_CONTROL);
4590 tmp |= DC_HPDx_INT_ACK;
4591 WREG32(DC_HPD3_INT_CONTROL, tmp);
4593 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4594 tmp = RREG32(DC_HPD4_INT_CONTROL);
4595 tmp |= DC_HPDx_INT_ACK;
4596 WREG32(DC_HPD4_INT_CONTROL, tmp);
4598 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4599 tmp = RREG32(DC_HPD5_INT_CONTROL);
4600 tmp |= DC_HPDx_INT_ACK;
4601 WREG32(DC_HPD5_INT_CONTROL, tmp);
4603 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4604 tmp = RREG32(DC_HPD5_INT_CONTROL);
4605 tmp |= DC_HPDx_INT_ACK;
4606 WREG32(DC_HPD6_INT_CONTROL, tmp);
4610 static void si_irq_disable(struct radeon_device *rdev)
4612 si_disable_interrupts(rdev);
4613 /* Wait and acknowledge irq */
4616 si_disable_interrupt_state(rdev);
4619 static void si_irq_suspend(struct radeon_device *rdev)
4621 si_irq_disable(rdev);
4625 static void si_irq_fini(struct radeon_device *rdev)
4627 si_irq_suspend(rdev);
4628 r600_ih_ring_fini(rdev);
4631 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
4635 if (rdev->wb.enabled)
4636 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4638 wptr = RREG32(IH_RB_WPTR);
4640 if (wptr & RB_OVERFLOW) {
4641 /* When a ring buffer overflow happen start parsing interrupt
4642 * from the last not overwritten vector (wptr + 16). Hopefully
4643 * this should allow us to catchup.
4645 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4646 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4647 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4648 tmp = RREG32(IH_RB_CNTL);
4649 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4650 WREG32(IH_RB_CNTL, tmp);
4652 return (wptr & rdev->ih.ptr_mask);
4656 * Each IV ring entry is 128 bits:
4657 * [7:0] - interrupt source id
4659 * [59:32] - interrupt source data
4660 * [63:60] - reserved
4663 * [127:80] - reserved
4665 int si_irq_process(struct radeon_device *rdev)
4669 u32 src_id, src_data, ring_id;
4671 bool queue_hotplug = false;
4673 if (!rdev->ih.enabled || rdev->shutdown)
4676 wptr = si_get_ih_wptr(rdev);
4679 /* is somebody else already processing irqs? */
4680 if (atomic_xchg(&rdev->ih.lock, 1))
4683 rptr = rdev->ih.rptr;
4684 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4686 /* Order reading of wptr vs. reading of IH ring data */
4689 /* display interrupts */
4692 while (rptr != wptr) {
4693 /* wptr/rptr are in bytes! */
4694 ring_index = rptr / 4;
4695 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4696 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4697 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
4700 case 1: /* D1 vblank/vline */
4702 case 0: /* D1 vblank */
4703 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4704 if (rdev->irq.crtc_vblank_int[0]) {
4705 drm_handle_vblank(rdev->ddev, 0);
4706 rdev->pm.vblank_sync = true;
4707 wake_up(&rdev->irq.vblank_queue);
4709 if (atomic_read(&rdev->irq.pflip[0]))
4710 radeon_crtc_handle_flip(rdev, 0);
4711 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4712 DRM_DEBUG("IH: D1 vblank\n");
4715 case 1: /* D1 vline */
4716 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4717 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4718 DRM_DEBUG("IH: D1 vline\n");
4722 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4726 case 2: /* D2 vblank/vline */
4728 case 0: /* D2 vblank */
4729 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4730 if (rdev->irq.crtc_vblank_int[1]) {
4731 drm_handle_vblank(rdev->ddev, 1);
4732 rdev->pm.vblank_sync = true;
4733 wake_up(&rdev->irq.vblank_queue);
4735 if (atomic_read(&rdev->irq.pflip[1]))
4736 radeon_crtc_handle_flip(rdev, 1);
4737 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4738 DRM_DEBUG("IH: D2 vblank\n");
4741 case 1: /* D2 vline */
4742 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4743 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4744 DRM_DEBUG("IH: D2 vline\n");
4748 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4752 case 3: /* D3 vblank/vline */
4754 case 0: /* D3 vblank */
4755 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4756 if (rdev->irq.crtc_vblank_int[2]) {
4757 drm_handle_vblank(rdev->ddev, 2);
4758 rdev->pm.vblank_sync = true;
4759 wake_up(&rdev->irq.vblank_queue);
4761 if (atomic_read(&rdev->irq.pflip[2]))
4762 radeon_crtc_handle_flip(rdev, 2);
4763 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4764 DRM_DEBUG("IH: D3 vblank\n");
4767 case 1: /* D3 vline */
4768 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4769 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4770 DRM_DEBUG("IH: D3 vline\n");
4774 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4778 case 4: /* D4 vblank/vline */
4780 case 0: /* D4 vblank */
4781 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4782 if (rdev->irq.crtc_vblank_int[3]) {
4783 drm_handle_vblank(rdev->ddev, 3);
4784 rdev->pm.vblank_sync = true;
4785 wake_up(&rdev->irq.vblank_queue);
4787 if (atomic_read(&rdev->irq.pflip[3]))
4788 radeon_crtc_handle_flip(rdev, 3);
4789 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4790 DRM_DEBUG("IH: D4 vblank\n");
4793 case 1: /* D4 vline */
4794 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4795 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4796 DRM_DEBUG("IH: D4 vline\n");
4800 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4804 case 5: /* D5 vblank/vline */
4806 case 0: /* D5 vblank */
4807 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4808 if (rdev->irq.crtc_vblank_int[4]) {
4809 drm_handle_vblank(rdev->ddev, 4);
4810 rdev->pm.vblank_sync = true;
4811 wake_up(&rdev->irq.vblank_queue);
4813 if (atomic_read(&rdev->irq.pflip[4]))
4814 radeon_crtc_handle_flip(rdev, 4);
4815 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4816 DRM_DEBUG("IH: D5 vblank\n");
4819 case 1: /* D5 vline */
4820 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4821 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4822 DRM_DEBUG("IH: D5 vline\n");
4826 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4830 case 6: /* D6 vblank/vline */
4832 case 0: /* D6 vblank */
4833 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4834 if (rdev->irq.crtc_vblank_int[5]) {
4835 drm_handle_vblank(rdev->ddev, 5);
4836 rdev->pm.vblank_sync = true;
4837 wake_up(&rdev->irq.vblank_queue);
4839 if (atomic_read(&rdev->irq.pflip[5]))
4840 radeon_crtc_handle_flip(rdev, 5);
4841 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4842 DRM_DEBUG("IH: D6 vblank\n");
4845 case 1: /* D6 vline */
4846 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4847 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4848 DRM_DEBUG("IH: D6 vline\n");
4852 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4856 case 42: /* HPD hotplug */
4859 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4860 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4861 queue_hotplug = true;
4862 DRM_DEBUG("IH: HPD1\n");
4866 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4867 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4868 queue_hotplug = true;
4869 DRM_DEBUG("IH: HPD2\n");
4873 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4874 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4875 queue_hotplug = true;
4876 DRM_DEBUG("IH: HPD3\n");
4880 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4881 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4882 queue_hotplug = true;
4883 DRM_DEBUG("IH: HPD4\n");
4887 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4888 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4889 queue_hotplug = true;
4890 DRM_DEBUG("IH: HPD5\n");
4894 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4895 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4896 queue_hotplug = true;
4897 DRM_DEBUG("IH: HPD6\n");
4901 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4907 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4908 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4909 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4910 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4911 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4912 /* reset addr and status */
4913 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4915 case 176: /* RINGID0 CP_INT */
4916 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4918 case 177: /* RINGID1 CP_INT */
4919 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4921 case 178: /* RINGID2 CP_INT */
4922 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4924 case 181: /* CP EOP event */
4925 DRM_DEBUG("IH: CP EOP\n");
4928 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4931 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4934 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4938 case 224: /* DMA trap event */
4939 DRM_DEBUG("IH: DMA trap\n");
4940 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4942 case 233: /* GUI IDLE */
4943 DRM_DEBUG("IH: GUI idle\n");
4945 case 244: /* DMA trap event */
4946 DRM_DEBUG("IH: DMA1 trap\n");
4947 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4950 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4954 /* wptr/rptr are in bytes! */
4956 rptr &= rdev->ih.ptr_mask;
4959 schedule_work(&rdev->hotplug_work);
4960 rdev->ih.rptr = rptr;
4961 WREG32(IH_RB_RPTR, rdev->ih.rptr);
4962 atomic_set(&rdev->ih.lock, 0);
4964 /* make sure wptr hasn't changed while processing */
4965 wptr = si_get_ih_wptr(rdev);
4973 * si_copy_dma - copy pages using the DMA engine
4975 * @rdev: radeon_device pointer
4976 * @src_offset: src GPU address
4977 * @dst_offset: dst GPU address
4978 * @num_gpu_pages: number of GPU pages to xfer
4979 * @fence: radeon fence object
4981 * Copy GPU paging using the DMA engine (SI).
4982 * Used by the radeon ttm implementation to move pages if
4983 * registered as the asic copy callback.
4985 int si_copy_dma(struct radeon_device *rdev,
4986 uint64_t src_offset, uint64_t dst_offset,
4987 unsigned num_gpu_pages,
4988 struct radeon_fence **fence)
4990 struct radeon_semaphore *sem = NULL;
4991 int ring_index = rdev->asic->copy.dma_ring_index;
4992 struct radeon_ring *ring = &rdev->ring[ring_index];
4993 u32 size_in_bytes, cur_size_in_bytes;
4997 r = radeon_semaphore_create(rdev, &sem);
4999 DRM_ERROR("radeon: moving bo (%d).\n", r);
5003 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
5004 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
5005 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5007 DRM_ERROR("radeon: moving bo (%d).\n", r);
5008 radeon_semaphore_free(rdev, &sem, NULL);
5012 if (radeon_fence_need_sync(*fence, ring->idx)) {
5013 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5015 radeon_fence_note_sync(*fence, ring->idx);
5017 radeon_semaphore_free(rdev, &sem, NULL);
5020 for (i = 0; i < num_loops; i++) {
5021 cur_size_in_bytes = size_in_bytes;
5022 if (cur_size_in_bytes > 0xFFFFF)
5023 cur_size_in_bytes = 0xFFFFF;
5024 size_in_bytes -= cur_size_in_bytes;
5025 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
5026 radeon_ring_write(ring, dst_offset & 0xffffffff);
5027 radeon_ring_write(ring, src_offset & 0xffffffff);
5028 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5029 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5030 src_offset += cur_size_in_bytes;
5031 dst_offset += cur_size_in_bytes;
5034 r = radeon_fence_emit(rdev, fence, ring->idx);
5036 radeon_ring_unlock_undo(rdev, ring);
5040 radeon_ring_unlock_commit(rdev, ring);
5041 radeon_semaphore_free(rdev, &sem, *fence);
5047 * startup/shutdown callbacks
5049 static int si_startup(struct radeon_device *rdev)
5051 struct radeon_ring *ring;
5054 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5055 !rdev->rlc_fw || !rdev->mc_fw) {
5056 r = si_init_microcode(rdev);
5058 DRM_ERROR("Failed to load firmware!\n");
5063 r = si_mc_load_microcode(rdev);
5065 DRM_ERROR("Failed to load MC firmware!\n");
5069 r = r600_vram_scratch_init(rdev);
5073 si_mc_program(rdev);
5074 r = si_pcie_gart_enable(rdev);
5079 /* allocate rlc buffers */
5080 r = si_rlc_init(rdev);
5082 DRM_ERROR("Failed to init rlc BOs!\n");
5086 /* allocate wb buffer */
5087 r = radeon_wb_init(rdev);
5091 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5093 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5097 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5099 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5103 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5105 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5109 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5111 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5115 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5117 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5121 r = rv770_uvd_resume(rdev);
5123 r = radeon_fence_driver_start_ring(rdev,
5124 R600_RING_TYPE_UVD_INDEX);
5126 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5129 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5132 r = si_irq_init(rdev);
5134 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5135 radeon_irq_kms_fini(rdev);
5140 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5141 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5142 CP_RB0_RPTR, CP_RB0_WPTR,
5143 0, 0xfffff, RADEON_CP_PACKET2);
5147 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5148 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5149 CP_RB1_RPTR, CP_RB1_WPTR,
5150 0, 0xfffff, RADEON_CP_PACKET2);
5154 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5155 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5156 CP_RB2_RPTR, CP_RB2_WPTR,
5157 0, 0xfffff, RADEON_CP_PACKET2);
5161 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5162 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5163 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
5164 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
5165 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5169 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5170 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5171 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
5172 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
5173 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5177 r = si_cp_load_microcode(rdev);
5180 r = si_cp_resume(rdev);
5184 r = cayman_dma_resume(rdev);
5188 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5189 if (ring->ring_size) {
5190 r = radeon_ring_init(rdev, ring, ring->ring_size,
5191 R600_WB_UVD_RPTR_OFFSET,
5192 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5193 0, 0xfffff, RADEON_CP_PACKET2);
5195 r = r600_uvd_init(rdev);
5197 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
5200 r = radeon_ib_pool_init(rdev);
5202 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5206 r = radeon_vm_manager_init(rdev);
5208 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
5215 int si_resume(struct radeon_device *rdev)
5219 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5220 * posting will perform necessary task to bring back GPU into good
5224 atom_asic_init(rdev->mode_info.atom_context);
5226 /* init golden registers */
5227 si_init_golden_registers(rdev);
5229 rdev->accel_working = true;
5230 r = si_startup(rdev);
5232 DRM_ERROR("si startup failed on resume\n");
5233 rdev->accel_working = false;
5241 int si_suspend(struct radeon_device *rdev)
5243 radeon_vm_manager_fini(rdev);
5244 si_cp_enable(rdev, false);
5245 cayman_dma_stop(rdev);
5246 r600_uvd_rbc_stop(rdev);
5247 radeon_uvd_suspend(rdev);
5248 si_irq_suspend(rdev);
5249 radeon_wb_disable(rdev);
5250 si_pcie_gart_disable(rdev);
5254 /* Plan is to move initialization in that function and use
5255 * helper function so that radeon_device_init pretty much
5256 * do nothing more than calling asic specific function. This
5257 * should also allow to remove a bunch of callback function
5260 int si_init(struct radeon_device *rdev)
5262 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5266 if (!radeon_get_bios(rdev)) {
5267 if (ASIC_IS_AVIVO(rdev))
5270 /* Must be an ATOMBIOS */
5271 if (!rdev->is_atom_bios) {
5272 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
5275 r = radeon_atombios_init(rdev);
5279 /* Post card if necessary */
5280 if (!radeon_card_posted(rdev)) {
5282 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5285 DRM_INFO("GPU not posted. posting now...\n");
5286 atom_asic_init(rdev->mode_info.atom_context);
5288 /* init golden registers */
5289 si_init_golden_registers(rdev);
5290 /* Initialize scratch registers */
5291 si_scratch_init(rdev);
5292 /* Initialize surface registers */
5293 radeon_surface_init(rdev);
5294 /* Initialize clocks */
5295 radeon_get_clock_info(rdev->ddev);
5298 r = radeon_fence_driver_init(rdev);
5302 /* initialize memory controller */
5303 r = si_mc_init(rdev);
5306 /* Memory manager */
5307 r = radeon_bo_init(rdev);
5311 r = radeon_irq_kms_init(rdev);
5315 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5316 ring->ring_obj = NULL;
5317 r600_ring_init(rdev, ring, 1024 * 1024);
5319 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5320 ring->ring_obj = NULL;
5321 r600_ring_init(rdev, ring, 1024 * 1024);
5323 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5324 ring->ring_obj = NULL;
5325 r600_ring_init(rdev, ring, 1024 * 1024);
5327 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5328 ring->ring_obj = NULL;
5329 r600_ring_init(rdev, ring, 64 * 1024);
5331 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5332 ring->ring_obj = NULL;
5333 r600_ring_init(rdev, ring, 64 * 1024);
5335 r = radeon_uvd_init(rdev);
5337 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5338 ring->ring_obj = NULL;
5339 r600_ring_init(rdev, ring, 4096);
5342 rdev->ih.ring_obj = NULL;
5343 r600_ih_ring_init(rdev, 64 * 1024);
5345 r = r600_pcie_gart_init(rdev);
5349 rdev->accel_working = true;
5350 r = si_startup(rdev);
5352 dev_err(rdev->dev, "disabling GPU acceleration\n");
5354 cayman_dma_fini(rdev);
5357 radeon_wb_fini(rdev);
5358 radeon_ib_pool_fini(rdev);
5359 radeon_vm_manager_fini(rdev);
5360 radeon_irq_kms_fini(rdev);
5361 si_pcie_gart_fini(rdev);
5362 rdev->accel_working = false;
5365 /* Don't start up if the MC ucode is missing.
5366 * The default clocks and voltages before the MC ucode
5367 * is loaded are not suffient for advanced operations.
5370 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5377 void si_fini(struct radeon_device *rdev)
5380 cayman_dma_fini(rdev);
5383 radeon_wb_fini(rdev);
5384 radeon_vm_manager_fini(rdev);
5385 radeon_ib_pool_fini(rdev);
5386 radeon_irq_kms_fini(rdev);
5387 radeon_uvd_fini(rdev);
5388 si_pcie_gart_fini(rdev);
5389 r600_vram_scratch_fini(rdev);
5390 radeon_gem_fini(rdev);
5391 radeon_fence_driver_fini(rdev);
5392 radeon_bo_fini(rdev);
5393 radeon_atombios_fini(rdev);
5399 * si_get_gpu_clock_counter - return GPU clock counter snapshot
5401 * @rdev: radeon_device pointer
5403 * Fetches a GPU clock counter snapshot (SI).
5404 * Returns the 64 bit clock counter snapshot.
5406 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
5410 mutex_lock(&rdev->gpu_clock_mutex);
5411 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5412 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
5413 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5414 mutex_unlock(&rdev->gpu_clock_mutex);
5418 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
5420 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
5423 /* bypass vclk and dclk with bclk */
5424 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5425 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
5426 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5428 /* put PLL in bypass mode */
5429 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
5431 if (!vclk || !dclk) {
5432 /* keep the Bypass mode, put PLL to sleep */
5433 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5437 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
5438 16384, 0x03FFFFFF, 0, 128, 5,
5439 &fb_div, &vclk_div, &dclk_div);
5443 /* set RESET_ANTI_MUX to 0 */
5444 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
5446 /* set VCO_MODE to 1 */
5447 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
5449 /* toggle UPLL_SLEEP to 1 then back to 0 */
5450 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5451 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
5453 /* deassert UPLL_RESET */
5454 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5458 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
5462 /* assert UPLL_RESET again */
5463 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
5465 /* disable spread spectrum. */
5466 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
5468 /* set feedback divider */
5469 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
5471 /* set ref divider to 0 */
5472 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
5474 if (fb_div < 307200)
5475 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
5477 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
5479 /* set PDIV_A and PDIV_B */
5480 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5481 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
5482 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
5484 /* give the PLL some time to settle */
5487 /* deassert PLL_RESET */
5488 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5492 /* switch from bypass mode to normal mode */
5493 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
5495 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
5499 /* switch VCLK and DCLK selection */
5500 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5501 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
5502 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));