drivers/gpu/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "cikd.h"
  31 #include "atom.h"
  32 #include "cik_blit_shaders.h"
  33 #include "radeon_ucode.h"
  34 #include "clearstate_ci.h"
  35
  36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  66
  67 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  68 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  69 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  70 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  71 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  72 extern void sumo_rlc_fini(struct radeon_device *rdev);
  73 extern int sumo_rlc_init(struct radeon_device *rdev);
  74 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
  75 extern void si_rlc_reset(struct radeon_device *rdev);
  76 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
  77 extern int cik_sdma_resume(struct radeon_device *rdev);
  78 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
  79 extern void cik_sdma_fini(struct radeon_device *rdev);
  80 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
  81 static void cik_rlc_stop(struct radeon_device *rdev);
  82 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
  83 static void cik_program_aspm(struct radeon_device *rdev);
  84 static void cik_init_pg(struct radeon_device *rdev);
  85 static void cik_init_cg(struct radeon_device *rdev);
  86 static void cik_fini_pg(struct radeon_device *rdev);
  87 static void cik_fini_cg(struct radeon_device *rdev);
  88 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
  89                                           bool enable);
  90
  91 /* get temperature in millidegrees */
  92 int ci_get_temp(struct radeon_device *rdev)
  93 {
  94         u32 temp;
  95         int actual_temp = 0;
  96
  97         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
  98                 CTF_TEMP_SHIFT;
  99
 100         if (temp & 0x200)
 101                 actual_temp = 255;
 102         else
 103                 actual_temp = temp & 0x1ff;
 104
 105         actual_temp = actual_temp * 1000;
 106
 107         return actual_temp;
 108 }
 109
 110 /* get temperature in millidegrees */
 111 int kv_get_temp(struct radeon_device *rdev)
 112 {
 113         u32 temp;
 114         int actual_temp = 0;
 115
 116         temp = RREG32_SMC(0xC0300E0C);
 117
 118         if (temp)
 119                 actual_temp = (temp / 8) - 49;
 120         else
 121                 actual_temp = 0;
 122
 123         actual_temp = actual_temp * 1000;
 124
 125         return actual_temp;
 126 }
 127
 128 /*
 129  * Indirect registers accessor
 130  */
 131 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 132 {
 133         unsigned long flags;
 134         u32 r;
 135
 136         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 137         WREG32(PCIE_INDEX, reg);
 138         (void)RREG32(PCIE_INDEX);
 139         r = RREG32(PCIE_DATA);
 140         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 141         return r;
 142 }
 143
 144 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 145 {
 146         unsigned long flags;
 147
 148         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 149         WREG32(PCIE_INDEX, reg);
 150         (void)RREG32(PCIE_INDEX);
 151         WREG32(PCIE_DATA, v);
 152         (void)RREG32(PCIE_DATA);
 153         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 154 }
 155
 156 static const u32 spectre_rlc_save_restore_register_list[] =
 157 {
 158         (0x0e00 << 16) | (0xc12c >> 2),
 159         0x00000000,
 160         (0x0e00 << 16) | (0xc140 >> 2),
 161         0x00000000,
 162         (0x0e00 << 16) | (0xc150 >> 2),
 163         0x00000000,
 164         (0x0e00 << 16) | (0xc15c >> 2),
 165         0x00000000,
 166         (0x0e00 << 16) | (0xc168 >> 2),
 167         0x00000000,
 168         (0x0e00 << 16) | (0xc170 >> 2),
 169         0x00000000,
 170         (0x0e00 << 16) | (0xc178 >> 2),
 171         0x00000000,
 172         (0x0e00 << 16) | (0xc204 >> 2),
 173         0x00000000,
 174         (0x0e00 << 16) | (0xc2b4 >> 2),
 175         0x00000000,
 176         (0x0e00 << 16) | (0xc2b8 >> 2),
 177         0x00000000,
 178         (0x0e00 << 16) | (0xc2bc >> 2),
 179         0x00000000,
 180         (0x0e00 << 16) | (0xc2c0 >> 2),
 181         0x00000000,
 182         (0x0e00 << 16) | (0x8228 >> 2),
 183         0x00000000,
 184         (0x0e00 << 16) | (0x829c >> 2),
 185         0x00000000,
 186         (0x0e00 << 16) | (0x869c >> 2),
 187         0x00000000,
 188         (0x0600 << 16) | (0x98f4 >> 2),
 189         0x00000000,
 190         (0x0e00 << 16) | (0x98f8 >> 2),
 191         0x00000000,
 192         (0x0e00 << 16) | (0x9900 >> 2),
 193         0x00000000,
 194         (0x0e00 << 16) | (0xc260 >> 2),
 195         0x00000000,
 196         (0x0e00 << 16) | (0x90e8 >> 2),
 197         0x00000000,
 198         (0x0e00 << 16) | (0x3c000 >> 2),
 199         0x00000000,
 200         (0x0e00 << 16) | (0x3c00c >> 2),
 201         0x00000000,
 202         (0x0e00 << 16) | (0x8c1c >> 2),
 203         0x00000000,
 204         (0x0e00 << 16) | (0x9700 >> 2),
 205         0x00000000,
 206         (0x0e00 << 16) | (0xcd20 >> 2),
 207         0x00000000,
 208         (0x4e00 << 16) | (0xcd20 >> 2),
 209         0x00000000,
 210         (0x5e00 << 16) | (0xcd20 >> 2),
 211         0x00000000,
 212         (0x6e00 << 16) | (0xcd20 >> 2),
 213         0x00000000,
 214         (0x7e00 << 16) | (0xcd20 >> 2),
 215         0x00000000,
 216         (0x8e00 << 16) | (0xcd20 >> 2),
 217         0x00000000,
 218         (0x9e00 << 16) | (0xcd20 >> 2),
 219         0x00000000,
 220         (0xae00 << 16) | (0xcd20 >> 2),
 221         0x00000000,
 222         (0xbe00 << 16) | (0xcd20 >> 2),
 223         0x00000000,
 224         (0x0e00 << 16) | (0x89bc >> 2),
 225         0x00000000,
 226         (0x0e00 << 16) | (0x8900 >> 2),
 227         0x00000000,
 228         0x3,
 229         (0x0e00 << 16) | (0xc130 >> 2),
 230         0x00000000,
 231         (0x0e00 << 16) | (0xc134 >> 2),
 232         0x00000000,
 233         (0x0e00 << 16) | (0xc1fc >> 2),
 234         0x00000000,
 235         (0x0e00 << 16) | (0xc208 >> 2),
 236         0x00000000,
 237         (0x0e00 << 16) | (0xc264 >> 2),
 238         0x00000000,
 239         (0x0e00 << 16) | (0xc268 >> 2),
 240         0x00000000,
 241         (0x0e00 << 16) | (0xc26c >> 2),
 242         0x00000000,
 243         (0x0e00 << 16) | (0xc270 >> 2),
 244         0x00000000,
 245         (0x0e00 << 16) | (0xc274 >> 2),
 246         0x00000000,
 247         (0x0e00 << 16) | (0xc278 >> 2),
 248         0x00000000,
 249         (0x0e00 << 16) | (0xc27c >> 2),
 250         0x00000000,
 251         (0x0e00 << 16) | (0xc280 >> 2),
 252         0x00000000,
 253         (0x0e00 << 16) | (0xc284 >> 2),
 254         0x00000000,
 255         (0x0e00 << 16) | (0xc288 >> 2),
 256         0x00000000,
 257         (0x0e00 << 16) | (0xc28c >> 2),
 258         0x00000000,
 259         (0x0e00 << 16) | (0xc290 >> 2),
 260         0x00000000,
 261         (0x0e00 << 16) | (0xc294 >> 2),
 262         0x00000000,
 263         (0x0e00 << 16) | (0xc298 >> 2),
 264         0x00000000,
 265         (0x0e00 << 16) | (0xc29c >> 2),
 266         0x00000000,
 267         (0x0e00 << 16) | (0xc2a0 >> 2),
 268         0x00000000,
 269         (0x0e00 << 16) | (0xc2a4 >> 2),
 270         0x00000000,
 271         (0x0e00 << 16) | (0xc2a8 >> 2),
 272         0x00000000,
 273         (0x0e00 << 16) | (0xc2ac  >> 2),
 274         0x00000000,
 275         (0x0e00 << 16) | (0xc2b0 >> 2),
 276         0x00000000,
 277         (0x0e00 << 16) | (0x301d0 >> 2),
 278         0x00000000,
 279         (0x0e00 << 16) | (0x30238 >> 2),
 280         0x00000000,
 281         (0x0e00 << 16) | (0x30250 >> 2),
 282         0x00000000,
 283         (0x0e00 << 16) | (0x30254 >> 2),
 284         0x00000000,
 285         (0x0e00 << 16) | (0x30258 >> 2),
 286         0x00000000,
 287         (0x0e00 << 16) | (0x3025c >> 2),
 288         0x00000000,
 289         (0x4e00 << 16) | (0xc900 >> 2),
 290         0x00000000,
 291         (0x5e00 << 16) | (0xc900 >> 2),
 292         0x00000000,
 293         (0x6e00 << 16) | (0xc900 >> 2),
 294         0x00000000,
 295         (0x7e00 << 16) | (0xc900 >> 2),
 296         0x00000000,
 297         (0x8e00 << 16) | (0xc900 >> 2),
 298         0x00000000,
 299         (0x9e00 << 16) | (0xc900 >> 2),
 300         0x00000000,
 301         (0xae00 << 16) | (0xc900 >> 2),
 302         0x00000000,
 303         (0xbe00 << 16) | (0xc900 >> 2),
 304         0x00000000,
 305         (0x4e00 << 16) | (0xc904 >> 2),
 306         0x00000000,
 307         (0x5e00 << 16) | (0xc904 >> 2),
 308         0x00000000,
 309         (0x6e00 << 16) | (0xc904 >> 2),
 310         0x00000000,
 311         (0x7e00 << 16) | (0xc904 >> 2),
 312         0x00000000,
 313         (0x8e00 << 16) | (0xc904 >> 2),
 314         0x00000000,
 315         (0x9e00 << 16) | (0xc904 >> 2),
 316         0x00000000,
 317         (0xae00 << 16) | (0xc904 >> 2),
 318         0x00000000,
 319         (0xbe00 << 16) | (0xc904 >> 2),
 320         0x00000000,
 321         (0x4e00 << 16) | (0xc908 >> 2),
 322         0x00000000,
 323         (0x5e00 << 16) | (0xc908 >> 2),
 324         0x00000000,
 325         (0x6e00 << 16) | (0xc908 >> 2),
 326         0x00000000,
 327         (0x7e00 << 16) | (0xc908 >> 2),
 328         0x00000000,
 329         (0x8e00 << 16) | (0xc908 >> 2),
 330         0x00000000,
 331         (0x9e00 << 16) | (0xc908 >> 2),
 332         0x00000000,
 333         (0xae00 << 16) | (0xc908 >> 2),
 334         0x00000000,
 335         (0xbe00 << 16) | (0xc908 >> 2),
 336         0x00000000,
 337         (0x4e00 << 16) | (0xc90c >> 2),
 338         0x00000000,
 339         (0x5e00 << 16) | (0xc90c >> 2),
 340         0x00000000,
 341         (0x6e00 << 16) | (0xc90c >> 2),
 342         0x00000000,
 343         (0x7e00 << 16) | (0xc90c >> 2),
 344         0x00000000,
 345         (0x8e00 << 16) | (0xc90c >> 2),
 346         0x00000000,
 347         (0x9e00 << 16) | (0xc90c >> 2),
 348         0x00000000,
 349         (0xae00 << 16) | (0xc90c >> 2),
 350         0x00000000,
 351         (0xbe00 << 16) | (0xc90c >> 2),
 352         0x00000000,
 353         (0x4e00 << 16) | (0xc910 >> 2),
 354         0x00000000,
 355         (0x5e00 << 16) | (0xc910 >> 2),
 356         0x00000000,
 357         (0x6e00 << 16) | (0xc910 >> 2),
 358         0x00000000,
 359         (0x7e00 << 16) | (0xc910 >> 2),
 360         0x00000000,
 361         (0x8e00 << 16) | (0xc910 >> 2),
 362         0x00000000,
 363         (0x9e00 << 16) | (0xc910 >> 2),
 364         0x00000000,
 365         (0xae00 << 16) | (0xc910 >> 2),
 366         0x00000000,
 367         (0xbe00 << 16) | (0xc910 >> 2),
 368         0x00000000,
 369         (0x0e00 << 16) | (0xc99c >> 2),
 370         0x00000000,
 371         (0x0e00 << 16) | (0x9834 >> 2),
 372         0x00000000,
 373         (0x0000 << 16) | (0x30f00 >> 2),
 374         0x00000000,
 375         (0x0001 << 16) | (0x30f00 >> 2),
 376         0x00000000,
 377         (0x0000 << 16) | (0x30f04 >> 2),
 378         0x00000000,
 379         (0x0001 << 16) | (0x30f04 >> 2),
 380         0x00000000,
 381         (0x0000 << 16) | (0x30f08 >> 2),
 382         0x00000000,
 383         (0x0001 << 16) | (0x30f08 >> 2),
 384         0x00000000,
 385         (0x0000 << 16) | (0x30f0c >> 2),
 386         0x00000000,
 387         (0x0001 << 16) | (0x30f0c >> 2),
 388         0x00000000,
 389         (0x0600 << 16) | (0x9b7c >> 2),
 390         0x00000000,
 391         (0x0e00 << 16) | (0x8a14 >> 2),
 392         0x00000000,
 393         (0x0e00 << 16) | (0x8a18 >> 2),
 394         0x00000000,
 395         (0x0600 << 16) | (0x30a00 >> 2),
 396         0x00000000,
 397         (0x0e00 << 16) | (0x8bf0 >> 2),
 398         0x00000000,
 399         (0x0e00 << 16) | (0x8bcc >> 2),
 400         0x00000000,
 401         (0x0e00 << 16) | (0x8b24 >> 2),
 402         0x00000000,
 403         (0x0e00 << 16) | (0x30a04 >> 2),
 404         0x00000000,
 405         (0x0600 << 16) | (0x30a10 >> 2),
 406         0x00000000,
 407         (0x0600 << 16) | (0x30a14 >> 2),
 408         0x00000000,
 409         (0x0600 << 16) | (0x30a18 >> 2),
 410         0x00000000,
 411         (0x0600 << 16) | (0x30a2c >> 2),
 412         0x00000000,
 413         (0x0e00 << 16) | (0xc700 >> 2),
 414         0x00000000,
 415         (0x0e00 << 16) | (0xc704 >> 2),
 416         0x00000000,
 417         (0x0e00 << 16) | (0xc708 >> 2),
 418         0x00000000,
 419         (0x0e00 << 16) | (0xc768 >> 2),
 420         0x00000000,
 421         (0x0400 << 16) | (0xc770 >> 2),
 422         0x00000000,
 423         (0x0400 << 16) | (0xc774 >> 2),
 424         0x00000000,
 425         (0x0400 << 16) | (0xc778 >> 2),
 426         0x00000000,
 427         (0x0400 << 16) | (0xc77c >> 2),
 428         0x00000000,
 429         (0x0400 << 16) | (0xc780 >> 2),
 430         0x00000000,
 431         (0x0400 << 16) | (0xc784 >> 2),
 432         0x00000000,
 433         (0x0400 << 16) | (0xc788 >> 2),
 434         0x00000000,
 435         (0x0400 << 16) | (0xc78c >> 2),
 436         0x00000000,
 437         (0x0400 << 16) | (0xc798 >> 2),
 438         0x00000000,
 439         (0x0400 << 16) | (0xc79c >> 2),
 440         0x00000000,
 441         (0x0400 << 16) | (0xc7a0 >> 2),
 442         0x00000000,
 443         (0x0400 << 16) | (0xc7a4 >> 2),
 444         0x00000000,
 445         (0x0400 << 16) | (0xc7a8 >> 2),
 446         0x00000000,
 447         (0x0400 << 16) | (0xc7ac >> 2),
 448         0x00000000,
 449         (0x0400 << 16) | (0xc7b0 >> 2),
 450         0x00000000,
 451         (0x0400 << 16) | (0xc7b4 >> 2),
 452         0x00000000,
 453         (0x0e00 << 16) | (0x9100 >> 2),
 454         0x00000000,
 455         (0x0e00 << 16) | (0x3c010 >> 2),
 456         0x00000000,
 457         (0x0e00 << 16) | (0x92a8 >> 2),
 458         0x00000000,
 459         (0x0e00 << 16) | (0x92ac >> 2),
 460         0x00000000,
 461         (0x0e00 << 16) | (0x92b4 >> 2),
 462         0x00000000,
 463         (0x0e00 << 16) | (0x92b8 >> 2),
 464         0x00000000,
 465         (0x0e00 << 16) | (0x92bc >> 2),
 466         0x00000000,
 467         (0x0e00 << 16) | (0x92c0 >> 2),
 468         0x00000000,
 469         (0x0e00 << 16) | (0x92c4 >> 2),
 470         0x00000000,
 471         (0x0e00 << 16) | (0x92c8 >> 2),
 472         0x00000000,
 473         (0x0e00 << 16) | (0x92cc >> 2),
 474         0x00000000,
 475         (0x0e00 << 16) | (0x92d0 >> 2),
 476         0x00000000,
 477         (0x0e00 << 16) | (0x8c00 >> 2),
 478         0x00000000,
 479         (0x0e00 << 16) | (0x8c04 >> 2),
 480         0x00000000,
 481         (0x0e00 << 16) | (0x8c20 >> 2),
 482         0x00000000,
 483         (0x0e00 << 16) | (0x8c38 >> 2),
 484         0x00000000,
 485         (0x0e00 << 16) | (0x8c3c >> 2),
 486         0x00000000,
 487         (0x0e00 << 16) | (0xae00 >> 2),
 488         0x00000000,
 489         (0x0e00 << 16) | (0x9604 >> 2),
 490         0x00000000,
 491         (0x0e00 << 16) | (0xac08 >> 2),
 492         0x00000000,
 493         (0x0e00 << 16) | (0xac0c >> 2),
 494         0x00000000,
 495         (0x0e00 << 16) | (0xac10 >> 2),
 496         0x00000000,
 497         (0x0e00 << 16) | (0xac14 >> 2),
 498         0x00000000,
 499         (0x0e00 << 16) | (0xac58 >> 2),
 500         0x00000000,
 501         (0x0e00 << 16) | (0xac68 >> 2),
 502         0x00000000,
 503         (0x0e00 << 16) | (0xac6c >> 2),
 504         0x00000000,
 505         (0x0e00 << 16) | (0xac70 >> 2),
 506         0x00000000,
 507         (0x0e00 << 16) | (0xac74 >> 2),
 508         0x00000000,
 509         (0x0e00 << 16) | (0xac78 >> 2),
 510         0x00000000,
 511         (0x0e00 << 16) | (0xac7c >> 2),
 512         0x00000000,
 513         (0x0e00 << 16) | (0xac80 >> 2),
 514         0x00000000,
 515         (0x0e00 << 16) | (0xac84 >> 2),
 516         0x00000000,
 517         (0x0e00 << 16) | (0xac88 >> 2),
 518         0x00000000,
 519         (0x0e00 << 16) | (0xac8c >> 2),
 520         0x00000000,
 521         (0x0e00 << 16) | (0x970c >> 2),
 522         0x00000000,
 523         (0x0e00 << 16) | (0x9714 >> 2),
 524         0x00000000,
 525         (0x0e00 << 16) | (0x9718 >> 2),
 526         0x00000000,
 527         (0x0e00 << 16) | (0x971c >> 2),
 528         0x00000000,
 529         (0x0e00 << 16) | (0x31068 >> 2),
 530         0x00000000,
 531         (0x4e00 << 16) | (0x31068 >> 2),
 532         0x00000000,
 533         (0x5e00 << 16) | (0x31068 >> 2),
 534         0x00000000,
 535         (0x6e00 << 16) | (0x31068 >> 2),
 536         0x00000000,
 537         (0x7e00 << 16) | (0x31068 >> 2),
 538         0x00000000,
 539         (0x8e00 << 16) | (0x31068 >> 2),
 540         0x00000000,
 541         (0x9e00 << 16) | (0x31068 >> 2),
 542         0x00000000,
 543         (0xae00 << 16) | (0x31068 >> 2),
 544         0x00000000,
 545         (0xbe00 << 16) | (0x31068 >> 2),
 546         0x00000000,
 547         (0x0e00 << 16) | (0xcd10 >> 2),
 548         0x00000000,
 549         (0x0e00 << 16) | (0xcd14 >> 2),
 550         0x00000000,
 551         (0x0e00 << 16) | (0x88b0 >> 2),
 552         0x00000000,
 553         (0x0e00 << 16) | (0x88b4 >> 2),
 554         0x00000000,
 555         (0x0e00 << 16) | (0x88b8 >> 2),
 556         0x00000000,
 557         (0x0e00 << 16) | (0x88bc >> 2),
 558         0x00000000,
 559         (0x0400 << 16) | (0x89c0 >> 2),
 560         0x00000000,
 561         (0x0e00 << 16) | (0x88c4 >> 2),
 562         0x00000000,
 563         (0x0e00 << 16) | (0x88c8 >> 2),
 564         0x00000000,
 565         (0x0e00 << 16) | (0x88d0 >> 2),
 566         0x00000000,
 567         (0x0e00 << 16) | (0x88d4 >> 2),
 568         0x00000000,
 569         (0x0e00 << 16) | (0x88d8 >> 2),
 570         0x00000000,
 571         (0x0e00 << 16) | (0x8980 >> 2),
 572         0x00000000,
 573         (0x0e00 << 16) | (0x30938 >> 2),
 574         0x00000000,
 575         (0x0e00 << 16) | (0x3093c >> 2),
 576         0x00000000,
 577         (0x0e00 << 16) | (0x30940 >> 2),
 578         0x00000000,
 579         (0x0e00 << 16) | (0x89a0 >> 2),
 580         0x00000000,
 581         (0x0e00 << 16) | (0x30900 >> 2),
 582         0x00000000,
 583         (0x0e00 << 16) | (0x30904 >> 2),
 584         0x00000000,
 585         (0x0e00 << 16) | (0x89b4 >> 2),
 586         0x00000000,
 587         (0x0e00 << 16) | (0x3c210 >> 2),
 588         0x00000000,
 589         (0x0e00 << 16) | (0x3c214 >> 2),
 590         0x00000000,
 591         (0x0e00 << 16) | (0x3c218 >> 2),
 592         0x00000000,
 593         (0x0e00 << 16) | (0x8904 >> 2),
 594         0x00000000,
 595         0x5,
 596         (0x0e00 << 16) | (0x8c28 >> 2),
 597         (0x0e00 << 16) | (0x8c2c >> 2),
 598         (0x0e00 << 16) | (0x8c30 >> 2),
 599         (0x0e00 << 16) | (0x8c34 >> 2),
 600         (0x0e00 << 16) | (0x9600 >> 2),
 601 };
 602
 603 static const u32 kalindi_rlc_save_restore_register_list[] =
 604 {
 605         (0x0e00 << 16) | (0xc12c >> 2),
 606         0x00000000,
 607         (0x0e00 << 16) | (0xc140 >> 2),
 608         0x00000000,
 609         (0x0e00 << 16) | (0xc150 >> 2),
 610         0x00000000,
 611         (0x0e00 << 16) | (0xc15c >> 2),
 612         0x00000000,
 613         (0x0e00 << 16) | (0xc168 >> 2),
 614         0x00000000,
 615         (0x0e00 << 16) | (0xc170 >> 2),
 616         0x00000000,
 617         (0x0e00 << 16) | (0xc204 >> 2),
 618         0x00000000,
 619         (0x0e00 << 16) | (0xc2b4 >> 2),
 620         0x00000000,
 621         (0x0e00 << 16) | (0xc2b8 >> 2),
 622         0x00000000,
 623         (0x0e00 << 16) | (0xc2bc >> 2),
 624         0x00000000,
 625         (0x0e00 << 16) | (0xc2c0 >> 2),
 626         0x00000000,
 627         (0x0e00 << 16) | (0x8228 >> 2),
 628         0x00000000,
 629         (0x0e00 << 16) | (0x829c >> 2),
 630         0x00000000,
 631         (0x0e00 << 16) | (0x869c >> 2),
 632         0x00000000,
 633         (0x0600 << 16) | (0x98f4 >> 2),
 634         0x00000000,
 635         (0x0e00 << 16) | (0x98f8 >> 2),
 636         0x00000000,
 637         (0x0e00 << 16) | (0x9900 >> 2),
 638         0x00000000,
 639         (0x0e00 << 16) | (0xc260 >> 2),
 640         0x00000000,
 641         (0x0e00 << 16) | (0x90e8 >> 2),
 642         0x00000000,
 643         (0x0e00 << 16) | (0x3c000 >> 2),
 644         0x00000000,
 645         (0x0e00 << 16) | (0x3c00c >> 2),
 646         0x00000000,
 647         (0x0e00 << 16) | (0x8c1c >> 2),
 648         0x00000000,
 649         (0x0e00 << 16) | (0x9700 >> 2),
 650         0x00000000,
 651         (0x0e00 << 16) | (0xcd20 >> 2),
 652         0x00000000,
 653         (0x4e00 << 16) | (0xcd20 >> 2),
 654         0x00000000,
 655         (0x5e00 << 16) | (0xcd20 >> 2),
 656         0x00000000,
 657         (0x6e00 << 16) | (0xcd20 >> 2),
 658         0x00000000,
 659         (0x7e00 << 16) | (0xcd20 >> 2),
 660         0x00000000,
 661         (0x0e00 << 16) | (0x89bc >> 2),
 662         0x00000000,
 663         (0x0e00 << 16) | (0x8900 >> 2),
 664         0x00000000,
 665         0x3,
 666         (0x0e00 << 16) | (0xc130 >> 2),
 667         0x00000000,
 668         (0x0e00 << 16) | (0xc134 >> 2),
 669         0x00000000,
 670         (0x0e00 << 16) | (0xc1fc >> 2),
 671         0x00000000,
 672         (0x0e00 << 16) | (0xc208 >> 2),
 673         0x00000000,
 674         (0x0e00 << 16) | (0xc264 >> 2),
 675         0x00000000,
 676         (0x0e00 << 16) | (0xc268 >> 2),
 677         0x00000000,
 678         (0x0e00 << 16) | (0xc26c >> 2),
 679         0x00000000,
 680         (0x0e00 << 16) | (0xc270 >> 2),
 681         0x00000000,
 682         (0x0e00 << 16) | (0xc274 >> 2),
 683         0x00000000,
 684         (0x0e00 << 16) | (0xc28c >> 2),
 685         0x00000000,
 686         (0x0e00 << 16) | (0xc290 >> 2),
 687         0x00000000,
 688         (0x0e00 << 16) | (0xc294 >> 2),
 689         0x00000000,
 690         (0x0e00 << 16) | (0xc298 >> 2),
 691         0x00000000,
 692         (0x0e00 << 16) | (0xc2a0 >> 2),
 693         0x00000000,
 694         (0x0e00 << 16) | (0xc2a4 >> 2),
 695         0x00000000,
 696         (0x0e00 << 16) | (0xc2a8 >> 2),
 697         0x00000000,
 698         (0x0e00 << 16) | (0xc2ac >> 2),
 699         0x00000000,
 700         (0x0e00 << 16) | (0x301d0 >> 2),
 701         0x00000000,
 702         (0x0e00 << 16) | (0x30238 >> 2),
 703         0x00000000,
 704         (0x0e00 << 16) | (0x30250 >> 2),
 705         0x00000000,
 706         (0x0e00 << 16) | (0x30254 >> 2),
 707         0x00000000,
 708         (0x0e00 << 16) | (0x30258 >> 2),
 709         0x00000000,
 710         (0x0e00 << 16) | (0x3025c >> 2),
 711         0x00000000,
 712         (0x4e00 << 16) | (0xc900 >> 2),
 713         0x00000000,
 714         (0x5e00 << 16) | (0xc900 >> 2),
 715         0x00000000,
 716         (0x6e00 << 16) | (0xc900 >> 2),
 717         0x00000000,
 718         (0x7e00 << 16) | (0xc900 >> 2),
 719         0x00000000,
 720         (0x4e00 << 16) | (0xc904 >> 2),
 721         0x00000000,
 722         (0x5e00 << 16) | (0xc904 >> 2),
 723         0x00000000,
 724         (0x6e00 << 16) | (0xc904 >> 2),
 725         0x00000000,
 726         (0x7e00 << 16) | (0xc904 >> 2),
 727         0x00000000,
 728         (0x4e00 << 16) | (0xc908 >> 2),
 729         0x00000000,
 730         (0x5e00 << 16) | (0xc908 >> 2),
 731         0x00000000,
 732         (0x6e00 << 16) | (0xc908 >> 2),
 733         0x00000000,
 734         (0x7e00 << 16) | (0xc908 >> 2),
 735         0x00000000,
 736         (0x4e00 << 16) | (0xc90c >> 2),
 737         0x00000000,
 738         (0x5e00 << 16) | (0xc90c >> 2),
 739         0x00000000,
 740         (0x6e00 << 16) | (0xc90c >> 2),
 741         0x00000000,
 742         (0x7e00 << 16) | (0xc90c >> 2),
 743         0x00000000,
 744         (0x4e00 << 16) | (0xc910 >> 2),
 745         0x00000000,
 746         (0x5e00 << 16) | (0xc910 >> 2),
 747         0x00000000,
 748         (0x6e00 << 16) | (0xc910 >> 2),
 749         0x00000000,
 750         (0x7e00 << 16) | (0xc910 >> 2),
 751         0x00000000,
 752         (0x0e00 << 16) | (0xc99c >> 2),
 753         0x00000000,
 754         (0x0e00 << 16) | (0x9834 >> 2),
 755         0x00000000,
 756         (0x0000 << 16) | (0x30f00 >> 2),
 757         0x00000000,
 758         (0x0000 << 16) | (0x30f04 >> 2),
 759         0x00000000,
 760         (0x0000 << 16) | (0x30f08 >> 2),
 761         0x00000000,
 762         (0x0000 << 16) | (0x30f0c >> 2),
 763         0x00000000,
 764         (0x0600 << 16) | (0x9b7c >> 2),
 765         0x00000000,
 766         (0x0e00 << 16) | (0x8a14 >> 2),
 767         0x00000000,
 768         (0x0e00 << 16) | (0x8a18 >> 2),
 769         0x00000000,
 770         (0x0600 << 16) | (0x30a00 >> 2),
 771         0x00000000,
 772         (0x0e00 << 16) | (0x8bf0 >> 2),
 773         0x00000000,
 774         (0x0e00 << 16) | (0x8bcc >> 2),
 775         0x00000000,
 776         (0x0e00 << 16) | (0x8b24 >> 2),
 777         0x00000000,
 778         (0x0e00 << 16) | (0x30a04 >> 2),
 779         0x00000000,
 780         (0x0600 << 16) | (0x30a10 >> 2),
 781         0x00000000,
 782         (0x0600 << 16) | (0x30a14 >> 2),
 783         0x00000000,
 784         (0x0600 << 16) | (0x30a18 >> 2),
 785         0x00000000,
 786         (0x0600 << 16) | (0x30a2c >> 2),
 787         0x00000000,
 788         (0x0e00 << 16) | (0xc700 >> 2),
 789         0x00000000,
 790         (0x0e00 << 16) | (0xc704 >> 2),
 791         0x00000000,
 792         (0x0e00 << 16) | (0xc708 >> 2),
 793         0x00000000,
 794         (0x0e00 << 16) | (0xc768 >> 2),
 795         0x00000000,
 796         (0x0400 << 16) | (0xc770 >> 2),
 797         0x00000000,
 798         (0x0400 << 16) | (0xc774 >> 2),
 799         0x00000000,
 800         (0x0400 << 16) | (0xc798 >> 2),
 801         0x00000000,
 802         (0x0400 << 16) | (0xc79c >> 2),
 803         0x00000000,
 804         (0x0e00 << 16) | (0x9100 >> 2),
 805         0x00000000,
 806         (0x0e00 << 16) | (0x3c010 >> 2),
 807         0x00000000,
 808         (0x0e00 << 16) | (0x8c00 >> 2),
 809         0x00000000,
 810         (0x0e00 << 16) | (0x8c04 >> 2),
 811         0x00000000,
 812         (0x0e00 << 16) | (0x8c20 >> 2),
 813         0x00000000,
 814         (0x0e00 << 16) | (0x8c38 >> 2),
 815         0x00000000,
 816         (0x0e00 << 16) | (0x8c3c >> 2),
 817         0x00000000,
 818         (0x0e00 << 16) | (0xae00 >> 2),
 819         0x00000000,
 820         (0x0e00 << 16) | (0x9604 >> 2),
 821         0x00000000,
 822         (0x0e00 << 16) | (0xac08 >> 2),
 823         0x00000000,
 824         (0x0e00 << 16) | (0xac0c >> 2),
 825         0x00000000,
 826         (0x0e00 << 16) | (0xac10 >> 2),
 827         0x00000000,
 828         (0x0e00 << 16) | (0xac14 >> 2),
 829         0x00000000,
 830         (0x0e00 << 16) | (0xac58 >> 2),
 831         0x00000000,
 832         (0x0e00 << 16) | (0xac68 >> 2),
 833         0x00000000,
 834         (0x0e00 << 16) | (0xac6c >> 2),
 835         0x00000000,
 836         (0x0e00 << 16) | (0xac70 >> 2),
 837         0x00000000,
 838         (0x0e00 << 16) | (0xac74 >> 2),
 839         0x00000000,
 840         (0x0e00 << 16) | (0xac78 >> 2),
 841         0x00000000,
 842         (0x0e00 << 16) | (0xac7c >> 2),
 843         0x00000000,
 844         (0x0e00 << 16) | (0xac80 >> 2),
 845         0x00000000,
 846         (0x0e00 << 16) | (0xac84 >> 2),
 847         0x00000000,
 848         (0x0e00 << 16) | (0xac88 >> 2),
 849         0x00000000,
 850         (0x0e00 << 16) | (0xac8c >> 2),
 851         0x00000000,
 852         (0x0e00 << 16) | (0x970c >> 2),
 853         0x00000000,
 854         (0x0e00 << 16) | (0x9714 >> 2),
 855         0x00000000,
 856         (0x0e00 << 16) | (0x9718 >> 2),
 857         0x00000000,
 858         (0x0e00 << 16) | (0x971c >> 2),
 859         0x00000000,
 860         (0x0e00 << 16) | (0x31068 >> 2),
 861         0x00000000,
 862         (0x4e00 << 16) | (0x31068 >> 2),
 863         0x00000000,
 864         (0x5e00 << 16) | (0x31068 >> 2),
 865         0x00000000,
 866         (0x6e00 << 16) | (0x31068 >> 2),
 867         0x00000000,
 868         (0x7e00 << 16) | (0x31068 >> 2),
 869         0x00000000,
 870         (0x0e00 << 16) | (0xcd10 >> 2),
 871         0x00000000,
 872         (0x0e00 << 16) | (0xcd14 >> 2),
 873         0x00000000,
 874         (0x0e00 << 16) | (0x88b0 >> 2),
 875         0x00000000,
 876         (0x0e00 << 16) | (0x88b4 >> 2),
 877         0x00000000,
 878         (0x0e00 << 16) | (0x88b8 >> 2),
 879         0x00000000,
 880         (0x0e00 << 16) | (0x88bc >> 2),
 881         0x00000000,
 882         (0x0400 << 16) | (0x89c0 >> 2),
 883         0x00000000,
 884         (0x0e00 << 16) | (0x88c4 >> 2),
 885         0x00000000,
 886         (0x0e00 << 16) | (0x88c8 >> 2),
 887         0x00000000,
 888         (0x0e00 << 16) | (0x88d0 >> 2),
 889         0x00000000,
 890         (0x0e00 << 16) | (0x88d4 >> 2),
 891         0x00000000,
 892         (0x0e00 << 16) | (0x88d8 >> 2),
 893         0x00000000,
 894         (0x0e00 << 16) | (0x8980 >> 2),
 895         0x00000000,
 896         (0x0e00 << 16) | (0x30938 >> 2),
 897         0x00000000,
 898         (0x0e00 << 16) | (0x3093c >> 2),
 899         0x00000000,
 900         (0x0e00 << 16) | (0x30940 >> 2),
 901         0x00000000,
 902         (0x0e00 << 16) | (0x89a0 >> 2),
 903         0x00000000,
 904         (0x0e00 << 16) | (0x30900 >> 2),
 905         0x00000000,
 906         (0x0e00 << 16) | (0x30904 >> 2),
 907         0x00000000,
 908         (0x0e00 << 16) | (0x89b4 >> 2),
 909         0x00000000,
 910         (0x0e00 << 16) | (0x3e1fc >> 2),
 911         0x00000000,
 912         (0x0e00 << 16) | (0x3c210 >> 2),
 913         0x00000000,
 914         (0x0e00 << 16) | (0x3c214 >> 2),
 915         0x00000000,
 916         (0x0e00 << 16) | (0x3c218 >> 2),
 917         0x00000000,
 918         (0x0e00 << 16) | (0x8904 >> 2),
 919         0x00000000,
 920         0x5,
 921         (0x0e00 << 16) | (0x8c28 >> 2),
 922         (0x0e00 << 16) | (0x8c2c >> 2),
 923         (0x0e00 << 16) | (0x8c30 >> 2),
 924         (0x0e00 << 16) | (0x8c34 >> 2),
 925         (0x0e00 << 16) | (0x9600 >> 2),
 926 };
 927
 928 static const u32 bonaire_golden_spm_registers[] =
 929 {
 930         0x30800, 0xe0ffffff, 0xe0000000
 931 };
 932
 933 static const u32 bonaire_golden_common_registers[] =
 934 {
 935         0xc770, 0xffffffff, 0x00000800,
 936         0xc774, 0xffffffff, 0x00000800,
 937         0xc798, 0xffffffff, 0x00007fbf,
 938         0xc79c, 0xffffffff, 0x00007faf
 939 };
 940
 941 static const u32 bonaire_golden_registers[] =
 942 {
 943         0x3354, 0x00000333, 0x00000333,
 944         0x3350, 0x000c0fc0, 0x00040200,
 945         0x9a10, 0x00010000, 0x00058208,
 946         0x3c000, 0xffff1fff, 0x00140000,
 947         0x3c200, 0xfdfc0fff, 0x00000100,
 948         0x3c234, 0x40000000, 0x40000200,
 949         0x9830, 0xffffffff, 0x00000000,
 950         0x9834, 0xf00fffff, 0x00000400,
 951         0x9838, 0x0002021c, 0x00020200,
 952         0xc78, 0x00000080, 0x00000000,
 953         0x5bb0, 0x000000f0, 0x00000070,
 954         0x5bc0, 0xf0311fff, 0x80300000,
 955         0x98f8, 0x73773777, 0x12010001,
 956         0x350c, 0x00810000, 0x408af000,
 957         0x7030, 0x31000111, 0x00000011,
 958         0x2f48, 0x73773777, 0x12010001,
 959         0x220c, 0x00007fb6, 0x0021a1b1,
 960         0x2210, 0x00007fb6, 0x002021b1,
 961         0x2180, 0x00007fb6, 0x00002191,
 962         0x2218, 0x00007fb6, 0x002121b1,
 963         0x221c, 0x00007fb6, 0x002021b1,
 964         0x21dc, 0x00007fb6, 0x00002191,
 965         0x21e0, 0x00007fb6, 0x00002191,
 966         0x3628, 0x0000003f, 0x0000000a,
 967         0x362c, 0x0000003f, 0x0000000a,
 968         0x2ae4, 0x00073ffe, 0x000022a2,
 969         0x240c, 0x000007ff, 0x00000000,
 970         0x8a14, 0xf000003f, 0x00000007,
 971         0x8bf0, 0x00002001, 0x00000001,
 972         0x8b24, 0xffffffff, 0x00ffffff,
 973         0x30a04, 0x0000ff0f, 0x00000000,
 974         0x28a4c, 0x07ffffff, 0x06000000,
 975         0x4d8, 0x00000fff, 0x00000100,
 976         0x3e78, 0x00000001, 0x00000002,
 977         0x9100, 0x03000000, 0x0362c688,
 978         0x8c00, 0x000000ff, 0x00000001,
 979         0xe40, 0x00001fff, 0x00001fff,
 980         0x9060, 0x0000007f, 0x00000020,
 981         0x9508, 0x00010000, 0x00010000,
 982         0xac14, 0x000003ff, 0x000000f3,
 983         0xac0c, 0xffffffff, 0x00001032
 984 };
 985
 986 static const u32 bonaire_mgcg_cgcg_init[] =
 987 {
 988         0xc420, 0xffffffff, 0xfffffffc,
 989         0x30800, 0xffffffff, 0xe0000000,
 990         0x3c2a0, 0xffffffff, 0x00000100,
 991         0x3c208, 0xffffffff, 0x00000100,
 992         0x3c2c0, 0xffffffff, 0xc0000100,
 993         0x3c2c8, 0xffffffff, 0xc0000100,
 994         0x3c2c4, 0xffffffff, 0xc0000100,
 995         0x55e4, 0xffffffff, 0x00600100,
 996         0x3c280, 0xffffffff, 0x00000100,
 997         0x3c214, 0xffffffff, 0x06000100,
 998         0x3c220, 0xffffffff, 0x00000100,
 999         0x3c218, 0xffffffff, 0x06000100,
1000         0x3c204, 0xffffffff, 0x00000100,
1001         0x3c2e0, 0xffffffff, 0x00000100,
1002         0x3c224, 0xffffffff, 0x00000100,
1003         0x3c200, 0xffffffff, 0x00000100,
1004         0x3c230, 0xffffffff, 0x00000100,
1005         0x3c234, 0xffffffff, 0x00000100,
1006         0x3c250, 0xffffffff, 0x00000100,
1007         0x3c254, 0xffffffff, 0x00000100,
1008         0x3c258, 0xffffffff, 0x00000100,
1009         0x3c25c, 0xffffffff, 0x00000100,
1010         0x3c260, 0xffffffff, 0x00000100,
1011         0x3c27c, 0xffffffff, 0x00000100,
1012         0x3c278, 0xffffffff, 0x00000100,
1013         0x3c210, 0xffffffff, 0x06000100,
1014         0x3c290, 0xffffffff, 0x00000100,
1015         0x3c274, 0xffffffff, 0x00000100,
1016         0x3c2b4, 0xffffffff, 0x00000100,
1017         0x3c2b0, 0xffffffff, 0x00000100,
1018         0x3c270, 0xffffffff, 0x00000100,
1019         0x30800, 0xffffffff, 0xe0000000,
1020         0x3c020, 0xffffffff, 0x00010000,
1021         0x3c024, 0xffffffff, 0x00030002,
1022         0x3c028, 0xffffffff, 0x00040007,
1023         0x3c02c, 0xffffffff, 0x00060005,
1024         0x3c030, 0xffffffff, 0x00090008,
1025         0x3c034, 0xffffffff, 0x00010000,
1026         0x3c038, 0xffffffff, 0x00030002,
1027         0x3c03c, 0xffffffff, 0x00040007,
1028         0x3c040, 0xffffffff, 0x00060005,
1029         0x3c044, 0xffffffff, 0x00090008,
1030         0x3c048, 0xffffffff, 0x00010000,
1031         0x3c04c, 0xffffffff, 0x00030002,
1032         0x3c050, 0xffffffff, 0x00040007,
1033         0x3c054, 0xffffffff, 0x00060005,
1034         0x3c058, 0xffffffff, 0x00090008,
1035         0x3c05c, 0xffffffff, 0x00010000,
1036         0x3c060, 0xffffffff, 0x00030002,
1037         0x3c064, 0xffffffff, 0x00040007,
1038         0x3c068, 0xffffffff, 0x00060005,
1039         0x3c06c, 0xffffffff, 0x00090008,
1040         0x3c070, 0xffffffff, 0x00010000,
1041         0x3c074, 0xffffffff, 0x00030002,
1042         0x3c078, 0xffffffff, 0x00040007,
1043         0x3c07c, 0xffffffff, 0x00060005,
1044         0x3c080, 0xffffffff, 0x00090008,
1045         0x3c084, 0xffffffff, 0x00010000,
1046         0x3c088, 0xffffffff, 0x00030002,
1047         0x3c08c, 0xffffffff, 0x00040007,
1048         0x3c090, 0xffffffff, 0x00060005,
1049         0x3c094, 0xffffffff, 0x00090008,
1050         0x3c098, 0xffffffff, 0x00010000,
1051         0x3c09c, 0xffffffff, 0x00030002,
1052         0x3c0a0, 0xffffffff, 0x00040007,
1053         0x3c0a4, 0xffffffff, 0x00060005,
1054         0x3c0a8, 0xffffffff, 0x00090008,
1055         0x3c000, 0xffffffff, 0x96e00200,
1056         0x8708, 0xffffffff, 0x00900100,
1057         0xc424, 0xffffffff, 0x0020003f,
1058         0x38, 0xffffffff, 0x0140001c,
1059         0x3c, 0x000f0000, 0x000f0000,
1060         0x220, 0xffffffff, 0xC060000C,
1061         0x224, 0xc0000fff, 0x00000100,
1062         0xf90, 0xffffffff, 0x00000100,
1063         0xf98, 0x00000101, 0x00000000,
1064         0x20a8, 0xffffffff, 0x00000104,
1065         0x55e4, 0xff000fff, 0x00000100,
1066         0x30cc, 0xc0000fff, 0x00000104,
1067         0xc1e4, 0x00000001, 0x00000001,
1068         0xd00c, 0xff000ff0, 0x00000100,
1069         0xd80c, 0xff000ff0, 0x00000100
1070 };
1071
1072 static const u32 spectre_golden_spm_registers[] =
1073 {
1074         0x30800, 0xe0ffffff, 0xe0000000
1075 };
1076
1077 static const u32 spectre_golden_common_registers[] =
1078 {
1079         0xc770, 0xffffffff, 0x00000800,
1080         0xc774, 0xffffffff, 0x00000800,
1081         0xc798, 0xffffffff, 0x00007fbf,
1082         0xc79c, 0xffffffff, 0x00007faf
1083 };
1084
1085 static const u32 spectre_golden_registers[] =
1086 {
1087         0x3c000, 0xffff1fff, 0x96940200,
1088         0x3c00c, 0xffff0001, 0xff000000,
1089         0x3c200, 0xfffc0fff, 0x00000100,
1090         0x6ed8, 0x00010101, 0x00010000,
1091         0x9834, 0xf00fffff, 0x00000400,
1092         0x9838, 0xfffffffc, 0x00020200,
1093         0x5bb0, 0x000000f0, 0x00000070,
1094         0x5bc0, 0xf0311fff, 0x80300000,
1095         0x98f8, 0x73773777, 0x12010001,
1096         0x9b7c, 0x00ff0000, 0x00fc0000,
1097         0x2f48, 0x73773777, 0x12010001,
1098         0x8a14, 0xf000003f, 0x00000007,
1099         0x8b24, 0xffffffff, 0x00ffffff,
1100         0x28350, 0x3f3f3fff, 0x00000082,
1101         0x28354, 0x0000003f, 0x00000000,
1102         0x3e78, 0x00000001, 0x00000002,
1103         0x913c, 0xffff03df, 0x00000004,
1104         0xc768, 0x00000008, 0x00000008,
1105         0x8c00, 0x000008ff, 0x00000800,
1106         0x9508, 0x00010000, 0x00010000,
1107         0xac0c, 0xffffffff, 0x54763210,
1108         0x214f8, 0x01ff01ff, 0x00000002,
1109         0x21498, 0x007ff800, 0x00200000,
1110         0x2015c, 0xffffffff, 0x00000f40,
1111         0x30934, 0xffffffff, 0x00000001
1112 };
1113
1114 static const u32 spectre_mgcg_cgcg_init[] =
1115 {
1116         0xc420, 0xffffffff, 0xfffffffc,
1117         0x30800, 0xffffffff, 0xe0000000,
1118         0x3c2a0, 0xffffffff, 0x00000100,
1119         0x3c208, 0xffffffff, 0x00000100,
1120         0x3c2c0, 0xffffffff, 0x00000100,
1121         0x3c2c8, 0xffffffff, 0x00000100,
1122         0x3c2c4, 0xffffffff, 0x00000100,
1123         0x55e4, 0xffffffff, 0x00600100,
1124         0x3c280, 0xffffffff, 0x00000100,
1125         0x3c214, 0xffffffff, 0x06000100,
1126         0x3c220, 0xffffffff, 0x00000100,
1127         0x3c218, 0xffffffff, 0x06000100,
1128         0x3c204, 0xffffffff, 0x00000100,
1129         0x3c2e0, 0xffffffff, 0x00000100,
1130         0x3c224, 0xffffffff, 0x00000100,
1131         0x3c200, 0xffffffff, 0x00000100,
1132         0x3c230, 0xffffffff, 0x00000100,
1133         0x3c234, 0xffffffff, 0x00000100,
1134         0x3c250, 0xffffffff, 0x00000100,
1135         0x3c254, 0xffffffff, 0x00000100,
1136         0x3c258, 0xffffffff, 0x00000100,
1137         0x3c25c, 0xffffffff, 0x00000100,
1138         0x3c260, 0xffffffff, 0x00000100,
1139         0x3c27c, 0xffffffff, 0x00000100,
1140         0x3c278, 0xffffffff, 0x00000100,
1141         0x3c210, 0xffffffff, 0x06000100,
1142         0x3c290, 0xffffffff, 0x00000100,
1143         0x3c274, 0xffffffff, 0x00000100,
1144         0x3c2b4, 0xffffffff, 0x00000100,
1145         0x3c2b0, 0xffffffff, 0x00000100,
1146         0x3c270, 0xffffffff, 0x00000100,
1147         0x30800, 0xffffffff, 0xe0000000,
1148         0x3c020, 0xffffffff, 0x00010000,
1149         0x3c024, 0xffffffff, 0x00030002,
1150         0x3c028, 0xffffffff, 0x00040007,
1151         0x3c02c, 0xffffffff, 0x00060005,
1152         0x3c030, 0xffffffff, 0x00090008,
1153         0x3c034, 0xffffffff, 0x00010000,
1154         0x3c038, 0xffffffff, 0x00030002,
1155         0x3c03c, 0xffffffff, 0x00040007,
1156         0x3c040, 0xffffffff, 0x00060005,
1157         0x3c044, 0xffffffff, 0x00090008,
1158         0x3c048, 0xffffffff, 0x00010000,
1159         0x3c04c, 0xffffffff, 0x00030002,
1160         0x3c050, 0xffffffff, 0x00040007,
1161         0x3c054, 0xffffffff, 0x00060005,
1162         0x3c058, 0xffffffff, 0x00090008,
1163         0x3c05c, 0xffffffff, 0x00010000,
1164         0x3c060, 0xffffffff, 0x00030002,
1165         0x3c064, 0xffffffff, 0x00040007,
1166         0x3c068, 0xffffffff, 0x00060005,
1167         0x3c06c, 0xffffffff, 0x00090008,
1168         0x3c070, 0xffffffff, 0x00010000,
1169         0x3c074, 0xffffffff, 0x00030002,
1170         0x3c078, 0xffffffff, 0x00040007,
1171         0x3c07c, 0xffffffff, 0x00060005,
1172         0x3c080, 0xffffffff, 0x00090008,
1173         0x3c084, 0xffffffff, 0x00010000,
1174         0x3c088, 0xffffffff, 0x00030002,
1175         0x3c08c, 0xffffffff, 0x00040007,
1176         0x3c090, 0xffffffff, 0x00060005,
1177         0x3c094, 0xffffffff, 0x00090008,
1178         0x3c098, 0xffffffff, 0x00010000,
1179         0x3c09c, 0xffffffff, 0x00030002,
1180         0x3c0a0, 0xffffffff, 0x00040007,
1181         0x3c0a4, 0xffffffff, 0x00060005,
1182         0x3c0a8, 0xffffffff, 0x00090008,
1183         0x3c0ac, 0xffffffff, 0x00010000,
1184         0x3c0b0, 0xffffffff, 0x00030002,
1185         0x3c0b4, 0xffffffff, 0x00040007,
1186         0x3c0b8, 0xffffffff, 0x00060005,
1187         0x3c0bc, 0xffffffff, 0x00090008,
1188         0x3c000, 0xffffffff, 0x96e00200,
1189         0x8708, 0xffffffff, 0x00900100,
1190         0xc424, 0xffffffff, 0x0020003f,
1191         0x38, 0xffffffff, 0x0140001c,
1192         0x3c, 0x000f0000, 0x000f0000,
1193         0x220, 0xffffffff, 0xC060000C,
1194         0x224, 0xc0000fff, 0x00000100,
1195         0xf90, 0xffffffff, 0x00000100,
1196         0xf98, 0x00000101, 0x00000000,
1197         0x20a8, 0xffffffff, 0x00000104,
1198         0x55e4, 0xff000fff, 0x00000100,
1199         0x30cc, 0xc0000fff, 0x00000104,
1200         0xc1e4, 0x00000001, 0x00000001,
1201         0xd00c, 0xff000ff0, 0x00000100,
1202         0xd80c, 0xff000ff0, 0x00000100
1203 };
1204
1205 static const u32 kalindi_golden_spm_registers[] =
1206 {
1207         0x30800, 0xe0ffffff, 0xe0000000
1208 };
1209
1210 static const u32 kalindi_golden_common_registers[] =
1211 {
1212         0xc770, 0xffffffff, 0x00000800,
1213         0xc774, 0xffffffff, 0x00000800,
1214         0xc798, 0xffffffff, 0x00007fbf,
1215         0xc79c, 0xffffffff, 0x00007faf
1216 };
1217
1218 static const u32 kalindi_golden_registers[] =
1219 {
1220         0x3c000, 0xffffdfff, 0x6e944040,
1221         0x55e4, 0xff607fff, 0xfc000100,
1222         0x3c220, 0xff000fff, 0x00000100,
1223         0x3c224, 0xff000fff, 0x00000100,
1224         0x3c200, 0xfffc0fff, 0x00000100,
1225         0x6ed8, 0x00010101, 0x00010000,
1226         0x9830, 0xffffffff, 0x00000000,
1227         0x9834, 0xf00fffff, 0x00000400,
1228         0x5bb0, 0x000000f0, 0x00000070,
1229         0x5bc0, 0xf0311fff, 0x80300000,
1230         0x98f8, 0x73773777, 0x12010001,
1231         0x98fc, 0xffffffff, 0x00000010,
1232         0x9b7c, 0x00ff0000, 0x00fc0000,
1233         0x8030, 0x00001f0f, 0x0000100a,
1234         0x2f48, 0x73773777, 0x12010001,
1235         0x2408, 0x000fffff, 0x000c007f,
1236         0x8a14, 0xf000003f, 0x00000007,
1237         0x8b24, 0x3fff3fff, 0x00ffcfff,
1238         0x30a04, 0x0000ff0f, 0x00000000,
1239         0x28a4c, 0x07ffffff, 0x06000000,
1240         0x4d8, 0x00000fff, 0x00000100,
1241         0x3e78, 0x00000001, 0x00000002,
1242         0xc768, 0x00000008, 0x00000008,
1243         0x8c00, 0x000000ff, 0x00000003,
1244         0x214f8, 0x01ff01ff, 0x00000002,
1245         0x21498, 0x007ff800, 0x00200000,
1246         0x2015c, 0xffffffff, 0x00000f40,
1247         0x88c4, 0x001f3ae3, 0x00000082,
1248         0x88d4, 0x0000001f, 0x00000010,
1249         0x30934, 0xffffffff, 0x00000000
1250 };
1251
1252 static const u32 kalindi_mgcg_cgcg_init[] =
1253 {
1254         0xc420, 0xffffffff, 0xfffffffc,
1255         0x30800, 0xffffffff, 0xe0000000,
1256         0x3c2a0, 0xffffffff, 0x00000100,
1257         0x3c208, 0xffffffff, 0x00000100,
1258         0x3c2c0, 0xffffffff, 0x00000100,
1259         0x3c2c8, 0xffffffff, 0x00000100,
1260         0x3c2c4, 0xffffffff, 0x00000100,
1261         0x55e4, 0xffffffff, 0x00600100,
1262         0x3c280, 0xffffffff, 0x00000100,
1263         0x3c214, 0xffffffff, 0x06000100,
1264         0x3c220, 0xffffffff, 0x00000100,
1265         0x3c218, 0xffffffff, 0x06000100,
1266         0x3c204, 0xffffffff, 0x00000100,
1267         0x3c2e0, 0xffffffff, 0x00000100,
1268         0x3c224, 0xffffffff, 0x00000100,
1269         0x3c200, 0xffffffff, 0x00000100,
1270         0x3c230, 0xffffffff, 0x00000100,
1271         0x3c234, 0xffffffff, 0x00000100,
1272         0x3c250, 0xffffffff, 0x00000100,
1273         0x3c254, 0xffffffff, 0x00000100,
1274         0x3c258, 0xffffffff, 0x00000100,
1275         0x3c25c, 0xffffffff, 0x00000100,
1276         0x3c260, 0xffffffff, 0x00000100,
1277         0x3c27c, 0xffffffff, 0x00000100,
1278         0x3c278, 0xffffffff, 0x00000100,
1279         0x3c210, 0xffffffff, 0x06000100,
1280         0x3c290, 0xffffffff, 0x00000100,
1281         0x3c274, 0xffffffff, 0x00000100,
1282         0x3c2b4, 0xffffffff, 0x00000100,
1283         0x3c2b0, 0xffffffff, 0x00000100,
1284         0x3c270, 0xffffffff, 0x00000100,
1285         0x30800, 0xffffffff, 0xe0000000,
1286         0x3c020, 0xffffffff, 0x00010000,
1287         0x3c024, 0xffffffff, 0x00030002,
1288         0x3c028, 0xffffffff, 0x00040007,
1289         0x3c02c, 0xffffffff, 0x00060005,
1290         0x3c030, 0xffffffff, 0x00090008,
1291         0x3c034, 0xffffffff, 0x00010000,
1292         0x3c038, 0xffffffff, 0x00030002,
1293         0x3c03c, 0xffffffff, 0x00040007,
1294         0x3c040, 0xffffffff, 0x00060005,
1295         0x3c044, 0xffffffff, 0x00090008,
1296         0x3c000, 0xffffffff, 0x96e00200,
1297         0x8708, 0xffffffff, 0x00900100,
1298         0xc424, 0xffffffff, 0x0020003f,
1299         0x38, 0xffffffff, 0x0140001c,
1300         0x3c, 0x000f0000, 0x000f0000,
1301         0x220, 0xffffffff, 0xC060000C,
1302         0x224, 0xc0000fff, 0x00000100,
1303         0x20a8, 0xffffffff, 0x00000104,
1304         0x55e4, 0xff000fff, 0x00000100,
1305         0x30cc, 0xc0000fff, 0x00000104,
1306         0xc1e4, 0x00000001, 0x00000001,
1307         0xd00c, 0xff000ff0, 0x00000100,
1308         0xd80c, 0xff000ff0, 0x00000100
1309 };
1310
1311 static const u32 hawaii_golden_spm_registers[] =
1312 {
1313         0x30800, 0xe0ffffff, 0xe0000000
1314 };
1315
1316 static const u32 hawaii_golden_common_registers[] =
1317 {
1318         0x30800, 0xffffffff, 0xe0000000,
1319         0x28350, 0xffffffff, 0x3a00161a,
1320         0x28354, 0xffffffff, 0x0000002e,
1321         0x9a10, 0xffffffff, 0x00018208,
1322         0x98f8, 0xffffffff, 0x12011003
1323 };
1324
1325 static const u32 hawaii_golden_registers[] =
1326 {
1327         0x3354, 0x00000333, 0x00000333,
1328         0x9a10, 0x00010000, 0x00058208,
1329         0x9830, 0xffffffff, 0x00000000,
1330         0x9834, 0xf00fffff, 0x00000400,
1331         0x9838, 0x0002021c, 0x00020200,
1332         0xc78, 0x00000080, 0x00000000,
1333         0x5bb0, 0x000000f0, 0x00000070,
1334         0x5bc0, 0xf0311fff, 0x80300000,
1335         0x350c, 0x00810000, 0x408af000,
1336         0x7030, 0x31000111, 0x00000011,
1337         0x2f48, 0x73773777, 0x12010001,
1338         0x2120, 0x0000007f, 0x0000001b,
1339         0x21dc, 0x00007fb6, 0x00002191,
1340         0x3628, 0x0000003f, 0x0000000a,
1341         0x362c, 0x0000003f, 0x0000000a,
1342         0x2ae4, 0x00073ffe, 0x000022a2,
1343         0x240c, 0x000007ff, 0x00000000,
1344         0x8bf0, 0x00002001, 0x00000001,
1345         0x8b24, 0xffffffff, 0x00ffffff,
1346         0x30a04, 0x0000ff0f, 0x00000000,
1347         0x28a4c, 0x07ffffff, 0x06000000,
1348         0x3e78, 0x00000001, 0x00000002,
1349         0xc768, 0x00000008, 0x00000008,
1350         0xc770, 0x00000f00, 0x00000800,
1351         0xc774, 0x00000f00, 0x00000800,
1352         0xc798, 0x00ffffff, 0x00ff7fbf,
1353         0xc79c, 0x00ffffff, 0x00ff7faf,
1354         0x8c00, 0x000000ff, 0x00000800,
1355         0xe40, 0x00001fff, 0x00001fff,
1356         0x9060, 0x0000007f, 0x00000020,
1357         0x9508, 0x00010000, 0x00010000,
1358         0xae00, 0x00100000, 0x000ff07c,
1359         0xac14, 0x000003ff, 0x0000000f,
1360         0xac10, 0xffffffff, 0x7564fdec,
1361         0xac0c, 0xffffffff, 0x3120b9a8,
1362         0xac08, 0x20000000, 0x0f9c0000
1363 };
1364
1365 static const u32 hawaii_mgcg_cgcg_init[] =
1366 {
1367         0xc420, 0xffffffff, 0xfffffffd,
1368         0x30800, 0xffffffff, 0xe0000000,
1369         0x3c2a0, 0xffffffff, 0x00000100,
1370         0x3c208, 0xffffffff, 0x00000100,
1371         0x3c2c0, 0xffffffff, 0x00000100,
1372         0x3c2c8, 0xffffffff, 0x00000100,
1373         0x3c2c4, 0xffffffff, 0x00000100,
1374         0x55e4, 0xffffffff, 0x00200100,
1375         0x3c280, 0xffffffff, 0x00000100,
1376         0x3c214, 0xffffffff, 0x06000100,
1377         0x3c220, 0xffffffff, 0x00000100,
1378         0x3c218, 0xffffffff, 0x06000100,
1379         0x3c204, 0xffffffff, 0x00000100,
1380         0x3c2e0, 0xffffffff, 0x00000100,
1381         0x3c224, 0xffffffff, 0x00000100,
1382         0x3c200, 0xffffffff, 0x00000100,
1383         0x3c230, 0xffffffff, 0x00000100,
1384         0x3c234, 0xffffffff, 0x00000100,
1385         0x3c250, 0xffffffff, 0x00000100,
1386         0x3c254, 0xffffffff, 0x00000100,
1387         0x3c258, 0xffffffff, 0x00000100,
1388         0x3c25c, 0xffffffff, 0x00000100,
1389         0x3c260, 0xffffffff, 0x00000100,
1390         0x3c27c, 0xffffffff, 0x00000100,
1391         0x3c278, 0xffffffff, 0x00000100,
1392         0x3c210, 0xffffffff, 0x06000100,
1393         0x3c290, 0xffffffff, 0x00000100,
1394         0x3c274, 0xffffffff, 0x00000100,
1395         0x3c2b4, 0xffffffff, 0x00000100,
1396         0x3c2b0, 0xffffffff, 0x00000100,
1397         0x3c270, 0xffffffff, 0x00000100,
1398         0x30800, 0xffffffff, 0xe0000000,
1399         0x3c020, 0xffffffff, 0x00010000,
1400         0x3c024, 0xffffffff, 0x00030002,
1401         0x3c028, 0xffffffff, 0x00040007,
1402         0x3c02c, 0xffffffff, 0x00060005,
1403         0x3c030, 0xffffffff, 0x00090008,
1404         0x3c034, 0xffffffff, 0x00010000,
1405         0x3c038, 0xffffffff, 0x00030002,
1406         0x3c03c, 0xffffffff, 0x00040007,
1407         0x3c040, 0xffffffff, 0x00060005,
1408         0x3c044, 0xffffffff, 0x00090008,
1409         0x3c048, 0xffffffff, 0x00010000,
1410         0x3c04c, 0xffffffff, 0x00030002,
1411         0x3c050, 0xffffffff, 0x00040007,
1412         0x3c054, 0xffffffff, 0x00060005,
1413         0x3c058, 0xffffffff, 0x00090008,
1414         0x3c05c, 0xffffffff, 0x00010000,
1415         0x3c060, 0xffffffff, 0x00030002,
1416         0x3c064, 0xffffffff, 0x00040007,
1417         0x3c068, 0xffffffff, 0x00060005,
1418         0x3c06c, 0xffffffff, 0x00090008,
1419         0x3c070, 0xffffffff, 0x00010000,
1420         0x3c074, 0xffffffff, 0x00030002,
1421         0x3c078, 0xffffffff, 0x00040007,
1422         0x3c07c, 0xffffffff, 0x00060005,
1423         0x3c080, 0xffffffff, 0x00090008,
1424         0x3c084, 0xffffffff, 0x00010000,
1425         0x3c088, 0xffffffff, 0x00030002,
1426         0x3c08c, 0xffffffff, 0x00040007,
1427         0x3c090, 0xffffffff, 0x00060005,
1428         0x3c094, 0xffffffff, 0x00090008,
1429         0x3c098, 0xffffffff, 0x00010000,
1430         0x3c09c, 0xffffffff, 0x00030002,
1431         0x3c0a0, 0xffffffff, 0x00040007,
1432         0x3c0a4, 0xffffffff, 0x00060005,
1433         0x3c0a8, 0xffffffff, 0x00090008,
1434         0x3c0ac, 0xffffffff, 0x00010000,
1435         0x3c0b0, 0xffffffff, 0x00030002,
1436         0x3c0b4, 0xffffffff, 0x00040007,
1437         0x3c0b8, 0xffffffff, 0x00060005,
1438         0x3c0bc, 0xffffffff, 0x00090008,
1439         0x3c0c0, 0xffffffff, 0x00010000,
1440         0x3c0c4, 0xffffffff, 0x00030002,
1441         0x3c0c8, 0xffffffff, 0x00040007,
1442         0x3c0cc, 0xffffffff, 0x00060005,
1443         0x3c0d0, 0xffffffff, 0x00090008,
1444         0x3c0d4, 0xffffffff, 0x00010000,
1445         0x3c0d8, 0xffffffff, 0x00030002,
1446         0x3c0dc, 0xffffffff, 0x00040007,
1447         0x3c0e0, 0xffffffff, 0x00060005,
1448         0x3c0e4, 0xffffffff, 0x00090008,
1449         0x3c0e8, 0xffffffff, 0x00010000,
1450         0x3c0ec, 0xffffffff, 0x00030002,
1451         0x3c0f0, 0xffffffff, 0x00040007,
1452         0x3c0f4, 0xffffffff, 0x00060005,
1453         0x3c0f8, 0xffffffff, 0x00090008,
1454         0xc318, 0xffffffff, 0x00020200,
1455         0x3350, 0xffffffff, 0x00000200,
1456         0x15c0, 0xffffffff, 0x00000400,
1457         0x55e8, 0xffffffff, 0x00000000,
1458         0x2f50, 0xffffffff, 0x00000902,
1459         0x3c000, 0xffffffff, 0x96940200,
1460         0x8708, 0xffffffff, 0x00900100,
1461         0xc424, 0xffffffff, 0x0020003f,
1462         0x38, 0xffffffff, 0x0140001c,
1463         0x3c, 0x000f0000, 0x000f0000,
1464         0x220, 0xffffffff, 0xc060000c,
1465         0x224, 0xc0000fff, 0x00000100,
1466         0xf90, 0xffffffff, 0x00000100,
1467         0xf98, 0x00000101, 0x00000000,
1468         0x20a8, 0xffffffff, 0x00000104,
1469         0x55e4, 0xff000fff, 0x00000100,
1470         0x30cc, 0xc0000fff, 0x00000104,
1471         0xc1e4, 0x00000001, 0x00000001,
1472         0xd00c, 0xff000ff0, 0x00000100,
1473         0xd80c, 0xff000ff0, 0x00000100
1474 };
1475
1476 static void cik_init_golden_registers(struct radeon_device *rdev)
1477 {
1478         switch (rdev->family) {
1479         case CHIP_BONAIRE:
1480                 radeon_program_register_sequence(rdev,
1481                                                  bonaire_mgcg_cgcg_init,
1482                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1483                 radeon_program_register_sequence(rdev,
1484                                                  bonaire_golden_registers,
1485                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1486                 radeon_program_register_sequence(rdev,
1487                                                  bonaire_golden_common_registers,
1488                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1489                 radeon_program_register_sequence(rdev,
1490                                                  bonaire_golden_spm_registers,
1491                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1492                 break;
1493         case CHIP_KABINI:
1494                 radeon_program_register_sequence(rdev,
1495                                                  kalindi_mgcg_cgcg_init,
1496                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1497                 radeon_program_register_sequence(rdev,
1498                                                  kalindi_golden_registers,
1499                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1500                 radeon_program_register_sequence(rdev,
1501                                                  kalindi_golden_common_registers,
1502                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1503                 radeon_program_register_sequence(rdev,
1504                                                  kalindi_golden_spm_registers,
1505                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1506                 break;
1507         case CHIP_KAVERI:
1508                 radeon_program_register_sequence(rdev,
1509                                                  spectre_mgcg_cgcg_init,
1510                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1511                 radeon_program_register_sequence(rdev,
1512                                                  spectre_golden_registers,
1513                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1514                 radeon_program_register_sequence(rdev,
1515                                                  spectre_golden_common_registers,
1516                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1517                 radeon_program_register_sequence(rdev,
1518                                                  spectre_golden_spm_registers,
1519                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1520                 break;
1521         case CHIP_HAWAII:
1522                 radeon_program_register_sequence(rdev,
1523                                                  hawaii_mgcg_cgcg_init,
1524                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1525                 radeon_program_register_sequence(rdev,
1526                                                  hawaii_golden_registers,
1527                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
1528                 radeon_program_register_sequence(rdev,
1529                                                  hawaii_golden_common_registers,
1530                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1531                 radeon_program_register_sequence(rdev,
1532                                                  hawaii_golden_spm_registers,
1533                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1534                 break;
1535         default:
1536                 break;
1537         }
1538 }
1539
1540 /**
1541  * cik_get_xclk - get the xclk
1542  *
1543  * @rdev: radeon_device pointer
1544  *
1545  * Returns the reference clock used by the gfx engine
1546  * (CIK).
1547  */
1548 u32 cik_get_xclk(struct radeon_device *rdev)
1549 {
1550         u32 reference_clock = rdev->clock.spll.reference_freq;
1551
1552         if (rdev->flags & RADEON_IS_IGP) {
1553                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1554                         return reference_clock / 2;
1555         } else {
1556                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1557                         return reference_clock / 4;
1558         }
1559         return reference_clock;
1560 }
1561
1562 /**
1563  * cik_mm_rdoorbell - read a doorbell dword
1564  *
1565  * @rdev: radeon_device pointer
1566  * @index: doorbell index
1567  *
1568  * Returns the value in the doorbell aperture at the
1569  * requested doorbell index (CIK).
1570  */
1571 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1572 {
1573         if (index < rdev->doorbell.num_doorbells) {
1574                 return readl(rdev->doorbell.ptr + index);
1575         } else {
1576                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1577                 return 0;
1578         }
1579 }
1580
1581 /**
1582  * cik_mm_wdoorbell - write a doorbell dword
1583  *
1584  * @rdev: radeon_device pointer
1585  * @index: doorbell index
1586  * @v: value to write
1587  *
1588  * Writes @v to the doorbell aperture at the
1589  * requested doorbell index (CIK).
1590  */
1591 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1592 {
1593         if (index < rdev->doorbell.num_doorbells) {
1594                 writel(v, rdev->doorbell.ptr + index);
1595         } else {
1596                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1597         }
1598 }
1599
1600 #define BONAIRE_IO_MC_REGS_SIZE 36
1601
1602 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1603 {
1604         {0x00000070, 0x04400000},
1605         {0x00000071, 0x80c01803},
1606         {0x00000072, 0x00004004},
1607         {0x00000073, 0x00000100},
1608         {0x00000074, 0x00ff0000},
1609         {0x00000075, 0x34000000},
1610         {0x00000076, 0x08000014},
1611         {0x00000077, 0x00cc08ec},
1612         {0x00000078, 0x00000400},
1613         {0x00000079, 0x00000000},
1614         {0x0000007a, 0x04090000},
1615         {0x0000007c, 0x00000000},
1616         {0x0000007e, 0x4408a8e8},
1617         {0x0000007f, 0x00000304},
1618         {0x00000080, 0x00000000},
1619         {0x00000082, 0x00000001},
1620         {0x00000083, 0x00000002},
1621         {0x00000084, 0xf3e4f400},
1622         {0x00000085, 0x052024e3},
1623         {0x00000087, 0x00000000},
1624         {0x00000088, 0x01000000},
1625         {0x0000008a, 0x1c0a0000},
1626         {0x0000008b, 0xff010000},
1627         {0x0000008d, 0xffffefff},
1628         {0x0000008e, 0xfff3efff},
1629         {0x0000008f, 0xfff3efbf},
1630         {0x00000092, 0xf7ffffff},
1631         {0x00000093, 0xffffff7f},
1632         {0x00000095, 0x00101101},
1633         {0x00000096, 0x00000fff},
1634         {0x00000097, 0x00116fff},
1635         {0x00000098, 0x60010000},
1636         {0x00000099, 0x10010000},
1637         {0x0000009a, 0x00006000},
1638         {0x0000009b, 0x00001000},
1639         {0x0000009f, 0x00b48000}
1640 };
1641
1642 #define HAWAII_IO_MC_REGS_SIZE 22
1643
1644 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1645 {
1646         {0x0000007d, 0x40000000},
1647         {0x0000007e, 0x40180304},
1648         {0x0000007f, 0x0000ff00},
1649         {0x00000081, 0x00000000},
1650         {0x00000083, 0x00000800},
1651         {0x00000086, 0x00000000},
1652         {0x00000087, 0x00000100},
1653         {0x00000088, 0x00020100},
1654         {0x00000089, 0x00000000},
1655         {0x0000008b, 0x00040000},
1656         {0x0000008c, 0x00000100},
1657         {0x0000008e, 0xff010000},
1658         {0x00000090, 0xffffefff},
1659         {0x00000091, 0xfff3efff},
1660         {0x00000092, 0xfff3efbf},
1661         {0x00000093, 0xf7ffffff},
1662         {0x00000094, 0xffffff7f},
1663         {0x00000095, 0x00000fff},
1664         {0x00000096, 0x00116fff},
1665         {0x00000097, 0x60010000},
1666         {0x00000098, 0x10010000},
1667         {0x0000009f, 0x00c79000}
1668 };
1669
1670
1671 /**
1672  * cik_srbm_select - select specific register instances
1673  *
1674  * @rdev: radeon_device pointer
1675  * @me: selected ME (micro engine)
1676  * @pipe: pipe
1677  * @queue: queue
1678  * @vmid: VMID
1679  *
1680  * Switches the currently active registers instances.  Some
1681  * registers are instanced per VMID, others are instanced per
1682  * me/pipe/queue combination.
1683  */
1684 static void cik_srbm_select(struct radeon_device *rdev,
1685                             u32 me, u32 pipe, u32 queue, u32 vmid)
1686 {
1687         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1688                              MEID(me & 0x3) |
1689                              VMID(vmid & 0xf) |
1690                              QUEUEID(queue & 0x7));
1691         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1692 }
1693
1694 /* ucode loading */
1695 /**
1696  * ci_mc_load_microcode - load MC ucode into the hw
1697  *
1698  * @rdev: radeon_device pointer
1699  *
1700  * Load the GDDR MC ucode into the hw (CIK).
1701  * Returns 0 on success, error on failure.
1702  */
1703 int ci_mc_load_microcode(struct radeon_device *rdev)
1704 {
1705         const __be32 *fw_data;
1706         u32 running, blackout = 0;
1707         u32 *io_mc_regs;
1708         int i, regs_size, ucode_size;
1709
1710         if (!rdev->mc_fw)
1711                 return -EINVAL;
1712
1713         ucode_size = rdev->mc_fw->size / 4;
1714
1715         switch (rdev->family) {
1716         case CHIP_BONAIRE:
1717                 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1718                 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1719                 break;
1720         case CHIP_HAWAII:
1721                 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1722                 regs_size = HAWAII_IO_MC_REGS_SIZE;
1723                 break;
1724         default:
1725                 return -EINVAL;
1726         }
1727
1728         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1729
1730         if (running == 0) {
1731                 if (running) {
1732                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1733                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1734                 }
1735
1736                 /* reset the engine and set to writable */
1737                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1738                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1739
1740                 /* load mc io regs */
1741                 for (i = 0; i < regs_size; i++) {
1742                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1743                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1744                 }
1745                 /* load the MC ucode */
1746                 fw_data = (const __be32 *)rdev->mc_fw->data;
1747                 for (i = 0; i < ucode_size; i++)
1748                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1749
1750                 /* put the engine back into the active state */
1751                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1752                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1753                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1754
1755                 /* wait for training to complete */
1756                 for (i = 0; i < rdev->usec_timeout; i++) {
1757                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1758                                 break;
1759                         udelay(1);
1760                 }
1761                 for (i = 0; i < rdev->usec_timeout; i++) {
1762                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1763                                 break;
1764                         udelay(1);
1765                 }
1766
1767                 if (running)
1768                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1769         }
1770
1771         return 0;
1772 }
1773
1774 /**
1775  * cik_init_microcode - load ucode images from disk
1776  *
1777  * @rdev: radeon_device pointer
1778  *
1779  * Use the firmware interface to load the ucode images into
1780  * the driver (not loaded into hw).
1781  * Returns 0 on success, error on failure.
1782  */
1783 static int cik_init_microcode(struct radeon_device *rdev)
1784 {
1785         const char *chip_name;
1786         size_t pfp_req_size, me_req_size, ce_req_size,
1787                 mec_req_size, rlc_req_size, mc_req_size = 0,
1788                 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1789         char fw_name[30];
1790         int err;
1791
1792         DRM_DEBUG("\n");
1793
1794         switch (rdev->family) {
1795         case CHIP_BONAIRE:
1796                 chip_name = "BONAIRE";
1797                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1798                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1799                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1800                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1801                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1802                 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1803                 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1804                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1805                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1806                 break;
1807         case CHIP_HAWAII:
1808                 chip_name = "HAWAII";
1809                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1810                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1811                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1812                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1813                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1814                 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1815                 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1816                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1817                 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1818                 break;
1819         case CHIP_KAVERI:
1820                 chip_name = "KAVERI";
1821                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1822                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1823                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1824                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1825                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1826                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1827                 break;
1828         case CHIP_KABINI:
1829                 chip_name = "KABINI";
1830                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1831                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1832                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1833                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1834                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1835                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1836                 break;
1837         default: BUG();
1838         }
1839
1840         DRM_INFO("Loading %s Microcode\n", chip_name);
1841
1842         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1843         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1844         if (err)
1845                 goto out;
1846         if (rdev->pfp_fw->size != pfp_req_size) {
1847                 printk(KERN_ERR
1848                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1849                        rdev->pfp_fw->size, fw_name);
1850                 err = -EINVAL;
1851                 goto out;
1852         }
1853
1854         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1855         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1856         if (err)
1857                 goto out;
1858         if (rdev->me_fw->size != me_req_size) {
1859                 printk(KERN_ERR
1860                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1861                        rdev->me_fw->size, fw_name);
1862                 err = -EINVAL;
1863         }
1864
1865         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1866         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1867         if (err)
1868                 goto out;
1869         if (rdev->ce_fw->size != ce_req_size) {
1870                 printk(KERN_ERR
1871                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1872                        rdev->ce_fw->size, fw_name);
1873                 err = -EINVAL;
1874         }
1875
1876         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1877         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1878         if (err)
1879                 goto out;
1880         if (rdev->mec_fw->size != mec_req_size) {
1881                 printk(KERN_ERR
1882                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1883                        rdev->mec_fw->size, fw_name);
1884                 err = -EINVAL;
1885         }
1886
1887         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1888         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1889         if (err)
1890                 goto out;
1891         if (rdev->rlc_fw->size != rlc_req_size) {
1892                 printk(KERN_ERR
1893                        "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1894                        rdev->rlc_fw->size, fw_name);
1895                 err = -EINVAL;
1896         }
1897
1898         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1899         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1900         if (err)
1901                 goto out;
1902         if (rdev->sdma_fw->size != sdma_req_size) {
1903                 printk(KERN_ERR
1904                        "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1905                        rdev->sdma_fw->size, fw_name);
1906                 err = -EINVAL;
1907         }
1908
1909         /* No SMC, MC ucode on APUs */
1910         if (!(rdev->flags & RADEON_IS_IGP)) {
1911                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1912                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1913                 if (err) {
1914                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1915                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1916                         if (err)
1917                                 goto out;
1918                 }
1919                 if ((rdev->mc_fw->size != mc_req_size) &&
1920                     (rdev->mc_fw->size != mc2_req_size)){
1921                         printk(KERN_ERR
1922                                "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1923                                rdev->mc_fw->size, fw_name);
1924                         err = -EINVAL;
1925                 }
1926                 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1927
1928                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1929                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1930                 if (err) {
1931                         printk(KERN_ERR
1932                                "smc: error loading firmware \"%s\"\n",
1933                                fw_name);
1934                         release_firmware(rdev->smc_fw);
1935                         rdev->smc_fw = NULL;
1936                         err = 0;
1937                 } else if (rdev->smc_fw->size != smc_req_size) {
1938                         printk(KERN_ERR
1939                                "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1940                                rdev->smc_fw->size, fw_name);
1941                         err = -EINVAL;
1942                 }
1943         }
1944
1945 out:
1946         if (err) {
1947                 if (err != -EINVAL)
1948                         printk(KERN_ERR
1949                                "cik_cp: Failed to load firmware \"%s\"\n",
1950                                fw_name);
1951                 release_firmware(rdev->pfp_fw);
1952                 rdev->pfp_fw = NULL;
1953                 release_firmware(rdev->me_fw);
1954                 rdev->me_fw = NULL;
1955                 release_firmware(rdev->ce_fw);
1956                 rdev->ce_fw = NULL;
1957                 release_firmware(rdev->rlc_fw);
1958                 rdev->rlc_fw = NULL;
1959                 release_firmware(rdev->mc_fw);
1960                 rdev->mc_fw = NULL;
1961                 release_firmware(rdev->smc_fw);
1962                 rdev->smc_fw = NULL;
1963         }
1964         return err;
1965 }
1966
1967 /*
1968  * Core functions
1969  */
1970 /**
1971  * cik_tiling_mode_table_init - init the hw tiling table
1972  *
1973  * @rdev: radeon_device pointer
1974  *
1975  * Starting with SI, the tiling setup is done globally in a
1976  * set of 32 tiling modes.  Rather than selecting each set of
1977  * parameters per surface as on older asics, we just select
1978  * which index in the tiling table we want to use, and the
1979  * surface uses those parameters (CIK).
1980  */
1981 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1982 {
1983         const u32 num_tile_mode_states = 32;
1984         const u32 num_secondary_tile_mode_states = 16;
1985         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1986         u32 num_pipe_configs;
1987         u32 num_rbs = rdev->config.cik.max_backends_per_se *
1988                 rdev->config.cik.max_shader_engines;
1989
1990         switch (rdev->config.cik.mem_row_size_in_kb) {
1991         case 1:
1992                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1993                 break;
1994         case 2:
1995         default:
1996                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1997                 break;
1998         case 4:
1999                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2000                 break;
2001         }
2002
2003         num_pipe_configs = rdev->config.cik.max_tile_pipes;
2004         if (num_pipe_configs > 8)
2005                 num_pipe_configs = 16;
2006
2007         if (num_pipe_configs == 16) {
2008                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2009                         switch (reg_offset) {
2010                         case 0:
2011                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2012                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2013                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2014                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2015                                 break;
2016                         case 1:
2017                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2018                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2019                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2020                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2021                                 break;
2022                         case 2:
2023                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2024                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2025                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2026                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2027                                 break;
2028                         case 3:
2029                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2030                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2031                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2032                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2033                                 break;
2034                         case 4:
2035                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2036                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2037                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2038                                                  TILE_SPLIT(split_equal_to_row_size));
2039                                 break;
2040                         case 5:
2041                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2042                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2044                                 break;
2045                         case 6:
2046                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2047                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2048                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2049                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2050                                 break;
2051                         case 7:
2052                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2053                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2054                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2055                                                  TILE_SPLIT(split_equal_to_row_size));
2056                                 break;
2057                         case 8:
2058                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2059                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2060                                 break;
2061                         case 9:
2062                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2063                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2064                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2065                                 break;
2066                         case 10:
2067                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2068                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2069                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2070                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2071                                 break;
2072                         case 11:
2073                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2074                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2075                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2076                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2077                                 break;
2078                         case 12:
2079                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2080                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2081                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2082                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2083                                 break;
2084                         case 13:
2085                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2086                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2087                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2088                                 break;
2089                         case 14:
2090                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2091                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2092                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2093                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2094                                 break;
2095                         case 16:
2096                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2097                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2098                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2099                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2100                                 break;
2101                         case 17:
2102                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2103                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2104                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2105                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2106                                 break;
2107                         case 27:
2108                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2109                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2110                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2111                                 break;
2112                         case 28:
2113                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2114                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2115                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2116                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2117                                 break;
2118                         case 29:
2119                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2120                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2121                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2122                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2123                                 break;
2124                         case 30:
2125                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2126                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2127                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2128                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2129                                 break;
2130                         default:
2131                                 gb_tile_moden = 0;
2132                                 break;
2133                         }
2134                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2135                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2136                 }
2137                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2138                         switch (reg_offset) {
2139                         case 0:
2140                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2141                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2142                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2143                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2144                                 break;
2145                         case 1:
2146                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2147                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2148                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2149                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2150                                 break;
2151                         case 2:
2152                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2153                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2154                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2155                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2156                                 break;
2157                         case 3:
2158                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2159                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2160                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2161                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2162                                 break;
2163                         case 4:
2164                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2165                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2166                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2167                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2168                                 break;
2169                         case 5:
2170                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2171                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2172                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2173                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2174                                 break;
2175                         case 6:
2176                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2177                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2178                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2179                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2180                                 break;
2181                         case 8:
2182                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2183                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2184                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2185                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2186                                 break;
2187                         case 9:
2188                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2189                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2190                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2191                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2192                                 break;
2193                         case 10:
2194                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2195                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2196                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2197                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2198                                 break;
2199                         case 11:
2200                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2201                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2202                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2203                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2204                                 break;
2205                         case 12:
2206                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2207                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2208                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2209                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2210                                 break;
2211                         case 13:
2212                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2213                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2214                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2215                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2216                                 break;
2217                         case 14:
2218                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2219                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2220                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2221                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2222                                 break;
2223                         default:
2224                                 gb_tile_moden = 0;
2225                                 break;
2226                         }
2227                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2228                 }
2229         } else if (num_pipe_configs == 8) {
2230                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2231                         switch (reg_offset) {
2232                         case 0:
2233                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2234                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2235                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2236                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2237                                 break;
2238                         case 1:
2239                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2240                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2241                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2242                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2243                                 break;
2244                         case 2:
2245                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2246                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2247                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2248                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2249                                 break;
2250                         case 3:
2251                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2252                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2253                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2254                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2255                                 break;
2256                         case 4:
2257                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2258                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2259                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2260                                                  TILE_SPLIT(split_equal_to_row_size));
2261                                 break;
2262                         case 5:
2263                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2264                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2265                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2266                                 break;
2267                         case 6:
2268                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2269                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2270                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2271                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2272                                 break;
2273                         case 7:
2274                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2275                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2276                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2277                                                  TILE_SPLIT(split_equal_to_row_size));
2278                                 break;
2279                         case 8:
2280                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2281                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2282                                 break;
2283                         case 9:
2284                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2285                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2286                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2287                                 break;
2288                         case 10:
2289                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2290                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2291                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2292                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2293                                 break;
2294                         case 11:
2295                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2296                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2297                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2298                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2299                                 break;
2300                         case 12:
2301                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2302                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2303                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2304                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2305                                 break;
2306                         case 13:
2307                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2308                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2310                                 break;
2311                         case 14:
2312                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2313                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2314                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2315                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2316                                 break;
2317                         case 16:
2318                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2319                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2320                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2321                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2322                                 break;
2323                         case 17:
2324                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2325                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2326                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2327                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2328                                 break;
2329                         case 27:
2330                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2331                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2332                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2333                                 break;
2334                         case 28:
2335                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2336                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2337                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2338                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2339                                 break;
2340                         case 29:
2341                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2342                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2343                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2344                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2345                                 break;
2346                         case 30:
2347                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2348                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2349                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2350                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2351                                 break;
2352                         default:
2353                                 gb_tile_moden = 0;
2354                                 break;
2355                         }
2356                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2357                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2358                 }
2359                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2360                         switch (reg_offset) {
2361                         case 0:
2362                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2363                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2364                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2365                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2366                                 break;
2367                         case 1:
2368                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2370                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2371                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2372                                 break;
2373                         case 2:
2374                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2375                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2376                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2377                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2378                                 break;
2379                         case 3:
2380                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2381                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2382                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2383                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2384                                 break;
2385                         case 4:
2386                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2388                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2389                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2390                                 break;
2391                         case 5:
2392                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2393                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2394                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2395                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2396                                 break;
2397                         case 6:
2398                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2399                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2400                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2401                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2402                                 break;
2403                         case 8:
2404                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2405                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2406                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2407                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2408                                 break;
2409                         case 9:
2410                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2411                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2412                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2413                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2414                                 break;
2415                         case 10:
2416                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2418                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2419                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2420                                 break;
2421                         case 11:
2422                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2423                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2424                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2425                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2426                                 break;
2427                         case 12:
2428                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2429                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2430                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2431                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2432                                 break;
2433                         case 13:
2434                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2437                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2438                                 break;
2439                         case 14:
2440                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2442                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2443                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2444                                 break;
2445                         default:
2446                                 gb_tile_moden = 0;
2447                                 break;
2448                         }
2449                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2450                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2451                 }
2452         } else if (num_pipe_configs == 4) {
2453                 if (num_rbs == 4) {
2454                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2455                                 switch (reg_offset) {
2456                                 case 0:
2457                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2458                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2459                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2460                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2461                                         break;
2462                                 case 1:
2463                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2465                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2466                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2467                                         break;
2468                                 case 2:
2469                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2470                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2471                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2472                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2473                                         break;
2474                                 case 3:
2475                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2476                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2477                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2478                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2479                                         break;
2480                                 case 4:
2481                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2483                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2484                                                          TILE_SPLIT(split_equal_to_row_size));
2485                                         break;
2486                                 case 5:
2487                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2488                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2489                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2490                                         break;
2491                                 case 6:
2492                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2493                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2494                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2496                                         break;
2497                                 case 7:
2498                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2499                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2500                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2501                                                          TILE_SPLIT(split_equal_to_row_size));
2502                                         break;
2503                                 case 8:
2504                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2505                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
2506                                         break;
2507                                 case 9:
2508                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2509                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2510                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2511                                         break;
2512                                 case 10:
2513                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2515                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2516                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2517                                         break;
2518                                 case 11:
2519                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2520                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2521                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2522                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2523                                         break;
2524                                 case 12:
2525                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2526                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2527                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2529                                         break;
2530                                 case 13:
2531                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2532                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2533                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2534                                         break;
2535                                 case 14:
2536                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2538                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540                                         break;
2541                                 case 16:
2542                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2544                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2546                                         break;
2547                                 case 17:
2548                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2550                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552                                         break;
2553                                 case 27:
2554                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2555                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2556                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2557                                         break;
2558                                 case 28:
2559                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2561                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2562                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2563                                         break;
2564                                 case 29:
2565                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2566                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2567                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2568                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2569                                         break;
2570                                 case 30:
2571                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2572                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2573                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2574                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2575                                         break;
2576                                 default:
2577                                         gb_tile_moden = 0;
2578                                         break;
2579                                 }
2580                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2581                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2582                         }
2583                 } else if (num_rbs < 4) {
2584                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2585                                 switch (reg_offset) {
2586                                 case 0:
2587                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2589                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2591                                         break;
2592                                 case 1:
2593                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2594                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2595                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2596                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2597                                         break;
2598                                 case 2:
2599                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2600                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2601                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2602                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2603                                         break;
2604                                 case 3:
2605                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2607                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2608                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2609                                         break;
2610                                 case 4:
2611                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2612                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2613                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2614                                                          TILE_SPLIT(split_equal_to_row_size));
2615                                         break;
2616                                 case 5:
2617                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2618                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2619                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2620                                         break;
2621                                 case 6:
2622                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2623                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2624                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2625                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2626                                         break;
2627                                 case 7:
2628                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2629                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2630                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2631                                                          TILE_SPLIT(split_equal_to_row_size));
2632                                         break;
2633                                 case 8:
2634                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2635                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
2636                                         break;
2637                                 case 9:
2638                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2639                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2640                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2641                                         break;
2642                                 case 10:
2643                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2645                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2646                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2647                                         break;
2648                                 case 11:
2649                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2650                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2651                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2652                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2653                                         break;
2654                                 case 12:
2655                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2656                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2657                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2658                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2659                                         break;
2660                                 case 13:
2661                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2662                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2663                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2664                                         break;
2665                                 case 14:
2666                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2668                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2669                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2670                                         break;
2671                                 case 16:
2672                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2673                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2674                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2675                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2676                                         break;
2677                                 case 17:
2678                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2679                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2680                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2681                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2682                                         break;
2683                                 case 27:
2684                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2685                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2686                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2687                                         break;
2688                                 case 28:
2689                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2690                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2691                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2692                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2693                                         break;
2694                                 case 29:
2695                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2696                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2697                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2698                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2699                                         break;
2700                                 case 30:
2701                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2702                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2703                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2704                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2705                                         break;
2706                                 default:
2707                                         gb_tile_moden = 0;
2708                                         break;
2709                                 }
2710                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2711                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2712                         }
2713                 }
2714                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2715                         switch (reg_offset) {
2716                         case 0:
2717                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2719                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2720                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2721                                 break;
2722                         case 1:
2723                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2724                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2725                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2726                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2727                                 break;
2728                         case 2:
2729                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2730                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2731                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2732                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2733                                 break;
2734                         case 3:
2735                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2736                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2737                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2738                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2739                                 break;
2740                         case 4:
2741                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2742                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2743                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2744                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2745                                 break;
2746                         case 5:
2747                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2748                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2749                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2750                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2751                                 break;
2752                         case 6:
2753                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2755                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2756                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2757                                 break;
2758                         case 8:
2759                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2760                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2761                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2762                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2763                                 break;
2764                         case 9:
2765                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2766                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2767                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2768                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2769                                 break;
2770                         case 10:
2771                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2773                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2774                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2775                                 break;
2776                         case 11:
2777                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2778                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2779                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2780                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2781                                 break;
2782                         case 12:
2783                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2784                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2785                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2786                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2787                                 break;
2788                         case 13:
2789                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2791                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2792                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2793                                 break;
2794                         case 14:
2795                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2796                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2797                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2798                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2799                                 break;
2800                         default:
2801                                 gb_tile_moden = 0;
2802                                 break;
2803                         }
2804                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2805                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2806                 }
2807         } else if (num_pipe_configs == 2) {
2808                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2809                         switch (reg_offset) {
2810                         case 0:
2811                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2813                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2814                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2815                                 break;
2816                         case 1:
2817                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2818                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2819                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2820                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2821                                 break;
2822                         case 2:
2823                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2824                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2825                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2826                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2827                                 break;
2828                         case 3:
2829                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2830                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2831                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2832                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2833                                 break;
2834                         case 4:
2835                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2836                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2837                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2838                                                  TILE_SPLIT(split_equal_to_row_size));
2839                                 break;
2840                         case 5:
2841                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2842                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2843                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2844                                 break;
2845                         case 6:
2846                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2847                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2848                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2849                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2850                                 break;
2851                         case 7:
2852                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2853                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2854                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2855                                                  TILE_SPLIT(split_equal_to_row_size));
2856                                 break;
2857                         case 8:
2858                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2859                                                 PIPE_CONFIG(ADDR_SURF_P2);
2860                                 break;
2861                         case 9:
2862                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2863                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2864                                                  PIPE_CONFIG(ADDR_SURF_P2));
2865                                 break;
2866                         case 10:
2867                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2868                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2869                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2870                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2871                                 break;
2872                         case 11:
2873                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2874                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2875                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2876                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2877                                 break;
2878                         case 12:
2879                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2880                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2881                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2882                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2883                                 break;
2884                         case 13:
2885                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2886                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2887                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2888                                 break;
2889                         case 14:
2890                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2891                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2892                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2893                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2894                                 break;
2895                         case 16:
2896                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2897                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2898                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2899                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2900                                 break;
2901                         case 17:
2902                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2903                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2904                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2905                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2906                                 break;
2907                         case 27:
2908                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2909                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2910                                                  PIPE_CONFIG(ADDR_SURF_P2));
2911                                 break;
2912                         case 28:
2913                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2914                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2915                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2916                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2917                                 break;
2918                         case 29:
2919                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2920                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2921                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2922                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2923                                 break;
2924                         case 30:
2925                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2926                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2927                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2928                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2929                                 break;
2930                         default:
2931                                 gb_tile_moden = 0;
2932                                 break;
2933                         }
2934                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2935                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2936                 }
2937                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2938                         switch (reg_offset) {
2939                         case 0:
2940                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2941                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2942                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2943                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2944                                 break;
2945                         case 1:
2946                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2947                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2948                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2949                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2950                                 break;
2951                         case 2:
2952                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2953                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2954                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2955                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2956                                 break;
2957                         case 3:
2958                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2959                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2960                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2961                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2962                                 break;
2963                         case 4:
2964                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2965                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2966                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2967                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2968                                 break;
2969                         case 5:
2970                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2971                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2972                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2973                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2974                                 break;
2975                         case 6:
2976                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2977                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2978                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2979                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2980                                 break;
2981                         case 8:
2982                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2983                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2984                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2985                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2986                                 break;
2987                         case 9:
2988                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2989                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2990                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2991                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2992                                 break;
2993                         case 10:
2994                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2995                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2996                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2997                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2998                                 break;
2999                         case 11:
3000                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3001                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3002                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3003                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3004                                 break;
3005                         case 12:
3006                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3007                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3008                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3009                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3010                                 break;
3011                         case 13:
3012                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3013                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3014                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3015                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3016                                 break;
3017                         case 14:
3018                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3019                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3020                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3021                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3022                                 break;
3023                         default:
3024                                 gb_tile_moden = 0;
3025                                 break;
3026                         }
3027                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3028                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3029                 }
3030         } else
3031                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3032 }
3033
3034 /**
3035  * cik_select_se_sh - select which SE, SH to address
3036  *
3037  * @rdev: radeon_device pointer
3038  * @se_num: shader engine to address
3039  * @sh_num: sh block to address
3040  *
3041  * Select which SE, SH combinations to address. Certain
3042  * registers are instanced per SE or SH.  0xffffffff means
3043  * broadcast to all SEs or SHs (CIK).
3044  */
3045 static void cik_select_se_sh(struct radeon_device *rdev,
3046                              u32 se_num, u32 sh_num)
3047 {
3048         u32 data = INSTANCE_BROADCAST_WRITES;
3049
3050         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3051                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3052         else if (se_num == 0xffffffff)
3053                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3054         else if (sh_num == 0xffffffff)
3055                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3056         else
3057                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3058         WREG32(GRBM_GFX_INDEX, data);
3059 }
3060
3061 /**
3062  * cik_create_bitmask - create a bitmask
3063  *
3064  * @bit_width: length of the mask
3065  *
3066  * create a variable length bit mask (CIK).
3067  * Returns the bitmask.
3068  */
3069 static u32 cik_create_bitmask(u32 bit_width)
3070 {
3071         u32 i, mask = 0;
3072
3073         for (i = 0; i < bit_width; i++) {
3074                 mask <<= 1;
3075                 mask |= 1;
3076         }
3077         return mask;
3078 }
3079
3080 /**
3081  * cik_get_rb_disabled - computes the mask of disabled RBs
3082  *
3083  * @rdev: radeon_device pointer
3084  * @max_rb_num: max RBs (render backends) for the asic
3085  * @se_num: number of SEs (shader engines) for the asic
3086  * @sh_per_se: number of SH blocks per SE for the asic
3087  *
3088  * Calculates the bitmask of disabled RBs (CIK).
3089  * Returns the disabled RB bitmask.
3090  */
3091 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3092                               u32 max_rb_num_per_se,
3093                               u32 sh_per_se)
3094 {
3095         u32 data, mask;
3096
3097         data = RREG32(CC_RB_BACKEND_DISABLE);
3098         if (data & 1)
3099                 data &= BACKEND_DISABLE_MASK;
3100         else
3101                 data = 0;
3102         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3103
3104         data >>= BACKEND_DISABLE_SHIFT;
3105
3106         mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3107
3108         return data & mask;
3109 }
3110
3111 /**
3112  * cik_setup_rb - setup the RBs on the asic
3113  *
3114  * @rdev: radeon_device pointer
3115  * @se_num: number of SEs (shader engines) for the asic
3116  * @sh_per_se: number of SH blocks per SE for the asic
3117  * @max_rb_num: max RBs (render backends) for the asic
3118  *
3119  * Configures per-SE/SH RB registers (CIK).
3120  */
3121 static void cik_setup_rb(struct radeon_device *rdev,
3122                          u32 se_num, u32 sh_per_se,
3123                          u32 max_rb_num_per_se)
3124 {
3125         int i, j;
3126         u32 data, mask;
3127         u32 disabled_rbs = 0;
3128         u32 enabled_rbs = 0;
3129
3130         for (i = 0; i < se_num; i++) {
3131                 for (j = 0; j < sh_per_se; j++) {
3132                         cik_select_se_sh(rdev, i, j);
3133                         data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3134                         if (rdev->family == CHIP_HAWAII)
3135                                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3136                         else
3137                                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3138                 }
3139         }
3140         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3141
3142         mask = 1;
3143         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3144                 if (!(disabled_rbs & mask))
3145                         enabled_rbs |= mask;
3146                 mask <<= 1;
3147         }
3148
3149         rdev->config.cik.backend_enable_mask = enabled_rbs;
3150
3151         for (i = 0; i < se_num; i++) {
3152                 cik_select_se_sh(rdev, i, 0xffffffff);
3153                 data = 0;
3154                 for (j = 0; j < sh_per_se; j++) {
3155                         switch (enabled_rbs & 3) {
3156                         case 0:
3157                                 if (j == 0)
3158                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3159                                 else
3160                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3161                                 break;
3162                         case 1:
3163                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3164                                 break;
3165                         case 2:
3166                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3167                                 break;
3168                         case 3:
3169                         default:
3170                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3171                                 break;
3172                         }
3173                         enabled_rbs >>= 2;
3174                 }
3175                 WREG32(PA_SC_RASTER_CONFIG, data);
3176         }
3177         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3178 }
3179
3180 /**
3181  * cik_gpu_init - setup the 3D engine
3182  *
3183  * @rdev: radeon_device pointer
3184  *
3185  * Configures the 3D engine and tiling configuration
3186  * registers so that the 3D engine is usable.
3187  */
3188 static void cik_gpu_init(struct radeon_device *rdev)
3189 {
3190         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3191         u32 mc_shared_chmap, mc_arb_ramcfg;
3192         u32 hdp_host_path_cntl;
3193         u32 tmp;
3194         int i, j;
3195
3196         switch (rdev->family) {
3197         case CHIP_BONAIRE:
3198                 rdev->config.cik.max_shader_engines = 2;
3199                 rdev->config.cik.max_tile_pipes = 4;
3200                 rdev->config.cik.max_cu_per_sh = 7;
3201                 rdev->config.cik.max_sh_per_se = 1;
3202                 rdev->config.cik.max_backends_per_se = 2;
3203                 rdev->config.cik.max_texture_channel_caches = 4;
3204                 rdev->config.cik.max_gprs = 256;
3205                 rdev->config.cik.max_gs_threads = 32;
3206                 rdev->config.cik.max_hw_contexts = 8;
3207
3208                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3209                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3210                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3211                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3212                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3213                 break;
3214         case CHIP_HAWAII:
3215                 rdev->config.cik.max_shader_engines = 4;
3216                 rdev->config.cik.max_tile_pipes = 16;
3217                 rdev->config.cik.max_cu_per_sh = 11;
3218                 rdev->config.cik.max_sh_per_se = 1;
3219                 rdev->config.cik.max_backends_per_se = 4;
3220                 rdev->config.cik.max_texture_channel_caches = 16;
3221                 rdev->config.cik.max_gprs = 256;
3222                 rdev->config.cik.max_gs_threads = 32;
3223                 rdev->config.cik.max_hw_contexts = 8;
3224
3225                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3226                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3227                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3228                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3229                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3230                 break;
3231         case CHIP_KAVERI:
3232                 rdev->config.cik.max_shader_engines = 1;
3233                 rdev->config.cik.max_tile_pipes = 4;
3234                 if ((rdev->pdev->device == 0x1304) ||
3235                     (rdev->pdev->device == 0x1305) ||
3236                     (rdev->pdev->device == 0x130C) ||
3237                     (rdev->pdev->device == 0x130F) ||
3238                     (rdev->pdev->device == 0x1310) ||
3239                     (rdev->pdev->device == 0x1311) ||
3240                     (rdev->pdev->device == 0x131C)) {
3241                         rdev->config.cik.max_cu_per_sh = 8;
3242                         rdev->config.cik.max_backends_per_se = 2;
3243                 } else if ((rdev->pdev->device == 0x1309) ||
3244                            (rdev->pdev->device == 0x130A) ||
3245                            (rdev->pdev->device == 0x130D) ||
3246                            (rdev->pdev->device == 0x1313) ||
3247                            (rdev->pdev->device == 0x131D)) {
3248                         rdev->config.cik.max_cu_per_sh = 6;
3249                         rdev->config.cik.max_backends_per_se = 2;
3250                 } else if ((rdev->pdev->device == 0x1306) ||
3251                            (rdev->pdev->device == 0x1307) ||
3252                            (rdev->pdev->device == 0x130B) ||
3253                            (rdev->pdev->device == 0x130E) ||
3254                            (rdev->pdev->device == 0x1315) ||
3255                            (rdev->pdev->device == 0x131B)) {
3256                         rdev->config.cik.max_cu_per_sh = 4;
3257                         rdev->config.cik.max_backends_per_se = 1;
3258                 } else {
3259                         rdev->config.cik.max_cu_per_sh = 3;
3260                         rdev->config.cik.max_backends_per_se = 1;
3261                 }
3262                 rdev->config.cik.max_sh_per_se = 1;
3263                 rdev->config.cik.max_texture_channel_caches = 4;
3264                 rdev->config.cik.max_gprs = 256;
3265                 rdev->config.cik.max_gs_threads = 16;
3266                 rdev->config.cik.max_hw_contexts = 8;
3267
3268                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3269                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3270                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3271                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3272                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3273                 break;
3274         case CHIP_KABINI:
3275         default:
3276                 rdev->config.cik.max_shader_engines = 1;
3277                 rdev->config.cik.max_tile_pipes = 2;
3278                 rdev->config.cik.max_cu_per_sh = 2;
3279                 rdev->config.cik.max_sh_per_se = 1;
3280                 rdev->config.cik.max_backends_per_se = 1;
3281                 rdev->config.cik.max_texture_channel_caches = 2;
3282                 rdev->config.cik.max_gprs = 256;
3283                 rdev->config.cik.max_gs_threads = 16;
3284                 rdev->config.cik.max_hw_contexts = 8;
3285
3286                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3287                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3288                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3289                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3290                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3291                 break;
3292         }
3293
3294         /* Initialize HDP */
3295         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3296                 WREG32((0x2c14 + j), 0x00000000);
3297                 WREG32((0x2c18 + j), 0x00000000);
3298                 WREG32((0x2c1c + j), 0x00000000);
3299                 WREG32((0x2c20 + j), 0x00000000);
3300                 WREG32((0x2c24 + j), 0x00000000);
3301         }
3302
3303         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3304
3305         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3306
3307         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3308         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3309
3310         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3311         rdev->config.cik.mem_max_burst_length_bytes = 256;
3312         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3313         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3314         if (rdev->config.cik.mem_row_size_in_kb > 4)
3315                 rdev->config.cik.mem_row_size_in_kb = 4;
3316         /* XXX use MC settings? */
3317         rdev->config.cik.shader_engine_tile_size = 32;
3318         rdev->config.cik.num_gpus = 1;
3319         rdev->config.cik.multi_gpu_tile_size = 64;
3320
3321         /* fix up row size */
3322         gb_addr_config &= ~ROW_SIZE_MASK;
3323         switch (rdev->config.cik.mem_row_size_in_kb) {
3324         case 1:
3325         default:
3326                 gb_addr_config |= ROW_SIZE(0);
3327                 break;
3328         case 2:
3329                 gb_addr_config |= ROW_SIZE(1);
3330                 break;
3331         case 4:
3332                 gb_addr_config |= ROW_SIZE(2);
3333                 break;
3334         }
3335
3336         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3337          * not have bank info, so create a custom tiling dword.
3338          * bits 3:0   num_pipes
3339          * bits 7:4   num_banks
3340          * bits 11:8  group_size
3341          * bits 15:12 row_size
3342          */
3343         rdev->config.cik.tile_config = 0;
3344         switch (rdev->config.cik.num_tile_pipes) {
3345         case 1:
3346                 rdev->config.cik.tile_config |= (0 << 0);
3347                 break;
3348         case 2:
3349                 rdev->config.cik.tile_config |= (1 << 0);
3350                 break;
3351         case 4:
3352                 rdev->config.cik.tile_config |= (2 << 0);
3353                 break;
3354         case 8:
3355         default:
3356                 /* XXX what about 12? */
3357                 rdev->config.cik.tile_config |= (3 << 0);
3358                 break;
3359         }
3360         rdev->config.cik.tile_config |=
3361                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3362         rdev->config.cik.tile_config |=
3363                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3364         rdev->config.cik.tile_config |=
3365                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3366
3367         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3368         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3369         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3370         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3371         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3372         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3373         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3374         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3375
3376         cik_tiling_mode_table_init(rdev);
3377
3378         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3379                      rdev->config.cik.max_sh_per_se,
3380                      rdev->config.cik.max_backends_per_se);
3381
3382         /* set HW defaults for 3D engine */
3383         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3384
3385         WREG32(SX_DEBUG_1, 0x20);
3386
3387         WREG32(TA_CNTL_AUX, 0x00010000);
3388
3389         tmp = RREG32(SPI_CONFIG_CNTL);
3390         tmp |= 0x03000000;
3391         WREG32(SPI_CONFIG_CNTL, tmp);
3392
3393         WREG32(SQ_CONFIG, 1);
3394
3395         WREG32(DB_DEBUG, 0);
3396
3397         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3398         tmp |= 0x00000400;
3399         WREG32(DB_DEBUG2, tmp);
3400
3401         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3402         tmp |= 0x00020200;
3403         WREG32(DB_DEBUG3, tmp);
3404
3405         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3406         tmp |= 0x00018208;
3407         WREG32(CB_HW_CONTROL, tmp);
3408
3409         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3410
3411         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3412                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3413                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3414                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3415
3416         WREG32(VGT_NUM_INSTANCES, 1);
3417
3418         WREG32(CP_PERFMON_CNTL, 0);
3419
3420         WREG32(SQ_CONFIG, 0);
3421
3422         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3423                                           FORCE_EOV_MAX_REZ_CNT(255)));
3424
3425         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3426                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3427
3428         WREG32(VGT_GS_VERTEX_REUSE, 16);
3429         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3430
3431         tmp = RREG32(HDP_MISC_CNTL);
3432         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3433         WREG32(HDP_MISC_CNTL, tmp);
3434
3435         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3436         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3437
3438         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3439         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3440
3441         udelay(50);
3442 }
3443
3444 /*
3445  * GPU scratch registers helpers function.
3446  */
3447 /**
3448  * cik_scratch_init - setup driver info for CP scratch regs
3449  *
3450  * @rdev: radeon_device pointer
3451  *
3452  * Set up the number and offset of the CP scratch registers.
3453  * NOTE: use of CP scratch registers is a legacy inferface and
3454  * is not used by default on newer asics (r6xx+).  On newer asics,
3455  * memory buffers are used for fences rather than scratch regs.
3456  */
3457 static void cik_scratch_init(struct radeon_device *rdev)
3458 {
3459         int i;
3460
3461         rdev->scratch.num_reg = 7;
3462         rdev->scratch.reg_base = SCRATCH_REG0;
3463         for (i = 0; i < rdev->scratch.num_reg; i++) {
3464                 rdev->scratch.free[i] = true;
3465                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3466         }
3467 }
3468
3469 /**
3470  * cik_ring_test - basic gfx ring test
3471  *
3472  * @rdev: radeon_device pointer
3473  * @ring: radeon_ring structure holding ring information
3474  *
3475  * Allocate a scratch register and write to it using the gfx ring (CIK).
3476  * Provides a basic gfx ring test to verify that the ring is working.
3477  * Used by cik_cp_gfx_resume();
3478  * Returns 0 on success, error on failure.
3479  */
3480 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3481 {
3482         uint32_t scratch;
3483         uint32_t tmp = 0;
3484         unsigned i;
3485         int r;
3486
3487         r = radeon_scratch_get(rdev, &scratch);
3488         if (r) {
3489                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3490                 return r;
3491         }
3492         WREG32(scratch, 0xCAFEDEAD);
3493         r = radeon_ring_lock(rdev, ring, 3);
3494         if (r) {
3495                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3496                 radeon_scratch_free(rdev, scratch);
3497                 return r;
3498         }
3499         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3500         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3501         radeon_ring_write(ring, 0xDEADBEEF);
3502         radeon_ring_unlock_commit(rdev, ring);
3503
3504         for (i = 0; i < rdev->usec_timeout; i++) {
3505                 tmp = RREG32(scratch);
3506                 if (tmp == 0xDEADBEEF)
3507                         break;
3508                 DRM_UDELAY(1);
3509         }
3510         if (i < rdev->usec_timeout) {
3511                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3512         } else {
3513                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3514                           ring->idx, scratch, tmp);
3515                 r = -EINVAL;
3516         }
3517         radeon_scratch_free(rdev, scratch);
3518         return r;
3519 }
3520
3521 /**
3522  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3523  *
3524  * @rdev: radeon_device pointer
3525  * @ridx: radeon ring index
3526  *
3527  * Emits an hdp flush on the cp.
3528  */
3529 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3530                                        int ridx)
3531 {
3532         struct radeon_ring *ring = &rdev->ring[ridx];
3533         u32 ref_and_mask;
3534
3535         switch (ring->idx) {
3536         case CAYMAN_RING_TYPE_CP1_INDEX:
3537         case CAYMAN_RING_TYPE_CP2_INDEX:
3538         default:
3539                 switch (ring->me) {
3540                 case 0:
3541                         ref_and_mask = CP2 << ring->pipe;
3542                         break;
3543                 case 1:
3544                         ref_and_mask = CP6 << ring->pipe;
3545                         break;
3546                 default:
3547                         return;
3548                 }
3549                 break;
3550         case RADEON_RING_TYPE_GFX_INDEX:
3551                 ref_and_mask = CP0;
3552                 break;
3553         }
3554
3555         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3556         radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3557                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
3558                                  WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3559         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3560         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3561         radeon_ring_write(ring, ref_and_mask);
3562         radeon_ring_write(ring, ref_and_mask);
3563         radeon_ring_write(ring, 0x20); /* poll interval */
3564 }
3565
3566 /**
3567  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3568  *
3569  * @rdev: radeon_device pointer
3570  * @fence: radeon fence object
3571  *
3572  * Emits a fence sequnce number on the gfx ring and flushes
3573  * GPU caches.
3574  */
3575 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3576                              struct radeon_fence *fence)
3577 {
3578         struct radeon_ring *ring = &rdev->ring[fence->ring];
3579         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3580
3581         /* EVENT_WRITE_EOP - flush caches, send int */
3582         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3583         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3584                                  EOP_TC_ACTION_EN |
3585                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3586                                  EVENT_INDEX(5)));
3587         radeon_ring_write(ring, addr & 0xfffffffc);
3588         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3589         radeon_ring_write(ring, fence->seq);
3590         radeon_ring_write(ring, 0);
3591         /* HDP flush */
3592         cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3593 }
3594
3595 /**
3596  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3597  *
3598  * @rdev: radeon_device pointer
3599  * @fence: radeon fence object
3600  *
3601  * Emits a fence sequnce number on the compute ring and flushes
3602  * GPU caches.
3603  */
3604 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3605                                  struct radeon_fence *fence)
3606 {
3607         struct radeon_ring *ring = &rdev->ring[fence->ring];
3608         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3609
3610         /* RELEASE_MEM - flush caches, send int */
3611         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3612         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3613                                  EOP_TC_ACTION_EN |
3614                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3615                                  EVENT_INDEX(5)));
3616         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3617         radeon_ring_write(ring, addr & 0xfffffffc);
3618         radeon_ring_write(ring, upper_32_bits(addr));
3619         radeon_ring_write(ring, fence->seq);
3620         radeon_ring_write(ring, 0);
3621         /* HDP flush */
3622         cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3623 }
3624
3625 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3626                              struct radeon_ring *ring,
3627                              struct radeon_semaphore *semaphore,
3628                              bool emit_wait)
3629 {
3630         uint64_t addr = semaphore->gpu_addr;
3631         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3632
3633         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3634         radeon_ring_write(ring, addr & 0xffffffff);
3635         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3636
3637         return true;
3638 }
3639
3640 /**
3641  * cik_copy_cpdma - copy pages using the CP DMA engine
3642  *
3643  * @rdev: radeon_device pointer
3644  * @src_offset: src GPU address
3645  * @dst_offset: dst GPU address
3646  * @num_gpu_pages: number of GPU pages to xfer
3647  * @fence: radeon fence object
3648  *
3649  * Copy GPU paging using the CP DMA engine (CIK+).
3650  * Used by the radeon ttm implementation to move pages if
3651  * registered as the asic copy callback.
3652  */
3653 int cik_copy_cpdma(struct radeon_device *rdev,
3654                    uint64_t src_offset, uint64_t dst_offset,
3655                    unsigned num_gpu_pages,
3656                    struct radeon_fence **fence)
3657 {
3658         struct radeon_semaphore *sem = NULL;
3659         int ring_index = rdev->asic->copy.blit_ring_index;
3660         struct radeon_ring *ring = &rdev->ring[ring_index];
3661         u32 size_in_bytes, cur_size_in_bytes, control;
3662         int i, num_loops;
3663         int r = 0;
3664
3665         r = radeon_semaphore_create(rdev, &sem);
3666         if (r) {
3667                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3668                 return r;
3669         }
3670
3671         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3672         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3673         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3674         if (r) {
3675                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3676                 radeon_semaphore_free(rdev, &sem, NULL);
3677                 return r;
3678         }
3679
3680         radeon_semaphore_sync_to(sem, *fence);
3681         radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3682
3683         for (i = 0; i < num_loops; i++) {
3684                 cur_size_in_bytes = size_in_bytes;
3685                 if (cur_size_in_bytes > 0x1fffff)
3686                         cur_size_in_bytes = 0x1fffff;
3687                 size_in_bytes -= cur_size_in_bytes;
3688                 control = 0;
3689                 if (size_in_bytes == 0)
3690                         control |= PACKET3_DMA_DATA_CP_SYNC;
3691                 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3692                 radeon_ring_write(ring, control);
3693                 radeon_ring_write(ring, lower_32_bits(src_offset));
3694                 radeon_ring_write(ring, upper_32_bits(src_offset));
3695                 radeon_ring_write(ring, lower_32_bits(dst_offset));
3696                 radeon_ring_write(ring, upper_32_bits(dst_offset));
3697                 radeon_ring_write(ring, cur_size_in_bytes);
3698                 src_offset += cur_size_in_bytes;
3699                 dst_offset += cur_size_in_bytes;
3700         }
3701
3702         r = radeon_fence_emit(rdev, fence, ring->idx);
3703         if (r) {
3704                 radeon_ring_unlock_undo(rdev, ring);
3705                 radeon_semaphore_free(rdev, &sem, NULL);
3706                 return r;
3707         }
3708
3709         radeon_ring_unlock_commit(rdev, ring);
3710         radeon_semaphore_free(rdev, &sem, *fence);
3711
3712         return r;
3713 }
3714
3715 /*
3716  * IB stuff
3717  */
3718 /**
3719  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3720  *
3721  * @rdev: radeon_device pointer
3722  * @ib: radeon indirect buffer object
3723  *
3724  * Emits an DE (drawing engine) or CE (constant engine) IB
3725  * on the gfx ring.  IBs are usually generated by userspace
3726  * acceleration drivers and submitted to the kernel for
3727  * sheduling on the ring.  This function schedules the IB
3728  * on the gfx ring for execution by the GPU.
3729  */
3730 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3731 {
3732         struct radeon_ring *ring = &rdev->ring[ib->ring];
3733         u32 header, control = INDIRECT_BUFFER_VALID;
3734
3735         if (ib->is_const_ib) {
3736                 /* set switch buffer packet before const IB */
3737                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3738                 radeon_ring_write(ring, 0);
3739
3740                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3741         } else {
3742                 u32 next_rptr;
3743                 if (ring->rptr_save_reg) {
3744                         next_rptr = ring->wptr + 3 + 4;
3745                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3746                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3747                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
3748                         radeon_ring_write(ring, next_rptr);
3749                 } else if (rdev->wb.enabled) {
3750                         next_rptr = ring->wptr + 5 + 4;
3751                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3752                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3753                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3754                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3755                         radeon_ring_write(ring, next_rptr);
3756                 }
3757
3758                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3759         }
3760
3761         control |= ib->length_dw |
3762                 (ib->vm ? (ib->vm->id << 24) : 0);
3763
3764         radeon_ring_write(ring, header);
3765         radeon_ring_write(ring,
3766 #ifdef __BIG_ENDIAN
3767                           (2 << 0) |
3768 #endif
3769                           (ib->gpu_addr & 0xFFFFFFFC));
3770         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3771         radeon_ring_write(ring, control);
3772 }
3773
3774 /**
3775  * cik_ib_test - basic gfx ring IB test
3776  *
3777  * @rdev: radeon_device pointer
3778  * @ring: radeon_ring structure holding ring information
3779  *
3780  * Allocate an IB and execute it on the gfx ring (CIK).
3781  * Provides a basic gfx ring test to verify that IBs are working.
3782  * Returns 0 on success, error on failure.
3783  */
3784 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3785 {
3786         struct radeon_ib ib;
3787         uint32_t scratch;
3788         uint32_t tmp = 0;
3789         unsigned i;
3790         int r;
3791
3792         r = radeon_scratch_get(rdev, &scratch);
3793         if (r) {
3794                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3795                 return r;
3796         }
3797         WREG32(scratch, 0xCAFEDEAD);
3798         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3799         if (r) {
3800                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3801                 radeon_scratch_free(rdev, scratch);
3802                 return r;
3803         }
3804         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3805         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3806         ib.ptr[2] = 0xDEADBEEF;
3807         ib.length_dw = 3;
3808         r = radeon_ib_schedule(rdev, &ib, NULL);
3809         if (r) {
3810                 radeon_scratch_free(rdev, scratch);
3811                 radeon_ib_free(rdev, &ib);
3812                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3813                 return r;
3814         }
3815         r = radeon_fence_wait(ib.fence, false);
3816         if (r) {
3817                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3818                 radeon_scratch_free(rdev, scratch);
3819                 radeon_ib_free(rdev, &ib);
3820                 return r;
3821         }
3822         for (i = 0; i < rdev->usec_timeout; i++) {
3823                 tmp = RREG32(scratch);
3824                 if (tmp == 0xDEADBEEF)
3825                         break;
3826                 DRM_UDELAY(1);
3827         }
3828         if (i < rdev->usec_timeout) {
3829                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3830         } else {
3831                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3832                           scratch, tmp);
3833                 r = -EINVAL;
3834         }
3835         radeon_scratch_free(rdev, scratch);
3836         radeon_ib_free(rdev, &ib);
3837         return r;
3838 }
3839
3840 /*
3841  * CP.
3842  * On CIK, gfx and compute now have independant command processors.
3843  *
3844  * GFX
3845  * Gfx consists of a single ring and can process both gfx jobs and
3846  * compute jobs.  The gfx CP consists of three microengines (ME):
3847  * PFP - Pre-Fetch Parser
3848  * ME - Micro Engine
3849  * CE - Constant Engine
3850  * The PFP and ME make up what is considered the Drawing Engine (DE).
3851  * The CE is an asynchronous engine used for updating buffer desciptors
3852  * used by the DE so that they can be loaded into cache in parallel
3853  * while the DE is processing state update packets.
3854  *
3855  * Compute
3856  * The compute CP consists of two microengines (ME):
3857  * MEC1 - Compute MicroEngine 1
3858  * MEC2 - Compute MicroEngine 2
3859  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3860  * The queues are exposed to userspace and are programmed directly
3861  * by the compute runtime.
3862  */
3863 /**
3864  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3865  *
3866  * @rdev: radeon_device pointer
3867  * @enable: enable or disable the MEs
3868  *
3869  * Halts or unhalts the gfx MEs.
3870  */
3871 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3872 {
3873         if (enable)
3874                 WREG32(CP_ME_CNTL, 0);
3875         else {
3876                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3877                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3878                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3879                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3880         }
3881         udelay(50);
3882 }
3883
3884 /**
3885  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3886  *
3887  * @rdev: radeon_device pointer
3888  *
3889  * Loads the gfx PFP, ME, and CE ucode.
3890  * Returns 0 for success, -EINVAL if the ucode is not available.
3891  */
3892 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3893 {
3894         const __be32 *fw_data;
3895         int i;
3896
3897         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3898                 return -EINVAL;
3899
3900         cik_cp_gfx_enable(rdev, false);
3901
3902         /* PFP */
3903         fw_data = (const __be32 *)rdev->pfp_fw->data;
3904         WREG32(CP_PFP_UCODE_ADDR, 0);
3905         for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3906                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3907         WREG32(CP_PFP_UCODE_ADDR, 0);
3908
3909         /* CE */
3910         fw_data = (const __be32 *)rdev->ce_fw->data;
3911         WREG32(CP_CE_UCODE_ADDR, 0);
3912         for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3913                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3914         WREG32(CP_CE_UCODE_ADDR, 0);
3915
3916         /* ME */
3917         fw_data = (const __be32 *)rdev->me_fw->data;
3918         WREG32(CP_ME_RAM_WADDR, 0);
3919         for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3920                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3921         WREG32(CP_ME_RAM_WADDR, 0);
3922
3923         WREG32(CP_PFP_UCODE_ADDR, 0);
3924         WREG32(CP_CE_UCODE_ADDR, 0);
3925         WREG32(CP_ME_RAM_WADDR, 0);
3926         WREG32(CP_ME_RAM_RADDR, 0);
3927         return 0;
3928 }
3929
3930 /**
3931  * cik_cp_gfx_start - start the gfx ring
3932  *
3933  * @rdev: radeon_device pointer
3934  *
3935  * Enables the ring and loads the clear state context and other
3936  * packets required to init the ring.
3937  * Returns 0 for success, error for failure.
3938  */
3939 static int cik_cp_gfx_start(struct radeon_device *rdev)
3940 {
3941         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3942         int r, i;
3943
3944         /* init the CP */
3945         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3946         WREG32(CP_ENDIAN_SWAP, 0);
3947         WREG32(CP_DEVICE_ID, 1);
3948
3949         cik_cp_gfx_enable(rdev, true);
3950
3951         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3952         if (r) {
3953                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3954                 return r;
3955         }
3956
3957         /* init the CE partitions.  CE only used for gfx on CIK */
3958         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3959         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3960         radeon_ring_write(ring, 0xc000);
3961         radeon_ring_write(ring, 0xc000);
3962
3963         /* setup clear context state */
3964         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3965         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3966
3967         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3968         radeon_ring_write(ring, 0x80000000);
3969         radeon_ring_write(ring, 0x80000000);
3970
3971         for (i = 0; i < cik_default_size; i++)
3972                 radeon_ring_write(ring, cik_default_state[i]);
3973
3974         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3975         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3976
3977         /* set clear context state */
3978         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3979         radeon_ring_write(ring, 0);
3980
3981         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3982         radeon_ring_write(ring, 0x00000316);
3983         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3984         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3985
3986         radeon_ring_unlock_commit(rdev, ring);
3987
3988         return 0;
3989 }
3990
3991 /**
3992  * cik_cp_gfx_fini - stop the gfx ring
3993  *
3994  * @rdev: radeon_device pointer
3995  *
3996  * Stop the gfx ring and tear down the driver ring
3997  * info.
3998  */
3999 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4000 {
4001         cik_cp_gfx_enable(rdev, false);
4002         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4003 }
4004
4005 /**
4006  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4007  *
4008  * @rdev: radeon_device pointer
4009  *
4010  * Program the location and size of the gfx ring buffer
4011  * and test it to make sure it's working.
4012  * Returns 0 for success, error for failure.
4013  */
4014 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4015 {
4016         struct radeon_ring *ring;
4017         u32 tmp;
4018         u32 rb_bufsz;
4019         u64 rb_addr;
4020         int r;
4021
4022         WREG32(CP_SEM_WAIT_TIMER, 0x0);
4023         if (rdev->family != CHIP_HAWAII)
4024                 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4025
4026         /* Set the write pointer delay */
4027         WREG32(CP_RB_WPTR_DELAY, 0);
4028
4029         /* set the RB to use vmid 0 */
4030         WREG32(CP_RB_VMID, 0);
4031
4032         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4033
4034         /* ring 0 - compute and gfx */
4035         /* Set ring buffer size */
4036         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4037         rb_bufsz = order_base_2(ring->ring_size / 8);
4038         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4039 #ifdef __BIG_ENDIAN
4040         tmp |= BUF_SWAP_32BIT;
4041 #endif
4042         WREG32(CP_RB0_CNTL, tmp);
4043
4044         /* Initialize the ring buffer's read and write pointers */
4045         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4046         ring->wptr = 0;
4047         WREG32(CP_RB0_WPTR, ring->wptr);
4048
4049         /* set the wb address wether it's enabled or not */
4050         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4051         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4052
4053         /* scratch register shadowing is no longer supported */
4054         WREG32(SCRATCH_UMSK, 0);
4055
4056         if (!rdev->wb.enabled)
4057                 tmp |= RB_NO_UPDATE;
4058
4059         mdelay(1);
4060         WREG32(CP_RB0_CNTL, tmp);
4061
4062         rb_addr = ring->gpu_addr >> 8;
4063         WREG32(CP_RB0_BASE, rb_addr);
4064         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4065
4066         /* start the ring */
4067         cik_cp_gfx_start(rdev);
4068         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4069         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4070         if (r) {
4071                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4072                 return r;
4073         }
4074
4075         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4076                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4077
4078         return 0;
4079 }
4080
4081 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4082                      struct radeon_ring *ring)
4083 {
4084         u32 rptr;
4085
4086         if (rdev->wb.enabled)
4087                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4088         else
4089                 rptr = RREG32(CP_RB0_RPTR);
4090
4091         return rptr;
4092 }
4093
4094 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4095                      struct radeon_ring *ring)
4096 {
4097         u32 wptr;
4098
4099         wptr = RREG32(CP_RB0_WPTR);
4100
4101         return wptr;
4102 }
4103
4104 void cik_gfx_set_wptr(struct radeon_device *rdev,
4105                       struct radeon_ring *ring)
4106 {
4107         WREG32(CP_RB0_WPTR, ring->wptr);
4108         (void)RREG32(CP_RB0_WPTR);
4109 }
4110
4111 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4112                          struct radeon_ring *ring)
4113 {
4114         u32 rptr;
4115
4116         if (rdev->wb.enabled) {
4117                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4118         } else {
4119                 mutex_lock(&rdev->srbm_mutex);
4120                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4121                 rptr = RREG32(CP_HQD_PQ_RPTR);
4122                 cik_srbm_select(rdev, 0, 0, 0, 0);
4123                 mutex_unlock(&rdev->srbm_mutex);
4124         }
4125
4126         return rptr;
4127 }
4128
4129 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4130                          struct radeon_ring *ring)
4131 {
4132         u32 wptr;
4133
4134         if (rdev->wb.enabled) {
4135                 /* XXX check if swapping is necessary on BE */
4136                 wptr = rdev->wb.wb[ring->wptr_offs/4];
4137         } else {
4138                 mutex_lock(&rdev->srbm_mutex);
4139                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4140                 wptr = RREG32(CP_HQD_PQ_WPTR);
4141                 cik_srbm_select(rdev, 0, 0, 0, 0);
4142                 mutex_unlock(&rdev->srbm_mutex);
4143         }
4144
4145         return wptr;
4146 }
4147
4148 void cik_compute_set_wptr(struct radeon_device *rdev,
4149                           struct radeon_ring *ring)
4150 {
4151         /* XXX check if swapping is necessary on BE */
4152         rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4153         WDOORBELL32(ring->doorbell_index, ring->wptr);
4154 }
4155
4156 /**
4157  * cik_cp_compute_enable - enable/disable the compute CP MEs
4158  *
4159  * @rdev: radeon_device pointer
4160  * @enable: enable or disable the MEs
4161  *
4162  * Halts or unhalts the compute MEs.
4163  */
4164 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4165 {
4166         if (enable)
4167                 WREG32(CP_MEC_CNTL, 0);
4168         else {
4169                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4170                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4171                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4172         }
4173         udelay(50);
4174 }
4175
4176 /**
4177  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4178  *
4179  * @rdev: radeon_device pointer
4180  *
4181  * Loads the compute MEC1&2 ucode.
4182  * Returns 0 for success, -EINVAL if the ucode is not available.
4183  */
4184 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4185 {
4186         const __be32 *fw_data;
4187         int i;
4188
4189         if (!rdev->mec_fw)
4190                 return -EINVAL;
4191
4192         cik_cp_compute_enable(rdev, false);
4193
4194         /* MEC1 */
4195         fw_data = (const __be32 *)rdev->mec_fw->data;
4196         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4197         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4198                 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4199         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4200
4201         if (rdev->family == CHIP_KAVERI) {
4202                 /* MEC2 */
4203                 fw_data = (const __be32 *)rdev->mec_fw->data;
4204                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4205                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4206                         WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4207                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4208         }
4209
4210         return 0;
4211 }
4212
4213 /**
4214  * cik_cp_compute_start - start the compute queues
4215  *
4216  * @rdev: radeon_device pointer
4217  *
4218  * Enable the compute queues.
4219  * Returns 0 for success, error for failure.
4220  */
4221 static int cik_cp_compute_start(struct radeon_device *rdev)
4222 {
4223         cik_cp_compute_enable(rdev, true);
4224
4225         return 0;
4226 }
4227
4228 /**
4229  * cik_cp_compute_fini - stop the compute queues
4230  *
4231  * @rdev: radeon_device pointer
4232  *
4233  * Stop the compute queues and tear down the driver queue
4234  * info.
4235  */
4236 static void cik_cp_compute_fini(struct radeon_device *rdev)
4237 {
4238         int i, idx, r;
4239
4240         cik_cp_compute_enable(rdev, false);
4241
4242         for (i = 0; i < 2; i++) {
4243                 if (i == 0)
4244                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4245                 else
4246                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4247
4248                 if (rdev->ring[idx].mqd_obj) {
4249                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4250                         if (unlikely(r != 0))
4251                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4252
4253                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4254                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4255
4256                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4257                         rdev->ring[idx].mqd_obj = NULL;
4258                 }
4259         }
4260 }
4261
4262 static void cik_mec_fini(struct radeon_device *rdev)
4263 {
4264         int r;
4265
4266         if (rdev->mec.hpd_eop_obj) {
4267                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4268                 if (unlikely(r != 0))
4269                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4270                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4271                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4272
4273                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4274                 rdev->mec.hpd_eop_obj = NULL;
4275         }
4276 }
4277
4278 #define MEC_HPD_SIZE 2048
4279
4280 static int cik_mec_init(struct radeon_device *rdev)
4281 {
4282         int r;
4283         u32 *hpd;
4284
4285         /*
4286          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4287          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4288          */
4289         if (rdev->family == CHIP_KAVERI)
4290                 rdev->mec.num_mec = 2;
4291         else
4292                 rdev->mec.num_mec = 1;
4293         rdev->mec.num_pipe = 4;
4294         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4295
4296         if (rdev->mec.hpd_eop_obj == NULL) {
4297                 r = radeon_bo_create(rdev,
4298                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4299                                      PAGE_SIZE, true,
4300                                      RADEON_GEM_DOMAIN_GTT, NULL,
4301                                      &rdev->mec.hpd_eop_obj);
4302                 if (r) {
4303                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4304                         return r;
4305                 }
4306         }
4307
4308         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4309         if (unlikely(r != 0)) {
4310                 cik_mec_fini(rdev);
4311                 return r;
4312         }
4313         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4314                           &rdev->mec.hpd_eop_gpu_addr);
4315         if (r) {
4316                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4317                 cik_mec_fini(rdev);
4318                 return r;
4319         }
4320         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4321         if (r) {
4322                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4323                 cik_mec_fini(rdev);
4324                 return r;
4325         }
4326
4327         /* clear memory.  Not sure if this is required or not */
4328         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4329
4330         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4331         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4332
4333         return 0;
4334 }
4335
4336 struct hqd_registers
4337 {
4338         u32 cp_mqd_base_addr;
4339         u32 cp_mqd_base_addr_hi;
4340         u32 cp_hqd_active;
4341         u32 cp_hqd_vmid;
4342         u32 cp_hqd_persistent_state;
4343         u32 cp_hqd_pipe_priority;
4344         u32 cp_hqd_queue_priority;
4345         u32 cp_hqd_quantum;
4346         u32 cp_hqd_pq_base;
4347         u32 cp_hqd_pq_base_hi;
4348         u32 cp_hqd_pq_rptr;
4349         u32 cp_hqd_pq_rptr_report_addr;
4350         u32 cp_hqd_pq_rptr_report_addr_hi;
4351         u32 cp_hqd_pq_wptr_poll_addr;
4352         u32 cp_hqd_pq_wptr_poll_addr_hi;
4353         u32 cp_hqd_pq_doorbell_control;
4354         u32 cp_hqd_pq_wptr;
4355         u32 cp_hqd_pq_control;
4356         u32 cp_hqd_ib_base_addr;
4357         u32 cp_hqd_ib_base_addr_hi;
4358         u32 cp_hqd_ib_rptr;
4359         u32 cp_hqd_ib_control;
4360         u32 cp_hqd_iq_timer;
4361         u32 cp_hqd_iq_rptr;
4362         u32 cp_hqd_dequeue_request;
4363         u32 cp_hqd_dma_offload;
4364         u32 cp_hqd_sema_cmd;
4365         u32 cp_hqd_msg_type;
4366         u32 cp_hqd_atomic0_preop_lo;
4367         u32 cp_hqd_atomic0_preop_hi;
4368         u32 cp_hqd_atomic1_preop_lo;
4369         u32 cp_hqd_atomic1_preop_hi;
4370         u32 cp_hqd_hq_scheduler0;
4371         u32 cp_hqd_hq_scheduler1;
4372         u32 cp_mqd_control;
4373 };
4374
4375 struct bonaire_mqd
4376 {
4377         u32 header;
4378         u32 dispatch_initiator;
4379         u32 dimensions[3];
4380         u32 start_idx[3];
4381         u32 num_threads[3];
4382         u32 pipeline_stat_enable;
4383         u32 perf_counter_enable;
4384         u32 pgm[2];
4385         u32 tba[2];
4386         u32 tma[2];
4387         u32 pgm_rsrc[2];
4388         u32 vmid;
4389         u32 resource_limits;
4390         u32 static_thread_mgmt01[2];
4391         u32 tmp_ring_size;
4392         u32 static_thread_mgmt23[2];
4393         u32 restart[3];
4394         u32 thread_trace_enable;
4395         u32 reserved1;
4396         u32 user_data[16];
4397         u32 vgtcs_invoke_count[2];
4398         struct hqd_registers queue_state;
4399         u32 dequeue_cntr;
4400         u32 interrupt_queue[64];
4401 };
4402
4403 /**
4404  * cik_cp_compute_resume - setup the compute queue registers
4405  *
4406  * @rdev: radeon_device pointer
4407  *
4408  * Program the compute queues and test them to make sure they
4409  * are working.
4410  * Returns 0 for success, error for failure.
4411  */
4412 static int cik_cp_compute_resume(struct radeon_device *rdev)
4413 {
4414         int r, i, idx;
4415         u32 tmp;
4416         bool use_doorbell = true;
4417         u64 hqd_gpu_addr;
4418         u64 mqd_gpu_addr;
4419         u64 eop_gpu_addr;
4420         u64 wb_gpu_addr;
4421         u32 *buf;
4422         struct bonaire_mqd *mqd;
4423
4424         r = cik_cp_compute_start(rdev);
4425         if (r)
4426                 return r;
4427
4428         /* fix up chicken bits */
4429         tmp = RREG32(CP_CPF_DEBUG);
4430         tmp |= (1 << 23);
4431         WREG32(CP_CPF_DEBUG, tmp);
4432
4433         /* init the pipes */
4434         mutex_lock(&rdev->srbm_mutex);
4435         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4436                 int me = (i < 4) ? 1 : 2;
4437                 int pipe = (i < 4) ? i : (i - 4);
4438
4439                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4440
4441                 cik_srbm_select(rdev, me, pipe, 0, 0);
4442
4443                 /* write the EOP addr */
4444                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4445                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4446
4447                 /* set the VMID assigned */
4448                 WREG32(CP_HPD_EOP_VMID, 0);
4449
4450                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4451                 tmp = RREG32(CP_HPD_EOP_CONTROL);
4452                 tmp &= ~EOP_SIZE_MASK;
4453                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4454                 WREG32(CP_HPD_EOP_CONTROL, tmp);
4455         }
4456         cik_srbm_select(rdev, 0, 0, 0, 0);
4457         mutex_unlock(&rdev->srbm_mutex);
4458
4459         /* init the queues.  Just two for now. */
4460         for (i = 0; i < 2; i++) {
4461                 if (i == 0)
4462                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4463                 else
4464                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4465
4466                 if (rdev->ring[idx].mqd_obj == NULL) {
4467                         r = radeon_bo_create(rdev,
4468                                              sizeof(struct bonaire_mqd),
4469                                              PAGE_SIZE, true,
4470                                              RADEON_GEM_DOMAIN_GTT, NULL,
4471                                              &rdev->ring[idx].mqd_obj);
4472                         if (r) {
4473                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4474                                 return r;
4475                         }
4476                 }
4477
4478                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4479                 if (unlikely(r != 0)) {
4480                         cik_cp_compute_fini(rdev);
4481                         return r;
4482                 }
4483                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4484                                   &mqd_gpu_addr);
4485                 if (r) {
4486                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4487                         cik_cp_compute_fini(rdev);
4488                         return r;
4489                 }
4490                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4491                 if (r) {
4492                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4493                         cik_cp_compute_fini(rdev);
4494                         return r;
4495                 }
4496
4497                 /* init the mqd struct */
4498                 memset(buf, 0, sizeof(struct bonaire_mqd));
4499
4500                 mqd = (struct bonaire_mqd *)buf;
4501                 mqd->header = 0xC0310800;
4502                 mqd->static_thread_mgmt01[0] = 0xffffffff;
4503                 mqd->static_thread_mgmt01[1] = 0xffffffff;
4504                 mqd->static_thread_mgmt23[0] = 0xffffffff;
4505                 mqd->static_thread_mgmt23[1] = 0xffffffff;
4506
4507                 mutex_lock(&rdev->srbm_mutex);
4508                 cik_srbm_select(rdev, rdev->ring[idx].me,
4509                                 rdev->ring[idx].pipe,
4510                                 rdev->ring[idx].queue, 0);
4511
4512                 /* disable wptr polling */
4513                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4514                 tmp &= ~WPTR_POLL_EN;
4515                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4516
4517                 /* enable doorbell? */
4518                 mqd->queue_state.cp_hqd_pq_doorbell_control =
4519                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4520                 if (use_doorbell)
4521                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4522                 else
4523                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4524                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4525                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4526
4527                 /* disable the queue if it's active */
4528                 mqd->queue_state.cp_hqd_dequeue_request = 0;
4529                 mqd->queue_state.cp_hqd_pq_rptr = 0;
4530                 mqd->queue_state.cp_hqd_pq_wptr= 0;
4531                 if (RREG32(CP_HQD_ACTIVE) & 1) {
4532                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4533                         for (i = 0; i < rdev->usec_timeout; i++) {
4534                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4535                                         break;
4536                                 udelay(1);
4537                         }
4538                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4539                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4540                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4541                 }
4542
4543                 /* set the pointer to the MQD */
4544                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4545                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4546                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4547                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4548                 /* set MQD vmid to 0 */
4549                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4550                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4551                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4552
4553                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4554                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4555                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4556                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4557                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4558                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4559
4560                 /* set up the HQD, this is similar to CP_RB0_CNTL */
4561                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4562                 mqd->queue_state.cp_hqd_pq_control &=
4563                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4564
4565                 mqd->queue_state.cp_hqd_pq_control |=
4566                         order_base_2(rdev->ring[idx].ring_size / 8);
4567                 mqd->queue_state.cp_hqd_pq_control |=
4568                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4569 #ifdef __BIG_ENDIAN
4570                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4571 #endif
4572                 mqd->queue_state.cp_hqd_pq_control &=
4573                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4574                 mqd->queue_state.cp_hqd_pq_control |=
4575                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4576                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4577
4578                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4579                 if (i == 0)
4580                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4581                 else
4582                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4583                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4584                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4585                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4586                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4587                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4588
4589                 /* set the wb address wether it's enabled or not */
4590                 if (i == 0)
4591                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4592                 else
4593                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4594                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4595                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4596                         upper_32_bits(wb_gpu_addr) & 0xffff;
4597                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4598                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4599                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4600                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4601
4602                 /* enable the doorbell if requested */
4603                 if (use_doorbell) {
4604                         mqd->queue_state.cp_hqd_pq_doorbell_control =
4605                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4606                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4607                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
4608                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4609                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4610                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
4611                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4612
4613                 } else {
4614                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4615                 }
4616                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4617                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4618
4619                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4620                 rdev->ring[idx].wptr = 0;
4621                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4622                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4623                 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4624
4625                 /* set the vmid for the queue */
4626                 mqd->queue_state.cp_hqd_vmid = 0;
4627                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4628
4629                 /* activate the queue */
4630                 mqd->queue_state.cp_hqd_active = 1;
4631                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4632
4633                 cik_srbm_select(rdev, 0, 0, 0, 0);
4634                 mutex_unlock(&rdev->srbm_mutex);
4635
4636                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4637                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4638
4639                 rdev->ring[idx].ready = true;
4640                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4641                 if (r)
4642                         rdev->ring[idx].ready = false;
4643         }
4644
4645         return 0;
4646 }
4647
4648 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4649 {
4650         cik_cp_gfx_enable(rdev, enable);
4651         cik_cp_compute_enable(rdev, enable);
4652 }
4653
4654 static int cik_cp_load_microcode(struct radeon_device *rdev)
4655 {
4656         int r;
4657
4658         r = cik_cp_gfx_load_microcode(rdev);
4659         if (r)
4660                 return r;
4661         r = cik_cp_compute_load_microcode(rdev);
4662         if (r)
4663                 return r;
4664
4665         return 0;
4666 }
4667
4668 static void cik_cp_fini(struct radeon_device *rdev)
4669 {
4670         cik_cp_gfx_fini(rdev);
4671         cik_cp_compute_fini(rdev);
4672 }
4673
4674 static int cik_cp_resume(struct radeon_device *rdev)
4675 {
4676         int r;
4677
4678         cik_enable_gui_idle_interrupt(rdev, false);
4679
4680         r = cik_cp_load_microcode(rdev);
4681         if (r)
4682                 return r;
4683
4684         r = cik_cp_gfx_resume(rdev);
4685         if (r)
4686                 return r;
4687         r = cik_cp_compute_resume(rdev);
4688         if (r)
4689                 return r;
4690
4691         cik_enable_gui_idle_interrupt(rdev, true);
4692
4693         return 0;
4694 }
4695
4696 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4697 {
4698         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4699                 RREG32(GRBM_STATUS));
4700         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4701                 RREG32(GRBM_STATUS2));
4702         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4703                 RREG32(GRBM_STATUS_SE0));
4704         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4705                 RREG32(GRBM_STATUS_SE1));
4706         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4707                 RREG32(GRBM_STATUS_SE2));
4708         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4709                 RREG32(GRBM_STATUS_SE3));
4710         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4711                 RREG32(SRBM_STATUS));
4712         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4713                 RREG32(SRBM_STATUS2));
4714         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4715                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4716         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4717                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4718         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4719         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4720                  RREG32(CP_STALLED_STAT1));
4721         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4722                  RREG32(CP_STALLED_STAT2));
4723         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4724                  RREG32(CP_STALLED_STAT3));
4725         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4726                  RREG32(CP_CPF_BUSY_STAT));
4727         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4728                  RREG32(CP_CPF_STALLED_STAT1));
4729         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4730         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4731         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4732                  RREG32(CP_CPC_STALLED_STAT1));
4733         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4734 }
4735
4736 /**
4737  * cik_gpu_check_soft_reset - check which blocks are busy
4738  *
4739  * @rdev: radeon_device pointer
4740  *
4741  * Check which blocks are busy and return the relevant reset
4742  * mask to be used by cik_gpu_soft_reset().
4743  * Returns a mask of the blocks to be reset.
4744  */
4745 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4746 {
4747         u32 reset_mask = 0;
4748         u32 tmp;
4749
4750         /* GRBM_STATUS */
4751         tmp = RREG32(GRBM_STATUS);
4752         if (tmp & (PA_BUSY | SC_BUSY |
4753                    BCI_BUSY | SX_BUSY |
4754                    TA_BUSY | VGT_BUSY |
4755                    DB_BUSY | CB_BUSY |
4756                    GDS_BUSY | SPI_BUSY |
4757                    IA_BUSY | IA_BUSY_NO_DMA))
4758                 reset_mask |= RADEON_RESET_GFX;
4759
4760         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4761                 reset_mask |= RADEON_RESET_CP;
4762
4763         /* GRBM_STATUS2 */
4764         tmp = RREG32(GRBM_STATUS2);
4765         if (tmp & RLC_BUSY)
4766                 reset_mask |= RADEON_RESET_RLC;
4767
4768         /* SDMA0_STATUS_REG */
4769         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4770         if (!(tmp & SDMA_IDLE))
4771                 reset_mask |= RADEON_RESET_DMA;
4772
4773         /* SDMA1_STATUS_REG */
4774         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4775         if (!(tmp & SDMA_IDLE))
4776                 reset_mask |= RADEON_RESET_DMA1;
4777
4778         /* SRBM_STATUS2 */
4779         tmp = RREG32(SRBM_STATUS2);
4780         if (tmp & SDMA_BUSY)
4781                 reset_mask |= RADEON_RESET_DMA;
4782
4783         if (tmp & SDMA1_BUSY)
4784                 reset_mask |= RADEON_RESET_DMA1;
4785
4786         /* SRBM_STATUS */
4787         tmp = RREG32(SRBM_STATUS);
4788
4789         if (tmp & IH_BUSY)
4790                 reset_mask |= RADEON_RESET_IH;
4791
4792         if (tmp & SEM_BUSY)
4793                 reset_mask |= RADEON_RESET_SEM;
4794
4795         if (tmp & GRBM_RQ_PENDING)
4796                 reset_mask |= RADEON_RESET_GRBM;
4797
4798         if (tmp & VMC_BUSY)
4799                 reset_mask |= RADEON_RESET_VMC;
4800
4801         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4802                    MCC_BUSY | MCD_BUSY))
4803                 reset_mask |= RADEON_RESET_MC;
4804
4805         if (evergreen_is_display_hung(rdev))
4806                 reset_mask |= RADEON_RESET_DISPLAY;
4807
4808         /* Skip MC reset as it's mostly likely not hung, just busy */
4809         if (reset_mask & RADEON_RESET_MC) {
4810                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4811                 reset_mask &= ~RADEON_RESET_MC;
4812         }
4813
4814         return reset_mask;
4815 }
4816
4817 /**
4818  * cik_gpu_soft_reset - soft reset GPU
4819  *
4820  * @rdev: radeon_device pointer
4821  * @reset_mask: mask of which blocks to reset
4822  *
4823  * Soft reset the blocks specified in @reset_mask.
4824  */
4825 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4826 {
4827         struct evergreen_mc_save save;
4828         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4829         u32 tmp;
4830
4831         if (reset_mask == 0)
4832                 return;
4833
4834         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4835
4836         cik_print_gpu_status_regs(rdev);
4837         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4838                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4839         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4840                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4841
4842         /* disable CG/PG */
4843         cik_fini_pg(rdev);
4844         cik_fini_cg(rdev);
4845
4846         /* stop the rlc */
4847         cik_rlc_stop(rdev);
4848
4849         /* Disable GFX parsing/prefetching */
4850         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4851
4852         /* Disable MEC parsing/prefetching */
4853         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4854
4855         if (reset_mask & RADEON_RESET_DMA) {
4856                 /* sdma0 */
4857                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4858                 tmp |= SDMA_HALT;
4859                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4860         }
4861         if (reset_mask & RADEON_RESET_DMA1) {
4862                 /* sdma1 */
4863                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4864                 tmp |= SDMA_HALT;
4865                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4866         }
4867
4868         evergreen_mc_stop(rdev, &save);
4869         if (evergreen_mc_wait_for_idle(rdev)) {
4870                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4871         }
4872
4873         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4874                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4875
4876         if (reset_mask & RADEON_RESET_CP) {
4877                 grbm_soft_reset |= SOFT_RESET_CP;
4878
4879                 srbm_soft_reset |= SOFT_RESET_GRBM;
4880         }
4881
4882         if (reset_mask & RADEON_RESET_DMA)
4883                 srbm_soft_reset |= SOFT_RESET_SDMA;
4884
4885         if (reset_mask & RADEON_RESET_DMA1)
4886                 srbm_soft_reset |= SOFT_RESET_SDMA1;
4887
4888         if (reset_mask & RADEON_RESET_DISPLAY)
4889                 srbm_soft_reset |= SOFT_RESET_DC;
4890
4891         if (reset_mask & RADEON_RESET_RLC)
4892                 grbm_soft_reset |= SOFT_RESET_RLC;
4893
4894         if (reset_mask & RADEON_RESET_SEM)
4895                 srbm_soft_reset |= SOFT_RESET_SEM;
4896
4897         if (reset_mask & RADEON_RESET_IH)
4898                 srbm_soft_reset |= SOFT_RESET_IH;
4899
4900         if (reset_mask & RADEON_RESET_GRBM)
4901                 srbm_soft_reset |= SOFT_RESET_GRBM;
4902
4903         if (reset_mask & RADEON_RESET_VMC)
4904                 srbm_soft_reset |= SOFT_RESET_VMC;
4905
4906         if (!(rdev->flags & RADEON_IS_IGP)) {
4907                 if (reset_mask & RADEON_RESET_MC)
4908                         srbm_soft_reset |= SOFT_RESET_MC;
4909         }
4910
4911         if (grbm_soft_reset) {
4912                 tmp = RREG32(GRBM_SOFT_RESET);
4913                 tmp |= grbm_soft_reset;
4914                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4915                 WREG32(GRBM_SOFT_RESET, tmp);
4916                 tmp = RREG32(GRBM_SOFT_RESET);
4917
4918                 udelay(50);
4919
4920                 tmp &= ~grbm_soft_reset;
4921                 WREG32(GRBM_SOFT_RESET, tmp);
4922                 tmp = RREG32(GRBM_SOFT_RESET);
4923         }
4924
4925         if (srbm_soft_reset) {
4926                 tmp = RREG32(SRBM_SOFT_RESET);
4927                 tmp |= srbm_soft_reset;
4928                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4929                 WREG32(SRBM_SOFT_RESET, tmp);
4930                 tmp = RREG32(SRBM_SOFT_RESET);
4931
4932                 udelay(50);
4933
4934                 tmp &= ~srbm_soft_reset;
4935                 WREG32(SRBM_SOFT_RESET, tmp);
4936                 tmp = RREG32(SRBM_SOFT_RESET);
4937         }
4938
4939         /* Wait a little for things to settle down */
4940         udelay(50);
4941
4942         evergreen_mc_resume(rdev, &save);
4943         udelay(50);
4944
4945         cik_print_gpu_status_regs(rdev);
4946 }
4947
4948 struct kv_reset_save_regs {
4949         u32 gmcon_reng_execute;
4950         u32 gmcon_misc;
4951         u32 gmcon_misc3;
4952 };
4953
4954 static void kv_save_regs_for_reset(struct radeon_device *rdev,
4955                                    struct kv_reset_save_regs *save)
4956 {
4957         save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4958         save->gmcon_misc = RREG32(GMCON_MISC);
4959         save->gmcon_misc3 = RREG32(GMCON_MISC3);
4960
4961         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4962         WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4963                                                 STCTRL_STUTTER_EN));
4964 }
4965
4966 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4967                                       struct kv_reset_save_regs *save)
4968 {
4969         int i;
4970
4971         WREG32(GMCON_PGFSM_WRITE, 0);
4972         WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4973
4974         for (i = 0; i < 5; i++)
4975                 WREG32(GMCON_PGFSM_WRITE, 0);
4976
4977         WREG32(GMCON_PGFSM_WRITE, 0);
4978         WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4979
4980         for (i = 0; i < 5; i++)
4981                 WREG32(GMCON_PGFSM_WRITE, 0);
4982
4983         WREG32(GMCON_PGFSM_WRITE, 0x210000);
4984         WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4985
4986         for (i = 0; i < 5; i++)
4987                 WREG32(GMCON_PGFSM_WRITE, 0);
4988
4989         WREG32(GMCON_PGFSM_WRITE, 0x21003);
4990         WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4991
4992         for (i = 0; i < 5; i++)
4993                 WREG32(GMCON_PGFSM_WRITE, 0);
4994
4995         WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4996         WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4997
4998         for (i = 0; i < 5; i++)
4999                 WREG32(GMCON_PGFSM_WRITE, 0);
5000
5001         WREG32(GMCON_PGFSM_WRITE, 0);
5002         WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5003
5004         for (i = 0; i < 5; i++)
5005                 WREG32(GMCON_PGFSM_WRITE, 0);
5006
5007         WREG32(GMCON_PGFSM_WRITE, 0x420000);
5008         WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5009
5010         for (i = 0; i < 5; i++)
5011                 WREG32(GMCON_PGFSM_WRITE, 0);
5012
5013         WREG32(GMCON_PGFSM_WRITE, 0x120202);
5014         WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5015
5016         for (i = 0; i < 5; i++)
5017                 WREG32(GMCON_PGFSM_WRITE, 0);
5018
5019         WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5020         WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5021
5022         for (i = 0; i < 5; i++)
5023                 WREG32(GMCON_PGFSM_WRITE, 0);
5024
5025         WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5026         WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5027
5028         for (i = 0; i < 5; i++)
5029                 WREG32(GMCON_PGFSM_WRITE, 0);
5030
5031         WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5032         WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5033
5034         WREG32(GMCON_MISC3, save->gmcon_misc3);
5035         WREG32(GMCON_MISC, save->gmcon_misc);
5036         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5037 }
5038
5039 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5040 {
5041         struct evergreen_mc_save save;
5042         struct kv_reset_save_regs kv_save = { 0 };
5043         u32 tmp, i;
5044
5045         dev_info(rdev->dev, "GPU pci config reset\n");
5046
5047         /* disable dpm? */
5048
5049         /* disable cg/pg */
5050         cik_fini_pg(rdev);
5051         cik_fini_cg(rdev);
5052
5053         /* Disable GFX parsing/prefetching */
5054         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5055
5056         /* Disable MEC parsing/prefetching */
5057         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5058
5059         /* sdma0 */
5060         tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5061         tmp |= SDMA_HALT;
5062         WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5063         /* sdma1 */
5064         tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5065         tmp |= SDMA_HALT;
5066         WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5067         /* XXX other engines? */
5068
5069         /* halt the rlc, disable cp internal ints */
5070         cik_rlc_stop(rdev);
5071
5072         udelay(50);
5073
5074         /* disable mem access */
5075         evergreen_mc_stop(rdev, &save);
5076         if (evergreen_mc_wait_for_idle(rdev)) {
5077                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5078         }
5079
5080         if (rdev->flags & RADEON_IS_IGP)
5081                 kv_save_regs_for_reset(rdev, &kv_save);
5082
5083         /* disable BM */
5084         pci_clear_master(rdev->pdev);
5085         /* reset */
5086         radeon_pci_config_reset(rdev);
5087
5088         udelay(100);
5089
5090         /* wait for asic to come out of reset */
5091         for (i = 0; i < rdev->usec_timeout; i++) {
5092                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5093                         break;
5094                 udelay(1);
5095         }
5096
5097         /* does asic init need to be run first??? */
5098         if (rdev->flags & RADEON_IS_IGP)
5099                 kv_restore_regs_for_reset(rdev, &kv_save);
5100 }
5101
5102 /**
5103  * cik_asic_reset - soft reset GPU
5104  *
5105  * @rdev: radeon_device pointer
5106  *
5107  * Look up which blocks are hung and attempt
5108  * to reset them.
5109  * Returns 0 for success.
5110  */
5111 int cik_asic_reset(struct radeon_device *rdev)
5112 {
5113         u32 reset_mask;
5114
5115         reset_mask = cik_gpu_check_soft_reset(rdev);
5116
5117         if (reset_mask)
5118                 r600_set_bios_scratch_engine_hung(rdev, true);
5119
5120         /* try soft reset */
5121         cik_gpu_soft_reset(rdev, reset_mask);
5122
5123         reset_mask = cik_gpu_check_soft_reset(rdev);
5124
5125         /* try pci config reset */
5126         if (reset_mask && radeon_hard_reset)
5127                 cik_gpu_pci_config_reset(rdev);
5128
5129         reset_mask = cik_gpu_check_soft_reset(rdev);
5130
5131         if (!reset_mask)
5132                 r600_set_bios_scratch_engine_hung(rdev, false);
5133
5134         return 0;
5135 }
5136
5137 /**
5138  * cik_gfx_is_lockup - check if the 3D engine is locked up
5139  *
5140  * @rdev: radeon_device pointer
5141  * @ring: radeon_ring structure holding ring information
5142  *
5143  * Check if the 3D engine is locked up (CIK).
5144  * Returns true if the engine is locked, false if not.
5145  */
5146 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5147 {
5148         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5149
5150         if (!(reset_mask & (RADEON_RESET_GFX |
5151                             RADEON_RESET_COMPUTE |
5152                             RADEON_RESET_CP))) {
5153                 radeon_ring_lockup_update(rdev, ring);
5154                 return false;
5155         }
5156         return radeon_ring_test_lockup(rdev, ring);
5157 }
5158
5159 /* MC */
5160 /**
5161  * cik_mc_program - program the GPU memory controller
5162  *
5163  * @rdev: radeon_device pointer
5164  *
5165  * Set the location of vram, gart, and AGP in the GPU's
5166  * physical address space (CIK).
5167  */
5168 static void cik_mc_program(struct radeon_device *rdev)
5169 {
5170         struct evergreen_mc_save save;
5171         u32 tmp;
5172         int i, j;
5173
5174         /* Initialize HDP */
5175         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5176                 WREG32((0x2c14 + j), 0x00000000);
5177                 WREG32((0x2c18 + j), 0x00000000);
5178                 WREG32((0x2c1c + j), 0x00000000);
5179                 WREG32((0x2c20 + j), 0x00000000);
5180                 WREG32((0x2c24 + j), 0x00000000);
5181         }
5182         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5183
5184         evergreen_mc_stop(rdev, &save);
5185         if (radeon_mc_wait_for_idle(rdev)) {
5186                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5187         }
5188         /* Lockout access through VGA aperture*/
5189         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5190         /* Update configuration */
5191         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5192                rdev->mc.vram_start >> 12);
5193         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5194                rdev->mc.vram_end >> 12);
5195         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5196                rdev->vram_scratch.gpu_addr >> 12);
5197         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5198         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5199         WREG32(MC_VM_FB_LOCATION, tmp);
5200         /* XXX double check these! */
5201         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5202         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5203         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5204         WREG32(MC_VM_AGP_BASE, 0);
5205         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5206         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5207         if (radeon_mc_wait_for_idle(rdev)) {
5208                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5209         }
5210         evergreen_mc_resume(rdev, &save);
5211         /* we need to own VRAM, so turn off the VGA renderer here
5212          * to stop it overwriting our objects */
5213         rv515_vga_render_disable(rdev);
5214 }
5215
5216 /**
5217  * cik_mc_init - initialize the memory controller driver params
5218  *
5219  * @rdev: radeon_device pointer
5220  *
5221  * Look up the amount of vram, vram width, and decide how to place
5222  * vram and gart within the GPU's physical address space (CIK).
5223  * Returns 0 for success.
5224  */
5225 static int cik_mc_init(struct radeon_device *rdev)
5226 {
5227         u32 tmp;
5228         int chansize, numchan;
5229
5230         /* Get VRAM informations */
5231         rdev->mc.vram_is_ddr = true;
5232         tmp = RREG32(MC_ARB_RAMCFG);
5233         if (tmp & CHANSIZE_MASK) {
5234                 chansize = 64;
5235         } else {
5236                 chansize = 32;
5237         }
5238         tmp = RREG32(MC_SHARED_CHMAP);
5239         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5240         case 0:
5241         default:
5242                 numchan = 1;
5243                 break;
5244         case 1:
5245                 numchan = 2;
5246                 break;
5247         case 2:
5248                 numchan = 4;
5249                 break;
5250         case 3:
5251                 numchan = 8;
5252                 break;
5253         case 4:
5254                 numchan = 3;
5255                 break;
5256         case 5:
5257                 numchan = 6;
5258                 break;
5259         case 6:
5260                 numchan = 10;
5261                 break;
5262         case 7:
5263                 numchan = 12;
5264                 break;
5265         case 8:
5266                 numchan = 16;
5267                 break;
5268         }
5269         rdev->mc.vram_width = numchan * chansize;
5270         /* Could aper size report 0 ? */
5271         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5272         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5273         /* size in MB on si */
5274         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5275         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5276         rdev->mc.visible_vram_size = rdev->mc.aper_size;
5277         si_vram_gtt_location(rdev, &rdev->mc);
5278         radeon_update_bandwidth_info(rdev);
5279
5280         return 0;
5281 }
5282
5283 /*
5284  * GART
5285  * VMID 0 is the physical GPU addresses as used by the kernel.
5286  * VMIDs 1-15 are used for userspace clients and are handled
5287  * by the radeon vm/hsa code.
5288  */
5289 /**
5290  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5291  *
5292  * @rdev: radeon_device pointer
5293  *
5294  * Flush the TLB for the VMID 0 page table (CIK).
5295  */
5296 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5297 {
5298         /* flush hdp cache */
5299         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5300
5301         /* bits 0-15 are the VM contexts0-15 */
5302         WREG32(VM_INVALIDATE_REQUEST, 0x1);
5303 }
5304
5305 /**
5306  * cik_pcie_gart_enable - gart enable
5307  *
5308  * @rdev: radeon_device pointer
5309  *
5310  * This sets up the TLBs, programs the page tables for VMID0,
5311  * sets up the hw for VMIDs 1-15 which are allocated on
5312  * demand, and sets up the global locations for the LDS, GDS,
5313  * and GPUVM for FSA64 clients (CIK).
5314  * Returns 0 for success, errors for failure.
5315  */
5316 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5317 {
5318         int r, i;
5319
5320         if (rdev->gart.robj == NULL) {
5321                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5322                 return -EINVAL;
5323         }
5324         r = radeon_gart_table_vram_pin(rdev);
5325         if (r)
5326                 return r;
5327         radeon_gart_restore(rdev);
5328         /* Setup TLB control */
5329         WREG32(MC_VM_MX_L1_TLB_CNTL,
5330                (0xA << 7) |
5331                ENABLE_L1_TLB |
5332                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5333                ENABLE_ADVANCED_DRIVER_MODEL |
5334                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5335         /* Setup L2 cache */
5336         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5337                ENABLE_L2_FRAGMENT_PROCESSING |
5338                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5339                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5340                EFFECTIVE_L2_QUEUE_SIZE(7) |
5341                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5342         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5343         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5344                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5345         /* setup context0 */
5346         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5347         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5348         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5349         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5350                         (u32)(rdev->dummy_page.addr >> 12));
5351         WREG32(VM_CONTEXT0_CNTL2, 0);
5352         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5353                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5354
5355         WREG32(0x15D4, 0);
5356         WREG32(0x15D8, 0);
5357         WREG32(0x15DC, 0);
5358
5359         /* empty context1-15 */
5360         /* FIXME start with 4G, once using 2 level pt switch to full
5361          * vm size space
5362          */
5363         /* set vm size, must be a multiple of 4 */
5364         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5365         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5366         for (i = 1; i < 16; i++) {
5367                 if (i < 8)
5368                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5369                                rdev->gart.table_addr >> 12);
5370                 else
5371                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5372                                rdev->gart.table_addr >> 12);
5373         }
5374
5375         /* enable context1-15 */
5376         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5377                (u32)(rdev->dummy_page.addr >> 12));
5378         WREG32(VM_CONTEXT1_CNTL2, 4);
5379         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5380                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5381                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5382                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5383                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5384                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5385                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5386                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5387                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5388                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5389                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5390                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5391                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5392
5393         if (rdev->family == CHIP_KAVERI) {
5394                 u32 tmp = RREG32(CHUB_CONTROL);
5395                 tmp &= ~BYPASS_VM;
5396                 WREG32(CHUB_CONTROL, tmp);
5397         }
5398
5399         /* XXX SH_MEM regs */
5400         /* where to put LDS, scratch, GPUVM in FSA64 space */
5401         mutex_lock(&rdev->srbm_mutex);
5402         for (i = 0; i < 16; i++) {
5403                 cik_srbm_select(rdev, 0, 0, 0, i);
5404                 /* CP and shaders */
5405                 WREG32(SH_MEM_CONFIG, 0);
5406                 WREG32(SH_MEM_APE1_BASE, 1);
5407                 WREG32(SH_MEM_APE1_LIMIT, 0);
5408                 WREG32(SH_MEM_BASES, 0);
5409                 /* SDMA GFX */
5410                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5411                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5412                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5413                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5414                 /* XXX SDMA RLC - todo */
5415         }
5416         cik_srbm_select(rdev, 0, 0, 0, 0);
5417         mutex_unlock(&rdev->srbm_mutex);
5418
5419         cik_pcie_gart_tlb_flush(rdev);
5420         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5421                  (unsigned)(rdev->mc.gtt_size >> 20),
5422                  (unsigned long long)rdev->gart.table_addr);
5423         rdev->gart.ready = true;
5424         return 0;
5425 }
5426
5427 /**
5428  * cik_pcie_gart_disable - gart disable
5429  *
5430  * @rdev: radeon_device pointer
5431  *
5432  * This disables all VM page table (CIK).
5433  */
5434 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5435 {
5436         /* Disable all tables */
5437         WREG32(VM_CONTEXT0_CNTL, 0);
5438         WREG32(VM_CONTEXT1_CNTL, 0);
5439         /* Setup TLB control */
5440         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5441                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5442         /* Setup L2 cache */
5443         WREG32(VM_L2_CNTL,
5444                ENABLE_L2_FRAGMENT_PROCESSING |
5445                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5446                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5447                EFFECTIVE_L2_QUEUE_SIZE(7) |
5448                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5449         WREG32(VM_L2_CNTL2, 0);
5450         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5451                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5452         radeon_gart_table_vram_unpin(rdev);
5453 }
5454
5455 /**
5456  * cik_pcie_gart_fini - vm fini callback
5457  *
5458  * @rdev: radeon_device pointer
5459  *
5460  * Tears down the driver GART/VM setup (CIK).
5461  */
5462 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5463 {
5464         cik_pcie_gart_disable(rdev);
5465         radeon_gart_table_vram_free(rdev);
5466         radeon_gart_fini(rdev);
5467 }
5468
5469 /* vm parser */
5470 /**
5471  * cik_ib_parse - vm ib_parse callback
5472  *
5473  * @rdev: radeon_device pointer
5474  * @ib: indirect buffer pointer
5475  *
5476  * CIK uses hw IB checking so this is a nop (CIK).
5477  */
5478 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5479 {
5480         return 0;
5481 }
5482
5483 /*
5484  * vm
5485  * VMID 0 is the physical GPU addresses as used by the kernel.
5486  * VMIDs 1-15 are used for userspace clients and are handled
5487  * by the radeon vm/hsa code.
5488  */
5489 /**
5490  * cik_vm_init - cik vm init callback
5491  *
5492  * @rdev: radeon_device pointer
5493  *
5494  * Inits cik specific vm parameters (number of VMs, base of vram for
5495  * VMIDs 1-15) (CIK).
5496  * Returns 0 for success.
5497  */
5498 int cik_vm_init(struct radeon_device *rdev)
5499 {
5500         /* number of VMs */
5501         rdev->vm_manager.nvm = 16;
5502         /* base offset of vram pages */
5503         if (rdev->flags & RADEON_IS_IGP) {
5504                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5505                 tmp <<= 22;
5506                 rdev->vm_manager.vram_base_offset = tmp;
5507         } else
5508                 rdev->vm_manager.vram_base_offset = 0;
5509
5510         return 0;
5511 }
5512
5513 /**
5514  * cik_vm_fini - cik vm fini callback
5515  *
5516  * @rdev: radeon_device pointer
5517  *
5518  * Tear down any asic specific VM setup (CIK).
5519  */
5520 void cik_vm_fini(struct radeon_device *rdev)
5521 {
5522 }
5523
5524 /**
5525  * cik_vm_decode_fault - print human readable fault info
5526  *
5527  * @rdev: radeon_device pointer
5528  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5529  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5530  *
5531  * Print human readable fault information (CIK).
5532  */
5533 static void cik_vm_decode_fault(struct radeon_device *rdev,
5534                                 u32 status, u32 addr, u32 mc_client)
5535 {
5536         u32 mc_id;
5537         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5538         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5539         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5540                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5541
5542         if (rdev->family == CHIP_HAWAII)
5543                 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5544         else
5545                 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5546
5547         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5548                protections, vmid, addr,
5549                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5550                block, mc_client, mc_id);
5551 }
5552
5553 /**
5554  * cik_vm_flush - cik vm flush using the CP
5555  *
5556  * @rdev: radeon_device pointer
5557  *
5558  * Update the page table base and flush the VM TLB
5559  * using the CP (CIK).
5560  */
5561 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5562 {
5563         struct radeon_ring *ring = &rdev->ring[ridx];
5564
5565         if (vm == NULL)
5566                 return;
5567
5568         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5569         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5570                                  WRITE_DATA_DST_SEL(0)));
5571         if (vm->id < 8) {
5572                 radeon_ring_write(ring,
5573                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5574         } else {
5575                 radeon_ring_write(ring,
5576                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5577         }
5578         radeon_ring_write(ring, 0);
5579         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5580
5581         /* update SH_MEM_* regs */
5582         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5583         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5584                                  WRITE_DATA_DST_SEL(0)));
5585         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5586         radeon_ring_write(ring, 0);
5587         radeon_ring_write(ring, VMID(vm->id));
5588
5589         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5590         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5591                                  WRITE_DATA_DST_SEL(0)));
5592         radeon_ring_write(ring, SH_MEM_BASES >> 2);
5593         radeon_ring_write(ring, 0);
5594
5595         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5596         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5597         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5598         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5599
5600         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5601         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5602                                  WRITE_DATA_DST_SEL(0)));
5603         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5604         radeon_ring_write(ring, 0);
5605         radeon_ring_write(ring, VMID(0));
5606
5607         /* HDP flush */
5608         cik_hdp_flush_cp_ring_emit(rdev, ridx);
5609
5610         /* bits 0-15 are the VM contexts0-15 */
5611         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5612         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5613                                  WRITE_DATA_DST_SEL(0)));
5614         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5615         radeon_ring_write(ring, 0);
5616         radeon_ring_write(ring, 1 << vm->id);
5617
5618         /* compute doesn't have PFP */
5619         if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5620                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5621                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5622                 radeon_ring_write(ring, 0x0);
5623         }
5624 }
5625
5626 /*
5627  * RLC
5628  * The RLC is a multi-purpose microengine that handles a
5629  * variety of functions, the most important of which is
5630  * the interrupt controller.
5631  */
5632 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5633                                           bool enable)
5634 {
5635         u32 tmp = RREG32(CP_INT_CNTL_RING0);
5636
5637         if (enable)
5638                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5639         else
5640                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5641         WREG32(CP_INT_CNTL_RING0, tmp);
5642 }
5643
5644 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5645 {
5646         u32 tmp;
5647
5648         tmp = RREG32(RLC_LB_CNTL);
5649         if (enable)
5650                 tmp |= LOAD_BALANCE_ENABLE;
5651         else
5652                 tmp &= ~LOAD_BALANCE_ENABLE;
5653         WREG32(RLC_LB_CNTL, tmp);
5654 }
5655
5656 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5657 {
5658         u32 i, j, k;
5659         u32 mask;
5660
5661         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5662                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5663                         cik_select_se_sh(rdev, i, j);
5664                         for (k = 0; k < rdev->usec_timeout; k++) {
5665                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5666                                         break;
5667                                 udelay(1);
5668                         }
5669                 }
5670         }
5671         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5672
5673         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5674         for (k = 0; k < rdev->usec_timeout; k++) {
5675                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5676                         break;
5677                 udelay(1);
5678         }
5679 }
5680
5681 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5682 {
5683         u32 tmp;
5684
5685         tmp = RREG32(RLC_CNTL);
5686         if (tmp != rlc)
5687                 WREG32(RLC_CNTL, rlc);
5688 }
5689
5690 static u32 cik_halt_rlc(struct radeon_device *rdev)
5691 {
5692         u32 data, orig;
5693
5694         orig = data = RREG32(RLC_CNTL);
5695
5696         if (data & RLC_ENABLE) {
5697                 u32 i;
5698
5699                 data &= ~RLC_ENABLE;
5700                 WREG32(RLC_CNTL, data);
5701
5702                 for (i = 0; i < rdev->usec_timeout; i++) {
5703                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5704                                 break;
5705                         udelay(1);
5706                 }
5707
5708                 cik_wait_for_rlc_serdes(rdev);
5709         }
5710
5711         return orig;
5712 }
5713
5714 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5715 {
5716         u32 tmp, i, mask;
5717
5718         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5719         WREG32(RLC_GPR_REG2, tmp);
5720
5721         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5722         for (i = 0; i < rdev->usec_timeout; i++) {
5723                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5724                         break;
5725                 udelay(1);
5726         }
5727
5728         for (i = 0; i < rdev->usec_timeout; i++) {
5729                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5730                         break;
5731                 udelay(1);
5732         }
5733 }
5734
5735 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5736 {
5737         u32 tmp;
5738
5739         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5740         WREG32(RLC_GPR_REG2, tmp);
5741 }
5742
5743 /**
5744  * cik_rlc_stop - stop the RLC ME
5745  *
5746  * @rdev: radeon_device pointer
5747  *
5748  * Halt the RLC ME (MicroEngine) (CIK).
5749  */
5750 static void cik_rlc_stop(struct radeon_device *rdev)
5751 {
5752         WREG32(RLC_CNTL, 0);
5753
5754         cik_enable_gui_idle_interrupt(rdev, false);
5755
5756         cik_wait_for_rlc_serdes(rdev);
5757 }
5758
5759 /**
5760  * cik_rlc_start - start the RLC ME
5761  *
5762  * @rdev: radeon_device pointer
5763  *
5764  * Unhalt the RLC ME (MicroEngine) (CIK).
5765  */
5766 static void cik_rlc_start(struct radeon_device *rdev)
5767 {
5768         WREG32(RLC_CNTL, RLC_ENABLE);
5769
5770         cik_enable_gui_idle_interrupt(rdev, true);
5771
5772         udelay(50);
5773 }
5774
5775 /**
5776  * cik_rlc_resume - setup the RLC hw
5777  *
5778  * @rdev: radeon_device pointer
5779  *
5780  * Initialize the RLC registers, load the ucode,
5781  * and start the RLC (CIK).
5782  * Returns 0 for success, -EINVAL if the ucode is not available.
5783  */
5784 static int cik_rlc_resume(struct radeon_device *rdev)
5785 {
5786         u32 i, size, tmp;
5787         const __be32 *fw_data;
5788
5789         if (!rdev->rlc_fw)
5790                 return -EINVAL;
5791
5792         switch (rdev->family) {
5793         case CHIP_BONAIRE:
5794         case CHIP_HAWAII:
5795         default:
5796                 size = BONAIRE_RLC_UCODE_SIZE;
5797                 break;
5798         case CHIP_KAVERI:
5799                 size = KV_RLC_UCODE_SIZE;
5800                 break;
5801         case CHIP_KABINI:
5802                 size = KB_RLC_UCODE_SIZE;
5803                 break;
5804         }
5805
5806         cik_rlc_stop(rdev);
5807
5808         /* disable CG */
5809         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5810         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5811
5812         si_rlc_reset(rdev);
5813
5814         cik_init_pg(rdev);
5815
5816         cik_init_cg(rdev);
5817
5818         WREG32(RLC_LB_CNTR_INIT, 0);
5819         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5820
5821         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5822         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5823         WREG32(RLC_LB_PARAMS, 0x00600408);
5824         WREG32(RLC_LB_CNTL, 0x80000004);
5825
5826         WREG32(RLC_MC_CNTL, 0);
5827         WREG32(RLC_UCODE_CNTL, 0);
5828
5829         fw_data = (const __be32 *)rdev->rlc_fw->data;
5830                 WREG32(RLC_GPM_UCODE_ADDR, 0);
5831         for (i = 0; i < size; i++)
5832                 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5833         WREG32(RLC_GPM_UCODE_ADDR, 0);
5834
5835         /* XXX - find out what chips support lbpw */
5836         cik_enable_lbpw(rdev, false);
5837
5838         if (rdev->family == CHIP_BONAIRE)
5839                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5840
5841         cik_rlc_start(rdev);
5842
5843         return 0;
5844 }
5845
5846 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5847 {
5848         u32 data, orig, tmp, tmp2;
5849
5850         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5851
5852         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5853                 cik_enable_gui_idle_interrupt(rdev, true);
5854
5855                 tmp = cik_halt_rlc(rdev);
5856
5857                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5858                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5859                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5860                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5861                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5862
5863                 cik_update_rlc(rdev, tmp);
5864
5865                 data |= CGCG_EN | CGLS_EN;
5866         } else {
5867                 cik_enable_gui_idle_interrupt(rdev, false);
5868
5869                 RREG32(CB_CGTT_SCLK_CTRL);
5870                 RREG32(CB_CGTT_SCLK_CTRL);
5871                 RREG32(CB_CGTT_SCLK_CTRL);
5872                 RREG32(CB_CGTT_SCLK_CTRL);
5873
5874                 data &= ~(CGCG_EN | CGLS_EN);
5875         }
5876
5877         if (orig != data)
5878                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5879
5880 }
5881
5882 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5883 {
5884         u32 data, orig, tmp = 0;
5885
5886         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5887                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5888                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5889                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
5890                                 data |= CP_MEM_LS_EN;
5891                                 if (orig != data)
5892                                         WREG32(CP_MEM_SLP_CNTL, data);
5893                         }
5894                 }
5895
5896                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5897                 data &= 0xfffffffd;
5898                 if (orig != data)
5899                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5900
5901                 tmp = cik_halt_rlc(rdev);
5902
5903                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5904                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5905                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5906                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5907                 WREG32(RLC_SERDES_WR_CTRL, data);
5908
5909                 cik_update_rlc(rdev, tmp);
5910
5911                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5912                         orig = data = RREG32(CGTS_SM_CTRL_REG);
5913                         data &= ~SM_MODE_MASK;
5914                         data |= SM_MODE(0x2);
5915                         data |= SM_MODE_ENABLE;
5916                         data &= ~CGTS_OVERRIDE;
5917                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5918                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5919                                 data &= ~CGTS_LS_OVERRIDE;
5920                         data &= ~ON_MONITOR_ADD_MASK;
5921                         data |= ON_MONITOR_ADD_EN;
5922                         data |= ON_MONITOR_ADD(0x96);
5923                         if (orig != data)
5924                                 WREG32(CGTS_SM_CTRL_REG, data);
5925                 }
5926         } else {
5927                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5928                 data |= 0x00000002;
5929                 if (orig != data)
5930                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5931
5932                 data = RREG32(RLC_MEM_SLP_CNTL);
5933                 if (data & RLC_MEM_LS_EN) {
5934                         data &= ~RLC_MEM_LS_EN;
5935                         WREG32(RLC_MEM_SLP_CNTL, data);
5936                 }
5937
5938                 data = RREG32(CP_MEM_SLP_CNTL);
5939                 if (data & CP_MEM_LS_EN) {
5940                         data &= ~CP_MEM_LS_EN;
5941                         WREG32(CP_MEM_SLP_CNTL, data);
5942                 }
5943
5944                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5945                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5946                 if (orig != data)
5947                         WREG32(CGTS_SM_CTRL_REG, data);
5948
5949                 tmp = cik_halt_rlc(rdev);
5950
5951                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5952                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5953                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5954                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5955                 WREG32(RLC_SERDES_WR_CTRL, data);
5956
5957                 cik_update_rlc(rdev, tmp);
5958         }
5959 }
5960
5961 static const u32 mc_cg_registers[] =
5962 {
5963         MC_HUB_MISC_HUB_CG,
5964         MC_HUB_MISC_SIP_CG,
5965         MC_HUB_MISC_VM_CG,
5966         MC_XPB_CLK_GAT,
5967         ATC_MISC_CG,
5968         MC_CITF_MISC_WR_CG,
5969         MC_CITF_MISC_RD_CG,
5970         MC_CITF_MISC_VM_CG,
5971         VM_L2_CG,
5972 };
5973
5974 static void cik_enable_mc_ls(struct radeon_device *rdev,
5975                              bool enable)
5976 {
5977         int i;
5978         u32 orig, data;
5979
5980         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5981                 orig = data = RREG32(mc_cg_registers[i]);
5982                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5983                         data |= MC_LS_ENABLE;
5984                 else
5985                         data &= ~MC_LS_ENABLE;
5986                 if (data != orig)
5987                         WREG32(mc_cg_registers[i], data);
5988         }
5989 }
5990
5991 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5992                                bool enable)
5993 {
5994         int i;
5995         u32 orig, data;
5996
5997         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5998                 orig = data = RREG32(mc_cg_registers[i]);
5999                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6000                         data |= MC_CG_ENABLE;
6001                 else
6002                         data &= ~MC_CG_ENABLE;
6003                 if (data != orig)
6004                         WREG32(mc_cg_registers[i], data);
6005         }
6006 }
6007
6008 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6009                                  bool enable)
6010 {
6011         u32 orig, data;
6012
6013         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6014                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6015                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6016         } else {
6017                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6018                 data |= 0xff000000;
6019                 if (data != orig)
6020                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6021
6022                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6023                 data |= 0xff000000;
6024                 if (data != orig)
6025                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6026         }
6027 }
6028
6029 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6030                                  bool enable)
6031 {
6032         u32 orig, data;
6033
6034         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6035                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6036                 data |= 0x100;
6037                 if (orig != data)
6038                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6039
6040                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6041                 data |= 0x100;
6042                 if (orig != data)
6043                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6044         } else {
6045                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6046                 data &= ~0x100;
6047                 if (orig != data)
6048                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6049
6050                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6051                 data &= ~0x100;
6052                 if (orig != data)
6053                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6054         }
6055 }
6056
6057 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6058                                 bool enable)
6059 {
6060         u32 orig, data;
6061
6062         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6063                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6064                 data = 0xfff;
6065                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6066
6067                 orig = data = RREG32(UVD_CGC_CTRL);
6068                 data |= DCM;
6069                 if (orig != data)
6070                         WREG32(UVD_CGC_CTRL, data);
6071         } else {
6072                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6073                 data &= ~0xfff;
6074                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6075
6076                 orig = data = RREG32(UVD_CGC_CTRL);
6077                 data &= ~DCM;
6078                 if (orig != data)
6079                         WREG32(UVD_CGC_CTRL, data);
6080         }
6081 }
6082
6083 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6084                                bool enable)
6085 {
6086         u32 orig, data;
6087
6088         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6089
6090         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6091                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6092                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6093         else
6094                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6095                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6096
6097         if (orig != data)
6098                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6099 }
6100
6101 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6102                                 bool enable)
6103 {
6104         u32 orig, data;
6105
6106         orig = data = RREG32(HDP_HOST_PATH_CNTL);
6107
6108         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6109                 data &= ~CLOCK_GATING_DIS;
6110         else
6111                 data |= CLOCK_GATING_DIS;
6112
6113         if (orig != data)
6114                 WREG32(HDP_HOST_PATH_CNTL, data);
6115 }
6116
6117 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6118                               bool enable)
6119 {
6120         u32 orig, data;
6121
6122         orig = data = RREG32(HDP_MEM_POWER_LS);
6123
6124         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6125                 data |= HDP_LS_ENABLE;
6126         else
6127                 data &= ~HDP_LS_ENABLE;
6128
6129         if (orig != data)
6130                 WREG32(HDP_MEM_POWER_LS, data);
6131 }
6132
6133 void cik_update_cg(struct radeon_device *rdev,
6134                    u32 block, bool enable)
6135 {
6136
6137         if (block & RADEON_CG_BLOCK_GFX) {
6138                 cik_enable_gui_idle_interrupt(rdev, false);
6139                 /* order matters! */
6140                 if (enable) {
6141                         cik_enable_mgcg(rdev, true);
6142                         cik_enable_cgcg(rdev, true);
6143                 } else {
6144                         cik_enable_cgcg(rdev, false);
6145                         cik_enable_mgcg(rdev, false);
6146                 }
6147                 cik_enable_gui_idle_interrupt(rdev, true);
6148         }
6149
6150         if (block & RADEON_CG_BLOCK_MC) {
6151                 if (!(rdev->flags & RADEON_IS_IGP)) {
6152                         cik_enable_mc_mgcg(rdev, enable);
6153                         cik_enable_mc_ls(rdev, enable);
6154                 }
6155         }
6156
6157         if (block & RADEON_CG_BLOCK_SDMA) {
6158                 cik_enable_sdma_mgcg(rdev, enable);
6159                 cik_enable_sdma_mgls(rdev, enable);
6160         }
6161
6162         if (block & RADEON_CG_BLOCK_BIF) {
6163                 cik_enable_bif_mgls(rdev, enable);
6164         }
6165
6166         if (block & RADEON_CG_BLOCK_UVD) {
6167                 if (rdev->has_uvd)
6168                         cik_enable_uvd_mgcg(rdev, enable);
6169         }
6170
6171         if (block & RADEON_CG_BLOCK_HDP) {
6172                 cik_enable_hdp_mgcg(rdev, enable);
6173                 cik_enable_hdp_ls(rdev, enable);
6174         }
6175
6176         if (block & RADEON_CG_BLOCK_VCE) {
6177                 vce_v2_0_enable_mgcg(rdev, enable);
6178         }
6179 }
6180
6181 static void cik_init_cg(struct radeon_device *rdev)
6182 {
6183
6184         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6185
6186         if (rdev->has_uvd)
6187                 si_init_uvd_internal_cg(rdev);
6188
6189         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6190                              RADEON_CG_BLOCK_SDMA |
6191                              RADEON_CG_BLOCK_BIF |
6192                              RADEON_CG_BLOCK_UVD |
6193                              RADEON_CG_BLOCK_HDP), true);
6194 }
6195
6196 static void cik_fini_cg(struct radeon_device *rdev)
6197 {
6198         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6199                              RADEON_CG_BLOCK_SDMA |
6200                              RADEON_CG_BLOCK_BIF |
6201                              RADEON_CG_BLOCK_UVD |
6202                              RADEON_CG_BLOCK_HDP), false);
6203
6204         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6205 }
6206
6207 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6208                                           bool enable)
6209 {
6210         u32 data, orig;
6211
6212         orig = data = RREG32(RLC_PG_CNTL);
6213         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6214                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6215         else
6216                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6217         if (orig != data)
6218                 WREG32(RLC_PG_CNTL, data);
6219 }
6220
6221 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6222                                           bool enable)
6223 {
6224         u32 data, orig;
6225
6226         orig = data = RREG32(RLC_PG_CNTL);
6227         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6228                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6229         else
6230                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6231         if (orig != data)
6232                 WREG32(RLC_PG_CNTL, data);
6233 }
6234
6235 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6236 {
6237         u32 data, orig;
6238
6239         orig = data = RREG32(RLC_PG_CNTL);
6240         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6241                 data &= ~DISABLE_CP_PG;
6242         else
6243                 data |= DISABLE_CP_PG;
6244         if (orig != data)
6245                 WREG32(RLC_PG_CNTL, data);
6246 }
6247
6248 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6249 {
6250         u32 data, orig;
6251
6252         orig = data = RREG32(RLC_PG_CNTL);
6253         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6254                 data &= ~DISABLE_GDS_PG;
6255         else
6256                 data |= DISABLE_GDS_PG;
6257         if (orig != data)
6258                 WREG32(RLC_PG_CNTL, data);
6259 }
6260
6261 #define CP_ME_TABLE_SIZE    96
6262 #define CP_ME_TABLE_OFFSET  2048
6263 #define CP_MEC_TABLE_OFFSET 4096
6264
6265 void cik_init_cp_pg_table(struct radeon_device *rdev)
6266 {
6267         const __be32 *fw_data;
6268         volatile u32 *dst_ptr;
6269         int me, i, max_me = 4;
6270         u32 bo_offset = 0;
6271         u32 table_offset;
6272
6273         if (rdev->family == CHIP_KAVERI)
6274                 max_me = 5;
6275
6276         if (rdev->rlc.cp_table_ptr == NULL)
6277                 return;
6278
6279         /* write the cp table buffer */
6280         dst_ptr = rdev->rlc.cp_table_ptr;
6281         for (me = 0; me < max_me; me++) {
6282                 if (me == 0) {
6283                         fw_data = (const __be32 *)rdev->ce_fw->data;
6284                         table_offset = CP_ME_TABLE_OFFSET;
6285                 } else if (me == 1) {
6286                         fw_data = (const __be32 *)rdev->pfp_fw->data;
6287                         table_offset = CP_ME_TABLE_OFFSET;
6288                 } else if (me == 2) {
6289                         fw_data = (const __be32 *)rdev->me_fw->data;
6290                         table_offset = CP_ME_TABLE_OFFSET;
6291                 } else {
6292                         fw_data = (const __be32 *)rdev->mec_fw->data;
6293                         table_offset = CP_MEC_TABLE_OFFSET;
6294                 }
6295
6296                 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6297                         dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6298                 }
6299                 bo_offset += CP_ME_TABLE_SIZE;
6300         }
6301 }
6302
6303 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6304                                 bool enable)
6305 {
6306         u32 data, orig;
6307
6308         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6309                 orig = data = RREG32(RLC_PG_CNTL);
6310                 data |= GFX_PG_ENABLE;
6311                 if (orig != data)
6312                         WREG32(RLC_PG_CNTL, data);
6313
6314                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6315                 data |= AUTO_PG_EN;
6316                 if (orig != data)
6317                         WREG32(RLC_AUTO_PG_CTRL, data);
6318         } else {
6319                 orig = data = RREG32(RLC_PG_CNTL);
6320                 data &= ~GFX_PG_ENABLE;
6321                 if (orig != data)
6322                         WREG32(RLC_PG_CNTL, data);
6323
6324                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6325                 data &= ~AUTO_PG_EN;
6326                 if (orig != data)
6327                         WREG32(RLC_AUTO_PG_CTRL, data);
6328
6329                 data = RREG32(DB_RENDER_CONTROL);
6330         }
6331 }
6332
6333 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6334 {
6335         u32 mask = 0, tmp, tmp1;
6336         int i;
6337
6338         cik_select_se_sh(rdev, se, sh);
6339         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6340         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6341         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6342
6343         tmp &= 0xffff0000;
6344
6345         tmp |= tmp1;
6346         tmp >>= 16;
6347
6348         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6349                 mask <<= 1;
6350                 mask |= 1;
6351         }
6352
6353         return (~tmp) & mask;
6354 }
6355
6356 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6357 {
6358         u32 i, j, k, active_cu_number = 0;
6359         u32 mask, counter, cu_bitmap;
6360         u32 tmp = 0;
6361
6362         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6363                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6364                         mask = 1;
6365                         cu_bitmap = 0;
6366                         counter = 0;
6367                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6368                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6369                                         if (counter < 2)
6370                                                 cu_bitmap |= mask;
6371                                         counter ++;
6372                                 }
6373                                 mask <<= 1;
6374                         }
6375
6376                         active_cu_number += counter;
6377                         tmp |= (cu_bitmap << (i * 16 + j * 8));
6378                 }
6379         }
6380
6381         WREG32(RLC_PG_AO_CU_MASK, tmp);
6382
6383         tmp = RREG32(RLC_MAX_PG_CU);
6384         tmp &= ~MAX_PU_CU_MASK;
6385         tmp |= MAX_PU_CU(active_cu_number);
6386         WREG32(RLC_MAX_PG_CU, tmp);
6387 }
6388
6389 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6390                                        bool enable)
6391 {
6392         u32 data, orig;
6393
6394         orig = data = RREG32(RLC_PG_CNTL);
6395         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6396                 data |= STATIC_PER_CU_PG_ENABLE;
6397         else
6398                 data &= ~STATIC_PER_CU_PG_ENABLE;
6399         if (orig != data)
6400                 WREG32(RLC_PG_CNTL, data);
6401 }
6402
6403 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6404                                         bool enable)
6405 {
6406         u32 data, orig;
6407
6408         orig = data = RREG32(RLC_PG_CNTL);
6409         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6410                 data |= DYN_PER_CU_PG_ENABLE;
6411         else
6412                 data &= ~DYN_PER_CU_PG_ENABLE;
6413         if (orig != data)
6414                 WREG32(RLC_PG_CNTL, data);
6415 }
6416
6417 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6418 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6419
6420 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6421 {
6422         u32 data, orig;
6423         u32 i;
6424
6425         if (rdev->rlc.cs_data) {
6426                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6427                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6428                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6429                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6430         } else {
6431                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6432                 for (i = 0; i < 3; i++)
6433                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
6434         }
6435         if (rdev->rlc.reg_list) {
6436                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6437                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6438                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6439         }
6440
6441         orig = data = RREG32(RLC_PG_CNTL);
6442         data |= GFX_PG_SRC;
6443         if (orig != data)
6444                 WREG32(RLC_PG_CNTL, data);
6445
6446         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6447         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6448
6449         data = RREG32(CP_RB_WPTR_POLL_CNTL);
6450         data &= ~IDLE_POLL_COUNT_MASK;
6451         data |= IDLE_POLL_COUNT(0x60);
6452         WREG32(CP_RB_WPTR_POLL_CNTL, data);
6453
6454         data = 0x10101010;
6455         WREG32(RLC_PG_DELAY, data);
6456
6457         data = RREG32(RLC_PG_DELAY_2);
6458         data &= ~0xff;
6459         data |= 0x3;
6460         WREG32(RLC_PG_DELAY_2, data);
6461
6462         data = RREG32(RLC_AUTO_PG_CTRL);
6463         data &= ~GRBM_REG_SGIT_MASK;
6464         data |= GRBM_REG_SGIT(0x700);
6465         WREG32(RLC_AUTO_PG_CTRL, data);
6466
6467 }
6468
6469 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6470 {
6471         cik_enable_gfx_cgpg(rdev, enable);
6472         cik_enable_gfx_static_mgpg(rdev, enable);
6473         cik_enable_gfx_dynamic_mgpg(rdev, enable);
6474 }
6475
6476 u32 cik_get_csb_size(struct radeon_device *rdev)
6477 {
6478         u32 count = 0;
6479         const struct cs_section_def *sect = NULL;
6480         const struct cs_extent_def *ext = NULL;
6481
6482         if (rdev->rlc.cs_data == NULL)
6483                 return 0;
6484
6485         /* begin clear state */
6486         count += 2;
6487         /* context control state */
6488         count += 3;
6489
6490         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6491                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6492                         if (sect->id == SECT_CONTEXT)
6493                                 count += 2 + ext->reg_count;
6494                         else
6495                                 return 0;
6496                 }
6497         }
6498         /* pa_sc_raster_config/pa_sc_raster_config1 */
6499         count += 4;
6500         /* end clear state */
6501         count += 2;
6502         /* clear state */
6503         count += 2;
6504
6505         return count;
6506 }
6507
6508 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6509 {
6510         u32 count = 0, i;
6511         const struct cs_section_def *sect = NULL;
6512         const struct cs_extent_def *ext = NULL;
6513
6514         if (rdev->rlc.cs_data == NULL)
6515                 return;
6516         if (buffer == NULL)
6517                 return;
6518
6519         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6520         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6521
6522         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6523         buffer[count++] = cpu_to_le32(0x80000000);
6524         buffer[count++] = cpu_to_le32(0x80000000);
6525
6526         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6527                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6528                         if (sect->id == SECT_CONTEXT) {
6529                                 buffer[count++] =
6530                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6531                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6532                                 for (i = 0; i < ext->reg_count; i++)
6533                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
6534                         } else {
6535                                 return;
6536                         }
6537                 }
6538         }
6539
6540         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6541         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6542         switch (rdev->family) {
6543         case CHIP_BONAIRE:
6544                 buffer[count++] = cpu_to_le32(0x16000012);
6545                 buffer[count++] = cpu_to_le32(0x00000000);
6546                 break;
6547         case CHIP_KAVERI:
6548                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6549                 buffer[count++] = cpu_to_le32(0x00000000);
6550                 break;
6551         case CHIP_KABINI:
6552                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6553                 buffer[count++] = cpu_to_le32(0x00000000);
6554                 break;
6555         case CHIP_HAWAII:
6556                 buffer[count++] = cpu_to_le32(0x3a00161a);
6557                 buffer[count++] = cpu_to_le32(0x0000002e);
6558                 break;
6559         default:
6560                 buffer[count++] = cpu_to_le32(0x00000000);
6561                 buffer[count++] = cpu_to_le32(0x00000000);
6562                 break;
6563         }
6564
6565         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6566         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6567
6568         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6569         buffer[count++] = cpu_to_le32(0);
6570 }
6571
6572 static void cik_init_pg(struct radeon_device *rdev)
6573 {
6574         if (rdev->pg_flags) {
6575                 cik_enable_sck_slowdown_on_pu(rdev, true);
6576                 cik_enable_sck_slowdown_on_pd(rdev, true);
6577                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6578                         cik_init_gfx_cgpg(rdev);
6579                         cik_enable_cp_pg(rdev, true);
6580                         cik_enable_gds_pg(rdev, true);
6581                 }
6582                 cik_init_ao_cu_mask(rdev);
6583                 cik_update_gfx_pg(rdev, true);
6584         }
6585 }
6586
6587 static void cik_fini_pg(struct radeon_device *rdev)
6588 {
6589         if (rdev->pg_flags) {
6590                 cik_update_gfx_pg(rdev, false);
6591                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6592                         cik_enable_cp_pg(rdev, false);
6593                         cik_enable_gds_pg(rdev, false);
6594                 }
6595         }
6596 }
6597
6598 /*
6599  * Interrupts
6600  * Starting with r6xx, interrupts are handled via a ring buffer.
6601  * Ring buffers are areas of GPU accessible memory that the GPU
6602  * writes interrupt vectors into and the host reads vectors out of.
6603  * There is a rptr (read pointer) that determines where the
6604  * host is currently reading, and a wptr (write pointer)
6605  * which determines where the GPU has written.  When the
6606  * pointers are equal, the ring is idle.  When the GPU
6607  * writes vectors to the ring buffer, it increments the
6608  * wptr.  When there is an interrupt, the host then starts
6609  * fetching commands and processing them until the pointers are
6610  * equal again at which point it updates the rptr.
6611  */
6612
6613 /**
6614  * cik_enable_interrupts - Enable the interrupt ring buffer
6615  *
6616  * @rdev: radeon_device pointer
6617  *
6618  * Enable the interrupt ring buffer (CIK).
6619  */
6620 static void cik_enable_interrupts(struct radeon_device *rdev)
6621 {
6622         u32 ih_cntl = RREG32(IH_CNTL);
6623         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6624
6625         ih_cntl |= ENABLE_INTR;
6626         ih_rb_cntl |= IH_RB_ENABLE;
6627         WREG32(IH_CNTL, ih_cntl);
6628         WREG32(IH_RB_CNTL, ih_rb_cntl);
6629         rdev->ih.enabled = true;
6630 }
6631
6632 /**
6633  * cik_disable_interrupts - Disable the interrupt ring buffer
6634  *
6635  * @rdev: radeon_device pointer
6636  *
6637  * Disable the interrupt ring buffer (CIK).
6638  */
6639 static void cik_disable_interrupts(struct radeon_device *rdev)
6640 {
6641         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6642         u32 ih_cntl = RREG32(IH_CNTL);
6643
6644         ih_rb_cntl &= ~IH_RB_ENABLE;
6645         ih_cntl &= ~ENABLE_INTR;
6646         WREG32(IH_RB_CNTL, ih_rb_cntl);
6647         WREG32(IH_CNTL, ih_cntl);
6648         /* set rptr, wptr to 0 */
6649         WREG32(IH_RB_RPTR, 0);
6650         WREG32(IH_RB_WPTR, 0);
6651         rdev->ih.enabled = false;
6652         rdev->ih.rptr = 0;
6653 }
6654
6655 /**
6656  * cik_disable_interrupt_state - Disable all interrupt sources
6657  *
6658  * @rdev: radeon_device pointer
6659  *
6660  * Clear all interrupt enable bits used by the driver (CIK).
6661  */
6662 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6663 {
6664         u32 tmp;
6665
6666         /* gfx ring */
6667         tmp = RREG32(CP_INT_CNTL_RING0) &
6668                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6669         WREG32(CP_INT_CNTL_RING0, tmp);
6670         /* sdma */
6671         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6672         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6673         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6674         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6675         /* compute queues */
6676         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6677         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6678         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6679         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6680         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6681         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6682         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6683         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6684         /* grbm */
6685         WREG32(GRBM_INT_CNTL, 0);
6686         /* vline/vblank, etc. */
6687         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6688         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6689         if (rdev->num_crtc >= 4) {
6690                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6691                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6692         }
6693         if (rdev->num_crtc >= 6) {
6694                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6695                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6696         }
6697         /* pflip */
6698         if (rdev->num_crtc >= 2) {
6699                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6700                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6701         }
6702         if (rdev->num_crtc >= 4) {
6703                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6704                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6705         }
6706         if (rdev->num_crtc >= 6) {
6707                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6708                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6709         }
6710
6711         /* dac hotplug */
6712         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6713
6714         /* digital hotplug */
6715         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6716         WREG32(DC_HPD1_INT_CONTROL, tmp);
6717         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6718         WREG32(DC_HPD2_INT_CONTROL, tmp);
6719         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6720         WREG32(DC_HPD3_INT_CONTROL, tmp);
6721         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6722         WREG32(DC_HPD4_INT_CONTROL, tmp);
6723         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6724         WREG32(DC_HPD5_INT_CONTROL, tmp);
6725         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6726         WREG32(DC_HPD6_INT_CONTROL, tmp);
6727
6728 }
6729
6730 /**
6731  * cik_irq_init - init and enable the interrupt ring
6732  *
6733  * @rdev: radeon_device pointer
6734  *
6735  * Allocate a ring buffer for the interrupt controller,
6736  * enable the RLC, disable interrupts, enable the IH
6737  * ring buffer and enable it (CIK).
6738  * Called at device load and reume.
6739  * Returns 0 for success, errors for failure.
6740  */
6741 static int cik_irq_init(struct radeon_device *rdev)
6742 {
6743         int ret = 0;
6744         int rb_bufsz;
6745         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6746
6747         /* allocate ring */
6748         ret = r600_ih_ring_alloc(rdev);
6749         if (ret)
6750                 return ret;
6751
6752         /* disable irqs */
6753         cik_disable_interrupts(rdev);
6754
6755         /* init rlc */
6756         ret = cik_rlc_resume(rdev);
6757         if (ret) {
6758                 r600_ih_ring_fini(rdev);
6759                 return ret;
6760         }
6761
6762         /* setup interrupt control */
6763         /* XXX this should actually be a bus address, not an MC address. same on older asics */
6764         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6765         interrupt_cntl = RREG32(INTERRUPT_CNTL);
6766         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6767          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6768          */
6769         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6770         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6771         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6772         WREG32(INTERRUPT_CNTL, interrupt_cntl);
6773
6774         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6775         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6776
6777         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6778                       IH_WPTR_OVERFLOW_CLEAR |
6779                       (rb_bufsz << 1));
6780
6781         if (rdev->wb.enabled)
6782                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6783
6784         /* set the writeback address whether it's enabled or not */
6785         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6786         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6787
6788         WREG32(IH_RB_CNTL, ih_rb_cntl);
6789
6790         /* set rptr, wptr to 0 */
6791         WREG32(IH_RB_RPTR, 0);
6792         WREG32(IH_RB_WPTR, 0);
6793
6794         /* Default settings for IH_CNTL (disabled at first) */
6795         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6796         /* RPTR_REARM only works if msi's are enabled */
6797         if (rdev->msi_enabled)
6798                 ih_cntl |= RPTR_REARM;
6799         WREG32(IH_CNTL, ih_cntl);
6800
6801         /* force the active interrupt state to all disabled */
6802         cik_disable_interrupt_state(rdev);
6803
6804         pci_set_master(rdev->pdev);
6805
6806         /* enable irqs */
6807         cik_enable_interrupts(rdev);
6808
6809         return ret;
6810 }
6811
6812 /**
6813  * cik_irq_set - enable/disable interrupt sources
6814  *
6815  * @rdev: radeon_device pointer
6816  *
6817  * Enable interrupt sources on the GPU (vblanks, hpd,
6818  * etc.) (CIK).
6819  * Returns 0 for success, errors for failure.
6820  */
6821 int cik_irq_set(struct radeon_device *rdev)
6822 {
6823         u32 cp_int_cntl;
6824         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6825         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6826         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6827         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6828         u32 grbm_int_cntl = 0;
6829         u32 dma_cntl, dma_cntl1;
6830         u32 thermal_int;
6831
6832         if (!rdev->irq.installed) {
6833                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6834                 return -EINVAL;
6835         }
6836         /* don't enable anything if the ih is disabled */
6837         if (!rdev->ih.enabled) {
6838                 cik_disable_interrupts(rdev);
6839                 /* force the active interrupt state to all disabled */
6840                 cik_disable_interrupt_state(rdev);
6841                 return 0;
6842         }
6843
6844         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6845                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6846         cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6847
6848         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6849         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6850         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6851         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6852         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6853         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6854
6855         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6856         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6857
6858         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6859         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6860         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6861         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6862         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6863         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6864         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6865         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6866
6867         if (rdev->flags & RADEON_IS_IGP)
6868                 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6869                         ~(THERM_INTH_MASK | THERM_INTL_MASK);
6870         else
6871                 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6872                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6873
6874         /* enable CP interrupts on all rings */
6875         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6876                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6877                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6878         }
6879         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6880                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6881                 DRM_DEBUG("si_irq_set: sw int cp1\n");
6882                 if (ring->me == 1) {
6883                         switch (ring->pipe) {
6884                         case 0:
6885                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6886                                 break;
6887                         case 1:
6888                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6889                                 break;
6890                         case 2:
6891                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6892                                 break;
6893                         case 3:
6894                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6895                                 break;
6896                         default:
6897                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6898                                 break;
6899                         }
6900                 } else if (ring->me == 2) {
6901                         switch (ring->pipe) {
6902                         case 0:
6903                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6904                                 break;
6905                         case 1:
6906                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6907                                 break;
6908                         case 2:
6909                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6910                                 break;
6911                         case 3:
6912                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6913                                 break;
6914                         default:
6915                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6916                                 break;
6917                         }
6918                 } else {
6919                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6920                 }
6921         }
6922         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6923                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6924                 DRM_DEBUG("si_irq_set: sw int cp2\n");
6925                 if (ring->me == 1) {
6926                         switch (ring->pipe) {
6927                         case 0:
6928                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6929                                 break;
6930                         case 1:
6931                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6932                                 break;
6933                         case 2:
6934                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6935                                 break;
6936                         case 3:
6937                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6938                                 break;
6939                         default:
6940                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6941                                 break;
6942                         }
6943                 } else if (ring->me == 2) {
6944                         switch (ring->pipe) {
6945                         case 0:
6946                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6947                                 break;
6948                         case 1:
6949                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6950                                 break;
6951                         case 2:
6952                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6953                                 break;
6954                         case 3:
6955                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6956                                 break;
6957                         default:
6958                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6959                                 break;
6960                         }
6961                 } else {
6962                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6963                 }
6964         }
6965
6966         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6967                 DRM_DEBUG("cik_irq_set: sw int dma\n");
6968                 dma_cntl |= TRAP_ENABLE;
6969         }
6970
6971         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6972                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6973                 dma_cntl1 |= TRAP_ENABLE;
6974         }
6975
6976         if (rdev->irq.crtc_vblank_int[0] ||
6977             atomic_read(&rdev->irq.pflip[0])) {
6978                 DRM_DEBUG("cik_irq_set: vblank 0\n");
6979                 crtc1 |= VBLANK_INTERRUPT_MASK;
6980         }
6981         if (rdev->irq.crtc_vblank_int[1] ||
6982             atomic_read(&rdev->irq.pflip[1])) {
6983                 DRM_DEBUG("cik_irq_set: vblank 1\n");
6984                 crtc2 |= VBLANK_INTERRUPT_MASK;
6985         }
6986         if (rdev->irq.crtc_vblank_int[2] ||
6987             atomic_read(&rdev->irq.pflip[2])) {
6988                 DRM_DEBUG("cik_irq_set: vblank 2\n");
6989                 crtc3 |= VBLANK_INTERRUPT_MASK;
6990         }
6991         if (rdev->irq.crtc_vblank_int[3] ||
6992             atomic_read(&rdev->irq.pflip[3])) {
6993                 DRM_DEBUG("cik_irq_set: vblank 3\n");
6994                 crtc4 |= VBLANK_INTERRUPT_MASK;
6995         }
6996         if (rdev->irq.crtc_vblank_int[4] ||
6997             atomic_read(&rdev->irq.pflip[4])) {
6998                 DRM_DEBUG("cik_irq_set: vblank 4\n");
6999                 crtc5 |= VBLANK_INTERRUPT_MASK;
7000         }
7001         if (rdev->irq.crtc_vblank_int[5] ||
7002             atomic_read(&rdev->irq.pflip[5])) {
7003                 DRM_DEBUG("cik_irq_set: vblank 5\n");
7004                 crtc6 |= VBLANK_INTERRUPT_MASK;
7005         }
7006         if (rdev->irq.hpd[0]) {
7007                 DRM_DEBUG("cik_irq_set: hpd 1\n");
7008                 hpd1 |= DC_HPDx_INT_EN;
7009         }
7010         if (rdev->irq.hpd[1]) {
7011                 DRM_DEBUG("cik_irq_set: hpd 2\n");
7012                 hpd2 |= DC_HPDx_INT_EN;
7013         }
7014         if (rdev->irq.hpd[2]) {
7015                 DRM_DEBUG("cik_irq_set: hpd 3\n");
7016                 hpd3 |= DC_HPDx_INT_EN;
7017         }
7018         if (rdev->irq.hpd[3]) {
7019                 DRM_DEBUG("cik_irq_set: hpd 4\n");
7020                 hpd4 |= DC_HPDx_INT_EN;
7021         }
7022         if (rdev->irq.hpd[4]) {
7023                 DRM_DEBUG("cik_irq_set: hpd 5\n");
7024                 hpd5 |= DC_HPDx_INT_EN;
7025         }
7026         if (rdev->irq.hpd[5]) {
7027                 DRM_DEBUG("cik_irq_set: hpd 6\n");
7028                 hpd6 |= DC_HPDx_INT_EN;
7029         }
7030
7031         if (rdev->irq.dpm_thermal) {
7032                 DRM_DEBUG("dpm thermal\n");
7033                 if (rdev->flags & RADEON_IS_IGP)
7034                         thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7035                 else
7036                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7037         }
7038
7039         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7040
7041         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7042         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7043
7044         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7045         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7046         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7047         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7048         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7049         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7050         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7051         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7052
7053         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7054
7055         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7056         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7057         if (rdev->num_crtc >= 4) {
7058                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7059                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7060         }
7061         if (rdev->num_crtc >= 6) {
7062                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7063                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7064         }
7065
7066         if (rdev->num_crtc >= 2) {
7067                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7068                        GRPH_PFLIP_INT_MASK);
7069                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7070                        GRPH_PFLIP_INT_MASK);
7071         }
7072         if (rdev->num_crtc >= 4) {
7073                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7074                        GRPH_PFLIP_INT_MASK);
7075                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7076                        GRPH_PFLIP_INT_MASK);
7077         }
7078         if (rdev->num_crtc >= 6) {
7079                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7080                        GRPH_PFLIP_INT_MASK);
7081                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7082                        GRPH_PFLIP_INT_MASK);
7083         }
7084
7085         WREG32(DC_HPD1_INT_CONTROL, hpd1);
7086         WREG32(DC_HPD2_INT_CONTROL, hpd2);
7087         WREG32(DC_HPD3_INT_CONTROL, hpd3);
7088         WREG32(DC_HPD4_INT_CONTROL, hpd4);
7089         WREG32(DC_HPD5_INT_CONTROL, hpd5);
7090         WREG32(DC_HPD6_INT_CONTROL, hpd6);
7091
7092         if (rdev->flags & RADEON_IS_IGP)
7093                 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7094         else
7095                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7096
7097         return 0;
7098 }
7099
7100 /**
7101  * cik_irq_ack - ack interrupt sources
7102  *
7103  * @rdev: radeon_device pointer
7104  *
7105  * Ack interrupt sources on the GPU (vblanks, hpd,
7106  * etc.) (CIK).  Certain interrupts sources are sw
7107  * generated and do not require an explicit ack.
7108  */
7109 static inline void cik_irq_ack(struct radeon_device *rdev)
7110 {
7111         u32 tmp;
7112
7113         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7114         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7115         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7116         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7117         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7118         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7119         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7120
7121         rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7122                 EVERGREEN_CRTC0_REGISTER_OFFSET);
7123         rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7124                 EVERGREEN_CRTC1_REGISTER_OFFSET);
7125         if (rdev->num_crtc >= 4) {
7126                 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7127                         EVERGREEN_CRTC2_REGISTER_OFFSET);
7128                 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7129                         EVERGREEN_CRTC3_REGISTER_OFFSET);
7130         }
7131         if (rdev->num_crtc >= 6) {
7132                 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7133                         EVERGREEN_CRTC4_REGISTER_OFFSET);
7134                 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7135                         EVERGREEN_CRTC5_REGISTER_OFFSET);
7136         }
7137
7138         if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7139                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7140                        GRPH_PFLIP_INT_CLEAR);
7141         if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7142                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7143                        GRPH_PFLIP_INT_CLEAR);
7144         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7145                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7146         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7147                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7148         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7149                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7150         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7151                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7152
7153         if (rdev->num_crtc >= 4) {
7154                 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7155                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7156                                GRPH_PFLIP_INT_CLEAR);
7157                 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7158                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7159                                GRPH_PFLIP_INT_CLEAR);
7160                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7161                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7162                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7163                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7164                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7165                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7166                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7167                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7168         }
7169
7170         if (rdev->num_crtc >= 6) {
7171                 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7172                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7173                                GRPH_PFLIP_INT_CLEAR);
7174                 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7175                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7176                                GRPH_PFLIP_INT_CLEAR);
7177                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7178                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7179                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7180                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7181                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7182                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7183                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7184                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7185         }
7186
7187         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7188                 tmp = RREG32(DC_HPD1_INT_CONTROL);
7189                 tmp |= DC_HPDx_INT_ACK;
7190                 WREG32(DC_HPD1_INT_CONTROL, tmp);
7191         }
7192         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7193                 tmp = RREG32(DC_HPD2_INT_CONTROL);
7194                 tmp |= DC_HPDx_INT_ACK;
7195                 WREG32(DC_HPD2_INT_CONTROL, tmp);
7196         }
7197         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7198                 tmp = RREG32(DC_HPD3_INT_CONTROL);
7199                 tmp |= DC_HPDx_INT_ACK;
7200                 WREG32(DC_HPD3_INT_CONTROL, tmp);
7201         }
7202         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7203                 tmp = RREG32(DC_HPD4_INT_CONTROL);
7204                 tmp |= DC_HPDx_INT_ACK;
7205                 WREG32(DC_HPD4_INT_CONTROL, tmp);
7206         }
7207         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7208                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7209                 tmp |= DC_HPDx_INT_ACK;
7210                 WREG32(DC_HPD5_INT_CONTROL, tmp);
7211         }
7212         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7213                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7214                 tmp |= DC_HPDx_INT_ACK;
7215                 WREG32(DC_HPD6_INT_CONTROL, tmp);
7216         }
7217 }
7218
7219 /**
7220  * cik_irq_disable - disable interrupts
7221  *
7222  * @rdev: radeon_device pointer
7223  *
7224  * Disable interrupts on the hw (CIK).
7225  */
7226 static void cik_irq_disable(struct radeon_device *rdev)
7227 {
7228         cik_disable_interrupts(rdev);
7229         /* Wait and acknowledge irq */
7230         mdelay(1);
7231         cik_irq_ack(rdev);
7232         cik_disable_interrupt_state(rdev);
7233 }
7234
7235 /**
7236  * cik_irq_disable - disable interrupts for suspend
7237  *
7238  * @rdev: radeon_device pointer
7239  *
7240  * Disable interrupts and stop the RLC (CIK).
7241  * Used for suspend.
7242  */
7243 static void cik_irq_suspend(struct radeon_device *rdev)
7244 {
7245         cik_irq_disable(rdev);
7246         cik_rlc_stop(rdev);
7247 }
7248
7249 /**
7250  * cik_irq_fini - tear down interrupt support
7251  *
7252  * @rdev: radeon_device pointer
7253  *
7254  * Disable interrupts on the hw and free the IH ring
7255  * buffer (CIK).
7256  * Used for driver unload.
7257  */
7258 static void cik_irq_fini(struct radeon_device *rdev)
7259 {
7260         cik_irq_suspend(rdev);
7261         r600_ih_ring_fini(rdev);
7262 }
7263
7264 /**
7265  * cik_get_ih_wptr - get the IH ring buffer wptr
7266  *
7267  * @rdev: radeon_device pointer
7268  *
7269  * Get the IH ring buffer wptr from either the register
7270  * or the writeback memory buffer (CIK).  Also check for
7271  * ring buffer overflow and deal with it.
7272  * Used by cik_irq_process().
7273  * Returns the value of the wptr.
7274  */
7275 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7276 {
7277         u32 wptr, tmp;
7278
7279         if (rdev->wb.enabled)
7280                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7281         else
7282                 wptr = RREG32(IH_RB_WPTR);
7283
7284         if (wptr & RB_OVERFLOW) {
7285                 /* When a ring buffer overflow happen start parsing interrupt
7286                  * from the last not overwritten vector (wptr + 16). Hopefully
7287                  * this should allow us to catchup.
7288                  */
7289                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7290                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7291                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7292                 tmp = RREG32(IH_RB_CNTL);
7293                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7294                 WREG32(IH_RB_CNTL, tmp);
7295         }
7296         return (wptr & rdev->ih.ptr_mask);
7297 }
7298
7299 /*        CIK IV Ring
7300  * Each IV ring entry is 128 bits:
7301  * [7:0]    - interrupt source id
7302  * [31:8]   - reserved
7303  * [59:32]  - interrupt source data
7304  * [63:60]  - reserved
7305  * [71:64]  - RINGID
7306  *            CP:
7307  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7308  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7309  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7310  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7311  *            PIPE_ID - ME0 0=3D
7312  *                    - ME1&2 compute dispatcher (4 pipes each)
7313  *            SDMA:
7314  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7315  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7316  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7317  * [79:72]  - VMID
7318  * [95:80]  - PASID
7319  * [127:96] - reserved
7320  */
7321 /**
7322  * cik_irq_process - interrupt handler
7323  *
7324  * @rdev: radeon_device pointer
7325  *
7326  * Interrupt hander (CIK).  Walk the IH ring,
7327  * ack interrupts and schedule work to handle
7328  * interrupt events.
7329  * Returns irq process return code.
7330  */
7331 int cik_irq_process(struct radeon_device *rdev)
7332 {
7333         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7334         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7335         u32 wptr;
7336         u32 rptr;
7337         u32 src_id, src_data, ring_id;
7338         u8 me_id, pipe_id, queue_id;
7339         u32 ring_index;
7340         bool queue_hotplug = false;
7341         bool queue_reset = false;
7342         u32 addr, status, mc_client;
7343         bool queue_thermal = false;
7344
7345         if (!rdev->ih.enabled || rdev->shutdown)
7346                 return IRQ_NONE;
7347
7348         wptr = cik_get_ih_wptr(rdev);
7349
7350 restart_ih:
7351         /* is somebody else already processing irqs? */
7352         if (atomic_xchg(&rdev->ih.lock, 1))
7353                 return IRQ_NONE;
7354
7355         rptr = rdev->ih.rptr;
7356         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7357
7358         /* Order reading of wptr vs. reading of IH ring data */
7359         rmb();
7360
7361         /* display interrupts */
7362         cik_irq_ack(rdev);
7363
7364         while (rptr != wptr) {
7365                 /* wptr/rptr are in bytes! */
7366                 ring_index = rptr / 4;
7367                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7368                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7369                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7370
7371                 switch (src_id) {
7372                 case 1: /* D1 vblank/vline */
7373                         switch (src_data) {
7374                         case 0: /* D1 vblank */
7375                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7376                                         if (rdev->irq.crtc_vblank_int[0]) {
7377                                                 drm_handle_vblank(rdev->ddev, 0);
7378                                                 rdev->pm.vblank_sync = true;
7379                                                 wake_up(&rdev->irq.vblank_queue);
7380                                         }
7381                                         if (atomic_read(&rdev->irq.pflip[0]))
7382                                                 radeon_crtc_handle_flip(rdev, 0);
7383                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7384                                         DRM_DEBUG("IH: D1 vblank\n");
7385                                 }
7386                                 break;
7387                         case 1: /* D1 vline */
7388                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7389                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7390                                         DRM_DEBUG("IH: D1 vline\n");
7391                                 }
7392                                 break;
7393                         default:
7394                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7395                                 break;
7396                         }
7397                         break;
7398                 case 2: /* D2 vblank/vline */
7399                         switch (src_data) {
7400                         case 0: /* D2 vblank */
7401                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7402                                         if (rdev->irq.crtc_vblank_int[1]) {
7403                                                 drm_handle_vblank(rdev->ddev, 1);
7404                                                 rdev->pm.vblank_sync = true;
7405                                                 wake_up(&rdev->irq.vblank_queue);
7406                                         }
7407                                         if (atomic_read(&rdev->irq.pflip[1]))
7408                                                 radeon_crtc_handle_flip(rdev, 1);
7409                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7410                                         DRM_DEBUG("IH: D2 vblank\n");
7411                                 }
7412                                 break;
7413                         case 1: /* D2 vline */
7414                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7415                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7416                                         DRM_DEBUG("IH: D2 vline\n");
7417                                 }
7418                                 break;
7419                         default:
7420                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7421                                 break;
7422                         }
7423                         break;
7424                 case 3: /* D3 vblank/vline */
7425                         switch (src_data) {
7426                         case 0: /* D3 vblank */
7427                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7428                                         if (rdev->irq.crtc_vblank_int[2]) {
7429                                                 drm_handle_vblank(rdev->ddev, 2);
7430                                                 rdev->pm.vblank_sync = true;
7431                                                 wake_up(&rdev->irq.vblank_queue);
7432                                         }
7433                                         if (atomic_read(&rdev->irq.pflip[2]))
7434                                                 radeon_crtc_handle_flip(rdev, 2);
7435                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7436                                         DRM_DEBUG("IH: D3 vblank\n");
7437                                 }
7438                                 break;
7439                         case 1: /* D3 vline */
7440                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7441                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7442                                         DRM_DEBUG("IH: D3 vline\n");
7443                                 }
7444                                 break;
7445                         default:
7446                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7447                                 break;
7448                         }
7449                         break;
7450                 case 4: /* D4 vblank/vline */
7451                         switch (src_data) {
7452                         case 0: /* D4 vblank */
7453                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7454                                         if (rdev->irq.crtc_vblank_int[3]) {
7455                                                 drm_handle_vblank(rdev->ddev, 3);
7456                                                 rdev->pm.vblank_sync = true;
7457                                                 wake_up(&rdev->irq.vblank_queue);
7458                                         }
7459                                         if (atomic_read(&rdev->irq.pflip[3]))
7460                                                 radeon_crtc_handle_flip(rdev, 3);
7461                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7462                                         DRM_DEBUG("IH: D4 vblank\n");
7463                                 }
7464                                 break;
7465                         case 1: /* D4 vline */
7466                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7467                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7468                                         DRM_DEBUG("IH: D4 vline\n");
7469                                 }
7470                                 break;
7471                         default:
7472                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7473                                 break;
7474                         }
7475                         break;
7476                 case 5: /* D5 vblank/vline */
7477                         switch (src_data) {
7478                         case 0: /* D5 vblank */
7479                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7480                                         if (rdev->irq.crtc_vblank_int[4]) {
7481                                                 drm_handle_vblank(rdev->ddev, 4);
7482                                                 rdev->pm.vblank_sync = true;
7483                                                 wake_up(&rdev->irq.vblank_queue);
7484                                         }
7485                                         if (atomic_read(&rdev->irq.pflip[4]))
7486                                                 radeon_crtc_handle_flip(rdev, 4);
7487                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7488                                         DRM_DEBUG("IH: D5 vblank\n");
7489                                 }
7490                                 break;
7491                         case 1: /* D5 vline */
7492                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7493                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7494                                         DRM_DEBUG("IH: D5 vline\n");
7495                                 }
7496                                 break;
7497                         default:
7498                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7499                                 break;
7500                         }
7501                         break;
7502                 case 6: /* D6 vblank/vline */
7503                         switch (src_data) {
7504                         case 0: /* D6 vblank */
7505                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7506                                         if (rdev->irq.crtc_vblank_int[5]) {
7507                                                 drm_handle_vblank(rdev->ddev, 5);
7508                                                 rdev->pm.vblank_sync = true;
7509                                                 wake_up(&rdev->irq.vblank_queue);
7510                                         }
7511                                         if (atomic_read(&rdev->irq.pflip[5]))
7512                                                 radeon_crtc_handle_flip(rdev, 5);
7513                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7514                                         DRM_DEBUG("IH: D6 vblank\n");
7515                                 }
7516                                 break;
7517                         case 1: /* D6 vline */
7518                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7519                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7520                                         DRM_DEBUG("IH: D6 vline\n");
7521                                 }
7522                                 break;
7523                         default:
7524                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7525                                 break;
7526                         }
7527                         break;
7528                 case 8: /* D1 page flip */
7529                 case 10: /* D2 page flip */
7530                 case 12: /* D3 page flip */
7531                 case 14: /* D4 page flip */
7532                 case 16: /* D5 page flip */
7533                 case 18: /* D6 page flip */
7534                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7535                         radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7536                         break;
7537                 case 42: /* HPD hotplug */
7538                         switch (src_data) {
7539                         case 0:
7540                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7541                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7542                                         queue_hotplug = true;
7543                                         DRM_DEBUG("IH: HPD1\n");
7544                                 }
7545                                 break;
7546                         case 1:
7547                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7548                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7549                                         queue_hotplug = true;
7550                                         DRM_DEBUG("IH: HPD2\n");
7551                                 }
7552                                 break;
7553                         case 2:
7554                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7555                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7556                                         queue_hotplug = true;
7557                                         DRM_DEBUG("IH: HPD3\n");
7558                                 }
7559                                 break;
7560                         case 3:
7561                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7562                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7563                                         queue_hotplug = true;
7564                                         DRM_DEBUG("IH: HPD4\n");
7565                                 }
7566                                 break;
7567                         case 4:
7568                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7569                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7570                                         queue_hotplug = true;
7571                                         DRM_DEBUG("IH: HPD5\n");
7572                                 }
7573                                 break;
7574                         case 5:
7575                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7576                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7577                                         queue_hotplug = true;
7578                                         DRM_DEBUG("IH: HPD6\n");
7579                                 }
7580                                 break;
7581                         default:
7582                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7583                                 break;
7584                         }
7585                         break;
7586                 case 124: /* UVD */
7587                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7588                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7589                         break;
7590                 case 146:
7591                 case 147:
7592                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7593                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7594                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7595                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7596                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
7597                                 addr);
7598                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7599                                 status);
7600                         cik_vm_decode_fault(rdev, status, addr, mc_client);
7601                         /* reset addr and status */
7602                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7603                         break;
7604                 case 167: /* VCE */
7605                         DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7606                         switch (src_data) {
7607                         case 0:
7608                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7609                                 break;
7610                         case 1:
7611                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7612                                 break;
7613                         default:
7614                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7615                                 break;
7616                         }
7617                         break;
7618                 case 176: /* GFX RB CP_INT */
7619                 case 177: /* GFX IB CP_INT */
7620                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7621                         break;
7622                 case 181: /* CP EOP event */
7623                         DRM_DEBUG("IH: CP EOP\n");
7624                         /* XXX check the bitfield order! */
7625                         me_id = (ring_id & 0x60) >> 5;
7626                         pipe_id = (ring_id & 0x18) >> 3;
7627                         queue_id = (ring_id & 0x7) >> 0;
7628                         switch (me_id) {
7629                         case 0:
7630                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7631                                 break;
7632                         case 1:
7633                         case 2:
7634                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7635                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7636                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7637                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7638                                 break;
7639                         }
7640                         break;
7641                 case 184: /* CP Privileged reg access */
7642                         DRM_ERROR("Illegal register access in command stream\n");
7643                         /* XXX check the bitfield order! */
7644                         me_id = (ring_id & 0x60) >> 5;
7645                         pipe_id = (ring_id & 0x18) >> 3;
7646                         queue_id = (ring_id & 0x7) >> 0;
7647                         switch (me_id) {
7648                         case 0:
7649                                 /* This results in a full GPU reset, but all we need to do is soft
7650                                  * reset the CP for gfx
7651                                  */
7652                                 queue_reset = true;
7653                                 break;
7654                         case 1:
7655                                 /* XXX compute */
7656                                 queue_reset = true;
7657                                 break;
7658                         case 2:
7659                                 /* XXX compute */
7660                                 queue_reset = true;
7661                                 break;
7662                         }
7663                         break;
7664                 case 185: /* CP Privileged inst */
7665                         DRM_ERROR("Illegal instruction in command stream\n");
7666                         /* XXX check the bitfield order! */
7667                         me_id = (ring_id & 0x60) >> 5;
7668                         pipe_id = (ring_id & 0x18) >> 3;
7669                         queue_id = (ring_id & 0x7) >> 0;
7670                         switch (me_id) {
7671                         case 0:
7672                                 /* This results in a full GPU reset, but all we need to do is soft
7673                                  * reset the CP for gfx
7674                                  */
7675                                 queue_reset = true;
7676                                 break;
7677                         case 1:
7678                                 /* XXX compute */
7679                                 queue_reset = true;
7680                                 break;
7681                         case 2:
7682                                 /* XXX compute */
7683                                 queue_reset = true;
7684                                 break;
7685                         }
7686                         break;
7687                 case 224: /* SDMA trap event */
7688                         /* XXX check the bitfield order! */
7689                         me_id = (ring_id & 0x3) >> 0;
7690                         queue_id = (ring_id & 0xc) >> 2;
7691                         DRM_DEBUG("IH: SDMA trap\n");
7692                         switch (me_id) {
7693                         case 0:
7694                                 switch (queue_id) {
7695                                 case 0:
7696                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7697                                         break;
7698                                 case 1:
7699                                         /* XXX compute */
7700                                         break;
7701                                 case 2:
7702                                         /* XXX compute */
7703                                         break;
7704                                 }
7705                                 break;
7706                         case 1:
7707                                 switch (queue_id) {
7708                                 case 0:
7709                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7710                                         break;
7711                                 case 1:
7712                                         /* XXX compute */
7713                                         break;
7714                                 case 2:
7715                                         /* XXX compute */
7716                                         break;
7717                                 }
7718                                 break;
7719                         }
7720                         break;
7721                 case 230: /* thermal low to high */
7722                         DRM_DEBUG("IH: thermal low to high\n");
7723                         rdev->pm.dpm.thermal.high_to_low = false;
7724                         queue_thermal = true;
7725                         break;
7726                 case 231: /* thermal high to low */
7727                         DRM_DEBUG("IH: thermal high to low\n");
7728                         rdev->pm.dpm.thermal.high_to_low = true;
7729                         queue_thermal = true;
7730                         break;
7731                 case 233: /* GUI IDLE */
7732                         DRM_DEBUG("IH: GUI idle\n");
7733                         break;
7734                 case 241: /* SDMA Privileged inst */
7735                 case 247: /* SDMA Privileged inst */
7736                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
7737                         /* XXX check the bitfield order! */
7738                         me_id = (ring_id & 0x3) >> 0;
7739                         queue_id = (ring_id & 0xc) >> 2;
7740                         switch (me_id) {
7741                         case 0:
7742                                 switch (queue_id) {
7743                                 case 0:
7744                                         queue_reset = true;
7745                                         break;
7746                                 case 1:
7747                                         /* XXX compute */
7748                                         queue_reset = true;
7749                                         break;
7750                                 case 2:
7751                                         /* XXX compute */
7752                                         queue_reset = true;
7753                                         break;
7754                                 }
7755                                 break;
7756                         case 1:
7757                                 switch (queue_id) {
7758                                 case 0:
7759                                         queue_reset = true;
7760                                         break;
7761                                 case 1:
7762                                         /* XXX compute */
7763                                         queue_reset = true;
7764                                         break;
7765                                 case 2:
7766                                         /* XXX compute */
7767                                         queue_reset = true;
7768                                         break;
7769                                 }
7770                                 break;
7771                         }
7772                         break;
7773                 default:
7774                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7775                         break;
7776                 }
7777
7778                 /* wptr/rptr are in bytes! */
7779                 rptr += 16;
7780                 rptr &= rdev->ih.ptr_mask;
7781         }
7782         if (queue_hotplug)
7783                 schedule_work(&rdev->hotplug_work);
7784         if (queue_reset)
7785                 schedule_work(&rdev->reset_work);
7786         if (queue_thermal)
7787                 schedule_work(&rdev->pm.dpm.thermal.work);
7788         rdev->ih.rptr = rptr;
7789         WREG32(IH_RB_RPTR, rdev->ih.rptr);
7790         atomic_set(&rdev->ih.lock, 0);
7791
7792         /* make sure wptr hasn't changed while processing */
7793         wptr = cik_get_ih_wptr(rdev);
7794         if (wptr != rptr)
7795                 goto restart_ih;
7796
7797         return IRQ_HANDLED;
7798 }
7799
7800 /*
7801  * startup/shutdown callbacks
7802  */
7803 /**
7804  * cik_startup - program the asic to a functional state
7805  *
7806  * @rdev: radeon_device pointer
7807  *
7808  * Programs the asic to a functional state (CIK).
7809  * Called by cik_init() and cik_resume().
7810  * Returns 0 for success, error for failure.
7811  */
7812 static int cik_startup(struct radeon_device *rdev)
7813 {
7814         struct radeon_ring *ring;
7815         int r;
7816
7817         /* enable pcie gen2/3 link */
7818         cik_pcie_gen3_enable(rdev);
7819         /* enable aspm */
7820         cik_program_aspm(rdev);
7821
7822         /* scratch needs to be initialized before MC */
7823         r = r600_vram_scratch_init(rdev);
7824         if (r)
7825                 return r;
7826
7827         cik_mc_program(rdev);
7828
7829         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7830                 r = ci_mc_load_microcode(rdev);
7831                 if (r) {
7832                         DRM_ERROR("Failed to load MC firmware!\n");
7833                         return r;
7834                 }
7835         }
7836
7837         r = cik_pcie_gart_enable(rdev);
7838         if (r)
7839                 return r;
7840         cik_gpu_init(rdev);
7841
7842         /* allocate rlc buffers */
7843         if (rdev->flags & RADEON_IS_IGP) {
7844                 if (rdev->family == CHIP_KAVERI) {
7845                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7846                         rdev->rlc.reg_list_size =
7847                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7848                 } else {
7849                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7850                         rdev->rlc.reg_list_size =
7851                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7852                 }
7853         }
7854         rdev->rlc.cs_data = ci_cs_data;
7855         rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7856         r = sumo_rlc_init(rdev);
7857         if (r) {
7858                 DRM_ERROR("Failed to init rlc BOs!\n");
7859                 return r;
7860         }
7861
7862         /* allocate wb buffer */
7863         r = radeon_wb_init(rdev);
7864         if (r)
7865                 return r;
7866
7867         /* allocate mec buffers */
7868         r = cik_mec_init(rdev);
7869         if (r) {
7870                 DRM_ERROR("Failed to init MEC BOs!\n");
7871                 return r;
7872         }
7873
7874         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7875         if (r) {
7876                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7877                 return r;
7878         }
7879
7880         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7881         if (r) {
7882                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7883                 return r;
7884         }
7885
7886         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7887         if (r) {
7888                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7889                 return r;
7890         }
7891
7892         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7893         if (r) {
7894                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7895                 return r;
7896         }
7897
7898         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7899         if (r) {
7900                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7901                 return r;
7902         }
7903
7904         r = radeon_uvd_resume(rdev);
7905         if (!r) {
7906                 r = uvd_v4_2_resume(rdev);
7907                 if (!r) {
7908                         r = radeon_fence_driver_start_ring(rdev,
7909                                                            R600_RING_TYPE_UVD_INDEX);
7910                         if (r)
7911                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7912                 }
7913         }
7914         if (r)
7915                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7916
7917         r = radeon_vce_resume(rdev);
7918         if (!r) {
7919                 r = vce_v2_0_resume(rdev);
7920                 if (!r)
7921                         r = radeon_fence_driver_start_ring(rdev,
7922                                                            TN_RING_TYPE_VCE1_INDEX);
7923                 if (!r)
7924                         r = radeon_fence_driver_start_ring(rdev,
7925                                                            TN_RING_TYPE_VCE2_INDEX);
7926         }
7927         if (r) {
7928                 dev_err(rdev->dev, "VCE init error (%d).\n", r);
7929                 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
7930                 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
7931         }
7932
7933         /* Enable IRQ */
7934         if (!rdev->irq.installed) {
7935                 r = radeon_irq_kms_init(rdev);
7936                 if (r)
7937                         return r;
7938         }
7939
7940         r = cik_irq_init(rdev);
7941         if (r) {
7942                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7943                 radeon_irq_kms_fini(rdev);
7944                 return r;
7945         }
7946         cik_irq_set(rdev);
7947
7948         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7949         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7950                              PACKET3(PACKET3_NOP, 0x3FFF));
7951         if (r)
7952                 return r;
7953
7954         /* set up the compute queues */
7955         /* type-2 packets are deprecated on MEC, use type-3 instead */
7956         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7957         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7958                              PACKET3(PACKET3_NOP, 0x3FFF));
7959         if (r)
7960                 return r;
7961         ring->me = 1; /* first MEC */
7962         ring->pipe = 0; /* first pipe */
7963         ring->queue = 0; /* first queue */
7964         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7965
7966         /* type-2 packets are deprecated on MEC, use type-3 instead */
7967         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7968         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7969                              PACKET3(PACKET3_NOP, 0x3FFF));
7970         if (r)
7971                 return r;
7972         /* dGPU only have 1 MEC */
7973         ring->me = 1; /* first MEC */
7974         ring->pipe = 0; /* first pipe */
7975         ring->queue = 1; /* second queue */
7976         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7977
7978         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7979         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7980                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7981         if (r)
7982                 return r;
7983
7984         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7985         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7986                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7987         if (r)
7988                 return r;
7989
7990         r = cik_cp_resume(rdev);
7991         if (r)
7992                 return r;
7993
7994         r = cik_sdma_resume(rdev);
7995         if (r)
7996                 return r;
7997
7998         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7999         if (ring->ring_size) {
8000                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8001                                      RADEON_CP_PACKET2);
8002                 if (!r)
8003                         r = uvd_v1_0_init(rdev);
8004                 if (r)
8005                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8006         }
8007
8008         r = -ENOENT;
8009
8010         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8011         if (ring->ring_size)
8012                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8013                                      VCE_CMD_NO_OP);
8014
8015         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8016         if (ring->ring_size)
8017                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8018                                      VCE_CMD_NO_OP);
8019
8020         if (!r)
8021                 r = vce_v1_0_init(rdev);
8022         else if (r != -ENOENT)
8023                 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8024
8025         r = radeon_ib_pool_init(rdev);
8026         if (r) {
8027                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8028                 return r;
8029         }
8030
8031         r = radeon_vm_manager_init(rdev);
8032         if (r) {
8033                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8034                 return r;
8035         }
8036
8037         r = dce6_audio_init(rdev);
8038         if (r)
8039                 return r;
8040
8041         return 0;
8042 }
8043
8044 /**
8045  * cik_resume - resume the asic to a functional state
8046  *
8047  * @rdev: radeon_device pointer
8048  *
8049  * Programs the asic to a functional state (CIK).
8050  * Called at resume.
8051  * Returns 0 for success, error for failure.
8052  */
8053 int cik_resume(struct radeon_device *rdev)
8054 {
8055         int r;
8056
8057         /* post card */
8058         atom_asic_init(rdev->mode_info.atom_context);
8059
8060         /* init golden registers */
8061         cik_init_golden_registers(rdev);
8062
8063         if (rdev->pm.pm_method == PM_METHOD_DPM)
8064                 radeon_pm_resume(rdev);
8065
8066         rdev->accel_working = true;
8067         r = cik_startup(rdev);
8068         if (r) {
8069                 DRM_ERROR("cik startup failed on resume\n");
8070                 rdev->accel_working = false;
8071                 return r;
8072         }
8073
8074         return r;
8075
8076 }
8077
8078 /**
8079  * cik_suspend - suspend the asic
8080  *
8081  * @rdev: radeon_device pointer
8082  *
8083  * Bring the chip into a state suitable for suspend (CIK).
8084  * Called at suspend.
8085  * Returns 0 for success.
8086  */
8087 int cik_suspend(struct radeon_device *rdev)
8088 {
8089         radeon_pm_suspend(rdev);
8090         dce6_audio_fini(rdev);
8091         radeon_vm_manager_fini(rdev);
8092         cik_cp_enable(rdev, false);
8093         cik_sdma_enable(rdev, false);
8094         uvd_v1_0_fini(rdev);
8095         radeon_uvd_suspend(rdev);
8096         radeon_vce_suspend(rdev);
8097         cik_fini_pg(rdev);
8098         cik_fini_cg(rdev);
8099         cik_irq_suspend(rdev);
8100         radeon_wb_disable(rdev);
8101         cik_pcie_gart_disable(rdev);
8102         return 0;
8103 }
8104
8105 /* Plan is to move initialization in that function and use
8106  * helper function so that radeon_device_init pretty much
8107  * do nothing more than calling asic specific function. This
8108  * should also allow to remove a bunch of callback function
8109  * like vram_info.
8110  */
8111 /**
8112  * cik_init - asic specific driver and hw init
8113  *
8114  * @rdev: radeon_device pointer
8115  *
8116  * Setup asic specific driver variables and program the hw
8117  * to a functional state (CIK).
8118  * Called at driver startup.
8119  * Returns 0 for success, errors for failure.
8120  */
8121 int cik_init(struct radeon_device *rdev)
8122 {
8123         struct radeon_ring *ring;
8124         int r;
8125
8126         /* Read BIOS */
8127         if (!radeon_get_bios(rdev)) {
8128                 if (ASIC_IS_AVIVO(rdev))
8129                         return -EINVAL;
8130         }
8131         /* Must be an ATOMBIOS */
8132         if (!rdev->is_atom_bios) {
8133                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8134                 return -EINVAL;
8135         }
8136         r = radeon_atombios_init(rdev);
8137         if (r)
8138                 return r;
8139
8140         /* Post card if necessary */
8141         if (!radeon_card_posted(rdev)) {
8142                 if (!rdev->bios) {
8143                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8144                         return -EINVAL;
8145                 }
8146                 DRM_INFO("GPU not posted. posting now...\n");
8147                 atom_asic_init(rdev->mode_info.atom_context);
8148         }
8149         /* init golden registers */
8150         cik_init_golden_registers(rdev);
8151         /* Initialize scratch registers */
8152         cik_scratch_init(rdev);
8153         /* Initialize surface registers */
8154         radeon_surface_init(rdev);
8155         /* Initialize clocks */
8156         radeon_get_clock_info(rdev->ddev);
8157
8158         /* Fence driver */
8159         r = radeon_fence_driver_init(rdev);
8160         if (r)
8161                 return r;
8162
8163         /* initialize memory controller */
8164         r = cik_mc_init(rdev);
8165         if (r)
8166                 return r;
8167         /* Memory manager */
8168         r = radeon_bo_init(rdev);
8169         if (r)
8170                 return r;
8171
8172         if (rdev->flags & RADEON_IS_IGP) {
8173                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8174                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8175                         r = cik_init_microcode(rdev);
8176                         if (r) {
8177                                 DRM_ERROR("Failed to load firmware!\n");
8178                                 return r;
8179                         }
8180                 }
8181         } else {
8182                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8183                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8184                     !rdev->mc_fw) {
8185                         r = cik_init_microcode(rdev);
8186                         if (r) {
8187                                 DRM_ERROR("Failed to load firmware!\n");
8188                                 return r;
8189                         }
8190                 }
8191         }
8192
8193         /* Initialize power management */
8194         radeon_pm_init(rdev);
8195
8196         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8197         ring->ring_obj = NULL;
8198         r600_ring_init(rdev, ring, 1024 * 1024);
8199
8200         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8201         ring->ring_obj = NULL;
8202         r600_ring_init(rdev, ring, 1024 * 1024);
8203         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8204         if (r)
8205                 return r;
8206
8207         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8208         ring->ring_obj = NULL;
8209         r600_ring_init(rdev, ring, 1024 * 1024);
8210         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8211         if (r)
8212                 return r;
8213
8214         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8215         ring->ring_obj = NULL;
8216         r600_ring_init(rdev, ring, 256 * 1024);
8217
8218         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8219         ring->ring_obj = NULL;
8220         r600_ring_init(rdev, ring, 256 * 1024);
8221
8222         r = radeon_uvd_init(rdev);
8223         if (!r) {
8224                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8225                 ring->ring_obj = NULL;
8226                 r600_ring_init(rdev, ring, 4096);
8227         }
8228
8229         r = radeon_vce_init(rdev);
8230         if (!r) {
8231                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8232                 ring->ring_obj = NULL;
8233                 r600_ring_init(rdev, ring, 4096);
8234
8235                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8236                 ring->ring_obj = NULL;
8237                 r600_ring_init(rdev, ring, 4096);
8238         }
8239
8240         rdev->ih.ring_obj = NULL;
8241         r600_ih_ring_init(rdev, 64 * 1024);
8242
8243         r = r600_pcie_gart_init(rdev);
8244         if (r)
8245                 return r;
8246
8247         rdev->accel_working = true;
8248         r = cik_startup(rdev);
8249         if (r) {
8250                 dev_err(rdev->dev, "disabling GPU acceleration\n");
8251                 cik_cp_fini(rdev);
8252                 cik_sdma_fini(rdev);
8253                 cik_irq_fini(rdev);
8254                 sumo_rlc_fini(rdev);
8255                 cik_mec_fini(rdev);
8256                 radeon_wb_fini(rdev);
8257                 radeon_ib_pool_fini(rdev);
8258                 radeon_vm_manager_fini(rdev);
8259                 radeon_irq_kms_fini(rdev);
8260                 cik_pcie_gart_fini(rdev);
8261                 rdev->accel_working = false;
8262         }
8263
8264         /* Don't start up if the MC ucode is missing.
8265          * The default clocks and voltages before the MC ucode
8266          * is loaded are not suffient for advanced operations.
8267          */
8268         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8269                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8270                 return -EINVAL;
8271         }
8272
8273         return 0;
8274 }
8275
8276 /**
8277  * cik_fini - asic specific driver and hw fini
8278  *
8279  * @rdev: radeon_device pointer
8280  *
8281  * Tear down the asic specific driver variables and program the hw
8282  * to an idle state (CIK).
8283  * Called at driver unload.
8284  */
8285 void cik_fini(struct radeon_device *rdev)
8286 {
8287         radeon_pm_fini(rdev);
8288         cik_cp_fini(rdev);
8289         cik_sdma_fini(rdev);
8290         cik_fini_pg(rdev);
8291         cik_fini_cg(rdev);
8292         cik_irq_fini(rdev);
8293         sumo_rlc_fini(rdev);
8294         cik_mec_fini(rdev);
8295         radeon_wb_fini(rdev);
8296         radeon_vm_manager_fini(rdev);
8297         radeon_ib_pool_fini(rdev);
8298         radeon_irq_kms_fini(rdev);
8299         uvd_v1_0_fini(rdev);
8300         radeon_uvd_fini(rdev);
8301         radeon_vce_fini(rdev);
8302         cik_pcie_gart_fini(rdev);
8303         r600_vram_scratch_fini(rdev);
8304         radeon_gem_fini(rdev);
8305         radeon_fence_driver_fini(rdev);
8306         radeon_bo_fini(rdev);
8307         radeon_atombios_fini(rdev);
8308         kfree(rdev->bios);
8309         rdev->bios = NULL;
8310 }
8311
8312 void dce8_program_fmt(struct drm_encoder *encoder)
8313 {
8314         struct drm_device *dev = encoder->dev;
8315         struct radeon_device *rdev = dev->dev_private;
8316         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8317         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8318         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8319         int bpc = 0;
8320         u32 tmp = 0;
8321         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8322
8323         if (connector) {
8324                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8325                 bpc = radeon_get_monitor_bpc(connector);
8326                 dither = radeon_connector->dither;
8327         }
8328
8329         /* LVDS/eDP FMT is set up by atom */
8330         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8331                 return;
8332
8333         /* not needed for analog */
8334         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8335             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8336                 return;
8337
8338         if (bpc == 0)
8339                 return;
8340
8341         switch (bpc) {
8342         case 6:
8343                 if (dither == RADEON_FMT_DITHER_ENABLE)
8344                         /* XXX sort out optimal dither settings */
8345                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8346                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8347                 else
8348                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8349                 break;
8350         case 8:
8351                 if (dither == RADEON_FMT_DITHER_ENABLE)
8352                         /* XXX sort out optimal dither settings */
8353                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8354                                 FMT_RGB_RANDOM_ENABLE |
8355                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8356                 else
8357                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8358                 break;
8359         case 10:
8360                 if (dither == RADEON_FMT_DITHER_ENABLE)
8361                         /* XXX sort out optimal dither settings */
8362                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8363                                 FMT_RGB_RANDOM_ENABLE |
8364                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8365                 else
8366                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8367                 break;
8368         default:
8369                 /* not needed */
8370                 break;
8371         }
8372
8373         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8374 }
8375
8376 /* display watermark setup */
8377 /**
8378  * dce8_line_buffer_adjust - Set up the line buffer
8379  *
8380  * @rdev: radeon_device pointer
8381  * @radeon_crtc: the selected display controller
8382  * @mode: the current display mode on the selected display
8383  * controller
8384  *
8385  * Setup up the line buffer allocation for
8386  * the selected display controller (CIK).
8387  * Returns the line buffer size in pixels.
8388  */
8389 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8390                                    struct radeon_crtc *radeon_crtc,
8391                                    struct drm_display_mode *mode)
8392 {
8393         u32 tmp, buffer_alloc, i;
8394         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8395         /*
8396          * Line Buffer Setup
8397          * There are 6 line buffers, one for each display controllers.
8398          * There are 3 partitions per LB. Select the number of partitions
8399          * to enable based on the display width.  For display widths larger
8400          * than 4096, you need use to use 2 display controllers and combine
8401          * them using the stereo blender.
8402          */
8403         if (radeon_crtc->base.enabled && mode) {
8404                 if (mode->crtc_hdisplay < 1920) {
8405                         tmp = 1;
8406                         buffer_alloc = 2;
8407                 } else if (mode->crtc_hdisplay < 2560) {
8408                         tmp = 2;
8409                         buffer_alloc = 2;
8410                 } else if (mode->crtc_hdisplay < 4096) {
8411                         tmp = 0;
8412                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8413                 } else {
8414                         DRM_DEBUG_KMS("Mode too big for LB!\n");
8415                         tmp = 0;
8416                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8417                 }
8418         } else {
8419                 tmp = 1;
8420                 buffer_alloc = 0;
8421         }
8422
8423         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8424                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8425
8426         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8427                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8428         for (i = 0; i < rdev->usec_timeout; i++) {
8429                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8430                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
8431                         break;
8432                 udelay(1);
8433         }
8434
8435         if (radeon_crtc->base.enabled && mode) {
8436                 switch (tmp) {
8437                 case 0:
8438                 default:
8439                         return 4096 * 2;
8440                 case 1:
8441                         return 1920 * 2;
8442                 case 2:
8443                         return 2560 * 2;
8444                 }
8445         }
8446
8447         /* controller not enabled, so no lb used */
8448         return 0;
8449 }
8450
8451 /**
8452  * cik_get_number_of_dram_channels - get the number of dram channels
8453  *
8454  * @rdev: radeon_device pointer
8455  *
8456  * Look up the number of video ram channels (CIK).
8457  * Used for display watermark bandwidth calculations
8458  * Returns the number of dram channels
8459  */
8460 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8461 {
8462         u32 tmp = RREG32(MC_SHARED_CHMAP);
8463
8464         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8465         case 0:
8466         default:
8467                 return 1;
8468         case 1:
8469                 return 2;
8470         case 2:
8471                 return 4;
8472         case 3:
8473                 return 8;
8474         case 4:
8475                 return 3;
8476         case 5:
8477                 return 6;
8478         case 6:
8479                 return 10;
8480         case 7:
8481                 return 12;
8482         case 8:
8483                 return 16;
8484         }
8485 }
8486
8487 struct dce8_wm_params {
8488         u32 dram_channels; /* number of dram channels */
8489         u32 yclk;          /* bandwidth per dram data pin in kHz */
8490         u32 sclk;          /* engine clock in kHz */
8491         u32 disp_clk;      /* display clock in kHz */
8492         u32 src_width;     /* viewport width */
8493         u32 active_time;   /* active display time in ns */
8494         u32 blank_time;    /* blank time in ns */
8495         bool interlaced;    /* mode is interlaced */
8496         fixed20_12 vsc;    /* vertical scale ratio */
8497         u32 num_heads;     /* number of active crtcs */
8498         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8499         u32 lb_size;       /* line buffer allocated to pipe */
8500         u32 vtaps;         /* vertical scaler taps */
8501 };
8502
8503 /**
8504  * dce8_dram_bandwidth - get the dram bandwidth
8505  *
8506  * @wm: watermark calculation data
8507  *
8508  * Calculate the raw dram bandwidth (CIK).
8509  * Used for display watermark bandwidth calculations
8510  * Returns the dram bandwidth in MBytes/s
8511  */
8512 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8513 {
8514         /* Calculate raw DRAM Bandwidth */
8515         fixed20_12 dram_efficiency; /* 0.7 */
8516         fixed20_12 yclk, dram_channels, bandwidth;
8517         fixed20_12 a;
8518
8519         a.full = dfixed_const(1000);
8520         yclk.full = dfixed_const(wm->yclk);
8521         yclk.full = dfixed_div(yclk, a);
8522         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8523         a.full = dfixed_const(10);
8524         dram_efficiency.full = dfixed_const(7);
8525         dram_efficiency.full = dfixed_div(dram_efficiency, a);
8526         bandwidth.full = dfixed_mul(dram_channels, yclk);
8527         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8528
8529         return dfixed_trunc(bandwidth);
8530 }
8531
8532 /**
8533  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8534  *
8535  * @wm: watermark calculation data
8536  *
8537  * Calculate the dram bandwidth used for display (CIK).
8538  * Used for display watermark bandwidth calculations
8539  * Returns the dram bandwidth for display in MBytes/s
8540  */
8541 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8542 {
8543         /* Calculate DRAM Bandwidth and the part allocated to display. */
8544         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8545         fixed20_12 yclk, dram_channels, bandwidth;
8546         fixed20_12 a;
8547
8548         a.full = dfixed_const(1000);
8549         yclk.full = dfixed_const(wm->yclk);
8550         yclk.full = dfixed_div(yclk, a);
8551         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8552         a.full = dfixed_const(10);
8553         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8554         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8555         bandwidth.full = dfixed_mul(dram_channels, yclk);
8556         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8557
8558         return dfixed_trunc(bandwidth);
8559 }
8560
8561 /**
8562  * dce8_data_return_bandwidth - get the data return bandwidth
8563  *
8564  * @wm: watermark calculation data
8565  *
8566  * Calculate the data return bandwidth used for display (CIK).
8567  * Used for display watermark bandwidth calculations
8568  * Returns the data return bandwidth in MBytes/s
8569  */
8570 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8571 {
8572         /* Calculate the display Data return Bandwidth */
8573         fixed20_12 return_efficiency; /* 0.8 */
8574         fixed20_12 sclk, bandwidth;
8575         fixed20_12 a;
8576
8577         a.full = dfixed_const(1000);
8578         sclk.full = dfixed_const(wm->sclk);
8579         sclk.full = dfixed_div(sclk, a);
8580         a.full = dfixed_const(10);
8581         return_efficiency.full = dfixed_const(8);
8582         return_efficiency.full = dfixed_div(return_efficiency, a);
8583         a.full = dfixed_const(32);
8584         bandwidth.full = dfixed_mul(a, sclk);
8585         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8586
8587         return dfixed_trunc(bandwidth);
8588 }
8589
8590 /**
8591  * dce8_dmif_request_bandwidth - get the dmif bandwidth
8592  *
8593  * @wm: watermark calculation data
8594  *
8595  * Calculate the dmif bandwidth used for display (CIK).
8596  * Used for display watermark bandwidth calculations
8597  * Returns the dmif bandwidth in MBytes/s
8598  */
8599 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8600 {
8601         /* Calculate the DMIF Request Bandwidth */
8602         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8603         fixed20_12 disp_clk, bandwidth;
8604         fixed20_12 a, b;
8605
8606         a.full = dfixed_const(1000);
8607         disp_clk.full = dfixed_const(wm->disp_clk);
8608         disp_clk.full = dfixed_div(disp_clk, a);
8609         a.full = dfixed_const(32);
8610         b.full = dfixed_mul(a, disp_clk);
8611
8612         a.full = dfixed_const(10);
8613         disp_clk_request_efficiency.full = dfixed_const(8);
8614         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8615
8616         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8617
8618         return dfixed_trunc(bandwidth);
8619 }
8620
8621 /**
8622  * dce8_available_bandwidth - get the min available bandwidth
8623  *
8624  * @wm: watermark calculation data
8625  *
8626  * Calculate the min available bandwidth used for display (CIK).
8627  * Used for display watermark bandwidth calculations
8628  * Returns the min available bandwidth in MBytes/s
8629  */
8630 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8631 {
8632         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8633         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8634         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8635         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8636
8637         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8638 }
8639
8640 /**
8641  * dce8_average_bandwidth - get the average available bandwidth
8642  *
8643  * @wm: watermark calculation data
8644  *
8645  * Calculate the average available bandwidth used for display (CIK).
8646  * Used for display watermark bandwidth calculations
8647  * Returns the average available bandwidth in MBytes/s
8648  */
8649 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8650 {
8651         /* Calculate the display mode Average Bandwidth
8652          * DisplayMode should contain the source and destination dimensions,
8653          * timing, etc.
8654          */
8655         fixed20_12 bpp;
8656         fixed20_12 line_time;
8657         fixed20_12 src_width;
8658         fixed20_12 bandwidth;
8659         fixed20_12 a;
8660
8661         a.full = dfixed_const(1000);
8662         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8663         line_time.full = dfixed_div(line_time, a);
8664         bpp.full = dfixed_const(wm->bytes_per_pixel);
8665         src_width.full = dfixed_const(wm->src_width);
8666         bandwidth.full = dfixed_mul(src_width, bpp);
8667         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8668         bandwidth.full = dfixed_div(bandwidth, line_time);
8669
8670         return dfixed_trunc(bandwidth);
8671 }
8672
8673 /**
8674  * dce8_latency_watermark - get the latency watermark
8675  *
8676  * @wm: watermark calculation data
8677  *
8678  * Calculate the latency watermark (CIK).
8679  * Used for display watermark bandwidth calculations
8680  * Returns the latency watermark in ns
8681  */
8682 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8683 {
8684         /* First calculate the latency in ns */
8685         u32 mc_latency = 2000; /* 2000 ns. */
8686         u32 available_bandwidth = dce8_available_bandwidth(wm);
8687         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8688         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8689         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8690         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8691                 (wm->num_heads * cursor_line_pair_return_time);
8692         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8693         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8694         u32 tmp, dmif_size = 12288;
8695         fixed20_12 a, b, c;
8696
8697         if (wm->num_heads == 0)
8698                 return 0;
8699
8700         a.full = dfixed_const(2);
8701         b.full = dfixed_const(1);
8702         if ((wm->vsc.full > a.full) ||
8703             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8704             (wm->vtaps >= 5) ||
8705             ((wm->vsc.full >= a.full) && wm->interlaced))
8706                 max_src_lines_per_dst_line = 4;
8707         else
8708                 max_src_lines_per_dst_line = 2;
8709
8710         a.full = dfixed_const(available_bandwidth);
8711         b.full = dfixed_const(wm->num_heads);
8712         a.full = dfixed_div(a, b);
8713
8714         b.full = dfixed_const(mc_latency + 512);
8715         c.full = dfixed_const(wm->disp_clk);
8716         b.full = dfixed_div(b, c);
8717
8718         c.full = dfixed_const(dmif_size);
8719         b.full = dfixed_div(c, b);
8720
8721         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8722
8723         b.full = dfixed_const(1000);
8724         c.full = dfixed_const(wm->disp_clk);
8725         b.full = dfixed_div(c, b);
8726         c.full = dfixed_const(wm->bytes_per_pixel);
8727         b.full = dfixed_mul(b, c);
8728
8729         lb_fill_bw = min(tmp, dfixed_trunc(b));
8730
8731         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8732         b.full = dfixed_const(1000);
8733         c.full = dfixed_const(lb_fill_bw);
8734         b.full = dfixed_div(c, b);
8735         a.full = dfixed_div(a, b);
8736         line_fill_time = dfixed_trunc(a);
8737
8738         if (line_fill_time < wm->active_time)
8739                 return latency;
8740         else
8741                 return latency + (line_fill_time - wm->active_time);
8742
8743 }
8744
8745 /**
8746  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8747  * average and available dram bandwidth
8748  *
8749  * @wm: watermark calculation data
8750  *
8751  * Check if the display average bandwidth fits in the display
8752  * dram bandwidth (CIK).
8753  * Used for display watermark bandwidth calculations
8754  * Returns true if the display fits, false if not.
8755  */
8756 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8757 {
8758         if (dce8_average_bandwidth(wm) <=
8759             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8760                 return true;
8761         else
8762                 return false;
8763 }
8764
8765 /**
8766  * dce8_average_bandwidth_vs_available_bandwidth - check
8767  * average and available bandwidth
8768  *
8769  * @wm: watermark calculation data
8770  *
8771  * Check if the display average bandwidth fits in the display
8772  * available bandwidth (CIK).
8773  * Used for display watermark bandwidth calculations
8774  * Returns true if the display fits, false if not.
8775  */
8776 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8777 {
8778         if (dce8_average_bandwidth(wm) <=
8779             (dce8_available_bandwidth(wm) / wm->num_heads))
8780                 return true;
8781         else
8782                 return false;
8783 }
8784
8785 /**
8786  * dce8_check_latency_hiding - check latency hiding
8787  *
8788  * @wm: watermark calculation data
8789  *
8790  * Check latency hiding (CIK).
8791  * Used for display watermark bandwidth calculations
8792  * Returns true if the display fits, false if not.
8793  */
8794 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8795 {
8796         u32 lb_partitions = wm->lb_size / wm->src_width;
8797         u32 line_time = wm->active_time + wm->blank_time;
8798         u32 latency_tolerant_lines;
8799         u32 latency_hiding;
8800         fixed20_12 a;
8801
8802         a.full = dfixed_const(1);
8803         if (wm->vsc.full > a.full)
8804                 latency_tolerant_lines = 1;
8805         else {
8806                 if (lb_partitions <= (wm->vtaps + 1))
8807                         latency_tolerant_lines = 1;
8808                 else
8809                         latency_tolerant_lines = 2;
8810         }
8811
8812         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8813
8814         if (dce8_latency_watermark(wm) <= latency_hiding)
8815                 return true;
8816         else
8817                 return false;
8818 }
8819
8820 /**
8821  * dce8_program_watermarks - program display watermarks
8822  *
8823  * @rdev: radeon_device pointer
8824  * @radeon_crtc: the selected display controller
8825  * @lb_size: line buffer size
8826  * @num_heads: number of display controllers in use
8827  *
8828  * Calculate and program the display watermarks for the
8829  * selected display controller (CIK).
8830  */
8831 static void dce8_program_watermarks(struct radeon_device *rdev,
8832                                     struct radeon_crtc *radeon_crtc,
8833                                     u32 lb_size, u32 num_heads)
8834 {
8835         struct drm_display_mode *mode = &radeon_crtc->base.mode;
8836         struct dce8_wm_params wm_low, wm_high;
8837         u32 pixel_period;
8838         u32 line_time = 0;
8839         u32 latency_watermark_a = 0, latency_watermark_b = 0;
8840         u32 tmp, wm_mask;
8841
8842         if (radeon_crtc->base.enabled && num_heads && mode) {
8843                 pixel_period = 1000000 / (u32)mode->clock;
8844                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8845
8846                 /* watermark for high clocks */
8847                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8848                     rdev->pm.dpm_enabled) {
8849                         wm_high.yclk =
8850                                 radeon_dpm_get_mclk(rdev, false) * 10;
8851                         wm_high.sclk =
8852                                 radeon_dpm_get_sclk(rdev, false) * 10;
8853                 } else {
8854                         wm_high.yclk = rdev->pm.current_mclk * 10;
8855                         wm_high.sclk = rdev->pm.current_sclk * 10;
8856                 }
8857
8858                 wm_high.disp_clk = mode->clock;
8859                 wm_high.src_width = mode->crtc_hdisplay;
8860                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8861                 wm_high.blank_time = line_time - wm_high.active_time;
8862                 wm_high.interlaced = false;
8863                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8864                         wm_high.interlaced = true;
8865                 wm_high.vsc = radeon_crtc->vsc;
8866                 wm_high.vtaps = 1;
8867                 if (radeon_crtc->rmx_type != RMX_OFF)
8868                         wm_high.vtaps = 2;
8869                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8870                 wm_high.lb_size = lb_size;
8871                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8872                 wm_high.num_heads = num_heads;
8873
8874                 /* set for high clocks */
8875                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8876
8877                 /* possibly force display priority to high */
8878                 /* should really do this at mode validation time... */
8879                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8880                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8881                     !dce8_check_latency_hiding(&wm_high) ||
8882                     (rdev->disp_priority == 2)) {
8883                         DRM_DEBUG_KMS("force priority to high\n");
8884                 }
8885
8886                 /* watermark for low clocks */
8887                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8888                     rdev->pm.dpm_enabled) {
8889                         wm_low.yclk =
8890                                 radeon_dpm_get_mclk(rdev, true) * 10;
8891                         wm_low.sclk =
8892                                 radeon_dpm_get_sclk(rdev, true) * 10;
8893                 } else {
8894                         wm_low.yclk = rdev->pm.current_mclk * 10;
8895                         wm_low.sclk = rdev->pm.current_sclk * 10;
8896                 }
8897
8898                 wm_low.disp_clk = mode->clock;
8899                 wm_low.src_width = mode->crtc_hdisplay;
8900                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8901                 wm_low.blank_time = line_time - wm_low.active_time;
8902                 wm_low.interlaced = false;
8903                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8904                         wm_low.interlaced = true;
8905                 wm_low.vsc = radeon_crtc->vsc;
8906                 wm_low.vtaps = 1;
8907                 if (radeon_crtc->rmx_type != RMX_OFF)
8908                         wm_low.vtaps = 2;
8909                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8910                 wm_low.lb_size = lb_size;
8911                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8912                 wm_low.num_heads = num_heads;
8913
8914                 /* set for low clocks */
8915                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8916
8917                 /* possibly force display priority to high */
8918                 /* should really do this at mode validation time... */
8919                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8920                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8921                     !dce8_check_latency_hiding(&wm_low) ||
8922                     (rdev->disp_priority == 2)) {
8923                         DRM_DEBUG_KMS("force priority to high\n");
8924                 }
8925         }
8926
8927         /* select wm A */
8928         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8929         tmp = wm_mask;
8930         tmp &= ~LATENCY_WATERMARK_MASK(3);
8931         tmp |= LATENCY_WATERMARK_MASK(1);
8932         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8933         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8934                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8935                 LATENCY_HIGH_WATERMARK(line_time)));
8936         /* select wm B */
8937         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8938         tmp &= ~LATENCY_WATERMARK_MASK(3);
8939         tmp |= LATENCY_WATERMARK_MASK(2);
8940         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8941         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8942                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8943                 LATENCY_HIGH_WATERMARK(line_time)));
8944         /* restore original selection */
8945         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
8946
8947         /* save values for DPM */
8948         radeon_crtc->line_time = line_time;
8949         radeon_crtc->wm_high = latency_watermark_a;
8950         radeon_crtc->wm_low = latency_watermark_b;
8951 }
8952
8953 /**
8954  * dce8_bandwidth_update - program display watermarks
8955  *
8956  * @rdev: radeon_device pointer
8957  *
8958  * Calculate and program the display watermarks and line
8959  * buffer allocation (CIK).
8960  */
8961 void dce8_bandwidth_update(struct radeon_device *rdev)
8962 {
8963         struct drm_display_mode *mode = NULL;
8964         u32 num_heads = 0, lb_size;
8965         int i;
8966
8967         radeon_update_display_priority(rdev);
8968
8969         for (i = 0; i < rdev->num_crtc; i++) {
8970                 if (rdev->mode_info.crtcs[i]->base.enabled)
8971                         num_heads++;
8972         }
8973         for (i = 0; i < rdev->num_crtc; i++) {
8974                 mode = &rdev->mode_info.crtcs[i]->base.mode;
8975                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8976                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8977         }
8978 }
8979
8980 /**
8981  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8982  *
8983  * @rdev: radeon_device pointer
8984  *
8985  * Fetches a GPU clock counter snapshot (SI).
8986  * Returns the 64 bit clock counter snapshot.
8987  */
8988 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8989 {
8990         uint64_t clock;
8991
8992         mutex_lock(&rdev->gpu_clock_mutex);
8993         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8994         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8995                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8996         mutex_unlock(&rdev->gpu_clock_mutex);
8997         return clock;
8998 }
8999
9000 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9001                               u32 cntl_reg, u32 status_reg)
9002 {
9003         int r, i;
9004         struct atom_clock_dividers dividers;
9005         uint32_t tmp;
9006
9007         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9008                                            clock, false, &dividers);
9009         if (r)
9010                 return r;
9011
9012         tmp = RREG32_SMC(cntl_reg);
9013         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9014         tmp |= dividers.post_divider;
9015         WREG32_SMC(cntl_reg, tmp);
9016
9017         for (i = 0; i < 100; i++) {
9018                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9019                         break;
9020                 mdelay(10);
9021         }
9022         if (i == 100)
9023                 return -ETIMEDOUT;
9024
9025         return 0;
9026 }
9027
9028 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9029 {
9030         int r = 0;
9031
9032         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9033         if (r)
9034                 return r;
9035
9036         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9037         return r;
9038 }
9039
9040 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9041 {
9042         int r, i;
9043         struct atom_clock_dividers dividers;
9044         u32 tmp;
9045
9046         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9047                                            ecclk, false, &dividers);
9048         if (r)
9049                 return r;
9050
9051         for (i = 0; i < 100; i++) {
9052                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9053                         break;
9054                 mdelay(10);
9055         }
9056         if (i == 100)
9057                 return -ETIMEDOUT;
9058
9059         tmp = RREG32_SMC(CG_ECLK_CNTL);
9060         tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9061         tmp |= dividers.post_divider;
9062         WREG32_SMC(CG_ECLK_CNTL, tmp);
9063
9064         for (i = 0; i < 100; i++) {
9065                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9066                         break;
9067                 mdelay(10);
9068         }
9069         if (i == 100)
9070                 return -ETIMEDOUT;
9071
9072         return 0;
9073 }
9074
9075 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9076 {
9077         struct pci_dev *root = rdev->pdev->bus->self;
9078         int bridge_pos, gpu_pos;
9079         u32 speed_cntl, mask, current_data_rate;
9080         int ret, i;
9081         u16 tmp16;
9082
9083         if (radeon_pcie_gen2 == 0)
9084                 return;
9085
9086         if (rdev->flags & RADEON_IS_IGP)
9087                 return;
9088
9089         if (!(rdev->flags & RADEON_IS_PCIE))
9090                 return;
9091
9092         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9093         if (ret != 0)
9094                 return;
9095
9096         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9097                 return;
9098
9099         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9100         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9101                 LC_CURRENT_DATA_RATE_SHIFT;
9102         if (mask & DRM_PCIE_SPEED_80) {
9103                 if (current_data_rate == 2) {
9104                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9105                         return;
9106                 }
9107                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9108         } else if (mask & DRM_PCIE_SPEED_50) {
9109                 if (current_data_rate == 1) {
9110                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9111                         return;
9112                 }
9113                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9114         }
9115
9116         bridge_pos = pci_pcie_cap(root);
9117         if (!bridge_pos)
9118                 return;
9119
9120         gpu_pos = pci_pcie_cap(rdev->pdev);
9121         if (!gpu_pos)
9122                 return;
9123
9124         if (mask & DRM_PCIE_SPEED_80) {
9125                 /* re-try equalization if gen3 is not already enabled */
9126                 if (current_data_rate != 2) {
9127                         u16 bridge_cfg, gpu_cfg;
9128                         u16 bridge_cfg2, gpu_cfg2;
9129                         u32 max_lw, current_lw, tmp;
9130
9131                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9132                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9133
9134                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9135                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9136
9137                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9138                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9139
9140                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9141                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9142                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9143
9144                         if (current_lw < max_lw) {
9145                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9146                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9147                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9148                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9149                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9150                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9151                                 }
9152                         }
9153
9154                         for (i = 0; i < 10; i++) {
9155                                 /* check status */
9156                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9157                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9158                                         break;
9159
9160                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9161                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9162
9163                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9164                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9165
9166                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9167                                 tmp |= LC_SET_QUIESCE;
9168                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9169
9170                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9171                                 tmp |= LC_REDO_EQ;
9172                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9173
9174                                 mdelay(100);
9175
9176                                 /* linkctl */
9177                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9178                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9179                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9180                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9181
9182                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9183                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9184                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9185                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9186
9187                                 /* linkctl2 */
9188                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9189                                 tmp16 &= ~((1 << 4) | (7 << 9));
9190                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9191                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9192
9193                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9194                                 tmp16 &= ~((1 << 4) | (7 << 9));
9195                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9196                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9197
9198                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9199                                 tmp &= ~LC_SET_QUIESCE;
9200                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9201                         }
9202                 }
9203         }
9204
9205         /* set the link speed */
9206         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9207         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9208         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9209
9210         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9211         tmp16 &= ~0xf;
9212         if (mask & DRM_PCIE_SPEED_80)
9213                 tmp16 |= 3; /* gen3 */
9214         else if (mask & DRM_PCIE_SPEED_50)
9215                 tmp16 |= 2; /* gen2 */
9216         else
9217                 tmp16 |= 1; /* gen1 */
9218         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9219
9220         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9221         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9222         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9223
9224         for (i = 0; i < rdev->usec_timeout; i++) {
9225                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9226                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9227                         break;
9228                 udelay(1);
9229         }
9230 }
9231
9232 static void cik_program_aspm(struct radeon_device *rdev)
9233 {
9234         u32 data, orig;
9235         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9236         bool disable_clkreq = false;
9237
9238         if (radeon_aspm == 0)
9239                 return;
9240
9241         /* XXX double check IGPs */
9242         if (rdev->flags & RADEON_IS_IGP)
9243                 return;
9244
9245         if (!(rdev->flags & RADEON_IS_PCIE))
9246                 return;
9247
9248         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9249         data &= ~LC_XMIT_N_FTS_MASK;
9250         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9251         if (orig != data)
9252                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9253
9254         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9255         data |= LC_GO_TO_RECOVERY;
9256         if (orig != data)
9257                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9258
9259         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9260         data |= P_IGNORE_EDB_ERR;
9261         if (orig != data)
9262                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9263
9264         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9265         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9266         data |= LC_PMI_TO_L1_DIS;
9267         if (!disable_l0s)
9268                 data |= LC_L0S_INACTIVITY(7);
9269
9270         if (!disable_l1) {
9271                 data |= LC_L1_INACTIVITY(7);
9272                 data &= ~LC_PMI_TO_L1_DIS;
9273                 if (orig != data)
9274                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9275
9276                 if (!disable_plloff_in_l1) {
9277                         bool clk_req_support;
9278
9279                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9280                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9281                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9282                         if (orig != data)
9283                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9284
9285                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9286                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9287                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9288                         if (orig != data)
9289                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9290
9291                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9292                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9293                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9294                         if (orig != data)
9295                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9296
9297                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9298                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9299                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9300                         if (orig != data)
9301                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9302
9303                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9304                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9305                         data |= LC_DYN_LANES_PWR_STATE(3);
9306                         if (orig != data)
9307                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9308
9309                         if (!disable_clkreq) {
9310                                 struct pci_dev *root = rdev->pdev->bus->self;
9311                                 u32 lnkcap;
9312
9313                                 clk_req_support = false;
9314                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9315                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9316                                         clk_req_support = true;
9317                         } else {
9318                                 clk_req_support = false;
9319                         }
9320
9321                         if (clk_req_support) {
9322                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9323                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9324                                 if (orig != data)
9325                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9326
9327                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
9328                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9329                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9330                                 if (orig != data)
9331                                         WREG32_SMC(THM_CLK_CNTL, data);
9332
9333                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9334                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9335                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9336                                 if (orig != data)
9337                                         WREG32_SMC(MISC_CLK_CTRL, data);
9338
9339                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9340                                 data &= ~BCLK_AS_XCLK;
9341                                 if (orig != data)
9342                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
9343
9344                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9345                                 data &= ~FORCE_BIF_REFCLK_EN;
9346                                 if (orig != data)
9347                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9348
9349                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9350                                 data &= ~MPLL_CLKOUT_SEL_MASK;
9351                                 data |= MPLL_CLKOUT_SEL(4);
9352                                 if (orig != data)
9353                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9354                         }
9355                 }
9356         } else {
9357                 if (orig != data)
9358                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9359         }
9360
9361         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9362         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9363         if (orig != data)
9364                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9365
9366         if (!disable_l0s) {
9367                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9368                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9369                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9370                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9371                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9372                                 data &= ~LC_L0S_INACTIVITY_MASK;
9373                                 if (orig != data)
9374                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9375                         }
9376                 }
9377         }
9378 }