drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 static const u32 tn_rlc_save_restore_register_list[] =
  40 {
  41         0x98fc,
  42         0x98f0,
  43         0x9834,
  44         0x9838,
  45         0x9870,
  46         0x9874,
  47         0x8a14,
  48         0x8b24,
  49         0x8bcc,
  50         0x8b10,
  51         0x8c30,
  52         0x8d00,
  53         0x8d04,
  54         0x8c00,
  55         0x8c04,
  56         0x8c10,
  57         0x8c14,
  58         0x8d8c,
  59         0x8cf0,
  60         0x8e38,
  61         0x9508,
  62         0x9688,
  63         0x9608,
  64         0x960c,
  65         0x9610,
  66         0x9614,
  67         0x88c4,
  68         0x8978,
  69         0x88d4,
  70         0x900c,
  71         0x9100,
  72         0x913c,
  73         0x90e8,
  74         0x9354,
  75         0xa008,
  76         0x98f8,
  77         0x9148,
  78         0x914c,
  79         0x3f94,
  80         0x98f4,
  81         0x9b7c,
  82         0x3f8c,
  83         0x8950,
  84         0x8954,
  85         0x8a18,
  86         0x8b28,
  87         0x9144,
  88         0x3f90,
  89         0x915c,
  90         0x9160,
  91         0x9178,
  92         0x917c,
  93         0x9180,
  94         0x918c,
  95         0x9190,
  96         0x9194,
  97         0x9198,
  98         0x919c,
  99         0x91a8,
 100         0x91ac,
 101         0x91b0,
 102         0x91b4,
 103         0x91b8,
 104         0x91c4,
 105         0x91c8,
 106         0x91cc,
 107         0x91d0,
 108         0x91d4,
 109         0x91e0,
 110         0x91e4,
 111         0x91ec,
 112         0x91f0,
 113         0x91f4,
 114         0x9200,
 115         0x9204,
 116         0x929c,
 117         0x8030,
 118         0x9150,
 119         0x9a60,
 120         0x920c,
 121         0x9210,
 122         0x9228,
 123         0x922c,
 124         0x9244,
 125         0x9248,
 126         0x91e8,
 127         0x9294,
 128         0x9208,
 129         0x9224,
 130         0x9240,
 131         0x9220,
 132         0x923c,
 133         0x9258,
 134         0x9744,
 135         0xa200,
 136         0xa204,
 137         0xa208,
 138         0xa20c,
 139         0x8d58,
 140         0x9030,
 141         0x9034,
 142         0x9038,
 143         0x903c,
 144         0x9040,
 145         0x9654,
 146         0x897c,
 147         0xa210,
 148         0xa214,
 149         0x9868,
 150         0xa02c,
 151         0x9664,
 152         0x9698,
 153         0x949c,
 154         0x8e10,
 155         0x8e18,
 156         0x8c50,
 157         0x8c58,
 158         0x8c60,
 159         0x8c68,
 160         0x89b4,
 161         0x9830,
 162         0x802c,
 163 };
 164
 165 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 166 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 167 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 168 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 169 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 170 extern void evergreen_mc_program(struct radeon_device *rdev);
 171 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 172 extern int evergreen_mc_init(struct radeon_device *rdev);
 173 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 174 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 175 extern void evergreen_program_aspm(struct radeon_device *rdev);
 176 extern void sumo_rlc_fini(struct radeon_device *rdev);
 177 extern int sumo_rlc_init(struct radeon_device *rdev);
 178 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 179
 180 /* Firmware Names */
 181 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 182 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 183 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 184 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 185 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 186 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 187 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 188 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 189 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 190 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 191 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 192 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 193 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 194 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 195 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 196 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 197 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 198 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 199 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 200 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 201 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 202
 203
 204 static const u32 cayman_golden_registers2[] =
 205 {
 206         0x3e5c, 0xffffffff, 0x00000000,
 207         0x3e48, 0xffffffff, 0x00000000,
 208         0x3e4c, 0xffffffff, 0x00000000,
 209         0x3e64, 0xffffffff, 0x00000000,
 210         0x3e50, 0xffffffff, 0x00000000,
 211         0x3e60, 0xffffffff, 0x00000000
 212 };
 213
 214 static const u32 cayman_golden_registers[] =
 215 {
 216         0x5eb4, 0xffffffff, 0x00000002,
 217         0x5e78, 0x8f311ff1, 0x001000f0,
 218         0x3f90, 0xffff0000, 0xff000000,
 219         0x9148, 0xffff0000, 0xff000000,
 220         0x3f94, 0xffff0000, 0xff000000,
 221         0x914c, 0xffff0000, 0xff000000,
 222         0xc78, 0x00000080, 0x00000080,
 223         0xbd4, 0x70073777, 0x00011003,
 224         0xd02c, 0xbfffff1f, 0x08421000,
 225         0xd0b8, 0x73773777, 0x02011003,
 226         0x5bc0, 0x00200000, 0x50100000,
 227         0x98f8, 0x33773777, 0x02011003,
 228         0x98fc, 0xffffffff, 0x76541032,
 229         0x7030, 0x31000311, 0x00000011,
 230         0x2f48, 0x33773777, 0x42010001,
 231         0x6b28, 0x00000010, 0x00000012,
 232         0x7728, 0x00000010, 0x00000012,
 233         0x10328, 0x00000010, 0x00000012,
 234         0x10f28, 0x00000010, 0x00000012,
 235         0x11b28, 0x00000010, 0x00000012,
 236         0x12728, 0x00000010, 0x00000012,
 237         0x240c, 0x000007ff, 0x00000000,
 238         0x8a14, 0xf000001f, 0x00000007,
 239         0x8b24, 0x3fff3fff, 0x00ff0fff,
 240         0x8b10, 0x0000ff0f, 0x00000000,
 241         0x28a4c, 0x07ffffff, 0x06000000,
 242         0x10c, 0x00000001, 0x00010003,
 243         0xa02c, 0xffffffff, 0x0000009b,
 244         0x913c, 0x0000010f, 0x01000100,
 245         0x8c04, 0xf8ff00ff, 0x40600060,
 246         0x28350, 0x00000f01, 0x00000000,
 247         0x9508, 0x3700001f, 0x00000002,
 248         0x960c, 0xffffffff, 0x54763210,
 249         0x88c4, 0x001f3ae3, 0x00000082,
 250         0x88d0, 0xffffffff, 0x0f40df40,
 251         0x88d4, 0x0000001f, 0x00000010,
 252         0x8974, 0xffffffff, 0x00000000
 253 };
 254
 255 static const u32 dvst_golden_registers2[] =
 256 {
 257         0x8f8, 0xffffffff, 0,
 258         0x8fc, 0x00380000, 0,
 259         0x8f8, 0xffffffff, 1,
 260         0x8fc, 0x0e000000, 0
 261 };
 262
 263 static const u32 dvst_golden_registers[] =
 264 {
 265         0x690, 0x3fff3fff, 0x20c00033,
 266         0x918c, 0x0fff0fff, 0x00010006,
 267         0x91a8, 0x0fff0fff, 0x00010006,
 268         0x9150, 0xffffdfff, 0x6e944040,
 269         0x917c, 0x0fff0fff, 0x00030002,
 270         0x9198, 0x0fff0fff, 0x00030002,
 271         0x915c, 0x0fff0fff, 0x00010000,
 272         0x3f90, 0xffff0001, 0xff000000,
 273         0x9178, 0x0fff0fff, 0x00070000,
 274         0x9194, 0x0fff0fff, 0x00070000,
 275         0x9148, 0xffff0001, 0xff000000,
 276         0x9190, 0x0fff0fff, 0x00090008,
 277         0x91ac, 0x0fff0fff, 0x00090008,
 278         0x3f94, 0xffff0000, 0xff000000,
 279         0x914c, 0xffff0000, 0xff000000,
 280         0x929c, 0x00000fff, 0x00000001,
 281         0x55e4, 0xff607fff, 0xfc000100,
 282         0x8a18, 0xff000fff, 0x00000100,
 283         0x8b28, 0xff000fff, 0x00000100,
 284         0x9144, 0xfffc0fff, 0x00000100,
 285         0x6ed8, 0x00010101, 0x00010000,
 286         0x9830, 0xffffffff, 0x00000000,
 287         0x9834, 0xf00fffff, 0x00000400,
 288         0x9838, 0xfffffffe, 0x00000000,
 289         0xd0c0, 0xff000fff, 0x00000100,
 290         0xd02c, 0xbfffff1f, 0x08421000,
 291         0xd0b8, 0x73773777, 0x12010001,
 292         0x5bb0, 0x000000f0, 0x00000070,
 293         0x98f8, 0x73773777, 0x12010001,
 294         0x98fc, 0xffffffff, 0x00000010,
 295         0x9b7c, 0x00ff0000, 0x00fc0000,
 296         0x8030, 0x00001f0f, 0x0000100a,
 297         0x2f48, 0x73773777, 0x12010001,
 298         0x2408, 0x00030000, 0x000c007f,
 299         0x8a14, 0xf000003f, 0x00000007,
 300         0x8b24, 0x3fff3fff, 0x00ff0fff,
 301         0x8b10, 0x0000ff0f, 0x00000000,
 302         0x28a4c, 0x07ffffff, 0x06000000,
 303         0x4d8, 0x00000fff, 0x00000100,
 304         0xa008, 0xffffffff, 0x00010000,
 305         0x913c, 0xffff03ff, 0x01000100,
 306         0x8c00, 0x000000ff, 0x00000003,
 307         0x8c04, 0xf8ff00ff, 0x40600060,
 308         0x8cf0, 0x1fff1fff, 0x08e00410,
 309         0x28350, 0x00000f01, 0x00000000,
 310         0x9508, 0xf700071f, 0x00000002,
 311         0x960c, 0xffffffff, 0x54763210,
 312         0x20ef8, 0x01ff01ff, 0x00000002,
 313         0x20e98, 0xfffffbff, 0x00200000,
 314         0x2015c, 0xffffffff, 0x00000f40,
 315         0x88c4, 0x001f3ae3, 0x00000082,
 316         0x8978, 0x3fffffff, 0x04050140,
 317         0x88d4, 0x0000001f, 0x00000010,
 318         0x8974, 0xffffffff, 0x00000000
 319 };
 320
 321 static const u32 scrapper_golden_registers[] =
 322 {
 323         0x690, 0x3fff3fff, 0x20c00033,
 324         0x918c, 0x0fff0fff, 0x00010006,
 325         0x918c, 0x0fff0fff, 0x00010006,
 326         0x91a8, 0x0fff0fff, 0x00010006,
 327         0x91a8, 0x0fff0fff, 0x00010006,
 328         0x9150, 0xffffdfff, 0x6e944040,
 329         0x9150, 0xffffdfff, 0x6e944040,
 330         0x917c, 0x0fff0fff, 0x00030002,
 331         0x917c, 0x0fff0fff, 0x00030002,
 332         0x9198, 0x0fff0fff, 0x00030002,
 333         0x9198, 0x0fff0fff, 0x00030002,
 334         0x915c, 0x0fff0fff, 0x00010000,
 335         0x915c, 0x0fff0fff, 0x00010000,
 336         0x3f90, 0xffff0001, 0xff000000,
 337         0x3f90, 0xffff0001, 0xff000000,
 338         0x9178, 0x0fff0fff, 0x00070000,
 339         0x9178, 0x0fff0fff, 0x00070000,
 340         0x9194, 0x0fff0fff, 0x00070000,
 341         0x9194, 0x0fff0fff, 0x00070000,
 342         0x9148, 0xffff0001, 0xff000000,
 343         0x9148, 0xffff0001, 0xff000000,
 344         0x9190, 0x0fff0fff, 0x00090008,
 345         0x9190, 0x0fff0fff, 0x00090008,
 346         0x91ac, 0x0fff0fff, 0x00090008,
 347         0x91ac, 0x0fff0fff, 0x00090008,
 348         0x3f94, 0xffff0000, 0xff000000,
 349         0x3f94, 0xffff0000, 0xff000000,
 350         0x914c, 0xffff0000, 0xff000000,
 351         0x914c, 0xffff0000, 0xff000000,
 352         0x929c, 0x00000fff, 0x00000001,
 353         0x929c, 0x00000fff, 0x00000001,
 354         0x55e4, 0xff607fff, 0xfc000100,
 355         0x8a18, 0xff000fff, 0x00000100,
 356         0x8a18, 0xff000fff, 0x00000100,
 357         0x8b28, 0xff000fff, 0x00000100,
 358         0x8b28, 0xff000fff, 0x00000100,
 359         0x9144, 0xfffc0fff, 0x00000100,
 360         0x9144, 0xfffc0fff, 0x00000100,
 361         0x6ed8, 0x00010101, 0x00010000,
 362         0x9830, 0xffffffff, 0x00000000,
 363         0x9830, 0xffffffff, 0x00000000,
 364         0x9834, 0xf00fffff, 0x00000400,
 365         0x9834, 0xf00fffff, 0x00000400,
 366         0x9838, 0xfffffffe, 0x00000000,
 367         0x9838, 0xfffffffe, 0x00000000,
 368         0xd0c0, 0xff000fff, 0x00000100,
 369         0xd02c, 0xbfffff1f, 0x08421000,
 370         0xd02c, 0xbfffff1f, 0x08421000,
 371         0xd0b8, 0x73773777, 0x12010001,
 372         0xd0b8, 0x73773777, 0x12010001,
 373         0x5bb0, 0x000000f0, 0x00000070,
 374         0x98f8, 0x73773777, 0x12010001,
 375         0x98f8, 0x73773777, 0x12010001,
 376         0x98fc, 0xffffffff, 0x00000010,
 377         0x98fc, 0xffffffff, 0x00000010,
 378         0x9b7c, 0x00ff0000, 0x00fc0000,
 379         0x9b7c, 0x00ff0000, 0x00fc0000,
 380         0x8030, 0x00001f0f, 0x0000100a,
 381         0x8030, 0x00001f0f, 0x0000100a,
 382         0x2f48, 0x73773777, 0x12010001,
 383         0x2f48, 0x73773777, 0x12010001,
 384         0x2408, 0x00030000, 0x000c007f,
 385         0x8a14, 0xf000003f, 0x00000007,
 386         0x8a14, 0xf000003f, 0x00000007,
 387         0x8b24, 0x3fff3fff, 0x00ff0fff,
 388         0x8b24, 0x3fff3fff, 0x00ff0fff,
 389         0x8b10, 0x0000ff0f, 0x00000000,
 390         0x8b10, 0x0000ff0f, 0x00000000,
 391         0x28a4c, 0x07ffffff, 0x06000000,
 392         0x28a4c, 0x07ffffff, 0x06000000,
 393         0x4d8, 0x00000fff, 0x00000100,
 394         0x4d8, 0x00000fff, 0x00000100,
 395         0xa008, 0xffffffff, 0x00010000,
 396         0xa008, 0xffffffff, 0x00010000,
 397         0x913c, 0xffff03ff, 0x01000100,
 398         0x913c, 0xffff03ff, 0x01000100,
 399         0x90e8, 0x001fffff, 0x010400c0,
 400         0x8c00, 0x000000ff, 0x00000003,
 401         0x8c00, 0x000000ff, 0x00000003,
 402         0x8c04, 0xf8ff00ff, 0x40600060,
 403         0x8c04, 0xf8ff00ff, 0x40600060,
 404         0x8c30, 0x0000000f, 0x00040005,
 405         0x8cf0, 0x1fff1fff, 0x08e00410,
 406         0x8cf0, 0x1fff1fff, 0x08e00410,
 407         0x900c, 0x00ffffff, 0x0017071f,
 408         0x28350, 0x00000f01, 0x00000000,
 409         0x28350, 0x00000f01, 0x00000000,
 410         0x9508, 0xf700071f, 0x00000002,
 411         0x9508, 0xf700071f, 0x00000002,
 412         0x9688, 0x00300000, 0x0017000f,
 413         0x960c, 0xffffffff, 0x54763210,
 414         0x960c, 0xffffffff, 0x54763210,
 415         0x20ef8, 0x01ff01ff, 0x00000002,
 416         0x20e98, 0xfffffbff, 0x00200000,
 417         0x2015c, 0xffffffff, 0x00000f40,
 418         0x88c4, 0x001f3ae3, 0x00000082,
 419         0x88c4, 0x001f3ae3, 0x00000082,
 420         0x8978, 0x3fffffff, 0x04050140,
 421         0x8978, 0x3fffffff, 0x04050140,
 422         0x88d4, 0x0000001f, 0x00000010,
 423         0x88d4, 0x0000001f, 0x00000010,
 424         0x8974, 0xffffffff, 0x00000000,
 425         0x8974, 0xffffffff, 0x00000000
 426 };
 427
 428 static void ni_init_golden_registers(struct radeon_device *rdev)
 429 {
 430         switch (rdev->family) {
 431         case CHIP_CAYMAN:
 432                 radeon_program_register_sequence(rdev,
 433                                                  cayman_golden_registers,
 434                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 435                 radeon_program_register_sequence(rdev,
 436                                                  cayman_golden_registers2,
 437                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 438                 break;
 439         case CHIP_ARUBA:
 440                 if ((rdev->pdev->device == 0x9900) ||
 441                     (rdev->pdev->device == 0x9901) ||
 442                     (rdev->pdev->device == 0x9903) ||
 443                     (rdev->pdev->device == 0x9904) ||
 444                     (rdev->pdev->device == 0x9905) ||
 445                     (rdev->pdev->device == 0x9906) ||
 446                     (rdev->pdev->device == 0x9907) ||
 447                     (rdev->pdev->device == 0x9908) ||
 448                     (rdev->pdev->device == 0x9909) ||
 449                     (rdev->pdev->device == 0x990A) ||
 450                     (rdev->pdev->device == 0x990B) ||
 451                     (rdev->pdev->device == 0x990C) ||
 452                     (rdev->pdev->device == 0x990D) ||
 453                     (rdev->pdev->device == 0x990E) ||
 454                     (rdev->pdev->device == 0x990F) ||
 455                     (rdev->pdev->device == 0x9910) ||
 456                     (rdev->pdev->device == 0x9913) ||
 457                     (rdev->pdev->device == 0x9917) ||
 458                     (rdev->pdev->device == 0x9918)) {
 459                         radeon_program_register_sequence(rdev,
 460                                                          dvst_golden_registers,
 461                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 462                         radeon_program_register_sequence(rdev,
 463                                                          dvst_golden_registers2,
 464                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 465                 } else {
 466                         radeon_program_register_sequence(rdev,
 467                                                          scrapper_golden_registers,
 468                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 469                         radeon_program_register_sequence(rdev,
 470                                                          dvst_golden_registers2,
 471                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 472                 }
 473                 break;
 474         default:
 475                 break;
 476         }
 477 }
 478
 479 #define BTC_IO_MC_REGS_SIZE 29
 480
 481 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 482         {0x00000077, 0xff010100},
 483         {0x00000078, 0x00000000},
 484         {0x00000079, 0x00001434},
 485         {0x0000007a, 0xcc08ec08},
 486         {0x0000007b, 0x00040000},
 487         {0x0000007c, 0x000080c0},
 488         {0x0000007d, 0x09000000},
 489         {0x0000007e, 0x00210404},
 490         {0x00000081, 0x08a8e800},
 491         {0x00000082, 0x00030444},
 492         {0x00000083, 0x00000000},
 493         {0x00000085, 0x00000001},
 494         {0x00000086, 0x00000002},
 495         {0x00000087, 0x48490000},
 496         {0x00000088, 0x20244647},
 497         {0x00000089, 0x00000005},
 498         {0x0000008b, 0x66030000},
 499         {0x0000008c, 0x00006603},
 500         {0x0000008d, 0x00000100},
 501         {0x0000008f, 0x00001c0a},
 502         {0x00000090, 0xff000001},
 503         {0x00000094, 0x00101101},
 504         {0x00000095, 0x00000fff},
 505         {0x00000096, 0x00116fff},
 506         {0x00000097, 0x60010000},
 507         {0x00000098, 0x10010000},
 508         {0x00000099, 0x00006000},
 509         {0x0000009a, 0x00001000},
 510         {0x0000009f, 0x00946a00}
 511 };
 512
 513 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 514         {0x00000077, 0xff010100},
 515         {0x00000078, 0x00000000},
 516         {0x00000079, 0x00001434},
 517         {0x0000007a, 0xcc08ec08},
 518         {0x0000007b, 0x00040000},
 519         {0x0000007c, 0x000080c0},
 520         {0x0000007d, 0x09000000},
 521         {0x0000007e, 0x00210404},
 522         {0x00000081, 0x08a8e800},
 523         {0x00000082, 0x00030444},
 524         {0x00000083, 0x00000000},
 525         {0x00000085, 0x00000001},
 526         {0x00000086, 0x00000002},
 527         {0x00000087, 0x48490000},
 528         {0x00000088, 0x20244647},
 529         {0x00000089, 0x00000005},
 530         {0x0000008b, 0x66030000},
 531         {0x0000008c, 0x00006603},
 532         {0x0000008d, 0x00000100},
 533         {0x0000008f, 0x00001c0a},
 534         {0x00000090, 0xff000001},
 535         {0x00000094, 0x00101101},
 536         {0x00000095, 0x00000fff},
 537         {0x00000096, 0x00116fff},
 538         {0x00000097, 0x60010000},
 539         {0x00000098, 0x10010000},
 540         {0x00000099, 0x00006000},
 541         {0x0000009a, 0x00001000},
 542         {0x0000009f, 0x00936a00}
 543 };
 544
 545 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 546         {0x00000077, 0xff010100},
 547         {0x00000078, 0x00000000},
 548         {0x00000079, 0x00001434},
 549         {0x0000007a, 0xcc08ec08},
 550         {0x0000007b, 0x00040000},
 551         {0x0000007c, 0x000080c0},
 552         {0x0000007d, 0x09000000},
 553         {0x0000007e, 0x00210404},
 554         {0x00000081, 0x08a8e800},
 555         {0x00000082, 0x00030444},
 556         {0x00000083, 0x00000000},
 557         {0x00000085, 0x00000001},
 558         {0x00000086, 0x00000002},
 559         {0x00000087, 0x48490000},
 560         {0x00000088, 0x20244647},
 561         {0x00000089, 0x00000005},
 562         {0x0000008b, 0x66030000},
 563         {0x0000008c, 0x00006603},
 564         {0x0000008d, 0x00000100},
 565         {0x0000008f, 0x00001c0a},
 566         {0x00000090, 0xff000001},
 567         {0x00000094, 0x00101101},
 568         {0x00000095, 0x00000fff},
 569         {0x00000096, 0x00116fff},
 570         {0x00000097, 0x60010000},
 571         {0x00000098, 0x10010000},
 572         {0x00000099, 0x00006000},
 573         {0x0000009a, 0x00001000},
 574         {0x0000009f, 0x00916a00}
 575 };
 576
 577 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 578         {0x00000077, 0xff010100},
 579         {0x00000078, 0x00000000},
 580         {0x00000079, 0x00001434},
 581         {0x0000007a, 0xcc08ec08},
 582         {0x0000007b, 0x00040000},
 583         {0x0000007c, 0x000080c0},
 584         {0x0000007d, 0x09000000},
 585         {0x0000007e, 0x00210404},
 586         {0x00000081, 0x08a8e800},
 587         {0x00000082, 0x00030444},
 588         {0x00000083, 0x00000000},
 589         {0x00000085, 0x00000001},
 590         {0x00000086, 0x00000002},
 591         {0x00000087, 0x48490000},
 592         {0x00000088, 0x20244647},
 593         {0x00000089, 0x00000005},
 594         {0x0000008b, 0x66030000},
 595         {0x0000008c, 0x00006603},
 596         {0x0000008d, 0x00000100},
 597         {0x0000008f, 0x00001c0a},
 598         {0x00000090, 0xff000001},
 599         {0x00000094, 0x00101101},
 600         {0x00000095, 0x00000fff},
 601         {0x00000096, 0x00116fff},
 602         {0x00000097, 0x60010000},
 603         {0x00000098, 0x10010000},
 604         {0x00000099, 0x00006000},
 605         {0x0000009a, 0x00001000},
 606         {0x0000009f, 0x00976b00}
 607 };
 608
 609 int ni_mc_load_microcode(struct radeon_device *rdev)
 610 {
 611         const __be32 *fw_data;
 612         u32 mem_type, running, blackout = 0;
 613         u32 *io_mc_regs;
 614         int i, ucode_size, regs_size;
 615
 616         if (!rdev->mc_fw)
 617                 return -EINVAL;
 618
 619         switch (rdev->family) {
 620         case CHIP_BARTS:
 621                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 622                 ucode_size = BTC_MC_UCODE_SIZE;
 623                 regs_size = BTC_IO_MC_REGS_SIZE;
 624                 break;
 625         case CHIP_TURKS:
 626                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 627                 ucode_size = BTC_MC_UCODE_SIZE;
 628                 regs_size = BTC_IO_MC_REGS_SIZE;
 629                 break;
 630         case CHIP_CAICOS:
 631         default:
 632                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 633                 ucode_size = BTC_MC_UCODE_SIZE;
 634                 regs_size = BTC_IO_MC_REGS_SIZE;
 635                 break;
 636         case CHIP_CAYMAN:
 637                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 638                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 639                 regs_size = BTC_IO_MC_REGS_SIZE;
 640                 break;
 641         }
 642
 643         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 644         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 645
 646         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 647                 if (running) {
 648                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 649                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 650                 }
 651
 652                 /* reset the engine and set to writable */
 653                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 654                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 655
 656                 /* load mc io regs */
 657                 for (i = 0; i < regs_size; i++) {
 658                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 659                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 660                 }
 661                 /* load the MC ucode */
 662                 fw_data = (const __be32 *)rdev->mc_fw->data;
 663                 for (i = 0; i < ucode_size; i++)
 664                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 665
 666                 /* put the engine back into the active state */
 667                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 668                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 669                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 670
 671                 /* wait for training to complete */
 672                 for (i = 0; i < rdev->usec_timeout; i++) {
 673                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 674                                 break;
 675                         udelay(1);
 676                 }
 677
 678                 if (running)
 679                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 680         }
 681
 682         return 0;
 683 }
 684
 685 int ni_init_microcode(struct radeon_device *rdev)
 686 {
 687         const char *chip_name;
 688         const char *rlc_chip_name;
 689         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 690         size_t smc_req_size = 0;
 691         char fw_name[30];
 692         int err;
 693
 694         DRM_DEBUG("\n");
 695
 696         switch (rdev->family) {
 697         case CHIP_BARTS:
 698                 chip_name = "BARTS";
 699                 rlc_chip_name = "BTC";
 700                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 701                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 702                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 703                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 704                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 705                 break;
 706         case CHIP_TURKS:
 707                 chip_name = "TURKS";
 708                 rlc_chip_name = "BTC";
 709                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 710                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 711                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 712                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 713                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 714                 break;
 715         case CHIP_CAICOS:
 716                 chip_name = "CAICOS";
 717                 rlc_chip_name = "BTC";
 718                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 719                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 720                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 721                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 722                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 723                 break;
 724         case CHIP_CAYMAN:
 725                 chip_name = "CAYMAN";
 726                 rlc_chip_name = "CAYMAN";
 727                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 728                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 729                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 730                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 731                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 732                 break;
 733         case CHIP_ARUBA:
 734                 chip_name = "ARUBA";
 735                 rlc_chip_name = "ARUBA";
 736                 /* pfp/me same size as CAYMAN */
 737                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 738                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 739                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 740                 mc_req_size = 0;
 741                 break;
 742         default: BUG();
 743         }
 744
 745         DRM_INFO("Loading %s Microcode\n", chip_name);
 746
 747         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 748         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 749         if (err)
 750                 goto out;
 751         if (rdev->pfp_fw->size != pfp_req_size) {
 752                 printk(KERN_ERR
 753                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 754                        rdev->pfp_fw->size, fw_name);
 755                 err = -EINVAL;
 756                 goto out;
 757         }
 758
 759         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 760         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 761         if (err)
 762                 goto out;
 763         if (rdev->me_fw->size != me_req_size) {
 764                 printk(KERN_ERR
 765                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 766                        rdev->me_fw->size, fw_name);
 767                 err = -EINVAL;
 768         }
 769
 770         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 771         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 772         if (err)
 773                 goto out;
 774         if (rdev->rlc_fw->size != rlc_req_size) {
 775                 printk(KERN_ERR
 776                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 777                        rdev->rlc_fw->size, fw_name);
 778                 err = -EINVAL;
 779         }
 780
 781         /* no MC ucode on TN */
 782         if (!(rdev->flags & RADEON_IS_IGP)) {
 783                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 784                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 785                 if (err)
 786                         goto out;
 787                 if (rdev->mc_fw->size != mc_req_size) {
 788                         printk(KERN_ERR
 789                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 790                                rdev->mc_fw->size, fw_name);
 791                         err = -EINVAL;
 792                 }
 793         }
 794
 795         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 796                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 797                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 798                 if (err) {
 799                         printk(KERN_ERR
 800                                "smc: error loading firmware \"%s\"\n",
 801                                fw_name);
 802                         release_firmware(rdev->smc_fw);
 803                         rdev->smc_fw = NULL;
 804                         err = 0;
 805                 } else if (rdev->smc_fw->size != smc_req_size) {
 806                         printk(KERN_ERR
 807                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 808                                rdev->mc_fw->size, fw_name);
 809                         err = -EINVAL;
 810                 }
 811         }
 812
 813 out:
 814         if (err) {
 815                 if (err != -EINVAL)
 816                         printk(KERN_ERR
 817                                "ni_cp: Failed to load firmware \"%s\"\n",
 818                                fw_name);
 819                 release_firmware(rdev->pfp_fw);
 820                 rdev->pfp_fw = NULL;
 821                 release_firmware(rdev->me_fw);
 822                 rdev->me_fw = NULL;
 823                 release_firmware(rdev->rlc_fw);
 824                 rdev->rlc_fw = NULL;
 825                 release_firmware(rdev->mc_fw);
 826                 rdev->mc_fw = NULL;
 827         }
 828         return err;
 829 }
 830
 831 int tn_get_temp(struct radeon_device *rdev)
 832 {
 833         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 834         int actual_temp = (temp / 8) - 49;
 835
 836         return actual_temp * 1000;
 837 }
 838
 839 /*
 840  * Core functions
 841  */
 842 static void cayman_gpu_init(struct radeon_device *rdev)
 843 {
 844         u32 gb_addr_config = 0;
 845         u32 mc_shared_chmap, mc_arb_ramcfg;
 846         u32 cgts_tcc_disable;
 847         u32 sx_debug_1;
 848         u32 smx_dc_ctl0;
 849         u32 cgts_sm_ctrl_reg;
 850         u32 hdp_host_path_cntl;
 851         u32 tmp;
 852         u32 disabled_rb_mask;
 853         int i, j;
 854
 855         switch (rdev->family) {
 856         case CHIP_CAYMAN:
 857                 rdev->config.cayman.max_shader_engines = 2;
 858                 rdev->config.cayman.max_pipes_per_simd = 4;
 859                 rdev->config.cayman.max_tile_pipes = 8;
 860                 rdev->config.cayman.max_simds_per_se = 12;
 861                 rdev->config.cayman.max_backends_per_se = 4;
 862                 rdev->config.cayman.max_texture_channel_caches = 8;
 863                 rdev->config.cayman.max_gprs = 256;
 864                 rdev->config.cayman.max_threads = 256;
 865                 rdev->config.cayman.max_gs_threads = 32;
 866                 rdev->config.cayman.max_stack_entries = 512;
 867                 rdev->config.cayman.sx_num_of_sets = 8;
 868                 rdev->config.cayman.sx_max_export_size = 256;
 869                 rdev->config.cayman.sx_max_export_pos_size = 64;
 870                 rdev->config.cayman.sx_max_export_smx_size = 192;
 871                 rdev->config.cayman.max_hw_contexts = 8;
 872                 rdev->config.cayman.sq_num_cf_insts = 2;
 873
 874                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 875                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 876                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 877                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 878                 break;
 879         case CHIP_ARUBA:
 880         default:
 881                 rdev->config.cayman.max_shader_engines = 1;
 882                 rdev->config.cayman.max_pipes_per_simd = 4;
 883                 rdev->config.cayman.max_tile_pipes = 2;
 884                 if ((rdev->pdev->device == 0x9900) ||
 885                     (rdev->pdev->device == 0x9901) ||
 886                     (rdev->pdev->device == 0x9905) ||
 887                     (rdev->pdev->device == 0x9906) ||
 888                     (rdev->pdev->device == 0x9907) ||
 889                     (rdev->pdev->device == 0x9908) ||
 890                     (rdev->pdev->device == 0x9909) ||
 891                     (rdev->pdev->device == 0x990B) ||
 892                     (rdev->pdev->device == 0x990C) ||
 893                     (rdev->pdev->device == 0x990F) ||
 894                     (rdev->pdev->device == 0x9910) ||
 895                     (rdev->pdev->device == 0x9917) ||
 896                     (rdev->pdev->device == 0x9999) ||
 897                     (rdev->pdev->device == 0x999C)) {
 898                         rdev->config.cayman.max_simds_per_se = 6;
 899                         rdev->config.cayman.max_backends_per_se = 2;
 900                         rdev->config.cayman.max_hw_contexts = 8;
 901                         rdev->config.cayman.sx_max_export_size = 256;
 902                         rdev->config.cayman.sx_max_export_pos_size = 64;
 903                         rdev->config.cayman.sx_max_export_smx_size = 192;
 904                 } else if ((rdev->pdev->device == 0x9903) ||
 905                            (rdev->pdev->device == 0x9904) ||
 906                            (rdev->pdev->device == 0x990A) ||
 907                            (rdev->pdev->device == 0x990D) ||
 908                            (rdev->pdev->device == 0x990E) ||
 909                            (rdev->pdev->device == 0x9913) ||
 910                            (rdev->pdev->device == 0x9918) ||
 911                            (rdev->pdev->device == 0x999D)) {
 912                         rdev->config.cayman.max_simds_per_se = 4;
 913                         rdev->config.cayman.max_backends_per_se = 2;
 914                         rdev->config.cayman.max_hw_contexts = 8;
 915                         rdev->config.cayman.sx_max_export_size = 256;
 916                         rdev->config.cayman.sx_max_export_pos_size = 64;
 917                         rdev->config.cayman.sx_max_export_smx_size = 192;
 918                 } else if ((rdev->pdev->device == 0x9919) ||
 919                            (rdev->pdev->device == 0x9990) ||
 920                            (rdev->pdev->device == 0x9991) ||
 921                            (rdev->pdev->device == 0x9994) ||
 922                            (rdev->pdev->device == 0x9995) ||
 923                            (rdev->pdev->device == 0x9996) ||
 924                            (rdev->pdev->device == 0x999A) ||
 925                            (rdev->pdev->device == 0x99A0)) {
 926                         rdev->config.cayman.max_simds_per_se = 3;
 927                         rdev->config.cayman.max_backends_per_se = 1;
 928                         rdev->config.cayman.max_hw_contexts = 4;
 929                         rdev->config.cayman.sx_max_export_size = 128;
 930                         rdev->config.cayman.sx_max_export_pos_size = 32;
 931                         rdev->config.cayman.sx_max_export_smx_size = 96;
 932                 } else {
 933                         rdev->config.cayman.max_simds_per_se = 2;
 934                         rdev->config.cayman.max_backends_per_se = 1;
 935                         rdev->config.cayman.max_hw_contexts = 4;
 936                         rdev->config.cayman.sx_max_export_size = 128;
 937                         rdev->config.cayman.sx_max_export_pos_size = 32;
 938                         rdev->config.cayman.sx_max_export_smx_size = 96;
 939                 }
 940                 rdev->config.cayman.max_texture_channel_caches = 2;
 941                 rdev->config.cayman.max_gprs = 256;
 942                 rdev->config.cayman.max_threads = 256;
 943                 rdev->config.cayman.max_gs_threads = 32;
 944                 rdev->config.cayman.max_stack_entries = 512;
 945                 rdev->config.cayman.sx_num_of_sets = 8;
 946                 rdev->config.cayman.sq_num_cf_insts = 2;
 947
 948                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 949                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 950                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 951                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 952                 break;
 953         }
 954
 955         /* Initialize HDP */
 956         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 957                 WREG32((0x2c14 + j), 0x00000000);
 958                 WREG32((0x2c18 + j), 0x00000000);
 959                 WREG32((0x2c1c + j), 0x00000000);
 960                 WREG32((0x2c20 + j), 0x00000000);
 961                 WREG32((0x2c24 + j), 0x00000000);
 962         }
 963
 964         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 965
 966         evergreen_fix_pci_max_read_req_size(rdev);
 967
 968         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 969         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 970
 971         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 972         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 973         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 974                 rdev->config.cayman.mem_row_size_in_kb = 4;
 975         /* XXX use MC settings? */
 976         rdev->config.cayman.shader_engine_tile_size = 32;
 977         rdev->config.cayman.num_gpus = 1;
 978         rdev->config.cayman.multi_gpu_tile_size = 64;
 979
 980         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 981         rdev->config.cayman.num_tile_pipes = (1 << tmp);
 982         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
 983         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
 984         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
 985         rdev->config.cayman.num_shader_engines = tmp + 1;
 986         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
 987         rdev->config.cayman.num_gpus = tmp + 1;
 988         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
 989         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
 990         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
 991         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
 992
 993
 994         /* setup tiling info dword.  gb_addr_config is not adequate since it does
 995          * not have bank info, so create a custom tiling dword.
 996          * bits 3:0   num_pipes
 997          * bits 7:4   num_banks
 998          * bits 11:8  group_size
 999          * bits 15:12 row_size
1000          */
1001         rdev->config.cayman.tile_config = 0;
1002         switch (rdev->config.cayman.num_tile_pipes) {
1003         case 1:
1004         default:
1005                 rdev->config.cayman.tile_config |= (0 << 0);
1006                 break;
1007         case 2:
1008                 rdev->config.cayman.tile_config |= (1 << 0);
1009                 break;
1010         case 4:
1011                 rdev->config.cayman.tile_config |= (2 << 0);
1012                 break;
1013         case 8:
1014                 rdev->config.cayman.tile_config |= (3 << 0);
1015                 break;
1016         }
1017
1018         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1019         if (rdev->flags & RADEON_IS_IGP)
1020                 rdev->config.cayman.tile_config |= 1 << 4;
1021         else {
1022                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1023                 case 0: /* four banks */
1024                         rdev->config.cayman.tile_config |= 0 << 4;
1025                         break;
1026                 case 1: /* eight banks */
1027                         rdev->config.cayman.tile_config |= 1 << 4;
1028                         break;
1029                 case 2: /* sixteen banks */
1030                 default:
1031                         rdev->config.cayman.tile_config |= 2 << 4;
1032                         break;
1033                 }
1034         }
1035         rdev->config.cayman.tile_config |=
1036                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1037         rdev->config.cayman.tile_config |=
1038                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1039
1040         tmp = 0;
1041         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1042                 u32 rb_disable_bitmap;
1043
1044                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1045                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1046                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1047                 tmp <<= 4;
1048                 tmp |= rb_disable_bitmap;
1049         }
1050         /* enabled rb are just the one not disabled :) */
1051         disabled_rb_mask = tmp;
1052         tmp = 0;
1053         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1054                 tmp |= (1 << i);
1055         /* if all the backends are disabled, fix it up here */
1056         if ((disabled_rb_mask & tmp) == tmp) {
1057                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1058                         disabled_rb_mask &= ~(1 << i);
1059         }
1060
1061         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1062                 u32 simd_disable_bitmap;
1063
1064                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1065                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1066                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1067                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1068                 tmp <<= 16;
1069                 tmp |= simd_disable_bitmap;
1070         }
1071         rdev->config.cayman.active_simds = hweight32(~tmp);
1072
1073         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1074         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1075
1076         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1077         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1078         if (ASIC_IS_DCE6(rdev))
1079                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1080         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1081         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1082         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1083         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1084         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1085         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1086
1087         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1088             (rdev->flags & RADEON_IS_IGP)) {
1089                 if ((disabled_rb_mask & 3) == 1) {
1090                         /* RB0 disabled, RB1 enabled */
1091                         tmp = 0x11111111;
1092                 } else {
1093                         /* RB1 disabled, RB0 enabled */
1094                         tmp = 0x00000000;
1095                 }
1096         } else {
1097                 tmp = gb_addr_config & NUM_PIPES_MASK;
1098                 tmp = r6xx_remap_render_backend(rdev, tmp,
1099                                                 rdev->config.cayman.max_backends_per_se *
1100                                                 rdev->config.cayman.max_shader_engines,
1101                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1102         }
1103         WREG32(GB_BACKEND_MAP, tmp);
1104
1105         cgts_tcc_disable = 0xffff0000;
1106         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1107                 cgts_tcc_disable &= ~(1 << (16 + i));
1108         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1109         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1110         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1111         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1112
1113         /* reprogram the shader complex */
1114         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1115         for (i = 0; i < 16; i++)
1116                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1117         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1118
1119         /* set HW defaults for 3D engine */
1120         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1121
1122         sx_debug_1 = RREG32(SX_DEBUG_1);
1123         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1124         WREG32(SX_DEBUG_1, sx_debug_1);
1125
1126         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1127         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1128         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1129         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1130
1131         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1132
1133         /* need to be explicitly zero-ed */
1134         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1135         WREG32(SQ_LSTMP_RING_BASE, 0);
1136         WREG32(SQ_HSTMP_RING_BASE, 0);
1137         WREG32(SQ_ESTMP_RING_BASE, 0);
1138         WREG32(SQ_GSTMP_RING_BASE, 0);
1139         WREG32(SQ_VSTMP_RING_BASE, 0);
1140         WREG32(SQ_PSTMP_RING_BASE, 0);
1141
1142         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1143
1144         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1145                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1146                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1147
1148         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1149                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1150                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1151
1152
1153         WREG32(VGT_NUM_INSTANCES, 1);
1154
1155         WREG32(CP_PERFMON_CNTL, 0);
1156
1157         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1158                                   FETCH_FIFO_HIWATER(0x4) |
1159                                   DONE_FIFO_HIWATER(0xe0) |
1160                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1161
1162         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1163         WREG32(SQ_CONFIG, (VC_ENABLE |
1164                            EXPORT_SRC_C |
1165                            GFX_PRIO(0) |
1166                            CS1_PRIO(0) |
1167                            CS2_PRIO(1)));
1168         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1169
1170         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1171                                           FORCE_EOV_MAX_REZ_CNT(255)));
1172
1173         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1174                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1175
1176         WREG32(VGT_GS_VERTEX_REUSE, 16);
1177         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1178
1179         WREG32(CB_PERF_CTR0_SEL_0, 0);
1180         WREG32(CB_PERF_CTR0_SEL_1, 0);
1181         WREG32(CB_PERF_CTR1_SEL_0, 0);
1182         WREG32(CB_PERF_CTR1_SEL_1, 0);
1183         WREG32(CB_PERF_CTR2_SEL_0, 0);
1184         WREG32(CB_PERF_CTR2_SEL_1, 0);
1185         WREG32(CB_PERF_CTR3_SEL_0, 0);
1186         WREG32(CB_PERF_CTR3_SEL_1, 0);
1187
1188         tmp = RREG32(HDP_MISC_CNTL);
1189         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1190         WREG32(HDP_MISC_CNTL, tmp);
1191
1192         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1193         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1194
1195         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1196
1197         udelay(50);
1198
1199         /* set clockgating golden values on TN */
1200         if (rdev->family == CHIP_ARUBA) {
1201                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1202                 tmp &= ~0x00380000;
1203                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1204                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1205                 tmp &= ~0x0e000000;
1206                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1207         }
1208 }
1209
1210 /*
1211  * GART
1212  */
1213 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1214 {
1215         /* flush hdp cache */
1216         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1217
1218         /* bits 0-7 are the VM contexts0-7 */
1219         WREG32(VM_INVALIDATE_REQUEST, 1);
1220 }
1221
1222 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1223 {
1224         int i, r;
1225
1226         if (rdev->gart.robj == NULL) {
1227                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1228                 return -EINVAL;
1229         }
1230         r = radeon_gart_table_vram_pin(rdev);
1231         if (r)
1232                 return r;
1233         /* Setup TLB control */
1234         WREG32(MC_VM_MX_L1_TLB_CNTL,
1235                (0xA << 7) |
1236                ENABLE_L1_TLB |
1237                ENABLE_L1_FRAGMENT_PROCESSING |
1238                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1239                ENABLE_ADVANCED_DRIVER_MODEL |
1240                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1241         /* Setup L2 cache */
1242         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1243                ENABLE_L2_FRAGMENT_PROCESSING |
1244                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1245                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1246                EFFECTIVE_L2_QUEUE_SIZE(7) |
1247                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1248         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1249         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1250                BANK_SELECT(6) |
1251                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1252         /* setup context0 */
1253         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1254         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1255         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1256         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1257                         (u32)(rdev->dummy_page.addr >> 12));
1258         WREG32(VM_CONTEXT0_CNTL2, 0);
1259         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1260                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1261
1262         WREG32(0x15D4, 0);
1263         WREG32(0x15D8, 0);
1264         WREG32(0x15DC, 0);
1265
1266         /* empty context1-7 */
1267         /* Assign the pt base to something valid for now; the pts used for
1268          * the VMs are determined by the application and setup and assigned
1269          * on the fly in the vm part of radeon_gart.c
1270          */
1271         for (i = 1; i < 8; i++) {
1272                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1273                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1274                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1275                        rdev->vm_manager.saved_table_addr[i]);
1276         }
1277
1278         /* enable context1-7 */
1279         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1280                (u32)(rdev->dummy_page.addr >> 12));
1281         WREG32(VM_CONTEXT1_CNTL2, 4);
1282         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1283                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1284                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1285                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1286                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1287                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1288                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1289                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1290                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1291                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1292                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1293                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1294                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1295                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1296
1297         cayman_pcie_gart_tlb_flush(rdev);
1298         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1299                  (unsigned)(rdev->mc.gtt_size >> 20),
1300                  (unsigned long long)rdev->gart.table_addr);
1301         rdev->gart.ready = true;
1302         return 0;
1303 }
1304
1305 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1306 {
1307         unsigned i;
1308
1309         for (i = 1; i < 8; ++i) {
1310                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1311                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1312         }
1313
1314         /* Disable all tables */
1315         WREG32(VM_CONTEXT0_CNTL, 0);
1316         WREG32(VM_CONTEXT1_CNTL, 0);
1317         /* Setup TLB control */
1318         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1319                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1320                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1321         /* Setup L2 cache */
1322         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1323                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1324                EFFECTIVE_L2_QUEUE_SIZE(7) |
1325                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1326         WREG32(VM_L2_CNTL2, 0);
1327         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1328                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1329         radeon_gart_table_vram_unpin(rdev);
1330 }
1331
1332 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1333 {
1334         cayman_pcie_gart_disable(rdev);
1335         radeon_gart_table_vram_free(rdev);
1336         radeon_gart_fini(rdev);
1337 }
1338
1339 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1340                               int ring, u32 cp_int_cntl)
1341 {
1342         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1343
1344         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1345         WREG32(CP_INT_CNTL, cp_int_cntl);
1346 }
1347
1348 /*
1349  * CP.
1350  */
1351 void cayman_fence_ring_emit(struct radeon_device *rdev,
1352                             struct radeon_fence *fence)
1353 {
1354         struct radeon_ring *ring = &rdev->ring[fence->ring];
1355         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1356         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1357                 PACKET3_SH_ACTION_ENA;
1358
1359         /* flush read cache over gart for this vmid */
1360         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1361         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1362         radeon_ring_write(ring, 0xFFFFFFFF);
1363         radeon_ring_write(ring, 0);
1364         radeon_ring_write(ring, 10); /* poll interval */
1365         /* EVENT_WRITE_EOP - flush caches, send int */
1366         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1367         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1368         radeon_ring_write(ring, lower_32_bits(addr));
1369         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1370         radeon_ring_write(ring, fence->seq);
1371         radeon_ring_write(ring, 0);
1372 }
1373
1374 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1375 {
1376         struct radeon_ring *ring = &rdev->ring[ib->ring];
1377         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1378         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1379                 PACKET3_SH_ACTION_ENA;
1380
1381         /* set to DX10/11 mode */
1382         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1383         radeon_ring_write(ring, 1);
1384
1385         if (ring->rptr_save_reg) {
1386                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1387                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1388                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1389                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1390                 radeon_ring_write(ring, next_rptr);
1391         }
1392
1393         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1394         radeon_ring_write(ring,
1395 #ifdef __BIG_ENDIAN
1396                           (2 << 0) |
1397 #endif
1398                           (ib->gpu_addr & 0xFFFFFFFC));
1399         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1400         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1401
1402         /* flush read cache over gart for this vmid */
1403         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1404         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1405         radeon_ring_write(ring, 0xFFFFFFFF);
1406         radeon_ring_write(ring, 0);
1407         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1408 }
1409
1410 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1411 {
1412         if (enable)
1413                 WREG32(CP_ME_CNTL, 0);
1414         else {
1415                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1416                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1417                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1418                 WREG32(SCRATCH_UMSK, 0);
1419                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1420         }
1421 }
1422
1423 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1424                         struct radeon_ring *ring)
1425 {
1426         u32 rptr;
1427
1428         if (rdev->wb.enabled)
1429                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1430         else {
1431                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1432                         rptr = RREG32(CP_RB0_RPTR);
1433                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1434                         rptr = RREG32(CP_RB1_RPTR);
1435                 else
1436                         rptr = RREG32(CP_RB2_RPTR);
1437         }
1438
1439         return rptr;
1440 }
1441
1442 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1443                         struct radeon_ring *ring)
1444 {
1445         u32 wptr;
1446
1447         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1448                 wptr = RREG32(CP_RB0_WPTR);
1449         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1450                 wptr = RREG32(CP_RB1_WPTR);
1451         else
1452                 wptr = RREG32(CP_RB2_WPTR);
1453
1454         return wptr;
1455 }
1456
1457 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1458                          struct radeon_ring *ring)
1459 {
1460         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1461                 WREG32(CP_RB0_WPTR, ring->wptr);
1462                 (void)RREG32(CP_RB0_WPTR);
1463         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1464                 WREG32(CP_RB1_WPTR, ring->wptr);
1465                 (void)RREG32(CP_RB1_WPTR);
1466         } else {
1467                 WREG32(CP_RB2_WPTR, ring->wptr);
1468                 (void)RREG32(CP_RB2_WPTR);
1469         }
1470 }
1471
1472 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1473 {
1474         const __be32 *fw_data;
1475         int i;
1476
1477         if (!rdev->me_fw || !rdev->pfp_fw)
1478                 return -EINVAL;
1479
1480         cayman_cp_enable(rdev, false);
1481
1482         fw_data = (const __be32 *)rdev->pfp_fw->data;
1483         WREG32(CP_PFP_UCODE_ADDR, 0);
1484         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1485                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1486         WREG32(CP_PFP_UCODE_ADDR, 0);
1487
1488         fw_data = (const __be32 *)rdev->me_fw->data;
1489         WREG32(CP_ME_RAM_WADDR, 0);
1490         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1491                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1492
1493         WREG32(CP_PFP_UCODE_ADDR, 0);
1494         WREG32(CP_ME_RAM_WADDR, 0);
1495         WREG32(CP_ME_RAM_RADDR, 0);
1496         return 0;
1497 }
1498
1499 static int cayman_cp_start(struct radeon_device *rdev)
1500 {
1501         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1502         int r, i;
1503
1504         r = radeon_ring_lock(rdev, ring, 7);
1505         if (r) {
1506                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1507                 return r;
1508         }
1509         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1510         radeon_ring_write(ring, 0x1);
1511         radeon_ring_write(ring, 0x0);
1512         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1513         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1514         radeon_ring_write(ring, 0);
1515         radeon_ring_write(ring, 0);
1516         radeon_ring_unlock_commit(rdev, ring, false);
1517
1518         cayman_cp_enable(rdev, true);
1519
1520         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1521         if (r) {
1522                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1523                 return r;
1524         }
1525
1526         /* setup clear context state */
1527         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1528         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1529
1530         for (i = 0; i < cayman_default_size; i++)
1531                 radeon_ring_write(ring, cayman_default_state[i]);
1532
1533         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1534         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1535
1536         /* set clear context state */
1537         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1538         radeon_ring_write(ring, 0);
1539
1540         /* SQ_VTX_BASE_VTX_LOC */
1541         radeon_ring_write(ring, 0xc0026f00);
1542         radeon_ring_write(ring, 0x00000000);
1543         radeon_ring_write(ring, 0x00000000);
1544         radeon_ring_write(ring, 0x00000000);
1545
1546         /* Clear consts */
1547         radeon_ring_write(ring, 0xc0036f00);
1548         radeon_ring_write(ring, 0x00000bc4);
1549         radeon_ring_write(ring, 0xffffffff);
1550         radeon_ring_write(ring, 0xffffffff);
1551         radeon_ring_write(ring, 0xffffffff);
1552
1553         radeon_ring_write(ring, 0xc0026900);
1554         radeon_ring_write(ring, 0x00000316);
1555         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1556         radeon_ring_write(ring, 0x00000010); /*  */
1557
1558         radeon_ring_unlock_commit(rdev, ring, false);
1559
1560         /* XXX init other rings */
1561
1562         return 0;
1563 }
1564
1565 static void cayman_cp_fini(struct radeon_device *rdev)
1566 {
1567         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1568         cayman_cp_enable(rdev, false);
1569         radeon_ring_fini(rdev, ring);
1570         radeon_scratch_free(rdev, ring->rptr_save_reg);
1571 }
1572
1573 static int cayman_cp_resume(struct radeon_device *rdev)
1574 {
1575         static const int ridx[] = {
1576                 RADEON_RING_TYPE_GFX_INDEX,
1577                 CAYMAN_RING_TYPE_CP1_INDEX,
1578                 CAYMAN_RING_TYPE_CP2_INDEX
1579         };
1580         static const unsigned cp_rb_cntl[] = {
1581                 CP_RB0_CNTL,
1582                 CP_RB1_CNTL,
1583                 CP_RB2_CNTL,
1584         };
1585         static const unsigned cp_rb_rptr_addr[] = {
1586                 CP_RB0_RPTR_ADDR,
1587                 CP_RB1_RPTR_ADDR,
1588                 CP_RB2_RPTR_ADDR
1589         };
1590         static const unsigned cp_rb_rptr_addr_hi[] = {
1591                 CP_RB0_RPTR_ADDR_HI,
1592                 CP_RB1_RPTR_ADDR_HI,
1593                 CP_RB2_RPTR_ADDR_HI
1594         };
1595         static const unsigned cp_rb_base[] = {
1596                 CP_RB0_BASE,
1597                 CP_RB1_BASE,
1598                 CP_RB2_BASE
1599         };
1600         static const unsigned cp_rb_rptr[] = {
1601                 CP_RB0_RPTR,
1602                 CP_RB1_RPTR,
1603                 CP_RB2_RPTR
1604         };
1605         static const unsigned cp_rb_wptr[] = {
1606                 CP_RB0_WPTR,
1607                 CP_RB1_WPTR,
1608                 CP_RB2_WPTR
1609         };
1610         struct radeon_ring *ring;
1611         int i, r;
1612
1613         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1614         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1615                                  SOFT_RESET_PA |
1616                                  SOFT_RESET_SH |
1617                                  SOFT_RESET_VGT |
1618                                  SOFT_RESET_SPI |
1619                                  SOFT_RESET_SX));
1620         RREG32(GRBM_SOFT_RESET);
1621         mdelay(15);
1622         WREG32(GRBM_SOFT_RESET, 0);
1623         RREG32(GRBM_SOFT_RESET);
1624
1625         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1626         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1627
1628         /* Set the write pointer delay */
1629         WREG32(CP_RB_WPTR_DELAY, 0);
1630
1631         WREG32(CP_DEBUG, (1 << 27));
1632
1633         /* set the wb address whether it's enabled or not */
1634         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1635         WREG32(SCRATCH_UMSK, 0xff);
1636
1637         for (i = 0; i < 3; ++i) {
1638                 uint32_t rb_cntl;
1639                 uint64_t addr;
1640
1641                 /* Set ring buffer size */
1642                 ring = &rdev->ring[ridx[i]];
1643                 rb_cntl = order_base_2(ring->ring_size / 8);
1644                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1645 #ifdef __BIG_ENDIAN
1646                 rb_cntl |= BUF_SWAP_32BIT;
1647 #endif
1648                 WREG32(cp_rb_cntl[i], rb_cntl);
1649
1650                 /* set the wb address whether it's enabled or not */
1651                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1652                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1653                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1654         }
1655
1656         /* set the rb base addr, this causes an internal reset of ALL rings */
1657         for (i = 0; i < 3; ++i) {
1658                 ring = &rdev->ring[ridx[i]];
1659                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1660         }
1661
1662         for (i = 0; i < 3; ++i) {
1663                 /* Initialize the ring buffer's read and write pointers */
1664                 ring = &rdev->ring[ridx[i]];
1665                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1666
1667                 ring->wptr = 0;
1668                 WREG32(cp_rb_rptr[i], 0);
1669                 WREG32(cp_rb_wptr[i], ring->wptr);
1670
1671                 mdelay(1);
1672                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1673         }
1674
1675         /* start the rings */
1676         cayman_cp_start(rdev);
1677         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1678         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1679         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1680         /* this only test cp0 */
1681         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1682         if (r) {
1683                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1684                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1685                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1686                 return r;
1687         }
1688
1689         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1690                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1691
1692         return 0;
1693 }
1694
1695 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1696 {
1697         u32 reset_mask = 0;
1698         u32 tmp;
1699
1700         /* GRBM_STATUS */
1701         tmp = RREG32(GRBM_STATUS);
1702         if (tmp & (PA_BUSY | SC_BUSY |
1703                    SH_BUSY | SX_BUSY |
1704                    TA_BUSY | VGT_BUSY |
1705                    DB_BUSY | CB_BUSY |
1706                    GDS_BUSY | SPI_BUSY |
1707                    IA_BUSY | IA_BUSY_NO_DMA))
1708                 reset_mask |= RADEON_RESET_GFX;
1709
1710         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1711                    CP_BUSY | CP_COHERENCY_BUSY))
1712                 reset_mask |= RADEON_RESET_CP;
1713
1714         if (tmp & GRBM_EE_BUSY)
1715                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1716
1717         /* DMA_STATUS_REG 0 */
1718         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1719         if (!(tmp & DMA_IDLE))
1720                 reset_mask |= RADEON_RESET_DMA;
1721
1722         /* DMA_STATUS_REG 1 */
1723         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1724         if (!(tmp & DMA_IDLE))
1725                 reset_mask |= RADEON_RESET_DMA1;
1726
1727         /* SRBM_STATUS2 */
1728         tmp = RREG32(SRBM_STATUS2);
1729         if (tmp & DMA_BUSY)
1730                 reset_mask |= RADEON_RESET_DMA;
1731
1732         if (tmp & DMA1_BUSY)
1733                 reset_mask |= RADEON_RESET_DMA1;
1734
1735         /* SRBM_STATUS */
1736         tmp = RREG32(SRBM_STATUS);
1737         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1738                 reset_mask |= RADEON_RESET_RLC;
1739
1740         if (tmp & IH_BUSY)
1741                 reset_mask |= RADEON_RESET_IH;
1742
1743         if (tmp & SEM_BUSY)
1744                 reset_mask |= RADEON_RESET_SEM;
1745
1746         if (tmp & GRBM_RQ_PENDING)
1747                 reset_mask |= RADEON_RESET_GRBM;
1748
1749         if (tmp & VMC_BUSY)
1750                 reset_mask |= RADEON_RESET_VMC;
1751
1752         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1753                    MCC_BUSY | MCD_BUSY))
1754                 reset_mask |= RADEON_RESET_MC;
1755
1756         if (evergreen_is_display_hung(rdev))
1757                 reset_mask |= RADEON_RESET_DISPLAY;
1758
1759         /* VM_L2_STATUS */
1760         tmp = RREG32(VM_L2_STATUS);
1761         if (tmp & L2_BUSY)
1762                 reset_mask |= RADEON_RESET_VMC;
1763
1764         /* Skip MC reset as it's mostly likely not hung, just busy */
1765         if (reset_mask & RADEON_RESET_MC) {
1766                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1767                 reset_mask &= ~RADEON_RESET_MC;
1768         }
1769
1770         return reset_mask;
1771 }
1772
1773 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1774 {
1775         struct evergreen_mc_save save;
1776         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1777         u32 tmp;
1778
1779         if (reset_mask == 0)
1780                 return;
1781
1782         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1783
1784         evergreen_print_gpu_status_regs(rdev);
1785         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1786                  RREG32(0x14F8));
1787         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1788                  RREG32(0x14D8));
1789         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1790                  RREG32(0x14FC));
1791         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1792                  RREG32(0x14DC));
1793
1794         /* Disable CP parsing/prefetching */
1795         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1796
1797         if (reset_mask & RADEON_RESET_DMA) {
1798                 /* dma0 */
1799                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1800                 tmp &= ~DMA_RB_ENABLE;
1801                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1802         }
1803
1804         if (reset_mask & RADEON_RESET_DMA1) {
1805                 /* dma1 */
1806                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1807                 tmp &= ~DMA_RB_ENABLE;
1808                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1809         }
1810
1811         udelay(50);
1812
1813         evergreen_mc_stop(rdev, &save);
1814         if (evergreen_mc_wait_for_idle(rdev)) {
1815                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1816         }
1817
1818         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1819                 grbm_soft_reset = SOFT_RESET_CB |
1820                         SOFT_RESET_DB |
1821                         SOFT_RESET_GDS |
1822                         SOFT_RESET_PA |
1823                         SOFT_RESET_SC |
1824                         SOFT_RESET_SPI |
1825                         SOFT_RESET_SH |
1826                         SOFT_RESET_SX |
1827                         SOFT_RESET_TC |
1828                         SOFT_RESET_TA |
1829                         SOFT_RESET_VGT |
1830                         SOFT_RESET_IA;
1831         }
1832
1833         if (reset_mask & RADEON_RESET_CP) {
1834                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1835
1836                 srbm_soft_reset |= SOFT_RESET_GRBM;
1837         }
1838
1839         if (reset_mask & RADEON_RESET_DMA)
1840                 srbm_soft_reset |= SOFT_RESET_DMA;
1841
1842         if (reset_mask & RADEON_RESET_DMA1)
1843                 srbm_soft_reset |= SOFT_RESET_DMA1;
1844
1845         if (reset_mask & RADEON_RESET_DISPLAY)
1846                 srbm_soft_reset |= SOFT_RESET_DC;
1847
1848         if (reset_mask & RADEON_RESET_RLC)
1849                 srbm_soft_reset |= SOFT_RESET_RLC;
1850
1851         if (reset_mask & RADEON_RESET_SEM)
1852                 srbm_soft_reset |= SOFT_RESET_SEM;
1853
1854         if (reset_mask & RADEON_RESET_IH)
1855                 srbm_soft_reset |= SOFT_RESET_IH;
1856
1857         if (reset_mask & RADEON_RESET_GRBM)
1858                 srbm_soft_reset |= SOFT_RESET_GRBM;
1859
1860         if (reset_mask & RADEON_RESET_VMC)
1861                 srbm_soft_reset |= SOFT_RESET_VMC;
1862
1863         if (!(rdev->flags & RADEON_IS_IGP)) {
1864                 if (reset_mask & RADEON_RESET_MC)
1865                         srbm_soft_reset |= SOFT_RESET_MC;
1866         }
1867
1868         if (grbm_soft_reset) {
1869                 tmp = RREG32(GRBM_SOFT_RESET);
1870                 tmp |= grbm_soft_reset;
1871                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1872                 WREG32(GRBM_SOFT_RESET, tmp);
1873                 tmp = RREG32(GRBM_SOFT_RESET);
1874
1875                 udelay(50);
1876
1877                 tmp &= ~grbm_soft_reset;
1878                 WREG32(GRBM_SOFT_RESET, tmp);
1879                 tmp = RREG32(GRBM_SOFT_RESET);
1880         }
1881
1882         if (srbm_soft_reset) {
1883                 tmp = RREG32(SRBM_SOFT_RESET);
1884                 tmp |= srbm_soft_reset;
1885                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1886                 WREG32(SRBM_SOFT_RESET, tmp);
1887                 tmp = RREG32(SRBM_SOFT_RESET);
1888
1889                 udelay(50);
1890
1891                 tmp &= ~srbm_soft_reset;
1892                 WREG32(SRBM_SOFT_RESET, tmp);
1893                 tmp = RREG32(SRBM_SOFT_RESET);
1894         }
1895
1896         /* Wait a little for things to settle down */
1897         udelay(50);
1898
1899         evergreen_mc_resume(rdev, &save);
1900         udelay(50);
1901
1902         evergreen_print_gpu_status_regs(rdev);
1903 }
1904
1905 int cayman_asic_reset(struct radeon_device *rdev)
1906 {
1907         u32 reset_mask;
1908
1909         reset_mask = cayman_gpu_check_soft_reset(rdev);
1910
1911         if (reset_mask)
1912                 r600_set_bios_scratch_engine_hung(rdev, true);
1913
1914         cayman_gpu_soft_reset(rdev, reset_mask);
1915
1916         reset_mask = cayman_gpu_check_soft_reset(rdev);
1917
1918         if (reset_mask)
1919                 evergreen_gpu_pci_config_reset(rdev);
1920
1921         r600_set_bios_scratch_engine_hung(rdev, false);
1922
1923         return 0;
1924 }
1925
1926 /**
1927  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1928  *
1929  * @rdev: radeon_device pointer
1930  * @ring: radeon_ring structure holding ring information
1931  *
1932  * Check if the GFX engine is locked up.
1933  * Returns true if the engine appears to be locked up, false if not.
1934  */
1935 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1936 {
1937         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1938
1939         if (!(reset_mask & (RADEON_RESET_GFX |
1940                             RADEON_RESET_COMPUTE |
1941                             RADEON_RESET_CP))) {
1942                 radeon_ring_lockup_update(rdev, ring);
1943                 return false;
1944         }
1945         return radeon_ring_test_lockup(rdev, ring);
1946 }
1947
1948 static int cayman_startup(struct radeon_device *rdev)
1949 {
1950         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1951         int r;
1952
1953         /* enable pcie gen2 link */
1954         evergreen_pcie_gen2_enable(rdev);
1955         /* enable aspm */
1956         evergreen_program_aspm(rdev);
1957
1958         /* scratch needs to be initialized before MC */
1959         r = r600_vram_scratch_init(rdev);
1960         if (r)
1961                 return r;
1962
1963         evergreen_mc_program(rdev);
1964
1965         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
1966                 r = ni_mc_load_microcode(rdev);
1967                 if (r) {
1968                         DRM_ERROR("Failed to load MC firmware!\n");
1969                         return r;
1970                 }
1971         }
1972
1973         r = cayman_pcie_gart_enable(rdev);
1974         if (r)
1975                 return r;
1976         cayman_gpu_init(rdev);
1977
1978         /* allocate rlc buffers */
1979         if (rdev->flags & RADEON_IS_IGP) {
1980                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
1981                 rdev->rlc.reg_list_size =
1982                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
1983                 rdev->rlc.cs_data = cayman_cs_data;
1984                 r = sumo_rlc_init(rdev);
1985                 if (r) {
1986                         DRM_ERROR("Failed to init rlc BOs!\n");
1987                         return r;
1988                 }
1989         }
1990
1991         /* allocate wb buffer */
1992         r = radeon_wb_init(rdev);
1993         if (r)
1994                 return r;
1995
1996         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1997         if (r) {
1998                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1999                 return r;
2000         }
2001
2002         r = uvd_v2_2_resume(rdev);
2003         if (!r) {
2004                 r = radeon_fence_driver_start_ring(rdev,
2005                                                    R600_RING_TYPE_UVD_INDEX);
2006                 if (r)
2007                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2008         }
2009         if (r)
2010                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2011
2012         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2013         if (r) {
2014                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2015                 return r;
2016         }
2017
2018         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2019         if (r) {
2020                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2021                 return r;
2022         }
2023
2024         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2025         if (r) {
2026                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2027                 return r;
2028         }
2029
2030         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2031         if (r) {
2032                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2033                 return r;
2034         }
2035
2036         /* Enable IRQ */
2037         if (!rdev->irq.installed) {
2038                 r = radeon_irq_kms_init(rdev);
2039                 if (r)
2040                         return r;
2041         }
2042
2043         r = r600_irq_init(rdev);
2044         if (r) {
2045                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2046                 radeon_irq_kms_fini(rdev);
2047                 return r;
2048         }
2049         evergreen_irq_set(rdev);
2050
2051         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2052                              RADEON_CP_PACKET2);
2053         if (r)
2054                 return r;
2055
2056         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2057         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2058                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2059         if (r)
2060                 return r;
2061
2062         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2063         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2064                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2065         if (r)
2066                 return r;
2067
2068         r = cayman_cp_load_microcode(rdev);
2069         if (r)
2070                 return r;
2071         r = cayman_cp_resume(rdev);
2072         if (r)
2073                 return r;
2074
2075         r = cayman_dma_resume(rdev);
2076         if (r)
2077                 return r;
2078
2079         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2080         if (ring->ring_size) {
2081                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2082                                      RADEON_CP_PACKET2);
2083                 if (!r)
2084                         r = uvd_v1_0_init(rdev);
2085                 if (r)
2086                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2087         }
2088
2089         r = radeon_ib_pool_init(rdev);
2090         if (r) {
2091                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2092                 return r;
2093         }
2094
2095         r = radeon_vm_manager_init(rdev);
2096         if (r) {
2097                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2098                 return r;
2099         }
2100
2101         r = radeon_audio_init(rdev);
2102         if (r)
2103                 return r;
2104
2105         return 0;
2106 }
2107
2108 int cayman_resume(struct radeon_device *rdev)
2109 {
2110         int r;
2111
2112         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2113          * posting will perform necessary task to bring back GPU into good
2114          * shape.
2115          */
2116         /* post card */
2117         atom_asic_init(rdev->mode_info.atom_context);
2118
2119         /* init golden registers */
2120         ni_init_golden_registers(rdev);
2121
2122         if (rdev->pm.pm_method == PM_METHOD_DPM)
2123                 radeon_pm_resume(rdev);
2124
2125         rdev->accel_working = true;
2126         r = cayman_startup(rdev);
2127         if (r) {
2128                 DRM_ERROR("cayman startup failed on resume\n");
2129                 rdev->accel_working = false;
2130                 return r;
2131         }
2132         return r;
2133 }
2134
2135 int cayman_suspend(struct radeon_device *rdev)
2136 {
2137         radeon_pm_suspend(rdev);
2138         radeon_audio_fini(rdev);
2139         radeon_vm_manager_fini(rdev);
2140         cayman_cp_enable(rdev, false);
2141         cayman_dma_stop(rdev);
2142         uvd_v1_0_fini(rdev);
2143         radeon_uvd_suspend(rdev);
2144         evergreen_irq_suspend(rdev);
2145         radeon_wb_disable(rdev);
2146         cayman_pcie_gart_disable(rdev);
2147         return 0;
2148 }
2149
2150 /* Plan is to move initialization in that function and use
2151  * helper function so that radeon_device_init pretty much
2152  * do nothing more than calling asic specific function. This
2153  * should also allow to remove a bunch of callback function
2154  * like vram_info.
2155  */
2156 int cayman_init(struct radeon_device *rdev)
2157 {
2158         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2159         int r;
2160
2161         /* Read BIOS */
2162         if (!radeon_get_bios(rdev)) {
2163                 if (ASIC_IS_AVIVO(rdev))
2164                         return -EINVAL;
2165         }
2166         /* Must be an ATOMBIOS */
2167         if (!rdev->is_atom_bios) {
2168                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2169                 return -EINVAL;
2170         }
2171         r = radeon_atombios_init(rdev);
2172         if (r)
2173                 return r;
2174
2175         /* Post card if necessary */
2176         if (!radeon_card_posted(rdev)) {
2177                 if (!rdev->bios) {
2178                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2179                         return -EINVAL;
2180                 }
2181                 DRM_INFO("GPU not posted. posting now...\n");
2182                 atom_asic_init(rdev->mode_info.atom_context);
2183         }
2184         /* init golden registers */
2185         ni_init_golden_registers(rdev);
2186         /* Initialize scratch registers */
2187         r600_scratch_init(rdev);
2188         /* Initialize surface registers */
2189         radeon_surface_init(rdev);
2190         /* Initialize clocks */
2191         radeon_get_clock_info(rdev->ddev);
2192         /* Fence driver */
2193         r = radeon_fence_driver_init(rdev);
2194         if (r)
2195                 return r;
2196         /* initialize memory controller */
2197         r = evergreen_mc_init(rdev);
2198         if (r)
2199                 return r;
2200         /* Memory manager */
2201         r = radeon_bo_init(rdev);
2202         if (r)
2203                 return r;
2204
2205         if (rdev->flags & RADEON_IS_IGP) {
2206                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2207                         r = ni_init_microcode(rdev);
2208                         if (r) {
2209                                 DRM_ERROR("Failed to load firmware!\n");
2210                                 return r;
2211                         }
2212                 }
2213         } else {
2214                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2215                         r = ni_init_microcode(rdev);
2216                         if (r) {
2217                                 DRM_ERROR("Failed to load firmware!\n");
2218                                 return r;
2219                         }
2220                 }
2221         }
2222
2223         /* Initialize power management */
2224         radeon_pm_init(rdev);
2225
2226         ring->ring_obj = NULL;
2227         r600_ring_init(rdev, ring, 1024 * 1024);
2228
2229         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2230         ring->ring_obj = NULL;
2231         r600_ring_init(rdev, ring, 64 * 1024);
2232
2233         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2234         ring->ring_obj = NULL;
2235         r600_ring_init(rdev, ring, 64 * 1024);
2236
2237         r = radeon_uvd_init(rdev);
2238         if (!r) {
2239                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2240                 ring->ring_obj = NULL;
2241                 r600_ring_init(rdev, ring, 4096);
2242         }
2243
2244         rdev->ih.ring_obj = NULL;
2245         r600_ih_ring_init(rdev, 64 * 1024);
2246
2247         r = r600_pcie_gart_init(rdev);
2248         if (r)
2249                 return r;
2250
2251         rdev->accel_working = true;
2252         r = cayman_startup(rdev);
2253         if (r) {
2254                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2255                 cayman_cp_fini(rdev);
2256                 cayman_dma_fini(rdev);
2257                 r600_irq_fini(rdev);
2258                 if (rdev->flags & RADEON_IS_IGP)
2259                         sumo_rlc_fini(rdev);
2260                 radeon_wb_fini(rdev);
2261                 radeon_ib_pool_fini(rdev);
2262                 radeon_vm_manager_fini(rdev);
2263                 radeon_irq_kms_fini(rdev);
2264                 cayman_pcie_gart_fini(rdev);
2265                 rdev->accel_working = false;
2266         }
2267
2268         /* Don't start up if the MC ucode is missing.
2269          * The default clocks and voltages before the MC ucode
2270          * is loaded are not suffient for advanced operations.
2271          *
2272          * We can skip this check for TN, because there is no MC
2273          * ucode.
2274          */
2275         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2276                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2277                 return -EINVAL;
2278         }
2279
2280         return 0;
2281 }
2282
2283 void cayman_fini(struct radeon_device *rdev)
2284 {
2285         radeon_pm_fini(rdev);
2286         cayman_cp_fini(rdev);
2287         cayman_dma_fini(rdev);
2288         r600_irq_fini(rdev);
2289         if (rdev->flags & RADEON_IS_IGP)
2290                 sumo_rlc_fini(rdev);
2291         radeon_wb_fini(rdev);
2292         radeon_vm_manager_fini(rdev);
2293         radeon_ib_pool_fini(rdev);
2294         radeon_irq_kms_fini(rdev);
2295         uvd_v1_0_fini(rdev);
2296         radeon_uvd_fini(rdev);
2297         cayman_pcie_gart_fini(rdev);
2298         r600_vram_scratch_fini(rdev);
2299         radeon_gem_fini(rdev);
2300         radeon_fence_driver_fini(rdev);
2301         radeon_bo_fini(rdev);
2302         radeon_atombios_fini(rdev);
2303         kfree(rdev->bios);
2304         rdev->bios = NULL;
2305 }
2306
2307 /*
2308  * vm
2309  */
2310 int cayman_vm_init(struct radeon_device *rdev)
2311 {
2312         /* number of VMs */
2313         rdev->vm_manager.nvm = 8;
2314         /* base offset of vram pages */
2315         if (rdev->flags & RADEON_IS_IGP) {
2316                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2317                 tmp <<= 22;
2318                 rdev->vm_manager.vram_base_offset = tmp;
2319         } else
2320                 rdev->vm_manager.vram_base_offset = 0;
2321         return 0;
2322 }
2323
2324 void cayman_vm_fini(struct radeon_device *rdev)
2325 {
2326 }
2327
2328 /**
2329  * cayman_vm_decode_fault - print human readable fault info
2330  *
2331  * @rdev: radeon_device pointer
2332  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2333  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2334  *
2335  * Print human readable fault information (cayman/TN).
2336  */
2337 void cayman_vm_decode_fault(struct radeon_device *rdev,
2338                             u32 status, u32 addr)
2339 {
2340         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2341         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2342         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2343         char *block;
2344
2345         switch (mc_id) {
2346         case 32:
2347         case 16:
2348         case 96:
2349         case 80:
2350         case 160:
2351         case 144:
2352         case 224:
2353         case 208:
2354                 block = "CB";
2355                 break;
2356         case 33:
2357         case 17:
2358         case 97:
2359         case 81:
2360         case 161:
2361         case 145:
2362         case 225:
2363         case 209:
2364                 block = "CB_FMASK";
2365                 break;
2366         case 34:
2367         case 18:
2368         case 98:
2369         case 82:
2370         case 162:
2371         case 146:
2372         case 226:
2373         case 210:
2374                 block = "CB_CMASK";
2375                 break;
2376         case 35:
2377         case 19:
2378         case 99:
2379         case 83:
2380         case 163:
2381         case 147:
2382         case 227:
2383         case 211:
2384                 block = "CB_IMMED";
2385                 break;
2386         case 36:
2387         case 20:
2388         case 100:
2389         case 84:
2390         case 164:
2391         case 148:
2392         case 228:
2393         case 212:
2394                 block = "DB";
2395                 break;
2396         case 37:
2397         case 21:
2398         case 101:
2399         case 85:
2400         case 165:
2401         case 149:
2402         case 229:
2403         case 213:
2404                 block = "DB_HTILE";
2405                 break;
2406         case 38:
2407         case 22:
2408         case 102:
2409         case 86:
2410         case 166:
2411         case 150:
2412         case 230:
2413         case 214:
2414                 block = "SX";
2415                 break;
2416         case 39:
2417         case 23:
2418         case 103:
2419         case 87:
2420         case 167:
2421         case 151:
2422         case 231:
2423         case 215:
2424                 block = "DB_STEN";
2425                 break;
2426         case 40:
2427         case 24:
2428         case 104:
2429         case 88:
2430         case 232:
2431         case 216:
2432         case 168:
2433         case 152:
2434                 block = "TC_TFETCH";
2435                 break;
2436         case 41:
2437         case 25:
2438         case 105:
2439         case 89:
2440         case 233:
2441         case 217:
2442         case 169:
2443         case 153:
2444                 block = "TC_VFETCH";
2445                 break;
2446         case 42:
2447         case 26:
2448         case 106:
2449         case 90:
2450         case 234:
2451         case 218:
2452         case 170:
2453         case 154:
2454                 block = "VC";
2455                 break;
2456         case 112:
2457                 block = "CP";
2458                 break;
2459         case 113:
2460         case 114:
2461                 block = "SH";
2462                 break;
2463         case 115:
2464                 block = "VGT";
2465                 break;
2466         case 178:
2467                 block = "IH";
2468                 break;
2469         case 51:
2470                 block = "RLC";
2471                 break;
2472         case 55:
2473                 block = "DMA";
2474                 break;
2475         case 56:
2476                 block = "HDP";
2477                 break;
2478         default:
2479                 block = "unknown";
2480                 break;
2481         }
2482
2483         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2484                protections, vmid, addr,
2485                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2486                block, mc_id);
2487 }
2488
2489 /**
2490  * cayman_vm_flush - vm flush using the CP
2491  *
2492  * @rdev: radeon_device pointer
2493  *
2494  * Update the page table base and flush the VM TLB
2495  * using the CP (cayman-si).
2496  */
2497 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2498                      unsigned vm_id, uint64_t pd_addr)
2499 {
2500         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2501         radeon_ring_write(ring, pd_addr >> 12);
2502
2503         /* flush hdp cache */
2504         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2505         radeon_ring_write(ring, 0x1);
2506
2507         /* bits 0-7 are the VM contexts0-7 */
2508         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2509         radeon_ring_write(ring, 1 << vm_id);
2510
2511         /* wait for the invalidate to complete */
2512         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2513         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2514                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2515         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2516         radeon_ring_write(ring, 0);
2517         radeon_ring_write(ring, 0); /* ref */
2518         radeon_ring_write(ring, 0); /* mask */
2519         radeon_ring_write(ring, 0x20); /* poll interval */
2520
2521         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2522         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2523         radeon_ring_write(ring, 0x0);
2524 }