drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "drmP.h"
  25 #include "radeon.h"
  26 #include "nid.h"
  27 #include "r600_dpm.h"
  28 #include "ni_dpm.h"
  29 #include "atom.h"
  30 #include <linux/math64.h>
  31 #include <linux/seq_file.h>
  32
  33 #define MC_CG_ARB_FREQ_F0           0x0a
  34 #define MC_CG_ARB_FREQ_F1           0x0b
  35 #define MC_CG_ARB_FREQ_F2           0x0c
  36 #define MC_CG_ARB_FREQ_F3           0x0d
  37
  38 #define SMC_RAM_END 0xC000
  39
  40 static const struct ni_cac_weights cac_weights_cayman_xt =
  41 {
  42         0x15,
  43         0x2,
  44         0x19,
  45         0x2,
  46         0x8,
  47         0x14,
  48         0x2,
  49         0x16,
  50         0xE,
  51         0x17,
  52         0x13,
  53         0x2B,
  54         0x10,
  55         0x7,
  56         0x5,
  57         0x5,
  58         0x5,
  59         0x2,
  60         0x3,
  61         0x9,
  62         0x10,
  63         0x10,
  64         0x2B,
  65         0xA,
  66         0x9,
  67         0x4,
  68         0xD,
  69         0xD,
  70         0x3E,
  71         0x18,
  72         0x14,
  73         0,
  74         0x3,
  75         0x3,
  76         0x5,
  77         0,
  78         0x2,
  79         0,
  80         0,
  81         0,
  82         0,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0,
  88         0x1CC,
  89         0,
  90         0x164,
  91         1,
  92         1,
  93         1,
  94         1,
  95         12,
  96         12,
  97         12,
  98         0x12,
  99         0x1F,
 100         132,
 101         5,
 102         7,
 103         0,
 104         { 0, 0, 0, 0, 0, 0, 0, 0 },
 105         { 0, 0, 0, 0 },
 106         true
 107 };
 108
 109 static const struct ni_cac_weights cac_weights_cayman_pro =
 110 {
 111         0x16,
 112         0x4,
 113         0x10,
 114         0x2,
 115         0xA,
 116         0x16,
 117         0x2,
 118         0x18,
 119         0x10,
 120         0x1A,
 121         0x16,
 122         0x2D,
 123         0x12,
 124         0xA,
 125         0x6,
 126         0x6,
 127         0x6,
 128         0x2,
 129         0x4,
 130         0xB,
 131         0x11,
 132         0x11,
 133         0x2D,
 134         0xC,
 135         0xC,
 136         0x7,
 137         0x10,
 138         0x10,
 139         0x3F,
 140         0x1A,
 141         0x16,
 142         0,
 143         0x7,
 144         0x4,
 145         0x6,
 146         1,
 147         0x2,
 148         0x1,
 149         0,
 150         0,
 151         0,
 152         0,
 153         0,
 154         0,
 155         0x30,
 156         0,
 157         0x1CF,
 158         0,
 159         0x166,
 160         1,
 161         1,
 162         1,
 163         1,
 164         12,
 165         12,
 166         12,
 167         0x15,
 168         0x1F,
 169         132,
 170         6,
 171         6,
 172         0,
 173         { 0, 0, 0, 0, 0, 0, 0, 0 },
 174         { 0, 0, 0, 0 },
 175         true
 176 };
 177
 178 static const struct ni_cac_weights cac_weights_cayman_le =
 179 {
 180         0x7,
 181         0xE,
 182         0x1,
 183         0xA,
 184         0x1,
 185         0x3F,
 186         0x2,
 187         0x18,
 188         0x10,
 189         0x1A,
 190         0x1,
 191         0x3F,
 192         0x1,
 193         0xE,
 194         0x6,
 195         0x6,
 196         0x6,
 197         0x2,
 198         0x4,
 199         0x9,
 200         0x1A,
 201         0x1A,
 202         0x2C,
 203         0xA,
 204         0x11,
 205         0x8,
 206         0x19,
 207         0x19,
 208         0x1,
 209         0x1,
 210         0x1A,
 211         0,
 212         0x8,
 213         0x5,
 214         0x8,
 215         0x1,
 216         0x3,
 217         0x1,
 218         0,
 219         0,
 220         0,
 221         0,
 222         0,
 223         0,
 224         0x38,
 225         0x38,
 226         0x239,
 227         0x3,
 228         0x18A,
 229         1,
 230         1,
 231         1,
 232         1,
 233         12,
 234         12,
 235         12,
 236         0x15,
 237         0x22,
 238         132,
 239         6,
 240         6,
 241         0,
 242         { 0, 0, 0, 0, 0, 0, 0, 0 },
 243         { 0, 0, 0, 0 },
 244         true
 245 };
 246
 247 #define NISLANDS_MGCG_SEQUENCE  300
 248
 249 static const u32 cayman_cgcg_cgls_default[] =
 250 {
 251         0x000008f8, 0x00000010, 0xffffffff,
 252         0x000008fc, 0x00000000, 0xffffffff,
 253         0x000008f8, 0x00000011, 0xffffffff,
 254         0x000008fc, 0x00000000, 0xffffffff,
 255         0x000008f8, 0x00000012, 0xffffffff,
 256         0x000008fc, 0x00000000, 0xffffffff,
 257         0x000008f8, 0x00000013, 0xffffffff,
 258         0x000008fc, 0x00000000, 0xffffffff,
 259         0x000008f8, 0x00000014, 0xffffffff,
 260         0x000008fc, 0x00000000, 0xffffffff,
 261         0x000008f8, 0x00000015, 0xffffffff,
 262         0x000008fc, 0x00000000, 0xffffffff,
 263         0x000008f8, 0x00000016, 0xffffffff,
 264         0x000008fc, 0x00000000, 0xffffffff,
 265         0x000008f8, 0x00000017, 0xffffffff,
 266         0x000008fc, 0x00000000, 0xffffffff,
 267         0x000008f8, 0x00000018, 0xffffffff,
 268         0x000008fc, 0x00000000, 0xffffffff,
 269         0x000008f8, 0x00000019, 0xffffffff,
 270         0x000008fc, 0x00000000, 0xffffffff,
 271         0x000008f8, 0x0000001a, 0xffffffff,
 272         0x000008fc, 0x00000000, 0xffffffff,
 273         0x000008f8, 0x0000001b, 0xffffffff,
 274         0x000008fc, 0x00000000, 0xffffffff,
 275         0x000008f8, 0x00000020, 0xffffffff,
 276         0x000008fc, 0x00000000, 0xffffffff,
 277         0x000008f8, 0x00000021, 0xffffffff,
 278         0x000008fc, 0x00000000, 0xffffffff,
 279         0x000008f8, 0x00000022, 0xffffffff,
 280         0x000008fc, 0x00000000, 0xffffffff,
 281         0x000008f8, 0x00000023, 0xffffffff,
 282         0x000008fc, 0x00000000, 0xffffffff,
 283         0x000008f8, 0x00000024, 0xffffffff,
 284         0x000008fc, 0x00000000, 0xffffffff,
 285         0x000008f8, 0x00000025, 0xffffffff,
 286         0x000008fc, 0x00000000, 0xffffffff,
 287         0x000008f8, 0x00000026, 0xffffffff,
 288         0x000008fc, 0x00000000, 0xffffffff,
 289         0x000008f8, 0x00000027, 0xffffffff,
 290         0x000008fc, 0x00000000, 0xffffffff,
 291         0x000008f8, 0x00000028, 0xffffffff,
 292         0x000008fc, 0x00000000, 0xffffffff,
 293         0x000008f8, 0x00000029, 0xffffffff,
 294         0x000008fc, 0x00000000, 0xffffffff,
 295         0x000008f8, 0x0000002a, 0xffffffff,
 296         0x000008fc, 0x00000000, 0xffffffff,
 297         0x000008f8, 0x0000002b, 0xffffffff,
 298         0x000008fc, 0x00000000, 0xffffffff
 299 };
 300 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 301
 302 static const u32 cayman_cgcg_cgls_disable[] =
 303 {
 304         0x000008f8, 0x00000010, 0xffffffff,
 305         0x000008fc, 0xffffffff, 0xffffffff,
 306         0x000008f8, 0x00000011, 0xffffffff,
 307         0x000008fc, 0xffffffff, 0xffffffff,
 308         0x000008f8, 0x00000012, 0xffffffff,
 309         0x000008fc, 0xffffffff, 0xffffffff,
 310         0x000008f8, 0x00000013, 0xffffffff,
 311         0x000008fc, 0xffffffff, 0xffffffff,
 312         0x000008f8, 0x00000014, 0xffffffff,
 313         0x000008fc, 0xffffffff, 0xffffffff,
 314         0x000008f8, 0x00000015, 0xffffffff,
 315         0x000008fc, 0xffffffff, 0xffffffff,
 316         0x000008f8, 0x00000016, 0xffffffff,
 317         0x000008fc, 0xffffffff, 0xffffffff,
 318         0x000008f8, 0x00000017, 0xffffffff,
 319         0x000008fc, 0xffffffff, 0xffffffff,
 320         0x000008f8, 0x00000018, 0xffffffff,
 321         0x000008fc, 0xffffffff, 0xffffffff,
 322         0x000008f8, 0x00000019, 0xffffffff,
 323         0x000008fc, 0xffffffff, 0xffffffff,
 324         0x000008f8, 0x0000001a, 0xffffffff,
 325         0x000008fc, 0xffffffff, 0xffffffff,
 326         0x000008f8, 0x0000001b, 0xffffffff,
 327         0x000008fc, 0xffffffff, 0xffffffff,
 328         0x000008f8, 0x00000020, 0xffffffff,
 329         0x000008fc, 0x00000000, 0xffffffff,
 330         0x000008f8, 0x00000021, 0xffffffff,
 331         0x000008fc, 0x00000000, 0xffffffff,
 332         0x000008f8, 0x00000022, 0xffffffff,
 333         0x000008fc, 0x00000000, 0xffffffff,
 334         0x000008f8, 0x00000023, 0xffffffff,
 335         0x000008fc, 0x00000000, 0xffffffff,
 336         0x000008f8, 0x00000024, 0xffffffff,
 337         0x000008fc, 0x00000000, 0xffffffff,
 338         0x000008f8, 0x00000025, 0xffffffff,
 339         0x000008fc, 0x00000000, 0xffffffff,
 340         0x000008f8, 0x00000026, 0xffffffff,
 341         0x000008fc, 0x00000000, 0xffffffff,
 342         0x000008f8, 0x00000027, 0xffffffff,
 343         0x000008fc, 0x00000000, 0xffffffff,
 344         0x000008f8, 0x00000028, 0xffffffff,
 345         0x000008fc, 0x00000000, 0xffffffff,
 346         0x000008f8, 0x00000029, 0xffffffff,
 347         0x000008fc, 0x00000000, 0xffffffff,
 348         0x000008f8, 0x0000002a, 0xffffffff,
 349         0x000008fc, 0x00000000, 0xffffffff,
 350         0x000008f8, 0x0000002b, 0xffffffff,
 351         0x000008fc, 0x00000000, 0xffffffff,
 352         0x00000644, 0x000f7902, 0x001f4180,
 353         0x00000644, 0x000f3802, 0x001f4180
 354 };
 355 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 356
 357 static const u32 cayman_cgcg_cgls_enable[] =
 358 {
 359         0x00000644, 0x000f7882, 0x001f4080,
 360         0x000008f8, 0x00000010, 0xffffffff,
 361         0x000008fc, 0x00000000, 0xffffffff,
 362         0x000008f8, 0x00000011, 0xffffffff,
 363         0x000008fc, 0x00000000, 0xffffffff,
 364         0x000008f8, 0x00000012, 0xffffffff,
 365         0x000008fc, 0x00000000, 0xffffffff,
 366         0x000008f8, 0x00000013, 0xffffffff,
 367         0x000008fc, 0x00000000, 0xffffffff,
 368         0x000008f8, 0x00000014, 0xffffffff,
 369         0x000008fc, 0x00000000, 0xffffffff,
 370         0x000008f8, 0x00000015, 0xffffffff,
 371         0x000008fc, 0x00000000, 0xffffffff,
 372         0x000008f8, 0x00000016, 0xffffffff,
 373         0x000008fc, 0x00000000, 0xffffffff,
 374         0x000008f8, 0x00000017, 0xffffffff,
 375         0x000008fc, 0x00000000, 0xffffffff,
 376         0x000008f8, 0x00000018, 0xffffffff,
 377         0x000008fc, 0x00000000, 0xffffffff,
 378         0x000008f8, 0x00000019, 0xffffffff,
 379         0x000008fc, 0x00000000, 0xffffffff,
 380         0x000008f8, 0x0000001a, 0xffffffff,
 381         0x000008fc, 0x00000000, 0xffffffff,
 382         0x000008f8, 0x0000001b, 0xffffffff,
 383         0x000008fc, 0x00000000, 0xffffffff,
 384         0x000008f8, 0x00000020, 0xffffffff,
 385         0x000008fc, 0xffffffff, 0xffffffff,
 386         0x000008f8, 0x00000021, 0xffffffff,
 387         0x000008fc, 0xffffffff, 0xffffffff,
 388         0x000008f8, 0x00000022, 0xffffffff,
 389         0x000008fc, 0xffffffff, 0xffffffff,
 390         0x000008f8, 0x00000023, 0xffffffff,
 391         0x000008fc, 0xffffffff, 0xffffffff,
 392         0x000008f8, 0x00000024, 0xffffffff,
 393         0x000008fc, 0xffffffff, 0xffffffff,
 394         0x000008f8, 0x00000025, 0xffffffff,
 395         0x000008fc, 0xffffffff, 0xffffffff,
 396         0x000008f8, 0x00000026, 0xffffffff,
 397         0x000008fc, 0xffffffff, 0xffffffff,
 398         0x000008f8, 0x00000027, 0xffffffff,
 399         0x000008fc, 0xffffffff, 0xffffffff,
 400         0x000008f8, 0x00000028, 0xffffffff,
 401         0x000008fc, 0xffffffff, 0xffffffff,
 402         0x000008f8, 0x00000029, 0xffffffff,
 403         0x000008fc, 0xffffffff, 0xffffffff,
 404         0x000008f8, 0x0000002a, 0xffffffff,
 405         0x000008fc, 0xffffffff, 0xffffffff,
 406         0x000008f8, 0x0000002b, 0xffffffff,
 407         0x000008fc, 0xffffffff, 0xffffffff
 408 };
 409 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 410
 411 static const u32 cayman_mgcg_default[] =
 412 {
 413         0x0000802c, 0xc0000000, 0xffffffff,
 414         0x00003fc4, 0xc0000000, 0xffffffff,
 415         0x00005448, 0x00000100, 0xffffffff,
 416         0x000055e4, 0x00000100, 0xffffffff,
 417         0x0000160c, 0x00000100, 0xffffffff,
 418         0x00008984, 0x06000100, 0xffffffff,
 419         0x0000c164, 0x00000100, 0xffffffff,
 420         0x00008a18, 0x00000100, 0xffffffff,
 421         0x0000897c, 0x06000100, 0xffffffff,
 422         0x00008b28, 0x00000100, 0xffffffff,
 423         0x00009144, 0x00800200, 0xffffffff,
 424         0x00009a60, 0x00000100, 0xffffffff,
 425         0x00009868, 0x00000100, 0xffffffff,
 426         0x00008d58, 0x00000100, 0xffffffff,
 427         0x00009510, 0x00000100, 0xffffffff,
 428         0x0000949c, 0x00000100, 0xffffffff,
 429         0x00009654, 0x00000100, 0xffffffff,
 430         0x00009030, 0x00000100, 0xffffffff,
 431         0x00009034, 0x00000100, 0xffffffff,
 432         0x00009038, 0x00000100, 0xffffffff,
 433         0x0000903c, 0x00000100, 0xffffffff,
 434         0x00009040, 0x00000100, 0xffffffff,
 435         0x0000a200, 0x00000100, 0xffffffff,
 436         0x0000a204, 0x00000100, 0xffffffff,
 437         0x0000a208, 0x00000100, 0xffffffff,
 438         0x0000a20c, 0x00000100, 0xffffffff,
 439         0x00009744, 0x00000100, 0xffffffff,
 440         0x00003f80, 0x00000100, 0xffffffff,
 441         0x0000a210, 0x00000100, 0xffffffff,
 442         0x0000a214, 0x00000100, 0xffffffff,
 443         0x000004d8, 0x00000100, 0xffffffff,
 444         0x00009664, 0x00000100, 0xffffffff,
 445         0x00009698, 0x00000100, 0xffffffff,
 446         0x000004d4, 0x00000200, 0xffffffff,
 447         0x000004d0, 0x00000000, 0xffffffff,
 448         0x000030cc, 0x00000104, 0xffffffff,
 449         0x0000d0c0, 0x00000100, 0xffffffff,
 450         0x0000d8c0, 0x00000100, 0xffffffff,
 451         0x0000802c, 0x40000000, 0xffffffff,
 452         0x00003fc4, 0x40000000, 0xffffffff,
 453         0x0000915c, 0x00010000, 0xffffffff,
 454         0x00009160, 0x00030002, 0xffffffff,
 455         0x00009164, 0x00050004, 0xffffffff,
 456         0x00009168, 0x00070006, 0xffffffff,
 457         0x00009178, 0x00070000, 0xffffffff,
 458         0x0000917c, 0x00030002, 0xffffffff,
 459         0x00009180, 0x00050004, 0xffffffff,
 460         0x0000918c, 0x00010006, 0xffffffff,
 461         0x00009190, 0x00090008, 0xffffffff,
 462         0x00009194, 0x00070000, 0xffffffff,
 463         0x00009198, 0x00030002, 0xffffffff,
 464         0x0000919c, 0x00050004, 0xffffffff,
 465         0x000091a8, 0x00010006, 0xffffffff,
 466         0x000091ac, 0x00090008, 0xffffffff,
 467         0x000091b0, 0x00070000, 0xffffffff,
 468         0x000091b4, 0x00030002, 0xffffffff,
 469         0x000091b8, 0x00050004, 0xffffffff,
 470         0x000091c4, 0x00010006, 0xffffffff,
 471         0x000091c8, 0x00090008, 0xffffffff,
 472         0x000091cc, 0x00070000, 0xffffffff,
 473         0x000091d0, 0x00030002, 0xffffffff,
 474         0x000091d4, 0x00050004, 0xffffffff,
 475         0x000091e0, 0x00010006, 0xffffffff,
 476         0x000091e4, 0x00090008, 0xffffffff,
 477         0x000091e8, 0x00000000, 0xffffffff,
 478         0x000091ec, 0x00070000, 0xffffffff,
 479         0x000091f0, 0x00030002, 0xffffffff,
 480         0x000091f4, 0x00050004, 0xffffffff,
 481         0x00009200, 0x00010006, 0xffffffff,
 482         0x00009204, 0x00090008, 0xffffffff,
 483         0x00009208, 0x00070000, 0xffffffff,
 484         0x0000920c, 0x00030002, 0xffffffff,
 485         0x00009210, 0x00050004, 0xffffffff,
 486         0x0000921c, 0x00010006, 0xffffffff,
 487         0x00009220, 0x00090008, 0xffffffff,
 488         0x00009224, 0x00070000, 0xffffffff,
 489         0x00009228, 0x00030002, 0xffffffff,
 490         0x0000922c, 0x00050004, 0xffffffff,
 491         0x00009238, 0x00010006, 0xffffffff,
 492         0x0000923c, 0x00090008, 0xffffffff,
 493         0x00009240, 0x00070000, 0xffffffff,
 494         0x00009244, 0x00030002, 0xffffffff,
 495         0x00009248, 0x00050004, 0xffffffff,
 496         0x00009254, 0x00010006, 0xffffffff,
 497         0x00009258, 0x00090008, 0xffffffff,
 498         0x0000925c, 0x00070000, 0xffffffff,
 499         0x00009260, 0x00030002, 0xffffffff,
 500         0x00009264, 0x00050004, 0xffffffff,
 501         0x00009270, 0x00010006, 0xffffffff,
 502         0x00009274, 0x00090008, 0xffffffff,
 503         0x00009278, 0x00070000, 0xffffffff,
 504         0x0000927c, 0x00030002, 0xffffffff,
 505         0x00009280, 0x00050004, 0xffffffff,
 506         0x0000928c, 0x00010006, 0xffffffff,
 507         0x00009290, 0x00090008, 0xffffffff,
 508         0x000092a8, 0x00070000, 0xffffffff,
 509         0x000092ac, 0x00030002, 0xffffffff,
 510         0x000092b0, 0x00050004, 0xffffffff,
 511         0x000092bc, 0x00010006, 0xffffffff,
 512         0x000092c0, 0x00090008, 0xffffffff,
 513         0x000092c4, 0x00070000, 0xffffffff,
 514         0x000092c8, 0x00030002, 0xffffffff,
 515         0x000092cc, 0x00050004, 0xffffffff,
 516         0x000092d8, 0x00010006, 0xffffffff,
 517         0x000092dc, 0x00090008, 0xffffffff,
 518         0x00009294, 0x00000000, 0xffffffff,
 519         0x0000802c, 0x40010000, 0xffffffff,
 520         0x00003fc4, 0x40010000, 0xffffffff,
 521         0x0000915c, 0x00010000, 0xffffffff,
 522         0x00009160, 0x00030002, 0xffffffff,
 523         0x00009164, 0x00050004, 0xffffffff,
 524         0x00009168, 0x00070006, 0xffffffff,
 525         0x00009178, 0x00070000, 0xffffffff,
 526         0x0000917c, 0x00030002, 0xffffffff,
 527         0x00009180, 0x00050004, 0xffffffff,
 528         0x0000918c, 0x00010006, 0xffffffff,
 529         0x00009190, 0x00090008, 0xffffffff,
 530         0x00009194, 0x00070000, 0xffffffff,
 531         0x00009198, 0x00030002, 0xffffffff,
 532         0x0000919c, 0x00050004, 0xffffffff,
 533         0x000091a8, 0x00010006, 0xffffffff,
 534         0x000091ac, 0x00090008, 0xffffffff,
 535         0x000091b0, 0x00070000, 0xffffffff,
 536         0x000091b4, 0x00030002, 0xffffffff,
 537         0x000091b8, 0x00050004, 0xffffffff,
 538         0x000091c4, 0x00010006, 0xffffffff,
 539         0x000091c8, 0x00090008, 0xffffffff,
 540         0x000091cc, 0x00070000, 0xffffffff,
 541         0x000091d0, 0x00030002, 0xffffffff,
 542         0x000091d4, 0x00050004, 0xffffffff,
 543         0x000091e0, 0x00010006, 0xffffffff,
 544         0x000091e4, 0x00090008, 0xffffffff,
 545         0x000091e8, 0x00000000, 0xffffffff,
 546         0x000091ec, 0x00070000, 0xffffffff,
 547         0x000091f0, 0x00030002, 0xffffffff,
 548         0x000091f4, 0x00050004, 0xffffffff,
 549         0x00009200, 0x00010006, 0xffffffff,
 550         0x00009204, 0x00090008, 0xffffffff,
 551         0x00009208, 0x00070000, 0xffffffff,
 552         0x0000920c, 0x00030002, 0xffffffff,
 553         0x00009210, 0x00050004, 0xffffffff,
 554         0x0000921c, 0x00010006, 0xffffffff,
 555         0x00009220, 0x00090008, 0xffffffff,
 556         0x00009224, 0x00070000, 0xffffffff,
 557         0x00009228, 0x00030002, 0xffffffff,
 558         0x0000922c, 0x00050004, 0xffffffff,
 559         0x00009238, 0x00010006, 0xffffffff,
 560         0x0000923c, 0x00090008, 0xffffffff,
 561         0x00009240, 0x00070000, 0xffffffff,
 562         0x00009244, 0x00030002, 0xffffffff,
 563         0x00009248, 0x00050004, 0xffffffff,
 564         0x00009254, 0x00010006, 0xffffffff,
 565         0x00009258, 0x00090008, 0xffffffff,
 566         0x0000925c, 0x00070000, 0xffffffff,
 567         0x00009260, 0x00030002, 0xffffffff,
 568         0x00009264, 0x00050004, 0xffffffff,
 569         0x00009270, 0x00010006, 0xffffffff,
 570         0x00009274, 0x00090008, 0xffffffff,
 571         0x00009278, 0x00070000, 0xffffffff,
 572         0x0000927c, 0x00030002, 0xffffffff,
 573         0x00009280, 0x00050004, 0xffffffff,
 574         0x0000928c, 0x00010006, 0xffffffff,
 575         0x00009290, 0x00090008, 0xffffffff,
 576         0x000092a8, 0x00070000, 0xffffffff,
 577         0x000092ac, 0x00030002, 0xffffffff,
 578         0x000092b0, 0x00050004, 0xffffffff,
 579         0x000092bc, 0x00010006, 0xffffffff,
 580         0x000092c0, 0x00090008, 0xffffffff,
 581         0x000092c4, 0x00070000, 0xffffffff,
 582         0x000092c8, 0x00030002, 0xffffffff,
 583         0x000092cc, 0x00050004, 0xffffffff,
 584         0x000092d8, 0x00010006, 0xffffffff,
 585         0x000092dc, 0x00090008, 0xffffffff,
 586         0x00009294, 0x00000000, 0xffffffff,
 587         0x0000802c, 0xc0000000, 0xffffffff,
 588         0x00003fc4, 0xc0000000, 0xffffffff,
 589         0x000008f8, 0x00000010, 0xffffffff,
 590         0x000008fc, 0x00000000, 0xffffffff,
 591         0x000008f8, 0x00000011, 0xffffffff,
 592         0x000008fc, 0x00000000, 0xffffffff,
 593         0x000008f8, 0x00000012, 0xffffffff,
 594         0x000008fc, 0x00000000, 0xffffffff,
 595         0x000008f8, 0x00000013, 0xffffffff,
 596         0x000008fc, 0x00000000, 0xffffffff,
 597         0x000008f8, 0x00000014, 0xffffffff,
 598         0x000008fc, 0x00000000, 0xffffffff,
 599         0x000008f8, 0x00000015, 0xffffffff,
 600         0x000008fc, 0x00000000, 0xffffffff,
 601         0x000008f8, 0x00000016, 0xffffffff,
 602         0x000008fc, 0x00000000, 0xffffffff,
 603         0x000008f8, 0x00000017, 0xffffffff,
 604         0x000008fc, 0x00000000, 0xffffffff,
 605         0x000008f8, 0x00000018, 0xffffffff,
 606         0x000008fc, 0x00000000, 0xffffffff,
 607         0x000008f8, 0x00000019, 0xffffffff,
 608         0x000008fc, 0x00000000, 0xffffffff,
 609         0x000008f8, 0x0000001a, 0xffffffff,
 610         0x000008fc, 0x00000000, 0xffffffff,
 611         0x000008f8, 0x0000001b, 0xffffffff,
 612         0x000008fc, 0x00000000, 0xffffffff
 613 };
 614 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 615
 616 static const u32 cayman_mgcg_disable[] =
 617 {
 618         0x0000802c, 0xc0000000, 0xffffffff,
 619         0x000008f8, 0x00000000, 0xffffffff,
 620         0x000008fc, 0xffffffff, 0xffffffff,
 621         0x000008f8, 0x00000001, 0xffffffff,
 622         0x000008fc, 0xffffffff, 0xffffffff,
 623         0x000008f8, 0x00000002, 0xffffffff,
 624         0x000008fc, 0xffffffff, 0xffffffff,
 625         0x000008f8, 0x00000003, 0xffffffff,
 626         0x000008fc, 0xffffffff, 0xffffffff,
 627         0x00009150, 0x00600000, 0xffffffff
 628 };
 629 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 630
 631 static const u32 cayman_mgcg_enable[] =
 632 {
 633         0x0000802c, 0xc0000000, 0xffffffff,
 634         0x000008f8, 0x00000000, 0xffffffff,
 635         0x000008fc, 0x00000000, 0xffffffff,
 636         0x000008f8, 0x00000001, 0xffffffff,
 637         0x000008fc, 0x00000000, 0xffffffff,
 638         0x000008f8, 0x00000002, 0xffffffff,
 639         0x000008fc, 0x00600000, 0xffffffff,
 640         0x000008f8, 0x00000003, 0xffffffff,
 641         0x000008fc, 0x00000000, 0xffffffff,
 642         0x00009150, 0x96944200, 0xffffffff
 643 };
 644
 645 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 646
 647 #define NISLANDS_SYSLS_SEQUENCE  100
 648
 649 static const u32 cayman_sysls_default[] =
 650 {
 651         /* Register,   Value,     Mask bits */
 652         0x000055e8, 0x00000000, 0xffffffff,
 653         0x0000d0bc, 0x00000000, 0xffffffff,
 654         0x0000d8bc, 0x00000000, 0xffffffff,
 655         0x000015c0, 0x000c1401, 0xffffffff,
 656         0x0000264c, 0x000c0400, 0xffffffff,
 657         0x00002648, 0x000c0400, 0xffffffff,
 658         0x00002650, 0x000c0400, 0xffffffff,
 659         0x000020b8, 0x000c0400, 0xffffffff,
 660         0x000020bc, 0x000c0400, 0xffffffff,
 661         0x000020c0, 0x000c0c80, 0xffffffff,
 662         0x0000f4a0, 0x000000c0, 0xffffffff,
 663         0x0000f4a4, 0x00680fff, 0xffffffff,
 664         0x00002f50, 0x00000404, 0xffffffff,
 665         0x000004c8, 0x00000001, 0xffffffff,
 666         0x000064ec, 0x00000000, 0xffffffff,
 667         0x00000c7c, 0x00000000, 0xffffffff,
 668         0x00008dfc, 0x00000000, 0xffffffff
 669 };
 670 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 671
 672 static const u32 cayman_sysls_disable[] =
 673 {
 674         /* Register,   Value,     Mask bits */
 675         0x0000d0c0, 0x00000000, 0xffffffff,
 676         0x0000d8c0, 0x00000000, 0xffffffff,
 677         0x000055e8, 0x00000000, 0xffffffff,
 678         0x0000d0bc, 0x00000000, 0xffffffff,
 679         0x0000d8bc, 0x00000000, 0xffffffff,
 680         0x000015c0, 0x00041401, 0xffffffff,
 681         0x0000264c, 0x00040400, 0xffffffff,
 682         0x00002648, 0x00040400, 0xffffffff,
 683         0x00002650, 0x00040400, 0xffffffff,
 684         0x000020b8, 0x00040400, 0xffffffff,
 685         0x000020bc, 0x00040400, 0xffffffff,
 686         0x000020c0, 0x00040c80, 0xffffffff,
 687         0x0000f4a0, 0x000000c0, 0xffffffff,
 688         0x0000f4a4, 0x00680000, 0xffffffff,
 689         0x00002f50, 0x00000404, 0xffffffff,
 690         0x000004c8, 0x00000001, 0xffffffff,
 691         0x000064ec, 0x00007ffd, 0xffffffff,
 692         0x00000c7c, 0x0000ff00, 0xffffffff,
 693         0x00008dfc, 0x0000007f, 0xffffffff
 694 };
 695 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 696
 697 static const u32 cayman_sysls_enable[] =
 698 {
 699         /* Register,   Value,     Mask bits */
 700         0x000055e8, 0x00000001, 0xffffffff,
 701         0x0000d0bc, 0x00000100, 0xffffffff,
 702         0x0000d8bc, 0x00000100, 0xffffffff,
 703         0x000015c0, 0x000c1401, 0xffffffff,
 704         0x0000264c, 0x000c0400, 0xffffffff,
 705         0x00002648, 0x000c0400, 0xffffffff,
 706         0x00002650, 0x000c0400, 0xffffffff,
 707         0x000020b8, 0x000c0400, 0xffffffff,
 708         0x000020bc, 0x000c0400, 0xffffffff,
 709         0x000020c0, 0x000c0c80, 0xffffffff,
 710         0x0000f4a0, 0x000000c0, 0xffffffff,
 711         0x0000f4a4, 0x00680fff, 0xffffffff,
 712         0x00002f50, 0x00000903, 0xffffffff,
 713         0x000004c8, 0x00000000, 0xffffffff,
 714         0x000064ec, 0x00000000, 0xffffffff,
 715         0x00000c7c, 0x00000000, 0xffffffff,
 716         0x00008dfc, 0x00000000, 0xffffffff
 717 };
 718 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 719
 720 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
 721 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
 722
 723 extern int ni_mc_load_microcode(struct radeon_device *rdev);
 724
 725 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 726 {
 727         struct ni_power_info *pi = rdev->pm.dpm.priv;
 728
 729         return pi;
 730 }
 731
 732 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 733 {
 734         struct ni_ps *ps = rps->ps_priv;
 735
 736         return ps;
 737 }
 738
 739 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 740                                                      u16 v, s32 t,
 741                                                      u32 ileakage,
 742                                                      u32 *leakage)
 743 {
 744         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 745
 746         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 747         vddc = div64_s64(drm_int2fixp(v), 1000);
 748         temperature = div64_s64(drm_int2fixp(t), 1000);
 749
 750         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 751                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 752         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 753                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 754
 755         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 756
 757         *leakage = drm_fixp2int(leakage_w * 1000);
 758 }
 759
 760 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 761                                              const struct ni_leakage_coeffients *coeff,
 762                                              u16 v,
 763                                              s32 t,
 764                                              u32 i_leakage,
 765                                              u32 *leakage)
 766 {
 767         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 768 }
 769
 770 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
 771 {
 772         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 773         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
 774         /* we never hit the non-gddr5 limit so disable it */
 775         u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
 776
 777         if (vblank_time < switch_limit)
 778                 return true;
 779         else
 780                 return false;
 781
 782 }
 783
 784 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 785                                         struct radeon_ps *rps)
 786 {
 787         struct ni_ps *ps = ni_get_ps(rps);
 788         struct radeon_clock_and_voltage_limits *max_limits;
 789         bool disable_mclk_switching;
 790         u32 mclk;
 791         u16 vddci;
 792         int i;
 793
 794         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
 795             ni_dpm_vblank_too_short(rdev))
 796                 disable_mclk_switching = true;
 797         else
 798                 disable_mclk_switching = false;
 799
 800         if (rdev->pm.dpm.ac_power)
 801                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 802         else
 803                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 804
 805         if (rdev->pm.dpm.ac_power == false) {
 806                 for (i = 0; i < ps->performance_level_count; i++) {
 807                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 808                                 ps->performance_levels[i].mclk = max_limits->mclk;
 809                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 810                                 ps->performance_levels[i].sclk = max_limits->sclk;
 811                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 812                                 ps->performance_levels[i].vddc = max_limits->vddc;
 813                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 814                                 ps->performance_levels[i].vddci = max_limits->vddci;
 815                 }
 816         }
 817
 818         /* XXX validate the min clocks required for display */
 819
 820         /* adjust low state */
 821         if (disable_mclk_switching) {
 822                 ps->performance_levels[0].mclk =
 823                         ps->performance_levels[ps->performance_level_count - 1].mclk;
 824                 ps->performance_levels[0].vddci =
 825                         ps->performance_levels[ps->performance_level_count - 1].vddci;
 826         }
 827
 828         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 829                                   &ps->performance_levels[0].sclk,
 830                                   &ps->performance_levels[0].mclk);
 831
 832         for (i = 1; i < ps->performance_level_count; i++) {
 833                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 834                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 835                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 836                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 837         }
 838
 839         /* adjust remaining states */
 840         if (disable_mclk_switching) {
 841                 mclk = ps->performance_levels[0].mclk;
 842                 vddci = ps->performance_levels[0].vddci;
 843                 for (i = 1; i < ps->performance_level_count; i++) {
 844                         if (mclk < ps->performance_levels[i].mclk)
 845                                 mclk = ps->performance_levels[i].mclk;
 846                         if (vddci < ps->performance_levels[i].vddci)
 847                                 vddci = ps->performance_levels[i].vddci;
 848                 }
 849                 for (i = 0; i < ps->performance_level_count; i++) {
 850                         ps->performance_levels[i].mclk = mclk;
 851                         ps->performance_levels[i].vddci = vddci;
 852                 }
 853         } else {
 854                 for (i = 1; i < ps->performance_level_count; i++) {
 855                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 856                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 857                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 858                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 859                 }
 860         }
 861
 862         for (i = 1; i < ps->performance_level_count; i++)
 863                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 864                                           &ps->performance_levels[i].sclk,
 865                                           &ps->performance_levels[i].mclk);
 866
 867         for (i = 0; i < ps->performance_level_count; i++)
 868                 btc_adjust_clock_combinations(rdev, max_limits,
 869                                               &ps->performance_levels[i]);
 870
 871         for (i = 0; i < ps->performance_level_count; i++) {
 872                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 873                                                    ps->performance_levels[i].sclk,
 874                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 875                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 876                                                    ps->performance_levels[i].mclk,
 877                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 878                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 879                                                    ps->performance_levels[i].mclk,
 880                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 881                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
 882                                                    rdev->clock.current_dispclk,
 883                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 884         }
 885
 886         for (i = 0; i < ps->performance_level_count; i++) {
 887                 btc_apply_voltage_delta_rules(rdev,
 888                                               max_limits->vddc, max_limits->vddci,
 889                                               &ps->performance_levels[i].vddc,
 890                                               &ps->performance_levels[i].vddci);
 891         }
 892
 893         ps->dc_compatible = true;
 894         for (i = 0; i < ps->performance_level_count; i++) {
 895                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 896                         ps->dc_compatible = false;
 897
 898                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 899                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 900         }
 901 }
 902
 903 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 904 {
 905         u32 count;
 906         const u32 *ps = NULL;
 907
 908         ps = (const u32 *)&cayman_cgcg_cgls_default;
 909         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 910
 911         btc_program_mgcg_hw_sequence(rdev, ps, count);
 912 }
 913
 914 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 915                                       bool enable)
 916 {
 917         u32 count;
 918         const u32 *ps = NULL;
 919
 920         if (enable) {
 921                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 922                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 923         } else {
 924                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 925                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 926         }
 927
 928         btc_program_mgcg_hw_sequence(rdev, ps, count);
 929 }
 930
 931 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 932 {
 933         u32 count;
 934         const u32 *ps = NULL;
 935
 936         ps = (const u32 *)&cayman_mgcg_default;
 937         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 938
 939         btc_program_mgcg_hw_sequence(rdev, ps, count);
 940 }
 941
 942 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 943                                      bool enable)
 944 {
 945         u32 count;
 946         const u32 *ps = NULL;
 947
 948         if (enable) {
 949                 ps = (const u32 *)&cayman_mgcg_enable;
 950                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 951         } else {
 952                 ps = (const u32 *)&cayman_mgcg_disable;
 953                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 954         }
 955
 956         btc_program_mgcg_hw_sequence(rdev, ps, count);
 957 }
 958
 959 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 960 {
 961         u32 count;
 962         const u32 *ps = NULL;
 963
 964         ps = (const u32 *)&cayman_sysls_default;
 965         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 966
 967         btc_program_mgcg_hw_sequence(rdev, ps, count);
 968 }
 969
 970 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 971                                      bool enable)
 972 {
 973         u32 count;
 974         const u32 *ps = NULL;
 975
 976         if (enable) {
 977                 ps = (const u32 *)&cayman_sysls_enable;
 978                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
 979         } else {
 980                 ps = (const u32 *)&cayman_sysls_disable;
 981                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
 982         }
 983
 984         btc_program_mgcg_hw_sequence(rdev, ps, count);
 985
 986 }
 987
 988 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
 989                                                              struct radeon_clock_voltage_dependency_table *table)
 990 {
 991         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 992         u32 i;
 993
 994         if (table) {
 995                 for (i = 0; i < table->count; i++) {
 996                         if (0xff01 == table->entries[i].v) {
 997                                 if (pi->max_vddc == 0)
 998                                         return -EINVAL;
 999                                 table->entries[i].v = pi->max_vddc;
1000                         }
1001                 }
1002         }
1003         return 0;
1004 }
1005
1006 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1007 {
1008         int ret = 0;
1009
1010         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1011                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1012
1013         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1014                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1015         return ret;
1016 }
1017
1018 static void ni_stop_dpm(struct radeon_device *rdev)
1019 {
1020         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1021 }
1022
1023 #if 0
1024 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1025                                         bool ac_power)
1026 {
1027         if (ac_power)
1028                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1029                         0 : -EINVAL;
1030
1031         return 0;
1032 }
1033 #endif
1034
1035 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1036                                                       PPSMC_Msg msg, u32 parameter)
1037 {
1038         WREG32(SMC_SCRATCH0, parameter);
1039         return rv770_send_msg_to_smc(rdev, msg);
1040 }
1041
1042 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1043 {
1044         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1045                 return -EINVAL;
1046
1047         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1048                 0 : -EINVAL;
1049 }
1050
1051 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1052                                    enum radeon_dpm_forced_level level)
1053 {
1054         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1055                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1056                         return -EINVAL;
1057
1058                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1059                         return -EINVAL;
1060         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1061                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1062                         return -EINVAL;
1063
1064                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1065                         return -EINVAL;
1066         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1067                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1068                         return -EINVAL;
1069
1070                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1071                         return -EINVAL;
1072         }
1073
1074         rdev->pm.dpm.forced_level = level;
1075
1076         return 0;
1077 }
1078
1079 static void ni_stop_smc(struct radeon_device *rdev)
1080 {
1081         u32 tmp;
1082         int i;
1083
1084         for (i = 0; i < rdev->usec_timeout; i++) {
1085                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1086                 if (tmp != 1)
1087                         break;
1088                 udelay(1);
1089         }
1090
1091         udelay(100);
1092
1093         r7xx_stop_smc(rdev);
1094 }
1095
1096 static int ni_process_firmware_header(struct radeon_device *rdev)
1097 {
1098         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1099         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1100         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1101         u32 tmp;
1102         int ret;
1103
1104         ret = rv770_read_smc_sram_dword(rdev,
1105                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1106                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1107                                         &tmp, pi->sram_end);
1108
1109         if (ret)
1110                 return ret;
1111
1112         pi->state_table_start = (u16)tmp;
1113
1114         ret = rv770_read_smc_sram_dword(rdev,
1115                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1116                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1117                                         &tmp, pi->sram_end);
1118
1119         if (ret)
1120                 return ret;
1121
1122         pi->soft_regs_start = (u16)tmp;
1123
1124         ret = rv770_read_smc_sram_dword(rdev,
1125                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1126                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1127                                         &tmp, pi->sram_end);
1128
1129         if (ret)
1130                 return ret;
1131
1132         eg_pi->mc_reg_table_start = (u16)tmp;
1133
1134         ret = rv770_read_smc_sram_dword(rdev,
1135                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1136                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1137                                         &tmp, pi->sram_end);
1138
1139         if (ret)
1140                 return ret;
1141
1142         ni_pi->fan_table_start = (u16)tmp;
1143
1144         ret = rv770_read_smc_sram_dword(rdev,
1145                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1146                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1147                                         &tmp, pi->sram_end);
1148
1149         if (ret)
1150                 return ret;
1151
1152         ni_pi->arb_table_start = (u16)tmp;
1153
1154         ret = rv770_read_smc_sram_dword(rdev,
1155                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1156                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1157                                         &tmp, pi->sram_end);
1158
1159         if (ret)
1160                 return ret;
1161
1162         ni_pi->cac_table_start = (u16)tmp;
1163
1164         ret = rv770_read_smc_sram_dword(rdev,
1165                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1166                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1167                                         &tmp, pi->sram_end);
1168
1169         if (ret)
1170                 return ret;
1171
1172         ni_pi->spll_table_start = (u16)tmp;
1173
1174
1175         return ret;
1176 }
1177
1178 static void ni_read_clock_registers(struct radeon_device *rdev)
1179 {
1180         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1181
1182         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1183         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1184         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1185         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1186         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1187         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1188         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1189         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1190         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1191         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1192         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1193         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1194         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1195         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1196 }
1197
1198 #if 0
1199 static int ni_enter_ulp_state(struct radeon_device *rdev)
1200 {
1201         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1202
1203         if (pi->gfx_clock_gating) {
1204                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1205                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1206                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1207                 RREG32(GB_ADDR_CONFIG);
1208         }
1209
1210         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1211                  ~HOST_SMC_MSG_MASK);
1212
1213         udelay(25000);
1214
1215         return 0;
1216 }
1217 #endif
1218
1219 static void ni_program_response_times(struct radeon_device *rdev)
1220 {
1221         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1222         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1223         u32 reference_clock;
1224
1225         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1226
1227         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1228         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1229
1230         if (voltage_response_time == 0)
1231                 voltage_response_time = 1000;
1232
1233         if (backbias_response_time == 0)
1234                 backbias_response_time = 1000;
1235
1236         acpi_delay_time = 15000;
1237         vbi_time_out = 100000;
1238
1239         reference_clock = radeon_get_xclk(rdev);
1240
1241         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1242         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1243         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1244         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1245
1246         mclk_switch_limit = (460 * reference_clock) / 100;
1247
1248         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1249         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1250         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1251         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1252         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1253         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1254 }
1255
1256 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1257                                           struct atom_voltage_table *voltage_table,
1258                                           NISLANDS_SMC_STATETABLE *table)
1259 {
1260         unsigned int i;
1261
1262         for (i = 0; i < voltage_table->count; i++) {
1263                 table->highSMIO[i] = 0;
1264                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1265         }
1266 }
1267
1268 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1269                                            NISLANDS_SMC_STATETABLE *table)
1270 {
1271         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1272         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1273         unsigned char i;
1274
1275         if (eg_pi->vddc_voltage_table.count) {
1276                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1277                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1278                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1279                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1280
1281                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1282                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1283                                 table->maxVDDCIndexInPPTable = i;
1284                                 break;
1285                         }
1286                 }
1287         }
1288
1289         if (eg_pi->vddci_voltage_table.count) {
1290                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1291
1292                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1293                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1294                         cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
1295         }
1296 }
1297
1298 static int ni_populate_voltage_value(struct radeon_device *rdev,
1299                                      struct atom_voltage_table *table,
1300                                      u16 value,
1301                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1302 {
1303         unsigned int i;
1304
1305         for (i = 0; i < table->count; i++) {
1306                 if (value <= table->entries[i].value) {
1307                         voltage->index = (u8)i;
1308                         voltage->value = cpu_to_be16(table->entries[i].value);
1309                         break;
1310                 }
1311         }
1312
1313         if (i >= table->count)
1314                 return -EINVAL;
1315
1316         return 0;
1317 }
1318
1319 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1320                                    u32 mclk,
1321                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1322 {
1323         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1324         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1325
1326         if (!pi->mvdd_control) {
1327                 voltage->index = eg_pi->mvdd_high_index;
1328                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1329                 return;
1330         }
1331
1332         if (mclk <= pi->mvdd_split_frequency) {
1333                 voltage->index = eg_pi->mvdd_low_index;
1334                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1335         } else {
1336                 voltage->index = eg_pi->mvdd_high_index;
1337                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1338         }
1339 }
1340
1341 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1342                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1343                                     u16 *std_voltage)
1344 {
1345         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1346             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1347                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1348         else
1349                 *std_voltage = be16_to_cpu(voltage->value);
1350
1351         return 0;
1352 }
1353
1354 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1355                                           u16 value, u8 index,
1356                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1357 {
1358         voltage->index = index;
1359         voltage->value = cpu_to_be16(value);
1360 }
1361
1362 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1363 {
1364         u32 xclk_period;
1365         u32 xclk = radeon_get_xclk(rdev);
1366         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1367
1368         xclk_period = (1000000000UL / xclk);
1369         xclk_period /= 10000UL;
1370
1371         return tmp * xclk_period;
1372 }
1373
1374 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1375 {
1376         return (power_in_watts * scaling_factor) << 2;
1377 }
1378
1379 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1380                                           struct radeon_ps *radeon_state,
1381                                           u32 near_tdp_limit)
1382 {
1383         struct ni_ps *state = ni_get_ps(radeon_state);
1384         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1385         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1386         u32 power_boost_limit = 0;
1387         int ret;
1388
1389         if (ni_pi->enable_power_containment &&
1390             ni_pi->use_power_boost_limit) {
1391                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1392                 u16 std_vddc_med;
1393                 u16 std_vddc_high;
1394                 u64 tmp, n, d;
1395
1396                 if (state->performance_level_count < 3)
1397                         return 0;
1398
1399                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1400                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1401                                                 &vddc);
1402                 if (ret)
1403                         return 0;
1404
1405                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1406                 if (ret)
1407                         return 0;
1408
1409                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1410                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1411                                                 &vddc);
1412                 if (ret)
1413                         return 0;
1414
1415                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1416                 if (ret)
1417                         return 0;
1418
1419                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1420                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1421                 tmp = div64_u64(n, d);
1422
1423                 if (tmp >> 32)
1424                         return 0;
1425                 power_boost_limit = (u32)tmp;
1426         }
1427
1428         return power_boost_limit;
1429 }
1430
1431 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1432                                             bool adjust_polarity,
1433                                             u32 tdp_adjustment,
1434                                             u32 *tdp_limit,
1435                                             u32 *near_tdp_limit)
1436 {
1437         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1438                 return -EINVAL;
1439
1440         if (adjust_polarity) {
1441                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1442                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1443         } else {
1444                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1445                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1446         }
1447
1448         return 0;
1449 }
1450
1451 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1452                                       struct radeon_ps *radeon_state)
1453 {
1454         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1455         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1456
1457         if (ni_pi->enable_power_containment) {
1458                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1459                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1460                 u32 tdp_limit;
1461                 u32 near_tdp_limit;
1462                 u32 power_boost_limit;
1463                 int ret;
1464
1465                 if (scaling_factor == 0)
1466                         return -EINVAL;
1467
1468                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1469
1470                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1471                                                        false, /* ??? */
1472                                                        rdev->pm.dpm.tdp_adjustment,
1473                                                        &tdp_limit,
1474                                                        &near_tdp_limit);
1475                 if (ret)
1476                         return ret;
1477
1478                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1479                                                                    near_tdp_limit);
1480
1481                 smc_table->dpm2Params.TDPLimit =
1482                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1483                 smc_table->dpm2Params.NearTDPLimit =
1484                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1485                 smc_table->dpm2Params.SafePowerLimit =
1486                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1487                                                            scaling_factor));
1488                 smc_table->dpm2Params.PowerBoostLimit =
1489                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1490
1491                 ret = rv770_copy_bytes_to_smc(rdev,
1492                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1493                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1494                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1495                                               sizeof(u32) * 4, pi->sram_end);
1496                 if (ret)
1497                         return ret;
1498         }
1499
1500         return 0;
1501 }
1502
1503 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1504                                 u32 arb_freq_src, u32 arb_freq_dest)
1505 {
1506         u32 mc_arb_dram_timing;
1507         u32 mc_arb_dram_timing2;
1508         u32 burst_time;
1509         u32 mc_cg_config;
1510
1511         switch (arb_freq_src) {
1512         case MC_CG_ARB_FREQ_F0:
1513                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1514                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1515                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1516                 break;
1517         case MC_CG_ARB_FREQ_F1:
1518                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1519                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1520                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1521                 break;
1522         case MC_CG_ARB_FREQ_F2:
1523                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1524                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1525                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1526                 break;
1527         case MC_CG_ARB_FREQ_F3:
1528                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1529                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1530                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1531                 break;
1532         default:
1533                 return -EINVAL;
1534         }
1535
1536         switch (arb_freq_dest) {
1537         case MC_CG_ARB_FREQ_F0:
1538                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1539                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1540                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1541                 break;
1542         case MC_CG_ARB_FREQ_F1:
1543                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1544                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1545                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1546                 break;
1547         case MC_CG_ARB_FREQ_F2:
1548                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1549                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1550                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1551                 break;
1552         case MC_CG_ARB_FREQ_F3:
1553                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1554                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1555                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1556                 break;
1557         default:
1558                 return -EINVAL;
1559         }
1560
1561         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1562         WREG32(MC_CG_CONFIG, mc_cg_config);
1563         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1564
1565         return 0;
1566 }
1567
1568 static int ni_init_arb_table_index(struct radeon_device *rdev)
1569 {
1570         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1571         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1572         u32 tmp;
1573         int ret;
1574
1575         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1576                                         &tmp, pi->sram_end);
1577         if (ret)
1578                 return ret;
1579
1580         tmp &= 0x00FFFFFF;
1581         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1582
1583         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1584                                           tmp, pi->sram_end);
1585 }
1586
1587 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1588 {
1589         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1590 }
1591
1592 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1593 {
1594         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1595         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1596         u32 tmp;
1597         int ret;
1598
1599         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1600                                         &tmp, pi->sram_end);
1601         if (ret)
1602                 return ret;
1603
1604         tmp = (tmp >> 24) & 0xff;
1605
1606         if (tmp == MC_CG_ARB_FREQ_F0)
1607                 return 0;
1608
1609         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1610 }
1611
1612 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1613                                                 struct rv7xx_pl *pl,
1614                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1615 {
1616         u32 dram_timing;
1617         u32 dram_timing2;
1618
1619         arb_regs->mc_arb_rfsh_rate =
1620                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1621
1622
1623         radeon_atom_set_engine_dram_timings(rdev,
1624                                             pl->sclk,
1625                                             pl->mclk);
1626
1627         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1628         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1629
1630         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1631         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1632
1633         return 0;
1634 }
1635
1636 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1637                                                   struct radeon_ps *radeon_state,
1638                                                   unsigned int first_arb_set)
1639 {
1640         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1641         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1642         struct ni_ps *state = ni_get_ps(radeon_state);
1643         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1644         int i, ret = 0;
1645
1646         for (i = 0; i < state->performance_level_count; i++) {
1647                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1648                 if (ret)
1649                         break;
1650
1651                 ret = rv770_copy_bytes_to_smc(rdev,
1652                                               (u16)(ni_pi->arb_table_start +
1653                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1654                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1655                                               (u8 *)&arb_regs,
1656                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1657                                               pi->sram_end);
1658                 if (ret)
1659                         break;
1660         }
1661         return ret;
1662 }
1663
1664 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1665                                                struct radeon_ps *radeon_new_state)
1666 {
1667         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1668                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1669 }
1670
1671 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1672                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1673 {
1674         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1675
1676         voltage->index = eg_pi->mvdd_high_index;
1677         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1678 }
1679
1680 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1681                                          struct radeon_ps *radeon_initial_state,
1682                                          NISLANDS_SMC_STATETABLE *table)
1683 {
1684         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1685         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1686         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1687         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1688         u32 reg;
1689         int ret;
1690
1691         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
1692                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1693         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
1694                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1695         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
1696                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1697         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
1698                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1699         table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
1700                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1701         table->initialState.levels[0].mclk.vDLL_CNTL =
1702                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1703         table->initialState.levels[0].mclk.vMPLL_SS =
1704                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1705         table->initialState.levels[0].mclk.vMPLL_SS2 =
1706                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1707         table->initialState.levels[0].mclk.mclk_value =
1708                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1709
1710         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1711                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1712         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1713                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1714         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1715                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1716         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
1717                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1718         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1719                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1720         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1721                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1722         table->initialState.levels[0].sclk.sclk_value =
1723                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1724         table->initialState.levels[0].arbRefreshState =
1725                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1726
1727         table->initialState.levels[0].ACIndex = 0;
1728
1729         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1730                                         initial_state->performance_levels[0].vddc,
1731                                         &table->initialState.levels[0].vddc);
1732         if (!ret) {
1733                 u16 std_vddc;
1734
1735                 ret = ni_get_std_voltage_value(rdev,
1736                                                &table->initialState.levels[0].vddc,
1737                                                &std_vddc);
1738                 if (!ret)
1739                         ni_populate_std_voltage_value(rdev, std_vddc,
1740                                                       table->initialState.levels[0].vddc.index,
1741                                                       &table->initialState.levels[0].std_vddc);
1742         }
1743
1744         if (eg_pi->vddci_control)
1745                 ni_populate_voltage_value(rdev,
1746                                           &eg_pi->vddci_voltage_table,
1747                                           initial_state->performance_levels[0].vddci,
1748                                           &table->initialState.levels[0].vddci);
1749
1750         ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
1751
1752         reg = CG_R(0xffff) | CG_L(0);
1753         table->initialState.levels[0].aT = cpu_to_be32(reg);
1754
1755         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1756
1757         if (pi->boot_in_gen2)
1758                 table->initialState.levels[0].gen2PCIE = 1;
1759         else
1760                 table->initialState.levels[0].gen2PCIE = 0;
1761
1762         if (pi->mem_gddr5) {
1763                 table->initialState.levels[0].strobeMode =
1764                         cypress_get_strobe_mode_settings(rdev,
1765                                                          initial_state->performance_levels[0].mclk);
1766
1767                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1768                         table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1769                 else
1770                         table->initialState.levels[0].mcFlags =  0;
1771         }
1772
1773         table->initialState.levelCount = 1;
1774
1775         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1776
1777         table->initialState.levels[0].dpm2.MaxPS = 0;
1778         table->initialState.levels[0].dpm2.NearTDPDec = 0;
1779         table->initialState.levels[0].dpm2.AboveSafeInc = 0;
1780         table->initialState.levels[0].dpm2.BelowSafeInc = 0;
1781
1782         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1783         table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1784
1785         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1786         table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1787
1788         return 0;
1789 }
1790
1791 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1792                                       NISLANDS_SMC_STATETABLE *table)
1793 {
1794         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1795         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1796         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1797         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1798         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1799         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1800         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1801         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1802         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1803         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1804         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1805         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1806         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1807         u32 reg;
1808         int ret;
1809
1810         table->ACPIState = table->initialState;
1811
1812         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1813
1814         if (pi->acpi_vddc) {
1815                 ret = ni_populate_voltage_value(rdev,
1816                                                 &eg_pi->vddc_voltage_table,
1817                                                 pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
1818                 if (!ret) {
1819                         u16 std_vddc;
1820
1821                         ret = ni_get_std_voltage_value(rdev,
1822                                                        &table->ACPIState.levels[0].vddc, &std_vddc);
1823                         if (!ret)
1824                                 ni_populate_std_voltage_value(rdev, std_vddc,
1825                                                               table->ACPIState.levels[0].vddc.index,
1826                                                               &table->ACPIState.levels[0].std_vddc);
1827                 }
1828
1829                 if (pi->pcie_gen2) {
1830                         if (pi->acpi_pcie_gen2)
1831                                 table->ACPIState.levels[0].gen2PCIE = 1;
1832                         else
1833                                 table->ACPIState.levels[0].gen2PCIE = 0;
1834                 } else {
1835                         table->ACPIState.levels[0].gen2PCIE = 0;
1836                 }
1837         } else {
1838                 ret = ni_populate_voltage_value(rdev,
1839                                                 &eg_pi->vddc_voltage_table,
1840                                                 pi->min_vddc_in_table,
1841                                                 &table->ACPIState.levels[0].vddc);
1842                 if (!ret) {
1843                         u16 std_vddc;
1844
1845                         ret = ni_get_std_voltage_value(rdev,
1846                                                        &table->ACPIState.levels[0].vddc,
1847                                                        &std_vddc);
1848                         if (!ret)
1849                                 ni_populate_std_voltage_value(rdev, std_vddc,
1850                                                               table->ACPIState.levels[0].vddc.index,
1851                                                               &table->ACPIState.levels[0].std_vddc);
1852                 }
1853                 table->ACPIState.levels[0].gen2PCIE = 0;
1854         }
1855
1856         if (eg_pi->acpi_vddci) {
1857                 if (eg_pi->vddci_control)
1858                         ni_populate_voltage_value(rdev,
1859                                                   &eg_pi->vddci_voltage_table,
1860                                                   eg_pi->acpi_vddci,
1861                                                   &table->ACPIState.levels[0].vddci);
1862         }
1863
1864
1865         mpll_ad_func_cntl &= ~PDNB;
1866
1867         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1868
1869         if (pi->mem_gddr5)
1870                 mpll_dq_func_cntl &= ~PDNB;
1871         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1872
1873
1874         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1875                              MRDCKA1_RESET |
1876                              MRDCKB0_RESET |
1877                              MRDCKB1_RESET |
1878                              MRDCKC0_RESET |
1879                              MRDCKC1_RESET |
1880                              MRDCKD0_RESET |
1881                              MRDCKD1_RESET);
1882
1883         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1884                               MRDCKA1_PDNB |
1885                               MRDCKB0_PDNB |
1886                               MRDCKB1_PDNB |
1887                               MRDCKC0_PDNB |
1888                               MRDCKC1_PDNB |
1889                               MRDCKD0_PDNB |
1890                               MRDCKD1_PDNB);
1891
1892         dll_cntl |= (MRDCKA0_BYPASS |
1893                      MRDCKA1_BYPASS |
1894                      MRDCKB0_BYPASS |
1895                      MRDCKB1_BYPASS |
1896                      MRDCKC0_BYPASS |
1897                      MRDCKC1_BYPASS |
1898                      MRDCKD0_BYPASS |
1899                      MRDCKD1_BYPASS);
1900
1901         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1902         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1903
1904         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1905         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1906         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1907         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1908         table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1909         table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1910
1911         table->ACPIState.levels[0].mclk.mclk_value = 0;
1912
1913         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1914         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1915         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1916         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1917
1918         table->ACPIState.levels[0].sclk.sclk_value = 0;
1919
1920         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1921
1922         if (eg_pi->dynamic_ac_timing)
1923                 table->ACPIState.levels[0].ACIndex = 1;
1924
1925         table->ACPIState.levels[0].dpm2.MaxPS = 0;
1926         table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
1927         table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
1928         table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
1929
1930         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1931         table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1932
1933         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1934         table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1935
1936         return 0;
1937 }
1938
1939 static int ni_init_smc_table(struct radeon_device *rdev)
1940 {
1941         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1942         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1943         int ret;
1944         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1945         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1946
1947         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1948
1949         ni_populate_smc_voltage_tables(rdev, table);
1950
1951         switch (rdev->pm.int_thermal_type) {
1952         case THERMAL_TYPE_NI:
1953         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1954                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1955                 break;
1956         case THERMAL_TYPE_NONE:
1957                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1958                 break;
1959         default:
1960                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1961                 break;
1962         }
1963
1964         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1965                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1966
1967         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1968                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1969
1970         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1971                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1972
1973         if (pi->mem_gddr5)
1974                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1975
1976         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1977         if (ret)
1978                 return ret;
1979
1980         ret = ni_populate_smc_acpi_state(rdev, table);
1981         if (ret)
1982                 return ret;
1983
1984         table->driverState = table->initialState;
1985
1986         table->ULVState = table->initialState;
1987
1988         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1989                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
1990         if (ret)
1991                 return ret;
1992
1993         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1994                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1995 }
1996
1997 static int ni_calculate_sclk_params(struct radeon_device *rdev,
1998                                     u32 engine_clock,
1999                                     NISLANDS_SMC_SCLK_VALUE *sclk)
2000 {
2001         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2002         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2003         struct atom_clock_dividers dividers;
2004         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2005         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2006         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2007         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2008         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2009         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2010         u64 tmp;
2011         u32 reference_clock = rdev->clock.spll.reference_freq;
2012         u32 reference_divider;
2013         u32 fbdiv;
2014         int ret;
2015
2016         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2017                                              engine_clock, false, &dividers);
2018         if (ret)
2019                 return ret;
2020
2021         reference_divider = 1 + dividers.ref_div;
2022
2023
2024         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2025         do_div(tmp, reference_clock);
2026         fbdiv = (u32) tmp;
2027
2028         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2029         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2030         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2031
2032         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2033         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2034
2035         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2036         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2037         spll_func_cntl_3 |= SPLL_DITHEN;
2038
2039         if (pi->sclk_ss) {
2040                 struct radeon_atom_ss ss;
2041                 u32 vco_freq = engine_clock * dividers.post_div;
2042
2043                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2044                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2045                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2046                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2047
2048                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2049                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2050                         cg_spll_spread_spectrum |= SSEN;
2051
2052                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2053                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2054                 }
2055         }
2056
2057         sclk->sclk_value = engine_clock;
2058         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2059         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2060         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2061         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2062         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2063         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2064
2065         return 0;
2066 }
2067
2068 static int ni_populate_sclk_value(struct radeon_device *rdev,
2069                                   u32 engine_clock,
2070                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2071 {
2072         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2073         int ret;
2074
2075         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2076         if (!ret) {
2077                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2078                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2079                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2080                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2081                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2082                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2083                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2084         }
2085
2086         return ret;
2087 }
2088
2089 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2090 {
2091         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2092         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2093         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2094         NISLANDS_SMC_SCLK_VALUE sclk_params;
2095         u32 fb_div;
2096         u32 p_div;
2097         u32 clk_s;
2098         u32 clk_v;
2099         u32 sclk = 0;
2100         int i, ret;
2101         u32 tmp;
2102
2103         if (ni_pi->spll_table_start == 0)
2104                 return -EINVAL;
2105
2106         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2107         if (spll_table == NULL)
2108                 return -ENOMEM;
2109
2110         for (i = 0; i < 256; i++) {
2111                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2112                 if (ret)
2113                         break;
2114
2115                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2116                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2117                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2118                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2119
2120                 fb_div &= ~0x00001FFF;
2121                 fb_div >>= 1;
2122                 clk_v >>= 6;
2123
2124                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2125                         ret = -EINVAL;
2126
2127                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2128                         ret = -EINVAL;
2129
2130                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2131                         ret = -EINVAL;
2132
2133                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2134                         ret = -EINVAL;
2135
2136                 if (ret)
2137                         break;
2138
2139                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2140                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2141                 spll_table->freq[i] = cpu_to_be32(tmp);
2142
2143                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2144                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2145                 spll_table->ss[i] = cpu_to_be32(tmp);
2146
2147                 sclk += 512;
2148         }
2149
2150         if (!ret)
2151                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2152                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2153
2154         kfree(spll_table);
2155
2156         return ret;
2157 }
2158
2159 static int ni_populate_mclk_value(struct radeon_device *rdev,
2160                                   u32 engine_clock,
2161                                   u32 memory_clock,
2162                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2163                                   bool strobe_mode,
2164                                   bool dll_state_on)
2165 {
2166         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2167         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2168         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2169         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2170         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2171         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2172         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2173         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2174         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2175         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2176         struct atom_clock_dividers dividers;
2177         u32 ibias;
2178         u32 dll_speed;
2179         int ret;
2180         u32 mc_seq_misc7;
2181
2182         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2183                                              memory_clock, strobe_mode, &dividers);
2184         if (ret)
2185                 return ret;
2186
2187         if (!strobe_mode) {
2188                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2189
2190                 if (mc_seq_misc7 & 0x8000000)
2191                         dividers.post_div = 1;
2192         }
2193
2194         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2195
2196         mpll_ad_func_cntl &= ~(CLKR_MASK |
2197                                YCLK_POST_DIV_MASK |
2198                                CLKF_MASK |
2199                                CLKFRAC_MASK |
2200                                IBIAS_MASK);
2201         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2202         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2203         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2204         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2205         mpll_ad_func_cntl |= IBIAS(ibias);
2206
2207         if (dividers.vco_mode)
2208                 mpll_ad_func_cntl_2 |= VCO_MODE;
2209         else
2210                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2211
2212         if (pi->mem_gddr5) {
2213                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2214                                        YCLK_POST_DIV_MASK |
2215                                        CLKF_MASK |
2216                                        CLKFRAC_MASK |
2217                                        IBIAS_MASK);
2218                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2219                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2220                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2221                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2222                 mpll_dq_func_cntl |= IBIAS(ibias);
2223
2224                 if (strobe_mode)
2225                         mpll_dq_func_cntl &= ~PDNB;
2226                 else
2227                         mpll_dq_func_cntl |= PDNB;
2228
2229                 if (dividers.vco_mode)
2230                         mpll_dq_func_cntl_2 |= VCO_MODE;
2231                 else
2232                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2233         }
2234
2235         if (pi->mclk_ss) {
2236                 struct radeon_atom_ss ss;
2237                 u32 vco_freq = memory_clock * dividers.post_div;
2238
2239                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2240                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2241                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2242                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2243                         u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2244                         u32 clk_v = ss.percentage *
2245                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2246
2247                         mpll_ss1 &= ~CLKV_MASK;
2248                         mpll_ss1 |= CLKV(clk_v);
2249
2250                         mpll_ss2 &= ~CLKS_MASK;
2251                         mpll_ss2 |= CLKS(clk_s);
2252                 }
2253         }
2254
2255         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2256                                         memory_clock);
2257
2258         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2259         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2260         if (dll_state_on)
2261                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2262                                      MRDCKA1_PDNB |
2263                                      MRDCKB0_PDNB |
2264                                      MRDCKB1_PDNB |
2265                                      MRDCKC0_PDNB |
2266                                      MRDCKC1_PDNB |
2267                                      MRDCKD0_PDNB |
2268                                      MRDCKD1_PDNB);
2269         else
2270                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2271                                       MRDCKA1_PDNB |
2272                                       MRDCKB0_PDNB |
2273                                       MRDCKB1_PDNB |
2274                                       MRDCKC0_PDNB |
2275                                       MRDCKC1_PDNB |
2276                                       MRDCKD0_PDNB |
2277                                       MRDCKD1_PDNB);
2278
2279
2280         mclk->mclk_value = cpu_to_be32(memory_clock);
2281         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2282         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2283         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2284         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2285         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2286         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2287         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2288         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2289
2290         return 0;
2291 }
2292
2293 static void ni_populate_smc_sp(struct radeon_device *rdev,
2294                                struct radeon_ps *radeon_state,
2295                                NISLANDS_SMC_SWSTATE *smc_state)
2296 {
2297         struct ni_ps *ps = ni_get_ps(radeon_state);
2298         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2299         int i;
2300
2301         for (i = 0; i < ps->performance_level_count - 1; i++)
2302                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2303
2304         smc_state->levels[ps->performance_level_count - 1].bSP =
2305                 cpu_to_be32(pi->psp);
2306 }
2307
2308 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2309                                          struct rv7xx_pl *pl,
2310                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2311 {
2312         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2313         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2314         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2315         int ret;
2316         bool dll_state_on;
2317         u16 std_vddc;
2318         u32 tmp = RREG32(DC_STUTTER_CNTL);
2319
2320         level->gen2PCIE = pi->pcie_gen2 ?
2321                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2322
2323         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2324         if (ret)
2325                 return ret;
2326
2327         level->mcFlags =  0;
2328         if (pi->mclk_stutter_mode_threshold &&
2329             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2330             !eg_pi->uvd_enabled &&
2331             (tmp & DC_STUTTER_ENABLE_A) &&
2332             (tmp & DC_STUTTER_ENABLE_B))
2333                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2334
2335         if (pi->mem_gddr5) {
2336                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2337                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2338                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2339                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2340
2341                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2342
2343                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2344                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2345                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2346                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2347                         else
2348                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2349                 } else {
2350                         dll_state_on = false;
2351                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2352                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2353                 }
2354
2355                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2356                                              &level->mclk,
2357                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2358                                              dll_state_on);
2359         } else
2360                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2361
2362         if (ret)
2363                 return ret;
2364
2365         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2366                                         pl->vddc, &level->vddc);
2367         if (ret)
2368                 return ret;
2369
2370         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2371         if (ret)
2372                 return ret;
2373
2374         ni_populate_std_voltage_value(rdev, std_vddc,
2375                                       level->vddc.index, &level->std_vddc);
2376
2377         if (eg_pi->vddci_control) {
2378                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2379                                                 pl->vddci, &level->vddci);
2380                 if (ret)
2381                         return ret;
2382         }
2383
2384         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2385
2386         return ret;
2387 }
2388
2389 static int ni_populate_smc_t(struct radeon_device *rdev,
2390                              struct radeon_ps *radeon_state,
2391                              NISLANDS_SMC_SWSTATE *smc_state)
2392 {
2393         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2394         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2395         struct ni_ps *state = ni_get_ps(radeon_state);
2396         u32 a_t;
2397         u32 t_l, t_h;
2398         u32 high_bsp;
2399         int i, ret;
2400
2401         if (state->performance_level_count >= 9)
2402                 return -EINVAL;
2403
2404         if (state->performance_level_count < 2) {
2405                 a_t = CG_R(0xffff) | CG_L(0);
2406                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2407                 return 0;
2408         }
2409
2410         smc_state->levels[0].aT = cpu_to_be32(0);
2411
2412         for (i = 0; i <= state->performance_level_count - 2; i++) {
2413                 if (eg_pi->uvd_enabled)
2414                         ret = r600_calculate_at(
2415                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2416                                 100 * R600_AH_DFLT,
2417                                 state->performance_levels[i + 1].sclk,
2418                                 state->performance_levels[i].sclk,
2419                                 &t_l,
2420                                 &t_h);
2421                 else
2422                         ret = r600_calculate_at(
2423                                 1000 * (i + 1),
2424                                 100 * R600_AH_DFLT,
2425                                 state->performance_levels[i + 1].sclk,
2426                                 state->performance_levels[i].sclk,
2427                                 &t_l,
2428                                 &t_h);
2429
2430                 if (ret) {
2431                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2432                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2433                 }
2434
2435                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2436                 a_t |= CG_R(t_l * pi->bsp / 20000);
2437                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2438
2439                 high_bsp = (i == state->performance_level_count - 2) ?
2440                         pi->pbsp : pi->bsp;
2441
2442                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2443                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2444         }
2445
2446         return 0;
2447 }
2448
2449 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2450                                                 struct radeon_ps *radeon_state,
2451                                                 NISLANDS_SMC_SWSTATE *smc_state)
2452 {
2453         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2454         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2455         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2456         struct ni_ps *state = ni_get_ps(radeon_state);
2457         u32 prev_sclk;
2458         u32 max_sclk;
2459         u32 min_sclk;
2460         int i, ret;
2461         u32 tdp_limit;
2462         u32 near_tdp_limit;
2463         u32 power_boost_limit;
2464         u8 max_ps_percent;
2465
2466         if (ni_pi->enable_power_containment == false)
2467                 return 0;
2468
2469         if (state->performance_level_count == 0)
2470                 return -EINVAL;
2471
2472         if (smc_state->levelCount != state->performance_level_count)
2473                 return -EINVAL;
2474
2475         ret = ni_calculate_adjusted_tdp_limits(rdev,
2476                                                false, /* ??? */
2477                                                rdev->pm.dpm.tdp_adjustment,
2478                                                &tdp_limit,
2479                                                &near_tdp_limit);
2480         if (ret)
2481                 return ret;
2482
2483         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2484
2485         ret = rv770_write_smc_sram_dword(rdev,
2486                                          pi->state_table_start +
2487                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2488                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2489                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2490                                          pi->sram_end);
2491         if (ret)
2492                 power_boost_limit = 0;
2493
2494         smc_state->levels[0].dpm2.MaxPS = 0;
2495         smc_state->levels[0].dpm2.NearTDPDec = 0;
2496         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2497         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2498         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2499
2500         for (i = 1; i < state->performance_level_count; i++) {
2501                 prev_sclk = state->performance_levels[i-1].sclk;
2502                 max_sclk  = state->performance_levels[i].sclk;
2503                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2504                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2505
2506                 if (max_sclk < prev_sclk)
2507                         return -EINVAL;
2508
2509                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2510                         min_sclk = max_sclk;
2511                 else if (1 == i)
2512                         min_sclk = prev_sclk;
2513                 else
2514                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2515
2516                 if (min_sclk < state->performance_levels[0].sclk)
2517                         min_sclk = state->performance_levels[0].sclk;
2518
2519                 if (min_sclk == 0)
2520                         return -EINVAL;
2521
2522                 smc_state->levels[i].dpm2.MaxPS =
2523                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2524                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2525                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2526                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2527                 smc_state->levels[i].stateFlags |=
2528                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2529                         PPSMC_STATEFLAG_POWERBOOST : 0;
2530         }
2531
2532         return 0;
2533 }
2534
2535 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2536                                          struct radeon_ps *radeon_state,
2537                                          NISLANDS_SMC_SWSTATE *smc_state)
2538 {
2539         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2540         struct ni_ps *state = ni_get_ps(radeon_state);
2541         u32 sq_power_throttle;
2542         u32 sq_power_throttle2;
2543         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2544         int i;
2545
2546         if (state->performance_level_count == 0)
2547                 return -EINVAL;
2548
2549         if (smc_state->levelCount != state->performance_level_count)
2550                 return -EINVAL;
2551
2552         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2553                 return -EINVAL;
2554
2555         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2556                 enable_sq_ramping = false;
2557
2558         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2559                 enable_sq_ramping = false;
2560
2561         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2562                 enable_sq_ramping = false;
2563
2564         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2565                 enable_sq_ramping = false;
2566
2567         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2568                 enable_sq_ramping = false;
2569
2570         for (i = 0; i < state->performance_level_count; i++) {
2571                 sq_power_throttle  = 0;
2572                 sq_power_throttle2 = 0;
2573
2574                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2575                     enable_sq_ramping) {
2576                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2577                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2578                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2579                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2580                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2581                 } else {
2582                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2583                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2584                 }
2585
2586                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2587                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2588         }
2589
2590         return 0;
2591 }
2592
2593 static int ni_enable_power_containment(struct radeon_device *rdev,
2594                                        struct radeon_ps *radeon_new_state,
2595                                        bool enable)
2596 {
2597         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2598         PPSMC_Result smc_result;
2599         int ret = 0;
2600
2601         if (ni_pi->enable_power_containment) {
2602                 if (enable) {
2603                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2604                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2605                                 if (smc_result != PPSMC_Result_OK) {
2606                                         ret = -EINVAL;
2607                                         ni_pi->pc_enabled = false;
2608                                 } else {
2609                                         ni_pi->pc_enabled = true;
2610                                 }
2611                         }
2612                 } else {
2613                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2614                         if (smc_result != PPSMC_Result_OK)
2615                                 ret = -EINVAL;
2616                         ni_pi->pc_enabled = false;
2617                 }
2618         }
2619
2620         return ret;
2621 }
2622
2623 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2624                                          struct radeon_ps *radeon_state,
2625                                          NISLANDS_SMC_SWSTATE *smc_state)
2626 {
2627         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2628         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2629         struct ni_ps *state = ni_get_ps(radeon_state);
2630         int i, ret;
2631         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2632
2633         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2634                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2635
2636         smc_state->levelCount = 0;
2637
2638         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2639                 return -EINVAL;
2640
2641         for (i = 0; i < state->performance_level_count; i++) {
2642                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2643                                                     &smc_state->levels[i]);
2644                 smc_state->levels[i].arbRefreshState =
2645                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2646
2647                 if (ret)
2648                         return ret;
2649
2650                 if (ni_pi->enable_power_containment)
2651                         smc_state->levels[i].displayWatermark =
2652                                 (state->performance_levels[i].sclk < threshold) ?
2653                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2654                 else
2655                         smc_state->levels[i].displayWatermark = (i < 2) ?
2656                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2657
2658                 if (eg_pi->dynamic_ac_timing)
2659                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2660                 else
2661                         smc_state->levels[i].ACIndex = 0;
2662
2663                 smc_state->levelCount++;
2664         }
2665
2666         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2667                                       cpu_to_be32(threshold / 512));
2668
2669         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2670
2671         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2672         if (ret)
2673                 ni_pi->enable_power_containment = false;
2674
2675         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2676         if (ret)
2677                 ni_pi->enable_sq_ramping = false;
2678
2679         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2680 }
2681
2682 static int ni_upload_sw_state(struct radeon_device *rdev,
2683                               struct radeon_ps *radeon_new_state)
2684 {
2685         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2686         u16 address = pi->state_table_start +
2687                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2688         u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) +
2689                 ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL));
2690         int ret;
2691         NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL);
2692
2693         if (smc_state == NULL)
2694                 return -ENOMEM;
2695
2696         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2697         if (ret)
2698                 goto done;
2699
2700         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2701
2702 done:
2703         kfree(smc_state);
2704
2705         return ret;
2706 }
2707
2708 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2709                                        struct ni_mc_reg_table *table)
2710 {
2711         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2712         u8 i, j, k;
2713         u32 temp_reg;
2714
2715         for (i = 0, j = table->last; i < table->last; i++) {
2716                 switch (table->mc_reg_address[i].s1) {
2717                 case MC_SEQ_MISC1 >> 2:
2718                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2719                                 return -EINVAL;
2720                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2721                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2722                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2723                         for (k = 0; k < table->num_entries; k++)
2724                                 table->mc_reg_table_entry[k].mc_data[j] =
2725                                         ((temp_reg & 0xffff0000)) |
2726                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2727                         j++;
2728                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2729                                 return -EINVAL;
2730
2731                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2732                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2733                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2734                         for(k = 0; k < table->num_entries; k++) {
2735                                 table->mc_reg_table_entry[k].mc_data[j] =
2736                                         (temp_reg & 0xffff0000) |
2737                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2738                                 if (!pi->mem_gddr5)
2739                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2740                         }
2741                         j++;
2742                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2743                                 return -EINVAL;
2744                         break;
2745                 case MC_SEQ_RESERVE_M >> 2:
2746                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2747                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2748                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2749                         for (k = 0; k < table->num_entries; k++)
2750                                 table->mc_reg_table_entry[k].mc_data[j] =
2751                                         (temp_reg & 0xffff0000) |
2752                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2753                         j++;
2754                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2755                                 return -EINVAL;
2756                         break;
2757                 default:
2758                         break;
2759                 }
2760         }
2761
2762         table->last = j;
2763
2764         return 0;
2765 }
2766
2767 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2768 {
2769         bool result = true;
2770
2771         switch (in_reg) {
2772         case  MC_SEQ_RAS_TIMING >> 2:
2773                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2774                 break;
2775         case MC_SEQ_CAS_TIMING >> 2:
2776                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2777                 break;
2778         case MC_SEQ_MISC_TIMING >> 2:
2779                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2780                 break;
2781         case MC_SEQ_MISC_TIMING2 >> 2:
2782                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2783                 break;
2784         case MC_SEQ_RD_CTL_D0 >> 2:
2785                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2786                 break;
2787         case MC_SEQ_RD_CTL_D1 >> 2:
2788                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2789                 break;
2790         case MC_SEQ_WR_CTL_D0 >> 2:
2791                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2792                 break;
2793         case MC_SEQ_WR_CTL_D1 >> 2:
2794                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2795                 break;
2796         case MC_PMG_CMD_EMRS >> 2:
2797                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2798                 break;
2799         case MC_PMG_CMD_MRS >> 2:
2800                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2801                 break;
2802         case MC_PMG_CMD_MRS1 >> 2:
2803                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2804                 break;
2805         case MC_SEQ_PMG_TIMING >> 2:
2806                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2807                 break;
2808         case MC_PMG_CMD_MRS2 >> 2:
2809                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2810                 break;
2811         default:
2812                 result = false;
2813                 break;
2814         }
2815
2816         return result;
2817 }
2818
2819 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2820 {
2821         u8 i, j;
2822
2823         for (i = 0; i < table->last; i++) {
2824                 for (j = 1; j < table->num_entries; j++) {
2825                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2826                                 table->valid_flag |= 1 << i;
2827                                 break;
2828                         }
2829                 }
2830         }
2831 }
2832
2833 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2834 {
2835         u32 i;
2836         u16 address;
2837
2838         for (i = 0; i < table->last; i++)
2839                 table->mc_reg_address[i].s0 =
2840                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2841                         address : table->mc_reg_address[i].s1;
2842 }
2843
2844 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2845                                       struct ni_mc_reg_table *ni_table)
2846 {
2847         u8 i, j;
2848
2849         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2850                 return -EINVAL;
2851         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2852                 return -EINVAL;
2853
2854         for (i = 0; i < table->last; i++)
2855                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2856         ni_table->last = table->last;
2857
2858         for (i = 0; i < table->num_entries; i++) {
2859                 ni_table->mc_reg_table_entry[i].mclk_max =
2860                         table->mc_reg_table_entry[i].mclk_max;
2861                 for (j = 0; j < table->last; j++)
2862                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2863                                 table->mc_reg_table_entry[i].mc_data[j];
2864         }
2865         ni_table->num_entries = table->num_entries;
2866
2867         return 0;
2868 }
2869
2870 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2871 {
2872         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2873         int ret;
2874         struct atom_mc_reg_table *table;
2875         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2876         u8 module_index = rv770_get_memory_module_index(rdev);
2877
2878         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2879         if (!table)
2880                 return -ENOMEM;
2881
2882         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2883         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2884         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2885         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2886         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2887         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2888         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2889         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2890         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2891         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2892         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2893         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2894         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2895
2896         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2897
2898         if (ret)
2899                 goto init_mc_done;
2900
2901         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2902
2903         if (ret)
2904                 goto init_mc_done;
2905
2906         ni_set_s0_mc_reg_index(ni_table);
2907
2908         ret = ni_set_mc_special_registers(rdev, ni_table);
2909
2910         if (ret)
2911                 goto init_mc_done;
2912
2913         ni_set_valid_flag(ni_table);
2914
2915 init_mc_done:
2916         kfree(table);
2917
2918         return ret;
2919 }
2920
2921 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2922                                          SMC_NIslands_MCRegisters *mc_reg_table)
2923 {
2924         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2925         u32 i, j;
2926
2927         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2928                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2929                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2930                                 break;
2931                         mc_reg_table->address[i].s0 =
2932                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2933                         mc_reg_table->address[i].s1 =
2934                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2935                         i++;
2936                 }
2937         }
2938         mc_reg_table->last = (u8)i;
2939 }
2940
2941
2942 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2943                                     SMC_NIslands_MCRegisterSet *data,
2944                                     u32 num_entries, u32 valid_flag)
2945 {
2946         u32 i, j;
2947
2948         for (i = 0, j = 0; j < num_entries; j++) {
2949                 if (valid_flag & (1 << j)) {
2950                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2951                         i++;
2952                 }
2953         }
2954 }
2955
2956 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2957                                                  struct rv7xx_pl *pl,
2958                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2959 {
2960         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2961         u32 i = 0;
2962
2963         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2964                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2965                         break;
2966         }
2967
2968         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2969                 --i;
2970
2971         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2972                                 mc_reg_table_data,
2973                                 ni_pi->mc_reg_table.last,
2974                                 ni_pi->mc_reg_table.valid_flag);
2975 }
2976
2977 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2978                                            struct radeon_ps *radeon_state,
2979                                            SMC_NIslands_MCRegisters *mc_reg_table)
2980 {
2981         struct ni_ps *state = ni_get_ps(radeon_state);
2982         int i;
2983
2984         for (i = 0; i < state->performance_level_count; i++) {
2985                 ni_convert_mc_reg_table_entry_to_smc(rdev,
2986                                                      &state->performance_levels[i],
2987                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2988         }
2989 }
2990
2991 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2992                                     struct radeon_ps *radeon_boot_state)
2993 {
2994         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2995         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2996         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2997         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
2998         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
2999
3000         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3001
3002         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3003
3004         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3005
3006         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3007                                              &mc_reg_table->data[0]);
3008
3009         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3010                                 &mc_reg_table->data[1],
3011                                 ni_pi->mc_reg_table.last,
3012                                 ni_pi->mc_reg_table.valid_flag);
3013
3014         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3015
3016         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3017                                        (u8 *)mc_reg_table,
3018                                        sizeof(SMC_NIslands_MCRegisters),
3019                                        pi->sram_end);
3020 }
3021
3022 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3023                                   struct radeon_ps *radeon_new_state)
3024 {
3025         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3026         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3027         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3028         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3029         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3030         u16 address;
3031
3032         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3033
3034         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3035
3036         address = eg_pi->mc_reg_table_start +
3037                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3038
3039         return rv770_copy_bytes_to_smc(rdev, address,
3040                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3041                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3042                                        pi->sram_end);
3043 }
3044
3045 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3046                                                    PP_NIslands_CACTABLES *cac_tables)
3047 {
3048         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3049         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3050         u32 leakage = 0;
3051         unsigned int i, j, table_size;
3052         s32 t;
3053         u32 smc_leakage, max_leakage = 0;
3054         u32 scaling_factor;
3055
3056         table_size = eg_pi->vddc_voltage_table.count;
3057
3058         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3059                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3060
3061         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3062
3063         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3064                 for (j = 0; j < table_size; j++) {
3065                         t = (1000 * ((i + 1) * 8));
3066
3067                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3068                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3069
3070                         ni_calculate_leakage_for_v_and_t(rdev,
3071                                                          &ni_pi->cac_data.leakage_coefficients,
3072                                                          eg_pi->vddc_voltage_table.entries[j].value,
3073                                                          t,
3074                                                          ni_pi->cac_data.i_leakage,
3075                                                          &leakage);
3076
3077                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3078                         if (smc_leakage > max_leakage)
3079                                 max_leakage = smc_leakage;
3080
3081                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3082                 }
3083         }
3084
3085         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3086                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3087                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3088         }
3089         return 0;
3090 }
3091
3092 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3093                                             PP_NIslands_CACTABLES *cac_tables)
3094 {
3095         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3096         struct radeon_cac_leakage_table *leakage_table =
3097                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3098         u32 i, j, table_size;
3099         u32 smc_leakage, max_leakage = 0;
3100         u32 scaling_factor;
3101
3102         if (!leakage_table)
3103                 return -EINVAL;
3104
3105         table_size = leakage_table->count;
3106
3107         if (eg_pi->vddc_voltage_table.count != table_size)
3108                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3109                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3110
3111         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3112                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3113
3114         if (table_size == 0)
3115                 return -EINVAL;
3116
3117         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3118
3119         for (j = 0; j < table_size; j++) {
3120                 smc_leakage = leakage_table->entries[j].leakage;
3121
3122                 if (smc_leakage > max_leakage)
3123                         max_leakage = smc_leakage;
3124
3125                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3126                         cac_tables->cac_lkge_lut[i][j] =
3127                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3128         }
3129
3130         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3131                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3132                         cac_tables->cac_lkge_lut[i][j] =
3133                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3134         }
3135         return 0;
3136 }
3137
3138 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3139 {
3140         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3141         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3142         PP_NIslands_CACTABLES *cac_tables = NULL;
3143         int i, ret;
3144         u32 reg;
3145
3146         if (ni_pi->enable_cac == false)
3147                 return 0;
3148
3149         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3150         if (!cac_tables)
3151                 return -ENOMEM;
3152
3153         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3154         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3155                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3156         WREG32(CG_CAC_CTRL, reg);
3157
3158         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3159                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3160
3161         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3162                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3163
3164         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3165         ni_pi->cac_data.pwr_const = 0;
3166         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3167         ni_pi->cac_data.bif_cac_value = 0;
3168         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3169         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3170         ni_pi->cac_data.allow_ovrflw = 0;
3171         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3172         ni_pi->cac_data.num_win_tdp = 0;
3173         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3174
3175         if (ni_pi->driver_calculate_cac_leakage)
3176                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3177         else
3178                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3179
3180         if (ret)
3181                 goto done_free;
3182
3183         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3184         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3185         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3186         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3187         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3188         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3189         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3190         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3191         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3192
3193         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3194                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3195
3196 done_free:
3197         if (ret) {
3198                 ni_pi->enable_cac = false;
3199                 ni_pi->enable_power_containment = false;
3200         }
3201
3202         kfree(cac_tables);
3203
3204         return 0;
3205 }
3206
3207 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3208 {
3209         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3210         u32 reg;
3211
3212         if (!ni_pi->enable_cac ||
3213             !ni_pi->cac_configuration_required)
3214                 return 0;
3215
3216         if (ni_pi->cac_weights == NULL)
3217                 return -EINVAL;
3218
3219         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3220                                                       WEIGHT_TCP_SIG1_MASK |
3221                                                       WEIGHT_TA_SIG_MASK);
3222         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3223                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3224                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3225         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3226
3227         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3228                                                       WEIGHT_TCC_EN1_MASK |
3229                                                       WEIGHT_TCC_EN2_MASK);
3230         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3231                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3232                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3233         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3234
3235         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3236                                                       WEIGHT_CB_EN1_MASK |
3237                                                       WEIGHT_CB_EN2_MASK |
3238                                                       WEIGHT_CB_EN3_MASK);
3239         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3240                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3241                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3242                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3243         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3244
3245         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3246                                                       WEIGHT_DB_SIG1_MASK |
3247                                                       WEIGHT_DB_SIG2_MASK |
3248                                                       WEIGHT_DB_SIG3_MASK);
3249         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3250                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3251                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3252                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3253         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3254
3255         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3256                                                       WEIGHT_SXM_SIG1_MASK |
3257                                                       WEIGHT_SXM_SIG2_MASK |
3258                                                       WEIGHT_SXS_SIG0_MASK |
3259                                                       WEIGHT_SXS_SIG1_MASK);
3260         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3261                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3262                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3263                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3264                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3265         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3266
3267         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3268                                                       WEIGHT_XBR_1_MASK |
3269                                                       WEIGHT_XBR_2_MASK |
3270                                                       WEIGHT_SPI_SIG0_MASK);
3271         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3272                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3273                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3274                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3275         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3276
3277         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3278                                                       WEIGHT_SPI_SIG2_MASK |
3279                                                       WEIGHT_SPI_SIG3_MASK |
3280                                                       WEIGHT_SPI_SIG4_MASK |
3281                                                       WEIGHT_SPI_SIG5_MASK);
3282         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3283                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3284                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3285                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3286                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3287         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3288
3289         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3290                                                       WEIGHT_LDS_SIG1_MASK |
3291                                                       WEIGHT_SC_MASK);
3292         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3293                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3294                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3295         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3296
3297         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3298                                                       WEIGHT_CP_MASK |
3299                                                       WEIGHT_PA_SIG0_MASK |
3300                                                       WEIGHT_PA_SIG1_MASK |
3301                                                       WEIGHT_VGT_SIG0_MASK);
3302         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3303                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3304                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3305                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3306                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3307         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3308
3309         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3310                                                       WEIGHT_VGT_SIG2_MASK |
3311                                                       WEIGHT_DC_SIG0_MASK |
3312                                                       WEIGHT_DC_SIG1_MASK |
3313                                                       WEIGHT_DC_SIG2_MASK);
3314         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3315                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3316                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3317                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3318                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3319         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3320
3321         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3322                                                       WEIGHT_UVD_SIG0_MASK |
3323                                                       WEIGHT_UVD_SIG1_MASK |
3324                                                       WEIGHT_SPARE0_MASK |
3325                                                       WEIGHT_SPARE1_MASK);
3326         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3327                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3328                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3329                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3330                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3331         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3332
3333         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3334                                                       WEIGHT_SQ_VSP0_MASK);
3335         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3336                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3337         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3338
3339         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3340         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3341         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3342
3343         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3344                                                         OVR_VAL_SPARE_0_MASK |
3345                                                         OVR_MODE_SPARE_1_MASK |
3346                                                         OVR_VAL_SPARE_1_MASK);
3347         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3348                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3349                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3350                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3351         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3352
3353         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3354                                            VSP0_MASK |
3355                                            GPR_MASK);
3356         reg |= (VSP(ni_pi->cac_weights->vsp) |
3357                 VSP0(ni_pi->cac_weights->vsp0) |
3358                 GPR(ni_pi->cac_weights->gpr));
3359         WREG32(SQ_CAC_THRESHOLD, reg);
3360
3361         reg = (MCDW_WR_ENABLE |
3362                MCDX_WR_ENABLE |
3363                MCDY_WR_ENABLE |
3364                MCDZ_WR_ENABLE |
3365                INDEX(0x09D4));
3366         WREG32(MC_CG_CONFIG, reg);
3367
3368         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3369                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3370                ALLOW_OVERFLOW);
3371         WREG32(MC_CG_DATAPORT, reg);
3372
3373         return 0;
3374 }
3375
3376 static int ni_enable_smc_cac(struct radeon_device *rdev,
3377                              struct radeon_ps *radeon_new_state,
3378                              bool enable)
3379 {
3380         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3381         int ret = 0;
3382         PPSMC_Result smc_result;
3383
3384         if (ni_pi->enable_cac) {
3385                 if (enable) {
3386                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3387                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3388
3389                                 if (ni_pi->support_cac_long_term_average) {
3390                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3391                                         if (PPSMC_Result_OK != smc_result)
3392                                                 ni_pi->support_cac_long_term_average = false;
3393                                 }
3394
3395                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3396                                 if (PPSMC_Result_OK != smc_result)
3397                                         ret = -EINVAL;
3398
3399                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3400                         }
3401                 } else if (ni_pi->cac_enabled) {
3402                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3403
3404                         ni_pi->cac_enabled = false;
3405
3406                         if (ni_pi->support_cac_long_term_average) {
3407                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3408                                 if (PPSMC_Result_OK != smc_result)
3409                                         ni_pi->support_cac_long_term_average = false;
3410                         }
3411                 }
3412         }
3413
3414         return ret;
3415 }
3416
3417 static int ni_pcie_performance_request(struct radeon_device *rdev,
3418                                        u8 perf_req, bool advertise)
3419 {
3420 #if defined(CONFIG_ACPI)
3421         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3422
3423         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3424             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3425                 if (eg_pi->pcie_performance_request_registered == false)
3426                         radeon_acpi_pcie_notify_device_ready(rdev);
3427                 eg_pi->pcie_performance_request_registered = true;
3428                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3429         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3430                    eg_pi->pcie_performance_request_registered) {
3431                 eg_pi->pcie_performance_request_registered = false;
3432                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3433         }
3434 #endif
3435         return 0;
3436 }
3437
3438 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3439 {
3440         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3441         u32 tmp;
3442
3443         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3444
3445         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3446             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3447                 pi->pcie_gen2 = true;
3448         else
3449                 pi->pcie_gen2 = false;
3450
3451         if (!pi->pcie_gen2)
3452                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3453
3454         return 0;
3455 }
3456
3457 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3458                                             bool enable)
3459 {
3460         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3461         u32 tmp, bif;
3462
3463         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3464
3465         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3466             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3467                 if (enable) {
3468                         if (!pi->boot_in_gen2) {
3469                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3470                                 bif |= CG_CLIENT_REQ(0xd);
3471                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3472                         }
3473                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3474                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3475                         tmp |= LC_GEN2_EN_STRAP;
3476
3477                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3478                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3479                         udelay(10);
3480                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3481                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3482                 } else {
3483                         if (!pi->boot_in_gen2) {
3484                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3485                                 bif |= CG_CLIENT_REQ(0xd);
3486                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3487
3488                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3489                                 tmp &= ~LC_GEN2_EN_STRAP;
3490                         }
3491                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3492                 }
3493         }
3494 }
3495
3496 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3497                                         bool enable)
3498 {
3499         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3500
3501         if (enable)
3502                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3503         else
3504                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3505 }
3506
3507 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3508                                            struct radeon_ps *new_ps,
3509                                            struct radeon_ps *old_ps)
3510 {
3511         struct ni_ps *new_state = ni_get_ps(new_ps);
3512         struct ni_ps *current_state = ni_get_ps(old_ps);
3513
3514         if ((new_ps->vclk == old_ps->vclk) &&
3515             (new_ps->dclk == old_ps->dclk))
3516                 return;
3517
3518         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3519             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3520                 return;
3521
3522         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3523 }
3524
3525 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3526                                           struct radeon_ps *new_ps,
3527                                           struct radeon_ps *old_ps)
3528 {
3529         struct ni_ps *new_state = ni_get_ps(new_ps);
3530         struct ni_ps *current_state = ni_get_ps(old_ps);
3531
3532         if ((new_ps->vclk == old_ps->vclk) &&
3533             (new_ps->dclk == old_ps->dclk))
3534                 return;
3535
3536         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3537             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3538                 return;
3539
3540         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3541 }
3542
3543 void ni_dpm_setup_asic(struct radeon_device *rdev)
3544 {
3545         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3546         int r;
3547
3548         r = ni_mc_load_microcode(rdev);
3549         if (r)
3550                 DRM_ERROR("Failed to load MC firmware!\n");
3551         ni_read_clock_registers(rdev);
3552         btc_read_arb_registers(rdev);
3553         rv770_get_memory_type(rdev);
3554         if (eg_pi->pcie_performance_request)
3555                 ni_advertise_gen2_capability(rdev);
3556         rv770_get_pcie_gen2_status(rdev);
3557         rv770_enable_acpi_pm(rdev);
3558 }
3559
3560 void ni_update_current_ps(struct radeon_device *rdev,
3561                           struct radeon_ps *rps)
3562 {
3563         struct ni_ps *new_ps = ni_get_ps(rps);
3564         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3565         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3566
3567         eg_pi->current_rps = *rps;
3568         ni_pi->current_ps = *new_ps;
3569         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3570 }
3571
3572 void ni_update_requested_ps(struct radeon_device *rdev,
3573                             struct radeon_ps *rps)
3574 {
3575         struct ni_ps *new_ps = ni_get_ps(rps);
3576         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3577         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3578
3579         eg_pi->requested_rps = *rps;
3580         ni_pi->requested_ps = *new_ps;
3581         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3582 }
3583
3584 int ni_dpm_enable(struct radeon_device *rdev)
3585 {
3586         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3587         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3588         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3589         int ret;
3590
3591         if (pi->gfx_clock_gating)
3592                 ni_cg_clockgating_default(rdev);
3593         if (btc_dpm_enabled(rdev))
3594                 return -EINVAL;
3595         if (pi->mg_clock_gating)
3596                 ni_mg_clockgating_default(rdev);
3597         if (eg_pi->ls_clock_gating)
3598                 ni_ls_clockgating_default(rdev);
3599         if (pi->voltage_control) {
3600                 rv770_enable_voltage_control(rdev, true);
3601                 ret = cypress_construct_voltage_tables(rdev);
3602                 if (ret) {
3603                         DRM_ERROR("cypress_construct_voltage_tables failed\n");
3604                         return ret;
3605                 }
3606         }
3607         if (eg_pi->dynamic_ac_timing) {
3608                 ret = ni_initialize_mc_reg_table(rdev);
3609                 if (ret)
3610                         eg_pi->dynamic_ac_timing = false;
3611         }
3612         if (pi->dynamic_ss)
3613                 cypress_enable_spread_spectrum(rdev, true);
3614         if (pi->thermal_protection)
3615                 rv770_enable_thermal_protection(rdev, true);
3616         rv770_setup_bsp(rdev);
3617         rv770_program_git(rdev);
3618         rv770_program_tp(rdev);
3619         rv770_program_tpp(rdev);
3620         rv770_program_sstp(rdev);
3621         cypress_enable_display_gap(rdev);
3622         rv770_program_vc(rdev);
3623         if (pi->dynamic_pcie_gen2)
3624                 ni_enable_dynamic_pcie_gen2(rdev, true);
3625         ret = rv770_upload_firmware(rdev);
3626         if (ret) {
3627                 DRM_ERROR("rv770_upload_firmware failed\n");
3628                 return ret;
3629         }
3630         ret = ni_process_firmware_header(rdev);
3631         if (ret) {
3632                 DRM_ERROR("ni_process_firmware_header failed\n");
3633                 return ret;
3634         }
3635         ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3636         if (ret) {
3637                 DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3638                 return ret;
3639         }
3640         ret = ni_init_smc_table(rdev);
3641         if (ret) {
3642                 DRM_ERROR("ni_init_smc_table failed\n");
3643                 return ret;
3644         }
3645         ret = ni_init_smc_spll_table(rdev);
3646         if (ret) {
3647                 DRM_ERROR("ni_init_smc_spll_table failed\n");
3648                 return ret;
3649         }
3650         ret = ni_init_arb_table_index(rdev);
3651         if (ret) {
3652                 DRM_ERROR("ni_init_arb_table_index failed\n");
3653                 return ret;
3654         }
3655         if (eg_pi->dynamic_ac_timing) {
3656                 ret = ni_populate_mc_reg_table(rdev, boot_ps);
3657                 if (ret) {
3658                         DRM_ERROR("ni_populate_mc_reg_table failed\n");
3659                         return ret;
3660                 }
3661         }
3662         ret = ni_initialize_smc_cac_tables(rdev);
3663         if (ret) {
3664                 DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3665                 return ret;
3666         }
3667         ret = ni_initialize_hardware_cac_manager(rdev);
3668         if (ret) {
3669                 DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3670                 return ret;
3671         }
3672         ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3673         if (ret) {
3674                 DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3675                 return ret;
3676         }
3677         ni_program_response_times(rdev);
3678         r7xx_start_smc(rdev);
3679         ret = cypress_notify_smc_display_change(rdev, false);
3680         if (ret) {
3681                 DRM_ERROR("cypress_notify_smc_display_change failed\n");
3682                 return ret;
3683         }
3684         cypress_enable_sclk_control(rdev, true);
3685         if (eg_pi->memory_transition)
3686                 cypress_enable_mclk_control(rdev, true);
3687         cypress_start_dpm(rdev);
3688         if (pi->gfx_clock_gating)
3689                 ni_gfx_clockgating_enable(rdev, true);
3690         if (pi->mg_clock_gating)
3691                 ni_mg_clockgating_enable(rdev, true);
3692         if (eg_pi->ls_clock_gating)
3693                 ni_ls_clockgating_enable(rdev, true);
3694
3695         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3696
3697         ni_update_current_ps(rdev, boot_ps);
3698
3699         return 0;
3700 }
3701
3702 void ni_dpm_disable(struct radeon_device *rdev)
3703 {
3704         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3705         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3706         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3707
3708         if (!btc_dpm_enabled(rdev))
3709                 return;
3710         rv770_clear_vc(rdev);
3711         if (pi->thermal_protection)
3712                 rv770_enable_thermal_protection(rdev, false);
3713         ni_enable_power_containment(rdev, boot_ps, false);
3714         ni_enable_smc_cac(rdev, boot_ps, false);
3715         cypress_enable_spread_spectrum(rdev, false);
3716         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3717         if (pi->dynamic_pcie_gen2)
3718                 ni_enable_dynamic_pcie_gen2(rdev, false);
3719
3720         if (rdev->irq.installed &&
3721             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3722                 rdev->irq.dpm_thermal = false;
3723                 radeon_irq_set(rdev);
3724         }
3725
3726         if (pi->gfx_clock_gating)
3727                 ni_gfx_clockgating_enable(rdev, false);
3728         if (pi->mg_clock_gating)
3729                 ni_mg_clockgating_enable(rdev, false);
3730         if (eg_pi->ls_clock_gating)
3731                 ni_ls_clockgating_enable(rdev, false);
3732         ni_stop_dpm(rdev);
3733         btc_reset_to_default(rdev);
3734         ni_stop_smc(rdev);
3735         ni_force_switch_to_arb_f0(rdev);
3736
3737         ni_update_current_ps(rdev, boot_ps);
3738 }
3739
3740 static int ni_power_control_set_level(struct radeon_device *rdev)
3741 {
3742         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3743         int ret;
3744
3745         ret = ni_restrict_performance_levels_before_switch(rdev);
3746         if (ret)
3747                 return ret;
3748         ret = rv770_halt_smc(rdev);
3749         if (ret)
3750                 return ret;
3751         ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3752         if (ret)
3753                 return ret;
3754         ret = rv770_resume_smc(rdev);
3755         if (ret)
3756                 return ret;
3757         ret = rv770_set_sw_state(rdev);
3758         if (ret)
3759                 return ret;
3760
3761         return 0;
3762 }
3763
3764 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3765 {
3766         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3767         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3768         struct radeon_ps *new_ps = &requested_ps;
3769
3770         ni_update_requested_ps(rdev, new_ps);
3771
3772         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3773
3774         return 0;
3775 }
3776
3777 int ni_dpm_set_power_state(struct radeon_device *rdev)
3778 {
3779         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3780         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3781         struct radeon_ps *old_ps = &eg_pi->current_rps;
3782         int ret;
3783
3784         ret = ni_restrict_performance_levels_before_switch(rdev);
3785         if (ret) {
3786                 DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3787                 return ret;
3788         }
3789         ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3790         ret = ni_enable_power_containment(rdev, new_ps, false);
3791         if (ret) {
3792                 DRM_ERROR("ni_enable_power_containment failed\n");
3793                 return ret;
3794         }
3795         ret = ni_enable_smc_cac(rdev, new_ps, false);
3796         if (ret) {
3797                 DRM_ERROR("ni_enable_smc_cac failed\n");
3798                 return ret;
3799         }
3800         ret = rv770_halt_smc(rdev);
3801         if (ret) {
3802                 DRM_ERROR("rv770_halt_smc failed\n");
3803                 return ret;
3804         }
3805         if (eg_pi->smu_uvd_hs)
3806                 btc_notify_uvd_to_smc(rdev, new_ps);
3807         ret = ni_upload_sw_state(rdev, new_ps);
3808         if (ret) {
3809                 DRM_ERROR("ni_upload_sw_state failed\n");
3810                 return ret;
3811         }
3812         if (eg_pi->dynamic_ac_timing) {
3813                 ret = ni_upload_mc_reg_table(rdev, new_ps);
3814                 if (ret) {
3815                         DRM_ERROR("ni_upload_mc_reg_table failed\n");
3816                         return ret;
3817                 }
3818         }
3819         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3820         if (ret) {
3821                 DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3822                 return ret;
3823         }
3824         ret = rv770_resume_smc(rdev);
3825         if (ret) {
3826                 DRM_ERROR("rv770_resume_smc failed\n");
3827                 return ret;
3828         }
3829         ret = rv770_set_sw_state(rdev);
3830         if (ret) {
3831                 DRM_ERROR("rv770_set_sw_state failed\n");
3832                 return ret;
3833         }
3834         ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3835         ret = ni_enable_smc_cac(rdev, new_ps, true);
3836         if (ret) {
3837                 DRM_ERROR("ni_enable_smc_cac failed\n");
3838                 return ret;
3839         }
3840         ret = ni_enable_power_containment(rdev, new_ps, true);
3841         if (ret) {
3842                 DRM_ERROR("ni_enable_power_containment failed\n");
3843                 return ret;
3844         }
3845
3846         /* update tdp */
3847         ret = ni_power_control_set_level(rdev);
3848         if (ret) {
3849                 DRM_ERROR("ni_power_control_set_level failed\n");
3850                 return ret;
3851         }
3852
3853         return 0;
3854 }
3855
3856 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3857 {
3858         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3859         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3860
3861         ni_update_current_ps(rdev, new_ps);
3862 }
3863
3864 void ni_dpm_reset_asic(struct radeon_device *rdev)
3865 {
3866         ni_restrict_performance_levels_before_switch(rdev);
3867         rv770_set_boot_state(rdev);
3868 }
3869
3870 union power_info {
3871         struct _ATOM_POWERPLAY_INFO info;
3872         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3873         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3874         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3875         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3876         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3877 };
3878
3879 union pplib_clock_info {
3880         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3881         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3882         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3883         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3884 };
3885
3886 union pplib_power_state {
3887         struct _ATOM_PPLIB_STATE v1;
3888         struct _ATOM_PPLIB_STATE_V2 v2;
3889 };
3890
3891 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3892                                           struct radeon_ps *rps,
3893                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3894                                           u8 table_rev)
3895 {
3896         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3897         rps->class = le16_to_cpu(non_clock_info->usClassification);
3898         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3899
3900         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3901                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3902                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3903         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3904                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3905                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3906         } else {
3907                 rps->vclk = 0;
3908                 rps->dclk = 0;
3909         }
3910
3911         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3912                 rdev->pm.dpm.boot_ps = rps;
3913         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3914                 rdev->pm.dpm.uvd_ps = rps;
3915 }
3916
3917 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3918                                       struct radeon_ps *rps, int index,
3919                                       union pplib_clock_info *clock_info)
3920 {
3921         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3922         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3923         struct ni_ps *ps = ni_get_ps(rps);
3924         struct rv7xx_pl *pl = &ps->performance_levels[index];
3925
3926         ps->performance_level_count = index + 1;
3927
3928         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3929         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3930         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3931         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3932
3933         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3934         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3935         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3936
3937         /* patch up vddc if necessary */
3938         if (pl->vddc == 0xff01) {
3939                 if (pi->max_vddc)
3940                         pl->vddc = pi->max_vddc;
3941         }
3942
3943         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3944                 pi->acpi_vddc = pl->vddc;
3945                 eg_pi->acpi_vddci = pl->vddci;
3946                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3947                         pi->acpi_pcie_gen2 = true;
3948                 else
3949                         pi->acpi_pcie_gen2 = false;
3950         }
3951
3952         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3953                 eg_pi->ulv.supported = true;
3954                 eg_pi->ulv.pl = pl;
3955         }
3956
3957         if (pi->min_vddc_in_table > pl->vddc)
3958                 pi->min_vddc_in_table = pl->vddc;
3959
3960         if (pi->max_vddc_in_table < pl->vddc)
3961                 pi->max_vddc_in_table = pl->vddc;
3962
3963         /* patch up boot state */
3964         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3965                 u16 vddc, vddci, mvdd;
3966                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3967                 pl->mclk = rdev->clock.default_mclk;
3968                 pl->sclk = rdev->clock.default_sclk;
3969                 pl->vddc = vddc;
3970                 pl->vddci = vddci;
3971         }
3972
3973         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3974             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3975                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3976                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3977                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3978                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
3979         }
3980 }
3981
3982 static int ni_parse_power_table(struct radeon_device *rdev)
3983 {
3984         struct radeon_mode_info *mode_info = &rdev->mode_info;
3985         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
3986         union pplib_power_state *power_state;
3987         int i, j;
3988         union pplib_clock_info *clock_info;
3989         union power_info *power_info;
3990         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
3991         u16 data_offset;
3992         u8 frev, crev;
3993         struct ni_ps *ps;
3994
3995         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
3996                                    &frev, &crev, &data_offset))
3997                 return -EINVAL;
3998         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
3999
4000         rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
4001                                   power_info->pplib.ucNumStates, GFP_KERNEL);
4002         if (!rdev->pm.dpm.ps)
4003                 return -ENOMEM;
4004
4005         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4006                 power_state = (union pplib_power_state *)
4007                         (mode_info->atom_context->bios + data_offset +
4008                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4009                          i * power_info->pplib.ucStateEntrySize);
4010                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4011                         (mode_info->atom_context->bios + data_offset +
4012                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4013                          (power_state->v1.ucNonClockStateIndex *
4014                           power_info->pplib.ucNonClockSize));
4015                 if (power_info->pplib.ucStateEntrySize - 1) {
4016                         u8 *idx;
4017                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4018                         if (ps == NULL) {
4019                                 kfree(rdev->pm.dpm.ps);
4020                                 return -ENOMEM;
4021                         }
4022                         rdev->pm.dpm.ps[i].ps_priv = ps;
4023                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4024                                                          non_clock_info,
4025                                                          power_info->pplib.ucNonClockSize);
4026                         idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
4027                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4028                                 clock_info = (union pplib_clock_info *)
4029                                         (mode_info->atom_context->bios + data_offset +
4030                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4031                                          (idx[j] * power_info->pplib.ucClockInfoSize));
4032                                 ni_parse_pplib_clock_info(rdev,
4033                                                           &rdev->pm.dpm.ps[i], j,
4034                                                           clock_info);
4035                         }
4036                 }
4037         }
4038         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4039         return 0;
4040 }
4041
4042 int ni_dpm_init(struct radeon_device *rdev)
4043 {
4044         struct rv7xx_power_info *pi;
4045         struct evergreen_power_info *eg_pi;
4046         struct ni_power_info *ni_pi;
4047         struct atom_clock_dividers dividers;
4048         int ret;
4049
4050         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4051         if (ni_pi == NULL)
4052                 return -ENOMEM;
4053         rdev->pm.dpm.priv = ni_pi;
4054         eg_pi = &ni_pi->eg;
4055         pi = &eg_pi->rv7xx;
4056
4057         rv770_get_max_vddc(rdev);
4058
4059         eg_pi->ulv.supported = false;
4060         pi->acpi_vddc = 0;
4061         eg_pi->acpi_vddci = 0;
4062         pi->min_vddc_in_table = 0;
4063         pi->max_vddc_in_table = 0;
4064
4065         ret = r600_get_platform_caps(rdev);
4066         if (ret)
4067                 return ret;
4068
4069         ret = ni_parse_power_table(rdev);
4070         if (ret)
4071                 return ret;
4072         ret = r600_parse_extended_power_table(rdev);
4073         if (ret)
4074                 return ret;
4075
4076         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4077                 kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL);
4078         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4079                 r600_free_extended_power_table(rdev);
4080                 return -ENOMEM;
4081         }
4082         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4083         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4084         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4085         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4086         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4087         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4088         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4089         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4090         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4091
4092         ni_patch_dependency_tables_based_on_leakage(rdev);
4093
4094         if (rdev->pm.dpm.voltage_response_time == 0)
4095                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4096         if (rdev->pm.dpm.backbias_response_time == 0)
4097                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4098
4099         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4100                                              0, false, &dividers);
4101         if (ret)
4102                 pi->ref_div = dividers.ref_div + 1;
4103         else
4104                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4105
4106         pi->rlp = RV770_RLP_DFLT;
4107         pi->rmp = RV770_RMP_DFLT;
4108         pi->lhp = RV770_LHP_DFLT;
4109         pi->lmp = RV770_LMP_DFLT;
4110
4111         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4112         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4113         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4114         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4115
4116         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4117         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4118         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4119         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4120
4121         eg_pi->smu_uvd_hs = true;
4122
4123         if (rdev->pdev->device == 0x6707) {
4124                 pi->mclk_strobe_mode_threshold = 55000;
4125                 pi->mclk_edc_enable_threshold = 55000;
4126                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
4127         } else {
4128                 pi->mclk_strobe_mode_threshold = 40000;
4129                 pi->mclk_edc_enable_threshold = 40000;
4130                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
4131         }
4132         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4133
4134         pi->voltage_control =
4135                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4136
4137         pi->mvdd_control =
4138                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4139
4140         eg_pi->vddci_control =
4141                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4142
4143         rv770_get_engine_memory_ss(rdev);
4144
4145         pi->asi = RV770_ASI_DFLT;
4146         pi->pasi = CYPRESS_HASI_DFLT;
4147         pi->vrc = CYPRESS_VRC_DFLT;
4148
4149         pi->power_gating = false;
4150
4151         pi->gfx_clock_gating = true;
4152
4153         pi->mg_clock_gating = true;
4154         pi->mgcgtssm = true;
4155         eg_pi->ls_clock_gating = false;
4156         eg_pi->sclk_deep_sleep = false;
4157
4158         pi->dynamic_pcie_gen2 = true;
4159
4160         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4161                 pi->thermal_protection = true;
4162         else
4163                 pi->thermal_protection = false;
4164
4165         pi->display_gap = true;
4166
4167         pi->dcodt = true;
4168
4169         pi->ulps = true;
4170
4171         eg_pi->dynamic_ac_timing = true;
4172         eg_pi->abm = true;
4173         eg_pi->mcls = true;
4174         eg_pi->light_sleep = true;
4175         eg_pi->memory_transition = true;
4176 #if defined(CONFIG_ACPI)
4177         eg_pi->pcie_performance_request =
4178                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4179 #else
4180         eg_pi->pcie_performance_request = false;
4181 #endif
4182
4183         eg_pi->dll_default_on = false;
4184
4185         eg_pi->sclk_deep_sleep = false;
4186
4187         pi->mclk_stutter_mode_threshold = 0;
4188
4189         pi->sram_end = SMC_RAM_END;
4190
4191         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4192         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4193         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4194         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4195         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4196         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4197         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4198         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4199
4200         ni_pi->cac_data.leakage_coefficients.at = 516;
4201         ni_pi->cac_data.leakage_coefficients.bt = 18;
4202         ni_pi->cac_data.leakage_coefficients.av = 51;
4203         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4204
4205         switch (rdev->pdev->device) {
4206         case 0x6700:
4207         case 0x6701:
4208         case 0x6702:
4209         case 0x6703:
4210         case 0x6718:
4211                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4212                 break;
4213         case 0x6705:
4214         case 0x6719:
4215         case 0x671D:
4216         case 0x671C:
4217         default:
4218                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4219                 break;
4220         case 0x6704:
4221         case 0x6706:
4222         case 0x6707:
4223         case 0x6708:
4224         case 0x6709:
4225                 ni_pi->cac_weights = &cac_weights_cayman_le;
4226                 break;
4227         }
4228
4229         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4230                 ni_pi->enable_power_containment = true;
4231                 ni_pi->enable_cac = true;
4232                 ni_pi->enable_sq_ramping = true;
4233         } else {
4234                 ni_pi->enable_power_containment = false;
4235                 ni_pi->enable_cac = false;
4236                 ni_pi->enable_sq_ramping = false;
4237         }
4238
4239         ni_pi->driver_calculate_cac_leakage = false;
4240         ni_pi->cac_configuration_required = true;
4241
4242         if (ni_pi->cac_configuration_required) {
4243                 ni_pi->support_cac_long_term_average = true;
4244                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4245                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4246         } else {
4247                 ni_pi->support_cac_long_term_average = false;
4248                 ni_pi->lta_window_size = 0;
4249                 ni_pi->lts_truncate = 0;
4250         }
4251
4252         ni_pi->use_power_boost_limit = true;
4253
4254         /* make sure dc limits are valid */
4255         if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
4256             (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
4257                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
4258                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4259
4260         return 0;
4261 }
4262
4263 void ni_dpm_fini(struct radeon_device *rdev)
4264 {
4265         int i;
4266
4267         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4268                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4269         }
4270         kfree(rdev->pm.dpm.ps);
4271         kfree(rdev->pm.dpm.priv);
4272         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4273         r600_free_extended_power_table(rdev);
4274 }
4275
4276 void ni_dpm_print_power_state(struct radeon_device *rdev,
4277                               struct radeon_ps *rps)
4278 {
4279         struct ni_ps *ps = ni_get_ps(rps);
4280         struct rv7xx_pl *pl;
4281         int i;
4282
4283         r600_dpm_print_class_info(rps->class, rps->class2);
4284         r600_dpm_print_cap_info(rps->caps);
4285         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4286         for (i = 0; i < ps->performance_level_count; i++) {
4287                 pl = &ps->performance_levels[i];
4288                 if (rdev->family >= CHIP_TAHITI)
4289                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4290                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4291                 else
4292                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4293                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4294         }
4295         r600_dpm_print_ps_status(rdev, rps);
4296 }
4297
4298 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4299                                                     struct seq_file *m)
4300 {
4301         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4302         struct radeon_ps *rps = &eg_pi->current_rps;
4303         struct ni_ps *ps = ni_get_ps(rps);
4304         struct rv7xx_pl *pl;
4305         u32 current_index =
4306                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4307                 CURRENT_STATE_INDEX_SHIFT;
4308
4309         if (current_index >= ps->performance_level_count) {
4310                 seq_printf(m, "invalid dpm profile %d\n", current_index);
4311         } else {
4312                 pl = &ps->performance_levels[current_index];
4313                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4314                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4315                            current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4316         }
4317 }
4318
4319 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4320 {
4321         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4322         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4323
4324         if (low)
4325                 return requested_state->performance_levels[0].sclk;
4326         else
4327                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4328 }
4329
4330 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4331 {
4332         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4333         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4334
4335         if (low)
4336                 return requested_state->performance_levels[0].mclk;
4337         else
4338                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4339 }
4340