arch/sparc/kernel/perf_counter.c

   1 /* Performance counter support for sparc64.
   2  *
   3  * Copyright (C) 2009 David S. Miller <davem@davemloft.net>
   4  *
   5  * This code is based almost entirely upon the x86 perf counter
   6  * code, which is:
   7  *
   8  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   9  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  10  *  Copyright (C) 2009 Jaswinder Singh Rajput
  11  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  12  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  13  */
  14
  15 #include <linux/perf_counter.h>
  16 #include <linux/kprobes.h>
  17 #include <linux/kernel.h>
  18 #include <linux/kdebug.h>
  19 #include <linux/mutex.h>
  20
  21 #include <asm/cpudata.h>
  22 #include <asm/atomic.h>
  23 #include <asm/nmi.h>
  24 #include <asm/pcr.h>
  25
  26 /* Sparc64 chips have two performance counters, 32-bits each, with
  27  * overflow interrupts generated on transition from 0xffffffff to 0.
  28  * The counters are accessed in one go using a 64-bit register.
  29  *
  30  * Both counters are controlled using a single control register.  The
  31  * only way to stop all sampling is to clear all of the context (user,
  32  * supervisor, hypervisor) sampling enable bits.  But these bits apply
  33  * to both counters, thus the two counters can't be enabled/disabled
  34  * individually.
  35  *
  36  * The control register has two event fields, one for each of the two
  37  * counters.  It's thus nearly impossible to have one counter going
  38  * while keeping the other one stopped.  Therefore it is possible to
  39  * get overflow interrupts for counters not currently "in use" and
  40  * that condition must be checked in the overflow interrupt handler.
  41  *
  42  * So we use a hack, in that we program inactive counters with the
  43  * "sw_count0" and "sw_count1" events.  These count how many times
  44  * the instruction "sethi %hi(0xfc000), %g0" is executed.  It's an
  45  * unusual way to encode a NOP and therefore will not trigger in
  46  * normal code.
  47  */
  48
  49 #define MAX_HWCOUNTERS                  2
  50 #define MAX_PERIOD                      ((1UL << 32) - 1)
  51
  52 #define PIC_UPPER_INDEX                 0
  53 #define PIC_LOWER_INDEX                 1
  54
  55 struct cpu_hw_counters {
  56         struct perf_counter     *counters[MAX_HWCOUNTERS];
  57         unsigned long           used_mask[BITS_TO_LONGS(MAX_HWCOUNTERS)];
  58         unsigned long           active_mask[BITS_TO_LONGS(MAX_HWCOUNTERS)];
  59         int enabled;
  60 };
  61 DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = { .enabled = 1, };
  62
  63 struct perf_event_map {
  64         u16     encoding;
  65         u8      pic_mask;
  66 #define PIC_NONE        0x00
  67 #define PIC_UPPER       0x01
  68 #define PIC_LOWER       0x02
  69 };
  70
  71 struct sparc_pmu {
  72         const struct perf_event_map     *(*event_map)(int);
  73         int                             max_events;
  74         int                             upper_shift;
  75         int                             lower_shift;
  76         int                             event_mask;
  77         int                             hv_bit;
  78         int                             irq_bit;
  79         int                             upper_nop;
  80         int                             lower_nop;
  81 };
  82
  83 static const struct perf_event_map ultra3i_perfmon_event_map[] = {
  84         [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
  85         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
  86         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
  87         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
  88 };
  89
  90 static const struct perf_event_map *ultra3i_event_map(int event)
  91 {
  92         return &ultra3i_perfmon_event_map[event];
  93 }
  94
  95 static const struct sparc_pmu ultra3i_pmu = {
  96         .event_map      = ultra3i_event_map,
  97         .max_events     = ARRAY_SIZE(ultra3i_perfmon_event_map),
  98         .upper_shift    = 11,
  99         .lower_shift    = 4,
 100         .event_mask     = 0x3f,
 101         .upper_nop      = 0x1c,
 102         .lower_nop      = 0x14,
 103 };
 104
 105 static const struct sparc_pmu *sparc_pmu __read_mostly;
 106
 107 static u64 event_encoding(u64 event, int idx)
 108 {
 109         if (idx == PIC_UPPER_INDEX)
 110                 event <<= sparc_pmu->upper_shift;
 111         else
 112                 event <<= sparc_pmu->lower_shift;
 113         return event;
 114 }
 115
 116 static u64 mask_for_index(int idx)
 117 {
 118         return event_encoding(sparc_pmu->event_mask, idx);
 119 }
 120
 121 static u64 nop_for_index(int idx)
 122 {
 123         return event_encoding(idx == PIC_UPPER_INDEX ?
 124                               sparc_pmu->upper_nop :
 125                               sparc_pmu->lower_nop, idx);
 126 }
 127
 128 static inline void sparc_pmu_enable_counter(struct hw_perf_counter *hwc,
 129                                             int idx)
 130 {
 131         u64 val, mask = mask_for_index(idx);
 132
 133         val = pcr_ops->read();
 134         pcr_ops->write((val & ~mask) | hwc->config);
 135 }
 136
 137 static inline void sparc_pmu_disable_counter(struct hw_perf_counter *hwc,
 138                                              int idx)
 139 {
 140         u64 mask = mask_for_index(idx);
 141         u64 nop = nop_for_index(idx);
 142         u64 val = pcr_ops->read();
 143
 144         pcr_ops->write((val & ~mask) | nop);
 145 }
 146
 147 void hw_perf_enable(void)
 148 {
 149         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 150         u64 val;
 151         int i;
 152
 153         if (cpuc->enabled)
 154                 return;
 155
 156         cpuc->enabled = 1;
 157         barrier();
 158
 159         val = pcr_ops->read();
 160
 161         for (i = 0; i < MAX_HWCOUNTERS; i++) {
 162                 struct perf_counter *cp = cpuc->counters[i];
 163                 struct hw_perf_counter *hwc;
 164
 165                 if (!cp)
 166                         continue;
 167                 hwc = &cp->hw;
 168                 val |= hwc->config_base;
 169         }
 170
 171         pcr_ops->write(val);
 172 }
 173
 174 void hw_perf_disable(void)
 175 {
 176         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 177         u64 val;
 178
 179         if (!cpuc->enabled)
 180                 return;
 181
 182         cpuc->enabled = 0;
 183
 184         val = pcr_ops->read();
 185         val &= ~(PCR_UTRACE | PCR_STRACE |
 186                  sparc_pmu->hv_bit | sparc_pmu->irq_bit);
 187         pcr_ops->write(val);
 188 }
 189
 190 static u32 read_pmc(int idx)
 191 {
 192         u64 val;
 193
 194         read_pic(val);
 195         if (idx == PIC_UPPER_INDEX)
 196                 val >>= 32;
 197
 198         return val & 0xffffffff;
 199 }
 200
 201 static void write_pmc(int idx, u64 val)
 202 {
 203         u64 shift, mask, pic;
 204
 205         shift = 0;
 206         if (idx == PIC_UPPER_INDEX)
 207                 shift = 32;
 208
 209         mask = ((u64) 0xffffffff) << shift;
 210         val <<= shift;
 211
 212         read_pic(pic);
 213         pic &= ~mask;
 214         pic |= val;
 215         write_pic(pic);
 216 }
 217
 218 static int sparc_perf_counter_set_period(struct perf_counter *counter,
 219                                          struct hw_perf_counter *hwc, int idx)
 220 {
 221         s64 left = atomic64_read(&hwc->period_left);
 222         s64 period = hwc->sample_period;
 223         int ret = 0;
 224
 225         if (unlikely(left <= -period)) {
 226                 left = period;
 227                 atomic64_set(&hwc->period_left, left);
 228                 hwc->last_period = period;
 229                 ret = 1;
 230         }
 231
 232         if (unlikely(left <= 0)) {
 233                 left += period;
 234                 atomic64_set(&hwc->period_left, left);
 235                 hwc->last_period = period;
 236                 ret = 1;
 237         }
 238         if (left > MAX_PERIOD)
 239                 left = MAX_PERIOD;
 240
 241         atomic64_set(&hwc->prev_count, (u64)-left);
 242
 243         write_pmc(idx, (u64)(-left) & 0xffffffff);
 244
 245         perf_counter_update_userpage(counter);
 246
 247         return ret;
 248 }
 249
 250 static int sparc_pmu_enable(struct perf_counter *counter)
 251 {
 252         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 253         struct hw_perf_counter *hwc = &counter->hw;
 254         int idx = hwc->idx;
 255
 256         if (test_and_set_bit(idx, cpuc->used_mask))
 257                 return -EAGAIN;
 258
 259         sparc_pmu_disable_counter(hwc, idx);
 260
 261         cpuc->counters[idx] = counter;
 262         set_bit(idx, cpuc->active_mask);
 263
 264         sparc_perf_counter_set_period(counter, hwc, idx);
 265         sparc_pmu_enable_counter(hwc, idx);
 266         perf_counter_update_userpage(counter);
 267         return 0;
 268 }
 269
 270 static u64 sparc_perf_counter_update(struct perf_counter *counter,
 271                                      struct hw_perf_counter *hwc, int idx)
 272 {
 273         int shift = 64 - 32;
 274         u64 prev_raw_count, new_raw_count;
 275         s64 delta;
 276
 277 again:
 278         prev_raw_count = atomic64_read(&hwc->prev_count);
 279         new_raw_count = read_pmc(idx);
 280
 281         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 282                              new_raw_count) != prev_raw_count)
 283                 goto again;
 284
 285         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 286         delta >>= shift;
 287
 288         atomic64_add(delta, &counter->count);
 289         atomic64_sub(delta, &hwc->period_left);
 290
 291         return new_raw_count;
 292 }
 293
 294 static void sparc_pmu_disable(struct perf_counter *counter)
 295 {
 296         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 297         struct hw_perf_counter *hwc = &counter->hw;
 298         int idx = hwc->idx;
 299
 300         clear_bit(idx, cpuc->active_mask);
 301         sparc_pmu_disable_counter(hwc, idx);
 302
 303         barrier();
 304
 305         sparc_perf_counter_update(counter, hwc, idx);
 306         cpuc->counters[idx] = NULL;
 307         clear_bit(idx, cpuc->used_mask);
 308
 309         perf_counter_update_userpage(counter);
 310 }
 311
 312 static void sparc_pmu_read(struct perf_counter *counter)
 313 {
 314         struct hw_perf_counter *hwc = &counter->hw;
 315         sparc_perf_counter_update(counter, hwc, hwc->idx);
 316 }
 317
 318 static void sparc_pmu_unthrottle(struct perf_counter *counter)
 319 {
 320         struct hw_perf_counter *hwc = &counter->hw;
 321         sparc_pmu_enable_counter(hwc, hwc->idx);
 322 }
 323
 324 static atomic_t active_counters = ATOMIC_INIT(0);
 325 static DEFINE_MUTEX(pmc_grab_mutex);
 326
 327 void perf_counter_grab_pmc(void)
 328 {
 329         if (atomic_inc_not_zero(&active_counters))
 330                 return;
 331
 332         mutex_lock(&pmc_grab_mutex);
 333         if (atomic_read(&active_counters) == 0) {
 334                 if (atomic_read(&nmi_active) > 0) {
 335                         on_each_cpu(stop_nmi_watchdog, NULL, 1);
 336                         BUG_ON(atomic_read(&nmi_active) != 0);
 337                 }
 338                 atomic_inc(&active_counters);
 339         }
 340         mutex_unlock(&pmc_grab_mutex);
 341 }
 342
 343 void perf_counter_release_pmc(void)
 344 {
 345         if (atomic_dec_and_mutex_lock(&active_counters, &pmc_grab_mutex)) {
 346                 if (atomic_read(&nmi_active) == 0)
 347                         on_each_cpu(start_nmi_watchdog, NULL, 1);
 348                 mutex_unlock(&pmc_grab_mutex);
 349         }
 350 }
 351
 352 static void hw_perf_counter_destroy(struct perf_counter *counter)
 353 {
 354         perf_counter_release_pmc();
 355 }
 356
 357 static int __hw_perf_counter_init(struct perf_counter *counter)
 358 {
 359         struct perf_counter_attr *attr = &counter->attr;
 360         struct hw_perf_counter *hwc = &counter->hw;
 361         const struct perf_event_map *pmap;
 362         u64 enc;
 363
 364         if (atomic_read(&nmi_active) < 0)
 365                 return -ENODEV;
 366
 367         if (attr->type != PERF_TYPE_HARDWARE)
 368                 return -EOPNOTSUPP;
 369
 370         if (attr->config >= sparc_pmu->max_events)
 371                 return -EINVAL;
 372
 373         perf_counter_grab_pmc();
 374         counter->destroy = hw_perf_counter_destroy;
 375
 376         /* We save the enable bits in the config_base.  So to
 377          * turn off sampling just write 'config', and to enable
 378          * things write 'config | config_base'.
 379          */
 380         hwc->config_base = sparc_pmu->irq_bit;
 381         if (!attr->exclude_user)
 382                 hwc->config_base |= PCR_UTRACE;
 383         if (!attr->exclude_kernel)
 384                 hwc->config_base |= PCR_STRACE;
 385         if (!attr->exclude_hv)
 386                 hwc->config_base |= sparc_pmu->hv_bit;
 387
 388         if (!hwc->sample_period) {
 389                 hwc->sample_period = MAX_PERIOD;
 390                 hwc->last_period = hwc->sample_period;
 391                 atomic64_set(&hwc->period_left, hwc->sample_period);
 392         }
 393
 394         pmap = sparc_pmu->event_map(attr->config);
 395
 396         enc = pmap->encoding;
 397         if (pmap->pic_mask & PIC_UPPER) {
 398                 hwc->idx = PIC_UPPER_INDEX;
 399                 enc <<= sparc_pmu->upper_shift;
 400         } else {
 401                 hwc->idx = PIC_LOWER_INDEX;
 402                 enc <<= sparc_pmu->lower_shift;
 403         }
 404
 405         hwc->config |= enc;
 406         return 0;
 407 }
 408
 409 static const struct pmu pmu = {
 410         .enable         = sparc_pmu_enable,
 411         .disable        = sparc_pmu_disable,
 412         .read           = sparc_pmu_read,
 413         .unthrottle     = sparc_pmu_unthrottle,
 414 };
 415
 416 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
 417 {
 418         int err = __hw_perf_counter_init(counter);
 419
 420         if (err)
 421                 return ERR_PTR(err);
 422         return &pmu;
 423 }
 424
 425 void perf_counter_print_debug(void)
 426 {
 427         unsigned long flags;
 428         u64 pcr, pic;
 429         int cpu;
 430
 431         if (!sparc_pmu)
 432                 return;
 433
 434         local_irq_save(flags);
 435
 436         cpu = smp_processor_id();
 437
 438         pcr = pcr_ops->read();
 439         read_pic(pic);
 440
 441         pr_info("\n");
 442         pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
 443                 cpu, pcr, pic);
 444
 445         local_irq_restore(flags);
 446 }
 447
 448 static int __kprobes perf_counter_nmi_handler(struct notifier_block *self,
 449                                               unsigned long cmd, void *__args)
 450 {
 451         struct die_args *args = __args;
 452         struct perf_sample_data data;
 453         struct cpu_hw_counters *cpuc;
 454         struct pt_regs *regs;
 455         int idx;
 456
 457         if (!atomic_read(&active_counters))
 458                 return NOTIFY_DONE;
 459
 460         switch (cmd) {
 461         case DIE_NMI:
 462                 break;
 463
 464         default:
 465                 return NOTIFY_DONE;
 466         }
 467
 468         regs = args->regs;
 469
 470         data.regs = regs;
 471         data.addr = 0;
 472
 473         cpuc = &__get_cpu_var(cpu_hw_counters);
 474         for (idx = 0; idx < MAX_HWCOUNTERS; idx++) {
 475                 struct perf_counter *counter = cpuc->counters[idx];
 476                 struct hw_perf_counter *hwc;
 477                 u64 val;
 478
 479                 if (!test_bit(idx, cpuc->active_mask))
 480                         continue;
 481                 hwc = &counter->hw;
 482                 val = sparc_perf_counter_update(counter, hwc, idx);
 483                 if (val & (1ULL << 31))
 484                         continue;
 485
 486                 data.period = counter->hw.last_period;
 487                 if (!sparc_perf_counter_set_period(counter, hwc, idx))
 488                         continue;
 489
 490                 if (perf_counter_overflow(counter, 1, &data))
 491                         sparc_pmu_disable_counter(hwc, idx);
 492         }
 493
 494         return NOTIFY_STOP;
 495 }
 496
 497 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
 498         .notifier_call          = perf_counter_nmi_handler,
 499 };
 500
 501 static bool __init supported_pmu(void)
 502 {
 503         if (!strcmp(sparc_pmu_type, "ultra3i")) {
 504                 sparc_pmu = &ultra3i_pmu;
 505                 return true;
 506         }
 507         return false;
 508 }
 509
 510 void __init init_hw_perf_counters(void)
 511 {
 512         pr_info("Performance counters: ");
 513
 514         if (!supported_pmu()) {
 515                 pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
 516                 return;
 517         }
 518
 519         pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
 520
 521         /* All sparc64 PMUs currently have 2 counters.  But this simple
 522          * driver only supports one active counter at a time.
 523          */
 524         perf_max_counters = 1;
 525
 526         register_die_notifier(&perf_counter_nmi_notifier);
 527 }