2 * Copyright (C) ARM Limited 2012-2013. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
12 #include <linux/module.h>
13 #include <linux/time.h>
14 #include <linux/math64.h>
15 #include <linux/slab.h>
18 /* Mali T6xx DDK includes */
19 #include "linux/mali_linux_trace.h"
20 #include "kbase/src/common/mali_kbase.h"
21 #include "kbase/src/linux/mali_kbase_mem_linux.h"
23 #include "gator_events_mali_common.h"
25 /* If API version is not specified then assume API version 1. */
26 #ifndef MALI_DDK_GATOR_API_VERSION
27 #define MALI_DDK_GATOR_API_VERSION 1
30 #if (MALI_DDK_GATOR_API_VERSION != 1) && (MALI_DDK_GATOR_API_VERSION != 2)
31 #error MALI_DDK_GATOR_API_VERSION is invalid (must be 1 for r1/r2 DDK, or 2 for r3 DDK).
37 typedef struct kbase_device *kbase_find_device_type(int);
38 typedef kbase_context *kbase_create_context_type(kbase_device *);
39 typedef void kbase_destroy_context_type(kbase_context *);
41 #if MALI_DDK_GATOR_API_VERSION == 1
42 typedef void *kbase_va_alloc_type(kbase_context *, u32);
43 typedef void kbase_va_free_type(kbase_context *, void *);
44 #elif MALI_DDK_GATOR_API_VERSION == 2
45 typedef void *kbase_va_alloc_type(kbase_context *, u32, kbase_hwc_dma_mapping * handle);
46 typedef void kbase_va_free_type(kbase_context *, kbase_hwc_dma_mapping * handle);
49 typedef mali_error kbase_instr_hwcnt_enable_type(kbase_context *, kbase_uk_hwcnt_setup *);
50 typedef mali_error kbase_instr_hwcnt_disable_type(kbase_context *);
51 typedef mali_error kbase_instr_hwcnt_clear_type(kbase_context *);
52 typedef mali_error kbase_instr_hwcnt_dump_irq_type(kbase_context *);
53 typedef mali_bool kbase_instr_hwcnt_dump_complete_type(kbase_context *, mali_bool *);
55 static kbase_find_device_type *kbase_find_device_symbol;
56 static kbase_create_context_type *kbase_create_context_symbol;
57 static kbase_va_alloc_type *kbase_va_alloc_symbol;
58 static kbase_instr_hwcnt_enable_type *kbase_instr_hwcnt_enable_symbol;
59 static kbase_instr_hwcnt_clear_type *kbase_instr_hwcnt_clear_symbol;
60 static kbase_instr_hwcnt_dump_irq_type *kbase_instr_hwcnt_dump_irq_symbol;
61 static kbase_instr_hwcnt_dump_complete_type *kbase_instr_hwcnt_dump_complete_symbol;
62 static kbase_instr_hwcnt_disable_type *kbase_instr_hwcnt_disable_symbol;
63 static kbase_va_free_type *kbase_va_free_symbol;
64 static kbase_destroy_context_type *kbase_destroy_context_symbol;
66 static long shader_present_low = 0;
68 /** The interval between reads, in ns.
70 * Earlier we introduced
71 * a 'hold off for 1ms after last read' to resolve MIDBASE-2178 and MALINE-724.
72 * However, the 1ms hold off is too long if no context switches occur as there is a race
73 * between this value and the tick of the read clock in gator which is also 1ms. If we 'miss' the
74 * current read, the counter values are effectively 'spread' over 2ms and the values seen are half
75 * what they should be (since Streamline averages over sample time). In the presence of context switches
76 * this spread can vary and markedly affect the counters. Currently there is no 'proper' solution to
77 * this, but empirically we have found that reducing the minimum read interval to 950us causes the
78 * counts to be much more stable.
80 static const int READ_INTERVAL_NSEC = 950000;
83 #include "gator_events_mali_t6xx_hw_test.c"
86 /* Blocks for HW counters */
94 /* Counters for Mali-T6xx:
96 * - HW counters, 4 blocks
97 * For HW counters we need strings to create /dev/gator/events files.
98 * Enums are not needed because the position of the HW name in the array is the same
99 * of the corresponding value in the received block of memory.
100 * HW counters are requested by calculating a bitmask, passed then to the driver.
101 * Every millisecond a HW counters dump is requested, and if the previous has been completed they are read.
104 /* Hardware Counters */
105 static const char *const hardware_counter_names[] = {
113 "GPU_ACTIVE", /* 6 */
242 "SHADER_CORE_ACTIVE",
245 "FRAG_PRIMATIVES_DROPPED",
249 "FRAG_CYCLES_TRISETUP",
252 "FRAG_DUMMY_THREADS",
254 "FRAG_QUADS_EZS_TEST",
255 "FRAG_QUADS_EZS_KILLED",
256 "FRAG_QUADS_LZS_TEST",
257 "FRAG_QUADS_LZS_KILLED",
258 "FRAG_CYCLE_NO_TILE",
264 "COMPUTE_CYCLES_DESC",
275 "LS_REISSUE_ATTRIB_MISS",
298 "AXI_TLB_TRANSACTION",
357 "L2_EXT_WRITE_SMALL",
361 "L2_EXT_RD_BUF_FULL",
371 #define NUMBER_OF_HARDWARE_COUNTERS (sizeof(hardware_counter_names) / sizeof(hardware_counter_names[0]))
373 #define GET_HW_BLOCK(c) (((c) >> 6) & 0x3)
374 #define GET_COUNTER_OFFSET(c) ((c) & 0x3f)
376 /* Memory to dump hardware counters into */
377 static void *kernel_dump_buffer;
379 #if MALI_DDK_GATOR_API_VERSION == 2
380 /* DMA state used to manage lifetime of the buffer */
381 kbase_hwc_dma_mapping kernel_dump_buffer_handle;
384 /* kbase context and device */
385 static kbase_context *kbcontext = NULL;
386 static struct kbase_device *kbdevice = NULL;
389 * The following function has no external prototype in older DDK revisions. When the DDK
390 * is updated then this should be removed.
392 struct kbase_device *kbase_find_device(int minor);
394 static volatile bool kbase_device_busy = false;
395 static unsigned int num_hardware_counters_enabled;
398 * gatorfs variables for counter enable state
400 static mali_counter counters[NUMBER_OF_HARDWARE_COUNTERS];
402 /* An array used to return the data we recorded
403 * as key,value pairs hence the *2
405 static unsigned long counter_dump[NUMBER_OF_HARDWARE_COUNTERS * 2];
407 #define SYMBOL_GET(FUNCTION, ERROR_COUNT) \
408 if(FUNCTION ## _symbol) \
410 printk("gator: mali " #FUNCTION " symbol was already registered\n"); \
415 FUNCTION ## _symbol = symbol_get(FUNCTION); \
416 if(! FUNCTION ## _symbol) \
418 printk("gator: mali online " #FUNCTION " symbol not found\n"); \
423 #define SYMBOL_CLEANUP(FUNCTION) \
424 if(FUNCTION ## _symbol) \
426 symbol_put(FUNCTION); \
427 FUNCTION ## _symbol = NULL; \
431 * Execute symbol_get for all the Mali symbols and check for success.
432 * @return the number of symbols not loaded.
434 static int init_symbols(void)
437 SYMBOL_GET(kbase_find_device, error_count);
438 SYMBOL_GET(kbase_create_context, error_count);
439 SYMBOL_GET(kbase_va_alloc, error_count);
440 SYMBOL_GET(kbase_instr_hwcnt_enable, error_count);
441 SYMBOL_GET(kbase_instr_hwcnt_clear, error_count);
442 SYMBOL_GET(kbase_instr_hwcnt_dump_irq, error_count);
443 SYMBOL_GET(kbase_instr_hwcnt_dump_complete, error_count);
444 SYMBOL_GET(kbase_instr_hwcnt_disable, error_count);
445 SYMBOL_GET(kbase_va_free, error_count);
446 SYMBOL_GET(kbase_destroy_context, error_count);
452 * Execute symbol_put for all the registered Mali symbols.
454 static void clean_symbols(void)
456 SYMBOL_CLEANUP(kbase_find_device);
457 SYMBOL_CLEANUP(kbase_create_context);
458 SYMBOL_CLEANUP(kbase_va_alloc);
459 SYMBOL_CLEANUP(kbase_instr_hwcnt_enable);
460 SYMBOL_CLEANUP(kbase_instr_hwcnt_clear);
461 SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_irq);
462 SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_complete);
463 SYMBOL_CLEANUP(kbase_instr_hwcnt_disable);
464 SYMBOL_CLEANUP(kbase_va_free);
465 SYMBOL_CLEANUP(kbase_destroy_context);
469 * Determines whether a read should take place
470 * @param current_time The current time, obtained from getnstimeofday()
471 * @param prev_time_s The number of seconds at the previous read attempt.
472 * @param next_read_time_ns The time (in ns) when the next read should be allowed.
474 * Note that this function has been separated out here to allow it to be tested.
476 static int is_read_scheduled(const struct timespec *current_time, u32 *prev_time_s, s32 *next_read_time_ns)
478 /* If the current ns count rolls over a second, roll the next read time too. */
479 if (current_time->tv_sec != *prev_time_s) {
480 *next_read_time_ns = *next_read_time_ns - NSEC_PER_SEC;
483 /* Abort the read if the next read time has not arrived. */
484 if (current_time->tv_nsec < *next_read_time_ns) {
488 /* Set the next read some fixed time after this one, and update the read timestamp. */
489 *next_read_time_ns = current_time->tv_nsec + READ_INTERVAL_NSEC;
491 *prev_time_s = current_time->tv_sec;
495 static int start(void)
497 kbase_uk_hwcnt_setup setup;
500 u16 bitmask[] = { 0, 0, 0, 0 };
501 unsigned long long shadersPresent = 0;
503 /* Setup HW counters */
504 num_hardware_counters_enabled = 0;
506 if (NUMBER_OF_HARDWARE_COUNTERS != 256) {
507 pr_debug("Unexpected number of hardware counters defined: expecting 256, got %d\n", NUMBER_OF_HARDWARE_COUNTERS);
510 /* Calculate enable bitmasks based on counters_enabled array */
511 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
512 const mali_counter *counter = &counters[cnt];
513 if (counter->enabled) {
514 int block = GET_HW_BLOCK(cnt);
515 int enable_bit = GET_COUNTER_OFFSET(cnt) / 4;
516 bitmask[block] |= (1 << enable_bit);
517 pr_debug("gator: Mali-T6xx: hardware counter %s selected [%d]\n", hardware_counter_names[cnt], cnt);
518 num_hardware_counters_enabled++;
522 /* Create a kbase context for HW counters */
523 if (num_hardware_counters_enabled > 0) {
524 if (init_symbols() > 0) {
526 /* No Mali driver code entrypoints found - not a fault. */
530 kbdevice = kbase_find_device_symbol(-1);
532 /* If we already got a context, fail */
534 pr_debug("gator: Mali-T6xx: error context already present\n");
538 /* kbcontext will only be valid after all the Mali symbols are loaded successfully */
539 kbcontext = kbase_create_context_symbol(kbdevice);
541 pr_debug("gator: Mali-T6xx: error creating kbase context\n");
546 /* See if we can get the number of shader cores */
547 shadersPresent = kbdevice->shader_present_bitmap;
548 shader_present_low = (unsigned long)shadersPresent;
551 * The amount of memory needed to store the dump (bytes)
552 * DUMP_SIZE = number of core groups
553 * * number of blocks (always 8 for midgard)
554 * * number of counters per block (always 64 for midgard)
555 * * number of bytes per counter (always 4 in midgard)
556 * For a Mali-T6xx with a single core group = 1 * 8 * 64 * 4 = 2048
557 * For a Mali-T6xx with a dual core group = 2 * 8 * 64 * 4 = 4096
559 #if MALI_DDK_GATOR_API_VERSION == 1
560 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096);
561 #elif MALI_DDK_GATOR_API_VERSION == 2
562 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096, &kernel_dump_buffer_handle);
564 if (!kernel_dump_buffer) {
565 pr_debug("gator: Mali-T6xx: error trying to allocate va\n");
566 goto destroy_context;
569 setup.dump_buffer = (uintptr_t)kernel_dump_buffer;
570 setup.jm_bm = bitmask[JM_BLOCK];
571 setup.tiler_bm = bitmask[TILER_BLOCK];
572 setup.shader_bm = bitmask[SHADER_BLOCK];
573 setup.mmu_l2_bm = bitmask[MMU_BLOCK];
574 /* These counters do not exist on Mali-T60x */
575 setup.l3_cache_bm = 0;
577 /* Use kbase API to enable hardware counters and provide dump buffer */
578 err = kbase_instr_hwcnt_enable_symbol(kbcontext, &setup);
579 if (err != MALI_ERROR_NONE) {
580 pr_debug("gator: Mali-T6xx: can't setup hardware counters\n");
583 pr_debug("gator: Mali-T6xx: hardware counters enabled\n");
584 kbase_instr_hwcnt_clear_symbol(kbcontext);
585 pr_debug("gator: Mali-T6xx: hardware counters cleared \n");
587 kbase_device_busy = false;
593 #if MALI_DDK_GATOR_API_VERSION == 1
594 kbase_va_free_symbol(kbcontext, kernel_dump_buffer);
595 #elif MALI_DDK_GATOR_API_VERSION == 2
596 kbase_va_free_symbol(kbcontext, &kernel_dump_buffer_handle);
600 kbase_destroy_context_symbol(kbcontext);
607 static void stop(void)
610 kbase_context *temp_kbcontext;
612 pr_debug("gator: Mali-T6xx: stop\n");
614 /* Set all counters as disabled */
615 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
616 counters[cnt].enabled = 0;
619 /* Destroy the context for HW counters */
620 if (num_hardware_counters_enabled > 0 && kbcontext != NULL) {
622 * Set the global variable to NULL before destroying it, because
623 * other function will check this before using it.
625 temp_kbcontext = kbcontext;
628 kbase_instr_hwcnt_disable_symbol(temp_kbcontext);
630 #if MALI_DDK_GATOR_API_VERSION == 1
631 kbase_va_free_symbol(temp_kbcontext, kernel_dump_buffer);
632 #elif MALI_DDK_GATOR_API_VERSION == 2
633 kbase_va_free_symbol(temp_kbcontext, &kernel_dump_buffer_handle);
636 kbase_destroy_context_symbol(temp_kbcontext);
638 pr_debug("gator: Mali-T6xx: hardware counters stopped\n");
644 static int read(int **buffer)
651 struct timespec current_time;
652 static u32 prev_time_s = 0;
653 static s32 next_read_time_ns = 0;
655 if (!on_primary_core()) {
659 getnstimeofday(¤t_time);
662 * Discard reads unless a respectable time has passed. This reduces the load on the GPU without sacrificing
663 * accuracy on the Streamline display.
665 if (!is_read_scheduled(¤t_time, &prev_time_s, &next_read_time_ns)) {
670 * Report the HW counters
671 * Only process hardware counters if at least one of the hardware counters is enabled.
673 if (num_hardware_counters_enabled > 0) {
674 const unsigned int vithar_blocks[] = {
675 0x700, /* VITHAR_JOB_MANAGER, Block 0 */
676 0x400, /* VITHAR_TILER, Block 1 */
677 0x000, /* VITHAR_SHADER_CORE, Block 2 */
678 0x500 /* VITHAR_MEMORY_SYSTEM, Block 3 */
685 /* Mali symbols can be called safely since a kbcontext is valid */
686 if (kbase_instr_hwcnt_dump_complete_symbol(kbcontext, &success) == MALI_TRUE) {
687 kbase_device_busy = false;
689 if (success == MALI_TRUE) {
690 /* Cycle through hardware counters and accumulate totals */
691 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
692 const mali_counter *counter = &counters[cnt];
693 if (counter->enabled) {
694 const int block = GET_HW_BLOCK(cnt);
695 const int counter_offset = GET_COUNTER_OFFSET(cnt);
697 const char* block_base_address = (char*)kernel_dump_buffer + vithar_blocks[block];
699 /* If counter belongs to shader block need to take into account all cores */
700 if (block == SHADER_BLOCK) {
702 int shader_core_count = 0;
705 for (i = 0; i < 4; i++) {
706 if ((shader_present_low >> i) & 1) {
707 value += *((u32*) (block_base_address + (0x100 * i)) + counter_offset);
712 for (i = 0; i < 4; i++) {
713 if((shader_present_low >> (i+4)) & 1) {
714 value += *((u32*)(block_base_address + (0x100 * i) + 0x800) + counter_offset);
719 /* Need to total by number of cores to produce an average */
720 if (shader_core_count != 0) {
721 value /= shader_core_count;
724 value = *((u32*)block_base_address + counter_offset);
727 counter_dump[len++] = counter->key;
728 counter_dump[len++] = value;
734 if (!kbase_device_busy) {
735 kbase_device_busy = true;
736 kbase_instr_hwcnt_dump_irq_symbol(kbcontext);
740 /* Update the buffer */
742 *buffer = (int *)counter_dump;
748 static int create_files(struct super_block *sb, struct dentry *root)
752 * Create the filesystem for all events
754 int counter_index = 0;
755 const char *mali_name = gator_mali_get_mali_name();
757 for (event = 0; event < NUMBER_OF_HARDWARE_COUNTERS; event++) {
758 if (gator_mali_create_file_system(mali_name, hardware_counter_names[counter_index], sb, root, &counters[event], NULL) != 0)
766 static struct gator_interface gator_events_mali_t6xx_interface = {
767 .create_files = create_files,
773 int gator_events_mali_t6xx_hw_init(void)
775 pr_debug("gator: Mali-T6xx: sw_counters init\n");
778 test_all_is_read_scheduled();
781 gator_mali_initialise_counters(counters, NUMBER_OF_HARDWARE_COUNTERS);
783 return gator_events_install(&gator_events_mali_t6xx_interface);