driver/gator_events_mali_t6xx_hw.c

   1 /**
   2  * Copyright (C) ARM Limited 2012-2013. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #include "gator.h"
  11
  12 #include <linux/module.h>
  13 #include <linux/time.h>
  14 #include <linux/math64.h>
  15 #include <linux/slab.h>
  16 #include <asm/io.h>
  17
  18 /* Mali T6xx DDK includes */
  19 #include "linux/mali_linux_trace.h"
  20 #include "kbase/src/common/mali_kbase.h"
  21 #include "kbase/src/linux/mali_kbase_mem_linux.h"
  22
  23 #include "gator_events_mali_common.h"
  24
  25 /*
  26  * Mali-T6xx
  27  */
  28 typedef struct kbase_device *kbase_find_device_type(int);
  29 typedef kbase_context *kbase_create_context_type(kbase_device *);
  30 typedef void kbase_destroy_context_type(kbase_context *);
  31 typedef void *kbase_va_alloc_type(kbase_context *, u32);
  32 typedef void kbase_va_free_type(kbase_context *, void *);
  33 typedef mali_error kbase_instr_hwcnt_enable_type(kbase_context *, kbase_uk_hwcnt_setup *);
  34 typedef mali_error kbase_instr_hwcnt_disable_type(kbase_context *);
  35 typedef mali_error kbase_instr_hwcnt_clear_type(kbase_context *);
  36 typedef mali_error kbase_instr_hwcnt_dump_irq_type(kbase_context *);
  37 typedef mali_bool kbase_instr_hwcnt_dump_complete_type(kbase_context *, mali_bool *);
  38
  39 static kbase_find_device_type *kbase_find_device_symbol;
  40 static kbase_create_context_type *kbase_create_context_symbol;
  41 static kbase_va_alloc_type *kbase_va_alloc_symbol;
  42 static kbase_instr_hwcnt_enable_type *kbase_instr_hwcnt_enable_symbol;
  43 static kbase_instr_hwcnt_clear_type *kbase_instr_hwcnt_clear_symbol;
  44 static kbase_instr_hwcnt_dump_irq_type *kbase_instr_hwcnt_dump_irq_symbol;
  45 static kbase_instr_hwcnt_dump_complete_type *kbase_instr_hwcnt_dump_complete_symbol;
  46 static kbase_instr_hwcnt_disable_type *kbase_instr_hwcnt_disable_symbol;
  47 static kbase_va_free_type *kbase_va_free_symbol;
  48 static kbase_destroy_context_type *kbase_destroy_context_symbol;
  49
  50 /** The interval between reads, in ns.
  51  *
  52  * Earlier we introduced
  53  * a 'hold off for 1ms after last read' to resolve MIDBASE-2178 and MALINE-724.
  54  * However, the 1ms hold off is too long if no context switches occur as there is a race
  55  * between this value and the tick of the read clock in gator which is also 1ms. If we 'miss' the
  56  * current read, the counter values are effectively 'spread' over 2ms and the values seen are half
  57  * what they should be (since Streamline averages over sample time). In the presence of context switches
  58  * this spread can vary and markedly affect the counters.  Currently there is no 'proper' solution to
  59  * this, but empirically we have found that reducing the minimum read interval to 950us causes the
  60  * counts to be much more stable.
  61  */
  62 static const int READ_INTERVAL_NSEC = 950000;
  63
  64 #if GATOR_TEST
  65 #include "gator_events_mali_t6xx_hw_test.c"
  66 #endif
  67
  68 /* Blocks for HW counters */
  69 enum {
  70         JM_BLOCK = 0,
  71         TILER_BLOCK,
  72         SHADER_BLOCK,
  73         MMU_BLOCK
  74 };
  75
  76 /* Counters for Mali-T6xx:
  77  *
  78  *  - HW counters, 4 blocks
  79  *    For HW counters we need strings to create /dev/gator/events files.
  80  *    Enums are not needed because the position of the HW name in the array is the same
  81  *    of the corresponding value in the received block of memory.
  82  *    HW counters are requested by calculating a bitmask, passed then to the driver.
  83  *    Every millisecond a HW counters dump is requested, and if the previous has been completed they are read.
  84  */
  85
  86 /* Hardware Counters */
  87 static const char *const hardware_counter_names[] = {
  88         /* Job Manager */
  89         "",
  90         "",
  91         "",
  92         "",
  93         "MESSAGES_SENT",
  94         "MESSAGES_RECEIVED",
  95         "GPU_ACTIVE",           /* 6 */
  96         "IRQ_ACTIVE",
  97         "JS0_JOBS",
  98         "JS0_TASKS",
  99         "JS0_ACTIVE",
 100         "",
 101         "JS0_WAIT_READ",
 102         "JS0_WAIT_ISSUE",
 103         "JS0_WAIT_DEPEND",
 104         "JS0_WAIT_FINISH",
 105         "JS1_JOBS",
 106         "JS1_TASKS",
 107         "JS1_ACTIVE",
 108         "",
 109         "JS1_WAIT_READ",
 110         "JS1_WAIT_ISSUE",
 111         "JS1_WAIT_DEPEND",
 112         "JS1_WAIT_FINISH",
 113         "JS2_JOBS",
 114         "JS2_TASKS",
 115         "JS2_ACTIVE",
 116         "",
 117         "JS2_WAIT_READ",
 118         "JS2_WAIT_ISSUE",
 119         "JS2_WAIT_DEPEND",
 120         "JS2_WAIT_FINISH",
 121         "JS3_JOBS",
 122         "JS3_TASKS",
 123         "JS3_ACTIVE",
 124         "",
 125         "JS3_WAIT_READ",
 126         "JS3_WAIT_ISSUE",
 127         "JS3_WAIT_DEPEND",
 128         "JS3_WAIT_FINISH",
 129         "JS4_JOBS",
 130         "JS4_TASKS",
 131         "JS4_ACTIVE",
 132         "",
 133         "JS4_WAIT_READ",
 134         "JS4_WAIT_ISSUE",
 135         "JS4_WAIT_DEPEND",
 136         "JS4_WAIT_FINISH",
 137         "JS5_JOBS",
 138         "JS5_TASKS",
 139         "JS5_ACTIVE",
 140         "",
 141         "JS5_WAIT_READ",
 142         "JS5_WAIT_ISSUE",
 143         "JS5_WAIT_DEPEND",
 144         "JS5_WAIT_FINISH",
 145         "JS6_JOBS",
 146         "JS6_TASKS",
 147         "JS6_ACTIVE",
 148         "",
 149         "JS6_WAIT_READ",
 150         "JS6_WAIT_ISSUE",
 151         "JS6_WAIT_DEPEND",
 152         "JS6_WAIT_FINISH",
 153
 154         /*Tiler */
 155         "",
 156         "",
 157         "",
 158         "JOBS_PROCESSED",
 159         "TRIANGLES",
 160         "QUADS",
 161         "POLYGONS",
 162         "POINTS",
 163         "LINES",
 164         "VCACHE_HIT",
 165         "VCACHE_MISS",
 166         "FRONT_FACING",
 167         "BACK_FACING",
 168         "PRIM_VISIBLE",
 169         "PRIM_CULLED",
 170         "PRIM_CLIPPED",
 171         "LEVEL0",
 172         "LEVEL1",
 173         "LEVEL2",
 174         "LEVEL3",
 175         "LEVEL4",
 176         "LEVEL5",
 177         "LEVEL6",
 178         "LEVEL7",
 179         "COMMAND_1",
 180         "COMMAND_2",
 181         "COMMAND_3",
 182         "COMMAND_4",
 183         "COMMAND_4_7",
 184         "COMMAND_8_15",
 185         "COMMAND_16_63",
 186         "COMMAND_64",
 187         "COMPRESS_IN",
 188         "COMPRESS_OUT",
 189         "COMPRESS_FLUSH",
 190         "TIMESTAMPS",
 191         "PCACHE_HIT",
 192         "PCACHE_MISS",
 193         "PCACHE_LINE",
 194         "PCACHE_STALL",
 195         "WRBUF_HIT",
 196         "WRBUF_MISS",
 197         "WRBUF_LINE",
 198         "WRBUF_PARTIAL",
 199         "WRBUF_STALL",
 200         "ACTIVE",
 201         "LOADING_DESC",
 202         "INDEX_WAIT",
 203         "INDEX_RANGE_WAIT",
 204         "VERTEX_WAIT",
 205         "PCACHE_WAIT",
 206         "WRBUF_WAIT",
 207         "BUS_READ",
 208         "BUS_WRITE",
 209         "",
 210         "",
 211         "",
 212         "",
 213         "",
 214         "UTLB_STALL",
 215         "UTLB_REPLAY_MISS",
 216         "UTLB_REPLAY_FULL",
 217         "UTLB_NEW_MISS",
 218         "UTLB_HIT",
 219
 220         /* Shader Core */
 221         "",
 222         "",
 223         "",
 224         "SHADER_CORE_ACTIVE",
 225         "FRAG_ACTIVE",
 226         "FRAG_PRIMATIVES",
 227         "FRAG_PRIMATIVES_DROPPED",
 228         "FRAG_CYCLE_DESC",
 229         "FRAG_CYCLES_PLR",
 230         "FRAG_CYCLES_VERT",
 231         "FRAG_CYCLES_TRISETUP",
 232         "FRAG_CYCLES_RAST",
 233         "FRAG_THREADS",
 234         "FRAG_DUMMY_THREADS",
 235         "FRAG_QUADS_RAST",
 236         "FRAG_QUADS_EZS_TEST",
 237         "FRAG_QUADS_EZS_KILLED",
 238         "FRAG_QUADS_LZS_TEST",
 239         "FRAG_QUADS_LZS_KILLED",
 240         "FRAG_CYCLE_NO_TILE",
 241         "FRAG_NUM_TILES",
 242         "FRAG_TRANS_ELIM",
 243         "COMPUTE_ACTIVE",
 244         "COMPUTE_TASKS",
 245         "COMPUTE_THREADS",
 246         "COMPUTE_CYCLES_DESC",
 247         "TRIPIPE_ACTIVE",
 248         "ARITH_WORDS",
 249         "ARITH_CYCLES_REG",
 250         "ARITH_CYCLES_L0",
 251         "ARITH_FRAG_DEPEND",
 252         "LS_WORDS",
 253         "LS_ISSUES",
 254         "LS_RESTARTS",
 255         "LS_REISSUES_MISS",
 256         "LS_REISSUES_VD",
 257         "LS_REISSUE_ATTRIB_MISS",
 258         "LS_NO_WB",
 259         "TEX_WORDS",
 260         "TEX_BUBBLES",
 261         "TEX_WORDS_L0",
 262         "TEX_WORDS_DESC",
 263         "TEX_THREADS",
 264         "TEX_RECIRC_FMISS",
 265         "TEX_RECIRC_DESC",
 266         "TEX_RECIRC_MULTI",
 267         "TEX_RECIRC_PMISS",
 268         "TEX_RECIRC_CONF",
 269         "LSC_READ_HITS",
 270         "LSC_READ_MISSES",
 271         "LSC_WRITE_HITS",
 272         "LSC_WRITE_MISSES",
 273         "LSC_ATOMIC_HITS",
 274         "LSC_ATOMIC_MISSES",
 275         "LSC_LINE_FETCHES",
 276         "LSC_DIRTY_LINE",
 277         "LSC_SNOOPS",
 278         "AXI_TLB_STALL",
 279         "AXI_TLB_MIESS",
 280         "AXI_TLB_TRANSACTION",
 281         "LS_TLB_MISS",
 282         "LS_TLB_HIT",
 283         "AXI_BEATS_READ",
 284         "AXI_BEATS_WRITTEN",
 285
 286         /*L2 and MMU */
 287         "",
 288         "",
 289         "",
 290         "",
 291         "MMU_TABLE_WALK",
 292         "MMU_REPLAY_MISS",
 293         "MMU_REPLAY_FULL",
 294         "MMU_NEW_MISS",
 295         "MMU_HIT",
 296         "",
 297         "",
 298         "",
 299         "",
 300         "",
 301         "",
 302         "",
 303         "UTLB_STALL",
 304         "UTLB_REPLAY_MISS",
 305         "UTLB_REPLAY_FULL",
 306         "UTLB_NEW_MISS",
 307         "UTLB_HIT",
 308         "",
 309         "",
 310         "",
 311         "",
 312         "",
 313         "",
 314         "",
 315         "",
 316         "",
 317         "L2_WRITE_BEATS",
 318         "L2_READ_BEATS",
 319         "L2_ANY_LOOKUP",
 320         "L2_READ_LOOKUP",
 321         "L2_SREAD_LOOKUP",
 322         "L2_READ_REPLAY",
 323         "L2_READ_SNOOP",
 324         "L2_READ_HIT",
 325         "L2_CLEAN_MISS",
 326         "L2_WRITE_LOOKUP",
 327         "L2_SWRITE_LOOKUP",
 328         "L2_WRITE_REPLAY",
 329         "L2_WRITE_SNOOP",
 330         "L2_WRITE_HIT",
 331         "L2_EXT_READ_FULL",
 332         "L2_EXT_READ_HALF",
 333         "L2_EXT_WRITE_FULL",
 334         "L2_EXT_WRITE_HALF",
 335         "L2_EXT_READ",
 336         "L2_EXT_READ_LINE",
 337         "L2_EXT_WRITE",
 338         "L2_EXT_WRITE_LINE",
 339         "L2_EXT_WRITE_SMALL",
 340         "L2_EXT_BARRIER",
 341         "L2_EXT_AR_STALL",
 342         "L2_EXT_R_BUF_FULL",
 343         "L2_EXT_RD_BUF_FULL",
 344         "L2_EXT_R_RAW",
 345         "L2_EXT_W_STALL",
 346         "L2_EXT_W_BUF_FULL",
 347         "L2_EXT_R_W_HAZARD",
 348         "L2_TAG_HAZARD",
 349         "L2_SNOOP_FULL",
 350         "L2_REPLAY_FULL"
 351 };
 352
 353 #define NUMBER_OF_HARDWARE_COUNTERS (sizeof(hardware_counter_names) / sizeof(hardware_counter_names[0]))
 354
 355 #define GET_HW_BLOCK(c) (((c) >> 6) & 0x3)
 356 #define GET_COUNTER_OFFSET(c) ((c) & 0x3f)
 357
 358 /* Memory to dump hardware counters into */
 359 static void *kernel_dump_buffer;
 360
 361 /* kbase context and device */
 362 static kbase_context *kbcontext = NULL;
 363 static struct kbase_device *kbdevice = NULL;
 364
 365 /*
 366  * The following function has no external prototype in older DDK revisions.  When the DDK
 367  * is updated then this should be removed.
 368  */
 369 struct kbase_device *kbase_find_device(int minor);
 370
 371 static volatile bool kbase_device_busy = false;
 372 static unsigned int num_hardware_counters_enabled;
 373
 374 /*
 375  * gatorfs variables for counter enable state
 376  */
 377 static mali_counter counters[NUMBER_OF_HARDWARE_COUNTERS];
 378
 379 /* An array used to return the data we recorded
 380  * as key,value pairs hence the *2
 381  */
 382 static unsigned long counter_dump[NUMBER_OF_HARDWARE_COUNTERS * 2];
 383
 384 #define SYMBOL_GET(FUNCTION, ERROR_COUNT) \
 385         if(FUNCTION ## _symbol) \
 386         { \
 387                 printk("gator: mali " #FUNCTION " symbol was already registered\n"); \
 388                 (ERROR_COUNT)++; \
 389         } \
 390         else \
 391         { \
 392                 FUNCTION ## _symbol = symbol_get(FUNCTION); \
 393                 if(! FUNCTION ## _symbol) \
 394                 { \
 395                         printk("gator: mali online " #FUNCTION " symbol not found\n"); \
 396                         (ERROR_COUNT)++; \
 397                 } \
 398         }
 399
 400 #define SYMBOL_CLEANUP(FUNCTION) \
 401         if(FUNCTION ## _symbol) \
 402         { \
 403         symbol_put(FUNCTION); \
 404         FUNCTION ## _symbol = NULL; \
 405         }
 406
 407 /**
 408  * Execute symbol_get for all the Mali symbols and check for success.
 409  * @return the number of symbols not loaded.
 410  */
 411 static int init_symbols(void)
 412 {
 413         int error_count = 0;
 414         SYMBOL_GET(kbase_find_device, error_count);
 415         SYMBOL_GET(kbase_create_context, error_count);
 416         SYMBOL_GET(kbase_va_alloc, error_count);
 417         SYMBOL_GET(kbase_instr_hwcnt_enable, error_count);
 418         SYMBOL_GET(kbase_instr_hwcnt_clear, error_count);
 419         SYMBOL_GET(kbase_instr_hwcnt_dump_irq, error_count);
 420         SYMBOL_GET(kbase_instr_hwcnt_dump_complete, error_count);
 421         SYMBOL_GET(kbase_instr_hwcnt_disable, error_count);
 422         SYMBOL_GET(kbase_va_free, error_count);
 423         SYMBOL_GET(kbase_destroy_context, error_count);
 424
 425         return error_count;
 426 }
 427
 428 /**
 429  * Execute symbol_put for all the registered Mali symbols.
 430  */
 431 static void clean_symbols(void)
 432 {
 433         SYMBOL_CLEANUP(kbase_find_device);
 434         SYMBOL_CLEANUP(kbase_create_context);
 435         SYMBOL_CLEANUP(kbase_va_alloc);
 436         SYMBOL_CLEANUP(kbase_instr_hwcnt_enable);
 437         SYMBOL_CLEANUP(kbase_instr_hwcnt_clear);
 438         SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_irq);
 439         SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_complete);
 440         SYMBOL_CLEANUP(kbase_instr_hwcnt_disable);
 441         SYMBOL_CLEANUP(kbase_va_free);
 442         SYMBOL_CLEANUP(kbase_destroy_context);
 443 }
 444
 445 /**
 446  * Determines whether a read should take place
 447  * @param current_time The current time, obtained from getnstimeofday()
 448  * @param prev_time_s The number of seconds at the previous read attempt.
 449  * @param next_read_time_ns The time (in ns) when the next read should be allowed.
 450  *
 451  * Note that this function has been separated out here to allow it to be tested.
 452  */
 453 static int is_read_scheduled(const struct timespec *current_time, u32 *prev_time_s, s32 *next_read_time_ns)
 454 {
 455         /* If the current ns count rolls over a second, roll the next read time too. */
 456         if (current_time->tv_sec != *prev_time_s) {
 457                 *next_read_time_ns = *next_read_time_ns - NSEC_PER_SEC;
 458         }
 459
 460         /* Abort the read if the next read time has not arrived. */
 461         if (current_time->tv_nsec < *next_read_time_ns) {
 462                 return 0;
 463         }
 464
 465         /* Set the next read some fixed time after this one, and update the read timestamp. */
 466         *next_read_time_ns = current_time->tv_nsec + READ_INTERVAL_NSEC;
 467
 468         *prev_time_s = current_time->tv_sec;
 469         return 1;
 470 }
 471
 472 static int start(void)
 473 {
 474         kbase_uk_hwcnt_setup setup;
 475         mali_error err;
 476         int cnt;
 477         u16 bitmask[] = { 0, 0, 0, 0 };
 478
 479         /* Setup HW counters */
 480         num_hardware_counters_enabled = 0;
 481
 482         if (NUMBER_OF_HARDWARE_COUNTERS != 256) {
 483                 pr_debug("Unexpected number of hardware counters defined: expecting 256, got %d\n", NUMBER_OF_HARDWARE_COUNTERS);
 484         }
 485
 486         /* Calculate enable bitmasks based on counters_enabled array */
 487         for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
 488                 const mali_counter *counter = &counters[cnt];
 489                 if (counter->enabled) {
 490                         int block = GET_HW_BLOCK(cnt);
 491                         int enable_bit = GET_COUNTER_OFFSET(cnt) / 4;
 492                         bitmask[block] |= (1 << enable_bit);
 493                         pr_debug("gator: Mali-T6xx: hardware counter %s selected [%d]\n", hardware_counter_names[cnt], cnt);
 494                         num_hardware_counters_enabled++;
 495                 }
 496         }
 497
 498         /* Create a kbase context for HW counters */
 499         if (num_hardware_counters_enabled > 0) {
 500                 if (init_symbols() > 0) {
 501                         clean_symbols();
 502                         /* No Mali driver code entrypoints found - not a fault. */
 503                         return 0;
 504                 }
 505
 506                 kbdevice = kbase_find_device_symbol(-1);
 507
 508                 /* If we already got a context, fail */
 509                 if (kbcontext) {
 510                         pr_debug("gator: Mali-T6xx: error context already present\n");
 511                         goto out;
 512                 }
 513
 514                 /* kbcontext will only be valid after all the Mali symbols are loaded successfully */
 515                 kbcontext = kbase_create_context_symbol(kbdevice);
 516                 if (!kbcontext) {
 517                         pr_debug("gator: Mali-T6xx: error creating kbase context\n");
 518                         goto out;
 519                 }
 520
 521                 /*
 522                  * The amount of memory needed to store the dump (bytes)
 523                  * DUMP_SIZE = number of core groups
 524                  *             * number of blocks (always 8 for midgard)
 525                  *             * number of counters per block (always 64 for midgard)
 526                  *             * number of bytes per counter (always 4 in midgard)
 527                  * For a Mali-T6xx with a single core group = 1 * 8 * 64 * 4
 528                  */
 529                 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 2048);
 530                 if (!kernel_dump_buffer) {
 531                         pr_debug("gator: Mali-T6xx: error trying to allocate va\n");
 532                         goto destroy_context;
 533                 }
 534
 535                 setup.dump_buffer = (uintptr_t)kernel_dump_buffer;
 536                 setup.jm_bm = bitmask[JM_BLOCK];
 537                 setup.tiler_bm = bitmask[TILER_BLOCK];
 538                 setup.shader_bm = bitmask[SHADER_BLOCK];
 539                 setup.mmu_l2_bm = bitmask[MMU_BLOCK];
 540                 /* These counters do not exist on Mali-T60x */
 541                 setup.l3_cache_bm = 0;
 542
 543                 /* Use kbase API to enable hardware counters and provide dump buffer */
 544                 err = kbase_instr_hwcnt_enable_symbol(kbcontext, &setup);
 545                 if (err != MALI_ERROR_NONE) {
 546                         pr_debug("gator: Mali-T6xx: can't setup hardware counters\n");
 547                         goto free_buffer;
 548                 }
 549                 pr_debug("gator: Mali-T6xx: hardware counters enabled\n");
 550                 kbase_instr_hwcnt_clear_symbol(kbcontext);
 551                 pr_debug("gator: Mali-T6xx: hardware counters cleared \n");
 552
 553                 kbase_device_busy = false;
 554         }
 555
 556         return 0;
 557
 558 free_buffer:
 559         kbase_va_free_symbol(kbcontext, kernel_dump_buffer);
 560
 561 destroy_context:
 562         kbase_destroy_context_symbol(kbcontext);
 563
 564 out:
 565         clean_symbols();
 566         return -1;
 567 }
 568
 569 static void stop(void)
 570 {
 571         unsigned int cnt;
 572         kbase_context *temp_kbcontext;
 573
 574         pr_debug("gator: Mali-T6xx: stop\n");
 575
 576         /* Set all counters as disabled */
 577         for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
 578                 counters[cnt].enabled = 0;
 579         }
 580
 581         /* Destroy the context for HW counters */
 582         if (num_hardware_counters_enabled > 0 && kbcontext != NULL) {
 583                 /*
 584                  * Set the global variable to NULL before destroying it, because
 585                  * other function will check this before using it.
 586                  */
 587                 temp_kbcontext = kbcontext;
 588                 kbcontext = NULL;
 589
 590                 kbase_instr_hwcnt_disable_symbol(temp_kbcontext);
 591                 kbase_va_free_symbol(temp_kbcontext, kernel_dump_buffer);
 592                 kbase_destroy_context_symbol(temp_kbcontext);
 593
 594                 pr_debug("gator: Mali-T6xx: hardware counters stopped\n");
 595
 596                 clean_symbols();
 597         }
 598 }
 599
 600 static int read(int **buffer)
 601 {
 602         int cnt;
 603         int len = 0;
 604         u32 value = 0;
 605         mali_bool success;
 606
 607         struct timespec current_time;
 608         static u32 prev_time_s = 0;
 609         static s32 next_read_time_ns = 0;
 610
 611         if (!on_primary_core()) {
 612                 return 0;
 613         }
 614
 615         getnstimeofday(&current_time);
 616
 617         /*
 618          * Discard reads unless a respectable time has passed.  This reduces the load on the GPU without sacrificing
 619          * accuracy on the Streamline display.
 620          */
 621         if (!is_read_scheduled(&current_time, &prev_time_s, &next_read_time_ns)) {
 622                 return 0;
 623         }
 624
 625         /*
 626          * Report the HW counters
 627          * Only process hardware counters if at least one of the hardware counters is enabled.
 628          */
 629         if (num_hardware_counters_enabled > 0) {
 630                 const unsigned int vithar_blocks[] = {
 631                         0x700,  /* VITHAR_JOB_MANAGER,     Block 0 */
 632                         0x400,  /* VITHAR_TILER,           Block 1 */
 633                         0x000,  /* VITHAR_SHADER_CORE,     Block 2 */
 634                         0x500   /* VITHAR_MEMORY_SYSTEM,   Block 3 */
 635                 };
 636
 637                 if (!kbcontext) {
 638                         return -1;
 639                 }
 640
 641                 /* Mali symbols can be called safely since a kbcontext is valid */
 642                 if (kbase_instr_hwcnt_dump_complete_symbol(kbcontext, &success) == MALI_TRUE) {
 643                         kbase_device_busy = false;
 644
 645                         if (success == MALI_TRUE) {
 646                                 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
 647                                         const mali_counter *counter = &counters[cnt];
 648                                         if (counter->enabled) {
 649                                                 const int block = GET_HW_BLOCK(cnt);
 650                                                 const int counter_offset = GET_COUNTER_OFFSET(cnt);
 651                                                 const u32 *counter_block = (u32 *) ((uintptr_t)kernel_dump_buffer + vithar_blocks[block]);
 652                                                 const u32 *counter_address = counter_block + counter_offset;
 653
 654                                                 value = *counter_address;
 655
 656                                                 if (block == SHADER_BLOCK) {
 657                                                         /* (counter_address + 0x000) has already been accounted-for above. */
 658                                                         value += *(counter_address + 0x100);
 659                                                         value += *(counter_address + 0x200);
 660                                                         value += *(counter_address + 0x300);
 661                                                 }
 662
 663                                                 counter_dump[len++] = counter->key;
 664                                                 counter_dump[len++] = value;
 665                                         }
 666                                 }
 667                         }
 668                 }
 669
 670                 if (!kbase_device_busy) {
 671                         kbase_device_busy = true;
 672                         kbase_instr_hwcnt_dump_irq_symbol(kbcontext);
 673                 }
 674         }
 675
 676         /* Update the buffer */
 677         if (buffer) {
 678                 *buffer = (int *)counter_dump;
 679         }
 680
 681         return len;
 682 }
 683
 684 static int create_files(struct super_block *sb, struct dentry *root)
 685 {
 686         unsigned int event;
 687         /*
 688          * Create the filesystem for all events
 689          */
 690         int counter_index = 0;
 691         const char *mali_name = gator_mali_get_mali_name();
 692
 693         for (event = 0; event < NUMBER_OF_HARDWARE_COUNTERS; event++) {
 694                 if (gator_mali_create_file_system(mali_name, hardware_counter_names[counter_index], sb, root, &counters[event]) != 0)
 695                         return -1;
 696                 counter_index++;
 697         }
 698
 699         return 0;
 700 }
 701
 702 static struct gator_interface gator_events_mali_t6xx_interface = {
 703         .create_files = create_files,
 704         .start = start,
 705         .stop = stop,
 706         .read = read
 707 };
 708
 709 int gator_events_mali_t6xx_hw_init(void)
 710 {
 711         pr_debug("gator: Mali-T6xx: sw_counters init\n");
 712
 713 #if GATOR_TEST
 714         test_all_is_read_scheduled();
 715 #endif
 716
 717         gator_mali_initialise_counters(counters, NUMBER_OF_HARDWARE_COUNTERS);
 718
 719         return gator_events_install(&gator_events_mali_t6xx_interface);
 720 }
 721
 722 gator_events_init(gator_events_mali_t6xx_hw_init);