3 * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
22 * Base structures shared with the kernel.
25 #ifndef _BASE_KERNEL_H_
26 #define _BASE_KERNEL_H_
32 /* Support UK6 IOCTLS */
33 #define BASE_LEGACY_UK6_SUPPORT 1
35 /* Support UK7 IOCTLS */
36 /* NB: To support UK6 we also need to support UK7 */
37 #define BASE_LEGACY_UK7_SUPPORT 1
39 /* Support UK8 IOCTLS */
40 #define BASE_LEGACY_UK8_SUPPORT 1
42 /* Support UK9 IOCTLS */
43 #define BASE_LEGACY_UK9_SUPPORT 1
45 typedef struct base_mem_handle {
51 #include "mali_base_mem_priv.h"
52 #include "mali_kbase_profiling_gator_api.h"
53 #include "mali_midg_coherency.h"
54 #include "mali_kbase_gpu_id.h"
57 * Dependency stuff, keep it private for now. May want to expose it if
58 * we decide to make the number of semaphores a configurable
61 #define BASE_JD_ATOM_COUNT 256
63 #define BASEP_JD_SEM_PER_WORD_LOG2 5
64 #define BASEP_JD_SEM_PER_WORD (1 << BASEP_JD_SEM_PER_WORD_LOG2)
65 #define BASEP_JD_SEM_WORD_NR(x) ((x) >> BASEP_JD_SEM_PER_WORD_LOG2)
66 #define BASEP_JD_SEM_MASK_IN_WORD(x) (1 << ((x) & (BASEP_JD_SEM_PER_WORD - 1)))
67 #define BASEP_JD_SEM_ARRAY_SIZE BASEP_JD_SEM_WORD_NR(BASE_JD_ATOM_COUNT)
69 /* Set/reset values for a software event */
70 #define BASE_JD_SOFT_EVENT_SET ((unsigned char)1)
71 #define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0)
73 #define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3
75 #define BASE_MAX_COHERENT_GROUPS 16
77 #if defined CDBG_ASSERT
78 #define LOCAL_ASSERT CDBG_ASSERT
79 #elif defined KBASE_DEBUG_ASSERT
80 #define LOCAL_ASSERT KBASE_DEBUG_ASSERT
82 #error assert macro not defined!
86 #define LOCAL_PAGE_LSB ~PAGE_MASK
88 #include <osu/mali_osu.h>
90 #if defined OSU_CONFIG_CPU_PAGE_SIZE_LOG2
91 #define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1)
93 #error Failed to find page size
97 /** 32/64-bit neutral way to represent pointers */
98 typedef union kbase_pointer {
99 void __user *value; /**< client should store their pointers here */
100 u32 compat_value; /**< 64-bit kernels should fetch value here when handling 32-bit clients */
101 u64 sizer; /**< Force 64-bit storage for all clients regardless */
105 * @addtogroup base_user_api User-side Base APIs
110 * @addtogroup base_user_api_memory User-side Base Memory APIs
115 * @brief Memory allocation, access/hint flags
117 * A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator
118 * in order to determine the best cache policy. Some combinations are
119 * of course invalid (eg @c MEM_PROT_CPU_WR | @c MEM_HINT_CPU_RD),
120 * which defines a @a write-only region on the CPU side, which is
121 * heavily read by the CPU...
122 * Other flags are only meaningful to a particular allocator.
123 * More flags can be added to this list, as long as they don't clash
124 * (see ::BASE_MEM_FLAGS_NR_BITS for the number of the first free bit).
126 typedef u32 base_mem_alloc_flags;
129 * @brief Memory allocation, access/hint flags
131 * See ::base_mem_alloc_flags.
136 BASE_MEM_PROT_CPU_RD = (1U << 0), /**< Read access CPU side */
137 BASE_MEM_PROT_CPU_WR = (1U << 1), /**< Write access CPU side */
138 BASE_MEM_PROT_GPU_RD = (1U << 2), /**< Read access GPU side */
139 BASE_MEM_PROT_GPU_WR = (1U << 3), /**< Write access GPU side */
140 BASE_MEM_PROT_GPU_EX = (1U << 4), /**< Execute allowed on the GPU
143 /* BASE_MEM_HINT flags have been removed, but their values are reserved
144 * for backwards compatibility with older user-space drivers. The values
145 * can be re-used once support for r5p0 user-space drivers is removed,
146 * presumably in r7p0.
148 * RESERVED: (1U << 5)
149 * RESERVED: (1U << 6)
150 * RESERVED: (1U << 7)
151 * RESERVED: (1U << 8)
154 BASE_MEM_GROW_ON_GPF = (1U << 9), /**< Grow backing store on GPU
157 BASE_MEM_COHERENT_SYSTEM = (1U << 10), /**< Page coherence Outer
158 shareable, if available */
159 BASE_MEM_COHERENT_LOCAL = (1U << 11), /**< Page coherence Inner
161 BASE_MEM_CACHED_CPU = (1U << 12), /**< Should be cached on the
165 BASE_MEM_SAME_VA = (1U << 13), /**< Must have same VA on both the GPU
168 BASE_MEM_NEED_MMAP = (1U << 14), /**< Must call mmap to aquire a GPU
169 address for the alloc */
171 BASE_MEM_COHERENT_SYSTEM_REQUIRED = (1U << 15), /**< Page coherence
172 Outer shareable, required. */
173 BASE_MEM_SECURE = (1U << 16), /**< Secure memory */
174 BASE_MEM_DONT_NEED = (1U << 17), /**< Not needed physical
176 BASE_MEM_IMPORT_SHARED = (1U << 18), /**< Must use shared CPU/GPU zone
177 (SAME_VA zone) but doesn't
178 require the addresses to
183 * @brief Number of bits used as flags for base memory management
185 * Must be kept in sync with the ::base_mem_alloc_flags flags
187 #define BASE_MEM_FLAGS_NR_BITS 19
190 * A mask for all output bits, excluding IN/OUT bits.
192 #define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP
195 * A mask for all input bits, including IN/OUT bits.
197 #define BASE_MEM_FLAGS_INPUT_MASK \
198 (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK)
201 * A mask for all the flags which are modifiable via the base_mem_set_flags
204 #define BASE_MEM_FLAGS_MODIFIABLE \
205 (BASE_MEM_DONT_NEED | BASE_MEM_COHERENT_SYSTEM | \
206 BASE_MEM_COHERENT_LOCAL)
209 * enum base_mem_import_type - Memory types supported by @a base_mem_import
211 * @BASE_MEM_IMPORT_TYPE_INVALID: Invalid type
212 * @BASE_MEM_IMPORT_TYPE_UMP: UMP import. Handle type is ump_secure_id.
213 * @BASE_MEM_IMPORT_TYPE_UMM: UMM import. Handle type is a file descriptor (int)
214 * @BASE_MEM_IMPORT_TYPE_USER_BUFFER: User buffer import. Handle is a
215 * base_mem_import_user_buffer
217 * Each type defines what the supported handle type is.
219 * If any new type is added here ARM must be contacted
220 * to allocate a numeric value for it.
221 * Do not just add a new type without synchronizing with ARM
222 * as future releases from ARM might include other new types
223 * which could clash with your custom types.
225 typedef enum base_mem_import_type {
226 BASE_MEM_IMPORT_TYPE_INVALID = 0,
227 BASE_MEM_IMPORT_TYPE_UMP = 1,
228 BASE_MEM_IMPORT_TYPE_UMM = 2,
229 BASE_MEM_IMPORT_TYPE_USER_BUFFER = 3
230 } base_mem_import_type;
233 * struct base_mem_import_user_buffer - Handle of an imported user buffer
235 * @ptr: kbase_pointer to imported user buffer
236 * @length: length of imported user buffer in bytes
238 * This structure is used to represent a handle of an imported user buffer.
241 struct base_mem_import_user_buffer {
247 * @brief Invalid memory handle.
249 * Return value from functions returning @ref base_mem_handle on error.
251 * @warning @ref base_mem_handle_new_invalid must be used instead of this macro
252 * in C++ code or other situations where compound literals cannot be used.
254 #define BASE_MEM_INVALID_HANDLE ((base_mem_handle) { {BASEP_MEM_INVALID_HANDLE} })
257 * @brief Special write-alloc memory handle.
259 * A special handle is used to represent a region where a special page is mapped
260 * with a write-alloc cache setup, typically used when the write result of the
261 * GPU isn't needed, but the GPU must write anyway.
263 * @warning @ref base_mem_handle_new_write_alloc must be used instead of this macro
264 * in C++ code or other situations where compound literals cannot be used.
266 #define BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ((base_mem_handle) { {BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE} })
268 #define BASEP_MEM_INVALID_HANDLE (0ull << 12)
269 #define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12)
270 #define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12)
271 #define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12)
272 #define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12)
273 /* reserved handles ..-64<<PAGE_SHIFT> for future special handles */
274 #define BASE_MEM_COOKIE_BASE (64ul << 12)
275 #define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \
276 BASE_MEM_COOKIE_BASE)
278 /* Mask to detect 4GB boundary alignment */
279 #define BASE_MEM_MASK_4GB 0xfffff000UL
282 /* Bit mask of cookies used for for memory allocation setup */
283 #define KBASE_COOKIE_MASK ~1UL /* bit 0 is reserved */
287 * @brief Result codes of changing the size of the backing store allocated to a tmem region
289 typedef enum base_backing_threshold_status {
290 BASE_BACKING_THRESHOLD_OK = 0, /**< Resize successful */
291 BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE = -1, /**< Not a growable tmem object */
292 BASE_BACKING_THRESHOLD_ERROR_OOM = -2, /**< Increase failed due to an out-of-memory condition */
293 BASE_BACKING_THRESHOLD_ERROR_MAPPED = -3, /**< Resize attempted on buffer while it was mapped, which is not permitted */
294 BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS = -4 /**< Invalid arguments (not tmem, illegal size request, etc.) */
295 } base_backing_threshold_status;
298 * @addtogroup base_user_api_memory_defered User-side Base Defered Memory Coherency APIs
303 * @brief a basic memory operation (sync-set).
305 * The content of this structure is private, and should only be used
308 typedef struct base_syncset {
309 struct basep_syncset basep_sset;
312 /** @} end group base_user_api_memory_defered */
315 * Handle to represent imported memory object.
316 * Simple opague handle to imported memory, can't be used
317 * with anything but base_external_resource_init to bind to an atom.
319 typedef struct base_import_handle {
323 } base_import_handle;
325 /** @} end group base_user_api_memory */
328 * @addtogroup base_user_api_job_dispatch User-side Base Job Dispatcher APIs
332 typedef int platform_fence_type;
333 #define INVALID_PLATFORM_FENCE ((platform_fence_type)-1)
336 * Base stream handle.
338 * References an underlying base stream object.
340 typedef struct base_stream {
349 * References an underlying base fence object.
351 typedef struct base_fence {
359 * @brief Per-job data
361 * This structure is used to store per-job data, and is completly unused
362 * by the Base driver. It can be used to store things such as callback
363 * function pointer, data to handle job completion. It is guaranteed to be
364 * untouched by the Base driver.
366 typedef struct base_jd_udata {
367 u64 blob[2]; /**< per-job data array */
371 * @brief Memory aliasing info
373 * Describes a memory handle to be aliased.
374 * A subset of the handle can be chosen for aliasing, given an offset and a
376 * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a
377 * region where a special page is mapped with a write-alloc cache setup,
378 * typically used when the write result of the GPU isn't needed, but the GPU
381 * Offset and length are specified in pages.
382 * Offset must be within the size of the handle.
383 * Offset+length must not overrun the size of the handle.
385 * @handle Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE
386 * @offset Offset within the handle to start aliasing from, in pages.
387 * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE.
388 * @length Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE
389 * specifies the number of times the special page is needed.
391 struct base_mem_aliasing_info {
392 base_mem_handle handle;
398 * struct base_jit_alloc_info - Structure which describes a JIT allocation
400 * @gpu_alloc_addr: The GPU virtual address to write the JIT
401 * allocated GPU virtual address to.
402 * @va_pages: The minimum number of virtual pages required.
403 * @commit_pages: The minimum number of physical pages which
404 * should back the allocation.
405 * @extent: Granularity of physical pages to grow the
406 * allocation by during a fault.
407 * @id: Unique ID provided by the caller, this is used
408 * to pair allocation and free requests.
409 * Zero is not a valid value.
411 struct base_jit_alloc_info {
420 * @brief Job dependency type.
422 * A flags field will be inserted into the atom structure to specify whether a dependency is a data or
423 * ordering dependency (by putting it before/after 'core_req' in the structure it should be possible to add without
424 * changing the structure size).
425 * When the flag is set for a particular dependency to signal that it is an ordering only dependency then
426 * errors will not be propagated.
428 typedef u8 base_jd_dep_type;
431 #define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */
432 #define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */
433 #define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */
436 * @brief Job chain hardware requirements.
438 * A job chain must specify what GPU features it needs to allow the
439 * driver to schedule the job correctly. By not specifying the
440 * correct settings can/will cause an early job termination. Multiple
441 * values can be ORed together to specify multiple requirements.
442 * Special case is ::BASE_JD_REQ_DEP, which is used to express complex
443 * dependencies, and that doesn't execute anything on the hardware.
445 typedef u16 base_jd_core_req;
447 /* Requirements that come from the HW */
448 #define BASE_JD_REQ_DEP 0 /**< No requirement, dependency only */
449 #define BASE_JD_REQ_FS (1U << 0) /**< Requires fragment shaders */
451 * Requires compute shaders
452 * This covers any of the following Midgard Job types:
453 * - Vertex Shader Job
454 * - Geometry Shader Job
455 * - An actual Compute Shader Job
457 * Compare this with @ref BASE_JD_REQ_ONLY_COMPUTE, which specifies that the
458 * job is specifically just the "Compute Shader" job type, and not the "Vertex
459 * Shader" nor the "Geometry Shader" job type.
461 #define BASE_JD_REQ_CS (1U << 1)
462 #define BASE_JD_REQ_T (1U << 2) /**< Requires tiling */
463 #define BASE_JD_REQ_CF (1U << 3) /**< Requires cache flushes */
464 #define BASE_JD_REQ_V (1U << 4) /**< Requires value writeback */
466 /* SW-only requirements - the HW does not expose these as part of the job slot capabilities */
468 /* Requires fragment job with AFBC encoding */
469 #define BASE_JD_REQ_FS_AFBC (1U << 13)
472 * SW-only requirement: coalesce completion events.
473 * If this bit is set then completion of this atom will not cause an event to
474 * be sent to userspace, whether successful or not; completion events will be
475 * deferred until an atom completes which does not have this bit set.
477 * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES.
479 #define BASE_JD_REQ_EVENT_COALESCE (1U << 5)
482 * SW Only requirement: the job chain requires a coherent core group. We don't
483 * mind which coherent core group is used.
485 #define BASE_JD_REQ_COHERENT_GROUP (1U << 6)
488 * SW Only requirement: The performance counters should be enabled only when
489 * they are needed, to reduce power consumption.
492 #define BASE_JD_REQ_PERMON (1U << 7)
495 * SW Only requirement: External resources are referenced by this atom.
496 * When external resources are referenced no syncsets can be bundled with the atom
497 * but should instead be part of a NULL jobs inserted into the dependency tree.
498 * The first pre_dep object must be configured for the external resouces to use,
499 * the second pre_dep object can be used to create other dependencies.
501 * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE.
503 #define BASE_JD_REQ_EXTERNAL_RESOURCES (1U << 8)
506 * SW Only requirement: Software defined job. Jobs with this bit set will not be submitted
507 * to the hardware but will cause some action to happen within the driver
509 #define BASE_JD_REQ_SOFT_JOB (1U << 9)
511 #define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1)
512 #define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2)
513 #define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3)
516 * SW Only requirement : Replay job.
518 * If the preceeding job fails, the replay job will cause the jobs specified in
519 * the list of base_jd_replay_payload pointed to by the jc pointer to be
522 * A replay job will only cause jobs to be replayed up to BASEP_JD_REPLAY_LIMIT
523 * times. If a job fails more than BASEP_JD_REPLAY_LIMIT times then the replay
524 * job is failed, as well as any following dependencies.
526 * The replayed jobs will require a number of atom IDs. If there are not enough
527 * free atom IDs then the replay job will fail.
529 * If the preceeding job does not fail, then the replay job is returned as
532 * The replayed jobs will never be returned to userspace. The preceeding failed
533 * job will be returned to userspace as failed; the status of this job should
534 * be ignored. Completion should be determined by the status of the replay soft
537 * In order for the jobs to be replayed, the job headers will have to be
538 * modified. The Status field will be reset to NOT_STARTED. If the Job Type
539 * field indicates a Vertex Shader Job then it will be changed to Null Job.
541 * The replayed jobs have the following assumptions :
543 * - No external resources. Any required external resources will be held by the
545 * - Pre-dependencies are created based on job order.
546 * - Atom numbers are automatically assigned.
547 * - device_nr is set to 0. This is not relevant as
548 * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set.
549 * - Priority is inherited from the replay job.
551 #define BASE_JD_REQ_SOFT_REPLAY (BASE_JD_REQ_SOFT_JOB | 0x4)
553 * SW only requirement: event wait/trigger job.
555 * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set.
556 * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the
557 * other waiting jobs. It completes immediately.
558 * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it
559 * possible for other jobs to wait upon. It completes immediately.
561 #define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5)
562 #define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6)
563 #define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7)
565 #define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8)
568 * SW only requirement: Just In Time allocation
570 * This job requests a JIT allocation based on the request in the
571 * @base_jit_alloc_info structure which is passed via the jc element of
574 * It should be noted that the id entry in @base_jit_alloc_info must not
575 * be reused until it has been released via @BASE_JD_REQ_SOFT_JIT_FREE.
577 * Should this soft job fail it is expected that a @BASE_JD_REQ_SOFT_JIT_FREE
578 * soft job to free the JIT allocation is still made.
580 * The job will complete immediately.
582 #define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9)
584 * SW only requirement: Just In Time free
586 * This job requests a JIT allocation created by @BASE_JD_REQ_SOFT_JIT_ALLOC
587 * to be freed. The ID of the JIT allocation is passed via the jc element of
590 * The job will complete immediately.
592 #define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa)
595 * SW only requirement: Map external resource
597 * This job requests external resource(s) are mapped once the dependencies
598 * of the job have been satisfied. The list of external resources are
599 * passed via the jc element of the atom which is a pointer to a
600 * @base_external_resource_list.
602 #define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb)
604 * SW only requirement: Unmap external resource
606 * This job requests external resource(s) are unmapped once the dependencies
607 * of the job has been satisfied. The list of external resources are
608 * passed via the jc element of the atom which is a pointer to a
609 * @base_external_resource_list.
611 #define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc)
614 * HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders)
616 * This indicates that the Job Chain contains Midgard Jobs of the 'Compute Shaders' type.
618 * In contrast to @ref BASE_JD_REQ_CS, this does \b not indicate that the Job
619 * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs.
621 #define BASE_JD_REQ_ONLY_COMPUTE (1U << 10)
624 * HW Requirement: Use the base_jd_atom::device_nr field to specify a
625 * particular core group
627 * If both BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag takes priority
629 * This is only guaranteed to work for BASE_JD_REQ_ONLY_COMPUTE atoms.
631 * If the core availability policy is keeping the required core group turned off, then
632 * the job will fail with a BASE_JD_EVENT_PM_EVENT error code.
634 #define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP (1U << 11)
637 * SW Flag: If this bit is set then the successful completion of this atom
638 * will not cause an event to be sent to userspace
640 #define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE (1U << 12)
643 * SW Flag: If this bit is set then completion of this atom will not cause an
644 * event to be sent to userspace, whether successful or not.
646 #define BASEP_JD_REQ_EVENT_NEVER (1U << 14)
649 * These requirement bits are currently unused in base_jd_core_req (currently a u16)
652 #define BASEP_JD_REQ_RESERVED (1U << 15)
655 * Mask of all bits in base_jd_core_req that control the type of the atom.
657 * This allows dependency only atoms to have flags set
659 #define BASEP_JD_REQ_ATOM_TYPE (~(BASEP_JD_REQ_RESERVED |\
660 BASE_JD_REQ_EVENT_ONLY_ON_FAILURE |\
661 BASE_JD_REQ_EXTERNAL_RESOURCES |\
662 BASEP_JD_REQ_EVENT_NEVER |\
663 BASE_JD_REQ_EVENT_COALESCE))
666 * @brief States to model state machine processed by kbasep_js_job_check_ref_cores(), which
667 * handles retaining cores for power management and affinity management.
669 * The state @ref KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY prevents an attack
670 * where lots of atoms could be submitted before powerup, and each has an
671 * affinity chosen that causes other atoms to have an affinity
672 * violation. Whilst the affinity was not causing violations at the time it
673 * was chosen, it could cause violations thereafter. For example, 1000 jobs
674 * could have had their affinity chosen during the powerup time, so any of
675 * those 1000 jobs could cause an affinity violation later on.
677 * The attack would otherwise occur because other atoms/contexts have to wait for:
678 * -# the currently running atoms (which are causing the violation) to
680 * -# and, the atoms that had their affinity chosen during powerup to
681 * finish. These are run preferrentially because they don't cause a
682 * violation, but instead continue to cause the violation in others.
683 * -# or, the attacker is scheduled out (which might not happen for just 2
686 * By re-choosing the affinity (which is designed to avoid violations at the
687 * time it's chosen), we break condition (2) of the wait, which minimizes the
688 * problem to just waiting for current jobs to finish (which can be bounded if
689 * the Job Scheduling Policy has a timer).
691 enum kbase_atom_coreref_state {
692 /** Starting state: No affinity chosen, and cores must be requested. kbase_jd_atom::affinity==0 */
693 KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED,
694 /** Cores requested, but waiting for them to be powered. Requested cores given by kbase_jd_atom::affinity */
695 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES,
696 /** Cores given by kbase_jd_atom::affinity are powered, but affinity might be out-of-date, so must recheck */
697 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY,
698 /** Cores given by kbase_jd_atom::affinity are powered, and affinity is up-to-date, but must check for violations */
699 KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS,
700 /** Cores are powered, kbase_jd_atom::affinity up-to-date, no affinity violations: atom can be submitted to HW */
701 KBASE_ATOM_COREREF_STATE_READY
707 * Only certain priority levels are actually implemented, as specified by the
708 * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority
709 * level that is not one of those defined below.
711 * Priority levels only affect scheduling between atoms of the same type within
712 * a base context, and only after the atoms have had dependencies resolved.
713 * Fragment atoms does not affect non-frament atoms with lower priorities, and
714 * the other way around. For example, a low priority atom that has had its
715 * dependencies resolved might run before a higher priority atom that has not
716 * had its dependencies resolved.
718 * The scheduling between base contexts/processes and between atoms from
719 * different base contexts/processes is unaffected by atom priority.
721 * The atoms are scheduled as follows with respect to their priorities:
722 * - Let atoms 'X' and 'Y' be for the same job slot who have dependencies
723 * resolved, and atom 'X' has a higher priority than atom 'Y'
724 * - If atom 'Y' is currently running on the HW, then it is interrupted to
725 * allow atom 'X' to run soon after
726 * - If instead neither atom 'Y' nor atom 'X' are running, then when choosing
727 * the next atom to run, atom 'X' will always be chosen instead of atom 'Y'
728 * - Any two atoms that have the same priority could run in any order with
729 * respect to each other. That is, there is no ordering constraint between
730 * atoms of the same priority.
732 typedef u8 base_jd_prio;
734 /* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */
735 #define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0)
736 /* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and
737 * BASE_JD_PRIO_LOW */
738 #define BASE_JD_PRIO_HIGH ((base_jd_prio)1)
739 /* Low atom priority. */
740 #define BASE_JD_PRIO_LOW ((base_jd_prio)2)
742 /* Count of the number of priority levels. This itself is not a valid
743 * base_jd_prio setting */
744 #define BASE_JD_NR_PRIO_LEVELS 3
746 enum kbase_jd_atom_state {
747 /** Atom is not used */
748 KBASE_JD_ATOM_STATE_UNUSED,
749 /** Atom is queued in JD */
750 KBASE_JD_ATOM_STATE_QUEUED,
751 /** Atom has been given to JS (is runnable/running) */
752 KBASE_JD_ATOM_STATE_IN_JS,
753 /** Atom has been completed, but not yet handed back to job dispatcher
754 * for dependency resolution */
755 KBASE_JD_ATOM_STATE_HW_COMPLETED,
756 /** Atom has been completed, but not yet handed back to userspace */
757 KBASE_JD_ATOM_STATE_COMPLETED
760 typedef u8 base_atom_id; /**< Type big enough to store an atom number in */
762 struct base_dependency {
763 base_atom_id atom_id; /**< An atom number */
764 base_jd_dep_type dependency_type; /**< Dependency type */
767 typedef struct base_jd_atom_v2 {
768 u64 jc; /**< job-chain GPU address */
769 struct base_jd_udata udata; /**< user data */
770 kbase_pointer extres_list; /**< list of external resources */
771 u16 nr_extres; /**< nr of external resources */
772 base_jd_core_req core_req; /**< core requirements */
773 struct base_dependency pre_dep[2]; /**< pre-dependencies, one need to use SETTER function to assign this field,
774 this is done in order to reduce possibility of improper assigment of a dependency field */
775 base_atom_id atom_number; /**< unique number to identify the atom */
776 base_jd_prio prio; /**< Atom priority. Refer to @ref base_jd_prio for more details */
777 u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */
781 #ifdef BASE_LEGACY_UK6_SUPPORT
782 struct base_jd_atom_v2_uk6 {
783 u64 jc; /**< job-chain GPU address */
784 struct base_jd_udata udata; /**< user data */
785 kbase_pointer extres_list; /**< list of external resources */
786 u16 nr_extres; /**< nr of external resources */
787 base_jd_core_req core_req; /**< core requirements */
788 base_atom_id pre_dep[2]; /**< pre-dependencies */
789 base_atom_id atom_number; /**< unique number to identify the atom */
790 base_jd_prio prio; /**< priority - smaller is higher priority */
791 u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */
796 typedef enum base_external_resource_access {
797 BASE_EXT_RES_ACCESS_SHARED,
798 BASE_EXT_RES_ACCESS_EXCLUSIVE
799 } base_external_resource_access;
801 typedef struct base_external_resource {
803 } base_external_resource;
807 * The maximum number of external resources which can be mapped/unmapped
808 * in a single request.
810 #define BASE_EXT_RES_COUNT_MAX 10
813 * struct base_external_resource_list - Structure which describes a list of
814 * external resources.
815 * @count: The number of resources.
816 * @ext_res: Array of external resources which is
817 * sized at allocation time.
819 struct base_external_resource_list {
821 struct base_external_resource ext_res[1];
824 struct base_jd_debug_copy_buffer {
827 struct base_external_resource extres;
831 * @brief Setter for a dependency structure
833 * @param[in] dep The kbase jd atom dependency to be initialized.
834 * @param id The atom_id to be assigned.
835 * @param dep_type The dep_type to be assigned.
838 static inline void base_jd_atom_dep_set(struct base_dependency *dep,
839 base_atom_id id, base_jd_dep_type dep_type)
841 LOCAL_ASSERT(dep != NULL);
844 * make sure we don't set not allowed combinations
845 * of atom_id/dependency_type.
847 LOCAL_ASSERT((id == 0 && dep_type == BASE_JD_DEP_TYPE_INVALID) ||
848 (id > 0 && dep_type != BASE_JD_DEP_TYPE_INVALID));
851 dep->dependency_type = dep_type;
855 * @brief Make a copy of a dependency structure
857 * @param[in,out] dep The kbase jd atom dependency to be written.
858 * @param[in] from The dependency to make a copy from.
861 static inline void base_jd_atom_dep_copy(struct base_dependency *dep,
862 const struct base_dependency *from)
864 LOCAL_ASSERT(dep != NULL);
866 base_jd_atom_dep_set(dep, from->atom_id, from->dependency_type);
870 * @brief Soft-atom fence trigger setup.
872 * Sets up an atom to be a SW-only atom signaling a fence
873 * when it reaches the run state.
875 * Using the existing base dependency system the fence can
876 * be set to trigger when a GPU job has finished.
878 * The base fence object must not be terminated until the atom
879 * has been submitted to @a base_jd_submit_bag and @a base_jd_submit_bag has returned.
881 * @a fence must be a valid fence set up with @a base_fence_init.
882 * Calling this function with a uninitialized fence results in undefined behavior.
884 * @param[out] atom A pre-allocated atom to configure as a fence trigger SW atom
885 * @param[in] fence The base fence object to trigger.
887 static inline void base_jd_fence_trigger_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence)
891 LOCAL_ASSERT(fence->basep.fd == INVALID_PLATFORM_FENCE);
892 LOCAL_ASSERT(fence->basep.stream_fd >= 0);
893 atom->jc = (uintptr_t) fence;
894 atom->core_req = BASE_JD_REQ_SOFT_FENCE_TRIGGER;
898 * @brief Soft-atom fence wait setup.
900 * Sets up an atom to be a SW-only atom waiting on a fence.
901 * When the fence becomes triggered the atom becomes runnable
902 * and completes immediately.
904 * Using the existing base dependency system the fence can
905 * be set to block a GPU job until it has been triggered.
907 * The base fence object must not be terminated until the atom
908 * has been submitted to @a base_jd_submit_bag and @a base_jd_submit_bag has returned.
910 * @a fence must be a valid fence set up with @a base_fence_init or @a base_fence_import.
911 * Calling this function with a uninitialized fence results in undefined behavior.
913 * @param[out] atom A pre-allocated atom to configure as a fence wait SW atom
914 * @param[in] fence The base fence object to wait on
916 static inline void base_jd_fence_wait_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence)
920 LOCAL_ASSERT(fence->basep.fd >= 0);
921 atom->jc = (uintptr_t) fence;
922 atom->core_req = BASE_JD_REQ_SOFT_FENCE_WAIT;
926 * @brief External resource info initialization.
928 * Sets up an external resource object to reference
929 * a memory allocation and the type of access requested.
931 * @param[in] res The resource object to initialize
932 * @param handle The handle to the imported memory object, must be
933 * obtained by calling @ref base_mem_as_import_handle().
934 * @param access The type of access requested
936 static inline void base_external_resource_init(struct base_external_resource *res, struct base_import_handle handle, base_external_resource_access access)
940 address = handle.basep.handle;
942 LOCAL_ASSERT(res != NULL);
943 LOCAL_ASSERT(0 == (address & LOCAL_PAGE_LSB));
944 LOCAL_ASSERT(access == BASE_EXT_RES_ACCESS_SHARED || access == BASE_EXT_RES_ACCESS_EXCLUSIVE);
946 res->ext_resource = address | (access & LOCAL_PAGE_LSB);
950 * @brief Job chain event code bits
951 * Defines the bits used to create ::base_jd_event_code
954 BASE_JD_SW_EVENT_KERNEL = (1u << 15), /**< Kernel side event */
955 BASE_JD_SW_EVENT = (1u << 14), /**< SW defined event */
956 BASE_JD_SW_EVENT_SUCCESS = (1u << 13), /**< Event idicates success (SW events only) */
957 BASE_JD_SW_EVENT_JOB = (0u << 11), /**< Job related event */
958 BASE_JD_SW_EVENT_BAG = (1u << 11), /**< Bag related event */
959 BASE_JD_SW_EVENT_INFO = (2u << 11), /**< Misc/info event */
960 BASE_JD_SW_EVENT_RESERVED = (3u << 11), /**< Reserved event type */
961 BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) /**< Mask to extract the type from an event code */
965 * @brief Job chain event codes
967 * HW and low-level SW events are represented by event codes.
968 * The status of jobs which succeeded are also represented by
969 * an event code (see ::BASE_JD_EVENT_DONE).
970 * Events are usually reported as part of a ::base_jd_event.
972 * The event codes are encoded in the following way:
975 * @li 13 - SW success (only valid if the SW bit is set)
976 * @li 14 - SW event (HW event if not set)
977 * @li 15 - Kernel event (should never be seen in userspace)
979 * Events are split up into ranges as follows:
980 * - BASE_JD_EVENT_RANGE_\<description\>_START
981 * - BASE_JD_EVENT_RANGE_\<description\>_END
983 * \a code is in \<description\>'s range when:
984 * - <tt>BASE_JD_EVENT_RANGE_\<description\>_START <= code < BASE_JD_EVENT_RANGE_\<description\>_END </tt>
986 * Ranges can be asserted for adjacency by testing that the END of the previous
987 * is equal to the START of the next. This is useful for optimizing some tests
990 * A limitation is that the last member of this enum must explicitly be handled
991 * (with an assert-unreachable statement) in switch statements that use
992 * variables of this type. Otherwise, the compiler warns that we have not
993 * handled that enum value.
995 typedef enum base_jd_event_code {
996 /* HW defined exceptions */
998 /** Start of HW Non-fault status codes
1000 * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault,
1001 * because the job was hard-stopped
1003 BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0,
1005 /* non-fatal exceptions */
1006 BASE_JD_EVENT_NOT_STARTED = 0x00, /**< Can't be seen by userspace, treated as 'previous job done' */
1007 BASE_JD_EVENT_DONE = 0x01,
1008 BASE_JD_EVENT_STOPPED = 0x03, /**< Can't be seen by userspace, becomes TERMINATED, DONE or JOB_CANCELLED */
1009 BASE_JD_EVENT_TERMINATED = 0x04, /**< This is actually a fault status code - the job was hard stopped */
1010 BASE_JD_EVENT_ACTIVE = 0x08, /**< Can't be seen by userspace, jobs only returned on complete/fail/cancel */
1012 /** End of HW Non-fault status codes
1014 * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault,
1015 * because the job was hard-stopped
1017 BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40,
1019 /** Start of HW fault and SW Error status codes */
1020 BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40,
1022 /* job exceptions */
1023 BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40,
1024 BASE_JD_EVENT_JOB_POWER_FAULT = 0x41,
1025 BASE_JD_EVENT_JOB_READ_FAULT = 0x42,
1026 BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43,
1027 BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44,
1028 BASE_JD_EVENT_JOB_BUS_FAULT = 0x48,
1029 BASE_JD_EVENT_INSTR_INVALID_PC = 0x50,
1030 BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51,
1031 BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52,
1032 BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53,
1033 BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54,
1034 BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55,
1035 BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56,
1036 BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58,
1037 BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59,
1038 BASE_JD_EVENT_STATE_FAULT = 0x5A,
1039 BASE_JD_EVENT_OUT_OF_MEMORY = 0x60,
1040 BASE_JD_EVENT_UNKNOWN = 0x7F,
1042 /* GPU exceptions */
1043 BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80,
1044 BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88,
1046 /* MMU exceptions */
1047 BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1,
1048 BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2,
1049 BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3,
1050 BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4,
1051 BASE_JD_EVENT_PERMISSION_FAULT = 0xC8,
1052 BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1,
1053 BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2,
1054 BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3,
1055 BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4,
1056 BASE_JD_EVENT_ACCESS_FLAG = 0xD8,
1058 /* SW defined exceptions */
1059 BASE_JD_EVENT_MEM_GROWTH_FAILED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000,
1060 BASE_JD_EVENT_TIMED_OUT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x001,
1061 BASE_JD_EVENT_JOB_CANCELLED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002,
1062 BASE_JD_EVENT_JOB_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003,
1063 BASE_JD_EVENT_PM_EVENT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x004,
1064 BASE_JD_EVENT_FORCE_REPLAY = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x005,
1066 BASE_JD_EVENT_BAG_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_BAG | 0x003,
1068 /** End of HW fault and SW Error status codes */
1069 BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_RESERVED | 0x3FF,
1071 /** Start of SW Success status codes */
1072 BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | 0x000,
1074 BASE_JD_EVENT_PROGRESS_REPORT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_JOB | 0x000,
1075 BASE_JD_EVENT_BAG_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_BAG | 0x000,
1076 BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_INFO | 0x000,
1078 /** End of SW Success status codes */
1079 BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_RESERVED | 0x3FF,
1081 /** Start of Kernel-only status codes. Such codes are never returned to user-space */
1082 BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000,
1083 BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000,
1085 /** End of Kernel-only status codes. */
1086 BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF
1087 } base_jd_event_code;
1090 * @brief Event reporting structure
1092 * This structure is used by the kernel driver to report information
1093 * about GPU events. The can either be HW-specific events or low-level
1094 * SW events, such as job-chain completion.
1096 * The event code contains an event type field which can be extracted
1097 * by ANDing with ::BASE_JD_SW_EVENT_TYPE_MASK.
1099 * Based on the event type base_jd_event::data holds:
1100 * @li ::BASE_JD_SW_EVENT_JOB : the offset in the ring-buffer for the completed
1102 * @li ::BASE_JD_SW_EVENT_BAG : The address of the ::base_jd_bag that has
1103 * been completed (ie all contained job-chains have been completed).
1104 * @li ::BASE_JD_SW_EVENT_INFO : base_jd_event::data not used
1106 typedef struct base_jd_event_v2 {
1107 base_jd_event_code event_code; /**< event code */
1108 base_atom_id atom_number; /**< the atom number that has completed */
1109 struct base_jd_udata udata; /**< user data */
1113 * Padding required to ensure that the @ref struct base_dump_cpu_gpu_counters structure fills
1114 * a full cache line.
1117 #define BASE_CPU_GPU_CACHE_LINE_PADDING (36)
1121 * @brief Structure for BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS jobs.
1123 * This structure is stored into the memory pointed to by the @c jc field of @ref base_jd_atom.
1125 * This structure must be padded to ensure that it will occupy whole cache lines. This is to avoid
1126 * cases where access to pages containing the structure is shared between cached and un-cached
1127 * memory regions, which would cause memory corruption. Here we set the structure size to be 64 bytes
1128 * which is the cache line for ARM A15 processors.
1131 typedef struct base_dump_cpu_gpu_counters {
1136 u8 padding[BASE_CPU_GPU_CACHE_LINE_PADDING];
1137 } base_dump_cpu_gpu_counters;
1141 /** @} end group base_user_api_job_dispatch */
1143 #define GPU_MAX_JOB_SLOTS 16
1146 * @page page_base_user_api_gpuprops User-side Base GPU Property Query API
1148 * The User-side Base GPU Property Query API encapsulates two
1151 * - @ref base_user_api_gpuprops_dyn "Dynamic GPU Properties"
1152 * - @ref base_plat_config_gpuprops "Base Platform Config GPU Properties"
1154 * There is a related third module outside of Base, which is owned by the MIDG
1156 * - @ref gpu_props_static "Midgard Compile-time GPU Properties"
1158 * Base only deals with properties that vary between different Midgard
1159 * implementations - the Dynamic GPU properties and the Platform Config
1162 * For properties that are constant for the Midgard Architecture, refer to the
1163 * MIDG module. However, we will discuss their relevance here <b>just to
1164 * provide background information.</b>
1166 * @section sec_base_user_api_gpuprops_about About the GPU Properties in Base and MIDG modules
1168 * The compile-time properties (Platform Config, Midgard Compile-time
1169 * properties) are exposed as pre-processor macros.
1171 * Complementing the compile-time properties are the Dynamic GPU
1172 * Properties, which act as a conduit for the Midgard Configuration
1175 * In general, the dynamic properties are present to verify that the platform
1176 * has been configured correctly with the right set of Platform Config
1177 * Compile-time Properties.
1179 * As a consistant guide across the entire DDK, the choice for dynamic or
1180 * compile-time should consider the following, in order:
1181 * -# Can the code be written so that it doesn't need to know the
1182 * implementation limits at all?
1183 * -# If you need the limits, get the information from the Dynamic Property
1184 * lookup. This should be done once as you fetch the context, and then cached
1185 * as part of the context data structure, so it's cheap to access.
1186 * -# If there's a clear and arguable inefficiency in using Dynamic Properties,
1187 * then use a Compile-Time Property (Platform Config, or Midgard Compile-time
1188 * property). Examples of where this might be sensible follow:
1189 * - Part of a critical inner-loop
1190 * - Frequent re-use throughout the driver, causing significant extra load
1191 * instructions or control flow that would be worthwhile optimizing out.
1193 * We cannot provide an exhaustive set of examples, neither can we provide a
1194 * rule for every possible situation. Use common sense, and think about: what
1195 * the rest of the driver will be doing; how the compiler might represent the
1196 * value if it is a compile-time constant; whether an OEM shipping multiple
1197 * devices would benefit much more from a single DDK binary, instead of
1198 * insignificant micro-optimizations.
1200 * @section sec_base_user_api_gpuprops_dyn Dynamic GPU Properties
1202 * Dynamic GPU properties are presented in two sets:
1203 * -# the commonly used properties in @ref base_gpu_props, which have been
1204 * unpacked from GPU register bitfields.
1205 * -# The full set of raw, unprocessed properties in @ref gpu_raw_gpu_props
1206 * (also a member of @ref base_gpu_props). All of these are presented in
1207 * the packed form, as presented by the GPU registers themselves.
1209 * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to
1210 * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device
1211 * behaving differently?". In this case, all information about the
1212 * configuration is potentially useful, but it <b>does not need to be processed
1213 * by the driver</b>. Instead, the raw registers can be processed by the Mali
1214 * Tools software on the host PC.
1216 * The properties returned extend the Midgard Configuration Discovery
1217 * registers. For example, GPU clock speed is not specified in the Midgard
1218 * Architecture, but is <b>necessary for OpenCL's clGetDeviceInfo() function</b>.
1220 * The GPU properties are obtained by a call to
1221 * _mali_base_get_gpu_props(). This simply returns a pointer to a const
1222 * base_gpu_props structure. It is constant for the life of a base
1223 * context. Multiple calls to _mali_base_get_gpu_props() to a base context
1224 * return the same pointer to a constant structure. This avoids cache pollution
1225 * of the common data.
1227 * This pointer must not be freed, because it does not point to the start of a
1228 * region allocated by the memory allocator; instead, just close the @ref
1232 * @section sec_base_user_api_gpuprops_config Platform Config Compile-time Properties
1234 * The Platform Config File sets up gpu properties that are specific to a
1235 * certain platform. Properties that are 'Implementation Defined' in the
1236 * Midgard Architecture spec are placed here.
1238 * @note Reference configurations are provided for Midgard Implementations, such as
1239 * the Mali-T600 family. The customer need not repeat this information, and can select one of
1240 * these reference configurations. For example, VA_BITS, PA_BITS and the
1241 * maximum number of samples per pixel might vary between Midgard Implementations, but
1242 * \b not for platforms using the Mali-T604. This information is placed in
1243 * the reference configuration files.
1245 * The System Integrator creates the following structure:
1247 * - platform_XYZ/plat
1248 * - platform_XYZ/plat/plat_config.h
1250 * They then edit plat_config.h, using the example plat_config.h files as a
1253 * At the very least, the customer must set @ref CONFIG_GPU_CORE_TYPE, and will
1254 * receive a helpful \#error message if they do not do this correctly. This
1255 * selects the Reference Configuration for the Midgard Implementation. The rationale
1256 * behind this decision (against asking the customer to write \#include
1257 * <gpus/mali_t600.h> in their plat_config.h) is as follows:
1258 * - This mechanism 'looks' like a regular config file (such as Linux's
1260 * - It is difficult to get wrong in a way that will produce strange build
1262 * - They need not know where the mali_t600.h, other_midg_gpu.h etc. files are stored - and
1263 * so they won't accidentally pick another file with 'mali_t600' in its name
1264 * - When the build doesn't work, the System Integrator may think the DDK is
1265 * doesn't work, and attempt to fix it themselves:
1266 * - For the @ref CONFIG_GPU_CORE_TYPE mechanism, the only way to get past the
1267 * error is to set @ref CONFIG_GPU_CORE_TYPE, and this is what the \#error tells
1269 * - For a \#include mechanism, checks must still be made elsewhere, which the
1270 * System Integrator may try working around by setting \#defines (such as
1271 * VA_BITS) themselves in their plat_config.h. In the worst case, they may
1272 * set the prevention-mechanism \#define of
1273 * "A_CORRECT_MIDGARD_CORE_WAS_CHOSEN".
1274 * - In this case, they would believe they are on the right track, because
1275 * the build progresses with their fix, but with errors elsewhere.
1277 * However, there is nothing to prevent the customer using \#include to organize
1278 * their own configurations files hierarchically.
1280 * The mechanism for the header file processing is as follows:
1283 digraph plat_config_mechanism {
1292 rank = same; ordering = out;
1294 "gpu/mali_gpu_props.h";
1295 "base/midg_gpus/mali_t600.h";
1296 "base/midg_gpus/other_midg_gpu.h";
1298 { rank = same; "plat/plat_config.h"; }
1301 "gpu/mali_gpu.h" [ shape=box ];
1302 gpu_chooser [ label="" style="invisible" width=0 height=0 fixedsize=true ];
1303 select_gpu [ label="Mali-T600 | Other\n(select_gpu.h)" shape=polygon,sides=4,distortion=0.25 width=3.3 height=0.99 fixedsize=true ] ;
1306 { rank = same; "plat/plat_config.h"; }
1307 { rank = same; "mali_base.h"; }
1309 "mali_base.h" -> "gpu/mali_gpu.h" -> "gpu/mali_gpu_props.h";
1310 "mali_base.h" -> "plat/plat_config.h" ;
1311 "mali_base.h" -> select_gpu ;
1313 "plat/plat_config.h" -> gpu_chooser [style="dotted,bold" dir=none weight=4] ;
1314 gpu_chooser -> select_gpu [style="dotted,bold"] ;
1316 select_gpu -> "base/midg_gpus/mali_t600.h" ;
1317 select_gpu -> "base/midg_gpus/other_midg_gpu.h" ;
1322 * @section sec_base_user_api_gpuprops_kernel Kernel Operation
1324 * During Base Context Create time, user-side makes a single kernel call:
1325 * - A call to fill user memory with GPU information structures
1327 * The kernel-side will fill the provided the entire processed @ref base_gpu_props
1328 * structure, because this information is required in both
1329 * user and kernel side; it does not make sense to decode it twice.
1331 * Coherency groups must be derived from the bitmasks, but this can be done
1332 * kernel side, and just once at kernel startup: Coherency groups must already
1333 * be known kernel-side, to support chains that specify a 'Only Coherent Group'
1334 * SW requirement, or 'Only Coherent Group with Tiler' SW requirement.
1336 * @section sec_base_user_api_gpuprops_cocalc Coherency Group calculation
1337 * Creation of the coherent group data is done at device-driver startup, and so
1338 * is one-time. This will most likely involve a loop with CLZ, shifting, and
1339 * bit clearing on the L2_PRESENT mask, depending on whether the
1340 * system is L2 Coherent. The number of shader cores is done by a
1341 * population count, since faulty cores may be disabled during production,
1342 * producing a non-contiguous mask.
1344 * The memory requirements for this algoirthm can be determined either by a u64
1345 * population count on the L2_PRESENT mask (a LUT helper already is
1346 * requried for the above), or simple assumption that there can be no more than
1347 * 16 coherent groups, since core groups are typically 4 cores.
1351 * @addtogroup base_user_api_gpuprops User-side Base GPU Property Query APIs
1356 * @addtogroup base_user_api_gpuprops_dyn Dynamic HW Properties
1360 #define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3
1362 #define BASE_MAX_COHERENT_GROUPS 16
1364 struct mali_base_gpu_core_props {
1366 * Product specific value.
1371 * Status of the GPU release.
1372 * No defined values, but starts at 0 and increases by one for each release
1373 * status (alpha, beta, EAC, etc.).
1374 * 4 bit values (0-15).
1379 * Minor release number of the GPU. "P" part of an "RnPn" release number.
1380 * 8 bit values (0-255).
1385 * Major release number of the GPU. "R" part of an "RnPn" release number.
1386 * 4 bit values (0-15).
1393 * @usecase GPU clock speed is not specified in the Midgard Architecture, but is
1394 * <b>necessary for OpenCL's clGetDeviceInfo() function</b>.
1399 * @usecase GPU clock max/min speed is required for computing best/worst case
1400 * in tasks as job scheduling ant irq_throttling. (It is not specified in the
1401 * Midgard Architecture).
1403 u32 gpu_freq_khz_max;
1404 u32 gpu_freq_khz_min;
1407 * Size of the shader program counter, in bits.
1409 u32 log2_program_counter_size;
1412 * TEXTURE_FEATURES_x registers, as exposed by the GPU. This is a
1413 * bitpattern where a set bit indicates that the format is supported.
1415 * Before using a texture format, it is recommended that the corresponding
1418 u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS];
1421 * Theoretical maximum memory available to the GPU. It is unlikely that a
1422 * client will be able to allocate all of this memory for their own
1423 * purposes, but this at least provides an upper bound on the memory
1424 * available to the GPU.
1426 * This is required for OpenCL's clGetDeviceInfo() call when
1427 * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The
1428 * client will not be expecting to allocate anywhere near this value.
1430 u64 gpu_available_memory_size;
1435 * More information is possible - but associativity and bus width are not
1436 * required by upper-level apis.
1438 struct mali_base_gpu_l2_cache_props {
1441 u8 num_l2_slices; /* Number of L2C slices. 1 or higher */
1445 struct mali_base_gpu_tiler_props {
1446 u32 bin_size_bytes; /* Max is 4*2^15 */
1447 u32 max_active_levels; /* Max is 2^15 */
1451 * GPU threading system details.
1453 struct mali_base_gpu_thread_props {
1454 u32 max_threads; /* Max. number of threads per core */
1455 u32 max_workgroup_size; /* Max. number of threads per workgroup */
1456 u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */
1457 u16 max_registers; /* Total size [1..65535] of the register file available per core. */
1458 u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */
1459 u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */
1460 u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */
1465 * @brief descriptor for a coherent group
1467 * \c core_mask exposes all cores in that coherent group, and \c num_cores
1468 * provides a cached population-count for that mask.
1470 * @note Whilst all cores are exposed in the mask, not all may be available to
1471 * the application, depending on the Kernel Power policy.
1473 * @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage.
1475 struct mali_base_gpu_coherent_group {
1476 u64 core_mask; /**< Core restriction mask required for the group */
1477 u16 num_cores; /**< Number of cores in the group */
1482 * @brief Coherency group information
1484 * Note that the sizes of the members could be reduced. However, the \c group
1485 * member might be 8-byte aligned to ensure the u64 core_mask is 8-byte
1486 * aligned, thus leading to wastage if the other members sizes were reduced.
1488 * The groups are sorted by core mask. The core masks are non-repeating and do
1491 struct mali_base_gpu_coherent_group_info {
1495 * Number of core groups (coherent or not) in the GPU. Equivalent to the number of L2 Caches.
1497 * The GPU Counter dumping writes 2048 bytes per core group, regardless of
1498 * whether the core groups are coherent or not. Hence this member is needed
1499 * to calculate how much memory is required for dumping.
1501 * @note Do not use it to work out how many valid elements are in the
1502 * group[] member. Use num_groups instead.
1504 u32 num_core_groups;
1507 * Coherency features of the memory, accessed by @ref gpu_mem_features
1515 * Descriptors of coherent groups
1517 struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS];
1521 * A complete description of the GPU's Hardware Configuration Discovery
1524 * The information is presented inefficiently for access. For frequent access,
1525 * the values should be better expressed in an unpacked form in the
1526 * base_gpu_props structure.
1528 * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to
1529 * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device
1530 * behaving differently?". In this case, all information about the
1531 * configuration is potentially useful, but it <b>does not need to be processed
1532 * by the driver</b>. Instead, the raw registers can be processed by the Mali
1533 * Tools software on the host PC.
1536 struct gpu_raw_gpu_props {
1540 u64 unused_1; /* keep for backward compatibility */
1543 u32 suspend_size; /* API 8.2+ */
1550 u32 js_features[GPU_MAX_JOB_SLOTS];
1552 u32 texture_features[3];
1556 u32 thread_max_threads;
1557 u32 thread_max_workgroup_size;
1558 u32 thread_max_barrier_size;
1559 u32 thread_features;
1562 * Note: This is the _selected_ coherency mode rather than the
1563 * available modes as exposed in the coherency_features register.
1569 * Return structure for _mali_base_get_gpu_props().
1571 * NOTE: the raw_props member in this datastructure contains the register
1572 * values from which the value of the other members are derived. The derived
1573 * members exist to allow for efficient access and/or shielding the details
1574 * of the layout of the registers.
1577 typedef struct mali_base_gpu_props {
1578 struct mali_base_gpu_core_props core_props;
1579 struct mali_base_gpu_l2_cache_props l2_props;
1580 u64 unused_1; /* keep for backwards compatibility */
1581 struct mali_base_gpu_tiler_props tiler_props;
1582 struct mali_base_gpu_thread_props thread_props;
1584 /** This member is large, likely to be 128 bytes */
1585 struct gpu_raw_gpu_props raw_props;
1587 /** This must be last member of the structure */
1588 struct mali_base_gpu_coherent_group_info coherency_info;
1591 /** @} end group base_user_api_gpuprops_dyn */
1593 /** @} end group base_user_api_gpuprops */
1596 * @addtogroup base_user_api_core User-side Base core APIs
1601 * \enum base_context_create_flags
1603 * Flags to pass to ::base_context_init.
1604 * Flags can be ORed together to enable multiple things.
1606 * These share the same space as @ref basep_context_private_flags, and so must
1607 * not collide with them.
1609 enum base_context_create_flags {
1611 BASE_CONTEXT_CREATE_FLAG_NONE = 0,
1613 /** Base context is embedded in a cctx object (flag used for CINSTR software counter macros) */
1614 BASE_CONTEXT_CCTX_EMBEDDED = (1u << 0),
1616 /** Base context is a 'System Monitor' context for Hardware counters.
1618 * One important side effect of this is that job submission is disabled. */
1619 BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED = (1u << 1)
1623 * Bitpattern describing the ::base_context_create_flags that can be passed to base_context_init()
1625 #define BASE_CONTEXT_CREATE_ALLOWED_FLAGS \
1626 (((u32)BASE_CONTEXT_CCTX_EMBEDDED) | \
1627 ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED))
1630 * Bitpattern describing the ::base_context_create_flags that can be passed to the kernel
1632 #define BASE_CONTEXT_CREATE_KERNEL_FLAGS \
1633 ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED)
1636 * Private flags used on the base context
1638 * These start at bit 31, and run down to zero.
1640 * They share the same space as @ref base_context_create_flags, and so must
1641 * not collide with them.
1643 enum basep_context_private_flags {
1644 /** Private flag tracking whether job descriptor dumping is disabled */
1645 BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED = (1 << 31)
1648 /** @} end group base_user_api_core */
1650 /** @} end group base_user_api */
1653 * @addtogroup base_plat_config_gpuprops Base Platform Config GPU Properties
1656 * C Pre-processor macros are exposed here to do with Platform
1660 * - GPU Properties that are constant on a particular Midgard Family
1661 * Implementation e.g. Maximum samples per pixel on Mali-T600.
1662 * - General platform config for the GPU, such as the GPU major and minor
1666 /** @} end group base_plat_config_gpuprops */
1669 * @addtogroup base_api Base APIs
1674 * @brief The payload for a replay job. This must be in GPU memory.
1676 typedef struct base_jd_replay_payload {
1678 * Pointer to the first entry in the base_jd_replay_jc list. These
1679 * will be replayed in @b reverse order (so that extra ones can be added
1680 * to the head in future soft jobs without affecting this soft job)
1685 * Pointer to the fragment job chain.
1690 * Pointer to the tiler heap free FBD field to be modified.
1692 u64 tiler_heap_free;
1695 * Hierarchy mask for the replayed fragment jobs. May be zero.
1697 u16 fragment_hierarchy_mask;
1700 * Hierarchy mask for the replayed tiler jobs. May be zero.
1702 u16 tiler_hierarchy_mask;
1705 * Default weight to be used for hierarchy levels not in the original
1708 u32 hierarchy_default_weight;
1711 * Core requirements for the tiler job chain
1713 base_jd_core_req tiler_core_req;
1716 * Core requirements for the fragment job chain
1718 base_jd_core_req fragment_core_req;
1721 } base_jd_replay_payload;
1724 * @brief An entry in the linked list of job chains to be replayed. This must
1727 typedef struct base_jd_replay_jc {
1729 * Pointer to next entry in the list. A setting of NULL indicates the
1735 * Pointer to the job chain.
1739 } base_jd_replay_jc;
1741 /* Maximum number of jobs allowed in a fragment chain in the payload of a
1743 #define BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT 256
1745 /** @} end group base_api */
1747 typedef struct base_profiling_controls {
1748 u32 profiling_controls[FBDUMP_CONTROL_MAX];
1749 } base_profiling_controls;
1751 #endif /* _BASE_KERNEL_H_ */