# Driver version string which is returned to userspace via an ioctl
-MALI_RELEASE_NAME ?= "r5p0-06rel0"
+MALI_RELEASE_NAME ?= "r6p0-02rel0"
# Paths required for build
KBASE_PATH = $(src)
mali_kbase_cpuprops.c \
mali_kbase_gpuprops.c \
mali_kbase_js.c \
- mali_kbase_js_affinity.c \
mali_kbase_js_ctx_attr.c \
mali_kbase_event.c \
mali_kbase_context.c \
mali_kbase_pm.c \
- mali_kbase_pm_driver.c \
- mali_kbase_pm_metrics.c \
- mali_kbase_pm_ca.c \
- mali_kbase_pm_ca_fixed.c \
- mali_kbase_pm_always_on.c \
- mali_kbase_pm_coarse_demand.c \
- mali_kbase_pm_demand.c \
- mali_kbase_pm_policy.c \
mali_kbase_config.c \
mali_kbase_security.c \
mali_kbase_instr.c \
+ mali_kbase_vinstr.c \
mali_kbase_softjobs.c \
mali_kbase_10969_workaround.c \
mali_kbase_hw.c \
mali_kbase_utility.c \
mali_kbase_debug.c \
mali_kbase_trace_timeline.c \
+ mali_kbase_gpu_memory_debugfs.c \
mali_kbase_mem_linux.c \
mali_kbase_core_linux.c \
mali_kbase_sync.c \
mali_kbase_sync_user.c \
mali_kbase_replay.c \
mali_kbase_mem_profile_debugfs.c \
- mali_kbase_mmu_hw_direct.c \
+ mali_kbase_mmu_mode_lpae.c \
mali_kbase_disjoint_events.c \
- mali_kbase_gator_api.c
-
-ifeq ($(CONFIG_DEBUG_FS),y)
- SRC += mali_kbase_gpu_memory_debugfs.c
-endif
-
-ifeq ($(MALI_CUSTOMER_RELEASE),0)
-SRC += \
- mali_kbase_pm_ca_random.c \
- mali_kbase_pm_demand_always_powered.c \
- mali_kbase_pm_fast_start.c
+ mali_kbase_gator_api.c \
+ mali_kbase_debug_mem_view.c \
+ mali_kbase_smc.c
+
+ifeq ($(CONFIG_MALI_MIPE_ENABLED),y)
+ SRC += mali_kbase_tlstream.c
+ ifeq ($(MALI_UNIT_TEST),1)
+ SRC += mali_kbase_tlstream_test.c
+ endif
endif
# Job Scheduler Policy: Completely Fair Scheduler
SRC += mali_kbase_mem_alloc.c
endif
-# ensure GPL version of malisw gets pulled in
ccflags-y += -I$(KBASE_PATH)
-ifeq ($(CONFIG_MALI_NO_MALI),y)
- # Dummy model
- SRC += mali_kbase_model_dummy.c
- SRC += mali_kbase_model_linux.c
- # HW error simulation
- SRC += mali_kbase_model_error_generator.c
-endif
-
-ifeq ($(MALI_MOCK_TEST),1)
- # Test functionality
- SRC += tests/internal/src/mock/mali_kbase_pm_driver_mock.c
-endif
-
# in-tree/out-of-tree logic needs to be slightly different to determine if a file is present
ifeq ($(KBUILD_EXTMOD),)
# in-tree
MALI_METRICS_PATH = $(KBUILD_EXTMOD)
endif
-# Use vsync metrics example using PL111 driver, if available
-ifeq ($(wildcard $(MALI_METRICS_PATH)/mali_kbase_pm_metrics_linux.c),)
- SRC += mali_kbase_pm_metrics_dummy.c
-else
- SRC += mali_kbase_pm_metrics_linux.c
-endif
-
ifeq ($(CONFIG_MALI_PLATFORM_FAKE),y)
SRC += mali_kbase_platform_fake.c
ccflags-y += -I$(src)/platform/vexpress_1xv7_a57
endif
- ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_VIRTEX7_40MHZ),y)
- SRC += platform/vexpress_virtex7_40mhz/mali_kbase_config_vexpress.c \
- platform/vexpress_virtex7_40mhz/mali_kbase_cpu_vexpress.c
- ccflags-y += -I$(src)/platform/vexpress_virtex7_40mhz
- endif
-
ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_6XVIRTEX7_10MHZ),y)
SRC += platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c \
platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c
ccflags-y += -I$(src)/platform/vexpress_6xvirtex7_10mhz
endif
- ifeq ($(CONFIG_MALI_PLATFORM_GOLDFISH),y)
- SRC += platform/goldfish/mali_kbase_config_goldfish.c
- ccflags-y += -I$(src)/platform/goldfish
- endif
-
- ifeq ($(CONFIG_MALI_PLATFORM_PBX),y)
- SRC += platform/pbx/mali_kbase_config_pbx.c
- ccflags-y += -I$(src)/platform/pbx
- endif
-
- ifeq ($(CONFIG_MALI_PLATFORM_PANDA),y)
- SRC += platform/panda/mali_kbase_config_panda.c
- ccflags-y += -I$(src)/platform/panda
+ ifeq ($(CONFIG_MALI_PLATFORM_A7_KIPLING),y)
+ SRC += platform/a7_kipling/mali_kbase_config_a7_kipling.c \
+ platform/a7_kipling/mali_kbase_cpu_a7_kipling.c
+ ccflags-y += -I$(src)/platform/a7_kipling
endif
ifeq ($(CONFIG_MALI_PLATFORM_THIRDPARTY),y)
endif
endif
+ifeq ($(CONFIG_MALI_PLATFORM_DEVICETREE),y)
+ SRC += platform/devicetree/mali_kbase_runtime_pm.c
+ SRC += platform/devicetree/mali_kbase_config_devicetree.c
+ ccflags-y += -I$(src)/platform/devicetree
+endif
+
# Tell the Linux build system from which .o file to create the kernel module
obj-$(CONFIG_MALI_MIDGARD) += mali_kbase.o
# Tell the Linux build system to enable building of our .c files
mali_kbase-y := $(SRC:.c=.o)
-mali_kbase-$(CONFIG_MALI_DEVFREQ) += mali_kbase_devfreq.o
-mali_kbase-$(CONFIG_MALI_POWER_ACTOR) += mali_kbase_power_actor.o
-
ifneq ($(wildcard $(src)/internal/Kbuild),)
ifeq ($(MALI_CUSTOMER_RELEASE),0)
+# This include may set MALI_BACKEND_PATH and CONFIG_MALI_BACKEND_REAL
include $(src)/internal/Kbuild
mali_kbase-y += $(INTERNAL:.c=.o)
endif
endif
+
+MALI_BACKEND_PATH ?= backend
+CONFIG_MALI_BACKEND ?= gpu
+CONFIG_MALI_BACKEND_REAL ?= $(CONFIG_MALI_BACKEND)
+
+ifeq ($(MALI_MOCK_TEST),1)
+ifeq ($(CONFIG_MALI_BACKEND_REAL),gpu)
+# Test functionality
+mali_kbase-y += tests/internal/src/mock/mali_kbase_pm_driver_mock.o
+endif
+endif
+
+include $(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL)/Kbuild
+mali_kbase-y += $(BACKEND:.c=.o)
+
+ccflags-y += -I$(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL)
+subdir-ccflags-y += -I$(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL)
#
-# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
To compile this driver as a module, choose M here:
this will generate a single module, called mali_kbase.
-config MALI_GATOR_SUPPORT
- bool "Streamline Debug support"
+choice
+ prompt "Streamline support"
depends on MALI_MIDGARD
- default n
+ default MALI_TIMELINE_DISABLED
+ help
+ Select streamline support configuration.
+
+config MALI_TIMELINE_DISABLED
+ bool "Streamline support disabled"
+ help
+ Disable support for ARM Streamline Performance Analyzer.
+
+ Timeline support will not be included in
+ kernel code.
+ Debug stream will not be generated.
+
+config MALI_GATOR_SUPPORT
+ bool "Streamline support via Gator"
help
Adds diagnostic support for use with the ARM Streamline Performance Analyzer.
You will need the Gator device driver already loaded before loading this driver when enabling
Streamline debug support.
+config MALI_MIPE_ENABLED
+ bool "Streamline support via MIPE"
+ help
+ Adds diagnostic support for use with the ARM Streamline Performance Analyzer.
+
+ Stream will be transmitted directly to Mali GPU library.
+ Compatible version of the library is required to read debug stream generated by kernel.
+
+endchoice
+
config MALI_MIDGARD_DVFS
bool "Enable DVFS"
depends on MALI_MIDGARD
help
Enables sysfs for the Mali Midgard DDK. Set/Monitor the Mali Midgard DDK
-# MALI_EXPERT configuration options
-
-menuconfig MALI_EXPERT
- depends on MALI_MIDGARD
- bool "Enable Expert Settings"
- default n
- help
- Enabling this option and modifying the default settings may produce a driver with performance or
- other limitations.
-
config MALI_DEVFREQ
bool "devfreq support for Mali"
depends on MALI_MIDGARD && PM_DEVFREQ
governor, the frequency of Mali will be dynamically selected from the
available OPPs.
-config MALI_POWER_ACTOR
- bool "Thermal API support for Mali"
- depends on MALI_MIDGARD && DEVFREQ_THERMAL && THERMAL_POWER_ACTOR
- help
- Support the thermal API for Mali.
- This can be used with the power allocator thermal governor to
- dynamically allocate the power budget to Mali.
+# MALI_EXPERT configuration options
+
+menuconfig MALI_EXPERT
+ depends on MALI_MIDGARD
+ bool "Enable Expert Settings"
+ default n
+ help
+ Enabling this option and modifying the default settings may produce a driver with performance or
+ other limitations.
config MALI_DEBUG_SHADER_SPLIT_FS
bool "Allow mapping of shader cores via sysfs"
#
-# (C) COPYRIGHT 2010-2013 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
ifeq ($(MALI_UNIT_TEST), 1)
EXTRA_SYMBOLS += $(KBASE_PATH_RELATIVE)/tests/internal/src/kernel_assert_module/linux/Module.symvers
+ EXTRA_SYMBOLS += $(KBASE_PATH_RELATIVE)/../../../../tests/kutf/Module.symvers
endif
# GPL driver supports KDS
# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions
all:
- $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules
+ $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules
clean:
$(MAKE) -C $(KDIR) M=$(CURDIR) clean
--- /dev/null
+#
+# (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+BACKEND += \
+ backend/gpu/mali_kbase_cache_policy_backend.c \
+ backend/gpu/mali_kbase_device_hw.c \
+ backend/gpu/mali_kbase_gpu.c \
+ backend/gpu/mali_kbase_gpuprops_backend.c \
+ backend/gpu/mali_kbase_irq_linux.c \
+ backend/gpu/mali_kbase_instr_backend.c \
+ backend/gpu/mali_kbase_jm_as.c \
+ backend/gpu/mali_kbase_jm_hw.c \
+ backend/gpu/mali_kbase_jm_rb.c \
+ backend/gpu/mali_kbase_js_affinity.c \
+ backend/gpu/mali_kbase_js_backend.c \
+ backend/gpu/mali_kbase_mmu_hw_direct.c \
+ backend/gpu/mali_kbase_pm_backend.c \
+ backend/gpu/mali_kbase_pm_driver.c \
+ backend/gpu/mali_kbase_pm_metrics.c \
+ backend/gpu/mali_kbase_pm_ca.c \
+ backend/gpu/mali_kbase_pm_ca_fixed.c \
+ backend/gpu/mali_kbase_pm_always_on.c \
+ backend/gpu/mali_kbase_pm_coarse_demand.c \
+ backend/gpu/mali_kbase_pm_demand.c \
+ backend/gpu/mali_kbase_pm_policy.c \
+ backend/gpu/mali_kbase_time.c
+
+ifeq ($(MALI_CUSTOMER_RELEASE),0)
+BACKEND += \
+ backend/gpu/mali_kbase_pm_ca_random.c \
+ backend/gpu/mali_kbase_pm_demand_always_powered.c \
+ backend/gpu/mali_kbase_pm_fast_start.c
+endif
+
+ifeq ($(CONFIG_MALI_DEVFREQ),y)
+BACKEND += backend/gpu/mali_kbase_devfreq.c
+endif
+
+ifeq ($(CONFIG_MALI_NO_MALI),y)
+ # Dummy model
+ BACKEND += backend/gpu/mali_kbase_model_dummy.c
+ BACKEND += backend/gpu/mali_kbase_model_linux.c
+ # HW error simulation
+ BACKEND += backend/gpu/mali_kbase_model_error_generator.c
+endif
+
+ifeq ($(wildcard $(MALI_METRICS_PATH)/backend/gpu/mali_kbase_pm_metrics_linux.c),)
+ BACKEND += backend/gpu/mali_kbase_pm_metrics_dummy.c
+else
+ BACKEND += backend/gpu/mali_kbase_pm_metrics_linux.c
+endif
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * Backend specific configuration
+ */
+
+#ifndef _KBASE_BACKEND_CONFIG_H_
+#define _KBASE_BACKEND_CONFIG_H_
+
+/* Enable GPU reset API */
+#define KBASE_GPU_RESET_EN 1
+
+#endif /* _KBASE_BACKEND_CONFIG_H_ */
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include "backend/gpu/mali_kbase_cache_policy_backend.h"
+#include <backend/gpu/mali_kbase_pm_internal.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
+
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+#ifndef _KBASE_CACHE_POLICY_BACKEND_H_
+#define _KBASE_CACHE_POLICY_BACKEND_H_
+
+#include "mali_kbase.h"
+#include "mali_base_kernel.h"
+
+
+#endif /* _KBASE_CACHE_POLICY_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include <mali_kbase.h>
+#include <mali_kbase_config_defaults.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+#include <linux/clk.h>
+#include <linux/devfreq.h>
+#ifdef CONFIG_DEVFREQ_THERMAL
+#include <linux/devfreq_cooling.h>
+#endif
+
+#include <linux/version.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
+#include <linux/pm_opp.h>
+#else /* Linux >= 3.13 */
+/* In 3.13 the OPP include header file, types, and functions were all
+ * renamed. Use the old filename for the include, and define the new names to
+ * the old, when an old kernel is detected.
+ */
+#include <linux/opp.h>
+#define dev_pm_opp opp
+#define dev_pm_opp_get_voltage opp_get_voltage
+#define dev_pm_opp_get_opp_count opp_get_opp_count
+#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil
+#endif /* Linux >= 3.13 */
+
+
+static int
+kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags)
+{
+ struct kbase_device *kbdev = dev_get_drvdata(dev);
+ struct dev_pm_opp *opp;
+ unsigned long freq = 0;
+ unsigned long voltage;
+ int err;
+
+ freq = *target_freq;
+
+ rcu_read_lock();
+ opp = devfreq_recommended_opp(dev, &freq, flags);
+ voltage = dev_pm_opp_get_voltage(opp);
+ rcu_read_unlock();
+ if (IS_ERR_OR_NULL(opp)) {
+ dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp));
+ return PTR_ERR(opp);
+ }
+
+ /*
+ * Only update if there is a change of frequency
+ */
+ if (kbdev->current_freq == freq) {
+ *target_freq = freq;
+ return 0;
+ }
+
+#ifdef CONFIG_REGULATOR
+ if (kbdev->regulator && kbdev->current_voltage != voltage
+ && kbdev->current_freq < freq) {
+ err = regulator_set_voltage(kbdev->regulator, voltage, voltage);
+ if (err) {
+ dev_err(dev, "Failed to increase voltage (%d)\n", err);
+ return err;
+ }
+ }
+#endif
+
+ err = clk_set_rate(kbdev->clock, freq);
+ if (err) {
+ dev_err(dev, "Failed to set clock %lu (target %lu)\n",
+ freq, *target_freq);
+ return err;
+ }
+
+#ifdef CONFIG_REGULATOR
+ if (kbdev->regulator && kbdev->current_voltage != voltage
+ && kbdev->current_freq > freq) {
+ err = regulator_set_voltage(kbdev->regulator, voltage, voltage);
+ if (err) {
+ dev_err(dev, "Failed to decrease voltage (%d)\n", err);
+ return err;
+ }
+ }
+#endif
+
+ *target_freq = freq;
+ kbdev->current_voltage = voltage;
+ kbdev->current_freq = freq;
+
+ kbase_pm_reset_dvfs_utilisation(kbdev);
+
+ return err;
+}
+
+static int
+kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq)
+{
+ struct kbase_device *kbdev = dev_get_drvdata(dev);
+
+ *freq = kbdev->current_freq;
+
+ return 0;
+}
+
+static int
+kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat)
+{
+ struct kbase_device *kbdev = dev_get_drvdata(dev);
+
+ stat->current_frequency = kbdev->current_freq;
+
+ kbase_pm_get_dvfs_utilisation(kbdev,
+ &stat->total_time, &stat->busy_time);
+
+ stat->private_data = NULL;
+
+#ifdef CONFIG_DEVFREQ_THERMAL
+ memcpy(&kbdev->devfreq_cooling->last_status, stat, sizeof(*stat));
+#endif
+
+ return 0;
+}
+
+static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev,
+ struct devfreq_dev_profile *dp)
+{
+ int count;
+ int i = 0;
+ unsigned long freq = 0;
+ struct dev_pm_opp *opp;
+
+ rcu_read_lock();
+ count = dev_pm_opp_get_opp_count(kbdev->dev);
+ if (count < 0) {
+ rcu_read_unlock();
+ return count;
+ }
+ rcu_read_unlock();
+
+ dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]),
+ GFP_KERNEL);
+ if (!dp->freq_table)
+ return -ENOMEM;
+
+ rcu_read_lock();
+ for (i = 0; i < count; i++, freq++) {
+ opp = dev_pm_opp_find_freq_ceil(kbdev->dev, &freq);
+ if (IS_ERR(opp))
+ break;
+
+ dp->freq_table[i] = freq;
+ }
+ rcu_read_unlock();
+
+ if (count != i)
+ dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n",
+ count, i);
+
+ dp->max_state = i;
+
+ return 0;
+}
+
+static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev)
+{
+ struct devfreq_dev_profile *dp = kbdev->devfreq->profile;
+
+ kfree(dp->freq_table);
+}
+
+static void kbase_devfreq_exit(struct device *dev)
+{
+ struct kbase_device *kbdev = dev_get_drvdata(dev);
+
+ kbase_devfreq_term_freq_table(kbdev);
+}
+
+int kbase_devfreq_init(struct kbase_device *kbdev)
+{
+#ifdef CONFIG_DEVFREQ_THERMAL
+ struct devfreq_cooling_ops *callbacks = POWER_MODEL_CALLBACKS;
+#endif
+ struct devfreq_dev_profile *dp;
+ int err;
+
+ dev_dbg(kbdev->dev, "Init Mali devfreq\n");
+
+ if (!kbdev->clock)
+ return -ENODEV;
+
+ kbdev->current_freq = clk_get_rate(kbdev->clock);
+
+ dp = &kbdev->devfreq_profile;
+
+ dp->initial_freq = kbdev->current_freq;
+ dp->polling_ms = 100;
+ dp->target = kbase_devfreq_target;
+ dp->get_dev_status = kbase_devfreq_status;
+ dp->get_cur_freq = kbase_devfreq_cur_freq;
+ dp->exit = kbase_devfreq_exit;
+
+ if (kbase_devfreq_init_freq_table(kbdev, dp))
+ return -EFAULT;
+
+ kbdev->devfreq = devfreq_add_device(kbdev->dev, dp,
+ "simple_ondemand", NULL);
+ if (IS_ERR(kbdev->devfreq)) {
+ kbase_devfreq_term_freq_table(kbdev);
+ return PTR_ERR(kbdev->devfreq);
+ }
+
+ err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq);
+ if (err) {
+ dev_err(kbdev->dev,
+ "Failed to register OPP notifier (%d)\n", err);
+ goto opp_notifier_failed;
+ }
+
+#ifdef CONFIG_DEVFREQ_THERMAL
+ if (callbacks) {
+
+ kbdev->devfreq_cooling = of_devfreq_cooling_register_power(
+ kbdev->dev->of_node,
+ kbdev->devfreq,
+ callbacks);
+ if (IS_ERR_OR_NULL(kbdev->devfreq_cooling)) {
+ err = PTR_ERR(kbdev->devfreq_cooling);
+ dev_err(kbdev->dev,
+ "Failed to register cooling device (%d)\n",
+ err);
+ goto cooling_failed;
+ }
+ }
+#endif
+
+ return 0;
+
+#ifdef CONFIG_DEVFREQ_THERMAL
+cooling_failed:
+ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
+#endif /* CONFIG_DEVFREQ_THERMAL */
+opp_notifier_failed:
+ err = devfreq_remove_device(kbdev->devfreq);
+ if (err)
+ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
+ else
+ kbdev->devfreq = NULL;
+
+ return err;
+}
+
+void kbase_devfreq_term(struct kbase_device *kbdev)
+{
+ int err;
+
+ dev_dbg(kbdev->dev, "Term Mali devfreq\n");
+
+#ifdef CONFIG_DEVFREQ_THERMAL
+ devfreq_cooling_unregister(kbdev->devfreq_cooling);
+#endif
+
+ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
+
+ err = devfreq_remove_device(kbdev->devfreq);
+ if (err)
+ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
+ else
+ kbdev->devfreq = NULL;
+}
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _BASE_DEVFREQ_H_
+#define _BASE_DEVFREQ_H_
+
+int kbase_devfreq_init(struct kbase_device *kbdev);
+void kbase_devfreq_term(struct kbase_device *kbdev);
+
+#endif /* _BASE_DEVFREQ_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/*
+ *
+ */
+#include <mali_kbase.h>
+#include <backend/gpu/mali_kbase_instr_internal.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+#include <backend/gpu/mali_kbase_device_internal.h>
+
+#if !defined(CONFIG_MALI_NO_MALI)
+void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value,
+ struct kbase_context *kctx)
+{
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
+ KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID);
+ KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
+ dev_dbg(kbdev->dev, "w: reg %04x val %08x", offset, value);
+ writel(value, kbdev->reg + offset);
+ if (kctx && kctx->jctx.tb)
+ kbase_device_trace_register_access(kctx, REG_WRITE, offset,
+ value);
+}
+
+KBASE_EXPORT_TEST_API(kbase_reg_write);
+
+u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset,
+ struct kbase_context *kctx)
+{
+ u32 val;
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
+ KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID);
+ KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
+ val = readl(kbdev->reg + offset);
+ dev_dbg(kbdev->dev, "r: reg %04x val %08x", offset, val);
+ if (kctx && kctx->jctx.tb)
+ kbase_device_trace_register_access(kctx, REG_READ, offset, val);
+ return val;
+}
+
+KBASE_EXPORT_TEST_API(kbase_reg_read);
+#endif /* !defined(CONFIG_MALI_NO_MALI) */
+
+/**
+ * kbase_report_gpu_fault - Report a GPU fault.
+ * @kbdev: Kbase device pointer
+ * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS
+ * was also set
+ *
+ * This function is called from the interrupt handler when a GPU fault occurs.
+ * It reports the details of the fault using dev_warn().
+ */
+static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple)
+{
+ u32 status;
+ u64 address;
+
+ status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL);
+ address = (u64) kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI), NULL) << 32;
+ address |= kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO), NULL);
+
+ dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx",
+ status & 0xFF,
+ kbase_exception_name(kbdev, status),
+ address);
+ if (multiple)
+ dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n");
+}
+
+void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
+{
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val);
+ if (val & GPU_FAULT)
+ kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS);
+
+ if (val & RESET_COMPLETED)
+ kbase_pm_reset_done(kbdev);
+
+ if (val & PRFCNT_SAMPLE_COMPLETED)
+ kbase_instr_hwcnt_sample_done(kbdev);
+
+ if (val & CLEAN_CACHES_COMPLETED)
+ kbase_clean_caches_done(kbdev);
+
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, NULL);
+
+ /* kbase_pm_check_transitions must be called after the IRQ has been
+ * cleared. This is because it might trigger further power transitions
+ * and we don't want to miss the interrupt raised to notify us that
+ * these further transitions have finished.
+ */
+ if (val & POWER_CHANGED_ALL)
+ kbase_pm_power_changed(kbdev);
+
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val);
+}
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * Backend-specific HW access device APIs
+ */
+
+#ifndef _KBASE_DEVICE_INTERNAL_H_
+#define _KBASE_DEVICE_INTERNAL_H_
+
+/**
+ * kbase_reg_write - write to GPU register
+ * @kbdev: Kbase device pointer
+ * @offset: Offset of register
+ * @value: Value to write
+ * @kctx: Kbase context pointer. May be NULL
+ *
+ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If
+ * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr
+ * != KBASEP_AS_NR_INVALID).
+ */
+void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value,
+ struct kbase_context *kctx);
+
+/**
+ * kbase_reg_read - read from GPU register
+ * @kbdev: Kbase device pointer
+ * @offset: Offset of register
+ * @kctx: Kbase context pointer. May be NULL
+ *
+ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If
+ * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr
+ * != KBASEP_AS_NR_INVALID).
+ *
+ * Return: Value in desired register
+ */
+u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset,
+ struct kbase_context *kctx);
+
+
+/**
+ * kbase_gpu_interrupt - GPU interrupt handler
+ * @kbdev: Kbase device pointer
+ * @val: The value of the GPU IRQ status register which triggered the call
+ *
+ * This function is called from the interrupt handler when a GPU irq is to be
+ * handled.
+ */
+void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val);
+
+#endif /* _KBASE_DEVICE_INTERNAL_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/*
+ * Register-based HW access backend APIs
+ */
+#include <mali_kbase.h>
+#include <mali_kbase_hwaccess_jm.h>
+#include <mali_kbase_hwaccess_backend.h>
+#include <backend/gpu/mali_kbase_irq_internal.h>
+#include <backend/gpu/mali_kbase_jm_internal.h>
+#include <backend/gpu/mali_kbase_js_internal.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+int kbase_backend_early_init(struct kbase_device *kbdev)
+{
+ int err;
+
+ err = kbasep_platform_device_init(kbdev);
+ if (err)
+ return err;
+
+ /* Ensure we can access the GPU registers */
+ kbase_pm_register_access_enable(kbdev);
+
+ /* Find out GPU properties based on the GPU feature registers */
+ kbase_gpuprops_set(kbdev);
+
+ /* We're done accessing the GPU registers for now. */
+ kbase_pm_register_access_disable(kbdev);
+
+ err = kbase_hwaccess_pm_init(kbdev);
+ if (err)
+ goto fail_pm;
+
+ err = kbase_install_interrupts(kbdev);
+ if (err)
+ goto fail_interrupts;
+
+ return 0;
+
+fail_interrupts:
+ kbase_hwaccess_pm_term(kbdev);
+fail_pm:
+ kbasep_platform_device_term(kbdev);
+
+ return err;
+}
+
+void kbase_backend_early_term(struct kbase_device *kbdev)
+{
+ kbase_release_interrupts(kbdev);
+ kbase_hwaccess_pm_term(kbdev);
+ kbasep_platform_device_term(kbdev);
+}
+
+int kbase_backend_late_init(struct kbase_device *kbdev)
+{
+ int err;
+
+ err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT);
+ if (err)
+ return err;
+
+ err = kbase_backend_timer_init(kbdev);
+ if (err)
+ goto fail_timer;
+
+/* Currently disabled on the prototype */
+#ifdef CONFIG_MALI_DEBUG
+#ifndef CONFIG_MALI_NO_MALI
+ if (kbasep_common_test_interrupt_handlers(kbdev) != 0) {
+ dev_err(kbdev->dev, "Interrupt assigment check failed.\n");
+ err = -EINVAL;
+ goto fail_interrupt_test;
+ }
+#endif /* !CONFIG_MALI_NO_MALI */
+#endif /* CONFIG_MALI_DEBUG */
+
+ err = kbase_job_slot_init(kbdev);
+ if (err)
+ goto fail_job_slot;
+
+ init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait);
+
+ return 0;
+
+fail_job_slot:
+/* Currently disabled on the prototype */
+#ifdef CONFIG_MALI_DEBUG
+#ifndef CONFIG_MALI_NO_MALI
+fail_interrupt_test:
+#endif /* !CONFIG_MALI_NO_MALI */
+#endif /* CONFIG_MALI_DEBUG */
+ kbase_backend_timer_term(kbdev);
+fail_timer:
+ kbase_hwaccess_pm_halt(kbdev);
+
+ return err;
+}
+
+void kbase_backend_late_term(struct kbase_device *kbdev)
+{
+ kbase_job_slot_halt(kbdev);
+ kbase_job_slot_term(kbdev);
+ kbase_backend_timer_term(kbdev);
+ kbase_hwaccess_pm_halt(kbdev);
+}
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * Base kernel property query backend APIs
+ */
+
+#include <mali_kbase.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+#include <mali_kbase_hwaccess_gpuprops.h>
+
+void kbase_backend_gpuprops_get(struct kbase_device *kbdev,
+ struct kbase_gpuprops_regdump *regdump)
+{
+ int i;
+
+ /* Fill regdump with the content of the relevant registers */
+ regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID), NULL);
+
+ regdump->l2_features = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(L2_FEATURES), NULL);
+ regdump->suspend_size = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(SUSPEND_SIZE), NULL);
+ regdump->tiler_features = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(TILER_FEATURES), NULL);
+ regdump->mem_features = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(MEM_FEATURES), NULL);
+ regdump->mmu_features = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(MMU_FEATURES), NULL);
+ regdump->as_present = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(AS_PRESENT), NULL);
+ regdump->js_present = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(JS_PRESENT), NULL);
+
+ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++)
+ regdump->js_features[i] = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(JS_FEATURES_REG(i)), NULL);
+
+ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++)
+ regdump->texture_features[i] = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)), NULL);
+
+ regdump->thread_max_threads = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(THREAD_MAX_THREADS), NULL);
+ regdump->thread_max_workgroup_size = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE),
+ NULL);
+ regdump->thread_max_barrier_size = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE), NULL);
+ regdump->thread_features = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(THREAD_FEATURES), NULL);
+
+ regdump->shader_present_lo = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(SHADER_PRESENT_LO), NULL);
+ regdump->shader_present_hi = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(SHADER_PRESENT_HI), NULL);
+
+ regdump->tiler_present_lo = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(TILER_PRESENT_LO), NULL);
+ regdump->tiler_present_hi = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(TILER_PRESENT_HI), NULL);
+
+ regdump->l2_present_lo = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(L2_PRESENT_LO), NULL);
+ regdump->l2_present_hi = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(L2_PRESENT_HI), NULL);
+}
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * GPU backend instrumentation APIs.
+ */
+
+#include <mali_kbase.h>
+#include <mali_midg_regmap.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+#include <backend/gpu/mali_kbase_instr_internal.h>
+
+/**
+ * kbasep_instr_hwcnt_cacheclean - Issue Cache Clean & Invalidate command to
+ * hardware
+ *
+ * @kbdev: Kbase device
+ */
+static void kbasep_instr_hwcnt_cacheclean(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+ unsigned long pm_flags;
+ u32 irq_mask;
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ /* Wait for any reset to complete */
+ while (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_RESETTING) {
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.backend.cache_clean_wait,
+ kbdev->hwcnt.backend.state !=
+ KBASE_INSTR_STATE_RESETTING);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state ==
+ KBASE_INSTR_STATE_REQUEST_CLEAN);
+
+ /* Enable interrupt */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
+ irq_mask | CLEAN_CACHES_COMPLETED, NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
+
+ /* clean&invalidate the caches so we're sure the mmu tables for the dump
+ * buffer is valid */
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_CLEAN_INV_CACHES, NULL);
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANING;
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+}
+
+int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ struct kbase_uk_hwcnt_setup *setup)
+{
+ unsigned long flags, pm_flags;
+ int err = -EINVAL;
+ struct kbasep_js_device_data *js_devdata;
+ u32 irq_mask;
+ int ret;
+ u64 shader_cores_needed;
+
+ KBASE_DEBUG_ASSERT(NULL == kbdev->hwcnt.suspended_kctx);
+
+ shader_cores_needed = kbase_pm_get_present_cores(kbdev,
+ KBASE_PM_CORE_SHADER);
+
+ js_devdata = &kbdev->js_data;
+
+ /* alignment failure */
+ if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1)))
+ goto out_err;
+
+ /* Override core availability policy to ensure all cores are available
+ */
+ kbase_pm_ca_instr_enable(kbdev);
+
+ /* Request the cores early on synchronously - we'll release them on any
+ * errors (e.g. instrumentation already active) */
+ kbase_pm_request_cores_sync(kbdev, true, shader_cores_needed);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_RESETTING) {
+ /* GPU is being reset */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.backend.wait,
+ kbdev->hwcnt.backend.triggered != 0);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+
+ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) {
+ /* Instrumentation is already enabled */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ goto out_unrequest_cores;
+ }
+
+ /* Enable interrupt */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask |
+ PRFCNT_SAMPLE_COMPLETED, NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
+
+ /* In use, this context is the owner */
+ kbdev->hwcnt.kctx = kctx;
+ /* Remember the dump address so we can reprogram it later */
+ kbdev->hwcnt.addr = setup->dump_buffer;
+ /* Remember all the settings for suspend/resume */
+ if (&kbdev->hwcnt.suspended_state != setup)
+ memcpy(&kbdev->hwcnt.suspended_state, setup,
+ sizeof(kbdev->hwcnt.suspended_state));
+
+ /* Request the clean */
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
+ kbdev->hwcnt.backend.triggered = 0;
+ /* Clean&invalidate the caches so we're sure the mmu tables for the dump
+ * buffer is valid */
+ ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq,
+ &kbdev->hwcnt.backend.cache_clean_work);
+ KBASE_DEBUG_ASSERT(ret);
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ /* Wait for cacheclean to complete */
+ wait_event(kbdev->hwcnt.backend.wait,
+ kbdev->hwcnt.backend.triggered != 0);
+
+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state ==
+ KBASE_INSTR_STATE_IDLE);
+
+ kbase_pm_request_l2_caches(kbdev);
+
+ /* Configure */
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
+ (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT)
+ | PRFCNT_CONFIG_MODE_OFF, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
+ setup->dump_buffer & 0xFFFFFFFF, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
+ setup->dump_buffer >> 32, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN),
+ setup->jm_bm, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN),
+ setup->shader_bm, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN),
+ setup->mmu_l2_bm, kctx);
+ /* Due to PRLAM-8186 we need to disable the Tiler before we enable the
+ * HW counter dump. */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0,
+ kctx);
+ else
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
+ setup->tiler_bm, kctx);
+
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
+ (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) |
+ PRFCNT_CONFIG_MODE_MANUAL, kctx);
+
+ /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump
+ */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
+ setup->tiler_bm, kctx);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_RESETTING) {
+ /* GPU is being reset */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.backend.wait,
+ kbdev->hwcnt.backend.triggered != 0);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
+ kbdev->hwcnt.backend.triggered = 1;
+ wake_up(&kbdev->hwcnt.backend.wait);
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ err = 0;
+
+ dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx);
+ return err;
+ out_unrequest_cores:
+ kbase_pm_unrequest_cores(kbdev, true, shader_cores_needed);
+ out_err:
+ return err;
+}
+
+int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx)
+{
+ unsigned long flags, pm_flags;
+ int err = -EINVAL;
+ u32 irq_mask;
+ struct kbase_device *kbdev = kctx->kbdev;
+
+ while (1) {
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) {
+ /* Instrumentation is not enabled */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ goto out;
+ }
+
+ if (kbdev->hwcnt.kctx != kctx) {
+ /* Instrumentation has been setup for another context */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ goto out;
+ }
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE)
+ break;
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ /* Ongoing dump/setup - wait for its completion */
+ wait_event(kbdev->hwcnt.backend.wait,
+ kbdev->hwcnt.backend.triggered != 0);
+ }
+
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
+ kbdev->hwcnt.backend.triggered = 0;
+
+ /* Disable interrupt */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
+ irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
+
+ /* Disable the counters */
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx);
+
+ kbdev->hwcnt.kctx = NULL;
+ kbdev->hwcnt.addr = 0ULL;
+
+ kbase_pm_ca_instr_disable(kbdev);
+
+ kbase_pm_unrequest_cores(kbdev, true,
+ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER));
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ kbase_pm_release_l2_caches(kbdev);
+
+ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p",
+ kctx);
+
+ err = 0;
+
+ out:
+ return err;
+}
+
+int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx)
+{
+ unsigned long flags;
+ int err = -EINVAL;
+ struct kbase_device *kbdev = kctx->kbdev;
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.kctx != kctx) {
+ /* The instrumentation has been setup for another context */
+ goto unlock;
+ }
+
+ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) {
+ /* HW counters are disabled or another dump is ongoing, or we're
+ * resetting */
+ goto unlock;
+ }
+
+ kbdev->hwcnt.backend.triggered = 0;
+
+ /* Mark that we're dumping - the PF handler can signal that we faulted
+ */
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING;
+
+ /* Reconfigure the dump address */
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
+ kbdev->hwcnt.addr & 0xFFFFFFFF, NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
+ kbdev->hwcnt.addr >> 32, NULL);
+
+ /* Start dumping */
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL,
+ kbdev->hwcnt.addr, 0);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_PRFCNT_SAMPLE, kctx);
+
+ dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx);
+
+ err = 0;
+
+ unlock:
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ return err;
+}
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump);
+
+bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx,
+ bool * const success)
+{
+ unsigned long flags;
+ bool complete = false;
+ struct kbase_device *kbdev = kctx->kbdev;
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) {
+ *success = true;
+ complete = true;
+ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
+ *success = false;
+ complete = true;
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
+ }
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ return complete;
+}
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete);
+
+void kbasep_cache_clean_worker(struct work_struct *data)
+{
+ struct kbase_device *kbdev;
+ unsigned long flags;
+
+ kbdev = container_of(data, struct kbase_device,
+ hwcnt.backend.cache_clean_work);
+
+ mutex_lock(&kbdev->cacheclean_lock);
+ kbasep_instr_hwcnt_cacheclean(kbdev);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ /* Wait for our condition, and any reset to complete */
+ while (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_RESETTING ||
+ kbdev->hwcnt.backend.state ==
+ KBASE_INSTR_STATE_CLEANING) {
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.backend.cache_clean_wait,
+ (kbdev->hwcnt.backend.state !=
+ KBASE_INSTR_STATE_RESETTING &&
+ kbdev->hwcnt.backend.state !=
+ KBASE_INSTR_STATE_CLEANING));
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state ==
+ KBASE_INSTR_STATE_CLEANED);
+
+ /* All finished and idle */
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
+ kbdev->hwcnt.backend.triggered = 1;
+ wake_up(&kbdev->hwcnt.backend.wait);
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ mutex_unlock(&kbdev->cacheclean_lock);
+}
+
+void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
+ kbdev->hwcnt.backend.triggered = 1;
+ wake_up(&kbdev->hwcnt.backend.wait);
+ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) {
+ int ret;
+ /* Always clean and invalidate the cache after a successful dump
+ */
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
+ ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq,
+ &kbdev->hwcnt.backend.cache_clean_work);
+ KBASE_DEBUG_ASSERT(ret);
+ }
+ /* NOTE: In the state KBASE_INSTR_STATE_RESETTING, We're in a reset,
+ * and the instrumentation state hasn't been restored yet -
+ * kbasep_reset_timeout_worker() will do the rest of the work */
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+}
+
+void kbase_clean_caches_done(struct kbase_device *kbdev)
+{
+ u32 irq_mask;
+
+ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) {
+ unsigned long flags;
+ unsigned long pm_flags;
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ /* Disable interrupt */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
+ NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
+ irq_mask & ~CLEAN_CACHES_COMPLETED, NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
+
+ /* Wakeup... */
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) {
+ /* Only wake if we weren't resetting */
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANED;
+ wake_up(&kbdev->hwcnt.backend.cache_clean_wait);
+ }
+ /* NOTE: In the state KBASE_INSTR_STATE_RESETTING, We're in a
+ * reset, and the instrumentation state hasn't been restored yet
+ * - kbasep_reset_timeout_worker() will do the rest of the work
+ */
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ }
+}
+
+int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx)
+{
+ struct kbase_device *kbdev = kctx->kbdev;
+ unsigned long flags;
+ int err;
+
+ /* Wait for dump & cacheclean to complete */
+ wait_event(kbdev->hwcnt.backend.wait,
+ kbdev->hwcnt.backend.triggered != 0);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_RESETTING) {
+ /* GPU is being reset */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.backend.wait,
+ kbdev->hwcnt.backend.triggered != 0);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
+ err = -EINVAL;
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
+ } else {
+ /* Dump done */
+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state ==
+ KBASE_INSTR_STATE_IDLE);
+ err = 0;
+ }
+
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ return err;
+}
+
+int kbase_instr_hwcnt_clear(struct kbase_context *kctx)
+{
+ unsigned long flags;
+ int err = -EINVAL;
+ struct kbase_device *kbdev = kctx->kbdev;
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_RESETTING) {
+ /* GPU is being reset */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.backend.wait,
+ kbdev->hwcnt.backend.triggered != 0);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+
+ /* Check it's the context previously set up and we're not already
+ * dumping */
+ if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state !=
+ KBASE_INSTR_STATE_IDLE)
+ goto out;
+
+ /* Clear the counters */
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_PRFCNT_CLEAR, kctx);
+
+ err = 0;
+
+out:
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ return err;
+}
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear);
+
+int kbase_instr_backend_init(struct kbase_device *kbdev)
+{
+ int ret = 0;
+
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
+
+ init_waitqueue_head(&kbdev->hwcnt.backend.wait);
+ init_waitqueue_head(&kbdev->hwcnt.backend.cache_clean_wait);
+ INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work,
+ kbasep_cache_clean_worker);
+ kbdev->hwcnt.backend.triggered = 0;
+
+ kbdev->hwcnt.backend.cache_clean_wq =
+ alloc_workqueue("Mali cache cleaning workqueue", 0, 1);
+ if (NULL == kbdev->hwcnt.backend.cache_clean_wq)
+ ret = -EINVAL;
+
+ return ret;
+}
+
+void kbase_instr_backend_term(struct kbase_device *kbdev)
+{
+ destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq);
+}
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * Backend-specific instrumentation definitions
+ */
+
+#ifndef _KBASE_INSTR_DEFS_H_
+#define _KBASE_INSTR_DEFS_H_
+
+/*
+ * Instrumentation State Machine States
+ */
+enum kbase_instr_state {
+ /* State where instrumentation is not active */
+ KBASE_INSTR_STATE_DISABLED = 0,
+ /* State machine is active and ready for a command. */
+ KBASE_INSTR_STATE_IDLE,
+ /* Hardware is currently dumping a frame. */
+ KBASE_INSTR_STATE_DUMPING,
+ /* We've requested a clean to occur on a workqueue */
+ KBASE_INSTR_STATE_REQUEST_CLEAN,
+ /* Hardware is currently cleaning and invalidating caches. */
+ KBASE_INSTR_STATE_CLEANING,
+ /* Cache clean completed, and either a) a dump is complete, or
+ * b) instrumentation can now be setup. */
+ KBASE_INSTR_STATE_CLEANED,
+ /* kbasep_reset_timeout_worker() has started (but not compelted) a
+ * reset. This generally indicates the current action should be aborted,
+ * and kbasep_reset_timeout_worker() will handle the cleanup */
+ KBASE_INSTR_STATE_RESETTING,
+ /* An error has occured during DUMPING (page fault). */
+ KBASE_INSTR_STATE_FAULT
+};
+
+/* Structure used for instrumentation and HW counters dumping */
+struct kbase_instr_backend {
+ wait_queue_head_t wait;
+ int triggered;
+
+ enum kbase_instr_state state;
+ wait_queue_head_t cache_clean_wait;
+ struct workqueue_struct *cache_clean_wq;
+ struct work_struct cache_clean_work;
+};
+
+#endif /* _KBASE_INSTR_DEFS_H_ */
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * Backend-specific HW access instrumentation APIs
+ */
+
+#ifndef _KBASE_INSTR_INTERNAL_H_
+#define _KBASE_INSTR_INTERNAL_H_
+
+/**
+ * kbasep_cache_clean_worker() - Workqueue for handling cache cleaning
+ * @data: a &struct work_struct
+ */
+void kbasep_cache_clean_worker(struct work_struct *data);
+
+/**
+ * kbase_clean_caches_done() - Cache clean interrupt received
+ * @kbdev: Kbase device
+ */
+void kbase_clean_caches_done(struct kbase_device *kbdev);
+
+/**
+ * kbase_instr_hwcnt_sample_done() - Dump complete interrupt received
+ * @kbdev: Kbase device
+ */
+void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev);
+
+#endif /* _KBASE_INSTR_INTERNAL_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * Backend specific IRQ APIs
+ */
+
+#ifndef _KBASE_IRQ_INTERNAL_H_
+#define _KBASE_IRQ_INTERNAL_H_
+
+int kbase_install_interrupts(struct kbase_device *kbdev);
+
+void kbase_release_interrupts(struct kbase_device *kbdev);
+
+/**
+ * kbase_synchronize_irqs - Ensure that all IRQ handlers have completed
+ * execution
+ * @kbdev: The kbase device
+ */
+void kbase_synchronize_irqs(struct kbase_device *kbdev);
+
+int kbasep_common_test_interrupt_handlers(
+ struct kbase_device * const kbdev);
+
+#endif /* _KBASE_IRQ_INTERNAL_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include <mali_kbase.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
+#include <backend/gpu/mali_kbase_irq_internal.h>
+
+#include <linux/interrupt.h>
+
+#if !defined(CONFIG_MALI_NO_MALI)
+
+/* GPU IRQ Tags */
+#define JOB_IRQ_TAG 0
+#define MMU_IRQ_TAG 1
+#define GPU_IRQ_TAG 2
+
+
+static void *kbase_tag(void *ptr, u32 tag)
+{
+ return (void *)(((uintptr_t) ptr) | tag);
+}
+
+static void *kbase_untag(void *ptr)
+{
+ return (void *)(((uintptr_t) ptr) & ~3);
+}
+
+
+
+
+static irqreturn_t kbase_job_irq_handler(int irq, void *data)
+{
+ unsigned long flags;
+ struct kbase_device *kbdev = kbase_untag(data);
+ u32 val;
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!kbdev->pm.backend.gpu_powered) {
+ /* GPU is turned off - IRQ is not for us */
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
+ flags);
+ return IRQ_NONE;
+ }
+
+ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL);
+
+#ifdef CONFIG_MALI_DEBUG
+ if (!kbdev->pm.backend.driver_ready_for_irqs)
+ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n",
+ __func__, irq, val);
+#endif /* CONFIG_MALI_DEBUG */
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!val)
+ return IRQ_NONE;
+
+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
+
+ kbase_job_done(kbdev, val);
+
+ return IRQ_HANDLED;
+}
+
+KBASE_EXPORT_TEST_API(kbase_job_irq_handler);
+
+static irqreturn_t kbase_mmu_irq_handler(int irq, void *data)
+{
+ unsigned long flags;
+ struct kbase_device *kbdev = kbase_untag(data);
+ u32 val;
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!kbdev->pm.backend.gpu_powered) {
+ /* GPU is turned off - IRQ is not for us */
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
+ flags);
+ return IRQ_NONE;
+ }
+
+ atomic_inc(&kbdev->faults_pending);
+
+ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL);
+
+#ifdef CONFIG_MALI_DEBUG
+ if (!kbdev->pm.backend.driver_ready_for_irqs)
+ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n",
+ __func__, irq, val);
+#endif /* CONFIG_MALI_DEBUG */
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!val) {
+ atomic_dec(&kbdev->faults_pending);
+ return IRQ_NONE;
+ }
+
+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
+
+ kbase_mmu_interrupt(kbdev, val);
+
+ atomic_dec(&kbdev->faults_pending);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t kbase_gpu_irq_handler(int irq, void *data)
+{
+ unsigned long flags;
+ struct kbase_device *kbdev = kbase_untag(data);
+ u32 val;
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!kbdev->pm.backend.gpu_powered) {
+ /* GPU is turned off - IRQ is not for us */
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
+ flags);
+ return IRQ_NONE;
+ }
+
+ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL);
+
+#ifdef CONFIG_MALI_DEBUG
+ if (!kbdev->pm.backend.driver_ready_for_irqs)
+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n",
+ __func__, irq, val);
+#endif /* CONFIG_MALI_DEBUG */
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!val)
+ return IRQ_NONE;
+
+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
+
+ kbase_gpu_interrupt(kbdev, val);
+
+ return IRQ_HANDLED;
+}
+static irq_handler_t kbase_handler_table[] = {
+ [JOB_IRQ_TAG] = kbase_job_irq_handler,
+ [MMU_IRQ_TAG] = kbase_mmu_irq_handler,
+ [GPU_IRQ_TAG] = kbase_gpu_irq_handler,
+};
+
+
+#ifdef CONFIG_MALI_DEBUG
+#define JOB_IRQ_HANDLER JOB_IRQ_TAG
+#define MMU_IRQ_HANDLER MMU_IRQ_TAG
+#define GPU_IRQ_HANDLER GPU_IRQ_TAG
+
+/**
+ * kbase_set_custom_irq_handler - Set a custom IRQ handler
+ * @kbdev: Device for which the handler is to be registered
+ * @custom_handler: Handler to be registered
+ * @irq_type: Interrupt type
+ *
+ * Registers given interrupt handler for requested interrupt type
+ * In the case where irq handler is not specified, the default handler shall be
+ * registered
+ *
+ * Return: 0 case success, error code otherwise
+ */
+int kbase_set_custom_irq_handler(struct kbase_device *kbdev,
+ irq_handler_t custom_handler,
+ int irq_type)
+{
+ int result = 0;
+ irq_handler_t requested_irq_handler = NULL;
+
+ KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) &&
+ (GPU_IRQ_HANDLER >= irq_type));
+
+ /* Release previous handler */
+ if (kbdev->irqs[irq_type].irq)
+ free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type));
+
+ requested_irq_handler = (NULL != custom_handler) ? custom_handler :
+ kbase_handler_table[irq_type];
+
+ if (0 != request_irq(kbdev->irqs[irq_type].irq,
+ requested_irq_handler,
+ kbdev->irqs[irq_type].flags | IRQF_SHARED,
+ dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) {
+ result = -EINVAL;
+ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n",
+ kbdev->irqs[irq_type].irq, irq_type);
+#ifdef CONFIG_SPARSE_IRQ
+ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n");
+#endif /* CONFIG_SPARSE_IRQ */
+ }
+
+ return result;
+}
+
+KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler);
+
+/* test correct interrupt assigment and reception by cpu */
+struct kbasep_irq_test {
+ struct hrtimer timer;
+ wait_queue_head_t wait;
+ int triggered;
+ u32 timeout;
+};
+
+static struct kbasep_irq_test kbasep_irq_test_data;
+
+#define IRQ_TEST_TIMEOUT 500
+
+static irqreturn_t kbase_job_irq_test_handler(int irq, void *data)
+{
+ unsigned long flags;
+ struct kbase_device *kbdev = kbase_untag(data);
+ u32 val;
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!kbdev->pm.backend.gpu_powered) {
+ /* GPU is turned off - IRQ is not for us */
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
+ flags);
+ return IRQ_NONE;
+ }
+
+ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL);
+
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!val)
+ return IRQ_NONE;
+
+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
+
+ kbasep_irq_test_data.triggered = 1;
+ wake_up(&kbasep_irq_test_data.wait);
+
+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val, NULL);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data)
+{
+ unsigned long flags;
+ struct kbase_device *kbdev = kbase_untag(data);
+ u32 val;
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!kbdev->pm.backend.gpu_powered) {
+ /* GPU is turned off - IRQ is not for us */
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
+ flags);
+ return IRQ_NONE;
+ }
+
+ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL);
+
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (!val)
+ return IRQ_NONE;
+
+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
+
+ kbasep_irq_test_data.triggered = 1;
+ wake_up(&kbasep_irq_test_data.wait);
+
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val, NULL);
+
+ return IRQ_HANDLED;
+}
+
+static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer)
+{
+ struct kbasep_irq_test *test_data = container_of(timer,
+ struct kbasep_irq_test, timer);
+
+ test_data->timeout = 1;
+ test_data->triggered = 1;
+ wake_up(&test_data->wait);
+ return HRTIMER_NORESTART;
+}
+
+static int kbasep_common_test_interrupt(
+ struct kbase_device * const kbdev, u32 tag)
+{
+ int err = 0;
+ irq_handler_t test_handler;
+
+ u32 old_mask_val;
+ u16 mask_offset;
+ u16 rawstat_offset;
+
+ switch (tag) {
+ case JOB_IRQ_TAG:
+ test_handler = kbase_job_irq_test_handler;
+ rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT);
+ mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK);
+ break;
+ case MMU_IRQ_TAG:
+ test_handler = kbase_mmu_irq_test_handler;
+ rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT);
+ mask_offset = MMU_REG(MMU_IRQ_MASK);
+ break;
+ case GPU_IRQ_TAG:
+ /* already tested by pm_driver - bail out */
+ default:
+ return 0;
+ }
+
+ /* store old mask */
+ old_mask_val = kbase_reg_read(kbdev, mask_offset, NULL);
+ /* mask interrupts */
+ kbase_reg_write(kbdev, mask_offset, 0x0, NULL);
+
+ if (kbdev->irqs[tag].irq) {
+ /* release original handler and install test handler */
+ if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) {
+ err = -EINVAL;
+ } else {
+ kbasep_irq_test_data.timeout = 0;
+ hrtimer_init(&kbasep_irq_test_data.timer,
+ CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ kbasep_irq_test_data.timer.function =
+ kbasep_test_interrupt_timeout;
+
+ /* trigger interrupt */
+ kbase_reg_write(kbdev, mask_offset, 0x1, NULL);
+ kbase_reg_write(kbdev, rawstat_offset, 0x1, NULL);
+
+ hrtimer_start(&kbasep_irq_test_data.timer,
+ HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT),
+ HRTIMER_MODE_REL);
+
+ wait_event(kbasep_irq_test_data.wait,
+ kbasep_irq_test_data.triggered != 0);
+
+ if (kbasep_irq_test_data.timeout != 0) {
+ dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n",
+ kbdev->irqs[tag].irq, tag);
+ err = -EINVAL;
+ } else {
+ dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n",
+ kbdev->irqs[tag].irq, tag);
+ }
+
+ hrtimer_cancel(&kbasep_irq_test_data.timer);
+ kbasep_irq_test_data.triggered = 0;
+
+ /* mask interrupts */
+ kbase_reg_write(kbdev, mask_offset, 0x0, NULL);
+
+ /* release test handler */
+ free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag));
+ }
+
+ /* restore original interrupt */
+ if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag],
+ kbdev->irqs[tag].flags | IRQF_SHARED,
+ dev_name(kbdev->dev), kbase_tag(kbdev, tag))) {
+ dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n",
+ kbdev->irqs[tag].irq, tag);
+ err = -EINVAL;
+ }
+ }
+ /* restore old mask */
+ kbase_reg_write(kbdev, mask_offset, old_mask_val, NULL);
+
+ return err;
+}
+
+int kbasep_common_test_interrupt_handlers(
+ struct kbase_device * const kbdev)
+{
+ int err;
+
+ init_waitqueue_head(&kbasep_irq_test_data.wait);
+ kbasep_irq_test_data.triggered = 0;
+
+ /* A suspend won't happen during startup/insmod */
+ kbase_pm_context_active(kbdev);
+
+ err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG);
+ if (err) {
+ dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n");
+ goto out;
+ }
+
+ err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG);
+ if (err) {
+ dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n");
+ goto out;
+ }
+
+ dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n");
+
+ out:
+ kbase_pm_context_idle(kbdev);
+
+ return err;
+}
+#endif /* CONFIG_MALI_DEBUG */
+
+int kbase_install_interrupts(struct kbase_device *kbdev)
+{
+ u32 nr = ARRAY_SIZE(kbase_handler_table);
+ int err;
+ u32 i;
+
+ for (i = 0; i < nr; i++) {
+ err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i],
+ kbdev->irqs[i].flags | IRQF_SHARED,
+ dev_name(kbdev->dev),
+ kbase_tag(kbdev, i));
+ if (err) {
+ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n",
+ kbdev->irqs[i].irq, i);
+#ifdef CONFIG_SPARSE_IRQ
+ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n");
+#endif /* CONFIG_SPARSE_IRQ */
+ goto release;
+ }
+ }
+
+ return 0;
+
+ release:
+ while (i-- > 0)
+ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i));
+
+ return err;
+}
+
+void kbase_release_interrupts(struct kbase_device *kbdev)
+{
+ u32 nr = ARRAY_SIZE(kbase_handler_table);
+ u32 i;
+
+ for (i = 0; i < nr; i++) {
+ if (kbdev->irqs[i].irq)
+ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i));
+ }
+}
+
+void kbase_synchronize_irqs(struct kbase_device *kbdev)
+{
+ u32 nr = ARRAY_SIZE(kbase_handler_table);
+ u32 i;
+
+ for (i = 0; i < nr; i++) {
+ if (kbdev->irqs[i].irq)
+ synchronize_irq(kbdev->irqs[i].irq);
+ }
+}
+
+#endif /* !defined(CONFIG_MALI_NO_MALI) */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/*
+ * Register backend context / address space management
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_hwaccess_jm.h>
+
+/**
+ * assign_and_activate_kctx_addr_space - Assign an AS to a context
+ * @kbdev: Kbase device
+ * @kctx: Kbase context
+ * @current_as: Address Space to assign
+ *
+ * Assign an Address Space (AS) to a context, and add the context to the Policy.
+ *
+ * This includes
+ * setting up the global runpool_irq structure and the context on the AS,
+ * Activating the MMU on the AS,
+ * Allowing jobs to be submitted on the AS.
+ *
+ * Context:
+ * kbasep_js_kctx_info.jsctx_mutex held,
+ * kbasep_js_device_data.runpool_mutex held,
+ * AS transaction mutex held,
+ * Runpool IRQ lock held
+ */
+static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ struct kbase_as *current_as)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ struct kbasep_js_per_as_data *js_per_as_data;
+ int as_nr = current_as->number;
+
+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
+ lockdep_assert_held(&js_devdata->runpool_mutex);
+ lockdep_assert_held(¤t_as->transaction_mutex);
+ lockdep_assert_held(&js_devdata->runpool_irq.lock);
+
+ js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
+
+ /* Attribute handling */
+ kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx);
+
+ /* Assign addr space */
+ kctx->as_nr = as_nr;
+
+ /* If the GPU is currently powered, activate this address space on the
+ * MMU */
+ if (kbdev->pm.backend.gpu_powered)
+ kbase_mmu_update(kctx);
+ /* If the GPU was not powered then the MMU will be reprogrammed on the
+ * next pm_context_active() */
+
+ /* Allow it to run jobs */
+ kbasep_js_set_submit_allowed(js_devdata, kctx);
+
+ /* Book-keeping */
+ js_per_as_data->kctx = kctx;
+ js_per_as_data->as_busy_refcount = 0;
+
+ kbase_js_runpool_inc_context_count(kbdev, kctx);
+}
+
+/**
+ * release_addr_space - Release an address space
+ * @kbdev: Kbase device
+ * @kctx_as_nr: Address space of context to release
+ * @kctx: Context being released
+ *
+ * Context: kbasep_js_device_data.runpool_mutex must be held
+ *
+ * Release an address space, making it available for being picked again.
+ */
+static void release_addr_space(struct kbase_device *kbdev, int kctx_as_nr,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_device_data *js_devdata;
+ u16 as_bit = (1u << kctx_as_nr);
+
+ js_devdata = &kbdev->js_data;
+ lockdep_assert_held(&js_devdata->runpool_mutex);
+
+ /* The address space must not already be free */
+ KBASE_DEBUG_ASSERT(!(js_devdata->as_free & as_bit));
+
+ js_devdata->as_free |= as_bit;
+
+ kbase_js_runpool_dec_context_count(kbdev, kctx);
+}
+
+bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ int i;
+
+ if (kbdev->hwaccess.active_kctx == kctx) {
+ /* Context is already active */
+ return true;
+ }
+
+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
+ struct kbasep_js_per_as_data *js_per_as_data =
+ &kbdev->js_data.runpool_irq.per_as_data[i];
+
+ if (js_per_as_data->kctx == kctx) {
+ /* Context already has ASID - mark as active */
+ return true;
+ }
+ }
+
+ /* Context does not have address space assigned */
+ return false;
+}
+
+void kbase_backend_release_ctx_irq(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_per_as_data *js_per_as_data;
+ int as_nr = kctx->as_nr;
+
+ if (as_nr == KBASEP_AS_NR_INVALID) {
+ WARN(1, "Attempting to release context without ASID\n");
+ return;
+ }
+
+ lockdep_assert_held(&kbdev->as[as_nr].transaction_mutex);
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ js_per_as_data = &kbdev->js_data.runpool_irq.per_as_data[kctx->as_nr];
+ if (js_per_as_data->as_busy_refcount != 0) {
+ WARN(1, "Attempting to release active ASID\n");
+ return;
+ }
+
+ /* Release context from address space */
+ js_per_as_data->kctx = NULL;
+
+ kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx);
+ /* If the GPU is currently powered, de-activate this address space on
+ * the MMU */
+ if (kbdev->pm.backend.gpu_powered)
+ kbase_mmu_disable(kctx);
+ /* If the GPU was not powered then the MMU will be reprogrammed on the
+ * next pm_context_active() */
+
+ release_addr_space(kbdev, as_nr, kctx);
+ kctx->as_nr = KBASEP_AS_NR_INVALID;
+}
+
+void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+}
+
+void kbase_backend_release_free_address_space(struct kbase_device *kbdev,
+ int as_nr)
+{
+ struct kbasep_js_device_data *js_devdata;
+
+ js_devdata = &kbdev->js_data;
+
+ lockdep_assert_held(&js_devdata->runpool_mutex);
+
+ js_devdata->as_free |= (1 << as_nr);
+}
+
+/**
+ * check_is_runpool_full - check whether the runpool is full for a specified
+ * context
+ * @kbdev: Kbase device
+ * @kctx: Kbase context
+ *
+ * If kctx == NULL, then this makes the least restrictive check on the
+ * runpool. A specific context that is supplied immediately after could fail
+ * the check, even under the same conditions.
+ *
+ * Therefore, once a context is obtained you \b must re-check it with this
+ * function, since the return value could change to false.
+ *
+ * Context:
+ * In all cases, the caller must hold kbasep_js_device_data.runpool_mutex.
+ * When kctx != NULL the caller must hold the
+ * kbasep_js_kctx_info.ctx.jsctx_mutex.
+ * When kctx == NULL, then the caller need not hold any jsctx_mutex locks (but
+ * it doesn't do any harm to do so).
+ *
+ * Return: true if the runpool is full
+ */
+static bool check_is_runpool_full(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_device_data *js_devdata;
+ bool is_runpool_full;
+
+ js_devdata = &kbdev->js_data;
+ lockdep_assert_held(&js_devdata->runpool_mutex);
+
+ /* Regardless of whether a context is submitting or not, can't have more
+ * than there are HW address spaces */
+ is_runpool_full = (bool) (js_devdata->nr_all_contexts_running >=
+ kbdev->nr_hw_address_spaces);
+
+ if (kctx != NULL && (kctx->jctx.sched_info.ctx.flags &
+ KBASE_CTX_FLAG_SUBMIT_DISABLED) == 0) {
+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
+ /* Contexts that submit might use less of the address spaces
+ * available, due to HW workarounds. In which case, the runpool
+ * is also full when the number of submitting contexts exceeds
+ * the number of submittable address spaces.
+ *
+ * Both checks must be made: can have nr_user_address_spaces ==
+ * nr_hw_address spaces, and at the same time can have
+ * nr_user_contexts_running < nr_all_contexts_running. */
+ is_runpool_full |= (bool)
+ (js_devdata->nr_user_contexts_running >=
+ kbdev->nr_user_address_spaces);
+ }
+
+ return is_runpool_full;
+}
+
+int kbase_backend_find_free_address_space(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_device_data *js_devdata;
+ struct kbasep_js_kctx_info *js_kctx_info;
+ unsigned long flags;
+ int i;
+
+ js_devdata = &kbdev->js_data;
+ js_kctx_info = &kctx->jctx.sched_info;
+
+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_lock(&js_devdata->runpool_mutex);
+
+ /* First try to find a free address space */
+ if (check_is_runpool_full(kbdev, kctx))
+ i = -1;
+ else
+ i = ffs(js_devdata->as_free) - 1;
+
+ if (i >= 0 && i < kbdev->nr_hw_address_spaces) {
+ js_devdata->as_free &= ~(1 << i);
+
+ mutex_unlock(&js_devdata->runpool_mutex);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+
+ return i;
+ }
+
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ /* No address space currently free, see if we can release one */
+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
+ struct kbasep_js_per_as_data *js_per_as_data;
+ struct kbasep_js_kctx_info *as_js_kctx_info;
+ struct kbase_context *as_kctx;
+
+ js_per_as_data = &kbdev->js_data.runpool_irq.per_as_data[i];
+ as_kctx = js_per_as_data->kctx;
+ as_js_kctx_info = &as_kctx->jctx.sched_info;
+
+ /* Don't release privileged or active contexts, or contexts with
+ * jobs running */
+ if (as_kctx && !(as_kctx->jctx.sched_info.ctx.flags &
+ KBASE_CTX_FLAG_PRIVILEGED) &&
+ js_per_as_data->as_busy_refcount == 0) {
+ if (!kbasep_js_runpool_retain_ctx_nolock(kbdev,
+ as_kctx)) {
+ WARN(1, "Failed to retain active context\n");
+
+ spin_unlock_irqrestore(
+ &js_devdata->runpool_irq.lock,
+ flags);
+ mutex_unlock(&js_devdata->runpool_mutex);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+
+ return KBASEP_AS_NR_INVALID;
+ }
+
+ kbasep_js_clear_submit_allowed(js_devdata, as_kctx);
+
+ /* Drop and retake locks to take the jsctx_mutex on the
+ * context we're about to release without violating lock
+ * ordering
+ */
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
+ flags);
+ mutex_unlock(&js_devdata->runpool_mutex);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+
+
+ /* Release context from address space */
+ mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex);
+ mutex_lock(&js_devdata->runpool_mutex);
+
+ kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx);
+
+ if (!as_js_kctx_info->ctx.is_scheduled) {
+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev,
+ as_kctx,
+ true);
+
+ js_devdata->as_free &= ~(1 << i);
+
+ mutex_unlock(&js_devdata->runpool_mutex);
+ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex);
+
+ return i;
+ }
+
+ /* Context was retained while locks were dropped,
+ * continue looking for free AS */
+
+ mutex_unlock(&js_devdata->runpool_mutex);
+ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex);
+
+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_lock(&js_devdata->runpool_mutex);
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ }
+ }
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ mutex_unlock(&js_devdata->runpool_mutex);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+
+ return KBASEP_AS_NR_INVALID;
+}
+
+bool kbase_backend_use_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ int as_nr)
+{
+ struct kbasep_js_device_data *js_devdata;
+ struct kbasep_js_kctx_info *js_kctx_info;
+ struct kbase_as *new_address_space = NULL;
+
+ js_devdata = &kbdev->js_data;
+ js_kctx_info = &kctx->jctx.sched_info;
+
+ if (kbdev->hwaccess.active_kctx == kctx ||
+ kctx->as_nr != KBASEP_AS_NR_INVALID ||
+ as_nr == KBASEP_AS_NR_INVALID) {
+ WARN(1, "Invalid parameters to use_ctx()\n");
+ return false;
+ }
+
+ new_address_space = &kbdev->as[as_nr];
+
+ lockdep_assert_held(&js_devdata->runpool_mutex);
+ lockdep_assert_held(&new_address_space->transaction_mutex);
+ lockdep_assert_held(&js_devdata->runpool_irq.lock);
+
+ assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space);
+
+ if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_PRIVILEGED) != 0) {
+ /* We need to retain it to keep the corresponding address space
+ */
+ kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
+ }
+
+ return true;
+}
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/*
+ * Register-based HW access backend specific definitions
+ */
+
+#ifndef _KBASE_HWACCESS_GPU_DEFS_H_
+#define _KBASE_HWACCESS_GPU_DEFS_H_
+
+/* SLOT_RB_SIZE must be < 256 */
+#define SLOT_RB_SIZE 2
+#define SLOT_RB_MASK (SLOT_RB_SIZE - 1)
+
+/**
+ * struct rb_entry - Ringbuffer entry
+ * @katom: Atom associated with this entry
+ */
+struct rb_entry {
+ struct kbase_jd_atom *katom;
+};
+
+/**
+ * struct slot_rb - Slot ringbuffer
+ * @entries: Ringbuffer entries
+ * @last_context: The last context to submit a job on this slot
+ * @read_idx: Current read index of buffer
+ * @write_idx: Current write index of buffer
+ * @job_chain_flag: Flag used to implement jobchain disambiguation
+ */
+struct slot_rb {
+ struct rb_entry entries[SLOT_RB_SIZE];
+
+ struct kbase_context *last_context;
+
+ u8 read_idx;
+ u8 write_idx;
+
+ u8 job_chain_flag;
+};
+
+/**
+ * struct kbase_backend_data - GPU backend specific data for HW access layer
+ * @slot_rb: Slot ringbuffers
+ * @rmu_workaround_flag: When PRLAM-8987 is present, this flag determines
+ * whether slots 0/1 or slot 2 are currently being
+ * pulled from
+ * @scheduling_timer: The timer tick used for rescheduling jobs
+ * @timer_running: Is the timer running? The runpool_mutex must be
+ * held whilst modifying this.
+ *
+ * The kbasep_js_device_data::runpool_irq::lock (a spinlock) must be held when
+ * accessing this structure
+ */
+struct kbase_backend_data {
+ struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS];
+
+ bool rmu_workaround_flag;
+
+ struct hrtimer scheduling_timer;
+
+ bool timer_running;
+
+ /* Set when we're about to reset the GPU */
+ atomic_t reset_gpu;
+
+/* The GPU reset isn't pending */
+#define KBASE_RESET_GPU_NOT_PENDING 0
+/* kbase_prepare_to_reset_gpu has been called */
+#define KBASE_RESET_GPU_PREPARED 1
+/* kbase_reset_gpu has been called - the reset will now definitely happen
+ * within the timeout period */
+#define KBASE_RESET_GPU_COMMITTED 2
+/* The GPU reset process is currently occuring (timeout has expired or
+ * kbasep_try_reset_gpu_early was called) */
+#define KBASE_RESET_GPU_HAPPENING 3
+
+ /* Work queue and work item for performing the reset in */
+ struct workqueue_struct *reset_workq;
+ struct work_struct reset_work;
+ wait_queue_head_t reset_wait;
+ struct hrtimer reset_timer;
+};
+
+/**
+ * struct kbase_jd_atom_backend - GPU backend specific katom data
+ */
+struct kbase_jd_atom_backend {
+};
+
+/**
+ * struct kbase_context_backend - GPU backend specific context data
+ */
+struct kbase_context_backend {
+};
+
+#endif /* _KBASE_HWACCESS_GPU_DEFS_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * Base kernel job manager APIs
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_config.h>
+#include <mali_midg_regmap.h>
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
+#include <mali_kbase_gator.h>
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+#include <mali_kbase_tlstream.h>
+#endif
+#include <mali_kbase_hw.h>
+#include <mali_kbase_config_defaults.h>
+#include <mali_kbase_hwaccess_jm.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
+#include <backend/gpu/mali_kbase_irq_internal.h>
+#include <backend/gpu/mali_kbase_js_affinity.h>
+#include <backend/gpu/mali_kbase_jm_internal.h>
+
+#define beenthere(kctx, f, a...) \
+ dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
+
+#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
+u64 mali_js0_affinity_mask = 0xFFFFFFFFFFFFFFFFULL;
+u64 mali_js1_affinity_mask = 0xFFFFFFFFFFFFFFFFULL;
+u64 mali_js2_affinity_mask = 0xFFFFFFFFFFFFFFFFULL;
+#endif
+
+#if KBASE_GPU_RESET_EN
+static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev);
+static void kbasep_reset_timeout_worker(struct work_struct *data);
+static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer);
+#endif /* KBASE_GPU_RESET_EN */
+
+static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js,
+ struct kbase_context *kctx)
+{
+ return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx);
+}
+
+void kbase_job_hw_submit(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom,
+ int js)
+{
+ struct kbase_context *kctx;
+ u32 cfg;
+ u64 jc_head = katom->jc;
+
+ KBASE_DEBUG_ASSERT(kbdev);
+ KBASE_DEBUG_ASSERT(katom);
+
+ kctx = katom->kctx;
+
+ /* Command register must be available */
+ KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx));
+ /* Affinity is not violating */
+ kbase_js_debug_log_current_affinities(kbdev);
+ KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js,
+ katom->affinity));
+
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO),
+ jc_head & 0xFFFFFFFF, kctx);
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI),
+ jc_head >> 32, kctx);
+
+#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
+ {
+ u64 mask;
+ u32 value;
+
+ if (0 == js)
+ mask = mali_js0_affinity_mask;
+ else if (1 == js)
+ mask = mali_js1_affinity_mask;
+ else
+ mask = mali_js2_affinity_mask;
+
+ value = katom->affinity & (mask & 0xFFFFFFFF);
+
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO),
+ value, kctx);
+
+ value = (katom->affinity >> 32) & ((mask>>32) & 0xFFFFFFFF);
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI),
+ value, kctx);
+ }
+#else
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO),
+ katom->affinity & 0xFFFFFFFF, kctx);
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI),
+ katom->affinity >> 32, kctx);
+#endif
+
+ /* start MMU, medium priority, cache clean/flush on end, clean/flush on
+ * start */
+ cfg = kctx->as_nr;
+
+ cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE;
+ cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE;
+
+ cfg |= JS_CONFIG_START_MMU;
+ cfg |= JS_CONFIG_THREAD_PRI(8);
+
+ if (kbase_hw_has_feature(kbdev,
+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
+ if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) {
+ cfg |= JS_CONFIG_JOB_CHAIN_FLAG;
+ katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
+ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
+ true;
+ } else {
+ katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
+ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
+ false;
+ }
+ }
+
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx);
+
+
+ /* Write an approximate start timestamp.
+ * It's approximate because there might be a job in the HEAD register.
+ * In such cases, we'll try to make a better approximation in the IRQ
+ * handler (up to the KBASE_JS_IRQ_THROTTLE_TIME_US). */
+ katom->start_timestamp = ktime_get();
+
+ /* GO ! */
+ dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx",
+ katom, kctx, js, jc_head, katom->affinity);
+
+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js,
+ (u32) katom->affinity);
+
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
+ kbase_trace_mali_job_slots_event(
+ GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js),
+ kctx, kbase_jd_atom_id(kctx, katom));
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_tl_ret_atom_lpu(
+ katom,
+ &kbdev->gpu_props.props.raw_props.js_features[js]);
+#endif
+#ifdef CONFIG_GPU_TRACEPOINTS
+ if (kbase_backend_nr_atoms_submitted(kbdev, js) == 1) {
+ /* If this is the only job on the slot, trace it as starting */
+ char js_string[16];
+
+ trace_gpu_sched_switch(
+ kbasep_make_job_slot_string(js, js_string),
+ ktime_to_ns(katom->start_timestamp),
+ (u32)katom->kctx, 0, katom->work_id);
+ kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx;
+ }
+#endif
+ kbase_timeline_job_slot_submit(kbdev, kctx, katom, js);
+
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
+ JS_COMMAND_START, katom->kctx);
+}
+
+/**
+ * kbasep_job_slot_update_head_start_timestamp - Update timestamp
+ * @kbdev: kbase device
+ * @js: job slot
+ * @end_timestamp: timestamp
+ *
+ * Update the start_timestamp of the job currently in the HEAD, based on the
+ * fact that we got an IRQ for the previous set of completed jobs.
+ *
+ * The estimate also takes into account the %KBASE_JS_IRQ_THROTTLE_TIME_US and
+ * the time the job was submitted, to work out the best estimate (which might
+ * still result in an over-estimate to the calculated time spent)
+ */
+static void kbasep_job_slot_update_head_start_timestamp(
+ struct kbase_device *kbdev,
+ int js,
+ ktime_t end_timestamp)
+{
+ if (kbase_backend_nr_atoms_on_slot(kbdev, js) > 0) {
+ struct kbase_jd_atom *katom;
+ ktime_t new_timestamp;
+ ktime_t timestamp_diff;
+ /* The atom in the HEAD */
+ katom = kbase_gpu_inspect(kbdev, js, 0);
+
+ KBASE_DEBUG_ASSERT(katom != NULL);
+
+ /* Account for any IRQ Throttle time - makes an overestimate of
+ * the time spent by the job */
+ new_timestamp = ktime_sub_ns(end_timestamp,
+ KBASE_JS_IRQ_THROTTLE_TIME_US * 1000);
+ timestamp_diff = ktime_sub(new_timestamp,
+ katom->start_timestamp);
+ if (ktime_to_ns(timestamp_diff) >= 0) {
+ /* Only update the timestamp if it's a better estimate
+ * than what's currently stored. This is because our
+ * estimate that accounts for the throttle time may be
+ * too much of an overestimate */
+ katom->start_timestamp = new_timestamp;
+ }
+ }
+}
+
+void kbase_job_done(struct kbase_device *kbdev, u32 done)
+{
+ unsigned long flags;
+ int i;
+ u32 count = 0;
+ ktime_t end_timestamp = ktime_get();
+ struct kbasep_js_device_data *js_devdata;
+
+ KBASE_DEBUG_ASSERT(kbdev);
+ js_devdata = &kbdev->js_data;
+
+ KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done);
+
+ memset(&kbdev->slot_submit_count_irq[0], 0,
+ sizeof(kbdev->slot_submit_count_irq));
+
+ /* write irq throttle register, this will prevent irqs from occurring
+ * until the given number of gpu clock cycles have passed */
+ {
+ int irq_throttle_cycles =
+ atomic_read(&kbdev->irq_throttle_cycles);
+
+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE),
+ irq_throttle_cycles, NULL);
+ }
+
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ while (done) {
+ u32 failed = done >> 16;
+
+ /* treat failed slots as finished slots */
+ u32 finished = (done & 0xFFFF) | failed;
+
+ /* Note: This is inherently unfair, as we always check
+ * for lower numbered interrupts before the higher
+ * numbered ones.*/
+ i = ffs(finished) - 1;
+ KBASE_DEBUG_ASSERT(i >= 0);
+
+ do {
+ int nr_done;
+ u32 active;
+ u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */
+ u64 job_tail = 0;
+
+ if (failed & (1u << i)) {
+ /* read out the job slot status code if the job
+ * slot reported failure */
+ completion_code = kbase_reg_read(kbdev,
+ JOB_SLOT_REG(i, JS_STATUS), NULL);
+
+ switch (completion_code) {
+ case BASE_JD_EVENT_STOPPED:
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
+ kbase_trace_mali_job_slots_event(
+ GATOR_MAKE_EVENT(
+ GATOR_JOB_SLOT_SOFT_STOPPED, i),
+ NULL, 0);
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_aux_job_softstop(i);
+#endif
+ /* Soft-stopped job - read the value of
+ * JS<n>_TAIL so that the job chain can
+ * be resumed */
+ job_tail = (u64)kbase_reg_read(kbdev,
+ JOB_SLOT_REG(i, JS_TAIL_LO),
+ NULL) |
+ ((u64)kbase_reg_read(kbdev,
+ JOB_SLOT_REG(i, JS_TAIL_HI),
+ NULL) << 32);
+ break;
+ case BASE_JD_EVENT_NOT_STARTED:
+ /* PRLAM-10673 can cause a TERMINATED
+ * job to come back as NOT_STARTED, but
+ * the error interrupt helps us detect
+ * it */
+ completion_code =
+ BASE_JD_EVENT_TERMINATED;
+ /* fall through */
+ default:
+ dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)",
+ i, completion_code,
+ kbase_exception_name
+ (kbdev,
+ completion_code));
+ }
+
+ kbase_gpu_irq_evict(kbdev, i);
+ }
+
+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR),
+ done & ((1 << i) | (1 << (i + 16))),
+ NULL);
+ active = kbase_reg_read(kbdev,
+ JOB_CONTROL_REG(JOB_IRQ_JS_STATE),
+ NULL);
+
+ if (((active >> i) & 1) == 0 &&
+ (((done >> (i + 16)) & 1) == 0)) {
+ /* There is a potential race we must work
+ * around:
+ *
+ * 1. A job slot has a job in both current and
+ * next registers
+ * 2. The job in current completes
+ * successfully, the IRQ handler reads
+ * RAWSTAT and calls this function with the
+ * relevant bit set in "done"
+ * 3. The job in the next registers becomes the
+ * current job on the GPU
+ * 4. Sometime before the JOB_IRQ_CLEAR line
+ * above the job on the GPU _fails_
+ * 5. The IRQ_CLEAR clears the done bit but not
+ * the failed bit. This atomically sets
+ * JOB_IRQ_JS_STATE. However since both jobs
+ * have now completed the relevant bits for
+ * the slot are set to 0.
+ *
+ * If we now did nothing then we'd incorrectly
+ * assume that _both_ jobs had completed
+ * successfully (since we haven't yet observed
+ * the fail bit being set in RAWSTAT).
+ *
+ * So at this point if there are no active jobs
+ * left we check to see if RAWSTAT has a failure
+ * bit set for the job slot. If it does we know
+ * that there has been a new failure that we
+ * didn't previously know about, so we make sure
+ * that we record this in active (but we wait
+ * for the next loop to deal with it).
+ *
+ * If we were handling a job failure (i.e. done
+ * has the relevant high bit set) then we know
+ * that the value read back from
+ * JOB_IRQ_JS_STATE is the correct number of
+ * remaining jobs because the failed job will
+ * have prevented any futher jobs from starting
+ * execution.
+ */
+ u32 rawstat = kbase_reg_read(kbdev,
+ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
+
+ if ((rawstat >> (i + 16)) & 1) {
+ /* There is a failed job that we've
+ * missed - add it back to active */
+ active |= (1u << i);
+ }
+ }
+
+ dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n",
+ completion_code);
+
+ nr_done = kbase_backend_nr_atoms_submitted(kbdev, i);
+ nr_done -= (active >> i) & 1;
+ nr_done -= (active >> (i + 16)) & 1;
+
+ if (nr_done <= 0) {
+ dev_warn(kbdev->dev, "Spurious interrupt on slot %d",
+ i);
+
+ goto spurious;
+ }
+
+ count += nr_done;
+
+ while (nr_done) {
+ if (nr_done == 1) {
+ kbase_gpu_complete_hw(kbdev, i,
+ completion_code,
+ job_tail,
+ &end_timestamp);
+ kbase_jm_try_kick_all(kbdev);
+ } else {
+ /* More than one job has completed.
+ * Since this is not the last job being
+ * reported this time it must have
+ * passed. This is because the hardware
+ * will not allow further jobs in a job
+ * slot to complete until the failed job
+ * is cleared from the IRQ status.
+ */
+ kbase_gpu_complete_hw(kbdev, i,
+ BASE_JD_EVENT_DONE,
+ 0,
+ &end_timestamp);
+ }
+ nr_done--;
+ }
+ spurious:
+ done = kbase_reg_read(kbdev,
+ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) {
+ /* Workaround for missing interrupt caused by
+ * PRLAM-10883 */
+ if (((active >> i) & 1) && (0 ==
+ kbase_reg_read(kbdev,
+ JOB_SLOT_REG(i,
+ JS_STATUS), NULL))) {
+ /* Force job slot to be processed again
+ */
+ done |= (1u << i);
+ }
+ }
+
+ failed = done >> 16;
+ finished = (done & 0xFFFF) | failed;
+ } while (finished & (1 << i));
+
+ kbasep_job_slot_update_head_start_timestamp(kbdev, i,
+ end_timestamp);
+ }
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+#if KBASE_GPU_RESET_EN
+ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
+ KBASE_RESET_GPU_COMMITTED) {
+ /* If we're trying to reset the GPU then we might be able to do
+ * it early (without waiting for a timeout) because some jobs
+ * have completed
+ */
+ kbasep_try_reset_gpu_early(kbdev);
+ }
+#endif /* KBASE_GPU_RESET_EN */
+ KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
+}
+KBASE_EXPORT_TEST_API(kbase_job_done);
+
+static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev,
+ u16 core_reqs)
+{
+ bool soft_stops_allowed = true;
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) {
+ if ((core_reqs & BASE_JD_REQ_T) != 0)
+ soft_stops_allowed = false;
+ }
+ return soft_stops_allowed;
+}
+
+static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev,
+ u16 core_reqs)
+{
+ bool hard_stops_allowed = true;
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) {
+ if ((core_reqs & BASE_JD_REQ_T) != 0)
+ hard_stops_allowed = false;
+ }
+ return hard_stops_allowed;
+}
+
+void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
+ int js,
+ u32 action,
+ u16 core_reqs,
+ struct kbase_jd_atom *target_katom)
+{
+ struct kbase_context *kctx = target_katom->kctx;
+#if KBASE_TRACE_ENABLE
+ u32 status_reg_before;
+ u64 job_in_head_before;
+ u32 status_reg_after;
+
+ KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK)));
+
+ /* Check the head pointer */
+ job_in_head_before = ((u64) kbase_reg_read(kbdev,
+ JOB_SLOT_REG(js, JS_HEAD_LO), NULL))
+ | (((u64) kbase_reg_read(kbdev,
+ JOB_SLOT_REG(js, JS_HEAD_HI), NULL))
+ << 32);
+ status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS),
+ NULL);
+#endif
+
+ if (action == JS_COMMAND_SOFT_STOP) {
+ bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev,
+ core_reqs);
+
+ if (!soft_stop_allowed) {
+#ifdef CONFIG_MALI_DEBUG
+ dev_dbg(kbdev->dev, "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X",
+ (unsigned int)core_reqs);
+#endif /* CONFIG_MALI_DEBUG */
+ return;
+ }
+
+ /* We are about to issue a soft stop, so mark the atom as having
+ * been soft stopped */
+ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED;
+ }
+
+ if (action == JS_COMMAND_HARD_STOP) {
+ bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev,
+ core_reqs);
+
+ if (!hard_stop_allowed) {
+ /* Jobs can be hard-stopped for the following reasons:
+ * * CFS decides the job has been running too long (and
+ * soft-stop has not occurred). In this case the GPU
+ * will be reset by CFS if the job remains on the
+ * GPU.
+ *
+ * * The context is destroyed, kbase_jd_zap_context
+ * will attempt to hard-stop the job. However it also
+ * has a watchdog which will cause the GPU to be
+ * reset if the job remains on the GPU.
+ *
+ * * An (unhandled) MMU fault occurred. As long as
+ * BASE_HW_ISSUE_8245 is defined then the GPU will be
+ * reset.
+ *
+ * All three cases result in the GPU being reset if the
+ * hard-stop fails, so it is safe to just return and
+ * ignore the hard-stop request.
+ */
+ dev_warn(kbdev->dev, "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X",
+ (unsigned int)core_reqs);
+ return;
+ }
+ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED;
+ }
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316) &&
+ action == JS_COMMAND_SOFT_STOP) {
+ int i;
+
+ for (i = 0; i < kbase_backend_nr_atoms_submitted(kbdev, js);
+ i++) {
+ struct kbase_jd_atom *katom;
+
+ katom = kbase_gpu_inspect(kbdev, js, i);
+
+ KBASE_DEBUG_ASSERT(katom);
+
+ /* For HW_ISSUE_8316, only 'bad' jobs attacking the
+ * system can cause this issue: normally, all memory
+ * should be allocated in multiples of 4 pages, and
+ * growable memory should be changed size in multiples
+ * of 4 pages.
+ *
+ * Whilst such 'bad' jobs can be cleared by a GPU reset,
+ * the locking up of a uTLB entry caused by the bad job
+ * could also stall other ASs, meaning that other ASs'
+ * jobs don't complete in the 'grace' period before the
+ * reset. We don't want to lose other ASs' jobs when
+ * they would normally complete fine, so we must 'poke'
+ * the MMU regularly to help other ASs complete */
+ kbase_as_poking_timer_retain_atom(kbdev, katom->kctx,
+ katom);
+ }
+ }
+
+ if (kbase_hw_has_feature(kbdev,
+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
+ if (action == JS_COMMAND_SOFT_STOP)
+ action = (target_katom->atom_flags &
+ KBASE_KATOM_FLAGS_JOBCHAIN) ?
+ JS_COMMAND_SOFT_STOP_1 :
+ JS_COMMAND_SOFT_STOP_0;
+ else
+ action = (target_katom->atom_flags &
+ KBASE_KATOM_FLAGS_JOBCHAIN) ?
+ JS_COMMAND_HARD_STOP_1 :
+ JS_COMMAND_HARD_STOP_0;
+ }
+
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx);
+
+#if KBASE_TRACE_ENABLE
+ status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS),
+ NULL);
+ if (status_reg_after == BASE_JD_EVENT_ACTIVE) {
+ struct kbase_jd_atom *head;
+ struct kbase_context *head_kctx;
+
+ head = kbase_gpu_inspect(kbdev, js, 0);
+ head_kctx = head->kctx;
+
+ if (status_reg_before == BASE_JD_EVENT_ACTIVE)
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx,
+ head, job_in_head_before, js);
+ else
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
+ 0, js);
+
+ switch (action) {
+ case JS_COMMAND_SOFT_STOP:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx,
+ head, head->jc, js);
+ break;
+ case JS_COMMAND_SOFT_STOP_0:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx,
+ head, head->jc, js);
+ break;
+ case JS_COMMAND_SOFT_STOP_1:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx,
+ head, head->jc, js);
+ break;
+ case JS_COMMAND_HARD_STOP:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx,
+ head, head->jc, js);
+ break;
+ case JS_COMMAND_HARD_STOP_0:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx,
+ head, head->jc, js);
+ break;
+ case JS_COMMAND_HARD_STOP_1:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx,
+ head, head->jc, js);
+ break;
+ default:
+ BUG();
+ break;
+ }
+ } else {
+ if (status_reg_before == BASE_JD_EVENT_ACTIVE)
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
+ job_in_head_before, js);
+ else
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
+ 0, js);
+
+ switch (action) {
+ case JS_COMMAND_SOFT_STOP:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0,
+ js);
+ break;
+ case JS_COMMAND_SOFT_STOP_0:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL,
+ 0, js);
+ break;
+ case JS_COMMAND_SOFT_STOP_1:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL,
+ 0, js);
+ break;
+ case JS_COMMAND_HARD_STOP:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0,
+ js);
+ break;
+ case JS_COMMAND_HARD_STOP_0:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL,
+ 0, js);
+ break;
+ case JS_COMMAND_HARD_STOP_1:
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL,
+ 0, js);
+ break;
+ default:
+ BUG();
+ break;
+ }
+ }
+#endif
+}
+
+void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx)
+{
+ unsigned long flags;
+ struct kbase_device *kbdev;
+ struct kbasep_js_device_data *js_devdata;
+ int i;
+
+ KBASE_DEBUG_ASSERT(kctx != NULL);
+ kbdev = kctx->kbdev;
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ js_devdata = &kbdev->js_data;
+
+ /* Cancel any remaining running jobs for this kctx */
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ /* Invalidate all jobs in context, to prevent re-submitting */
+ for (i = 0; i < BASE_JD_ATOM_COUNT; i++) {
+ if (!work_pending(&kctx->jctx.atoms[i].work))
+ kctx->jctx.atoms[i].event_code =
+ BASE_JD_EVENT_JOB_CANCELLED;
+ }
+
+ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
+ kbase_job_slot_hardstop(kctx, i, NULL);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+}
+
+void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
+ struct kbase_jd_atom *target_katom)
+{
+ struct kbase_device *kbdev;
+ struct kbasep_js_device_data *js_devdata;
+ int js = target_katom->slot_nr;
+ int priority = target_katom->sched_priority;
+ int i;
+
+ KBASE_DEBUG_ASSERT(kctx != NULL);
+ kbdev = kctx->kbdev;
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ js_devdata = &kbdev->js_data;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) {
+ struct kbase_jd_atom *katom;
+
+ katom = kbase_gpu_inspect(kbdev, js, i);
+ if (!katom)
+ continue;
+
+ if (katom->kctx != kctx)
+ continue;
+
+ if (katom->sched_priority > priority)
+ kbase_job_slot_softstop(kbdev, js, katom);
+ }
+}
+
+struct zap_reset_data {
+ /* The stages are:
+ * 1. The timer has never been called
+ * 2. The zap has timed out, all slots are soft-stopped - the GPU reset
+ * will happen. The GPU has been reset when
+ * kbdev->hwaccess.backend.reset_waitq is signalled
+ *
+ * (-1 - The timer has been cancelled)
+ */
+ int stage;
+ struct kbase_device *kbdev;
+ struct hrtimer timer;
+ spinlock_t lock; /* protects updates to stage member */
+};
+
+static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer)
+{
+ struct zap_reset_data *reset_data = container_of(timer,
+ struct zap_reset_data, timer);
+ struct kbase_device *kbdev = reset_data->kbdev;
+ unsigned long flags;
+
+ spin_lock_irqsave(&reset_data->lock, flags);
+
+ if (reset_data->stage == -1)
+ goto out;
+
+#if KBASE_GPU_RESET_EN
+ if (kbase_prepare_to_reset_gpu(kbdev)) {
+ dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n",
+ ZAP_TIMEOUT);
+ kbase_reset_gpu(kbdev);
+ }
+#endif /* KBASE_GPU_RESET_EN */
+ reset_data->stage = 2;
+
+ out:
+ spin_unlock_irqrestore(&reset_data->lock, flags);
+
+ return HRTIMER_NORESTART;
+}
+
+void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx)
+{
+ struct kbase_device *kbdev = kctx->kbdev;
+ struct zap_reset_data reset_data;
+ unsigned long flags;
+
+ hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC,
+ HRTIMER_MODE_REL);
+ reset_data.timer.function = zap_timeout_callback;
+
+ spin_lock_init(&reset_data.lock);
+
+ reset_data.kbdev = kbdev;
+ reset_data.stage = 1;
+
+ hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT),
+ HRTIMER_MODE_REL);
+
+ /* Wait for all jobs to finish, and for the context to be not-scheduled
+ * (due to kbase_job_zap_context(), we also guarentee it's not in the JS
+ * policy queue either */
+ wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0);
+ wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait,
+ kctx->jctx.sched_info.ctx.is_scheduled == false);
+
+ spin_lock_irqsave(&reset_data.lock, flags);
+ if (reset_data.stage == 1) {
+ /* The timer hasn't run yet - so cancel it */
+ reset_data.stage = -1;
+ }
+ spin_unlock_irqrestore(&reset_data.lock, flags);
+
+ hrtimer_cancel(&reset_data.timer);
+
+ if (reset_data.stage == 2) {
+ /* The reset has already started.
+ * Wait for the reset to complete
+ */
+ wait_event(kbdev->hwaccess.backend.reset_wait,
+ atomic_read(&kbdev->hwaccess.backend.reset_gpu)
+ == KBASE_RESET_GPU_NOT_PENDING);
+ }
+ destroy_hrtimer_on_stack(&reset_data.timer);
+
+ dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx);
+
+ /* Ensure that the signallers of the waitqs have finished */
+ mutex_lock(&kctx->jctx.lock);
+ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
+ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
+ mutex_unlock(&kctx->jctx.lock);
+}
+
+int kbase_job_slot_init(struct kbase_device *kbdev)
+{
+#if KBASE_GPU_RESET_EN
+ kbdev->hwaccess.backend.reset_workq = alloc_workqueue(
+ "Mali reset workqueue", 0, 1);
+ if (NULL == kbdev->hwaccess.backend.reset_workq)
+ return -EINVAL;
+
+ KBASE_DEBUG_ASSERT(0 ==
+ object_is_on_stack(&kbdev->hwaccess.backend.reset_work));
+ INIT_WORK(&kbdev->hwaccess.backend.reset_work,
+ kbasep_reset_timeout_worker);
+
+ hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC,
+ HRTIMER_MODE_REL);
+ kbdev->hwaccess.backend.reset_timer.function =
+ kbasep_reset_timer_callback;
+#endif
+
+ return 0;
+}
+KBASE_EXPORT_TEST_API(kbase_job_slot_init);
+
+void kbase_job_slot_halt(struct kbase_device *kbdev)
+{
+ CSTD_UNUSED(kbdev);
+}
+
+void kbase_job_slot_term(struct kbase_device *kbdev)
+{
+#if KBASE_GPU_RESET_EN
+ destroy_workqueue(kbdev->hwaccess.backend.reset_workq);
+#endif
+}
+KBASE_EXPORT_TEST_API(kbase_job_slot_term);
+
+/**
+ * kbase_job_slot_softstop_swflags - Soft-stop a job with flags
+ * @kbdev: The kbase device
+ * @js: The job slot to soft-stop
+ * @target_katom: The job that should be soft-stopped (or NULL for any job)
+ * @sw_flags: Flags to pass in about the soft-stop
+ *
+ * Context:
+ * The job slot lock must be held when calling this function.
+ * The job slot must not already be in the process of being soft-stopped.
+ *
+ * Soft-stop the specified job slot, with extra information about the stop
+ *
+ * Where possible any job in the next register is evicted before the soft-stop.
+ */
+void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
+ struct kbase_jd_atom *target_katom, u32 sw_flags)
+{
+ KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK));
+ kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom,
+ JS_COMMAND_SOFT_STOP | sw_flags);
+}
+
+/**
+ * kbase_job_slot_softstop - Soft-stop the specified job slot
+ * @kbdev: The kbase device
+ * @js: The job slot to soft-stop
+ * @target_katom: The job that should be soft-stopped (or NULL for any job)
+ * Context:
+ * The job slot lock must be held when calling this function.
+ * The job slot must not already be in the process of being soft-stopped.
+ *
+ * Where possible any job in the next register is evicted before the soft-stop.
+ */
+void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
+ struct kbase_jd_atom *target_katom)
+{
+ kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u);
+}
+
+/**
+ * kbase_job_slot_hardstop - Hard-stop the specified job slot
+ * @kctx: The kbase context that contains the job(s) that should
+ * be hard-stopped
+ * @js: The job slot to hard-stop
+ * @target_katom: The job that should be hard-stopped (or NULL for all
+ * jobs from the context)
+ * Context:
+ * The job slot lock must be held when calling this function.
+ */
+void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
+ struct kbase_jd_atom *target_katom)
+{
+ struct kbase_device *kbdev = kctx->kbdev;
+
+ bool stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js,
+ target_katom,
+ JS_COMMAND_HARD_STOP);
+#if KBASE_GPU_RESET_EN
+ if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) ||
+ kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) ||
+ (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_T76X_3542) &&
+ (target_katom == NULL || target_katom->core_req &
+ BASE_JD_REQ_FS_AFBC)))) {
+ /* MIDBASE-2916 if a fragment job with AFBC encoding is
+ * hardstopped, ensure to do a soft reset also in order to
+ * clear the GPU status.
+ * Workaround for HW issue 8401 has an issue,so after
+ * hard-stopping just reset the GPU. This will ensure that the
+ * jobs leave the GPU.*/
+ if (kbase_prepare_to_reset_gpu_locked(kbdev)) {
+ dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue");
+ kbase_reset_gpu_locked(kbdev);
+ }
+ }
+#endif
+}
+
+/**
+ * kbase_job_check_enter_disjoint - potentiall enter disjoint mode
+ * @kbdev: kbase device
+ * @action: the event which has occurred
+ * @core_reqs: core requirements of the atom
+ * @target_katom: the atom which is being affected
+ *
+ * For a certain soft/hard-stop action, work out whether to enter disjoint
+ * state.
+ *
+ * This does not register multiple disjoint events if the atom has already
+ * started a disjoint period
+ *
+ * @core_reqs can be supplied as 0 if the atom had not started on the hardware
+ * (and so a 'real' soft/hard-stop was not required, but it still interrupted
+ * flow, perhaps on another context)
+ *
+ * kbase_job_check_leave_disjoint() should be used to end the disjoint
+ * state when the soft/hard-stop action is complete
+ */
+void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
+ u16 core_reqs, struct kbase_jd_atom *target_katom)
+{
+ u32 hw_action = action & JS_COMMAND_MASK;
+
+ /* For hard-stop, don't enter if hard-stop not allowed */
+ if (hw_action == JS_COMMAND_HARD_STOP &&
+ !kbasep_hard_stop_allowed(kbdev, core_reqs))
+ return;
+
+ /* For soft-stop, don't enter if soft-stop not allowed, or isn't
+ * causing disjoint */
+ if (hw_action == JS_COMMAND_SOFT_STOP &&
+ !(kbasep_soft_stop_allowed(kbdev, core_reqs) &&
+ (action & JS_COMMAND_SW_CAUSES_DISJOINT)))
+ return;
+
+ /* Nothing to do if already logged disjoint state on this atom */
+ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT)
+ return;
+
+ target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT;
+ kbase_disjoint_state_up(kbdev);
+}
+
+/**
+ * kbase_job_check_enter_disjoint - potentially leave disjoint state
+ * @kbdev: kbase device
+ * @target_katom: atom which is finishing
+ *
+ * Work out whether to leave disjoint state when finishing an atom that was
+ * originated by kbase_job_check_enter_disjoint().
+ */
+void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
+ struct kbase_jd_atom *target_katom)
+{
+ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) {
+ target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT;
+ kbase_disjoint_state_down(kbdev);
+ }
+}
+
+
+#if KBASE_GPU_RESET_EN
+static void kbase_debug_dump_registers(struct kbase_device *kbdev)
+{
+ int i;
+
+ dev_err(kbdev->dev, "Register state:");
+ dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x",
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL),
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL));
+ dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x JOB_IRQ_THROTTLE=0x%08x",
+ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL),
+ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL),
+ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE), NULL));
+ for (i = 0; i < 3; i++) {
+ dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x",
+ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS),
+ NULL),
+ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO),
+ NULL));
+ }
+ dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x",
+ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL),
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL));
+ dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x",
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL),
+ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL),
+ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL));
+ dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x",
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL),
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL));
+ dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x",
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL),
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL));
+}
+
+static void kbasep_save_hwcnt_setup(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ struct kbase_uk_hwcnt_setup *hwcnt_setup)
+{
+ hwcnt_setup->dump_buffer =
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), kctx) &
+ 0xffffffff;
+ hwcnt_setup->dump_buffer |= (u64)
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), kctx) <<
+ 32;
+ hwcnt_setup->jm_bm =
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), kctx);
+ hwcnt_setup->shader_bm =
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), kctx);
+ hwcnt_setup->tiler_bm =
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), kctx);
+ hwcnt_setup->mmu_l2_bm =
+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), kctx);
+}
+
+static void kbasep_reset_timeout_worker(struct work_struct *data)
+{
+ unsigned long flags;
+ struct kbase_device *kbdev;
+ int i;
+ ktime_t end_timestamp = ktime_get();
+ struct kbasep_js_device_data *js_devdata;
+ struct kbase_uk_hwcnt_setup hwcnt_setup = { {0} };
+ enum kbase_instr_state bckp_state;
+ bool try_schedule = false;
+ bool restore_hwc = false;
+
+ u32 mmu_irq_mask;
+
+ KBASE_DEBUG_ASSERT(data);
+
+ kbdev = container_of(data, struct kbase_device,
+ hwaccess.backend.reset_work);
+
+ KBASE_DEBUG_ASSERT(kbdev);
+ js_devdata = &kbdev->js_data;
+
+ KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
+
+ /* Make sure the timer has completed - this cannot be done from
+ * interrupt context, so this cannot be done within
+ * kbasep_try_reset_gpu_early. */
+ hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer);
+
+ if (kbase_pm_context_active_handle_suspend(kbdev,
+ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
+ /* This would re-activate the GPU. Since it's already idle,
+ * there's no need to reset it */
+ atomic_set(&kbdev->hwaccess.backend.reset_gpu,
+ KBASE_RESET_GPU_NOT_PENDING);
+ kbase_disjoint_state_down(kbdev);
+ wake_up(&kbdev->hwaccess.backend.reset_wait);
+ return;
+ }
+
+ mutex_lock(&kbdev->pm.lock);
+ /* We hold the pm lock, so there ought to be a current policy */
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy);
+
+ /* All slot have been soft-stopped and we've waited
+ * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we
+ * assume that anything that is still left on the GPU is stuck there and
+ * we'll kill it when we reset the GPU */
+
+ dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)",
+ RESET_TIMEOUT);
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+
+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_RESETTING) {
+ /* the same interrupt handler preempted itself */
+ /* GPU is being reset */
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ wait_event(kbdev->hwcnt.backend.wait,
+ kbdev->hwcnt.backend.triggered != 0);
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ }
+ /* Save the HW counters setup */
+ if (kbdev->hwcnt.kctx != NULL) {
+ struct kbase_context *kctx = kbdev->hwcnt.kctx;
+
+ if (kctx->jctx.sched_info.ctx.is_scheduled) {
+ kbasep_save_hwcnt_setup(kbdev, kctx, &hwcnt_setup);
+
+ restore_hwc = true;
+ }
+ }
+
+ /* Output the state of some interesting registers to help in the
+ * debugging of GPU resets */
+ kbase_debug_dump_registers(kbdev);
+
+ bckp_state = kbdev->hwcnt.backend.state;
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_RESETTING;
+ kbdev->hwcnt.backend.triggered = 0;
+
+ mmu_irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL);
+ /* Disable IRQ to avoid IRQ handlers to kick in after releasing the
+ * spinlock; this also clears any outstanding interrupts */
+ kbase_pm_disable_interrupts(kbdev);
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ /* Ensure that any IRQ handlers have finished
+ * Must be done without any locks IRQ handlers will take */
+ kbase_synchronize_irqs(kbdev);
+
+ /* Reset the GPU */
+ kbase_pm_init_hw(kbdev, 0);
+ kbase_pm_enable_interrupts_mmu_mask(kbdev, mmu_irq_mask);
+ /* IRQs were re-enabled by kbase_pm_init_hw, and GPU is still powered */
+
+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ /* Restore the HW counters setup */
+ if (restore_hwc) {
+ struct kbase_context *kctx = kbdev->hwcnt.kctx;
+
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
+ (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) |
+ PRFCNT_CONFIG_MODE_OFF, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
+ hwcnt_setup.dump_buffer & 0xFFFFFFFF, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
+ hwcnt_setup.dump_buffer >> 32, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN),
+ hwcnt_setup.jm_bm, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN),
+ hwcnt_setup.shader_bm, kctx);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN),
+ hwcnt_setup.mmu_l2_bm, kctx);
+
+ /* Due to PRLAM-8186 we need to disable the Tiler before we
+ * enable the HW counter dump. */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
+ 0, kctx);
+ else
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
+ hwcnt_setup.tiler_bm, kctx);
+
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
+ (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) |
+ PRFCNT_CONFIG_MODE_MANUAL, kctx);
+
+ /* If HW has PRLAM-8186 we can now re-enable the tiler HW
+ * counters dump */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
+ hwcnt_setup.tiler_bm, kctx);
+ }
+ kbdev->hwcnt.backend.state = bckp_state;
+ switch (kbdev->hwcnt.backend.state) {
+ /* Cases for waking kbasep_cache_clean_worker worker */
+ case KBASE_INSTR_STATE_CLEANED:
+ /* Cache-clean IRQ occurred, but we reset:
+ * Wakeup incase the waiter saw RESETTING */
+ case KBASE_INSTR_STATE_REQUEST_CLEAN:
+ /* After a clean was requested, but before the regs were
+ * written:
+ * Wakeup incase the waiter saw RESETTING */
+ wake_up(&kbdev->hwcnt.backend.cache_clean_wait);
+ break;
+ case KBASE_INSTR_STATE_CLEANING:
+ /* Either:
+ * 1) We've not got the Cache-clean IRQ yet: it was lost, or:
+ * 2) We got it whilst resetting: it was voluntarily lost
+ *
+ * So, move to the next state and wakeup: */
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANED;
+ wake_up(&kbdev->hwcnt.backend.cache_clean_wait);
+ break;
+
+ /* Cases for waking anyone else */
+ case KBASE_INSTR_STATE_DUMPING:
+ /* If dumping, abort the dump, because we may've lost the IRQ */
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
+ kbdev->hwcnt.backend.triggered = 1;
+ wake_up(&kbdev->hwcnt.backend.wait);
+ break;
+ case KBASE_INSTR_STATE_DISABLED:
+ case KBASE_INSTR_STATE_IDLE:
+ case KBASE_INSTR_STATE_FAULT:
+ /* Every other reason: wakeup in that state */
+ kbdev->hwcnt.backend.triggered = 1;
+ wake_up(&kbdev->hwcnt.backend.wait);
+ break;
+
+ /* Unhandled cases */
+ case KBASE_INSTR_STATE_RESETTING:
+ default:
+ BUG();
+ break;
+ }
+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+
+ /* Complete any jobs that were still on the GPU */
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ kbase_backend_reset(kbdev, &end_timestamp);
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ mutex_unlock(&kbdev->pm.lock);
+
+ mutex_lock(&js_devdata->runpool_mutex);
+
+ /* Reprogram the GPU's MMU */
+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
+ struct kbase_as *as = &kbdev->as[i];
+
+ mutex_lock(&as->transaction_mutex);
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ if (js_devdata->runpool_irq.per_as_data[i].kctx)
+ kbase_mmu_update(
+ js_devdata->runpool_irq.per_as_data[i].kctx);
+ else
+ kbase_mmu_disable_as(kbdev, i);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ mutex_unlock(&as->transaction_mutex);
+ }
+
+ atomic_set(&kbdev->hwaccess.backend.reset_gpu,
+ KBASE_RESET_GPU_NOT_PENDING);
+
+ kbase_disjoint_state_down(kbdev);
+
+ wake_up(&kbdev->hwaccess.backend.reset_wait);
+ dev_err(kbdev->dev, "Reset complete");
+
+ if (js_devdata->nr_contexts_pullable > 0 && !kbdev->poweroff_pending)
+ try_schedule = true;
+
+ mutex_unlock(&js_devdata->runpool_mutex);
+
+ mutex_lock(&kbdev->pm.lock);
+
+ /* Find out what cores are required now */
+ kbase_pm_update_cores_state(kbdev);
+
+ /* Synchronously request and wait for those cores, because if
+ * instrumentation is enabled it would need them immediately. */
+ kbase_pm_check_transitions_sync(kbdev);
+
+ mutex_unlock(&kbdev->pm.lock);
+
+ /* Try submitting some jobs to restart processing */
+ if (try_schedule) {
+ KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u,
+ 0);
+ kbase_js_sched_all(kbdev);
+ }
+
+ kbase_pm_context_idle(kbdev);
+ KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
+}
+
+static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer)
+{
+ struct kbase_device *kbdev = container_of(timer, struct kbase_device,
+ hwaccess.backend.reset_timer);
+
+ KBASE_DEBUG_ASSERT(kbdev);
+
+ /* Reset still pending? */
+ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
+ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) ==
+ KBASE_RESET_GPU_COMMITTED)
+ queue_work(kbdev->hwaccess.backend.reset_workq,
+ &kbdev->hwaccess.backend.reset_work);
+
+ return HRTIMER_NORESTART;
+}
+
+/*
+ * If all jobs are evicted from the GPU then we can reset the GPU
+ * immediately instead of waiting for the timeout to elapse
+ */
+
+static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev)
+{
+ int i;
+ int pending_jobs = 0;
+
+ KBASE_DEBUG_ASSERT(kbdev);
+
+ /* Count the number of jobs */
+ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
+ pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i);
+
+ if (pending_jobs > 0) {
+ /* There are still jobs on the GPU - wait */
+ return;
+ }
+
+ /* Check that the reset has been committed to (i.e. kbase_reset_gpu has
+ * been called), and that no other thread beat this thread to starting
+ * the reset */
+ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
+ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) !=
+ KBASE_RESET_GPU_COMMITTED) {
+ /* Reset has already occurred */
+ return;
+ }
+
+ queue_work(kbdev->hwaccess.backend.reset_workq,
+ &kbdev->hwaccess.backend.reset_work);
+}
+
+static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+ struct kbasep_js_device_data *js_devdata;
+
+ js_devdata = &kbdev->js_data;
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ kbasep_try_reset_gpu_early_locked(kbdev);
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+}
+
+/**
+ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU
+ * @kbdev: kbase device
+ *
+ * This function just soft-stops all the slots to ensure that as many jobs as
+ * possible are saved.
+ *
+ * Return:
+ * The function returns a boolean which should be interpreted as follows:
+ * true - Prepared for reset, kbase_reset_gpu should be called.
+ * false - Another thread is performing a reset, kbase_reset_gpu should
+ * not be called.
+ */
+bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev)
+{
+ int i;
+
+ KBASE_DEBUG_ASSERT(kbdev);
+
+ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
+ KBASE_RESET_GPU_NOT_PENDING,
+ KBASE_RESET_GPU_PREPARED) !=
+ KBASE_RESET_GPU_NOT_PENDING) {
+ /* Some other thread is already resetting the GPU */
+ return false;
+ }
+
+ kbase_disjoint_state_up(kbdev);
+
+ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
+ kbase_job_slot_softstop(kbdev, i, NULL);
+
+ return true;
+}
+
+bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+ bool ret;
+ struct kbasep_js_device_data *js_devdata;
+
+ js_devdata = &kbdev->js_data;
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ ret = kbase_prepare_to_reset_gpu_locked(kbdev);
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ return ret;
+}
+KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu);
+
+/*
+ * This function should be called after kbase_prepare_to_reset_gpu if it
+ * returns true. It should never be called without a corresponding call to
+ * kbase_prepare_to_reset_gpu.
+ *
+ * After this function is called (or not called if kbase_prepare_to_reset_gpu
+ * returned false), the caller should wait for
+ * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset
+ * has completed.
+ */
+void kbase_reset_gpu(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev);
+
+ /* Note this is an assert/atomic_set because it is a software issue for
+ * a race to be occuring here */
+ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
+ KBASE_RESET_GPU_PREPARED);
+ atomic_set(&kbdev->hwaccess.backend.reset_gpu,
+ KBASE_RESET_GPU_COMMITTED);
+
+ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
+ kbdev->reset_timeout_ms);
+
+ hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
+ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
+ HRTIMER_MODE_REL);
+
+ /* Try resetting early */
+ kbasep_try_reset_gpu_early(kbdev);
+}
+KBASE_EXPORT_TEST_API(kbase_reset_gpu);
+
+void kbase_reset_gpu_locked(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev);
+
+ /* Note this is an assert/atomic_set because it is a software issue for
+ * a race to be occuring here */
+ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
+ KBASE_RESET_GPU_PREPARED);
+ atomic_set(&kbdev->hwaccess.backend.reset_gpu,
+ KBASE_RESET_GPU_COMMITTED);
+
+ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
+ kbdev->reset_timeout_ms);
+ hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
+ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
+ HRTIMER_MODE_REL);
+
+ /* Try resetting early */
+ kbasep_try_reset_gpu_early_locked(kbdev);
+}
+#endif /* KBASE_GPU_RESET_EN */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * Job Manager backend-specific low-level APIs.
+ */
+
+#ifndef _KBASE_JM_HWACCESS_H_
+#define _KBASE_JM_HWACCESS_H_
+
+#include <mali_kbase_hw.h>
+#include <mali_kbase_debug.h>
+#include <linux/atomic.h>
+
+#include <backend/gpu/mali_kbase_jm_rb.h>
+
+/**
+ * kbase_job_submit_nolock() - Submit a job to a certain job-slot
+ * @kbdev: Device pointer
+ * @katom: Atom to submit
+ * @js: Job slot to submit on
+ *
+ * The caller must check kbasep_jm_is_submit_slots_free() != false before
+ * calling this.
+ *
+ * The following locking conditions are made on the caller:
+ * - it must hold the kbasep_js_device_data::runpoool_irq::lock
+ */
+void kbase_job_submit_nolock(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom, int js);
+
+/**
+ * kbase_job_done_slot() - Complete the head job on a particular job-slot
+ * @kbdev: Device pointer
+ * @s: Job slot
+ * @completion_code: Completion code of job reported by GPU
+ * @job_tail: Job tail address reported by GPU
+ * @end_timestamp: Timestamp of job completion
+ */
+void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code,
+ u64 job_tail, ktime_t *end_timestamp);
+
+#ifdef CONFIG_GPU_TRACEPOINTS
+static inline char *kbasep_make_job_slot_string(int js, char *js_string)
+{
+ sprintf(js_string, "job_slot_%i", js);
+ return js_string;
+}
+#endif
+
+/**
+ * kbase_job_hw_submit() - Submit a job to the GPU
+ * @kbdev: Device pointer
+ * @katom: Atom to submit
+ * @js: Job slot to submit on
+ *
+ * The caller must check kbasep_jm_is_submit_slots_free() != false before
+ * calling this.
+ *
+ * The following locking conditions are made on the caller:
+ * - it must hold the kbasep_js_device_data::runpoool_irq::lock
+ */
+void kbase_job_hw_submit(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom,
+ int js);
+
+/**
+ * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop
+ * on the specified atom
+ * @kbdev: Device pointer
+ * @js: Job slot to stop on
+ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or
+ * JSn_COMMAND_SOFT_STOP
+ * @core_reqs: Core requirements of atom to stop
+ * @target_katom: Atom to stop
+ *
+ * The following locking conditions are made on the caller:
+ * - it must hold the kbasep_js_device_data::runpool_irq::lock
+ */
+void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
+ int js,
+ u32 action,
+ u16 core_reqs,
+ struct kbase_jd_atom *target_katom);
+
+/**
+ * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job
+ * slot belonging to a given context.
+ * @kbdev: Device pointer
+ * @kctx: Context pointer. May be NULL
+ * @katom: Specific atom to stop. May be NULL
+ * @js: Job slot to hard stop
+ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or
+ * JSn_COMMAND_SOFT_STOP
+ *
+ * If no context is provided then all jobs on the slot will be soft or hard
+ * stopped.
+ *
+ * If a katom is provided then only that specific atom will be stopped. In this
+ * case the kctx parameter is ignored.
+ *
+ * Jobs that are on the slot but are not yet on the GPU will be unpulled and
+ * returned to the job scheduler.
+ *
+ * Return: true if an atom was stopped, false otherwise
+ */
+bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ int js,
+ struct kbase_jd_atom *katom,
+ u32 action);
+
+/**
+ * kbase_job_slot_init - Initialise job slot framework
+ * @kbdev: Device pointer
+ *
+ * Called on driver initialisation
+ *
+ * Return: 0 on success
+ */
+int kbase_job_slot_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_job_slot_halt - Halt the job slot framework
+ * @kbdev: Device pointer
+ *
+ * Should prevent any further job slot processing
+ */
+void kbase_job_slot_halt(struct kbase_device *kbdev);
+
+/**
+ * kbase_job_slot_term - Terminate job slot framework
+ * @kbdev: Device pointer
+ *
+ * Called on driver termination
+ */
+void kbase_job_slot_term(struct kbase_device *kbdev);
+
+#if KBASE_GPU_RESET_EN
+/**
+ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU.
+ * @kbdev: Device pointer
+ *
+ * This function just soft-stops all the slots to ensure that as many jobs as
+ * possible are saved.
+ *
+ * Return: a boolean which should be interpreted as follows:
+ * - true - Prepared for reset, kbase_reset_gpu should be called.
+ * - false - Another thread is performing a reset, kbase_reset_gpu should
+ * not be called.
+ */
+bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev);
+
+/**
+ * kbase_reset_gpu_locked - Reset the GPU
+ * @kbdev: Device pointer
+ *
+ * This function should be called after kbase_prepare_to_reset_gpu if it
+ * returns true. It should never be called without a corresponding call to
+ * kbase_prepare_to_reset_gpu.
+ *
+ * After this function is called (or not called if kbase_prepare_to_reset_gpu
+ * returned false), the caller should wait for kbdev->reset_waitq to be
+ * signalled to know when the reset has completed.
+ */
+void kbase_reset_gpu_locked(struct kbase_device *kbdev);
+#endif
+
+#endif /* _KBASE_JM_HWACCESS_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/*
+ * Register-based HW access backend specific APIs
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_hwaccess_jm.h>
+#include <mali_kbase_jm.h>
+#include <mali_kbase_js.h>
+#include <mali_kbase_10969_workaround.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
+#include <backend/gpu/mali_kbase_jm_internal.h>
+#include <backend/gpu/mali_kbase_js_affinity.h>
+#include <backend/gpu/mali_kbase_js_internal.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+/* Return whether the specified ringbuffer is empty. HW access lock must be
+ * held */
+#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx)
+/* Return number of atoms currently in the specified ringbuffer. HW access lock
+ * must be held */
+#define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx)
+
+static void kbase_gpu_release_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom);
+
+/**
+ * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer
+ * @kbdev: Device pointer
+ * @katom: Atom to enqueue
+ *
+ * Context: Caller must hold the HW access lock
+ */
+static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr];
+
+ WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE);
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom;
+ rb->write_idx++;
+
+ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
+}
+
+/**
+ * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once
+ * it has been completed
+ * @kbdev: Device pointer
+ * @js: Job slot to remove atom from
+ *
+ * Context: Caller must hold the HW access lock
+ *
+ * Return: Atom removed from ringbuffer
+ */
+static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev,
+ int js)
+{
+ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
+ struct kbase_jd_atom *katom;
+
+ if (SLOT_RB_EMPTY(rb)) {
+ WARN(1, "GPU ringbuffer unexpectedly empty\n");
+ return NULL;
+ }
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom;
+
+ kbase_gpu_release_atom(kbdev, katom);
+
+ rb->read_idx++;
+
+ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB;
+
+ kbase_js_debug_log_current_affinities(kbdev);
+
+ return katom;
+}
+
+struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js,
+ int idx)
+{
+ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ if ((SLOT_RB_ENTRIES(rb) - 1) < idx)
+ return NULL; /* idx out of range */
+
+ return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom;
+}
+
+struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev,
+ int js)
+{
+ return kbase_gpu_inspect(kbdev, js, 0);
+}
+
+struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev,
+ int js)
+{
+ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
+
+ if (SLOT_RB_EMPTY(rb))
+ return NULL;
+
+ return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom;
+}
+
+/**
+ * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently
+ * on the GPU
+ * @kbdev: Device pointer
+ * @js: Job slot to inspect
+ *
+ * Return: true if there are atoms on the GPU for slot js,
+ * false otherwise
+ */
+static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js)
+{
+ int i;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ for (i = 0; i < SLOT_RB_SIZE; i++) {
+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
+
+ if (!katom)
+ return false;
+ if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED ||
+ katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY)
+ return true;
+ }
+
+ return false;
+}
+
+/**
+ * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms
+ * currently on the GPU
+ * @kbdev: Device pointer
+ *
+ * Return: true if there are any atoms on the GPU, false otherwise
+ */
+static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev)
+{
+ int js;
+ int i;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
+ for (i = 0; i < SLOT_RB_SIZE; i++) {
+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
+
+ if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)
+ return true;
+ }
+ }
+ return false;
+}
+
+int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js)
+{
+ int nr = 0;
+ int i;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ for (i = 0; i < SLOT_RB_SIZE; i++) {
+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
+
+ if (katom && (katom->gpu_rb_state ==
+ KBASE_ATOM_GPU_RB_SUBMITTED))
+ nr++;
+ }
+
+ return nr;
+}
+
+int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js)
+{
+ int nr = 0;
+ int i;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ for (i = 0; i < SLOT_RB_SIZE; i++) {
+ if (kbase_gpu_inspect(kbdev, js, i))
+ nr++;
+ }
+
+ return nr;
+}
+
+static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js,
+ enum kbase_atom_gpu_rb_state min_rb_state)
+{
+ int nr = 0;
+ int i;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ for (i = 0; i < SLOT_RB_SIZE; i++) {
+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
+
+ if (katom && (katom->gpu_rb_state >= min_rb_state))
+ nr++;
+ }
+
+ return nr;
+}
+
+int kbase_backend_slot_free(struct kbase_device *kbdev, int js)
+{
+ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) !=
+ KBASE_RESET_GPU_NOT_PENDING) {
+ /* The GPU is being reset - so prevent submission */
+ return 0;
+ }
+
+ return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js);
+}
+
+
+static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom);
+
+static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev,
+ int js,
+ struct kbase_jd_atom *katom)
+{
+ /* The most recently checked affinity. Having this at this scope allows
+ * us to guarantee that we've checked the affinity in this function
+ * call.
+ */
+ u64 recently_chosen_affinity = 0;
+ bool chosen_affinity = false;
+ bool retry;
+
+ do {
+ retry = false;
+
+ /* NOTE: The following uses a number of FALLTHROUGHs to optimize
+ * the calls to this function. Ending of the function is
+ * indicated by BREAK OUT */
+ switch (katom->coreref_state) {
+ /* State when job is first attempted to be run */
+ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
+ KBASE_DEBUG_ASSERT(katom->affinity == 0);
+
+ /* Compute affinity */
+ if (false == kbase_js_choose_affinity(
+ &recently_chosen_affinity, kbdev, katom,
+ js)) {
+ /* No cores are currently available */
+ /* *** BREAK OUT: No state transition *** */
+ break;
+ }
+
+ chosen_affinity = true;
+
+ /* Request the cores */
+ kbase_pm_request_cores(kbdev,
+ katom->core_req & BASE_JD_REQ_T,
+ recently_chosen_affinity);
+
+ katom->affinity = recently_chosen_affinity;
+
+ /* Proceed to next state */
+ katom->coreref_state =
+ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
+
+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+
+ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
+ {
+ enum kbase_pm_cores_ready cores_ready;
+
+ KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
+ (katom->core_req & BASE_JD_REQ_T));
+
+ cores_ready = kbase_pm_register_inuse_cores(
+ kbdev,
+ katom->core_req & BASE_JD_REQ_T,
+ katom->affinity);
+ if (cores_ready == KBASE_NEW_AFFINITY) {
+ /* Affinity no longer valid - return to
+ * previous state */
+ kbasep_js_job_check_deref_cores(kbdev,
+ katom);
+ KBASE_TRACE_ADD_SLOT_INFO(kbdev,
+ JS_CORE_REF_REGISTER_INUSE_FAILED,
+ katom->kctx, katom,
+ katom->jc, js,
+ (u32) katom->affinity);
+ /* *** BREAK OUT: Return to previous
+ * state, retry *** */
+ retry = true;
+ break;
+ }
+ if (cores_ready == KBASE_CORES_NOT_READY) {
+ /* Stay in this state and return, to
+ * retry at this state later */
+ KBASE_TRACE_ADD_SLOT_INFO(kbdev,
+ JS_CORE_REF_REGISTER_INUSE_FAILED,
+ katom->kctx, katom,
+ katom->jc, js,
+ (u32) katom->affinity);
+ /* *** BREAK OUT: No state transition
+ * *** */
+ break;
+ }
+ /* Proceed to next state */
+ katom->coreref_state =
+ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
+ }
+
+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+
+ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
+ KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
+ (katom->core_req & BASE_JD_REQ_T));
+
+ /* Optimize out choosing the affinity twice in the same
+ * function call */
+ if (chosen_affinity == false) {
+ /* See if the affinity changed since a previous
+ * call. */
+ if (false == kbase_js_choose_affinity(
+ &recently_chosen_affinity,
+ kbdev, katom, js)) {
+ /* No cores are currently available */
+ kbasep_js_job_check_deref_cores(kbdev,
+ katom);
+ KBASE_TRACE_ADD_SLOT_INFO(kbdev,
+ JS_CORE_REF_REQUEST_ON_RECHECK_FAILED,
+ katom->kctx, katom,
+ katom->jc, js,
+ (u32) recently_chosen_affinity);
+ /* *** BREAK OUT: Transition to lower
+ * state *** */
+ break;
+ }
+ chosen_affinity = true;
+ }
+
+ /* Now see if this requires a different set of cores */
+ if (recently_chosen_affinity != katom->affinity) {
+ enum kbase_pm_cores_ready cores_ready;
+
+ kbase_pm_request_cores(kbdev,
+ katom->core_req & BASE_JD_REQ_T,
+ recently_chosen_affinity);
+
+ /* Register new cores whilst we still hold the
+ * old ones, to minimize power transitions */
+ cores_ready =
+ kbase_pm_register_inuse_cores(kbdev,
+ katom->core_req & BASE_JD_REQ_T,
+ recently_chosen_affinity);
+ kbasep_js_job_check_deref_cores(kbdev, katom);
+
+ /* Fixup the state that was reduced by
+ * deref_cores: */
+ katom->coreref_state =
+ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
+ katom->affinity = recently_chosen_affinity;
+ if (cores_ready == KBASE_NEW_AFFINITY) {
+ /* Affinity no longer valid - return to
+ * previous state */
+ katom->coreref_state =
+ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
+
+ kbasep_js_job_check_deref_cores(kbdev,
+ katom);
+
+ KBASE_TRACE_ADD_SLOT_INFO(kbdev,
+ JS_CORE_REF_REGISTER_INUSE_FAILED,
+ katom->kctx, katom,
+ katom->jc, js,
+ (u32) katom->affinity);
+ /* *** BREAK OUT: Return to previous
+ * state, retry *** */
+ retry = true;
+ break;
+ }
+ /* Now might be waiting for powerup again, with
+ * a new affinity */
+ if (cores_ready == KBASE_CORES_NOT_READY) {
+ /* Return to previous state */
+ katom->coreref_state =
+ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
+ KBASE_TRACE_ADD_SLOT_INFO(kbdev,
+ JS_CORE_REF_REGISTER_ON_RECHECK_FAILED,
+ katom->kctx, katom,
+ katom->jc, js,
+ (u32) katom->affinity);
+ /* *** BREAK OUT: Transition to lower
+ * state *** */
+ break;
+ }
+ }
+ /* Proceed to next state */
+ katom->coreref_state =
+ KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS;
+
+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+ case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS:
+ KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
+ (katom->core_req & BASE_JD_REQ_T));
+ KBASE_DEBUG_ASSERT(katom->affinity ==
+ recently_chosen_affinity);
+
+ /* Note: this is where the caller must've taken the
+ * runpool_irq.lock */
+
+ /* Check for affinity violations - if there are any,
+ * then we just ask the caller to requeue and try again
+ * later */
+ if (kbase_js_affinity_would_violate(kbdev, js,
+ katom->affinity) != false) {
+ /* Return to previous state */
+ katom->coreref_state =
+ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
+ /* *** BREAK OUT: Transition to lower state ***
+ */
+ KBASE_TRACE_ADD_SLOT_INFO(kbdev,
+ JS_CORE_REF_AFFINITY_WOULD_VIOLATE,
+ katom->kctx, katom, katom->jc, js,
+ (u32) katom->affinity);
+ break;
+ }
+
+ /* No affinity violations would result, so the cores are
+ * ready */
+ katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY;
+ /* *** BREAK OUT: Cores Ready *** */
+ break;
+
+ default:
+ KBASE_DEBUG_ASSERT_MSG(false,
+ "Unhandled kbase_atom_coreref_state %d",
+ katom->coreref_state);
+ break;
+ }
+ } while (retry != false);
+
+ return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY);
+}
+
+static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(katom != NULL);
+
+ switch (katom->coreref_state) {
+ case KBASE_ATOM_COREREF_STATE_READY:
+ /* State where atom was submitted to the HW - just proceed to
+ * power-down */
+ KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
+ (katom->core_req & BASE_JD_REQ_T));
+
+ /* *** FALLTHROUGH *** */
+
+ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
+ /* State where cores were registered */
+ KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
+ (katom->core_req & BASE_JD_REQ_T));
+ kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
+ katom->affinity);
+
+ break;
+
+ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
+ /* State where cores were requested, but not registered */
+ KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
+ (katom->core_req & BASE_JD_REQ_T));
+ kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
+ katom->affinity);
+ break;
+
+ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
+ /* Initial state - nothing required */
+ KBASE_DEBUG_ASSERT(katom->affinity == 0);
+ break;
+
+ default:
+ KBASE_DEBUG_ASSERT_MSG(false,
+ "Unhandled coreref_state: %d",
+ katom->coreref_state);
+ break;
+ }
+
+ katom->affinity = 0;
+ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
+}
+
+
+static void kbase_gpu_release_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ switch (katom->gpu_rb_state) {
+ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
+ /* Should be impossible */
+ WARN(1, "Attempting to release atom not in ringbuffer\n");
+ break;
+
+ case KBASE_ATOM_GPU_RB_SUBMITTED:
+ /* Inform power management at start/finish of atom
+ * so it can update its GPU utilisation metrics. */
+ kbase_pm_metrics_release_atom(kbdev, katom);
+
+ if (katom->core_req & BASE_JD_REQ_PERMON)
+ kbase_pm_release_gpu_cycle_counter(kbdev);
+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
+
+ case KBASE_ATOM_GPU_RB_READY:
+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
+
+ case KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE:
+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
+
+ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
+ kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr,
+ katom->affinity);
+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
+
+ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
+ kbasep_js_job_check_deref_cores(kbdev, katom);
+ break;
+
+ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
+
+ case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
+ break;
+ }
+
+ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
+}
+
+static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ kbase_gpu_release_atom(kbdev, katom);
+ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS;
+}
+
+static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js)
+{
+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
+ bool slot_busy[3];
+
+ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
+ return true;
+ slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0,
+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
+ slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1,
+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
+ slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2,
+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
+
+ if ((js == 2 && !(slot_busy[0] || slot_busy[1])) ||
+ (js != 2 && !slot_busy[2]))
+ return true;
+
+ /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */
+ if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) ||
+ kbase_gpu_atoms_submitted(kbdev, 1) ||
+ backend->rmu_workaround_flag))
+ return false;
+
+ /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */
+ if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) ||
+ !backend->rmu_workaround_flag))
+ return false;
+
+ backend->rmu_workaround_flag = !backend->rmu_workaround_flag;
+
+ return true;
+}
+
+static bool kbase_gpu_in_secure_mode(struct kbase_device *kbdev)
+{
+ return kbdev->js_data.runpool_irq.secure_mode;
+}
+
+static int kbase_gpu_secure_mode_enable(struct kbase_device *kbdev)
+{
+ int err = -EINVAL;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ WARN_ONCE(!kbdev->secure_ops,
+ "Cannot enable secure mode: secure callbacks not specified.\n");
+
+ if (kbdev->secure_ops) {
+ /* Switch GPU to secure mode */
+ err = kbdev->secure_ops->secure_mode_enable(kbdev);
+
+ if (err)
+ dev_warn(kbdev->dev, "Failed to enable secure mode: %d\n", err);
+ else
+ kbdev->js_data.runpool_irq.secure_mode = true;
+ }
+
+ return err;
+}
+
+static int kbase_gpu_secure_mode_disable(struct kbase_device *kbdev)
+{
+ int err = -EINVAL;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ WARN_ONCE(!kbdev->secure_ops,
+ "Cannot disable secure mode: secure callbacks not specified.\n");
+
+ if (kbdev->secure_ops) {
+ /* Switch GPU to non-secure mode */
+ err = kbdev->secure_ops->secure_mode_disable(kbdev);
+
+ if (err)
+ dev_warn(kbdev->dev, "Failed to disable secure mode: %d\n", err);
+ else
+ kbdev->js_data.runpool_irq.secure_mode = false;
+ }
+
+ return err;
+}
+
+void kbase_gpu_slot_update(struct kbase_device *kbdev)
+{
+ int js;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
+ struct kbase_jd_atom *katom[2];
+ int idx;
+
+ katom[0] = kbase_gpu_inspect(kbdev, js, 0);
+ katom[1] = kbase_gpu_inspect(kbdev, js, 1);
+ WARN_ON(katom[1] && !katom[0]);
+
+ for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
+ bool cores_ready;
+
+ if (!katom[idx])
+ continue;
+
+ switch (katom[idx]->gpu_rb_state) {
+ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
+ /* Should be impossible */
+ WARN(1, "Attempting to update atom not in ringbuffer\n");
+ break;
+
+ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
+ if (katom[idx]->atom_flags &
+ KBASE_KATOM_FLAG_X_DEP_BLOCKED)
+ break;
+
+ katom[idx]->gpu_rb_state =
+ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE;
+
+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
+ cores_ready =
+ kbasep_js_job_check_ref_cores(kbdev, js,
+ katom[idx]);
+
+ if (katom[idx]->event_code ==
+ BASE_JD_EVENT_PM_EVENT) {
+ katom[idx]->gpu_rb_state =
+ KBASE_ATOM_GPU_RB_RETURN_TO_JS;
+ break;
+ }
+
+ if (!cores_ready)
+ break;
+
+ kbase_js_affinity_retain_slot_cores(kbdev, js,
+ katom[idx]->affinity);
+ katom[idx]->gpu_rb_state =
+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY;
+
+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+
+ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
+ if (!kbase_gpu_rmu_workaround(kbdev, js))
+ break;
+
+ katom[idx]->gpu_rb_state =
+ KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE;
+
+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+
+ case KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE:
+ if (kbase_gpu_in_secure_mode(kbdev) != kbase_jd_katom_is_secure(katom[idx])) {
+ int err = 0;
+
+ /* Not in correct mode, take action */
+ if (kbase_gpu_atoms_submitted_any(kbdev)) {
+ /*
+ * We are not in the correct
+ * GPU mode for this job, and
+ * we can't switch now because
+ * there are jobs already
+ * running.
+ */
+ break;
+ }
+
+ /* No jobs running, so we can switch GPU mode right now */
+ if (kbase_jd_katom_is_secure(katom[idx])) {
+ err = kbase_gpu_secure_mode_enable(kbdev);
+ } else {
+ err = kbase_gpu_secure_mode_disable(kbdev);
+ }
+
+ if (err) {
+ /* Failed to switch secure mode, fail atom */
+ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
+ kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
+ break;
+ }
+ }
+
+ /* Secure mode sanity checks */
+ KBASE_DEBUG_ASSERT_MSG(
+ kbase_jd_katom_is_secure(katom[idx]) == kbase_gpu_in_secure_mode(kbdev),
+ "Secure mode of atom (%d) doesn't match secure mode of GPU (%d)",
+ kbase_jd_katom_is_secure(katom[idx]), kbase_gpu_in_secure_mode(kbdev));
+ KBASE_DEBUG_ASSERT_MSG(
+ (kbase_jd_katom_is_secure(katom[idx]) && js == 0) ||
+ !kbase_jd_katom_is_secure(katom[idx]),
+ "Secure atom on JS%d not supported", js);
+
+ katom[idx]->gpu_rb_state =
+ KBASE_ATOM_GPU_RB_READY;
+
+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+
+ case KBASE_ATOM_GPU_RB_READY:
+ /* Only submit if head atom or previous atom
+ * already submitted */
+ if (idx == 1 &&
+ (katom[0]->gpu_rb_state !=
+ KBASE_ATOM_GPU_RB_SUBMITTED &&
+ katom[0]->gpu_rb_state !=
+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
+ break;
+
+ /* Check if this job needs the cycle counter
+ * enabled before submission */
+ if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
+ kbase_pm_request_gpu_cycle_counter_l2_is_on(
+ kbdev);
+
+ /* Inform power management at start/finish of
+ * atom so it can update its GPU utilisation
+ * metrics. */
+ kbase_pm_metrics_run_atom(kbdev, katom[idx]);
+
+ kbase_job_hw_submit(kbdev, katom[idx], js);
+ katom[idx]->gpu_rb_state =
+ KBASE_ATOM_GPU_RB_SUBMITTED;
+
+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+
+ case KBASE_ATOM_GPU_RB_SUBMITTED:
+ /* Atom submitted to HW, nothing else to do */
+ break;
+
+ case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
+ /* Only return if head atom or previous atom
+ * already removed - as atoms must be returned
+ * in order */
+ if (idx == 0 || katom[0]->gpu_rb_state ==
+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
+ kbase_gpu_dequeue_atom(kbdev, js);
+ kbase_jm_return_atom_to_js(kbdev,
+ katom[idx]);
+ }
+ break;
+ }
+ }
+ }
+
+ /* Warn if PRLAM-8987 affinity restrictions are violated */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
+ WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) ||
+ kbase_gpu_atoms_submitted(kbdev, 1)) &&
+ kbase_gpu_atoms_submitted(kbdev, 2));
+}
+
+
+void kbase_backend_run_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ kbase_gpu_enqueue_atom(kbdev, katom);
+ kbase_gpu_slot_update(kbdev);
+}
+
+bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js)
+{
+ struct kbase_jd_atom *katom;
+ struct kbase_jd_atom *next_katom;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ katom = kbase_gpu_inspect(kbdev, js, 0);
+ next_katom = kbase_gpu_inspect(kbdev, js, 1);
+
+ if (next_katom && katom->kctx == next_katom->kctx &&
+ next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED &&
+ (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL)
+ != 0 ||
+ kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL)
+ != 0)) {
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
+ JS_COMMAND_NOP, NULL);
+ next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
+ return true;
+ }
+
+ return false;
+}
+
+void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
+ u32 completion_code,
+ u64 job_tail,
+ ktime_t *end_timestamp)
+{
+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0);
+ struct kbase_context *kctx = katom->kctx;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) &&
+ completion_code != BASE_JD_EVENT_DONE &&
+ !(completion_code & BASE_JD_SW_EVENT)) {
+ katom->need_cache_flush_cores_retained = katom->affinity;
+ kbase_pm_request_cores(kbdev, false, katom->affinity);
+ } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) {
+ if (kbdev->gpu_props.num_core_groups > 1 &&
+ !(katom->affinity &
+ kbdev->gpu_props.props.coherency_info.group[0].core_mask
+ ) &&
+ (katom->affinity &
+ kbdev->gpu_props.props.coherency_info.group[1].core_mask
+ )) {
+ dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n");
+ katom->need_cache_flush_cores_retained =
+ katom->affinity;
+ kbase_pm_request_cores(kbdev, false,
+ katom->affinity);
+ }
+ }
+
+ katom = kbase_gpu_dequeue_atom(kbdev, js);
+
+ kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0);
+
+ if (completion_code == BASE_JD_EVENT_STOPPED) {
+ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
+ 0);
+
+ /*
+ * Dequeue next atom from ringbuffers on same slot if required.
+ * This atom will already have been removed from the NEXT
+ * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that
+ * the atoms on this slot are returned in the correct order.
+ */
+ if (next_katom && katom->kctx == next_katom->kctx) {
+ kbase_gpu_dequeue_atom(kbdev, js);
+ kbase_jm_return_atom_to_js(kbdev, next_katom);
+ }
+ } else if (completion_code != BASE_JD_EVENT_DONE) {
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ int i;
+
+#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
+ KBASE_TRACE_DUMP(kbdev);
+#endif
+ kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
+
+ /*
+ * Remove all atoms on the same context from ringbuffers. This
+ * will not remove atoms that are already on the GPU, as these
+ * are guaranteed not to have fail dependencies on the failed
+ * atom.
+ */
+ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
+ struct kbase_jd_atom *katom_idx0 =
+ kbase_gpu_inspect(kbdev, i, 0);
+ struct kbase_jd_atom *katom_idx1 =
+ kbase_gpu_inspect(kbdev, i, 1);
+
+ if (katom_idx0 && katom_idx0->kctx == katom->kctx &&
+ katom_idx0->gpu_rb_state !=
+ KBASE_ATOM_GPU_RB_SUBMITTED) {
+ /* Dequeue katom_idx0 from ringbuffer */
+ kbase_gpu_dequeue_atom(kbdev, i);
+
+ if (katom_idx1 &&
+ katom_idx1->kctx == katom->kctx &&
+ katom_idx0->gpu_rb_state !=
+ KBASE_ATOM_GPU_RB_SUBMITTED) {
+ /* Dequeue katom_idx1 from ringbuffer */
+ kbase_gpu_dequeue_atom(kbdev, i);
+
+ katom_idx1->event_code =
+ BASE_JD_EVENT_STOPPED;
+ kbase_jm_return_atom_to_js(kbdev,
+ katom_idx1);
+ }
+ katom_idx0->event_code = BASE_JD_EVENT_STOPPED;
+ kbase_jm_return_atom_to_js(kbdev, katom_idx0);
+
+ } else if (katom_idx1 &&
+ katom_idx1->kctx == katom->kctx &&
+ katom_idx1->gpu_rb_state !=
+ KBASE_ATOM_GPU_RB_SUBMITTED) {
+ /* Can not dequeue this atom yet - will be
+ * dequeued when atom at idx0 completes */
+ katom_idx1->event_code = BASE_JD_EVENT_STOPPED;
+ kbase_gpu_mark_atom_for_return(kbdev,
+ katom_idx1);
+ }
+ }
+ }
+
+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc,
+ js, completion_code);
+
+ if (job_tail != 0 && job_tail != katom->jc) {
+ bool was_updated = (job_tail != katom->jc);
+
+ /* Some of the job has been executed, so we update the job chain
+ * address to where we should resume from */
+ katom->jc = job_tail;
+ if (was_updated)
+ KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
+ katom, job_tail, js);
+ }
+
+ /* Only update the event code for jobs that weren't cancelled */
+ if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
+ katom->event_code = (base_jd_event_code)completion_code;
+
+ kbase_device_trace_register_access(kctx, REG_WRITE,
+ JOB_CONTROL_REG(JOB_IRQ_CLEAR),
+ 1 << js);
+
+ /* Complete the job, and start new ones
+ *
+ * Also defer remaining work onto the workqueue:
+ * - Re-queue Soft-stopped jobs
+ * - For any other jobs, queue the job back into the dependency system
+ * - Schedule out the parent context if necessary, and schedule a new
+ * one in.
+ */
+#ifdef CONFIG_GPU_TRACEPOINTS
+ {
+ /* The atom in the HEAD */
+ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
+ 0);
+
+ if (next_katom && next_katom->gpu_rb_state ==
+ KBASE_ATOM_GPU_RB_SUBMITTED) {
+ char js_string[16];
+
+ trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
+ js_string),
+ ktime_to_ns(*end_timestamp),
+ (u32)next_katom->kctx, 0,
+ next_katom->work_id);
+ kbdev->hwaccess.backend.slot_rb[js].last_context =
+ next_katom->kctx;
+ } else {
+ char js_string[16];
+
+ trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
+ js_string),
+ ktime_to_ns(ktime_get()), 0, 0,
+ 0);
+ kbdev->hwaccess.backend.slot_rb[js].last_context = 0;
+ }
+ }
+#endif
+
+ if (completion_code == BASE_JD_EVENT_STOPPED)
+ kbase_jm_return_atom_to_js(kbdev, katom);
+ else
+ kbase_jm_complete(kbdev, katom, end_timestamp);
+
+ /* Job completion may have unblocked other atoms. Try to update all job
+ * slots */
+ kbase_gpu_slot_update(kbdev);
+}
+
+void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
+{
+ int js;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
+ int idx;
+
+ for (idx = 0; idx < 2; idx++) {
+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
+ js, 0);
+
+ if (katom) {
+ enum kbase_atom_gpu_rb_state gpu_rb_state =
+ katom->gpu_rb_state;
+
+ kbase_gpu_release_atom(kbdev, katom);
+ kbase_gpu_dequeue_atom(kbdev, js);
+
+ if (gpu_rb_state ==
+ KBASE_ATOM_GPU_RB_SUBMITTED) {
+ katom->event_code =
+ BASE_JD_EVENT_JOB_CANCELLED;
+ kbase_jm_complete(kbdev, katom,
+ end_timestamp);
+ } else {
+ katom->event_code =
+ BASE_JD_EVENT_STOPPED;
+ kbase_jm_return_atom_to_js(kbdev,
+ katom);
+ }
+ }
+ }
+ }
+}
+
+static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev,
+ int js,
+ struct kbase_jd_atom *katom,
+ u32 action)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ u32 hw_action = action & JS_COMMAND_MASK;
+
+ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom);
+ kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action,
+ katom->core_req, katom);
+ kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
+}
+
+static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom,
+ u32 action,
+ bool disjoint)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+
+ katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
+ kbase_gpu_mark_atom_for_return(kbdev, katom);
+ kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
+
+ if (disjoint)
+ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req,
+ katom);
+}
+
+static int should_stop_x_dep_slot(struct kbase_jd_atom *katom)
+{
+ if (katom->x_post_dep) {
+ struct kbase_jd_atom *dep_atom = katom->x_post_dep;
+
+ if (dep_atom->gpu_rb_state !=
+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB &&
+ dep_atom->gpu_rb_state !=
+ KBASE_ATOM_GPU_RB_RETURN_TO_JS)
+ return dep_atom->slot_nr;
+ }
+ return -1;
+}
+
+bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ int js,
+ struct kbase_jd_atom *katom,
+ u32 action)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+
+ struct kbase_jd_atom *katom_idx0;
+ struct kbase_jd_atom *katom_idx1;
+
+ bool katom_idx0_valid, katom_idx1_valid;
+
+ bool ret = false;
+
+ int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ katom_idx0 = kbase_gpu_inspect(kbdev, js, 0);
+ katom_idx1 = kbase_gpu_inspect(kbdev, js, 1);
+
+ if (katom) {
+ katom_idx0_valid = (katom_idx0 == katom);
+ /* If idx0 is to be removed and idx1 is on the same context,
+ * then idx1 must also be removed otherwise the atoms might be
+ * returned out of order */
+ if (katom_idx1)
+ katom_idx1_valid = (katom_idx1 == katom) ||
+ (katom_idx0_valid &&
+ (katom_idx0->kctx ==
+ katom_idx1->kctx));
+ else
+ katom_idx1_valid = false;
+ } else {
+ katom_idx0_valid = (katom_idx0 &&
+ (!kctx || katom_idx0->kctx == kctx));
+ katom_idx1_valid = (katom_idx1 &&
+ (!kctx || katom_idx1->kctx == kctx));
+ }
+
+ if (katom_idx0_valid)
+ stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0);
+ if (katom_idx1_valid)
+ stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1);
+
+ if (katom_idx0_valid) {
+ if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
+ /* Simple case - just dequeue and return */
+ kbase_gpu_dequeue_atom(kbdev, js);
+ if (katom_idx1_valid) {
+ kbase_gpu_dequeue_atom(kbdev, js);
+ katom_idx1->event_code =
+ BASE_JD_EVENT_REMOVED_FROM_NEXT;
+ kbase_jm_return_atom_to_js(kbdev, katom_idx1);
+ kbasep_js_clear_submit_allowed(js_devdata,
+ katom_idx1->kctx);
+ }
+
+ katom_idx0->event_code =
+ BASE_JD_EVENT_REMOVED_FROM_NEXT;
+ kbase_jm_return_atom_to_js(kbdev, katom_idx0);
+ kbasep_js_clear_submit_allowed(js_devdata,
+ katom_idx0->kctx);
+ } else {
+ /* katom_idx0 is on GPU */
+ if (katom_idx1 && katom_idx1->gpu_rb_state ==
+ KBASE_ATOM_GPU_RB_SUBMITTED) {
+ /* katom_idx0 and katom_idx1 are on GPU */
+
+ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
+ JS_COMMAND_NEXT), NULL) == 0) {
+ /* idx0 has already completed - stop
+ * idx1 if needed*/
+ if (katom_idx1_valid) {
+ kbase_gpu_stop_atom(kbdev, js,
+ katom_idx1,
+ action);
+ ret = true;
+ }
+ } else {
+ /* idx1 is in NEXT registers - attempt
+ * to remove */
+ kbase_reg_write(kbdev,
+ JOB_SLOT_REG(js,
+ JS_COMMAND_NEXT),
+ JS_COMMAND_NOP, NULL);
+
+ if (kbase_reg_read(kbdev,
+ JOB_SLOT_REG(js,
+ JS_HEAD_NEXT_LO), NULL)
+ != 0 ||
+ kbase_reg_read(kbdev,
+ JOB_SLOT_REG(js,
+ JS_HEAD_NEXT_HI), NULL)
+ != 0) {
+ /* idx1 removed successfully,
+ * will be handled in IRQ */
+ kbase_gpu_remove_atom(kbdev,
+ katom_idx1,
+ action, true);
+ stop_x_dep_idx1 =
+ should_stop_x_dep_slot(katom_idx1);
+
+ /* stop idx0 if still on GPU */
+ kbase_gpu_stop_atom(kbdev, js,
+ katom_idx0,
+ action);
+ ret = true;
+ } else if (katom_idx1_valid) {
+ /* idx0 has already completed,
+ * stop idx1 if needed */
+ kbase_gpu_stop_atom(kbdev, js,
+ katom_idx1,
+ action);
+ ret = true;
+ }
+ }
+ } else if (katom_idx1_valid) {
+ /* idx1 not on GPU but must be dequeued*/
+
+ /* idx1 will be handled in IRQ */
+ kbase_gpu_remove_atom(kbdev, katom_idx1, action,
+ false);
+ /* stop idx0 */
+ /* This will be repeated for anything removed
+ * from the next registers, since their normal
+ * flow was also interrupted, and this function
+ * might not enter disjoint state e.g. if we
+ * don't actually do a hard stop on the head
+ * atom */
+ kbase_gpu_stop_atom(kbdev, js, katom_idx0,
+ action);
+ ret = true;
+ } else {
+ /* no atom in idx1 */
+ /* just stop idx0 */
+ kbase_gpu_stop_atom(kbdev, js, katom_idx0,
+ action);
+ ret = true;
+ }
+ }
+ } else if (katom_idx1_valid) {
+ if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
+ /* Mark for return */
+ /* idx1 will be returned once idx0 completes */
+ kbase_gpu_remove_atom(kbdev, katom_idx1, action,
+ false);
+ } else {
+ /* idx1 is on GPU */
+ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
+ JS_COMMAND_NEXT), NULL) == 0) {
+ /* idx0 has already completed - stop idx1 */
+ kbase_gpu_stop_atom(kbdev, js, katom_idx1,
+ action);
+ ret = true;
+ } else {
+ /* idx1 is in NEXT registers - attempt to
+ * remove */
+ kbase_reg_write(kbdev, JOB_SLOT_REG(js,
+ JS_COMMAND_NEXT),
+ JS_COMMAND_NOP, NULL);
+
+ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
+ JS_HEAD_NEXT_LO), NULL) != 0 ||
+ kbase_reg_read(kbdev, JOB_SLOT_REG(js,
+ JS_HEAD_NEXT_HI), NULL) != 0) {
+ /* idx1 removed successfully, will be
+ * handled in IRQ once idx0 completes */
+ kbase_gpu_remove_atom(kbdev, katom_idx1,
+ action,
+ false);
+ } else {
+ /* idx0 has already completed - stop
+ * idx1 */
+ kbase_gpu_stop_atom(kbdev, js,
+ katom_idx1,
+ action);
+ ret = true;
+ }
+ }
+ }
+ }
+
+
+ if (stop_x_dep_idx0 != -1)
+ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0,
+ NULL, action);
+
+ if (stop_x_dep_idx1 != -1)
+ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1,
+ NULL, action);
+
+ return ret;
+}
+
+static void kbasep_gpu_cacheclean(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ /* Limit the number of loops to avoid a hang if the interrupt is missed
+ */
+ u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
+
+ mutex_lock(&kbdev->cacheclean_lock);
+
+ /* use GPU_COMMAND completion solution */
+ /* clean & invalidate the caches */
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_CLEAN_INV_CACHES, NULL);
+
+ /* wait for cache flush to complete before continuing */
+ while (--max_loops &&
+ (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
+ CLEAN_CACHES_COMPLETED) == 0)
+ ;
+
+ /* clear the CLEAN_CACHES_COMPLETED irq */
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u,
+ CLEAN_CACHES_COMPLETED);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
+ CLEAN_CACHES_COMPLETED, NULL);
+ KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state !=
+ KBASE_INSTR_STATE_CLEANING,
+ "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang.");
+
+ mutex_unlock(&kbdev->cacheclean_lock);
+
+ kbase_pm_unrequest_cores(kbdev, false,
+ katom->need_cache_flush_cores_retained);
+}
+
+void kbase_backend_complete_wq(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ /*
+ * If cache flush required due to HW workaround then perform the flush
+ * now
+ */
+ if (katom->need_cache_flush_cores_retained) {
+ kbasep_gpu_cacheclean(kbdev, katom);
+ katom->need_cache_flush_cores_retained = 0;
+ }
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) &&
+ (katom->core_req & BASE_JD_REQ_FS) &&
+ katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT &&
+ (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) &&
+ !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) {
+ dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n");
+ if (kbasep_10969_workaround_clamp_coordinates(katom)) {
+ /* The job had a TILE_RANGE_FAULT after was soft-stopped
+ * Due to an HW issue we try to execute the job again.
+ */
+ dev_dbg(kbdev->dev,
+ "Clamping has been executed, try to rerun the job\n"
+ );
+ katom->event_code = BASE_JD_EVENT_STOPPED;
+ katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN;
+ }
+ }
+}
+
+void kbase_gpu_dump_slots(struct kbase_device *kbdev)
+{
+ struct kbasep_js_device_data *js_devdata;
+ unsigned long flags;
+ int js;
+
+ js_devdata = &kbdev->js_data;
+
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n");
+
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
+ int idx;
+
+ for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
+ js,
+ idx);
+
+ if (katom)
+ dev_info(kbdev->dev,
+ " js%d idx%d : katom=%p gpu_rb_state=%d\n",
+ js, idx, katom, katom->gpu_rb_state);
+ else
+ dev_info(kbdev->dev, " js%d idx%d : empty\n",
+ js, idx);
+ }
+ }
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+}
+
+
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/**
+ * @file mali_kbase_hwaccess_gpu.h
+ * Register-based HW access backend specific APIs
+ */
+
+#ifndef _KBASE_HWACCESS_GPU_H_
+#define _KBASE_HWACCESS_GPU_H_
+
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+/**
+ * Evict the atom in the NEXT slot for the specified job slot. This function is
+ * called from the job complete IRQ handler when the previous job has failed.
+ *
+ * @param[in] kbdev Device pointer
+ * @param[in] js Job slot to evict from
+ * @return true if job evicted from NEXT registers
+ * false otherwise
+ */
+bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js);
+
+/**
+ * Complete an atom on job slot js
+ *
+ * @param[in] kbdev Device pointer
+ * @param[in] js Job slot that has completed
+ * @param[in] event_code Event code from job that has completed
+ * @param[in] end_timestamp Time of completion
+ */
+void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
+ u32 completion_code,
+ u64 job_tail,
+ ktime_t *end_timestamp);
+
+/**
+ * Inspect the contents of the HW access ringbuffer
+ *
+ * @param[in] kbdev Device pointer
+ * @param[in] js Job slot to inspect
+ * @param[in] idx Index into ringbuffer. 0 is the job currently running on
+ * the slot, 1 is the job waiting, all other values are
+ * invalid.
+ * @return The atom at that position in the ringbuffer
+ * NULL if no atom present
+ */
+struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js,
+ int idx);
+
+/**
+ * Inspect the jobs in the slot ringbuffers and update state.
+ *
+ * This will cause jobs to be submitted to hardware if they are unblocked
+ *
+ * @param[in] kbdev Device pointer
+ */
+void kbase_gpu_slot_update(struct kbase_device *kbdev);
+
+/**
+ * Print the contents of the slot ringbuffers
+ *
+ * @param[in] kbdev Device pointer
+ */
+void kbase_gpu_dump_slots(struct kbase_device *kbdev);
+
+#endif /* _KBASE_HWACCESS_GPU_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_js_affinity.c
+ * Base kernel affinity manager APIs
+ */
+
+#include <mali_kbase.h>
+#include "mali_kbase_js_affinity.h"
+
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+
+bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev,
+ int js)
+{
+ /*
+ * Here are the reasons for using job slot 2:
+ * - BASE_HW_ISSUE_8987 (which is entirely used for that purpose)
+ * - In absence of the above, then:
+ * - Atoms with BASE_JD_REQ_COHERENT_GROUP
+ * - But, only when there aren't contexts with
+ * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, because the atoms that run on
+ * all cores on slot 1 could be blocked by those using a coherent group
+ * on slot 2
+ * - And, only when you actually have 2 or more coregroups - if you
+ * only have 1 coregroup, then having jobs for slot 2 implies they'd
+ * also be for slot 1, meaning you'll get interference from them. Jobs
+ * able to run on slot 2 could also block jobs that can only run on
+ * slot 1 (tiler jobs)
+ */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
+ return true;
+
+ if (js != 2)
+ return true;
+
+ /* Only deal with js==2 now: */
+ if (kbdev->gpu_props.num_core_groups > 1) {
+ /* Only use slot 2 in the 2+ coregroup case */
+ if (kbasep_js_ctx_attr_is_attr_on_runpool(kbdev,
+ KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) ==
+ false) {
+ /* ...But only when we *don't* have atoms that run on
+ * all cores */
+
+ /* No specific check for BASE_JD_REQ_COHERENT_GROUP
+ * atoms - the policy will sort that out */
+ return true;
+ }
+ }
+
+ /* Above checks failed mean we shouldn't use slot 2 */
+ return false;
+}
+
+/*
+ * As long as it has been decided to have a deeper modification of
+ * what job scheduler, power manager and affinity manager will
+ * implement, this function is just an intermediate step that
+ * assumes:
+ * - all working cores will be powered on when this is called.
+ * - largest current configuration is 2 core groups.
+ * - It has been decided not to have hardcoded values so the low
+ * and high cores in a core split will be evently distributed.
+ * - Odd combinations of core requirements have been filtered out
+ * and do not get to this function (e.g. CS+T+NSS is not
+ * supported here).
+ * - This function is frequently called and can be optimized,
+ * (see notes in loops), but as the functionallity will likely
+ * be modified, optimization has not been addressed.
+*/
+bool kbase_js_choose_affinity(u64 * const affinity,
+ struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom, int js)
+{
+ base_jd_core_req core_req = katom->core_req;
+ unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
+ u64 core_availability_mask;
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ core_availability_mask = kbase_pm_ca_get_core_mask(kbdev);
+
+ /*
+ * If no cores are currently available (core availability policy is
+ * transitioning) then fail.
+ */
+ if (0 == core_availability_mask) {
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+ *affinity = 0;
+ return false;
+ }
+
+ KBASE_DEBUG_ASSERT(js >= 0);
+
+ if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) ==
+ BASE_JD_REQ_T) {
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+ /* Tiler only job, bit 0 needed to enable tiler but no shader
+ * cores required */
+ *affinity = 1;
+ return true;
+ }
+
+ if (1 == kbdev->gpu_props.num_cores) {
+ /* trivial case only one core, nothing to do */
+ *affinity = core_availability_mask;
+ } else {
+ if ((core_req & (BASE_JD_REQ_COHERENT_GROUP |
+ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) {
+ if (js == 0 || num_core_groups == 1) {
+ /* js[0] and single-core-group systems just get
+ * the first core group */
+ *affinity =
+ kbdev->gpu_props.props.coherency_info.group[0].core_mask
+ & core_availability_mask;
+ } else {
+ /* js[1], js[2] use core groups 0, 1 for
+ * dual-core-group systems */
+ u32 core_group_idx = ((u32) js) - 1;
+
+ KBASE_DEBUG_ASSERT(core_group_idx <
+ num_core_groups);
+ *affinity =
+ kbdev->gpu_props.props.coherency_info.group[core_group_idx].core_mask
+ & core_availability_mask;
+
+ /* If the job is specifically targeting core
+ * group 1 and the core availability policy is
+ * keeping that core group off, then fail */
+ if (*affinity == 0 && core_group_idx == 1 &&
+ kbdev->pm.backend.cg1_disabled
+ == true)
+ katom->event_code =
+ BASE_JD_EVENT_PM_EVENT;
+ }
+ } else {
+ /* All cores are available when no core split is
+ * required */
+ *affinity = core_availability_mask;
+ }
+ }
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ /*
+ * If no cores are currently available in the desired core group(s)
+ * (core availability policy is transitioning) then fail.
+ */
+ if (*affinity == 0)
+ return false;
+
+ /* Enable core 0 if tiler required */
+ if (core_req & BASE_JD_REQ_T)
+ *affinity = *affinity | 1;
+
+ return true;
+}
+
+static inline bool kbase_js_affinity_is_violating(
+ struct kbase_device *kbdev,
+ u64 *affinities)
+{
+ /* This implementation checks whether the two slots involved in Generic
+ * thread creation have intersecting affinity. This is due to micro-
+ * architectural issues where a job in slot A targetting cores used by
+ * slot B could prevent the job in slot B from making progress until the
+ * job in slot A has completed.
+ */
+ u64 affinity_set_left;
+ u64 affinity_set_right;
+ u64 intersection;
+
+ KBASE_DEBUG_ASSERT(affinities != NULL);
+
+ affinity_set_left = affinities[1];
+
+ affinity_set_right = affinities[2];
+
+ /* A violation occurs when any bit in the left_set is also in the
+ * right_set */
+ intersection = affinity_set_left & affinity_set_right;
+
+ return (bool) (intersection != (u64) 0u);
+}
+
+bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js,
+ u64 affinity)
+{
+ struct kbasep_js_device_data *js_devdata;
+ u64 new_affinities[BASE_JM_MAX_NR_SLOTS];
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
+ js_devdata = &kbdev->js_data;
+
+ memcpy(new_affinities, js_devdata->runpool_irq.slot_affinities,
+ sizeof(js_devdata->runpool_irq.slot_affinities));
+
+ new_affinities[js] |= affinity;
+
+ return kbase_js_affinity_is_violating(kbdev, new_affinities);
+}
+
+void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js,
+ u64 affinity)
+{
+ struct kbasep_js_device_data *js_devdata;
+ u64 cores;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
+ js_devdata = &kbdev->js_data;
+
+ KBASE_DEBUG_ASSERT(kbase_js_affinity_would_violate(kbdev, js, affinity)
+ == false);
+
+ cores = affinity;
+ while (cores) {
+ int bitnum = fls64(cores) - 1;
+ u64 bit = 1ULL << bitnum;
+ s8 cnt;
+
+ cnt =
+ ++(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]);
+
+ if (cnt == 1)
+ js_devdata->runpool_irq.slot_affinities[js] |= bit;
+
+ cores &= ~bit;
+ }
+}
+
+void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js,
+ u64 affinity)
+{
+ struct kbasep_js_device_data *js_devdata;
+ u64 cores;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
+ js_devdata = &kbdev->js_data;
+
+ cores = affinity;
+ while (cores) {
+ int bitnum = fls64(cores) - 1;
+ u64 bit = 1ULL << bitnum;
+ s8 cnt;
+
+ KBASE_DEBUG_ASSERT(
+ js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum] > 0);
+
+ cnt =
+ --(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]);
+
+ if (0 == cnt)
+ js_devdata->runpool_irq.slot_affinities[js] &= ~bit;
+
+ cores &= ~bit;
+ }
+}
+
+#if KBASE_TRACE_ENABLE
+void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev)
+{
+ struct kbasep_js_device_data *js_devdata;
+ int slot_nr;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ js_devdata = &kbdev->js_data;
+
+ for (slot_nr = 0; slot_nr < 3; ++slot_nr)
+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_AFFINITY_CURRENT, NULL,
+ NULL, 0u, slot_nr,
+ (u32) js_devdata->runpool_irq.slot_affinities[slot_nr]);
+}
+#endif /* KBASE_TRACE_ENABLE */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_js_affinity.h
+ * Affinity Manager internal APIs.
+ */
+
+#ifndef _KBASE_JS_AFFINITY_H_
+#define _KBASE_JS_AFFINITY_H_
+
+#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
+/* Import the external affinity mask variables */
+extern u64 mali_js0_affinity_mask;
+extern u64 mali_js1_affinity_mask;
+extern u64 mali_js2_affinity_mask;
+#endif /* CONFIG_MALI_DEBUG_SHADER_SPLIT_FS */
+
+
+/**
+ * @addtogroup base_api
+ * @{
+ */
+
+/**
+ * @addtogroup base_kbase_api
+ * @{
+ */
+
+/**
+ * @addtogroup kbase_js_affinity Affinity Manager internal APIs.
+ * @{
+ *
+ */
+
+/**
+ * @brief Decide whether it is possible to submit a job to a particular job slot
+ * in the current status
+ *
+ * Will check if submitting to the given job slot is allowed in the current
+ * status. For example using job slot 2 while in soft-stoppable state and only
+ * having 1 coregroup is not allowed by the policy. This function should be
+ * called prior to submitting a job to a slot to make sure policy rules are not
+ * violated.
+ *
+ * The following locking conditions are made on the caller:
+ * - it must hold kbasep_js_device_data::runpool_irq::lock
+ *
+ * @param kbdev The kbase device structure of the device
+ * @param js Job slot number to check for allowance
+ */
+bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev,
+ int js);
+
+/**
+ * @brief Compute affinity for a given job.
+ *
+ * Currently assumes an all-on/all-off power management policy.
+ * Also assumes there is at least one core with tiler available.
+ *
+ * Returns true if a valid affinity was chosen, false if
+ * no cores were available.
+ *
+ * @param[out] affinity Affinity bitmap computed
+ * @param kbdev The kbase device structure of the device
+ * @param katom Job chain of which affinity is going to be found
+ * @param js Slot the job chain is being submitted
+ */
+bool kbase_js_choose_affinity(u64 * const affinity,
+ struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom,
+ int js);
+
+/**
+ * @brief Determine whether a proposed \a affinity on job slot \a js would
+ * cause a violation of affinity restrictions.
+ *
+ * The following locks must be held by the caller:
+ * - kbasep_js_device_data::runpool_irq::lock
+ */
+bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js,
+ u64 affinity);
+
+/**
+ * @brief Affinity tracking: retain cores used by a slot
+ *
+ * The following locks must be held by the caller:
+ * - kbasep_js_device_data::runpool_irq::lock
+ */
+void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js,
+ u64 affinity);
+
+/**
+ * @brief Affinity tracking: release cores used by a slot
+ *
+ * Cores \b must be released as soon as a job is dequeued from a slot's 'submit
+ * slots', and before another job is submitted to those slots. Otherwise, the
+ * refcount could exceed the maximum number submittable to a slot,
+ * BASE_JM_SUBMIT_SLOTS.
+ *
+ * The following locks must be held by the caller:
+ * - kbasep_js_device_data::runpool_irq::lock
+ */
+void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js,
+ u64 affinity);
+
+/**
+ * @brief Output to the Trace log the current tracked affinities on all slots
+ */
+#if KBASE_TRACE_ENABLE
+void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev);
+#else /* KBASE_TRACE_ENABLE */
+static inline void
+kbase_js_debug_log_current_affinities(struct kbase_device *kbdev)
+{
+}
+#endif /* KBASE_TRACE_ENABLE */
+
+ /** @} *//* end group kbase_js_affinity */
+ /** @} *//* end group base_kbase_api */
+ /** @} *//* end group base_api */
+
+
+#endif /* _KBASE_JS_AFFINITY_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/*
+ * Register-based HW access backend specific job scheduler APIs
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_hwaccess_jm.h>
+#include <backend/gpu/mali_kbase_jm_internal.h>
+#include <backend/gpu/mali_kbase_js_internal.h>
+
+/*
+ * Define for when dumping is enabled.
+ * This should not be based on the instrumentation level as whether dumping is
+ * enabled for a particular level is down to the integrator. However this is
+ * being used for now as otherwise the cinstr headers would be needed.
+ */
+#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL)
+
+/*
+ * Hold the runpool_mutex for this
+ */
+static inline bool timer_callback_should_run(struct kbase_device *kbdev)
+{
+ s8 nr_running_ctxs;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_mutex);
+
+ /* nr_contexts_pullable is updated with the runpool_mutex. However, the
+ * locking in the caller gives us a barrier that ensures
+ * nr_contexts_pullable is up-to-date for reading */
+ nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable);
+
+#ifdef CONFIG_MALI_DEBUG
+ if (kbdev->js_data.softstop_always) {
+ /* Debug support for allowing soft-stop on a single context */
+ return true;
+ }
+#endif /* CONFIG_MALI_DEBUG */
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) {
+ /* Timeouts would have to be 4x longer (due to micro-
+ * architectural design) to support OpenCL conformance tests, so
+ * only run the timer when there's:
+ * - 2 or more CL contexts
+ * - 1 or more GLES contexts
+ *
+ * NOTE: We will treat a context that has both Compute and Non-
+ * Compute jobs will be treated as an OpenCL context (hence, we
+ * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE).
+ */
+ {
+ s8 nr_compute_ctxs =
+ kbasep_js_ctx_attr_count_on_runpool(kbdev,
+ KBASEP_JS_CTX_ATTR_COMPUTE);
+ s8 nr_noncompute_ctxs = nr_running_ctxs -
+ nr_compute_ctxs;
+
+ return (bool) (nr_compute_ctxs >= 2 ||
+ nr_noncompute_ctxs > 0);
+ }
+ } else {
+ /* Run the timer callback whenever you have at least 1 context
+ */
+ return (bool) (nr_running_ctxs > 0);
+ }
+}
+
+static enum hrtimer_restart timer_callback(struct hrtimer *timer)
+{
+ unsigned long flags;
+ struct kbase_device *kbdev;
+ struct kbasep_js_device_data *js_devdata;
+ struct kbase_backend_data *backend;
+ int s;
+ bool reset_needed = false;
+
+ KBASE_DEBUG_ASSERT(timer != NULL);
+
+ backend = container_of(timer, struct kbase_backend_data,
+ scheduling_timer);
+ kbdev = container_of(backend, struct kbase_device, hwaccess.backend);
+ js_devdata = &kbdev->js_data;
+
+ /* Loop through the slots */
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) {
+ struct kbase_jd_atom *atom = NULL;
+
+ if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) {
+ atom = kbase_gpu_inspect(kbdev, s, 0);
+ KBASE_DEBUG_ASSERT(atom != NULL);
+ }
+
+ if (atom != NULL) {
+ /* The current version of the model doesn't support
+ * Soft-Stop */
+ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) {
+ u32 ticks = atom->sched_info.cfs.ticks++;
+
+#if !CINSTR_DUMPING_ENABLED
+ u32 soft_stop_ticks, hard_stop_ticks,
+ gpu_reset_ticks;
+ if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
+ soft_stop_ticks =
+ js_devdata->soft_stop_ticks_cl;
+ hard_stop_ticks =
+ js_devdata->hard_stop_ticks_cl;
+ gpu_reset_ticks =
+ js_devdata->gpu_reset_ticks_cl;
+ } else {
+ soft_stop_ticks =
+ js_devdata->soft_stop_ticks;
+ hard_stop_ticks =
+ js_devdata->hard_stop_ticks_ss;
+ gpu_reset_ticks =
+ js_devdata->gpu_reset_ticks_ss;
+ }
+
+ /* Job is Soft-Stoppable */
+ if (ticks == soft_stop_ticks) {
+ int disjoint_threshold =
+ KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD;
+ u32 softstop_flags = 0u;
+ /* Job has been scheduled for at least
+ * js_devdata->soft_stop_ticks ticks.
+ * Soft stop the slot so we can run
+ * other jobs.
+ */
+ dev_dbg(kbdev->dev, "Soft-stop");
+#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS
+ /* nr_user_contexts_running is updated
+ * with the runpool_mutex, but we can't
+ * take that here.
+ *
+ * However, if it's about to be
+ * increased then the new context can't
+ * run any jobs until they take the
+ * runpool_irq lock, so it's OK to
+ * observe the older value.
+ *
+ * Similarly, if it's about to be
+ * decreased, the last job from another
+ * context has already finished, so it's
+ * not too bad that we observe the older
+ * value and register a disjoint event
+ * when we try soft-stopping */
+ if (js_devdata->nr_user_contexts_running
+ >= disjoint_threshold)
+ softstop_flags |=
+ JS_COMMAND_SW_CAUSES_DISJOINT;
+
+ kbase_job_slot_softstop_swflags(kbdev,
+ s, atom, softstop_flags);
+#endif
+ } else if (ticks == hard_stop_ticks) {
+ /* Job has been scheduled for at least
+ * js_devdata->hard_stop_ticks_ss ticks.
+ * It should have been soft-stopped by
+ * now. Hard stop the slot.
+ */
+#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
+ int ms =
+ js_devdata->scheduling_period_ns
+ / 1000000u;
+ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
+ (unsigned long)ticks,
+ (unsigned long)ms);
+ kbase_job_slot_hardstop(atom->kctx, s,
+ atom);
+#endif
+ } else if (ticks == gpu_reset_ticks) {
+ /* Job has been scheduled for at least
+ * js_devdata->gpu_reset_ticks_ss ticks.
+ * It should have left the GPU by now.
+ * Signal that the GPU needs to be
+ * reset.
+ */
+ reset_needed = true;
+ }
+#else /* !CINSTR_DUMPING_ENABLED */
+ /* NOTE: During CINSTR_DUMPING_ENABLED, we use
+ * the alternate timeouts, which makes the hard-
+ * stop and GPU reset timeout much longer. We
+ * also ensure that we don't soft-stop at all.
+ */
+ if (ticks == js_devdata->soft_stop_ticks) {
+ /* Job has been scheduled for at least
+ * js_devdata->soft_stop_ticks. We do
+ * not soft-stop during
+ * CINSTR_DUMPING_ENABLED, however.
+ */
+ dev_dbg(kbdev->dev, "Soft-stop");
+ } else if (ticks ==
+ js_devdata->hard_stop_ticks_dumping) {
+ /* Job has been scheduled for at least
+ * js_devdata->hard_stop_ticks_dumping
+ * ticks. Hard stop the slot.
+ */
+#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
+ int ms =
+ js_devdata->scheduling_period_ns
+ / 1000000u;
+ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
+ (unsigned long)ticks,
+ (unsigned long)ms);
+ kbase_job_slot_hardstop(atom->kctx, s,
+ atom);
+#endif
+ } else if (ticks ==
+ js_devdata->gpu_reset_ticks_dumping) {
+ /* Job has been scheduled for at least
+ * js_devdata->gpu_reset_ticks_dumping
+ * ticks. It should have left the GPU by
+ * now. Signal that the GPU needs to be
+ * reset.
+ */
+ reset_needed = true;
+ }
+#endif /* !CINSTR_DUMPING_ENABLED */
+ }
+ }
+ }
+#if KBASE_GPU_RESET_EN
+ if (reset_needed) {
+ dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve.");
+
+ if (kbase_prepare_to_reset_gpu_locked(kbdev))
+ kbase_reset_gpu_locked(kbdev);
+ }
+#endif /* KBASE_GPU_RESET_EN */
+ /* the timer is re-issued if there is contexts in the run-pool */
+
+ if (backend->timer_running)
+ hrtimer_start(&backend->scheduling_timer,
+ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
+ HRTIMER_MODE_REL);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ return HRTIMER_NORESTART;
+}
+
+void kbase_backend_ctx_count_changed(struct kbase_device *kbdev)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
+ unsigned long flags;
+
+ lockdep_assert_held(&js_devdata->runpool_mutex);
+
+ if (!timer_callback_should_run(kbdev)) {
+ /* Take spinlock to force synchronisation with timer */
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ backend->timer_running = false;
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ /* From now on, return value of timer_callback_should_run() will
+ * also cause the timer to not requeue itself. Its return value
+ * cannot change, because it depends on variables updated with
+ * the runpool_mutex held, which the caller of this must also
+ * hold */
+ hrtimer_cancel(&backend->scheduling_timer);
+ }
+
+ if (timer_callback_should_run(kbdev) && !backend->timer_running) {
+ /* Take spinlock to force synchronisation with timer */
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ backend->timer_running = true;
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ hrtimer_start(&backend->scheduling_timer,
+ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
+ HRTIMER_MODE_REL);
+
+ KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u,
+ 0u);
+ }
+}
+
+int kbase_backend_timer_init(struct kbase_device *kbdev)
+{
+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
+
+ hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC,
+ HRTIMER_MODE_REL);
+ backend->scheduling_timer.function = timer_callback;
+
+ backend->timer_running = false;
+
+ return 0;
+}
+
+void kbase_backend_timer_term(struct kbase_device *kbdev)
+{
+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
+
+ hrtimer_cancel(&backend->scheduling_timer);
+}
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/*
+ * Register-based HW access backend specific job scheduler APIs
+ */
+
+#ifndef _KBASE_JS_BACKEND_H_
+#define _KBASE_JS_BACKEND_H_
+
+/**
+ * kbase_backend_timer_init() - Initialise the JS scheduling timer
+ * @kbdev: Device pointer
+ *
+ * This function should be called at driver initialisation
+ *
+ * Return: 0 on success
+ */
+int kbase_backend_timer_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_backend_timer_term() - Terminate the JS scheduling timer
+ * @kbdev: Device pointer
+ *
+ * This function should be called at driver termination
+ */
+void kbase_backend_timer_term(struct kbase_device *kbdev);
+
+#endif /* _KBASE_JS_BACKEND_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include <linux/bitops.h>
+
+#include <mali_kbase.h>
+#include <mali_kbase_mem.h>
+#include <mali_kbase_mmu_hw.h>
+#include <backend/gpu/mali_kbase_mmu_hw_direct.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
+
+static inline u64 lock_region(struct kbase_device *kbdev, u64 pfn,
+ u32 num_pages)
+{
+ u64 region;
+
+ /* can't lock a zero sized range */
+ KBASE_DEBUG_ASSERT(num_pages);
+
+ region = pfn << PAGE_SHIFT;
+ /*
+ * fls returns (given the ASSERT above):
+ * 1 .. 32
+ *
+ * 10 + fls(num_pages)
+ * results in the range (11 .. 42)
+ */
+
+ /* gracefully handle num_pages being zero */
+ if (0 == num_pages) {
+ region |= 11;
+ } else {
+ u8 region_width;
+
+ region_width = 10 + fls(num_pages);
+ if (num_pages != (1ul << (region_width - 11))) {
+ /* not pow2, so must go up to the next pow2 */
+ region_width += 1;
+ }
+ KBASE_DEBUG_ASSERT(region_width <= KBASE_LOCK_REGION_MAX_SIZE);
+ KBASE_DEBUG_ASSERT(region_width >= KBASE_LOCK_REGION_MIN_SIZE);
+ region |= region_width;
+ }
+
+ return region;
+}
+
+static int wait_ready(struct kbase_device *kbdev,
+ unsigned int as_nr, struct kbase_context *kctx)
+{
+ unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
+
+ /* Wait for the MMU status to indicate there is no active command. */
+ while (--max_loops && kbase_reg_read(kbdev,
+ MMU_AS_REG(as_nr, AS_STATUS),
+ kctx) & AS_STATUS_AS_ACTIVE) {
+ ;
+ }
+
+ if (max_loops == 0) {
+ dev_err(kbdev->dev, "AS_ACTIVE bit stuck\n");
+ return -1;
+ }
+
+ return 0;
+}
+
+static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd,
+ struct kbase_context *kctx)
+{
+ int status;
+
+ /* write AS_COMMAND when MMU is ready to accept another command */
+ status = wait_ready(kbdev, as_nr, kctx);
+ if (status == 0)
+ kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd,
+ kctx);
+
+ return status;
+}
+
+void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat)
+{
+ const int num_as = 16;
+ const int busfault_shift = MMU_PAGE_FAULT_FLAGS;
+ const int pf_shift = 0;
+ const unsigned long as_bit_mask = (1UL << num_as) - 1;
+ unsigned long flags;
+ u32 new_mask;
+ u32 tmp;
+
+ /* bus faults */
+ u32 bf_bits = (irq_stat >> busfault_shift) & as_bit_mask;
+ /* page faults (note: Ignore ASes with both pf and bf) */
+ u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits;
+
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+
+ /* remember current mask */
+ spin_lock_irqsave(&kbdev->mmu_mask_change, flags);
+ new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL);
+ /* mask interrupts for now */
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL);
+ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
+
+ while (bf_bits | pf_bits) {
+ struct kbase_as *as;
+ int as_no;
+ struct kbase_context *kctx;
+
+ /*
+ * the while logic ensures we have a bit set, no need to check
+ * for not-found here
+ */
+ as_no = ffs(bf_bits | pf_bits) - 1;
+ as = &kbdev->as[as_no];
+
+ /*
+ * Refcount the kctx ASAP - it shouldn't disappear anyway, since
+ * Bus/Page faults _should_ only occur whilst jobs are running,
+ * and a job causing the Bus/Page fault shouldn't complete until
+ * the MMU is updated
+ */
+ kctx = kbasep_js_runpool_lookup_ctx(kbdev, as_no);
+
+ /* find faulting address */
+ as->fault_addr = kbase_reg_read(kbdev,
+ MMU_AS_REG(as_no,
+ AS_FAULTADDRESS_HI),
+ kctx);
+ as->fault_addr <<= 32;
+ as->fault_addr |= kbase_reg_read(kbdev,
+ MMU_AS_REG(as_no,
+ AS_FAULTADDRESS_LO),
+ kctx);
+
+ /* record the fault status */
+ as->fault_status = kbase_reg_read(kbdev,
+ MMU_AS_REG(as_no,
+ AS_FAULTSTATUS),
+ kctx);
+
+ /* find the fault type */
+ as->fault_type = (bf_bits & (1 << as_no)) ?
+ KBASE_MMU_FAULT_TYPE_BUS :
+ KBASE_MMU_FAULT_TYPE_PAGE;
+
+
+ if (kbase_as_has_bus_fault(as)) {
+ /* Mark bus fault as handled.
+ * Note that a bus fault is processed first in case
+ * where both a bus fault and page fault occur.
+ */
+ bf_bits &= ~(1UL << as_no);
+
+ /* remove the queued BF (and PF) from the mask */
+ new_mask &= ~(MMU_BUS_ERROR(as_no) |
+ MMU_PAGE_FAULT(as_no));
+ } else {
+ /* Mark page fault as handled */
+ pf_bits &= ~(1UL << as_no);
+
+ /* remove the queued PF from the mask */
+ new_mask &= ~MMU_PAGE_FAULT(as_no);
+ }
+
+ /* Process the interrupt for this address space */
+ spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
+ kbase_mmu_interrupt_process(kbdev, kctx, as);
+ spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock,
+ flags);
+ }
+
+ /* reenable interrupts */
+ spin_lock_irqsave(&kbdev->mmu_mask_change, flags);
+ tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL);
+ new_mask |= tmp;
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask, NULL);
+ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
+}
+
+void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as,
+ struct kbase_context *kctx)
+{
+ struct kbase_mmu_setup *current_setup = &as->current_setup;
+
+
+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO),
+ current_setup->transtab & 0xFFFFFFFFUL, kctx);
+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI),
+ (current_setup->transtab >> 32) & 0xFFFFFFFFUL, kctx);
+
+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO),
+ current_setup->memattr & 0xFFFFFFFFUL, kctx);
+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI),
+ (current_setup->memattr >> 32) & 0xFFFFFFFFUL, kctx);
+
+ write_cmd(kbdev, as->number, AS_COMMAND_UPDATE, kctx);
+}
+
+int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as,
+ struct kbase_context *kctx, u64 vpfn, u32 nr, u32 op,
+ unsigned int handling_irq)
+{
+ int ret;
+
+ if (op == AS_COMMAND_UNLOCK) {
+ /* Unlock doesn't require a lock first */
+ ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx);
+ } else {
+ u64 lock_addr = lock_region(kbdev, vpfn, nr);
+
+ /* Lock the region that needs to be updated */
+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_LO),
+ lock_addr & 0xFFFFFFFFUL, kctx);
+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_HI),
+ (lock_addr >> 32) & 0xFFFFFFFFUL, kctx);
+ write_cmd(kbdev, as->number, AS_COMMAND_LOCK, kctx);
+
+ /* Run the MMU operation */
+ write_cmd(kbdev, as->number, op, kctx);
+
+ /* Wait for the flush to complete */
+ ret = wait_ready(kbdev, as->number, kctx);
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9630)) {
+ /* Issue an UNLOCK command to ensure that valid page
+ tables are re-read by the GPU after an update.
+ Note that, the FLUSH command should perform all the
+ actions necessary, however the bus logs show that if
+ multiple page faults occur within an 8 page region
+ the MMU does not always re-read the updated page
+ table entries for later faults or is only partially
+ read, it subsequently raises the page fault IRQ for
+ the same addresses, the unlock ensures that the MMU
+ cache is flushed, so updates can be re-read. As the
+ region is now unlocked we need to issue 2 UNLOCK
+ commands in order to flush the MMU/uTLB,
+ see PRLAM-8812.
+ */
+ write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx);
+ write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx);
+ }
+ }
+
+ return ret;
+}
+
+void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as,
+ struct kbase_context *kctx, enum kbase_mmu_fault_type type)
+{
+ u32 pf_bf_mask;
+
+ /* Clear the page (and bus fault IRQ as well in case one occurred) */
+ pf_bf_mask = MMU_PAGE_FAULT(as->number);
+ if (type == KBASE_MMU_FAULT_TYPE_BUS)
+ pf_bf_mask |= MMU_BUS_ERROR(as->number);
+
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask, kctx);
+}
+
+void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as,
+ struct kbase_context *kctx, enum kbase_mmu_fault_type type)
+{
+ unsigned long flags;
+ u32 irq_mask;
+
+ /* Enable the page fault IRQ (and bus fault IRQ as well in case one
+ * occurred) */
+ spin_lock_irqsave(&kbdev->mmu_mask_change, flags);
+
+ irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), kctx) |
+ MMU_PAGE_FAULT(as->number);
+
+ if (type == KBASE_MMU_FAULT_TYPE_BUS)
+ irq_mask |= MMU_BUS_ERROR(as->number);
+
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask, kctx);
+
+ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
+}
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * @file
+ * Interface file for the direct implementation for MMU hardware access
+ */
+
+/**
+ * @page mali_kbase_mmu_hw_direct_page Direct MMU hardware interface
+ *
+ * @section mali_kbase_mmu_hw_direct_intro_sec Introduction
+ * This module provides the interface(s) that are required by the direct
+ * register access implementation of the MMU hardware interface
+ * @ref mali_kbase_mmu_hw_page .
+ */
+
+#ifndef _MALI_KBASE_MMU_HW_DIRECT_H_
+#define _MALI_KBASE_MMU_HW_DIRECT_H_
+
+#include <mali_kbase_defs.h>
+
+/**
+ * @addtogroup mali_kbase_mmu_hw
+ * @{
+ */
+
+/**
+ * @addtogroup mali_kbase_mmu_hw_direct Direct register access to MMU
+ * @{
+ */
+
+/** @brief Process an MMU interrupt.
+ *
+ * Process the MMU interrupt that was reported by the @ref kbase_device.
+ *
+ * @param[in] kbdev kbase context to clear the fault from.
+ * @param[in] irq_stat Value of the MMU_IRQ_STATUS register
+ */
+void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat);
+
+/** @} *//* end group mali_kbase_mmu_hw_direct */
+/** @} *//* end group mali_kbase_mmu_hw */
+
+#endif /* _MALI_KBASE_MMU_HW_DIRECT_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * "Always on" power management policy
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_pm.h>
+
+static u64 always_on_get_core_mask(struct kbase_device *kbdev)
+{
+ return kbdev->shader_present_bitmap;
+}
+
+static bool always_on_get_core_active(struct kbase_device *kbdev)
+{
+ return true;
+}
+
+static void always_on_init(struct kbase_device *kbdev)
+{
+ CSTD_UNUSED(kbdev);
+}
+
+static void always_on_term(struct kbase_device *kbdev)
+{
+ CSTD_UNUSED(kbdev);
+}
+
+/*
+ * The struct kbase_pm_policy structure for the demand power policy.
+ *
+ * This is the static structure that defines the demand power policy's callback
+ * and name.
+ */
+const struct kbase_pm_policy kbase_pm_always_on_policy_ops = {
+ "always_on", /* name */
+ always_on_init, /* init */
+ always_on_term, /* term */
+ always_on_get_core_mask, /* get_core_mask */
+ always_on_get_core_active, /* get_core_active */
+ 0u, /* flags */
+ KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */
+};
+
+KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops);
--- /dev/null
+
+/*
+ *
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * "Always on" power management policy
+ */
+
+#ifndef MALI_KBASE_PM_ALWAYS_ON_H
+#define MALI_KBASE_PM_ALWAYS_ON_H
+
+/**
+ * DOC:
+ * The "Always on" power management policy has the following
+ * characteristics:
+ *
+ * - When KBase indicates that the GPU will be powered up, but we don't yet
+ * know which Job Chains are to be run:
+ * All Shader Cores are powered up, regardless of whether or not they will
+ * be needed later.
+ *
+ * - When KBase indicates that a set of Shader Cores are needed to submit the
+ * currently queued Job Chains:
+ * All Shader Cores are kept powered, regardless of whether or not they will
+ * be needed
+ *
+ * - When KBase indicates that the GPU need not be powered:
+ * The Shader Cores are kept powered, regardless of whether or not they will
+ * be needed. The GPU itself is also kept powered, even though it is not
+ * needed.
+ *
+ * This policy is automatically overridden during system suspend: the desired
+ * core state is ignored, and the cores are forced off regardless of what the
+ * policy requests. After resuming from suspend, new changes to the desired
+ * core state made by the policy are honored.
+ *
+ * Note:
+ *
+ * - KBase indicates the GPU will be powered up when it has a User Process that
+ * has just started to submit Job Chains.
+ *
+ * - KBase indicates the GPU need not be powered when all the Job Chains from
+ * User Processes have finished, and it is waiting for a User Process to
+ * submit some more Job Chains.
+ */
+
+/**
+ * struct kbasep_pm_policy_always_on - Private struct for policy instance data
+ * @dummy: unused dummy variable
+ *
+ * This contains data that is private to the particular power policy that is
+ * active.
+ */
+struct kbasep_pm_policy_always_on {
+ int dummy;
+};
+
+extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops;
+
+#endif /* MALI_KBASE_PM_ALWAYS_ON_H */
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/**
+ * @file mali_kbase_pm_hwaccess.c
+ * GPU backend implementation of base kernel power management APIs
+ */
+
+#include <mali_kbase.h>
+#include <mali_midg_regmap.h>
+#include <mali_kbase_config_defaults.h>
+#ifdef CONFIG_MALI_PLATFORM_DEVICETREE
+#include <linux/pm_runtime.h>
+#endif /* CONFIG_MALI_PLATFORM_DEVICETREE */
+
+#include <mali_kbase_pm.h>
+#include <backend/gpu/mali_kbase_jm_internal.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+void kbase_pm_register_access_enable(struct kbase_device *kbdev)
+{
+ struct kbase_pm_callback_conf *callbacks;
+
+#ifdef CONFIG_MALI_PLATFORM_DEVICETREE
+ pm_runtime_enable(kbdev->dev);
+#endif /* CONFIG_MALI_PLATFORM_DEVICETREE */
+ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
+
+ if (callbacks)
+ callbacks->power_on_callback(kbdev);
+
+ kbdev->pm.backend.gpu_powered = true;
+}
+
+void kbase_pm_register_access_disable(struct kbase_device *kbdev)
+{
+ struct kbase_pm_callback_conf *callbacks;
+
+ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
+
+ if (callbacks)
+ callbacks->power_off_callback(kbdev);
+
+ kbdev->pm.backend.gpu_powered = false;
+#ifdef CONFIG_MALI_PLATFORM_DEVICETREE
+ pm_runtime_disable(kbdev->dev);
+#endif
+}
+
+int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
+{
+ int ret = 0;
+ struct kbase_pm_callback_conf *callbacks;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ mutex_init(&kbdev->pm.lock);
+
+ kbdev->pm.backend.gpu_powered = false;
+ kbdev->pm.suspending = false;
+#ifdef CONFIG_MALI_DEBUG
+ kbdev->pm.backend.driver_ready_for_irqs = false;
+#endif /* CONFIG_MALI_DEBUG */
+ kbdev->pm.backend.gpu_in_desired_state = true;
+ init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait);
+
+ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
+ if (callbacks) {
+ kbdev->pm.backend.callback_power_on =
+ callbacks->power_on_callback;
+ kbdev->pm.backend.callback_power_off =
+ callbacks->power_off_callback;
+ kbdev->pm.backend.callback_power_suspend =
+ callbacks->power_suspend_callback;
+ kbdev->pm.backend.callback_power_resume =
+ callbacks->power_resume_callback;
+ kbdev->pm.callback_power_runtime_init =
+ callbacks->power_runtime_init_callback;
+ kbdev->pm.callback_power_runtime_term =
+ callbacks->power_runtime_term_callback;
+ kbdev->pm.backend.callback_power_runtime_on =
+ callbacks->power_runtime_on_callback;
+ kbdev->pm.backend.callback_power_runtime_off =
+ callbacks->power_runtime_off_callback;
+ } else {
+ kbdev->pm.backend.callback_power_on = NULL;
+ kbdev->pm.backend.callback_power_off = NULL;
+ kbdev->pm.backend.callback_power_suspend = NULL;
+ kbdev->pm.backend.callback_power_resume = NULL;
+ kbdev->pm.callback_power_runtime_init = NULL;
+ kbdev->pm.callback_power_runtime_term = NULL;
+ kbdev->pm.backend.callback_power_runtime_on = NULL;
+ kbdev->pm.backend.callback_power_runtime_off = NULL;
+ }
+
+ /* Initialise the metrics subsystem */
+ ret = kbasep_pm_metrics_init(kbdev);
+ if (ret)
+ return ret;
+
+ init_waitqueue_head(&kbdev->pm.backend.l2_powered_wait);
+ kbdev->pm.backend.l2_powered = 0;
+
+ init_waitqueue_head(&kbdev->pm.backend.reset_done_wait);
+ kbdev->pm.backend.reset_done = false;
+
+ init_waitqueue_head(&kbdev->pm.zero_active_count_wait);
+ kbdev->pm.active_count = 0;
+
+ spin_lock_init(&kbdev->pm.power_change_lock);
+ spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock);
+ spin_lock_init(&kbdev->pm.backend.gpu_powered_lock);
+
+ if (kbase_pm_ca_init(kbdev) != 0)
+ goto workq_fail;
+
+ if (kbase_pm_policy_init(kbdev) != 0)
+ goto pm_policy_fail;
+
+ return 0;
+
+pm_policy_fail:
+ kbase_pm_ca_term(kbdev);
+workq_fail:
+ kbasep_pm_metrics_term(kbdev);
+ return -EINVAL;
+}
+
+void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume)
+{
+ lockdep_assert_held(&kbdev->pm.lock);
+
+ /* Turn clocks and interrupts on - no-op if we haven't done a previous
+ * kbase_pm_clock_off() */
+ kbase_pm_clock_on(kbdev, is_resume);
+
+ /* Update core status as required by the policy */
+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
+ SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START);
+ kbase_pm_update_cores_state(kbdev);
+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
+ SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END);
+
+ /* NOTE: We don't wait to reach the desired state, since running atoms
+ * will wait for that state to be reached anyway */
+}
+
+bool kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend)
+{
+ unsigned long flags;
+ bool cores_are_available;
+
+ lockdep_assert_held(&kbdev->pm.lock);
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ /* Force all cores off */
+ kbdev->pm.backend.desired_shader_state = 0;
+
+ /* Force all cores to be unavailable, in the situation where
+ * transitions are in progress for some cores but not others,
+ * and kbase_pm_check_transitions_nolock can not immediately
+ * power off the cores */
+ kbdev->shader_available_bitmap = 0;
+ kbdev->tiler_available_bitmap = 0;
+ kbdev->l2_available_bitmap = 0;
+
+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
+ SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START);
+ cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
+ SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END);
+ /* Don't need 'cores_are_available', because we don't return anything */
+ CSTD_UNUSED(cores_are_available);
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ /* NOTE: We won't wait to reach the core's desired state, even if we're
+ * powering off the GPU itself too. It's safe to cut the power whilst
+ * they're transitioning to off, because the cores should be idle and
+ * all cache flushes should already have occurred */
+
+ /* Consume any change-state events */
+ kbase_timeline_pm_check_handle_event(kbdev,
+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
+ /* Disable interrupts and turn the clock off */
+ return kbase_pm_clock_off(kbdev, is_suspend);
+}
+
+int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
+ unsigned int flags)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ unsigned long irq_flags;
+ int ret;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ mutex_lock(&js_devdata->runpool_mutex);
+ mutex_lock(&kbdev->pm.lock);
+
+ /* A suspend won't happen during startup/insmod */
+ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
+
+ /* Power up the GPU, don't enable IRQs as we are not ready to receive
+ * them. */
+ ret = kbase_pm_init_hw(kbdev, flags);
+ if (ret) {
+ mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
+ return ret;
+ }
+
+ kbasep_pm_read_present_cores(kbdev);
+
+ kbdev->pm.debug_core_mask = kbdev->shader_present_bitmap;
+
+ /* Pretend the GPU is active to prevent a power policy turning the GPU
+ * cores off */
+ kbdev->pm.active_count = 1;
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
+ irq_flags);
+ /* Ensure cycle counter is off */
+ kbdev->pm.backend.gpu_cycle_counter_requests = 0;
+ spin_unlock_irqrestore(
+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
+ irq_flags);
+
+ /* We are ready to receive IRQ's now as power policy is set up, so
+ * enable them now. */
+#ifdef CONFIG_MALI_DEBUG
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, irq_flags);
+ kbdev->pm.backend.driver_ready_for_irqs = true;
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, irq_flags);
+#endif
+ kbase_pm_enable_interrupts(kbdev);
+
+ /* Turn on the GPU and any cores needed by the policy */
+ kbase_pm_do_poweron(kbdev, false);
+ mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
+
+ /* Idle the GPU and/or cores, if the policy wants it to */
+ kbase_pm_context_idle(kbdev);
+
+ return 0;
+}
+
+void kbase_hwaccess_pm_halt(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ mutex_lock(&kbdev->pm.lock);
+ kbase_pm_cancel_deferred_poweroff(kbdev);
+ if (!kbase_pm_do_poweroff(kbdev, false)) {
+ /* Page/bus faults are pending, must drop pm.lock to process.
+ * Interrupts are disabled so no more faults should be
+ * generated at this point */
+ mutex_unlock(&kbdev->pm.lock);
+ kbase_flush_mmu_wqs(kbdev);
+ mutex_lock(&kbdev->pm.lock);
+ WARN_ON(!kbase_pm_do_poweroff(kbdev, false));
+ }
+ mutex_unlock(&kbdev->pm.lock);
+}
+
+KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt);
+
+void kbase_hwaccess_pm_term(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0);
+
+ /* Free any resources the policy allocated */
+ kbase_pm_policy_term(kbdev);
+ kbase_pm_ca_term(kbdev);
+
+ /* Shut down the metrics subsystem */
+ kbasep_pm_metrics_term(kbdev);
+}
+
+void kbase_pm_power_changed(struct kbase_device *kbdev)
+{
+ bool cores_are_available;
+ unsigned long flags;
+
+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
+ SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START);
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
+ SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END);
+
+ if (cores_are_available) {
+ /* Log timelining information that a change in state has
+ * completed */
+ kbase_timeline_pm_handle_event(kbdev,
+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
+
+ spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
+ kbase_gpu_slot_update(kbdev);
+ spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
+ }
+}
+
+void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask)
+{
+ kbdev->pm.debug_core_mask = new_core_mask;
+
+ kbase_pm_update_cores_state_nolock(kbdev);
+}
+
+void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev)
+{
+ kbase_pm_update_active(kbdev);
+}
+
+void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev)
+{
+ kbase_pm_update_active(kbdev);
+}
+
+void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+
+ /* Force power off the GPU and all cores (regardless of policy), only
+ * after the PM active count reaches zero (otherwise, we risk turning it
+ * off prematurely) */
+ mutex_lock(&js_devdata->runpool_mutex);
+ mutex_lock(&kbdev->pm.lock);
+ kbase_pm_cancel_deferred_poweroff(kbdev);
+ if (!kbase_pm_do_poweroff(kbdev, true)) {
+ /* Page/bus faults are pending, must drop pm.lock to process.
+ * Interrupts are disabled so no more faults should be
+ * generated at this point */
+ mutex_unlock(&kbdev->pm.lock);
+ kbase_flush_mmu_wqs(kbdev);
+ mutex_lock(&kbdev->pm.lock);
+ WARN_ON(!kbase_pm_do_poweroff(kbdev, false));
+ }
+
+ mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
+}
+
+void kbase_hwaccess_pm_resume(struct kbase_device *kbdev)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+
+ mutex_lock(&js_devdata->runpool_mutex);
+ mutex_lock(&kbdev->pm.lock);
+ kbdev->pm.suspending = false;
+ kbase_pm_do_poweron(kbdev, true);
+ mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
+}
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * @file mali_kbase_pm_ca.c
+ * Base kernel core availability APIs
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_pm.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+static const struct kbase_pm_ca_policy *const policy_list[] = {
+ &kbase_pm_ca_fixed_policy_ops,
+#if !MALI_CUSTOMER_RELEASE
+ &kbase_pm_ca_random_policy_ops
+#endif
+};
+
+/** The number of policies available in the system.
+ * This is derived from the number of functions listed in policy_get_functions.
+ */
+#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
+
+int kbase_pm_ca_init(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ kbdev->pm.backend.ca_current_policy = policy_list[0];
+
+ kbdev->pm.backend.ca_current_policy->init(kbdev);
+
+ return 0;
+}
+
+void kbase_pm_ca_term(struct kbase_device *kbdev)
+{
+ kbdev->pm.backend.ca_current_policy->term(kbdev);
+}
+
+int kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **list)
+{
+ if (!list)
+ return POLICY_COUNT;
+
+ *list = policy_list;
+
+ return POLICY_COUNT;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_ca_list_policies);
+
+const struct kbase_pm_ca_policy
+*kbase_pm_ca_get_policy(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ return kbdev->pm.backend.ca_current_policy;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_ca_get_policy);
+
+void kbase_pm_ca_set_policy(struct kbase_device *kbdev,
+ const struct kbase_pm_ca_policy *new_policy)
+{
+ const struct kbase_pm_ca_policy *old_policy;
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(new_policy != NULL);
+
+ KBASE_TRACE_ADD(kbdev, PM_CA_SET_POLICY, NULL, NULL, 0u,
+ new_policy->id);
+
+ /* During a policy change we pretend the GPU is active */
+ /* A suspend won't happen here, because we're in a syscall from a
+ * userspace thread */
+ kbase_pm_context_active(kbdev);
+
+ mutex_lock(&kbdev->pm.lock);
+
+ /* Remove the policy to prevent IRQ handlers from working on it */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ old_policy = kbdev->pm.backend.ca_current_policy;
+ kbdev->pm.backend.ca_current_policy = NULL;
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ if (old_policy->term)
+ old_policy->term(kbdev);
+
+ if (new_policy->init)
+ new_policy->init(kbdev);
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ kbdev->pm.backend.ca_current_policy = new_policy;
+
+ /* If any core power state changes were previously attempted, but
+ * couldn't be made because the policy was changing (current_policy was
+ * NULL), then re-try them here. */
+ kbase_pm_update_cores_state_nolock(kbdev);
+
+ kbdev->pm.backend.ca_current_policy->update_core_status(kbdev,
+ kbdev->shader_ready_bitmap,
+ kbdev->shader_transitioning_bitmap);
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ mutex_unlock(&kbdev->pm.lock);
+
+ /* Now the policy change is finished, we release our fake context active
+ * reference */
+ kbase_pm_context_idle(kbdev);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_ca_set_policy);
+
+u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev)
+{
+ lockdep_assert_held(&kbdev->pm.power_change_lock);
+
+ /* All cores must be enabled when instrumentation is in use */
+ if (kbdev->pm.backend.instr_enabled)
+ return kbdev->shader_present_bitmap & kbdev->pm.debug_core_mask;
+
+ if (kbdev->pm.backend.ca_current_policy == NULL)
+ return kbdev->shader_present_bitmap & kbdev->pm.debug_core_mask;
+
+ return kbdev->pm.backend.ca_current_policy->get_core_mask(kbdev) &
+ kbdev->pm.debug_core_mask;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask);
+
+void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready,
+ u64 cores_transitioning)
+{
+ lockdep_assert_held(&kbdev->pm.power_change_lock);
+
+ if (kbdev->pm.backend.ca_current_policy != NULL)
+ kbdev->pm.backend.ca_current_policy->update_core_status(kbdev,
+ cores_ready,
+ cores_transitioning);
+}
+
+void kbase_pm_ca_instr_enable(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ kbdev->pm.backend.instr_enabled = true;
+
+ kbase_pm_update_cores_state_nolock(kbdev);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+}
+
+void kbase_pm_ca_instr_disable(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ kbdev->pm.backend.instr_enabled = false;
+
+ kbase_pm_update_cores_state_nolock(kbdev);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+}
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * Base kernel core availability APIs
+ */
+
+#ifndef _KBASE_PM_CA_H_
+#define _KBASE_PM_CA_H_
+
+/**
+ * kbase_pm_ca_init - Initialize core availability framework
+ *
+ * Must be called before calling any other core availability function
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Return: 0 if the core availability framework was successfully initialized,
+ * -errno otherwise
+ */
+int kbase_pm_ca_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_ca_term - Terminate core availability framework
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ */
+void kbase_pm_ca_term(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_ca_get_core_mask - Get currently available shaders core mask
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Returns a mask of the currently available shader cores.
+ * Calls into the core availability policy
+ *
+ * Return: The bit mask of available cores
+ */
+u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_ca_update_core_status - Update core status
+ *
+ * @kbdev: The kbase device structure for the device (must be
+ * a valid pointer)
+ * @cores_ready: The bit mask of cores ready for job submission
+ * @cores_transitioning: The bit mask of cores that are transitioning power
+ * state
+ *
+ * Update core availability policy with current core power status
+ *
+ * Calls into the core availability policy
+ */
+void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready,
+ u64 cores_transitioning);
+
+/**
+ * kbase_pm_ca_instr_enable - Enable override for instrumentation
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * This overrides the output of the core availability policy, ensuring that all
+ * cores are available
+ */
+void kbase_pm_ca_instr_enable(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_ca_instr_disable - Disable override for instrumentation
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * This disables any previously enabled override, and resumes normal policy
+ * functionality
+ */
+void kbase_pm_ca_instr_disable(struct kbase_device *kbdev);
+
+#endif /* _KBASE_PM_CA_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * @file mali_kbase_pm_ca_fixed.c
+ * A power policy implementing fixed core availability
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_pm.h>
+
+static void fixed_init(struct kbase_device *kbdev)
+{
+ kbdev->pm.backend.ca_in_transition = false;
+}
+
+static void fixed_term(struct kbase_device *kbdev)
+{
+ CSTD_UNUSED(kbdev);
+}
+
+static u64 fixed_get_core_mask(struct kbase_device *kbdev)
+{
+ return kbdev->shader_present_bitmap;
+}
+
+static void fixed_update_core_status(struct kbase_device *kbdev,
+ u64 cores_ready,
+ u64 cores_transitioning)
+{
+ CSTD_UNUSED(kbdev);
+ CSTD_UNUSED(cores_ready);
+ CSTD_UNUSED(cores_transitioning);
+}
+
+/** The @ref struct kbase_pm_policy structure for the fixed power policy.
+ *
+ * This is the static structure that defines the fixed power policy's callback
+ * and name.
+ */
+const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops = {
+ "fixed", /* name */
+ fixed_init, /* init */
+ fixed_term, /* term */
+ fixed_get_core_mask, /* get_core_mask */
+ fixed_update_core_status, /* update_core_status */
+ 0u, /* flags */
+ KBASE_PM_CA_POLICY_ID_FIXED, /* id */
+};
+
+KBASE_EXPORT_TEST_API(kbase_pm_ca_fixed_policy_ops);
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * A power policy implementing fixed core availability
+ */
+
+#ifndef MALI_KBASE_PM_CA_FIXED_H
+#define MALI_KBASE_PM_CA_FIXED_H
+
+/**
+ * struct kbasep_pm_ca_policy_fixed - Private structure for policy instance data
+ *
+ * @dummy: Dummy member - no state is needed
+ *
+ * This contains data that is private to the particular power policy that is
+ * active.
+ */
+struct kbasep_pm_ca_policy_fixed {
+ int dummy;
+};
+
+extern const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops;
+
+#endif /* MALI_KBASE_PM_CA_FIXED_H */
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_pm_coarse_demand.c
+ * "Coarse Demand" power management policy
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_pm.h>
+
+static u64 coarse_demand_get_core_mask(struct kbase_device *kbdev)
+{
+ if (kbdev->pm.active_count == 0)
+ return 0;
+
+ return kbdev->shader_present_bitmap;
+}
+
+static bool coarse_demand_get_core_active(struct kbase_device *kbdev)
+{
+ if (kbdev->pm.active_count == 0)
+ return false;
+
+ return true;
+}
+
+static void coarse_demand_init(struct kbase_device *kbdev)
+{
+ CSTD_UNUSED(kbdev);
+}
+
+static void coarse_demand_term(struct kbase_device *kbdev)
+{
+ CSTD_UNUSED(kbdev);
+}
+
+/** The @ref struct kbase_pm_policy structure for the demand power policy.
+ *
+ * This is the static structure that defines the demand power policy's callback
+ * and name.
+ */
+const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = {
+ "coarse_demand", /* name */
+ coarse_demand_init, /* init */
+ coarse_demand_term, /* term */
+ coarse_demand_get_core_mask, /* get_core_mask */
+ coarse_demand_get_core_active, /* get_core_active */
+ 0u, /* flags */
+ KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */
+};
+
+KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops);
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_pm_coarse_demand.h
+ * "Coarse Demand" power management policy
+ */
+
+#ifndef MALI_KBASE_PM_COARSE_DEMAND_H
+#define MALI_KBASE_PM_COARSE_DEMAND_H
+
+/**
+ * The "Coarse" demand power management policy has the following
+ * characteristics:
+ * - When KBase indicates that the GPU will be powered up, but we don't yet
+ * know which Job Chains are to be run:
+ * - All Shader Cores are powered up, regardless of whether or not they will
+ * be needed later.
+ * - When KBase indicates that a set of Shader Cores are needed to submit the
+ * currently queued Job Chains:
+ * - All Shader Cores are kept powered, regardless of whether or not they will
+ * be needed
+ * - When KBase indicates that the GPU need not be powered:
+ * - The Shader Cores are powered off, and the GPU itself is powered off too.
+ *
+ * @note:
+ * - KBase indicates the GPU will be powered up when it has a User Process that
+ * has just started to submit Job Chains.
+ * - KBase indicates the GPU need not be powered when all the Job Chains from
+ * User Processes have finished, and it is waiting for a User Process to
+ * submit some more Job Chains.
+ */
+
+/**
+ * Private structure for policy instance data.
+ *
+ * This contains data that is private to the particular power policy that is
+ * active.
+ */
+struct kbasep_pm_policy_coarse_demand {
+ /** No state needed - just have a dummy variable here */
+ int dummy;
+};
+
+extern const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops;
+
+#endif /* MALI_KBASE_PM_COARSE_DEMAND_H */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * @file mali_kbase_pm_hwaccess_defs.h
+ * Backend-specific Power Manager definitions
+ */
+
+#ifndef _KBASE_PM_HWACCESS_DEFS_H_
+#define _KBASE_PM_HWACCESS_DEFS_H_
+
+#include "mali_kbase_pm_ca_fixed.h"
+#if !MALI_CUSTOMER_RELEASE
+#include "mali_kbase_pm_ca_random.h"
+#endif
+
+#include "mali_kbase_pm_always_on.h"
+#include "mali_kbase_pm_coarse_demand.h"
+#include "mali_kbase_pm_demand.h"
+#if !MALI_CUSTOMER_RELEASE
+#include "mali_kbase_pm_demand_always_powered.h"
+#include "mali_kbase_pm_fast_start.h"
+#endif
+
+/* Forward definition - see mali_kbase.h */
+struct kbase_device;
+struct kbase_jd_atom;
+
+/** The types of core in a GPU.
+ *
+ * These enumerated values are used in calls to:
+ * - @ref kbase_pm_get_present_cores
+ * - @ref kbase_pm_get_active_cores
+ * - @ref kbase_pm_get_trans_cores
+ * - @ref kbase_pm_get_ready_cores.
+ *
+ * They specify which type of core should be acted on. These values are set in
+ * a manner that allows @ref core_type_to_reg function to be simpler and more
+ * efficient.
+ */
+enum kbase_pm_core_type {
+ KBASE_PM_CORE_L2 = L2_PRESENT_LO, /**< The L2 cache */
+ KBASE_PM_CORE_SHADER = SHADER_PRESENT_LO, /**< Shader cores */
+ KBASE_PM_CORE_TILER = TILER_PRESENT_LO /**< Tiler cores */
+};
+
+/**
+ * struct kbasep_pm_metrics_data - Metrics data collected for use by the power management framework.
+ *
+ * @vsync_hit: indicates if a framebuffer update occured since the last vsync.
+ * A framebuffer driver is expected to provide this information by
+ * checking at each vsync if the framebuffer was updated and calling
+ * kbase_pm_vsync_callback() if there was a change of status.
+ * @utilisation: percentage indicating GPU load (0-100).
+ * The utilisation is the fraction of time the GPU was powered up
+ * and busy. This is based on the time_busy and time_idle metrics.
+ * @util_gl_share: percentage of GPU load related to OpenGL jobs (0-100).
+ * This is based on the busy_gl and time_busy metrics.
+ * @util_cl_share: percentage of GPU load related to OpenCL jobs (0-100).
+ * This is based on the busy_cl and time_busy metrics.
+ * @time_period_start: time at which busy/idle measurements started
+ * @time_busy: number of ns the GPU was busy executing jobs since the
+ * @time_period_start timestamp.
+ * @time_idle: number of ns since time_period_start the GPU was not executing
+ * jobs since the @time_period_start timestamp.
+ * @prev_busy: busy time in ns of previous time period.
+ * Updated when metrics are reset.
+ * @prev_idle: idle time in ns of previous time period
+ * Updated when metrics are reset.
+ * @gpu_active: true when the GPU is executing jobs. false when
+ * not. Updated when the job scheduler informs us a job in submitted
+ * or removed from a GPU slot.
+ * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that
+ * if two CL jobs were active for 400ns, this value would be updated
+ * with 800.
+ * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that
+ * if two GL jobs were active for 400ns, this value would be updated
+ * with 800.
+ * @active_cl_ctx: number of CL jobs active on the GPU. This is a portion of
+ * the @nr_in_slots value.
+ * @active_gl_ctx: number of GL jobs active on the GPU. This is a portion of
+ * the @nr_in_slots value.
+ * @nr_in_slots: Total number of jobs currently submitted to the GPU across
+ * all job slots. Maximum value would be 2*BASE_JM_MAX_NR_SLOTS
+ * (one in flight and one in the JSn_HEAD_NEXT register for each
+ * job slot).
+ * @lock: spinlock protecting the kbasep_pm_metrics_data structure
+ * @timer: timer to regularly make DVFS decisions based on the power
+ * management metrics.
+ * @timer_active: boolean indicating @timer is running
+ * @platform_data: pointer to data controlled by platform specific code
+ * @kbdev: pointer to kbase device for which metrics are collected
+ *
+ */
+struct kbasep_pm_metrics_data {
+ int vsync_hit;
+ int utilisation;
+ int util_gl_share;
+ int util_cl_share[2]; /* 2 is a max number of core groups we can have */
+ ktime_t time_period_start;
+ u32 time_busy;
+ u32 time_idle;
+ u32 prev_busy;
+ u32 prev_idle;
+ bool gpu_active;
+ u32 busy_cl[2];
+ u32 busy_gl;
+ u32 active_cl_ctx[2];
+ u32 active_gl_ctx;
+ u8 nr_in_slots;
+ spinlock_t lock;
+
+#ifdef CONFIG_MALI_MIDGARD_DVFS
+ struct hrtimer timer;
+ bool timer_active;
+#endif
+
+ void *platform_data;
+ struct kbase_device *kbdev;
+};
+
+/** Actions for DVFS.
+ *
+ * kbase_pm_get_dvfs_action will return one of these enumerated values to
+ * describe the action that the DVFS system should take.
+ */
+enum kbase_pm_dvfs_action {
+ KBASE_PM_DVFS_NOP, /* < No change in clock frequency is
+ * requested */
+ KBASE_PM_DVFS_CLOCK_UP, /* < The clock frequency should be increased
+ * if possible */
+ KBASE_PM_DVFS_CLOCK_DOWN /* < The clock frequency should be decreased
+ * if possible */
+};
+
+union kbase_pm_policy_data {
+ struct kbasep_pm_policy_always_on always_on;
+ struct kbasep_pm_policy_coarse_demand coarse_demand;
+ struct kbasep_pm_policy_demand demand;
+#if !MALI_CUSTOMER_RELEASE
+ struct kbasep_pm_policy_demand_always_powered demand_always_powered;
+ struct kbasep_pm_policy_fast_start fast_start;
+#endif
+};
+
+union kbase_pm_ca_policy_data {
+ struct kbasep_pm_ca_policy_fixed fixed;
+#if !MALI_CUSTOMER_RELEASE
+ struct kbasep_pm_ca_policy_random random;
+#endif
+};
+
+/**
+ * Data stored per device for power management.
+ *
+ * This structure contains data for the power management framework. There is one
+ * instance of this structure per device in the system.
+ */
+struct kbase_pm_backend_data {
+ /**
+ * The policy that is currently actively controlling core availability.
+ *
+ * @note: During an IRQ, this can be NULL when the policy is being
+ * changed with kbase_pm_ca_set_policy(). The change is protected under
+ * kbase_device::pm::power_change_lock. Direct access to this from IRQ
+ * context must therefore check for NULL. If NULL, then
+ * kbase_pm_ca_set_policy() will re-issue the policy functions that
+ * would've been done under IRQ.
+ */
+ const struct kbase_pm_ca_policy *ca_current_policy;
+
+ /**
+ * The policy that is currently actively controlling the power state.
+ *
+ * @note: During an IRQ, this can be NULL when the policy is being
+ * changed with kbase_pm_set_policy(). The change is protected under
+ * kbase_device::pm::power_change_lock. Direct access to this from IRQ
+ * context must therefore check for NULL. If NULL, then
+ * kbase_pm_set_policy() will re-issue the policy functions that
+ * would've been done under IRQ.
+ */
+ const struct kbase_pm_policy *pm_current_policy;
+
+ /** Private data for current CA policy */
+ union kbase_pm_ca_policy_data ca_policy_data;
+
+ /** Private data for current PM policy */
+ union kbase_pm_policy_data pm_policy_data;
+
+ /**
+ * Flag indicating when core availability policy is transitioning cores.
+ * The core availability policy must set this when a change in core
+ * availability is occuring.
+ *
+ * power_change_lock must be held when accessing this. */
+ bool ca_in_transition;
+
+ /** Waiting for reset and a queue to wait for changes */
+ bool reset_done;
+ wait_queue_head_t reset_done_wait;
+
+ /** Wait queue for whether the l2 cache has been powered as requested */
+ wait_queue_head_t l2_powered_wait;
+ /** State indicating whether all the l2 caches are powered.
+ * Non-zero indicates they're *all* powered
+ * Zero indicates that some (or all) are not powered */
+ int l2_powered;
+
+ /** The reference count of active gpu cycle counter users */
+ int gpu_cycle_counter_requests;
+ /** Lock to protect gpu_cycle_counter_requests */
+ spinlock_t gpu_cycle_counter_requests_lock;
+
+ /**
+ * A bit mask identifying the shader cores that the power policy would
+ * like to be on. The current state of the cores may be different, but
+ * there should be transitions in progress that will eventually achieve
+ * this state (assuming that the policy doesn't change its mind in the
+ * mean time).
+ */
+ u64 desired_shader_state;
+ /**
+ * A bit mask indicating which shader cores are currently in a power-on
+ * transition
+ */
+ u64 powering_on_shader_state;
+ /**
+ * A bit mask identifying the tiler cores that the power policy would
+ * like to be on. @see kbase_pm_device_data:desired_shader_state
+ */
+ u64 desired_tiler_state;
+ /**
+ * A bit mask indicating which tiler core are currently in a power-on
+ * transition
+ */
+ u64 powering_on_tiler_state;
+
+ /**
+ * A bit mask indicating which l2-caches are currently in a power-on
+ * transition
+ */
+ u64 powering_on_l2_state;
+
+ /**
+ * This flag is set if the GPU is powered as requested by the
+ * desired_xxx_state variables
+ */
+ bool gpu_in_desired_state;
+ /* Wait queue set when gpu_in_desired_state != 0 */
+ wait_queue_head_t gpu_in_desired_state_wait;
+
+ /**
+ * Set to true when the GPU is powered and register accesses are
+ * possible, false otherwise
+ */
+ bool gpu_powered;
+
+ /** Set to true when instrumentation is enabled, false otherwise */
+ bool instr_enabled;
+
+ bool cg1_disabled;
+
+#ifdef CONFIG_MALI_DEBUG
+ /**
+ * Debug state indicating whether sufficient initialization of the
+ * driver has occurred to handle IRQs
+ */
+ bool driver_ready_for_irqs;
+#endif /* CONFIG_MALI_DEBUG */
+
+ /**
+ * Spinlock that must be held when:
+ * - writing gpu_powered
+ * - accessing driver_ready_for_irqs (in CONFIG_MALI_DEBUG builds)
+ */
+ spinlock_t gpu_powered_lock;
+
+ /** Structure to hold metrics for the GPU */
+
+ struct kbasep_pm_metrics_data metrics;
+
+ /**
+ * Set to the number of poweroff timer ticks until the GPU is powered
+ * off
+ */
+ int gpu_poweroff_pending;
+
+ /**
+ * Set to the number of poweroff timer ticks until shaders are powered
+ * off
+ */
+ int shader_poweroff_pending_time;
+
+ /** Timer for powering off GPU */
+ struct hrtimer gpu_poweroff_timer;
+
+ struct workqueue_struct *gpu_poweroff_wq;
+
+ struct work_struct gpu_poweroff_work;
+
+ /** Bit mask of shaders to be powered off on next timer callback */
+ u64 shader_poweroff_pending;
+
+ /**
+ * Set to true if the poweroff timer is currently running,
+ * false otherwise
+ */
+ bool poweroff_timer_needed;
+
+ /**
+ * Callback when the GPU needs to be turned on. See
+ * @ref kbase_pm_callback_conf
+ *
+ * @param kbdev The kbase device
+ *
+ * @return 1 if GPU state was lost, 0 otherwise
+ */
+ int (*callback_power_on)(struct kbase_device *kbdev);
+
+ /**
+ * Callback when the GPU may be turned off. See
+ * @ref kbase_pm_callback_conf
+ *
+ * @param kbdev The kbase device
+ */
+ void (*callback_power_off)(struct kbase_device *kbdev);
+
+ /**
+ * Callback when a suspend occurs and the GPU needs to be turned off.
+ * See @ref kbase_pm_callback_conf
+ *
+ * @param kbdev The kbase device
+ */
+ void (*callback_power_suspend)(struct kbase_device *kbdev);
+
+ /**
+ * Callback when a resume occurs and the GPU needs to be turned on.
+ * See @ref kbase_pm_callback_conf
+ *
+ * @param kbdev The kbase device
+ */
+ void (*callback_power_resume)(struct kbase_device *kbdev);
+
+ /**
+ * Callback when the GPU needs to be turned on. See
+ * @ref kbase_pm_callback_conf
+ *
+ * @param kbdev The kbase device
+ *
+ * @return 1 if GPU state was lost, 0 otherwise
+ */
+ int (*callback_power_runtime_on)(struct kbase_device *kbdev);
+
+ /**
+ * Callback when the GPU may be turned off. See
+ * @ref kbase_pm_callback_conf
+ *
+ * @param kbdev The kbase device
+ */
+ void (*callback_power_runtime_off)(struct kbase_device *kbdev);
+
+};
+
+
+/** List of policy IDs */
+enum kbase_pm_policy_id {
+ KBASE_PM_POLICY_ID_DEMAND = 1,
+ KBASE_PM_POLICY_ID_ALWAYS_ON,
+ KBASE_PM_POLICY_ID_COARSE_DEMAND,
+#if !MALI_CUSTOMER_RELEASE
+ KBASE_PM_POLICY_ID_DEMAND_ALWAYS_POWERED,
+ KBASE_PM_POLICY_ID_FAST_START
+#endif
+};
+
+typedef u32 kbase_pm_policy_flags;
+
+/**
+ * Power policy structure.
+ *
+ * Each power policy exposes a (static) instance of this structure which
+ * contains function pointers to the policy's methods.
+ */
+struct kbase_pm_policy {
+ /** The name of this policy */
+ char *name;
+
+ /**
+ * Function called when the policy is selected
+ *
+ * This should initialize the kbdev->pm.pm_policy_data structure. It
+ * should not attempt to make any changes to hardware state.
+ *
+ * It is undefined what state the cores are in when the function is
+ * called.
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ */
+ void (*init)(struct kbase_device *kbdev);
+
+ /**
+ * Function called when the policy is unselected.
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ */
+ void (*term)(struct kbase_device *kbdev);
+
+ /**
+ * Function called to get the current shader core mask
+ *
+ * The returned mask should meet or exceed (kbdev->shader_needed_bitmap
+ * | kbdev->shader_inuse_bitmap).
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ *
+ * @return The mask of shader cores to be powered
+ */
+ u64 (*get_core_mask)(struct kbase_device *kbdev);
+
+ /**
+ * Function called to get the current overall GPU power state
+ *
+ * This function should consider the state of kbdev->pm.active_count. If
+ * this count is greater than 0 then there is at least one active
+ * context on the device and the GPU should be powered. If it is equal
+ * to 0 then there are no active contexts and the GPU could be powered
+ * off if desired.
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ *
+ * @return true if the GPU should be powered, false otherwise
+ */
+ bool (*get_core_active)(struct kbase_device *kbdev);
+
+ /** Field indicating flags for this policy */
+ kbase_pm_policy_flags flags;
+
+ /**
+ * Field indicating an ID for this policy. This is not necessarily the
+ * same as its index in the list returned by kbase_pm_list_policies().
+ * It is used purely for debugging.
+ */
+ enum kbase_pm_policy_id id;
+};
+
+
+enum kbase_pm_ca_policy_id {
+ KBASE_PM_CA_POLICY_ID_FIXED = 1,
+ KBASE_PM_CA_POLICY_ID_RANDOM
+};
+
+typedef u32 kbase_pm_ca_policy_flags;
+
+/**
+ * Core availability policy structure.
+ *
+ * Each core availability policy exposes a (static) instance of this structure
+ * which contains function pointers to the policy's methods.
+ */
+struct kbase_pm_ca_policy {
+ /** The name of this policy */
+ char *name;
+
+ /**
+ * Function called when the policy is selected
+ *
+ * This should initialize the kbdev->pm.ca_policy_data structure. It
+ * should not attempt to make any changes to hardware state.
+ *
+ * It is undefined what state the cores are in when the function is
+ * called.
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ */
+ void (*init)(struct kbase_device *kbdev);
+
+ /**
+ * Function called when the policy is unselected.
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ */
+ void (*term)(struct kbase_device *kbdev);
+
+ /**
+ * Function called to get the current shader core availability mask
+ *
+ * When a change in core availability is occuring, the policy must set
+ * kbdev->pm.ca_in_transition to true. This is to indicate that
+ * reporting changes in power state cannot be optimized out, even if
+ * kbdev->pm.desired_shader_state remains unchanged. This must be done
+ * by any functions internal to the Core Availability Policy that change
+ * the return value of kbase_pm_ca_policy::get_core_mask.
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ *
+ * @return The current core availability mask
+ */
+ u64 (*get_core_mask)(struct kbase_device *kbdev);
+
+ /**
+ * Function called to update the current core status
+ *
+ * If none of the cores in core group 0 are ready or transitioning, then
+ * the policy must ensure that the next call to get_core_mask does not
+ * return 0 for all cores in core group 0. It is an error to disable
+ * core group 0 through the core availability policy.
+ *
+ * When a change in core availability has finished, the policy must set
+ * kbdev->pm.ca_in_transition to false. This is to indicate that
+ * changes in power state can once again be optimized out when
+ * kbdev->pm.desired_shader_state is unchanged.
+ *
+ * @param kbdev The kbase device structure for the device
+ * (must be a valid pointer)
+ * @param cores_ready The mask of cores currently powered and
+ * ready to run jobs
+ * @param cores_transitioning The mask of cores currently transitioning
+ * power state
+ */
+ void (*update_core_status)(struct kbase_device *kbdev, u64 cores_ready,
+ u64 cores_transitioning);
+
+ /** Field indicating flags for this policy */
+ kbase_pm_ca_policy_flags flags;
+
+ /**
+ * Field indicating an ID for this policy. This is not necessarily the
+ * same as its index in the list returned by kbase_pm_list_policies().
+ * It is used purely for debugging.
+ */
+ enum kbase_pm_ca_policy_id id;
+};
+
+#endif /* _KBASE_PM_HWACCESS_DEFS_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_pm_demand.c
+ * A simple demand based power management policy
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_pm.h>
+
+static u64 demand_get_core_mask(struct kbase_device *kbdev)
+{
+ u64 desired = kbdev->shader_needed_bitmap | kbdev->shader_inuse_bitmap;
+
+ if (0 == kbdev->pm.active_count)
+ return 0;
+
+ return desired;
+}
+
+static bool demand_get_core_active(struct kbase_device *kbdev)
+{
+ if (0 == kbdev->pm.active_count)
+ return false;
+
+ return true;
+}
+
+static void demand_init(struct kbase_device *kbdev)
+{
+ CSTD_UNUSED(kbdev);
+}
+
+static void demand_term(struct kbase_device *kbdev)
+{
+ CSTD_UNUSED(kbdev);
+}
+
+/**
+ * The @ref struct kbase_pm_policy structure for the demand power policy.
+ *
+ * This is the static structure that defines the demand power policy's callback
+ * and name.
+ */
+const struct kbase_pm_policy kbase_pm_demand_policy_ops = {
+ "demand", /* name */
+ demand_init, /* init */
+ demand_term, /* term */
+ demand_get_core_mask, /* get_core_mask */
+ demand_get_core_active, /* get_core_active */
+ 0u, /* flags */
+ KBASE_PM_POLICY_ID_DEMAND, /* id */
+};
+
+KBASE_EXPORT_TEST_API(kbase_pm_demand_policy_ops);
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_pm_demand.h
+ * A simple demand based power management policy
+ */
+
+#ifndef MALI_KBASE_PM_DEMAND_H
+#define MALI_KBASE_PM_DEMAND_H
+
+/**
+ * The demand power management policy has the following characteristics:
+ * - When KBase indicates that the GPU will be powered up, but we don't yet
+ * know which Job Chains are to be run:
+ * - The Shader Cores are not powered up
+ * - When KBase indicates that a set of Shader Cores are needed to submit the
+ * currently queued Job Chains:
+ * - Only those Shader Cores are powered up
+ * - When KBase indicates that the GPU need not be powered:
+ * - The Shader Cores are powered off, and the GPU itself is powered off too.
+ *
+ * @note:
+ * - KBase indicates the GPU will be powered up when it has a User Process that
+ * has just started to submit Job Chains.
+ * - KBase indicates the GPU need not be powered when all the Job Chains from
+ * User Processes have finished, and it is waiting for a User Process to
+ * submit some more Job Chains.
+ */
+
+/**
+ * Private structure for policy instance data.
+ *
+ * This contains data that is private to the particular power policy that is
+ * active.
+ */
+struct kbasep_pm_policy_demand {
+ /** No state needed - just have a dummy variable here */
+ int dummy;
+};
+
+extern const struct kbase_pm_policy kbase_pm_demand_policy_ops;
+
+#endif /* MALI_KBASE_PM_DEMAND_H */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_pm_driver.c
+ * Base kernel Power Management hardware control
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_config_defaults.h>
+#include <mali_midg_regmap.h>
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
+#include <mali_kbase_gator.h>
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+#include <mali_kbase_tlstream.h>
+#endif
+#include <mali_kbase_pm.h>
+#include <mali_kbase_cache_policy.h>
+#include <mali_kbase_config_defaults.h>
+#include <mali_kbase_smc.h>
+#include <backend/gpu/mali_kbase_cache_policy_backend.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
+#include <backend/gpu/mali_kbase_irq_internal.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+#if MALI_MOCK_TEST
+#define MOCKABLE(function) function##_original
+#else
+#define MOCKABLE(function) function
+#endif /* MALI_MOCK_TEST */
+
+/**
+ * Actions that can be performed on a core.
+ *
+ * This enumeration is private to the file. Its values are set to allow
+ * @ref core_type_to_reg function, which decodes this enumeration, to be simpler
+ * and more efficient.
+ */
+enum kbasep_pm_action {
+ ACTION_PRESENT = 0,
+ ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO),
+ ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO),
+ ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO),
+ ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO),
+ ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO)
+};
+
+static u64 kbase_pm_get_state(
+ struct kbase_device *kbdev,
+ enum kbase_pm_core_type core_type,
+ enum kbasep_pm_action action);
+
+/**
+ * Decode a core type and action to a register.
+ *
+ * Given a core type (defined by @ref kbase_pm_core_type) and an action (defined
+ * by @ref kbasep_pm_action) this function will return the register offset that
+ * will perform the action on the core type. The register returned is the \c _LO
+ * register and an offset must be applied to use the \c _HI register.
+ *
+ * @param core_type The type of core
+ * @param action The type of action
+ *
+ * @return The register offset of the \c _LO register that performs an action of
+ * type \c action on a core of type \c core_type.
+ */
+static u32 core_type_to_reg(enum kbase_pm_core_type core_type,
+ enum kbasep_pm_action action)
+{
+ return (u32)core_type + (u32)action;
+}
+
+
+/** Invokes an action on a core set
+ *
+ * This function performs the action given by \c action on a set of cores of a
+ * type given by \c core_type. It is a static function used by
+ * @ref kbase_pm_transition_core_type
+ *
+ * @param kbdev The kbase device structure of the device
+ * @param core_type The type of core that the action should be performed on
+ * @param cores A bit mask of cores to perform the action on (low 32 bits)
+ * @param action The action to perform on the cores
+ */
+static void kbase_pm_invoke(struct kbase_device *kbdev,
+ enum kbase_pm_core_type core_type,
+ u64 cores,
+ enum kbasep_pm_action action)
+{
+ u32 reg;
+ u32 lo = cores & 0xFFFFFFFF;
+ u32 hi = (cores >> 32) & 0xFFFFFFFF;
+
+ lockdep_assert_held(&kbdev->pm.power_change_lock);
+
+ reg = core_type_to_reg(core_type, action);
+
+ KBASE_DEBUG_ASSERT(reg);
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
+ if (cores) {
+ if (action == ACTION_PWRON)
+ kbase_trace_mali_pm_power_on(core_type, cores);
+ else if (action == ACTION_PWROFF)
+ kbase_trace_mali_pm_power_off(core_type, cores);
+ }
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ if (cores) {
+ u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY);
+
+ if (action == ACTION_PWRON)
+ state |= cores;
+ else if (action == ACTION_PWROFF)
+ state &= ~cores;
+ kbase_tlstream_aux_pm_state(core_type, state);
+ }
+#endif
+ /* Tracing */
+ if (cores) {
+ if (action == ACTION_PWRON)
+ switch (core_type) {
+ case KBASE_PM_CORE_SHADER:
+ KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u,
+ lo);
+ break;
+ case KBASE_PM_CORE_TILER:
+ KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL,
+ NULL, 0u, lo);
+ break;
+ case KBASE_PM_CORE_L2:
+ KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL,
+ 0u, lo);
+ break;
+ default:
+ break;
+ }
+ else if (action == ACTION_PWROFF)
+ switch (core_type) {
+ case KBASE_PM_CORE_SHADER:
+ KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL,
+ 0u, lo);
+ break;
+ case KBASE_PM_CORE_TILER:
+ KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL,
+ NULL, 0u, lo);
+ break;
+ case KBASE_PM_CORE_L2:
+ KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL,
+ 0u, lo);
+ break;
+ default:
+ break;
+ }
+ }
+
+ if (lo != 0)
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL);
+
+ if (hi != 0)
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL);
+}
+
+/**
+ * Get information about a core set
+ *
+ * This function gets information (chosen by \c action) about a set of cores of
+ * a type given by \c core_type. It is a static function used by @ref
+ * kbase_pm_get_present_cores, @ref kbase_pm_get_active_cores, @ref
+ * kbase_pm_get_trans_cores and @ref kbase_pm_get_ready_cores.
+ *
+ * @param kbdev The kbase device structure of the device
+ * @param core_type The type of core that the should be queried
+ * @param action The property of the cores to query
+ *
+ * @return A bit mask specifying the state of the cores
+ */
+static u64 kbase_pm_get_state(struct kbase_device *kbdev,
+ enum kbase_pm_core_type core_type,
+ enum kbasep_pm_action action)
+{
+ u32 reg;
+ u32 lo, hi;
+
+ reg = core_type_to_reg(core_type, action);
+
+ KBASE_DEBUG_ASSERT(reg);
+
+ lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL);
+ hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL);
+
+ return (((u64) hi) << 32) | ((u64) lo);
+}
+
+void kbasep_pm_read_present_cores(struct kbase_device *kbdev)
+{
+ kbdev->shader_present_bitmap =
+ kbase_pm_get_state(kbdev, KBASE_PM_CORE_SHADER, ACTION_PRESENT);
+ kbdev->tiler_present_bitmap =
+ kbase_pm_get_state(kbdev, KBASE_PM_CORE_TILER, ACTION_PRESENT);
+ kbdev->l2_present_bitmap =
+ kbase_pm_get_state(kbdev, KBASE_PM_CORE_L2, ACTION_PRESENT);
+ kbdev->shader_inuse_bitmap = 0;
+ kbdev->shader_needed_bitmap = 0;
+ kbdev->shader_available_bitmap = 0;
+ kbdev->tiler_available_bitmap = 0;
+ kbdev->l2_users_count = 0;
+ kbdev->l2_available_bitmap = 0;
+ kbdev->tiler_needed_cnt = 0;
+ kbdev->tiler_inuse_cnt = 0;
+
+ memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt));
+}
+
+KBASE_EXPORT_TEST_API(kbasep_pm_read_present_cores);
+
+/**
+ * Get the cores that are present
+ */
+u64 kbase_pm_get_present_cores(struct kbase_device *kbdev,
+ enum kbase_pm_core_type type)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ switch (type) {
+ case KBASE_PM_CORE_L2:
+ return kbdev->l2_present_bitmap;
+ case KBASE_PM_CORE_SHADER:
+ return kbdev->shader_present_bitmap;
+ case KBASE_PM_CORE_TILER:
+ return kbdev->tiler_present_bitmap;
+ }
+ KBASE_DEBUG_ASSERT(0);
+ return 0;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores);
+
+/**
+ * Get the cores that are "active" (busy processing work)
+ */
+u64 kbase_pm_get_active_cores(struct kbase_device *kbdev,
+ enum kbase_pm_core_type type)
+{
+ return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores);
+
+/**
+ * Get the cores that are transitioning between power states
+ */
+u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev,
+ enum kbase_pm_core_type type)
+{
+ return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores);
+
+/**
+ * Get the cores that are powered on
+ */
+u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev,
+ enum kbase_pm_core_type type)
+{
+ u64 result;
+
+ result = kbase_pm_get_state(kbdev, type, ACTION_READY);
+
+ switch (type) {
+ case KBASE_PM_CORE_SHADER:
+ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u,
+ (u32) result);
+ break;
+ case KBASE_PM_CORE_TILER:
+ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u,
+ (u32) result);
+ break;
+ case KBASE_PM_CORE_L2:
+ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u,
+ (u32) result);
+ break;
+ default:
+ break;
+ }
+
+ return result;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores);
+
+/**
+ * Perform power transitions for a particular core type.
+ *
+ * This function will perform any available power transitions to make the actual
+ * hardware state closer to the desired state. If a core is currently
+ * transitioning then changes to the power state of that call cannot be made
+ * until the transition has finished. Cores which are not present in the
+ * hardware are ignored if they are specified in the desired_state bitmask,
+ * however the return value will always be 0 in this case.
+ *
+ * @param kbdev The kbase device
+ * @param type The core type to perform transitions for
+ * @param desired_state A bit mask of the desired state of the cores
+ * @param in_use A bit mask of the cores that are currently running
+ * jobs. These cores have to be kept powered up because
+ * there are jobs running (or about to run) on them.
+ * @param[out] available Receives a bit mask of the cores that the job
+ * scheduler can use to submit jobs to. May be NULL if
+ * this is not needed.
+ * @param[in,out] powering_on Bit mask to update with cores that are
+ * transitioning to a power-on state.
+ *
+ * @return true if the desired state has been reached, false otherwise
+ */
+static bool kbase_pm_transition_core_type(struct kbase_device *kbdev,
+ enum kbase_pm_core_type type,
+ u64 desired_state,
+ u64 in_use,
+ u64 * const available,
+ u64 *powering_on)
+{
+ u64 present;
+ u64 ready;
+ u64 trans;
+ u64 powerup;
+ u64 powerdown;
+ u64 powering_on_trans;
+ u64 desired_state_in_use;
+
+ lockdep_assert_held(&kbdev->pm.power_change_lock);
+
+ /* Get current state */
+ present = kbase_pm_get_present_cores(kbdev, type);
+ trans = kbase_pm_get_trans_cores(kbdev, type);
+ ready = kbase_pm_get_ready_cores(kbdev, type);
+ /* mask off ready from trans in case transitions finished between the
+ * register reads */
+ trans &= ~ready;
+
+ powering_on_trans = trans & *powering_on;
+ *powering_on = powering_on_trans;
+
+ if (available != NULL)
+ *available = (ready | powering_on_trans) & desired_state;
+
+ /* Update desired state to include the in-use cores. These have to be
+ * kept powered up because there are jobs running or about to run on
+ * these cores
+ */
+ desired_state_in_use = desired_state | in_use;
+
+ /* Update state of whether l2 caches are powered */
+ if (type == KBASE_PM_CORE_L2) {
+ if ((ready == present) && (desired_state_in_use == ready) &&
+ (trans == 0)) {
+ /* All are ready, none will be turned off, and none are
+ * transitioning */
+ kbdev->pm.backend.l2_powered = 1;
+ if (kbdev->l2_users_count > 0) {
+ /* Notify any registered l2 cache users
+ * (optimized out when no users waiting) */
+ wake_up(&kbdev->pm.backend.l2_powered_wait);
+ }
+ } else
+ kbdev->pm.backend.l2_powered = 0;
+ }
+
+ if (desired_state_in_use == ready && (trans == 0))
+ return true;
+
+ /* Restrict the cores to those that are actually present */
+ powerup = desired_state_in_use & present;
+ powerdown = (~desired_state_in_use) & present;
+
+ /* Restrict to cores that are not already in the desired state */
+ powerup &= ~ready;
+ powerdown &= ready;
+
+ /* Don't transition any cores that are already transitioning, except for
+ * Mali cores that support the following case:
+ *
+ * If the SHADER_PWRON or TILER_PWRON registers are written to turn on
+ * a core that is currently transitioning to power off, then this is
+ * remembered and the shader core is automatically powered up again once
+ * the original transition completes. Once the automatic power on is
+ * complete any job scheduled on the shader core should start.
+ */
+ powerdown &= ~trans;
+
+ if (kbase_hw_has_feature(kbdev,
+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS))
+ if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type)
+ trans = powering_on_trans; /* for exception cases, only
+ * mask off cores in power on
+ * transitions */
+
+ powerup &= ~trans;
+
+ /* Perform transitions if any */
+ kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON);
+ kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF);
+
+ /* Recalculate cores transitioning on, and re-evaluate our state */
+ powering_on_trans |= powerup;
+ *powering_on = powering_on_trans;
+ if (available != NULL)
+ *available = (ready | powering_on_trans) & desired_state;
+
+ return false;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type);
+
+/**
+ * Determine which caches should be on for a particular core state.
+ *
+ * This function takes a bit mask of the present caches and the cores (or
+ * caches) that are attached to the caches that will be powered. It then
+ * computes which caches should be turned on to allow the cores requested to be
+ * powered up.
+ *
+ * @param present The bit mask of present caches
+ * @param cores_powered A bit mask of cores (or L2 caches) that are desired to
+ * be powered
+ *
+ * @return A bit mask of the caches that should be turned on
+ */
+static u64 get_desired_cache_status(u64 present, u64 cores_powered)
+{
+ u64 desired = 0;
+
+ while (present) {
+ /* Find out which is the highest set bit */
+ u64 bit = fls64(present) - 1;
+ u64 bit_mask = 1ull << bit;
+ /* Create a mask which has all bits from 'bit' upwards set */
+
+ u64 mask = ~(bit_mask - 1);
+
+ /* If there are any cores powered at this bit or above (that
+ * haven't previously been processed) then we need this core on
+ */
+ if (cores_powered & mask)
+ desired |= bit_mask;
+
+ /* Remove bits from cores_powered and present */
+ cores_powered &= ~mask;
+ present &= ~bit_mask;
+ }
+
+ return desired;
+}
+
+KBASE_EXPORT_TEST_API(get_desired_cache_status);
+
+bool
+MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev)
+{
+ bool cores_are_available = false;
+ bool in_desired_state = true;
+ u64 desired_l2_state;
+ u64 cores_powered;
+ u64 tiler_available_bitmap;
+ u64 shader_available_bitmap;
+ u64 shader_ready_bitmap;
+ u64 shader_transitioning_bitmap;
+ u64 l2_available_bitmap;
+ u64 prev_l2_available_bitmap;
+
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ lockdep_assert_held(&kbdev->pm.power_change_lock);
+
+ spin_lock(&kbdev->pm.backend.gpu_powered_lock);
+ if (kbdev->pm.backend.gpu_powered == false) {
+ spin_unlock(&kbdev->pm.backend.gpu_powered_lock);
+ if (kbdev->pm.backend.desired_shader_state == 0 &&
+ kbdev->pm.backend.desired_tiler_state == 0)
+ return true;
+ return false;
+ }
+
+ /* Trace that a change-state is being requested, and that it took
+ * (effectively) no time to start it. This is useful for counting how
+ * many state changes occurred, in a way that's backwards-compatible
+ * with processing the trace data */
+ kbase_timeline_pm_send_event(kbdev,
+ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
+ kbase_timeline_pm_handle_event(kbdev,
+ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
+
+ /* If any cores are already powered then, we must keep the caches on */
+ cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
+
+ cores_powered |= kbdev->pm.backend.desired_shader_state;
+
+ /* If there are l2 cache users registered, keep all l2s powered even if
+ * all other cores are off. */
+ if (kbdev->l2_users_count > 0)
+ cores_powered |= kbdev->l2_present_bitmap;
+
+ desired_l2_state = get_desired_cache_status(kbdev->l2_present_bitmap,
+ cores_powered);
+
+ /* If any l2 cache is on, then enable l2 #0, for use by job manager */
+ if (0 != desired_l2_state) {
+ desired_l2_state |= 1;
+ /* Also enable tiler if l2 cache is powered */
+ kbdev->pm.backend.desired_tiler_state =
+ kbdev->tiler_present_bitmap;
+ } else {
+ kbdev->pm.backend.desired_tiler_state = 0;
+ }
+
+ prev_l2_available_bitmap = kbdev->l2_available_bitmap;
+ in_desired_state &= kbase_pm_transition_core_type(kbdev,
+ KBASE_PM_CORE_L2, desired_l2_state, 0,
+ &l2_available_bitmap,
+ &kbdev->pm.backend.powering_on_l2_state);
+
+ if (kbdev->l2_available_bitmap != l2_available_bitmap)
+ KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap);
+
+ kbdev->l2_available_bitmap = l2_available_bitmap;
+
+ if (in_desired_state) {
+ in_desired_state &= kbase_pm_transition_core_type(kbdev,
+ KBASE_PM_CORE_TILER,
+ kbdev->pm.backend.desired_tiler_state,
+ 0, &tiler_available_bitmap,
+ &kbdev->pm.backend.powering_on_tiler_state);
+ in_desired_state &= kbase_pm_transition_core_type(kbdev,
+ KBASE_PM_CORE_SHADER,
+ kbdev->pm.backend.desired_shader_state,
+ kbdev->shader_inuse_bitmap,
+ &shader_available_bitmap,
+ &kbdev->pm.backend.powering_on_shader_state);
+
+ if (kbdev->shader_available_bitmap != shader_available_bitmap) {
+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
+ NULL, 0u,
+ (u32) shader_available_bitmap);
+ KBASE_TIMELINE_POWER_SHADER(kbdev,
+ shader_available_bitmap);
+ }
+
+ kbdev->shader_available_bitmap = shader_available_bitmap;
+
+ if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
+ NULL, NULL, 0u,
+ (u32) tiler_available_bitmap);
+ KBASE_TIMELINE_POWER_TILER(kbdev,
+ tiler_available_bitmap);
+ }
+
+ kbdev->tiler_available_bitmap = tiler_available_bitmap;
+
+ } else if ((l2_available_bitmap & kbdev->tiler_present_bitmap) !=
+ kbdev->tiler_present_bitmap) {
+ tiler_available_bitmap = 0;
+
+ if (kbdev->tiler_available_bitmap != tiler_available_bitmap)
+ KBASE_TIMELINE_POWER_TILER(kbdev,
+ tiler_available_bitmap);
+
+ kbdev->tiler_available_bitmap = tiler_available_bitmap;
+ }
+
+ /* State updated for slow-path waiters */
+ kbdev->pm.backend.gpu_in_desired_state = in_desired_state;
+
+ shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev,
+ KBASE_PM_CORE_SHADER);
+ shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev,
+ KBASE_PM_CORE_SHADER);
+
+ /* Determine whether the cores are now available (even if the set of
+ * available cores is empty). Note that they can be available even if
+ * we've not finished transitioning to the desired state */
+ if ((kbdev->shader_available_bitmap &
+ kbdev->pm.backend.desired_shader_state)
+ == kbdev->pm.backend.desired_shader_state &&
+ (kbdev->tiler_available_bitmap &
+ kbdev->pm.backend.desired_tiler_state)
+ == kbdev->pm.backend.desired_tiler_state) {
+ cores_are_available = true;
+
+ KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u,
+ (u32)(kbdev->shader_available_bitmap &
+ kbdev->pm.backend.desired_shader_state));
+ KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u,
+ (u32)(kbdev->tiler_available_bitmap &
+ kbdev->pm.backend.desired_tiler_state));
+
+ /* Log timelining information about handling events that power
+ * up cores, to match up either with immediate submission either
+ * because cores already available, or from PM IRQ */
+ if (!in_desired_state)
+ kbase_timeline_pm_send_event(kbdev,
+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
+ }
+
+ if (in_desired_state) {
+ KBASE_DEBUG_ASSERT(cores_are_available);
+
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
+ kbase_trace_mali_pm_status(KBASE_PM_CORE_L2,
+ kbase_pm_get_ready_cores(kbdev,
+ KBASE_PM_CORE_L2));
+ kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER,
+ kbase_pm_get_ready_cores(kbdev,
+ KBASE_PM_CORE_SHADER));
+ kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER,
+ kbase_pm_get_ready_cores(kbdev,
+ KBASE_PM_CORE_TILER));
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_aux_pm_state(
+ KBASE_PM_CORE_L2,
+ kbase_pm_get_ready_cores(
+ kbdev, KBASE_PM_CORE_L2));
+ kbase_tlstream_aux_pm_state(
+ KBASE_PM_CORE_SHADER,
+ kbase_pm_get_ready_cores(
+ kbdev, KBASE_PM_CORE_SHADER));
+ kbase_tlstream_aux_pm_state(
+ KBASE_PM_CORE_TILER,
+ kbase_pm_get_ready_cores(
+ kbdev,
+ KBASE_PM_CORE_TILER));
+#endif
+
+ KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL,
+ kbdev->pm.backend.gpu_in_desired_state,
+ (u32)kbdev->pm.backend.desired_shader_state);
+ KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u,
+ (u32)kbdev->pm.backend.desired_tiler_state);
+
+ /* Log timelining information for synchronous waiters */
+ kbase_timeline_pm_send_event(kbdev,
+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
+ /* Wake slow-path waiters. Job scheduler does not use this. */
+ KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0);
+ wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
+ }
+
+ spin_unlock(&kbdev->pm.backend.gpu_powered_lock);
+
+ /* kbase_pm_ca_update_core_status can cause one-level recursion into
+ * this function, so it must only be called once all changes to kbdev
+ * have been committed, and after the gpu_powered_lock has been
+ * dropped. */
+ if (kbdev->shader_ready_bitmap != shader_ready_bitmap ||
+ kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) {
+ kbdev->shader_ready_bitmap = shader_ready_bitmap;
+ kbdev->shader_transitioning_bitmap =
+ shader_transitioning_bitmap;
+
+ kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap,
+ shader_transitioning_bitmap);
+ }
+
+ /* The core availability policy is not allowed to keep core group 0
+ * turned off (unless it was changing the l2 power state) */
+ if (!((shader_ready_bitmap | shader_transitioning_bitmap) &
+ kbdev->gpu_props.props.coherency_info.group[0].core_mask) &&
+ (prev_l2_available_bitmap == desired_l2_state) &&
+ !(kbase_pm_ca_get_core_mask(kbdev) &
+ kbdev->gpu_props.props.coherency_info.group[0].core_mask))
+ BUG();
+
+ /* The core availability policy is allowed to keep core group 1 off,
+ * but all jobs specifically targeting CG1 must fail */
+ if (!((shader_ready_bitmap | shader_transitioning_bitmap) &
+ kbdev->gpu_props.props.coherency_info.group[1].core_mask) &&
+ !(kbase_pm_ca_get_core_mask(kbdev) &
+ kbdev->gpu_props.props.coherency_info.group[1].core_mask))
+ kbdev->pm.backend.cg1_disabled = true;
+ else
+ kbdev->pm.backend.cg1_disabled = false;
+
+ return cores_are_available;
+}
+KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock);
+
+void kbase_pm_check_transitions_sync(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+ bool cores_are_available;
+ /* Force the transition to be checked and reported - the cores may be
+ * 'available' (for job submission) but not fully powered up. */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
+ /* Don't need 'cores_are_available', because we don't return anything */
+ CSTD_UNUSED(cores_are_available);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ /* Wait for cores */
+ wait_event(kbdev->pm.backend.gpu_in_desired_state_wait,
+ kbdev->pm.backend.gpu_in_desired_state);
+
+ /* Log timelining information that a change in state has completed */
+ kbase_timeline_pm_handle_event(kbdev,
+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
+}
+KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync);
+
+void kbase_pm_enable_interrupts(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ /*
+ * Clear all interrupts,
+ * and unmask them all.
+ */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL,
+ NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL,
+ NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF,
+ NULL);
+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL);
+
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts);
+
+void kbase_pm_enable_interrupts_mmu_mask(struct kbase_device *kbdev, u32 mask)
+{
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ /*
+ * Clear all interrupts,
+ * and unmask them all.
+ */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL,
+ NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL,
+ NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF,
+ NULL);
+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL);
+
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), mask, NULL);
+}
+
+void kbase_pm_disable_interrupts(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ /*
+ * Mask all interrupts,
+ * and clear them all.
+ */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL,
+ NULL);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL);
+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF,
+ NULL);
+
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL);
+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts);
+
+/*
+ * pmu layout:
+ * 0x0000: PMU TAG (RO) (0xCAFECAFE)
+ * 0x0004: PMU VERSION ID (RO) (0x00000000)
+ * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE)
+ */
+void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
+{
+ bool reset_required = is_resume;
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ unsigned long flags;
+ int i;
+
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ lockdep_assert_held(&js_devdata->runpool_mutex);
+ lockdep_assert_held(&kbdev->pm.lock);
+
+ if (kbdev->pm.backend.gpu_powered) {
+ /* Already turned on */
+ if (kbdev->poweroff_pending)
+ kbase_pm_enable_interrupts(kbdev);
+ kbdev->poweroff_pending = false;
+ KBASE_DEBUG_ASSERT(!is_resume);
+ return;
+ }
+
+ kbdev->poweroff_pending = false;
+
+ KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u);
+
+ if (is_resume && kbdev->pm.backend.callback_power_resume) {
+ kbdev->pm.backend.callback_power_resume(kbdev);
+ } else if (kbdev->pm.backend.callback_power_on) {
+ kbdev->pm.backend.callback_power_on(kbdev);
+ /* If your platform properly keeps the GPU state you may use the
+ * return value of the callback_power_on function to
+ * conditionally reset the GPU on power up. Currently we are
+ * conservative and always reset the GPU. */
+ reset_required = true;
+ }
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
+ kbdev->pm.backend.gpu_powered = true;
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (reset_required) {
+ /* GPU state was lost, reset GPU to ensure it is in a
+ * consistent state */
+ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS);
+ }
+
+ /* Reprogram the GPU's MMU */
+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
+ struct kbase_as *as = &kbdev->as[i];
+
+ mutex_lock(&as->transaction_mutex);
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ if (js_devdata->runpool_irq.per_as_data[i].kctx)
+ kbase_mmu_update(
+ js_devdata->runpool_irq.per_as_data[i].kctx);
+ else
+ kbase_mmu_disable_as(kbdev, i);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ mutex_unlock(&as->transaction_mutex);
+ }
+
+ /* Lastly, enable the interrupts */
+ kbase_pm_enable_interrupts(kbdev);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_clock_on);
+
+bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend)
+{
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ lockdep_assert_held(&kbdev->pm.lock);
+
+ /* ASSERT that the cores should now be unavailable. No lock needed. */
+ KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u);
+
+ kbdev->poweroff_pending = true;
+
+ if (!kbdev->pm.backend.gpu_powered) {
+ /* Already turned off */
+ if (is_suspend && kbdev->pm.backend.callback_power_suspend)
+ kbdev->pm.backend.callback_power_suspend(kbdev);
+ return true;
+ }
+
+ KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u);
+
+ /* Disable interrupts. This also clears any outstanding interrupts */
+ kbase_pm_disable_interrupts(kbdev);
+ /* Ensure that any IRQ handlers have finished */
+ kbase_synchronize_irqs(kbdev);
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (atomic_read(&kbdev->faults_pending)) {
+ /* Page/bus faults are still being processed. The GPU can not
+ * be powered off until they have completed */
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
+ flags);
+ return false;
+ }
+
+ /* The GPU power may be turned off from this point */
+ kbdev->pm.backend.gpu_powered = false;
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
+
+ if (is_suspend && kbdev->pm.backend.callback_power_suspend)
+ kbdev->pm.backend.callback_power_suspend(kbdev);
+ else if (kbdev->pm.backend.callback_power_off)
+ kbdev->pm.backend.callback_power_off(kbdev);
+ return true;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_clock_off);
+
+struct kbasep_reset_timeout_data {
+ struct hrtimer timer;
+ bool timed_out;
+ struct kbase_device *kbdev;
+};
+
+void kbase_pm_reset_done(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ kbdev->pm.backend.reset_done = true;
+ wake_up(&kbdev->pm.backend.reset_done_wait);
+}
+
+/**
+ * Wait for the RESET_COMPLETED IRQ to occur, then reset the waiting state.
+ */
+static void kbase_pm_wait_for_reset(struct kbase_device *kbdev)
+{
+ lockdep_assert_held(&kbdev->pm.lock);
+
+ wait_event(kbdev->pm.backend.reset_done_wait,
+ (kbdev->pm.backend.reset_done));
+ kbdev->pm.backend.reset_done = false;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_reset_done);
+
+static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
+{
+ struct kbasep_reset_timeout_data *rtdata =
+ container_of(timer, struct kbasep_reset_timeout_data, timer);
+
+ rtdata->timed_out = 1;
+
+ /* Set the wait queue to wake up kbase_pm_init_hw even though the reset
+ * hasn't completed */
+ kbase_pm_reset_done(rtdata->kbdev);
+
+ return HRTIMER_NORESTART;
+}
+
+static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
+{
+ kbdev->hw_quirks_sc = 0;
+
+ /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443.
+ * and needed due to MIDGLES-3539. See PRLAM-11035 */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) ||
+ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035))
+ kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE;
+
+ /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327.
+ */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327))
+ kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD;
+
+ /* Enable alternative hardware counter selection if configured. */
+ if (DEFAULT_ALTERNATIVE_HWC)
+ kbdev->hw_quirks_sc |= SC_ALT_COUNTERS;
+
+ /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797))
+ kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS;
+
+ kbdev->hw_quirks_tiler = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(TILER_CONFIG), NULL);
+
+ /* Set tiler clock gate override if required */
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953))
+ kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
+
+ /* Limit the GPU bus bandwidth if the platform needs this. */
+ kbdev->hw_quirks_mmu = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(L2_MMU_CONFIG), NULL);
+
+ /* Limit read ID width for AXI */
+ kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS);
+ kbdev->hw_quirks_mmu |= (DEFAULT_ARID_LIMIT & 0x3) <<
+ L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT;
+
+ /* Limit write ID width for AXI */
+ kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES);
+ kbdev->hw_quirks_mmu |= (DEFAULT_AWID_LIMIT & 0x3) <<
+ L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT;
+}
+
+static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev)
+{
+ if (kbdev->hw_quirks_sc)
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG),
+ kbdev->hw_quirks_sc, NULL);
+
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG),
+ kbdev->hw_quirks_tiler, NULL);
+
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG),
+ kbdev->hw_quirks_mmu, NULL);
+}
+
+
+int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
+{
+ unsigned long irq_flags;
+ struct kbasep_reset_timeout_data rtdata;
+
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ lockdep_assert_held(&kbdev->pm.lock);
+
+ /* Ensure the clock is on before attempting to access the hardware */
+ if (!kbdev->pm.backend.gpu_powered) {
+ if (kbdev->pm.backend.callback_power_on)
+ kbdev->pm.backend.callback_power_on(kbdev);
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock,
+ irq_flags);
+ kbdev->pm.backend.gpu_powered = true;
+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
+ irq_flags);
+ }
+
+ /* Ensure interrupts are off to begin with, this also clears any
+ * outstanding interrupts */
+ kbase_pm_disable_interrupts(kbdev);
+ /* Prepare for the soft-reset */
+ kbdev->pm.backend.reset_done = false;
+
+ /* The cores should be made unavailable due to the reset */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, irq_flags);
+ if (kbdev->shader_available_bitmap != 0u)
+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
+ NULL, 0u, (u32)0u);
+ if (kbdev->tiler_available_bitmap != 0u)
+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
+ NULL, NULL, 0u, (u32)0u);
+ kbdev->shader_available_bitmap = 0u;
+ kbdev->tiler_available_bitmap = 0u;
+ kbdev->l2_available_bitmap = 0u;
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, irq_flags);
+
+ /* Soft reset the GPU */
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_SOFT_RESET, NULL);
+
+ /* Unmask the reset complete interrupt only */
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED,
+ NULL);
+
+ /* Initialize a structure for tracking the status of the reset */
+ rtdata.kbdev = kbdev;
+ rtdata.timed_out = 0;
+
+ /* Create a timer to use as a timeout on the reset */
+ hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ rtdata.timer.function = kbasep_reset_timeout;
+
+ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
+ HRTIMER_MODE_REL);
+
+ /* Wait for the RESET_COMPLETED interrupt to be raised */
+ kbase_pm_wait_for_reset(kbdev);
+
+ if (rtdata.timed_out == 0) {
+ /* GPU has been reset */
+ hrtimer_cancel(&rtdata.timer);
+ destroy_hrtimer_on_stack(&rtdata.timer);
+ goto out;
+ }
+
+ /* No interrupt has been received - check if the RAWSTAT register says
+ * the reset has completed */
+ if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
+ RESET_COMPLETED) {
+ /* The interrupt is set in the RAWSTAT; this suggests that the
+ * interrupts are not getting to the CPU */
+ dev_warn(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
+ /* If interrupts aren't working we can't continue. */
+ destroy_hrtimer_on_stack(&rtdata.timer);
+ goto out;
+ }
+
+ /* The GPU doesn't seem to be responding to the reset so try a hard
+ * reset */
+ dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n",
+ RESET_TIMEOUT);
+ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_HARD_RESET, NULL);
+
+ /* Restart the timer to wait for the hard reset to complete */
+ rtdata.timed_out = 0;
+
+ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
+ HRTIMER_MODE_REL);
+
+ /* Wait for the RESET_COMPLETED interrupt to be raised */
+ kbase_pm_wait_for_reset(kbdev);
+
+ if (rtdata.timed_out == 0) {
+ /* GPU has been reset */
+ hrtimer_cancel(&rtdata.timer);
+ destroy_hrtimer_on_stack(&rtdata.timer);
+ goto out;
+ }
+
+ destroy_hrtimer_on_stack(&rtdata.timer);
+
+ dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n",
+ RESET_TIMEOUT);
+
+ /* The GPU still hasn't reset, give up */
+ return -EINVAL;
+
+out:
+
+ if (flags & PM_HW_ISSUES_DETECT)
+ kbase_pm_hw_issues_detect(kbdev);
+
+ kbase_pm_hw_issues_apply(kbdev);
+
+
+ /* If cycle counter was in use re-enable it, enable_irqs will only be
+ * false when called from kbase_pm_powerup */
+ if (kbdev->pm.backend.gpu_cycle_counter_requests &&
+ (flags & PM_ENABLE_IRQS)) {
+ /* enable interrupts as the L2 may have to be powered on */
+ kbase_pm_enable_interrupts(kbdev);
+ kbase_pm_request_l2_caches(kbdev);
+
+ /* Re-enable the counters if we need to */
+ spin_lock_irqsave(
+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
+ irq_flags);
+ if (kbdev->pm.backend.gpu_cycle_counter_requests)
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_CYCLE_COUNT_START, NULL);
+ spin_unlock_irqrestore(
+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
+ irq_flags);
+
+ kbase_pm_release_l2_caches(kbdev);
+ kbase_pm_disable_interrupts(kbdev);
+ }
+
+ if (flags & PM_ENABLE_IRQS)
+ kbase_pm_enable_interrupts(kbdev);
+
+ return 0;
+}
+
+/**
+ * Increase the count of cycle counter users and turn the cycle counters on if
+ * they were previously off
+ *
+ * This function is designed to be called by
+ * @ref kbase_pm_request_gpu_cycle_counter or
+ * @ref kbase_pm_request_gpu_cycle_counter_l2_is_on only
+ *
+ * When this function is called the l2 cache must be on and the l2 cache users
+ * count must have been incremented by a call to (@ref
+ * kbase_pm_request_l2_caches or @ref kbase_pm_request_l2_caches_l2_on)
+ *
+ * @param kbdev The kbase device structure of the device
+ *
+ */
+static void
+kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
+ flags);
+
+ ++kbdev->pm.backend.gpu_cycle_counter_requests;
+
+ if (1 == kbdev->pm.backend.gpu_cycle_counter_requests)
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_CYCLE_COUNT_START, NULL);
+
+ spin_unlock_irqrestore(
+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
+ flags);
+}
+
+void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
+
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
+ INT_MAX);
+
+ kbase_pm_request_l2_caches(kbdev);
+
+ kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter);
+
+void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
+
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
+ INT_MAX);
+
+ kbase_pm_request_l2_caches_l2_is_on(kbdev);
+
+ kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on);
+
+void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
+ flags);
+
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0);
+
+ --kbdev->pm.backend.gpu_cycle_counter_requests;
+
+ if (0 == kbdev->pm.backend.gpu_cycle_counter_requests)
+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
+ GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
+
+ spin_unlock_irqrestore(
+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
+ flags);
+
+ kbase_pm_release_l2_caches(kbdev);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter);
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_pm_hwaccess_internal.h
+ * Power management API definitions used internally by GPU backend
+ */
+
+#ifndef _KBASE_BACKEND_PM_INTERNAL_H_
+#define _KBASE_BACKEND_PM_INTERNAL_H_
+
+#include <mali_kbase_hwaccess_pm.h>
+
+#include "mali_kbase_pm_ca.h"
+#include "mali_kbase_pm_policy.h"
+
+
+/**
+ * The GPU is idle.
+ *
+ * The OS may choose to turn off idle devices
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_dev_idle(struct kbase_device *kbdev);
+
+/**
+ * The GPU is active.
+ *
+ * The OS should avoid opportunistically turning off the GPU while it is active
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_dev_activate(struct kbase_device *kbdev);
+
+/**
+ * Get details of the cores that are present in the device.
+ *
+ * This function can be called by the active power policy to return a bitmask of
+ * the cores (of a specified type) present in the GPU device and also a count of
+ * the number of cores.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param type The type of core (see the @ref enum kbase_pm_core_type
+ * enumeration)
+ *
+ * @return The bit mask of cores present
+ */
+u64 kbase_pm_get_present_cores(struct kbase_device *kbdev,
+ enum kbase_pm_core_type type);
+
+/**
+ * Get details of the cores that are currently active in the device.
+ *
+ * This function can be called by the active power policy to return a bitmask of
+ * the cores (of a specified type) that are actively processing work (i.e.
+ * turned on *and* busy).
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param type The type of core (see the @ref enum kbase_pm_core_type
+ * enumeration)
+ *
+ * @return The bit mask of active cores
+ */
+u64 kbase_pm_get_active_cores(struct kbase_device *kbdev,
+ enum kbase_pm_core_type type);
+
+/**
+ * Get details of the cores that are currently transitioning between power
+ * states.
+ *
+ * This function can be called by the active power policy to return a bitmask of
+ * the cores (of a specified type) that are currently transitioning between
+ * power states.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param type The type of core (see the @ref enum kbase_pm_core_type
+ * enumeration)
+ *
+ * @return The bit mask of transitioning cores
+ */
+u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev,
+ enum kbase_pm_core_type type);
+
+/**
+ * Get details of the cores that are currently powered and ready for jobs.
+ *
+ * This function can be called by the active power policy to return a bitmask of
+ * the cores (of a specified type) that are powered and ready for jobs (they may
+ * or may not be currently executing jobs).
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param type The type of core (see the @ref enum kbase_pm_core_type
+ * enumeration)
+ *
+ * @return The bit mask of ready cores
+ */
+u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev,
+ enum kbase_pm_core_type type);
+
+/**
+ * Turn the clock for the device on, and enable device interrupts.
+ *
+ * This function can be used by a power policy to turn the clock for the GPU on.
+ * It should be modified during integration to perform the necessary actions to
+ * ensure that the GPU is fully powered and clocked.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param is_resume true if clock on due to resume after suspend,
+ * false otherwise
+ */
+void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume);
+
+/**
+ * Disable device interrupts, and turn the clock for the device off.
+ *
+ * This function can be used by a power policy to turn the clock for the GPU
+ * off. It should be modified during integration to perform the necessary
+ * actions to turn the clock off (if this is possible in the integration).
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param is_suspend true if clock off due to suspend, false otherwise
+ *
+ * @return true if clock was turned off
+ * false if clock can not be turned off due to pending page/bus fault
+ * workers. Caller must flush MMU workqueues and retry
+ */
+bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend);
+
+/**
+ * Enable interrupts on the device.
+ *
+ * Interrupts are also enabled after a call to kbase_pm_clock_on().
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_enable_interrupts(struct kbase_device *kbdev);
+
+/**
+ * Enable interrupts on the device, using the provided mask to set MMU_IRQ_MASK.
+ *
+ * Interrupts are also enabled after a call to kbase_pm_clock_on().
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param mask The mask to use for MMU_IRQ_MASK
+ */
+void kbase_pm_enable_interrupts_mmu_mask(struct kbase_device *kbdev, u32 mask);
+
+/**
+ * Disable interrupts on the device.
+ *
+ * This prevents delivery of Power Management interrupts to the CPU so that
+ * kbase_pm_check_transitions_nolock() will not be called from the IRQ handler
+ * until @ref kbase_pm_enable_interrupts or kbase_pm_clock_on() is called.
+ *
+ * Interrupts are also disabled after a call to kbase_pm_clock_off().
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_disable_interrupts(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_init_hw - Initialize the hardware.
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ * @flags: Flags specifying the type of PM init
+ *
+ * This function checks the GPU ID register to ensure that the GPU is supported
+ * by the driver and performs a reset on the device so that it is in a known
+ * state before the device is used.
+ *
+ * Return: 0 if the device is supported and successfully reset.
+ */
+int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags);
+
+/**
+ * The GPU has been reset successfully.
+ *
+ * This function must be called by the GPU interrupt handler when the
+ * RESET_COMPLETED bit is set. It signals to the power management initialization
+ * code that the GPU has been successfully reset.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_reset_done(struct kbase_device *kbdev);
+
+
+/**
+ * Check if there are any power transitions to make, and if so start them.
+ *
+ * This function will check the desired_xx_state members of
+ * struct kbase_pm_device_data and the actual status of the hardware to see if
+ * any power transitions can be made at this time to make the hardware state
+ * closer to the state desired by the power policy.
+ *
+ * The return value can be used to check whether all the desired cores are
+ * available, and so whether it's worth submitting a job (e.g. from a Power
+ * Management IRQ).
+ *
+ * Note that this still returns true when desired_xx_state has no
+ * cores. That is: of the no cores desired, none were <em>un</em>available. In
+ * this case, the caller may still need to try submitting jobs. This is because
+ * the Core Availability Policy might have taken us to an intermediate state
+ * where no cores are powered, before powering on more cores (e.g. for core
+ * rotation)
+ *
+ * The caller must hold kbase_device::pm::power_change_lock
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @return non-zero when all desired cores are available. That is,
+ * it's worthwhile for the caller to submit a job.
+ * @return false otherwise
+ */
+bool kbase_pm_check_transitions_nolock(struct kbase_device *kbdev);
+
+/**
+ * Synchronous and locking variant of kbase_pm_check_transitions_nolock()
+ *
+ * On returning, the desired state at the time of the call will have been met.
+ *
+ * @note There is nothing to stop the core being switched off by calls to
+ * kbase_pm_release_cores() or kbase_pm_unrequest_cores(). Therefore, the
+ * caller must have already made a call to
+ * kbase_pm_request_cores()/kbase_pm_request_cores_sync() previously.
+ *
+ * The usual use-case for this is to ensure cores are 'READY' after performing
+ * a GPU Reset.
+ *
+ * Unlike kbase_pm_check_transitions_nolock(), the caller must not hold
+ * kbase_device::pm::power_change_lock, because this function will take that
+ * lock itself.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_check_transitions_sync(struct kbase_device *kbdev);
+
+/**
+ * Variant of kbase_pm_update_cores_state() where the caller must hold
+ * kbase_device::pm::power_change_lock
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev);
+
+/**
+ * Update the desired state of shader cores from the Power Policy, and begin
+ * any power transitions.
+ *
+ * This function will update the desired_xx_state members of
+ * struct kbase_pm_device_data by calling into the current Power Policy. It will
+ * then begin power transitions to make the hardware acheive the desired shader
+ * core state.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_update_cores_state(struct kbase_device *kbdev);
+
+/**
+ * Cancel any pending requests to power off the GPU and/or shader cores.
+ *
+ * This should be called by any functions which directly power off the GPU.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev);
+
+/**
+ * Read the bitmasks of present cores.
+ *
+ * This information is cached to avoid having to perform register reads whenever
+ * the information is required.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbasep_pm_read_present_cores(struct kbase_device *kbdev);
+
+/**
+ * Initialize the metrics gathering framework.
+ *
+ * This must be called before other metric gathering APIs are called.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ *
+ * @return 0 on success, error code on error
+ */
+int kbasep_pm_metrics_init(struct kbase_device *kbdev);
+
+/**
+ * Terminate the metrics gathering framework.
+ *
+ * This must be called when metric gathering is no longer required. It is an
+ * error to call any metrics gathering function (other than
+ * kbasep_pm_metrics_init) after calling this function.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbasep_pm_metrics_term(struct kbase_device *kbdev);
+
+/**
+ * Function to be called by the frame buffer driver to update the vsync metric.
+ *
+ * This function should be called by the frame buffer driver to update whether
+ * the system is hitting the vsync target or not. buffer_updated should be true
+ * if the vsync corresponded with a new frame being displayed, otherwise it
+ * should be false. This function does not need to be called every vsync, but
+ * only when the value of buffer_updated differs from a previous call.
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ * @param buffer_updated True if the buffer has been updated on this VSync,
+ * false otherwise
+ */
+void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated);
+
+/**
+ * Configure the frame buffer device to set the vsync callback.
+ *
+ * This function should do whatever is necessary for this integration to ensure
+ * that kbase_pm_report_vsync is called appropriately.
+ *
+ * This function will need porting as part of the integration for a device.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_register_vsync_callback(struct kbase_device *kbdev);
+
+/**
+ * Free any resources that kbase_pm_register_vsync_callback allocated.
+ *
+ * This function should perform any cleanup required from the call to
+ * kbase_pm_register_vsync_callback. No call backs should occur after this
+ * function has returned.
+ *
+ * This function will need porting as part of the integration for a device.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_unregister_vsync_callback(struct kbase_device *kbdev);
+
+/**
+ * Determine whether the DVFS system should change the clock speed of the GPU.
+ *
+ * This function should be called regularly by the DVFS system to check whether
+ * the clock speed of the GPU needs updating. It will return one of three
+ * enumerated values of kbase_pm_dvfs_action:
+ *
+ * @param kbdev The kbase device structure for the device
+ * (must be a valid pointer)
+ * @retval KBASE_PM_DVFS_NOP The clock does not need changing
+ * @retval KBASE_PM_DVFS_CLOCK_UP The clock frequency should be increased if
+ * possible.
+ * @retval KBASE_PM_DVFS_CLOCK_DOWN The clock frequency should be decreased if
+ * possible.
+ */
+enum kbase_pm_dvfs_action kbase_pm_get_dvfs_action(struct kbase_device *kbdev);
+
+/**
+ * Mark that the GPU cycle counter is needed, if the caller is the first caller
+ * then the GPU cycle counters will be enabled along with the l2 cache
+ *
+ * The GPU must be powered when calling this function (i.e.
+ * @ref kbase_pm_context_active must have been called).
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev);
+
+/**
+ * This is a version of the above function
+ * (@ref kbase_pm_request_gpu_cycle_counter) suitable for being called when the
+ * l2 cache is known to be on and assured to be on until the subsequent call of
+ * kbase_pm_release_gpu_cycle_counter such as when a job is submitted. It does
+ * not sleep and can be called from atomic functions.
+ *
+ * The GPU must be powered when calling this function (i.e.
+ * @ref kbase_pm_context_active must have been called) and the l2 cache must be
+ * powered on.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev);
+
+/**
+ * Mark that the GPU cycle counter is no longer in use, if the caller is the
+ * last caller then the GPU cycle counters will be disabled. A request must have
+ * been made before a call to this.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev);
+
+/**
+ * Enables access to the GPU registers before power management has powered up
+ * the GPU with kbase_pm_powerup().
+ *
+ * Access to registers should be done using kbase_os_reg_read/write() at this
+ * stage, not kbase_reg_read/write().
+ *
+ * This results in the power management callbacks provided in the driver
+ * configuration to get called to turn on power and/or clocks to the GPU. See
+ * @ref kbase_pm_callback_conf.
+ *
+ * This should only be used before power management is powered up with
+ * kbase_pm_powerup()
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_register_access_enable(struct kbase_device *kbdev);
+
+/**
+ * Disables access to the GPU registers enabled earlier by a call to
+ * kbase_pm_register_access_enable().
+ *
+ * This results in the power management callbacks provided in the driver
+ * configuration to get called to turn off power and/or clocks to the GPU. See
+ * @ref kbase_pm_callback_conf
+ *
+ * This should only be used before power management is powered up with
+ * kbase_pm_powerup()
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_pm_register_access_disable(struct kbase_device *kbdev);
+
+/* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline
+ * function */
+
+/**
+ * Check if the power management metrics collection is active.
+ *
+ * Note that this returns if the power management metrics collection was
+ * active at the time of calling, it is possible that after the call the metrics
+ * collection enable may have changed state.
+ *
+ * The caller must handle the consequence that the state may have changed.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @return true if metrics collection was active else false.
+ */
+bool kbase_pm_metrics_is_active(struct kbase_device *kbdev);
+
+/**
+ * Power on the GPU, and any cores that are requested.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param is_resume true if power on due to resume after suspend,
+ * false otherwise
+ */
+void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume);
+
+/**
+ * Power off the GPU, and any cores that have been requested.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param is_suspend true if power off due to suspend,
+ * false otherwise
+ * @return true if power was turned off
+ * false if power can not be turned off due to pending page/bus
+ * fault workers. Caller must flush MMU workqueues and retry
+ */
+bool kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend);
+
+#ifdef CONFIG_PM_DEVFREQ
+void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev,
+ unsigned long *total, unsigned long *busy);
+void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev);
+#endif
+
+#ifdef CONFIG_MALI_MIDGARD_DVFS
+
+/**
+ * Function provided by platform specific code when DVFS is enabled to allow
+ * the power management metrics system to report utilisation.
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ * @param utilisation The current calculated utilisation by the metrics
+ * system.
+ * @param util_gl_share The current calculated gl share of utilisation.
+ * @param util_cl_share The current calculated cl share of utilisation per core
+ * group.
+ * @return Returns 0 on failure and non zero on success.
+ */
+
+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation,
+ u32 util_gl_share, u32 util_cl_share[2]);
+#endif
+
+void kbase_pm_power_changed(struct kbase_device *kbdev);
+
+/**
+ * Inform the metrics system that an atom is about to be run.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param katom The atom that is about to be run
+ */
+void kbase_pm_metrics_run_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom);
+
+/**
+ * Inform the metrics system that an atom has been run and is being released.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param katom The atom that is about to be released
+ */
+void kbase_pm_metrics_release_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom);
+
+
+#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_pm_metrics.c
+ * Metrics for power management
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_pm.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+/* When VSync is being hit aim for utilisation between 70-90% */
+#define KBASE_PM_VSYNC_MIN_UTILISATION 70
+#define KBASE_PM_VSYNC_MAX_UTILISATION 90
+/* Otherwise aim for 10-40% */
+#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10
+#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40
+
+/* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns
+ * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly
+ * under 11s. Exceeding this will cause overflow */
+#define KBASE_PM_TIME_SHIFT 8
+
+/* Maximum time between sampling of utilization data, without resetting the
+ * counters. */
+#define MALI_UTILIZATION_MAX_PERIOD 100000 /* ns = 100ms */
+
+#ifdef CONFIG_MALI_MIDGARD_DVFS
+static enum hrtimer_restart dvfs_callback(struct hrtimer *timer)
+{
+ unsigned long flags;
+ enum kbase_pm_dvfs_action action;
+ struct kbasep_pm_metrics_data *metrics;
+
+ KBASE_DEBUG_ASSERT(timer != NULL);
+
+ metrics = container_of(timer, struct kbasep_pm_metrics_data, timer);
+ action = kbase_pm_get_dvfs_action(metrics->kbdev);
+
+ spin_lock_irqsave(&metrics->lock, flags);
+
+ if (metrics->timer_active)
+ hrtimer_start(timer,
+ HR_TIMER_DELAY_MSEC(metrics->kbdev->pm.dvfs_period),
+ HRTIMER_MODE_REL);
+
+ spin_unlock_irqrestore(&metrics->lock, flags);
+
+ return HRTIMER_NORESTART;
+}
+#endif /* CONFIG_MALI_MIDGARD_DVFS */
+
+int kbasep_pm_metrics_init(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ kbdev->pm.backend.metrics.kbdev = kbdev;
+ kbdev->pm.backend.metrics.vsync_hit = 0;
+ kbdev->pm.backend.metrics.utilisation = 0;
+ kbdev->pm.backend.metrics.util_cl_share[0] = 0;
+ kbdev->pm.backend.metrics.util_cl_share[1] = 0;
+ kbdev->pm.backend.metrics.util_gl_share = 0;
+
+ kbdev->pm.backend.metrics.time_period_start = ktime_get();
+ kbdev->pm.backend.metrics.time_busy = 0;
+ kbdev->pm.backend.metrics.time_idle = 0;
+ kbdev->pm.backend.metrics.prev_busy = 0;
+ kbdev->pm.backend.metrics.prev_idle = 0;
+ kbdev->pm.backend.metrics.gpu_active = false;
+ kbdev->pm.backend.metrics.active_cl_ctx[0] = 0;
+ kbdev->pm.backend.metrics.active_cl_ctx[1] = 0;
+ kbdev->pm.backend.metrics.active_gl_ctx = 0;
+ kbdev->pm.backend.metrics.busy_cl[0] = 0;
+ kbdev->pm.backend.metrics.busy_cl[1] = 0;
+ kbdev->pm.backend.metrics.busy_gl = 0;
+ kbdev->pm.backend.metrics.nr_in_slots = 0;
+
+ spin_lock_init(&kbdev->pm.backend.metrics.lock);
+
+#ifdef CONFIG_MALI_MIDGARD_DVFS
+ kbdev->pm.backend.metrics.timer_active = true;
+ hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC,
+ HRTIMER_MODE_REL);
+ kbdev->pm.backend.metrics.timer.function = dvfs_callback;
+
+ hrtimer_start(&kbdev->pm.backend.metrics.timer,
+ HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period),
+ HRTIMER_MODE_REL);
+#endif /* CONFIG_MALI_MIDGARD_DVFS */
+
+ kbase_pm_register_vsync_callback(kbdev);
+
+ return 0;
+}
+
+KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init);
+
+void kbasep_pm_metrics_term(struct kbase_device *kbdev)
+{
+#ifdef CONFIG_MALI_MIDGARD_DVFS
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+ kbdev->pm.backend.metrics.timer_active = false;
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+
+ hrtimer_cancel(&kbdev->pm.backend.metrics.timer);
+#endif /* CONFIG_MALI_MIDGARD_DVFS */
+
+ kbase_pm_unregister_vsync_callback(kbdev);
+}
+
+KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term);
+
+/**
+ * kbasep_pm_record_gpu_active - update metrics tracking GPU active time
+ *
+ * This function updates the time the GPU was busy executing jobs in
+ * general and specifically for CL and GL jobs. Call this function when
+ * a job is submitted or removed from the GPU (job issue) slots.
+ *
+ * The busy time recorded is the time passed since the last time the
+ * busy/idle metrics were updated (e.g. by this function,
+ * kbasep_pm_record_gpu_idle or others).
+ *
+ * Note that the time we record towards CL and GL jobs accounts for
+ * the total number of CL and GL jobs active at that time. If 20ms
+ * has passed and 3 GL jobs were active, we account 3*20 ms towards
+ * the GL busy time. The number of CL/GL jobs active is tracked by
+ * kbase_pm_metrics_run_atom() / kbase_pm_metrics_release_atom().
+ *
+ * The kbdev->pm.backend.metrics.lock needs to be held when calling
+ * this function.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+static void kbasep_pm_record_gpu_active(struct kbase_device *kbdev)
+{
+ ktime_t now = ktime_get();
+ ktime_t diff;
+ u32 ns_time;
+
+ lockdep_assert_held(&kbdev->pm.backend.metrics.lock);
+
+ /* Record active time */
+ diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start);
+ ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT);
+ kbdev->pm.backend.metrics.time_busy += ns_time;
+ kbdev->pm.backend.metrics.busy_gl += ns_time *
+ kbdev->pm.backend.metrics.active_gl_ctx;
+ kbdev->pm.backend.metrics.busy_cl[0] += ns_time *
+ kbdev->pm.backend.metrics.active_cl_ctx[0];
+ kbdev->pm.backend.metrics.busy_cl[1] += ns_time *
+ kbdev->pm.backend.metrics.active_cl_ctx[1];
+ /* Reset time period */
+ kbdev->pm.backend.metrics.time_period_start = now;
+}
+
+/**
+ * kbasep_pm_record_gpu_idle - update metrics tracking GPU idle time
+ *
+ * This function updates the time the GPU was idle (not executing any
+ * jobs) based on the time passed when kbasep_pm_record_gpu_active()
+ * was called last to record the last job on the GPU finishing.
+ *
+ * Call this function when no jobs are in the job slots of the GPU and
+ * a job is about to be submitted to a job slot.
+ *
+ * The kbdev->pm.backend.metrics.lock needs to be held when calling
+ * this function.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+static void kbasep_pm_record_gpu_idle(struct kbase_device *kbdev)
+{
+ ktime_t now = ktime_get();
+ ktime_t diff;
+ u32 ns_time;
+
+ lockdep_assert_held(&kbdev->pm.backend.metrics.lock);
+
+ /* Record idle time */
+ diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start);
+ ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT);
+ kbdev->pm.backend.metrics.time_idle += ns_time;
+ /* Reset time period */
+ kbdev->pm.backend.metrics.time_period_start = now;
+}
+
+void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated)
+{
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+ kbdev->pm.backend.metrics.vsync_hit = buffer_updated;
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_report_vsync);
+
+#if defined(CONFIG_PM_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS)
+/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this
+ * function
+ */
+static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev,
+ ktime_t now)
+{
+ ktime_t diff;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start);
+
+ if (kbdev->pm.backend.metrics.gpu_active) {
+ u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT);
+
+ kbdev->pm.backend.metrics.time_busy += ns_time;
+ kbdev->pm.backend.metrics.busy_cl[0] += ns_time *
+ kbdev->pm.backend.metrics.active_cl_ctx[0];
+ kbdev->pm.backend.metrics.busy_cl[1] += ns_time *
+ kbdev->pm.backend.metrics.active_cl_ctx[1];
+ kbdev->pm.backend.metrics.busy_gl += ns_time *
+ kbdev->pm.backend.metrics.active_gl_ctx;
+ } else {
+ kbdev->pm.backend.metrics.time_idle += (u32) (ktime_to_ns(diff)
+ >> KBASE_PM_TIME_SHIFT);
+ }
+}
+
+/* Caller needs to hold kbdev->pm.backend.metrics.lock before calling this
+ * function.
+ */
+static void kbase_pm_reset_dvfs_utilisation_unlocked(struct kbase_device *kbdev,
+ ktime_t now)
+{
+ /* Store previous value */
+ kbdev->pm.backend.metrics.prev_idle =
+ kbdev->pm.backend.metrics.time_idle;
+ kbdev->pm.backend.metrics.prev_busy =
+ kbdev->pm.backend.metrics.time_busy;
+
+ /* Reset current values */
+ kbdev->pm.backend.metrics.time_period_start = now;
+ kbdev->pm.backend.metrics.time_idle = 0;
+ kbdev->pm.backend.metrics.time_busy = 0;
+ kbdev->pm.backend.metrics.busy_cl[0] = 0;
+ kbdev->pm.backend.metrics.busy_cl[1] = 0;
+ kbdev->pm.backend.metrics.busy_gl = 0;
+}
+
+void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, ktime_get());
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+}
+
+void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev,
+ unsigned long *total_out, unsigned long *busy_out)
+{
+ ktime_t now = ktime_get();
+ unsigned long flags, busy, total;
+
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+ kbase_pm_get_dvfs_utilisation_calc(kbdev, now);
+
+ busy = kbdev->pm.backend.metrics.time_busy;
+ total = busy + kbdev->pm.backend.metrics.time_idle;
+
+ /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default
+ * 100ms) */
+ if (total >= MALI_UTILIZATION_MAX_PERIOD) {
+ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now);
+ } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) {
+ total += kbdev->pm.backend.metrics.prev_idle +
+ kbdev->pm.backend.metrics.prev_busy;
+ busy += kbdev->pm.backend.metrics.prev_busy;
+ }
+
+ *total_out = total;
+ *busy_out = busy;
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+}
+#endif
+
+#ifdef CONFIG_MALI_MIDGARD_DVFS
+
+/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this
+ * function
+ */
+int kbase_pm_get_dvfs_utilisation_old(struct kbase_device *kbdev,
+ int *util_gl_share,
+ int util_cl_share[2])
+{
+ int utilisation;
+ int busy;
+ ktime_t now = ktime_get();
+
+ kbase_pm_get_dvfs_utilisation_calc(kbdev, now);
+
+ if (kbdev->pm.backend.metrics.time_idle +
+ kbdev->pm.backend.metrics.time_busy == 0) {
+ /* No data - so we return NOP */
+ utilisation = -1;
+ if (util_gl_share)
+ *util_gl_share = -1;
+ if (util_cl_share) {
+ util_cl_share[0] = -1;
+ util_cl_share[1] = -1;
+ }
+ goto out;
+ }
+
+ utilisation = (100 * kbdev->pm.backend.metrics.time_busy) /
+ (kbdev->pm.backend.metrics.time_idle +
+ kbdev->pm.backend.metrics.time_busy);
+
+ busy = kbdev->pm.backend.metrics.busy_gl +
+ kbdev->pm.backend.metrics.busy_cl[0] +
+ kbdev->pm.backend.metrics.busy_cl[1];
+
+ if (busy != 0) {
+ if (util_gl_share)
+ *util_gl_share =
+ (100 * kbdev->pm.backend.metrics.busy_gl) /
+ busy;
+ if (util_cl_share) {
+ util_cl_share[0] =
+ (100 * kbdev->pm.backend.metrics.busy_cl[0]) /
+ busy;
+ util_cl_share[1] =
+ (100 * kbdev->pm.backend.metrics.busy_cl[1]) /
+ busy;
+ }
+ } else {
+ if (util_gl_share)
+ *util_gl_share = -1;
+ if (util_cl_share) {
+ util_cl_share[0] = -1;
+ util_cl_share[1] = -1;
+ }
+ }
+
+out:
+ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now);
+
+ return utilisation;
+}
+
+enum kbase_pm_dvfs_action kbase_pm_get_dvfs_action(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+ int utilisation, util_gl_share;
+ int util_cl_share[2];
+ enum kbase_pm_dvfs_action action;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+
+ utilisation = kbase_pm_get_dvfs_utilisation_old(kbdev, &util_gl_share,
+ util_cl_share);
+
+ if (utilisation < 0 || util_gl_share < 0 || util_cl_share[0] < 0 ||
+ util_cl_share[1] < 0) {
+ action = KBASE_PM_DVFS_NOP;
+ utilisation = 0;
+ util_gl_share = 0;
+ util_cl_share[0] = 0;
+ util_cl_share[1] = 0;
+ goto out;
+ }
+
+ if (kbdev->pm.backend.metrics.vsync_hit) {
+ /* VSync is being met */
+ if (utilisation < KBASE_PM_VSYNC_MIN_UTILISATION)
+ action = KBASE_PM_DVFS_CLOCK_DOWN;
+ else if (utilisation > KBASE_PM_VSYNC_MAX_UTILISATION)
+ action = KBASE_PM_DVFS_CLOCK_UP;
+ else
+ action = KBASE_PM_DVFS_NOP;
+ } else {
+ /* VSync is being missed */
+ if (utilisation < KBASE_PM_NO_VSYNC_MIN_UTILISATION)
+ action = KBASE_PM_DVFS_CLOCK_DOWN;
+ else if (utilisation > KBASE_PM_NO_VSYNC_MAX_UTILISATION)
+ action = KBASE_PM_DVFS_CLOCK_UP;
+ else
+ action = KBASE_PM_DVFS_NOP;
+ }
+
+ kbdev->pm.backend.metrics.utilisation = utilisation;
+ kbdev->pm.backend.metrics.util_cl_share[0] = util_cl_share[0];
+ kbdev->pm.backend.metrics.util_cl_share[1] = util_cl_share[1];
+ kbdev->pm.backend.metrics.util_gl_share = util_gl_share;
+out:
+#ifdef CONFIG_MALI_MIDGARD_DVFS
+ kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share,
+ util_cl_share);
+#endif /*CONFIG_MALI_MIDGARD_DVFS */
+
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+
+ return action;
+}
+KBASE_EXPORT_TEST_API(kbase_pm_get_dvfs_action);
+
+bool kbase_pm_metrics_is_active(struct kbase_device *kbdev)
+{
+ bool isactive;
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+ isactive = kbdev->pm.backend.metrics.timer_active;
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+
+ return isactive;
+}
+KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active);
+
+#endif /* CONFIG_MALI_MIDGARD_DVFS */
+
+/* called when job is submitted to a GPU slot */
+void kbase_pm_metrics_run_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+
+ /* We may have been idle before */
+ if (kbdev->pm.backend.metrics.nr_in_slots == 0) {
+ WARN_ON(kbdev->pm.backend.metrics.active_cl_ctx[0] != 0);
+ WARN_ON(kbdev->pm.backend.metrics.active_cl_ctx[1] != 0);
+ WARN_ON(kbdev->pm.backend.metrics.active_gl_ctx != 0);
+
+ /* Record idle time */
+ kbasep_pm_record_gpu_idle(kbdev);
+
+ /* We are now active */
+ WARN_ON(kbdev->pm.backend.metrics.gpu_active);
+ kbdev->pm.backend.metrics.gpu_active = true;
+
+ } else {
+ /* Record active time */
+ kbasep_pm_record_gpu_active(kbdev);
+ }
+
+ /* Track number of jobs in GPU slots */
+ WARN_ON(kbdev->pm.backend.metrics.nr_in_slots == U8_MAX);
+ kbdev->pm.backend.metrics.nr_in_slots++;
+
+ /* Track if it was a CL or GL one that was submitted to a GPU slot */
+ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
+ int device_nr = (katom->core_req &
+ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)
+ ? katom->device_nr : 0;
+ KBASE_DEBUG_ASSERT(device_nr < 2);
+ kbdev->pm.backend.metrics.active_cl_ctx[device_nr]++;
+ } else {
+ kbdev->pm.backend.metrics.active_gl_ctx++;
+ }
+
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+}
+
+/* called when job is removed from a GPU slot */
+void kbase_pm_metrics_release_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+
+ /* Track how long CL and/or GL jobs have been busy for */
+ kbasep_pm_record_gpu_active(kbdev);
+
+ /* Track number of jobs in GPU slots */
+ WARN_ON(kbdev->pm.backend.metrics.nr_in_slots == 0);
+ kbdev->pm.backend.metrics.nr_in_slots--;
+
+ /* We may become idle */
+ if (kbdev->pm.backend.metrics.nr_in_slots == 0) {
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.metrics.gpu_active);
+ kbdev->pm.backend.metrics.gpu_active = false;
+ }
+
+ /* Track of the GPU jobs that are active which ones are CL and
+ * which GL */
+ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
+ int device_nr = (katom->core_req &
+ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)
+ ? katom->device_nr : 0;
+ KBASE_DEBUG_ASSERT(device_nr < 2);
+
+ WARN_ON(kbdev->pm.backend.metrics.active_cl_ctx[device_nr]
+ == 0);
+ kbdev->pm.backend.metrics.active_cl_ctx[device_nr]--;
+ } else {
+ WARN_ON(kbdev->pm.backend.metrics.active_gl_ctx == 0);
+ kbdev->pm.backend.metrics.active_gl_ctx--;
+ }
+
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+}
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/**
+ * @file mali_kbase_pm_metrics_dummy.c
+ * Dummy Metrics for power management.
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_pm.h>
+
+void kbase_pm_register_vsync_callback(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ /* no VSync metrics will be available */
+ kbdev->pm.backend.metrics.platform_data = NULL;
+}
+
+void kbase_pm_unregister_vsync_callback(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+}
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * @file mali_kbase_pm_policy.c
+ * Power policy API implementations
+ */
+
+#include <mali_kbase.h>
+#include <mali_midg_regmap.h>
+#include <mali_kbase_gator.h>
+#include <mali_kbase_pm.h>
+#include <mali_kbase_config_defaults.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+static const struct kbase_pm_policy *const policy_list[] = {
+#ifdef CONFIG_MALI_NO_MALI
+ &kbase_pm_always_on_policy_ops,
+ &kbase_pm_demand_policy_ops,
+ &kbase_pm_coarse_demand_policy_ops,
+#if !MALI_CUSTOMER_RELEASE
+ &kbase_pm_demand_always_powered_policy_ops,
+ &kbase_pm_fast_start_policy_ops,
+#endif
+#else /* CONFIG_MALI_NO_MALI */
+ &kbase_pm_demand_policy_ops,
+ &kbase_pm_always_on_policy_ops,
+ &kbase_pm_coarse_demand_policy_ops,
+#if !MALI_CUSTOMER_RELEASE
+ &kbase_pm_demand_always_powered_policy_ops,
+ &kbase_pm_fast_start_policy_ops,
+#endif
+#endif /* CONFIG_MALI_NO_MALI */
+};
+
+/** The number of policies available in the system.
+ * This is derived from the number of functions listed in policy_get_functions.
+ */
+#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
+
+
+/* Function IDs for looking up Timeline Trace codes in
+ * kbase_pm_change_state_trace_code */
+enum kbase_pm_func_id {
+ KBASE_PM_FUNC_ID_REQUEST_CORES_START,
+ KBASE_PM_FUNC_ID_REQUEST_CORES_END,
+ KBASE_PM_FUNC_ID_RELEASE_CORES_START,
+ KBASE_PM_FUNC_ID_RELEASE_CORES_END,
+ /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither
+ * expect to hit it nor tend to hit it very much anyway. We can detect
+ * whether we need more instrumentation by a difference between
+ * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */
+
+ /* Must be the last */
+ KBASE_PM_FUNC_ID_COUNT
+};
+
+
+/* State changes during request/unrequest/release-ing cores */
+enum {
+ KBASE_PM_CHANGE_STATE_SHADER = (1u << 0),
+ KBASE_PM_CHANGE_STATE_TILER = (1u << 1),
+
+ /* These two must be last */
+ KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER |
+ KBASE_PM_CHANGE_STATE_SHADER),
+ KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1
+};
+typedef u32 kbase_pm_change_state;
+
+
+#ifdef CONFIG_MALI_TRACE_TIMELINE
+/* Timeline Trace code lookups for each function */
+static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT]
+ [KBASE_PM_CHANGE_STATE_COUNT] = {
+ /* kbase_pm_request_cores */
+ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0,
+ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START,
+ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START,
+ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
+ KBASE_PM_CHANGE_STATE_TILER] =
+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START,
+
+ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0,
+ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END,
+ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END,
+ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
+ KBASE_PM_CHANGE_STATE_TILER] =
+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END,
+
+ /* kbase_pm_release_cores */
+ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0,
+ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START,
+ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START,
+ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
+ KBASE_PM_CHANGE_STATE_TILER] =
+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START,
+
+ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0,
+ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END,
+ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END,
+ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
+ KBASE_PM_CHANGE_STATE_TILER] =
+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END
+};
+
+static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
+ enum kbase_pm_func_id func_id,
+ kbase_pm_change_state state)
+{
+ int trace_code;
+
+ KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT);
+ KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) ==
+ state);
+
+ trace_code = kbase_pm_change_state_trace_code[func_id][state];
+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code);
+}
+
+#else /* CONFIG_MALI_TRACE_TIMELINE */
+static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
+ enum kbase_pm_func_id func_id, kbase_pm_change_state state)
+{
+}
+
+#endif /* CONFIG_MALI_TRACE_TIMELINE */
+
+static enum hrtimer_restart
+kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer)
+{
+ struct kbase_device *kbdev;
+
+ kbdev = container_of(timer, struct kbase_device,
+ pm.backend.gpu_poweroff_timer);
+
+ /* It is safe for this call to do nothing if the work item is already
+ * queued. The worker function will read the must up-to-date state of
+ * kbdev->pm.backend.gpu_poweroff_pending under lock.
+ *
+ * If a state change occurs while the worker function is processing,
+ * this call will succeed as a work item can be requeued once it has
+ * started processing.
+ */
+ if (kbdev->pm.backend.gpu_poweroff_pending)
+ queue_work(kbdev->pm.backend.gpu_poweroff_wq,
+ &kbdev->pm.backend.gpu_poweroff_work);
+
+ if (kbdev->pm.backend.shader_poweroff_pending) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ if (kbdev->pm.backend.shader_poweroff_pending) {
+ kbdev->pm.backend.shader_poweroff_pending_time--;
+
+ KBASE_DEBUG_ASSERT(
+ kbdev->pm.backend.shader_poweroff_pending_time
+ >= 0);
+
+ if (!kbdev->pm.backend.shader_poweroff_pending_time) {
+ u64 prev_shader_state =
+ kbdev->pm.backend.desired_shader_state;
+
+ kbdev->pm.backend.desired_shader_state &=
+ ~kbdev->pm.backend.shader_poweroff_pending;
+
+ kbdev->pm.backend.shader_poweroff_pending = 0;
+
+ if (prev_shader_state !=
+ kbdev->pm.backend.desired_shader_state
+ || kbdev->pm.backend.ca_in_transition) {
+ bool cores_are_available;
+
+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START);
+ cores_are_available =
+ kbase_pm_check_transitions_nolock(
+ kbdev);
+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END);
+
+ /* Don't need 'cores_are_available',
+ * because we don't return anything */
+ CSTD_UNUSED(cores_are_available);
+ }
+ }
+ }
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+ }
+
+ if (kbdev->pm.backend.poweroff_timer_needed) {
+ hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time);
+
+ return HRTIMER_RESTART;
+ }
+
+ return HRTIMER_NORESTART;
+}
+
+static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data)
+{
+ unsigned long flags;
+ struct kbase_device *kbdev;
+ bool do_poweroff = false;
+
+ kbdev = container_of(data, struct kbase_device,
+ pm.backend.gpu_poweroff_work);
+
+ mutex_lock(&kbdev->pm.lock);
+
+ if (kbdev->pm.backend.gpu_poweroff_pending == 0) {
+ mutex_unlock(&kbdev->pm.lock);
+ return;
+ }
+
+ kbdev->pm.backend.gpu_poweroff_pending--;
+
+ if (kbdev->pm.backend.gpu_poweroff_pending > 0) {
+ mutex_unlock(&kbdev->pm.lock);
+ return;
+ }
+
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0);
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ /* Only power off the GPU if a request is still pending */
+ if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev))
+ do_poweroff = true;
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ if (do_poweroff) {
+ kbdev->pm.backend.poweroff_timer_needed = false;
+ hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
+ /* Power off the GPU */
+ if (!kbase_pm_do_poweroff(kbdev, false)) {
+ /* GPU can not be powered off at present */
+ kbdev->pm.backend.poweroff_timer_needed = true;
+ hrtimer_start(&kbdev->pm.backend.gpu_poweroff_timer,
+ kbdev->pm.gpu_poweroff_time,
+ HRTIMER_MODE_REL);
+ }
+ }
+
+ mutex_unlock(&kbdev->pm.lock);
+}
+
+int kbase_pm_policy_init(struct kbase_device *kbdev)
+{
+ struct workqueue_struct *wq;
+
+ wq = alloc_workqueue("kbase_pm_do_poweroff",
+ WQ_HIGHPRI | WQ_UNBOUND, 1);
+ if (!wq)
+ return -ENOMEM;
+
+ kbdev->pm.backend.gpu_poweroff_wq = wq;
+ INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work,
+ kbasep_pm_do_gpu_poweroff_wq);
+ hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer,
+ CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ kbdev->pm.backend.gpu_poweroff_timer.function =
+ kbasep_pm_do_gpu_poweroff_callback;
+ kbdev->pm.backend.pm_current_policy = policy_list[0];
+ kbdev->pm.backend.pm_current_policy->init(kbdev);
+ kbdev->pm.gpu_poweroff_time =
+ HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS);
+ kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
+ kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU;
+
+ return 0;
+}
+
+void kbase_pm_policy_term(struct kbase_device *kbdev)
+{
+ kbdev->pm.backend.pm_current_policy->term(kbdev);
+ destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq);
+}
+
+void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ lockdep_assert_held(&kbdev->pm.lock);
+
+ kbdev->pm.backend.poweroff_timer_needed = false;
+ hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
+
+ /* If wq is already running but is held off by pm.lock, make sure it has
+ * no effect */
+ kbdev->pm.backend.gpu_poweroff_pending = 0;
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ kbdev->pm.backend.shader_poweroff_pending = 0;
+ kbdev->pm.backend.shader_poweroff_pending_time = 0;
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+}
+
+void kbase_pm_update_active(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+ bool active;
+
+ lockdep_assert_held(&kbdev->pm.lock);
+
+ /* pm_current_policy will never be NULL while pm.lock is held */
+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy);
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ active = kbdev->pm.backend.pm_current_policy->get_core_active(kbdev);
+
+ if (active) {
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ if (kbdev->pm.backend.gpu_poweroff_pending) {
+ /* Cancel any pending power off request */
+ kbdev->pm.backend.gpu_poweroff_pending = 0;
+
+ /* If a request was pending then the GPU was still
+ * powered, so no need to continue */
+ if (!kbdev->poweroff_pending)
+ return;
+ }
+
+ if (!kbdev->pm.backend.poweroff_timer_needed &&
+ !kbdev->pm.backend.gpu_powered) {
+ kbdev->pm.backend.poweroff_timer_needed = true;
+ hrtimer_start(&kbdev->pm.backend.gpu_poweroff_timer,
+ kbdev->pm.gpu_poweroff_time,
+ HRTIMER_MODE_REL);
+ }
+
+ /* Power on the GPU and any cores requested by the policy */
+ kbase_pm_do_poweron(kbdev, false);
+ } else {
+ /* It is an error for the power policy to power off the GPU
+ * when there are contexts active */
+ KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
+
+ if (kbdev->pm.backend.shader_poweroff_pending) {
+ kbdev->pm.backend.shader_poweroff_pending = 0;
+ kbdev->pm.backend.shader_poweroff_pending_time = 0;
+ }
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+
+ /* Request power off */
+ if (kbdev->pm.backend.gpu_powered) {
+ kbdev->pm.backend.gpu_poweroff_pending =
+ kbdev->pm.poweroff_gpu_ticks;
+ if (!kbdev->pm.backend.poweroff_timer_needed) {
+ /* Start timer if not running (eg if power
+ * policy has been changed from always_on to
+ * something else). This will ensure the GPU is
+ * actually powered off */
+ kbdev->pm.backend.poweroff_timer_needed = true;
+ hrtimer_start(
+ &kbdev->pm.backend.gpu_poweroff_timer,
+ kbdev->pm.gpu_poweroff_time,
+ HRTIMER_MODE_REL);
+ }
+ }
+ }
+}
+
+void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
+{
+ u64 desired_bitmap;
+ bool cores_are_available;
+
+ lockdep_assert_held(&kbdev->pm.power_change_lock);
+
+ if (kbdev->pm.backend.pm_current_policy == NULL)
+ return;
+
+ desired_bitmap =
+ kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev);
+ desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);
+
+ /* Enable core 0 if tiler required, regardless of core availability */
+ if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
+ desired_bitmap |= 1;
+
+ if (kbdev->pm.backend.desired_shader_state != desired_bitmap)
+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u,
+ (u32)desired_bitmap);
+
+ /* Are any cores being powered on? */
+ if (~kbdev->pm.backend.desired_shader_state & desired_bitmap ||
+ kbdev->pm.backend.ca_in_transition) {
+ /* Check if we are powering off any cores before updating shader
+ * state */
+ if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap) {
+ /* Start timer to power off cores */
+ kbdev->pm.backend.shader_poweroff_pending |=
+ (kbdev->pm.backend.desired_shader_state &
+ ~desired_bitmap);
+ kbdev->pm.backend.shader_poweroff_pending_time =
+ kbdev->pm.poweroff_shader_ticks;
+ }
+
+ kbdev->pm.backend.desired_shader_state = desired_bitmap;
+
+ /* If any cores are being powered on, transition immediately */
+ cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
+ } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap) {
+ /* Start timer to power off cores */
+ kbdev->pm.backend.shader_poweroff_pending |=
+ (kbdev->pm.backend.desired_shader_state &
+ ~desired_bitmap);
+ kbdev->pm.backend.shader_poweroff_pending_time =
+ kbdev->pm.poweroff_shader_ticks;
+ } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 &&
+ kbdev->pm.backend.poweroff_timer_needed) {
+ /* If power policy is keeping cores on despite there being no
+ * active contexts then disable poweroff timer as it isn't
+ * required.
+ * Only reset poweroff_timer_needed if we're not in the middle
+ * of the power off callback */
+ kbdev->pm.backend.poweroff_timer_needed = false;
+ hrtimer_try_to_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
+ }
+
+ /* Ensure timer does not power off wanted cores and make sure to power
+ * off unwanted cores */
+ if (kbdev->pm.backend.shader_poweroff_pending != 0) {
+ kbdev->pm.backend.shader_poweroff_pending &=
+ ~(kbdev->pm.backend.desired_shader_state &
+ desired_bitmap);
+ if (kbdev->pm.backend.shader_poweroff_pending == 0)
+ kbdev->pm.backend.shader_poweroff_pending_time = 0;
+ }
+
+ /* Don't need 'cores_are_available', because we don't return anything */
+ CSTD_UNUSED(cores_are_available);
+}
+
+void kbase_pm_update_cores_state(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ kbase_pm_update_cores_state_nolock(kbdev);
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+}
+
+int kbase_pm_list_policies(const struct kbase_pm_policy * const **list)
+{
+ if (!list)
+ return POLICY_COUNT;
+
+ *list = policy_list;
+
+ return POLICY_COUNT;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_list_policies);
+
+const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev)
+{
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ return kbdev->pm.backend.pm_current_policy;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_get_policy);
+
+void kbase_pm_set_policy(struct kbase_device *kbdev,
+ const struct kbase_pm_policy *new_policy)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ const struct kbase_pm_policy *old_policy;
+ unsigned long flags;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(new_policy != NULL);
+
+ KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
+
+ /* During a policy change we pretend the GPU is active */
+ /* A suspend won't happen here, because we're in a syscall from a
+ * userspace thread */
+ kbase_pm_context_active(kbdev);
+
+ mutex_lock(&js_devdata->runpool_mutex);
+ mutex_lock(&kbdev->pm.lock);
+
+ /* Remove the policy to prevent IRQ handlers from working on it */
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ old_policy = kbdev->pm.backend.pm_current_policy;
+ kbdev->pm.backend.pm_current_policy = NULL;
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u,
+ old_policy->id);
+ if (old_policy->term)
+ old_policy->term(kbdev);
+
+ KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u,
+ new_policy->id);
+ if (new_policy->init)
+ new_policy->init(kbdev);
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+ kbdev->pm.backend.pm_current_policy = new_policy;
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ /* If any core power state changes were previously attempted, but
+ * couldn't be made because the policy was changing (current_policy was
+ * NULL), then re-try them here. */
+ kbase_pm_update_active(kbdev);
+ kbase_pm_update_cores_state(kbdev);
+
+ mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
+
+ /* Now the policy change is finished, we release our fake context active
+ * reference */
+ kbase_pm_context_idle(kbdev);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_set_policy);
+
+/** Check whether a state change has finished, and trace it as completed */
+static void
+kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev)
+{
+ if ((kbdev->shader_available_bitmap &
+ kbdev->pm.backend.desired_shader_state)
+ == kbdev->pm.backend.desired_shader_state &&
+ (kbdev->tiler_available_bitmap &
+ kbdev->pm.backend.desired_tiler_state)
+ == kbdev->pm.backend.desired_tiler_state)
+ kbase_timeline_pm_check_handle_event(kbdev,
+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
+}
+
+void kbase_pm_request_cores(struct kbase_device *kbdev,
+ bool tiler_required, u64 shader_cores)
+{
+ unsigned long flags;
+ u64 cores;
+
+ kbase_pm_change_state change_gpu_state = 0u;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ cores = shader_cores;
+ while (cores) {
+ int bitnum = fls64(cores) - 1;
+ u64 bit = 1ULL << bitnum;
+
+ /* It should be almost impossible for this to overflow. It would
+ * require 2^32 atoms to request a particular core, which would
+ * require 2^24 contexts to submit. This would require an amount
+ * of memory that is impossible on a 32-bit system and extremely
+ * unlikely on a 64-bit system. */
+ int cnt = ++kbdev->shader_needed_cnt[bitnum];
+
+ if (1 == cnt) {
+ kbdev->shader_needed_bitmap |= bit;
+ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
+ }
+
+ cores &= ~bit;
+ }
+
+ if (tiler_required) {
+ int cnt = ++kbdev->tiler_needed_cnt;
+
+ if (1 == cnt)
+ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
+
+ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
+ }
+
+ if (change_gpu_state) {
+ KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL,
+ NULL, 0u, (u32) kbdev->shader_needed_bitmap);
+
+ kbase_timeline_pm_cores_func(kbdev,
+ KBASE_PM_FUNC_ID_REQUEST_CORES_START,
+ change_gpu_state);
+ kbase_pm_update_cores_state_nolock(kbdev);
+ kbase_timeline_pm_cores_func(kbdev,
+ KBASE_PM_FUNC_ID_REQUEST_CORES_END,
+ change_gpu_state);
+ }
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_request_cores);
+
+void kbase_pm_unrequest_cores(struct kbase_device *kbdev,
+ bool tiler_required, u64 shader_cores)
+{
+ unsigned long flags;
+
+ kbase_pm_change_state change_gpu_state = 0u;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ while (shader_cores) {
+ int bitnum = fls64(shader_cores) - 1;
+ u64 bit = 1ULL << bitnum;
+ int cnt;
+
+ KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
+
+ cnt = --kbdev->shader_needed_cnt[bitnum];
+
+ if (0 == cnt) {
+ kbdev->shader_needed_bitmap &= ~bit;
+
+ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
+ }
+
+ shader_cores &= ~bit;
+ }
+
+ if (tiler_required) {
+ int cnt;
+
+ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
+
+ cnt = --kbdev->tiler_needed_cnt;
+
+ if (0 == cnt)
+ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
+ }
+
+ if (change_gpu_state) {
+ KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL,
+ NULL, 0u, (u32) kbdev->shader_needed_bitmap);
+
+ kbase_pm_update_cores_state_nolock(kbdev);
+
+ /* Trace that any state change effectively completes immediately
+ * - no-one will wait on the state change */
+ kbase_pm_trace_check_and_finish_state_change(kbdev);
+ }
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores);
+
+enum kbase_pm_cores_ready
+kbase_pm_register_inuse_cores(struct kbase_device *kbdev,
+ bool tiler_required, u64 shader_cores)
+{
+ unsigned long flags;
+ u64 prev_shader_needed; /* Just for tracing */
+ u64 prev_shader_inuse; /* Just for tracing */
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ prev_shader_needed = kbdev->shader_needed_bitmap;
+ prev_shader_inuse = kbdev->shader_inuse_bitmap;
+
+ /* If desired_shader_state does not contain the requested cores, then
+ * power management is not attempting to powering those cores (most
+ * likely due to core availability policy) and a new job affinity must
+ * be chosen */
+ if ((kbdev->pm.backend.desired_shader_state & shader_cores) !=
+ shader_cores) {
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ return KBASE_NEW_AFFINITY;
+ }
+
+ if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
+ (tiler_required && !kbdev->tiler_available_bitmap)) {
+ /* Trace ongoing core transition */
+ kbase_timeline_pm_l2_transition_start(kbdev);
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+ return KBASE_CORES_NOT_READY;
+ }
+
+ /* If we started to trace a state change, then trace it has being
+ * finished by now, at the very latest */
+ kbase_pm_trace_check_and_finish_state_change(kbdev);
+ /* Trace core transition done */
+ kbase_timeline_pm_l2_transition_done(kbdev);
+
+ while (shader_cores) {
+ int bitnum = fls64(shader_cores) - 1;
+ u64 bit = 1ULL << bitnum;
+ int cnt;
+
+ KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
+
+ cnt = --kbdev->shader_needed_cnt[bitnum];
+
+ if (0 == cnt)
+ kbdev->shader_needed_bitmap &= ~bit;
+
+ /* shader_inuse_cnt should not overflow because there can only
+ * be a very limited number of jobs on the h/w at one time */
+
+ kbdev->shader_inuse_cnt[bitnum]++;
+ kbdev->shader_inuse_bitmap |= bit;
+
+ shader_cores &= ~bit;
+ }
+
+ if (tiler_required) {
+ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
+
+ --kbdev->tiler_needed_cnt;
+
+ kbdev->tiler_inuse_cnt++;
+
+ KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
+ }
+
+ if (prev_shader_needed != kbdev->shader_needed_bitmap)
+ KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL,
+ NULL, 0u, (u32) kbdev->shader_needed_bitmap);
+
+ if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
+ KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL,
+ NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+
+ return KBASE_CORES_READY;
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores);
+
+void kbase_pm_release_cores(struct kbase_device *kbdev,
+ bool tiler_required, u64 shader_cores)
+{
+ unsigned long flags;
+ kbase_pm_change_state change_gpu_state = 0u;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ while (shader_cores) {
+ int bitnum = fls64(shader_cores) - 1;
+ u64 bit = 1ULL << bitnum;
+ int cnt;
+
+ KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);
+
+ cnt = --kbdev->shader_inuse_cnt[bitnum];
+
+ if (0 == cnt) {
+ kbdev->shader_inuse_bitmap &= ~bit;
+ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
+ }
+
+ shader_cores &= ~bit;
+ }
+
+ if (tiler_required) {
+ int cnt;
+
+ KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);
+
+ cnt = --kbdev->tiler_inuse_cnt;
+
+ if (0 == cnt)
+ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
+ }
+
+ if (change_gpu_state) {
+ KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL,
+ NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
+
+ kbase_timeline_pm_cores_func(kbdev,
+ KBASE_PM_FUNC_ID_RELEASE_CORES_START,
+ change_gpu_state);
+ kbase_pm_update_cores_state_nolock(kbdev);
+ kbase_timeline_pm_cores_func(kbdev,
+ KBASE_PM_FUNC_ID_RELEASE_CORES_END,
+ change_gpu_state);
+
+ /* Trace that any state change completed immediately */
+ kbase_pm_trace_check_and_finish_state_change(kbdev);
+ }
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_release_cores);
+
+void kbase_pm_request_cores_sync(struct kbase_device *kbdev,
+ bool tiler_required,
+ u64 shader_cores)
+{
+ kbase_pm_request_cores(kbdev, tiler_required, shader_cores);
+
+ kbase_pm_check_transitions_sync(kbdev);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync);
+
+void kbase_pm_request_l2_caches(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+ u32 prior_l2_users_count;
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ prior_l2_users_count = kbdev->l2_users_count++;
+
+ KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0);
+
+ /* if the GPU is reset while the l2 is on, l2 will be off but
+ * prior_l2_users_count will be > 0. l2_available_bitmap will have been
+ * set to 0 though by kbase_pm_init_hw */
+ if (!prior_l2_users_count || !kbdev->l2_available_bitmap)
+ kbase_pm_check_transitions_nolock(kbdev);
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+ wait_event(kbdev->pm.backend.l2_powered_wait,
+ kbdev->pm.backend.l2_powered == 1);
+
+ /* Trace that any state change completed immediately */
+ kbase_pm_trace_check_and_finish_state_change(kbdev);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches);
+
+void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ kbdev->l2_users_count++;
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on);
+
+void kbase_pm_release_l2_caches(struct kbase_device *kbdev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
+
+ KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0);
+
+ --kbdev->l2_users_count;
+
+ if (!kbdev->l2_users_count) {
+ kbase_pm_check_transitions_nolock(kbdev);
+ /* Trace that any state change completed immediately */
+ kbase_pm_trace_check_and_finish_state_change(kbdev);
+ }
+
+ spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
+}
+
+KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches);
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * Power policy API definitions
+ */
+
+#ifndef _KBASE_PM_POLICY_H_
+#define _KBASE_PM_POLICY_H_
+
+/**
+ * kbase_pm_policy_init - Initialize power policy framework
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Must be called before calling any other policy function
+ *
+ * Return: 0 if the power policy framework was successfully
+ * initialized, -errno otherwise.
+ */
+int kbase_pm_policy_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_policy_term - Terminate power policy framework
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ */
+void kbase_pm_policy_term(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_update_active - Update the active power state of the GPU
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Calls into the current power policy
+ */
+void kbase_pm_update_active(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_update_cores - Update the desired core state of the GPU
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Calls into the current power policy
+ */
+void kbase_pm_update_cores(struct kbase_device *kbdev);
+
+
+enum kbase_pm_cores_ready {
+ KBASE_CORES_NOT_READY = 0,
+ KBASE_NEW_AFFINITY = 1,
+ KBASE_CORES_READY = 2
+};
+
+
+/**
+ * kbase_pm_request_cores_sync - Synchronous variant of kbase_pm_request_cores()
+ *
+ * @kbdev: The kbase device structure for the device
+ * @tiler_required: true if the tiler is required, false otherwise
+ * @shader_cores: A bitmask of shader cores which are necessary for the job
+ *
+ * When this function returns, the @shader_cores will be in the READY state.
+ *
+ * This is safe variant of kbase_pm_check_transitions_sync(): it handles the
+ * work of ensuring the requested cores will remain powered until a matching
+ * call to kbase_pm_unrequest_cores()/kbase_pm_release_cores() (as appropriate)
+ * is made.
+ */
+void kbase_pm_request_cores_sync(struct kbase_device *kbdev,
+ bool tiler_required, u64 shader_cores);
+
+/**
+ * kbase_pm_request_cores - Mark one or more cores as being required
+ * for jobs to be submitted
+ *
+ * @kbdev: The kbase device structure for the device
+ * @tiler_required: true if the tiler is required, false otherwise
+ * @shader_cores: A bitmask of shader cores which are necessary for the job
+ *
+ * This function is called by the job scheduler to mark one or more cores as
+ * being required to submit jobs that are ready to run.
+ *
+ * The cores requested are reference counted and a subsequent call to
+ * kbase_pm_register_inuse_cores() or kbase_pm_unrequest_cores() should be
+ * made to dereference the cores as being 'needed'.
+ *
+ * The active power policy will meet or exceed the requirements of the
+ * requested cores in the system. Any core transitions needed will be begun
+ * immediately, but they might not complete/the cores might not be available
+ * until a Power Management IRQ.
+ *
+ * Return: 0 if the cores were successfully requested, or -errno otherwise.
+ */
+void kbase_pm_request_cores(struct kbase_device *kbdev,
+ bool tiler_required, u64 shader_cores);
+
+/**
+ * kbase_pm_unrequest_cores - Unmark one or more cores as being required for
+ * jobs to be submitted.
+ *
+ * @kbdev: The kbase device structure for the device
+ * @tiler_required: true if the tiler is required, false otherwise
+ * @shader_cores: A bitmask of shader cores (as given to
+ * kbase_pm_request_cores() )
+ *
+ * This function undoes the effect of kbase_pm_request_cores(). It should be
+ * used when a job is not going to be submitted to the hardware (e.g. the job is
+ * cancelled before it is enqueued).
+ *
+ * The active power policy will meet or exceed the requirements of the
+ * requested cores in the system. Any core transitions needed will be begun
+ * immediately, but they might not complete until a Power Management IRQ.
+ *
+ * The policy may use this as an indication that it can power down cores.
+ */
+void kbase_pm_unrequest_cores(struct kbase_device *kbdev,
+ bool tiler_required, u64 shader_cores);
+
+/**
+ * kbase_pm_register_inuse_cores - Register a set of cores as in use by a job
+ *
+ * @kbdev: The kbase device structure for the device
+ * @tiler_required: true if the tiler is required, false otherwise
+ * @shader_cores: A bitmask of shader cores (as given to
+ * kbase_pm_request_cores() )
+ *
+ * This function should be called after kbase_pm_request_cores() when the job
+ * is about to be submitted to the hardware. It will check that the necessary
+ * cores are available and if so update the 'needed' and 'inuse' bitmasks to
+ * reflect that the job is now committed to being run.
+ *
+ * If the necessary cores are not currently available then the function will
+ * return %KBASE_CORES_NOT_READY and have no effect.
+ *
+ * Return: true if the job can be submitted to the hardware or false
+ * if the job is not ready to run.
+ * Return: %KBASE_CORES_NOT_READY if the cores are not immediately ready,
+ * %KBASE_NEW_AFFINITY if the affinity requested is not allowed,
+ * %KBASE_CORES_READY if the cores requested are already available
+ */
+enum kbase_pm_cores_ready kbase_pm_register_inuse_cores(
+ struct kbase_device *kbdev,
+ bool tiler_required,
+ u64 shader_cores);
+
+/**
+ * kbase_pm_release_cores - Release cores after a job has run
+ *
+ * @kbdev: The kbase device structure for the device
+ * @tiler_required: true if the tiler is required, false otherwise
+ * @shader_cores: A bitmask of shader cores (as given to
+ * kbase_pm_register_inuse_cores() )
+ *
+ * This function should be called when a job has finished running on the
+ * hardware. A call to kbase_pm_register_inuse_cores() must have previously
+ * occurred. The reference counts of the specified cores will be decremented
+ * which may cause the bitmask of 'inuse' cores to be reduced. The power policy
+ * may then turn off any cores which are no longer 'inuse'.
+ */
+void kbase_pm_release_cores(struct kbase_device *kbdev,
+ bool tiler_required, u64 shader_cores);
+
+/**
+ * kbase_pm_request_l2_caches - Request l2 caches
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Request the use of l2 caches for all core groups, power up, wait and prevent
+ * the power manager from powering down the l2 caches.
+ *
+ * This tells the power management that the caches should be powered up, and
+ * they should remain powered, irrespective of the usage of shader cores. This
+ * does not return until the l2 caches are powered up.
+ *
+ * The caller must call kbase_pm_release_l2_caches() when they are finished
+ * to allow normal power management of the l2 caches to resume.
+ *
+ * This should only be used when power management is active.
+ */
+void kbase_pm_request_l2_caches(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_request_l2_caches_l2_is_on - Request l2 caches but don't power on
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Increment the count of l2 users but do not attempt to power on the l2
+ *
+ * It is the callers responsibility to ensure that the l2 is already powered up
+ * and to eventually call kbase_pm_release_l2_caches()
+ */
+void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev);
+
+/**
+ * kbase_pm_request_l2_caches - Release l2 caches
+ *
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ *
+ * Release the use of l2 caches for all core groups and allow the power manager
+ * to power them down when necessary.
+ *
+ * This tells the power management that the caches can be powered down if
+ * necessary, with respect to the usage of shader cores.
+ *
+ * The caller must have called kbase_pm_request_l2_caches() prior to a call
+ * to this.
+ *
+ * This should only be used when power management is active.
+ */
+void kbase_pm_release_l2_caches(struct kbase_device *kbdev);
+
+#endif /* _KBASE_PM_POLICY_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/**
+ *
+ */
+
+#include <mali_kbase.h>
+#include <mali_kbase_hwaccess_time.h>
+#include <backend/gpu/mali_kbase_device_internal.h>
+#include <backend/gpu/mali_kbase_pm_internal.h>
+
+void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
+ u64 *system_time, struct timespec *ts)
+{
+ u32 hi1, hi2;
+
+ kbase_pm_request_gpu_cycle_counter(kbdev);
+
+ /* Read hi, lo, hi to ensure that overflow from lo to hi is handled
+ * correctly */
+ do {
+ hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI),
+ NULL);
+ *cycle_counter = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
+ hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI),
+ NULL);
+ *cycle_counter |= (((u64) hi1) << 32);
+ } while (hi1 != hi2);
+
+ /* Read hi, lo, hi to ensure that overflow from lo to hi is handled
+ * correctly */
+ do {
+ hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI),
+ NULL);
+ *system_time = kbase_reg_read(kbdev,
+ GPU_CONTROL_REG(TIMESTAMP_LO), NULL);
+ hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI),
+ NULL);
+ *system_time |= (((u64) hi1) << 32);
+ } while (hi1 != hi2);
+
+ /* Record the CPU's idea of current time */
+ getrawmonotonic(ts);
+
+ kbase_pm_release_gpu_cycle_counter(kbdev);
+}
+
+/**
+ * kbase_wait_write_flush - Wait for GPU write flush
+ * @kctx: Context pointer
+ *
+ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush
+ * its write buffer.
+ *
+ * Only in use for BASE_HW_ISSUE_6367
+ *
+ * Note : If GPU resets occur then the counters are reset to zero, the delay may
+ * not be as expected.
+ */
+#ifndef CONFIG_MALI_NO_MALI
+void kbase_wait_write_flush(struct kbase_context *kctx)
+{
+ u32 base_count = 0;
+
+ /* A suspend won't happen here, because we're in a syscall from a
+ * userspace thread */
+
+ kbase_pm_context_active(kctx->kbdev);
+ kbase_pm_request_gpu_cycle_counter(kctx->kbdev);
+
+ while (true) {
+ u32 new_count;
+
+ new_count = kbase_reg_read(kctx->kbdev,
+ GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
+ /* First time around, just store the count. */
+ if (base_count == 0) {
+ base_count = new_count;
+ continue;
+ }
+
+ /* No need to handle wrapping, unsigned maths works for this. */
+ if ((new_count - base_count) > 1000)
+ break;
+ }
+
+ kbase_pm_release_gpu_cycle_counter(kctx->kbdev);
+ kbase_pm_context_idle(kctx->kbdev);
+}
+#endif /* CONFIG_MALI_NO_MALI */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/**
+ *
+ */
+
+#ifndef _KBASE_BACKEND_TIME_H_
+#define _KBASE_BACKEND_TIME_H_
+
+/**
+ * kbase_backend_get_gpu_time() - Get current GPU time
+ * @kbdev: Device pointer
+ * @cycle_counter: Pointer to u64 to store cycle counter in
+ * @system_time: Pointer to u64 to store system time in
+ * @ts: Pointer to struct timespec to store current monotonic
+ * time in
+ */
+void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
+ u64 *system_time, struct timespec *ts);
+
+/**
+ * kbase_wait_write_flush() - Wait for GPU write flush
+ * @kctx: Context pointer
+ *
+ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush
+ * its write buffer.
+ *
+ * If GPU resets occur then the counters are reset to zero, the delay may not be
+ * as expected.
+ *
+ * This function is only in use for BASE_HW_ISSUE_6367
+ */
+#ifdef CONFIG_MALI_NO_MALI
+static inline void kbase_wait_write_flush(struct kbase_context *kctx)
+{
+}
+#else
+void kbase_wait_write_flush(struct kbase_context *kctx);
+#endif
+
+#endif /* _KBASE_BACKEND_TIME_H_ */
#
-# (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2011-2013, 2015 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# The EXCLUDE tag can be used to specify files and/or directories that should
# excluded from the INPUT source files. This way you can easily exclude a
# subdirectory from a directory tree whose root is specified with the INPUT tag.
-EXCLUDE += ../../kernel/drivers/gpu/arm/midgard/malisw/ ../../kernel/drivers/gpu/arm/midgard/platform ../../kernel/drivers/gpu/arm/midgard/platform_dummy ../../kernel/drivers/gpu/arm/midgard/scripts ../../kernel/drivers/gpu/arm/midgard/tests ../../kernel/drivers/gpu/arm/midgard/Makefile ../../kernel/drivers/gpu/arm/midgard/Makefile.kbase ../../kernel/drivers/gpu/arm/midgard/Kbuild ../../kernel/drivers/gpu/arm/midgard/Kconfig ../../kernel/drivers/gpu/arm/midgard/sconscript ../../kernel/drivers/gpu/arm/midgard/docs ../../kernel/drivers/gpu/arm/midgard/pm_test_script.sh ../../kernel/drivers/gpu/arm/midgard/mali_uk.h ../../kernel/drivers/gpu/arm/midgard/Makefile
+EXCLUDE += ../../kernel/drivers/gpu/arm/midgard/platform ../../kernel/drivers/gpu/arm/midgard/platform_dummy ../../kernel/drivers/gpu/arm/midgard/scripts ../../kernel/drivers/gpu/arm/midgard/tests ../../kernel/drivers/gpu/arm/midgard/Makefile ../../kernel/drivers/gpu/arm/midgard/Makefile.kbase ../../kernel/drivers/gpu/arm/midgard/Kbuild ../../kernel/drivers/gpu/arm/midgard/Kconfig ../../kernel/drivers/gpu/arm/midgard/sconscript ../../kernel/drivers/gpu/arm/midgard/docs ../../kernel/drivers/gpu/arm/midgard/pm_test_script.sh ../../kernel/drivers/gpu/arm/midgard/mali_uk.h ../../kernel/drivers/gpu/arm/midgard/Makefile
# If the value of the INPUT tag contains directories, you can use the
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * @file
- * Software workarounds configuration for Hardware issues.
- */
-
-#ifndef _BASE_HWCONFIG_H_
-#define _BASE_HWCONFIG_H_
-
-#include <malisw/mali_malisw.h>
-
-#include "mali_base_hwconfig_issues.h"
-#include "mali_base_hwconfig_features.h"
-
-#endif /* _BASE_HWCONFIG_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features,
- * please update user/midgard/mali_base_hwconfig_{issues,features}.h and
- * re-run hwconfig_header_generator instead. This tool is available in
- * progs_install directory for host builds. More information is available in
- * base/tools/hwconfig_header_generator/README */
+ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py
+ * For more information see base/tools/hwconfig_generator/README
+ */
#ifndef _BASE_HWCONFIG_FEATURES_H_
#define _BASE_HWCONFIG_FEATURES_H_
BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE,
BASE_HW_FEATURE_BRNDOUT_KILL,
BASE_HW_FEATURE_WARPING,
+ BASE_HW_FEATURE_FLUSH_REDUCTION,
+ BASE_HW_FEATURE_V4,
BASE_HW_FEATURE_END
};
static const enum base_hw_feature base_hw_features_t60x[] = {
BASE_HW_FEATURE_LD_ST_LEA_TEX,
BASE_HW_FEATURE_LINEAR_FILTER_FLOAT,
+ BASE_HW_FEATURE_V4,
BASE_HW_FEATURE_END
};
BASE_HW_FEATURE_LD_ST_LEA_TEX,
BASE_HW_FEATURE_LINEAR_FILTER_FLOAT,
BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
+ BASE_HW_FEATURE_V4,
BASE_HW_FEATURE_END
};
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4,
BASE_HW_FEATURE_WARPING,
+ BASE_HW_FEATURE_V4,
BASE_HW_FEATURE_END
};
BASE_HW_FEATURE_END
};
+static const enum base_hw_feature base_hw_features_t83x[] = {
+ BASE_HW_FEATURE_33BIT_VA,
+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
+ BASE_HW_FEATURE_WARPING,
+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING,
+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS,
+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
+ BASE_HW_FEATURE_BRNDOUT_CC,
+ BASE_HW_FEATURE_BRNDOUT_KILL,
+ BASE_HW_FEATURE_LD_ST_LEA_TEX,
+ BASE_HW_FEATURE_LD_ST_TILEBUFFER,
+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT,
+ BASE_HW_FEATURE_MRT,
+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE,
+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC,
+ BASE_HW_FEATURE_T7XX_PAIRING_RULES,
+ BASE_HW_FEATURE_TEST4_DATUM_MODE,
+ BASE_HW_FEATURE_END
+};
+
+static const enum base_hw_feature base_hw_features_t82x[] = {
+ BASE_HW_FEATURE_33BIT_VA,
+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
+ BASE_HW_FEATURE_WARPING,
+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING,
+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS,
+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
+ BASE_HW_FEATURE_BRNDOUT_CC,
+ BASE_HW_FEATURE_BRNDOUT_KILL,
+ BASE_HW_FEATURE_LD_ST_LEA_TEX,
+ BASE_HW_FEATURE_LD_ST_TILEBUFFER,
+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT,
+ BASE_HW_FEATURE_MRT,
+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE,
+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC,
+ BASE_HW_FEATURE_T7XX_PAIRING_RULES,
+ BASE_HW_FEATURE_TEST4_DATUM_MODE,
+ BASE_HW_FEATURE_END
+};
+
#endif /* _BASE_HWCONFIG_FEATURES_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features,
- * please update user/midgard/mali_base_hwconfig_{issues,features}.h and
- * re-run hwconfig_header_generator instead. This tool is available in
- * progs_install directory for host builds. More information is available in
- * base/tools/hwconfig_header_generator/README */
+ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py
+ * For more information see base/tools/hwconfig_generator/README
+ */
#ifndef _BASE_HWCONFIG_ISSUES_H_
#define _BASE_HWCONFIG_ISSUES_H_
BASE_HW_ISSUE_10487,
BASE_HW_ISSUE_10607,
BASE_HW_ISSUE_10632,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10676,
BASE_HW_ISSUE_10682,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_11035,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_T76X_26,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
- BASE_HW_ISSUE_T76X_2121,
- BASE_HW_ISSUE_T76X_2315,
- BASE_HW_ISSUE_T76X_2686,
- BASE_HW_ISSUE_T76X_2712,
- BASE_HW_ISSUE_T76X_2772,
- BASE_HW_ISSUE_T76X_2906,
BASE_HW_ISSUE_T76X_3086,
- BASE_HW_ISSUE_T76X_3285,
BASE_HW_ISSUE_T76X_3542,
BASE_HW_ISSUE_T76X_3556,
BASE_HW_ISSUE_T76X_3700,
- BASE_HW_ISSUE_T76X_3759,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10487,
BASE_HW_ISSUE_10607,
BASE_HW_ISSUE_10632,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10676,
BASE_HW_ISSUE_10682,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_11012,
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11035,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10487,
BASE_HW_ISSUE_10607,
BASE_HW_ISSUE_10632,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10676,
BASE_HW_ISSUE_10682,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_11012,
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11035,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10487,
BASE_HW_ISSUE_10607,
BASE_HW_ISSUE_10632,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10676,
BASE_HW_ISSUE_10682,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_11012,
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11035,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_10487,
BASE_HW_ISSUE_10607,
BASE_HW_ISSUE_10632,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10676,
BASE_HW_ISSUE_10682,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11035,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_10471,
BASE_HW_ISSUE_10472,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_10471,
BASE_HW_ISSUE_10472,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10959,
BASE_HW_ISSUE_11012,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
-static const enum base_hw_issue base_hw_issues_t76x_r0p0_beta[] = {
- BASE_HW_ISSUE_8803,
- BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
- BASE_HW_ISSUE_10821,
- BASE_HW_ISSUE_10883,
- BASE_HW_ISSUE_10946,
- BASE_HW_ISSUE_10959,
- BASE_HW_ISSUE_11020,
- BASE_HW_ISSUE_11024,
- BASE_HW_ISSUE_11042,
- BASE_HW_ISSUE_T76X_26,
- BASE_HW_ISSUE_T76X_1963,
- BASE_HW_ISSUE_T76X_2121,
- BASE_HW_ISSUE_T76X_2315,
- BASE_HW_ISSUE_T76X_2686,
- BASE_HW_ISSUE_T76X_2712,
- BASE_HW_ISSUE_T76X_2772,
- BASE_HW_ISSUE_T76X_2906,
- BASE_HW_ISSUE_T76X_3285,
- BASE_HW_ISSUE_T76X_3700,
- BASE_HW_ISSUE_T76X_3759,
- BASE_HW_ISSUE_T76X_3793,
- BASE_HW_ISSUE_T76X_3953,
- BASE_HW_ISSUE_T76X_3960,
- BASE_HW_ISSUE_END
-};
-
static const enum base_hw_issue base_hw_issues_t76x_r0p0[] = {
BASE_HW_ISSUE_8803,
- BASE_HW_ISSUE_8778,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_T76X_26,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3542,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_t76x_r0p1[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_T76X_26,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3542,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_t76x_r0p1_50rel0[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_T76X_26,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3542,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_t76x_r0p2[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_T76X_26,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3542,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_t76x_r0p3[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
BASE_HW_ISSUE_T76X_26,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3542,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_t76x_r1p0[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_10471,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_10797,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
- BASE_HW_ISSUE_10931,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_10471,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_10797,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
- BASE_HW_ISSUE_10931,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_10471,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10684,
BASE_HW_ISSUE_10797,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
- BASE_HW_ISSUE_10931,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_10471,
BASE_HW_ISSUE_10797,
- BASE_HW_ISSUE_10931,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_11012,
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11024,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
BASE_HW_ISSUE_11020,
BASE_HW_ISSUE_11024,
BASE_HW_ISSUE_11042,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_END
};
-#if defined(MALI_INCLUDE_TFRX)
-static const enum base_hw_issue base_hw_issues_tFRx_r0p0[] = {
+static const enum base_hw_issue base_hw_issues_tFRx_r0p1[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_END
+};
+
+static const enum base_hw_issue base_hw_issues_tFRx_r0p2[] = {
+ BASE_HW_ISSUE_8803,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_10821,
+ BASE_HW_ISSUE_10883,
+ BASE_HW_ISSUE_10946,
+ BASE_HW_ISSUE_T76X_1909,
+ BASE_HW_ISSUE_T76X_1963,
+ BASE_HW_ISSUE_T76X_3086,
+ BASE_HW_ISSUE_T76X_3700,
+ BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_T76X_3953,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
-#endif /* defined(MALI_INCLUDE_TFRX) */
+static const enum base_hw_issue base_hw_issues_tFRx_r1p0[] = {
+ BASE_HW_ISSUE_8803,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_10821,
+ BASE_HW_ISSUE_10883,
+ BASE_HW_ISSUE_10946,
+ BASE_HW_ISSUE_T76X_1963,
+ BASE_HW_ISSUE_T76X_3086,
+ BASE_HW_ISSUE_T76X_3700,
+ BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_T76X_3953,
+ BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_END
+};
-#if defined(MALI_INCLUDE_TFRX)
-static const enum base_hw_issue base_hw_issues_tFRx_r0p1[] = {
+static const enum base_hw_issue base_hw_issues_tFRx_r2p0[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
- BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
-#endif /* defined(MALI_INCLUDE_TFRX) */
+static const enum base_hw_issue base_hw_issues_model_tFRx[] = {
+ BASE_HW_ISSUE_5736,
+ BASE_HW_ISSUE_9275,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_T76X_1963,
+ BASE_HW_ISSUE_T76X_3086,
+ BASE_HW_ISSUE_T76X_3700,
+ BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_END
+};
-#if defined(MALI_INCLUDE_TFRX)
-static const enum base_hw_issue base_hw_issues_tFRx_r0p2[] = {
+static const enum base_hw_issue base_hw_issues_t86x_r0p2[] = {
+ BASE_HW_ISSUE_8803,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_10821,
+ BASE_HW_ISSUE_10883,
+ BASE_HW_ISSUE_10946,
+ BASE_HW_ISSUE_T76X_1909,
+ BASE_HW_ISSUE_T76X_1963,
+ BASE_HW_ISSUE_T76X_3086,
+ BASE_HW_ISSUE_T76X_3700,
+ BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_T76X_3953,
+ BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_END
+};
+
+static const enum base_hw_issue base_hw_issues_t86x_r1p0[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
+ BASE_HW_ISSUE_T76X_3966,
BASE_HW_ISSUE_END
};
-#endif /* defined(MALI_INCLUDE_TFRX) */
+static const enum base_hw_issue base_hw_issues_t86x_r2p0[] = {
+ BASE_HW_ISSUE_8803,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_10821,
+ BASE_HW_ISSUE_10883,
+ BASE_HW_ISSUE_10946,
+ BASE_HW_ISSUE_T76X_1963,
+ BASE_HW_ISSUE_T76X_3086,
+ BASE_HW_ISSUE_T76X_3700,
+ BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_T76X_3953,
+ BASE_HW_ISSUE_T76X_3966,
+ BASE_HW_ISSUE_END
+};
-#if defined(MALI_INCLUDE_TFRX)
-static const enum base_hw_issue base_hw_issues_model_tFRx[] = {
+static const enum base_hw_issue base_hw_issues_model_t86x[] = {
BASE_HW_ISSUE_5736,
BASE_HW_ISSUE_9275,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10931,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_END
};
-#endif /* defined(MALI_INCLUDE_TFRX) */
+static const enum base_hw_issue base_hw_issues_t83x_r0p1[] = {
+ BASE_HW_ISSUE_8803,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_10821,
+ BASE_HW_ISSUE_10883,
+ BASE_HW_ISSUE_10946,
+ BASE_HW_ISSUE_T76X_1909,
+ BASE_HW_ISSUE_T76X_1963,
+ BASE_HW_ISSUE_T76X_3086,
+ BASE_HW_ISSUE_T76X_3700,
+ BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_T76X_3953,
+ BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_END
+};
-static const enum base_hw_issue base_hw_issues_t86x_r0p0[] = {
+static const enum base_hw_issue base_hw_issues_t83x_r1p0[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
BASE_HW_ISSUE_END
};
-static const enum base_hw_issue base_hw_issues_t86x_r0p2[] = {
+static const enum base_hw_issue base_hw_issues_model_t83x[] = {
+ BASE_HW_ISSUE_5736,
+ BASE_HW_ISSUE_9275,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_T76X_1909,
+ BASE_HW_ISSUE_T76X_1963,
+ BASE_HW_ISSUE_T76X_3086,
+ BASE_HW_ISSUE_T76X_3700,
+ BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_END
+};
+
+static const enum base_hw_issue base_hw_issues_t82x_r0p0[] = {
BASE_HW_ISSUE_8803,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10649,
BASE_HW_ISSUE_10821,
BASE_HW_ISSUE_10883,
BASE_HW_ISSUE_10946,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
BASE_HW_ISSUE_T76X_3793,
BASE_HW_ISSUE_T76X_3953,
+ BASE_HW_ISSUE_T76X_3960,
BASE_HW_ISSUE_END
};
-static const enum base_hw_issue base_hw_issues_model_t86x[] = {
+static const enum base_hw_issue base_hw_issues_t82x_r0p1[] = {
+ BASE_HW_ISSUE_8803,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_10821,
+ BASE_HW_ISSUE_10883,
+ BASE_HW_ISSUE_10946,
+ BASE_HW_ISSUE_T76X_1909,
+ BASE_HW_ISSUE_T76X_1963,
+ BASE_HW_ISSUE_T76X_3086,
+ BASE_HW_ISSUE_T76X_3700,
+ BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_T76X_3953,
+ BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_END
+};
+
+static const enum base_hw_issue base_hw_issues_t82x_r1p0[] = {
+ BASE_HW_ISSUE_8803,
+ BASE_HW_ISSUE_9435,
+ BASE_HW_ISSUE_10821,
+ BASE_HW_ISSUE_10883,
+ BASE_HW_ISSUE_10946,
+ BASE_HW_ISSUE_T76X_1963,
+ BASE_HW_ISSUE_T76X_3086,
+ BASE_HW_ISSUE_T76X_3700,
+ BASE_HW_ISSUE_T76X_3793,
+ BASE_HW_ISSUE_T76X_3953,
+ BASE_HW_ISSUE_T76X_3960,
+ BASE_HW_ISSUE_END
+};
+
+static const enum base_hw_issue base_hw_issues_model_t82x[] = {
BASE_HW_ISSUE_5736,
BASE_HW_ISSUE_9275,
BASE_HW_ISSUE_9435,
- BASE_HW_ISSUE_10931,
+ BASE_HW_ISSUE_T76X_1909,
BASE_HW_ISSUE_T76X_1963,
BASE_HW_ISSUE_T76X_3086,
BASE_HW_ISSUE_T76X_3700,
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/* NB: To support UK6 we also need to support UK7 */
#define BASE_LEGACY_UK7_SUPPORT 1
-typedef mali_addr64 base_mem_handle;
+typedef u64 base_mem_handle;
#include "mali_base_mem_priv.h"
#include "mali_kbase_profiling_gator_api.h"
* heavily read by the CPU...
* Other flags are only meaningful to a particular allocator.
* More flags can be added to this list, as long as they don't clash
- * (see ::BASE_MEM_FLAGS_NR_TOTAL_BITS for the number of the first free bit).
+ * (see ::BASE_MEM_FLAGS_NR_BITS for the number of the first free bit).
*/
typedef u32 base_mem_alloc_flags;
BASE_MEM_PROT_GPU_EX = (1U << 4), /**< Execute allowed on the GPU
side */
- /* Note that the HINT flags are obsolete now. If you want the memory
- * to be cached on the CPU please use the BASE_MEM_CACHED_CPU flag
+ /* BASE_MEM_HINT flags have been removed, but their values are reserved
+ * for backwards compatibility with older user-space drivers. The values
+ * can be re-used once support for r5p0 user-space drivers is removed,
+ * presumably in r7p0.
+ *
+ * RESERVED: (1U << 5)
+ * RESERVED: (1U << 6)
+ * RESERVED: (1U << 7)
+ * RESERVED: (1U << 8)
*/
- BASE_MEM_HINT_CPU_RD = (1U << 5), /**< Heavily read CPU side
- - OBSOLETE */
- BASE_MEM_HINT_CPU_WR = (1U << 6), /**< Heavily written CPU side
- - OBSOLETE */
- BASE_MEM_HINT_GPU_RD = (1U << 7), /**< Heavily read GPU side
- - OBSOLETE */
- BASE_MEM_HINT_GPU_WR = (1U << 8), /**< Heavily written GPU side
- - OBSOLETE */
BASE_MEM_GROW_ON_GPF = (1U << 9), /**< Grow backing store on GPU
Page Fault */
BASE_MEM_COHERENT_SYSTEM = (1U << 10), /**< Page coherence Outer
- shareable */
+ shareable, if available */
BASE_MEM_COHERENT_LOCAL = (1U << 11), /**< Page coherence Inner
shareable */
BASE_MEM_CACHED_CPU = (1U << 12), /**< Should be cached on the
BASE_MEM_SAME_VA = (1U << 13), /**< Must have same VA on both the GPU
and the CPU */
/* OUT */
- BASE_MEM_NEED_MMAP = (1U << 14) /**< Must call mmap to aquire a GPU
+ BASE_MEM_NEED_MMAP = (1U << 14), /**< Must call mmap to aquire a GPU
address for the alloc */
+
+/* IN */
+ BASE_MEM_COHERENT_SYSTEM_REQUIRED = (1U << 15), /**< Page coherence
+ Outer shareable, required. */
+ BASE_MEM_SECURE = (1U << 16) /**< Secure memory */
+
};
/**
*
* Must be kept in sync with the ::base_mem_alloc_flags flags
*/
-#define BASE_MEM_FLAGS_NR_INPUT_BITS 14
-#define BASE_MEM_FLAGS_NR_OUTPUT_BITS 1
-#define BASE_MEM_FLAGS_NR_TOTAL_BITS ((BASE_MEM_FLAGS_NR_INPUT_BITS) + (BASE_MEM_FLAGS_NR_OUTPUT_BITS))
-#define BASE_MEM_FLAGS_NR_BITS 15
+#define BASE_MEM_FLAGS_NR_BITS 17
+
+/**
+ * A mask for all output bits, excluding IN/OUT bits.
+ */
+#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP
+
+/**
+ * A mask for all input bits, including IN/OUT bits.
+ */
+#define BASE_MEM_FLAGS_INPUT_MASK \
+ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK)
-#if BASE_MEM_FLAGS_NR_TOTAL_BITS > BASE_MEM_FLAGS_NR_BITS
-#error "Too many flag bits, will require change in cmem"
-#endif
/**
* @brief Memory types supported by @a base_mem_import
*/
typedef struct base_import_handle {
struct {
- mali_addr64 handle;
+ u64 handle;
} basep;
} base_import_handle;
/**
* @brief Job dependency type.
*
- * A flags field will be inserted into the atom structure to specify whether a dependency is a data or
- * ordering dependency (by putting it before/after 'core_req' in the structure it should be possible to add without
+ * A flags field will be inserted into the atom structure to specify whether a dependency is a data or
+ * ordering dependency (by putting it before/after 'core_req' in the structure it should be possible to add without
* changing the structure size).
- * When the flag is set for a particular dependency to signal that it is an ordering only dependency then
+ * When the flag is set for a particular dependency to signal that it is an ordering only dependency then
* errors will not be propagated.
*/
typedef u8 base_jd_dep_type;
-#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */
-#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */
-#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */
+#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */
+#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */
+#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */
/**
* @brief Job chain hardware requirements.
*
* This is only guaranteed to work for BASE_JD_REQ_ONLY_COMPUTE atoms.
*
- * If the core availability policy is keeping the required core group turned off, then
+ * If the core availability policy is keeping the required core group turned off, then
* the job will fail with a BASE_JD_EVENT_PM_EVENT error code.
*/
#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP (1U << 11)
KBASE_ATOM_COREREF_STATE_READY
};
+/*
+ * Base Atom priority
+ *
+ * Only certain priority levels are actually implemented, as specified by the
+ * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority
+ * level that is not one of those defined below.
+ *
+ * Priority levels only affect scheduling between atoms of the same type within
+ * a base context, and only after the atoms have had dependencies resolved.
+ * Fragment atoms does not affect non-frament atoms with lower priorities, and
+ * the other way around. For example, a low priority atom that has had its
+ * dependencies resolved might run before a higher priority atom that has not
+ * had its dependencies resolved.
+ *
+ * The scheduling between base contexts/processes and between atoms from
+ * different base contexts/processes is unaffected by atom priority.
+ *
+ * The atoms are scheduled as follows with respect to their priorities:
+ * - Let atoms 'X' and 'Y' be for the same job slot who have dependencies
+ * resolved, and atom 'X' has a higher priority than atom 'Y'
+ * - If atom 'Y' is currently running on the HW, then it is interrupted to
+ * allow atom 'X' to run soon after
+ * - If instead neither atom 'Y' nor atom 'X' are running, then when choosing
+ * the next atom to run, atom 'X' will always be chosen instead of atom 'Y'
+ * - Any two atoms that have the same priority could run in any order with
+ * respect to each other. That is, there is no ordering constraint between
+ * atoms of the same priority.
+ */
+typedef u8 base_jd_prio;
+
+/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */
+#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0)
+/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and
+ * BASE_JD_PRIO_LOW */
+#define BASE_JD_PRIO_HIGH ((base_jd_prio)1)
+/* Low atom priority. */
+#define BASE_JD_PRIO_LOW ((base_jd_prio)2)
+
+/* Count of the number of priority levels. This itself is not a valid
+ * base_jd_prio setting */
+#define BASE_JD_NR_PRIO_LEVELS 3
+
enum kbase_jd_atom_state {
/** Atom is not used */
KBASE_JD_ATOM_STATE_UNUSED,
KBASE_JD_ATOM_STATE_QUEUED,
/** Atom has been given to JS (is runnable/running) */
KBASE_JD_ATOM_STATE_IN_JS,
+ /** Atom has been completed, but not yet handed back to job dispatcher
+ * for dependency resolution */
+ KBASE_JD_ATOM_STATE_HW_COMPLETED,
/** Atom has been completed, but not yet handed back to userspace */
KBASE_JD_ATOM_STATE_COMPLETED
};
struct base_dependency {
base_atom_id atom_id; /**< An atom number */
base_jd_dep_type dependency_type; /**< Dependency type */
-};
+};
typedef struct base_jd_atom_v2 {
- mali_addr64 jc; /**< job-chain GPU address */
+ u64 jc; /**< job-chain GPU address */
struct base_jd_udata udata; /**< user data */
kbase_pointer extres_list; /**< list of external resources */
u16 nr_extres; /**< nr of external resources */
const struct base_dependency pre_dep[2]; /**< pre-dependencies, one need to use SETTER function to assign this field,
this is done in order to reduce possibility of improper assigment of a dependency field */
base_atom_id atom_number; /**< unique number to identify the atom */
- s8 prio; /**< priority - smaller is higher priority */
+ base_jd_prio prio; /**< Atom priority. Refer to @ref base_jd_prio for more details */
u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */
u8 padding[5];
} base_jd_atom_v2;
#ifdef BASE_LEGACY_UK6_SUPPORT
struct base_jd_atom_v2_uk6 {
- mali_addr64 jc; /**< job-chain GPU address */
+ u64 jc; /**< job-chain GPU address */
struct base_jd_udata udata; /**< user data */
kbase_pointer extres_list; /**< list of external resources */
u16 nr_extres; /**< nr of external resources */
base_jd_core_req core_req; /**< core requirements */
base_atom_id pre_dep[2]; /**< pre-dependencies */
base_atom_id atom_number; /**< unique number to identify the atom */
- s8 prio; /**< priority - smaller is higher priority */
+ base_jd_prio prio; /**< priority - smaller is higher priority */
u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */
u8 padding[7];
};
* @param dep_type The dep_type to be assigned.
*
*/
-static INLINE void base_jd_atom_dep_set(const struct base_dependency* const_dep, base_atom_id id, base_jd_dep_type dep_type)
+static inline void base_jd_atom_dep_set(const struct base_dependency *const_dep, base_atom_id id, base_jd_dep_type dep_type)
{
- struct base_dependency* dep;
-
+ struct base_dependency *dep;
+
LOCAL_ASSERT(const_dep != NULL);
/* make sure we don't set not allowed combinations of atom_id/dependency_type */
- LOCAL_ASSERT( ( id == 0 && dep_type == BASE_JD_DEP_TYPE_INVALID) ||
- (id > 0 && dep_type != BASE_JD_DEP_TYPE_INVALID) );
+ LOCAL_ASSERT((id == 0 && dep_type == BASE_JD_DEP_TYPE_INVALID) ||
+ (id > 0 && dep_type != BASE_JD_DEP_TYPE_INVALID));
- dep = REINTERPRET_CAST(struct base_dependency*)const_dep;
+ dep = (struct base_dependency *)const_dep;
dep->atom_id = id;
dep->dependency_type = dep_type;
* @param[in] from The dependency to make a copy from.
*
*/
-static INLINE void base_jd_atom_dep_copy(const struct base_dependency* const_dep, const struct base_dependency* from)
+static inline void base_jd_atom_dep_copy(const struct base_dependency *const_dep, const struct base_dependency *from)
{
LOCAL_ASSERT(const_dep != NULL);
* @param[out] atom A pre-allocated atom to configure as a fence trigger SW atom
* @param[in] fence The base fence object to trigger.
*/
-static INLINE void base_jd_fence_trigger_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence)
+static inline void base_jd_fence_trigger_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence)
{
LOCAL_ASSERT(atom);
LOCAL_ASSERT(fence);
* @param[out] atom A pre-allocated atom to configure as a fence wait SW atom
* @param[in] fence The base fence object to wait on
*/
-static INLINE void base_jd_fence_wait_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence)
+static inline void base_jd_fence_wait_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence)
{
LOCAL_ASSERT(atom);
LOCAL_ASSERT(fence);
* @param handle The handle to the imported memory object
* @param access The type of access requested
*/
-static INLINE void base_external_resource_init(struct base_external_resource * res, struct base_import_handle handle, base_external_resource_access access)
+static inline void base_external_resource_init(struct base_external_resource *res, struct base_import_handle handle, base_external_resource_access access)
{
- mali_addr64 address;
+ u64 address;
+
address = handle.basep.handle;
LOCAL_ASSERT(res != NULL);
* The following typedefs should be removed when a midg types header is added.
* See MIDCOM-1657 for details.
*/
-typedef u32 midg_product_id;
-typedef u32 midg_cache_features;
-typedef u32 midg_tiler_features;
-typedef u32 midg_mem_features;
-typedef u32 midg_mmu_features;
-typedef u32 midg_js_features;
-typedef u32 midg_as_present;
-typedef u32 midg_js_present;
+typedef u32 gpu_product_id;
+typedef u32 gpu_cache_features;
+typedef u32 gpu_tiler_features;
+typedef u32 gpu_mem_features;
+typedef u32 gpu_mmu_features;
+typedef u32 gpu_js_features;
+typedef u32 gpu_as_present;
+typedef u32 gpu_js_present;
-#define MIDG_MAX_JOB_SLOTS 16
+#define GPU_MAX_JOB_SLOTS 16
#else
-#include <midg/mali_midg.h>
+#include <gpu/mali_gpu_registers.h>
+#include <gpu/mali_gpu_props.h>
#endif
/**
*
* There is a related third module outside of Base, which is owned by the MIDG
* module:
- * - @ref midg_gpuprops_static "Midgard Compile-time GPU Properties"
+ * - @ref gpu_props_static "Midgard Compile-time GPU Properties"
*
* Base only deals with properties that vary between different Midgard
* implementations - the Dynamic GPU properties and the Platform Config
* Dynamic GPU properties are presented in two sets:
* -# the commonly used properties in @ref base_gpu_props, which have been
* unpacked from GPU register bitfields.
- * -# The full set of raw, unprocessed properties in @ref midg_raw_gpu_props
+ * -# The full set of raw, unprocessed properties in @ref gpu_raw_gpu_props
* (also a member of @ref base_gpu_props). All of these are presented in
* the packed form, as presented by the GPU registers themselves.
*
- * @usecase The raw properties in @ref midg_raw_gpu_props are necessary to
+ * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to
* allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device
* behaving differently?". In this case, all information about the
* configuration is potentially useful, but it <b>does not need to be processed
size="6,6"
"mali_base.h";
- "midg/midg.h";
+ "gpu/mali_gpu.h";
node [ shape=box ];
{
rank = same; ordering = out;
- "midg/midg_gpu_props.h";
+ "gpu/mali_gpu_props.h";
"base/midg_gpus/mali_t600.h";
"base/midg_gpus/other_midg_gpu.h";
}
{ rank = same; "plat/plat_config.h"; }
{
rank = same;
- "midg/midg.h" [ shape=box ];
+ "gpu/mali_gpu.h" [ shape=box ];
gpu_chooser [ label="" style="invisible" width=0 height=0 fixedsize=true ];
select_gpu [ label="Mali-T600 | Other\n(select_gpu.h)" shape=polygon,sides=4,distortion=0.25 width=3.3 height=0.99 fixedsize=true ] ;
}
{ rank = same; "plat/plat_config.h"; }
{ rank = same; "mali_base.h"; }
- "mali_base.h" -> "midg/midg.h" -> "midg/midg_gpu_props.h";
+ "mali_base.h" -> "gpu/mali_gpu.h" -> "gpu/mali_gpu_props.h";
"mali_base.h" -> "plat/plat_config.h" ;
"mali_base.h" -> select_gpu ;
* @section sec_base_user_api_gpuprops_cocalc Coherency Group calculation
* Creation of the coherent group data is done at device-driver startup, and so
* is one-time. This will most likely involve a loop with CLZ, shifting, and
- * bit clearing on the L2_PRESENT or L3_PRESENT masks, depending on whether the
- * system is L2 or L2+L3 Coherent. The number of shader cores is done by a
+ * bit clearing on the L2_PRESENT mask, depending on whether the
+ * system is L2 Coherent. The number of shader cores is done by a
* population count, since faulty cores may be disabled during production,
* producing a non-contiguous mask.
*
* The memory requirements for this algoirthm can be determined either by a u64
- * population count on the L2/L3_PRESENT masks (a LUT helper already is
+ * population count on the L2_PRESENT mask (a LUT helper already is
* requried for the above), or simple assumption that there can be no more than
* 16 coherent groups, since core groups are typically 4 cores.
*/
/**
* Product specific value.
*/
- midg_product_id product_id;
+ gpu_product_id product_id;
/**
* Status of the GPU release.
u8 padding[5];
};
-struct mali_base_gpu_l3_cache_props {
- u8 log2_line_size;
- u8 log2_cache_size;
- u8 padding[6];
-};
-
struct mali_base_gpu_tiler_props {
u32 bin_size_bytes; /* Max is 4*2^15 */
u32 max_active_levels; /* Max is 2^15 */
};
/**
- * GPU threading system details.
+ * GPU threading system details.
*/
struct mali_base_gpu_thread_props {
- u32 max_threads; /* Max. number of threads per core */
+ u32 max_threads; /* Max. number of threads per core */
u32 max_workgroup_size; /* Max. number of threads per workgroup */
u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */
- u16 max_registers; /* Total size [1..65535] of the register file available per core. */
- u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */
- u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */
- u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */
+ u16 max_registers; /* Total size [1..65535] of the register file available per core. */
+ u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */
+ u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */
+ u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */
u8 padding[7];
};
* provides a cached population-count for that mask.
*
* @note Whilst all cores are exposed in the mask, not all may be available to
- * the application, depending on the Kernel Job Scheduler policy. Therefore,
- * the application should not further restrict the core mask itself, as it may
- * result in an empty core mask. However, it can guarentee that there will be
- * at least one core available for each core group exposed .
- *
- * @usecase Chains marked at certain user-side priorities (e.g. the Long-running
- * (batch) priority ) can be prevented from running on entire core groups by the
- * Kernel Chain Scheduler policy.
+ * the application, depending on the Kernel Power policy.
*
* @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage.
*/
u32 num_core_groups;
/**
- * Coherency features of the memory, accessed by @ref midg_mem_features
+ * Coherency features of the memory, accessed by @ref gpu_mem_features
* methods
*/
- midg_mem_features coherency;
+ gpu_mem_features coherency;
u32 padding;
* the values should be better expressed in an unpacked form in the
* base_gpu_props structure.
*
- * @usecase The raw properties in @ref midg_raw_gpu_props are necessary to
+ * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to
* allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device
* behaving differently?". In this case, all information about the
* configuration is potentially useful, but it <b>does not need to be processed
* Tools software on the host PC.
*
*/
-struct midg_raw_gpu_props {
+struct gpu_raw_gpu_props {
u64 shader_present;
u64 tiler_present;
u64 l2_present;
- u64 l3_present;
+ u64 unused_1; /* keep for backward compatibility */
- midg_cache_features l2_features;
- midg_cache_features l3_features;
- midg_mem_features mem_features;
- midg_mmu_features mmu_features;
+ gpu_cache_features l2_features;
+ u32 suspend_size; /* API 8.2+ */
+ gpu_mem_features mem_features;
+ gpu_mmu_features mmu_features;
- midg_as_present as_present;
+ gpu_as_present as_present;
u32 js_present;
- midg_js_features js_features[MIDG_MAX_JOB_SLOTS];
- midg_tiler_features tiler_features;
+ gpu_js_features js_features[GPU_MAX_JOB_SLOTS];
+ gpu_tiler_features tiler_features;
u32 texture_features[3];
u32 gpu_id;
-
+
u32 thread_max_threads;
u32 thread_max_workgroup_size;
u32 thread_max_barrier_size;
u32 thread_features;
- u32 padding;
+ u32 coherency_features;
};
/**
typedef struct mali_base_gpu_props {
struct mali_base_gpu_core_props core_props;
struct mali_base_gpu_l2_cache_props l2_props;
- struct mali_base_gpu_l3_cache_props l3_props;
+ u64 unused_1; /* keep for backwards compatibility */
struct mali_base_gpu_tiler_props tiler_props;
struct mali_base_gpu_thread_props thread_props;
/** This member is large, likely to be 128 bytes */
- struct midg_raw_gpu_props raw_props;
+ struct gpu_raw_gpu_props raw_props;
/** This must be last member of the structure */
struct mali_base_gpu_coherent_group_info coherency_info;
* Violation of these requirements will cause the Job-Chains to be rejected.
*
* In addition, it is inadvisable for the atom's Job-Chains to contain Jobs
- * of the following @ref midg_job_type (whilst it may work now, it may not
+ * of the following @ref gpu_job_type (whilst it may work now, it may not
* work in future) :
- * - @ref MIDG_JOB_VERTEX
- * - @ref MIDG_JOB_GEOMETRY
+ * - @ref GPU_JOB_VERTEX
+ * - @ref GPU_JOB_GEOMETRY
*
* @note An alternative to using this is to specify the BASE_JD_REQ_ONLY_COMPUTE
* requirement in atoms.
* will be replayed in @b reverse order (so that extra ones can be added
* to the head in future soft jobs without affecting this soft job)
*/
- mali_addr64 tiler_jc_list;
+ u64 tiler_jc_list;
/**
* Pointer to the fragment job chain.
*/
- mali_addr64 fragment_jc;
+ u64 fragment_jc;
/**
* Pointer to the tiler heap free FBD field to be modified.
*/
- mali_addr64 tiler_heap_free;
+ u64 tiler_heap_free;
/**
* Hierarchy mask for the replayed fragment jobs. May be zero.
* Pointer to next entry in the list. A setting of NULL indicates the
* end of the list.
*/
- mali_addr64 next;
+ u64 next;
/**
* Pointer to the job chain.
*/
- mali_addr64 jc;
+ u64 jc;
} base_jd_replay_jc;
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* - size = the amount of data to be synced, in bytes.
* - offset is ignored.
*/
-typedef struct basep_syncset {
+struct basep_syncset {
base_mem_handle mem_handle;
u64 user_addr;
u64 size;
u8 type;
u8 padding[7];
-} basep_syncset;
+};
#endif
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010, 2012-2013, 2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#ifndef _BASE_VENDOR_SPEC_FUNC_H_
#define _BASE_VENDOR_SPEC_FUNC_H_
-#include <malisw/mali_stdtypes.h>
-
-mali_error kbase_get_vendor_specific_cpu_clock_speed(u32 * const);
+int kbase_get_vendor_specific_cpu_clock_speed(u32 * const);
#endif /*_BASE_VENDOR_SPEC_FUNC_H_*/
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#ifndef _KBASE_H_
#define _KBASE_H_
-#include <malisw/mali_malisw.h>
+#include <mali_malisw.h>
#include <mali_kbase_debug.h>
#include "mali_kbase_jd_debugfs.h"
#include "mali_kbase_cpuprops.h"
#include "mali_kbase_gpuprops.h"
+#include "mali_kbase_jm.h"
+#include "mali_kbase_vinstr.h"
#ifdef CONFIG_GPU_TRACEPOINTS
#include <trace/events/gpu.h>
#endif
/* API to release the device list semaphore */
void kbase_dev_list_put(const struct list_head *dev_list);
-mali_error kbase_device_init(struct kbase_device * const kbdev);
+int kbase_device_init(struct kbase_device * const kbdev);
void kbase_device_term(struct kbase_device *kbdev);
void kbase_device_free(struct kbase_device *kbdev);
int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature);
-struct kbase_device *kbase_find_device(int minor); /* Only needed for gator integration */
+
+/* Needed for gator integration and for reporting vsync information */
+struct kbase_device *kbase_find_device(int minor);
void kbase_release_device(struct kbase_device *kbdev);
void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value);
u32 kbase_get_profiling_control(struct kbase_device *kbdev, u32 control);
-/**
- * Ensure that all IRQ handlers have completed execution
- *
- * @param kbdev The kbase device
- */
-void kbase_synchronize_irqs(struct kbase_device *kbdev);
-void kbase_synchronize_irqs(struct kbase_device *kbdev);
-
struct kbase_context *
kbase_create_context(struct kbase_device *kbdev, bool is_compat);
void kbase_destroy_context(struct kbase_context *kctx);
-mali_error kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags);
-
-mali_error kbase_instr_hwcnt_setup(struct kbase_context *kctx, struct kbase_uk_hwcnt_setup *setup);
-mali_error kbase_instr_hwcnt_enable(struct kbase_context *kctx, struct kbase_uk_hwcnt_setup *setup);
-mali_error kbase_instr_hwcnt_disable(struct kbase_context *kctx);
-mali_error kbase_instr_hwcnt_clear(struct kbase_context *kctx);
-mali_error kbase_instr_hwcnt_dump(struct kbase_context *kctx);
-mali_error kbase_instr_hwcnt_dump_irq(struct kbase_context *kctx);
-mali_bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, mali_bool * const success);
-void kbase_instr_hwcnt_suspend(struct kbase_device *kbdev);
-void kbase_instr_hwcnt_resume(struct kbase_device *kbdev);
+int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags);
-void kbasep_cache_clean_worker(struct work_struct *data);
-void kbase_clean_caches_done(struct kbase_device *kbdev);
-
-/**
- * The GPU has completed performance count sampling successfully.
- */
-void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev);
-
-mali_error kbase_jd_init(struct kbase_context *kctx);
+int kbase_jd_init(struct kbase_context *kctx);
void kbase_jd_exit(struct kbase_context *kctx);
#ifdef BASE_LEGACY_UK6_SUPPORT
-mali_error kbase_jd_submit(struct kbase_context *kctx,
+int kbase_jd_submit(struct kbase_context *kctx,
const struct kbase_uk_job_submit *submit_data,
int uk6_atom);
#else
-mali_error kbase_jd_submit(struct kbase_context *kctx,
+int kbase_jd_submit(struct kbase_context *kctx,
const struct kbase_uk_job_submit *submit_data);
#endif
void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp,
- kbasep_js_atom_done_code done_code);
+ kbasep_js_atom_done_code done_code);
void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom);
+void kbase_jd_evict(struct kbase_device *kbdev, struct kbase_jd_atom *katom);
void kbase_jd_zap_context(struct kbase_context *kctx);
-mali_bool jd_done_nolock(struct kbase_jd_atom *katom);
+bool jd_done_nolock(struct kbase_jd_atom *katom);
void kbase_jd_free_external_resources(struct kbase_jd_atom *katom);
-mali_bool jd_submit_atom(struct kbase_context *kctx,
+bool jd_submit_atom(struct kbase_context *kctx,
const struct base_jd_atom_v2 *user_atom,
struct kbase_jd_atom *katom);
-mali_error kbase_job_slot_init(struct kbase_device *kbdev);
-void kbase_job_slot_halt(struct kbase_device *kbdev);
-void kbase_job_slot_term(struct kbase_device *kbdev);
void kbase_job_done(struct kbase_device *kbdev, u32 done);
-void kbase_job_zap_context(struct kbase_context *kctx);
+
+/**
+ * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms
+ * and soft stop them
+ * @kctx: Pointer to context to check.
+ * @katom: Pointer to priority atom.
+ *
+ * Atoms from @kctx on the same job slot as @katom, which have lower priority
+ * than @katom will be soft stopped and put back in the queue, so that atoms
+ * with higher priority can run.
+ *
+ * The js_data.runpool_irq.lock must be held when calling this function.
+ */
+void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
+ struct kbase_jd_atom *katom);
void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
struct kbase_jd_atom *target_katom);
void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event);
int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent);
int kbase_event_pending(struct kbase_context *ctx);
-mali_error kbase_event_init(struct kbase_context *kctx);
+int kbase_event_init(struct kbase_context *kctx);
void kbase_event_close(struct kbase_context *kctx);
void kbase_event_cleanup(struct kbase_context *kctx);
void kbase_event_wakeup(struct kbase_context *kctx);
int kbase_process_soft_job(struct kbase_jd_atom *katom);
-mali_error kbase_prepare_soft_job(struct kbase_jd_atom *katom);
+int kbase_prepare_soft_job(struct kbase_jd_atom *katom);
void kbase_finish_soft_job(struct kbase_jd_atom *katom);
void kbase_cancel_soft_job(struct kbase_jd_atom *katom);
void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev);
bool kbase_replay_process(struct kbase_jd_atom *katom);
/* api used internally for register access. Contains validation and tracing */
-void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, struct kbase_context *kctx);
-u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, struct kbase_context *kctx);
void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value);
void kbase_device_trace_buffer_install(struct kbase_context *kctx, u32 *tb, size_t size);
void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx);
void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value);
u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset);
+
void kbasep_as_do_poke(struct work_struct *work);
-/** Report a GPU fault.
+/** Returns the name associated with a Mali exception code
*
* This function is called from the interrupt handler when a GPU fault occurs.
* It reports the details of the fault using KBASE_DEBUG_PRINT_WARN.
*
- * @param kbdev The kbase device that the GPU fault occurred from.
- * @param multiple Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS was also set
- */
-void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple);
-
-/** Kill all jobs that are currently running from a context
- *
- * This is used in response to a page fault to remove all jobs from the faulting context from the hardware.
- *
- * @param kctx The context to kill jobs from
- */
-void kbase_job_kill_jobs_from_context(struct kbase_context *kctx);
-
-/**
- * GPU interrupt handler
- *
- * This function is called from the interrupt handler when a GPU irq is to be handled.
- *
- * @param kbdev The kbase device to handle an IRQ for
- * @param val The value of the GPU IRQ status register which triggered the call
- */
-void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val);
-
-/**
- * Prepare for resetting the GPU.
- * This function just soft-stops all the slots to ensure that as many jobs as possible are saved.
- *
- * The function returns a boolean which should be interpreted as follows:
- * - MALI_TRUE - Prepared for reset, kbase_reset_gpu should be called.
- * - MALI_FALSE - Another thread is performing a reset, kbase_reset_gpu should not be called.
- *
- * @return See description
- */
-mali_bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev);
-
-/**
- * Pre-locked version of @a kbase_prepare_to_reset_gpu.
- *
- * Identical to @a kbase_prepare_to_reset_gpu, except that the
- * kbasep_js_device_data::runpool_irq::lock is externally locked.
- *
- * @see kbase_prepare_to_reset_gpu
- */
-mali_bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev);
-
-/** Reset the GPU
- *
- * This function should be called after kbase_prepare_to_reset_gpu iff it returns MALI_TRUE.
- * It should never be called without a corresponding call to kbase_prepare_to_reset_gpu.
- *
- * After this function is called (or not called if kbase_prepare_to_reset_gpu returned MALI_FALSE),
- * the caller should wait for kbdev->reset_waitq to be signalled to know when the reset has completed.
- */
-void kbase_reset_gpu(struct kbase_device *kbdev);
-
-/**
- * Pre-locked version of @a kbase_reset_gpu.
- *
- * Identical to @a kbase_reset_gpu, except that the
- * kbasep_js_device_data::runpool_irq::lock is externally locked.
- *
- * @see kbase_reset_gpu
- */
-void kbase_reset_gpu_locked(struct kbase_device *kbdev);
-
-/** Returns the name associated with a Mali exception code
- *
+ * @param[in] kbdev The kbase device that the GPU fault occurred from.
* @param[in] exception_code exception code
* @return name associated with the exception code
*/
-const char *kbase_exception_name(u32 exception_code);
+const char *kbase_exception_name(struct kbase_device *kbdev,
+ u32 exception_code);
/**
* Check whether a system suspend is in progress, or has already been suspended
* a dmb was executed recently (to ensure the value is most
* up-to-date). However, without a lock the value could change afterwards.
*
- * @return MALI_FALSE if a suspend is not in progress
- * @return !=MALI_FALSE otherwise
+ * @return false if a suspend is not in progress
+ * @return !=false otherwise
*/
-static INLINE mali_bool kbase_pm_is_suspending(struct kbase_device *kbdev) {
+static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev)
+{
return kbdev->pm.suspending;
}
* Return the atom's ID, as was originally supplied by userspace in
* base_jd_atom_v2::atom_number
*/
-static INLINE int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom)
+static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
int result;
+
KBASE_DEBUG_ASSERT(kctx);
KBASE_DEBUG_ASSERT(katom);
KBASE_DEBUG_ASSERT(katom->kctx == kctx);
return result;
}
+/**
+ * kbase_jd_atom_from_id - Return the atom structure for the given atom ID
+ * @kctx: Context pointer
+ * @id: ID of atom to retrieve
+ *
+ * Return: Pointer to struct kbase_jd_atom associated with the supplied ID
+ */
+static inline struct kbase_jd_atom *kbase_jd_atom_from_id(
+ struct kbase_context *kctx, int id)
+{
+ return &kctx->jctx.atoms[id];
+}
+
/**
* Initialize the disjoint state
*
#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2
#if KBASE_TRACE_ENABLE
+void kbasep_trace_debugfs_init(struct kbase_device *kbdev);
+
#ifndef CONFIG_MALI_SYSTEM_TRACE
/** Add trace values about a job-slot
*
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
if (!region || (region->flags & KBASE_REG_FREE))
goto out_unlock;
- page_array = kbase_get_phy_pages(region);
+ page_array = kbase_get_cpu_phy_pages(region);
if (!page_array)
goto out_unlock;
/* page_1 is a u32 pointer, offset is expressed in bytes */
page_1 += offset>>2;
- dma_sync_single_for_cpu(katom->kctx->kbdev->dev,
+
+ kbase_sync_single_for_cpu(katom->kctx->kbdev,
kbase_dma_addr(p) + offset,
copy_size, DMA_BIDIRECTIONAL);
+
memcpy(dst, page_1, copy_size);
/* The data needed overflows page the dimension,
p = pfn_to_page(PFN_DOWN(page_array[page_index + 1]));
page_2 = kmap_atomic(p);
- dma_sync_single_for_cpu(katom->kctx->kbdev->dev,
+ kbase_sync_single_for_cpu(katom->kctx->kbdev,
kbase_dma_addr(p),
JOB_HEADER_SIZE - copy_size, DMA_BIDIRECTIONAL);
+
memcpy(dst + copy_size, page_2, JOB_HEADER_SIZE - copy_size);
}
/* Flush CPU cache to update memory for future GPU reads*/
memcpy(page_1, dst, copy_size);
p = pfn_to_page(PFN_DOWN(page_array[page_index]));
- dma_sync_single_for_device(katom->kctx->kbdev->dev,
+
+ kbase_sync_single_for_device(katom->kctx->kbdev,
kbase_dma_addr(p) + offset,
copy_size, DMA_TO_DEVICE);
memcpy(page_2, dst + copy_size,
JOB_HEADER_SIZE - copy_size);
p = pfn_to_page(PFN_DOWN(page_array[page_index + 1]));
- dma_sync_single_for_device(katom->kctx->kbdev->dev,
+
+ kbase_sync_single_for_device(katom->kctx->kbdev,
kbase_dma_addr(p),
JOB_HEADER_SIZE - copy_size,
DMA_TO_DEVICE);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file mali_kbase_cache_policy.h
+/*
* Cache Policy API.
*/
/*
* The output flags should be a combination of the following values:
- * KBASE_REG_CPU_CACHED: CPU cache should be enabled
+ * KBASE_REG_CPU_CACHED: CPU cache should be enabled.
*/
u32 kbase_cache_enabled(u32 flags, u32 nr_pages)
{
CSTD_UNUSED(nr_pages);
+#ifdef CONFIG_MALI_CACHE_COHERENT
+ /* Cache is completely coherent at hardware level. So always allocate
+ * cached memory.
+ */
+ cache_flags |= KBASE_REG_CPU_CACHED;
+#else
if (flags & BASE_MEM_CACHED_CPU)
cache_flags |= KBASE_REG_CPU_CACHED;
+#endif /* (CONFIG_MALI_CACHE_COHERENT) */
return cache_flags;
}
+
+
+void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle,
+ size_t size, enum dma_data_direction dir)
+{
+ dma_sync_single_for_device(kbdev->dev, handle, size, dir);
+}
+
+
+void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle,
+ size_t size, enum dma_data_direction dir)
+{
+ dma_sync_single_for_cpu(kbdev->dev, handle, size, dir);
+}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file mali_kbase_cache_policy.h
+/*
* Cache Policy API.
*/
#ifndef _KBASE_CACHE_POLICY_H_
#define _KBASE_CACHE_POLICY_H_
-#include <malisw/mali_malisw.h>
#include "mali_kbase.h"
#include "mali_base_kernel.h"
-
+#include "mali_kbase_device_internal.h"
/**
- * @brief Choose the cache policy for a specific region
- *
- * Tells whether the CPU and GPU caches should be enabled or not for a specific region.
- * This function can be modified to customize the cache policy depending on the flags
- * and size of the region.
+ * kbase_cache_enabled - Choose the cache policy for a specific region
+ * @flags: flags describing attributes of the region
+ * @nr_pages: total number of pages (backed or not) for the region
*
- * @param[in] flags flags describing attributes of the region
- * @param[in] nr_pages total number of pages (backed or not) for the region
+ * Tells whether the CPU and GPU caches should be enabled or not for a specific
+ * region.
+ * This function can be modified to customize the cache policy depending on the
+ * flags and size of the region.
*
- * @return a combination of KBASE_REG_CPU_CACHED and KBASE_REG_GPU_CACHED depending
- * on the cache policy
+ * Return: a combination of %KBASE_REG_CPU_CACHED and %KBASE_REG_GPU_CACHED
+ * depending on the cache policy
*/
u32 kbase_cache_enabled(u32 flags, u32 nr_pages);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase_defs.h>
#include <mali_kbase_config_defaults.h>
-/* Specifies how many attributes are permitted in the config (excluding terminating attribute).
- * This is used in validation function so we can detect if configuration is properly terminated. This value can be
- * changed if we need to introduce more attributes or many memory regions need to be defined */
-#define ATTRIBUTE_COUNT_MAX 32
-
-int kbasep_get_config_attribute_count(const struct kbase_attribute *attributes)
+int kbasep_platform_device_init(struct kbase_device *kbdev)
{
- int count = 1;
-
- if (!attributes)
- return -EINVAL;
+ struct kbase_platform_funcs_conf *platform_funcs_p;
- while (attributes->id != KBASE_CONFIG_ATTR_END) {
- attributes++;
- count++;
- }
+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS;
+ if (platform_funcs_p && platform_funcs_p->platform_init_func)
+ return platform_funcs_p->platform_init_func(kbdev);
- return count;
+ return 0;
}
-const struct kbase_attribute *kbasep_get_next_attribute(const struct kbase_attribute *attributes, int attribute_id)
+void kbasep_platform_device_term(struct kbase_device *kbdev)
{
- KBASE_DEBUG_ASSERT(attributes != NULL);
-
- while (attributes->id != KBASE_CONFIG_ATTR_END) {
- if (attributes->id == attribute_id)
- return attributes;
+ struct kbase_platform_funcs_conf *platform_funcs_p;
- attributes++;
- }
- return NULL;
+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS;
+ if (platform_funcs_p && platform_funcs_p->platform_term_func)
+ platform_funcs_p->platform_term_func(kbdev);
}
-KBASE_EXPORT_TEST_API(kbasep_get_next_attribute)
-
int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed)
{
KBASE_DEBUG_ASSERT(NULL != clock_speed);
return 0;
}
-uintptr_t kbasep_get_config_value(struct kbase_device *kbdev, const struct kbase_attribute *attributes, int attribute_id)
-{
- const struct kbase_attribute *attr;
-
- KBASE_DEBUG_ASSERT(attributes != NULL);
-
- attr = kbasep_get_next_attribute(attributes, attribute_id);
- if (attr != NULL)
- return attr->data;
-
- /* default values */
- switch (attribute_id) {
- /* Begin scheduling defaults */
- case KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS:
- return DEFAULT_JS_SCHEDULING_TICK_NS;
- case KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS:
- return DEFAULT_JS_SOFT_STOP_TICKS;
- case KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL:
- return DEFAULT_JS_SOFT_STOP_TICKS_CL;
- case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS:
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
- return DEFAULT_JS_HARD_STOP_TICKS_SS_HW_ISSUE_8408;
- else
- return DEFAULT_JS_HARD_STOP_TICKS_SS;
- case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL:
- return DEFAULT_JS_HARD_STOP_TICKS_CL;
- case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS:
- return DEFAULT_JS_HARD_STOP_TICKS_NSS;
- case KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS:
- return DEFAULT_JS_CTX_TIMESLICE_NS;
- case KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS:
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
- return DEFAULT_JS_RESET_TICKS_SS_HW_ISSUE_8408;
- else
- return DEFAULT_JS_RESET_TICKS_SS;
- case KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL:
- return DEFAULT_JS_RESET_TICKS_CL;
- case KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS:
- return DEFAULT_JS_RESET_TICKS_NSS;
- case KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS:
- return DEFAULT_JS_RESET_TIMEOUT_MS;
- /* End scheduling defaults */
- case KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS:
- return 0;
- case KBASE_CONFIG_ATTR_PLATFORM_FUNCS:
- return 0;
- case KBASE_CONFIG_ATTR_CPU_SPEED_FUNC:
- return DEFAULT_CPU_SPEED_FUNC;
- case KBASE_CONFIG_ATTR_GPU_SPEED_FUNC:
- return 0;
- case KBASE_CONFIG_ATTR_POWER_MANAGEMENT_DVFS_FREQ:
- return DEFAULT_PM_DVFS_FREQ;
- case KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS:
- return DEFAULT_PM_GPU_POWEROFF_TICK_NS;
- case KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_SHADER:
- return DEFAULT_PM_POWEROFF_TICK_SHADER;
- case KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_GPU:
- return DEFAULT_PM_POWEROFF_TICK_GPU;
- case KBASE_CONFIG_ATTR_POWER_MODEL_CALLBACKS:
- return 0;
-
- default:
- dev_err(kbdev->dev, "kbasep_get_config_value. Cannot get value of attribute with id=%d and no default value defined", attribute_id);
- return 0;
- }
-}
-
-KBASE_EXPORT_TEST_API(kbasep_get_config_value)
-
-mali_bool kbasep_platform_device_init(struct kbase_device *kbdev)
-{
- struct kbase_platform_funcs_conf *platform_funcs;
-
- platform_funcs = (struct kbase_platform_funcs_conf *)kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PLATFORM_FUNCS);
- if (platform_funcs) {
- if (platform_funcs->platform_init_func)
- return platform_funcs->platform_init_func(kbdev);
- }
- return MALI_TRUE;
-}
-
-void kbasep_platform_device_term(struct kbase_device *kbdev)
-{
- struct kbase_platform_funcs_conf *platform_funcs;
-
- platform_funcs = (struct kbase_platform_funcs_conf *)kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PLATFORM_FUNCS);
- if (platform_funcs) {
- if (platform_funcs->platform_term_func)
- platform_funcs->platform_term_func(kbdev);
- }
-}
-
-static mali_bool kbasep_validate_pm_callback(const struct kbase_pm_callback_conf *callbacks, const struct kbase_device *kbdev)
-{
- if (callbacks == NULL) {
- /* Having no callbacks is valid */
- return MALI_TRUE;
- }
-
- if ((callbacks->power_off_callback != NULL && callbacks->power_on_callback == NULL) || (callbacks->power_off_callback == NULL && callbacks->power_on_callback != NULL)) {
- dev_warn(kbdev->dev, "Invalid power management callbacks: Only one of power_off_callback and power_on_callback was specified");
- return MALI_FALSE;
- }
- return MALI_TRUE;
-}
-
-static mali_bool kbasep_validate_cpu_speed_func(kbase_cpuprops_clock_speed_function fcn)
-{
- return fcn != NULL;
-}
-
-mali_bool kbasep_validate_configuration_attributes(struct kbase_device *kbdev, const struct kbase_attribute *attributes)
-{
- int i;
-
- KBASE_DEBUG_ASSERT(attributes);
-
- for (i = 0; attributes[i].id != KBASE_CONFIG_ATTR_END; i++) {
- if (i >= ATTRIBUTE_COUNT_MAX) {
- dev_warn(kbdev->dev, "More than ATTRIBUTE_COUNT_MAX=%d configuration attributes defined. Is attribute list properly terminated?", ATTRIBUTE_COUNT_MAX);
- return MALI_FALSE;
- }
-
- switch (attributes[i].id) {
- /* Only non-zero unsigned 32-bit values accepted */
- case KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS:
-#if CSTD_CPU_64BIT
- if (attributes[i].data == 0u || (u64) attributes[i].data > (u64) U32_MAX) {
-#else
- if (attributes[i].data == 0u) {
-#endif
- dev_warn(kbdev->dev, "Invalid Job Scheduling Configuration attribute for " "KBASE_CONFIG_ATTR_JS_SCHEDULING_TICKS_NS: %d", (int)attributes[i].data);
- return MALI_FALSE;
- }
- break;
-
- /* All these Job Scheduling attributes are FALLTHROUGH: only unsigned 32-bit values accepted */
- case KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS:
- case KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL:
- case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS:
- case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL:
- case KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS:
- case KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS:
- case KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL:
- case KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS:
- case KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS:
- case KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS:
-#if CSTD_CPU_64BIT
- if ((u64) attributes[i].data > (u64) U32_MAX) {
- dev_warn(kbdev->dev, "Job Scheduling Configuration attribute exceeds 32-bits: " "id==%d val==%d", attributes[i].id, (int)attributes[i].data);
- return MALI_FALSE;
- }
-#endif
- break;
-
- case KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS:
- if (MALI_FALSE == kbasep_validate_pm_callback((struct kbase_pm_callback_conf *)attributes[i].data, kbdev)) {
- /* Warning message handled by kbasep_validate_pm_callback() */
- return MALI_FALSE;
- }
- break;
-
- case KBASE_CONFIG_ATTR_CPU_SPEED_FUNC:
- if (MALI_FALSE == kbasep_validate_cpu_speed_func((kbase_cpuprops_clock_speed_function) attributes[i].data)) {
- dev_warn(kbdev->dev, "Invalid function pointer in KBASE_CONFIG_ATTR_CPU_SPEED_FUNC");
- return MALI_FALSE;
- }
- break;
-
- case KBASE_CONFIG_ATTR_GPU_SPEED_FUNC:
- if (0 == attributes[i].data) {
- dev_warn(kbdev->dev, "Invalid function pointer in KBASE_CONFIG_ATTR_GPU_SPEED_FUNC");
- return MALI_FALSE;
- }
- break;
-
- case KBASE_CONFIG_ATTR_PLATFORM_FUNCS:
- /* any value is allowed */
- break;
-
- case KBASE_CONFIG_ATTR_POWER_MANAGEMENT_DVFS_FREQ:
-#if CSTD_CPU_64BIT
- if ((u64) attributes[i].data > (u64) U32_MAX) {
- dev_warn(kbdev->dev, "PM DVFS interval exceeds 32-bits: " "id==%d val==%d", attributes[i].id, (int)attributes[i].data);
- return MALI_FALSE;
- }
-#endif
- break;
-
- case KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS:
-#if CSTD_CPU_64BIT
- if (attributes[i].data == 0u || (u64) attributes[i].data > (u64) U32_MAX) {
-#else
- if (attributes[i].data == 0u) {
-#endif
- dev_warn(kbdev->dev, "Invalid Power Manager Configuration attribute for " "KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS: %d", (int)attributes[i].data);
- return MALI_FALSE;
- }
- break;
-
- case KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_SHADER:
- case KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_GPU:
-#if CSTD_CPU_64BIT
- if ((u64) attributes[i].data > (u64) U32_MAX) {
- dev_warn(kbdev->dev, "Power Manager Configuration attribute exceeds 32-bits: " "id==%d val==%d", attributes[i].id, (int)attributes[i].data);
- return MALI_FALSE;
- }
-#endif
- break;
-
- case KBASE_CONFIG_ATTR_POWER_MODEL_CALLBACKS:
- if (0 == attributes[i].data) {
- dev_warn(kbdev->dev, "Power model callbacks is specified but NULL: " "id==%d val==%d",
- attributes[i].id, (int)attributes[i].data);
- return MALI_FALSE;
- }
- break;
-
- default:
- dev_warn(kbdev->dev, "Invalid attribute found in configuration: %d", attributes[i].id);
- return MALI_FALSE;
- }
- }
-
- return MALI_TRUE;
-}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#ifndef _KBASE_CONFIG_H_
#define _KBASE_CONFIG_H_
-#include <malisw/mali_stdtypes.h>
+#include <asm/page.h>
+
+#include <mali_malisw.h>
+#include <mali_kbase_backend_config.h>
/**
* @addtogroup base_api
#define KBASE_HWCNT_DUMP_BYPASS_ROOT 0
#endif
-/* Enable power management API, note that KBASE_PM_EN==0 is not supported */
-#define KBASE_PM_EN 1
-/* Enable GPU reset API, note that KBASE_GPU_RESET_EN==0 is not supported */
-#define KBASE_GPU_RESET_EN 1
-/* Enable HW MMU backend, note that KBASE_MMU_HW_BACKEND==0 is not supported */
-#define KBASE_MMU_HW_BACKEND 1
-
-
#include <linux/rbtree.h>
-
-/**
- * Device wide configuration
- */
-enum {
- /**
- * Invalid attribute ID (reserve 0).
- *
- * Attached value: Ignored
- * Default value: NA
- * */
- KBASE_CONFIG_ATTR_INVALID,
-
- /*** Begin Job Scheduling Configs ***/
- /**
- * Job Scheduler scheduling tick granuality. This is in nanoseconds to
- * allow HR timer support.
- *
- * On each scheduling tick, the scheduler may decide to:
- * -# soft stop a job (the job will be re-run later, and other jobs will
- * be able to run on the GPU now). This effectively controls the
- * 'timeslice' given to a job.
- * -# hard stop a job (to kill a job if it has spent too long on the GPU
- * and didn't soft-stop).
- *
- * The numbers of ticks for these events are controlled by:
- * - @ref KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS
- * - @ref KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS
- * - @ref KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS
- *
- * A soft-stopped job will later be resumed, allowing it to use more GPU
- * time <em>in total</em> than that defined by any of the above. However,
- * the scheduling policy attempts to limit the amount of \em uninterrupted
- * time spent on the GPU using the above values (that is, the 'timeslice'
- * of a job)
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit kbasep_js_device_data::scheduling_tick_ns.
- * The value might be rounded down to lower precision. Must be non-zero
- * after rounding.<br>
- * Default value: @ref DEFAULT_JS_SCHEDULING_TICK_NS
- *
- * @note this value is allowed to be greater than
- * @ref KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS. This allows jobs to run on (much)
- * longer than the job-timeslice, but once this happens, the context gets
- * scheduled in (much) less frequently than others that stay within the
- * ctx-timeslice.
- */
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
-
- /**
- * Job Scheduler minimum number of scheduling ticks before non-CL jobs
- * are soft-stopped.
- *
- * This defines the amount of time a job is allowed to stay on the GPU,
- * before it is soft-stopped to allow other jobs to run.
- *
- * That is, this defines the 'timeslice' of the job. It is separate from the
- * timeslice of the context that contains the job (see
- * @ref KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS).
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit kbasep_js_device_data::soft_stop_ticks<br>
- * Default value: @ref DEFAULT_JS_SOFT_STOP_TICKS
- *
- * @note a value of zero means "the quickest time to soft-stop a job",
- * which is somewhere between instant and one tick later.
- *
- * @note this value is allowed to be greater than
- * @ref KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS or
- * @ref KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS. This effectively disables
- * soft-stop, and just uses hard-stop instead. In this case, this value
- * should be much greater than any of the hard stop values (to avoid
- * soft-stop-after-hard-stop)
- *
- * @see KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS
- */
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
-
- /**
- * Job Scheduler minimum number of scheduling ticks before CL jobs
- * are soft-stopped.
- *
- * This defines the amount of time a job is allowed to stay on the GPU,
- * before it is soft-stopped to allow other jobs to run.
- *
- * That is, this defines the 'timeslice' of the job. It is separate
- * from the timeslice of the context that contains the job (see
- * @ref KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS).
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit
- * kbasep_js_device_data::soft_stop_ticks_cl<br>
- * Default value: @ref DEFAULT_JS_SOFT_STOP_TICKS_CL
- *
- * @note a value of zero means "the quickest time to soft-stop a job",
- * which is somewhere between instant and one tick later.
- *
- * @note this value is allowed to be greater than
- * @ref KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL. This effectively
- * disables soft-stop, and just uses hard-stop instead. In this case,
- * this value should be much greater than any of the hard stop values
- * (to avoid soft-stop-after-hard-stop)
- *
- * @see KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS
- */
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL,
-
- /**
- * Job Scheduler minimum number of scheduling ticks before non-CL jobs
- * are hard-stopped.
- *
- * This defines the amount of time a job is allowed to spend on the GPU before it
- * is killed. Such jobs won't be resumed if killed.
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit kbasep_js_device_data::hard_stop_ticks_ss<br>
- * Default value: @ref DEFAULT_JS_HARD_STOP_TICKS_SS
- *
- * @note a value of zero means "the quickest time to hard-stop a job",
- * which is somewhere between instant and one tick later.
- *
- * @see KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS
- */
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
-
- /**
- * Job Scheduler minimum number of scheduling ticks before CL jobs are hard-stopped.
- *
- * This defines the amount of time a job is allowed to spend on the GPU before it
- * is killed. Such jobs won't be resumed if killed.
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit kbasep_js_device_data::hard_stop_ticks_cl<br>
- * Default value: @ref DEFAULT_JS_HARD_STOP_TICKS_CL
- *
- * @note a value of zero means "the quickest time to hard-stop a job",
- * which is somewhere between instant and one tick later.
- *
- * @see KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS
- */
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL,
-
- /**
- * Job Scheduler minimum number of scheduling ticks before jobs are hard-stopped
- * when dumping.
- *
- * This defines the amount of time a job is allowed to spend on the GPU before it
- * is killed. Such jobs won't be resumed if killed.
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit kbasep_js_device_data::hard_stop_ticks_nss<br>
- * Default value: @ref DEFAULT_JS_HARD_STOP_TICKS_NSS
- *
- * @note a value of zero means "the quickest time to hard-stop a job",
- * which is somewhere between instant and one tick later.
- *
- * @see KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS
- */
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
-
- /**
- * Job Scheduler timeslice that a context is scheduled in for, in nanoseconds.
- *
- * When a context has used up this amount of time across its jobs, it is
- * scheduled out to let another run.
- *
- * @note the resolution is nanoseconds (ns) here, because that's the format
- * often used by the OS.
- *
- * This value controls affects the actual time defined by the following
- * defaults:
- * - @ref DEFAULT_JS_CFS_CTX_RUNTIME_INIT_SLICES
- * - @ref DEFAULT_JS_CFS_CTX_RUNTIME_MIN_SLICES
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit kbasep_js_device_data::ctx_timeslice_ns.
- * The value might be rounded down to lower precision.<br>
- * Default value: @ref DEFAULT_JS_CTX_TIMESLICE_NS
- *
- * @note a value of zero models a "Round Robin" scheduling policy, and
- * disables @ref DEFAULT_JS_CFS_CTX_RUNTIME_INIT_SLICES
- * (initially causing LIFO scheduling) and
- * @ref DEFAULT_JS_CFS_CTX_RUNTIME_MIN_SLICES (allowing
- * not-run-often contexts to get scheduled in quickly, but to only use
- * a single timeslice when they get scheduled in).
- */
- KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS,
-
- /**
- * Job Scheduler minimum number of scheduling ticks before non-CL jobs
- * cause the GPU to be reset.
- *
- * This defines the amount of time a job is allowed to spend on the GPU before it
- * is assumed that the GPU has hung and needs to be reset. The assumes that the job
- * has been hard-stopped already and so the presence of a job that has remained on
- * the GPU for so long indicates that the GPU has in some way hung.
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit kbasep_js_device_data::gpu_reset_ticks_nss<br>
- * Default value: @ref DEFAULT_JS_RESET_TICKS_SS
- *
- * @see KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS
- */
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
-
- /**
- * Job Scheduler minimum number of scheduling ticks before CL jobs
- * cause the GPU to be reset.
- *
- * This defines the amount of time a job is allowed to spend on the GPU before it
- * is assumed that the GPU has hung and needs to be reset. The assumes that the job
- * has been hard-stopped already and so the presence of a job that has remained on
- * the GPU for so long indicates that the GPU has in some way hung.
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit kbasep_js_device_data::gpu_reset_ticks_cl<br>
- * Default value: @ref DEFAULT_JS_RESET_TICKS_CL
- *
- * @see KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS
- */
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL,
-
- /**
- * Job Scheduler minimum number of scheduling ticks before jobs cause the GPU to be
- * reset when dumping.
- *
- * This defines the amount of time a job is allowed to spend on the GPU before it
- * is assumed that the GPU has hung and needs to be reset. The assumes that the job
- * has been hard-stopped already and so the presence of a job that has remained on
- * the GPU for so long indicates that the GPU has in some way hung.
- *
- * This value is supported by the following scheduling policies:
- * - The Completely Fair Share (CFS) policy
- *
- * Attached value: unsigned 32-bit kbasep_js_device_data::gpu_reset_ticks_nss<br>
- * Default value: @ref DEFAULT_JS_RESET_TICKS_NSS
- *
- * @see KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS
- */
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
-
- /**
- * Number of milliseconds given for other jobs on the GPU to be
- * soft-stopped when the GPU needs to be reset.
- *
- * Attached value: number in milliseconds
- * Default value: @ref DEFAULT_JS_RESET_TIMEOUT_MS
- */
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- /*** End Job Scheduling Configs ***/
-
- /** Power management configuration
- *
- * Attached value: pointer to @ref kbase_pm_callback_conf
- * Default value: See @ref kbase_pm_callback_conf
- */
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
-
- /**
- * A pointer to a function that calculates the CPU clock
- * speed of the platform in MHz - see
- * @ref kbase_cpuprops_clock_speed_function for the function
- * prototype.
- *
- * Attached value: A @ref kbase_cpuprops_clock_speed_function.
- * Default Value: Pointer to @ref DEFAULT_CPU_SPEED_FUNC -
- * returns a clock speed of 100 MHz.
- */
- KBASE_CONFIG_ATTR_CPU_SPEED_FUNC,
-
- /**
- * A pointer to a function that calculates the GPU clock
- * speed of the platform in MHz - see
- * @ref kbase_gpuprops_clock_speed_function for the function
- * prototype.
- *
- * Attached value: A @ref kbase_gpuprops_clock_speed_function.
- * Default Value: NULL (in which case the driver assumes maximum
- * GPU frequency stored in gpu_freq_khz_max)
- */
- KBASE_CONFIG_ATTR_GPU_SPEED_FUNC,
-
- /**
- * Platform specific configuration functions
- *
- * Attached value: pointer to @ref kbase_platform_funcs_conf
- * Default value: See @ref kbase_platform_funcs_conf
- */
- KBASE_CONFIG_ATTR_PLATFORM_FUNCS,
-
- /**
- * Rate at which dvfs data should be collected.
- *
- * Attached value: u32 value
- * Default value: 500 Milliseconds
- */
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_DVFS_FREQ,
-
- /**
- * Power Management poweroff tick granuality. This is in nanoseconds to
- * allow HR timer support.
- *
- * On each scheduling tick, the power manager core may decide to:
- * -# Power off one or more shader cores
- * -# Power off the entire GPU
- *
- * Attached value: number in nanoseconds
- * Default value: @ref DEFAULT_PM_GPU_POWEROFF_TICK_NS,
- */
- KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS,
-
- /**
- * Power Manager number of ticks before shader cores are powered off
- *
- * Attached value: unsigned 32-bit kbasep_pm_device_data::poweroff_shader_ticks<br>
- * Default value: @ref DEFAULT_PM_POWEROFF_TICK_SHADER
- *
- * @see KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS
- */
- KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_SHADER,
-
- /**
- * Power Manager number of ticks before GPU is powered off
- *
- * Attached value: unsigned 32-bit kbasep_pm_device_data::poweroff_gpu_ticks<br>
- * Default value: @ref DEFAULT_PM_POWEROFF_TICK_GPU
- *
- * @see KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS
- */
- KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_GPU,
-
- /** Power model for IPA
- *
- * Attached value: pointer to @ref mali_pa_model_ops
- */
- KBASE_CONFIG_ATTR_POWER_MODEL_CALLBACKS,
-
- /**
- * End of attribute list indicator.
- * The configuration loader will stop processing any more elements
- * when it encounters this attribute.
- *
- * Default value: NA
- */
- KBASE_CONFIG_ATTR_END = 0x1FFFUL
-};
-
-/*
- * @brief specifies a single attribute
- *
- * Attribute is identified by attr field. Data is either integer or a pointer to attribute-specific structure.
- */
-typedef struct kbase_attribute {
- int id;
- uintptr_t data;
-} kbase_attribute;
-
/* Forward declaration of struct kbase_device */
struct kbase_device;
-/*
- * @brief Specifies the functions for platform specific initialization and termination
+/**
+ * kbase_platform_funcs_conf - Specifies platform init/term function pointers
*
- * By default no functions are required. No additional platform specific control is necessary.
+ * Specifies the functions pointers for platform specific initialization and
+ * termination. By default no functions are required. No additional platform
+ * specific control is necessary.
*/
-typedef struct kbase_platform_funcs_conf {
+struct kbase_platform_funcs_conf {
/**
- * Function pointer for platform specific initialization or NULL if no initialization function is required.
- * This function will be called \em before any other callbacks listed in the struct kbase_attribute struct (such as
- * Power Management callbacks).
- * The platform specific private pointer kbase_device::platform_context can be accessed (and possibly initialized) in here.
+ * platform_init_func
+ * - platform specific init function pointer
+ * @kbdev - kbase_device pointer
+ *
+ * Returns 0 on success,
+ * negative error code otherwise.
+ *
+ * Function pointer
+ * for platform specific initialization
+ * or NULL if no initialization function is required.
+ * This function
+ * will be called
+ * before any other callbacks
+ * listed in the struct kbase_attribute struct
+ * (such as Power Management callbacks).
+ *
+ * The platform specific private pointer kbase_device::platform_context
+ * can be accessed (and possibly initialized) in here.
*/
- mali_bool(*platform_init_func) (struct kbase_device *kbdev);
+ int (*platform_init_func)(struct kbase_device *kbdev);
/**
- * Function pointer for platform specific termination or NULL if no termination function is required.
- * This function will be called \em after any other callbacks listed in the struct kbase_attribute struct (such as
- * Power Management callbacks).
- * The platform specific private pointer kbase_device::platform_context can be accessed (and possibly terminated) in here.
+ * platform_term_func - platform specific termination function pointer
+ * @kbdev - kbase_device pointer
+ *
+ * Function pointer for platform specific termination or NULL if no
+ * termination function is required. This function will be called
+ * after any other callbacks listed in the struct kbase_attribute struct
+ * (such as Power Management callbacks).
+ *
+ * The platform specific private pointer kbase_device::platform_context
+ * can be accessed (and possibly terminated) in here.
*/
void (*platform_term_func)(struct kbase_device *kbdev);
-
-} kbase_platform_funcs_conf;
+};
/*
* @brief Specifies the callbacks for power management
*
* By default no callbacks will be made and the GPU must not be powered off.
*/
-typedef struct kbase_pm_callback_conf {
+struct kbase_pm_callback_conf {
/** Callback for when the GPU is idle and the power to it can be switched off.
*
* The system integrator can decide whether to either do nothing, just switch off
* The runtime calls can be triggered by calls from @ref power_off_callback and @ref power_on_callback.
* Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature.
*
- * @return MALI_ERROR_NONE on success, else mali_error erro code.
+ * @return 0 on success, else int erro code.
*/
- mali_error(*power_runtime_init_callback) (struct kbase_device *kbdev);
+ int (*power_runtime_init_callback)(struct kbase_device *kbdev);
/** Callback for handling runtime power management termination.
*
* Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature.
*/
int (*power_runtime_on_callback)(struct kbase_device *kbdev);
-
-} kbase_pm_callback_conf;
+};
/**
- * @brief Default implementation of @ref KBASE_CONFIG_ATTR_CPU_SPEED_FUNC.
+ * kbase_cpuprops_get_default_clock_speed - default for CPU_SPEED_FUNC
+ * @clock_speed - see kbase_cpu_clk_speed_func for details on the parameters
*
- * This function sets clock_speed to 100, so will be an underestimate for
- * any real system.
+ * Returns 0 on success, negative error code otherwise.
*
- * See @ref kbase_cpuprops_clock_speed_function for details on the parameters
- * and return value.
+ * Default implementation of CPU_SPEED_FUNC. This function sets clock_speed
+ * to 100, so will be an underestimate for any real system.
*/
int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed);
/**
- * Type of the function pointer for KBASE_CONFIG_ATTR_CPU_SPEED_FUNC.
+ * kbase_cpu_clk_speed_func - Type of the function pointer for CPU_SPEED_FUNC
+ * @param clock_speed - pointer to store the current CPU clock speed in MHz
*
- * @param clock_speed [out] Once called this will contain the current CPU clock speed in MHz.
- * This is mainly used to implement OpenCL's clGetDeviceInfo().
+ * Returns 0 on success, otherwise negative error code.
*
- * @return 0 on success, 1 on error.
+ * This is mainly used to implement OpenCL's clGetDeviceInfo().
*/
-typedef int (*kbase_cpuprops_clock_speed_function) (u32 *clock_speed);
+typedef int (*kbase_cpu_clk_speed_func) (u32 *clock_speed);
/**
- * Type of the function pointer for KBASE_CONFIG_ATTR_GPU_SPEED_FUNC.
+ * kbase_gpu_clk_speed_func - Type of the function pointer for GPU_SPEED_FUNC
+ * @param clock_speed - pointer to store the current GPU clock speed in MHz
+ *
+ * Returns 0 on success, otherwise negative error code.
+ * When an error is returned the caller assumes maximum GPU speed stored in
+ * gpu_freq_khz_max.
*
- * @param clock_speed [out] Once called this will contain the current GPU clock speed in MHz.
- * If the system timer is not available then this function is required
- * for the OpenCL queue profiling to return correct timing information.
+ * If the system timer is not available then this function is required
+ * for the OpenCL queue profiling to return correct timing information.
*
- * @return 0 on success, 1 on error. When an error is returned the caller assumes maximum
- * GPU speed stored in gpu_freq_khz_max.
*/
-typedef int (*kbase_gpuprops_clock_speed_function) (u32 *clock_speed);
+typedef int (*kbase_gpu_clk_speed_func) (u32 *clock_speed);
#ifdef CONFIG_OF
-typedef struct kbase_platform_config {
- const struct kbase_attribute *attributes;
- u32 midgard_type;
-} kbase_platform_config;
+struct kbase_platform_config {
+};
#else
/*
* @brief Specifies start and end of I/O memory region.
*/
-typedef struct kbase_io_memory_region {
+struct kbase_io_memory_region {
u64 start;
u64 end;
-} kbase_io_memory_region;
+};
/*
* @brief Specifies I/O related resources like IRQs and memory region for I/O operations.
*/
-typedef struct kbase_io_resources {
-
+struct kbase_io_resources {
u32 job_irq_number;
u32 mmu_irq_number;
u32 gpu_irq_number;
struct kbase_io_memory_region io_memory_region;
-} kbase_io_resources;
+};
-typedef struct kbase_platform_config {
- const struct kbase_attribute *attributes;
+struct kbase_platform_config {
const struct kbase_io_resources *io_resources;
- u32 midgard_type;
-} kbase_platform_config;
+};
#endif /* CONFIG_OF */
/**
- * @brief Return character string associated with the given midgard type.
- *
- * @param[in] midgard_type - ID of midgard type
- *
- * @return Pointer to NULL-terminated character array associated with the given midgard type
- */
-const char *kbasep_midgard_type_to_string(u32 midgard_type);
-
-/**
- * @brief Gets the next config attribute with the specified ID from the array of attributes.
- *
- * Function gets the next attribute with specified attribute id within specified array. If no such attribute is found,
- * NULL is returned.
- *
- * @param[in] attributes Array of attributes in which lookup is performed
- * @param[in] attribute_id ID of attribute
+ * @brief Gets the pointer to platform config.
*
- * @return Pointer to the first attribute matching id or NULL if none is found.
+ * @return Pointer to the platform config
*/
-const struct kbase_attribute *kbasep_get_next_attribute(const struct kbase_attribute *attributes, int attribute_id);
+struct kbase_platform_config *kbase_get_platform_config(void);
/**
- * @brief Gets the value of a single config attribute.
+ * kbasep_platform_device_init: - Platform specific call to initialize hardware
+ * @kbdev: kbase device pointer
*
- * Function gets the value of attribute specified as parameter. If no such attribute is found in the array of
- * attributes, default value is used.
+ * Function calls a platform defined routine if specified in the configuration
+ * attributes. The routine can initialize any hardware and context state that
+ * is required for the GPU block to function.
*
- * @param[in] kbdev Kbase device pointer
- * @param[in] attributes Array of attributes in which lookup is performed
- * @param[in] attribute_id ID of attribute
- *
- * @return Value of attribute with the given id
+ * Return: 0 if no errors have been found in the config.
+ * Negative error code otherwise.
*/
-uintptr_t kbasep_get_config_value(struct kbase_device *kbdev, const struct kbase_attribute *attributes, int attribute_id);
+int kbasep_platform_device_init(struct kbase_device *kbdev);
/**
- * @brief Validates configuration attributes
- *
- * Function checks validity of given configuration attributes. It will fail on any attribute with unknown id, attribute
- * with invalid value or attribute list that is not correctly terminated.
+ * kbasep_platform_device_term - Platform specific call to terminate hardware
+ * @kbdev: Kbase device pointer
*
- * @param[in] kbdev Kbase device pointer
- * @param[in] attributes Array of attributes to validate
+ * Function calls a platform defined routine if specified in the configuration
+ * attributes. The routine can destroy any platform specific context state and
+ * shut down any hardware functionality that are outside of the Power Management
+ * callbacks.
*
- * @return MALI_TRUE if no errors have been found in the config. MALI_FALSE otherwise.
*/
-mali_bool kbasep_validate_configuration_attributes(struct kbase_device *kbdev, const struct kbase_attribute *attributes);
+void kbasep_platform_device_term(struct kbase_device *kbdev);
-/**
- * @brief Gets the pointer to platform config.
- *
- * @return Pointer to the platform config
- */
-struct kbase_platform_config *kbase_get_platform_config(void);
/**
- * @brief Gets the count of attributes in array
- *
- * Function gets the count of attributes in array. Note that end of list indicator is also included.
+ * kbase_platform_early_init - Early initialisation of the platform code
*
- * @param[in] attributes Array of attributes
+ * This function will be called when the module is loaded to perform any
+ * early initialisation required by the platform code. Such as reading
+ * platform specific device tree entries for the GPU.
*
- * @return Number of attributes in the array including end of list indicator.
+ * Return: 0 for success, any other fail causes module initialisation to fail
*/
-int kbasep_get_config_attribute_count(const struct kbase_attribute *attributes);
+int kbase_platform_early_init(void);
+#ifndef CONFIG_OF
+#ifdef CONFIG_MALI_PLATFORM_FAKE
/**
- * @brief Platform specific call to initialize hardware
- *
- * Function calls a platform defined routine if specified in the configuration attributes.
- * The routine can initialize any hardware and context state that is required for the GPU block to function.
+ * kbase_platform_fake_register - Register a platform device for the GPU
*
- * @param[in] kbdev Kbase device pointer
+ * This can be used to register a platform device on systems where device tree
+ * is not enabled and the platform initialisation code in the kernel doesn't
+ * create the GPU device. Where possible device tree should be used instead.
*
- * @return MALI_TRUE if no errors have been found in the config. MALI_FALSE otherwise.
+ * Return: 0 for success, any other fail causes module initialisation to fail
*/
-mali_bool kbasep_platform_device_init(struct kbase_device *kbdev);
+int kbase_platform_fake_register(void);
/**
- * @brief Platform specific call to terminate hardware
- *
- * Function calls a platform defined routine if specified in the configuration attributes.
- * The routine can destroy any platform specific context state and shut down any hardware functionality that are
- * outside of the Power Management callbacks.
- *
- * @param[in] kbdev Kbase device pointer
+ * kbase_platform_fake_unregister - Unregister a fake platform device
*
+ * Unregister the platform device created with kbase_platform_fake_register()
*/
-void kbasep_platform_device_term(struct kbase_device *kbdev);
+void kbase_platform_fake_unregister(void);
+#endif
+#endif
/** @} *//* end group kbase_config */
/** @} *//* end group base_kbase_api */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define _KBASE_CONFIG_DEFAULTS_H_
/* Include mandatory definitions per platform */
+#include <mali_kbase_config_platform.h>
/**
* Irq throttle. It is the minimum desired time in between two
* Attached value: number in micro seconds
*/
#define DEFAULT_IRQ_THROTTLE_TIME_US 20
-#define GPU_FREQ_KHZ_MAX 500000
-#define GPU_FREQ_KHZ_MIN 100000
-/*** Begin Scheduling defaults ***/
-
-/**
- * Default scheduling tick granuality, in nanoseconds
- */
-/* 50ms */
-#define DEFAULT_JS_SCHEDULING_TICK_NS 50000000u
-
-/**
- * Default minimum number of scheduling ticks before jobs are soft-stopped.
- *
- * This defines the time-slice for a job (which may be different from that of
- * a context)
- */
-/* Between 0.1 and 0.15s before soft-stop */
-#define DEFAULT_JS_SOFT_STOP_TICKS 2
-
-/**
- * Default minimum number of scheduling ticks before CL jobs are soft-stopped.
- */
-/* Between 0.05 and 0.1s before soft-stop */
-#define DEFAULT_JS_SOFT_STOP_TICKS_CL 1
-
-/**
- * Default minimum number of scheduling ticks before jobs are hard-stopped
- */
-/* 1.2s before hard-stop, for a certain GLES2 test at 128x128 (bound by
- * combined vertex+tiler job)
- */
-#define DEFAULT_JS_HARD_STOP_TICKS_SS_HW_ISSUE_8408 24
-/* Between 0.2 and 0.25s before hard-stop */
-#define DEFAULT_JS_HARD_STOP_TICKS_SS 4
-
-/**
- * Default minimum number of scheduling ticks before CL jobs are hard-stopped.
- */
-/* Between 0.1 and 0.15s before hard-stop */
-#define DEFAULT_JS_HARD_STOP_TICKS_CL 2
-
-/**
- * Default minimum number of scheduling ticks before jobs are hard-stopped
- * during dumping
- */
-/* 60s @ 50ms tick */
-#define DEFAULT_JS_HARD_STOP_TICKS_NSS 1200
-
-/**
- * Default minimum number of scheduling ticks before the GPU is reset
- * to clear a "stuck" job
- */
-/* 1.8s before resetting GPU, for a certain GLES2 test at 128x128 (bound by
- * combined vertex+tiler job)
- */
-#define DEFAULT_JS_RESET_TICKS_SS_HW_ISSUE_8408 36
-/* 0.3-0.35s before GPU is reset */
-#define DEFAULT_JS_RESET_TICKS_SS 6
-
-/**
- * Default minimum number of scheduling ticks before the GPU is reset
- * to clear a "stuck" CL job.
- */
-/* 0.2-0.25s before GPU is reset */
-#define DEFAULT_JS_RESET_TICKS_CL 4
-
-/**
- * Default minimum number of scheduling ticks before the GPU is reset
- * to clear a "stuck" job during dumping.
- */
-/* 60.1s @ 100ms tick */
-#define DEFAULT_JS_RESET_TICKS_NSS 1202
-
-/**
- * Number of milliseconds given for other jobs on the GPU to be
- * soft-stopped when the GPU needs to be reset.
- */
-#define DEFAULT_JS_RESET_TIMEOUT_MS 3000
-
-/**
- * Default timeslice that a context is scheduled in for, in nanoseconds.
- *
- * When a context has used up this amount of time across its jobs, it is
- * scheduled out to let another run.
- *
- * @note the resolution is nanoseconds (ns) here, because that's the format
- * often used by the OS.
- */
-/* 0.05s - at 20fps a ctx does at least 1 frame before being scheduled out.
- * At 40fps, 2 frames, etc
- */
-#define DEFAULT_JS_CTX_TIMESLICE_NS 50000000
/**
* Default Job Scheduler initial runtime of a context for the CFS Policy,
* On r0p0-15dev0 HW and earlier, there are tradeoffs between security and
* performance:
*
-* - When this is set to MALI_TRUE, the driver remains fully secure,
+* - When this is set to true, the driver remains fully secure,
* but potentially loses performance compared with setting this to
-* MALI_FALSE.
-* - When set to MALI_FALSE, the driver is open to certain security
+* false.
+* - When set to false, the driver is open to certain security
* attacks.
*
* From r0p0-00rel0 and onwards, there is no security loss by setting
-* this to MALI_FALSE, and no performance loss by setting it to
-* MALI_TRUE.
+* this to false, and no performance loss by setting it to
+* true.
*/
-#define DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE MALI_FALSE
+#define DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE false
enum {
/**
/**
* Default setting for using alternative hardware counters.
*/
-#define DEFAULT_ALTERNATIVE_HWC MALI_FALSE
+#define DEFAULT_ALTERNATIVE_HWC false
-/*** End Scheduling defaults ***/
+/**
+ * Default UMP device mapping. A UMP_DEVICE_<device>_SHIFT value which
+ * defines which UMP device this GPU should be mapped to.
+ */
+#define DEFAULT_UMP_GPU_DEVICE_SHIFT UMP_DEVICE_Z_SHIFT
-/*** Begin Power Manager defaults */
+/*
+ * Default period for DVFS sampling
+ */
+#define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */
-/* Milliseconds */
-#define DEFAULT_PM_DVFS_FREQ 20 /* 动态调频的周期, 为提高 UI 性能, 按照 4.4 的经验值减小. */
-// #define DEFAULT_PM_DVFS_FREQ 500
+/*
+ * Power Management poweroff tick granuality. This is in nanoseconds to
+ * allow HR timer support.
+ *
+ * On each scheduling tick, the power manager core may decide to:
+ * -# Power off one or more shader cores
+ * -# Power off the entire GPU
+ */
+#define DEFAULT_PM_GPU_POWEROFF_TICK_NS (400000) /* 400us */
-/**
- * Default poweroff tick granuality, in nanoseconds
+/*
+ * Power Manager number of ticks before shader cores are powered off
*/
-/* 400us */
-#define DEFAULT_PM_GPU_POWEROFF_TICK_NS 400000
+#define DEFAULT_PM_POWEROFF_TICK_SHADER (2) /* 400-800us */
-/**
- * Default number of poweroff ticks before shader cores are powered off
+/*
+ * Power Manager number of ticks before GPU is powered off
*/
-/* 400-800us */
-#define DEFAULT_PM_POWEROFF_TICK_SHADER 2
+#define DEFAULT_PM_POWEROFF_TICK_GPU (2) /* 400-800us */
-/**
- * Default number of poweroff ticks before GPU is powered off
+/*
+ * Default scheduling tick granuality
*/
-#define DEFAULT_PM_POWEROFF_TICK_GPU 2 /* 400-800us */
+#define DEFAULT_JS_SCHEDULING_PERIOD_NS (100000000u) /* 100ms */
-/*** End Power Manager defaults ***/
+/*
+ * Default minimum number of scheduling ticks before jobs are soft-stopped.
+ *
+ * This defines the time-slice for a job (which may be different from that of a
+ * context)
+ */
+#define DEFAULT_JS_SOFT_STOP_TICKS (1) /* 100ms-200ms */
+/*
+ * Default minimum number of scheduling ticks before CL jobs are soft-stopped.
+ */
+#define DEFAULT_JS_SOFT_STOP_TICKS_CL (1) /* 100ms-200ms */
-/**
- * Default UMP device mapping. A UMP_DEVICE_<device>_SHIFT value which
- * defines which UMP device this GPU should be mapped to.
+/*
+ * Default minimum number of scheduling ticks before jobs are hard-stopped
*/
-#define DEFAULT_UMP_GPU_DEVICE_SHIFT UMP_DEVICE_Z_SHIFT
+#define DEFAULT_JS_HARD_STOP_TICKS_SS (50) /* 5s */
+#define DEFAULT_JS_HARD_STOP_TICKS_SS_8408 (300) /* 30s */
-/**
- * Default value for KBASE_CONFIG_ATTR_CPU_SPEED_FUNC.
- * Points to @ref kbase_cpuprops_get_default_clock_speed.
+/*
+ * Default minimum number of scheduling ticks before CL jobs are hard-stopped.
+ */
+#define DEFAULT_JS_HARD_STOP_TICKS_CL (50) /* 5s */
+
+/*
+ * Default minimum number of scheduling ticks before jobs are hard-stopped
+ * during dumping
+ */
+#define DEFAULT_JS_HARD_STOP_TICKS_DUMPING (15000) /* 1500s */
+
+/*
+ * Default minimum number of scheduling ticks before the GPU is reset to clear a
+ * "stuck" job
+ */
+#define DEFAULT_JS_RESET_TICKS_SS (55) /* 5.5s */
+#define DEFAULT_JS_RESET_TICKS_SS_8408 (450) /* 45s */
+
+/*
+ * Default minimum number of scheduling ticks before the GPU is reset to clear a
+ * "stuck" CL job.
+ */
+#define DEFAULT_JS_RESET_TICKS_CL (55) /* 5.5s */
+
+/*
+ * Default minimum number of scheduling ticks before the GPU is reset to clear a
+ * "stuck" job during dumping.
+ */
+#define DEFAULT_JS_RESET_TICKS_DUMPING (15020) /* 1502s */
+
+/*
+ * Default number of milliseconds given for other jobs on the GPU to be
+ * soft-stopped when the GPU needs to be reset.
+ */
+#define DEFAULT_RESET_TIMEOUT_MS (3000) /* 3s */
+
+/*
+ * Default timeslice that a context is scheduled in for, in nanoseconds.
+ *
+ * When a context has used up this amount of time across its jobs, it is
+ * scheduled out to let another run.
+ *
+ * @note the resolution is nanoseconds (ns) here, because that's the format
+ * often used by the OS.
*/
-#define DEFAULT_CPU_SPEED_FUNC \
- ((uintptr_t)kbase_cpuprops_get_default_clock_speed)
+#define DEFAULT_JS_CTX_TIMESLICE_NS (50000000) /* 50ms */
#endif /* _KBASE_CONFIG_DEFAULTS_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file mali_kbase_context.c
+/*
* Base kernel context APIs
*/
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
+#include <mali_kbase_instr.h>
#define MEMPOOL_PAGES 16384
/**
- * @brief Create a kernel base context.
+ * kbase_create_context() - Create a kernel base context.
+ * @kbdev: Kbase device
+ * @is_compat: Force creation of a 32-bit context
*
* Allocate and init a kernel base context.
+ *
+ * Return: new kbase context
*/
struct kbase_context *
kbase_create_context(struct kbase_device *kbdev, bool is_compat)
{
struct kbase_context *kctx;
- mali_error mali_err;
+ int mali_err;
KBASE_DEBUG_ASSERT(kbdev != NULL);
#endif
atomic_set(&kctx->setup_complete, 0);
atomic_set(&kctx->setup_in_progress, 0);
- kctx->keep_gpu_powered = MALI_FALSE;
+ kctx->infinite_cache_active = 0;
spin_lock_init(&kctx->mm_update_lock);
kctx->process_mm = NULL;
atomic_set(&kctx->nonmapped_pages, 0);
+ kctx->slots_pullable = 0;
- if (MALI_ERROR_NONE != kbase_mem_allocator_init(&kctx->osalloc,
- MEMPOOL_PAGES,
- kctx->kbdev))
+ if (kbase_mem_allocator_init(&kctx->osalloc, MEMPOOL_PAGES, kctx->kbdev) != 0)
goto free_kctx;
kctx->pgd_allocator = &kctx->osalloc;
goto free_allocator;
mali_err = kbasep_js_kctx_init(kctx);
- if (MALI_ERROR_NONE != mali_err)
+ if (mali_err)
goto free_jd; /* safe to call kbasep_js_kctx_term in this case */
mali_err = kbase_event_init(kctx);
- if (MALI_ERROR_NONE != mali_err)
+ if (mali_err)
goto free_jd;
mutex_init(&kctx->reg_lock);
#endif
mali_err = kbase_mmu_init(kctx);
- if (MALI_ERROR_NONE != mali_err)
+ if (mali_err)
goto free_event;
kctx->pgd = kbase_mmu_alloc_pgd(kctx);
if (!kctx->pgd)
goto free_mmu;
- if (MALI_ERROR_NONE != kbase_mem_allocator_alloc(&kctx->osalloc, 1, &kctx->aliasing_sink_page))
+ if (kbase_mem_allocator_alloc(&kctx->osalloc, 1, &kctx->aliasing_sink_page) != 0)
goto no_sink_page;
kctx->tgid = current->tgid;
- kctx->pid = current->pid;
+ kctx->pid = current->pid;
init_waitqueue_head(&kctx->event_queue);
kctx->cookies = KBASE_COOKIE_MASK;
kctx->id = atomic_add_return(1, &(kbdev->ctx_num)) - 1;
- mali_err = kbasep_mem_profile_debugfs_add(kctx);
- if (MALI_ERROR_NONE != mali_err)
- goto no_region_tracker;
-
- if (kbasep_jd_debugfs_ctx_add(kctx))
- goto free_mem_profile;
+ mutex_init(&kctx->vinstr_cli_lock);
return kctx;
-free_mem_profile:
- kbasep_mem_profile_debugfs_remove(kctx);
no_region_tracker:
no_sink_page:
kbase_mem_allocator_free(&kctx->osalloc, 1, &kctx->aliasing_sink_page, 0);
+ /* VM lock needed for the call to kbase_mmu_free_pgd */
+ kbase_gpu_vm_lock(kctx);
kbase_mmu_free_pgd(kctx);
+ kbase_gpu_vm_unlock(kctx);
free_mmu:
kbase_mmu_term(kctx);
free_event:
vfree(kctx);
out:
return NULL;
-
}
-KBASE_EXPORT_SYMBOL(kbase_create_context)
+KBASE_EXPORT_SYMBOL(kbase_create_context);
static void kbase_reg_pending_dtor(struct kbase_va_region *reg)
{
dev_dbg(reg->kctx->kbdev->dev, "Freeing pending unmapped region\n");
- kbase_mem_phy_alloc_put(reg->alloc);
+ kbase_mem_phy_alloc_put(reg->cpu_alloc);
+ kbase_mem_phy_alloc_put(reg->gpu_alloc);
kfree(reg);
}
/**
- * @brief Destroy a kernel base context.
+ * kbase_destroy_context - Destroy a kernel base context.
+ * @kctx: Context to destroy
*
- * Destroy a kernel base context. Calls kbase_destroy_os_context() to
- * free OS specific structures. Will release all outstanding regions.
+ * Calls kbase_destroy_os_context() to free OS specific structures.
+ * Will release all outstanding regions.
*/
void kbase_destroy_context(struct kbase_context *kctx)
{
KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u);
- kbasep_jd_debugfs_ctx_remove(kctx);
-
- kbasep_mem_profile_debugfs_remove(kctx);
-
/* Ensure the core is powered up for the destroy process */
/* A suspend won't happen here, because we're in a syscall from a userspace
* thread. */
kbase_pm_context_active(kbdev);
- if (kbdev->hwcnt.kctx == kctx) {
- /* disable the use of the hw counters if the app didn't use the API correctly or crashed */
- KBASE_TRACE_ADD(kbdev, CORE_CTX_HWINSTR_TERM, kctx, NULL, 0u, 0u);
- dev_warn(kbdev->dev, "The privileged process asking for instrumentation forgot to disable it " "before exiting. Will end instrumentation for them");
- kbase_instr_hwcnt_disable(kctx);
- }
-
kbase_jd_zap_context(kctx);
kbase_event_cleanup(kctx);
pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK;
while (pending_regions_to_clean) {
unsigned int cookie = __ffs(pending_regions_to_clean);
+
BUG_ON(!kctx->pending_regions[cookie]);
kbase_reg_pending_dtor(kctx->pending_regions[cookie]);
if (pages != 0)
dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages);
- if (kctx->keep_gpu_powered) {
- atomic_dec(&kbdev->keep_gpu_powered_count);
- kbase_pm_context_idle(kbdev);
- }
-
kbase_mem_allocator_term(&kctx->osalloc);
WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0);
vfree(kctx);
}
-KBASE_EXPORT_SYMBOL(kbase_destroy_context)
+KBASE_EXPORT_SYMBOL(kbase_destroy_context);
/**
- * Set creation flags on a context
+ * kbase_context_set_create_flags - Set creation flags on a context
+ * @kctx: Kbase context
+ * @flags: Flags to set
+ *
+ * Return: 0 on success
*/
-mali_error kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags)
+int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags)
{
- mali_error err = MALI_ERROR_NONE;
+ int err = 0;
struct kbasep_js_kctx_info *js_kctx_info;
+ unsigned long irq_flags;
+
KBASE_DEBUG_ASSERT(NULL != kctx);
js_kctx_info = &kctx->jctx.sched_info;
/* Validate flags */
if (flags != (flags & BASE_CONTEXT_CREATE_KERNEL_FLAGS)) {
- err = MALI_ERROR_FUNCTION_FAILED;
+ err = -EINVAL;
goto out;
}
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
+ spin_lock_irqsave(&kctx->kbdev->js_data.runpool_irq.lock, irq_flags);
/* Translate the flags */
if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0)
/* Latch the initial attributes into the Job Scheduler */
kbasep_js_ctx_attr_set_initial_attrs(kctx->kbdev, kctx);
+ spin_unlock_irqrestore(&kctx->kbdev->js_data.runpool_irq.lock,
+ irq_flags);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
out:
return err;
}
-KBASE_EXPORT_SYMBOL(kbase_context_set_create_flags)
+KBASE_EXPORT_SYMBOL(kbase_context_set_create_flags);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-
-/**
- * @file mali_kbase_core_linux.c
- * Base kernel driver init.
- */
-
#include <mali_kbase.h>
+#include <mali_kbase_hwaccess_gpuprops.h>
#include <mali_kbase_config_defaults.h>
#include <mali_kbase_uku.h>
#include <mali_midg_regmap.h>
+#include <mali_kbase_instr.h>
#include <mali_kbase_gator.h>
+#include <backend/gpu/mali_kbase_js_affinity.h>
#include <mali_kbase_mem_linux.h>
#ifdef CONFIG_MALI_DEVFREQ
-#include "mali_kbase_devfreq.h"
+#include <backend/gpu/mali_kbase_devfreq.h>
#endif /* CONFIG_MALI_DEVFREQ */
#include <mali_kbase_cpuprops.h>
#ifdef CONFIG_MALI_NO_MALI
#include "mali_kbase_model_linux.h"
#endif /* CONFIG_MALI_NO_MALI */
#include "mali_kbase_mem_profile_debugfs_buf_size.h"
+#include "mali_kbase_debug_mem_view.h"
+#include <mali_kbase_hwaccess_backend.h>
#ifdef CONFIG_KDS
#include <linux/kds.h>
#include <linux/mm.h>
#include <linux/compat.h> /* is_compat_task */
#include <linux/version.h>
+#ifdef CONFIG_MALI_PLATFORM_DEVICETREE
+#include <linux/pm_runtime.h>
+#endif /* CONFIG_MALI_PLATFORM_DEVICETREE */
#include <mali_kbase_hw.h>
#include <platform/mali_kbase_platform_common.h>
+#ifdef CONFIG_MALI_PLATFORM_FAKE
+#include <platform/mali_kbase_platform_fake.h>
+#endif /*CONFIG_MALI_PLATFORM_FAKE */
#ifdef CONFIG_SYNC
#include <mali_kbase_sync.h>
#endif /* CONFIG_SYNC */
#endif /* CONFIG_PM_DEVFREQ */
#include <linux/clk.h>
-/*
- * This file is included since when we support device tree we don't
- * use the platform fake code for registering the kbase config attributes.
- */
-#ifdef CONFIG_OF
#include <mali_kbase_config.h>
-#endif
#ifdef CONFIG_MACH_MANTA
#include <plat/devs.h>
#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0))
+#include <linux/pm_opp.h>
+#else
+#include <linux/opp.h>
+#endif
+
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+#include <mali_kbase_tlstream.h>
+#endif
+
/* GPU IRQ Tags */
#define JOB_IRQ_TAG 0
#define MMU_IRQ_TAG 1
#define GPU_IRQ_TAG 2
-
-struct kbase_irq_table {
- u32 tag;
- irq_handler_t handler;
-};
#if MALI_UNIT_TEST
static struct kbase_exported_test_data shared_kernel_test_data;
EXPORT_SYMBOL(shared_kernel_test_data);
static int kbase_dev_nr;
-static DEFINE_SEMAPHORE(kbase_dev_list_lock);
+static DEFINE_MUTEX(kbase_dev_list_lock);
static LIST_HEAD(kbase_dev_list);
-KBASE_EXPORT_TEST_API(kbase_dev_list_lock)
-KBASE_EXPORT_TEST_API(kbase_dev_list)
#define KERNEL_SIDE_DDK_VERSION_STRING "K:" MALI_RELEASE_NAME "(GPL)"
-static INLINE void __compile_time_asserts(void)
+static inline void __compile_time_asserts(void)
{
CSTD_COMPILE_TIME_ASSERT(sizeof(KERNEL_SIDE_DDK_VERSION_STRING) <= KBASE_GET_VERSION_BUFFER_SIZE);
}
return 0;
}
-static mali_error kbasep_kds_allocate_resource_list_data(struct kbase_context *kctx, struct base_external_resource *ext_res, int num_elems, struct kbase_kds_resource_list_data *resources_list)
+static int kbasep_kds_allocate_resource_list_data(struct kbase_context *kctx, struct base_external_resource *ext_res, int num_elems, struct kbase_kds_resource_list_data *resources_list)
{
struct base_external_resource *res = ext_res;
int res_id;
sizeof(struct kds_resource *), GFP_KERNEL);
if (NULL == resources_list->kds_resources)
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
KBASE_DEBUG_ASSERT(0 != num_elems);
resources_list->kds_access_bitmap = kzalloc(
if (NULL == resources_list->kds_access_bitmap) {
kfree(resources_list->kds_access_bitmap);
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
}
kbase_gpu_vm_lock(kctx);
/* no need to check reg->alloc as only regions with an alloc has
* a size, and kbase_region_tracker_find_region_enclosing_address
* only returns regions with size > 0 */
- switch (reg->alloc->type) {
+ switch (reg->gpu_alloc->type) {
#if defined(CONFIG_UMP) && defined(CONFIG_KDS)
case KBASE_MEM_TYPE_IMPORTED_UMP:
- kds_res = ump_dd_kds_resource_get(reg->alloc->imported.ump_handle);
+ kds_res = ump_dd_kds_resource_get(reg->gpu_alloc->imported.ump_handle);
break;
#endif /* defined(CONFIG_UMP) && defined(CONFIG_KDS) */
default:
/* did the loop run to completion? */
if (res_id == num_elems)
- return MALI_ERROR_NONE;
+ return 0;
/* Clean up as the resource list is not valid. */
kfree(resources_list->kds_resources);
kfree(resources_list->kds_access_bitmap);
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
-static mali_bool kbasep_validate_kbase_pointer(
+static bool kbasep_validate_kbase_pointer(
struct kbase_context *kctx, union kbase_pointer *p)
{
if (kctx->is_compat) {
if (p->compat_value == 0)
- return MALI_FALSE;
+ return false;
} else {
if (NULL == p->value)
- return MALI_FALSE;
+ return false;
}
- return MALI_TRUE;
+ return true;
}
-static mali_error kbase_external_buffer_lock(struct kbase_context *kctx, struct kbase_uk_ext_buff_kds_data *args, u32 args_size)
+static int kbase_external_buffer_lock(struct kbase_context *kctx,
+ struct kbase_uk_ext_buff_kds_data *args, u32 args_size)
{
struct base_external_resource *ext_res_copy;
size_t ext_resource_size;
- mali_error return_error = MALI_ERROR_FUNCTION_FAILED;
- int fd;
+ int ret = -EINVAL;
+ int fd = -EBADF;
+ struct base_external_resource __user *ext_res_user;
+ int __user *file_desc_usr;
+ struct kbasep_kds_resource_set_file_data *fdata;
+ struct kbase_kds_resource_list_data resource_list_data;
if (args_size != sizeof(struct kbase_uk_ext_buff_kds_data))
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
/* Check user space has provided valid data */
if (!kbasep_validate_kbase_pointer(kctx, &args->external_resource) ||
!kbasep_validate_kbase_pointer(kctx, &args->file_descriptor) ||
(0 == args->num_res) ||
(args->num_res > KBASE_MAXIMUM_EXT_RESOURCES))
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
ext_resource_size = sizeof(struct base_external_resource) * args->num_res;
KBASE_DEBUG_ASSERT(0 != ext_resource_size);
ext_res_copy = kmalloc(ext_resource_size, GFP_KERNEL);
- if (NULL != ext_res_copy) {
- struct base_external_resource __user *ext_res_user;
- int __user *file_descriptor_user;
+ if (!ext_res_copy)
+ return -EINVAL;
#ifdef CONFIG_COMPAT
- if (kctx->is_compat) {
- ext_res_user = compat_ptr(args->external_resource.compat_value);
- file_descriptor_user = compat_ptr(args->file_descriptor.compat_value);
- } else {
+ if (kctx->is_compat) {
+ ext_res_user = compat_ptr(args->external_resource.compat_value);
+ file_desc_usr = compat_ptr(args->file_descriptor.compat_value);
+ } else {
#endif /* CONFIG_COMPAT */
- ext_res_user = args->external_resource.value;
- file_descriptor_user = args->file_descriptor.value;
+ ext_res_user = args->external_resource.value;
+ file_desc_usr = args->file_descriptor.value;
#ifdef CONFIG_COMPAT
- }
+ }
#endif /* CONFIG_COMPAT */
- /* Copy the external resources to lock from user space */
- if (0 == copy_from_user(ext_res_copy, ext_res_user, ext_resource_size)) {
- struct kbasep_kds_resource_set_file_data *fdata;
-
- /* Allocate data to be stored in the file */
- fdata = kmalloc(sizeof(*fdata), GFP_KERNEL);
+ /* Copy the external resources to lock from user space */
+ if (copy_from_user(ext_res_copy, ext_res_user, ext_resource_size))
+ goto out;
- if (NULL != fdata) {
- struct kbase_kds_resource_list_data resource_list_data;
- /* Parse given elements and create resource and access lists */
- return_error = kbasep_kds_allocate_resource_list_data(kctx, ext_res_copy, args->num_res, &resource_list_data);
- if (MALI_ERROR_NONE == return_error) {
- long err;
+ /* Allocate data to be stored in the file */
+ fdata = kmalloc(sizeof(*fdata), GFP_KERNEL);
- fdata->lock = NULL;
+ if (!fdata) {
+ ret = -ENOMEM;
+ goto out;
+ }
- fd = anon_inode_getfd("kds_ext", &kds_resource_fops, fdata, 0);
+ /* Parse given elements and create resource and access lists */
+ ret = kbasep_kds_allocate_resource_list_data(kctx,
+ ext_res_copy, args->num_res, &resource_list_data);
+ if (!ret) {
+ long err;
- err = copy_to_user(file_descriptor_user, &fd, sizeof(fd));
+ fdata->lock = NULL;
- /* If the file descriptor was valid and we successfully copied it to user space, then we
- * can try and lock the requested kds resources.
- */
- if ((fd >= 0) && (0 == err)) {
- struct kds_resource_set *lock;
+ fd = anon_inode_getfd("kds_ext", &kds_resource_fops, fdata, 0);
- lock = kds_waitall(args->num_res, resource_list_data.kds_access_bitmap,
- resource_list_data.kds_resources,
- KDS_WAIT_BLOCKING);
+ err = copy_to_user(file_desc_usr, &fd, sizeof(fd));
- if (IS_ERR_OR_NULL(lock)) {
- return_error = MALI_ERROR_FUNCTION_FAILED;
- } else {
- return_error = MALI_ERROR_NONE;
- fdata->lock = lock;
- }
- } else {
- return_error = MALI_ERROR_FUNCTION_FAILED;
- }
+ /* If the file descriptor was valid and we successfully copied
+ * it to user space, then we can try and lock the requested
+ * kds resources.
+ */
+ if ((fd >= 0) && (0 == err)) {
+ struct kds_resource_set *lock;
- kfree(resource_list_data.kds_resources);
- kfree(resource_list_data.kds_access_bitmap);
- }
+ lock = kds_waitall(args->num_res,
+ resource_list_data.kds_access_bitmap,
+ resource_list_data.kds_resources,
+ KDS_WAIT_BLOCKING);
- if (MALI_ERROR_NONE != return_error) {
- /* If the file was opened successfully then close it which will clean up
- * the file data, otherwise we clean up the file data ourself. */
- if (fd >= 0)
- sys_close(fd);
- else
- kfree(fdata);
- }
+ if (IS_ERR_OR_NULL(lock)) {
+ ret = -EINVAL;
} else {
- return_error = MALI_ERROR_OUT_OF_MEMORY;
+ ret = 0;
+ fdata->lock = lock;
}
+ } else {
+ ret = -EINVAL;
}
- kfree(ext_res_copy);
+
+ kfree(resource_list_data.kds_resources);
+ kfree(resource_list_data.kds_access_bitmap);
}
- return return_error;
+
+ if (ret) {
+ /* If the file was opened successfully then close it which will
+ * clean up the file data, otherwise we clean up the file data
+ * ourself.
+ */
+ if (fd >= 0)
+ sys_close(fd);
+ else
+ kfree(fdata);
+ }
+out:
+ kfree(ext_res_copy);
+
+ return ret;
}
#endif /* CONFIG_KDS */
-static mali_error kbase_dispatch(struct kbase_context *kctx, void * const args, u32 args_size)
+#ifdef CONFIG_MALI_MIPE_ENABLED
+static void kbase_create_timeline_objects(struct kbase_context *kctx)
+{
+ struct kbase_device *kbdev = kctx->kbdev;
+ unsigned int lpu_id;
+ struct kbasep_kctx_list_element *element;
+
+ /* Create LPU objects. */
+ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) {
+ gpu_js_features *lpu =
+ &kbdev->gpu_props.props.raw_props.js_features[lpu_id];
+ kbase_tlstream_tl_summary_new_lpu(lpu, lpu_id, (u32)*lpu);
+ }
+
+ /* Create GPU object and make it retain all LPUs. */
+ kbase_tlstream_tl_summary_new_gpu(
+ kbdev,
+ kbdev->gpu_props.props.raw_props.gpu_id,
+ kbdev->gpu_props.num_cores);
+
+ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) {
+ void *lpu =
+ &kbdev->gpu_props.props.raw_props.js_features[lpu_id];
+ kbase_tlstream_tl_summary_lifelink_lpu_gpu(lpu, kbdev);
+ }
+
+ /* Create object for each known context. */
+ mutex_lock(&kbdev->kctx_list_lock);
+ list_for_each_entry(element, &kbdev->kctx_list, link) {
+ kbase_tlstream_tl_summary_new_ctx(
+ element->kctx,
+ (u32)(element->kctx->id));
+ }
+ /* Before releasing the lock, reset body stream buffers.
+ * This will prevent context creation message to be directed to both
+ * summary and body stream. */
+ kbase_tlstream_reset_body_streams();
+ mutex_unlock(&kbdev->kctx_list_lock);
+ /* Static object are placed into summary packet that needs to be
+ * transmitted first. Flush all streams to make it available to
+ * user space. */
+ kbase_tlstream_flush_streams();
+}
+#endif
+
+static void kbase_api_handshake(struct uku_version_check_args *version)
+{
+ switch (version->major) {
+#ifdef BASE_LEGACY_UK6_SUPPORT
+ case 6:
+ /* We are backwards compatible with version 6,
+ * so pretend to be the old version */
+ version->major = 6;
+ version->minor = 1;
+ break;
+#endif /* BASE_LEGACY_UK6_SUPPORT */
+#ifdef BASE_LEGACY_UK7_SUPPORT
+ case 7:
+ /* We are backwards compatible with version 7,
+ * so pretend to be the old version */
+ version->major = 7;
+ version->minor = 1;
+ break;
+#endif /* BASE_LEGACY_UK7_SUPPORT */
+ case BASE_UK_VERSION_MAJOR:
+ /* set minor to be the lowest common */
+ version->minor = min_t(int, BASE_UK_VERSION_MINOR,
+ (int)version->minor);
+ break;
+ default:
+ /* We return our actual version regardless if it
+ * matches the version returned by userspace -
+ * userspace can bail if it can't handle this
+ * version */
+ version->major = BASE_UK_VERSION_MAJOR;
+ version->minor = BASE_UK_VERSION_MINOR;
+ break;
+ }
+}
+
+/**
+ * enum mali_error - Mali error codes shared with userspace
+ *
+ * This is subset of those common Mali errors that can be returned to userspace.
+ * Values of matching user and kernel space enumerators MUST be the same.
+ * MALI_ERROR_NONE is guaranteed to be 0.
+ */
+enum mali_error {
+ MALI_ERROR_NONE = 0,
+ MALI_ERROR_OUT_OF_GPU_MEMORY,
+ MALI_ERROR_OUT_OF_MEMORY,
+ MALI_ERROR_FUNCTION_FAILED,
+};
+
+static int kbase_dispatch(struct kbase_context *kctx, void * const args, u32 args_size)
{
struct kbase_device *kbdev;
union uk_header *ukh = args;
kbdev = kctx->kbdev;
id = ukh->id;
- ukh->ret = MALI_ERROR_NONE; /* Be optimistic */
+ ukh->ret = MALI_ERROR_NONE; /* Be optimistic */
if (UKP_FUNC_ID_CHECK_VERSION == id) {
- if (args_size == sizeof(struct uku_version_check_args)) {
- struct uku_version_check_args *version_check = (struct uku_version_check_args *)args;
-
- switch (version_check->major) {
-#ifdef BASE_LEGACY_UK6_SUPPORT
- case 6:
- /* We are backwards compatible with version 6,
- * so pretend to be the old version */
- version_check->major = 6;
- version_check->minor = 1;
- break;
-#endif /* BASE_LEGACY_UK6_SUPPORT */
-#ifdef BASE_LEGACY_UK7_SUPPORT
- case 7:
- /* We are backwards compatible with version 7,
- * so pretend to be the old version */
- version_check->major = 7;
- version_check->minor = 1;
- break;
-#endif /* BASE_LEGACY_UK7_SUPPORT */
- default:
- /* We return our actual version regardless if it
- * matches the version returned by userspace -
- * userspace can bail if it can't handle this
- * version */
- version_check->major = BASE_UK_VERSION_MAJOR;
- version_check->minor = BASE_UK_VERSION_MINOR;
- }
+ struct uku_version_check_args *version_check;
- ukh->ret = MALI_ERROR_NONE;
- } else {
+ if (args_size != sizeof(struct uku_version_check_args)) {
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ return 0;
}
- return MALI_ERROR_NONE;
+ version_check = (struct uku_version_check_args *)args;
+ kbase_api_handshake(version_check);
+ /* save the proposed version number for later use */
+ kctx->api_version = KBASE_API_VERSION(version_check->major,
+ version_check->minor);
+ ukh->ret = MALI_ERROR_NONE;
+ return 0;
}
+ /* block calls until version handshake */
+ if (kctx->api_version == 0)
+ return -EINVAL;
if (!atomic_read(&kctx->setup_complete)) {
- /* setup pending, try to signal that we'll do the setup */
- if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1)) {
- /* setup was already in progress, err this call */
- return MALI_ERROR_FUNCTION_FAILED;
- }
+ struct kbase_uk_set_flags *kbase_set_flags;
- /* we're the one doing setup */
+ /* setup pending, try to signal that we'll do the setup,
+ * if setup was already in progress, err this call
+ */
+ if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1))
+ return -EINVAL;
- /* is it the only call we accept? */
- if (id == KBASE_FUNC_SET_FLAGS) {
- struct kbase_uk_set_flags *kbase_set_flags = (struct kbase_uk_set_flags *)args;
+ /* if unexpected call, will stay stuck in setup mode
+ * (is it the only call we accept?)
+ */
+ if (id != KBASE_FUNC_SET_FLAGS)
+ return -EINVAL;
- if (sizeof(*kbase_set_flags) != args_size) {
- /* not matching the expected call, stay stuck in setup mode */
- goto bad_size;
- }
+ kbase_set_flags = (struct kbase_uk_set_flags *)args;
- if (MALI_ERROR_NONE != kbase_context_set_create_flags(kctx, kbase_set_flags->create_flags)) {
- ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- /* bad flags, will stay stuck in setup mode */
- return MALI_ERROR_NONE;
- } else {
- /* we've done the setup, all OK */
- atomic_set(&kctx->setup_complete, 1);
- return MALI_ERROR_NONE;
- }
- } else {
- /* unexpected call, will stay stuck in setup mode */
- return MALI_ERROR_FUNCTION_FAILED;
- }
+ /* if not matching the expected call, stay in setup mode */
+ if (sizeof(*kbase_set_flags) != args_size)
+ goto bad_size;
+
+ /* if bad flags, will stay stuck in setup mode */
+ if (kbase_context_set_create_flags(kctx,
+ kbase_set_flags->create_flags) != 0)
+ ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+
+ atomic_set(&kctx->setup_complete, 1);
+ return 0;
}
/* setup complete, perform normal operation */
if (sizeof(*mem) != args_size)
goto bad_size;
- reg = kbase_mem_alloc(kctx, mem->va_pages, mem->commit_pages, mem->extent, &mem->flags, &mem->gpu_va, &mem->va_alignment);
+ reg = kbase_mem_alloc(kctx, mem->va_pages,
+ mem->commit_pages, mem->extent,
+ &mem->flags, &mem->gpu_va,
+ &mem->va_alignment);
if (!reg)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
get_user(handle, phandle);
break;
default:
- goto bad_type;
+ mem_import->type = BASE_MEM_IMPORT_TYPE_INVALID;
break;
}
- if (kbase_mem_import(kctx, mem_import->type, handle, &mem_import->gpu_va, &mem_import->va_pages, &mem_import->flags)) {
-bad_type:
+ if (mem_import->type == BASE_MEM_IMPORT_TYPE_INVALID ||
+ kbase_mem_import(kctx, mem_import->type,
+ handle, &mem_import->gpu_va,
+ &mem_import->va_pages,
+ &mem_import->flags))
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
- }
break;
}
case KBASE_FUNC_MEM_ALIAS: {
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
}
+ if (!alias->nents) {
+ ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ break;
+ }
#ifdef CONFIG_COMPAT
if (kctx->is_compat)
break;
}
- ukh->ret = kbase_mem_query(kctx, query->gpu_addr, query->query, &query->value);
+ if (kbase_mem_query(kctx, query->gpu_addr,
+ query->query, &query->value) != 0)
+ ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ else
+ ukh->ret = MALI_ERROR_NONE;
break;
}
break;
goto bad_size;
#ifdef BASE_LEGACY_UK6_SUPPORT
- if (MALI_ERROR_NONE != kbase_jd_submit(kctx, job, 0))
+ if (kbase_jd_submit(kctx, job, 0) != 0)
#else
- if (MALI_ERROR_NONE != kbase_jd_submit(kctx, job))
+ if (kbase_jd_submit(kctx, job) != 0)
#endif /* BASE_LEGACY_UK6_SUPPORT */
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
if (sizeof(*job) != args_size)
goto bad_size;
- if (MALI_ERROR_NONE != kbase_jd_submit(kctx, job, 1))
+ if (kbase_jd_submit(kctx, job, 1) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
}
break;
}
- if (MALI_ERROR_NONE != kbase_sync_now(kctx, &sn->sset))
+#ifndef CONFIG_MALI_CACHE_COHERENT
+ if (kbase_sync_now(kctx, &sn->sset) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+#endif
break;
}
if (sizeof(*setup) != args_size)
goto bad_size;
- if (MALI_ERROR_NONE != kbase_instr_hwcnt_setup(kctx, setup))
+ mutex_lock(&kctx->vinstr_cli_lock);
+ if (kbase_instr_hwcnt_setup(kctx, setup) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ mutex_unlock(&kctx->vinstr_cli_lock);
break;
}
case KBASE_FUNC_HWCNT_DUMP:
{
/* args ignored */
- if (MALI_ERROR_NONE != kbase_instr_hwcnt_dump(kctx))
+ mutex_lock(&kctx->vinstr_cli_lock);
+ if (kbase_instr_hwcnt_dump(kctx) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ mutex_unlock(&kctx->vinstr_cli_lock);
break;
}
case KBASE_FUNC_HWCNT_CLEAR:
{
/* args ignored */
- if (MALI_ERROR_NONE != kbase_instr_hwcnt_clear(kctx))
+ mutex_lock(&kctx->vinstr_cli_lock);
+ if (kbase_vinstr_clear(kbdev->vinstr_ctx,
+ kctx->vinstr_cli) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ mutex_unlock(&kctx->vinstr_cli_lock);
break;
}
if (sizeof(*setup) != args_size)
goto bad_size;
- if (MALI_ERROR_NONE != kbase_cpuprops_uk_get_props(kctx, setup))
+ if (kbase_cpuprops_uk_get_props(kctx, setup) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
}
if (sizeof(*setup) != args_size)
goto bad_size;
- if (MALI_ERROR_NONE != kbase_gpuprops_uk_get_props(kctx, setup))
+ if (kbase_gpuprops_uk_get_props(kctx, setup) != 0)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
break;
}
if (find->size > SIZE_MAX || find->cpu_addr > ULONG_MAX) {
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
} else {
- mali_error err;
+ int err;
err = kbasep_find_enclosing_cpu_mapping_offset(
kctx,
(size_t) find->size,
&find->offset);
- if (err != MALI_ERROR_NONE)
+ if (err)
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
}
break;
break;
}
- ukh->ret = kbase_stream_create(screate->name, &screate->fd);
+ if (kbase_stream_create(screate->name, &screate->fd) != 0)
+ ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ else
+ ukh->ret = MALI_ERROR_NONE;
#else /* CONFIG_SYNC */
ukh->ret = MALI_ERROR_FUNCTION_FAILED;
#endif /* CONFIG_SYNC */
if (sizeof(*fence_validate) != args_size)
goto bad_size;
- ukh->ret = kbase_fence_validate(fence_validate->fd);
+ if (kbase_fence_validate(fence_validate->fd) != 0)
+ ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ else
+ ukh->ret = MALI_ERROR_NONE;
#endif /* CONFIG_SYNC */
break;
}
case KBASE_FUNC_EXT_BUFFER_LOCK:
{
#ifdef CONFIG_KDS
- ukh->ret = kbase_external_buffer_lock(kctx, (struct kbase_uk_ext_buff_kds_data *)args, args_size);
+ switch (kbase_external_buffer_lock(kctx,
+ (struct kbase_uk_ext_buff_kds_data *)args,
+ args_size)) {
+ case 0:
+ ukh->ret = MALI_ERROR_NONE;
+ break;
+ case -ENOMEM:
+ ukh->ret = MALI_ERROR_OUT_OF_MEMORY;
+ break;
+ default:
+ ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ }
#endif /* CONFIG_KDS */
break;
}
/*mutex lock */
spin_lock_irqsave(&kbdev->reg_op_lock, flags);
- ukh->ret = job_atom_inject_error(¶ms);
+ if (job_atom_inject_error(¶ms) != 0)
+ ukh->ret = MALI_ERROR_OUT_OF_MEMORY;
+ else
+ ukh->ret = MALI_ERROR_NONE;
spin_unlock_irqrestore(&kbdev->reg_op_lock, flags);
/*mutex unlock */
#endif /* CONFIG_MALI_ERROR_INJECT */
/*mutex lock */
spin_lock_irqsave(&kbdev->reg_op_lock, flags);
- ukh->ret = midg_model_control(kbdev->model, ¶ms);
+ if (gpu_model_control(kbdev->model, ¶ms) != 0)
+ ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ else
+ ukh->ret = MALI_ERROR_NONE;
spin_unlock_irqrestore(&kbdev->reg_op_lock, flags);
/*mutex unlock */
#endif /* CONFIG_MALI_NO_MALI */
break;
}
- case KBASE_FUNC_KEEP_GPU_POWERED:
- {
- struct kbase_uk_keep_gpu_powered *kgp =
- (struct kbase_uk_keep_gpu_powered *)args;
-
- /* A suspend won't happen here, because we're in a syscall from a
- * userspace thread.
- *
- * Nevertheless, we'd get the wrong pm_context_active/idle counting
- * here if a suspend did happen, so let's assert it won't: */
- KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
-
- if (kgp->enabled && !kctx->keep_gpu_powered) {
- kbase_pm_context_active(kbdev);
- atomic_inc(&kbdev->keep_gpu_powered_count);
- kctx->keep_gpu_powered = MALI_TRUE;
- } else if (!kgp->enabled && kctx->keep_gpu_powered) {
- atomic_dec(&kbdev->keep_gpu_powered_count);
- kbase_pm_context_idle(kbdev);
- kctx->keep_gpu_powered = MALI_FALSE;
- }
-
- break;
- }
-
case KBASE_FUNC_GET_PROFILING_CONTROLS:
{
struct kbase_uk_profiling_controls *controls =
break;
}
+#ifdef CONFIG_MALI_MIPE_ENABLED
+ case KBASE_FUNC_TLSTREAM_ACQUIRE:
+ {
+ struct kbase_uk_tlstream_acquire *tlstream_acquire =
+ args;
+
+ if (sizeof(*tlstream_acquire) != args_size)
+ goto bad_size;
+
+ if (0 != kbase_tlstream_acquire(
+ kctx,
+ &tlstream_acquire->fd)) {
+ ukh->ret = MALI_ERROR_FUNCTION_FAILED;
+ } else if (0 <= tlstream_acquire->fd) {
+ /* Summary stream was cleared during acquire.
+ * Create static timeline objects that will be
+ * read by client. */
+ kbase_create_timeline_objects(kctx);
+ }
+ break;
+ }
+ case KBASE_FUNC_TLSTREAM_FLUSH:
+ {
+ struct kbase_uk_tlstream_flush *tlstream_flush =
+ args;
+
+ if (sizeof(*tlstream_flush) != args_size)
+ goto bad_size;
+
+ kbase_tlstream_flush_streams();
+ break;
+ }
+#if MALI_UNIT_TEST
+ case KBASE_FUNC_TLSTREAM_TEST:
+ {
+ struct kbase_uk_tlstream_test *tlstream_test = args;
+
+ if (sizeof(*tlstream_test) != args_size)
+ goto bad_size;
+
+ kbase_tlstream_test(
+ tlstream_test->tpw_count,
+ tlstream_test->msg_delay,
+ tlstream_test->msg_count,
+ tlstream_test->aux_msg);
+ break;
+ }
+ case KBASE_FUNC_TLSTREAM_STATS:
+ {
+ struct kbase_uk_tlstream_stats *tlstream_stats = args;
+
+ if (sizeof(*tlstream_stats) != args_size)
+ goto bad_size;
+
+ kbase_tlstream_stats(
+ &tlstream_stats->bytes_collected,
+ &tlstream_stats->bytes_generated);
+ break;
+ }
+#endif /* MALI_UNIT_TEST */
+#endif /* CONFIG_MALI_MIPE_ENABLED */
+ /* used to signal the job core dump on fault has terminated and release the
+ * refcount of the context to let it be removed. It requires at least
+ * BASE_UK_VERSION_MAJOR to be 8 and BASE_UK_VERSION_MINOR to be 1 in the
+ * UK interface.
+ */
+ case KBASE_FUNC_DUMP_FAULT_TERM:
+ {
+#if 2 == MALI_INSTRUMENTATION_LEVEL
+ if (atomic_read(&kctx->jctx.sched_info.ctx.fault_count) > 0 &&
+ kctx->jctx.sched_info.ctx.is_scheduled)
+
+ kbasep_js_dump_fault_term(kbdev, kctx);
+
+ break;
+#endif /* 2 == MALI_INSTRUMENTATION_LEVEL */
+
+ /* This IOCTL should only be called when instr=2 at compile time. */
+ goto out_bad;
+ }
+
+ case KBASE_FUNC_GET_CONTEXT_ID:
+ {
+ struct kbase_uk_context_id *info = args;
+
+ info->id = kctx->id;
+ break;
+ }
default:
dev_err(kbdev->dev, "unknown ioctl %u", id);
goto out_bad;
}
- return MALI_ERROR_NONE;
+ return 0;
bad_size:
dev_err(kbdev->dev, "Wrong syscall size (%d) for %08x\n", args_size, id);
out_bad:
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
static struct kbase_device *to_kbase_device(struct device *dev)
}
/*
- * API to acquire device list semaphore and
+ * API to acquire device list mutex and
* return pointer to the device list head
*/
const struct list_head *kbase_dev_list_get(void)
{
- down(&kbase_dev_list_lock);
+ mutex_lock(&kbase_dev_list_lock);
return &kbase_dev_list;
}
+KBASE_EXPORT_TEST_API(kbase_dev_list_get);
-/* API to release the device list semaphore */
+/* API to release the device list mutex */
void kbase_dev_list_put(const struct list_head *dev_list)
{
- up(&kbase_dev_list_lock);
+ mutex_unlock(&kbase_dev_list_lock);
}
+KBASE_EXPORT_TEST_API(kbase_dev_list_put);
/* Find a particular kbase device (as specified by minor number), or find the "first" device if -1 is specified */
struct kbase_device *kbase_find_device(int minor)
{
struct kbase_device *kbdev = NULL;
struct list_head *entry;
+ const struct list_head *dev_list = kbase_dev_list_get();
- down(&kbase_dev_list_lock);
- list_for_each(entry, &kbase_dev_list) {
+ list_for_each(entry, dev_list) {
struct kbase_device *tmp;
tmp = list_entry(entry, struct kbase_device, entry);
break;
}
}
- up(&kbase_dev_list_lock);
+ kbase_dev_list_put(dev_list);
return kbdev;
}
struct kbase_device *kbdev = NULL;
struct kbase_context *kctx;
int ret = 0;
+#ifdef CONFIG_DEBUG_FS
+ char kctx_name[64];
+#endif
kbdev = kbase_find_device(iminor(inode));
init_waitqueue_head(&kctx->event_queue);
filp->private_data = kctx;
+ kctx->infinite_cache_active = kbdev->infinite_cache_active_default;
+
+#ifdef CONFIG_DEBUG_FS
+ snprintf(kctx_name, 64, "%d_%d", kctx->tgid, kctx->id);
+
+ kctx->kctx_dentry = debugfs_create_dir(kctx_name,
+ kbdev->debugfs_ctx_directory);
+
+ if (IS_ERR_OR_NULL(kctx->kctx_dentry)) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+#ifdef CONFIG_MALI_CACHE_COHERENT
+ /* if cache is completely coherent at hardware level, then remove the
+ * infinite cache control support from debugfs.
+ */
+#else
+ debugfs_create_bool("infinite_cache", 0644, kctx->kctx_dentry,
+ &kctx->infinite_cache_active);
+#endif /* CONFIG_MALI_CACHE_COHERENT */
+ kbasep_mem_profile_debugfs_add(kctx);
+
+ kbasep_jd_debugfs_ctx_add(kctx);
+ kbase_debug_mem_view_init(filp);
+#endif
+
dev_dbg(kbdev->dev, "created base context\n");
{
mutex_lock(&kbdev->kctx_list_lock);
element->kctx = kctx;
list_add(&element->link, &kbdev->kctx_list);
+#ifdef CONFIG_MALI_MIPE_ENABLED
+ kbase_tlstream_tl_new_ctx(
+ element->kctx,
+ (u32)(element->kctx->id));
+#endif
mutex_unlock(&kbdev->kctx_list_lock);
} else {
/* we don't treat this as a fail - just warn about it */
struct kbase_context *kctx = filp->private_data;
struct kbase_device *kbdev = kctx->kbdev;
struct kbasep_kctx_list_element *element, *tmp;
- mali_bool found_element = MALI_FALSE;
+ bool found_element = false;
+
+#ifdef CONFIG_MALI_MIPE_ENABLED
+ kbase_tlstream_tl_del_ctx(kctx);
+#endif
+
+#ifdef CONFIG_DEBUG_FS
+ debugfs_remove_recursive(kctx->kctx_dentry);
+ kbasep_mem_profile_debugfs_remove(kctx);
+#endif
mutex_lock(&kbdev->kctx_list_lock);
list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) {
if (element->kctx == kctx) {
list_del(&element->link);
kfree(element);
- found_element = MALI_TRUE;
+ found_element = true;
}
}
mutex_unlock(&kbdev->kctx_list_lock);
dev_warn(kbdev->dev, "kctx not in kctx_list\n");
filp->private_data = NULL;
+
+ mutex_lock(&kctx->vinstr_cli_lock);
+ /* If this client was performing hwcnt dumping and did not explicitly
+ * detach itself, remove it from the vinstr core now */
+ kbase_vinstr_detach_client(kctx->kbdev->vinstr_ctx, kctx->vinstr_cli);
+ mutex_unlock(&kctx->vinstr_cli_lock);
+
kbase_destroy_context(kctx);
dev_dbg(kbdev->dev, "deleted base context\n");
return -EFAULT;
}
- if (MALI_ERROR_NONE != kbase_dispatch(kctx, &msg, size))
+ if (kbase_dispatch(kctx, &msg, size) != 0)
return -EFAULT;
if (0 != copy_to_user((void __user *)arg, &msg, size)) {
wake_up_interruptible(&kctx->event_queue);
}
-KBASE_EXPORT_TEST_API(kbase_event_wakeup)
+KBASE_EXPORT_TEST_API(kbase_event_wakeup);
static int kbase_check_flags(int flags)
{
{
return readl(kbdev->reg + offset);
}
-#endif
-
-#ifndef CONFIG_MALI_NO_MALI
+#endif /* !CONFIG_MALI_NO_MALI */
-static void *kbase_tag(void *ptr, u32 tag)
-{
- return (void *)(((uintptr_t) ptr) | tag);
-}
-static void *kbase_untag(void *ptr)
+/** Show callback for the @c power_policy sysfs file.
+ *
+ * This function is called to get the contents of the @c power_policy sysfs
+ * file. This is a list of the available policies with the currently active one
+ * surrounded by square brackets.
+ *
+ * @param dev The device this sysfs file is for
+ * @param attr The attributes of the sysfs file
+ * @param buf The output buffer for the sysfs file contents
+ *
+ * @return The number of bytes output to @c buf.
+ */
+static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf)
{
- return (void *)(((uintptr_t) ptr) & ~3);
-}
-
+ struct kbase_device *kbdev;
+ const struct kbase_pm_policy *current_policy;
+ const struct kbase_pm_policy *const *policy_list;
+ int policy_count;
+ int i;
+ ssize_t ret = 0;
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
-static irqreturn_t kbase_job_irq_handler(int irq, void *data)
-{
- unsigned long flags;
- struct kbase_device *kbdev = kbase_untag(data);
- u32 val;
+ current_policy = kbase_pm_get_policy(kbdev);
- spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
+ policy_count = kbase_pm_list_policies(&policy_list);
- if (!kbdev->pm.gpu_powered) {
- /* GPU is turned off - IRQ is not for us */
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
- return IRQ_NONE;
+ for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) {
+ if (policy_list[i] == current_policy)
+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name);
+ else
+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name);
}
- val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL);
-
-#ifdef CONFIG_MALI_DEBUG
- if (!kbdev->pm.driver_ready_for_irqs)
- dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n",
- __func__, irq, val);
-#endif /* CONFIG_MALI_DEBUG */
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
-
- if (!val)
- return IRQ_NONE;
-
- dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
-
- kbase_job_done(kbdev, val);
-
- return IRQ_HANDLED;
-}
-
-KBASE_EXPORT_TEST_API(kbase_job_irq_handler);
-
-static irqreturn_t kbase_mmu_irq_handler(int irq, void *data)
-{
- unsigned long flags;
- struct kbase_device *kbdev = kbase_untag(data);
- u32 val;
-
- spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
-
- if (!kbdev->pm.gpu_powered) {
- /* GPU is turned off - IRQ is not for us */
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
- return IRQ_NONE;
+ if (ret < PAGE_SIZE - 1) {
+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
+ } else {
+ buf[PAGE_SIZE - 2] = '\n';
+ buf[PAGE_SIZE - 1] = '\0';
+ ret = PAGE_SIZE - 1;
}
- val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL);
-
-#ifdef CONFIG_MALI_DEBUG
- if (!kbdev->pm.driver_ready_for_irqs)
- dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n",
- __func__, irq, val);
-#endif /* CONFIG_MALI_DEBUG */
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
-
- if (!val)
- return IRQ_NONE;
-
- dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
-
- kbase_mmu_interrupt(kbdev, val);
-
- return IRQ_HANDLED;
-}
-
-static irqreturn_t kbase_gpu_irq_handler(int irq, void *data)
-{
- unsigned long flags;
- struct kbase_device *kbdev = kbase_untag(data);
- u32 val;
-
- spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
-
- if (!kbdev->pm.gpu_powered) {
- /* GPU is turned off - IRQ is not for us */
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
- return IRQ_NONE;
- }
-
- val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL);
-
-#ifdef CONFIG_MALI_DEBUG
- if (!kbdev->pm.driver_ready_for_irqs)
- dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n",
- __func__, irq, val);
-#endif /* CONFIG_MALI_DEBUG */
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
-
- if (!val)
- return IRQ_NONE;
-
- dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
-
- kbase_gpu_interrupt(kbdev, val);
-
- return IRQ_HANDLED;
-}
-static irq_handler_t kbase_handler_table[] = {
- [JOB_IRQ_TAG] = kbase_job_irq_handler,
- [MMU_IRQ_TAG] = kbase_mmu_irq_handler,
- [GPU_IRQ_TAG] = kbase_gpu_irq_handler,
-};
-
-
-#ifdef CONFIG_MALI_DEBUG
-#define JOB_IRQ_HANDLER JOB_IRQ_TAG
-#define MMU_IRQ_HANDLER MMU_IRQ_TAG
-#define GPU_IRQ_HANDLER GPU_IRQ_TAG
-
-/**
- * @brief Registers given interrupt handler for requested interrupt type
- * Case irq handler is not specified default handler shall be registered
- *
- * @param[in] kbdev - Device for which the handler is to be registered
- * @param[in] custom_handler - Handler to be registered
- * @param[in] irq_type - Interrupt type
- * @return MALI_ERROR_NONE case success, MALI_ERROR_FUNCTION_FAILED otherwise
- */
-static mali_error kbase_set_custom_irq_handler(struct kbase_device *kbdev, irq_handler_t custom_handler, int irq_type)
-{
- mali_error result = MALI_ERROR_NONE;
- irq_handler_t requested_irq_handler = NULL;
-
- KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && (GPU_IRQ_HANDLER >= irq_type));
-
- /* Release previous handler */
- if (kbdev->irqs[irq_type].irq)
- free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type));
-
- requested_irq_handler = (NULL != custom_handler) ? custom_handler : kbase_handler_table[irq_type];
-
- if (0 != request_irq(kbdev->irqs[irq_type].irq,
- requested_irq_handler,
- kbdev->irqs[irq_type].flags | IRQF_SHARED,
- dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) {
- result = MALI_ERROR_FUNCTION_FAILED;
- dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", kbdev->irqs[irq_type].irq, irq_type);
-#ifdef CONFIG_SPARSE_IRQ
- dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n");
-#endif /* CONFIG_SPARSE_IRQ */
- }
-
- return result;
-}
-
-KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler)
-
-/* test correct interrupt assigment and reception by cpu */
-struct kbasep_irq_test {
- struct hrtimer timer;
- wait_queue_head_t wait;
- int triggered;
- u32 timeout;
-};
-
-static struct kbasep_irq_test kbasep_irq_test_data;
-
-#define IRQ_TEST_TIMEOUT 500
-
-static irqreturn_t kbase_job_irq_test_handler(int irq, void *data)
-{
- unsigned long flags;
- struct kbase_device *kbdev = kbase_untag(data);
- u32 val;
-
- spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
-
- if (!kbdev->pm.gpu_powered) {
- /* GPU is turned off - IRQ is not for us */
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
- return IRQ_NONE;
- }
-
- val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL);
-
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
-
- if (!val)
- return IRQ_NONE;
-
- dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
-
- kbasep_irq_test_data.triggered = 1;
- wake_up(&kbasep_irq_test_data.wait);
-
- kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val, NULL);
-
- return IRQ_HANDLED;
-}
-
-static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data)
-{
- unsigned long flags;
- struct kbase_device *kbdev = kbase_untag(data);
- u32 val;
-
- spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
-
- if (!kbdev->pm.gpu_powered) {
- /* GPU is turned off - IRQ is not for us */
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
- return IRQ_NONE;
- }
-
- val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL);
-
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
-
- if (!val)
- return IRQ_NONE;
-
- dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
-
- kbasep_irq_test_data.triggered = 1;
- wake_up(&kbasep_irq_test_data.wait);
-
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val, NULL);
-
- return IRQ_HANDLED;
-}
-
-static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer)
-{
- struct kbasep_irq_test *test_data = container_of(timer, struct kbasep_irq_test, timer);
-
- test_data->timeout = 1;
- test_data->triggered = 1;
- wake_up(&test_data->wait);
- return HRTIMER_NORESTART;
-}
-
-static mali_error kbasep_common_test_interrupt(struct kbase_device * const kbdev, u32 tag)
-{
- mali_error err = MALI_ERROR_NONE;
- irq_handler_t test_handler;
-
- u32 old_mask_val;
- u16 mask_offset;
- u16 rawstat_offset;
-
- switch (tag) {
- case JOB_IRQ_TAG:
- test_handler = kbase_job_irq_test_handler;
- rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT);
- mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK);
- break;
- case MMU_IRQ_TAG:
- test_handler = kbase_mmu_irq_test_handler;
- rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT);
- mask_offset = MMU_REG(MMU_IRQ_MASK);
- break;
- case GPU_IRQ_TAG:
- /* already tested by pm_driver - bail out */
- default:
- return MALI_ERROR_NONE;
- }
-
- /* store old mask */
- old_mask_val = kbase_reg_read(kbdev, mask_offset, NULL);
- /* mask interrupts */
- kbase_reg_write(kbdev, mask_offset, 0x0, NULL);
-
- if (kbdev->irqs[tag].irq) {
- /* release original handler and install test handler */
- if (MALI_ERROR_NONE != kbase_set_custom_irq_handler(kbdev, test_handler, tag)) {
- err = MALI_ERROR_FUNCTION_FAILED;
- } else {
- kbasep_irq_test_data.timeout = 0;
- hrtimer_init(&kbasep_irq_test_data.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- kbasep_irq_test_data.timer.function = kbasep_test_interrupt_timeout;
-
- /* trigger interrupt */
- kbase_reg_write(kbdev, mask_offset, 0x1, NULL);
- kbase_reg_write(kbdev, rawstat_offset, 0x1, NULL);
-
- hrtimer_start(&kbasep_irq_test_data.timer, HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT), HRTIMER_MODE_REL);
-
- wait_event(kbasep_irq_test_data.wait, kbasep_irq_test_data.triggered != 0);
-
- if (kbasep_irq_test_data.timeout != 0) {
- dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n", kbdev->irqs[tag].irq, tag);
- err = MALI_ERROR_FUNCTION_FAILED;
- } else {
- dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n", kbdev->irqs[tag].irq, tag);
- }
-
- hrtimer_cancel(&kbasep_irq_test_data.timer);
- kbasep_irq_test_data.triggered = 0;
-
- /* mask interrupts */
- kbase_reg_write(kbdev, mask_offset, 0x0, NULL);
-
- /* release test handler */
- free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag));
- }
-
- /* restore original interrupt */
- if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag], kbdev->irqs[tag].flags | IRQF_SHARED, dev_name(kbdev->dev), kbase_tag(kbdev, tag))) {
- dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n", kbdev->irqs[tag].irq, tag);
- err = MALI_ERROR_FUNCTION_FAILED;
- }
- }
- /* restore old mask */
- kbase_reg_write(kbdev, mask_offset, old_mask_val, NULL);
-
- return err;
-}
-
-static mali_error kbasep_common_test_interrupt_handlers(struct kbase_device * const kbdev)
-{
- mali_error err;
-
- init_waitqueue_head(&kbasep_irq_test_data.wait);
- kbasep_irq_test_data.triggered = 0;
-
- /* A suspend won't happen during startup/insmod */
- kbase_pm_context_active(kbdev);
-
- err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG);
- if (MALI_ERROR_NONE != err) {
- dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n");
- goto out;
- }
-
- err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG);
- if (MALI_ERROR_NONE != err) {
- dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n");
- goto out;
- }
-
- dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n");
-
- out:
- kbase_pm_context_idle(kbdev);
-
- return err;
-}
-#endif /* CONFIG_MALI_DEBUG */
-
-static int kbase_install_interrupts(struct kbase_device *kbdev)
-{
- u32 nr = ARRAY_SIZE(kbase_handler_table);
- int err;
- u32 i;
-
- for (i = 0; i < nr; i++) {
- err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i], kbdev->irqs[i].flags | IRQF_SHARED, dev_name(kbdev->dev), kbase_tag(kbdev, i));
- if (err) {
- dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", kbdev->irqs[i].irq, i);
-#ifdef CONFIG_SPARSE_IRQ
- dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n");
-#endif /* CONFIG_SPARSE_IRQ */
- goto release;
- }
- }
-
- return 0;
-
- release:
- while (i-- > 0)
- free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i));
-
- return err;
-}
-
-static void kbase_release_interrupts(struct kbase_device *kbdev)
-{
- u32 nr = ARRAY_SIZE(kbase_handler_table);
- u32 i;
-
- for (i = 0; i < nr; i++) {
- if (kbdev->irqs[i].irq)
- free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i));
- }
-}
-
-void kbase_synchronize_irqs(struct kbase_device *kbdev)
-{
- u32 nr = ARRAY_SIZE(kbase_handler_table);
- u32 i;
-
- for (i = 0; i < nr; i++) {
- if (kbdev->irqs[i].irq)
- synchronize_irq(kbdev->irqs[i].irq);
- }
-}
-#endif /* CONFIG_MALI_NO_MALI */
-
-#if KBASE_PM_EN
-/** Show callback for the @c power_policy sysfs file.
- *
- * This function is called to get the contents of the @c power_policy sysfs
- * file. This is a list of the available policies with the currently active one
- * surrounded by square brackets.
- *
- * @param dev The device this sysfs file is for
- * @param attr The attributes of the sysfs file
- * @param buf The output buffer for the sysfs file contents
- *
- * @return The number of bytes output to @c buf.
- */
-static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf)
-{
- struct kbase_device *kbdev;
- const struct kbase_pm_policy *current_policy;
- const struct kbase_pm_policy *const *policy_list;
- int policy_count;
- int i;
- ssize_t ret = 0;
-
- kbdev = to_kbase_device(dev);
-
- if (!kbdev)
- return -ENODEV;
-
- current_policy = kbase_pm_get_policy(kbdev);
-
- policy_count = kbase_pm_list_policies(&policy_list);
-
- for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) {
- if (policy_list[i] == current_policy)
- ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name);
- else
- ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name);
- }
-
- if (ret < PAGE_SIZE - 1) {
- ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
- } else {
- buf[PAGE_SIZE - 2] = '\n';
- buf[PAGE_SIZE - 1] = '\0';
- ret = PAGE_SIZE - 1;
- }
-
- return ret;
+ return ret;
}
/** Store callback for the @c power_policy sysfs file.
spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- kbdev->pm.debug_core_mask = new_core_mask;
- kbase_pm_update_cores_state_nolock(kbdev);
+ kbase_pm_set_debug_core_mask(kbdev, new_core_mask);
spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
}
* Writing to it will set the current core mask.
*/
static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask);
-#endif /* KBASE_PM_EN */
#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
-/* Import the external affinity mask variables */
-extern u64 mali_js0_affinity_mask;
-extern u64 mali_js1_affinity_mask;
-extern u64 mali_js2_affinity_mask;
-
/**
+ * struct sc_split_config
+ * @tag: Short name
+ * @human_readable: Long name
+ * @js0_mask: Mask for job slot 0
+ * @js1_mask: Mask for job slot 1
+ * @js2_mask: Mask for job slot 2
+ *
* Structure containing a single shader affinity split configuration.
*/
struct sc_split_config {
u64 js2_mask;
};
-/**
+/*
* Array of available shader affinity split configurations.
*/
static struct sc_split_config const sc_split_configs[] = {
#endif /* CONFIG_MALI_DEBUG_SHADER_SPLIT_FS */
-#if !MALI_CUSTOMER_RELEASE
/** Store callback for the @c js_timeouts sysfs file.
*
* This function is called to get the contents of the @c js_timeouts sysfs
* file. This file contains five values separated by whitespace. The values
- * are basically the same as KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- * KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS, KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- * KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS, BASE_CONFIG_ATTR_JS_RESET_TICKS_NSS
+ * are basically the same as JS_SOFT_STOP_TICKS, JS_HARD_STOP_TICKS_SS,
+ * JS_HARD_STOP_TICKS_DUMPING, JS_RESET_TICKS_SS, JS_RESET_TICKS_DUMPING
* configuration values (in that order), with the difference that the js_timeout
- * valus are expressed in MILLISECONDS.
+ * values are expressed in MILLISECONDS.
*
* The js_timeouts sysfile file allows the current values in
* use by the job scheduler to get override. Note that a value needs to
{
struct kbase_device *kbdev;
int items;
- unsigned long js_soft_stop_ms;
- unsigned long js_soft_stop_ms_cl;
- unsigned long js_hard_stop_ms_ss;
- unsigned long js_hard_stop_ms_cl;
- unsigned long js_hard_stop_ms_nss;
- unsigned long js_reset_ms_ss;
- unsigned long js_reset_ms_cl;
- unsigned long js_reset_ms_nss;
+ long js_soft_stop_ms;
+ long js_soft_stop_ms_cl;
+ long js_hard_stop_ms_ss;
+ long js_hard_stop_ms_cl;
+ long js_hard_stop_ms_dumping;
+ long js_reset_ms_ss;
+ long js_reset_ms_cl;
+ long js_reset_ms_dumping;
kbdev = to_kbase_device(dev);
if (!kbdev)
return -ENODEV;
- items = sscanf(buf, "%lu %lu %lu %lu %lu %lu %lu %lu",
+ items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld",
&js_soft_stop_ms, &js_soft_stop_ms_cl,
&js_hard_stop_ms_ss, &js_hard_stop_ms_cl,
- &js_hard_stop_ms_nss, &js_reset_ms_ss,
- &js_reset_ms_cl, &js_reset_ms_nss);
+ &js_hard_stop_ms_dumping, &js_reset_ms_ss,
+ &js_reset_ms_cl, &js_reset_ms_dumping);
if (items == 8) {
u64 ticks;
- ticks = js_soft_stop_ms * 1000000ULL;
- do_div(ticks, kbdev->js_data.scheduling_tick_ns);
- kbdev->js_soft_stop_ticks = ticks;
+ if (js_soft_stop_ms >= 0) {
+ ticks = js_soft_stop_ms * 1000000ULL;
+ do_div(ticks, kbdev->js_data.scheduling_period_ns);
+ kbdev->js_soft_stop_ticks = ticks;
+ } else {
+ kbdev->js_soft_stop_ticks = -1;
+ }
- ticks = js_soft_stop_ms_cl * 1000000ULL;
- do_div(ticks, kbdev->js_data.scheduling_tick_ns);
- kbdev->js_soft_stop_ticks_cl = ticks;
+ if (js_soft_stop_ms_cl >= 0) {
+ ticks = js_soft_stop_ms_cl * 1000000ULL;
+ do_div(ticks, kbdev->js_data.scheduling_period_ns);
+ kbdev->js_soft_stop_ticks_cl = ticks;
+ } else {
+ kbdev->js_soft_stop_ticks_cl = -1;
+ }
- ticks = js_hard_stop_ms_ss * 1000000ULL;
- do_div(ticks, kbdev->js_data.scheduling_tick_ns);
- kbdev->js_hard_stop_ticks_ss = ticks;
+ if (js_hard_stop_ms_ss >= 0) {
+ ticks = js_hard_stop_ms_ss * 1000000ULL;
+ do_div(ticks, kbdev->js_data.scheduling_period_ns);
+ kbdev->js_hard_stop_ticks_ss = ticks;
+ } else {
+ kbdev->js_hard_stop_ticks_ss = -1;
+ }
- ticks = js_hard_stop_ms_cl * 1000000ULL;
- do_div(ticks, kbdev->js_data.scheduling_tick_ns);
- kbdev->js_hard_stop_ticks_cl = ticks;
+ if (js_hard_stop_ms_cl >= 0) {
+ ticks = js_hard_stop_ms_cl * 1000000ULL;
+ do_div(ticks, kbdev->js_data.scheduling_period_ns);
+ kbdev->js_hard_stop_ticks_cl = ticks;
+ } else {
+ kbdev->js_hard_stop_ticks_cl = -1;
+ }
- ticks = js_hard_stop_ms_nss * 1000000ULL;
- do_div(ticks, kbdev->js_data.scheduling_tick_ns);
- kbdev->js_hard_stop_ticks_nss = ticks;
+ if (js_hard_stop_ms_dumping >= 0) {
+ ticks = js_hard_stop_ms_dumping * 1000000ULL;
+ do_div(ticks, kbdev->js_data.scheduling_period_ns);
+ kbdev->js_hard_stop_ticks_dumping = ticks;
+ } else {
+ kbdev->js_hard_stop_ticks_dumping = -1;
+ }
- ticks = js_reset_ms_ss * 1000000ULL;
- do_div(ticks, kbdev->js_data.scheduling_tick_ns);
- kbdev->js_reset_ticks_ss = ticks;
+ if (js_reset_ms_ss >= 0) {
+ ticks = js_reset_ms_ss * 1000000ULL;
+ do_div(ticks, kbdev->js_data.scheduling_period_ns);
+ kbdev->js_reset_ticks_ss = ticks;
+ } else {
+ kbdev->js_reset_ticks_ss = -1;
+ }
- ticks = js_reset_ms_cl * 1000000ULL;
- do_div(ticks, kbdev->js_data.scheduling_tick_ns);
- kbdev->js_reset_ticks_cl = ticks;
+ if (js_reset_ms_cl >= 0) {
+ ticks = js_reset_ms_cl * 1000000ULL;
+ do_div(ticks, kbdev->js_data.scheduling_period_ns);
+ kbdev->js_reset_ticks_cl = ticks;
+ } else {
+ kbdev->js_reset_ticks_cl = -1;
+ }
- ticks = js_reset_ms_nss * 1000000ULL;
- do_div(ticks, kbdev->js_data.scheduling_tick_ns);
- kbdev->js_reset_ticks_nss = ticks;
+ if (js_reset_ms_dumping >= 0) {
+ ticks = js_reset_ms_dumping * 1000000ULL;
+ do_div(ticks, kbdev->js_data.scheduling_period_ns);
+ kbdev->js_reset_ticks_dumping = ticks;
+ } else {
+ kbdev->js_reset_ticks_dumping = -1;
+ }
- dev_dbg(kbdev->dev, "Overriding KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS with %lu ticks (%lu ms)\n", (unsigned long)kbdev->js_soft_stop_ticks, js_soft_stop_ms);
- dev_dbg(kbdev->dev, "Overriding KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL with %lu ticks (%lu ms)\n", (unsigned long)kbdev->js_soft_stop_ticks_cl, js_soft_stop_ms_cl);
- dev_dbg(kbdev->dev, "Overriding KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS with %lu ticks (%lu ms)\n", (unsigned long)kbdev->js_hard_stop_ticks_ss, js_hard_stop_ms_ss);
- dev_dbg(kbdev->dev, "Overriding KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL with %lu ticks (%lu ms)\n", (unsigned long)kbdev->js_hard_stop_ticks_cl, js_hard_stop_ms_cl);
- dev_dbg(kbdev->dev, "Overriding KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS with %lu ticks (%lu ms)\n", (unsigned long)kbdev->js_hard_stop_ticks_nss, js_hard_stop_ms_nss);
- dev_dbg(kbdev->dev, "Overriding KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS with %lu ticks (%lu ms)\n", (unsigned long)kbdev->js_reset_ticks_ss, js_reset_ms_ss);
- dev_dbg(kbdev->dev, "Overriding KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL with %lu ticks (%lu ms)\n", (unsigned long)kbdev->js_reset_ticks_cl, js_reset_ms_cl);
- dev_dbg(kbdev->dev, "Overriding KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS with %lu ticks (%lu ms)\n", (unsigned long)kbdev->js_reset_ticks_nss, js_reset_ms_nss);
+ kbdev->js_timeouts_updated = true;
+
+ dev_dbg(kbdev->dev, "Overriding JS_SOFT_STOP_TICKS with %lu ticks (%lu ms)\n",
+ (unsigned long)kbdev->js_soft_stop_ticks,
+ js_soft_stop_ms);
+ dev_dbg(kbdev->dev, "Overriding JS_SOFT_STOP_TICKS_CL with %lu ticks (%lu ms)\n",
+ (unsigned long)kbdev->js_soft_stop_ticks_cl,
+ js_soft_stop_ms_cl);
+ dev_dbg(kbdev->dev, "Overriding JS_HARD_STOP_TICKS_SS with %lu ticks (%lu ms)\n",
+ (unsigned long)kbdev->js_hard_stop_ticks_ss,
+ js_hard_stop_ms_ss);
+ dev_dbg(kbdev->dev, "Overriding JS_HARD_STOP_TICKS_CL with %lu ticks (%lu ms)\n",
+ (unsigned long)kbdev->js_hard_stop_ticks_cl,
+ js_hard_stop_ms_cl);
+ dev_dbg(kbdev->dev, "Overriding JS_HARD_STOP_TICKS_DUMPING with %lu ticks (%lu ms)\n",
+ (unsigned long)
+ kbdev->js_hard_stop_ticks_dumping,
+ js_hard_stop_ms_dumping);
+ dev_dbg(kbdev->dev, "Overriding JS_RESET_TICKS_SS with %lu ticks (%lu ms)\n",
+ (unsigned long)kbdev->js_reset_ticks_ss,
+ js_reset_ms_ss);
+ dev_dbg(kbdev->dev, "Overriding JS_RESET_TICKS_CL with %lu ticks (%lu ms)\n",
+ (unsigned long)kbdev->js_reset_ticks_cl,
+ js_reset_ms_cl);
+ dev_dbg(kbdev->dev, "Overriding JS_RESET_TICKS_DUMPING with %lu ticks (%lu ms)\n",
+ (unsigned long)kbdev->js_reset_ticks_dumping,
+ js_reset_ms_dumping);
return count;
- } else {
- dev_err(kbdev->dev, "Couldn't process js_timeouts write operation.\nUse format " "<soft_stop_ms> <hard_stop_ms_ss> <hard_stop_ms_nss> <reset_ms_ss> <reset_ms_nss>\n");
- return -EINVAL;
}
+
+ dev_err(kbdev->dev, "Couldn't process js_timeouts write operation.\n"
+ "Use format <soft_stop_ms> <soft_stop_ms_cl> <hard_stop_ms_ss> <hard_stop_ms_cl> <hard_stop_ms_dumping> <reset_ms_ss> <reset_ms_cl> <reset_ms_dumping>\n"
+ "Write 0 for no change, -1 to restore default timeout\n");
+ return -EINVAL;
}
/** Show callback for the @c js_timeouts sysfs file.
*
* This function is called to get the contents of the @c js_timeouts sysfs
* file. It returns the last set values written to the js_timeouts sysfs file.
- * If the file didn't get written yet, the values will be 0.
- *
+ * If the file didn't get written yet, the values will be current setting in
+ * use.
* @param dev The device this sysfs file is for
* @param attr The attributes of the sysfs file
* @param buf The output buffer for the sysfs file contents
unsigned long js_soft_stop_ms_cl;
unsigned long js_hard_stop_ms_ss;
unsigned long js_hard_stop_ms_cl;
- unsigned long js_hard_stop_ms_nss;
+ unsigned long js_hard_stop_ms_dumping;
unsigned long js_reset_ms_ss;
unsigned long js_reset_ms_cl;
- unsigned long js_reset_ms_nss;
+ unsigned long js_reset_ms_dumping;
+ unsigned long ticks;
+ u32 scheduling_period_ns;
kbdev = to_kbase_device(dev);
if (!kbdev)
return -ENODEV;
- ms = (u64) kbdev->js_soft_stop_ticks * kbdev->js_data.scheduling_tick_ns;
+ /* If no contexts have been scheduled since js_timeouts was last written
+ * to, the new timeouts might not have been latched yet. So check if an
+ * update is pending and use the new values if necessary. */
+ if (kbdev->js_timeouts_updated && kbdev->js_scheduling_period_ns > 0)
+ scheduling_period_ns = kbdev->js_scheduling_period_ns;
+ else
+ scheduling_period_ns = kbdev->js_data.scheduling_period_ns;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_soft_stop_ticks > 0)
+ ticks = kbdev->js_soft_stop_ticks;
+ else
+ ticks = kbdev->js_data.soft_stop_ticks;
+ ms = (u64)ticks * scheduling_period_ns;
do_div(ms, 1000000UL);
js_soft_stop_ms = (unsigned long)ms;
- ms = (u64) kbdev->js_soft_stop_ticks_cl * kbdev->js_data.scheduling_tick_ns;
+ if (kbdev->js_timeouts_updated && kbdev->js_soft_stop_ticks_cl > 0)
+ ticks = kbdev->js_soft_stop_ticks_cl;
+ else
+ ticks = kbdev->js_data.soft_stop_ticks_cl;
+ ms = (u64)ticks * scheduling_period_ns;
do_div(ms, 1000000UL);
js_soft_stop_ms_cl = (unsigned long)ms;
- ms = (u64) kbdev->js_hard_stop_ticks_ss * kbdev->js_data.scheduling_tick_ns;
+ if (kbdev->js_timeouts_updated && kbdev->js_hard_stop_ticks_ss > 0)
+ ticks = kbdev->js_hard_stop_ticks_ss;
+ else
+ ticks = kbdev->js_data.hard_stop_ticks_ss;
+ ms = (u64)ticks * scheduling_period_ns;
do_div(ms, 1000000UL);
js_hard_stop_ms_ss = (unsigned long)ms;
- ms = (u64) kbdev->js_hard_stop_ticks_cl * kbdev->js_data.scheduling_tick_ns;
+ if (kbdev->js_timeouts_updated && kbdev->js_hard_stop_ticks_cl > 0)
+ ticks = kbdev->js_hard_stop_ticks_cl;
+ else
+ ticks = kbdev->js_data.hard_stop_ticks_cl;
+ ms = (u64)ticks * scheduling_period_ns;
do_div(ms, 1000000UL);
js_hard_stop_ms_cl = (unsigned long)ms;
- ms = (u64) kbdev->js_hard_stop_ticks_nss * kbdev->js_data.scheduling_tick_ns;
+ if (kbdev->js_timeouts_updated && kbdev->js_hard_stop_ticks_dumping > 0)
+ ticks = kbdev->js_hard_stop_ticks_dumping;
+ else
+ ticks = kbdev->js_data.hard_stop_ticks_dumping;
+ ms = (u64)ticks * scheduling_period_ns;
do_div(ms, 1000000UL);
- js_hard_stop_ms_nss = (unsigned long)ms;
+ js_hard_stop_ms_dumping = (unsigned long)ms;
- ms = (u64) kbdev->js_reset_ticks_ss * kbdev->js_data.scheduling_tick_ns;
+ if (kbdev->js_timeouts_updated && kbdev->js_reset_ticks_ss > 0)
+ ticks = kbdev->js_reset_ticks_ss;
+ else
+ ticks = kbdev->js_data.gpu_reset_ticks_ss;
+ ms = (u64)ticks * scheduling_period_ns;
do_div(ms, 1000000UL);
js_reset_ms_ss = (unsigned long)ms;
- ms = (u64) kbdev->js_reset_ticks_cl * kbdev->js_data.scheduling_tick_ns;
+ if (kbdev->js_timeouts_updated && kbdev->js_reset_ticks_cl > 0)
+ ticks = kbdev->js_reset_ticks_cl;
+ else
+ ticks = kbdev->js_data.gpu_reset_ticks_cl;
+ ms = (u64)ticks * scheduling_period_ns;
do_div(ms, 1000000UL);
js_reset_ms_cl = (unsigned long)ms;
- ms = (u64) kbdev->js_reset_ticks_nss * kbdev->js_data.scheduling_tick_ns;
+ if (kbdev->js_timeouts_updated && kbdev->js_reset_ticks_dumping > 0)
+ ticks = kbdev->js_reset_ticks_dumping;
+ else
+ ticks = kbdev->js_data.gpu_reset_ticks_dumping;
+ ms = (u64)ticks * scheduling_period_ns;
do_div(ms, 1000000UL);
- js_reset_ms_nss = (unsigned long)ms;
+ js_reset_ms_dumping = (unsigned long)ms;
ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n",
js_soft_stop_ms, js_soft_stop_ms_cl,
js_hard_stop_ms_ss, js_hard_stop_ms_cl,
- js_hard_stop_ms_nss, js_reset_ms_ss,
- js_reset_ms_cl, js_reset_ms_nss);
+ js_hard_stop_ms_dumping, js_reset_ms_ss,
+ js_reset_ms_cl, js_reset_ms_dumping);
if (ret >= PAGE_SIZE) {
buf[PAGE_SIZE - 2] = '\n';
/** The sysfs file @c js_timeouts.
*
* This is used to override the current job scheduler values for
- * KBASE_CONFIG_ATTR_JS_STOP_STOP_TICKS_SS
- * KBASE_CONFIG_ATTR_JS_STOP_STOP_TICKS_CL
- * KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS
- * KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL
- * KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS
- * KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS
- * KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL
- * KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS.
+ * JS_STOP_STOP_TICKS_SS
+ * JS_STOP_STOP_TICKS_CL
+ * JS_HARD_STOP_TICKS_SS
+ * JS_HARD_STOP_TICKS_CL
+ * JS_HARD_STOP_TICKS_DUMPING
+ * JS_RESET_TICKS_SS
+ * JS_RESET_TICKS_CL
+ * JS_RESET_TICKS_DUMPING.
*/
static DEVICE_ATTR(js_timeouts, S_IRUGO | S_IWUSR, show_js_timeouts, set_js_timeouts);
+/**
+ * set_js_scheduling_period - Store callback for the js_scheduling_period sysfs
+ * file
+ * @dev: The device the sysfs file is for
+ * @attr: The attributes of the sysfs file
+ * @buf: The value written to the sysfs file
+ * @count: The number of bytes written to the sysfs file
+ *
+ * This function is called when the js_scheduling_period sysfs file is written
+ * to. It checks the data written, and if valid updates the js_scheduling_period
+ * value
+ *
+ * Return: @c count if the function succeeded. An error code on failure.
+ */
+static ssize_t set_js_scheduling_period(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct kbase_device *kbdev;
+ int ret;
+ unsigned int js_scheduling_period;
+ u32 new_scheduling_period_ns;
+ u32 old_period;
+ u64 ticks;
+
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
+
+ ret = kstrtouint(buf, 0, &js_scheduling_period);
+ if (ret || !js_scheduling_period) {
+ dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n"
+ "Use format <js_scheduling_period_ms>\n");
+ return -EINVAL;
+ }
+
+ new_scheduling_period_ns = js_scheduling_period * 1000000;
+
+ /* Update scheduling timeouts */
+ mutex_lock(&kbdev->js_data.runpool_mutex);
+
+ /* If no contexts have been scheduled since js_timeouts was last written
+ * to, the new timeouts might not have been latched yet. So check if an
+ * update is pending and use the new values if necessary. */
+
+ /* Use previous 'new' scheduling period as a base if present. */
+ if (kbdev->js_timeouts_updated && kbdev->js_scheduling_period_ns)
+ old_period = kbdev->js_scheduling_period_ns;
+ else
+ old_period = kbdev->js_data.scheduling_period_ns;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_soft_stop_ticks > 0)
+ ticks = (u64)kbdev->js_soft_stop_ticks * old_period;
+ else
+ ticks = (u64)kbdev->js_data.soft_stop_ticks *
+ kbdev->js_data.scheduling_period_ns;
+ do_div(ticks, new_scheduling_period_ns);
+ kbdev->js_soft_stop_ticks = ticks ? ticks : 1;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_soft_stop_ticks_cl > 0)
+ ticks = (u64)kbdev->js_soft_stop_ticks_cl * old_period;
+ else
+ ticks = (u64)kbdev->js_data.soft_stop_ticks_cl *
+ kbdev->js_data.scheduling_period_ns;
+ do_div(ticks, new_scheduling_period_ns);
+ kbdev->js_soft_stop_ticks_cl = ticks ? ticks : 1;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_hard_stop_ticks_ss > 0)
+ ticks = (u64)kbdev->js_hard_stop_ticks_ss * old_period;
+ else
+ ticks = (u64)kbdev->js_data.hard_stop_ticks_ss *
+ kbdev->js_data.scheduling_period_ns;
+ do_div(ticks, new_scheduling_period_ns);
+ kbdev->js_hard_stop_ticks_ss = ticks ? ticks : 1;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_hard_stop_ticks_cl > 0)
+ ticks = (u64)kbdev->js_hard_stop_ticks_cl * old_period;
+ else
+ ticks = (u64)kbdev->js_data.hard_stop_ticks_cl *
+ kbdev->js_data.scheduling_period_ns;
+ do_div(ticks, new_scheduling_period_ns);
+ kbdev->js_hard_stop_ticks_cl = ticks ? ticks : 1;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_hard_stop_ticks_dumping > 0)
+ ticks = (u64)kbdev->js_hard_stop_ticks_dumping * old_period;
+ else
+ ticks = (u64)kbdev->js_data.hard_stop_ticks_dumping *
+ kbdev->js_data.scheduling_period_ns;
+ do_div(ticks, new_scheduling_period_ns);
+ kbdev->js_hard_stop_ticks_dumping = ticks ? ticks : 1;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_reset_ticks_ss > 0)
+ ticks = (u64)kbdev->js_reset_ticks_ss * old_period;
+ else
+ ticks = (u64)kbdev->js_data.gpu_reset_ticks_ss *
+ kbdev->js_data.scheduling_period_ns;
+ do_div(ticks, new_scheduling_period_ns);
+ kbdev->js_reset_ticks_ss = ticks ? ticks : 1;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_reset_ticks_cl > 0)
+ ticks = (u64)kbdev->js_reset_ticks_cl * old_period;
+ else
+ ticks = (u64)kbdev->js_data.gpu_reset_ticks_cl *
+ kbdev->js_data.scheduling_period_ns;
+ do_div(ticks, new_scheduling_period_ns);
+ kbdev->js_reset_ticks_cl = ticks ? ticks : 1;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_reset_ticks_dumping > 0)
+ ticks = (u64)kbdev->js_reset_ticks_dumping * old_period;
+ else
+ ticks = (u64)kbdev->js_data.gpu_reset_ticks_dumping *
+ kbdev->js_data.scheduling_period_ns;
+ do_div(ticks, new_scheduling_period_ns);
+ kbdev->js_reset_ticks_dumping = ticks ? ticks : 1;
+
+ kbdev->js_scheduling_period_ns = new_scheduling_period_ns;
+ kbdev->js_timeouts_updated = true;
+ mutex_unlock(&kbdev->js_data.runpool_mutex);
+ dev_dbg(kbdev->dev, "JS scheduling period: %dms\n",
+ js_scheduling_period);
+
+ return count;
+}
+
+/**
+ * show_js_scheduling_period - Show callback for the js_scheduling_period sysfs
+ * entry.
+ * @dev: The device this sysfs file is for.
+ * @attr: The attributes of the sysfs file.
+ * @buf: The output buffer to receive the GPU information.
+ *
+ * This function is called to get the current period used for the JS scheduling
+ * period.
+ *
+ * Return: The number of bytes output to buf.
+ */
+static ssize_t show_js_scheduling_period(struct device *dev,
+ struct device_attribute *attr, char * const buf)
+{
+ struct kbase_device *kbdev;
+ u32 period;
+ ssize_t ret;
+
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
+
+ if (kbdev->js_timeouts_updated && kbdev->js_scheduling_period_ns > 0)
+ period = kbdev->js_scheduling_period_ns;
+ else
+ period = kbdev->js_data.scheduling_period_ns;
+
+ ret = scnprintf(buf, PAGE_SIZE, "%d\n",
+ period / 1000000);
+
+ return ret;
+}
+
+static DEVICE_ATTR(js_scheduling_period, S_IRUGO | S_IWUSR,
+ show_js_scheduling_period, set_js_scheduling_period);
+
+#if !MALI_CUSTOMER_RELEASE
/** Store callback for the @c force_replay sysfs file.
*
* @param dev The device with sysfs file is for
int items = sscanf(buf, "limit=%u", &force_replay_limit);
if (items == 1) {
- kbdev->force_replay_random = MALI_FALSE;
+ kbdev->force_replay_random = false;
kbdev->force_replay_limit = force_replay_limit;
kbdev->force_replay_count = 0;
return count;
}
} else if (!strncmp("random_limit", buf, MIN(12, count))) {
- kbdev->force_replay_random = MALI_TRUE;
+ kbdev->force_replay_random = true;
kbdev->force_replay_count = 0;
return count;
} else if (!strncmp("norandom_limit", buf, MIN(14, count))) {
- kbdev->force_replay_random = MALI_FALSE;
+ kbdev->force_replay_random = false;
kbdev->force_replay_limit = KBASEP_FORCE_REPLAY_DISABLED;
kbdev->force_replay_count = 0;
*
* @return The number of bytes output to @c buf.
*/
-static ssize_t show_force_replay(struct device *dev, struct device_attribute *attr, char * const buf)
+static ssize_t show_force_replay(struct device *dev,
+ struct device_attribute *attr, char * const buf)
{
struct kbase_device *kbdev;
ssize_t ret;
/** The sysfs file @c force_replay.
*
*/
-static DEVICE_ATTR(force_replay, S_IRUGO | S_IWUSR, show_force_replay, set_force_replay);
+static DEVICE_ATTR(force_replay, S_IRUGO | S_IWUSR, show_force_replay,
+ set_force_replay);
#endif /* !MALI_CUSTOMER_RELEASE */
#ifdef CONFIG_MALI_DEBUG
-static ssize_t set_js_softstop_always(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+static ssize_t set_js_softstop_always(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
{
struct kbase_device *kbdev;
- int items;
+ int ret;
int softstop_always;
kbdev = to_kbase_device(dev);
if (!kbdev)
return -ENODEV;
- items = sscanf(buf, "%d", &softstop_always);
- if ((items == 1) && ((softstop_always == 0) || (softstop_always == 1))) {
- kbdev->js_data.softstop_always = (mali_bool) softstop_always;
- dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", (kbdev->js_data.softstop_always == MALI_FALSE) ? "Disabled" : "Enabled");
- return count;
- } else {
- dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\nUse format " "<soft_stop_always>\n");
+ ret = kstrtoint(buf, 0, &softstop_always);
+ if (ret || ((softstop_always != 0) && (softstop_always != 1))) {
+ dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n"
+ "Use format <soft_stop_always>\n");
return -EINVAL;
}
+
+ kbdev->js_data.softstop_always = (bool) softstop_always;
+ dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n",
+ (kbdev->js_data.softstop_always) ?
+ "Enabled" : "Disabled");
+ return count;
}
-static ssize_t show_js_softstop_always(struct device *dev, struct device_attribute *attr, char * const buf)
+static ssize_t show_js_softstop_always(struct device *dev,
+ struct device_attribute *attr, char * const buf)
{
struct kbase_device *kbdev;
ssize_t ret;
return ret;
}
-/**
+/*
* By default, soft-stops are disabled when only a single context is present. The ability to
* enable soft-stop when only a single context is present can be used for debug and unit-testing purposes.
* (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.)
static DEVICE_ATTR(debug_command, S_IRUGO | S_IWUSR, show_debug, issue_debug);
#endif /* CONFIG_MALI_DEBUG */
+/**
+ * kbase_show_gpuinfo - Show callback for the gpuinfo sysfs entry.
+ * @dev: The device this sysfs file is for.
+ * @attr: The attributes of the sysfs file.
+ * @buf: The output buffer to receive the GPU information.
+ *
+ * This function is called to get a description of the present Mali
+ * GPU via the gpuinfo sysfs entry. This includes the GPU family, the
+ * number of cores, the hardware version and the raw product id. For
+ * example:
+ *
+ * Mali-T60x MP4 r0p0 0x6956
+ *
+ * Return: The number of bytes output to buf.
+ */
+static ssize_t kbase_show_gpuinfo(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ static const struct gpu_product_id_name {
+ unsigned id;
+ char *name;
+ } gpu_product_id_names[] = {
+ { .id = GPU_ID_PI_T60X, .name = "Mali-T60x" },
+ { .id = GPU_ID_PI_T62X, .name = "Mali-T62x" },
+ { .id = GPU_ID_PI_T72X, .name = "Mali-T72x" },
+ { .id = GPU_ID_PI_T76X, .name = "Mali-T76x" },
+ { .id = GPU_ID_PI_T82X, .name = "Mali-T82x" },
+ { .id = GPU_ID_PI_T83X, .name = "Mali-T83x" },
+ { .id = GPU_ID_PI_T86X, .name = "Mali-T86x" },
+ { .id = GPU_ID_PI_TFRX, .name = "Mali-T88x" },
+#ifdef MALI_INCLUDE_TMIX
+ { .id = GPU_ID_PI_TMIX, .name = "Mali-TMIx" },
+#endif /* MALI_INCLUDE_TMIX */
+ };
+ const char *product_name = "(Unknown Mali GPU)";
+ struct kbase_device *kbdev;
+ u32 gpu_id;
+ unsigned product_id;
+ unsigned i;
+
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
+
+ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
+ product_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT;
+
+ for (i = 0; i < ARRAY_SIZE(gpu_product_id_names); ++i) {
+ if (gpu_product_id_names[i].id == product_id) {
+ product_name = gpu_product_id_names[i].name;
+ break;
+ }
+ }
+
+ return scnprintf(buf, PAGE_SIZE, "%s MP%d r%dp%d 0x%04X\n",
+ product_name, kbdev->gpu_props.num_cores,
+ (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT,
+ (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT,
+ product_id);
+}
+static DEVICE_ATTR(gpuinfo, S_IRUGO, kbase_show_gpuinfo, NULL);
+
+/**
+ * set_dvfs_period - Store callback for the dvfs_period sysfs file.
+ * @dev: The device with sysfs file is for
+ * @attr: The attributes of the sysfs file
+ * @buf: The value written to the sysfs file
+ * @count: The number of bytes written to the sysfs file
+ *
+ * This function is called when the dvfs_period sysfs file is written to. It
+ * checks the data written, and if valid updates the DVFS period variable,
+ *
+ * Return: @c count if the function succeeded. An error code on failure.
+ */
+static ssize_t set_dvfs_period(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct kbase_device *kbdev;
+ int ret;
+ int dvfs_period;
+
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
+
+ ret = kstrtoint(buf, 0, &dvfs_period);
+ if (ret || dvfs_period <= 0) {
+ dev_err(kbdev->dev, "Couldn't process dvfs_period write operation.\n"
+ "Use format <dvfs_period_ms>\n");
+ return -EINVAL;
+ }
+
+ kbdev->pm.dvfs_period = dvfs_period;
+ dev_dbg(kbdev->dev, "DVFS period: %dms\n", dvfs_period);
+
+ return count;
+}
+
+/**
+ * show_dvfs_period - Show callback for the dvfs_period sysfs entry.
+ * @dev: The device this sysfs file is for.
+ * @attr: The attributes of the sysfs file.
+ * @buf: The output buffer to receive the GPU information.
+ *
+ * This function is called to get the current period used for the DVFS sample
+ * timer.
+ *
+ * Return: The number of bytes output to buf.
+ */
+static ssize_t show_dvfs_period(struct device *dev,
+ struct device_attribute *attr, char * const buf)
+{
+ struct kbase_device *kbdev;
+ ssize_t ret;
+
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
+
+ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->pm.dvfs_period);
+
+ return ret;
+}
+
+static DEVICE_ATTR(dvfs_period, S_IRUGO | S_IWUSR, show_dvfs_period,
+ set_dvfs_period);
+
+/**
+ * set_pm_poweroff - Store callback for the pm_poweroff sysfs file.
+ * @dev: The device with sysfs file is for
+ * @attr: The attributes of the sysfs file
+ * @buf: The value written to the sysfs file
+ * @count: The number of bytes written to the sysfs file
+ *
+ * This function is called when the pm_poweroff sysfs file is written to.
+ *
+ * This file contains three values separated by whitespace. The values
+ * are gpu_poweroff_time (the period of the poweroff timer, in ns),
+ * poweroff_shader_ticks (the number of poweroff timer ticks before an idle
+ * shader is powered off), and poweroff_gpu_ticks (the number of poweroff timer
+ * ticks before the GPU is powered off), in that order.
+ *
+ * Return: @c count if the function succeeded. An error code on failure.
+ */
+static ssize_t set_pm_poweroff(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct kbase_device *kbdev;
+ int items;
+ s64 gpu_poweroff_time;
+ int poweroff_shader_ticks, poweroff_gpu_ticks;
+
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
+
+ items = sscanf(buf, "%llu %u %u", &gpu_poweroff_time,
+ &poweroff_shader_ticks,
+ &poweroff_gpu_ticks);
+ if (items != 3) {
+ dev_err(kbdev->dev, "Couldn't process pm_poweroff write operation.\n"
+ "Use format <gpu_poweroff_time_ns> <poweroff_shader_ticks> <poweroff_gpu_ticks>\n");
+ return -EINVAL;
+ }
+
+ kbdev->pm.gpu_poweroff_time = HR_TIMER_DELAY_NSEC(gpu_poweroff_time);
+ kbdev->pm.poweroff_shader_ticks = poweroff_shader_ticks;
+ kbdev->pm.poweroff_gpu_ticks = poweroff_gpu_ticks;
+
+ return count;
+}
+
+/**
+ * show_pm_poweroff - Show callback for the pm_poweroff sysfs entry.
+ * @dev: The device this sysfs file is for.
+ * @attr: The attributes of the sysfs file.
+ * @buf: The output buffer to receive the GPU information.
+ *
+ * This function is called to get the current period used for the DVFS sample
+ * timer.
+ *
+ * Return: The number of bytes output to buf.
+ */
+static ssize_t show_pm_poweroff(struct device *dev,
+ struct device_attribute *attr, char * const buf)
+{
+ struct kbase_device *kbdev;
+ ssize_t ret;
+
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
+
+ ret = scnprintf(buf, PAGE_SIZE, "%llu %u %u\n",
+ ktime_to_ns(kbdev->pm.gpu_poweroff_time),
+ kbdev->pm.poweroff_shader_ticks,
+ kbdev->pm.poweroff_gpu_ticks);
+
+ return ret;
+}
+
+static DEVICE_ATTR(pm_poweroff, S_IRUGO | S_IWUSR, show_pm_poweroff,
+ set_pm_poweroff);
+
+/**
+ * set_reset_timeout - Store callback for the reset_timeout sysfs file.
+ * @dev: The device with sysfs file is for
+ * @attr: The attributes of the sysfs file
+ * @buf: The value written to the sysfs file
+ * @count: The number of bytes written to the sysfs file
+ *
+ * This function is called when the reset_timeout sysfs file is written to. It
+ * checks the data written, and if valid updates the reset timeout.
+ *
+ * Return: @c count if the function succeeded. An error code on failure.
+ */
+static ssize_t set_reset_timeout(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct kbase_device *kbdev;
+ int ret;
+ int reset_timeout;
+
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
+
+ ret = kstrtoint(buf, 0, &reset_timeout);
+ if (ret || reset_timeout <= 0) {
+ dev_err(kbdev->dev, "Couldn't process reset_timeout write operation.\n"
+ "Use format <reset_timeout_ms>\n");
+ return -EINVAL;
+ }
+
+ kbdev->reset_timeout_ms = reset_timeout;
+ dev_dbg(kbdev->dev, "Reset timeout: %dms\n", reset_timeout);
+
+ return count;
+}
+
+/**
+ * show_reset_timeout - Show callback for the reset_timeout sysfs entry.
+ * @dev: The device this sysfs file is for.
+ * @attr: The attributes of the sysfs file.
+ * @buf: The output buffer to receive the GPU information.
+ *
+ * This function is called to get the current reset timeout.
+ *
+ * Return: The number of bytes output to buf.
+ */
+static ssize_t show_reset_timeout(struct device *dev,
+ struct device_attribute *attr, char * const buf)
+{
+ struct kbase_device *kbdev;
+ ssize_t ret;
+
+ kbdev = to_kbase_device(dev);
+ if (!kbdev)
+ return -ENODEV;
+
+ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->reset_timeout_ms);
+
+ return ret;
+}
+
+static DEVICE_ATTR(reset_timeout, S_IRUGO | S_IWUSR, show_reset_timeout,
+ set_reset_timeout);
#ifdef CONFIG_MALI_NO_MALI
static int kbase_common_reg_map(struct kbase_device *kbdev)
{
int err = -ENOMEM;
- kbdev->reg_res = request_mem_region(kbdev->reg_start, kbdev->reg_size, dev_name(kbdev->dev));
- if (!kbdev->reg_res) {
- dev_err(kbdev->dev, "Register window unavailable\n");
- err = -EIO;
- goto out_region;
+ if (!request_mem_region(kbdev->reg_start, kbdev->reg_size, dev_name(kbdev->dev))) {
+ dev_err(kbdev->dev, "Register window unavailable\n");
+ err = -EIO;
+ goto out_region;
+ }
+
+ kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size);
+ if (!kbdev->reg) {
+ dev_err(kbdev->dev, "Can't remap register window\n");
+ err = -EINVAL;
+ goto out_ioremap;
+ }
+
+ return 0;
+
+ out_ioremap:
+ release_mem_region(kbdev->reg_start, kbdev->reg_size);
+ out_region:
+ return err;
+}
+
+static void kbase_common_reg_unmap(struct kbase_device * const kbdev)
+{
+ iounmap(kbdev->reg);
+ release_mem_region(kbdev->reg_start, kbdev->reg_size);
+}
+#endif /* CONFIG_MALI_NO_MALI */
+
+
+#ifdef CONFIG_DEBUG_FS
+
+#if KBASE_GPU_RESET_EN
+#include <mali_kbase_hwaccess_jm.h>
+
+static void trigger_quirks_reload(struct kbase_device *kbdev)
+{
+ kbase_pm_context_active(kbdev);
+ if (kbase_prepare_to_reset_gpu(kbdev))
+ kbase_reset_gpu(kbdev);
+ kbase_pm_context_idle(kbdev);
+}
+
+#define MAKE_QUIRK_ACCESSORS(type) \
+static int type##_quirks_set(void *data, u64 val) \
+{ \
+ struct kbase_device *kbdev; \
+ kbdev = (struct kbase_device *)data; \
+ kbdev->hw_quirks_##type = (u32)val; \
+ trigger_quirks_reload(kbdev); \
+ return 0;\
+} \
+\
+static int type##_quirks_get(void *data, u64 *val) \
+{ \
+ struct kbase_device *kbdev;\
+ kbdev = (struct kbase_device *)data;\
+ *val = kbdev->hw_quirks_##type;\
+ return 0;\
+} \
+DEFINE_SIMPLE_ATTRIBUTE(fops_##type##_quirks, type##_quirks_get,\
+ type##_quirks_set, "%llu\n")
+
+MAKE_QUIRK_ACCESSORS(sc);
+MAKE_QUIRK_ACCESSORS(tiler);
+MAKE_QUIRK_ACCESSORS(mmu);
+
+#endif /* KBASE_GPU_RESET_EN */
+
+static int kbase_device_debugfs_init(struct kbase_device *kbdev)
+{
+ struct dentry *debugfs_ctx_defaults_directory;
+ int err;
+
+ kbdev->mali_debugfs_directory = debugfs_create_dir(kbdev->devname,
+ NULL);
+ if (!kbdev->mali_debugfs_directory) {
+ dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n");
+ err = -ENOMEM;
+ goto out;
+ }
+
+ kbdev->debugfs_ctx_directory = debugfs_create_dir("ctx",
+ kbdev->mali_debugfs_directory);
+ if (!kbdev->debugfs_ctx_directory) {
+ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx directory\n");
+ err = -ENOMEM;
+ goto out;
}
- kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size);
- if (!kbdev->reg) {
- dev_err(kbdev->dev, "Can't remap register window\n");
- err = -EINVAL;
- goto out_ioremap;
+ debugfs_ctx_defaults_directory = debugfs_create_dir("defaults",
+ kbdev->debugfs_ctx_directory);
+ if (!debugfs_ctx_defaults_directory) {
+ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx defaults directory\n");
+ err = -ENOMEM;
+ goto out;
}
+ kbasep_gpu_memory_debugfs_init(kbdev);
+#if KBASE_GPU_RESET_EN
+ debugfs_create_file("quirks_sc", 0644,
+ kbdev->mali_debugfs_directory, kbdev,
+ &fops_sc_quirks);
+ debugfs_create_file("quirks_tiler", 0644,
+ kbdev->mali_debugfs_directory, kbdev,
+ &fops_tiler_quirks);
+ debugfs_create_file("quirks_mmu", 0644,
+ kbdev->mali_debugfs_directory, kbdev,
+ &fops_mmu_quirks);
+#endif /* KBASE_GPU_RESET_EN */
+
+#ifndef CONFIG_MALI_COH_USER
+ debugfs_create_bool("infinite_cache", 0644,
+ debugfs_ctx_defaults_directory,
+ &kbdev->infinite_cache_active_default);
+#endif /* CONFIG_MALI_COH_USER */
+
+#if KBASE_TRACE_ENABLE
+ kbasep_trace_debugfs_init(kbdev);
+#endif /* KBASE_TRACE_ENABLE */
+
+#ifdef CONFIG_MALI_TRACE_TIMELINE
+ kbasep_trace_timeline_debugfs_init(kbdev);
+#endif /* CONFIG_MALI_TRACE_TIMELINE */
+
return 0;
- out_ioremap:
- release_resource(kbdev->reg_res);
- kfree(kbdev->reg_res);
- out_region:
+out:
+ debugfs_remove_recursive(kbdev->mali_debugfs_directory);
return err;
}
-static void kbase_common_reg_unmap(struct kbase_device * const kbdev)
+static void kbase_device_debugfs_term(struct kbase_device *kbdev)
{
- iounmap(kbdev->reg);
- release_resource(kbdev->reg_res);
- kfree(kbdev->reg_res);
+ debugfs_remove_recursive(kbdev->mali_debugfs_directory);
}
-#endif /* CONFIG_MALI_NO_MALI */
+
+#else /* CONFIG_DEBUG_FS */
+static inline int kbase_device_debugfs_init(struct kbase_device *kbdev)
+{
+ return 0;
+}
+
+static inline void kbase_device_debugfs_term(struct kbase_device *kbdev) { }
+#endif /* CONFIG_DEBUG_FS */
+
static int kbase_common_device_init(struct kbase_device *kbdev)
{
- int err = -ENOMEM;
- mali_error mali_err;
+ int err;
+ struct mali_base_gpu_core_props *core_props;
enum {
inited_mem = (1u << 0),
- inited_job_slot = (1u << 1),
- inited_pm = (1u << 2),
- inited_js = (1u << 3),
- inited_irqs = (1u << 4),
- inited_debug = (1u << 5),
- inited_js_softstop = (1u << 6),
+ inited_js = (1u << 1),
+ inited_debug = (1u << 2),
+ inited_js_softstop = (1u << 3),
+ inited_js_timeouts = (1u << 4),
#if !MALI_CUSTOMER_RELEASE
- inited_js_timeouts = (1u << 7),
- inited_force_replay = (1u << 13),
+ inited_force_replay = (1u << 5),
#endif /* !MALI_CUSTOMER_RELEASE */
- inited_pm_runtime_init = (1u << 8),
-#ifdef CONFIG_DEBUG_FS
- inited_gpu_memory = (1u << 9),
-#endif /* CONFIG_DEBUG_FS */
+ inited_pm_runtime_init = (1u << 6),
#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
- inited_sc_split = (1u << 11),
+ inited_sc_split = (1u << 7),
#endif /* CONFIG_MALI_DEBUG_SHADER_SPLIT_FS */
#ifdef CONFIG_MALI_TRACE_TIMELINE
- inited_timeline = (1u << 12),
-#endif /* CONFIG_MALI_TRACE_LINE */
- inited_pm_powerup = (1u << 14),
+ inited_timeline = (1u << 8),
+#endif /* CONFIG_MALI_TRACE_TIMELINE */
+#ifdef CONFIG_MALI_DEVFREQ
+ inited_devfreq = (1u << 9),
+#endif /* CONFIG_MALI_DEVFREQ */
+#ifdef CONFIG_MALI_MIPE_ENABLED
+ inited_tlstream = (1u << 10),
+#endif /* CONFIG_MALI_MIPE_ENABLED */
+ inited_backend_early = (1u << 11),
+ inited_backend_late = (1u << 12),
+ inited_device = (1u << 13),
+ inited_gpuinfo = (1u << 14),
+ inited_dvfs_period = (1u << 15),
+ inited_pm_poweroff = (1u << 16),
+ inited_reset_timeout = (1u << 17),
+ inited_js_scheduling_period = (1u << 18),
+ inited_vinstr = (1u << 19)
};
int inited = 0;
+#if defined(CONFIG_MALI_PLATFORM_VEXPRESS)
+ u32 ve_logic_tile = 0;
+#endif /* CONFIG_MALI_PLATFORM_VEXPRESS */
dev_set_drvdata(kbdev->dev, kbdev);
- kbdev->mdev.minor = MISC_DYNAMIC_MINOR;
- kbdev->mdev.name = kbdev->devname;
- kbdev->mdev.fops = &kbase_fops;
- kbdev->mdev.parent = get_device(kbdev->dev);
+ err = kbase_backend_early_init(kbdev);
+ if (err)
+ goto out_partial;
+ inited |= inited_backend_early;
+
+ scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name,
+ kbase_dev_nr++);
kbase_disjoint_init(kbdev);
- scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name, kbase_dev_nr++);
+ /* obtain min/max configured gpu frequencies */
+ core_props = &(kbdev->gpu_props.props.core_props);
- if (misc_register(&kbdev->mdev)) {
- dev_err(kbdev->dev, "Couldn't register misc dev %s\n", kbdev->devname);
- err = -EINVAL;
- goto out_misc;
- }
-#if KBASE_PM_EN
- if (device_create_file(kbdev->dev, &dev_attr_power_policy)) {
- dev_err(kbdev->dev, "Couldn't create power_policy sysfs file\n");
- goto out_file;
+ /* For versatile express platforms, min and max values of GPU frequency
+ * depend on the type of the logic tile; these values may not be known
+ * at the build time so in some cases a platform config file with wrong
+ * GPU freguency values may be included; to ensure the correct value of
+ * min and max GPU frequency is obtained, the type of the logic tile is
+ * read from the corresponding register on the platform and frequency
+ * values assigned accordingly.*/
+#if defined(CONFIG_MALI_PLATFORM_VEXPRESS)
+ ve_logic_tile = kbase_get_platform_logic_tile_type();
+
+ switch (ve_logic_tile) {
+ case 0x217:
+ /* Virtex 6, HBI0217 */
+ core_props->gpu_freq_khz_min = VE_VIRTEX6_GPU_FREQ_MIN;
+ core_props->gpu_freq_khz_max = VE_VIRTEX6_GPU_FREQ_MAX;
+ break;
+ case 0x247:
+ /* Virtex 7, HBI0247 */
+ core_props->gpu_freq_khz_min = VE_VIRTEX7_GPU_FREQ_MIN;
+ core_props->gpu_freq_khz_max = VE_VIRTEX7_GPU_FREQ_MAX;
+ break;
+ default:
+ /* all other logic tiles, i.e., Virtex 5 HBI0192
+ * or unsuccessful reading from the platform -
+ * fall back to the config_platform default */
+ core_props->gpu_freq_khz_min = GPU_FREQ_KHZ_MIN;
+ core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX;
+ break;
}
+#else
+ core_props->gpu_freq_khz_min = GPU_FREQ_KHZ_MIN;
+ core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX;
+#endif /* CONFIG_MALI_PLATFORM_VEXPRESS */
- if (device_create_file(kbdev->dev, &dev_attr_core_availability_policy)) {
- dev_err(kbdev->dev, "Couldn't create core_availability_policy sysfs file\n");
- goto out_file_core_availability_policy;
- }
+ kbdev->gpu_props.irq_throttle_time_us = DEFAULT_IRQ_THROTTLE_TIME_US;
- if (device_create_file(kbdev->dev, &dev_attr_core_mask)) {
- dev_err(kbdev->dev, "Couldn't create core_mask sysfs file\n");
- goto out_file_core_mask;
+ err = kbase_device_init(kbdev);
+ if (err) {
+ dev_err(kbdev->dev, "Can't initialize device (%d)\n", err);
+ goto out_partial;
}
-#endif /* KBASE_PM_EN */
- down(&kbase_dev_list_lock);
- list_add(&kbdev->entry, &kbase_dev_list);
- up(&kbase_dev_list_lock);
- dev_info(kbdev->dev, "Probed as %s\n", dev_name(kbdev->mdev.this_device));
- mali_err = kbase_pm_init(kbdev);
- if (MALI_ERROR_NONE != mali_err)
+ inited |= inited_device;
+
+ kbdev->vinstr_ctx = kbase_vinstr_init(kbdev);
+ if (!kbdev->vinstr_ctx) {
+ dev_err(kbdev->dev, "Can't initialize virtual instrumentation core\n");
goto out_partial;
+ }
- inited |= inited_pm;
+ inited |= inited_vinstr;
if (kbdev->pm.callback_power_runtime_init) {
- mali_err = kbdev->pm.callback_power_runtime_init(kbdev);
- if (MALI_ERROR_NONE != mali_err)
+ err = kbdev->pm.callback_power_runtime_init(kbdev);
+ if (err)
goto out_partial;
inited |= inited_pm_runtime_init;
}
- mali_err = kbase_mem_init(kbdev);
- if (MALI_ERROR_NONE != mali_err)
+ err = kbase_mem_init(kbdev);
+ if (err)
goto out_partial;
inited |= inited_mem;
- mali_err = kbase_job_slot_init(kbdev);
- if (MALI_ERROR_NONE != mali_err)
- goto out_partial;
-
- inited |= inited_job_slot;
-
- mali_err = kbasep_js_devdata_init(kbdev);
- if (MALI_ERROR_NONE != mali_err)
- goto out_partial;
+ kbdev->system_coherency = COHERENCY_NONE;
- inited |= inited_js;
- err = kbase_install_interrupts(kbdev);
+ err = kbasep_js_devdata_init(kbdev);
if (err)
goto out_partial;
- inited |= inited_irqs;
+ inited |= inited_js;
#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
- if (device_create_file(kbdev->dev, &dev_attr_sc_split)) {
+ err = device_create_file(kbdev->dev, &dev_attr_sc_split);
+ if (err) {
dev_err(kbdev->dev, "Couldn't create sc_split sysfs file\n");
goto out_partial;
}
inited |= inited_sc_split;
#endif /* CONFIG_MALI_DEBUG_SHADER_SPLIT_FS */
-#ifdef CONFIG_DEBUG_FS
- if (kbasep_gpu_memory_debugfs_init(kbdev)) {
- dev_err(kbdev->dev, "Couldn't create gpu_memory debugfs file\n");
- goto out_partial;
- }
- inited |= inited_gpu_memory;
-#endif /* CONFIG_DEBUG_FS */
-
#ifdef CONFIG_MALI_DEBUG
- if (device_create_file(kbdev->dev, &dev_attr_debug_command)) {
+ err = device_create_file(kbdev->dev, &dev_attr_debug_command);
+ if (err) {
dev_err(kbdev->dev, "Couldn't create debug_command sysfs file\n");
goto out_partial;
}
inited |= inited_debug;
- if (device_create_file(kbdev->dev, &dev_attr_js_softstop_always)) {
+ err = device_create_file(kbdev->dev, &dev_attr_js_softstop_always);
+ if (err) {
dev_err(kbdev->dev, "Couldn't create js_softstop_always sysfs file\n");
goto out_partial;
}
inited |= inited_js_softstop;
#endif /* CONFIG_MALI_DEBUG */
-#if !MALI_CUSTOMER_RELEASE
- if (device_create_file(kbdev->dev, &dev_attr_js_timeouts)) {
+ err = device_create_file(kbdev->dev, &dev_attr_js_timeouts);
+ if (err) {
dev_err(kbdev->dev, "Couldn't create js_timeouts sysfs file\n");
goto out_partial;
}
inited |= inited_js_timeouts;
- if (device_create_file(kbdev->dev, &dev_attr_force_replay)) {
+#if !MALI_CUSTOMER_RELEASE
+ err = device_create_file(kbdev->dev, &dev_attr_force_replay);
+ if (err) {
dev_err(kbdev->dev, "Couldn't create force_replay sysfs file\n");
goto out_partial;
}
inited |= inited_force_replay;
#endif /* !MALI_CUSTOMER_RELEASE */
-#ifdef CONFIG_MALI_TRACE_TIMELINE
- if (kbasep_trace_timeline_debugfs_init(kbdev)) {
- dev_err(kbdev->dev, "Couldn't create mali_timeline_defs debugfs file\n");
+ err = device_create_file(kbdev->dev, &dev_attr_gpuinfo);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't create gpuinfo sysfs file\n");
goto out_partial;
}
- inited |= inited_timeline;
-#endif /* CONFIG_MALI_TRACE_TIMELINE */
+ inited |= inited_gpuinfo;
+
+ err = device_create_file(kbdev->dev, &dev_attr_dvfs_period);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't create dvfs_period sysfs file\n");
+ goto out_partial;
+ }
+ inited |= inited_dvfs_period;
+
+ err = device_create_file(kbdev->dev, &dev_attr_pm_poweroff);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't create pm_poweroff sysfs file\n");
+ goto out_partial;
+ }
+ inited |= inited_pm_poweroff;
+
+ err = device_create_file(kbdev->dev, &dev_attr_reset_timeout);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't create reset_timeout sysfs file\n");
+ goto out_partial;
+ }
+ inited |= inited_reset_timeout;
+
+ err = device_create_file(kbdev->dev, &dev_attr_js_scheduling_period);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't create js_scheduling_period sysfs file\n");
+ goto out_partial;
+ }
+ inited |= inited_js_scheduling_period;
+
+#ifdef CONFIG_MALI_MIPE_ENABLED
+ err = kbase_tlstream_init();
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't initialize timeline stream\n");
+ goto out_partial;
+ }
+ inited |= inited_tlstream;
+#endif /* CONFIG_MALI_MIPE_ENABLED */
+
+ err = kbase_backend_late_init(kbdev);
+ if (err)
+ goto out_partial;
+ inited |= inited_backend_late;
#ifdef CONFIG_MALI_DEVFREQ
- kbase_devfreq_init(kbdev);
+ err = kbase_devfreq_init(kbdev);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't initialize devfreq\n");
+ goto out_partial;
+ }
+ inited |= inited_devfreq;
+#endif /* CONFIG_MALI_DEVFREQ */
+
+#ifdef SECURE_CALLBACKS
+ kbdev->secure_ops = SECURE_CALLBACKS;
#endif
- mali_err = kbase_pm_powerup(kbdev);
- if (MALI_ERROR_NONE == mali_err) {
- inited |= inited_pm_powerup;
-#ifdef CONFIG_MALI_DEBUG
-#if !defined(CONFIG_MALI_NO_MALI)
- if (MALI_ERROR_NONE != kbasep_common_test_interrupt_handlers(kbdev)) {
- dev_err(kbdev->dev, "Interrupt assigment check failed.\n");
- err = -EINVAL;
- goto out_partial;
- }
-#endif /* CONFIG_MALI_NO_MALI */
-#endif /* CONFIG_MALI_DEBUG */
- /* intialise the kctx list */
- mutex_init(&kbdev->kctx_list_lock);
- INIT_LIST_HEAD(&kbdev->kctx_list);
- return 0;
- } else {
- /* Failed to power up the GPU. */
- dev_err(kbdev->dev, "GPU power up failed.\n");
- err = -ENODEV;
+ err = kbase_device_debugfs_init(kbdev);
+ if (err)
+ goto out_partial;
+
+ /* intialise the kctx list */
+ mutex_init(&kbdev->kctx_list_lock);
+ INIT_LIST_HEAD(&kbdev->kctx_list);
+
+ kbdev->mdev.minor = MISC_DYNAMIC_MINOR;
+ kbdev->mdev.name = kbdev->devname;
+ kbdev->mdev.fops = &kbase_fops;
+ kbdev->mdev.parent = get_device(kbdev->dev);
+
+ err = misc_register(&kbdev->mdev);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't register misc dev %s\n",
+ kbdev->devname);
+ goto out_misc;
}
- out_partial:
-#ifdef CONFIG_MALI_TRACE_TIMELINE
- if (inited & inited_timeline)
- kbasep_trace_timeline_debugfs_term(kbdev);
-#endif /* CONFIG_MALI_TRACE_TIMELINE */
+ err = device_create_file(kbdev->dev, &dev_attr_power_policy);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't create power_policy sysfs file\n");
+ goto out_file;
+ }
+
+ err = device_create_file(kbdev->dev,
+ &dev_attr_core_availability_policy);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't create core_availability_policy sysfs file\n");
+ goto out_file_core_availability_policy;
+ }
+
+ err = device_create_file(kbdev->dev, &dev_attr_core_mask);
+ if (err) {
+ dev_err(kbdev->dev, "Couldn't create core_mask sysfs file\n");
+ goto out_file_core_mask;
+ }
+
+ {
+ const struct list_head *dev_list = kbase_dev_list_get();
+
+ list_add(&kbdev->entry, &kbase_dev_list);
+ kbase_dev_list_put(dev_list);
+ }
+
+ dev_info(kbdev->dev, "Probed as %s\n",
+ dev_name(kbdev->mdev.this_device));
+
+ return 0;
+
+out_file_core_mask:
+ device_remove_file(kbdev->dev, &dev_attr_core_availability_policy);
+out_file_core_availability_policy:
+ device_remove_file(kbdev->dev, &dev_attr_power_policy);
+out_file:
+ misc_deregister(&kbdev->mdev);
+out_misc:
+ put_device(kbdev->dev);
+ kbase_device_debugfs_term(kbdev);
+out_partial:
+ if (inited & inited_vinstr)
+ kbase_vinstr_term(kbdev->vinstr_ctx);
+#ifdef CONFIG_MALI_DEVFREQ
+ if (inited & inited_devfreq)
+ kbase_devfreq_term(kbdev);
+#endif /* CONFIG_MALI_DEVFREQ */
+ if (inited & inited_backend_late)
+ kbase_backend_late_term(kbdev);
+#ifdef CONFIG_MALI_MIPE_ENABLED
+ if (inited & inited_tlstream)
+ kbase_tlstream_term();
+#endif /* CONFIG_MALI_MIPE_ENABLED */
+
+ if (inited & inited_js_scheduling_period)
+ device_remove_file(kbdev->dev, &dev_attr_js_scheduling_period);
+ if (inited & inited_reset_timeout)
+ device_remove_file(kbdev->dev, &dev_attr_reset_timeout);
+ if (inited & inited_pm_poweroff)
+ device_remove_file(kbdev->dev, &dev_attr_pm_poweroff);
+ if (inited & inited_dvfs_period)
+ device_remove_file(kbdev->dev, &dev_attr_dvfs_period);
#if !MALI_CUSTOMER_RELEASE
if (inited & inited_force_replay)
device_remove_file(kbdev->dev, &dev_attr_force_replay);
+#endif /* !MALI_CUSTOMER_RELEASE */
if (inited & inited_js_timeouts)
device_remove_file(kbdev->dev, &dev_attr_js_timeouts);
-#endif /* !MALI_CUSTOMER_RELEASE */
#ifdef CONFIG_MALI_DEBUG
if (inited & inited_js_softstop)
device_remove_file(kbdev->dev, &dev_attr_js_softstop_always);
#endif /* CONFIG_MALI_DEBUG */
-#ifdef CONFIG_DEBUG_FS
- if (inited & inited_gpu_memory)
- kbasep_gpu_memory_debugfs_term(kbdev);
-#endif /* CONFIG_DEBUG_FS */
-
#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
if (inited & inited_sc_split)
device_remove_file(kbdev->dev, &dev_attr_sc_split);
#endif /* CONFIG_MALI_DEBUG_SHADER_SPLIT_FS */
+ if (inited & inited_gpuinfo)
+ device_remove_file(kbdev->dev, &dev_attr_gpuinfo);
+
if (inited & inited_js)
kbasep_js_devdata_halt(kbdev);
- if (inited & inited_job_slot)
- kbase_job_slot_halt(kbdev);
-
if (inited & inited_mem)
kbase_mem_halt(kbdev);
- if (inited & inited_pm_powerup)
- kbase_pm_halt(kbdev);
-
- if (inited & inited_irqs)
- kbase_release_interrupts(kbdev);
-
if (inited & inited_js)
kbasep_js_devdata_term(kbdev);
- if (inited & inited_job_slot)
- kbase_job_slot_term(kbdev);
-
if (inited & inited_mem)
kbase_mem_term(kbdev);
kbdev->pm.callback_power_runtime_term(kbdev);
}
- if (inited & inited_pm)
- kbase_pm_term(kbdev);
+ if (inited & inited_device)
+ kbase_device_term(kbdev);
+
+ if (inited & inited_backend_early)
+ kbase_backend_early_term(kbdev);
- down(&kbase_dev_list_lock);
- list_del(&kbdev->entry);
- up(&kbase_dev_list_lock);
-#if KBASE_PM_EN
- device_remove_file(kbdev->dev, &dev_attr_core_mask);
- out_file_core_mask:
- device_remove_file(kbdev->dev, &dev_attr_core_availability_policy);
- out_file_core_availability_policy:
- device_remove_file(kbdev->dev, &dev_attr_power_policy);
- out_file:
-#endif /*KBASE_PM_EN*/
- misc_deregister(&kbdev->mdev);
- out_misc:
- put_device(kbdev->dev);
return err;
}
{
struct kbase_device *kbdev;
struct resource *reg_res;
- struct kbase_attribute *platform_data;
int err;
int i;
- struct mali_base_gpu_core_props *core_props;
-#ifdef CONFIG_MALI_NO_MALI
- mali_error mali_err;
-#endif /* CONFIG_MALI_NO_MALI */
-#ifdef CONFIG_OF
- struct kbase_platform_config *config;
- int attribute_count;
printk(KERN_INFO "arm_release_ver of this mali_ko is '%s', rk_ko_ver is '%d', built at '%s', on '%s'.",
MALI_RELEASE_NAME,
__TIME__,
__DATE__);
- config = kbase_get_platform_config();
- attribute_count = kbasep_get_config_attribute_count(config->attributes);
-
- err = platform_device_add_data(pdev, config->attributes,
- attribute_count * sizeof(config->attributes[0]));
- if (err)
+#ifdef CONFIG_OF
+ err = kbase_platform_early_init();
+ if (err) {
+ dev_err(&pdev->dev, "Early platform initialization failed\n");
return err;
-#endif /* CONFIG_OF */
+ }
+#endif
kbdev = kbase_device_alloc();
if (!kbdev) {
goto out;
}
#ifdef CONFIG_MALI_NO_MALI
- mali_err = midg_device_create(kbdev);
- if (MALI_ERROR_NONE != mali_err) {
+ err = gpu_device_create(kbdev);
+ if (err) {
dev_err(&pdev->dev, "Can't initialize dummy model\n");
- err = -ENOMEM;
goto out_midg;
}
#endif /* CONFIG_MALI_NO_MALI */
kbdev->dev = &pdev->dev;
- platform_data = (struct kbase_attribute *)kbdev->dev->platform_data;
-
- if (NULL == platform_data) {
- dev_err(kbdev->dev, "Platform data not specified\n");
- err = -ENOENT;
- goto out_free_dev;
- }
-
- if (MALI_TRUE != kbasep_validate_configuration_attributes(kbdev, platform_data)) {
- dev_err(kbdev->dev, "Configuration attributes failed to validate\n");
- err = -EINVAL;
- goto out_free_dev;
- }
- kbdev->config_attributes = platform_data;
/* 3 IRQ resources */
for (i = 0; i < 3; i++) {
if (!irq_res) {
dev_err(kbdev->dev, "No IRQ resource at index %d\n", i);
err = -ENOENT;
- goto out_free_dev;
+ goto out_platform_irq;
}
#ifdef CONFIG_OF
dev_err(&pdev->dev, "Invalid irq res name: '%s'\n",
irq_res->name);
err = -EINVAL;
- goto out_free_dev;
+ goto out_irq_name;
}
#else
irqtag = i;
if (!reg_res) {
dev_err(kbdev->dev, "Invalid register resource\n");
err = -ENOENT;
- goto out_free_dev;
+ goto out_platform_mem;
}
kbdev->reg_start = reg_res->start;
err = kbase_common_reg_map(kbdev);
if (err)
- goto out_free_dev;
+ goto out_reg_map;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \
+ && defined(CONFIG_REGULATOR)
+ kbdev->regulator = regulator_get_optional(kbdev->dev, "mali");
+ if (IS_ERR_OR_NULL(kbdev->regulator)) {
+ dev_info(kbdev->dev, "Continuing without Mali regulator control\n");
+ kbdev->regulator = NULL;
+ /* Allow probe to continue without regulator */
+ }
+#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */
+#ifdef CONFIG_MALI_PLATFORM_DEVICETREE
+ pm_runtime_enable(kbdev->dev);
+#endif
kbdev->clock = clk_get(kbdev->dev, "clk_mali");
if (IS_ERR_OR_NULL(kbdev->clock)) {
dev_info(kbdev->dev, "Continuing without Mali clock control\n");
if (err) {
dev_err(kbdev->dev,
"Failed to prepare and enable clock (%d)\n", err);
- goto out_clock_get;
+ goto out_clock_prepare;
}
}
-#ifdef CONFIG_DEBUG_FS
- kbdev->mali_debugfs_directory = debugfs_create_dir("mali", NULL);
- if (NULL == kbdev->mali_debugfs_directory) {
- dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n");
- goto out_clock_enable;
- }
- kbdev->memory_profile_directory = debugfs_create_dir("mem",
- kbdev->mali_debugfs_directory);
- if (NULL == kbdev->memory_profile_directory) {
- dev_err(kbdev->dev, "Couldn't create mali mem debugfs directory\n");
- goto out_mali_debugfs_remove;
- }
- if (kbasep_jd_debugfs_init(kbdev)) {
- dev_err(kbdev->dev, "Couldn't create mali jd debugfs entries\n");
- goto out_mem_profile_remove;
- }
-#endif /* CONFIG_DEBUG_FS */
-
-
- if (MALI_ERROR_NONE != kbase_device_init(kbdev)) {
- dev_err(kbdev->dev, "Can't initialize device\n");
-
- err = -ENOMEM;
- goto out_debugfs_remove;
- }
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) && defined(CONFIG_OF) \
+ && defined(CONFIG_PM_OPP)
+ /* Register the OPPs if they are available in device tree */
+ if (of_init_opp_table(kbdev->dev) < 0)
+ dev_dbg(kbdev->dev, "OPP table not found\n");
+#endif
- /* obtain min/max configured gpu frequencies */
- core_props = &(kbdev->gpu_props.props.core_props);
- core_props->gpu_freq_khz_min = GPU_FREQ_KHZ_MIN;
- core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX;
- kbdev->gpu_props.irq_throttle_time_us = DEFAULT_IRQ_THROTTLE_TIME_US;
err = kbase_common_device_init(kbdev);
if (err) {
dev_err(kbdev->dev, "Failed kbase_common_device_init\n");
- goto out_term_dev;
+ goto out_common_init;
}
return 0;
-out_term_dev:
- kbase_device_term(kbdev);
-out_debugfs_remove:
-#ifdef CONFIG_DEBUG_FS
- kbasep_jd_debugfs_term(kbdev);
-out_mem_profile_remove:
- debugfs_remove(kbdev->memory_profile_directory);
-out_mali_debugfs_remove:
- debugfs_remove(kbdev->mali_debugfs_directory);
-out_clock_enable:
-#endif /* CONFIG_DEBUG_FS */
+out_common_init:
clk_disable_unprepare(kbdev->clock);
-out_clock_get:
+out_clock_prepare:
clk_put(kbdev->clock);
+#ifdef CONFIG_MALI_PLATFORM_DEVICETREE
+ pm_runtime_disable(kbdev->dev);
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \
+ && defined(CONFIG_REGULATOR)
+ regulator_put(kbdev->regulator);
+#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */
kbase_common_reg_unmap(kbdev);
-out_free_dev:
+out_reg_map:
+out_platform_mem:
+#ifdef CONFIG_OF
+out_irq_name:
+#endif
+out_platform_irq:
#ifdef CONFIG_MALI_NO_MALI
- midg_device_destroy(kbdev);
+ gpu_device_destroy(kbdev);
out_midg:
#endif /* CONFIG_MALI_NO_MALI */
kbase_device_free(kbdev);
static int kbase_common_device_remove(struct kbase_device *kbdev)
{
+ kbase_vinstr_term(kbdev->vinstr_ctx);
+#ifdef CONFIG_DEBUG_FS
+ debugfs_remove_recursive(kbdev->mali_debugfs_directory);
+#endif
#ifdef CONFIG_MALI_DEVFREQ
kbase_devfreq_term(kbdev);
#endif
+ kbase_backend_late_term(kbdev);
+
if (kbdev->pm.callback_power_runtime_term)
kbdev->pm.callback_power_runtime_term(kbdev);
-#if KBASE_PM_EN
- /* Remove the sys power policy file */
+#ifdef CONFIG_MALI_PLATFORM_DEVICETREE
+ pm_runtime_disable(kbdev->dev);
+#endif
+ device_remove_file(kbdev->dev, &dev_attr_js_scheduling_period);
+ device_remove_file(kbdev->dev, &dev_attr_reset_timeout);
+ device_remove_file(kbdev->dev, &dev_attr_pm_poweroff);
+ device_remove_file(kbdev->dev, &dev_attr_dvfs_period);
device_remove_file(kbdev->dev, &dev_attr_power_policy);
device_remove_file(kbdev->dev, &dev_attr_core_availability_policy);
device_remove_file(kbdev->dev, &dev_attr_core_mask);
-#endif
-#ifdef CONFIG_MALI_TRACE_TIMELINE
- kbasep_trace_timeline_debugfs_term(kbdev);
-#endif /* CONFIG_MALI_TRACE_TIMELINE */
+
+#ifdef CONFIG_MALI_MIPE_ENABLED
+ kbase_tlstream_term();
+#endif /* CONFIG_MALI_MIPE_ENABLED */
#ifdef CONFIG_MALI_DEBUG
device_remove_file(kbdev->dev, &dev_attr_js_softstop_always);
device_remove_file(kbdev->dev, &dev_attr_debug_command);
#endif /* CONFIG_MALI_DEBUG */
-#if !MALI_CUSTOMER_RELEASE
device_remove_file(kbdev->dev, &dev_attr_js_timeouts);
+#if !MALI_CUSTOMER_RELEASE
device_remove_file(kbdev->dev, &dev_attr_force_replay);
#endif /* !MALI_CUSTOMER_RELEASE */
-#ifdef CONFIG_DEBUG_FS
- kbasep_gpu_memory_debugfs_term(kbdev);
-#endif
#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
device_remove_file(kbdev->dev, &dev_attr_sc_split);
#endif /* CONFIG_MALI_DEBUG_SHADER_SPLIT_FS */
+ device_remove_file(kbdev->dev, &dev_attr_gpuinfo);
kbasep_js_devdata_halt(kbdev);
- kbase_job_slot_halt(kbdev);
kbase_mem_halt(kbdev);
- kbase_pm_halt(kbdev);
-
- kbase_release_interrupts(kbdev);
kbasep_js_devdata_term(kbdev);
- kbase_job_slot_term(kbdev);
kbase_mem_term(kbdev);
- kbase_pm_term(kbdev);
+ kbase_backend_early_term(kbdev);
- down(&kbase_dev_list_lock);
- list_del(&kbdev->entry);
- up(&kbase_dev_list_lock);
+ {
+ const struct list_head *dev_list = kbase_dev_list_get();
+ list_del(&kbdev->entry);
+ kbase_dev_list_put(dev_list);
+ }
misc_deregister(&kbdev->mdev);
put_device(kbdev->dev);
kbase_common_reg_unmap(kbdev);
kbase_device_term(kbdev);
-#ifdef CONFIG_DEBUG_FS
- kbasep_jd_debugfs_term(kbdev);
- debugfs_remove(kbdev->memory_profile_directory);
- debugfs_remove(kbdev->mali_debugfs_directory);
-#endif /* CONFIG_DEBUG_FS */
if (kbdev->clock) {
clk_disable_unprepare(kbdev->clock);
clk_put(kbdev->clock);
kbdev->clock = NULL;
}
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \
+ && defined(CONFIG_REGULATOR)
+ regulator_put(kbdev->regulator);
+#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */
#ifdef CONFIG_MALI_NO_MALI
- midg_device_destroy(kbdev);
+ gpu_device_destroy(kbdev);
#endif /* CONFIG_MALI_NO_MALI */
kbase_device_free(kbdev);
devfreq_suspend_device(kbdev->devfreq);
#endif
- if (kbdev->pm.callback_power_runtime_off) {
- kbdev->pm.callback_power_runtime_off(kbdev);
+ if (kbdev->pm.backend.callback_power_runtime_off) {
+ kbdev->pm.backend.callback_power_runtime_off(kbdev);
dev_dbg(dev, "runtime suspend\n");
}
return 0;
if (!kbdev)
return -ENODEV;
- if (kbdev->pm.callback_power_runtime_on) {
- ret = kbdev->pm.callback_power_runtime_on(kbdev);
+ if (kbdev->pm.backend.callback_power_runtime_on) {
+ ret = kbdev->pm.backend.callback_power_runtime_on(kbdev);
dev_dbg(dev, "runtime resume\n");
}
* anymore when using Device Tree.
*/
#ifdef CONFIG_OF
-#if 0
module_platform_driver(kbase_platform_driver);
-#else
+#else
+
static int __init rockchip_gpu_init_driver(void)
{
return platform_driver_register(&kbase_platform_driver);
}
-
late_initcall(rockchip_gpu_init_driver);
-#endif
-#else
-#ifdef CONFIG_MALI_PLATFORM_FAKE
-extern int kbase_platform_fake_register(void);
-extern void kbase_platform_fake_unregister(void);
-#endif
static int __init kbase_driver_init(void)
{
if (ret)
return ret;
+#ifndef CONFIG_MACH_MANTA
#ifdef CONFIG_MALI_PLATFORM_FAKE
ret = kbase_platform_fake_register();
if (ret)
return ret;
+#endif
#endif
ret = platform_driver_register(&kbase_platform_driver);
+#ifndef CONFIG_MACH_MANTA
#ifdef CONFIG_MALI_PLATFORM_FAKE
if (ret)
kbase_platform_fake_unregister();
#endif
-
+#endif
return ret;
}
static void __exit kbase_driver_exit(void)
{
platform_driver_unregister(&kbase_platform_driver);
+#ifndef CONFIG_MACH_MANTA
#ifdef CONFIG_MALI_PLATFORM_FAKE
kbase_platform_fake_unregister();
#endif
+#endif
}
module_init(kbase_driver_init);
#endif /* CONFIG_OF */
MODULE_LICENSE("GPL");
-MODULE_VERSION(MALI_RELEASE_NAME);
+MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \
+ __stringify(BASE_UK_VERSION_MAJOR) "." \
+ __stringify(BASE_UK_VERSION_MINOR) ")");
#if defined(CONFIG_MALI_GATOR_SUPPORT) || defined(CONFIG_MALI_SYSTEM_TRACE)
#define CREATE_TRACE_POINTS
/* Create the trace points (otherwise we just get code to call a tracepoint) */
#include "mali_linux_trace.h"
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_job_slots_event);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_status);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_on);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_off);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_page_fault_insert_pages);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_in_use);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_released);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_total_alloc_pages_change);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counter);
+
void kbase_trace_mali_pm_status(u32 event, u64 value)
{
trace_mali_pm_status(event, value);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file mali_kbase_cpuprops.c
+/*
* Base kernel property query APIs
*/
#include "mali_kbase_cpuprops.h"
#include "mali_kbase_uku.h"
#include <mali_kbase_config.h>
+#include <mali_kbase_config_defaults.h>
#include <linux/cache.h>
#include <linux/cpufreq.h>
#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
#define L1_DCACHE_LINE_SIZE_LOG2 L1_CACHE_SHIFT
-/**
- * @brief Macros used to extract cpu id info
- * @see Doc's for Main ID register
+/*
+ * Macros used to extract cpu id info
+ * see documentation for Main ID register
*/
#define KBASE_CPUPROPS_ID_GET_REV(cpuid) ((cpuid) & 0x0F) /* [3:0] Revision */
#define KBASE_CPUPROPS_ID_GET_PART_NR(cpuid)(((cpuid) >> 4) & 0xFFF) /* [15:4] Part number */
#define L1_DCACHE_SIZE ((u32)0x00008000)
/**
- * @brief Retrieves detailed CPU info from given cpu_val ( ID reg )
- *
- * @param kbase_props CPU props to be filled-in with cpu id info
+ * kbasep_cpuprops_uk_get_cpu_id_info - Retrieves detailed CPU info from given
+ * cpu_val ( ID reg )
+ * @kbase_props: CPU props to be filled-in with cpu id info
*
*/
-#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
static void kbasep_cpuprops_uk_get_cpu_id_info(struct kbase_uk_cpuprops * const kbase_props)
{
kbase_props->props.cpu_id.id = read_cpuid_id();
}
#endif
-/**
+/*
* This function (and file!) is kept for the backward compatibility reasons.
* It shall be removed as soon as KBASE_FUNC_CPU_PROPS_REG_DUMP_OBSOLETE
* (previously KBASE_FUNC_CPU_PROPS_REG_DUMP) ioctl call
* the function for reading cpu properties moved from base to osu.
*/
-mali_error kbase_cpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_cpuprops * const kbase_props)
+int kbase_cpuprops_uk_get_props(struct kbase_context *kctx,
+ struct kbase_uk_cpuprops * const props)
{
unsigned int max_cpu_freq;
- kbase_props->props.cpu_l1_dcache_line_size_log2 = L1_DCACHE_LINE_SIZE_LOG2;
- kbase_props->props.cpu_l1_dcache_size = L1_DCACHE_SIZE;
- kbase_props->props.cpu_flags = BASE_CPU_PROPERTY_FLAG_LITTLE_ENDIAN;
+ props->props.cpu_l1_dcache_line_size_log2 = L1_DCACHE_LINE_SIZE_LOG2;
+ props->props.cpu_l1_dcache_size = L1_DCACHE_SIZE;
+ props->props.cpu_flags = BASE_CPU_PROPERTY_FLAG_LITTLE_ENDIAN;
- kbase_props->props.nr_cores = NR_CPUS;
- kbase_props->props.cpu_page_size_log2 = PAGE_SHIFT;
- kbase_props->props.available_memory_size = totalram_pages << PAGE_SHIFT;
+ props->props.nr_cores = num_possible_cpus();
+ props->props.cpu_page_size_log2 = PAGE_SHIFT;
+ props->props.available_memory_size = totalram_pages << PAGE_SHIFT;
- kbasep_cpuprops_uk_get_cpu_id_info(kbase_props);
+ kbasep_cpuprops_uk_get_cpu_id_info(props);
/* check if kernel supports dynamic frequency scaling */
max_cpu_freq = cpufreq_quick_get_max(KBASE_DEFAULT_CPU_NUM);
if (max_cpu_freq != 0) {
/* convert from kHz to mHz */
- kbase_props->props.max_cpu_clock_speed_mhz = max_cpu_freq / 1000;
+ props->props.max_cpu_clock_speed_mhz = max_cpu_freq / 1000;
} else {
/* fallback if CONFIG_CPU_FREQ turned off */
- int result;
- kbase_cpuprops_clock_speed_function kbase_cpuprops_uk_get_clock_speed;
+ int err;
+ kbase_cpu_clk_speed_func get_clock_speed;
- kbase_cpuprops_uk_get_clock_speed = (kbase_cpuprops_clock_speed_function) kbasep_get_config_value(kctx->kbdev, kctx->kbdev->config_attributes, KBASE_CONFIG_ATTR_CPU_SPEED_FUNC);
- result = kbase_cpuprops_uk_get_clock_speed(&kbase_props->props.max_cpu_clock_speed_mhz);
- if (result != 0)
- return MALI_ERROR_FUNCTION_FAILED;
+ get_clock_speed = (kbase_cpu_clk_speed_func) CPU_SPEED_FUNC;
+ err = get_clock_speed(&props->props.max_cpu_clock_speed_mhz);
+ if (err)
+ return err;
}
- return MALI_ERROR_NONE;
+ return 0;
}
#endif /* BASE_LEGACY_UK7_SUPPORT */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#ifndef _KBASE_CPUPROPS_H_
#define _KBASE_CPUPROPS_H_
-#include <malisw/mali_malisw.h>
-
/* Forward declarations */
struct kbase_uk_cpuprops;
* @param kctx The kbase context
* @param kbase_props A copy of the struct kbase_uk_cpuprops structure from userspace
*
- * @return MALI_ERROR_NONE on success. Any other value indicates failure.
+ * @return 0 on success. Any other value indicates failure.
*/
-mali_error kbase_cpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_cpuprops * const kbase_props);
+int kbase_cpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_cpuprops * const kbase_props);
#endif /*_KBASE_CPUPROPS_H_*/
#endif /* BASE_LEGACY_UK7_SUPPORT */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/** Function type that is called on an KBASE_DEBUG_ASSERT() or KBASE_DEBUG_ASSERT_MSG() */
typedef void (kbase_debug_assert_hook) (void *);
-typedef struct kbasep_debug_assert_cb {
+struct kbasep_debug_assert_cb {
kbase_debug_assert_hook *func;
void *param;
-} kbasep_debug_assert_cb;
+};
/**
* @def KBASEP_DEBUG_PRINT_TRACE
#define KBASEP_DEBUG_PRINT_TRACE \
"In file: " __FILE__ " line: " CSTD_STR2(__LINE__)
#if !KBASE_DEBUG_SKIP_FUNCTION_NAME
-#define KBASEP_DEBUG_PRINT_FUNCTION CSTD_FUNC
+#define KBASEP_DEBUG_PRINT_FUNCTION __func__
#else
#define KBASEP_DEBUG_PRINT_FUNCTION ""
#endif
pr_err("Mali<ASSERT>: %s function:%s ", trace, function);\
pr_err(__VA_ARGS__);\
pr_err("\n");\
- } while (MALI_FALSE)
+ } while (false)
#else
#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) CSTD_NOP()
#endif
#ifdef CONFIG_MALI_DEBUG
-#define KBASE_CALL_ASSERT_HOOK() kbasep_debug_assert_call_hook();
+#define KBASE_CALL_ASSERT_HOOK() kbasep_debug_assert_call_hook()
#else
-#define KBASE_CALL_ASSERT_HOOK() CSTD_NOP();
+#define KBASE_CALL_ASSERT_HOOK() CSTD_NOP()
#endif
/**
KBASE_CALL_ASSERT_HOOK();\
BUG();\
} \
- } while (MALI_FALSE)
+ } while (false)
#endif /* KBASE_DEBUG_DISABLE_ASSERTS */
/**
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/*
+ * Debugfs interface to dump the memory visible to the GPU
+ */
+
+#include "mali_kbase_debug_mem_view.h"
+#include "mali_kbase.h"
+
+#include <linux/list.h>
+#include <linux/file.h>
+
+#if CONFIG_DEBUG_FS
+
+struct debug_mem_mapping {
+ struct list_head node;
+
+ struct kbase_mem_phy_alloc *alloc;
+ unsigned long flags;
+
+ u64 start_pfn;
+ size_t nr_pages;
+};
+
+struct debug_mem_data {
+ struct list_head mapping_list;
+ struct kbase_context *kctx;
+};
+
+struct debug_mem_seq_off {
+ struct list_head *lh;
+ size_t offset;
+};
+
+static void *debug_mem_start(struct seq_file *m, loff_t *_pos)
+{
+ struct debug_mem_data *mem_data = m->private;
+ struct debug_mem_seq_off *data;
+ struct debug_mem_mapping *map;
+ loff_t pos = *_pos;
+
+ list_for_each_entry(map, &mem_data->mapping_list, node) {
+ if (pos >= map->nr_pages) {
+ pos -= map->nr_pages;
+ } else {
+ data = kmalloc(sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return NULL;
+ data->lh = &map->node;
+ data->offset = pos;
+ return data;
+ }
+ }
+
+ /* Beyond the end */
+ return NULL;
+}
+
+static void debug_mem_stop(struct seq_file *m, void *v)
+{
+ kfree(v);
+}
+
+static void *debug_mem_next(struct seq_file *m, void *v, loff_t *pos)
+{
+ struct debug_mem_data *mem_data = m->private;
+ struct debug_mem_seq_off *data = v;
+ struct debug_mem_mapping *map;
+
+ map = list_entry(data->lh, struct debug_mem_mapping, node);
+
+ if (data->offset < map->nr_pages - 1) {
+ data->offset++;
+ ++*pos;
+ return data;
+ }
+
+ if (list_is_last(data->lh, &mem_data->mapping_list))
+ return NULL;
+
+ data->lh = data->lh->next;
+ data->offset = 0;
+ ++*pos;
+
+ return data;
+}
+
+static int debug_mem_show(struct seq_file *m, void *v)
+{
+ struct debug_mem_data *mem_data = m->private;
+ struct debug_mem_seq_off *data = v;
+ struct debug_mem_mapping *map;
+ int i, j;
+ struct page *page;
+ uint32_t *mapping;
+ pgprot_t prot = PAGE_KERNEL;
+
+ map = list_entry(data->lh, struct debug_mem_mapping, node);
+
+ kbase_gpu_vm_lock(mem_data->kctx);
+
+ if (data->offset >= map->alloc->nents) {
+ seq_printf(m, "%016llx: Unbacked page\n\n", (map->start_pfn +
+ data->offset) << PAGE_SHIFT);
+ goto out;
+ }
+
+ if (!(map->flags & KBASE_REG_CPU_CACHED))
+ prot = pgprot_writecombine(prot);
+
+ page = pfn_to_page(PFN_DOWN(map->alloc->pages[data->offset]));
+ mapping = vmap(&page, 1, VM_MAP, prot);
+
+ for (i = 0; i < PAGE_SIZE; i += 4*sizeof(*mapping)) {
+ seq_printf(m, "%016llx:", i + ((map->start_pfn +
+ data->offset) << PAGE_SHIFT));
+
+ for (j = 0; j < 4*sizeof(*mapping); j += sizeof(*mapping))
+ seq_printf(m, " %08x", mapping[(i+j)/sizeof(*mapping)]);
+ seq_putc(m, '\n');
+ }
+
+ vunmap(mapping);
+
+ seq_putc(m, '\n');
+
+out:
+ kbase_gpu_vm_unlock(mem_data->kctx);
+ return 0;
+}
+
+static const struct seq_operations ops = {
+ .start = debug_mem_start,
+ .next = debug_mem_next,
+ .stop = debug_mem_stop,
+ .show = debug_mem_show,
+};
+
+static int debug_mem_open(struct inode *i, struct file *file)
+{
+ struct file *kctx_file = i->i_private;
+ struct kbase_context *kctx = kctx_file->private_data;
+ struct rb_node *p;
+ struct debug_mem_data *mem_data;
+ int ret;
+
+ ret = seq_open(file, &ops);
+
+ if (ret)
+ return ret;
+
+ mem_data = kmalloc(sizeof(*mem_data), GFP_KERNEL);
+ mem_data->kctx = kctx;
+
+ INIT_LIST_HEAD(&mem_data->mapping_list);
+
+ get_file(kctx_file);
+
+ kbase_gpu_vm_lock(kctx);
+
+ for (p = rb_first(&kctx->reg_rbtree); p; p = rb_next(p)) {
+ struct kbase_va_region *reg;
+ struct debug_mem_mapping *mapping;
+
+ reg = rb_entry(p, struct kbase_va_region, rblink);
+
+ if (reg->gpu_alloc == NULL)
+ /* Empty region - ignore */
+ continue;
+
+ mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
+
+ mapping->alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
+ mapping->start_pfn = reg->start_pfn;
+ mapping->nr_pages = reg->nr_pages;
+ mapping->flags = reg->flags;
+ list_add_tail(&mapping->node, &mem_data->mapping_list);
+ }
+
+ kbase_gpu_vm_unlock(kctx);
+
+ ((struct seq_file *)file->private_data)->private = mem_data;
+
+ return 0;
+}
+
+static int debug_mem_release(struct inode *inode, struct file *file)
+{
+ struct file *kctx_file = inode->i_private;
+ struct seq_file *sfile = file->private_data;
+ struct debug_mem_data *mem_data = sfile->private;
+ struct debug_mem_mapping *mapping;
+
+ seq_release(inode, file);
+
+ while (!list_empty(&mem_data->mapping_list)) {
+ mapping = list_first_entry(&mem_data->mapping_list,
+ struct debug_mem_mapping, node);
+ kbase_mem_phy_alloc_put(mapping->alloc);
+ list_del(&mapping->node);
+ kfree(mapping);
+ }
+
+ kfree(mem_data);
+
+ fput(kctx_file);
+
+ return 0;
+}
+
+static const struct file_operations kbase_debug_mem_view_fops = {
+ .open = debug_mem_open,
+ .release = debug_mem_release,
+ .read = seq_read,
+ .llseek = seq_lseek
+};
+
+/**
+ * kbase_debug_mem_view_init - Initialise the mem_view sysfs file
+ * @kctx_file: The /dev/mali0 file instance for the context
+ *
+ * This function creates a "mem_view" file which can be used to get a view of
+ * the context's memory as the GPU sees it (i.e. using the GPU's page tables).
+ *
+ * The file is cleaned up by a call to debugfs_remove_recursive() deleting the
+ * parent directory.
+ */
+void kbase_debug_mem_view_init(struct file *kctx_file)
+{
+ struct kbase_context *kctx = kctx_file->private_data;
+
+ debugfs_create_file("mem_view", S_IRUGO, kctx->kctx_dentry, kctx_file,
+ &kbase_debug_mem_view_fops);
+}
+
+#endif
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KBASE_DEBUG_MEM_VIEW_H
+#define _KBASE_DEBUG_MEM_VIEW_H
+
+#include <mali_kbase.h>
+
+void kbase_debug_mem_view_init(struct file *kctx_file);
+
+#endif
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define _KBASE_DEFS_H_
#include <mali_kbase_config.h>
-#include <mali_base_hwconfig.h>
+#include <mali_base_hwconfig_features.h>
+#include <mali_base_hwconfig_issues.h>
#include <mali_kbase_mem_lowlevel.h>
#include <mali_kbase_mem_alloc.h>
#include <mali_kbase_mmu_hw.h>
-
+#include <mali_kbase_mmu_mode.h>
+#include <mali_kbase_instr.h>
#include <linux/atomic.h>
#include <linux/mempool.h>
#include <linux/devfreq.h>
#endif /* CONFIG_DEVFREQ */
+#include <linux/clk.h>
+#include <linux/regulator/consumer.h>
+
/** Enable SW tracing when set */
#ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE
#define KBASE_TRACE_ENABLE 1
* You can optimize this down if your target devices will only ever support a
* small number of job slots.
*/
-#define BASE_JM_MAX_NR_SLOTS 16
+#define BASE_JM_MAX_NR_SLOTS 3
/**
* The maximum number of Address Spaces to support in the Hardware.
#define BASE_MAX_NR_AS 16
/* mmu */
-#define ENTRY_IS_ATE 1ULL
-#define ENTRY_IS_INVAL 2ULL
-#define ENTRY_IS_PTE 3ULL
-
#define MIDGARD_MMU_VA_BITS 48
-#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */
-#define ENTRY_RD_BIT (1ULL << 6)
-#define ENTRY_WR_BIT (1ULL << 7)
-#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */
-#define ENTRY_ACCESS_BIT (1ULL << 10)
-#define ENTRY_NX_BIT (1ULL << 54)
-
-#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT)
-
#if MIDGARD_MMU_VA_BITS > 39
#define MIDGARD_MMU_TOPLEVEL 0
#else
#define MIDGARD_MMU_TOPLEVEL 1
#endif
-#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW)
+#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR)
/** setting in kbase_context::as_nr that indicates it's invalid */
#define KBASEP_AS_NR_INVALID (-1)
#define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1)
#include "mali_kbase_js_defs.h"
+#include "mali_kbase_hwaccess_defs.h"
#define KBASEP_FORCE_REPLAY_DISABLED 0
#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4)
/** Atom has caused us to enter disjoint state */
#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5)
+/* Atom has fail dependency on same-slot dependency */
+#define KBASE_KATOM_FLAG_FAIL_PREV (1<<6)
+/* Atom blocked on cross-slot dependency */
+#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7)
+/* Atom has fail dependency on cross-slot dependency */
+#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8)
+/* Atom has been submitted to JSCTX ringbuffers */
+#define KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED (1<<9)
+/* Atom is currently holding a context reference */
+#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10)
+/* Atom requires GPU to be in secure mode */
+#define KBASE_KATOM_FLAG_SECURE (1<<11)
/* SW related flags about types of JS_COMMAND action
* NOTE: These must be masked off by JS_COMMAND_MASK */
#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \
(JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP)
+#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT
-struct kbase_jd_atom_dependency
-{
+struct kbase_jd_atom_dependency {
struct kbase_jd_atom *atom;
u8 dep_type;
};
/**
- * @brief The function retrieves a read-only reference to the atom field from
+ * @brief The function retrieves a read-only reference to the atom field from
* the kbase_jd_atom_dependency structure
*
* @param[in] dep kbase jd atom dependency.
*
* @return readonly reference to dependent ATOM.
*/
-static INLINE const struct kbase_jd_atom* const kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency* dep)
+static inline const struct kbase_jd_atom *const kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep)
{
LOCAL_ASSERT(dep != NULL);
-
- return (const struct kbase_jd_atom* const )(dep->atom);
+
+ return (const struct kbase_jd_atom * const)(dep->atom);
}
-
+
/**
- * @brief The function retrieves a read-only reference to the dependency type field from
+ * @brief The function retrieves a read-only reference to the dependency type field from
* the kbase_jd_atom_dependency structure
*
* @param[in] dep kbase jd atom dependency.
*
* @return A dependency type value.
*/
-static INLINE const u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency* dep)
+static inline const u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep)
{
LOCAL_ASSERT(dep != NULL);
* @param type The ATOM dependency type to be set.
*
*/
-static INLINE void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency* const_dep,
- struct kbase_jd_atom * a,
- u8 type)
+static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep,
+ struct kbase_jd_atom *a, u8 type)
{
- struct kbase_jd_atom_dependency* dep;
-
+ struct kbase_jd_atom_dependency *dep;
+
LOCAL_ASSERT(const_dep != NULL);
- dep = (REINTERPRET_CAST(struct kbase_jd_atom_dependency* )const_dep);
+ dep = (struct kbase_jd_atom_dependency *)const_dep;
dep->atom = a;
- dep->dep_type = type;
+ dep->dep_type = type;
}
/**
* @param[in] dep The kbase jd atom dependency to be cleared.
*
*/
-static INLINE void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency* const_dep)
+static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep)
{
- struct kbase_jd_atom_dependency* dep;
+ struct kbase_jd_atom_dependency *dep;
LOCAL_ASSERT(const_dep != NULL);
- dep = (REINTERPRET_CAST(struct kbase_jd_atom_dependency* )const_dep);
+ dep = (struct kbase_jd_atom_dependency *)const_dep;
dep->atom = NULL;
- dep->dep_type = BASE_JD_DEP_TYPE_INVALID;
+ dep->dep_type = BASE_JD_DEP_TYPE_INVALID;
}
-struct kbase_ext_res
-{
- mali_addr64 gpu_address;
- struct kbase_mem_phy_alloc * alloc;
+enum kbase_atom_gpu_rb_state {
+ /* Atom is not currently present in slot ringbuffer */
+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB,
+ /* Atom is in slot ringbuffer but is blocked on a previous atom */
+ KBASE_ATOM_GPU_RB_WAITING_BLOCKED,
+ /* Atom is in slot ringbuffer but is waiting for cores to become
+ * available */
+ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE,
+ /* Atom is in slot ringbuffer but is blocked on affinity */
+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY,
+ /* Atom is in slot ringbuffer but is waiting for secure mode switch */
+ KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE,
+ /* Atom is in slot ringbuffer and ready to run */
+ KBASE_ATOM_GPU_RB_READY,
+ /* Atom is in slot ringbuffer and has been submitted to the GPU */
+ KBASE_ATOM_GPU_RB_SUBMITTED,
+ /* Atom must be returned to JS as soon as it reaches the head of the
+ * ringbuffer due to a previous failure */
+ KBASE_ATOM_GPU_RB_RETURN_TO_JS
+};
+
+struct kbase_ext_res {
+ u64 gpu_address;
+ struct kbase_mem_phy_alloc *alloc;
};
struct kbase_jd_atom {
const struct kbase_jd_atom_dependency dep[2];
u16 nr_extres;
- struct kbase_ext_res * extres;
+ struct kbase_ext_res *extres;
u32 device_nr;
u64 affinity;
#ifdef CONFIG_KDS
struct list_head node;
struct kds_resource_set *kds_rset;
- mali_bool kds_dep_satisfied;
+ bool kds_dep_satisfied;
#endif /* CONFIG_KDS */
#ifdef CONFIG_SYNC
struct sync_fence *fence;
int retry_submit_on_slot;
union kbasep_js_policy_job_info sched_info;
- /* atom priority scaled to nice range with +20 offset 0..39 */
- int nice_prio;
+ /* JS atom priority with respect to other atoms on its kctx. */
+ int sched_priority;
int poking; /* BASE_HW_ISSUE_8316 */
/* Number of times this atom has been retried. Used by replay soft job.
*/
int retry_count;
+
+ enum kbase_atom_gpu_rb_state gpu_rb_state;
+
+ u64 need_cache_flush_cores_retained;
+
+ atomic_t blocked;
+
+ /* Pointer to atom that this atom has cross-slot dependency on */
+ struct kbase_jd_atom *x_pre_dep;
+ /* Pointer to atom that has cross-slot dependency on this atom */
+ struct kbase_jd_atom *x_post_dep;
+
+
+ struct kbase_jd_atom_backend backend;
};
+static inline bool kbase_jd_katom_is_secure(const struct kbase_jd_atom *katom)
+{
+ return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_SECURE);
+}
+
/*
* Theory of operations:
*
#endif
};
-struct kbase_jm_slot {
- /* The number of slots must be a power of two */
-#define BASE_JM_SUBMIT_SLOTS 16
-#define BASE_JM_SUBMIT_SLOTS_MASK (BASE_JM_SUBMIT_SLOTS - 1)
-
- struct kbase_jd_atom *submitted[BASE_JM_SUBMIT_SLOTS];
-
- struct kbase_context *last_context;
-
- u8 submitted_head;
- u8 submitted_nr;
- u8 job_chain_flag;
-
-};
-
struct kbase_device_info {
u32 features;
};
struct work_struct work_busfault;
enum kbase_mmu_fault_type fault_type;
u32 fault_status;
- mali_addr64 fault_addr;
+ u64 fault_addr;
struct mutex transaction_mutex;
struct kbase_mmu_setup current_setup;
return as->fault_type == KBASE_MMU_FAULT_TYPE_PAGE;
}
-/**
- * Instrumentation State Machine States
- */
-enum kbase_instr_state {
- /** State where instrumentation is not active */
- KBASE_INSTR_STATE_DISABLED = 0,
- /** State machine is active and ready for a command. */
- KBASE_INSTR_STATE_IDLE,
- /** Hardware is currently dumping a frame. */
- KBASE_INSTR_STATE_DUMPING,
- /** We've requested a clean to occur on a workqueue */
- KBASE_INSTR_STATE_REQUEST_CLEAN,
- /** Hardware is currently cleaning and invalidating caches. */
- KBASE_INSTR_STATE_CLEANING,
- /** Cache clean completed, and either a) a dump is complete, or
- * b) instrumentation can now be setup. */
- KBASE_INSTR_STATE_CLEANED,
- /** kbasep_reset_timeout_worker() has started (but not compelted) a
- * reset. This generally indicates the current action should be aborted, and
- * kbasep_reset_timeout_worker() will handle the cleanup */
- KBASE_INSTR_STATE_RESETTING,
- /** An error has occured during DUMPING (page fault). */
- KBASE_INSTR_STATE_FAULT
-};
-
-void kbasep_reset_timeout_worker(struct work_struct *data);
-enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *data);
-
struct kbasep_mem_device {
atomic_t used_pages; /* Tracks usage of OS shared memory. Updated
when OS memory is allocated/freed. */
};
-
-
#define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X
enum kbase_trace_code {
u32 thread_id;
u32 cpu;
void *ctx;
- mali_bool katom;
+ bool katom;
int atom_number;
u64 atom_udata[2];
u64 gpu_addr;
};
struct kbase_trace_kbdev_timeline {
- /** DebugFS entry */
- struct dentry *dentry;
-
/* Note: strictly speaking, not needed, because it's in sync with
* kbase_device::jm_slots[]::submitted_nr
*
*
* The caller must hold kbasep_js_device_data::runpool_irq::lock when
* accessing this */
- u8 slot_atoms_submitted[BASE_JM_SUBMIT_SLOTS];
+ u8 slot_atoms_submitted[BASE_JM_MAX_NR_SLOTS];
/* Last UID for each PM event */
atomic_t pm_event_uid[KBASEP_TIMELINE_PM_EVENT_LAST+1];
/* Counter for generating PM event UIDs */
atomic_t pm_event_uid_counter;
/*
- * L2 transition state - MALI_TRUE indicates that the transition is ongoing
+ * L2 transition state - true indicates that the transition is ongoing
* Expected to be protected by pm.power_change_lock */
- mali_bool l2_transitioning;
+ bool l2_transitioning;
};
#endif /* CONFIG_MALI_TRACE_TIMELINE */
struct kbasep_kctx_list_element {
struct list_head link;
- struct kbase_context *kctx;
+ struct kbase_context *kctx;
};
+/**
+ * Data stored per device for power management.
+ *
+ * This structure contains data for the power management framework. There is one
+ * instance of this structure per device in the system.
+ */
+struct kbase_pm_device_data {
+ /**
+ * The lock protecting Power Management structures accessed outside of
+ * IRQ.
+ *
+ * This lock must also be held whenever the GPU is being powered on or
+ * off.
+ */
+ struct mutex lock;
+
+ /** The reference count of active contexts on this device. */
+ int active_count;
+ /** Flag indicating suspending/suspended */
+ bool suspending;
+ /* Wait queue set when active_count == 0 */
+ wait_queue_head_t zero_active_count_wait;
+
+ /**
+ * A bit mask identifying the available shader cores that are specified
+ * via sysfs
+ */
+ u64 debug_core_mask;
+
+ /**
+ * Lock protecting the power state of the device.
+ *
+ * This lock must be held when accessing the shader_available_bitmap,
+ * tiler_available_bitmap, l2_available_bitmap, shader_inuse_bitmap and
+ * tiler_inuse_bitmap fields of kbase_device, and the ca_in_transition
+ * and shader_poweroff_pending fields of kbase_pm_device_data. It is
+ * also held when the hardware power registers are being written to, to
+ * ensure that two threads do not conflict over the power transitions
+ * that the hardware should make.
+ */
+ spinlock_t power_change_lock;
+
+ /**
+ * Callback for initializing the runtime power management.
+ *
+ * @param kbdev The kbase device
+ *
+ * @return 0 on success, else error code
+ */
+ int (*callback_power_runtime_init)(struct kbase_device *kbdev);
+
+ /**
+ * Callback for terminating the runtime power management.
+ *
+ * @param kbdev The kbase device
+ */
+ void (*callback_power_runtime_term)(struct kbase_device *kbdev);
+
+ /* Time in milliseconds between each dvfs sample */
+ u32 dvfs_period;
+
+ /* Period of GPU poweroff timer */
+ ktime_t gpu_poweroff_time;
+
+ /* Number of ticks of GPU poweroff timer before shader is powered off */
+ int poweroff_shader_ticks;
+
+ /* Number of ticks of GPU poweroff timer before GPU is powered off */
+ int poweroff_gpu_ticks;
+
+ struct kbase_pm_backend_data backend;
+};
+
+/**
+ * struct kbase_secure_ops - Platform specific functions for GPU secure mode
+ * operations
+ * @secure_mode_enable: Callback to enable secure mode on the GPU
+ * @secure_mode_disable: Callback to disable secure mode on the GPU
+ */
+struct kbase_secure_ops {
+ /**
+ * secure_mode_enable() - Enable secure mode on the GPU
+ * @kbdev: The kbase device
+ *
+ * Return: 0 on success, non-zero on error
+ */
+ int (*secure_mode_enable)(struct kbase_device *kbdev);
+
+ /**
+ * secure_mode_disable() - Disable secure mode on the GPU
+ * @kbdev: The kbase device
+ *
+ * Return: 0 on success, non-zero on error
+ */
+ int (*secure_mode_disable)(struct kbase_device *kbdev);
+};
+
+
#define DEVNAME_SIZE 16
struct kbase_device {
- /** jm_slots is protected by kbasep_js_device_data::runpool_irq::lock */
- struct kbase_jm_slot jm_slots[BASE_JM_MAX_NR_SLOTS];
s8 slot_submit_count_irq[BASE_JM_MAX_NR_SLOTS];
+ u32 hw_quirks_sc;
+ u32 hw_quirks_tiler;
+ u32 hw_quirks_mmu;
+
struct list_head entry;
struct device *dev;
unsigned int kbase_group_error;
u64 reg_start;
size_t reg_size;
void __iomem *reg;
- struct resource *reg_res;
struct {
int irq;
int flags;
} irqs[3];
#ifdef CONFIG_HAVE_CLK
struct clk *clock;
+#endif
+#ifdef CONFIG_REGULATOR
+ struct regulator *regulator;
#endif
char devname[DEVNAME_SIZE];
struct kbase_pm_device_data pm;
struct kbasep_js_device_data js_data;
struct kbasep_mem_device memdev;
+ struct kbase_mmu_mode const *mmu_mode;
+
struct kbase_as as[BASE_MAX_NR_AS];
- spinlock_t mmu_mask_change;
+ spinlock_t mmu_mask_change;
- kbase_gpu_props gpu_props;
+ struct kbase_gpu_props gpu_props;
/** List of SW workarounds for HW issues */
unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG];
u64 shader_present_bitmap;
u64 tiler_present_bitmap;
u64 l2_present_bitmap;
- u64 l3_present_bitmap;
/* Bitmaps of cores that are currently in use (running jobs).
* These should be kept up to date by the job scheduler.
struct kbase_context *kctx;
u64 addr;
- wait_queue_head_t wait;
- int triggered;
- enum kbase_instr_state state;
- wait_queue_head_t cache_clean_wait;
- struct workqueue_struct *cache_clean_wq;
- struct work_struct cache_clean_work;
struct kbase_context *suspended_kctx;
struct kbase_uk_hwcnt_setup suspended_state;
+
+ struct kbase_instr_backend backend;
} hwcnt;
- /* Set when we're about to reset the GPU */
- atomic_t reset_gpu;
-#define KBASE_RESET_GPU_NOT_PENDING 0 /* The GPU reset isn't pending */
-#define KBASE_RESET_GPU_PREPARED 1 /* kbase_prepare_to_reset_gpu has been called */
-#define KBASE_RESET_GPU_COMMITTED 2 /* kbase_reset_gpu has been called - the reset will now definitely happen
- * within the timeout period */
-#define KBASE_RESET_GPU_HAPPENING 3 /* The GPU reset process is currently occuring (timeout has expired or
- * kbasep_try_reset_gpu_early was called) */
-
- /* Work queue and work item for performing the reset in */
- struct workqueue_struct *reset_workq;
- struct work_struct reset_work;
- wait_queue_head_t reset_wait;
- struct hrtimer reset_timer;
+ struct kbase_vinstr_context *vinstr_ctx;
/*value to be written to the irq_throttle register each time an irq is served */
atomic_t irq_throttle_cycles;
- const struct kbase_attribute *config_attributes;
-
#if KBASE_TRACE_ENABLE
spinlock_t trace_lock;
u16 trace_first_out;
struct kbase_trace *trace_rbuf;
#endif
-#if !MALI_CUSTOMER_RELEASE
/* This is used to override the current job scheduler values for
- * KBASE_CONFIG_ATTR_JS_STOP_STOP_TICKS_SS
- * KBASE_CONFIG_ATTR_JS_STOP_STOP_TICKS_CL
- * KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS
- * KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL
- * KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS
- * KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS
- * KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL
- * KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS.
+ * JS_SCHEDULING_PERIOD_NS
+ * JS_SOFT_STOP_TICKS
+ * JS_SOFT_STOP_TICKS_CL
+ * JS_HARD_STOP_TICKS_SS
+ * JS_HARD_STOP_TICKS_CL
+ * JS_HARD_STOP_TICKS_DUMPING
+ * JS_RESET_TICKS_SS
+ * JS_RESET_TICKS_CL
+ * JS_RESET_TICKS_DUMPING.
*
* These values are set via the js_timeouts sysfs file.
*/
- u32 js_soft_stop_ticks;
- u32 js_soft_stop_ticks_cl;
- u32 js_hard_stop_ticks_ss;
- u32 js_hard_stop_ticks_cl;
- u32 js_hard_stop_ticks_nss;
- u32 js_reset_ticks_ss;
- u32 js_reset_ticks_cl;
- u32 js_reset_ticks_nss;
-#endif
+ u32 js_scheduling_period_ns;
+ int js_soft_stop_ticks;
+ int js_soft_stop_ticks_cl;
+ int js_hard_stop_ticks_ss;
+ int js_hard_stop_ticks_cl;
+ int js_hard_stop_ticks_dumping;
+ int js_reset_ticks_ss;
+ int js_reset_ticks_cl;
+ int js_reset_ticks_dumping;
+ bool js_timeouts_updated;
+
+ u32 reset_timeout_ms;
struct mutex cacheclean_lock;
/* Platform specific private data to be accessed by mali_kbase_config_xxx.c only */
void *platform_context;
- /** Count of contexts keeping the GPU powered */
- atomic_t keep_gpu_powered_count;
-
/* List of kbase_contexts created */
struct list_head kctx_list;
struct mutex kctx_list_lock;
#ifdef CONFIG_PM_DEVFREQ
struct devfreq_dev_profile devfreq_profile;
struct devfreq *devfreq;
- unsigned long freq;
+ unsigned long current_freq;
+ unsigned long current_voltage;
#ifdef CONFIG_DEVFREQ_THERMAL
struct devfreq_cooling_device *devfreq_cooling;
-#ifdef CONFIG_MALI_POWER_ACTOR
- struct power_actor *power_actor;
-#endif
#endif
#endif
#ifdef CONFIG_DEBUG_FS
/* directory for debugfs entries */
struct dentry *mali_debugfs_directory;
- /* debugfs entry for gpu_memory */
- struct dentry *gpu_memory_dentry;
- /* debugfs entry for trace */
- struct dentry *trace_dentry;
- /* directory for per-ctx memory profiling data */
- struct dentry *memory_profile_directory;
- /* Root directory for job dispatcher data */
- struct dentry *jd_directory;
+ /* Root directory for per context entry */
+ struct dentry *debugfs_ctx_directory;
#endif /* CONFIG_DEBUG_FS */
/* fbdump profiling controls set by gator */
int force_replay_count;
/* Core requirement for jobs to be failed and replayed. May be zero. */
base_jd_core_req force_replay_core_req;
- /* MALI_TRUE if force_replay_limit should be randomized. The random
+ /* true if force_replay_limit should be randomized. The random
* value will be in the range of 1 - KBASEP_FORCE_REPLAY_RANDOM_LIMIT.
*/
- mali_bool force_replay_random;
+ bool force_replay_random;
#endif
+
/* Total number of created contexts */
atomic_t ctx_num;
+
+ struct kbase_hwaccess_data hwaccess;
+
+ /* Count of page/bus faults waiting for workqueues to process */
+ atomic_t faults_pending;
+
+ /* true if GPU is powered off or power off operation is in progress */
+ bool poweroff_pending;
+
+
+ /* defaults for new context created for this device */
+ u32 infinite_cache_active_default;
+
+ /* system coherency mode */
+ u32 system_coherency;
+
+ /* Secure operations */
+ struct kbase_secure_ops *secure_ops;
+};
+
+/* JSCTX ringbuffer size must always be a power of 2 */
+#define JSCTX_RB_SIZE 256
+#define JSCTX_RB_MASK (JSCTX_RB_SIZE-1)
+
+/**
+ * struct jsctx_rb_entry - Entry in &struct jsctx_rb ring buffer
+ * @atom_id: Atom ID
+ */
+struct jsctx_rb_entry {
+ u16 atom_id;
};
+/**
+ * struct jsctx_rb - JS context atom ring buffer
+ * @entries: Array of size %JSCTX_RB_SIZE which holds the &struct
+ * kbase_jd_atom pointers which make up the contents of the ring
+ * buffer.
+ * @read_idx: Index into @entries. Indicates the next entry in @entries to
+ * read, and is incremented when pulling an atom, and decremented
+ * when unpulling.
+ * HW access lock must be held when accessing.
+ * @write_idx: Index into @entries. Indicates the next entry to use when
+ * adding atoms into the ring buffer, and is incremented when
+ * adding a new atom.
+ * jctx->lock must be held when accessing.
+ * @running_idx: Index into @entries. Indicates the last valid entry, and is
+ * incremented when remving atoms from the ring buffer.
+ * HW access lock must be held when accessing.
+ *
+ * &struct jsctx_rb is a ring buffer of &struct kbase_jd_atom.
+ */
+struct jsctx_rb {
+ struct jsctx_rb_entry entries[JSCTX_RB_SIZE];
+
+ u16 read_idx; /* HW access lock must be held when accessing */
+ u16 write_idx; /* jctx->lock must be held when accessing */
+ u16 running_idx; /* HW access lock must be held when accessing */
+};
+
+#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \
+ (((minor) & 0xFFF) << 8) | \
+ ((0 & 0xFF) << 0))
+
struct kbase_context {
struct kbase_device *kbdev;
int id; /* System wide unique id */
+ unsigned long api_version;
phys_addr_t pgd;
struct list_head event_list;
struct mutex event_mutex;
- mali_bool event_closed;
+ bool event_closed;
struct workqueue_struct *event_workq;
- u64 mem_attrs;
bool is_compat;
atomic_t setup_complete;
atomic_t setup_in_progress;
- mali_bool keep_gpu_powered;
-
u64 *mmu_teardown_pages;
phys_addr_t aliasing_sink_page;
unsigned long cookies;
struct kbase_va_region *pending_regions[BITS_PER_LONG];
-
+
wait_queue_head_t event_queue;
pid_t tgid;
pid_t pid;
atomic_t nonmapped_pages;
struct kbase_mem_allocator osalloc;
- struct kbase_mem_allocator * pgd_allocator;
+ struct kbase_mem_allocator *pgd_allocator;
struct list_head waiting_soft_jobs;
#ifdef CONFIG_KDS
*
* All other flags must be added there */
spinlock_t mm_update_lock;
- struct mm_struct * process_mm;
+ struct mm_struct *process_mm;
#ifdef CONFIG_MALI_TRACE_TIMELINE
struct kbase_trace_kctx_timeline timeline;
#endif
#ifdef CONFIG_DEBUG_FS
- /* debugfs entry for memory profile */
- struct dentry *mem_dentry;
/* Content of mem_profile file */
char *mem_profile_data;
/* Size of @c mem_profile_data */
size_t mem_profile_size;
/* Spinlock guarding data */
spinlock_t mem_profile_lock;
- /* Per-context directory for JD data */
- struct dentry *jd_ctx_dir;
+ struct dentry *kctx_dentry;
#endif /* CONFIG_DEBUG_FS */
+
+ struct jsctx_rb jsctx_rb
+ [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS];
+
+ /* Number of atoms currently pulled from this context */
+ atomic_t atoms_pulled;
+ /* Number of atoms currently pulled from this context, per slot */
+ atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS];
+ /* true if last kick() caused atoms to be pulled from this context */
+ bool pulled;
+ /* true if infinite cache is to be enabled for new allocations. Existing
+ * allocations will not change. bool stored as a u32 per Linux API */
+ u32 infinite_cache_active;
+ /* Bitmask of slots that can be pulled from */
+ u32 slots_pullable;
+
+ /* true if address space assignment is pending */
+ bool as_pending;
+
+ /* Backend specific data */
+ struct kbase_context_backend backend;
+
+ /* Work structure used for deferred ASID assignment */
+ struct work_struct work;
+
+ /* Only one userspace vinstr client per kbase context */
+ struct kbase_vinstr_client *vinstr_cli;
+ struct mutex vinstr_cli_lock;
};
enum kbase_reg_access_type {
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-#include <mali_kbase.h>
-#include <mali_kbase_config_defaults.h>
-
-#include <linux/devfreq.h>
-#ifdef CONFIG_DEVFREQ_THERMAL
-#include <linux/devfreq_cooling.h>
-#endif
-
-#include "mali_kbase_power_actor.h"
-
-static int
-kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags)
-{
- struct kbase_device *kbdev = dev_get_drvdata(dev);
- struct dev_pm_opp *opp;
- unsigned long freq = 0;
- int err;
-
- freq = *target_freq;
-
- rcu_read_lock();
- opp = devfreq_recommended_opp(dev, &freq, flags);
- rcu_read_unlock();
- if (IS_ERR_OR_NULL(opp)) {
- dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp));
- return PTR_ERR(opp);
- }
-
- /*
- * Only update if there is a change of frequency
- */
- if (kbdev->freq == freq) {
- *target_freq = freq;
- return 0;
- }
-
- err = clk_set_rate(kbdev->clock, freq);
- if (err) {
- dev_err(dev, "Failed to set clock %lu (target %lu)\n",
- freq, *target_freq);
- return err;
- }
-
- kbdev->freq = freq;
- *target_freq = freq;
-
- kbase_pm_reset_dvfs_utilisation(kbdev);
-
- return 0;
-}
-
-static int
-kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq)
-{
- struct kbase_device *kbdev = dev_get_drvdata(dev);
-
- *freq = kbdev->freq;
-
- return 0;
-}
-
-static int
-kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat)
-{
- struct kbase_device *kbdev = dev_get_drvdata(dev);
-
- stat->current_frequency = kbdev->freq;
-
- kbase_pm_get_dvfs_utilisation(kbdev,
- &stat->total_time, &stat->busy_time);
-
- stat->private_data = NULL;
-
- return 0;
-}
-
-static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev,
- struct devfreq_dev_profile *dp)
-{
- int count;
- int i = 0;
- unsigned long freq = 0;
- struct dev_pm_opp *opp;
-
- rcu_read_lock();
- count = dev_pm_opp_get_opp_count(kbdev->dev);
- if (count < 0) {
- rcu_read_unlock();
- return count;
- }
- rcu_read_unlock();
-
- dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]),
- GFP_KERNEL);
- if (!dp->freq_table)
- return -ENOMEM;
-
- rcu_read_lock();
- for (i = 0; i < count; i++, freq++) {
- opp = dev_pm_opp_find_freq_ceil(kbdev->dev, &freq);
- if (IS_ERR(opp))
- break;
-
- dp->freq_table[i] = freq;
- }
- rcu_read_unlock();
-
- if (count != i)
- dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n",
- count, i);
-
- dp->max_state = i;
-
- return 0;
-}
-
-static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev)
-{
- struct devfreq_dev_profile *dp = kbdev->devfreq->profile;
-
- kfree(dp->freq_table);
-}
-
-static void kbase_devfreq_exit(struct device *dev)
-{
- struct kbase_device *kbdev = dev_get_drvdata(dev);
-
- kbase_devfreq_term_freq_table(kbdev);
-}
-
-int kbase_devfreq_init(struct kbase_device *kbdev)
-{
- struct devfreq_dev_profile *dp;
- int err;
-
- dev_dbg(kbdev->dev, "Init Mali devfreq\n");
-
- if (!kbdev->clock)
- return -ENODEV;
-
- kbdev->freq = clk_get_rate(kbdev->clock);
-
- dp = &kbdev->devfreq_profile;
-
- dp->initial_freq = kbdev->freq;
- dp->polling_ms = 100;
- dp->target = kbase_devfreq_target;
- dp->get_dev_status = kbase_devfreq_status;
- dp->get_cur_freq = kbase_devfreq_cur_freq;
- dp->exit = kbase_devfreq_exit;
-
- if (kbase_devfreq_init_freq_table(kbdev, dp))
- return -EFAULT;
-
- kbdev->devfreq = devfreq_add_device(kbdev->dev, dp,
- "simple_ondemand", NULL);
- if (IS_ERR_OR_NULL(kbdev->devfreq)) {
- kbase_devfreq_term_freq_table(kbdev);
- return PTR_ERR(kbdev->devfreq);
- }
-
- err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq);
- if (err) {
- dev_err(kbdev->dev,
- "Failed to register OPP notifier (%d)\n", err);
- goto opp_notifier_failed;
- }
-
-#ifdef CONFIG_DEVFREQ_THERMAL
- kbdev->devfreq_cooling = of_devfreq_cooling_register(
- kbdev->dev->of_node,
- kbdev->devfreq);
- if (IS_ERR_OR_NULL(kbdev->devfreq_cooling)) {
- err = PTR_ERR(kbdev->devfreq_cooling);
- dev_err(kbdev->dev,
- "Failed to register cooling device (%d)\n", err);
- goto cooling_failed;
- }
-
-#ifdef CONFIG_MALI_POWER_ACTOR
- err = mali_pa_init(kbdev);
- if (err) {
- dev_err(kbdev->dev, "Failed to init power actor\n");
- goto pa_failed;
- }
-#endif
-#endif
-
- return 0;
-
-#ifdef CONFIG_DEVFREQ_THERMAL
-#ifdef CONFIG_MALI_POWER_ACTOR
-pa_failed:
- devfreq_cooling_unregister(kbdev->devfreq_cooling);
-#endif /* CONFIG_MALI_POWER_ACTOR */
-cooling_failed:
- devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
-#endif /* CONFIG_DEVFREQ_THERMAL */
-opp_notifier_failed:
- err = devfreq_remove_device(kbdev->devfreq);
- if (err)
- dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
- else
- kbdev->devfreq = NULL;
-
- return err;
-}
-
-void kbase_devfreq_term(struct kbase_device *kbdev)
-{
- int err;
-
- dev_dbg(kbdev->dev, "Term Mali devfreq\n");
-
-#ifdef CONFIG_DEVFREQ_THERMAL
-#ifdef CONFIG_MALI_POWER_ACTOR
- mali_pa_term(kbdev);
-#endif
-
- devfreq_cooling_unregister(kbdev->devfreq_cooling);
-#endif
-
- devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq);
-
- err = devfreq_remove_device(kbdev->devfreq);
- if (err)
- dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err);
- else
- kbdev->devfreq = NULL;
-}
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-#ifndef _BASE_DEVFREQ_H_
-#define _BASE_DEVFREQ_H_
-
-int kbase_devfreq_init(struct kbase_device *kbdev);
-void kbase_devfreq_term(struct kbase_device *kbdev);
-
-#endif /* _BASE_DEVFREQ_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file mali_kbase_device.c
+/*
* Base kernel device APIs
*/
#include <linux/seq_file.h>
#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/of_platform.h>
#include <mali_kbase.h>
#include <mali_kbase_defs.h>
#include <mali_kbase_hw.h>
+#include <mali_kbase_config_defaults.h>
#include <mali_kbase_profiling_gator_api.h>
*/
#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254
-#if defined(CONFIG_MALI_PLATFORM_VEXPRESS) || defined(CONFIG_MALI_PLATFORM_VEXPRESS_VIRTEX7_40MHZ)
-#ifdef CONFIG_MALI_PLATFORM_FAKE
-extern struct kbase_attribute config_attributes_hw_issue_8408[];
-#endif /* CONFIG_MALI_PLATFORM_FAKE */
-#endif /* CONFIG_MALI_PLATFORM_VEXPRESS || CONFIG_MALI_PLATFORM_VEXPRESS_VIRTEX7_40MHZ */
-
#if KBASE_TRACE_ENABLE
-STATIC CONST char *kbasep_trace_code_string[] = {
+static const char *kbasep_trace_code_string[] = {
/* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
* THIS MUST BE USED AT THE START OF THE ARRAY */
#define KBASE_TRACE_CODE_MAKE_CODE(X) # X
#define DEBUG_MESSAGE_SIZE 256
-STATIC mali_error kbasep_trace_init(struct kbase_device *kbdev);
-STATIC void kbasep_trace_term(struct kbase_device *kbdev);
-STATIC void kbasep_trace_hook_wrapper(void *param);
-#if KBASE_TRACE_ENABLE
-STATIC void kbasep_trace_debugfs_init(struct kbase_device *kbdev);
-STATIC void kbasep_trace_debugfs_term(struct kbase_device *kbdev);
-#endif
+static int kbasep_trace_init(struct kbase_device *kbdev);
+static void kbasep_trace_term(struct kbase_device *kbdev);
+static void kbasep_trace_hook_wrapper(void *param);
struct kbase_device *kbase_device_alloc(void)
{
return kzalloc(sizeof(struct kbase_device), GFP_KERNEL);
}
-mali_error kbase_device_init(struct kbase_device * const kbdev)
+static int kbase_device_as_init(struct kbase_device *kbdev, int i)
{
- int i; /* i used after the for loop, don't reuse ! */
+ const char format[] = "mali_mmu%d";
+ char name[sizeof(format)];
+ const char poke_format[] = "mali_mmu%d_poker";
+ char poke_name[sizeof(poke_format)];
- spin_lock_init(&kbdev->mmu_mask_change);
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
+ snprintf(poke_name, sizeof(poke_name), poke_format, i);
- /* Initialize platform specific context */
- if (MALI_FALSE == kbasep_platform_device_init(kbdev))
- goto fail;
+ snprintf(name, sizeof(name), format, i);
- /* Ensure we can access the GPU registers */
- kbase_pm_register_access_enable(kbdev);
+ kbdev->as[i].number = i;
+ kbdev->as[i].fault_addr = 0ULL;
- /* Find out GPU properties based on the GPU feature registers */
- kbase_gpuprops_set(kbdev);
+ kbdev->as[i].pf_wq = alloc_workqueue(name, 0, 1);
+ if (!kbdev->as[i].pf_wq)
+ return -EINVAL;
- /* Get the list of workarounds for issues on the current HW (identified by the GPU_ID register) */
- if (MALI_ERROR_NONE != kbase_hw_set_issues_mask(kbdev)) {
- kbase_pm_register_access_disable(kbdev);
- goto free_platform;
- }
- /* Set the list of features available on the current HW (identified by the GPU_ID register) */
- kbase_hw_set_features_mask(kbdev);
+ mutex_init(&kbdev->as[i].transaction_mutex);
+ INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker);
+ INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker);
-#if defined(CONFIG_ARM64)
- set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops);
-#endif
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) {
+ struct hrtimer *poke_timer = &kbdev->as[i].poke_timer;
+ struct work_struct *poke_work = &kbdev->as[i].poke_work;
- /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our
- * device structure was created by device-tree
- */
- if (!kbdev->dev->dma_mask)
- kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask;
+ kbdev->as[i].poke_wq = alloc_workqueue(poke_name, 0, 1);
+ if (!kbdev->as[i].poke_wq) {
+ destroy_workqueue(kbdev->as[i].pf_wq);
+ return -EINVAL;
+ }
+ KBASE_DEBUG_ASSERT(!object_is_on_stack(poke_work));
+ INIT_WORK(poke_work, kbasep_as_do_poke);
- if (dma_set_mask(kbdev->dev,
- DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)))
- goto dma_set_mask_failed;
+ hrtimer_init(poke_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- if (dma_set_coherent_mask(kbdev->dev,
- DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)))
- goto dma_set_mask_failed;
+ poke_timer->function = kbasep_as_poke_timer_callback;
- if (kbase_mem_lowlevel_init(kbdev))
- goto mem_lowlevel_init_failed;
+ kbdev->as[i].poke_refcount = 0;
+ kbdev->as[i].poke_state = 0u;
+ }
- kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces;
+ return 0;
+}
- /* We're done accessing the GPU registers for now. */
- kbase_pm_register_access_disable(kbdev);
+static void kbase_device_as_term(struct kbase_device *kbdev, int i)
+{
+ destroy_workqueue(kbdev->as[i].pf_wq);
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
+ destroy_workqueue(kbdev->as[i].poke_wq);
+}
- for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
- const char format[] = "mali_mmu%d";
- char name[sizeof(format)];
- const char poke_format[] = "mali_mmu%d_poker"; /* BASE_HW_ISSUE_8316 */
- char poke_name[sizeof(poke_format)]; /* BASE_HW_ISSUE_8316 */
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) {
- if (0 > snprintf(poke_name, sizeof(poke_name), poke_format, i))
- goto free_workqs;
- }
+static int kbase_device_all_as_init(struct kbase_device *kbdev)
+{
+ int i, err;
- if (0 > snprintf(name, sizeof(name), format, i))
+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
+ err = kbase_device_as_init(kbdev, i);
+ if (err)
goto free_workqs;
+ }
+
+ return 0;
- kbdev->as[i].number = i;
- kbdev->as[i].fault_addr = 0ULL;
+free_workqs:
+ for (; i > 0; i--)
+ kbase_device_as_term(kbdev, i);
- kbdev->as[i].pf_wq = alloc_workqueue(name, 0, 1);
- if (NULL == kbdev->as[i].pf_wq)
- goto free_workqs;
+ return err;
+}
- mutex_init(&kbdev->as[i].transaction_mutex);
+static void kbase_device_all_as_term(struct kbase_device *kbdev)
+{
+ int i;
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) {
- struct hrtimer *poking_timer = &kbdev->as[i].poke_timer;
+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++)
+ kbase_device_as_term(kbdev, i);
+}
- kbdev->as[i].poke_wq = alloc_workqueue(poke_name, 0, 1);
- if (NULL == kbdev->as[i].poke_wq) {
- destroy_workqueue(kbdev->as[i].pf_wq);
- goto free_workqs;
- }
- KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&kbdev->as[i].poke_work));
- INIT_WORK(&kbdev->as[i].poke_work, kbasep_as_do_poke);
+int kbase_device_init(struct kbase_device * const kbdev)
+{
+ int i, err;
- hrtimer_init(poking_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ spin_lock_init(&kbdev->mmu_mask_change);
+ /* Get the list of workarounds for issues on the current HW
+ * (identified by the GPU_ID register)
+ */
+ err = kbase_hw_set_issues_mask(kbdev);
+ if (err)
+ goto fail;
+ /* Set the list of features available on the current HW
+ * (identified by the GPU_ID register)
+ */
+ kbase_hw_set_features_mask(kbdev);
- poking_timer->function = kbasep_as_poke_timer_callback;
+#if defined(CONFIG_ARM64)
+ set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops);
+#endif /* CONFIG_ARM64 */
- kbdev->as[i].poke_refcount = 0;
- kbdev->as[i].poke_state = 0u;
- }
- }
- /* don't change i after this point */
+ /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our
+ * device structure was created by device-tree
+ */
+ if (!kbdev->dev->dma_mask)
+ kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask;
- spin_lock_init(&kbdev->hwcnt.lock);
+ err = dma_set_mask(kbdev->dev,
+ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits));
+ if (err)
+ goto dma_set_mask_failed;
- kbdev->hwcnt.state = KBASE_INSTR_STATE_DISABLED;
- init_waitqueue_head(&kbdev->reset_wait);
- init_waitqueue_head(&kbdev->hwcnt.wait);
- init_waitqueue_head(&kbdev->hwcnt.cache_clean_wait);
- INIT_WORK(&kbdev->hwcnt.cache_clean_work, kbasep_cache_clean_worker);
- kbdev->hwcnt.triggered = 0;
+ err = dma_set_coherent_mask(kbdev->dev,
+ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits));
+ if (err)
+ goto dma_set_mask_failed;
- kbdev->hwcnt.cache_clean_wq = alloc_workqueue("Mali cache cleaning workqueue",
- 0, 1);
- if (NULL == kbdev->hwcnt.cache_clean_wq)
- goto free_workqs;
+ err = kbase_mem_lowlevel_init(kbdev);
+ if (err)
+ goto mem_lowlevel_init_failed;
-#if KBASE_GPU_RESET_EN
- kbdev->reset_workq = alloc_workqueue("Mali reset workqueue", 0, 1);
- if (NULL == kbdev->reset_workq)
- goto free_cache_clean_workq;
+ kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces;
- KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&kbdev->reset_work));
- INIT_WORK(&kbdev->reset_work, kbasep_reset_timeout_worker);
+ err = kbase_device_all_as_init(kbdev);
+ if (err)
+ goto term_lowlevel_mem;
- hrtimer_init(&kbdev->reset_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- kbdev->reset_timer.function = kbasep_reset_timer_callback;
+ spin_lock_init(&kbdev->hwcnt.lock);
- if (kbasep_trace_init(kbdev) != MALI_ERROR_NONE)
- goto free_reset_workq;
-#else
- if (kbasep_trace_init(kbdev) != MALI_ERROR_NONE)
- goto free_cache_clean_workq;
-#endif /* KBASE_GPU_RESET_EN */
+ err = kbasep_trace_init(kbdev);
+ if (err)
+ goto term_as;
mutex_init(&kbdev->cacheclean_lock);
- atomic_set(&kbdev->keep_gpu_powered_count, 0);
#ifdef CONFIG_MALI_TRACE_TIMELINE
- for (i = 0; i < BASE_JM_SUBMIT_SLOTS; ++i)
+ for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i)
kbdev->timeline.slot_atoms_submitted[i] = 0;
for (i = 0; i <= KBASEP_TIMELINE_PM_EVENT_LAST; ++i)
kbase_debug_assert_register_hook(&kbasep_trace_hook_wrapper, kbdev);
-#if defined(CONFIG_MALI_PLATFORM_VEXPRESS) || defined(CONFIG_MALI_PLATFORM_VEXPRESS_VIRTEX7_40MHZ)
-#ifdef CONFIG_MALI_PLATFORM_FAKE
- /* BASE_HW_ISSUE_8408 requires a configuration with different timeouts for
- * the vexpress platform */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
- kbdev->config_attributes = config_attributes_hw_issue_8408;
-#endif /* CONFIG_MALI_PLATFORM_FAKE */
-#endif /* CONFIG_MALI_PLATFORM_VEXPRESS || CONFIG_MALI_PLATFORM_VEXPRESS_VIRTEX7_40MHZ */
-
atomic_set(&kbdev->ctx_num, 0);
- return MALI_ERROR_NONE;
-#if KBASE_GPU_RESET_EN
-free_reset_workq:
- destroy_workqueue(kbdev->reset_workq);
-#endif /* KBASE_GPU_RESET_EN */
-free_cache_clean_workq:
- destroy_workqueue(kbdev->hwcnt.cache_clean_wq);
- free_workqs:
- while (i > 0) {
- i--;
- destroy_workqueue(kbdev->as[i].pf_wq);
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
- destroy_workqueue(kbdev->as[i].poke_wq);
- }
+ err = kbase_instr_backend_init(kbdev);
+ if (err)
+ goto term_trace;
+
+ kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD;
+
+ kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS;
+
+ kbdev->mmu_mode = kbase_mmu_mode_get_lpae();
+
+ return 0;
+term_trace:
+ kbasep_trace_term(kbdev);
+term_as:
+ kbase_device_all_as_term(kbdev);
+term_lowlevel_mem:
kbase_mem_lowlevel_term(kbdev);
mem_lowlevel_init_failed:
dma_set_mask_failed:
-free_platform:
- kbasep_platform_device_term(kbdev);
fail:
- return MALI_ERROR_FUNCTION_FAILED;
+ return err;
}
void kbase_device_term(struct kbase_device *kbdev)
{
- int i;
-
KBASE_DEBUG_ASSERT(kbdev);
#if KBASE_TRACE_ENABLE
kbase_debug_assert_register_hook(NULL, NULL);
#endif
- kbasep_trace_term(kbdev);
-
-#if KBASE_GPU_RESET_EN
- destroy_workqueue(kbdev->reset_workq);
-#endif
+ kbase_instr_backend_term(kbdev);
- destroy_workqueue(kbdev->hwcnt.cache_clean_wq);
+ kbasep_trace_term(kbdev);
- for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
- destroy_workqueue(kbdev->as[i].pf_wq);
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
- destroy_workqueue(kbdev->as[i].poke_wq);
- }
+ kbase_device_all_as_term(kbdev);
kbase_mem_lowlevel_term(kbdev);
- kbasep_platform_device_term(kbdev);
}
void kbase_device_free(struct kbase_device *kbdev)
void kbase_device_trace_buffer_install(struct kbase_context *kctx, u32 *tb, size_t size)
{
unsigned long flags;
+
KBASE_DEBUG_ASSERT(kctx);
KBASE_DEBUG_ASSERT(tb);
void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx)
{
unsigned long flags;
+
KBASE_DEBUG_ASSERT(kctx);
spin_lock_irqsave(&kctx->jctx.tb_lock, flags);
kctx->jctx.tb = NULL;
void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value)
{
unsigned long flags;
+
spin_lock_irqsave(&kctx->jctx.tb_lock, flags);
if (kctx->jctx.tb) {
u16 wrap_count;
spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags);
}
-void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, struct kbase_context *kctx)
-{
- KBASE_DEBUG_ASSERT(kbdev->pm.gpu_powered);
- KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID);
- KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
- dev_dbg(kbdev->dev, "w: reg %04x val %08x", offset, value);
- kbase_os_reg_write(kbdev, offset, value);
- if (kctx && kctx->jctx.tb)
- kbase_device_trace_register_access(kctx, REG_WRITE, offset, value);
-}
-
-KBASE_EXPORT_TEST_API(kbase_reg_write)
-
-u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, struct kbase_context *kctx)
-{
- u32 val;
- KBASE_DEBUG_ASSERT(kbdev->pm.gpu_powered);
- KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID);
- KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
- val = kbase_os_reg_read(kbdev, offset);
- dev_dbg(kbdev->dev, "r: reg %04x val %08x", offset, val);
- if (kctx && kctx->jctx.tb)
- kbase_device_trace_register_access(kctx, REG_READ, offset, val);
- return val;
-}
-
-KBASE_EXPORT_TEST_API(kbase_reg_read)
-
-#if KBASE_PM_EN
-void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple)
-{
- u32 status;
- u64 address;
-
- status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL);
- address = (u64) kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTADDRESS_HI), NULL) << 32;
- address |= kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTADDRESS_LO), NULL);
-
- dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", status & 0xFF, kbase_exception_name(status), address);
- kbdev->kbase_group_error++;
- if (multiple)
- dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n");
-}
-
-void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
-{
- KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val);
- if (val & GPU_FAULT)
- kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS);
-
- if (val & RESET_COMPLETED)
- kbase_pm_reset_done(kbdev);
-
- if (val & PRFCNT_SAMPLE_COMPLETED)
- kbase_instr_hwcnt_sample_done(kbdev);
-
- if (val & CLEAN_CACHES_COMPLETED)
- kbase_clean_caches_done(kbdev);
-
- KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, NULL);
-
- /* kbase_pm_check_transitions must be called after the IRQ has been cleared. This is because it might trigger
- * further power transitions and we don't want to miss the interrupt raised to notify us that these further
- * transitions have finished.
- */
- if (val & POWER_CHANGED_ALL) {
- mali_bool cores_are_available;
- unsigned long flags;
-
- KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START);
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
- KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END);
-
- if (cores_are_available) {
- /* Fast-path Job Scheduling on PM IRQ */
- int js;
- /* Log timelining information that a change in state has completed */
- kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
-
- spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
- /* A simplified check to ensure the last context hasn't exited
- * after dropping the PM lock whilst doing a PM IRQ: any bits set
- * in 'submit_allowed' indicate that we have a context in the
- * runpool (which can't leave whilst we hold this lock). It is
- * sometimes zero even when we have a context in the runpool, but
- * that's no problem because we'll be unable to submit jobs
- * anyway */
- if (kbdev->js_data.runpool_irq.submit_allowed)
- for (js = 0; js < kbdev->gpu_props.num_job_slots; ++js) {
- mali_bool needs_retry;
- s8 submitted_count = 0;
- needs_retry = kbasep_js_try_run_next_job_on_slot_irq_nolock(kbdev, js, &submitted_count);
- /* Don't need to retry outside of IRQ context - this can
- * only happen if we submitted too many in one IRQ, such
- * that they were completing faster than we could
- * submit. In this case, a job IRQ will fire to cause more
- * work to be submitted in some way */
- CSTD_UNUSED(needs_retry);
- }
- spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
- }
- }
- KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val);
-}
-#endif /* KBASE_PM_EN */
/*
* Device trace functions
*/
#if KBASE_TRACE_ENABLE
-STATIC mali_error kbasep_trace_init(struct kbase_device *kbdev)
+static int kbasep_trace_init(struct kbase_device *kbdev)
{
- void *rbuf;
+ struct kbase_trace *rbuf;
- rbuf = kmalloc(sizeof(struct kbase_trace) * KBASE_TRACE_SIZE, GFP_KERNEL);
+ rbuf = kmalloc_array(KBASE_TRACE_SIZE, sizeof(*rbuf), GFP_KERNEL);
if (!rbuf)
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
kbdev->trace_rbuf = rbuf;
spin_lock_init(&kbdev->trace_lock);
- kbasep_trace_debugfs_init(kbdev);
- return MALI_ERROR_NONE;
+ return 0;
}
-STATIC void kbasep_trace_term(struct kbase_device *kbdev)
+static void kbasep_trace_term(struct kbase_device *kbdev)
{
- kbasep_trace_debugfs_term(kbdev);
kfree(kbdev->trace_rbuf);
}
/* Initial part of message */
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d.%.6d,%d,%d,%s,%p,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, kbasep_trace_code_string[trace_msg->code], trace_msg->ctx), 0);
- if (trace_msg->katom != MALI_FALSE) {
+ if (trace_msg->katom)
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "atom %d (ud: 0x%llx 0x%llx)", trace_msg->atom_number, trace_msg->atom_udata[0], trace_msg->atom_udata[1]), 0);
- }
written += MAX(snprintf(buffer + written, MAX(len - written, 0), ",%.8llx,", trace_msg->gpu_addr), 0);
/* NOTE: Could add function callbacks to handle different message types */
/* Jobslot present */
- if ((trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT) != MALI_FALSE)
+ if (trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT)
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->jobslot), 0);
written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0);
/* Refcount present */
- if ((trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT) != MALI_FALSE)
+ if (trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT)
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->refcount), 0);
written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0);
/* Rest of message */
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "0x%.8lx", trace_msg->info_val), 0);
-
}
static void kbasep_trace_dump_msg(struct kbase_device *kbdev, struct kbase_trace *trace_msg)
trace_msg->ctx = ctx;
if (NULL == katom) {
- trace_msg->katom = MALI_FALSE;
+ trace_msg->katom = false;
} else {
- trace_msg->katom = MALI_TRUE;
+ trace_msg->katom = true;
trace_msg->atom_number = kbase_jd_atom_id(katom->kctx, katom);
trace_msg->atom_udata[0] = katom->udata.blob[0];
trace_msg->atom_udata[1] = katom->udata.blob[1];
void kbasep_trace_clear(struct kbase_device *kbdev)
{
unsigned long flags;
+
spin_lock_irqsave(&kbdev->trace_lock, flags);
kbdev->trace_first_out = kbdev->trace_next_in;
spin_unlock_irqrestore(&kbdev->trace_lock, flags);
while (start != end) {
struct kbase_trace *trace_msg = &kbdev->trace_rbuf[start];
+
kbasep_trace_dump_msg(kbdev, trace_msg);
start = (start + 1) & KBASE_TRACE_MASK;
KBASE_TRACE_CLEAR(kbdev);
}
-STATIC void kbasep_trace_hook_wrapper(void *param)
+static void kbasep_trace_hook_wrapper(void *param)
{
struct kbase_device *kbdev = (struct kbase_device *)param;
+
kbasep_trace_dump(kbdev);
}
.release = seq_release_private,
};
-STATIC void kbasep_trace_debugfs_init(struct kbase_device *kbdev)
+void kbasep_trace_debugfs_init(struct kbase_device *kbdev)
{
- kbdev->trace_dentry = debugfs_create_file("mali_trace", S_IRUGO,
+ debugfs_create_file("mali_trace", S_IRUGO,
kbdev->mali_debugfs_directory, kbdev,
&kbasep_trace_debugfs_fops);
}
-STATIC void kbasep_trace_debugfs_term(struct kbase_device *kbdev)
-{
- debugfs_remove(kbdev->trace_dentry);
- kbdev->trace_dentry = NULL;
-}
#else
-STATIC void kbasep_trace_debugfs_init(struct kbase_device *kbdev)
-{
-
-}
-STATIC void kbasep_trace_debugfs_term(struct kbase_device *kbdev)
+void kbasep_trace_debugfs_init(struct kbase_device *kbdev)
{
-
}
#endif /* CONFIG_DEBUG_FS */
#else /* KBASE_TRACE_ENABLE */
-STATIC mali_error kbasep_trace_init(struct kbase_device *kbdev)
+static int kbasep_trace_init(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
- return MALI_ERROR_NONE;
+ return 0;
}
-STATIC void kbasep_trace_term(struct kbase_device *kbdev)
+static void kbasep_trace_term(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
}
-STATIC void kbasep_trace_hook_wrapper(void *param)
+static void kbasep_trace_hook_wrapper(void *param)
{
CSTD_UNUSED(param);
}
-void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val)
-{
- CSTD_UNUSED(kbdev);
- CSTD_UNUSED(code);
- CSTD_UNUSED(ctx);
- CSTD_UNUSED(katom);
- CSTD_UNUSED(gpu_addr);
- CSTD_UNUSED(flags);
- CSTD_UNUSED(refcount);
- CSTD_UNUSED(jobslot);
- CSTD_UNUSED(info_val);
-}
-
-void kbasep_trace_clear(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
void kbasep_trace_dump(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
/* find the first i.e. call with -1 */
kbdev = kbase_find_device(-1);
- if (NULL != kbdev) {
+ if (NULL != kbdev)
kbase_set_profiling_control(kbdev, action, value);
- }
}
-
KBASE_EXPORT_SYMBOL(_mali_profiling_control);
+
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file mali_kbase_disjoint_events.c
+/*
* Base kernel disjoint events helper functions
*/
return atomic_read(&kbdev->disjoint_event.count);
}
-KBASE_EXPORT_TEST_API(kbase_disjoint_event_get)
+KBASE_EXPORT_TEST_API(kbase_disjoint_event_get);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase.h>
#include <mali_kbase_debug.h>
-STATIC struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom)
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+#include <mali_kbase_tlstream.h>
+#endif
+
+static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
struct base_jd_udata data;
KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(1, &kctx->timeline.jd_atoms_in_flight));
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_tl_nret_atom_ctx(katom, kctx);
+ kbase_tlstream_tl_del_atom(katom);
+#endif
+
mutex_lock(&kctx->jctx.lock);
katom->status = KBASE_JD_ATOM_STATE_UNUSED;
mutex_unlock(&kctx->jctx.lock);
KBASE_DEBUG_ASSERT(ctx);
mutex_lock(&ctx->event_mutex);
- ret = (!list_empty(&ctx->event_list)) || (MALI_TRUE == ctx->event_closed);
+ ret = (!list_empty(&ctx->event_list)) || (true == ctx->event_closed);
mutex_unlock(&ctx->event_mutex);
return ret;
}
-KBASE_EXPORT_TEST_API(kbase_event_pending)
+KBASE_EXPORT_TEST_API(kbase_event_pending);
int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent)
{
mutex_lock(&ctx->event_mutex);
if (list_empty(&ctx->event_list)) {
- if (ctx->event_closed) {
- /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */
- mutex_unlock(&ctx->event_mutex);
- uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED;
- memset(&uevent->udata, 0, sizeof(uevent->udata));
- dev_dbg(ctx->kbdev->dev,
- "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n",
- BASE_JD_EVENT_DRV_TERMINATED);
- return 0;
- } else {
+ if (!ctx->event_closed) {
mutex_unlock(&ctx->event_mutex);
return -1;
}
+
+ /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */
+ mutex_unlock(&ctx->event_mutex);
+ uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED;
+ memset(&uevent->udata, 0, sizeof(uevent->udata));
+ dev_dbg(ctx->kbdev->dev,
+ "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n",
+ BASE_JD_EVENT_DRV_TERMINATED);
+ return 0;
}
/* normal event processing */
return 0;
}
-KBASE_EXPORT_TEST_API(kbase_event_dequeue)
+KBASE_EXPORT_TEST_API(kbase_event_dequeue);
static void kbase_event_post_worker(struct work_struct *data)
{
queue_work(ctx->event_workq, &atom->work);
}
-KBASE_EXPORT_TEST_API(kbase_event_post)
+KBASE_EXPORT_TEST_API(kbase_event_post);
void kbase_event_close(struct kbase_context *kctx)
{
mutex_lock(&kctx->event_mutex);
- kctx->event_closed = MALI_TRUE;
+ kctx->event_closed = true;
mutex_unlock(&kctx->event_mutex);
kbase_event_wakeup(kctx);
}
-mali_error kbase_event_init(struct kbase_context *kctx)
+int kbase_event_init(struct kbase_context *kctx)
{
KBASE_DEBUG_ASSERT(kctx);
INIT_LIST_HEAD(&kctx->event_list);
mutex_init(&kctx->event_mutex);
- kctx->event_closed = MALI_FALSE;
+ kctx->event_closed = false;
kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1);
if (NULL == kctx->event_workq)
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
- return MALI_ERROR_NONE;
+ return 0;
}
-KBASE_EXPORT_TEST_API(kbase_event_init)
+KBASE_EXPORT_TEST_API(kbase_event_init);
void kbase_event_cleanup(struct kbase_context *kctx)
{
}
}
-KBASE_EXPORT_TEST_API(kbase_event_cleanup)
+KBASE_EXPORT_TEST_API(kbase_event_cleanup);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/* NB taken from gator */
/*
- * List of possible actions to be controlled by Streamline.
- * The following numbers are used by gator to control the frame buffer dumping and s/w counter reporting.
- * We cannot use the enums in mali_uk_types.h because they are unknown inside gator.
+ * List of possible actions to be controlled by DS-5 Streamline.
+ * The following numbers are used by gator to control the frame buffer dumping
+ * and s/w counter reporting. We cannot use the enums in mali_uk_types.h because
+ * they are unknown inside gator.
*/
#ifndef _KBASE_GATOR_H_
#define _KBASE_GATOR_H_
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include "mali_kbase_mem_linux.h"
#include "mali_kbase_gator_api.h"
#include "mali_kbase_gator_hwcnt_names.h"
+#include "mali_kbase_instr.h"
#define MALI_MAX_CORES_PER_GROUP 4
#define MALI_MAX_NUM_BLOCKS_PER_GROUP 8
struct kbase_gator_hwcnt_handles {
struct kbase_device *kbdev;
struct kbase_context *kctx;
- mali_addr64 hwcnt_gpu_va;
+ u64 hwcnt_gpu_va;
void *hwcnt_cpu_va;
struct kbase_vmap_struct hwcnt_map;
};
-const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_number_of_counters)
+const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters)
{
uint32_t gpu_id;
- const char * const *hardware_counter_names;
+ const char * const *hardware_counters;
struct kbase_device *kbdev;
- if (!total_number_of_counters)
+ if (!total_counters)
return NULL;
/* Get the first device - it doesn't matter in this case */
switch (gpu_id) {
/* If we are using a Mali-T60x device */
case GPU_ID_PI_T60X:
- hardware_counter_names = hardware_counter_names_mali_t60x;
- *total_number_of_counters = ARRAY_SIZE(hardware_counter_names_mali_t60x);
- break;
+ hardware_counters = hardware_counters_mali_t60x;
+ *total_counters = ARRAY_SIZE(hardware_counters_mali_t60x);
+ break;
/* If we are using a Mali-T62x device */
case GPU_ID_PI_T62X:
- hardware_counter_names = hardware_counter_names_mali_t62x;
- *total_number_of_counters = ARRAY_SIZE(hardware_counter_names_mali_t62x);
- break;
+ hardware_counters = hardware_counters_mali_t62x;
+ *total_counters = ARRAY_SIZE(hardware_counters_mali_t62x);
+ break;
/* If we are using a Mali-T72x device */
case GPU_ID_PI_T72X:
- hardware_counter_names = hardware_counter_names_mali_t72x;
- *total_number_of_counters = ARRAY_SIZE(hardware_counter_names_mali_t72x);
- break;
+ hardware_counters = hardware_counters_mali_t72x;
+ *total_counters = ARRAY_SIZE(hardware_counters_mali_t72x);
+ break;
/* If we are using a Mali-T76x device */
case GPU_ID_PI_T76X:
- hardware_counter_names = hardware_counter_names_mali_t76x;
- *total_number_of_counters = ARRAY_SIZE(hardware_counter_names_mali_t76x);
- break;
-#ifdef MALI_INCLUDE_TFRX
- /* If we are using a Mali-TFRX device - for now just mimic the T760 counters */
- case GPU_ID_PI_TFRX:
- hardware_counter_names = hardware_counter_names_mali_t76x;
- *total_number_of_counters = ARRAY_SIZE(hardware_counter_names_mali_t76x);
- break;
-#endif /* MALI_INCLUDE_TRFX */
- /* If we are using a Mali-T86X device - for now just mimic the T760 counters */
+ hardware_counters = hardware_counters_mali_t76x;
+ *total_counters = ARRAY_SIZE(hardware_counters_mali_t76x);
+ break;
+ /* If we are using a Mali-T82x device */
+ case GPU_ID_PI_T82X:
+ hardware_counters = hardware_counters_mali_t82x;
+ *total_counters = ARRAY_SIZE(hardware_counters_mali_t82x);
+ break;
+ /* If we are using a Mali-T83x device */
+ case GPU_ID_PI_T83X:
+ hardware_counters = hardware_counters_mali_t83x;
+ *total_counters = ARRAY_SIZE(hardware_counters_mali_t83x);
+ break;
+ /* If we are using a Mali-T86x device */
case GPU_ID_PI_T86X:
- hardware_counter_names = hardware_counter_names_mali_t76x;
- *total_number_of_counters = ARRAY_SIZE(hardware_counter_names_mali_t76x);
- break;
+ hardware_counters = hardware_counters_mali_t86x;
+ *total_counters = ARRAY_SIZE(hardware_counters_mali_t86x);
+ break;
+ /* If we are using a Mali-T88x device */
+ case GPU_ID_PI_TFRX:
+ hardware_counters = hardware_counters_mali_t88x;
+ *total_counters = ARRAY_SIZE(hardware_counters_mali_t88x);
+ break;
default:
- hardware_counter_names = NULL;
- *total_number_of_counters = 0;
- dev_err(kbdev->dev, "Unrecognized gpu ID: %u\n", gpu_id);
- break;
+ hardware_counters = NULL;
+ *total_counters = 0;
+ dev_err(kbdev->dev, "Unrecognized gpu ID: %u\n", gpu_id);
+ break;
}
/* Release the kbdev reference. */
kbase_release_device(kbdev);
/* If we return a string array take a reference on the module (or fail). */
- if (hardware_counter_names && !try_module_get(THIS_MODULE))
+ if (hardware_counters && !try_module_get(THIS_MODULE))
return NULL;
- return hardware_counter_names;
+ return hardware_counters;
}
-KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init_names)
+KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init_names);
void kbase_gator_hwcnt_term_names(void)
{
/* Release the module reference. */
module_put(THIS_MODULE);
}
-KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term_names)
+KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term_names);
struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info)
{
struct kbase_gator_hwcnt_handles *hand;
struct kbase_uk_hwcnt_setup setup;
- mali_error err;
+ int err;
uint32_t dump_size = 0, i = 0;
struct kbase_va_region *reg;
u64 flags;
in_out_info->hwc_layout[i++] = TILER_BLOCK;
in_out_info->hwc_layout[i++] = MMU_L2_BLOCK;
- /* There are no implementation with L3 cache */
in_out_info->hwc_layout[i++] = RESERVED_BLOCK;
if (0 == cg)
else
in_out_info->hwc_layout[i++] = RESERVED_BLOCK;
}
- /* If we are using a Mali-T76x device */
- } else if (
- (in_out_info->gpu_id == GPU_ID_PI_T76X)
-#ifdef MALI_INCLUDE_TFRX
- || (in_out_info->gpu_id == GPU_ID_PI_TFRX)
-#endif /* MALI_INCLUDE_TFRX */
- || (in_out_info->gpu_id == GPU_ID_PI_T86X)
-#ifdef MALI_INCLUDE_TGAL
- || (in_out_info->gpu_id == GPU_ID_PI_TGAL)
-#endif
- ) {
+ /* If we are using any other device */
+ } else {
uint32_t nr_l2, nr_sc, j;
uint64_t core_mask;
nr_sc = hand->kbdev->gpu_props.props.coherency_info.group[0].num_cores;
- /* For Mali-T76x, the job manager and tiler sets of counters are always present */
+ /* The job manager and tiler sets of counters
+ * are always present */
in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * (2 + nr_sc + nr_l2), GFP_KERNEL);
if (!in_out_info->hwc_layout)
setup.shader_bm = in_out_info->bitmask[2];
setup.mmu_l2_bm = in_out_info->bitmask[3];
- /* There are no implementations with L3 cache */
- setup.l3_cache_bm = 0;
-
err = kbase_instr_hwcnt_enable(hand->kctx, &setup);
- if (err != MALI_ERROR_NONE)
+ if (err)
goto free_unmap;
kbase_instr_hwcnt_clear(hand->kctx);
return NULL;
}
-KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init)
+KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init);
void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles)
{
kfree(opaque_handles);
}
}
-KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term)
+KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term);
-uint32_t kbase_gator_instr_hwcnt_dump_complete(struct kbase_gator_hwcnt_handles *opaque_handles, uint32_t * const success)
+uint32_t kbase_gator_instr_hwcnt_dump_complete(
+ struct kbase_gator_hwcnt_handles *opaque_handles,
+ uint32_t * const success)
{
- if (opaque_handles && success)
- return (kbase_instr_hwcnt_dump_complete(opaque_handles->kctx, success) != 0);
+ bool ret_res, success_res;
+
+ if (opaque_handles && success) {
+ ret_res = kbase_instr_hwcnt_dump_complete(opaque_handles->kctx,
+ &success_res);
+ *success = (uint32_t)success_res;
+ return (uint32_t)(ret_res != 0);
+ }
return 0;
}
-KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_complete)
+KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_complete);
uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles)
{
if (opaque_handles)
- return (kbase_instr_hwcnt_dump_irq(opaque_handles->kctx) == MALI_ERROR_NONE);
+ return (kbase_instr_hwcnt_request_dump(
+ opaque_handles->kctx) == 0);
+
return 0;
}
-KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_irq)
+KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_irq);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define _KBASE_GATOR_API_H_
/**
- * @brief This file describes the API used by Gator to collect hardware counters data from a Mali device.
+ * @brief This file describes the API used by Gator to fetch hardware counters.
*/
/* This define is used by the gator kernel module compile to select which DDK
*
* 8] Release the dump resources by calling kbase_gator_hwcnt_term().
*
- * 9] Release the name table resources by calling kbase_gator_hwcnt_term_names().
- * This function must only be called if init_names() returned a non-NULL value.
+ * 9] Release the name table resources by calling
+ * kbase_gator_hwcnt_term_names(). This function must only be called if
+ * init_names() returned a non-NULL value.
**/
#define MALI_DDK_GATOR_API_VERSION 3
};
struct kbase_gator_hwcnt_info {
-
/* Passed from Gator to kbase */
/* the bitmask of enabled hardware counters for each counter block */
/**
* @brief This function is used to fetch the names table based on the Mali device in use.
*
- * @param[out] total_number_of_counters The total number of counters short names in the Mali devices' list.
+ * @param[out] total_counters The total number of counters short names in the Mali devices' list.
*
- * @return Pointer to an array of strings of length *total_number_of_counters.
+ * @return Pointer to an array of strings of length *total_counters.
*/
-extern const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_number_of_counters);
+extern const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters);
/**
* @brief This function is used to terminate the use of the names table.
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* where no counter exists.
*/
-static const char * const hardware_counter_names_mali_t60x[] = {
+static const char * const hardware_counters_mali_t60x[] = {
/* Job Manager */
"",
"",
"T60x_L2_SNOOP_FULL",
"T60x_L2_REPLAY_FULL"
};
-static const char * const hardware_counter_names_mali_t62x[] = {
+static const char * const hardware_counters_mali_t62x[] = {
/* Job Manager */
"",
"",
"T62x_L2_REPLAY_FULL"
};
-static const char * const hardware_counter_names_mali_t72x[] = {
+static const char * const hardware_counters_mali_t72x[] = {
/* Job Manager */
"",
"",
""
};
-static const char * const hardware_counter_names_mali_t76x[] = {
+static const char * const hardware_counters_mali_t76x[] = {
/* Job Manager */
"",
"",
"T76x_L2_REPLAY_FULL"
};
+static const char * const hardware_counters_mali_t82x[] = {
+ /* Job Manager */
+ "",
+ "",
+ "",
+ "",
+ "T82x_MESSAGES_SENT",
+ "T82x_MESSAGES_RECEIVED",
+ "T82x_GPU_ACTIVE",
+ "T82x_IRQ_ACTIVE",
+ "T82x_JS0_JOBS",
+ "T82x_JS0_TASKS",
+ "T82x_JS0_ACTIVE",
+ "",
+ "T82x_JS0_WAIT_READ",
+ "T82x_JS0_WAIT_ISSUE",
+ "T82x_JS0_WAIT_DEPEND",
+ "T82x_JS0_WAIT_FINISH",
+ "T82x_JS1_JOBS",
+ "T82x_JS1_TASKS",
+ "T82x_JS1_ACTIVE",
+ "",
+ "T82x_JS1_WAIT_READ",
+ "T82x_JS1_WAIT_ISSUE",
+ "T82x_JS1_WAIT_DEPEND",
+ "T82x_JS1_WAIT_FINISH",
+ "T82x_JS2_JOBS",
+ "T82x_JS2_TASKS",
+ "T82x_JS2_ACTIVE",
+ "",
+ "T82x_JS2_WAIT_READ",
+ "T82x_JS2_WAIT_ISSUE",
+ "T82x_JS2_WAIT_DEPEND",
+ "T82x_JS2_WAIT_FINISH",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+
+ /*Tiler */
+ "",
+ "",
+ "",
+ "T82x_TI_JOBS_PROCESSED",
+ "T82x_TI_TRIANGLES",
+ "T82x_TI_QUADS",
+ "T82x_TI_POLYGONS",
+ "T82x_TI_POINTS",
+ "T82x_TI_LINES",
+ "T82x_TI_FRONT_FACING",
+ "T82x_TI_BACK_FACING",
+ "T82x_TI_PRIM_VISIBLE",
+ "T82x_TI_PRIM_CULLED",
+ "T82x_TI_PRIM_CLIPPED",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "T82x_TI_ACTIVE",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+
+ /* Shader Core */
+ "",
+ "",
+ "",
+ "",
+ "T82x_FRAG_ACTIVE",
+ "T82x_FRAG_PRIMITIVES",
+ "T82x_FRAG_PRIMITIVES_DROPPED",
+ "T82x_FRAG_CYCLES_DESC",
+ "T82x_FRAG_CYCLES_FPKQ_ACTIVE",
+ "T82x_FRAG_CYCLES_VERT",
+ "T82x_FRAG_CYCLES_TRISETUP",
+ "T82x_FRAG_CYCLES_EZS_ACTIVE",
+ "T82x_FRAG_THREADS",
+ "T82x_FRAG_DUMMY_THREADS",
+ "T82x_FRAG_QUADS_RAST",
+ "T82x_FRAG_QUADS_EZS_TEST",
+ "T82x_FRAG_QUADS_EZS_KILLED",
+ "T82x_FRAG_THREADS_LZS_TEST",
+ "T82x_FRAG_THREADS_LZS_KILLED",
+ "T82x_FRAG_CYCLES_NO_TILE",
+ "T82x_FRAG_NUM_TILES",
+ "T82x_FRAG_TRANS_ELIM",
+ "T82x_COMPUTE_ACTIVE",
+ "T82x_COMPUTE_TASKS",
+ "T82x_COMPUTE_THREADS",
+ "T82x_COMPUTE_CYCLES_DESC",
+ "T82x_TRIPIPE_ACTIVE",
+ "T82x_ARITH_WORDS",
+ "T82x_ARITH_CYCLES_REG",
+ "T82x_ARITH_CYCLES_L0",
+ "T82x_ARITH_FRAG_DEPEND",
+ "T82x_LS_WORDS",
+ "T82x_LS_ISSUES",
+ "T82x_LS_REISSUE_ATTR",
+ "T82x_LS_REISSUES_VARY",
+ "T82x_LS_VARY_RV_MISS",
+ "T82x_LS_VARY_RV_HIT",
+ "T82x_LS_NO_UNPARK",
+ "T82x_TEX_WORDS",
+ "T82x_TEX_BUBBLES",
+ "T82x_TEX_WORDS_L0",
+ "T82x_TEX_WORDS_DESC",
+ "T82x_TEX_ISSUES",
+ "T82x_TEX_RECIRC_FMISS",
+ "T82x_TEX_RECIRC_DESC",
+ "T82x_TEX_RECIRC_MULTI",
+ "T82x_TEX_RECIRC_PMISS",
+ "T82x_TEX_RECIRC_CONF",
+ "T82x_LSC_READ_HITS",
+ "T82x_LSC_READ_OP",
+ "T82x_LSC_WRITE_HITS",
+ "T82x_LSC_WRITE_OP",
+ "T82x_LSC_ATOMIC_HITS",
+ "T82x_LSC_ATOMIC_OP",
+ "T82x_LSC_LINE_FETCHES",
+ "T82x_LSC_DIRTY_LINE",
+ "T82x_LSC_SNOOPS",
+ "T82x_AXI_TLB_STALL",
+ "T82x_AXI_TLB_MIESS",
+ "T82x_AXI_TLB_TRANSACTION",
+ "T82x_LS_TLB_MISS",
+ "T82x_LS_TLB_HIT",
+ "T82x_AXI_BEATS_READ",
+ "T82x_AXI_BEATS_WRITTEN",
+
+ /*L2 and MMU */
+ "",
+ "",
+ "",
+ "",
+ "T82x_MMU_HIT",
+ "T82x_MMU_NEW_MISS",
+ "T82x_MMU_REPLAY_FULL",
+ "T82x_MMU_REPLAY_MISS",
+ "T82x_MMU_TABLE_WALK",
+ "T82x_MMU_REQUESTS",
+ "",
+ "",
+ "T82x_UTLB_HIT",
+ "T82x_UTLB_NEW_MISS",
+ "T82x_UTLB_REPLAY_FULL",
+ "T82x_UTLB_REPLAY_MISS",
+ "T82x_UTLB_STALL",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "T82x_L2_EXT_WRITE_BEATS",
+ "T82x_L2_EXT_READ_BEATS",
+ "T82x_L2_ANY_LOOKUP",
+ "T82x_L2_READ_LOOKUP",
+ "T82x_L2_SREAD_LOOKUP",
+ "T82x_L2_READ_REPLAY",
+ "T82x_L2_READ_SNOOP",
+ "T82x_L2_READ_HIT",
+ "T82x_L2_CLEAN_MISS",
+ "T82x_L2_WRITE_LOOKUP",
+ "T82x_L2_SWRITE_LOOKUP",
+ "T82x_L2_WRITE_REPLAY",
+ "T82x_L2_WRITE_SNOOP",
+ "T82x_L2_WRITE_HIT",
+ "T82x_L2_EXT_READ_FULL",
+ "",
+ "T82x_L2_EXT_WRITE_FULL",
+ "T82x_L2_EXT_R_W_HAZARD",
+ "T82x_L2_EXT_READ",
+ "T82x_L2_EXT_READ_LINE",
+ "T82x_L2_EXT_WRITE",
+ "T82x_L2_EXT_WRITE_LINE",
+ "T82x_L2_EXT_WRITE_SMALL",
+ "T82x_L2_EXT_BARRIER",
+ "T82x_L2_EXT_AR_STALL",
+ "T82x_L2_EXT_R_BUF_FULL",
+ "T82x_L2_EXT_RD_BUF_FULL",
+ "T82x_L2_EXT_R_RAW",
+ "T82x_L2_EXT_W_STALL",
+ "T82x_L2_EXT_W_BUF_FULL",
+ "T82x_L2_EXT_R_BUF_FULL",
+ "T82x_L2_TAG_HAZARD",
+ "T82x_L2_SNOOP_FULL",
+ "T82x_L2_REPLAY_FULL"
+};
+
+static const char * const hardware_counters_mali_t83x[] = {
+ /* Job Manager */
+ "",
+ "",
+ "",
+ "",
+ "T83x_MESSAGES_SENT",
+ "T83x_MESSAGES_RECEIVED",
+ "T83x_GPU_ACTIVE",
+ "T83x_IRQ_ACTIVE",
+ "T83x_JS0_JOBS",
+ "T83x_JS0_TASKS",
+ "T83x_JS0_ACTIVE",
+ "",
+ "T83x_JS0_WAIT_READ",
+ "T83x_JS0_WAIT_ISSUE",
+ "T83x_JS0_WAIT_DEPEND",
+ "T83x_JS0_WAIT_FINISH",
+ "T83x_JS1_JOBS",
+ "T83x_JS1_TASKS",
+ "T83x_JS1_ACTIVE",
+ "",
+ "T83x_JS1_WAIT_READ",
+ "T83x_JS1_WAIT_ISSUE",
+ "T83x_JS1_WAIT_DEPEND",
+ "T83x_JS1_WAIT_FINISH",
+ "T83x_JS2_JOBS",
+ "T83x_JS2_TASKS",
+ "T83x_JS2_ACTIVE",
+ "",
+ "T83x_JS2_WAIT_READ",
+ "T83x_JS2_WAIT_ISSUE",
+ "T83x_JS2_WAIT_DEPEND",
+ "T83x_JS2_WAIT_FINISH",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+
+ /*Tiler */
+ "",
+ "",
+ "",
+ "T83x_TI_JOBS_PROCESSED",
+ "T83x_TI_TRIANGLES",
+ "T83x_TI_QUADS",
+ "T83x_TI_POLYGONS",
+ "T83x_TI_POINTS",
+ "T83x_TI_LINES",
+ "T83x_TI_FRONT_FACING",
+ "T83x_TI_BACK_FACING",
+ "T83x_TI_PRIM_VISIBLE",
+ "T83x_TI_PRIM_CULLED",
+ "T83x_TI_PRIM_CLIPPED",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "T83x_TI_ACTIVE",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+
+ /* Shader Core */
+ "",
+ "",
+ "",
+ "",
+ "T83x_FRAG_ACTIVE",
+ "T83x_FRAG_PRIMITIVES",
+ "T83x_FRAG_PRIMITIVES_DROPPED",
+ "T83x_FRAG_CYCLES_DESC",
+ "T83x_FRAG_CYCLES_FPKQ_ACTIVE",
+ "T83x_FRAG_CYCLES_VERT",
+ "T83x_FRAG_CYCLES_TRISETUP",
+ "T83x_FRAG_CYCLES_EZS_ACTIVE",
+ "T83x_FRAG_THREADS",
+ "T83x_FRAG_DUMMY_THREADS",
+ "T83x_FRAG_QUADS_RAST",
+ "T83x_FRAG_QUADS_EZS_TEST",
+ "T83x_FRAG_QUADS_EZS_KILLED",
+ "T83x_FRAG_THREADS_LZS_TEST",
+ "T83x_FRAG_THREADS_LZS_KILLED",
+ "T83x_FRAG_CYCLES_NO_TILE",
+ "T83x_FRAG_NUM_TILES",
+ "T83x_FRAG_TRANS_ELIM",
+ "T83x_COMPUTE_ACTIVE",
+ "T83x_COMPUTE_TASKS",
+ "T83x_COMPUTE_THREADS",
+ "T83x_COMPUTE_CYCLES_DESC",
+ "T83x_TRIPIPE_ACTIVE",
+ "T83x_ARITH_WORDS",
+ "T83x_ARITH_CYCLES_REG",
+ "T83x_ARITH_CYCLES_L0",
+ "T83x_ARITH_FRAG_DEPEND",
+ "T83x_LS_WORDS",
+ "T83x_LS_ISSUES",
+ "T83x_LS_REISSUE_ATTR",
+ "T83x_LS_REISSUES_VARY",
+ "T83x_LS_VARY_RV_MISS",
+ "T83x_LS_VARY_RV_HIT",
+ "T83x_LS_NO_UNPARK",
+ "T83x_TEX_WORDS",
+ "T83x_TEX_BUBBLES",
+ "T83x_TEX_WORDS_L0",
+ "T83x_TEX_WORDS_DESC",
+ "T83x_TEX_ISSUES",
+ "T83x_TEX_RECIRC_FMISS",
+ "T83x_TEX_RECIRC_DESC",
+ "T83x_TEX_RECIRC_MULTI",
+ "T83x_TEX_RECIRC_PMISS",
+ "T83x_TEX_RECIRC_CONF",
+ "T83x_LSC_READ_HITS",
+ "T83x_LSC_READ_OP",
+ "T83x_LSC_WRITE_HITS",
+ "T83x_LSC_WRITE_OP",
+ "T83x_LSC_ATOMIC_HITS",
+ "T83x_LSC_ATOMIC_OP",
+ "T83x_LSC_LINE_FETCHES",
+ "T83x_LSC_DIRTY_LINE",
+ "T83x_LSC_SNOOPS",
+ "T83x_AXI_TLB_STALL",
+ "T83x_AXI_TLB_MIESS",
+ "T83x_AXI_TLB_TRANSACTION",
+ "T83x_LS_TLB_MISS",
+ "T83x_LS_TLB_HIT",
+ "T83x_AXI_BEATS_READ",
+ "T83x_AXI_BEATS_WRITTEN",
+
+ /*L2 and MMU */
+ "",
+ "",
+ "",
+ "",
+ "T83x_MMU_HIT",
+ "T83x_MMU_NEW_MISS",
+ "T83x_MMU_REPLAY_FULL",
+ "T83x_MMU_REPLAY_MISS",
+ "T83x_MMU_TABLE_WALK",
+ "T83x_MMU_REQUESTS",
+ "",
+ "",
+ "T83x_UTLB_HIT",
+ "T83x_UTLB_NEW_MISS",
+ "T83x_UTLB_REPLAY_FULL",
+ "T83x_UTLB_REPLAY_MISS",
+ "T83x_UTLB_STALL",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "T83x_L2_EXT_WRITE_BEATS",
+ "T83x_L2_EXT_READ_BEATS",
+ "T83x_L2_ANY_LOOKUP",
+ "T83x_L2_READ_LOOKUP",
+ "T83x_L2_SREAD_LOOKUP",
+ "T83x_L2_READ_REPLAY",
+ "T83x_L2_READ_SNOOP",
+ "T83x_L2_READ_HIT",
+ "T83x_L2_CLEAN_MISS",
+ "T83x_L2_WRITE_LOOKUP",
+ "T83x_L2_SWRITE_LOOKUP",
+ "T83x_L2_WRITE_REPLAY",
+ "T83x_L2_WRITE_SNOOP",
+ "T83x_L2_WRITE_HIT",
+ "T83x_L2_EXT_READ_FULL",
+ "",
+ "T83x_L2_EXT_WRITE_FULL",
+ "T83x_L2_EXT_R_W_HAZARD",
+ "T83x_L2_EXT_READ",
+ "T83x_L2_EXT_READ_LINE",
+ "T83x_L2_EXT_WRITE",
+ "T83x_L2_EXT_WRITE_LINE",
+ "T83x_L2_EXT_WRITE_SMALL",
+ "T83x_L2_EXT_BARRIER",
+ "T83x_L2_EXT_AR_STALL",
+ "T83x_L2_EXT_R_BUF_FULL",
+ "T83x_L2_EXT_RD_BUF_FULL",
+ "T83x_L2_EXT_R_RAW",
+ "T83x_L2_EXT_W_STALL",
+ "T83x_L2_EXT_W_BUF_FULL",
+ "T83x_L2_EXT_R_BUF_FULL",
+ "T83x_L2_TAG_HAZARD",
+ "T83x_L2_SNOOP_FULL",
+ "T83x_L2_REPLAY_FULL"
+};
+
+static const char * const hardware_counters_mali_t86x[] = {
+ /* Job Manager */
+ "",
+ "",
+ "",
+ "",
+ "T86x_MESSAGES_SENT",
+ "T86x_MESSAGES_RECEIVED",
+ "T86x_GPU_ACTIVE",
+ "T86x_IRQ_ACTIVE",
+ "T86x_JS0_JOBS",
+ "T86x_JS0_TASKS",
+ "T86x_JS0_ACTIVE",
+ "",
+ "T86x_JS0_WAIT_READ",
+ "T86x_JS0_WAIT_ISSUE",
+ "T86x_JS0_WAIT_DEPEND",
+ "T86x_JS0_WAIT_FINISH",
+ "T86x_JS1_JOBS",
+ "T86x_JS1_TASKS",
+ "T86x_JS1_ACTIVE",
+ "",
+ "T86x_JS1_WAIT_READ",
+ "T86x_JS1_WAIT_ISSUE",
+ "T86x_JS1_WAIT_DEPEND",
+ "T86x_JS1_WAIT_FINISH",
+ "T86x_JS2_JOBS",
+ "T86x_JS2_TASKS",
+ "T86x_JS2_ACTIVE",
+ "",
+ "T86x_JS2_WAIT_READ",
+ "T86x_JS2_WAIT_ISSUE",
+ "T86x_JS2_WAIT_DEPEND",
+ "T86x_JS2_WAIT_FINISH",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+
+ /*Tiler */
+ "",
+ "",
+ "",
+ "T86x_TI_JOBS_PROCESSED",
+ "T86x_TI_TRIANGLES",
+ "T86x_TI_QUADS",
+ "T86x_TI_POLYGONS",
+ "T86x_TI_POINTS",
+ "T86x_TI_LINES",
+ "T86x_TI_VCACHE_HIT",
+ "T86x_TI_VCACHE_MISS",
+ "T86x_TI_FRONT_FACING",
+ "T86x_TI_BACK_FACING",
+ "T86x_TI_PRIM_VISIBLE",
+ "T86x_TI_PRIM_CULLED",
+ "T86x_TI_PRIM_CLIPPED",
+ "T86x_TI_LEVEL0",
+ "T86x_TI_LEVEL1",
+ "T86x_TI_LEVEL2",
+ "T86x_TI_LEVEL3",
+ "T86x_TI_LEVEL4",
+ "T86x_TI_LEVEL5",
+ "T86x_TI_LEVEL6",
+ "T86x_TI_LEVEL7",
+ "T86x_TI_COMMAND_1",
+ "T86x_TI_COMMAND_2",
+ "T86x_TI_COMMAND_3",
+ "T86x_TI_COMMAND_4",
+ "T86x_TI_COMMAND_5_7",
+ "T86x_TI_COMMAND_8_15",
+ "T86x_TI_COMMAND_16_63",
+ "T86x_TI_COMMAND_64",
+ "T86x_TI_COMPRESS_IN",
+ "T86x_TI_COMPRESS_OUT",
+ "T86x_TI_COMPRESS_FLUSH",
+ "T86x_TI_TIMESTAMPS",
+ "T86x_TI_PCACHE_HIT",
+ "T86x_TI_PCACHE_MISS",
+ "T86x_TI_PCACHE_LINE",
+ "T86x_TI_PCACHE_STALL",
+ "T86x_TI_WRBUF_HIT",
+ "T86x_TI_WRBUF_MISS",
+ "T86x_TI_WRBUF_LINE",
+ "T86x_TI_WRBUF_PARTIAL",
+ "T86x_TI_WRBUF_STALL",
+ "T86x_TI_ACTIVE",
+ "T86x_TI_LOADING_DESC",
+ "T86x_TI_INDEX_WAIT",
+ "T86x_TI_INDEX_RANGE_WAIT",
+ "T86x_TI_VERTEX_WAIT",
+ "T86x_TI_PCACHE_WAIT",
+ "T86x_TI_WRBUF_WAIT",
+ "T86x_TI_BUS_READ",
+ "T86x_TI_BUS_WRITE",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "T86x_TI_UTLB_HIT",
+ "T86x_TI_UTLB_NEW_MISS",
+ "T86x_TI_UTLB_REPLAY_FULL",
+ "T86x_TI_UTLB_REPLAY_MISS",
+ "T86x_TI_UTLB_STALL",
+
+ /* Shader Core */
+ "",
+ "",
+ "",
+ "",
+ "T86x_FRAG_ACTIVE",
+ "T86x_FRAG_PRIMITIVES",
+ "T86x_FRAG_PRIMITIVES_DROPPED",
+ "T86x_FRAG_CYCLES_DESC",
+ "T86x_FRAG_CYCLES_FPKQ_ACTIVE",
+ "T86x_FRAG_CYCLES_VERT",
+ "T86x_FRAG_CYCLES_TRISETUP",
+ "T86x_FRAG_CYCLES_EZS_ACTIVE",
+ "T86x_FRAG_THREADS",
+ "T86x_FRAG_DUMMY_THREADS",
+ "T86x_FRAG_QUADS_RAST",
+ "T86x_FRAG_QUADS_EZS_TEST",
+ "T86x_FRAG_QUADS_EZS_KILLED",
+ "T86x_FRAG_THREADS_LZS_TEST",
+ "T86x_FRAG_THREADS_LZS_KILLED",
+ "T86x_FRAG_CYCLES_NO_TILE",
+ "T86x_FRAG_NUM_TILES",
+ "T86x_FRAG_TRANS_ELIM",
+ "T86x_COMPUTE_ACTIVE",
+ "T86x_COMPUTE_TASKS",
+ "T86x_COMPUTE_THREADS",
+ "T86x_COMPUTE_CYCLES_DESC",
+ "T86x_TRIPIPE_ACTIVE",
+ "T86x_ARITH_WORDS",
+ "T86x_ARITH_CYCLES_REG",
+ "T86x_ARITH_CYCLES_L0",
+ "T86x_ARITH_FRAG_DEPEND",
+ "T86x_LS_WORDS",
+ "T86x_LS_ISSUES",
+ "T86x_LS_REISSUE_ATTR",
+ "T86x_LS_REISSUES_VARY",
+ "T86x_LS_VARY_RV_MISS",
+ "T86x_LS_VARY_RV_HIT",
+ "T86x_LS_NO_UNPARK",
+ "T86x_TEX_WORDS",
+ "T86x_TEX_BUBBLES",
+ "T86x_TEX_WORDS_L0",
+ "T86x_TEX_WORDS_DESC",
+ "T86x_TEX_ISSUES",
+ "T86x_TEX_RECIRC_FMISS",
+ "T86x_TEX_RECIRC_DESC",
+ "T86x_TEX_RECIRC_MULTI",
+ "T86x_TEX_RECIRC_PMISS",
+ "T86x_TEX_RECIRC_CONF",
+ "T86x_LSC_READ_HITS",
+ "T86x_LSC_READ_OP",
+ "T86x_LSC_WRITE_HITS",
+ "T86x_LSC_WRITE_OP",
+ "T86x_LSC_ATOMIC_HITS",
+ "T86x_LSC_ATOMIC_OP",
+ "T86x_LSC_LINE_FETCHES",
+ "T86x_LSC_DIRTY_LINE",
+ "T86x_LSC_SNOOPS",
+ "T86x_AXI_TLB_STALL",
+ "T86x_AXI_TLB_MIESS",
+ "T86x_AXI_TLB_TRANSACTION",
+ "T86x_LS_TLB_MISS",
+ "T86x_LS_TLB_HIT",
+ "T86x_AXI_BEATS_READ",
+ "T86x_AXI_BEATS_WRITTEN",
+
+ /*L2 and MMU */
+ "",
+ "",
+ "",
+ "",
+ "T86x_MMU_HIT",
+ "T86x_MMU_NEW_MISS",
+ "T86x_MMU_REPLAY_FULL",
+ "T86x_MMU_REPLAY_MISS",
+ "T86x_MMU_TABLE_WALK",
+ "T86x_MMU_REQUESTS",
+ "",
+ "",
+ "T86x_UTLB_HIT",
+ "T86x_UTLB_NEW_MISS",
+ "T86x_UTLB_REPLAY_FULL",
+ "T86x_UTLB_REPLAY_MISS",
+ "T86x_UTLB_STALL",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "T86x_L2_EXT_WRITE_BEATS",
+ "T86x_L2_EXT_READ_BEATS",
+ "T86x_L2_ANY_LOOKUP",
+ "T86x_L2_READ_LOOKUP",
+ "T86x_L2_SREAD_LOOKUP",
+ "T86x_L2_READ_REPLAY",
+ "T86x_L2_READ_SNOOP",
+ "T86x_L2_READ_HIT",
+ "T86x_L2_CLEAN_MISS",
+ "T86x_L2_WRITE_LOOKUP",
+ "T86x_L2_SWRITE_LOOKUP",
+ "T86x_L2_WRITE_REPLAY",
+ "T86x_L2_WRITE_SNOOP",
+ "T86x_L2_WRITE_HIT",
+ "T86x_L2_EXT_READ_FULL",
+ "",
+ "T86x_L2_EXT_WRITE_FULL",
+ "T86x_L2_EXT_R_W_HAZARD",
+ "T86x_L2_EXT_READ",
+ "T86x_L2_EXT_READ_LINE",
+ "T86x_L2_EXT_WRITE",
+ "T86x_L2_EXT_WRITE_LINE",
+ "T86x_L2_EXT_WRITE_SMALL",
+ "T86x_L2_EXT_BARRIER",
+ "T86x_L2_EXT_AR_STALL",
+ "T86x_L2_EXT_R_BUF_FULL",
+ "T86x_L2_EXT_RD_BUF_FULL",
+ "T86x_L2_EXT_R_RAW",
+ "T86x_L2_EXT_W_STALL",
+ "T86x_L2_EXT_W_BUF_FULL",
+ "T86x_L2_EXT_R_BUF_FULL",
+ "T86x_L2_TAG_HAZARD",
+ "T86x_L2_SNOOP_FULL",
+ "T86x_L2_REPLAY_FULL"
+};
+
+static const char * const hardware_counters_mali_t88x[] = {
+ /* Job Manager */
+ "",
+ "",
+ "",
+ "",
+ "T88x_MESSAGES_SENT",
+ "T88x_MESSAGES_RECEIVED",
+ "T88x_GPU_ACTIVE",
+ "T88x_IRQ_ACTIVE",
+ "T88x_JS0_JOBS",
+ "T88x_JS0_TASKS",
+ "T88x_JS0_ACTIVE",
+ "",
+ "T88x_JS0_WAIT_READ",
+ "T88x_JS0_WAIT_ISSUE",
+ "T88x_JS0_WAIT_DEPEND",
+ "T88x_JS0_WAIT_FINISH",
+ "T88x_JS1_JOBS",
+ "T88x_JS1_TASKS",
+ "T88x_JS1_ACTIVE",
+ "",
+ "T88x_JS1_WAIT_READ",
+ "T88x_JS1_WAIT_ISSUE",
+ "T88x_JS1_WAIT_DEPEND",
+ "T88x_JS1_WAIT_FINISH",
+ "T88x_JS2_JOBS",
+ "T88x_JS2_TASKS",
+ "T88x_JS2_ACTIVE",
+ "",
+ "T88x_JS2_WAIT_READ",
+ "T88x_JS2_WAIT_ISSUE",
+ "T88x_JS2_WAIT_DEPEND",
+ "T88x_JS2_WAIT_FINISH",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+
+ /*Tiler */
+ "",
+ "",
+ "",
+ "T88x_TI_JOBS_PROCESSED",
+ "T88x_TI_TRIANGLES",
+ "T88x_TI_QUADS",
+ "T88x_TI_POLYGONS",
+ "T88x_TI_POINTS",
+ "T88x_TI_LINES",
+ "T88x_TI_VCACHE_HIT",
+ "T88x_TI_VCACHE_MISS",
+ "T88x_TI_FRONT_FACING",
+ "T88x_TI_BACK_FACING",
+ "T88x_TI_PRIM_VISIBLE",
+ "T88x_TI_PRIM_CULLED",
+ "T88x_TI_PRIM_CLIPPED",
+ "T88x_TI_LEVEL0",
+ "T88x_TI_LEVEL1",
+ "T88x_TI_LEVEL2",
+ "T88x_TI_LEVEL3",
+ "T88x_TI_LEVEL4",
+ "T88x_TI_LEVEL5",
+ "T88x_TI_LEVEL6",
+ "T88x_TI_LEVEL7",
+ "T88x_TI_COMMAND_1",
+ "T88x_TI_COMMAND_2",
+ "T88x_TI_COMMAND_3",
+ "T88x_TI_COMMAND_4",
+ "T88x_TI_COMMAND_5_7",
+ "T88x_TI_COMMAND_8_15",
+ "T88x_TI_COMMAND_16_63",
+ "T88x_TI_COMMAND_64",
+ "T88x_TI_COMPRESS_IN",
+ "T88x_TI_COMPRESS_OUT",
+ "T88x_TI_COMPRESS_FLUSH",
+ "T88x_TI_TIMESTAMPS",
+ "T88x_TI_PCACHE_HIT",
+ "T88x_TI_PCACHE_MISS",
+ "T88x_TI_PCACHE_LINE",
+ "T88x_TI_PCACHE_STALL",
+ "T88x_TI_WRBUF_HIT",
+ "T88x_TI_WRBUF_MISS",
+ "T88x_TI_WRBUF_LINE",
+ "T88x_TI_WRBUF_PARTIAL",
+ "T88x_TI_WRBUF_STALL",
+ "T88x_TI_ACTIVE",
+ "T88x_TI_LOADING_DESC",
+ "T88x_TI_INDEX_WAIT",
+ "T88x_TI_INDEX_RANGE_WAIT",
+ "T88x_TI_VERTEX_WAIT",
+ "T88x_TI_PCACHE_WAIT",
+ "T88x_TI_WRBUF_WAIT",
+ "T88x_TI_BUS_READ",
+ "T88x_TI_BUS_WRITE",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "T88x_TI_UTLB_HIT",
+ "T88x_TI_UTLB_NEW_MISS",
+ "T88x_TI_UTLB_REPLAY_FULL",
+ "T88x_TI_UTLB_REPLAY_MISS",
+ "T88x_TI_UTLB_STALL",
+
+ /* Shader Core */
+ "",
+ "",
+ "",
+ "",
+ "T88x_FRAG_ACTIVE",
+ "T88x_FRAG_PRIMITIVES",
+ "T88x_FRAG_PRIMITIVES_DROPPED",
+ "T88x_FRAG_CYCLES_DESC",
+ "T88x_FRAG_CYCLES_FPKQ_ACTIVE",
+ "T88x_FRAG_CYCLES_VERT",
+ "T88x_FRAG_CYCLES_TRISETUP",
+ "T88x_FRAG_CYCLES_EZS_ACTIVE",
+ "T88x_FRAG_THREADS",
+ "T88x_FRAG_DUMMY_THREADS",
+ "T88x_FRAG_QUADS_RAST",
+ "T88x_FRAG_QUADS_EZS_TEST",
+ "T88x_FRAG_QUADS_EZS_KILLED",
+ "T88x_FRAG_THREADS_LZS_TEST",
+ "T88x_FRAG_THREADS_LZS_KILLED",
+ "T88x_FRAG_CYCLES_NO_TILE",
+ "T88x_FRAG_NUM_TILES",
+ "T88x_FRAG_TRANS_ELIM",
+ "T88x_COMPUTE_ACTIVE",
+ "T88x_COMPUTE_TASKS",
+ "T88x_COMPUTE_THREADS",
+ "T88x_COMPUTE_CYCLES_DESC",
+ "T88x_TRIPIPE_ACTIVE",
+ "T88x_ARITH_WORDS",
+ "T88x_ARITH_CYCLES_REG",
+ "T88x_ARITH_CYCLES_L0",
+ "T88x_ARITH_FRAG_DEPEND",
+ "T88x_LS_WORDS",
+ "T88x_LS_ISSUES",
+ "T88x_LS_REISSUE_ATTR",
+ "T88x_LS_REISSUES_VARY",
+ "T88x_LS_VARY_RV_MISS",
+ "T88x_LS_VARY_RV_HIT",
+ "T88x_LS_NO_UNPARK",
+ "T88x_TEX_WORDS",
+ "T88x_TEX_BUBBLES",
+ "T88x_TEX_WORDS_L0",
+ "T88x_TEX_WORDS_DESC",
+ "T88x_TEX_ISSUES",
+ "T88x_TEX_RECIRC_FMISS",
+ "T88x_TEX_RECIRC_DESC",
+ "T88x_TEX_RECIRC_MULTI",
+ "T88x_TEX_RECIRC_PMISS",
+ "T88x_TEX_RECIRC_CONF",
+ "T88x_LSC_READ_HITS",
+ "T88x_LSC_READ_OP",
+ "T88x_LSC_WRITE_HITS",
+ "T88x_LSC_WRITE_OP",
+ "T88x_LSC_ATOMIC_HITS",
+ "T88x_LSC_ATOMIC_OP",
+ "T88x_LSC_LINE_FETCHES",
+ "T88x_LSC_DIRTY_LINE",
+ "T88x_LSC_SNOOPS",
+ "T88x_AXI_TLB_STALL",
+ "T88x_AXI_TLB_MIESS",
+ "T88x_AXI_TLB_TRANSACTION",
+ "T88x_LS_TLB_MISS",
+ "T88x_LS_TLB_HIT",
+ "T88x_AXI_BEATS_READ",
+ "T88x_AXI_BEATS_WRITTEN",
+
+ /*L2 and MMU */
+ "",
+ "",
+ "",
+ "",
+ "T88x_MMU_HIT",
+ "T88x_MMU_NEW_MISS",
+ "T88x_MMU_REPLAY_FULL",
+ "T88x_MMU_REPLAY_MISS",
+ "T88x_MMU_TABLE_WALK",
+ "T88x_MMU_REQUESTS",
+ "",
+ "",
+ "T88x_UTLB_HIT",
+ "T88x_UTLB_NEW_MISS",
+ "T88x_UTLB_REPLAY_FULL",
+ "T88x_UTLB_REPLAY_MISS",
+ "T88x_UTLB_STALL",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "T88x_L2_EXT_WRITE_BEATS",
+ "T88x_L2_EXT_READ_BEATS",
+ "T88x_L2_ANY_LOOKUP",
+ "T88x_L2_READ_LOOKUP",
+ "T88x_L2_SREAD_LOOKUP",
+ "T88x_L2_READ_REPLAY",
+ "T88x_L2_READ_SNOOP",
+ "T88x_L2_READ_HIT",
+ "T88x_L2_CLEAN_MISS",
+ "T88x_L2_WRITE_LOOKUP",
+ "T88x_L2_SWRITE_LOOKUP",
+ "T88x_L2_WRITE_REPLAY",
+ "T88x_L2_WRITE_SNOOP",
+ "T88x_L2_WRITE_HIT",
+ "T88x_L2_EXT_READ_FULL",
+ "",
+ "T88x_L2_EXT_WRITE_FULL",
+ "T88x_L2_EXT_R_W_HAZARD",
+ "T88x_L2_EXT_READ",
+ "T88x_L2_EXT_READ_LINE",
+ "T88x_L2_EXT_WRITE",
+ "T88x_L2_EXT_WRITE_LINE",
+ "T88x_L2_EXT_WRITE_SMALL",
+ "T88x_L2_EXT_BARRIER",
+ "T88x_L2_EXT_AR_STALL",
+ "T88x_L2_EXT_R_BUF_FULL",
+ "T88x_L2_EXT_RD_BUF_FULL",
+ "T88x_L2_EXT_R_RAW",
+ "T88x_L2_EXT_W_STALL",
+ "T88x_L2_EXT_W_BUF_FULL",
+ "T88x_L2_EXT_R_BUF_FULL",
+ "T88x_L2_TAG_HAZARD",
+ "T88x_L2_SNOOP_FULL",
+ "T88x_L2_REPLAY_FULL"
+};
+
#endif
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
ssize_t ret = 0;
struct list_head *entry;
const struct list_head *kbdev_list;
+
kbdev_list = kbase_dev_list_get();
list_for_each(entry, kbdev_list) {
struct kbase_device *kbdev = NULL;
kbdev = list_entry(entry, struct kbase_device, entry);
/* output the total memory usage and cap for this device */
- ret = seq_printf(sfile, "%-16s %10u\n", \
- kbdev->devname, \
+ ret = seq_printf(sfile, "%-16s %10u\n",
+ kbdev->devname,
atomic_read(&(kbdev->memdev.used_pages)));
mutex_lock(&kbdev->kctx_list_lock);
list_for_each_entry(element, &kbdev->kctx_list, link) {
/* output the memory usage and cap for each kctx
* opened on this device */
- ret = seq_printf(sfile, " %s-0x%p %10u %10u %10u %10u\n", \
- "kctx", \
- element->kctx, \
- element->kctx->pid, \
- atomic_read(&(element->kctx->osalloc.free_list_size)), \
- atomic_read(&(element->kctx->used_pages)), \
- atomic_read(&(element->kctx->nonmapped_pages)));
+ ret = seq_printf(sfile, " %s-0x%p %10u\n",
+ "kctx",
+ element->kctx,
+ atomic_read(&(element->kctx->used_pages)));
}
mutex_unlock(&kbdev->kctx_list_lock);
}
/*
* File operations related to debugfs entry for gpu_memory
*/
-STATIC int kbasep_gpu_memory_debugfs_open(struct inode *in, struct file *file)
+static int kbasep_gpu_memory_debugfs_open(struct inode *in, struct file *file)
{
return single_open(file, kbasep_gpu_memory_seq_show , NULL);
}
.open = kbasep_gpu_memory_debugfs_open,
.read = seq_read,
.llseek = seq_lseek,
- .release = seq_release_private,
+ .release = single_release,
};
/*
* Initialize debugfs entry for gpu_memory
*/
-mali_error kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev)
+void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev)
{
- kbdev->gpu_memory_dentry = debugfs_create_file("gpu_memory", \
- S_IRUGO, \
- kbdev->mali_debugfs_directory, \
- NULL, \
- &kbasep_gpu_memory_debugfs_fops);
- if (IS_ERR(kbdev->gpu_memory_dentry))
- return MALI_ERROR_FUNCTION_FAILED;
-
- return MALI_ERROR_NONE;
+ debugfs_create_file("gpu_memory", S_IRUGO,
+ kbdev->mali_debugfs_directory, NULL,
+ &kbasep_gpu_memory_debugfs_fops);
+ return;
}
-/*
- * Terminate debugfs entry for gpu_memory
- */
-void kbasep_gpu_memory_debugfs_term(struct kbase_device *kbdev)
-{
- debugfs_remove(kbdev->gpu_memory_dentry);
-}
#else
/*
* Stub functions for when debugfs is disabled
*/
-mali_error kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev)
-{
- return MALI_ERROR_NONE;
-}
-void kbasep_gpu_memory_debugfs_term(struct kbase_device *kbdev)
+void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev)
{
+ return;
}
#endif
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/**
* @brief Initialize gpu_memory debugfs entry
*/
-mali_error kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev);
-
-/**
- * @brief Terminate gpu_memory debugfs entry
- */
-void kbasep_gpu_memory_debugfs_term(struct kbase_device *kbdev);
+void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev);
#endif /*_KBASE_GPU_MEMORY_H*/
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file mali_kbase_gpuprops.c
+/*
* Base kernel property query APIs
*/
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
#include <mali_kbase_gpuprops.h>
+#include <mali_kbase_config_defaults.h>
+#include <mali_kbase_hwaccess_gpuprops.h>
+#include <linux/clk.h>
/**
- * @brief Extracts bits from a 32-bit bitfield.
- * @hideinitializer
+ * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield.
+ * @value: The value from which to extract bits.
+ * @offset: The first bit to extract (0 being the LSB).
+ * @size: The number of bits to extract.
*
- * @param[in] value The value from which to extract bits.
- * @param[in] offset The first bit to extract (0 being the LSB).
- * @param[in] size The number of bits to extract.
- * @return Bits [@a offset, @a offset + @a size) from @a value.
+ * Context: @offset + @size <= 32.
*
- * @pre offset + size <= 32.
+ * Return: Bits [@offset, @offset + @size) from @value.
*/
/* from mali_cdsb.h */
#define KBASE_UBFX32(value, offset, size) \
(((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1))
-mali_error kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props)
+int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props)
{
- kbase_gpuprops_clock_speed_function get_gpu_speed_mhz;
+ kbase_gpu_clk_speed_func get_gpu_speed_mhz;
u32 gpu_speed_mhz;
int rc = 1;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(NULL != kbase_props);
- /* Current GPU speed is requested from the system integrator via the KBASE_CONFIG_ATTR_GPU_SPEED_FUNC function.
+ /* Current GPU speed is requested from the system integrator via the GPU_SPEED_FUNC function.
* If that function fails, or the function is not provided by the system integrator, we report the maximum
* GPU speed as specified by GPU_FREQ_KHZ_MAX.
*/
- get_gpu_speed_mhz = (kbase_gpuprops_clock_speed_function) kbasep_get_config_value(kctx->kbdev, kctx->kbdev->config_attributes, KBASE_CONFIG_ATTR_GPU_SPEED_FUNC);
+ get_gpu_speed_mhz = (kbase_gpu_clk_speed_func) GPU_SPEED_FUNC;
if (get_gpu_speed_mhz != NULL) {
rc = get_gpu_speed_mhz(&gpu_speed_mhz);
#ifdef CONFIG_MALI_DEBUG
/* Issue a warning message when the reported GPU speed falls outside the min/max range */
if (rc == 0) {
u32 gpu_speed_khz = gpu_speed_mhz * 1000;
- if (gpu_speed_khz < kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min || gpu_speed_khz > kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max)
- dev_warn(kctx->kbdev->dev, "GPU Speed is outside of min/max range (got %lu Khz, min %lu Khz, max %lu Khz)\n", (unsigned long)gpu_speed_khz, (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min, (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max);
+
+ if (gpu_speed_khz < kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min ||
+ gpu_speed_khz > kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max)
+ dev_warn(kctx->kbdev->dev, "GPU Speed is outside of min/max range (got %lu Khz, min %lu Khz, max %lu Khz)\n",
+ (unsigned long)gpu_speed_khz,
+ (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min,
+ (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max);
}
#endif /* CONFIG_MALI_DEBUG */
}
+ if (kctx->kbdev->clock) {
+ gpu_speed_mhz = clk_get_rate(kctx->kbdev->clock) / 1000000;
+ rc = 0;
+ }
if (rc != 0)
gpu_speed_mhz = kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max / 1000;
memcpy(&kbase_props->props, &kctx->kbdev->gpu_props.props, sizeof(kbase_props->props));
- return MALI_ERROR_NONE;
-}
+ /* Before API 8.2 they expect L3 cache info here, which was always 0 */
+ if (kctx->api_version < KBASE_API_VERSION(8, 2))
+ kbase_props->props.raw_props.suspend_size = 0;
-STATIC void kbase_gpuprops_dump_registers(struct kbase_device *kbdev, struct kbase_gpuprops_regdump *regdump)
-{
- int i;
- KBASE_DEBUG_ASSERT(NULL != kbdev);
- KBASE_DEBUG_ASSERT(NULL != regdump);
-
- /* Fill regdump with the content of the relevant registers */
- regdump->gpu_id = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
-
- regdump->l2_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L2_FEATURES));
- regdump->l3_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L3_FEATURES));
- regdump->tiler_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TILER_FEATURES));
- regdump->mem_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(MEM_FEATURES));
- regdump->mmu_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(MMU_FEATURES));
- regdump->as_present = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(AS_PRESENT));
- regdump->js_present = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(JS_PRESENT));
-
- for (i = 0; i < MIDG_MAX_JOB_SLOTS; i++)
- regdump->js_features[i] = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(JS_FEATURES_REG(i)));
-
- for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++)
- regdump->texture_features[i] = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)));
-
- regdump->thread_max_threads = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(THREAD_MAX_THREADS));
- regdump->thread_max_workgroup_size = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE));
- regdump->thread_max_barrier_size = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE));
- regdump->thread_features = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(THREAD_FEATURES));
-
- regdump->shader_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(SHADER_PRESENT_LO));
- regdump->shader_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(SHADER_PRESENT_HI));
-
- regdump->tiler_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TILER_PRESENT_LO));
- regdump->tiler_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(TILER_PRESENT_HI));
-
- regdump->l2_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L2_PRESENT_LO));
- regdump->l2_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L2_PRESENT_HI));
-
- regdump->l3_present_lo = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L3_PRESENT_LO));
- regdump->l3_present_hi = kbase_os_reg_read(kbdev, GPU_CONTROL_REG(L3_PRESENT_HI));
+ return 0;
}
-STATIC void kbase_gpuprops_construct_coherent_groups(base_gpu_props * const props)
+static void kbase_gpuprops_construct_coherent_groups(base_gpu_props * const props)
{
struct mali_base_gpu_coherent_group *current_group;
u64 group_present;
props->coherency_info.coherency = props->raw_props.mem_features;
props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present);
- if (props->coherency_info.coherency & GROUPS_L3_COHERENT) {
- /* Group is l3 coherent */
- group_present = props->raw_props.l3_present;
- } else if (props->coherency_info.coherency & GROUPS_L2_COHERENT) {
+ if (props->coherency_info.coherency & GROUPS_L2_COHERENT) {
/* Group is l2 coherent */
group_present = props->raw_props.l2_present;
} else {
}
/*
- * The coherent group mask can be computed from the l2/l3 present
+ * The coherent group mask can be computed from the l2 present
* register.
*
* For the coherent group n:
}
/**
- * @brief Get the GPU configuration
- *
- * Fill the base_gpu_props structure with values from the GPU configuration registers.
- * Only the raw properties are filled in this function
+ * kbase_gpuprops_get_props - Get the GPU configuration
+ * @gpu_props: The &base_gpu_props structure
+ * @kbdev: The &struct kbase_device structure for the device
*
- * @param gpu_props The base_gpu_props structure
- * @param kbdev The struct kbase_device structure for the device
+ * Fill the &base_gpu_props structure with values from the GPU configuration
+ * registers. Only the raw properties are filled in this function
*/
static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev)
{
KBASE_DEBUG_ASSERT(NULL != gpu_props);
/* Dump relevant registers */
- kbase_gpuprops_dump_registers(kbdev, ®dump);
+ kbase_backend_gpuprops_get(kbdev, ®dump);
+
gpu_props->raw_props.gpu_id = regdump.gpu_id;
gpu_props->raw_props.tiler_features = regdump.tiler_features;
gpu_props->raw_props.mem_features = regdump.mem_features;
gpu_props->raw_props.mmu_features = regdump.mmu_features;
gpu_props->raw_props.l2_features = regdump.l2_features;
- gpu_props->raw_props.l3_features = regdump.l3_features;
+ gpu_props->raw_props.suspend_size = regdump.suspend_size;
gpu_props->raw_props.as_present = regdump.as_present;
gpu_props->raw_props.js_present = regdump.js_present;
gpu_props->raw_props.shader_present = ((u64) regdump.shader_present_hi << 32) + regdump.shader_present_lo;
gpu_props->raw_props.tiler_present = ((u64) regdump.tiler_present_hi << 32) + regdump.tiler_present_lo;
gpu_props->raw_props.l2_present = ((u64) regdump.l2_present_hi << 32) + regdump.l2_present_lo;
- gpu_props->raw_props.l3_present = ((u64) regdump.l3_present_hi << 32) + regdump.l3_present_lo;
- for (i = 0; i < MIDG_MAX_JOB_SLOTS; i++)
+ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++)
gpu_props->raw_props.js_features[i] = regdump.js_features[i];
for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++)
gpu_props->raw_props.thread_max_threads = regdump.thread_max_threads;
gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size;
gpu_props->raw_props.thread_features = regdump.thread_features;
+
}
/**
- * @brief Calculate the derived properties
- *
- * Fill the base_gpu_props structure with values derived from the GPU configuration registers
+ * kbase_gpuprops_calculate_props - Calculate the derived properties
+ * @gpu_props: The &base_gpu_props structure
+ * @kbdev: The &struct kbase_device structure for the device
*
- * @param gpu_props The base_gpu_props structure
- * @param kbdev The struct kbase_device structure for the device
+ * Fill the &base_gpu_props structure with values derived from the GPU
+ * configuration registers
*/
static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev)
{
gpu_props->l2_props.log2_line_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 0U, 8);
gpu_props->l2_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8);
gpu_props->l2_props.num_l2_slices = 1;
- if (gpu_props->core_props.product_id == GPU_ID_PI_T76X) {
+ if (gpu_props->core_props.product_id == GPU_ID_PI_T76X)
gpu_props->l2_props.num_l2_slices = KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1;
- }
-
- gpu_props->l3_props.log2_line_size = KBASE_UBFX32(gpu_props->raw_props.l3_features, 0U, 8);
- gpu_props->l3_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l3_features, 16U, 8);
gpu_props->tiler_props.bin_size_bytes = 1 << KBASE_UBFX32(gpu_props->raw_props.tiler_features, 0U, 6);
gpu_props->tiler_props.max_active_levels = KBASE_UBFX32(gpu_props->raw_props.tiler_features, 8U, 4);
void kbase_gpuprops_set(struct kbase_device *kbdev)
{
- kbase_gpu_props *gpu_props;
- struct midg_raw_gpu_props *raw;
+ struct kbase_gpu_props *gpu_props;
+ struct gpu_raw_gpu_props *raw;
KBASE_DEBUG_ASSERT(NULL != kbdev);
gpu_props = &kbdev->gpu_props;
gpu_props->l2_props.associativity = KBASE_UBFX32(raw->l2_features, 8U, 8);
gpu_props->l2_props.external_bus_width = KBASE_UBFX32(raw->l2_features, 24U, 8);
- gpu_props->l3_props.associativity = KBASE_UBFX32(raw->l3_features, 8U, 8);
- gpu_props->l3_props.external_bus_width = KBASE_UBFX32(raw->l3_features, 24U, 8);
-
gpu_props->mem.core_group = KBASE_UBFX32(raw->mem_features, 0U, 1);
- gpu_props->mem.supergroup = KBASE_UBFX32(raw->mem_features, 1U, 1);
gpu_props->mmu.va_bits = KBASE_UBFX32(raw->mmu_features, 0U, 8);
gpu_props->mmu.pa_bits = KBASE_UBFX32(raw->mmu_features, 8U, 8);
gpu_props->num_cores = hweight64(raw->shader_present);
gpu_props->num_core_groups = hweight64(raw->l2_present);
- gpu_props->num_supergroups = hweight64(raw->l3_present);
gpu_props->num_address_spaces = hweight32(raw->as_present);
gpu_props->num_job_slots = hweight32(raw->js_present);
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* @param kctx The struct kbase_context structure
* @param kbase_props A copy of the struct kbase_uk_gpuprops structure from userspace
*
- * @return MALI_ERROR_NONE on success. Any other value indicates failure.
+ * @return 0 on success. Any other value indicates failure.
*/
-mali_error kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props);
+int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props);
#endif /* _KBASE_GPUPROPS_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define KBASE_GPU_SPEED_MHZ 123
#define KBASE_GPU_PC_SIZE_LOG2 24U
-typedef struct kbase_gpuprops_regdump {
+struct kbase_gpuprops_regdump {
u32 gpu_id;
u32 l2_features;
- u32 l3_features;
+ u32 suspend_size; /* API 8.2+ */
u32 tiler_features;
u32 mem_features;
u32 mmu_features;
u32 thread_max_barrier_size;
u32 thread_features;
u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS];
- u32 js_features[MIDG_MAX_JOB_SLOTS];
+ u32 js_features[GPU_MAX_JOB_SLOTS];
u32 shader_present_lo;
u32 shader_present_hi;
u32 tiler_present_lo;
u32 tiler_present_hi;
u32 l2_present_lo;
u32 l2_present_hi;
- u32 l3_present_lo;
- u32 l3_present_hi;
-} kbase_gpuprops_regdump;
+};
-typedef struct kbase_gpu_cache_props {
+struct kbase_gpu_cache_props {
u8 associativity;
u8 external_bus_width;
-} kbase_gpu_cache_props;
+};
-typedef struct kbase_gpu_mem_props {
+struct kbase_gpu_mem_props {
u8 core_group;
- u8 supergroup;
-} kbase_gpu_mem_props;
+};
-typedef struct kbase_gpu_mmu_props {
+struct kbase_gpu_mmu_props {
u8 va_bits;
u8 pa_bits;
-} kbase_gpu_mmu_props;
+};
-typedef struct mali_kbase_gpu_props {
+struct kbase_gpu_props {
/* kernel-only properties */
u8 num_cores;
u8 num_core_groups;
- u8 num_supergroups;
u8 num_address_spaces;
u8 num_job_slots;
struct kbase_gpu_cache_props l2_props;
- struct kbase_gpu_cache_props l3_props;
struct kbase_gpu_mem_props mem;
struct kbase_gpu_mmu_props mmu;
/* Properties shared with userspace */
base_gpu_props props;
-} kbase_gpu_props;
+};
#endif /* _KBASE_GPUPROPS_TYPES_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file
+/*
* Run-time work-arounds helpers
*/
-#include <mali_base_hwconfig.h>
+#include <mali_base_hwconfig_features.h>
+#include <mali_base_hwconfig_issues.h>
#include <mali_midg_regmap.h>
#include "mali_kbase.h"
#include "mali_kbase_hw.h"
gpu_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT;
switch (gpu_id) {
- case GPU_ID_PI_T76X:
- features = base_hw_features_t76x;
- break;
-#ifdef MALI_INCLUDE_TFRX
case GPU_ID_PI_TFRX:
- /* Fall through */
-#endif /* MALI_INCLUDE_TFRX */
+ /* FALLTHROUGH */
case GPU_ID_PI_T86X:
features = base_hw_features_tFxx;
break;
+ case GPU_ID_PI_T83X:
+ features = base_hw_features_t83x;
+ break;
+ case GPU_ID_PI_T82X:
+ features = base_hw_features_t82x;
+ break;
+ case GPU_ID_PI_T76X:
+ features = base_hw_features_t76x;
+ break;
case GPU_ID_PI_T72X:
features = base_hw_features_t72x;
break;
- case GPU_ID_MAKE(GPU_ID_PI_T62X, 0, 1, 0):
- case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 0):
- case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 1, 0):
+ case GPU_ID_PI_T62X:
features = base_hw_features_t62x;
break;
- case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_15DEV0):
- case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_EAC):
- case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 1, 0):
+ case GPU_ID_PI_T60X:
features = base_hw_features_t60x;
break;
default:
set_bit(*features, &kbdev->hw_features_mask[0]);
}
-mali_error kbase_hw_set_issues_mask(struct kbase_device *kbdev)
+int kbase_hw_set_issues_mask(struct kbase_device *kbdev)
{
const enum base_hw_issue *issues;
u32 gpu_id;
case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 1, 0):
issues = base_hw_issues_t62x_r1p1;
break;
- case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 0, 0):
- issues = base_hw_issues_t76x_r0p0_beta;
- break;
case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 0, 1):
issues = base_hw_issues_t76x_r0p0;
break;
case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 1, 0):
issues = base_hw_issues_t72x_r1p1;
break;
-#ifdef MALI_INCLUDE_TFRX
- case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 0, 0):
- case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 0, 1):
- issues = base_hw_issues_tFRx_r0p0;
- break;
case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 1, 2):
issues = base_hw_issues_tFRx_r0p1;
break;
case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 2, 0):
issues = base_hw_issues_tFRx_r0p2;
break;
-#endif /* MALI_INCLUDE_TFRX */
- case GPU_ID_MAKE(GPU_ID_PI_T86X, 0, 0, 1):
- issues = base_hw_issues_t86x_r0p0;
+ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 0):
+ issues = base_hw_issues_tFRx_r1p0;
+ break;
+ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 2, 0, 0):
+ issues = base_hw_issues_tFRx_r2p0;
break;
case GPU_ID_MAKE(GPU_ID_PI_T86X, 0, 2, 0):
issues = base_hw_issues_t86x_r0p2;
break;
+ case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 0):
+ issues = base_hw_issues_t86x_r1p0;
+ break;
+ case GPU_ID_MAKE(GPU_ID_PI_T86X, 2, 0, 0):
+ issues = base_hw_issues_t86x_r2p0;
+ break;
+ case GPU_ID_MAKE(GPU_ID_PI_T83X, 0, 1, 0):
+ issues = base_hw_issues_t83x_r0p1;
+ break;
+ case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 0):
+ issues = base_hw_issues_t83x_r1p0;
+ break;
+ case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 0, 0):
+ issues = base_hw_issues_t82x_r0p0;
+ break;
+ case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 1, 0):
+ issues = base_hw_issues_t82x_r0p1;
+ break;
+ case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 0):
+ issues = base_hw_issues_t82x_r1p0;
+ break;
default:
dev_err(kbdev->dev, "Unknown GPU ID %x", gpu_id);
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
} else {
/* Software model */
case GPU_ID_PI_T76X:
issues = base_hw_issues_model_t76x;
break;
-#ifdef MALI_INCLUDE_TFRX
case GPU_ID_PI_TFRX:
issues = base_hw_issues_model_tFRx;
break;
-#endif /* MALI_INCLUDE_TFRX */
case GPU_ID_PI_T86X:
issues = base_hw_issues_model_t86x;
break;
+ case GPU_ID_PI_T83X:
+ issues = base_hw_issues_model_t83x;
+ break;
+ case GPU_ID_PI_T82X:
+ issues = base_hw_issues_model_t82x;
+ break;
default:
dev_err(kbdev->dev, "Unknown GPU ID %x", gpu_id);
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
}
for (; *issues != BASE_HW_ISSUE_END; issues++)
set_bit(*issues, &kbdev->hw_issues_mask[0]);
- return MALI_ERROR_NONE;
+ return 0;
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/**
* @brief Set the HW issues mask depending on the GPU ID
*/
-mali_error kbase_hw_set_issues_mask(struct kbase_device *kbdev);
+int kbase_hw_set_issues_mask(struct kbase_device *kbdev);
/**
* @brief Set the features mask depending on the GPU ID
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/*
+ * HW access backend common APIs
+ */
+
+#ifndef _KBASE_HWACCESS_BACKEND_H_
+#define _KBASE_HWACCESS_BACKEND_H_
+
+/**
+ * kbase_backend_early_init - Perform any backend-specific initialization.
+ * @kbdev: Device pointer
+ *
+ * Return: 0 on success, or an error code on failure.
+ */
+int kbase_backend_early_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_backend_late_init - Perform any backend-specific initialization.
+ * @kbdev: Device pointer
+ *
+ * Return: 0 on success, or an error code on failure.
+ */
+int kbase_backend_late_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_backend_early_term - Perform any backend-specific termination.
+ * @kbdev: Device pointer
+ */
+void kbase_backend_early_term(struct kbase_device *kbdev);
+
+/**
+ * kbase_backend_late_term - Perform any backend-specific termination.
+ * @kbdev: Device pointer
+ */
+void kbase_backend_late_term(struct kbase_device *kbdev);
+
+#endif /* _KBASE_HWACCESS_BACKEND_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/**
+ * @file mali_kbase_hwaccess_gpu_defs.h
+ * HW access common definitions
+ */
+
+#ifndef _KBASE_HWACCESS_DEFS_H_
+#define _KBASE_HWACCESS_DEFS_H_
+
+#include <mali_kbase_jm_defs.h>
+
+/* The kbasep_js_device_data::runpool_irq::lock (a spinlock) must be held when
+ * accessing this structure */
+struct kbase_hwaccess_data {
+ struct kbase_context *active_kctx;
+
+ struct kbase_backend_data backend;
+};
+
+#endif /* _KBASE_HWACCESS_DEFS_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/**
+ * Base kernel property query backend APIs
+ */
+
+#ifndef _KBASE_HWACCESS_GPUPROPS_H_
+#define _KBASE_HWACCESS_GPUPROPS_H_
+
+/**
+ * kbase_backend_gpuprops_get() - Fill @regdump with GPU properties read from
+ * GPU
+ * @kbdev: Device pointer
+ * @regdump: Pointer to struct kbase_gpuprops_regdump structure
+ */
+void kbase_backend_gpuprops_get(struct kbase_device *kbdev,
+ struct kbase_gpuprops_regdump *regdump);
+
+#endif /* _KBASE_HWACCESS_GPUPROPS_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * HW Access instrumentation common APIs
+ */
+
+#ifndef _KBASE_HWACCESS_INSTR_H_
+#define _KBASE_HWACCESS_INSTR_H_
+
+#include <mali_kbase_instr_defs.h>
+
+/**
+ * kbase_instr_hwcnt_enable_internal - Enable HW counters collection
+ * @kbdev: Kbase device
+ * @kctx: Kbase context
+ * @setup: HW counter setup parameters
+ *
+ * Context: might sleep, waiting for reset to complete
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ struct kbase_uk_hwcnt_setup *setup);
+
+/**
+ * kbase_instr_hwcnt_disable_internal - Disable HW counters collection
+ * @kctx: Kbase context
+ *
+ * Context: might sleep, waiting for an ongoing dump to complete
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx);
+
+/**
+ * kbase_instr_hwcnt_request_dump() - Request HW counter dump from GPU
+ * @kctx: Kbase context
+ *
+ * Caller must either wait for kbase_instr_hwcnt_dump_complete() to return true,
+ * of call kbase_instr_hwcnt_wait_for_dump().
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx);
+
+/**
+ * kbase_instr_hwcnt_wait_for_dump() - Wait until pending HW counter dump has
+ * completed.
+ * @kctx: Kbase context
+ *
+ * Context: will sleep, waiting for dump to complete
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx);
+
+/**
+ * kbase_instr_hwcnt_dump_complete - Tell whether the HW counters dump has
+ * completed
+ * @kctx: Kbase context
+ * @success: Set to true if successful
+ *
+ * Context: does not sleep.
+ *
+ * Return: true if the dump is complete
+ */
+bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx,
+ bool * const success);
+
+/**
+ * kbase_instr_hwcnt_clear() - Clear HW counters
+ * @kctx: Kbase context
+ *
+ * Context: might sleep, waiting for reset to complete
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_hwcnt_clear(struct kbase_context *kctx);
+
+/**
+ * kbase_instr_backend_init() - Initialise the instrumentation backend
+ * @kbdev: Kbase device
+ *
+ * This function should be called during driver initialization.
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_backend_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_instr_backend_init() - Terminate the instrumentation backend
+ * @kbdev: Kbase device
+ *
+ * This function should be called during driver termination.
+ */
+void kbase_instr_backend_term(struct kbase_device *kbdev);
+
+#endif /* _KBASE_HWACCESS_INSTR_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/*
+ * HW access job manager common APIs
+ */
+
+#ifndef _KBASE_HWACCESS_JM_H_
+#define _KBASE_HWACCESS_JM_H_
+
+/**
+ * kbase_backend_run_atom() - Run an atom on the GPU
+ * @kbdev: Device pointer
+ * @atom: Atom to run
+ *
+ * Caller must hold the HW access lock
+ */
+void kbase_backend_run_atom(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom);
+
+/**
+ * kbase_backend_find_free_address_space() - Find a free address space.
+ * @kbdev: Device pointer
+ * @kctx: Context pointer
+ *
+ * If no address spaces are currently free, then this function can evict an
+ * idle context from the runpool, freeing up the address space it was using.
+ *
+ * The address space is marked as in use. The caller must either assign a
+ * context using kbase_gpu_use_ctx(), or release it using
+ * kbase_gpu_release_free_address_space()
+ *
+ * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none
+ * available
+ */
+int kbase_backend_find_free_address_space(struct kbase_device *kbdev,
+ struct kbase_context *kctx);
+
+/**
+ * kbase_backend_release_free_address_space() - Release an address space.
+ * @kbdev: Device pointer
+ * @as_nr: Address space to release
+ *
+ * The address space must have been returned by
+ * kbase_gpu_find_free_address_space().
+ */
+void kbase_backend_release_free_address_space(struct kbase_device *kbdev,
+ int as_nr);
+
+/**
+ * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the
+ * provided address space.
+ * @kbdev: Device pointer
+ * @kctx: Context pointer. May be NULL
+ * @as_nr: Free address space to use
+ *
+ * kbase_gpu_next_job() will pull atoms from the active context.
+ *
+ * Return: true if successful, false if ASID not assigned. If kctx->as_pending
+ * is true then ASID assignment will complete at some point in the
+ * future and will re-start scheduling, otherwise no ASIDs are available
+ */
+bool kbase_backend_use_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ int as_nr);
+
+/**
+ * kbase_backend_use_ctx_sched() - Activate a context.
+ * @kbdev: Device pointer
+ * @kctx: Context pointer
+ *
+ * kbase_gpu_next_job() will pull atoms from the active context.
+ *
+ * The context must already be scheduled and assigned to an address space. If
+ * the context is not scheduled, then kbase_gpu_use_ctx() should be used
+ * instead.
+ *
+ * Caller must hold runpool_irq.lock
+ *
+ * Return: true if context is now active, false otherwise (ie if context does
+ * not have an address space assigned)
+ */
+bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev,
+ struct kbase_context *kctx);
+
+/**
+ * kbase_backend_release_ctx_irq - Release a context from the GPU. This will
+ * de-assign the assigned address space.
+ * @kbdev: Device pointer
+ * @kctx: Context pointer
+ *
+ * Caller must hold as->transaction_mutex and runpool_irq.lock
+ */
+void kbase_backend_release_ctx_irq(struct kbase_device *kbdev,
+ struct kbase_context *kctx);
+
+/**
+ * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will
+ * de-assign the assigned address space.
+ * @kbdev: Device pointer
+ * @kctx: Context pointer
+ *
+ * Caller must hold as->transaction_mutex
+ *
+ * This function must perform any operations that could not be performed in IRQ
+ * context by kbase_backend_release_ctx_irq().
+ */
+void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev,
+ struct kbase_context *kctx);
+
+/**
+ * kbase_backend_complete_wq() - Perform backend-specific actions required on
+ * completing an atom.
+ * @kbdev: Device pointer
+ * @katom: Pointer to the atom to complete
+ *
+ * This function should only be called from jd_done_worker().
+ *
+ * Return: true if atom has completed, false if atom should be re-submitted
+ */
+void kbase_backend_complete_wq(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom);
+
+/**
+ * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU
+ * and remove any others from the ringbuffers.
+ * @kbdev: Device pointer
+ * @end_timestamp: Timestamp of reset
+ */
+void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp);
+
+/**
+ * kbase_backend_inspect_head() - Return the atom currently at the head of slot
+ * @js
+ * @kbdev: Device pointer
+ * @js: Job slot to inspect
+ *
+ * Return : Atom currently at the head of slot @js, or NULL
+ */
+struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev,
+ int js);
+
+/**
+ * kbase_backend_inspect_tail - Return the atom currently at the tail of slot
+ * @js
+ * @kbdev: Device pointer
+ * @js: Job slot to inspect
+ *
+ * Return : Atom currently at the head of slot @js, or NULL
+ */
+struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev,
+ int js);
+
+/**
+ * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a
+ * slot.
+ * @kbdev: Device pointer
+ * @js: Job slot to inspect
+ *
+ * Return : Number of atoms currently on slot
+ */
+int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js);
+
+/**
+ * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot
+ * that are currently on the GPU.
+ * @kbdev: Device pointer
+ * @js: Job slot to inspect
+ *
+ * Return : Number of atoms currently on slot @js that are currently on the GPU.
+ */
+int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js);
+
+/**
+ * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs
+ * has changed.
+ * @kbdev: Device pointer
+ *
+ * Perform any required backend-specific actions (eg starting/stopping
+ * scheduling timers).
+ */
+void kbase_backend_ctx_count_changed(struct kbase_device *kbdev);
+
+/**
+ * kbase_backend_slot_free() - Return the number of jobs that can be currently
+ * submitted to slot @js.
+ * @kbdev: Device pointer
+ * @js: Job slot to inspect
+ *
+ * Return : Number of jobs that can be submitted.
+ */
+int kbase_backend_slot_free(struct kbase_device *kbdev, int js);
+
+/**
+ * kbase_job_check_enter_disjoint - potentially leave disjoint state
+ * @kbdev: kbase device
+ * @target_katom: atom which is finishing
+ *
+ * Work out whether to leave disjoint state when finishing an atom that was
+ * originated by kbase_job_check_enter_disjoint().
+ */
+void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
+ struct kbase_jd_atom *target_katom);
+
+/**
+ * kbase_backend_jm_kill_jobs_from_kctx - Kill all jobs that are currently
+ * running from a context
+ * @kctx: Context pointer
+ *
+ * This is used in response to a page fault to remove all jobs from the faulting
+ * context from the hardware.
+ */
+void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx);
+
+/**
+ * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and
+ * to be descheduled.
+ * @kctx: Context pointer
+ *
+ * This should be called following kbase_js_zap_context(), to ensure the context
+ * can be safely destroyed.
+ */
+void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx);
+
+#if KBASE_GPU_RESET_EN
+/**
+ * kbase_prepare_to_reset_gpu - Prepare for resetting the GPU.
+ * @kbdev: Device pointer
+ *
+ * This function just soft-stops all the slots to ensure that as many jobs as
+ * possible are saved.
+ *
+ * Return: a boolean which should be interpreted as follows:
+ * - true - Prepared for reset, kbase_reset_gpu should be called.
+ * - false - Another thread is performing a reset, kbase_reset_gpu should
+ * not be called.
+ */
+bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev);
+
+/**
+ * kbase_reset_gpu - Reset the GPU
+ * @kbdev: Device pointer
+ *
+ * This function should be called after kbase_prepare_to_reset_gpu if it returns
+ * true. It should never be called without a corresponding call to
+ * kbase_prepare_to_reset_gpu.
+ *
+ * After this function is called (or not called if kbase_prepare_to_reset_gpu
+ * returned false), the caller should wait for kbdev->reset_waitq to be
+ * signalled to know when the reset has completed.
+ */
+void kbase_reset_gpu(struct kbase_device *kbdev);
+#endif
+
+/**
+ * kbase_job_slot_hardstop - Hard-stop the specified job slot
+ * @kctx: The kbase context that contains the job(s) that should
+ * be hard-stopped
+ * @js: The job slot to hard-stop
+ * @target_katom: The job that should be hard-stopped (or NULL for all
+ * jobs from the context)
+ * Context:
+ * The job slot lock must be held when calling this function.
+ */
+void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
+ struct kbase_jd_atom *target_katom);
+
+#endif /* _KBASE_HWACCESS_JM_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/**
+ * @file mali_kbase_hwaccess_pm.h
+ * HW access power manager common APIs
+ */
+
+#ifndef _KBASE_HWACCESS_PM_H_
+#define _KBASE_HWACCESS_PM_H_
+
+#include <mali_midg_regmap.h>
+#include <linux/atomic.h>
+
+#include <mali_kbase_pm_defs.h>
+
+/* Forward definition - see mali_kbase.h */
+struct kbase_device;
+
+/* Functions common to all HW access backends */
+
+/**
+ * Initialize the power management framework.
+ *
+ * Must be called before any other power management function
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ *
+ * @return 0 if the power management framework was successfully
+ * initialized.
+ */
+int kbase_hwaccess_pm_init(struct kbase_device *kbdev);
+
+/**
+ * Terminate the power management framework.
+ *
+ * No power management functions may be called after this (except
+ * @ref kbase_pm_init)
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_hwaccess_pm_term(struct kbase_device *kbdev);
+
+/**
+ * kbase_hwaccess_pm_powerup - Power up the GPU.
+ * @kbdev: The kbase device structure for the device (must be a valid pointer)
+ * @flags: Flags to pass on to kbase_pm_init_hw
+ *
+ * Power up GPU after all modules have been initialized and interrupt handlers
+ * installed.
+ *
+ * Return: 0 if powerup was successful.
+ */
+int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
+ unsigned int flags);
+
+/**
+ * Halt the power management framework.
+ *
+ * Should ensure that no new interrupts are generated, but allow any currently
+ * running interrupt handlers to complete successfully. The GPU is forced off by
+ * the time this function returns, regardless of whether or not the active power
+ * policy asks for the GPU to be powered off.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_hwaccess_pm_halt(struct kbase_device *kbdev);
+
+/**
+ * Perform any backend-specific actions to suspend the GPU
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev);
+
+/**
+ * Perform any backend-specific actions to resume the GPU from a suspend
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_hwaccess_pm_resume(struct kbase_device *kbdev);
+
+/**
+ * Perform any required actions for activating the GPU. Called when the first
+ * context goes active.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev);
+
+/**
+ * Perform any required actions for idling the GPU. Called when the last
+ * context goes idle.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ */
+void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev);
+
+
+/**
+ * Set the debug core mask.
+ *
+ * This determines which cores the power manager is allowed to use.
+ *
+ * @param kbdev The kbase device structure for the device (must be a
+ * valid pointer)
+ * @param new_core_mask The core mask to use
+ */
+void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev,
+ u64 new_core_mask);
+
+
+/**
+ * Get the current policy.
+ *
+ * Returns the policy that is currently active.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ *
+ * @return The current policy
+ */
+const struct kbase_pm_ca_policy
+*kbase_pm_ca_get_policy(struct kbase_device *kbdev);
+
+/**
+ * Change the policy to the one specified.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param policy The policy to change to (valid pointer returned from
+ * @ref kbase_pm_ca_list_policies)
+ */
+void kbase_pm_ca_set_policy(struct kbase_device *kbdev,
+ const struct kbase_pm_ca_policy *policy);
+
+/**
+ * Retrieve a static list of the available policies.
+ *
+ * @param[out] policies An array pointer to take the list of policies. This may
+ * be NULL. The contents of this array must not be
+ * modified.
+ *
+ * @return The number of policies
+ */
+int
+kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies);
+
+
+/**
+ * Get the current policy.
+ *
+ * Returns the policy that is currently active.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ *
+ * @return The current policy
+ */
+const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev);
+
+/**
+ * Change the policy to the one specified.
+ *
+ * @param kbdev The kbase device structure for the device (must be a valid
+ * pointer)
+ * @param policy The policy to change to (valid pointer returned from
+ * @ref kbase_pm_list_policies)
+ */
+void kbase_pm_set_policy(struct kbase_device *kbdev,
+ const struct kbase_pm_policy *policy);
+
+/**
+ * Retrieve a static list of the available policies.
+ *
+ * @param[out] policies An array pointer to take the list of policies. This may
+ * be NULL. The contents of this array must not be
+ * modified.
+ *
+ * @return The number of policies
+ */
+int kbase_pm_list_policies(const struct kbase_pm_policy * const **policies);
+
+#endif /* _KBASE_HWACCESS_PM_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+/**
+ *
+ */
+
+#ifndef _KBASE_BACKEND_TIME_H_
+#define _KBASE_BACKEND_TIME_H_
+
+/**
+ * kbase_backend_get_gpu_time() - Get current GPU time
+ * @kbdev: Device pointer
+ * @cycle_counter: Pointer to u64 to store cycle counter in
+ * @system_time: Pointer to u64 to store system time in
+ * @ts: Pointer to struct timespec to store current monotonic
+ * time in
+ */
+void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
+ u64 *system_time, struct timespec *ts);
+
+/**
+ * kbase_wait_write_flush() - Wait for GPU write flush
+ * @kctx: Context pointer
+ *
+ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush
+ * its write buffer.
+ *
+ * If GPU resets occur then the counters are reset to zero, the delay may not be
+ * as expected.
+ *
+ * This function is only in use for BASE_HW_ISSUE_6367
+ */
+#ifndef CONFIG_MALI_NO_MALI
+void kbase_wait_write_flush(struct kbase_context *kctx);
+#endif
+
+#endif /* _KBASE_BACKEND_TIME_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file mali_kbase_instr.c
+/*
* Base kernel instrumentation APIs.
*/
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
-/**
- * @brief Issue Cache Clean & Invalidate command to hardware
- */
-static void kbasep_instr_hwcnt_cacheclean(struct kbase_device *kbdev)
-{
- unsigned long flags;
- unsigned long pm_flags;
- u32 irq_mask;
-
- KBASE_DEBUG_ASSERT(NULL != kbdev);
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- /* Wait for any reset to complete */
- while (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- wait_event(kbdev->hwcnt.cache_clean_wait,
- kbdev->hwcnt.state != KBASE_INSTR_STATE_RESETTING);
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- }
- KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_REQUEST_CLEAN);
-
- /* Enable interrupt */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
- irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | CLEAN_CACHES_COMPLETED, NULL);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
-
- /* clean&invalidate the caches so we're sure the mmu tables for the dump buffer is valid */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CLEAN_INV_CACHES, NULL);
- kbdev->hwcnt.state = KBASE_INSTR_STATE_CLEANING;
-
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
-}
-
-STATIC mali_error kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_uk_hwcnt_setup *setup)
-{
- unsigned long flags, pm_flags;
- mali_error err = MALI_ERROR_FUNCTION_FAILED;
- struct kbasep_js_device_data *js_devdata;
- u32 irq_mask;
- int ret;
- u64 shader_cores_needed;
-
- KBASE_DEBUG_ASSERT(NULL != kctx);
- KBASE_DEBUG_ASSERT(NULL != kbdev);
- KBASE_DEBUG_ASSERT(NULL != setup);
- KBASE_DEBUG_ASSERT(NULL == kbdev->hwcnt.suspended_kctx);
-
- shader_cores_needed = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER);
-
- js_devdata = &kbdev->js_data;
-
- /* alignment failure */
- if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1)))
- goto out_err;
-
- /* Override core availability policy to ensure all cores are available */
- kbase_pm_ca_instr_enable(kbdev);
-
- /* Mark the context as active so the GPU is kept turned on */
- /* A suspend won't happen here, because we're in a syscall from a userspace
- * thread. */
- kbase_pm_context_active(kbdev);
-
- /* Request the cores early on synchronously - we'll release them on any errors
- * (e.g. instrumentation already active) */
- kbase_pm_request_cores_sync(kbdev, MALI_TRUE, shader_cores_needed);
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
-
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
- /* GPU is being reset */
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- }
-
- if (kbdev->hwcnt.state != KBASE_INSTR_STATE_DISABLED) {
- /* Instrumentation is already enabled */
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- goto out_unrequest_cores;
- }
-
- /* Enable interrupt */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
- irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | PRFCNT_SAMPLE_COMPLETED, NULL);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
-
- /* In use, this context is the owner */
- kbdev->hwcnt.kctx = kctx;
- /* Remember the dump address so we can reprogram it later */
- kbdev->hwcnt.addr = setup->dump_buffer;
- /* Remember all the settings for suspend/resume */
- if (&kbdev->hwcnt.suspended_state != setup)
- memcpy(&kbdev->hwcnt.suspended_state, setup, sizeof(kbdev->hwcnt.suspended_state));
-
- /* Request the clean */
- kbdev->hwcnt.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
- kbdev->hwcnt.triggered = 0;
- /* Clean&invalidate the caches so we're sure the mmu tables for the dump buffer is valid */
- ret = queue_work(kbdev->hwcnt.cache_clean_wq, &kbdev->hwcnt.cache_clean_work);
- KBASE_DEBUG_ASSERT(ret);
-
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
-
- /* Wait for cacheclean to complete */
- wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
-
- KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE);
-
- /* Schedule the context in */
- kbasep_js_schedule_privileged_ctx(kbdev, kctx);
-
- /* Configure */
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_OFF, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), setup->dump_buffer & 0xFFFFFFFF, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), setup->dump_buffer >> 32, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), setup->jm_bm, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), setup->shader_bm, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_L3_CACHE_EN), setup->l3_cache_bm, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), setup->mmu_l2_bm, kctx);
- /* Due to PRLAM-8186 we need to disable the Tiler before we enable the HW counter dump. */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, kctx);
- else
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
-
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, kctx);
-
- /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), setup->tiler_bm, kctx);
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
-
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
- /* GPU is being reset */
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- }
-
- kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
- kbdev->hwcnt.triggered = 1;
- wake_up(&kbdev->hwcnt.wait);
-
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
-
- err = MALI_ERROR_NONE;
-
- dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx);
- return err;
- out_unrequest_cores:
- kbase_pm_unrequest_cores(kbdev, MALI_TRUE, shader_cores_needed);
- kbase_pm_context_idle(kbdev);
- out_err:
- return err;
-}
-
-/**
- * @brief Enable HW counters collection
- *
- * Note: will wait for a cache clean to complete
- */
-mali_error kbase_instr_hwcnt_enable(struct kbase_context *kctx, struct kbase_uk_hwcnt_setup *setup)
-{
- struct kbase_device *kbdev;
- mali_bool access_allowed;
-
- kbdev = kctx->kbdev;
-
- KBASE_DEBUG_ASSERT(NULL != kctx);
- /* Determine if the calling task has access to this capability */
- access_allowed = kbase_security_has_capability(kctx, KBASE_SEC_INSTR_HW_COUNTERS_COLLECT, KBASE_SEC_FLAG_NOAUDIT);
- if (MALI_FALSE == access_allowed)
- return MALI_ERROR_FUNCTION_FAILED;
-
- return kbase_instr_hwcnt_enable_internal(kbdev, kctx, setup);
-}
-KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_enable)
-
-/**
- * @brief Disable HW counters collection
- *
- * Note: might sleep, waiting for an ongoing dump to complete
- */
-mali_error kbase_instr_hwcnt_disable(struct kbase_context *kctx)
-{
- unsigned long flags, pm_flags;
- mali_error err = MALI_ERROR_FUNCTION_FAILED;
- u32 irq_mask;
- struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(NULL != kctx);
- kbdev = kctx->kbdev;
- KBASE_DEBUG_ASSERT(NULL != kbdev);
-
- while (1) {
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
-
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_DISABLED) {
- /* Instrumentation is not enabled */
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- goto out;
- }
-
- if (kbdev->hwcnt.kctx != kctx) {
- /* Instrumentation has been setup for another context */
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- goto out;
- }
-
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE)
- break;
-
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
-
- /* Ongoing dump/setup - wait for its completion */
- wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
- }
-
- kbdev->hwcnt.state = KBASE_INSTR_STATE_DISABLED;
- kbdev->hwcnt.triggered = 0;
-
- /* Disable interrupt */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
- irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
-
- /* Disable the counters */
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx);
-
- kbdev->hwcnt.kctx = NULL;
- kbdev->hwcnt.addr = 0ULL;
-
- kbase_pm_ca_instr_disable(kbdev);
-
- kbase_pm_unrequest_cores(kbdev, MALI_TRUE, kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER));
-
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
-
- /* Release the context. This had its own Power Manager Active reference */
- kbasep_js_release_privileged_ctx(kbdev, kctx);
-
- /* Also release our Power Manager Active reference */
- kbase_pm_context_idle(kbdev);
-
- dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx);
-
- err = MALI_ERROR_NONE;
-
- out:
- return err;
-}
-KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_disable)
-
-/**
- * @brief Configure HW counters collection
- */
-mali_error kbase_instr_hwcnt_setup(struct kbase_context *kctx, struct kbase_uk_hwcnt_setup *setup)
+void kbase_instr_hwcnt_suspend(struct kbase_device *kbdev)
{
- mali_error err = MALI_ERROR_FUNCTION_FAILED;
- struct kbase_device *kbdev;
+ struct kbase_context *kctx;
- KBASE_DEBUG_ASSERT(NULL != kctx);
+ KBASE_DEBUG_ASSERT(kbdev);
+ KBASE_DEBUG_ASSERT(!kbdev->hwcnt.suspended_kctx);
- kbdev = kctx->kbdev;
- KBASE_DEBUG_ASSERT(NULL != kbdev);
+ kctx = kbdev->hwcnt.kctx;
+ kbdev->hwcnt.suspended_kctx = kctx;
- if (NULL == setup) {
- /* Bad parameter - abort */
- goto out;
- }
+ /* Relevant state was saved into hwcnt.suspended_state when enabling the
+ * counters */
- if (setup->dump_buffer != 0ULL) {
- /* Enable HW counters */
- err = kbase_instr_hwcnt_enable(kctx, setup);
- } else {
- /* Disable HW counters */
- err = kbase_instr_hwcnt_disable(kctx);
+ if (kctx) {
+ KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.flags &
+ KBASE_CTX_FLAG_PRIVILEGED);
+ kbase_instr_hwcnt_disable(kctx);
}
-
- out:
- return err;
}
-/**
- * @brief Issue Dump command to hardware
- *
- * Notes:
- * - does not sleep
- */
-mali_error kbase_instr_hwcnt_dump_irq(struct kbase_context *kctx)
+void kbase_instr_hwcnt_resume(struct kbase_device *kbdev)
{
- unsigned long flags;
- mali_error err = MALI_ERROR_FUNCTION_FAILED;
- struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(NULL != kctx);
- kbdev = kctx->kbdev;
- KBASE_DEBUG_ASSERT(NULL != kbdev);
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
-
- if (kbdev->hwcnt.kctx != kctx) {
- /* The instrumentation has been setup for another context */
- goto unlock;
- }
-
- if (kbdev->hwcnt.state != KBASE_INSTR_STATE_IDLE) {
- /* HW counters are disabled or another dump is ongoing, or we're resetting */
- goto unlock;
- }
-
- kbdev->hwcnt.triggered = 0;
-
- /* Mark that we're dumping - the PF handler can signal that we faulted */
- kbdev->hwcnt.state = KBASE_INSTR_STATE_DUMPING;
-
- /* Reconfigure the dump address */
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), kbdev->hwcnt.addr & 0xFFFFFFFF, NULL);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), kbdev->hwcnt.addr >> 32, NULL);
-
- /* Start dumping */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, kbdev->hwcnt.addr, 0);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_SAMPLE, kctx);
-
- dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx);
+ struct kbase_context *kctx;
- err = MALI_ERROR_NONE;
+ KBASE_DEBUG_ASSERT(kbdev);
- unlock:
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- return err;
-}
-KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_irq)
+ kctx = kbdev->hwcnt.suspended_kctx;
+ kbdev->hwcnt.suspended_kctx = NULL;
-/**
- * @brief Tell whether the HW counters dump has completed
- *
- * Notes:
- * - does not sleep
- * - success will be set to MALI_TRUE if the dump succeeded or
- * MALI_FALSE on failure
- */
-mali_bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, mali_bool * const success)
-{
- unsigned long flags;
- mali_bool complete = MALI_FALSE;
- struct kbase_device *kbdev;
+ if (kctx) {
+ int err;
- KBASE_DEBUG_ASSERT(NULL != kctx);
- kbdev = kctx->kbdev;
- KBASE_DEBUG_ASSERT(NULL != kbdev);
- KBASE_DEBUG_ASSERT(NULL != success);
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
-
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE) {
- *success = MALI_TRUE;
- complete = MALI_TRUE;
- } else if (kbdev->hwcnt.state == KBASE_INSTR_STATE_FAULT) {
- *success = MALI_FALSE;
- complete = MALI_TRUE;
- kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
+ err = kbase_instr_hwcnt_enable_internal(kbdev, kctx,
+ &kbdev->hwcnt.suspended_state);
+ WARN(err, "Failed to restore instrumented hardware counters on resume\n");
}
-
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
-
- return complete;
}
-KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete)
-/**
- * @brief Issue Dump command to hardware and wait for completion
- */
-mali_error kbase_instr_hwcnt_dump(struct kbase_context *kctx)
+int kbase_instr_hwcnt_enable(struct kbase_context *kctx,
+ struct kbase_uk_hwcnt_setup *setup)
{
- unsigned long flags;
- mali_error err = MALI_ERROR_FUNCTION_FAILED;
struct kbase_device *kbdev;
+ bool access_allowed;
+ int err;
- KBASE_DEBUG_ASSERT(NULL != kctx);
kbdev = kctx->kbdev;
- KBASE_DEBUG_ASSERT(NULL != kbdev);
-
- err = kbase_instr_hwcnt_dump_irq(kctx);
- if (MALI_ERROR_NONE != err) {
- /* Can't dump HW counters */
- goto out;
- }
- /* Wait for dump & cacheclean to complete */
- wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
+ /* Determine if the calling task has access to this capability */
+ access_allowed = kbase_security_has_capability(kctx,
+ KBASE_SEC_INSTR_HW_COUNTERS_COLLECT,
+ KBASE_SEC_FLAG_NOAUDIT);
+ if (!access_allowed)
+ return -EINVAL;
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ /* Mark the context as active so the GPU is kept turned on */
+ /* A suspend won't happen here, because we're in a syscall from a
+ * userspace thread. */
+ kbase_pm_context_active(kbdev);
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
- /* GPU is being reset */
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- }
+ /* Schedule the context in */
+ kbasep_js_schedule_privileged_ctx(kbdev, kctx);
+ err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, setup);
+ if (err) {
+ /* Release the context. This had its own Power Manager Active
+ * reference */
+ kbasep_js_release_privileged_ctx(kbdev, kctx);
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_FAULT) {
- err = MALI_ERROR_FUNCTION_FAILED;
- kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
- } else {
- /* Dump done */
- KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_IDLE);
- err = MALI_ERROR_NONE;
+ /* Also release our Power Manager Active reference */
+ kbase_pm_context_idle(kbdev);
}
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- out:
return err;
}
-KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump)
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_enable);
-/**
- * @brief Clear the HW counters
- */
-mali_error kbase_instr_hwcnt_clear(struct kbase_context *kctx)
+int kbase_instr_hwcnt_disable(struct kbase_context *kctx)
{
- unsigned long flags;
- mali_error err = MALI_ERROR_FUNCTION_FAILED;
- struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(NULL != kctx);
- kbdev = kctx->kbdev;
- KBASE_DEBUG_ASSERT(NULL != kbdev);
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
+ int err = -EINVAL;
+ struct kbase_device *kbdev = kctx->kbdev;
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) {
- /* GPU is being reset */
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- }
-
- /* Check it's the context previously set up and we're not already dumping */
- if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.state != KBASE_INSTR_STATE_IDLE)
+ err = kbase_instr_hwcnt_disable_internal(kctx);
+ if (err)
goto out;
- /* Clear the counters */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_PRFCNT_CLEAR, kctx);
+ /* Release the context. This had its own Power Manager Active reference
+ */
+ kbasep_js_release_privileged_ctx(kbdev, kctx);
- err = MALI_ERROR_NONE;
+ /* Also release our Power Manager Active reference */
+ kbase_pm_context_idle(kbdev);
- out:
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p",
+ kctx);
+out:
return err;
}
-KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear)
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_disable);
-/**
- * Workqueue for handling cache cleaning
- */
-void kbasep_cache_clean_worker(struct work_struct *data)
+int kbase_instr_hwcnt_setup(struct kbase_context *kctx,
+ struct kbase_uk_hwcnt_setup *setup)
{
- struct kbase_device *kbdev;
- unsigned long flags;
-
- kbdev = container_of(data, struct kbase_device, hwcnt.cache_clean_work);
-
- mutex_lock(&kbdev->cacheclean_lock);
- kbasep_instr_hwcnt_cacheclean(kbdev);
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- /* Wait for our condition, and any reset to complete */
- while (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING ||
- kbdev->hwcnt.state == KBASE_INSTR_STATE_CLEANING) {
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- wait_event(kbdev->hwcnt.cache_clean_wait,
- (kbdev->hwcnt.state != KBASE_INSTR_STATE_RESETTING
- && kbdev->hwcnt.state != KBASE_INSTR_STATE_CLEANING));
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- }
- KBASE_DEBUG_ASSERT(kbdev->hwcnt.state == KBASE_INSTR_STATE_CLEANED);
-
- /* All finished and idle */
- kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
- kbdev->hwcnt.triggered = 1;
- wake_up(&kbdev->hwcnt.wait);
-
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- mutex_unlock(&kbdev->cacheclean_lock);
-}
-
-/**
- * @brief Dump complete interrupt received
- */
-void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
-
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_FAULT) {
- kbdev->hwcnt.triggered = 1;
- wake_up(&kbdev->hwcnt.wait);
- } else if (kbdev->hwcnt.state == KBASE_INSTR_STATE_DUMPING) {
- int ret;
- /* Always clean and invalidate the cache after a successful dump */
- kbdev->hwcnt.state = KBASE_INSTR_STATE_REQUEST_CLEAN;
- ret = queue_work(kbdev->hwcnt.cache_clean_wq, &kbdev->hwcnt.cache_clean_work);
- KBASE_DEBUG_ASSERT(ret);
- }
- /* NOTE: In the state KBASE_INSTR_STATE_RESETTING, We're in a reset,
- * and the instrumentation state hasn't been restored yet -
- * kbasep_reset_timeout_worker() will do the rest of the work */
-
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
-}
-
-/**
- * @brief Cache clean interrupt received
- */
-void kbase_clean_caches_done(struct kbase_device *kbdev)
-{
- u32 irq_mask;
-
- if (kbdev->hwcnt.state != KBASE_INSTR_STATE_DISABLED) {
- unsigned long flags;
- unsigned long pm_flags;
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- /* Disable interrupt */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, pm_flags);
- irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~CLEAN_CACHES_COMPLETED, NULL);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, pm_flags);
-
- /* Wakeup... */
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_CLEANING) {
- /* Only wake if we weren't resetting */
- kbdev->hwcnt.state = KBASE_INSTR_STATE_CLEANED;
- wake_up(&kbdev->hwcnt.cache_clean_wait);
- }
- /* NOTE: In the state KBASE_INSTR_STATE_RESETTING, We're in a reset,
- * and the instrumentation state hasn't been restored yet -
- * kbasep_reset_timeout_worker() will do the rest of the work */
-
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
+ struct kbase_vinstr_context *vinstr_ctx = kctx->kbdev->vinstr_ctx;
+ u32 bitmap[4];
+
+ if (setup == NULL)
+ return -EINVAL;
+
+ bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
+ bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
+ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
+ bitmap[JM_HWCNT_BM] = setup->jm_bm;
+ if (setup->dump_buffer) {
+ if (kctx->vinstr_cli)
+ return -EBUSY;
+ kctx->vinstr_cli = kbase_vinstr_attach_client(vinstr_ctx, false,
+ setup->dump_buffer, bitmap);
+ if (!kctx->vinstr_cli)
+ return -ENOMEM;
+ } else {
+ kbase_vinstr_detach_client(vinstr_ctx, kctx->vinstr_cli);
+ kctx->vinstr_cli = NULL;
}
-}
-
-
-/* Disable instrumentation and wait for any existing dump to complete
- * It's assumed that there's only one privileged context
- * Safe to do this without lock when doing an OS suspend, because it only
- * changes in response to user-space IOCTLs */
-void kbase_instr_hwcnt_suspend(struct kbase_device *kbdev)
-{
- struct kbase_context *kctx;
- KBASE_DEBUG_ASSERT(kbdev);
- KBASE_DEBUG_ASSERT(!kbdev->hwcnt.suspended_kctx);
-
- kctx = kbdev->hwcnt.kctx;
- kbdev->hwcnt.suspended_kctx = kctx;
-
- /* Relevant state was saved into hwcnt.suspended_state when enabling the
- * counters */
-
- if (kctx) {
- KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_PRIVILEGED);
- kbase_instr_hwcnt_disable(kctx);
- }
+ return 0;
}
-void kbase_instr_hwcnt_resume(struct kbase_device *kbdev)
+int kbase_instr_hwcnt_dump(struct kbase_context *kctx)
{
- struct kbase_context *kctx;
-
- KBASE_DEBUG_ASSERT(kbdev);
-
- kctx = kbdev->hwcnt.suspended_kctx;
- kbdev->hwcnt.suspended_kctx = NULL;
-
- if (kctx) {
- mali_error err;
- err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &kbdev->hwcnt.suspended_state);
- WARN(err != MALI_ERROR_NONE,
- "Failed to restore instrumented hardware counters on resume\n");
- }
+ return kbase_vinstr_dump(kctx->kbdev->vinstr_ctx, kctx->vinstr_cli);
}
-
+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump);
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+/*
+ * Instrumentation API definitions
+ */
+
+#ifndef _KBASE_INSTR_H_
+#define _KBASE_INSTR_H_
+
+#include <mali_kbase_hwaccess_instr.h>
+
+/**
+ * kbase_instr_hwcnt_setup() - Configure HW counters collection
+ * @kctx: Kbase context
+ * @setup: &struct kbase_uk_hwcnt_setup containing configuration
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_hwcnt_setup(struct kbase_context *kctx,
+ struct kbase_uk_hwcnt_setup *setup);
+
+/**
+ * kbase_instr_hwcnt_enable() - Enable HW counters collection
+ * @kctx: Kbase context
+ * @setup: &struct kbase_uk_hwcnt_setup containing configuration
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_hwcnt_enable(struct kbase_context *kctx,
+ struct kbase_uk_hwcnt_setup *setup);
+
+/**
+ * kbase_instr_hwcnt_disable() - Disable HW counters collection
+ * @kctx: Kbase context
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_hwcnt_disable(struct kbase_context *kctx);
+
+/**
+ * kbase_instr_hwcnt_dump() - Trigger dump of HW counters and wait for
+ * completion
+ * @kctx: Kbase context
+ *
+ * Context: might sleep, waiting for dump to complete
+ *
+ * Return: 0 on success
+ */
+int kbase_instr_hwcnt_dump(struct kbase_context *kctx);
+
+/**
+ * kbase_instr_hwcnt_suspend() - GPU is suspending, stop HW counter collection
+ * @kbdev: Kbase device
+ *
+ * It's assumed that there's only one privileged context.
+ *
+ * Safe to do this without lock when doing an OS suspend, because it only
+ * changes in response to user-space IOCTLs
+ */
+void kbase_instr_hwcnt_suspend(struct kbase_device *kbdev);
+
+/**
+ * kbase_instr_hwcnt_resume() - GPU is resuming, resume HW counter collection
+ * @kbdev: Kbase device
+ */
+void kbase_instr_hwcnt_resume(struct kbase_device *kbdev);
+
+#endif /* _KBASE_INSTR_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#endif
#include <mali_kbase.h>
#include <mali_kbase_uku.h>
-#include <mali_kbase_js_affinity.h>
-#include <mali_kbase_10969_workaround.h>
#ifdef CONFIG_UMP
#include <linux/ump.h>
#endif /* CONFIG_UMP */
#include <linux/random.h>
+#include <linux/version.h>
+#include <linux/ratelimit.h>
+
+#include <mali_kbase_jm.h>
+#include <mali_kbase_hwaccess_jm.h>
+
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+#include <mali_kbase_tlstream.h>
+#endif
#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
#define prandom_u32 random32
#endif
+/* Return whether katom will run on the GPU or not. Currently only soft jobs and
+ * dependency-only atoms do not run on the GPU */
+#define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \
+ ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE) == \
+ BASE_JD_REQ_DEP)))
/*
* This is the kernel side of the API. Only entry points are:
* - kbase_jd_submit(): Called from userspace to submit a single bag
#ifdef CONFIG_COMPAT
if (kctx->is_compat)
return compat_ptr(p->compat_value);
- else
#endif
- return p->value;
+ return p->value;
}
/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs
goto out;
/* This atom's KDS dependency has now been met */
- katom->kds_dep_satisfied = MALI_TRUE;
-
- /* Check whether the atom's other dependencies were already met */
- if (!kbase_jd_katom_dep_atom(&katom->dep[0]) &&
- !kbase_jd_katom_dep_atom(&katom->dep[1])) {
+ katom->kds_dep_satisfied = true;
+
+ /* Check whether the atom's other dependencies were already met. If
+ * katom is a GPU atom then the job scheduler may be able to represent
+ * the dependencies, hence we may attempt to submit it before they are
+ * met. Other atoms must have had both dependencies resolved */
+ if (IS_GPU_ATOM(katom) ||
+ (!kbase_jd_katom_dep_atom(&katom->dep[0]) &&
+ !kbase_jd_katom_dep_atom(&katom->dep[1]))) {
/* katom dep complete, attempt to run it */
- mali_bool resched = MALI_FALSE;
+ bool resched = false;
resched = jd_run_atom(katom);
}
if (resched)
- kbasep_js_try_schedule_head_ctx(kbdev);
+ kbase_js_sched_all(kbdev);
}
out:
mutex_unlock(&ctx->lock);
/* Wait was cancelled - zap the atom */
katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
if (jd_done_nolock(katom))
- kbasep_js_try_schedule_head_ctx(katom->kctx->kbdev);
+ kbase_js_sched_all(katom->kctx->kbdev);
}
}
#endif /* CONFIG_KDS */
#ifdef CONFIG_DMA_SHARED_BUFFER
-static mali_error kbase_jd_umm_map(struct kbase_context *kctx, struct kbase_va_region *reg)
+static int kbase_jd_umm_map(struct kbase_context *kctx, struct kbase_va_region *reg)
{
struct sg_table *sgt;
struct scatterlist *s;
int i;
phys_addr_t *pa;
- mali_error err;
+ int err;
size_t count = 0;
+ struct kbase_mem_phy_alloc *alloc;
+
+ alloc = reg->gpu_alloc;
- KBASE_DEBUG_ASSERT(reg->alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM);
- KBASE_DEBUG_ASSERT(NULL == reg->alloc->imported.umm.sgt);
- sgt = dma_buf_map_attachment(reg->alloc->imported.umm.dma_attachment, DMA_BIDIRECTIONAL);
+ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM);
+ KBASE_DEBUG_ASSERT(NULL == alloc->imported.umm.sgt);
+ sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, DMA_BIDIRECTIONAL);
if (IS_ERR_OR_NULL(sgt))
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
/* save for later */
- reg->alloc->imported.umm.sgt = sgt;
+ alloc->imported.umm.sgt = sgt;
- pa = kbase_get_phy_pages(reg);
+ pa = kbase_get_gpu_phy_pages(reg);
KBASE_DEBUG_ASSERT(pa);
for_each_sg(sgt->sgl, s, sgt->nents, i) {
*pa++ = sg_dma_address(s) + (j << PAGE_SHIFT);
WARN_ONCE(j < pages,
"sg list from dma_buf_map_attachment > dma_buf->size=%zu\n",
- reg->alloc->imported.umm.dma_buf->size);
+ alloc->imported.umm.dma_buf->size);
}
if (WARN_ONCE(count < reg->nr_pages,
"sg list from dma_buf_map_attachment < dma_buf->size=%zu\n",
- reg->alloc->imported.umm.dma_buf->size)) {
- err = MALI_ERROR_FUNCTION_FAILED;
+ alloc->imported.umm.dma_buf->size)) {
+ err = -EINVAL;
goto out;
}
/* Update nents as we now have pages to map */
- reg->alloc->nents = count;
+ alloc->nents = count;
- err = kbase_mmu_insert_pages(kctx, reg->start_pfn, kbase_get_phy_pages(reg), kbase_reg_current_backed_size(reg), reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD);
+ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, kbase_get_gpu_phy_pages(reg), kbase_reg_current_backed_size(reg), reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD);
out:
- if (MALI_ERROR_NONE != err) {
- dma_buf_unmap_attachment(reg->alloc->imported.umm.dma_attachment, reg->alloc->imported.umm.sgt, DMA_BIDIRECTIONAL);
- reg->alloc->imported.umm.sgt = NULL;
+ if (err) {
+ dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, alloc->imported.umm.sgt, DMA_BIDIRECTIONAL);
+ alloc->imported.umm.sgt = NULL;
}
return err;
#ifdef CONFIG_KDS
/* Prevent the KDS resource from triggering the atom in case of zapping */
if (katom->kds_rset)
- katom->kds_dep_satisfied = MALI_TRUE;
+ katom->kds_dep_satisfied = true;
#endif /* CONFIG_KDS */
kbase_gpu_vm_lock(katom->kctx);
katom->kctx,
katom->extres[res_no].gpu_address);
- if (reg && reg->alloc == alloc)
+ if (reg && reg->gpu_alloc == alloc)
kbase_mmu_teardown_pages(
katom->kctx,
reg->start_pfn,
}
#if (defined(CONFIG_KDS) && defined(CONFIG_UMP)) || defined(CONFIG_DMA_SHARED_BUFFER_USES_KDS)
-static void add_kds_resource(struct kds_resource *kds_res, struct kds_resource **kds_resources, u32 *kds_res_count, unsigned long *kds_access_bitmap, mali_bool exclusive)
+static void add_kds_resource(struct kds_resource *kds_res, struct kds_resource **kds_resources, u32 *kds_res_count, unsigned long *kds_access_bitmap, bool exclusive)
{
u32 i;
* jctx.lock must be held when this is called.
*/
-static mali_error kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom)
+static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom)
{
- mali_error err_ret_val = MALI_ERROR_FUNCTION_FAILED;
+ int err_ret_val = -EINVAL;
u32 res_no;
#ifdef CONFIG_KDS
u32 kds_res_count = 0;
/* no resources encoded, early out */
if (!katom->nr_extres)
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL);
if (NULL == katom->extres) {
- err_ret_val = MALI_ERROR_OUT_OF_MEMORY;
+ err_ret_val = -ENOMEM;
goto early_err_out;
}
if (copy_from_user(input_extres,
get_compat_pointer(katom->kctx, &user_atom->extres_list),
sizeof(*input_extres) * katom->nr_extres) != 0) {
- err_ret_val = MALI_ERROR_FUNCTION_FAILED;
+ err_ret_val = -EINVAL;
goto early_err_out;
}
#ifdef CONFIG_KDS
kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL);
if (NULL == kds_resources) {
- err_ret_val = MALI_ERROR_OUT_OF_MEMORY;
+ err_ret_val = -ENOMEM;
goto early_err_out;
}
kds_access_bitmap = kzalloc(sizeof(unsigned long) * ((katom->nr_extres + BITS_PER_LONG - 1) / BITS_PER_LONG), GFP_KERNEL);
if (NULL == kds_access_bitmap) {
- err_ret_val = MALI_ERROR_OUT_OF_MEMORY;
+ err_ret_val = -ENOMEM;
goto early_err_out;
}
#endif /* CONFIG_KDS */
goto failed_loop;
}
+ if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) &&
+ (reg->flags & KBASE_REG_SECURE)) {
+ katom->atom_flags |= KBASE_KATOM_FLAG_SECURE;
+ if ((katom->core_req & BASE_JD_REQ_FS) == 0) {
+ WARN_RATELIMIT(1, "Secure non-fragment jobs not supported");
+ goto failed_loop;
+ }
+ }
+
/* decide what needs to happen for this resource */
- switch (reg->alloc->type) {
+ switch (reg->gpu_alloc->type) {
case BASE_TMEM_IMPORT_TYPE_UMP:
{
#if defined(CONFIG_KDS) && defined(CONFIG_UMP)
struct kds_resource *kds_res;
- kds_res = ump_dd_kds_resource_get(reg->alloc->imported.ump_handle);
+ kds_res = ump_dd_kds_resource_get(reg->gpu_alloc->imported.ump_handle);
if (kds_res)
add_kds_resource(kds_res, kds_resources, &kds_res_count,
kds_access_bitmap,
#ifdef CONFIG_DMA_SHARED_BUFFER_USES_KDS
struct kds_resource *kds_res;
- kds_res = get_dma_buf_kds_resource(reg->alloc->imported.umm.dma_buf);
+ kds_res = get_dma_buf_kds_resource(reg->gpu_alloc->imported.umm.dma_buf);
if (kds_res)
add_kds_resource(kds_res, kds_resources, &kds_res_count, kds_access_bitmap, res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE);
#endif
- reg->alloc->imported.umm.current_mapping_usage_count++;
- if (1 == reg->alloc->imported.umm.current_mapping_usage_count) {
+ reg->gpu_alloc->imported.umm.current_mapping_usage_count++;
+ if (1 == reg->gpu_alloc->imported.umm.current_mapping_usage_count) {
/* use a local variable to not pollute err_ret_val
* with a potential success value as some other gotos depend
* on the default error code stored in err_ret_val */
- mali_error tmp;
+ int tmp;
+
tmp = kbase_jd_umm_map(katom->kctx, reg);
- if (MALI_ERROR_NONE != tmp) {
+ if (tmp) {
/* failed to map this buffer, roll back */
err_ret_val = tmp;
- reg->alloc->imported.umm.current_mapping_usage_count--;
+ reg->gpu_alloc->imported.umm.current_mapping_usage_count--;
goto failed_loop;
}
}
* until the last read for an element.
* */
katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */
- katom->extres[res_no].alloc = kbase_mem_phy_alloc_get(reg->alloc);
+ katom->extres[res_no].alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
}
/* successfully parsed the extres array */
/* drop the vm lock before we call into kds */
int wait_failed;
/* We have resources to wait for with kds */
- katom->kds_dep_satisfied = MALI_FALSE;
+ katom->kds_dep_satisfied = false;
wait_failed = kds_async_waitall(&katom->kds_rset,
&katom->kctx->jctx.kds_cb, katom, NULL,
kbase_jd_kds_waiters_add(katom);
} else {
/* Nothing to wait for, so kds dep met */
- katom->kds_dep_satisfied = MALI_TRUE;
+ katom->kds_dep_satisfied = true;
}
kfree(kds_resources);
kfree(kds_access_bitmap);
#endif /* CONFIG_KDS */
/* all done OK */
- return MALI_ERROR_NONE;
+ return 0;
/* error handling section */
katom->kctx,
katom->extres[res_no].gpu_address);
- if (reg && reg->alloc == alloc)
+ if (reg && reg->gpu_alloc == alloc)
kbase_mmu_teardown_pages(katom->kctx,
reg->start_pfn,
- kbase_reg_current_backed_size(reg));
+ kbase_reg_current_backed_size(reg));
kbase_jd_umm_unmap(katom->kctx, alloc);
}
return err_ret_val;
}
-STATIC INLINE void jd_resolve_dep(struct list_head *out_list, struct kbase_jd_atom *katom, u8 d, bool ctx_is_dying)
+static inline void jd_resolve_dep(struct list_head *out_list,
+ struct kbase_jd_atom *katom,
+ u8 d,
+ bool ctx_is_dying)
{
u8 other_d = !d;
struct kbase_jd_atom *dep_atom;
u8 dep_type;
- dep_atom = list_entry(katom->dep_head[d].next,
+ dep_atom = list_entry(katom->dep_head[d].next,
struct kbase_jd_atom, dep_item[d]);
- dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]);
list_del(katom->dep_head[d].next);
+ dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]);
kbase_jd_katom_dep_clear(&dep_atom->dep[d]);
if (katom->event_code != BASE_JD_EVENT_DONE &&
/* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and
* do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up
*/
- dep_atom->kds_dep_satisfied = MALI_TRUE;
+ dep_atom->kds_dep_satisfied = true;
}
#endif
- /* at this point a dependency to the failed job is already removed */
+
dep_atom->event_code = katom->event_code;
- KBASE_DEBUG_ASSERT(dep_atom->status != KBASE_JD_ATOM_STATE_UNUSED);
+ KBASE_DEBUG_ASSERT(dep_atom->status !=
+ KBASE_JD_ATOM_STATE_UNUSED);
dep_atom->status = KBASE_JD_ATOM_STATE_COMPLETED;
list_add_tail(&dep_atom->dep_item[0], out_list);
}
}
-KBASE_EXPORT_TEST_API(jd_resolve_dep)
+KBASE_EXPORT_TEST_API(jd_resolve_dep);
#if MALI_CUSTOMER_RELEASE == 0
static void jd_force_failure(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
*
* The caller must hold the kbase_jd_context.lock.
*/
-mali_bool jd_done_nolock(struct kbase_jd_atom *katom)
+bool jd_done_nolock(struct kbase_jd_atom *katom)
{
struct kbase_context *kctx = katom->kctx;
- struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info;
struct kbase_device *kbdev = kctx->kbdev;
+ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info;
struct list_head completed_jobs;
struct list_head runnable_jobs;
- mali_bool need_to_try_schedule_context = MALI_FALSE;
+ bool need_to_try_schedule_context = false;
int i;
INIT_LIST_HEAD(&completed_jobs);
while (!list_empty(&runnable_jobs)) {
struct kbase_jd_atom *node;
- node = list_entry(runnable_jobs.prev, struct kbase_jd_atom, dep_item[0]);
-
- list_del(runnable_jobs.prev);
+ node = list_entry(runnable_jobs.next,
+ struct kbase_jd_atom, dep_item[0]);
+
+ list_del(runnable_jobs.next);
KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED);
return need_to_try_schedule_context;
}
-KBASE_EXPORT_TEST_API(jd_done_nolock)
+KBASE_EXPORT_TEST_API(jd_done_nolock);
#ifdef CONFIG_GPU_TRACEPOINTS
enum {
}
#endif
-mali_bool jd_submit_atom(struct kbase_context *kctx,
+bool jd_submit_atom(struct kbase_context *kctx,
const struct base_jd_atom_v2 *user_atom,
struct kbase_jd_atom *katom)
{
base_jd_core_req core_req;
int queued = 0;
int i;
- mali_bool ret;
+ int sched_prio;
+ bool ret;
/* Update the TOTAL number of jobs. This includes those not tracked by
* the scheduler: 'not ready to run' and 'dependency-only' jobs. */
katom->jc = user_atom->jc;
katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
katom->core_req = core_req;
- katom->nice_prio = user_atom->prio;
katom->atom_flags = 0;
katom->retry_count = 0;
-
+ katom->need_cache_flush_cores_retained = 0;
+ katom->x_pre_dep = NULL;
+ katom->x_post_dep = NULL;
#ifdef CONFIG_KDS
/* Start by assuming that the KDS dependencies are satisfied,
* kbase_jd_pre_external_resources will correct this if there are dependencies */
- katom->kds_dep_satisfied = MALI_TRUE;
+ katom->kds_dep_satisfied = true;
katom->kds_rset = NULL;
#endif /* CONFIG_KDS */
base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type;
if (dep_atom_number) {
- if ( dep_atom_type != BASE_JD_DEP_TYPE_ORDER &&
- dep_atom_type != BASE_JD_DEP_TYPE_DATA ) {
+ if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER &&
+ dep_atom_type != BASE_JD_DEP_TYPE_DATA) {
katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT;
katom->status = KBASE_JD_ATOM_STATE_COMPLETED;
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_tl_new_atom(
+ katom,
+ kbase_jd_atom_id(kctx, katom));
+ kbase_tlstream_tl_ret_atom_ctx(
+ katom, kctx);
+#endif
ret = jd_done_nolock(katom);
goto out;
}
/* Add dependencies */
for (i = 0; i < 2; i++) {
int dep_atom_number = user_atom->pre_dep[i].atom_id;
- base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type;
+ base_jd_dep_type dep_atom_type;
+ struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number];
+ dep_atom_type = user_atom->pre_dep[i].dependency_type;
kbase_jd_katom_dep_clear(&katom->dep[i]);
- if (dep_atom_number) {
- struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number];
-
- if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
- if (dep_atom->event_code != BASE_JD_EVENT_DONE) {
- /* don't stop this atom if it has an order dependency only to the failed one,
- try to submit it throught the normal path */
- if ( dep_atom_type == BASE_JD_DEP_TYPE_ORDER &&
- dep_atom->event_code > BASE_JD_EVENT_ACTIVE) {
- continue;
- }
+ if (!dep_atom_number)
+ continue;
- if (i == 1 && kbase_jd_katom_dep_atom(&katom->dep[0])) {
- /* Remove the previous dependency */
- list_del(&katom->dep_item[0]);
- kbase_jd_katom_dep_clear(&katom->dep[0]);
- }
+ if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED ||
+ dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
- /* Atom has completed, propagate the error code if any */
- katom->event_code = dep_atom->event_code;
- katom->status = KBASE_JD_ATOM_STATE_QUEUED;
- if ((katom->core_req &
- BASEP_JD_REQ_ATOM_TYPE)
- == BASE_JD_REQ_SOFT_REPLAY) {
- if (kbase_replay_process(katom)) {
- ret = MALI_FALSE;
- goto out;
- }
- }
- ret = jd_done_nolock(katom);
+ if (dep_atom->event_code == BASE_JD_EVENT_DONE)
+ continue;
+ /* don't stop this atom if it has an order dependency
+ * only to the failed one, try to submit it throught
+ * the normal path
+ */
+ if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER &&
+ dep_atom->event_code > BASE_JD_EVENT_ACTIVE) {
+ continue;
+ }
+ if (i == 1 && kbase_jd_katom_dep_atom(&katom->dep[0])) {
+ /* Remove the previous dependency */
+ list_del(&katom->dep_item[0]);
+ kbase_jd_katom_dep_clear(&katom->dep[0]);
+ }
+
+ /* Atom has completed, propagate the error code if any */
+ katom->event_code = dep_atom->event_code;
+ katom->status = KBASE_JD_ATOM_STATE_QUEUED;
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_tl_new_atom(
+ katom,
+ kbase_jd_atom_id(kctx, katom));
+ kbase_tlstream_tl_ret_atom_ctx(katom, kctx);
+#endif
+ if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE)
+ == BASE_JD_REQ_SOFT_REPLAY) {
+ if (kbase_replay_process(katom)) {
+ ret = false;
goto out;
}
- } else {
- /* Atom is in progress, add this atom to the list */
- list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]);
- kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type);
- queued = 1;
}
+ ret = jd_done_nolock(katom);
+
+ goto out;
+ } else {
+ /* Atom is in progress, add this atom to the list */
+ list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]);
+ kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type);
+ queued = 1;
}
}
katom->event_code = BASE_JD_EVENT_DONE;
katom->status = KBASE_JD_ATOM_STATE_QUEUED;
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_tl_new_atom(
+ katom,
+ kbase_jd_atom_id(kctx, katom));
+ kbase_tlstream_tl_ret_atom_ctx(katom, kctx);
+#endif
+
/* Reject atoms with job chain = NULL, as these cause issues with soft-stop */
- if (0 == katom->jc && (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
+ if (!katom->jc && (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
dev_warn(kctx->kbdev->dev, "Rejecting atom with jc = NULL");
katom->event_code = BASE_JD_EVENT_JOB_INVALID;
ret = jd_done_nolock(katom);
/* Reject atoms with an invalid device_nr */
if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) &&
(katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) {
- dev_warn(kctx->kbdev->dev, "Rejecting atom with invalid device_nr %d", katom->device_nr);
+ dev_warn(kctx->kbdev->dev,
+ "Rejecting atom with invalid device_nr %d",
+ katom->device_nr);
katom->event_code = BASE_JD_EVENT_JOB_INVALID;
ret = jd_done_nolock(katom);
goto out;
}
- /*
- * If the priority is increased we need to check the caller has security caps to do this, if
- * priority is decreased then this is ok as the result will have no negative impact on other
- * processes running.
- */
- if (0 > katom->nice_prio) {
- mali_bool access_allowed;
-
- access_allowed = kbase_security_has_capability(kctx,
- KBASE_SEC_MODIFY_PRIORITY, KBASE_SEC_FLAG_NOAUDIT);
-
- if (!access_allowed) {
- /* For unprivileged processes - a negative priority is interpreted as zero */
- katom->nice_prio = 0;
- }
- }
-
- /* Scale priority range to use NICE range */
- if (katom->nice_prio) {
- /* Remove sign for calculation */
- int nice_priority = katom->nice_prio + 128;
-
- /* Fixed point maths to scale from ..255 to 0..39 (NICE range with +20 offset) */
- katom->nice_prio = (((20 << 16) / 128) * nice_priority) >> 16;
- }
+ /* For invalid priority, be most lenient and choose the default */
+ sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio);
+ if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID)
+ sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT;
+ katom->sched_priority = sched_prio;
if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
/* handle what we need to do to access the external resources */
- if (MALI_ERROR_NONE != kbase_jd_pre_external_resources(katom, user_atom)) {
+ if (kbase_jd_pre_external_resources(katom, user_atom) != 0) {
/* setup failed (no access, bad resource, unknown resource types, etc.) */
katom->event_code = BASE_JD_EVENT_JOB_INVALID;
ret = jd_done_nolock(katom);
}
}
- /* Initialize the jobscheduler policy for this atom. Function will
- * return error if the atom is malformed.
+ /* Validate the atom. Function will return error if the atom is
+ * malformed.
*
* Soft-jobs never enter the job scheduler but have their own initialize method.
*
* If either fail then we immediately complete the atom with an error.
*/
if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) {
- union kbasep_js_policy *js_policy = &(kctx->kbdev->js_data.policy);
-
- if (MALI_ERROR_NONE != kbasep_js_policy_init_job(js_policy, kctx, katom)) {
+ if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) {
katom->event_code = BASE_JD_EVENT_JOB_INVALID;
ret = jd_done_nolock(katom);
goto out;
}
} else {
/* Soft-job */
- if (MALI_ERROR_NONE != kbase_prepare_soft_job(katom)) {
+ if (kbase_prepare_soft_job(katom) != 0) {
katom->event_code = BASE_JD_EVENT_JOB_INVALID;
ret = jd_done_nolock(katom);
goto out;
trace_gpu_job_enqueue((u32)kctx, katom->work_id, kbasep_map_core_reqs_to_string(katom->core_req));
#endif
- if (queued) {
- ret = MALI_FALSE;
+ if (queued && !IS_GPU_ATOM(katom)) {
+ ret = false;
goto out;
}
#ifdef CONFIG_KDS
if (!katom->kds_dep_satisfied) {
/* Queue atom due to KDS dependency */
- ret = MALI_FALSE;
+ ret = false;
goto out;
}
#endif /* CONFIG_KDS */
if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE)
== BASE_JD_REQ_SOFT_REPLAY) {
if (kbase_replay_process(katom))
- ret = MALI_FALSE;
+ ret = false;
else
ret = jd_done_nolock(katom);
}
/* The job has not yet completed */
list_add_tail(&katom->dep_item[0], &kctx->waiting_soft_jobs);
- ret = MALI_FALSE;
+ ret = false;
} else if ((katom->core_req & BASEP_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) {
katom->status = KBASE_JD_ATOM_STATE_IN_JS;
ret = kbasep_js_add_job(kctx, katom);
+ /* If job was cancelled then resolve immediately */
+ if (katom->event_code == BASE_JD_EVENT_JOB_CANCELLED)
+ ret = jd_done_nolock(katom);
} else {
/* This is a pure dependency. Resolve it immediately */
ret = jd_done_nolock(katom);
}
#ifdef BASE_LEGACY_UK6_SUPPORT
-mali_error kbase_jd_submit(struct kbase_context *kctx,
+int kbase_jd_submit(struct kbase_context *kctx,
const struct kbase_uk_job_submit *submit_data,
int uk6_atom)
#else
-mali_error kbase_jd_submit(struct kbase_context *kctx,
+int kbase_jd_submit(struct kbase_context *kctx,
const struct kbase_uk_job_submit *submit_data)
#endif /* BASE_LEGACY_UK6_SUPPORT */
{
struct kbase_jd_context *jctx = &kctx->jctx;
- mali_error err = MALI_ERROR_NONE;
+ int err = 0;
int i;
- mali_bool need_to_try_schedule_context = MALI_FALSE;
+ bool need_to_try_schedule_context = false;
struct kbase_device *kbdev;
void __user *user_addr;
if ((kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) != 0) {
dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
#ifdef BASE_LEGACY_UK6_SUPPORT
if (submit_data->stride != sizeof(base_jd_atom_v2)) {
#endif /* BASE_LEGACY_UK6_SUPPORT */
dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
user_addr = get_compat_pointer(kctx, &submit_data->addr);
if (copy_from_user(&user_atom_v6, user_addr,
sizeof(user_atom_v6))) {
- err = MALI_ERROR_FUNCTION_FAILED;
+ err = -EINVAL;
KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx,
atomic_sub_return(
submit_data->nr_atoms - i,
} else {
#endif /* BASE_LEGACY_UK6_SUPPORT */
if (copy_from_user(&user_atom, user_addr, sizeof(user_atom)) != 0) {
- err = MALI_ERROR_FUNCTION_FAILED;
+ err = -EINVAL;
KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(submit_data->nr_atoms - i, &kctx->timeline.jd_atoms_in_flight));
break;
}
user_addr = (void __user *)((uintptr_t) user_addr + submit_data->stride);
mutex_lock(&jctx->lock);
+#ifndef compiletime_assert
+#define compiletime_assert_defined
+#define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \
+while (false)
+#endif
+ compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) ==
+ BASE_JD_ATOM_COUNT,
+ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync");
+ compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) ==
+ sizeof(user_atom.atom_number),
+ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync");
+#ifdef compiletime_assert_defined
+#undef compiletime_assert
+#undef compiletime_assert_defined
+#endif
katom = &jctx->atoms[user_atom.atom_number];
while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) {
* complete
*/
mutex_unlock(&jctx->lock);
-
+
/* This thread will wait for the atom to complete. Due
* to thread scheduling we are not sure that the other
* thread that owns the atom will also schedule the
* hence eventually schedule this context at some point
* later.
*/
- kbasep_js_try_schedule_head_ctx(kctx->kbdev);
+ kbase_js_sched_all(kbdev);
+
if (wait_event_killable(katom->completed,
katom->status == KBASE_JD_ATOM_STATE_UNUSED)) {
/* We're being killed so the result code
* doesn't really matter
*/
- return MALI_ERROR_NONE;
+ return 0;
}
mutex_lock(&jctx->lock);
}
}
if (need_to_try_schedule_context)
- kbasep_js_try_schedule_head_ctx(kbdev);
+ kbase_js_sched_all(kbdev);
return err;
}
-KBASE_EXPORT_TEST_API(kbase_jd_submit)
-
-static void kbasep_jd_cacheclean(struct kbase_device *kbdev)
-{
- /* Limit the number of loops to avoid a hang if the interrupt is missed */
- u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
-
- mutex_lock(&kbdev->cacheclean_lock);
-
- /* use GPU_COMMAND completion solution */
- /* clean & invalidate the caches */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CLEAN_INV_CACHES, NULL);
-
- /* wait for cache flush to complete before continuing */
- while (--max_loops && (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & CLEAN_CACHES_COMPLETED) == 0)
- ;
-
- /* clear the CLEAN_CACHES_COMPLETED irq */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, CLEAN_CACHES_COMPLETED);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), CLEAN_CACHES_COMPLETED, NULL);
- KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.state != KBASE_INSTR_STATE_CLEANING,
- "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang.");
-
- mutex_unlock(&kbdev->cacheclean_lock);
-}
+KBASE_EXPORT_TEST_API(kbase_jd_submit);
/**
- * This function:
- * - requeues the job from the runpool (if it was soft-stopped/removed from NEXT registers)
- * - removes it from the system if it finished/failed/was cancelled.
- * - resolves dependencies to add dependent jobs to the context, potentially starting them if necessary (which may add more references to the context)
- * - releases the reference to the context from the no-longer-running job.
- * - Handles retrying submission outside of IRQ context if it failed from within IRQ context.
+ * jd_done_worker - Handle a job completion
+ * @data: a &struct work_struct
+ *
+ * This function requeues the job from the runpool (if it was soft-stopped or
+ * removed from NEXT registers).
+ *
+ * Removes it from the system if it finished/failed/was cancelled.
+ *
+ * Resolves dependencies to add dependent jobs to the context, potentially
+ * starting them if necessary (which may add more references to the context)
+ *
+ * Releases the reference to the context from the no-longer-running job.
+ *
+ * Handles retrying submission outside of IRQ context if it failed from within
+ * IRQ context.
*/
static void jd_done_worker(struct work_struct *data)
{
struct kbasep_js_device_data *js_devdata;
u64 cache_jc = katom->jc;
struct kbasep_js_atom_retained_state katom_retained_state;
+ bool schedule = false;
/* Soft jobs should never reach this function */
KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);
jctx = &kctx->jctx;
kbdev = kctx->kbdev;
js_kctx_info = &kctx->jctx.sched_info;
-
js_devdata = &kbdev->js_data;
js_policy = &kbdev->js_data.policy;
KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0);
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) && katom->event_code != BASE_JD_EVENT_DONE && !(katom->event_code & BASE_JD_SW_EVENT))
- kbasep_jd_cacheclean(kbdev); /* cache flush when jobs complete with non-done codes */
- else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) {
- if (kbdev->gpu_props.num_core_groups > 1 &&
- !(katom->affinity & kbdev->gpu_props.props.coherency_info.group[0].core_mask) &&
- (katom->affinity & kbdev->gpu_props.props.coherency_info.group[1].core_mask)) {
- dev_dbg(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n");
- kbasep_jd_cacheclean(kbdev);
- }
- }
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) &&
- (katom->core_req & BASE_JD_REQ_FS) &&
- katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT &&
- (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) &&
- !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) {
- dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n");
- if (kbasep_10969_workaround_clamp_coordinates(katom)) {
- /* The job had a TILE_RANGE_FAULT after was soft-stopped.
- * Due to an HW issue we try to execute the job
- * again.
- */
- dev_dbg(kbdev->dev, "Clamping has been executed, try to rerun the job\n");
- katom->event_code = BASE_JD_EVENT_STOPPED;
- katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN;
-
- /* The atom will be requeued, but requeing does not submit more
- * jobs. If this was the last job, we must also ensure that more
- * jobs will be run on slot 0 - this is a Fragment job. */
- kbasep_js_set_job_retry_submit_slot(katom, 0);
- }
- }
+ kbase_backend_complete_wq(kbdev, katom);
/*
* Begin transaction on JD context and JS context
*/
mutex_lock(&jctx->lock);
+ mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
/* This worker only gets called on contexts that are scheduled *in*. This is
* because it only happens in response to an IRQ from a job that was
* running.
*/
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled != MALI_FALSE);
-
- /* If job was rejected due to BASE_JD_EVENT_PM_EVENT but was not
- * specifically targeting core group 1, then re-submit targeting core
- * group 0 */
- if (katom->event_code == BASE_JD_EVENT_PM_EVENT && !(katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) {
- katom->event_code = BASE_JD_EVENT_STOPPED;
- /* Don't need to worry about any previously set retry-slot - it's
- * impossible for it to have been set previously, because we guarantee
- * kbase_jd_done() was called with done_code==0 on this atom */
- kbasep_js_set_job_retry_submit_slot(katom, 1);
+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled);
+
+ if (katom->event_code == BASE_JD_EVENT_STOPPED) {
+ /* Atom has been promoted to stopped */
+ unsigned long flags;
+
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_unlock(&js_devdata->queue_mutex);
+ mutex_unlock(&jctx->lock);
+
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ katom->status = KBASE_JD_ATOM_STATE_IN_JS;
+ kbase_js_unpull(kctx, katom);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ return;
}
+ if (katom->event_code != BASE_JD_EVENT_DONE)
+ dev_err(kbdev->dev,
+ "t6xx: GPU fault 0x%02lx from job slot %d\n",
+ (unsigned long)katom->event_code,
+ katom->slot_nr);
+
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
kbase_as_poking_timer_release_atom(kbdev, kctx, katom);
- /* Release cores this job was using (this might power down unused cores, and
- * cause extra latency if a job submitted here - such as depenedent jobs -
- * would use those cores) */
- kbasep_js_job_check_deref_cores(kbdev, katom);
-
/* Retain state before the katom disappears */
kbasep_js_atom_retained_state_copy(&katom_retained_state, katom);
if (!kbasep_js_has_atom_finished(&katom_retained_state)) {
- unsigned long flags;
-
- /* Requeue the atom on soft-stop / removed from NEXT registers */
- dev_dbg(kbdev->dev, "JS: Soft Stopped/Removed from next on Ctx %p; Requeuing", kctx);
-
mutex_lock(&js_devdata->runpool_mutex);
kbasep_js_clear_job_retry_submit(katom);
/* An atom that has been hard-stopped might have previously
* been soft-stopped and has just finished before the hard-stop
* occurred. For this reason, clear the hard-stopped flag */
katom->atom_flags &= ~(KBASE_KATOM_FLAG_BEEN_HARD_STOPPED);
+ mutex_unlock(&js_devdata->runpool_mutex);
+ }
- KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, katom));
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- kbasep_js_policy_enqueue_job(js_policy, katom);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
-
- /* A STOPPED/REMOVED job must cause a re-submit to happen, in case it
- * was the last job left. Crucially, work items on work queues can run
- * out of order e.g. on different CPUs, so being able to submit from
- * the IRQ handler is not a good indication that we don't need to run
- * jobs; the submitted job could be processed on the work-queue
- * *before* the stopped job, even though it was submitted after. */
- {
- int tmp;
- KBASE_DEBUG_ASSERT(kbasep_js_get_atom_retry_submit_slot(&katom_retained_state, &tmp) != MALI_FALSE);
- CSTD_UNUSED(tmp);
- }
+ if (kbasep_js_has_atom_finished(&katom_retained_state))
+ schedule = true;
- mutex_unlock(&js_devdata->runpool_mutex);
- mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
- } else {
- /* Remove the job from the system for all other reasons */
- mali_bool need_to_try_schedule_context;
+ kbase_js_complete_atom_wq(kctx, katom);
- kbasep_js_remove_job(kbdev, kctx, katom);
- mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
- /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */
+ KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state));
- need_to_try_schedule_context = jd_done_nolock(katom);
+ kbasep_js_remove_job(kbdev, kctx, katom);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_unlock(&js_devdata->queue_mutex);
+ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF;
+ /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */
+ schedule |= jd_done_nolock(katom);
- /* This ctx is already scheduled in, so return value guarenteed FALSE */
- KBASE_DEBUG_ASSERT(need_to_try_schedule_context == MALI_FALSE);
- }
/* katom may have been freed now, do not use! */
/*
/* Job is now no longer running, so can now safely release the context
* reference, and handle any actions that were logged against the atom's retained state */
+
kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state);
+ if (schedule)
+ kbase_js_sched_all(kbdev);
+
KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0);
}
/**
- * Work queue job cancel function
- * Only called as part of 'Zapping' a context (which occurs on termination)
+ * jd_cancel_worker - Work queue job cancel function.
+ * @data: a &struct work_struct
+ *
+ * Only called as part of 'Zapping' a context (which occurs on termination).
* Operates serially with the jd_done_worker() on the work queue.
*
* This can only be called on contexts that aren't scheduled.
*
- * @note We don't need to release most of the resources that would occur on
+ * We don't need to release most of the resources that would occur on
* kbase_jd_done() or jd_done_worker(), because the atoms here must not be
* running (by virtue of only being called on contexts that aren't
- * scheduled). The only resources that are an exception to this are:
- * - those held by kbasep_js_job_check_ref_cores(), because these resources are
- * held for non-running atoms as well as running atoms.
+ * scheduled).
*/
static void jd_cancel_worker(struct work_struct *data)
{
struct kbase_jd_context *jctx;
struct kbase_context *kctx;
struct kbasep_js_kctx_info *js_kctx_info;
- mali_bool need_to_try_schedule_context;
+ bool need_to_try_schedule_context;
+ bool attr_state_changed;
struct kbase_device *kbdev;
/* Soft jobs should never reach this function */
* any), nor must we try to schedule out the context (it's already
* scheduled out).
*/
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled == MALI_FALSE);
-
- /* Release cores this job was using (this might power down unused cores) */
- kbasep_js_job_check_deref_cores(kctx->kbdev, katom);
+ KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
/* Scheduler: Remove the job from the system */
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
- kbasep_js_remove_cancelled_job(kbdev, kctx, katom);
+ attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&jctx->lock);
/* Because we're zapping, we're not adding any more jobs to this ctx, so no need to
* schedule the context. There's also no need for the jsctx_mutex to have been taken
* around this too. */
- KBASE_DEBUG_ASSERT(need_to_try_schedule_context == MALI_FALSE);
+ KBASE_DEBUG_ASSERT(!need_to_try_schedule_context);
/* katom may have been freed now, do not use! */
mutex_unlock(&jctx->lock);
+
+ if (attr_state_changed)
+ kbase_js_sched_all(kbdev);
}
/**
- * @brief Complete a job that has been removed from the Hardware
+ * jd_evict_worker - Work queue job evict function
+ * @data: a &struct work_struct
*
- * This must be used whenever a job has been removed from the Hardware, e.g.:
- * - An IRQ indicates that the job finished (for both error and 'done' codes)
- * - The job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop.
+ * Only called as part of evicting failed jobs. This is only called on jobs that
+ * were never submitted to HW Access. Jobs that were submitted are handled
+ * through jd_done_worker().
+ * Operates serially with the jd_done_worker() on the work queue.
*
- * Some work is carried out immediately, and the rest is deferred onto a workqueue
+ * We don't need to release most of the resources that would occur on
+ * kbase_jd_done() or jd_done_worker(), because the atoms here must not be
+ * running (by virtue of having not been submitted to HW Access).
+ */
+static void jd_evict_worker(struct work_struct *data)
+{
+ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom,
+ work);
+ struct kbase_jd_context *jctx;
+ struct kbase_context *kctx;
+ struct kbasep_js_kctx_info *js_kctx_info;
+ struct kbase_device *kbdev;
+
+ /* Soft jobs should never reach this function */
+ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);
+
+ kctx = katom->kctx;
+ kbdev = kctx->kbdev;
+ jctx = &kctx->jctx;
+ js_kctx_info = &kctx->jctx.sched_info;
+
+ KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0);
+
+ /* Scheduler: Remove the job from the system */
+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
+ kbasep_js_remove_cancelled_job(kbdev, kctx, katom);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+
+ mutex_lock(&jctx->lock);
+ jd_done_nolock(katom);
+ /* katom may have been freed now, do not use! */
+ mutex_unlock(&jctx->lock);
+
+ kbase_js_sched_all(kbdev);
+}
+
+/**
+ * kbase_jd_done - Complete a job that has been removed from the Hardware
+ * @katom: atom which has been completed
+ * @slot_nr: slot the atom was on
+ * @end_timestamp: completion time
+ * @done_code: completion code
*
- * This can be called safely from atomic context.
+ * This must be used whenever a job has been removed from the Hardware, e.g.:
+ * An IRQ indicates that the job finished (for both error and 'done' codes), or
+ * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop.
*
- * The caller must hold kbasep_js_device_data::runpool_irq::lock
+ * Some work is carried out immediately, and the rest is deferred onto a
+ * workqueue
*
+ * Context:
+ * This can be called safely from atomic context.
+ * The caller must hold kbasep_js_device_data.runpool_irq.lock
*/
-void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp,
- kbasep_js_atom_done_code done_code)
+void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr,
+ ktime_t *end_timestamp, kbasep_js_atom_done_code done_code)
{
struct kbase_context *kctx;
struct kbase_device *kbdev;
if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT)
katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
- kbase_timeline_job_slot_done(kbdev, kctx, katom, slot_nr, done_code);
-
KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0);
kbase_job_check_leave_disjoint(kbdev, katom);
- kbasep_js_job_done_slot_irq(katom, slot_nr, end_timestamp, done_code);
katom->slot_nr = slot_nr;
+ WARN_ON(work_pending(&katom->work));
+
KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
INIT_WORK(&katom->work, jd_done_worker);
queue_work(kctx->jctx.job_done_wq, &katom->work);
}
-KBASE_EXPORT_TEST_API(kbase_jd_done)
+KBASE_EXPORT_TEST_API(kbase_jd_done);
void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
{
KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);
/* This should only be done from a context that is not scheduled */
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled == MALI_FALSE);
+ KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
+
+ WARN_ON(work_pending(&katom->work));
katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
queue_work(kctx->jctx.job_done_wq, &katom->work);
}
-struct zap_reset_data {
- /* The stages are:
- * 1. The timer has never been called
- * 2. The zap has timed out, all slots are soft-stopped - the GPU reset will happen.
- * The GPU has been reset when kbdev->reset_waitq is signalled
- *
- * (-1 - The timer has been cancelled)
- */
- int stage;
- struct kbase_device *kbdev;
- struct hrtimer timer;
- spinlock_t lock; /* protects updates to stage member */
-};
-
-static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer)
+void kbase_jd_evict(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
{
- struct zap_reset_data *reset_data = container_of(timer, struct zap_reset_data, timer);
- struct kbase_device *kbdev = reset_data->kbdev;
- unsigned long flags;
+ struct kbase_context *kctx;
+ struct kbasep_js_kctx_info *js_kctx_info;
- spin_lock_irqsave(&reset_data->lock, flags);
+ KBASE_DEBUG_ASSERT(NULL != kbdev);
+ KBASE_DEBUG_ASSERT(NULL != katom);
+ kctx = katom->kctx;
+ KBASE_DEBUG_ASSERT(NULL != kctx);
- if (reset_data->stage == -1)
- goto out;
+ js_kctx_info = &kctx->jctx.sched_info;
-#if KBASE_GPU_RESET_EN
- if (kbase_prepare_to_reset_gpu(kbdev)) {
- dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", ZAP_TIMEOUT);
- kbase_reset_gpu(kbdev);
- }
-#endif /* KBASE_GPU_RESET_EN */
- reset_data->stage = 2;
+ KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);
- out:
- spin_unlock_irqrestore(&reset_data->lock, flags);
+ /* This should only be done from a context that is currently scheduled
+ */
+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled);
+
+ WARN_ON(work_pending(&katom->work));
- return HRTIMER_NORESTART;
+ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
+ INIT_WORK(&katom->work, jd_evict_worker);
+ queue_work(kctx->jctx.job_done_wq, &katom->work);
}
void kbase_jd_zap_context(struct kbase_context *kctx)
struct kbase_jd_atom *katom;
struct list_head *entry, *tmp;
struct kbase_device *kbdev;
- struct zap_reset_data reset_data;
- unsigned long flags;
KBASE_DEBUG_ASSERT(kctx);
kbdev = kctx->kbdev;
KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u);
- kbase_job_zap_context(kctx);
+
+ kbase_js_zap_context(kctx);
+ kbase_jm_wait_for_zero_jobs(kctx);
mutex_lock(&kctx->jctx.lock);
#endif
mutex_unlock(&kctx->jctx.lock);
-
- hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- reset_data.timer.function = zap_timeout_callback;
-
- spin_lock_init(&reset_data.lock);
-
- reset_data.kbdev = kbdev;
- reset_data.stage = 1;
-
- hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT), HRTIMER_MODE_REL);
-
- /* Wait for all jobs to finish, and for the context to be not-scheduled
- * (due to kbase_job_zap_context(), we also guarentee it's not in the JS
- * policy queue either */
- wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0);
- wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, kctx->jctx.sched_info.ctx.is_scheduled == MALI_FALSE);
-
- spin_lock_irqsave(&reset_data.lock, flags);
- if (reset_data.stage == 1) {
- /* The timer hasn't run yet - so cancel it */
- reset_data.stage = -1;
- }
- spin_unlock_irqrestore(&reset_data.lock, flags);
-
- hrtimer_cancel(&reset_data.timer);
-
- if (reset_data.stage == 2) {
- /* The reset has already started.
- * Wait for the reset to complete
- */
- wait_event(kbdev->reset_wait, atomic_read(&kbdev->reset_gpu) == KBASE_RESET_GPU_NOT_PENDING);
- }
- destroy_hrtimer_on_stack(&reset_data.timer);
-
- dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx);
-
- /* Ensure that the signallers of the waitqs have finished */
- mutex_lock(&kctx->jctx.lock);
- mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
- mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
- mutex_unlock(&kctx->jctx.lock);
}
-KBASE_EXPORT_TEST_API(kbase_jd_zap_context)
+KBASE_EXPORT_TEST_API(kbase_jd_zap_context);
-mali_error kbase_jd_init(struct kbase_context *kctx)
+int kbase_jd_init(struct kbase_context *kctx)
{
int i;
- mali_error mali_err = MALI_ERROR_NONE;
+ int mali_err = 0;
#ifdef CONFIG_KDS
int err;
#endif /* CONFIG_KDS */
kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", 0, 1);
if (NULL == kctx->jctx.job_done_wq) {
- mali_err = MALI_ERROR_OUT_OF_MEMORY;
+ mali_err = -ENOMEM;
goto out1;
}
#ifdef CONFIG_KDS
err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear);
if (0 != err) {
- mali_err = MALI_ERROR_FUNCTION_FAILED;
+ mali_err = -EINVAL;
goto out2;
}
#endif /* CONFIG_KDS */
kctx->jctx.job_nr = 0;
- return MALI_ERROR_NONE;
+ return 0;
#ifdef CONFIG_KDS
out2:
return mali_err;
}
-KBASE_EXPORT_TEST_API(kbase_jd_init)
+KBASE_EXPORT_TEST_API(kbase_jd_init);
void kbase_jd_exit(struct kbase_context *kctx)
{
destroy_workqueue(kctx->jctx.job_done_wq);
}
-KBASE_EXPORT_TEST_API(kbase_jd_exit)
+KBASE_EXPORT_TEST_API(kbase_jd_exit);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#ifdef CONFIG_DEBUG_FS
/**
- * @brief Show callback for the @c JD atoms debugfs file.
+ * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file.
+ * @sfile: The debugfs entry
+ * @data: Data associated with the entry
*
- * This function is called to get the contents of the @c JD atoms debugfs file.
- * This is a report of all atoms managed by kbase_jd_context::atoms .
+ * This function is called to get the contents of the JD atoms debugfs file.
+ * This is a report of all atoms managed by kbase_jd_context.atoms
*
- * @param sfile The debugfs entry
- * @param data Data associated with the entry
- *
- * @return 0 if successfully prints data in debugfs entry file, failure
+ * Return: 0 if successfully prints data in debugfs entry file, failure
* otherwise
*/
static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data)
seq_printf(sfile,
"%i,%u,%u,%u,%u %u,%lli,%llu\n",
i, atom->core_req, atom->status, atom->coreref_state,
- atom->dep[0].atom ? atom->dep[0].atom - atoms : 0,
- atom->dep[1].atom ? atom->dep[1].atom - atoms : 0,
+ (unsigned)(atom->dep[0].atom ?
+ atom->dep[0].atom - atoms : 0),
+ (unsigned)(atom->dep[1].atom ?
+ atom->dep[1].atom - atoms : 0),
(signed long long)start_timestamp,
(unsigned long long)(atom->time_spent_us ?
atom->time_spent_us * 1000 : start_timestamp)
/**
- * @brief File operations related to debugfs entry for atoms
+ * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file
+ * @in: &struct inode pointer
+ * @file: &struct file pointer
+ *
+ * Return: file descriptor
*/
static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file)
{
.release = single_release,
};
-
-int kbasep_jd_debugfs_init(struct kbase_device *kbdev)
-{
- kbdev->jd_directory = debugfs_create_dir(
- "jd", kbdev->mali_debugfs_directory);
- if (IS_ERR(kbdev->jd_directory)) {
- dev_err(kbdev->dev, "Couldn't create mali jd debugfs directory\n");
- goto err;
- }
-
- return 0;
-
-err:
- return -1;
-}
-
-
-void kbasep_jd_debugfs_term(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- if (!IS_ERR(kbdev->jd_directory))
- debugfs_remove_recursive(kbdev->jd_directory);
-}
-
-
-int kbasep_jd_debugfs_ctx_add(struct kbase_context *kctx)
+void kbasep_jd_debugfs_ctx_add(struct kbase_context *kctx)
{
- /* Refer below for format string, %u is 10 chars max */
- char dir_name[10 * 2 + 2];
-
KBASE_DEBUG_ASSERT(kctx != NULL);
- /* Create per-context directory */
- scnprintf(dir_name, sizeof(dir_name), "%u_%u", kctx->pid, kctx->id);
- kctx->jd_ctx_dir = debugfs_create_dir(dir_name, kctx->kbdev->jd_directory);
- if (IS_ERR(kctx->jd_ctx_dir))
- goto err;
-
/* Expose all atoms */
- if (IS_ERR(debugfs_create_file("atoms", S_IRUGO,
- kctx->jd_ctx_dir, kctx, &kbasep_jd_debugfs_atoms_fops)))
- goto err_jd_ctx_dir;
-
- return 0;
+ debugfs_create_file("atoms", S_IRUGO, kctx->kctx_dentry, kctx,
+ &kbasep_jd_debugfs_atoms_fops);
-err_jd_ctx_dir:
- debugfs_remove_recursive(kctx->jd_ctx_dir);
-err:
- return -1;
-}
-
-
-void kbasep_jd_debugfs_ctx_remove(struct kbase_context *kctx)
-{
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- if (!IS_ERR(kctx->jd_ctx_dir))
- debugfs_remove_recursive(kctx->jd_ctx_dir);
-}
-
-#else /* CONFIG_DEBUG_FS */
-
-/**
- * @brief Stub functions for when debugfs is disabled
- */
-int kbasep_jd_debugfs_init(struct kbase_device *kbdev)
-{
- return 0;
-}
-void kbasep_jd_debugfs_term(struct kbase_device *kbdev)
-{
-}
-int kbasep_jd_debugfs_ctx_add(struct kbase_context *ctx)
-{
- return 0;
-}
-void kbasep_jd_debugfs_ctx_remove(struct kbase_context *ctx)
-{
}
#endif /* CONFIG_DEBUG_FS */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase.h>
/**
- * @brief Initialize JD debugfs entries
+ * kbasep_jd_debugfs_ctx_add() - Add debugfs entries for JD system
*
- * This should be called during device probing after the main mali debugfs
- * directory has been created.
- *
- * @param[in] kbdev Pointer to kbase_device
- */
-int kbasep_jd_debugfs_init(struct kbase_device *kbdev);
-
-/**
- * @brief Clean up all JD debugfs entries and related data
- *
- * This should be called during device removal before the main mali debugfs
- * directory will be removed.
- *
- * @param[in] kbdev Pointer to kbase_device
- */
-void kbasep_jd_debugfs_term(struct kbase_device *kbdev);
-
-/**
- * @brief Add new entry to JD debugfs
- *
- * @param[in] kctx Pointer to kbase_context
- *
- * @return 0 on success, failure otherwise
- */
-int kbasep_jd_debugfs_ctx_add(struct kbase_context *kctx);
-
-/**
- * @brief Remove entry from JD debugfs
- *
- * param[in] kctx Pointer to kbase_context
+ * @kctx Pointer to kbase_context
*/
-void kbasep_jd_debugfs_ctx_remove(struct kbase_context *kctx);
+void kbasep_jd_debugfs_ctx_add(struct kbase_context *kctx);
#endif /*_KBASE_JD_DEBUGFS_H*/
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-
-/**
- * @file mali_kbase_jm.c
- * Base kernel job manager APIs
+/*
+ * HW access job manager common APIs
*/
#include <mali_kbase.h>
-#include <mali_kbase_config.h>
-#include <mali_midg_regmap.h>
-#include <mali_kbase_gator.h>
-#include <mali_kbase_js_affinity.h>
-#include <mali_kbase_hw.h>
-
+#include "mali_kbase_hwaccess_jm.h"
#include "mali_kbase_jm.h"
-#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
-
-#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
-u64 mali_js0_affinity_mask = 0xFFFFFFFFFFFFFFFFULL;
-u64 mali_js1_affinity_mask = 0xFFFFFFFFFFFFFFFFULL;
-u64 mali_js2_affinity_mask = 0xFFFFFFFFFFFFFFFFULL;
-#endif
-
-#if KBASE_GPU_RESET_EN
-static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev);
-#endif /* KBASE_GPU_RESET_EN */
-
-#ifdef CONFIG_GPU_TRACEPOINTS
-static char *kbasep_make_job_slot_string(int js, char *js_string)
-{
- sprintf(js_string, "job_slot_%i", js);
- return js_string;
-}
-#endif
-
-static void kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, int js)
-{
- struct kbase_context *kctx;
- u32 cfg;
- u64 jc_head = katom->jc;
-
- KBASE_DEBUG_ASSERT(kbdev);
- KBASE_DEBUG_ASSERT(katom);
-
- kctx = katom->kctx;
-
- /* Command register must be available */
- KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx));
- /* Affinity is not violating */
- kbase_js_debug_log_current_affinities(kbdev);
- KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js, katom->affinity));
-
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), jc_head & 0xFFFFFFFF, kctx);
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), jc_head >> 32, kctx);
-
-#ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
- {
- u64 mask;
- u32 value;
-
- if (0 == js)
- {
- mask = mali_js0_affinity_mask;
- }
- else if (1 == js)
- {
- mask = mali_js1_affinity_mask;
- }
- else
- {
- mask = mali_js2_affinity_mask;
- }
-
- value = katom->affinity & (mask & 0xFFFFFFFF);
-
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), value, kctx);
-
- value = (katom->affinity >> 32) & ((mask>>32) & 0xFFFFFFFF);
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI), value, kctx);
- }
-#else
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), katom->affinity & 0xFFFFFFFF, kctx);
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI), katom->affinity >> 32, kctx);
-#endif
-
- /* start MMU, medium priority, cache clean/flush on end, clean/flush on start */
- cfg = kctx->as_nr | JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE | JS_CONFIG_START_MMU | JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE | JS_CONFIG_THREAD_PRI(8);
-
- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
- if (!kbdev->jm_slots[js].job_chain_flag) {
- cfg |= JS_CONFIG_JOB_CHAIN_FLAG;
- katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
- kbdev->jm_slots[js].job_chain_flag = MALI_TRUE;
- } else {
- katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
- kbdev->jm_slots[js].job_chain_flag = MALI_FALSE;
- }
- }
-
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx);
-
- /* Write an approximate start timestamp.
- * It's approximate because there might be a job in the HEAD register. In
- * such cases, we'll try to make a better approximation in the IRQ handler
- * (up to the KBASE_JS_IRQ_THROTTLE_TIME_US). */
- katom->start_timestamp = ktime_get();
-
- /* GO ! */
- dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx", katom, kctx, js, jc_head, katom->affinity);
-
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, (u32) katom->affinity);
-
-#ifdef CONFIG_MALI_GATOR_SUPPORT
- kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js), kctx, kbase_jd_atom_id(kctx, katom));
-#endif /* CONFIG_MALI_GATOR_SUPPORT */
-#ifdef CONFIG_GPU_TRACEPOINTS
- if (kbasep_jm_nr_jobs_submitted(&kbdev->jm_slots[js]) == 1)
- {
- /* If this is the only job on the slot, trace it as starting */
- char js_string[16];
- trace_gpu_sched_switch(kbasep_make_job_slot_string(js, js_string), ktime_to_ns(katom->start_timestamp), (u32)katom->kctx, 0, katom->work_id);
- kbdev->jm_slots[js].last_context = katom->kctx;
- }
-#endif
- kbase_timeline_job_slot_submit(kbdev, kctx, katom, js);
-
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_START, katom->kctx);
-}
-
-void kbase_job_submit_nolock(struct kbase_device *kbdev, struct kbase_jd_atom *katom, int js)
-{
- struct kbase_jm_slot *jm_slots;
-#if KBASE_PM_EN
- base_jd_core_req core_req;
-#endif
- KBASE_DEBUG_ASSERT(kbdev);
- KBASE_DEBUG_ASSERT(katom);
-
- jm_slots = kbdev->jm_slots;
-
-#if KBASE_PM_EN
- core_req = katom->core_req;
- if (core_req & BASE_JD_REQ_ONLY_COMPUTE) {
- unsigned long flags;
- int device_nr = (core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) ? katom->device_nr : 0;
- KBASE_DEBUG_ASSERT(device_nr < 2);
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbasep_pm_record_job_status(kbdev);
- kbdev->pm.metrics.active_cl_ctx[device_nr]++;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
- } else {
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbasep_pm_record_job_status(kbdev);
- kbdev->pm.metrics.active_gl_ctx++;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
- }
-#endif
-
- /*
- * We can have:
- * - one job already done (pending interrupt),
- * - one running,
- * - one ready to be run.
- * Hence a maximum of 3 inflight jobs. We have a 4 job
- * queue, which I hope will be enough...
- */
- kbasep_jm_enqueue_submit_slot(&jm_slots[js], katom);
- kbase_job_hw_submit(kbdev, katom, js);
-}
-
-void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, u64 job_tail, ktime_t *end_timestamp)
-{
- struct kbase_jm_slot *slot;
- struct kbase_jd_atom *katom;
- mali_addr64 jc_head;
- struct kbase_context *kctx;
-
- KBASE_DEBUG_ASSERT(kbdev);
-
- if (completion_code != BASE_JD_EVENT_DONE && completion_code != BASE_JD_EVENT_STOPPED)
- dev_err(kbdev->dev, "GPU fault 0x%02lx from job slot %d\n", (unsigned long)completion_code, s);
-
- /* IMPORTANT: this function must only contain work necessary to complete a
- * job from a Real IRQ (and not 'fake' completion, e.g. from
- * Soft-stop). For general work that must happen no matter how the job was
- * removed from the hardware, place it in kbase_jd_done() */
-
- slot = &kbdev->jm_slots[s];
- katom = kbasep_jm_dequeue_submit_slot(slot);
-
- /* If the katom completed is because it's a dummy job for HW workarounds, then take no further action */
- if (kbasep_jm_is_dummy_workaround_job(kbdev, katom)) {
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, NULL, NULL, 0, s, completion_code);
- return;
- }
-
- jc_head = katom->jc;
- kctx = katom->kctx;
-
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, jc_head, s, completion_code);
-
- if (completion_code != BASE_JD_EVENT_DONE && completion_code != BASE_JD_EVENT_STOPPED) {
-
-#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
- KBASE_TRACE_DUMP(kbdev);
-#endif
- }
- if (job_tail != 0) {
- mali_bool was_updated = (job_tail != jc_head);
- /* Some of the job has been executed, so we update the job chain address to where we should resume from */
- katom->jc = job_tail;
- if (was_updated)
- KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, kctx, katom, job_tail, s);
- }
-
- /* Only update the event code for jobs that weren't cancelled */
- if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
- katom->event_code = (base_jd_event_code) completion_code;
-
- kbase_device_trace_register_access(kctx, REG_WRITE, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 1 << s);
-
- /* Complete the job, and start new ones
- *
- * Also defer remaining work onto the workqueue:
- * - Re-queue Soft-stopped jobs
- * - For any other jobs, queue the job back into the dependency system
- * - Schedule out the parent context if necessary, and schedule a new one in.
- */
-#ifdef CONFIG_GPU_TRACEPOINTS
- if (kbasep_jm_nr_jobs_submitted(slot) != 0) {
- struct kbase_jd_atom *katom;
- char js_string[16];
- katom = kbasep_jm_peek_idx_submit_slot(slot, 0); /* The atom in the HEAD */
- trace_gpu_sched_switch(kbasep_make_job_slot_string(s, js_string), ktime_to_ns(*end_timestamp), (u32)katom->kctx, 0, katom->work_id);
- slot->last_context = katom->kctx;
- } else {
- char js_string[16];
- trace_gpu_sched_switch(kbasep_make_job_slot_string(s, js_string), ktime_to_ns(ktime_get()), 0, 0, 0);
- slot->last_context = 0;
- }
-#endif
- kbase_jd_done(katom, s, end_timestamp, KBASE_JS_ATOM_DONE_START_NEW_ATOMS);
-}
-
/**
- * Update the start_timestamp of the job currently in the HEAD, based on the
- * fact that we got an IRQ for the previous set of completed jobs.
+ * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot
+ * @js on the active context.
+ * @kbdev: Device pointer
+ * @js: Job slot to run on
+ * @nr_jobs_to_submit: Number of jobs to attempt to submit
*
- * The estimate also takes into account the KBASE_JS_IRQ_THROTTLE_TIME_US and
- * the time the job was submitted, to work out the best estimate (which might
- * still result in an over-estimate to the calculated time spent)
+ * Return: true if slot can still be submitted on, false if slot is now full.
*/
-STATIC void kbasep_job_slot_update_head_start_timestamp(struct kbase_device *kbdev, struct kbase_jm_slot *slot, ktime_t end_timestamp)
-{
- KBASE_DEBUG_ASSERT(slot);
-
- if (kbasep_jm_nr_jobs_submitted(slot) > 0) {
- struct kbase_jd_atom *katom;
- ktime_t new_timestamp;
- ktime_t timestamp_diff;
- katom = kbasep_jm_peek_idx_submit_slot(slot, 0); /* The atom in the HEAD */
-
- KBASE_DEBUG_ASSERT(katom != NULL);
-
- if (kbasep_jm_is_dummy_workaround_job(kbdev, katom) != MALI_FALSE) {
- /* Don't access the members of HW workaround 'dummy' jobs */
- return;
- }
-
- /* Account for any IRQ Throttle time - makes an overestimate of the time spent by the job */
- new_timestamp = ktime_sub_ns(end_timestamp, KBASE_JS_IRQ_THROTTLE_TIME_US * 1000);
- timestamp_diff = ktime_sub(new_timestamp, katom->start_timestamp);
- if (ktime_to_ns(timestamp_diff) >= 0) {
- /* Only update the timestamp if it's a better estimate than what's currently stored.
- * This is because our estimate that accounts for the throttle time may be too much
- * of an overestimate */
- katom->start_timestamp = new_timestamp;
- }
- }
-}
-
-void kbase_job_done(struct kbase_device *kbdev, u32 done)
+static bool kbase_jm_next_job(struct kbase_device *kbdev, int js,
+ int nr_jobs_to_submit)
{
- unsigned long flags;
+ struct kbase_context *kctx;
int i;
- u32 count = 0;
- ktime_t end_timestamp = ktime_get();
- struct kbasep_js_device_data *js_devdata;
-
- KBASE_DEBUG_ASSERT(kbdev);
- js_devdata = &kbdev->js_data;
-
- KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done);
-
- memset(&kbdev->slot_submit_count_irq[0], 0, sizeof(kbdev->slot_submit_count_irq));
-
- /* write irq throttle register, this will prevent irqs from occurring until
- * the given number of gpu clock cycles have passed */
- {
- int irq_throttle_cycles = atomic_read(&kbdev->irq_throttle_cycles);
- kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE), irq_throttle_cycles, NULL);
- }
-
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
-
- while (done) {
- struct kbase_jm_slot *slot;
- u32 failed = done >> 16;
-
- /* treat failed slots as finished slots */
- u32 finished = (done & 0xFFFF) | failed;
-
- /* Note: This is inherently unfair, as we always check
- * for lower numbered interrupts before the higher
- * numbered ones.*/
- i = ffs(finished) - 1;
- KBASE_DEBUG_ASSERT(i >= 0);
- slot = &kbdev->jm_slots[i];
+ kctx = kbdev->hwaccess.active_kctx;
- do {
- int nr_done;
- u32 active;
- u32 completion_code = BASE_JD_EVENT_DONE; /* assume OK */
- u64 job_tail = 0;
+ if (!kctx)
+ return true;
- if (failed & (1u << i)) {
- /* read out the job slot status code if the job slot reported failure */
- completion_code = kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS), NULL);
+ for (i = 0; i < nr_jobs_to_submit; i++) {
+ struct kbase_jd_atom *katom = kbase_js_pull(kctx, js);
- switch (completion_code) {
- case BASE_JD_EVENT_STOPPED:
-#ifdef CONFIG_MALI_GATOR_SUPPORT
- kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_SOFT_STOPPED, i), NULL, 0);
-#endif /* CONFIG_MALI_GATOR_SUPPORT */
- /* Soft-stopped job - read the value of JS<n>_TAIL so that the job chain can be resumed */
- job_tail = (u64) kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_TAIL_LO), NULL) | ((u64) kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_TAIL_HI), NULL) << 32);
- break;
- case BASE_JD_EVENT_NOT_STARTED:
- /* PRLAM-10673 can cause a TERMINATED job to come back as NOT_STARTED, but the error interrupt helps us detect it */
- completion_code = BASE_JD_EVENT_TERMINATED;
- /* fall throught */
- default:
- dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", i, completion_code, kbase_exception_name(completion_code));
- kbdev->kbase_group_error++;
- }
- }
+ if (!katom)
+ return true; /* Context has no jobs on this slot */
- kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), done & ((1 << i) | (1 << (i + 16))), NULL);
- active = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL);
-
- if (((active >> i) & 1) == 0 && (((done >> (i + 16)) & 1) == 0)) {
- /* There is a potential race we must work around:
- *
- * 1. A job slot has a job in both current and next registers
- * 2. The job in current completes successfully, the IRQ handler reads RAWSTAT
- * and calls this function with the relevant bit set in "done"
- * 3. The job in the next registers becomes the current job on the GPU
- * 4. Sometime before the JOB_IRQ_CLEAR line above the job on the GPU _fails_
- * 5. The IRQ_CLEAR clears the done bit but not the failed bit. This atomically sets
- * JOB_IRQ_JS_STATE. However since both jobs have now completed the relevant bits
- * for the slot are set to 0.
- *
- * If we now did nothing then we'd incorrectly assume that _both_ jobs had completed
- * successfully (since we haven't yet observed the fail bit being set in RAWSTAT).
- *
- * So at this point if there are no active jobs left we check to see if RAWSTAT has a failure
- * bit set for the job slot. If it does we know that there has been a new failure that we
- * didn't previously know about, so we make sure that we record this in active (but we wait
- * for the next loop to deal with it).
- *
- * If we were handling a job failure (i.e. done has the relevant high bit set) then we know that
- * the value read back from JOB_IRQ_JS_STATE is the correct number of remaining jobs because
- * the failed job will have prevented any futher jobs from starting execution.
- */
- u32 rawstat = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
-
- if ((rawstat >> (i + 16)) & 1) {
- /* There is a failed job that we've missed - add it back to active */
- active |= (1u << i);
- }
- }
-
- dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", completion_code);
-
- nr_done = kbasep_jm_nr_jobs_submitted(slot);
- nr_done -= (active >> i) & 1;
- nr_done -= (active >> (i + 16)) & 1;
-
- if (nr_done <= 0) {
- dev_warn(kbdev->dev, "Spurious interrupt on slot %d", i);
- goto spurious;
- }
-
- count += nr_done;
-
- while (nr_done) {
- if (nr_done == 1) {
- kbase_job_done_slot(kbdev, i, completion_code, job_tail, &end_timestamp);
- } else {
- /* More than one job has completed. Since this is not the last job being reported this time it
- * must have passed. This is because the hardware will not allow further jobs in a job slot to
- * complete until the faile job is cleared from the IRQ status.
- */
- kbase_job_done_slot(kbdev, i, BASE_JD_EVENT_DONE, 0, &end_timestamp);
- }
- nr_done--;
- }
-
- spurious:
- done = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) {
- /* Workaround for missing interrupt caused by PRLAM-10883 */
- if (((active >> i) & 1) && (0 == kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS), NULL))) {
- /* Force job slot to be processed again */
- done |= (1u << i);
- }
- }
-
- failed = done >> 16;
- finished = (done & 0xFFFF) | failed;
- } while (finished & (1 << i));
-
- kbasep_job_slot_update_head_start_timestamp(kbdev, slot, end_timestamp);
+ kbase_backend_run_atom(kbdev, katom);
}
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
-#if KBASE_GPU_RESET_EN
- if (atomic_read(&kbdev->reset_gpu) == KBASE_RESET_GPU_COMMITTED) {
- /* If we're trying to reset the GPU then we might be able to do it early
- * (without waiting for a timeout) because some jobs have completed
- */
- kbasep_try_reset_gpu_early(kbdev);
- }
-#endif /* KBASE_GPU_RESET_EN */
- KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
-}
-KBASE_EXPORT_TEST_API(kbase_job_done)
-
-static mali_bool kbasep_soft_stop_allowed(struct kbase_device *kbdev, u16 core_reqs)
-{
- mali_bool soft_stops_allowed = MALI_TRUE;
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) {
- if ((core_reqs & BASE_JD_REQ_T) != 0)
- soft_stops_allowed = MALI_FALSE;
- }
- return soft_stops_allowed;
-}
-static mali_bool kbasep_hard_stop_allowed(struct kbase_device *kbdev, u16 core_reqs)
-{
- mali_bool hard_stops_allowed = MALI_TRUE;
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) {
- if ((core_reqs & BASE_JD_REQ_T) != 0)
- hard_stops_allowed = MALI_FALSE;
- }
- return hard_stops_allowed;
+ return false; /* Slot ringbuffer should now be full */
}
-static void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, int js, u32 action, u16 core_reqs, struct kbase_jd_atom *target_katom)
+u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask)
{
- struct kbase_context *kctx = target_katom->kctx;
-#if KBASE_TRACE_ENABLE
- u32 status_reg_before;
- u64 job_in_head_before;
- u32 status_reg_after;
+ u32 ret_mask = 0;
- KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK)));
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
- /* Check the head pointer */
- job_in_head_before = ((u64) kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_LO), NULL))
- | (((u64) kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_HI), NULL)) << 32);
- status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), NULL);
-#endif
+ while (js_mask) {
+ int js = ffs(js_mask) - 1;
+ int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js);
- if (action == JS_COMMAND_SOFT_STOP) {
- mali_bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev, core_reqs);
- if (!soft_stop_allowed) {
-#ifdef CONFIG_MALI_DEBUG
- dev_dbg(kbdev->dev, "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X", (unsigned int)core_reqs);
-#endif /* CONFIG_MALI_DEBUG */
- return;
- }
+ if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit))
+ ret_mask |= (1 << js);
- /* We are about to issue a soft stop, so mark the atom as having been soft stopped */
- target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED;
+ js_mask &= ~(1 << js);
}
- if (action == JS_COMMAND_HARD_STOP) {
- mali_bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev, core_reqs);
- if (!hard_stop_allowed) {
- /* Jobs can be hard-stopped for the following reasons:
- * * CFS decides the job has been running too long (and soft-stop has not occurred).
- * In this case the GPU will be reset by CFS if the job remains on the GPU.
- *
- * * The context is destroyed, kbase_jd_zap_context will attempt to hard-stop the job. However
- * it also has a watchdog which will cause the GPU to be reset if the job remains on the GPU.
- *
- * * An (unhandled) MMU fault occurred. As long as BASE_HW_ISSUE_8245 is defined then
- * the GPU will be reset.
- *
- * All three cases result in the GPU being reset if the hard-stop fails,
- * so it is safe to just return and ignore the hard-stop request.
- */
- dev_warn(kbdev->dev, "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X", (unsigned int)core_reqs);
- return;
- }
- target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED;
- }
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316) && action == JS_COMMAND_SOFT_STOP) {
- int i;
- struct kbase_jm_slot *slot;
- slot = &kbdev->jm_slots[js];
-
- for (i = 0; i < kbasep_jm_nr_jobs_submitted(slot); i++) {
- struct kbase_jd_atom *katom;
-
- katom = kbasep_jm_peek_idx_submit_slot(slot, i);
-
- KBASE_DEBUG_ASSERT(katom);
-
- if (kbasep_jm_is_dummy_workaround_job(kbdev, katom) != MALI_FALSE) {
- /* Don't access the members of HW workaround 'dummy' jobs
- *
- * This assumes that such jobs can't cause HW_ISSUE_8316, and could only be blocked
- * by other jobs causing HW_ISSUE_8316 (which will get poked/or eventually get killed) */
- continue;
- }
-
- /* For HW_ISSUE_8316, only 'bad' jobs attacking the system can
- * cause this issue: normally, all memory should be allocated in
- * multiples of 4 pages, and growable memory should be changed size
- * in multiples of 4 pages.
- *
- * Whilst such 'bad' jobs can be cleared by a GPU reset, the
- * locking up of a uTLB entry caused by the bad job could also
- * stall other ASs, meaning that other ASs' jobs don't complete in
- * the 'grace' period before the reset. We don't want to lose other
- * ASs' jobs when they would normally complete fine, so we must
- * 'poke' the MMU regularly to help other ASs complete */
- kbase_as_poking_timer_retain_atom(kbdev, katom->kctx, katom);
- }
- }
-
- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
- if (action == JS_COMMAND_SOFT_STOP)
- action = (target_katom->atom_flags & KBASE_KATOM_FLAGS_JOBCHAIN) ?
- JS_COMMAND_SOFT_STOP_1 :
- JS_COMMAND_SOFT_STOP_0;
- else
- action = (target_katom->atom_flags & KBASE_KATOM_FLAGS_JOBCHAIN) ?
- JS_COMMAND_HARD_STOP_1 :
- JS_COMMAND_HARD_STOP_0;
- }
-
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx);
-
-#if KBASE_TRACE_ENABLE
- status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), NULL);
- if (status_reg_after == BASE_JD_EVENT_ACTIVE) {
- struct kbase_jm_slot *slot;
- struct kbase_jd_atom *head;
- struct kbase_context *head_kctx;
-
- slot = &kbdev->jm_slots[js];
- head = kbasep_jm_peek_idx_submit_slot(slot, slot->submitted_nr - 1);
- head_kctx = head->kctx;
-
- /* We don't need to check kbasep_jm_is_dummy_workaround_job( head ) here:
- * - Members are not indirected through
- * - The members will all be zero anyway
- */
- if (status_reg_before == BASE_JD_EVENT_ACTIVE)
- KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, head, job_in_head_before, js);
- else
- KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js);
-
- switch (action) {
- case JS_COMMAND_SOFT_STOP:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx, head, head->jc, js);
- break;
- case JS_COMMAND_SOFT_STOP_0:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, head, head->jc, js);
- break;
- case JS_COMMAND_SOFT_STOP_1:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, head, head->jc, js);
- break;
- case JS_COMMAND_HARD_STOP:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx, head, head->jc, js);
- break;
- case JS_COMMAND_HARD_STOP_0:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, head, head->jc, js);
- break;
- case JS_COMMAND_HARD_STOP_1:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, js);
- break;
- default:
- BUG();
- break;
- }
- } else {
- if (status_reg_before == BASE_JD_EVENT_ACTIVE)
- KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, job_in_head_before, js);
- else
- KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js);
-
- switch (action) {
- case JS_COMMAND_SOFT_STOP:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, js);
- break;
- case JS_COMMAND_SOFT_STOP_0:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, 0, js);
- break;
- case JS_COMMAND_SOFT_STOP_1:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, 0, js);
- break;
- case JS_COMMAND_HARD_STOP:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, js);
- break;
- case JS_COMMAND_HARD_STOP_0:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, 0, js);
- break;
- case JS_COMMAND_HARD_STOP_1:
- KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js);
- break;
- default:
- BUG();
- break;
- }
- }
-#endif
+ return ret_mask;
}
-/* Helper macros used by kbasep_job_slot_soft_or_hard_stop */
-#define JM_SLOT_MAX_JOB_SUBMIT_REGS 2
-#define JM_JOB_IS_CURRENT_JOB_INDEX(n) (1 == n) /* Index of the last job to process */
-#define JM_JOB_IS_NEXT_JOB_INDEX(n) (2 == n) /* Index of the prior to last job to process */
-
-/** Soft or hard-stop a slot
- *
- * This function safely ensures that the correct job is either hard or soft-stopped.
- * It deals with evicting jobs from the next registers where appropriate.
- *
- * This does not attempt to stop or evict jobs that are 'dummy' jobs for HW workarounds.
- *
- * @param kbdev The kbase device
- * @param kctx The context to soft/hard-stop job(s) from (or NULL is all jobs should be targeted)
- * @param js The slot that the job(s) are on
- * @param target_katom The atom that should be targeted (or NULL if all jobs from the context should be targeted)
- * @param action The action to perform, either JS_COMMAND_HARD_STOP or JS_COMMAND_SOFT_STOP
- */
-static void kbasep_job_slot_soft_or_hard_stop(struct kbase_device *kbdev, struct kbase_context *kctx, int js, struct kbase_jd_atom *target_katom, u32 action)
+void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask)
{
- struct kbase_jd_atom *katom;
- u8 i;
- u8 jobs_submitted;
- struct kbase_jm_slot *slot;
- u16 core_reqs;
- struct kbasep_js_device_data *js_devdata;
- mali_bool can_safely_stop = kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION);
- u32 hw_action = action & JS_COMMAND_MASK;
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
- KBASE_DEBUG_ASSERT(hw_action == JS_COMMAND_HARD_STOP ||
- hw_action == JS_COMMAND_SOFT_STOP);
- KBASE_DEBUG_ASSERT(kbdev);
- js_devdata = &kbdev->js_data;
-
- slot = &kbdev->jm_slots[js];
- KBASE_DEBUG_ASSERT(slot);
lockdep_assert_held(&js_devdata->runpool_irq.lock);
- jobs_submitted = kbasep_jm_nr_jobs_submitted(slot);
-
- KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, 1);
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SLOT_SOFT_OR_HARD_STOP, kctx, NULL, 0u, js, jobs_submitted);
-
- if (jobs_submitted > JM_SLOT_MAX_JOB_SUBMIT_REGS)
- i = jobs_submitted - JM_SLOT_MAX_JOB_SUBMIT_REGS;
- else
- i = 0;
-
- /* Loop through all jobs that have been submitted to the slot and haven't completed */
- for (; i < jobs_submitted; i++) {
- katom = kbasep_jm_peek_idx_submit_slot(slot, i);
-
- if (kctx && katom->kctx != kctx)
- continue;
-
- if (target_katom && katom != target_katom)
- continue;
-
- if (kbasep_jm_is_dummy_workaround_job(kbdev, katom))
- continue;
-
- core_reqs = katom->core_req;
-
- /* This will be repeated for anything removed from the next
- * registers, since their normal flow was also interrupted, and
- * this function might not enter disjoint state e.g. if we
- * don't actually do a hard stop on the head atom
- */
- kbase_job_check_enter_disjoint(kbdev, action, core_reqs, katom);
-
- if (JM_JOB_IS_CURRENT_JOB_INDEX(jobs_submitted - i)) {
- /* The last job in the slot, check if there is a job in the next register */
- if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), NULL) == 0) {
- kbasep_job_slot_soft_or_hard_stop_do_action(kbdev,
- js, hw_action, core_reqs, katom);
- } else {
- /* The job is in the next registers */
- beenthere(kctx, "clearing job from next registers on slot %d", js);
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_NOP, NULL);
- /* Check to see if we did remove a job from the next registers */
- if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) != 0 || kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) != 0) {
- /* The job was successfully cleared from the next registers, requeue it */
- struct kbase_jd_atom *dequeued_katom = kbasep_jm_dequeue_tail_submit_slot(slot);
- KBASE_DEBUG_ASSERT(dequeued_katom == katom);
- jobs_submitted--;
-
- /* Set the next registers to NULL */
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), 0, NULL);
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), 0, NULL);
-
- /* As the job is removed from the next registers we undo the associated
- * update to the job_chain_flag for the job slot. */
- if (can_safely_stop)
- slot->job_chain_flag = !slot->job_chain_flag;
-
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SLOT_EVICT, dequeued_katom->kctx, dequeued_katom, dequeued_katom->jc, js);
-
- kbase_job_check_enter_disjoint(kbdev, action, 0u, dequeued_katom);
- /* Complete the job, indicate it took no time, but don't submit any more at this point */
- kbase_jd_done(dequeued_katom, js, NULL, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
- } else {
- /* The job transitioned into the current registers before we managed to evict it,
- * in this case we fall back to soft/hard-stopping the job */
- beenthere(kctx, "missed job in next register, soft/hard-stopping slot %d", js);
- kbasep_job_slot_soft_or_hard_stop_do_action(kbdev,
- js, hw_action, core_reqs, katom);
- }
- }
- } else if (JM_JOB_IS_NEXT_JOB_INDEX(jobs_submitted - i)) {
- /* There's a job after this one, check to see if that
- * job is in the next registers. If so, we need to pay
- * attention to not accidently stop that one when
- * issueing the command to stop the one pointed to by
- * the head registers (as the one in the head may
- * finish in the mean time and the one in the next
- * moves to the head). Either the hardware has support
- * for this using job chain disambiguation or we need
- * to evict the job from the next registers first to
- * ensure we can safely stop the one pointed to by the
- * head registers.
- */
- if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), NULL) != 0) {
- struct kbase_jd_atom *check_next_atom;
- /* It is - we should remove that job and soft/hard-stop the slot */
-
- /* Only proceed when the next job isn't a HW workaround 'dummy' job
- *
- * This can't be an ASSERT due to MMU fault code:
- * - This first hard-stops the job that caused the fault
- * - Under HW Issue 8245, it will then reset the GPU
- * - This causes a Soft-stop to occur on all slots
- * - By the time of the soft-stop, we may (depending on timing) still have:
- * - The original job in HEAD, if it's not finished the hard-stop
- * - The dummy workaround job in NEXT
- *
- * Other cases could be coded in future that cause back-to-back Soft/Hard
- * stops with dummy workaround jobs in place, e.g. MMU handler code and Job
- * Scheduler watchdog timer running in parallel.
- *
- * Note, the index i+1 is valid to peek from: i == jobs_submitted-2, therefore
- * i+1 == jobs_submitted-1 */
- check_next_atom = kbasep_jm_peek_idx_submit_slot(slot, i + 1);
- if (kbasep_jm_is_dummy_workaround_job(kbdev, check_next_atom) != MALI_FALSE)
- continue;
-
- if (!can_safely_stop) {
- beenthere(kctx, "clearing job from next registers on slot %d", js);
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_NOP, NULL);
-
- /* Check to see if we did remove a job from the next registers */
- if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) != 0 || kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) != 0) {
- /* We did remove a job from the next registers, requeue it */
- struct kbase_jd_atom *dequeued_katom = kbasep_jm_dequeue_tail_submit_slot(slot);
- KBASE_DEBUG_ASSERT(dequeued_katom != NULL);
- jobs_submitted--;
-
- /* Set the next registers to NULL */
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), 0, NULL);
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), 0, NULL);
-
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SLOT_EVICT, dequeued_katom->kctx, dequeued_katom, dequeued_katom->jc, js);
-
- kbase_job_check_enter_disjoint(kbdev, action, 0u, dequeued_katom);
- /* Complete the job, indicate it took no time, but don't submit any more at this point */
- kbase_jd_done(dequeued_katom, js, NULL, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
- } else {
- /* We missed the job, that means the job we're interested in left the hardware before
- * we managed to do anything, so we can proceed to the next job */
- continue;
- }
- }
-
- /* Next is now free, so we can soft/hard-stop the slot */
- beenthere(kctx, "soft/hard-stopped slot %d (there was a job in next which was successfully cleared)\n", js);
- kbasep_job_slot_soft_or_hard_stop_do_action(kbdev,
- js, hw_action, core_reqs, katom);
- }
- /* If there was no job in the next registers, then the job we were
- * interested in has finished, so we need not take any action
- */
- }
+ if (!down_trylock(&js_devdata->schedule_sem)) {
+ kbase_jm_kick(kbdev, js_mask);
+ up(&js_devdata->schedule_sem);
}
-
- KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, 0);
-}
-
-void kbase_job_kill_jobs_from_context(struct kbase_context *kctx)
-{
- unsigned long flags;
- struct kbase_device *kbdev;
- struct kbasep_js_device_data *js_devdata;
- int i;
-
- KBASE_DEBUG_ASSERT(kctx != NULL);
- kbdev = kctx->kbdev;
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
-
- /* Cancel any remaining running jobs for this kctx */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
-
- /* Invalidate all jobs in context, to prevent re-submitting */
- for (i = 0; i < BASE_JD_ATOM_COUNT; i++)
- kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_CANCELLED;
-
- for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
- kbase_job_slot_hardstop(kctx, i, NULL);
-
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
}
-void kbase_job_zap_context(struct kbase_context *kctx)
+void kbase_jm_try_kick_all(struct kbase_device *kbdev)
{
- struct kbase_device *kbdev;
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_kctx_info *js_kctx_info;
- int i;
- mali_bool evict_success;
-
- KBASE_DEBUG_ASSERT(kctx != NULL);
- kbdev = kctx->kbdev;
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
- js_kctx_info = &kctx->jctx.sched_info;
-
- /*
- * Critical assumption: No more submission is possible outside of the
- * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs)
- * whilst the struct kbase_context is terminating.
- */
-
- /* First, atomically do the following:
- * - mark the context as dying
- * - try to evict it from the policy queue */
-
- mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
- js_kctx_info->ctx.is_dying = MALI_TRUE;
-
- dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx);
- mutex_lock(&js_devdata->queue_mutex);
- evict_success = kbasep_js_policy_try_evict_ctx(&js_devdata->policy, kctx);
- mutex_unlock(&js_devdata->queue_mutex);
-
- /*
- * At this point we know:
- * - If eviction succeeded, it was in the policy queue, but now no longer is
- * - We must cancel the jobs here. No Power Manager active reference to
- * release.
- * - This happens asynchronously - kbase_jd_zap_context() will wait for
- * those jobs to be killed.
- * - If eviction failed, then it wasn't in the policy queue. It is one of
- * the following:
- * - a. it didn't have any jobs, and so is not in the Policy Queue or the
- * Run Pool (not scheduled)
- * - Hence, no more work required to cancel jobs. No Power Manager active
- * reference to release.
- * - b. it was in the middle of a scheduling transaction (and thus must
- * have at least 1 job). This can happen from a syscall or a kernel thread.
- * We still hold the jsctx_mutex, and so the thread must be waiting inside
- * kbasep_js_try_schedule_head_ctx(), before checking whether the runpool
- * is full. That thread will continue after we drop the mutex, and will
- * notice the context is dying. It will rollback the transaction, killing
- * all jobs at the same time. kbase_jd_zap_context() will wait for those
- * jobs to be killed.
- * - Hence, no more work required to cancel jobs, or to release the Power
- * Manager active reference.
- * - c. it is scheduled, and may or may not be running jobs
- * - We must cause it to leave the runpool by stopping it from submitting
- * any more jobs. When it finally does leave,
- * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs
- * (because it is dying), release the Power Manager active reference, and
- * will not requeue the context in the policy queue. kbase_jd_zap_context()
- * will wait for those jobs to be killed.
- * - Hence, work required just to make it leave the runpool. Cancelling
- * jobs and releasing the Power manager active reference will be handled
- * when it leaves the runpool.
- */
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
- if (evict_success != MALI_FALSE || js_kctx_info->ctx.is_scheduled == MALI_FALSE) {
- /* The following events require us to kill off remaining jobs and
- * update PM book-keeping:
- * - we evicted it correctly (it must have jobs to be in the Policy Queue)
- *
- * These events need no action, but take this path anyway:
- * - Case a: it didn't have any jobs, and was never in the Queue
- * - Case b: scheduling transaction will be partially rolled-back (this
- * already cancels the jobs)
- */
-
- KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, js_kctx_info->ctx.is_scheduled);
-
- dev_dbg(kbdev->dev, "Zap: Ctx %p evict_success=%d, scheduled=%d", kctx, evict_success, js_kctx_info->ctx.is_scheduled);
-
- if (evict_success != MALI_FALSE) {
- /* Only cancel jobs when we evicted from the policy queue. No Power
- * Manager active reference was held.
- *
- * Having is_dying set ensures that this kills, and doesn't requeue */
- kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, MALI_FALSE);
- }
- mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
- } else {
- unsigned long flags;
- mali_bool was_retained;
- /* Case c: didn't evict, but it is scheduled - it's in the Run Pool */
- KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, js_kctx_info->ctx.is_scheduled);
- dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx);
-
- /* Disable the ctx from submitting any more jobs */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- kbasep_js_clear_submit_allowed(js_devdata, kctx);
-
- /* Retain and (later) release the context whilst it is is now disallowed from submitting
- * jobs - ensures that someone somewhere will be removing the context later on */
- was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
-
- /* Since it's scheduled and we have the jsctx_mutex, it must be retained successfully */
- KBASE_DEBUG_ASSERT(was_retained != MALI_FALSE);
-
- dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx);
- /* Cancel any remaining running jobs for this kctx - if any. Submit is disallowed
- * which takes effect immediately, so no more new jobs will appear after we do this. */
- for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
- kbase_job_slot_hardstop(kctx, i, NULL);
-
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
- mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
-
- dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", kctx);
- kbasep_js_runpool_release_ctx(kbdev, kctx);
- }
- KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u);
-
- /* After this, you must wait on both the kbase_jd_context::zero_jobs_wait
- * and the kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the
- * jobs to be destroyed, and the context to be de-scheduled (if it was on
- * the runpool).
- *
- * kbase_jd_zap_context() will do this. */
-}
-KBASE_EXPORT_TEST_API(kbase_job_zap_context)
-
-mali_error kbase_job_slot_init(struct kbase_device *kbdev)
-{
- int i;
- KBASE_DEBUG_ASSERT(kbdev);
-
- for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
- kbasep_jm_init_submit_slot(&kbdev->jm_slots[i]);
-
- return MALI_ERROR_NONE;
-}
-KBASE_EXPORT_TEST_API(kbase_job_slot_init)
-
-void kbase_job_slot_halt(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
-void kbase_job_slot_term(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev);
-}
-KBASE_EXPORT_TEST_API(kbase_job_slot_term)
-
-/**
- * Soft-stop the specified job slot, with extra information about the stop
- *
- * The job slot lock must be held when calling this function.
- * The job slot must not already be in the process of being soft-stopped.
- *
- * Where possible any job in the next register is evicted before the soft-stop.
- *
- * @param kbdev The kbase device
- * @param js The job slot to soft-stop
- * @param target_katom The job that should be soft-stopped (or NULL for any job)
- * @param sw_flags Flags to pass in about the soft-stop
- */
-void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
- struct kbase_jd_atom *target_katom, u32 sw_flags)
-{
- KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK));
- kbasep_job_slot_soft_or_hard_stop(kbdev, NULL, js, target_katom,
- JS_COMMAND_SOFT_STOP | sw_flags);
-}
-
-/**
- * Soft-stop the specified job slot
- *
- * The job slot lock must be held when calling this function.
- * The job slot must not already be in the process of being soft-stopped.
- *
- * Where possible any job in the next register is evicted before the soft-stop.
- *
- * @param kbdev The kbase device
- * @param js The job slot to soft-stop
- * @param target_katom The job that should be soft-stopped (or NULL for any job)
- */
-void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, struct kbase_jd_atom *target_katom)
-{
- kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u);
-}
-
-/**
- * Hard-stop the specified job slot
- *
- * The job slot lock must be held when calling this function.
- *
- * @param kctx The kbase context that contains the job(s) that should
- * be hard-stopped
- * @param js The job slot to hard-stop
- * @param target_katom The job that should be hard-stopped (or NULL for all
- * jobs from the context)
- */
-void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
- struct kbase_jd_atom *target_katom)
-{
- struct kbase_device *kbdev = kctx->kbdev;
-
- kbasep_job_slot_soft_or_hard_stop(kbdev, kctx, js, target_katom,
- JS_COMMAND_HARD_STOP);
-#if KBASE_GPU_RESET_EN
- if (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) ||
- kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) ||
- (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_T76X_3542) &&
- (target_katom == NULL || target_katom->core_req & BASE_JD_REQ_FS_AFBC))) {
- /* MIDBASE-2916 if a fragment job with AFBC encoding is
- * hardstopped, ensure to do a soft reset also in order to
- * clear the GPU status.
- * Workaround for HW issue 8401 has an issue,so after
- * hard-stopping just reset the GPU. This will ensure that the
- * jobs leave the GPU.*/
- if (kbase_prepare_to_reset_gpu_locked(kbdev)) {
- dev_err(kbdev->dev, "Issueing GPU\
- soft-reset after hard stopping due to hardware issue");
- kbase_reset_gpu_locked(kbdev);
- }
- }
-#endif
-}
-
-/**
- * For a certain soft/hard-stop action, work out whether to enter disjoint
- * state.
- *
- * This does not register multiple disjoint events if the atom has already
- * started a disjoint period
- *
- * core_reqs can be supplied as 0 if the atom had not started on the hardware
- * (and so a 'real' soft/hard-stop was not required, but it still interrupted
- * flow, perhaps on another context)
- *
- * kbase_job_check_leave_disjoint() should be used to end the disjoint
- * state when the soft/hard-stop action is complete
- */
-void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
- u16 core_reqs, struct kbase_jd_atom *target_katom)
-{
- u32 hw_action = action & JS_COMMAND_MASK;
-
- /* For hard-stop, don't enter if hard-stop not allowed */
- if (hw_action == JS_COMMAND_HARD_STOP &&
- !kbasep_hard_stop_allowed(kbdev, core_reqs))
- return;
-
- /* For soft-stop, don't enter if soft-stop not allowed, or isn't
- * causing disjoint */
- if (hw_action == JS_COMMAND_SOFT_STOP &&
- !(kbasep_soft_stop_allowed(kbdev, core_reqs) &&
- (action & JS_COMMAND_SW_CAUSES_DISJOINT)))
- return;
-
- /* Nothing to do if already logged disjoint state on this atom */
- if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT)
- return;
-
- target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT;
- kbase_disjoint_state_up(kbdev);
-}
-
-/**
- * Work out whether to leave disjoint state when finishing an atom that was
- * originated by kbase_job_check_enter_disjoint().
- */
-void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
- struct kbase_jd_atom *target_katom)
-{
- if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) {
- target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT;
- kbase_disjoint_state_down(kbdev);
- }
-}
-
-
-#if KBASE_GPU_RESET_EN
-static void kbase_debug_dump_registers(struct kbase_device *kbdev)
-{
- int i;
- dev_err(kbdev->dev, "Register state:");
- dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x",
- kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL),
- kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL));
- dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x JOB_IRQ_THROTTLE=0x%08x",
- kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL),
- kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL),
- kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE), NULL));
- for (i = 0; i < 3; i++) {
- dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x",
- i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS),
- NULL),
- i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO),
- NULL));
- }
- dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x",
- kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL),
- kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL));
- dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x",
- kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL),
- kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL),
- kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL));
- dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x",
- kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL),
- kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL));
- dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x",
- kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL),
- kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL));
-}
-
-void kbasep_reset_timeout_worker(struct work_struct *data)
-{
- unsigned long flags;
- struct kbase_device *kbdev;
- int i;
- ktime_t end_timestamp = ktime_get();
- struct kbasep_js_device_data *js_devdata;
- struct kbase_uk_hwcnt_setup hwcnt_setup = { {0} };
- enum kbase_instr_state bckp_state;
-
- KBASE_DEBUG_ASSERT(data);
-
- kbdev = container_of(data, struct kbase_device, reset_work);
-
- KBASE_DEBUG_ASSERT(kbdev);
- js_devdata = &kbdev->js_data;
-
- KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
-
- /* Make sure the timer has completed - this cannot be done from interrupt context,
- * so this cannot be done within kbasep_try_reset_gpu_early. */
- hrtimer_cancel(&kbdev->reset_timer);
-
- if (kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
- /* This would re-activate the GPU. Since it's already idle, there's no
- * need to reset it */
- atomic_set(&kbdev->reset_gpu, KBASE_RESET_GPU_NOT_PENDING);
- kbase_disjoint_state_down(kbdev);
- wake_up(&kbdev->reset_wait);
- return;
- }
-
- mutex_lock(&kbdev->pm.lock);
- /* We hold the pm lock, so there ought to be a current policy */
- KBASE_DEBUG_ASSERT(kbdev->pm.pm_current_policy);
-
- /* All slot have been soft-stopped and we've waited SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point
- * we assume that anything that is still left on the GPU is stuck there and we'll kill it when we reset the GPU */
-
- dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", RESET_TIMEOUT);
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
-
- if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) { /*the same interrupt handler preempted itself */
- /* GPU is being reset */
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
- wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- }
- /* Save the HW counters setup */
- if (kbdev->hwcnt.kctx != NULL) {
- struct kbase_context *kctx = kbdev->hwcnt.kctx;
- hwcnt_setup.dump_buffer = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), kctx) & 0xffffffff;
- hwcnt_setup.dump_buffer |= (mali_addr64) kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), kctx) << 32;
- hwcnt_setup.jm_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), kctx);
- hwcnt_setup.shader_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), kctx);
- hwcnt_setup.tiler_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), kctx);
- hwcnt_setup.l3_cache_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_L3_CACHE_EN), kctx);
- hwcnt_setup.mmu_l2_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), kctx);
- }
-
- /* Output the state of some interesting registers to help in the
- * debugging of GPU resets */
- kbase_debug_dump_registers(kbdev);
-
- bckp_state = kbdev->hwcnt.state;
- kbdev->hwcnt.state = KBASE_INSTR_STATE_RESETTING;
- kbdev->hwcnt.triggered = 0;
- /* Disable IRQ to avoid IRQ handlers to kick in after releaseing the spinlock;
- * this also clears any outstanding interrupts */
- kbase_pm_disable_interrupts(kbdev);
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
-
- /* Ensure that any IRQ handlers have finished
- * Must be done without any locks IRQ handlers will take */
- kbase_synchronize_irqs(kbdev);
-
- /* Reset the GPU */
- kbase_pm_init_hw(kbdev, MALI_TRUE);
- /* IRQs were re-enabled by kbase_pm_init_hw, and GPU is still powered */
-
- spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
- /* Restore the HW counters setup */
- if (kbdev->hwcnt.kctx != NULL) {
- struct kbase_context *kctx = kbdev->hwcnt.kctx;
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_OFF, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), hwcnt_setup.dump_buffer & 0xFFFFFFFF, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), hwcnt_setup.dump_buffer >> 32, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), hwcnt_setup.jm_bm, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), hwcnt_setup.shader_bm, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_L3_CACHE_EN), hwcnt_setup.l3_cache_bm, kctx);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), hwcnt_setup.mmu_l2_bm, kctx);
-
- /* Due to PRLAM-8186 we need to disable the Tiler before we enable the HW counter dump. */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, kctx);
- else
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), hwcnt_setup.tiler_bm, kctx);
-
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, kctx);
-
- /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
- kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), hwcnt_setup.tiler_bm, kctx);
- }
- kbdev->hwcnt.state = bckp_state;
- switch (kbdev->hwcnt.state) {
- /* Cases for waking kbasep_cache_clean_worker worker */
- case KBASE_INSTR_STATE_CLEANED:
- /* Cache-clean IRQ occurred, but we reset:
- * Wakeup incase the waiter saw RESETTING */
- case KBASE_INSTR_STATE_REQUEST_CLEAN:
- /* After a clean was requested, but before the regs were written:
- * Wakeup incase the waiter saw RESETTING */
- wake_up(&kbdev->hwcnt.cache_clean_wait);
- break;
- case KBASE_INSTR_STATE_CLEANING:
- /* Either:
- * 1) We've not got the Cache-clean IRQ yet: it was lost, or:
- * 2) We got it whilst resetting: it was voluntarily lost
- *
- * So, move to the next state and wakeup: */
- kbdev->hwcnt.state = KBASE_INSTR_STATE_CLEANED;
- wake_up(&kbdev->hwcnt.cache_clean_wait);
- break;
-
- /* Cases for waking anyone else */
- case KBASE_INSTR_STATE_DUMPING:
- /* If dumping, abort the dump, because we may've lost the IRQ */
- kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
- kbdev->hwcnt.triggered = 1;
- wake_up(&kbdev->hwcnt.wait);
- break;
- case KBASE_INSTR_STATE_DISABLED:
- case KBASE_INSTR_STATE_IDLE:
- case KBASE_INSTR_STATE_FAULT:
- /* Every other reason: wakeup in that state */
- kbdev->hwcnt.triggered = 1;
- wake_up(&kbdev->hwcnt.wait);
- break;
-
- /* Unhandled cases */
- case KBASE_INSTR_STATE_RESETTING:
- default:
- BUG();
- break;
- }
- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
-
- /* Complete any jobs that were still on the GPU */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
- int nr_done;
- struct kbase_jm_slot *slot = &kbdev->jm_slots[i];
-
- nr_done = kbasep_jm_nr_jobs_submitted(slot);
- while (nr_done) {
- dev_err(kbdev->dev, "Job stuck in slot %d on the GPU was cancelled", i);
- kbase_job_done_slot(kbdev, i, BASE_JD_EVENT_JOB_CANCELLED, 0, &end_timestamp);
- nr_done--;
- }
- }
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
-
- mutex_lock(&js_devdata->runpool_mutex);
-
- /* Reprogram the GPU's MMU */
- for (i = 0; i < BASE_MAX_NR_AS; i++) {
- if (js_devdata->runpool_irq.per_as_data[i].kctx) {
- struct kbase_as *as = &kbdev->as[i];
- mutex_lock(&as->transaction_mutex);
- kbase_mmu_update(js_devdata->runpool_irq.per_as_data[i].kctx);
- mutex_unlock(&as->transaction_mutex);
- }
- }
-
- atomic_set(&kbdev->reset_gpu, KBASE_RESET_GPU_NOT_PENDING);
-
- kbase_disjoint_state_down(kbdev);
- wake_up(&kbdev->reset_wait);
- dev_err(kbdev->dev, "Reset complete");
-
- /* Find out what cores are required now */
- kbase_pm_update_cores_state(kbdev);
-
- /* Synchronously request and wait for those cores, because if
- * instrumentation is enabled it would need them immediately. */
- kbase_pm_check_transitions_sync(kbdev);
-
- /* Try submitting some jobs to restart processing */
- if (js_devdata->nr_user_contexts_running > 0) {
- KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0);
+ lockdep_assert_held(&js_devdata->runpool_irq.lock);
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- kbasep_js_try_run_next_job_nolock(kbdev);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ if (!down_trylock(&js_devdata->schedule_sem)) {
+ kbase_jm_kick_all(kbdev);
+ up(&js_devdata->schedule_sem);
}
-
- mutex_unlock(&js_devdata->runpool_mutex);
- mutex_unlock(&kbdev->pm.lock);
-
- kbase_pm_context_idle(kbdev);
- KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
}
-enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer)
+void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
{
- struct kbase_device *kbdev = container_of(timer, struct kbase_device, reset_timer);
-
- KBASE_DEBUG_ASSERT(kbdev);
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
- /* Reset still pending? */
- if (atomic_cmpxchg(&kbdev->reset_gpu, KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) == KBASE_RESET_GPU_COMMITTED)
- queue_work(kbdev->reset_workq, &kbdev->reset_work);
+ WARN_ON(atomic_read(&kctx->atoms_pulled));
- return HRTIMER_NORESTART;
+ if (kbdev->hwaccess.active_kctx == kctx)
+ kbdev->hwaccess.active_kctx = NULL;
}
-/*
- * If all jobs are evicted from the GPU then we can reset the GPU
- * immediately instead of waiting for the timeout to elapse
- */
-
-static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev)
+void kbase_jm_return_atom_to_js(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
{
- int i;
- int pending_jobs = 0;
-
- KBASE_DEBUG_ASSERT(kbdev);
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
- /* Count the number of jobs */
- for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
- struct kbase_jm_slot *slot = &kbdev->jm_slots[i];
- pending_jobs += kbasep_jm_nr_jobs_submitted(slot);
- }
-
- if (pending_jobs > 0) {
- /* There are still jobs on the GPU - wait */
- return;
- }
-
- /* Check that the reset has been committed to (i.e. kbase_reset_gpu has been called), and that no other
- * thread beat this thread to starting the reset */
- if (atomic_cmpxchg(&kbdev->reset_gpu, KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != KBASE_RESET_GPU_COMMITTED) {
- /* Reset has already occurred */
- return;
- }
- queue_work(kbdev->reset_workq, &kbdev->reset_work);
-}
-
-static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev)
-{
- unsigned long flags;
- struct kbasep_js_device_data *js_devdata;
-
- js_devdata = &kbdev->js_data;
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- kbasep_try_reset_gpu_early_locked(kbdev);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
-}
-
-/*
- * Prepare for resetting the GPU.
- * This function just soft-stops all the slots to ensure that as many jobs as possible are saved.
- *
- * The function returns a boolean which should be interpreted as follows:
- * - MALI_TRUE - Prepared for reset, kbase_reset_gpu should be called.
- * - MALI_FALSE - Another thread is performing a reset, kbase_reset_gpu should not be called.
- *
- * @return See description
- */
-mali_bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev)
-{
- int i;
-
- KBASE_DEBUG_ASSERT(kbdev);
-
- if (atomic_cmpxchg(&kbdev->reset_gpu, KBASE_RESET_GPU_NOT_PENDING, KBASE_RESET_GPU_PREPARED) != KBASE_RESET_GPU_NOT_PENDING) {
- /* Some other thread is already resetting the GPU */
- return MALI_FALSE;
+ if (katom->event_code != BASE_JD_EVENT_STOPPED &&
+ katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) {
+ kbase_js_complete_atom(katom, NULL);
+ } else {
+ kbase_js_unpull(katom->kctx, katom);
}
-
- kbase_disjoint_state_up(kbdev);
-
- for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
- kbase_job_slot_softstop(kbdev, i, NULL);
-
- return MALI_TRUE;
-}
-
-mali_bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev)
-{
- unsigned long flags;
- mali_bool ret;
- struct kbasep_js_device_data *js_devdata;
-
- js_devdata = &kbdev->js_data;
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- ret = kbase_prepare_to_reset_gpu_locked(kbdev);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
-
- return ret;
}
-KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu)
-/*
- * This function should be called after kbase_prepare_to_reset_gpu iff it returns MALI_TRUE.
- * It should never be called without a corresponding call to kbase_prepare_to_reset_gpu.
- *
- * After this function is called (or not called if kbase_prepare_to_reset_gpu returned MALI_FALSE),
- * the caller should wait for kbdev->reset_waitq to be signalled to know when the reset has completed.
- */
-void kbase_reset_gpu(struct kbase_device *kbdev)
+void kbase_jm_complete(struct kbase_device *kbdev, struct kbase_jd_atom *katom,
+ ktime_t *end_timestamp)
{
- u32 timeout_ms;
-
- KBASE_DEBUG_ASSERT(kbdev);
-
- /* Note this is an assert/atomic_set because it is a software issue for a race to be occuring here */
- KBASE_DEBUG_ASSERT(atomic_read(&kbdev->reset_gpu) == KBASE_RESET_GPU_PREPARED);
- atomic_set(&kbdev->reset_gpu, KBASE_RESET_GPU_COMMITTED);
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
- timeout_ms = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS);
- dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", timeout_ms);
- hrtimer_start(&kbdev->reset_timer, HR_TIMER_DELAY_MSEC(timeout_ms), HRTIMER_MODE_REL);
-
- /* Try resetting early */
- kbasep_try_reset_gpu_early(kbdev);
+ kbase_js_complete_atom(katom, end_timestamp);
}
-KBASE_EXPORT_TEST_API(kbase_reset_gpu)
-
-void kbase_reset_gpu_locked(struct kbase_device *kbdev)
-{
- u32 timeout_ms;
-
- KBASE_DEBUG_ASSERT(kbdev);
-
- /* Note this is an assert/atomic_set because it is a software issue for a race to be occuring here */
- KBASE_DEBUG_ASSERT(atomic_read(&kbdev->reset_gpu) == KBASE_RESET_GPU_PREPARED);
- atomic_set(&kbdev->reset_gpu, KBASE_RESET_GPU_COMMITTED);
- timeout_ms = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS);
- dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", timeout_ms);
- hrtimer_start(&kbdev->reset_timer, HR_TIMER_DELAY_MSEC(timeout_ms), HRTIMER_MODE_REL);
-
- /* Try resetting early */
- kbasep_try_reset_gpu_early_locked(kbdev);
-}
-#endif /* KBASE_GPU_RESET_EN */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-
-/**
- * @file mali_kbase_jm.h
- * Job Manager Low-level APIs.
+/*
+ * Job manager common APIs
*/
#ifndef _KBASE_JM_H_
#define _KBASE_JM_H_
-#include <mali_kbase_hw.h>
-#include <mali_kbase_debug.h>
-#include <linux/atomic.h>
-
/**
- * @addtogroup base_api
- * @{
- */
-
-/**
- * @addtogroup base_kbase_api
- * @{
- */
-
-/**
- * @addtogroup kbase_jm Job Manager Low-level APIs
- * @{
+ * kbase_jm_kick() - Indicate that there are jobs ready to run.
+ * @kbdev: Device pointer
+ * @js_mask: Mask of the job slots that can be pulled from.
*
+ * Caller must hold the runpool_irq lock and schedule_sem semaphore
+ *
+ * Return: Mask of the job slots that can still be submitted to.
*/
-
-static INLINE int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, struct kbase_context *kctx)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(0 <= js && js < kbdev->gpu_props.num_job_slots);
-
- return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx);
-}
-
-/**
- * This checks that:
- * - there is enough space in the GPU's buffers (JS_NEXT and JS_HEAD registers) to accomodate the job.
- * - there is enough space to track the job in a our Submit Slots. Note that we have to maintain space to
- * requeue one job in case the next registers on the hardware need to be cleared.
- */
-static INLINE mali_bool kbasep_jm_is_submit_slots_free(struct kbase_device *kbdev, int js, struct kbase_context *kctx)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(0 <= js && js < kbdev->gpu_props.num_job_slots);
-
- if (atomic_read(&kbdev->reset_gpu) != KBASE_RESET_GPU_NOT_PENDING) {
- /* The GPU is being reset - so prevent submission */
- return MALI_FALSE;
- }
-
- return (mali_bool) (kbasep_jm_is_js_free(kbdev, js, kctx)
- && kbdev->jm_slots[js].submitted_nr < (BASE_JM_SUBMIT_SLOTS - 2));
-}
+u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask);
/**
- * Initialize a submit slot
+ * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job
+ * slots.
+ * @kbdev: Device pointer
+ *
+ * Caller must hold the runpool_irq lock and schedule_sem semaphore
+ *
+ * Return: Mask of the job slots that can still be submitted to.
*/
-static INLINE void kbasep_jm_init_submit_slot(struct kbase_jm_slot *slot)
+static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev)
{
- slot->submitted_nr = 0;
- slot->submitted_head = 0;
+ return kbase_jm_kick(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1);
}
/**
- * Find the atom at the idx'th element in the queue without removing it, starting at the head with idx==0.
- */
-static INLINE struct kbase_jd_atom *kbasep_jm_peek_idx_submit_slot(struct kbase_jm_slot *slot, u8 idx)
-{
- u8 pos;
- struct kbase_jd_atom *katom;
-
- KBASE_DEBUG_ASSERT(idx < BASE_JM_SUBMIT_SLOTS);
-
- pos = (slot->submitted_head + idx) & BASE_JM_SUBMIT_SLOTS_MASK;
- katom = slot->submitted[pos];
-
- return katom;
-}
-
-/**
- * Pop front of the submitted
- */
-static INLINE struct kbase_jd_atom *kbasep_jm_dequeue_submit_slot(struct kbase_jm_slot *slot)
-{
- u8 pos;
- struct kbase_jd_atom *katom;
-
- pos = slot->submitted_head & BASE_JM_SUBMIT_SLOTS_MASK;
- katom = slot->submitted[pos];
- slot->submitted[pos] = NULL; /* Just to catch bugs... */
- KBASE_DEBUG_ASSERT(katom);
-
- /* rotate the buffers */
- slot->submitted_head = (slot->submitted_head + 1) & BASE_JM_SUBMIT_SLOTS_MASK;
- slot->submitted_nr--;
-
- dev_dbg(katom->kctx->kbdev->dev, "katom %p new head %u", (void *)katom, (unsigned int)slot->submitted_head);
-
- return katom;
-}
-
-/* Pop back of the submitted queue (unsubmit a job)
+ * kbase_jm_try_kick - Attempt to call kbase_jm_kick
+ * @kbdev: Device pointer
+ * @js_mask: Mask of the job slots that can be pulled from
+ * Context: Caller must hold runpool_irq lock
+ *
+ * If schedule_sem can be immediately obtained then this function will call
+ * kbase_jm_kick() otherwise it will do nothing.
*/
-static INLINE struct kbase_jd_atom *kbasep_jm_dequeue_tail_submit_slot(struct kbase_jm_slot *slot)
-{
- u8 pos;
-
- slot->submitted_nr--;
-
- pos = (slot->submitted_head + slot->submitted_nr) & BASE_JM_SUBMIT_SLOTS_MASK;
-
- return slot->submitted[pos];
-}
-
-static INLINE u8 kbasep_jm_nr_jobs_submitted(struct kbase_jm_slot *slot)
-{
- return slot->submitted_nr;
-}
+void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask);
/**
- * Push back of the submitted
+ * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all
+ * @kbdev: Device pointer
+ * Context: Caller must hold runpool_irq lock
+ *
+ * If schedule_sem can be immediately obtained then this function will call
+ * kbase_jm_kick_all() otherwise it will do nothing.
*/
-static INLINE void kbasep_jm_enqueue_submit_slot(struct kbase_jm_slot *slot, struct kbase_jd_atom *katom)
-{
- u8 nr;
- u8 pos;
- nr = slot->submitted_nr++;
- KBASE_DEBUG_ASSERT(nr < BASE_JM_SUBMIT_SLOTS);
-
- pos = (slot->submitted_head + nr) & BASE_JM_SUBMIT_SLOTS_MASK;
- slot->submitted[pos] = katom;
-}
+void kbase_jm_try_kick_all(struct kbase_device *kbdev);
/**
- * @brief Query whether a job peeked/dequeued from the submit slots is a
- * 'dummy' job that is used for hardware workaround purposes.
- *
- * Any time a job is peeked/dequeued from the submit slots, this should be
- * queried on that job.
+ * kbase_jm_idle_ctx() - Mark a context as idle.
+ * @kbdev: Device pointer
+ * @kctx: Context to mark as idle
*
- * If a \a atom is indicated as being a dummy job, then you <b>must not attempt
- * to use \a atom</b>. This is because its members will not necessarily be
- * initialized, and so could lead to a fault if they were used.
+ * No more atoms will be pulled from this context until it is marked as active
+ * by kbase_js_use_ctx().
*
- * @param[in] kbdev kbase device pointer
- * @param[in] atom The atom to query
+ * The context should have no atoms currently pulled from it
+ * (kctx->atoms_pulled == 0).
*
- * @return MALI_TRUE if \a atom is for a dummy job, in which case you must not
- * attempt to use it.
- * @return MALI_FALSE otherwise, and \a atom is safe to use.
+ * Caller must hold the runpool_irq lock
*/
-static INLINE mali_bool kbasep_jm_is_dummy_workaround_job(struct kbase_device *kbdev, struct kbase_jd_atom *atom)
-{
- /* Query the set of workaround jobs here */
- /* none exists today */
- return MALI_FALSE;
-}
+void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
/**
- * @brief Submit a job to a certain job-slot
- *
- * The caller must check kbasep_jm_is_submit_slots_free() != MALI_FALSE before calling this.
- *
- * The following locking conditions are made on the caller:
- * - it must hold the kbasep_js_device_data::runpoool_irq::lock
+ * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has
+ * been soft-stopped or will fail due to a
+ * dependency
+ * @kbdev: Device pointer
+ * @katom: Atom that has been stopped or will be failed
*/
-void kbase_job_submit_nolock(struct kbase_device *kbdev, struct kbase_jd_atom *katom, int js);
+void kbase_jm_return_atom_to_js(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom);
/**
- * @brief Complete the head job on a particular job-slot
+ * kbase_jm_complete() - Complete an atom
+ * @kbdev: Device pointer
+ * @katom: Atom that has completed
+ * @end_timestamp: Timestamp of atom completion
*/
-void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, u64 job_tail, ktime_t *end_timestamp);
-
- /** @} *//* end group kbase_jm */
- /** @} *//* end group base_kbase_api */
- /** @} *//* end group base_api */
+void kbase_jm_complete(struct kbase_device *kbdev, struct kbase_jd_atom *katom,
+ ktime_t *end_timestamp);
-#endif /* _KBASE_JM_H_ */
+#endif /* _KBASE_JM_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
*/
#include <mali_kbase.h>
#include <mali_kbase_js.h>
-#include <mali_kbase_js_affinity.h>
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
#include <mali_kbase_gator.h>
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+#include <mali_kbase_tlstream.h>
+#endif
#include <mali_kbase_hw.h>
-#include "mali_kbase_jm.h"
#include <mali_kbase_defs.h>
#include <mali_kbase_config_defaults.h>
+#include "mali_kbase_jm.h"
+#include "mali_kbase_hwaccess_jm.h"
+
/*
* Private types
*/
-/** Bitpattern indicating the result of releasing a context */
+/* Bitpattern indicating the result of releasing a context */
enum {
- /** The context was descheduled - caller should try scheduling in a new one
- * to keep the runpool full */
+ /* The context was descheduled - caller should try scheduling in a new
+ * one to keep the runpool full */
KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0),
+ /* Ctx attributes were changed - caller should try scheduling all
+ * contexts */
+ KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1)
};
typedef u32 kbasep_js_release_result;
+const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = {
+ KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */
+ KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */
+ KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */
+};
+
+const base_jd_prio
+kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = {
+ BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */
+ BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */
+ BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */
+};
+
+
/*
* Private function prototypes
*/
-STATIC INLINE void kbasep_js_deref_permon_check_and_disable_cycle_counter(struct kbase_device *kbdev, struct kbase_jd_atom *katom);
+static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
+ struct kbase_device *kbdev, struct kbase_context *kctx,
+ struct kbasep_js_atom_retained_state *katom_retained_state);
-STATIC INLINE void kbasep_js_ref_permon_check_and_enable_cycle_counter(struct kbase_device *kbdev, struct kbase_jd_atom *katom);
+static int kbase_js_get_slot(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom);
-STATIC kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state);
+static void kbase_js_foreach_ctx_job(struct kbase_context *kctx,
+ kbasep_js_policy_ctx_job_cb callback);
-/** Helper for trace subcodes */
+static bool kbase_js_evict_atom(struct kbase_context *kctx,
+ struct kbase_jd_atom *katom_evict,
+ struct kbase_jd_atom *start_katom,
+ struct kbase_jd_atom *head_katom,
+ struct list_head *evict_list,
+ struct jsctx_rb *rb, int idx);
+
+/* Helper for trace subcodes */
#if KBASE_TRACE_ENABLE
-STATIC int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
+static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
return refcnt;
}
+
+static int kbasep_js_trace_get_refcnt_nolock(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_device_data *js_devdata;
+ int as_nr;
+ int refcnt = 0;
+
+ js_devdata = &kbdev->js_data;
+
+ as_nr = kctx->as_nr;
+ if (as_nr != KBASEP_AS_NR_INVALID) {
+ struct kbasep_js_per_as_data *js_per_as_data;
+
+ js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
+
+ refcnt = js_per_as_data->as_busy_refcount;
+ }
+
+ return refcnt;
+}
#else /* KBASE_TRACE_ENABLE */
-STATIC int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
+static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ CSTD_UNUSED(kbdev);
+ CSTD_UNUSED(kctx);
+ return 0;
+}
+static int kbasep_js_trace_get_refcnt_nolock(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
{
CSTD_UNUSED(kbdev);
CSTD_UNUSED(kctx);
*/
/**
- * Check if the job had performance monitoring enabled and decrement the count. If no jobs require
- * performance monitoring, then the cycle counters will be disabled in the GPU.
- *
- * No locks need to be held - locking is handled further down
- *
- * This function does not sleep.
- */
-
-STATIC INLINE void kbasep_js_deref_permon_check_and_disable_cycle_counter(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(katom != NULL);
-
- if (katom->core_req & BASE_JD_REQ_PERMON)
- kbase_pm_release_gpu_cycle_counter(kbdev);
-}
-
-/**
- * Check if the job has performance monitoring enabled and keep a count of it. If at least one
- * job requires performance monitoring, then the cycle counters will be enabled in the GPU.
+ * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements
+ * @features: JSn_FEATURE register value
*
- * No locks need to be held - locking is handled further down
+ * Given a JSn_FEATURE register value returns the core requirements that match
*
- * The L2 Cache must be ON when this function is called
- *
- * This function does not sleep.
+ * Return: Core requirement bit mask
*/
-
-STATIC INLINE void kbasep_js_ref_permon_check_and_enable_cycle_counter(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
+static base_jd_core_req core_reqs_from_jsn_features(u16 features)
{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(katom != NULL);
+ base_jd_core_req core_req = 0u;
- if (katom->core_req & BASE_JD_REQ_PERMON)
- kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev);
-}
+ if ((features & JS_FEATURE_SET_VALUE_JOB) != 0)
+ core_req |= BASE_JD_REQ_V;
-/*
- * The following locking conditions are made on the caller:
- * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
- * - The caller must hold the kbasep_js_device_data::runpool_mutex
- */
-STATIC INLINE void runpool_inc_context_count(struct kbase_device *kbdev, struct kbase_context *kctx)
-{
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_kctx_info *js_kctx_info;
+ if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0)
+ core_req |= BASE_JD_REQ_CF;
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
+ if ((features & JS_FEATURE_COMPUTE_JOB) != 0)
+ core_req |= BASE_JD_REQ_CS;
- js_devdata = &kbdev->js_data;
- js_kctx_info = &kctx->jctx.sched_info;
+ if ((features & JS_FEATURE_TILER_JOB) != 0)
+ core_req |= BASE_JD_REQ_T;
- BUG_ON(!mutex_is_locked(&js_kctx_info->ctx.jsctx_mutex));
- BUG_ON(!mutex_is_locked(&js_devdata->runpool_mutex));
+ if ((features & JS_FEATURE_FRAGMENT_JOB) != 0)
+ core_req |= BASE_JD_REQ_FS;
- /* Track total contexts */
- KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX);
- ++(js_devdata->nr_all_contexts_running);
+ return core_req;
+}
- if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) == 0) {
- /* Track contexts that can submit jobs */
- KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running < S8_MAX);
- ++(js_devdata->nr_user_contexts_running);
- }
+static void kbase_js_sync_timers(struct kbase_device *kbdev)
+{
+ mutex_lock(&kbdev->js_data.runpool_mutex);
+ kbase_backend_ctx_count_changed(kbdev);
+ mutex_unlock(&kbdev->js_data.runpool_mutex);
}
-/*
- * The following locking conditions are made on the caller:
- * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
- * - The caller must hold the kbasep_js_device_data::runpool_mutex
- */
-STATIC INLINE void runpool_dec_context_count(struct kbase_device *kbdev, struct kbase_context *kctx)
+/* Hold the kbasep_js_device_data::runpool_irq::lock for this */
+bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_kctx_info *js_kctx_info;
+ struct kbasep_js_per_as_data *js_per_as_data;
+ bool result = false;
+ int as_nr;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
-
js_devdata = &kbdev->js_data;
- js_kctx_info = &kctx->jctx.sched_info;
- BUG_ON(!mutex_is_locked(&js_kctx_info->ctx.jsctx_mutex));
- BUG_ON(!mutex_is_locked(&js_devdata->runpool_mutex));
+ as_nr = kctx->as_nr;
+ if (as_nr != KBASEP_AS_NR_INVALID) {
+ int new_refcnt;
+
+ KBASE_DEBUG_ASSERT(as_nr >= 0);
+ js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
+
+ KBASE_DEBUG_ASSERT(js_per_as_data->kctx != NULL);
- /* Track total contexts */
- --(js_devdata->nr_all_contexts_running);
- KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0);
+ new_refcnt = ++(js_per_as_data->as_busy_refcount);
- if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) == 0) {
- /* Track contexts that can submit jobs */
- --(js_devdata->nr_user_contexts_running);
- KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0);
+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx,
+ NULL, 0u, new_refcnt);
+ result = true;
}
+
+ return result;
}
/**
- * @brief check whether the runpool is full for a specified context
+ * jsctx_rb_is_empty_prio(): - Check if ring buffer is empty
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @js: Job slot id to check.
+ * @prio: Priority to check.
*
- * If kctx == NULL, then this makes the least restrictive check on the
- * runpool. A specific context that is supplied immediately after could fail
- * the check, even under the same conditions.
+ * Return true if there are no atoms to pull. There may be running atoms in the
+ * ring buffer even if there are no atoms to pull. It is also possible for the
+ * ring buffer to be full (with running atoms) when this functions returns
+ * true.
*
- * Therefore, once a context is obtained you \b must re-check it with this
- * function, since the return value could change to MALI_FALSE.
+ * Caller must hold runpool_irq.lock
*
- * The following locking conditions are made on the caller:
- * - In all cases, the caller must hold kbasep_js_device_data::runpool_mutex
- * - When kctx != NULL the caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
- * - When kctx == NULL, then the caller need not hold any jsctx_mutex locks (but it doesn't do any harm to do so).
+ * Return: true if there are no atoms to pull, false otherwise.
*/
-STATIC mali_bool check_is_runpool_full(struct kbase_device *kbdev, struct kbase_context *kctx)
+static inline bool
+jsctx_rb_is_empty_prio(struct kbase_context *kctx, int js, int prio)
{
- struct kbasep_js_device_data *js_devdata;
- mali_bool is_runpool_full;
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
- KBASE_DEBUG_ASSERT(kbdev != NULL);
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- js_devdata = &kbdev->js_data;
- BUG_ON(!mutex_is_locked(&js_devdata->runpool_mutex));
+ return rb->read_idx == rb->write_idx;
+}
+
+/**
+ * jsctx_rb_is_empty(): - Check if all priority ring buffers are empty
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @js: Job slot id to check.
+ *
+ * Caller must hold runpool_irq.lock
+ *
+ * Return: true if the ring buffers for all priorities are empty, false
+ * otherwise.
+ */
+static inline bool
+jsctx_rb_is_empty(struct kbase_context *kctx, int js)
+{
+ int prio;
- /* Regardless of whether a context is submitting or not, can't have more than there
- * are HW address spaces */
- is_runpool_full = (mali_bool) (js_devdata->nr_all_contexts_running >= kbdev->nr_hw_address_spaces);
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- if (kctx != NULL && (kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) == 0) {
- BUG_ON(!mutex_is_locked(&kctx->jctx.sched_info.ctx.jsctx_mutex));
- /* Contexts that submit might use less of the address spaces available, due to HW
- * workarounds. In which case, the runpool is also full when the number of
- * submitting contexts exceeds the number of submittable address spaces.
- *
- * Both checks must be made: can have nr_user_address_spaces == nr_hw_address spaces,
- * and at the same time can have nr_user_contexts_running < nr_all_contexts_running. */
- is_runpool_full |= (mali_bool) (js_devdata->nr_user_contexts_running >= kbdev->nr_user_address_spaces);
+ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) {
+ if (!jsctx_rb_is_empty_prio(kctx, js, prio))
+ return false;
}
- return is_runpool_full;
+ return true;
}
-STATIC base_jd_core_req core_reqs_from_jsn_features(u16 features) /* JS<n>_FEATURE register value */
+/**
+ * jsctx_rb_compact_prio(): - Compact a ring buffer
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @js: Job slot id to compact.
+ * @prio: Priority id to compact.
+ */
+static inline void
+jsctx_rb_compact_prio(struct kbase_context *kctx, int js, int prio)
{
- base_jd_core_req core_req = 0u;
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
+ u16 compact_idx = rb->write_idx - 1;
+ u16 end_idx = rb->running_idx - 1;
+ u16 i;
- if ((features & JS_FEATURE_SET_VALUE_JOB) != 0)
- core_req |= BASE_JD_REQ_V;
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
+ lockdep_assert_held(&kctx->jctx.lock);
- if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0)
- core_req |= BASE_JD_REQ_CF;
+ for (i = compact_idx; i != end_idx; i--) {
+ if (rb->entries[i & JSCTX_RB_MASK].atom_id !=
+ KBASEP_ATOM_ID_INVALID) {
+ WARN_ON(compact_idx < rb->running_idx);
+ rb->entries[compact_idx & JSCTX_RB_MASK].atom_id =
+ rb->entries[i & JSCTX_RB_MASK].atom_id;
- if ((features & JS_FEATURE_COMPUTE_JOB) != 0)
- core_req |= BASE_JD_REQ_CS;
+ compact_idx--;
+ }
+ if (rb->read_idx == i)
+ rb->read_idx = compact_idx + 1;
+ }
- if ((features & JS_FEATURE_TILER_JOB) != 0)
- core_req |= BASE_JD_REQ_T;
+ rb->running_idx = compact_idx + 1;
+}
- if ((features & JS_FEATURE_FRAGMENT_JOB) != 0)
- core_req |= BASE_JD_REQ_FS;
+/**
+ * jsctx_rb_compact(): - Compact all priority ring buffers
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @js: Job slot id to compact.
+ */
+static inline void
+jsctx_rb_compact(struct kbase_context *kctx, int js)
+{
+ int prio;
- return core_req;
+ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++)
+ jsctx_rb_compact_prio(kctx, js, prio);
}
/**
- * Picks and reserves an address space.
- *
- * When this function returns, the address space returned is reserved and
- * cannot be picked for another context until it is released.
+ * jsctx_rb_foreach_prio(): - Execute callback for each entry in ring buffer
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @js: Job slot id to iterate.
+ * @prio: Priority id to iterate.
+ * @callback: Function pointer to callback.
*
- * The caller must ensure there \b is a free address space before calling this.
+ * Iterate over a ring buffer and invoke @callback for each entry in buffer, and
+ * remove the entry from the buffer.
*
- * The following locking conditions are made on the caller:
- * - it must hold kbasep_js_device_data::runpool_mutex
+ * If entries are added to the ring buffer while this is running those entries
+ * may, or may not be covered. To ensure that all entries in the buffer have
+ * been enumerated when this function returns jsctx->lock must be held when
+ * calling this function.
*
- * @return a non-NULL pointer to a struct kbase_as that is not in use by any other context
+ * The HW access lock, js_data.runpool_irq.lock, must always be held when
+ * calling this function.
*/
-STATIC struct kbase_as *pick_free_addr_space(struct kbase_device *kbdev)
+static void
+jsctx_rb_foreach_prio(struct kbase_context *kctx, int js, int prio,
+ kbasep_js_policy_ctx_job_cb callback)
{
- struct kbasep_js_device_data *js_devdata;
- struct kbase_as *current_as;
- long ffs_result;
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
+ struct kbase_jd_atom *katom;
+ u16 write_idx = ACCESS_ONCE(rb->write_idx);
- js_devdata = &kbdev->js_data;
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- lockdep_assert_held(&js_devdata->runpool_mutex);
+ /* There must be no jobs currently in HW access */
+ WARN_ON(rb->read_idx != rb->running_idx);
- /* Find the free address space */
- ffs_result = ffs(js_devdata->as_free) - 1;
+ /* Invoke callback on all kbase_jd_atoms in the ring buffer, and
+ * removes them from the buffer */
+ while (rb->read_idx != write_idx) {
+ int id = rb->entries[rb->read_idx & JSCTX_RB_MASK].atom_id;
- /* ASSERT that we should've found a free one */
- KBASE_DEBUG_ASSERT(0 <= ffs_result && ffs_result < kbdev->nr_hw_address_spaces);
- /* Ensure no-one else picks this one */
- js_devdata->as_free &= ~((u16) (1u << ffs_result));
+ katom = kbase_jd_atom_from_id(kctx, id);
- current_as = &kbdev->as[ffs_result];
+ rb->read_idx++;
+ rb->running_idx++;
- return current_as;
+ callback(kctx->kbdev, katom);
+ }
}
/**
- * Release an address space, making it available for being picked again.
+ * jsctx_rb_foreach(): - Execute callback for each entry in all priority rb
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @js: Job slot id to iterate.
+ * @callback: Function pointer to callback.
*
- * The following locking conditions are made on the caller:
- * - it must hold kbasep_js_device_data::runpool_mutex
+ * Iterate over all the different priorities, and for each call
+ * jsctx_rb_foreach_prio() to iterate over the ring buffer and invoke @callback
+ * for each entry in buffer, and remove the entry from the buffer.
*/
-STATIC INLINE void release_addr_space(struct kbase_device *kbdev, int kctx_as_nr)
+static inline void
+jsctx_rb_foreach(struct kbase_context *kctx, int js,
+ kbasep_js_policy_ctx_job_cb callback)
{
- struct kbasep_js_device_data *js_devdata;
- u16 as_bit = (1u << kctx_as_nr);
-
- js_devdata = &kbdev->js_data;
- lockdep_assert_held(&js_devdata->runpool_mutex);
-
- /* The address space must not already be free */
- KBASE_DEBUG_ASSERT(!(js_devdata->as_free & as_bit));
+ int prio;
- js_devdata->as_free |= as_bit;
+ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++)
+ jsctx_rb_foreach_prio(kctx, js, prio, callback);
}
/**
- * Assign an Address Space (AS) to a context, and add the context to the Policy.
- *
- * This includes:
- * - setting up the global runpool_irq structure and the context on the AS
- * - Activating the MMU on the AS
- * - Allowing jobs to be submitted on the AS
- *
- * Locking conditions:
- * - Caller must hold the kbasep_js_kctx_info::jsctx_mutex
- * - Caller must hold the kbasep_js_device_data::runpool_mutex
- * - Caller must hold AS transaction mutex
- * - Caller must hold Runpool IRQ lock
+ * jsctx_rb_peek_prio(): - Check buffer and get next atom
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @js: Job slot id to check.
+ * @prio: Priority id to check.
+ *
+ * Check the ring buffer for the specified @js and @prio and return a pointer to
+ * the next atom, unless the ring buffer is empty.
+ *
+ * Return: Pointer to next atom in buffer, or NULL if there is no atom.
*/
-STATIC void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *current_as)
+static inline struct kbase_jd_atom *
+jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio)
{
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_per_as_data *js_per_as_data;
- int as_nr;
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
+ int id;
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
- KBASE_DEBUG_ASSERT(current_as != NULL);
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- js_devdata = &kbdev->js_data;
- as_nr = current_as->number;
+ if (jsctx_rb_is_empty_prio(kctx, js, prio))
+ return NULL;
- lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
- lockdep_assert_held(&js_devdata->runpool_mutex);
- lockdep_assert_held(¤t_as->transaction_mutex);
- lockdep_assert_held(&js_devdata->runpool_irq.lock);
+ id = rb->entries[rb->read_idx & JSCTX_RB_MASK].atom_id;
+ return kbase_jd_atom_from_id(kctx, id);
+}
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
+/**
+ * jsctx_rb_peek(): - Check all priority buffers and get next atom
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @js: Job slot id to check.
+ *
+ * Check the ring buffers for all priorities, starting from
+ * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a
+ * pointer to the next atom, unless all the priority's ring buffers are empty.
+ *
+ * Return: Pointer to next atom in buffer, or NULL if there is no atom.
+ */
+static inline struct kbase_jd_atom *
+jsctx_rb_peek(struct kbase_context *kctx, int js)
+{
+ int prio;
- /* Attribute handling */
- kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx);
+ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) {
+ struct kbase_jd_atom *katom;
- /* Assign addr space */
- kctx->as_nr = as_nr;
-#ifdef CONFIG_MALI_GATOR_SUPPORT
- kbase_trace_mali_mmu_as_in_use(kctx->as_nr);
-#endif /* CONFIG_MALI_GATOR_SUPPORT */
- /* Activate this address space on the MMU */
- kbase_mmu_update(kctx);
+ katom = jsctx_rb_peek_prio(kctx, js, prio);
+ if (katom)
+ return katom;
+ }
- /* Allow it to run jobs */
- kbasep_js_set_submit_allowed(js_devdata, kctx);
+ return NULL;
+}
+
+/**
+ * jsctx_rb_peek_last(): - Check a ring buffer and get the last atom
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @js: Job slot id to check.
+ * @prio: Priority id to check.
+ *
+ * Check the ring buffer for the specified @js and @prio and return a
+ * pointer to the last atom, unless all the priority's ring buffers are empty.
+ *
+ * The last atom is the atom that was added using jsctx_rb_add() most recently.
+ *
+ * Return: Pointer to last atom in buffer, or NULL if there is no atom.
+ */
+static inline struct kbase_jd_atom *
+jsctx_rb_peek_last(struct kbase_context *kctx, int js, int prio)
+{
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
+ int id;
+
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
+ lockdep_assert_held(&kctx->jctx.lock);
- /* Book-keeping */
- js_per_as_data->kctx = kctx;
- js_per_as_data->as_busy_refcount = 0;
+ if (jsctx_rb_is_empty_prio(kctx, js, prio))
+ return NULL;
- /* Lastly, add the context to the policy's runpool - this really allows it to run jobs */
- kbasep_js_policy_runpool_add_ctx(&js_devdata->policy, kctx);
+ id = rb->entries[(rb->write_idx - 1) & JSCTX_RB_MASK].atom_id;
+ return kbase_jd_atom_from_id(kctx, id);
}
-void kbasep_js_try_run_next_job_nolock(struct kbase_device *kbdev)
+/**
+ * jsctx_rb_pull(): - Mark atom in list as running
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @katom: Pointer to katom to pull.
+ *
+ * Mark an atom previously obtained from jsctx_rb_peek() as running.
+ *
+ * @katom must currently be at the head of the ring buffer.
+ */
+static inline void
+jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
- struct kbasep_js_device_data *js_devdata;
- int js;
+ int prio = katom->sched_priority;
+ int js = katom->slot_nr;
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
- lockdep_assert_held(&js_devdata->runpool_mutex);
- lockdep_assert_held(&js_devdata->runpool_irq.lock);
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- /* It's cheap and simple to retest this here - otherwise we burden the
- * caller with it. In some cases, we do this higher up to optimize out the
- * spinlock. */
- if (js_devdata->nr_user_contexts_running == 0)
- return; /* No contexts present - the GPU might be powered off, so just return */
+ /* Atoms must be pulled in the correct order. */
+ WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio));
- for (js = 0; js < kbdev->gpu_props.num_job_slots; ++js)
- kbasep_js_try_run_next_job_on_slot_nolock(kbdev, js);
+ rb->read_idx++;
}
-/** Hold the kbasep_js_device_data::runpool_irq::lock for this */
-mali_bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
+/**
+ * jsctx_rb_unpull(): - Undo marking of atom in list as running
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @katom: Pointer to katom to unpull.
+ *
+ * Undo jsctx_rb_pull() and put @katom back in the queue.
+ *
+ * jsctx_rb_unpull() must be called on atoms in the same order the atoms were
+ * pulled.
+ */
+static inline void
+jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_per_as_data *js_per_as_data;
- mali_bool result = MALI_FALSE;
- int as_nr;
+ int prio = katom->sched_priority;
+ int js = katom->slot_nr;
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
- js_devdata = &kbdev->js_data;
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- as_nr = kctx->as_nr;
- if (as_nr != KBASEP_AS_NR_INVALID) {
- int new_refcnt;
+ /* Atoms must be unpulled in correct order. */
+ WARN_ON(rb->entries[(rb->read_idx - 1) & JSCTX_RB_MASK].atom_id !=
+ kbase_jd_atom_id(kctx, katom));
- KBASE_DEBUG_ASSERT(as_nr >= 0);
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
+ rb->read_idx--;
+}
- KBASE_DEBUG_ASSERT(js_per_as_data->kctx != NULL);
+/**
+ * jsctx_rb_add(): - Add atom to ring buffer
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @katom: Pointer to katom to add.
+ *
+ * Add @katom to the ring buffer determined by the atom's priority and job slot
+ * number.
+ *
+ * If the ring buffer is full -EBUSY will be returned.
+ *
+ * Return: On success 0 is returned, on failure a negative error code.
+ */
+static int
+jsctx_rb_add_atom(struct kbase_context *kctx, struct kbase_jd_atom *katom)
+{
+ int prio = katom->sched_priority;
+ int js = katom->slot_nr;
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
- new_refcnt = ++(js_per_as_data->as_busy_refcount);
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx, NULL, 0u, new_refcnt);
- result = MALI_TRUE;
- }
+ lockdep_assert_held(&kctx->jctx.lock);
- return result;
+ /* Check if the ring buffer is full */
+ if ((rb->write_idx - rb->running_idx) >= JSCTX_RB_SIZE)
+ return -EBUSY;
+
+ rb->entries[rb->write_idx & JSCTX_RB_MASK].atom_id =
+ kbase_jd_atom_id(kctx, katom);
+ rb->write_idx++;
+
+ return 0;
}
-/*
- * Functions private to KBase ('Protected' functions)
+/**
+ * jsctx_rb_remove(): - Remove atom from ring buffer
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @katom: Pointer to katom to remove.
+ *
+ * Remove @katom from the ring buffer.
+ *
+ * @katom must have been pulled from the buffer earlier by jsctx_rb_pull(), and
+ * atoms must be removed in the same order they were pulled from the ring
+ * buffer.
*/
-void kbase_js_try_run_jobs(struct kbase_device *kbdev)
+static inline void
+jsctx_rb_remove(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
- struct kbasep_js_device_data *js_devdata;
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
+ int prio = katom->sched_priority;
+ int js = katom->slot_nr;
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
- mutex_lock(&js_devdata->runpool_mutex);
- if (js_devdata->nr_user_contexts_running != 0) {
- /* Only try running jobs when we have contexts present, otherwise the GPU might be powered off. */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- kbasep_js_try_run_next_job_nolock(kbdev);
+ /* Atoms must be completed in order. */
+ WARN_ON(rb->entries[rb->running_idx & JSCTX_RB_MASK].atom_id !=
+ kbase_jd_atom_id(kctx, katom));
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
- }
- mutex_unlock(&js_devdata->runpool_mutex);
+ rb->running_idx++;
}
-void kbase_js_try_run_jobs_on_slot(struct kbase_device *kbdev, int js)
+/**
+ * jsctx_rb_evict(): - Evict atom, and dependents, from ring buffer
+ * @kctx: Pointer to kbase context with ring buffer.
+ * @start_katom: Pointer to the first katom to evict.
+ * @head_katom: Pointer to head katom.
+ * @evict_list: Pointer to head of list where evicted atoms are added.
+ *
+ * Iterate over the ring buffer starting at @start_katom and evict @start_atom
+ * and dependent atoms in ring buffer.
+ *
+ * @evict_list and @head_katom is passed on to kbase_js_evict_atom() which will
+ * examine the atom dependencies.
+ *
+ * jsctx_rb_evict() is only called by kbase_js_evict_deps().
+ */
+static void
+jsctx_rb_evict(struct kbase_context *kctx,
+ struct kbase_jd_atom *start_katom,
+ struct kbase_jd_atom *head_katom,
+ struct list_head *evict_list)
{
- unsigned long flags;
- struct kbasep_js_device_data *js_devdata;
+ int prio = start_katom->sched_priority;
+ int js = start_katom->slot_nr;
+ struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js];
+ bool atom_in_rb = false;
+ u16 i, start_idx;
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
+ lockdep_assert_held(&kctx->jctx.lock);
- mutex_lock(&js_devdata->runpool_mutex);
- if (js_devdata->nr_user_contexts_running != 0) {
- /* Only try running jobs when we have contexts present, otherwise the GPU might be powered off. */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ for (i = rb->running_idx; i != rb->write_idx; i++) {
+ if (rb->entries[i & JSCTX_RB_MASK].atom_id ==
+ kbase_jd_atom_id(kctx, start_katom)) {
+ start_idx = i;
+ atom_in_rb = true;
+ break;
+ }
+ }
+
+ /* start_katom must still be in ring buffer. */
+ if (i == rb->write_idx || !atom_in_rb)
+ return;
- kbasep_js_try_run_next_job_on_slot_nolock(kbdev, js);
+ /* Evict all dependencies on same slot. */
+ for (i = start_idx; i != rb->write_idx; i++) {
+ u8 katom_evict;
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ katom_evict = rb->entries[i & JSCTX_RB_MASK].atom_id;
+ if (katom_evict != KBASEP_ATOM_ID_INVALID) {
+ if (!kbase_js_evict_atom(kctx,
+ &kctx->jctx.atoms[katom_evict],
+ start_katom, head_katom,
+ evict_list, rb, i))
+ break;
+ }
}
- mutex_unlock(&js_devdata->runpool_mutex);
}
-mali_error kbasep_js_devdata_init(struct kbase_device * const kbdev)
+/*
+ * Functions private to KBase ('Protected' functions)
+ */
+int kbasep_js_devdata_init(struct kbase_device * const kbdev)
{
- struct kbasep_js_device_data *js_devdata;
- mali_error err;
+ struct kbasep_js_device_data *jsdd;
+ int err;
int i;
u16 as_present;
KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
+ jsdd = &kbdev->js_data;
- KBASE_DEBUG_ASSERT(js_devdata->init_status == JS_DEVDATA_INIT_NONE);
+ KBASE_DEBUG_ASSERT(jsdd->init_status == JS_DEVDATA_INIT_NONE);
- /* These two must be recalculated if nr_hw_address_spaces changes (e.g. for HW workarounds) */
+ /* These two must be recalculated if nr_hw_address_spaces changes
+ * (e.g. for HW workarounds) */
as_present = (1U << kbdev->nr_hw_address_spaces) - 1;
kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
- mali_bool use_workaround_for_security;
+ bool use_workaround;
- use_workaround_for_security = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE;
- if (use_workaround_for_security != MALI_FALSE) {
+ use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE;
+ if (use_workaround) {
dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only");
kbdev->nr_user_address_spaces = 1;
}
}
#ifdef CONFIG_MALI_DEBUG
- /* Soft-stop will be disabled on a single context by default unless softstop_always is set */
- js_devdata->softstop_always = MALI_FALSE;
+ /* Soft-stop will be disabled on a single context by default unless
+ * softstop_always is set */
+ jsdd->softstop_always = false;
#endif /* CONFIG_MALI_DEBUG */
- js_devdata->nr_all_contexts_running = 0;
- js_devdata->nr_user_contexts_running = 0;
- js_devdata->as_free = as_present; /* All ASs initially free */
- js_devdata->runpool_irq.submit_allowed = 0u; /* No ctx allowed to submit */
- memset(js_devdata->runpool_irq.ctx_attr_ref_count, 0, sizeof(js_devdata->runpool_irq.ctx_attr_ref_count));
- memset(js_devdata->runpool_irq.slot_affinities, 0, sizeof(js_devdata->runpool_irq.slot_affinities));
- js_devdata->runpool_irq.slots_blocked_on_affinity = 0u;
- memset(js_devdata->runpool_irq.slot_affinity_refcount, 0, sizeof(js_devdata->runpool_irq.slot_affinity_refcount));
- INIT_LIST_HEAD(&js_devdata->suspended_soft_jobs_list);
+ jsdd->nr_all_contexts_running = 0;
+ jsdd->nr_user_contexts_running = 0;
+ jsdd->nr_contexts_pullable = 0;
+ atomic_set(&jsdd->nr_contexts_runnable, 0);
+ /* All ASs initially free */
+ jsdd->as_free = as_present;
+ /* No ctx allowed to submit */
+ jsdd->runpool_irq.submit_allowed = 0u;
+ memset(jsdd->runpool_irq.ctx_attr_ref_count, 0,
+ sizeof(jsdd->runpool_irq.ctx_attr_ref_count));
+ memset(jsdd->runpool_irq.slot_affinities, 0,
+ sizeof(jsdd->runpool_irq.slot_affinities));
+ memset(jsdd->runpool_irq.slot_affinity_refcount, 0,
+ sizeof(jsdd->runpool_irq.slot_affinity_refcount));
+ INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list);
/* Config attributes */
- js_devdata->scheduling_tick_ns = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS);
- js_devdata->soft_stop_ticks = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS);
- js_devdata->soft_stop_ticks_cl = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL);
- js_devdata->hard_stop_ticks_ss = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS);
- js_devdata->hard_stop_ticks_cl = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL);
- js_devdata->hard_stop_ticks_nss = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS);
- js_devdata->gpu_reset_ticks_ss = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS);
- js_devdata->gpu_reset_ticks_cl = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL);
- js_devdata->gpu_reset_ticks_nss = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS);
- js_devdata->ctx_timeslice_ns = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS);
- js_devdata->cfs_ctx_runtime_init_slices = DEFAULT_JS_CFS_CTX_RUNTIME_INIT_SLICES;
- js_devdata->cfs_ctx_runtime_min_slices = DEFAULT_JS_CFS_CTX_RUNTIME_MIN_SLICES;
+ jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS;
+ jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS;
+ jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL;
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
+ jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS_8408;
+ else
+ jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS;
+ jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL;
+ jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING;
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
+ jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS_8408;
+ else
+ jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS;
+ jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL;
+ jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING;
+ jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS;
+ jsdd->cfs_ctx_runtime_init_slices =
+ DEFAULT_JS_CFS_CTX_RUNTIME_INIT_SLICES;
+ jsdd->cfs_ctx_runtime_min_slices =
+ DEFAULT_JS_CFS_CTX_RUNTIME_MIN_SLICES;
dev_dbg(kbdev->dev, "JS Config Attribs: ");
- dev_dbg(kbdev->dev, "\tscheduling_tick_ns:%u", js_devdata->scheduling_tick_ns);
- dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", js_devdata->soft_stop_ticks);
- dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", js_devdata->soft_stop_ticks_cl);
- dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", js_devdata->hard_stop_ticks_ss);
- dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", js_devdata->hard_stop_ticks_cl);
- dev_dbg(kbdev->dev, "\thard_stop_ticks_nss:%u", js_devdata->hard_stop_ticks_nss);
- dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", js_devdata->gpu_reset_ticks_ss);
- dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", js_devdata->gpu_reset_ticks_cl);
- dev_dbg(kbdev->dev, "\tgpu_reset_ticks_nss:%u", js_devdata->gpu_reset_ticks_nss);
- dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", js_devdata->ctx_timeslice_ns);
- dev_dbg(kbdev->dev, "\tcfs_ctx_runtime_init_slices:%u", js_devdata->cfs_ctx_runtime_init_slices);
- dev_dbg(kbdev->dev, "\tcfs_ctx_runtime_min_slices:%u", js_devdata->cfs_ctx_runtime_min_slices);
+ dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u",
+ jsdd->scheduling_period_ns);
+ dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u",
+ jsdd->soft_stop_ticks);
+ dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u",
+ jsdd->soft_stop_ticks_cl);
+ dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u",
+ jsdd->hard_stop_ticks_ss);
+ dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u",
+ jsdd->hard_stop_ticks_cl);
+ dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u",
+ jsdd->hard_stop_ticks_dumping);
+ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u",
+ jsdd->gpu_reset_ticks_ss);
+ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u",
+ jsdd->gpu_reset_ticks_cl);
+ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u",
+ jsdd->gpu_reset_ticks_dumping);
+ dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u",
+ jsdd->ctx_timeslice_ns);
+ dev_dbg(kbdev->dev, "\tcfs_ctx_runtime_init_slices:%u",
+ jsdd->cfs_ctx_runtime_init_slices);
+ dev_dbg(kbdev->dev, "\tcfs_ctx_runtime_min_slices:%u",
+ jsdd->cfs_ctx_runtime_min_slices);
+
+ if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss &&
+ jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss &&
+ jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping &&
+ jsdd->hard_stop_ticks_dumping <
+ jsdd->gpu_reset_ticks_dumping)) {
+ dev_err(kbdev->dev, "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n");
+ return -EINVAL;
+ }
#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS
- dev_dbg(kbdev->dev, "Job Scheduling Policy Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", js_devdata->soft_stop_ticks, js_devdata->scheduling_tick_ns);
+ dev_dbg(kbdev->dev, "Job Scheduling Policy Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.",
+ jsdd->soft_stop_ticks,
+ jsdd->scheduling_period_ns);
#endif
#if KBASE_DISABLE_SCHEDULING_HARD_STOPS
- dev_dbg(kbdev->dev, "Job Scheduling Policy Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_nss==%u at %uns per tick. Other hard-stops may still occur.", js_devdata->hard_stop_ticks_ss, js_devdata->hard_stop_ticks_nss, js_devdata->scheduling_tick_ns);
+ dev_dbg(kbdev->dev, "Job Scheduling Policy Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.",
+ jsdd->hard_stop_ticks_ss,
+ jsdd->hard_stop_ticks_dumping,
+ jsdd->scheduling_period_ns);
#endif
#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS
dev_dbg(kbdev->dev, "Note: The JS policy's tick timer (if coded) will still be run, but do nothing.");
/* setup the number of irq throttle cycles base on given time */
{
- int irq_throttle_time_us = kbdev->gpu_props.irq_throttle_time_us;
- int irq_throttle_cycles = kbasep_js_convert_us_to_gpu_ticks_max_freq(kbdev, irq_throttle_time_us);
+ int time_us = kbdev->gpu_props.irq_throttle_time_us;
+ int cycles = kbasep_js_convert_us_to_gpu_ticks_max_freq(kbdev,
+ time_us);
- atomic_set(&kbdev->irq_throttle_cycles, irq_throttle_cycles);
+ atomic_set(&kbdev->irq_throttle_cycles, cycles);
}
/* Clear the AS data, including setting NULL pointers */
- memset(&js_devdata->runpool_irq.per_as_data[0], 0, sizeof(js_devdata->runpool_irq.per_as_data));
+ memset(&jsdd->runpool_irq.per_as_data[0], 0,
+ sizeof(jsdd->runpool_irq.per_as_data));
for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i)
- js_devdata->js_reqs[i] = core_reqs_from_jsn_features(kbdev->gpu_props.props.raw_props.js_features[i]);
+ jsdd->js_reqs[i] = core_reqs_from_jsn_features(
+ kbdev->gpu_props.props.raw_props.js_features[i]);
- js_devdata->init_status |= JS_DEVDATA_INIT_CONSTANTS;
+ jsdd->init_status |= JS_DEVDATA_INIT_CONSTANTS;
/* On error, we could continue on: providing none of the below resources
* rely on the ones above */
- mutex_init(&js_devdata->runpool_mutex);
- mutex_init(&js_devdata->queue_mutex);
- spin_lock_init(&js_devdata->runpool_irq.lock);
+ mutex_init(&jsdd->runpool_mutex);
+ mutex_init(&jsdd->queue_mutex);
+ spin_lock_init(&jsdd->runpool_irq.lock);
+ sema_init(&jsdd->schedule_sem, 1);
err = kbasep_js_policy_init(kbdev);
- if (err == MALI_ERROR_NONE)
- js_devdata->init_status |= JS_DEVDATA_INIT_POLICY;
+ if (!err)
+ jsdd->init_status |= JS_DEVDATA_INIT_POLICY;
+
+ for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) {
+ INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i]);
+ INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i]);
+ }
+
+ jsdd->runpool_irq.secure_mode = false;
+ if (kbdev->secure_ops) {
+ /* Make sure secure mode is disabled */
+ kbdev->secure_ops->secure_mode_disable(kbdev);
+ }
/* On error, do no cleanup; this will be handled by the caller(s), since
* we've designed this resource to be safe to terminate on init-fail */
- if (js_devdata->init_status != JS_DEVDATA_INIT_ALL)
- return MALI_ERROR_FUNCTION_FAILED;
+ if (jsdd->init_status != JS_DEVDATA_INIT_ALL)
+ return -EINVAL;
- return MALI_ERROR_NONE;
+ return 0;
}
void kbasep_js_devdata_halt(struct kbase_device *kbdev)
if ((js_devdata->init_status & JS_DEVDATA_INIT_CONSTANTS)) {
s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, };
- /* The caller must de-register all contexts before calling this */
+ /* The caller must de-register all contexts before calling this
+ */
KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0);
- KBASE_DEBUG_ASSERT(memcmp(js_devdata->runpool_irq.ctx_attr_ref_count, zero_ctx_attr_ref_count, sizeof(js_devdata->runpool_irq.ctx_attr_ref_count)) == 0);
+ KBASE_DEBUG_ASSERT(memcmp(
+ js_devdata->runpool_irq.ctx_attr_ref_count,
+ zero_ctx_attr_ref_count,
+ sizeof(zero_ctx_attr_ref_count)) == 0);
CSTD_UNUSED(zero_ctx_attr_ref_count);
}
if ((js_devdata->init_status & JS_DEVDATA_INIT_POLICY))
js_devdata->init_status = JS_DEVDATA_INIT_NONE;
}
-mali_error kbasep_js_kctx_init(struct kbase_context * const kctx)
+int kbasep_js_kctx_init(struct kbase_context * const kctx)
{
struct kbase_device *kbdev;
struct kbasep_js_kctx_info *js_kctx_info;
- mali_error err;
+ int err;
+ int i;
KBASE_DEBUG_ASSERT(kctx != NULL);
kbdev = kctx->kbdev;
KBASE_DEBUG_ASSERT(kbdev != NULL);
+ for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i)
+ INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]);
+
js_kctx_info = &kctx->jctx.sched_info;
KBASE_DEBUG_ASSERT(js_kctx_info->init_status == JS_KCTX_INIT_NONE);
js_kctx_info->ctx.nr_jobs = 0;
- js_kctx_info->ctx.is_scheduled = MALI_FALSE;
- js_kctx_info->ctx.is_dying = MALI_FALSE;
- memset(js_kctx_info->ctx.ctx_attr_ref_count, 0, sizeof(js_kctx_info->ctx.ctx_attr_ref_count));
-
- /* Initially, the context is disabled from submission until the create flags are set */
+ atomic_set(&js_kctx_info->ctx.fault_count, 0);
+ js_kctx_info->ctx.is_scheduled = false;
+ js_kctx_info->ctx.is_dying = false;
+ memset(js_kctx_info->ctx.ctx_attr_ref_count, 0,
+ sizeof(js_kctx_info->ctx.ctx_attr_ref_count));
+
+ /* Initially, the context is disabled from submission until the create
+ * flags are set */
js_kctx_info->ctx.flags = KBASE_CTX_FLAG_SUBMIT_DISABLED;
js_kctx_info->init_status |= JS_KCTX_INIT_CONSTANTS;
init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait);
err = kbasep_js_policy_init_ctx(kbdev, kctx);
- if (err == MALI_ERROR_NONE)
+ if (!err)
js_kctx_info->init_status |= JS_KCTX_INIT_POLICY;
/* On error, do no cleanup; this will be handled by the caller(s), since
* we've designed this resource to be safe to terminate on init-fail */
if (js_kctx_info->init_status != JS_KCTX_INIT_ALL)
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
- return MALI_ERROR_NONE;
+ return 0;
}
void kbasep_js_kctx_term(struct kbase_context *kctx)
struct kbase_device *kbdev;
struct kbasep_js_kctx_info *js_kctx_info;
union kbasep_js_policy *js_policy;
+ int js;
KBASE_DEBUG_ASSERT(kctx != NULL);
if ((js_kctx_info->init_status & JS_KCTX_INIT_CONSTANTS)) {
/* The caller must de-register all jobs before calling this */
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled == MALI_FALSE);
+ KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0);
}
+ mutex_lock(&kbdev->js_data.queue_mutex);
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++)
+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
+ mutex_unlock(&kbdev->js_data.queue_mutex);
+
if ((js_kctx_info->init_status & JS_KCTX_INIT_POLICY))
kbasep_js_policy_term_ctx(js_policy, kctx);
js_kctx_info->init_status = JS_KCTX_INIT_NONE;
}
-/* Evict jobs from the NEXT registers
+/**
+ * kbase_js_ctx_list_add_pullable - Add context to the tail of the per-slot
+ * pullable context queue
+ * @kbdev: Device pointer
+ * @kctx: Context to add to queue
+ * @js: Job slot to use
*
- * The caller must hold:
- * - kbasep_js_kctx_info::ctx::jsctx_mutex
- * - kbasep_js_device_data::runpool_mutex
+ * If the context is on either the pullable or unpullable queues, then it is
+ * removed before being added to the tail.
+ *
+ * This function should be used when queueing a context for the first time, or
+ * re-queueing a context that has been pulled from.
+ *
+ * Caller must hold kbasep_jd_device_data.queue_mutex
+ *
+ * Return: true if caller should call kbase_backend_ctx_count_changed()
*/
-STATIC void kbasep_js_runpool_evict_next_jobs(struct kbase_device *kbdev, struct kbase_context *kctx)
+static bool kbase_js_ctx_list_add_pullable(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ int js)
{
- unsigned long flags;
- int js;
- struct kbasep_js_device_data *js_devdata;
+ bool ret = false;
- js_devdata = &kbdev->js_data;
+ lockdep_assert_held(&kbdev->js_data.queue_mutex);
+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
- BUG_ON(!mutex_is_locked(&kctx->jctx.sched_info.ctx.jsctx_mutex));
- BUG_ON(!mutex_is_locked(&js_devdata->runpool_mutex));
+ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]))
+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
- /* Prevent contexts in the runpool from submitting jobs */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js],
+ &kbdev->js_data.ctx_list_pullable[js]);
- /* There's no need to prevent contexts in the runpool from submitting jobs,
- * because we complete this operation by the time we release the
- * runpool_irq.lock */
+ if (!kctx->slots_pullable) {
+ kbdev->js_data.nr_contexts_pullable++;
+ ret = true;
+ if (!atomic_read(&kctx->atoms_pulled))
+ atomic_inc(&kbdev->js_data.nr_contexts_runnable);
+ }
+ kctx->slots_pullable |= (1 << js);
- /* Evict jobs from the NEXT registers */
- for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
- struct kbase_jm_slot *slot;
- struct kbase_jd_atom *tail;
+ return ret;
+}
- if (!kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), NULL)) {
- /* No job in the NEXT register */
- continue;
- }
+/**
+ * kbase_js_ctx_list_add_pullable_head - Add context to the head of the
+ * per-slot pullable context queue
+ * @kbdev: Device pointer
+ * @kctx: Context to add to queue
+ * @js: Job slot to use
+ *
+ * If the context is on either the pullable or unpullable queues, then it is
+ * removed before being added to the head.
+ *
+ * This function should be used when a context has been scheduled, but no jobs
+ * can currently be pulled from it.
+ *
+ * Caller must hold kbasep_jd_device_data.queue_mutex
+ *
+ * Return: true if caller should call kbase_backend_ctx_count_changed()
+ */
+static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ int js)
+{
+ bool ret = false;
+
+ lockdep_assert_held(&kbdev->js_data.queue_mutex);
+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
+
+ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]))
+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
+
+ list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js],
+ &kbdev->js_data.ctx_list_pullable[js]);
- slot = &kbdev->jm_slots[js];
- tail = kbasep_jm_peek_idx_submit_slot(slot, slot->submitted_nr - 1);
+ if (!kctx->slots_pullable) {
+ kbdev->js_data.nr_contexts_pullable++;
+ ret = true;
+ if (!atomic_read(&kctx->atoms_pulled))
+ atomic_inc(&kbdev->js_data.nr_contexts_runnable);
+ }
+ kctx->slots_pullable |= (1 << js);
+
+ return ret;
+}
- KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, 1);
- /* Clearing job from next registers */
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_NOP, NULL);
+/**
+ * kbase_js_ctx_list_add_unpullable - Add context to the tail of the per-slot
+ * unpullable context queue
+ * @kbdev: Device pointer
+ * @kctx: Context to add to queue
+ * @js: Job slot to use
+ *
+ * The context must already be on the per-slot pullable queue. It will be
+ * removed from the pullable queue before being added to the unpullable queue.
+ *
+ * This function should be used when a context has been pulled from, and there
+ * are no jobs remaining on the specified slot.
+ *
+ * Caller must hold kbasep_jd_device_data.queue_mutex
+ *
+ * Return: true if caller should call kbase_backend_ctx_count_changed()
+ */
+static bool kbase_js_ctx_list_add_unpullable(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ int js)
+{
+ bool ret = false;
- /* Check to see if we did remove a job from the next registers */
- if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) != 0 || kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) != 0) {
- /* The job was successfully cleared from the next registers, requeue it */
- struct kbase_jd_atom *dequeued_katom = kbasep_jm_dequeue_tail_submit_slot(slot);
+ lockdep_assert_held(&kbdev->js_data.queue_mutex);
+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
- KBASE_DEBUG_ASSERT(dequeued_katom == tail);
+ list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js],
+ &kbdev->js_data.ctx_list_unpullable[js]);
- /* Set the next registers to NULL */
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), 0, NULL);
- kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), 0, NULL);
+ if (kctx->slots_pullable == (1 << js)) {
+ kbdev->js_data.nr_contexts_pullable--;
+ ret = true;
+ if (!atomic_read(&kctx->atoms_pulled))
+ atomic_dec(&kbdev->js_data.nr_contexts_runnable);
+ }
+ kctx->slots_pullable &= ~(1 << js);
- KBASE_TRACE_ADD_SLOT(kbdev, JM_SLOT_EVICT, dequeued_katom->kctx, dequeued_katom, dequeued_katom->jc, js);
+ return ret;
+}
- /* Complete the job, indicate that it took no time, and don't start
- * new atoms */
- kbase_jd_done(dequeued_katom, js, NULL, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
- }
- KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, 0);
+/**
+ * kbase_js_ctx_list_remove - Remove context from the per-slot pullable or
+ * unpullable context queues
+ * @kbdev: Device pointer
+ * @kctx: Context to remove from queue
+ * @js: Job slot to use
+ *
+ * The context must already be on one of the queues.
+ *
+ * This function should be used when a context has no jobs on the GPU, and no
+ * jobs remaining for the specified slot.
+ *
+ * Caller must hold kbasep_jd_device_data.queue_mutex
+ *
+ * Return: true if caller should call kbase_backend_ctx_count_changed()
+ */
+static bool kbase_js_ctx_list_remove(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ int js)
+{
+ bool ret = false;
+
+ lockdep_assert_held(&kbdev->js_data.queue_mutex);
+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
+
+ WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]));
+
+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
+
+ if (kctx->slots_pullable == (1 << js)) {
+ kbdev->js_data.nr_contexts_pullable--;
+ ret = true;
+ if (!atomic_read(&kctx->atoms_pulled))
+ atomic_dec(&kbdev->js_data.nr_contexts_runnable);
}
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ kctx->slots_pullable &= ~(1 << js);
+
+ return ret;
}
/**
- * Fast start a higher priority job
- * If the runpool is full, the lower priority contexts with no running jobs
- * will be evicted from the runpool
- *
- * If \a kctx_new is NULL, the first context with no running jobs will be evicted
- *
- * The following locking conditions are made on the caller:
- * - The caller must \b not hold \a kctx_new's
- * kbasep_js_kctx_info::ctx::jsctx_mutex, or that mutex of any ctx in the
- * runpool. This is because \a kctx_new's jsctx_mutex and one of the other
- * scheduled ctx's jsctx_mutex will be obtained internally.
- * - it must \em not hold kbasep_js_device_data::runpool_irq::lock (as this will be
- * obtained internally)
- * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
- * obtained internally)
- * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used
- * internally).
+ * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable
+ * queue.
+ * @kbdev: Device pointer
+ * @js: Job slot to use
+ *
+ * Caller must hold kbasep_jd_device_data::queue_mutex
+ *
+ * Return: Context to use for specified slot.
+ * NULL if no contexts present for specified slot
*/
-STATIC void kbasep_js_runpool_attempt_fast_start_ctx(struct kbase_device *kbdev, struct kbase_context *kctx_new)
+static struct kbase_context *kbase_js_ctx_list_pop_head(
+ struct kbase_device *kbdev,
+ int js)
+{
+ struct kbase_context *kctx;
+
+ lockdep_assert_held(&kbdev->js_data.queue_mutex);
+
+ if (list_empty(&kbdev->js_data.ctx_list_pullable[js]))
+ return NULL;
+
+ kctx = list_entry(kbdev->js_data.ctx_list_pullable[js].next,
+ struct kbase_context,
+ jctx.sched_info.ctx.ctx_list_entry[js]);
+
+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
+
+ return kctx;
+}
+
+/**
+ * kbase_js_ctx_pullable - Return if a context can be pulled from on the
+ * specified slot
+ * @kctx: Context pointer
+ * @js: Job slot to use
+ * @is_scheduled: true if the context is currently scheduled
+ *
+ * Caller must hold runpool_irq.lock
+ *
+ * Return: true if context can be pulled from on specified slot
+ * false otherwise
+ */
+static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js,
+ bool is_scheduled)
{
- unsigned long flags;
struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_kctx_info *js_kctx_new;
- union kbasep_js_policy *js_policy;
- struct kbasep_js_per_as_data *js_per_as_data;
- int evict_as_nr;
- struct kbasep_js_atom_retained_state katom_retained_state;
+ struct kbase_jd_atom *katom;
- KBASE_DEBUG_ASSERT(kbdev != NULL);
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- js_devdata = &kbdev->js_data;
- js_policy = &kbdev->js_data.policy;
+ js_devdata = &kctx->kbdev->js_data;
- if (kctx_new != NULL) {
- js_kctx_new = &kctx_new->jctx.sched_info;
- mutex_lock(&js_kctx_new->ctx.jsctx_mutex);
- } else {
- js_kctx_new = NULL;
- CSTD_UNUSED(js_kctx_new);
+ if (is_scheduled) {
+ if (!kbasep_js_is_submit_allowed(js_devdata, kctx))
+ return false;
+ }
+ katom = jsctx_rb_peek(kctx, js);
+ if (!katom)
+ return false; /* ringbuffer empty */
+ if (atomic_read(&katom->blocked))
+ return false; /* next atom blocked */
+ if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) {
+ if (katom->x_pre_dep->gpu_rb_state ==
+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)
+ return false;
+ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) &&
+ kbase_backend_nr_atoms_on_slot(kctx->kbdev, js))
+ return false;
}
- /* Setup a dummy katom_retained_state */
- kbasep_js_atom_retained_state_init_invalid(&katom_retained_state);
+ return true;
+}
- mutex_lock(&js_devdata->runpool_mutex);
+static bool kbase_js_dep_validate(struct kbase_context *kctx,
+ struct kbase_jd_atom *katom)
+{
+ struct kbase_device *kbdev = kctx->kbdev;
+ bool ret = true;
+ bool has_dep = false, has_x_dep = false;
+ int js = kbase_js_get_slot(kbdev, katom);
+ int prio = katom->sched_priority;
+ int i;
- /* If the runpool is full and either there is no specified context or the specified context is not dying, then
- attempt to fast start the specified context or evict the first context with no running jobs. */
- if (check_is_runpool_full(kbdev, kctx_new) &&
- (!js_kctx_new || (js_kctx_new &&
- !js_kctx_new->ctx.is_dying))) {
- /* No free address spaces - attempt to evict non-running lower priority context */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- for (evict_as_nr = 0; evict_as_nr < kbdev->nr_hw_address_spaces; evict_as_nr++) {
- struct kbase_context *kctx_evict;
-
- js_per_as_data = &js_devdata->runpool_irq.per_as_data[evict_as_nr];
- kctx_evict = js_per_as_data->kctx;
-
- /* Look for the AS which is not currently running */
- if (0 == js_per_as_data->as_busy_refcount && kctx_evict != NULL) {
- /* Now compare the scheduled priority we are considering evicting with the new ctx priority
- * and take into consideration if the scheduled priority is a realtime policy or not.
- * Note that the lower the number, the higher the priority
- */
- if ((kctx_new == NULL) || kbasep_js_policy_ctx_has_priority(js_policy, kctx_evict, kctx_new)) {
- mali_bool retain_result;
- kbasep_js_release_result release_result;
-
- KBASE_TRACE_ADD(kbdev, JS_FAST_START_EVICTS_CTX, kctx_evict, NULL, 0u, (uintptr_t)kctx_new);
-
- /* Retain the ctx to work on it - this shouldn't be able to fail */
- retain_result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx_evict);
- KBASE_DEBUG_ASSERT(retain_result != MALI_FALSE);
- CSTD_UNUSED(retain_result);
-
- /* This will cause the context to be scheduled out on the next runpool_release_ctx(),
- * and also stop its refcount increasing */
- kbasep_js_clear_submit_allowed(js_devdata, kctx_evict);
-
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
- mutex_unlock(&js_devdata->runpool_mutex);
- if (kctx_new != NULL)
- mutex_unlock(&js_kctx_new->ctx.jsctx_mutex);
-
- /* Stop working on the target context, start working on the kctx_evict context */
-
- mutex_lock(&kctx_evict->jctx.sched_info.ctx.jsctx_mutex);
- mutex_lock(&js_devdata->runpool_mutex);
- release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx_evict, &katom_retained_state);
- mutex_unlock(&js_devdata->runpool_mutex);
- /* Only requeue if actually descheduled, which is more robust in case
- * something else retains it (e.g. two high priority contexts racing
- * to evict the same lower priority context) */
- if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u)
- kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx_evict, MALI_TRUE);
-
- mutex_unlock(&kctx_evict->jctx.sched_info.ctx.jsctx_mutex);
-
- /* release_result isn't propogated further:
- * - the caller will be scheduling in a context anyway
- * - which will also cause new jobs to run */
-
- /* ctx fast start has taken place */
- return;
+ for (i = 0; i < 2; i++) {
+ struct kbase_jd_atom *dep_atom = katom->dep[i].atom;
+
+ if (dep_atom) {
+ int dep_js = kbase_js_get_slot(kbdev, dep_atom);
+ int dep_prio = dep_atom->sched_priority;
+
+ /* Dependent atom must already have been submitted */
+ if (!(dep_atom->atom_flags &
+ KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED)) {
+ ret = false;
+ break;
+ }
+
+ /* Dependencies with different priorities can't
+ be represented in the ringbuffer */
+ if (prio != dep_prio) {
+ ret = false;
+ break;
+ }
+
+ if (js == dep_js) {
+ /* Only one same-slot dependency can be
+ * represented in the ringbuffer */
+ if (has_dep) {
+ ret = false;
+ break;
+ }
+ has_dep = true;
+ } else {
+ /* Only one cross-slot dependency can be
+ * represented in the ringbuffer */
+ if (has_x_dep) {
+ ret = false;
+ break;
+ }
+ /* Each dependee atom can only have one
+ * cross-slot dependency */
+ if (dep_atom->x_post_dep) {
+ ret = false;
+ break;
+ }
+ /* The dependee atom can not already be in the
+ * HW access ringbuffer */
+ if (dep_atom->gpu_rb_state !=
+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
+ ret = false;
+ break;
+ }
+ /* The dependee atom can not already have
+ * completed */
+ if (dep_atom->status !=
+ KBASE_JD_ATOM_STATE_IN_JS) {
+ ret = false;
+ break;
+ }
+ /* Cross-slot dependencies must not violate
+ * PRLAM-8987 affinity restrictions */
+ if (kbase_hw_has_issue(kbdev,
+ BASE_HW_ISSUE_8987) &&
+ (js == 2 || dep_js == 2)) {
+ ret = false;
+ break;
+ }
+ has_x_dep = true;
+ }
+
+ if (kbase_jd_katom_dep_type(&katom->dep[i]) ==
+ BASE_JD_DEP_TYPE_DATA &&
+ js == dep_js) {
+ struct kbase_jd_atom *last_atom =
+ jsctx_rb_peek_last(kctx, js,
+ prio);
+
+ /* Last atom on slot must be pre-dep for this
+ * atom */
+ if (last_atom != dep_atom) {
+ ret = false;
+ break;
}
}
+
+ /* Dependency can be represented in ringbuffers */
}
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
}
- /* ctx fast start has not taken place */
- mutex_unlock(&js_devdata->runpool_mutex);
- if (kctx_new != NULL)
- mutex_unlock(&js_kctx_new->ctx.jsctx_mutex);
+ /* If dependencies can be represented by ringbuffer then clear them from
+ * atom structure */
+ if (ret) {
+ for (i = 0; i < 2; i++) {
+ struct kbase_jd_atom *dep_atom = katom->dep[i].atom;
+
+ if (dep_atom) {
+ int dep_js = kbase_js_get_slot(kbdev, dep_atom);
+
+ if ((js != dep_js) &&
+ (dep_atom->status !=
+ KBASE_JD_ATOM_STATE_COMPLETED)
+ && (dep_atom->status !=
+ KBASE_JD_ATOM_STATE_HW_COMPLETED)
+ && (dep_atom->status !=
+ KBASE_JD_ATOM_STATE_UNUSED)) {
+
+ katom->atom_flags |=
+ KBASE_KATOM_FLAG_X_DEP_BLOCKED;
+ katom->x_pre_dep = dep_atom;
+ dep_atom->x_post_dep = katom;
+ if (kbase_jd_katom_dep_type(
+ &katom->dep[i]) ==
+ BASE_JD_DEP_TYPE_DATA)
+ katom->atom_flags |=
+ KBASE_KATOM_FLAG_FAIL_BLOCKER;
+ }
+ if ((kbase_jd_katom_dep_type(&katom->dep[i])
+ == BASE_JD_DEP_TYPE_DATA) &&
+ (js == dep_js))
+ katom->atom_flags |=
+ KBASE_KATOM_FLAG_FAIL_PREV;
+
+ list_del(&katom->dep_item[i]);
+ kbase_jd_katom_dep_clear(&katom->dep[i]);
+ }
+ }
+ }
+
+ return ret;
}
-mali_bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom)
+bool kbasep_js_add_job(struct kbase_context *kctx,
+ struct kbase_jd_atom *atom)
{
unsigned long flags;
struct kbasep_js_kctx_info *js_kctx_info;
struct kbasep_js_device_data *js_devdata;
union kbasep_js_policy *js_policy;
- mali_bool policy_queue_updated = MALI_FALSE;
+ bool enqueue_required = false;
+ bool timer_sync = false;
KBASE_DEBUG_ASSERT(kctx != NULL);
KBASE_DEBUG_ASSERT(atom != NULL);
js_policy = &kbdev->js_data.policy;
js_kctx_info = &kctx->jctx.sched_info;
- KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, atom));
-
+ mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
- /* Policy-specific initialization of atoms (which cannot fail). Anything that
- * could've failed must've been done at kbasep_jd_policy_init_job() time. */
- kbasep_js_policy_register_job(js_policy, kctx, atom);
/*
* Begin Runpool transaction
*/
mutex_lock(&js_devdata->runpool_mutex);
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, kbasep_js_trace_get_refcnt(kbdev, kctx));
/* Refcount ctx.nr_jobs */
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX);
/* Lock for state available during IRQ */
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ if (!kbase_js_dep_validate(kctx, atom)) {
+ /* Dependencies could not be represented */
+ --(js_kctx_info->ctx.nr_jobs);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ mutex_unlock(&js_devdata->runpool_mutex);
+
+ goto out_unlock;
+ }
+
+ KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, atom));
+
+ if (kbase_js_dep_resolved_submit(kctx, atom, &enqueue_required) != 0) {
+ /* Ringbuffer was full (should be impossible) - fail the job */
+ --(js_kctx_info->ctx.nr_jobs);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ mutex_unlock(&js_devdata->runpool_mutex);
+
+ atom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
+
+ goto out_unlock;
+ }
+
+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc,
+ kbasep_js_trace_get_refcnt_nolock(kbdev, kctx));
+
/* Context Attribute Refcounting */
kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom);
- /* Enqueue the job in the policy, causing it to be scheduled if the
- * parent context gets scheduled */
- kbasep_js_policy_enqueue_job(js_policy, atom);
+ if (enqueue_required)
+ timer_sync = kbase_js_ctx_list_add_pullable(kbdev, kctx,
+ atom->slot_nr);
+
+ /* If this context is active and the atom is the first on its slot,
+ * kick the job manager to attempt to fast-start the atom */
+ if (enqueue_required && kctx == kbdev->hwaccess.active_kctx)
+ kbase_jm_try_kick(kbdev, 1 << atom->slot_nr);
- if (js_kctx_info->ctx.is_scheduled != MALI_FALSE) {
- /* Handle an already running context - try to run the new job, in case it
- * matches requirements that aren't matched by any other job in the Run
- * Pool */
- kbasep_js_try_run_next_job_nolock(kbdev);
- }
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ if (timer_sync)
+ kbase_backend_ctx_count_changed(kbdev);
mutex_unlock(&js_devdata->runpool_mutex);
/* End runpool transaction */
- if (js_kctx_info->ctx.is_scheduled == MALI_FALSE) {
+ if (!js_kctx_info->ctx.is_scheduled) {
if (js_kctx_info->ctx.is_dying) {
- /* A job got added while/after kbase_job_zap_context() was called
- * on a non-scheduled context (e.g. KDS dependency resolved). Kill
- * that job by killing the context. */
- kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, MALI_FALSE);
+ /* A job got added while/after kbase_job_zap_context()
+ * was called on a non-scheduled context (e.g. KDS
+ * dependency resolved). Kill that job by killing the
+ * context. */
+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx,
+ false);
} else if (js_kctx_info->ctx.nr_jobs == 1) {
- /* Handle Refcount going from 0 to 1: schedule the context on the Policy Queue */
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled == MALI_FALSE);
+ /* Handle Refcount going from 0 to 1: schedule the
+ * context on the Policy Queue */
+ KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx);
- mutex_lock(&js_devdata->queue_mutex);
- kbasep_js_policy_enqueue_ctx(js_policy, kctx);
- mutex_unlock(&js_devdata->queue_mutex);
-
- /* Policy Queue was updated - caller must try to schedule the head context
- * We also try to encourage a fast-start from here. */
- policy_queue_updated = MALI_TRUE;
+ /* Policy Queue was updated - caller must try to
+ * schedule the head context */
+ WARN_ON(!enqueue_required);
}
}
+out_unlock:
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
- /* If the runpool is full and this job has a higher priority than the
- * non-running job in the runpool - evict it so this higher priority job
- * starts faster. Fast-starting requires the jsctx_mutex to be dropped,
- * because it works on multiple ctxs
- *
- * Note: If the context is being killed with kbase_job_zap_context(), then
- * kctx can't disappear after the jsctx_mutex was dropped. This is because
- * the caller holds kctx->jctx.lock */
- if (policy_queue_updated)
- kbasep_js_runpool_attempt_fast_start_ctx(kbdev, kctx);
+ mutex_unlock(&js_devdata->queue_mutex);
- return policy_queue_updated;
+ return enqueue_required;
}
-void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *atom)
+void kbasep_js_remove_job(struct kbase_device *kbdev,
+ struct kbase_context *kctx, struct kbase_jd_atom *atom)
{
struct kbasep_js_kctx_info *js_kctx_info;
struct kbasep_js_device_data *js_devdata;
js_policy = &kbdev->js_data.policy;
js_kctx_info = &kctx->jctx.sched_info;
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, kbasep_js_trace_get_refcnt(kbdev, kctx));
+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc,
+ kbasep_js_trace_get_refcnt(kbdev, kctx));
/* De-refcount ctx.nr_jobs */
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0);
--(js_kctx_info->ctx.nr_jobs);
-
- /* De-register the job from the system */
- kbasep_js_policy_deregister_job(js_policy, kctx, atom);
}
-void kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
+bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev,
+ struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
unsigned long flags;
struct kbasep_js_atom_retained_state katom_retained_state;
struct kbasep_js_device_data *js_devdata;
- mali_bool attr_state_changed;
+ bool attr_state_changed;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
/* The atom has 'finished' (will not be re-run), so no need to call
* kbasep_js_has_atom_finished().
*
- * This is because it returns MALI_FALSE for soft-stopped atoms, but we
+ * This is because it returns false for soft-stopped atoms, but we
* want to override that, because we're cancelling an atom regardless of
* whether it was soft-stopped or not */
- attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, &katom_retained_state);
+ attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx,
+ &katom_retained_state);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
- if (attr_state_changed != MALI_FALSE) {
- /* A change in runpool ctx attributes might mean we can run more jobs
- * than before. */
- kbase_js_try_run_jobs(kbdev);
- }
+ return attr_state_changed;
}
-mali_bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
+bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
- mali_bool result;
+ bool result;
KBASE_DEBUG_ASSERT(kbdev != NULL);
js_devdata = &kbdev->js_data;
return result;
}
-struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr)
+struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev,
+ int as_nr)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
return found_kctx;
}
+struct kbase_context *kbasep_js_runpool_lookup_ctx_nolock(
+ struct kbase_device *kbdev, int as_nr)
+{
+ struct kbasep_js_device_data *js_devdata;
+ struct kbase_context *found_kctx = NULL;
+ struct kbasep_js_per_as_data *js_per_as_data;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS);
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ js_devdata = &kbdev->js_data;
+ js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
+
+ found_kctx = js_per_as_data->kctx;
+
+ if (found_kctx != NULL)
+ ++(js_per_as_data->as_busy_refcount);
+
+ return found_kctx;
+}
+
/**
- * @brief Try running more jobs after releasing a context and/or atom
+ * kbasep_js_release_result - Try running more jobs after releasing a context
+ * and/or atom
+ *
+ * @kbdev: The kbase_device to operate on
+ * @kctx: The kbase_context to operate on
+ * @katom_retained_state: Retained state from the atom
+ * @runpool_ctx_attr_change: True if the runpool context attributes have changed
*
* This collates a set of actions that must happen whilst
- * kbasep_js_device_data::runpool_irq::lock is held.
+ * kbasep_js_device_data.runpool_irq.lock is held.
*
* This includes running more jobs when:
- * - The previously released kctx caused a ctx attribute change
- * - The released atom caused a ctx attribute change
- * - Slots were previously blocked due to affinity restrictions
- * - Submission during IRQ handling failed
+ * - The previously released kctx caused a ctx attribute change,
+ * - The released atom caused a ctx attribute change,
+ * - Slots were previously blocked due to affinity restrictions,
+ * - Submission during IRQ handling failed.
+ *
+ * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were
+ * changed. The caller should try scheduling all contexts
*/
-STATIC void kbasep_js_run_jobs_after_ctx_and_atom_release(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state, mali_bool runpool_ctx_attr_change)
+static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release(
+ struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ struct kbasep_js_atom_retained_state *katom_retained_state,
+ bool runpool_ctx_attr_change)
{
struct kbasep_js_device_data *js_devdata;
+ kbasep_js_release_result result = 0;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
lockdep_assert_held(&js_devdata->runpool_irq.lock);
if (js_devdata->nr_user_contexts_running != 0) {
- mali_bool retry_submit;
+ bool retry_submit = false;
int retry_jobslot;
- retry_submit = kbasep_js_get_atom_retry_submit_slot(katom_retained_state, &retry_jobslot);
+ if (katom_retained_state)
+ retry_submit = kbasep_js_get_atom_retry_submit_slot(
+ katom_retained_state, &retry_jobslot);
- if (runpool_ctx_attr_change != MALI_FALSE) {
- /* A change in runpool ctx attributes might mean we can run more jobs
- * than before */
- kbasep_js_try_run_next_job_nolock(kbdev);
+ if (runpool_ctx_attr_change || retry_submit) {
+ /* A change in runpool ctx attributes might mean we can
+ * run more jobs than before */
+ result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL;
- /* A retry submit on all slots has now happened, so don't need to do it again */
- retry_submit = MALI_FALSE;
- }
-
- /* Submit on any slots that might've had atoms blocked by the affinity of
- * a completed atom.
- *
- * If no atom has recently completed, then this is harmelss */
- kbase_js_affinity_submit_to_blocked_slots(kbdev);
-
- /* If the IRQ handler failed to get a job from the policy, try again from
- * outside the IRQ handler
- * NOTE: We may've already cleared retry_submit from submitting above */
- if (retry_submit != MALI_FALSE) {
- KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, kctx, NULL, 0u, retry_jobslot);
- kbasep_js_try_run_next_job_on_slot_nolock(kbdev, retry_jobslot);
+ KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB,
+ kctx, NULL, 0u, retry_jobslot);
}
}
+ return result;
}
-/**
+/*
* Internal function to release the reference on a ctx and an atom's "retained
* state", only taking the runpool and as transaction mutexes
*
- * This also starts more jobs running in the case of an ctx-attribute state change
+ * This also starts more jobs running in the case of an ctx-attribute state
+ * change
*
* This does none of the followup actions for scheduling:
* - It does not schedule in a new context
* - Caller holds js_kctx_info->ctx.jsctx_mutex
* - Caller holds js_devdata->runpool_mutex
*/
-STATIC kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state)
+static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal(
+ struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ struct kbasep_js_atom_retained_state *katom_retained_state)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_per_as_data *js_per_as_data;
kbasep_js_release_result release_result = 0u;
- mali_bool runpool_ctx_attr_change = MALI_FALSE;
+ bool runpool_ctx_attr_change = false;
int kctx_as_nr;
struct kbase_as *current_as;
int new_ref_count;
js_policy = &kbdev->js_data.policy;
/* Ensure context really is scheduled in */
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled != MALI_FALSE);
+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled);
/* kctx->as_nr and js_per_as_data are only read from here. The caller's
* js_ctx_mutex provides a barrier that ensures they are up-to-date.
* Assert about out calling contract
*/
current_as = &kbdev->as[kctx_as_nr];
+ mutex_lock(&kbdev->pm.lock);
mutex_lock(¤t_as->transaction_mutex);
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr);
new_ref_count = --(js_per_as_data->as_busy_refcount);
/* Release the atom if it finished (i.e. wasn't soft-stopped) */
- if (kbasep_js_has_atom_finished(katom_retained_state) != MALI_FALSE)
- runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, katom_retained_state);
+ if (kbasep_js_has_atom_finished(katom_retained_state))
+ runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom(
+ kbdev, kctx, katom_retained_state);
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u, new_ref_count);
+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u,
+ new_ref_count);
- if (new_ref_count == 1 && kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_PRIVILEGED &&
+ if (new_ref_count == 1 && kctx->jctx.sched_info.ctx.flags &
+ KBASE_CTX_FLAG_PRIVILEGED &&
!kbase_pm_is_suspending(kbdev)) {
- /* Context is kept scheduled into an address space even when there are no jobs, in this case we have
- * to handle the situation where all jobs have been evicted from the GPU and submission is disabled.
+ /* Context is kept scheduled into an address space even when
+ * there are no jobs, in this case we have to handle the
+ * situation where all jobs have been evicted from the GPU and
+ * submission is disabled.
*
- * At this point we re-enable submission to allow further jobs to be executed
+ * At this point we re-enable submission to allow further jobs
+ * to be executed
*/
kbasep_js_set_submit_allowed(js_devdata, kctx);
}
/* Make a set of checks to see if the context should be scheduled out */
- if (new_ref_count == 0 && (kctx->jctx.sched_info.ctx.nr_jobs == 0 || kbasep_js_is_submit_allowed(js_devdata, kctx) == MALI_FALSE)) {
- /* Last reference, and we've been told to remove this context from the Run Pool */
- dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because as_busy_refcount=%d, jobs=%d, allowed=%d", kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, kbasep_js_is_submit_allowed(js_devdata, kctx));
-
- kbasep_js_policy_runpool_remove_ctx(js_policy, kctx);
-
- /* Stop any more refcounts occuring on the context */
- js_per_as_data->kctx = NULL;
-
- /* Ensure we prevent the context from submitting any new jobs
- * e.g. from kbasep_js_try_run_next_job_on_slot_irq_nolock() */
- kbasep_js_clear_submit_allowed(js_devdata, kctx);
-
- /* Disable the MMU on the affected address space, and indicate it's invalid */
- kbase_mmu_disable(kctx);
-
-#ifdef CONFIG_MALI_GATOR_SUPPORT
+ if (new_ref_count == 0 &&
+ (!kbasep_js_is_submit_allowed(js_devdata, kctx) ||
+ kbdev->pm.suspending)) {
+ /* Last reference, and we've been told to remove this context
+ * from the Run Pool */
+ dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because as_busy_refcount=%d, jobs=%d, allowed=%d",
+ kctx, new_ref_count, js_kctx_info->ctx.nr_jobs,
+ kbasep_js_is_submit_allowed(js_devdata, kctx));
+
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
kbase_trace_mali_mmu_as_released(kctx->as_nr);
-#endif /* CONFIG_MALI_GATOR_SUPPORT */
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_tl_nret_gpu_ctx(kbdev, kctx);
+#endif
- kctx->as_nr = KBASEP_AS_NR_INVALID;
+ kbase_backend_release_ctx_irq(kbdev, kctx);
+
+ if (kbdev->hwaccess.active_kctx == kctx)
+ kbdev->hwaccess.active_kctx = NULL;
/* Ctx Attribute handling
*
- * Releasing atoms attributes must either happen before this, or after
- * 'is_scheduled' is changed, otherwise we double-decount the attributes*/
- runpool_ctx_attr_change |= kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx);
-
- /* Early update of context count, to optimize the
- * kbasep_js_run_jobs_after_ctx_and_atom_release() call */
- runpool_dec_context_count(kbdev, kctx);
-
- /* Releasing the context and katom retained state can allow more jobs to run */
- kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, katom_retained_state, runpool_ctx_attr_change);
+ * Releasing atoms attributes must either happen before this, or
+ * after 'is_scheduled' is changed, otherwise we double-decount
+ * the attributes */
+ runpool_ctx_attr_change |=
+ kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx);
+
+ /* Releasing the context and katom retained state can allow
+ * more jobs to run */
+ release_result |=
+ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev,
+ kctx, katom_retained_state,
+ runpool_ctx_attr_change);
/*
* Transaction ends on AS and runpool_irq:
*
- * By this point, the AS-related data is now clear and ready for re-use.
+ * By this point, the AS-related data is now clear and ready
+ * for re-use.
*
- * Since releases only occur once for each previous successful retain, and no more
- * retains are allowed on this context, no other thread will be operating in this
+ * Since releases only occur once for each previous successful
+ * retain, and no more retains are allowed on this context, no
+ * other thread will be operating in this
* code whilst we are
*/
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ kbase_backend_release_ctx_noirq(kbdev, kctx);
+
mutex_unlock(¤t_as->transaction_mutex);
+ mutex_unlock(&kbdev->pm.lock);
- /* Free up the address space */
- release_addr_space(kbdev, kctx_as_nr);
/* Note: Don't reuse kctx_as_nr now */
/* Synchronize with any policy timers */
- kbasep_js_policy_runpool_timers_sync(js_policy);
+ kbase_backend_ctx_count_changed(kbdev);
/* update book-keeping info */
- js_kctx_info->ctx.is_scheduled = MALI_FALSE;
- /* Signal any waiter that the context is not scheduled, so is safe for
- * termination - once the jsctx_mutex is also dropped, and jobs have
- * finished. */
+ js_kctx_info->ctx.is_scheduled = false;
+ /* Signal any waiter that the context is not scheduled, so is
+ * safe for termination - once the jsctx_mutex is also dropped,
+ * and jobs have finished. */
wake_up(&js_kctx_info->ctx.is_scheduled_wait);
/* Queue an action to occur after we've dropped the lock */
release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED;
-
} else {
- kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, katom_retained_state, runpool_ctx_attr_change);
+ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx,
+ katom_retained_state, runpool_ctx_attr_change);
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
mutex_unlock(¤t_as->transaction_mutex);
+ mutex_unlock(&kbdev->pm.lock);
}
return release_result;
}
-void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, mali_bool has_pm_ref)
+void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_atom_retained_state katom_retained_state;
+
+ /* Setup a dummy katom_retained_state */
+ kbasep_js_atom_retained_state_init_invalid(&katom_retained_state);
+
+ kbasep_js_runpool_release_ctx_internal(kbdev, kctx,
+ &katom_retained_state);
+}
+
+void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx, bool has_pm_ref)
{
struct kbasep_js_device_data *js_devdata;
union kbasep_js_policy *js_policy;
js_devdata = &kbdev->js_data;
/* This is called if and only if you've you've detached the context from
- * the Runpool or the Policy Queue, and not added it back to the Runpool */
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled == MALI_FALSE);
-
- if (js_kctx_info->ctx.is_dying != MALI_FALSE) {
- /* Dying: don't requeue, but kill all jobs on the context. This happens
- * asynchronously */
- dev_dbg(kbdev->dev, "JS: ** Killing Context %p on RunPool Remove **", kctx);
- kbasep_js_policy_foreach_ctx_job(js_policy, kctx, &kbase_jd_cancel, MALI_TRUE);
- } else if (js_kctx_info->ctx.nr_jobs > 0) {
- /* Not dying, has jobs: de-ref core counts from each job before addding
- * back to the queue */
- kbasep_js_policy_foreach_ctx_job(js_policy, kctx, &kbasep_js_job_check_deref_cores, MALI_FALSE);
-
- dev_dbg(kbdev->dev, "JS: Requeue Context %p", kctx);
- mutex_lock(&js_devdata->queue_mutex);
- kbasep_js_policy_enqueue_ctx(js_policy, kctx);
- mutex_unlock(&js_devdata->queue_mutex);
- } else {
- /* Not dying, no jobs: don't add back to the queue */
- dev_dbg(kbdev->dev, "JS: Idling Context %p (not requeued)", kctx);
- }
-
- if (has_pm_ref) {
- /* In all cases where we had a pm active refcount, release it */
- kbase_pm_context_idle(kbdev);
+ * the Runpool or the Policy Queue, and not added it back to the Runpool
+ */
+ KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
+
+ if (js_kctx_info->ctx.is_dying) {
+ /* Dying: don't requeue, but kill all jobs on the context. This
+ * happens asynchronously */
+ dev_dbg(kbdev->dev,
+ "JS: ** Killing Context %p on RunPool Remove **", kctx);
+ kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel);
}
}
-void kbasep_js_runpool_release_ctx_and_katom_retained_state(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state)
+void kbasep_js_runpool_release_ctx_and_katom_retained_state(
+ struct kbase_device *kbdev, struct kbase_context *kctx,
+ struct kbasep_js_atom_retained_state *katom_retained_state)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
+ base_jd_event_code event_code;
kbasep_js_release_result release_result;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_kctx_info = &kctx->jctx.sched_info;
js_devdata = &kbdev->js_data;
+ event_code = katom_retained_state->event_code;
+ mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
- release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, katom_retained_state);
+
+#if 2 == MALI_INSTRUMENTATION_LEVEL
+ /* When any fault is detected, stop the context from submitting more
+ * and add a refcount to prevent the context from being removed while
+ * the job core dump is undergoing in the user space. Once the dump
+ * finish, it will release the refcount of the context and allow it
+ * to be removed. The test conditions are to ensure this mechanism
+ * will be triggered only in the cases that cmarp_event_handler
+ * handles.
+ *
+ * Currently, cmar event handler only handles job exceptions and
+ * assert the cases where the event code of the atom does not
+ * belong to the MMU exceptions or GPU exceptions class. In order to
+ * perform dump on error in those cases, changes in cmar event handler
+ * need to be made.
+ */
+ if ((BASE_JD_EVENT_NOT_STARTED != event_code) &&
+ (BASE_JD_EVENT_STOPPED != event_code) &&
+ (BASE_JD_EVENT_ACTIVE != event_code) &&
+ (!((event_code >= BASE_JD_EVENT_RANGE_KERNEL_ONLY_START) &&
+ (event_code <= BASE_JD_EVENT_RANGE_KERNEL_ONLY_END))) &&
+ ((event_code & BASE_JD_SW_EVENT) ||
+ event_code <= BASE_JD_EVENT_UNKNOWN) &&
+ (BASE_JD_EVENT_DONE != event_code)) {
+ unsigned long flags;
+
+ atomic_inc(&js_kctx_info->ctx.fault_count);
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ kbasep_js_clear_submit_allowed(js_devdata, kctx);
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ kbasep_js_runpool_retain_ctx(kbdev, kctx);
+ }
+#endif /* 2 == MALI_INSTRUMENTATION_LEVEL */
+
+ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx,
+ katom_retained_state);
/* Drop the runpool mutex to allow requeing kctx */
mutex_unlock(&js_devdata->runpool_mutex);
+
if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u)
- kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, MALI_TRUE);
+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true);
/* Drop the jsctx_mutex to allow scheduling in a new context */
+
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
- if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) {
- /* We've freed up an address space, so let's try to schedule in another
- * context
- *
- * Note: if there's a context to schedule in, then it also tries to run
- * another job, in case the new context has jobs satisfying requirements
- * that no other context/job in the runpool does */
- kbasep_js_try_schedule_head_ctx(kbdev);
- }
+ mutex_unlock(&js_devdata->queue_mutex);
+
+ if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL)
+ kbase_js_sched_all(kbdev);
+}
+
+void kbasep_js_dump_fault_term(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ unsigned long flags;
+ struct kbasep_js_device_data *js_devdata;
+
+ js_devdata = &kbdev->js_data;
+
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ kbasep_js_set_submit_allowed(js_devdata, kctx);
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ kbasep_js_runpool_release_ctx(kbdev, kctx);
+ atomic_dec(&kctx->jctx.sched_info.ctx.fault_count);
}
-void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
+
+void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
{
struct kbasep_js_atom_retained_state katom_retained_state;
kbasep_js_atom_retained_state_init_invalid(&katom_retained_state);
- kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state);
+ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx,
+ &katom_retained_state);
}
-/** Variant of kbasep_js_runpool_release_ctx() that doesn't call into
- * kbasep_js_try_schedule_head_ctx() */
-STATIC void kbasep_js_runpool_release_ctx_no_schedule(struct kbase_device *kbdev, struct kbase_context *kctx)
+/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into
+ * kbase_js_sched_all() */
+static void kbasep_js_runpool_release_ctx_no_schedule(
+ struct kbase_device *kbdev, struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
kbasep_js_release_result release_result;
struct kbasep_js_atom_retained_state katom_retained_state_struct;
- struct kbasep_js_atom_retained_state *katom_retained_state = &katom_retained_state_struct;
+ struct kbasep_js_atom_retained_state *katom_retained_state =
+ &katom_retained_state_struct;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
mutex_lock(&js_devdata->runpool_mutex);
- release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, katom_retained_state);
+
+ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx,
+ katom_retained_state);
/* Drop the runpool mutex to allow requeing kctx */
mutex_unlock(&js_devdata->runpool_mutex);
if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u)
- kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, MALI_TRUE);
+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true);
/* Drop the jsctx_mutex to allow scheduling in a new context */
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
/* NOTE: could return release_result if the caller would like to know
- * whether it should schedule a new context, but currently no callers do */
+ * whether it should schedule a new context, but currently no callers do
+ */
}
-
/**
- * @brief Handle retaining cores for power management and affinity management,
- * ensuring that cores are powered up and won't violate affinity restrictions.
- *
- * This function enters at the following @ref enum kbase_atom_coreref_state states:
- *
- * - NO_CORES_REQUESTED,
- * - WAITING_FOR_REQUESTED_CORES,
- * - RECHECK_AFFINITY,
- *
- * The transitions are as folows:
- * - NO_CORES_REQUESTED -> WAITING_FOR_REQUESTED_CORES
- * - WAITING_FOR_REQUESTED_CORES -> ( WAITING_FOR_REQUESTED_CORES or RECHECK_AFFINITY )
- * - RECHECK_AFFINITY -> ( WAITING_FOR_REQUESTED_CORES or CHECK_AFFINITY_VIOLATIONS )
- * - CHECK_AFFINITY_VIOLATIONS -> ( RECHECK_AFFINITY or READY )
- *
- * The caller must hold:
- * - kbasep_js_device_data::runpool_irq::lock
- *
- * @return MALI_FALSE when the function makes a transition to the same or lower state, indicating
- * that the cores are not ready.
- * @return MALI_TRUE once READY state is reached, indicating that the cores are 'ready' and won't
- * violate affinity restrictions.
+ * kbase_js_set_timeouts - update all JS timeouts with user specified data
+ * @kbdev: Device pointer
*
+ * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is
+ * set to a positive number then that becomes the new value used, if a timeout
+ * is negative then the default is set.
*/
-STATIC mali_bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev, int js, struct kbase_jd_atom *katom)
-{
- /* The most recently checked affinity. Having this at this scope allows us
- * to guarantee that we've checked the affinity in this function call. */
- u64 recently_chosen_affinity = 0;
- mali_bool chosen_affinity = MALI_FALSE;
- mali_bool retry;
-
- do {
- retry = MALI_FALSE;
-
- /* NOTE: The following uses a number of FALLTHROUGHs to optimize the
- * calls to this function. Ending of the function is indicated by BREAK OUT */
- switch (katom->coreref_state) {
- /* State when job is first attempted to be run */
- case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
- KBASE_DEBUG_ASSERT(katom->affinity == 0);
-
- /* Compute affinity */
- if (MALI_FALSE == kbase_js_choose_affinity(&recently_chosen_affinity, kbdev, katom, js)) {
- /* No cores are currently available */
- /* *** BREAK OUT: No state transition *** */
- break;
- }
-
- chosen_affinity = MALI_TRUE;
-
- /* Request the cores */
- kbase_pm_request_cores(kbdev, katom->core_req & BASE_JD_REQ_T, recently_chosen_affinity);
-
- katom->affinity = recently_chosen_affinity;
-
- /* Proceed to next state */
- katom->coreref_state = KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
+static void kbase_js_set_timeouts(struct kbase_device *kbdev)
+{
+ struct kbasep_js_device_data *js_data = &kbdev->js_data;
+
+ if (kbdev->js_scheduling_period_ns < 0)
+ js_data->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS;
+ else if (kbdev->js_scheduling_period_ns > 0)
+ js_data->scheduling_period_ns = kbdev->js_scheduling_period_ns;
+
+ if (kbdev->js_soft_stop_ticks < 0)
+ js_data->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS;
+ else if (kbdev->js_soft_stop_ticks > 0)
+ js_data->soft_stop_ticks = kbdev->js_soft_stop_ticks;
+
+ if (kbdev->js_soft_stop_ticks_cl < 0)
+ js_data->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL;
+ else if (kbdev->js_soft_stop_ticks_cl > 0)
+ js_data->soft_stop_ticks_cl = kbdev->js_soft_stop_ticks_cl;
+
+ if (kbdev->js_hard_stop_ticks_ss < 0) {
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
+ js_data->hard_stop_ticks_ss =
+ DEFAULT_JS_HARD_STOP_TICKS_SS_8408;
+ else
+ js_data->hard_stop_ticks_ss =
+ DEFAULT_JS_HARD_STOP_TICKS_SS;
+ } else if (kbdev->js_hard_stop_ticks_ss > 0) {
+ js_data->hard_stop_ticks_ss = kbdev->js_hard_stop_ticks_ss;
+ }
- /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+ if (kbdev->js_hard_stop_ticks_cl < 0)
+ js_data->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL;
+ else if (kbdev->js_hard_stop_ticks_cl > 0)
+ js_data->hard_stop_ticks_cl = kbdev->js_hard_stop_ticks_cl;
+
+ if (kbdev->js_hard_stop_ticks_dumping < 0)
+ js_data->hard_stop_ticks_dumping =
+ DEFAULT_JS_HARD_STOP_TICKS_DUMPING;
+ else if (kbdev->js_hard_stop_ticks_dumping > 0)
+ js_data->hard_stop_ticks_dumping =
+ kbdev->js_hard_stop_ticks_dumping;
+
+ if (kbdev->js_reset_ticks_ss < 0) {
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408))
+ js_data->gpu_reset_ticks_ss =
+ DEFAULT_JS_RESET_TICKS_SS_8408;
+ else
+ js_data->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS;
+ } else if (kbdev->js_reset_ticks_ss > 0) {
+ js_data->gpu_reset_ticks_ss = kbdev->js_reset_ticks_ss;
+ }
- case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
- {
- enum kbase_pm_cores_ready cores_ready;
- KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T));
+ if (kbdev->js_reset_ticks_cl < 0)
+ js_data->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL;
+ else if (kbdev->js_reset_ticks_cl > 0)
+ js_data->gpu_reset_ticks_cl = kbdev->js_reset_ticks_cl;
+
+ if (kbdev->js_reset_ticks_dumping < 0)
+ js_data->gpu_reset_ticks_dumping =
+ DEFAULT_JS_RESET_TICKS_DUMPING;
+ else if (kbdev->js_reset_ticks_dumping > 0)
+ js_data->gpu_reset_ticks_dumping =
+ kbdev->js_reset_ticks_dumping;
+}
- cores_ready = kbase_pm_register_inuse_cores(kbdev, katom->core_req & BASE_JD_REQ_T, katom->affinity);
- if (cores_ready == KBASE_NEW_AFFINITY) {
- /* Affinity no longer valid - return to previous state */
- kbasep_js_job_check_deref_cores(kbdev, katom);
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REGISTER_INUSE_FAILED, katom->kctx, katom, katom->jc, js, (u32) katom->affinity);
- /* *** BREAK OUT: Return to previous state, retry *** */
- retry = MALI_TRUE;
- break;
- }
- if (cores_ready == KBASE_CORES_NOT_READY) {
- /* Stay in this state and return, to retry at this state later */
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REGISTER_INUSE_FAILED, katom->kctx, katom, katom->jc, js, (u32) katom->affinity);
- /* *** BREAK OUT: No state transition *** */
- break;
- }
- /* Proceed to next state */
- katom->coreref_state = KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
- }
+static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_device_data *js_devdata;
+ struct kbasep_js_kctx_info *js_kctx_info;
+ union kbasep_js_policy *js_policy;
+ struct kbase_as *new_address_space = NULL;
+ unsigned long flags;
+ bool kctx_suspended = false;
+ int as_nr;
- /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
+ js_devdata = &kbdev->js_data;
+ js_policy = &kbdev->js_data.policy;
+ js_kctx_info = &kctx->jctx.sched_info;
- case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
- KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T));
+ /* Pick available address space for this context */
+ as_nr = kbase_backend_find_free_address_space(kbdev, kctx);
- /* Optimize out choosing the affinity twice in the same function call */
- if (chosen_affinity == MALI_FALSE) {
- /* See if the affinity changed since a previous call. */
- if (MALI_FALSE == kbase_js_choose_affinity(&recently_chosen_affinity, kbdev, katom, js)) {
- /* No cores are currently available */
- kbasep_js_job_check_deref_cores(kbdev, katom);
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REQUEST_ON_RECHECK_FAILED, katom->kctx, katom, katom->jc, js, (u32) recently_chosen_affinity);
- /* *** BREAK OUT: Transition to lower state *** */
- break;
- }
- chosen_affinity = MALI_TRUE;
- }
+ if (as_nr == KBASEP_AS_NR_INVALID)
+ return false; /* No address spaces currently available */
- /* Now see if this requires a different set of cores */
- if (recently_chosen_affinity != katom->affinity) {
- enum kbase_pm_cores_ready cores_ready;
-
- kbase_pm_request_cores(kbdev, katom->core_req & BASE_JD_REQ_T, recently_chosen_affinity);
-
- /* Register new cores whilst we still hold the old ones, to minimize power transitions */
- cores_ready = kbase_pm_register_inuse_cores(kbdev, katom->core_req & BASE_JD_REQ_T, recently_chosen_affinity);
- kbasep_js_job_check_deref_cores(kbdev, katom);
-
- /* Fixup the state that was reduced by deref_cores: */
- katom->coreref_state = KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
- katom->affinity = recently_chosen_affinity;
- if (cores_ready == KBASE_NEW_AFFINITY) {
- /* Affinity no longer valid - return to previous state */
- katom->coreref_state = KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
- kbasep_js_job_check_deref_cores(kbdev, katom);
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REGISTER_INUSE_FAILED, katom->kctx, katom, katom->jc, js, (u32) katom->affinity);
- /* *** BREAK OUT: Return to previous state, retry *** */
- retry = MALI_TRUE;
- break;
- }
- /* Now might be waiting for powerup again, with a new affinity */
- if (cores_ready == KBASE_CORES_NOT_READY) {
- /* Return to previous state */
- katom->coreref_state = KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REGISTER_ON_RECHECK_FAILED, katom->kctx, katom, katom->jc, js, (u32) katom->affinity);
- /* *** BREAK OUT: Transition to lower state *** */
- break;
- }
- }
- /* Proceed to next state */
- katom->coreref_state = KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS;
-
- /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
- case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS:
- KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T));
- KBASE_DEBUG_ASSERT(katom->affinity == recently_chosen_affinity);
-
- /* Note: this is where the caller must've taken the runpool_irq.lock */
-
- /* Check for affinity violations - if there are any, then we just ask
- * the caller to requeue and try again later */
- if (kbase_js_affinity_would_violate(kbdev, js, katom->affinity) != MALI_FALSE) {
- /* Cause a re-attempt to submit from this slot on the next job complete */
- kbase_js_affinity_slot_blocked_an_atom(kbdev, js);
- /* Return to previous state */
- katom->coreref_state = KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
- /* *** BREAK OUT: Transition to lower state *** */
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_AFFINITY_WOULD_VIOLATE, katom->kctx, katom, katom->jc, js, (u32) katom->affinity);
- break;
- }
+ new_address_space = &kbdev->as[as_nr];
- /* No affinity violations would result, so the cores are ready */
- katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY;
- /* *** BREAK OUT: Cores Ready *** */
- break;
+ /*
+ * Atomic transaction on the Context and Run Pool begins
+ */
+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_lock(&js_devdata->runpool_mutex);
- default:
- KBASE_DEBUG_ASSERT_MSG(MALI_FALSE, "Unhandled kbase_atom_coreref_state %d", katom->coreref_state);
- break;
+ /* Check to see if context is dying due to kbase_job_zap_context() */
+ if (js_kctx_info->ctx.is_dying) {
+ /* Roll back the transaction so far and return */
+ kbase_backend_release_free_address_space(kbdev, as_nr);
+
+ mutex_unlock(&js_devdata->runpool_mutex);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+
+ return false;
+ }
+
+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL,
+ 0u,
+ kbasep_js_trace_get_refcnt(kbdev, kctx));
+
+ if (js_devdata->nr_user_contexts_running == 0 &&
+ kbdev->js_timeouts_updated) {
+ /* Only when there are no other contexts submitting jobs:
+ * Latch in run-time job scheduler timeouts that were set
+ * through js_timeouts sysfs file */
+ kbase_js_set_timeouts(kbdev);
+
+ kbdev->js_timeouts_updated = false;
+ }
+
+ js_kctx_info->ctx.is_scheduled = true;
+
+ mutex_lock(&new_address_space->transaction_mutex);
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ /* Assign context to previously chosen address space */
+ if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) {
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ mutex_unlock(&new_address_space->transaction_mutex);
+ /* If address space is not pending, then kbase_backend_use_ctx()
+ * failed. Roll back the transaction so far and return */
+ if (!kctx->as_pending) {
+ js_kctx_info->ctx.is_scheduled = false;
+
+ kbase_backend_release_free_address_space(kbdev, as_nr);
}
- } while (retry != MALI_FALSE);
- return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY);
-}
+ mutex_unlock(&js_devdata->runpool_mutex);
-void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(katom != NULL);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ return false;
+ }
- switch (katom->coreref_state) {
- case KBASE_ATOM_COREREF_STATE_READY:
- /* State where atom was submitted to the HW - just proceed to power-down */
- KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T));
+ kbdev->hwaccess.active_kctx = kctx;
- /* *** FALLTHROUGH *** */
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
+ kbase_trace_mali_mmu_as_in_use(kctx->as_nr);
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_tl_ret_gpu_ctx(kbdev, kctx);
+#endif
- case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
- /* State where cores were registered */
- KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T));
- kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T, katom->affinity);
+ /* Cause any future waiter-on-termination to wait until the context is
+ * descheduled */
+ wake_up(&js_kctx_info->ctx.is_scheduled_wait);
- /* Note: We do not clear the state for kbase_js_affinity_slot_blocked_an_atom().
- * That is handled after finishing the job. This might be slightly
- * suboptimal for some corner cases, but is otherwise not a problem
- * (and resolves itself after the next job completes). */
+ /* Re-check for suspending: a suspend could've occurred, and all the
+ * contexts could've been removed from the runpool before we took this
+ * lock. In this case, we don't want to allow this context to run jobs,
+ * we just want it out immediately.
+ *
+ * The DMB required to read the suspend flag was issued recently as part
+ * of the runpool_irq locking. If a suspend occurs *after* that lock was
+ * taken (i.e. this condition doesn't execute), then the
+ * kbasep_js_suspend() code will cleanup this context instead (by virtue
+ * of it being called strictly after the suspend flag is set, and will
+ * wait for this lock to drop) */
+ if (kbase_pm_is_suspending(kbdev)) {
+ /* Cause it to leave at some later point */
+ bool retained;
+
+ retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
+ KBASE_DEBUG_ASSERT(retained);
+
+ kbasep_js_clear_submit_allowed(js_devdata, kctx);
+ kctx_suspended = true;
+ }
- break;
+ /* Transaction complete */
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ mutex_unlock(&new_address_space->transaction_mutex);
+
+ /* Synchronize with any policy timers */
+ kbase_backend_ctx_count_changed(kbdev);
+
+ mutex_unlock(&js_devdata->runpool_mutex);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ /* Note: after this point, the context could potentially get scheduled
+ * out immediately */
+
+ if (kctx_suspended) {
+ /* Finishing forcing out the context due to a suspend. Use a
+ * variant of kbasep_js_runpool_release_ctx() that doesn't
+ * schedule a new context, to prevent a risk of recursion back
+ * into this function */
+ kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx);
+ return false;
+ }
+ return true;
+}
- case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
- /* State where cores were requested, but not registered */
- KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T));
- kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T, katom->affinity);
- break;
+static bool kbase_js_use_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ unsigned long flags;
- case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
- /* Initial state - nothing required */
- KBASE_DEBUG_ASSERT(katom->affinity == 0);
- break;
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ if (kctx->as_pending) {
+ /* Context waiting for AS to be assigned */
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ return false;
+ }
+ if (kbase_backend_use_ctx_sched(kbdev, kctx)) {
+ /* Context already has ASID - mark as active */
+ kbdev->hwaccess.active_kctx = kctx;
- default:
- KBASE_DEBUG_ASSERT_MSG(MALI_FALSE, "Unhandled coreref_state: %d", katom->coreref_state);
- break;
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ return true; /* Context already scheduled */
}
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
- katom->affinity = 0;
- katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
+ return kbasep_js_schedule_ctx(kbdev, kctx);
}
-/*
- * Note: this function is quite similar to kbasep_js_try_run_next_job_on_slot()
- */
-mali_bool kbasep_js_try_run_next_job_on_slot_irq_nolock(struct kbase_device *kbdev, int js, s8 *submit_count)
+void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
{
+ struct kbasep_js_kctx_info *js_kctx_info;
struct kbasep_js_device_data *js_devdata;
- mali_bool cores_ready;
+ bool is_scheduled;
KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(kctx != NULL);
js_devdata = &kbdev->js_data;
+ js_kctx_info = &kctx->jctx.sched_info;
- /* The caller of this function may not be aware of Ctx Attribute state changes so we
- * must recheck if the given slot is still valid. Otherwise do not try to run.
- */
- if (kbase_js_can_run_job_on_slot_no_lock(kbdev, js)) {
- /* Keep submitting while there's space to run a job on this job-slot,
- * and there are jobs to get that match its requirements (see 'break'
- * statement below) */
- while (*submit_count < KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ && kbasep_jm_is_submit_slots_free(kbdev, js, NULL) != MALI_FALSE) {
- struct kbase_jd_atom *dequeued_atom;
- mali_bool has_job = MALI_FALSE;
-
- /* Dequeue a job that matches the requirements */
- has_job = kbasep_js_policy_dequeue_job(kbdev, js, &dequeued_atom);
-
- if (has_job != MALI_FALSE) {
- /* NOTE: since the runpool_irq lock is currently held and acts across
- * all address spaces, any context whose busy refcount has reached
- * zero won't yet be scheduled out whilst we're trying to run jobs
- * from it */
- struct kbase_context *parent_ctx = dequeued_atom->kctx;
- mali_bool retain_success;
-
- /* Retain/power up the cores it needs, check if cores are ready */
- cores_ready = kbasep_js_job_check_ref_cores(kbdev, js, dequeued_atom);
-
- if (dequeued_atom->event_code == BASE_JD_EVENT_PM_EVENT || cores_ready != MALI_TRUE) {
- /* The job either can't be submitted until the cores are ready, or
- * the job will fail due to the specified core group being unavailable.
- * To avoid recursion this will be handled outside of IRQ content by
- * kbasep_js_try_run_next_job_on_slot_nolock */
- kbasep_js_policy_enqueue_job(&kbdev->js_data.policy, dequeued_atom);
- return MALI_TRUE;
- }
+ /* This must never be attempted whilst suspending - i.e. it should only
+ * happen in response to a syscall from a user-space thread */
+ BUG_ON(kbase_pm_is_suspending(kbdev));
- /* ASSERT that the Policy picked a job from an allowed context */
- KBASE_DEBUG_ASSERT(kbasep_js_is_submit_allowed(js_devdata, parent_ctx));
+ mutex_lock(&js_devdata->queue_mutex);
+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
- /* Retain the context to stop it from being scheduled out
- * This is released when the job finishes */
- retain_success = kbasep_js_runpool_retain_ctx_nolock(kbdev, parent_ctx);
- KBASE_DEBUG_ASSERT(retain_success != MALI_FALSE);
- CSTD_UNUSED(retain_success);
+ /* Mark the context as privileged */
+ js_kctx_info->ctx.flags |= KBASE_CTX_FLAG_PRIVILEGED;
- /* Retain the affinity on the slot */
- kbase_js_affinity_retain_slot_cores(kbdev, js, dequeued_atom->affinity);
+ is_scheduled = js_kctx_info->ctx.is_scheduled;
+ if (!is_scheduled) {
+ /* Add the context to the pullable list */
+ if (kbase_js_ctx_list_add_pullable(kbdev, kctx, 0))
+ kbase_js_sync_timers(kbdev);
- /* Check if this job needs the cycle counter enabled before submission */
- kbasep_js_ref_permon_check_and_enable_cycle_counter(kbdev, dequeued_atom);
+ /* Fast-starting requires the jsctx_mutex to be dropped,
+ * because it works on multiple ctxs */
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_unlock(&js_devdata->queue_mutex);
- /* Submit the job */
- kbase_job_submit_nolock(kbdev, dequeued_atom, js);
+ /* Try to schedule the context in */
+ kbase_js_sched_all(kbdev);
- ++(*submit_count);
- } else {
- /* No more jobs - stop submitting for this slot */
- break;
- }
- }
+ /* Wait for the context to be scheduled in */
+ wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait,
+ kctx->jctx.sched_info.ctx.is_scheduled);
+ } else {
+ /* Already scheduled in - We need to retain it to keep the
+ * corresponding address space */
+ kbasep_js_runpool_retain_ctx(kbdev, kctx);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_unlock(&js_devdata->queue_mutex);
}
+}
- /* Indicate whether a retry in submission should be tried on a different
- * dequeue function. These are the reasons why it *must* happen:
- * - the KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ threshold was reached
- * and new scheduling must be performed outside of IRQ mode.
- *
- * Failure to indicate this correctly could stop further jobs being processed.
- *
- * However, we do not _need_ to indicate a retry for the following:
- * - kbasep_js_policy_dequeue_job() couldn't get a job. In which case,
- * there's no point re-trying outside of IRQ, because the result will be
- * the same until job dependencies are resolved, or user-space provides
- * more jobs. In both those cases, we try to run jobs anyway, so
- * processing does not stop.
- * - kbasep_jm_is_submit_slots_free() was MALI_FALSE, indicating jobs were
- * already running. When those jobs complete, that will still cause events
- * that cause us to resume job submission.
- * - kbase_js_can_run_job_on_slot_no_lock() was MALI_FALSE - this is for
- * Ctx Attribute handling. That _can_ change outside of IRQ context, but
- * is handled explicitly by kbasep_js_runpool_release_ctx_and_katom_retained_state().
- */
- return (mali_bool) (*submit_count >= KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ);
+void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_kctx_info *js_kctx_info;
+ bool pending;
+
+ KBASE_DEBUG_ASSERT(kctx != NULL);
+ js_kctx_info = &kctx->jctx.sched_info;
+
+ /* We don't need to use the address space anymore */
+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
+ js_kctx_info->ctx.flags &= (~KBASE_CTX_FLAG_PRIVILEGED);
+ pending = kctx->as_pending;
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+
+ /* Release the context - it will be scheduled out if there is no
+ * pending job */
+ if (!pending)
+ kbasep_js_runpool_release_ctx(kbdev, kctx);
+
+ kbase_js_sched_all(kbdev);
}
-void kbasep_js_try_run_next_job_on_slot_nolock(struct kbase_device *kbdev, int js)
+void kbasep_js_suspend(struct kbase_device *kbdev)
{
+ unsigned long flags;
struct kbasep_js_device_data *js_devdata;
- mali_bool has_job;
- mali_bool cores_ready;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
+ int i;
+ u16 retained = 0u;
+ int nr_privileged_ctx = 0;
+ KBASE_DEBUG_ASSERT(kbdev);
+ KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev));
js_devdata = &kbdev->js_data;
- KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running > 0);
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- /* Keep submitting while there's space to run a job on this job-slot,
- * and there are jobs to get that match its requirements (see 'break'
- * statement below) */
- if (kbasep_jm_is_submit_slots_free(kbdev, js, NULL) != MALI_FALSE) {
- /* The caller of this function may not be aware of Ctx Attribute state changes so we
- * must recheck if the given slot is still valid. Otherwise do not try to run.
- */
- if (kbase_js_can_run_job_on_slot_no_lock(kbdev, js)) {
- do {
- struct kbase_jd_atom *dequeued_atom;
-
- /* Dequeue a job that matches the requirements */
- has_job = kbasep_js_policy_dequeue_job(kbdev, js, &dequeued_atom);
-
- if (has_job != MALI_FALSE) {
- /* NOTE: since the runpool_irq lock is currently held and acts across
- * all address spaces, any context whose busy refcount has reached
- * zero won't yet be scheduled out whilst we're trying to run jobs
- * from it */
- struct kbase_context *parent_ctx = dequeued_atom->kctx;
- mali_bool retain_success;
-
- /* Retain/power up the cores it needs, check if cores are ready */
- cores_ready = kbasep_js_job_check_ref_cores(kbdev, js, dequeued_atom);
-
- if (cores_ready != MALI_TRUE && dequeued_atom->event_code != BASE_JD_EVENT_PM_EVENT) {
- /* The job can't be submitted until the cores are ready, requeue the job */
- kbasep_js_policy_enqueue_job(&kbdev->js_data.policy, dequeued_atom);
- break;
- }
- /* ASSERT that the Policy picked a job from an allowed context */
- KBASE_DEBUG_ASSERT(kbasep_js_is_submit_allowed(js_devdata, parent_ctx));
-
- /* Retain the context to stop it from being scheduled out
- * This is released when the job finishes */
- retain_success = kbasep_js_runpool_retain_ctx_nolock(kbdev, parent_ctx);
- KBASE_DEBUG_ASSERT(retain_success != MALI_FALSE);
- CSTD_UNUSED(retain_success);
-
- /* Retain the affinity on the slot */
- kbase_js_affinity_retain_slot_cores(kbdev, js, dequeued_atom->affinity);
-
- /* Check if this job needs the cycle counter enabled before submission */
- kbasep_js_ref_permon_check_and_enable_cycle_counter(kbdev, dequeued_atom);
-
- if (dequeued_atom->event_code == BASE_JD_EVENT_PM_EVENT) {
- dev_warn(kbdev->dev, "Rejecting atom due to BASE_JD_EVENT_PM_EVENT\n");
- /* The job has failed due to the specified core group being unavailable */
- kbase_jd_done(dequeued_atom, js, NULL, 0);
- } else {
- /* Submit the job */
- kbase_job_submit_nolock(kbdev, dequeued_atom, js);
- }
- }
+ /* Prevent all contexts from submitting */
+ js_devdata->runpool_irq.submit_allowed = 0;
+
+ /* Retain each of the contexts, so we can cause it to leave even if it
+ * had no refcount to begin with */
+ for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) {
+ struct kbasep_js_per_as_data *js_per_as_data =
+ &js_devdata->runpool_irq.per_as_data[i];
+ struct kbase_context *kctx = js_per_as_data->kctx;
- } while (kbasep_jm_is_submit_slots_free(kbdev, js, NULL) != MALI_FALSE && has_job != MALI_FALSE);
+ retained = retained << 1;
+
+ if (kctx) {
+ ++(js_per_as_data->as_busy_refcount);
+ retained |= 1u;
+ /* We can only cope with up to 1 privileged context -
+ * the instrumented context. It'll be suspended by
+ * disabling instrumentation */
+ if (kctx->jctx.sched_info.ctx.flags &
+ KBASE_CTX_FLAG_PRIVILEGED)
+ KBASE_DEBUG_ASSERT(++nr_privileged_ctx == 1);
}
}
+ CSTD_UNUSED(nr_privileged_ctx);
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+
+ /* De-ref the previous retain to ensure each context gets pulled out
+ * sometime later. */
+ for (i = 0;
+ i < BASE_MAX_NR_AS;
+ ++i, retained = retained >> 1) {
+ struct kbasep_js_per_as_data *js_per_as_data =
+ &js_devdata->runpool_irq.per_as_data[i];
+ struct kbase_context *kctx = js_per_as_data->kctx;
+
+ if (retained & 1u)
+ kbasep_js_runpool_release_ctx(kbdev, kctx);
+ }
+
+ /* Caller must wait for all Power Manager active references to be
+ * dropped */
}
-void kbasep_js_try_schedule_head_ctx(struct kbase_device *kbdev)
+void kbasep_js_resume(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata;
- mali_bool has_kctx;
- struct kbase_context *head_kctx;
- struct kbasep_js_kctx_info *js_kctx_info;
- mali_bool is_runpool_full;
- struct kbase_as *new_address_space;
- unsigned long flags;
- mali_bool head_kctx_suspended = MALI_FALSE;
- int pm_active_err;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
+ int js;
+ KBASE_DEBUG_ASSERT(kbdev);
js_devdata = &kbdev->js_data;
+ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
- /* We *don't* make a speculative check on whether we can fit a context in the
- * runpool, because most of our use-cases assume 2 or fewer contexts, and
- * so we will usually have enough address spaces free.
- *
- * In any case, the check will be done later on once we have a context */
-
- /* Grab the context off head of queue - if there is one */
mutex_lock(&js_devdata->queue_mutex);
- has_kctx = kbasep_js_policy_dequeue_head_ctx(&js_devdata->policy, &head_kctx);
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
+ struct kbase_context *kctx, *n;
+
+ list_for_each_entry_safe(kctx, n,
+ &kbdev->js_data.ctx_list_unpullable[js],
+ jctx.sched_info.ctx.ctx_list_entry[js]) {
+ struct kbasep_js_kctx_info *js_kctx_info;
+ unsigned long flags;
+ bool timer_sync = false;
+
+ js_kctx_info = &kctx->jctx.sched_info;
+
+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_lock(&js_devdata->runpool_mutex);
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ if (!js_kctx_info->ctx.is_scheduled &&
+ kbase_js_ctx_pullable(kctx, js, false))
+ timer_sync = kbase_js_ctx_list_add_pullable(
+ kbdev, kctx, js);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
+ flags);
+ if (timer_sync)
+ kbase_backend_ctx_count_changed(kbdev);
+ mutex_unlock(&js_devdata->runpool_mutex);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ }
+ }
mutex_unlock(&js_devdata->queue_mutex);
- if (has_kctx == MALI_FALSE) {
- /* No ctxs to run - nothing to do */
- return;
- }
- js_kctx_info = &head_kctx->jctx.sched_info;
+ /* Restart atom processing */
+ kbase_js_sched_all(kbdev);
- dev_dbg(kbdev->dev, "JS: Dequeue Context %p", head_kctx);
+ /* JS Resume complete */
+}
- pm_active_err = kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE);
+bool kbase_js_is_atom_valid(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ if ((katom->core_req & BASE_JD_REQ_FS) &&
+ (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE |
+ BASE_JD_REQ_T)))
+ return false;
- /*
- * Atomic transaction on the Context and Run Pool begins
- */
- mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
- mutex_lock(&js_devdata->runpool_mutex);
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987) &&
+ (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) &&
+ (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_T)))
+ return false;
- /* Check to see if we shouldn't add the context to run Run Pool:
- * - it can't take the specified context, and so is 'full'. This may be
- * 'full' even when there are addres spaces available, since some contexts
- * are allowed in whereas others may not due to HW workarounds
- * - A suspend is taking place
- * - The context is dying due to kbase_job_zap_context() */
- is_runpool_full = check_is_runpool_full(kbdev, head_kctx);
- if (is_runpool_full || pm_active_err || js_kctx_info->ctx.is_dying) {
- /* Roll back the transaction so far and return */
- mutex_unlock(&js_devdata->runpool_mutex);
+ return true;
+}
- /* Note: If a Power Management active reference was taken, it's released by
- * this: */
- kbasep_js_runpool_requeue_or_kill_ctx(kbdev, head_kctx, !pm_active_err);
+static int kbase_js_get_slot(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom)
+{
+ if (katom->core_req & BASE_JD_REQ_FS)
+ return 0;
+
+ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
+ if (katom->device_nr == 1 &&
+ kbdev->gpu_props.num_core_groups == 2)
+ return 2;
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
+ return 2;
+ }
- mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
- return;
+ return 1;
+}
+
+int kbase_js_dep_resolved_submit(struct kbase_context *kctx,
+ struct kbase_jd_atom *katom,
+ bool *enqueue_required)
+{
+ katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom);
+
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
+
+ /* If slot will transition from unpullable to pullable then add to
+ * pullable list */
+ if (jsctx_rb_is_empty(kctx, katom->slot_nr)) {
+ *enqueue_required = true;
+ } else {
+ *enqueue_required = false;
+ /* Check if there are lower priority jobs to soft stop */
+ kbase_job_slot_ctx_priority_check_locked(kctx, katom);
}
- /* From the point on, the Power Management active reference is released
- * only if kbasep_js_runpool_release_ctx() causes the context to be removed
- * from the runpool */
+ /* Add atom to ring buffer. */
+ if (unlikely(jsctx_rb_add_atom(kctx, katom))) {
+ /* The ring buffer is full. This should be impossible as the
+ * job dispatcher can not submit enough atoms to exceed the
+ * ring buffer size. Fail the job.
+ */
+ WARN(1, "Job submit while JSCTX ringbuffer already full\n");
+ return -EINVAL;
+ }
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, head_kctx, NULL, 0u, kbasep_js_trace_get_refcnt(kbdev, head_kctx));
+ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED;
-#if !MALI_CUSTOMER_RELEASE
- if (js_devdata->nr_user_contexts_running == 0) {
- /* Only when there are no other contexts submitting jobs:
- * Latch in run-time job scheduler timeouts that were set through js_timeouts sysfs file */
- if (kbdev->js_soft_stop_ticks != 0)
- js_devdata->soft_stop_ticks = kbdev->js_soft_stop_ticks;
+ return 0;
+}
- if (kbdev->js_soft_stop_ticks_cl != 0)
- js_devdata->soft_stop_ticks_cl = kbdev->js_soft_stop_ticks_cl;
+struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js)
+{
+ struct kbase_jd_atom *katom;
+ struct kbasep_js_device_data *js_devdata;
+ int pulled;
- if (kbdev->js_hard_stop_ticks_ss != 0)
- js_devdata->hard_stop_ticks_ss = kbdev->js_hard_stop_ticks_ss;
+ KBASE_DEBUG_ASSERT(kctx);
- if (kbdev->js_hard_stop_ticks_cl != 0)
- js_devdata->hard_stop_ticks_cl = kbdev->js_hard_stop_ticks_cl;
+ js_devdata = &kctx->kbdev->js_data;
+ lockdep_assert_held(&js_devdata->runpool_irq.lock);
- if (kbdev->js_hard_stop_ticks_nss != 0)
- js_devdata->hard_stop_ticks_nss = kbdev->js_hard_stop_ticks_nss;
+ if (!kbasep_js_is_submit_allowed(js_devdata, kctx))
+ return NULL;
+ if (kbase_pm_is_suspending(kctx->kbdev))
+ return NULL;
- if (kbdev->js_reset_ticks_ss != 0)
- js_devdata->gpu_reset_ticks_ss = kbdev->js_reset_ticks_ss;
+ katom = jsctx_rb_peek(kctx, js);
+ if (!katom)
+ return NULL;
- if (kbdev->js_reset_ticks_cl != 0)
- js_devdata->gpu_reset_ticks_cl = kbdev->js_reset_ticks_cl;
+ if (atomic_read(&katom->blocked))
+ return NULL;
- if (kbdev->js_reset_ticks_nss != 0)
- js_devdata->gpu_reset_ticks_nss = kbdev->js_reset_ticks_nss;
+ /* Due to ordering restrictions when unpulling atoms on failure, we do
+ * not allow multiple runs of fail-dep atoms from the same context to be
+ * present on the same slot */
+ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_PREV) &&
+ atomic_read(&kctx->atoms_pulled_slot[js])) {
+ struct kbase_jd_atom *prev_atom =
+ kbase_backend_inspect_tail(kctx->kbdev, js);
+
+ if (prev_atom && prev_atom->kctx != kctx)
+ return NULL;
}
-#endif
- runpool_inc_context_count(kbdev, head_kctx);
- /* Cause any future waiter-on-termination to wait until the context is
- * descheduled */
- js_kctx_info->ctx.is_scheduled = MALI_TRUE;
- wake_up(&js_kctx_info->ctx.is_scheduled_wait);
+ if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) {
+ if (katom->x_pre_dep->gpu_rb_state ==
+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)
+ return NULL;
+ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) &&
+ kbase_backend_nr_atoms_on_slot(kctx->kbdev, js))
+ return NULL;
+ }
- /* Pick the free address space (guaranteed free by check_is_runpool_full() ) */
- new_address_space = pick_free_addr_space(kbdev);
+ kctx->pulled = true;
+ pulled = atomic_inc_return(&kctx->atoms_pulled);
+ if (pulled == 1 && !kctx->slots_pullable)
+ atomic_inc(&kctx->kbdev->js_data.nr_contexts_runnable);
+ atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]);
+ jsctx_rb_pull(kctx, katom);
+
+ kbasep_js_runpool_retain_ctx_nolock(kctx->kbdev, kctx);
+ katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF;
+
+ katom->sched_info.cfs.ticks = 0;
+
+ return katom;
+}
+
+
+static void js_return_worker(struct work_struct *data)
+{
+ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom,
+ work);
+ struct kbase_context *kctx = katom->kctx;
+ struct kbase_device *kbdev = kctx->kbdev;
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info;
+ struct kbasep_js_atom_retained_state retained_state;
+ int js = katom->slot_nr;
+ bool timer_sync = false;
+ bool context_idle = false;
+ unsigned long flags;
+
+ kbase_backend_complete_wq(kbdev, katom);
+
+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316))
+ kbase_as_poking_timer_release_atom(kbdev, kctx, katom);
+
+ kbasep_js_atom_retained_state_copy(&retained_state, katom);
+
+ mutex_lock(&js_devdata->queue_mutex);
+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
+
+ atomic_dec(&kctx->atoms_pulled);
+ atomic_dec(&kctx->atoms_pulled_slot[js]);
+
+ atomic_dec(&katom->blocked);
- /* Lock the address space whilst working on it */
- mutex_lock(&new_address_space->transaction_mutex);
spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- /* Do all the necessaries to assign the address space (inc. update book-keeping info)
- * Add the context to the Run Pool, and allow it to run jobs */
- assign_and_activate_kctx_addr_space(kbdev, head_kctx, new_address_space);
+ if (!atomic_read(&kctx->atoms_pulled_slot[js]) &&
+ jsctx_rb_is_empty(kctx, js))
+ timer_sync |= kbase_js_ctx_list_remove(kbdev, kctx, js);
- /* NOTE: If Linux allows, then we can drop the new_address_space->transaction mutex here */
+ if (!atomic_read(&kctx->atoms_pulled)) {
+ if (!kctx->slots_pullable)
+ atomic_dec(&kbdev->js_data.nr_contexts_runnable);
- if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_PRIVILEGED) != 0) {
- /* We need to retain it to keep the corresponding address space */
- kbasep_js_runpool_retain_ctx_nolock(kbdev, head_kctx);
- }
+ if (kctx->as_nr != KBASEP_AS_NR_INVALID &&
+ !js_kctx_info->ctx.is_dying) {
+ int num_slots = kbdev->gpu_props.num_job_slots;
+ int slot;
- /* Re-check for suspending: a suspend could've occurred after we
- * pm_context_active'd, and all the contexts could've been removed from the
- * runpool before we took this lock. In this case, we don't want to allow
- * this context to run jobs, we just want it out immediately.
- *
- * The DMB required to read the suspend flag was issued recently as part of
- * the runpool_irq locking. If a suspend occurs *after* that lock was taken
- * (i.e. this condition doesn't execute), then the kbasep_js_suspend() code
- * will cleanup this context instead (by virtue of it being called strictly
- * after the suspend flag is set, and will wait for this lock to drop) */
- if (kbase_pm_is_suspending(kbdev)) {
- /* Cause it to leave at some later point */
- mali_bool retained;
+ if (!kbasep_js_is_submit_allowed(js_devdata, kctx))
+ kbasep_js_set_submit_allowed(js_devdata, kctx);
- retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, head_kctx);
- KBASE_DEBUG_ASSERT(retained);
- kbasep_js_clear_submit_allowed(js_devdata, head_kctx);
- head_kctx_suspended = MALI_TRUE;
- }
+ for (slot = 0; slot < num_slots; slot++) {
+ if (kbase_js_ctx_pullable(kctx, slot, true))
+ timer_sync |=
+ kbase_js_ctx_list_add_pullable(
+ kbdev, kctx, slot);
+ }
+ }
- /* Try to run the next job, in case this context has jobs that match the
- * job slot requirements, but none of the other currently running contexts
- * do */
- kbasep_js_try_run_next_job_nolock(kbdev);
+ kbase_jm_idle_ctx(kbdev, kctx);
+
+ context_idle = true;
+ }
- /* Transaction complete */
spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
- mutex_unlock(&new_address_space->transaction_mutex);
- mutex_unlock(&js_devdata->runpool_mutex);
+
+ if (context_idle)
+ kbase_pm_context_idle(kbdev);
+
+ if (timer_sync)
+ kbase_js_sync_timers(kbdev);
+
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
- /* Note: after this point, the context could potentially get scheduled out immediately */
+ mutex_unlock(&js_devdata->queue_mutex);
- if (head_kctx_suspended) {
- /* Finishing forcing out the context due to a suspend. Use a variant of
- * kbasep_js_runpool_release_ctx() that doesn't schedule a new context,
- * to prevent a risk of recursion back into this function */
- kbasep_js_runpool_release_ctx_no_schedule(kbdev, head_kctx);
- }
+ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF;
+ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx,
+ &retained_state);
+
+ kbase_js_sched_all(kbdev);
}
-void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
+void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
- struct kbasep_js_kctx_info *js_kctx_info;
- struct kbasep_js_device_data *js_devdata;
- mali_bool is_scheduled;
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
+ jsctx_rb_unpull(kctx, katom);
- js_devdata = &kbdev->js_data;
- js_kctx_info = &kctx->jctx.sched_info;
+ WARN_ON(work_pending(&katom->work));
- /* This must never be attempted whilst suspending - i.e. it should only
- * happen in response to a syscall from a user-space thread */
- BUG_ON(kbase_pm_is_suspending(kbdev));
+ /* Block re-submission until workqueue has run */
+ atomic_inc(&katom->blocked);
- kbase_pm_request_l2_caches(kbdev);
+ kbase_job_check_leave_disjoint(kctx->kbdev, katom);
- mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
- /* Mark the context as privileged */
- js_kctx_info->ctx.flags |= KBASE_CTX_FLAG_PRIVILEGED;
+ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work));
+ INIT_WORK(&katom->work, js_return_worker);
+ queue_work(kctx->jctx.job_done_wq, &katom->work);
+}
- is_scheduled = js_kctx_info->ctx.is_scheduled;
- if (is_scheduled == MALI_FALSE) {
- mali_bool is_runpool_full;
+static bool kbase_js_evict_atom(struct kbase_context *kctx,
+ struct kbase_jd_atom *katom_evict,
+ struct kbase_jd_atom *start_katom,
+ struct kbase_jd_atom *head_katom,
+ struct list_head *evict_list,
+ struct jsctx_rb *rb, int idx)
+{
+ struct kbase_jd_atom *x_dep = katom_evict->x_post_dep;
- /* Add the context to the runpool */
- mutex_lock(&js_devdata->queue_mutex);
- kbasep_js_policy_enqueue_ctx(&js_devdata->policy, kctx);
- mutex_unlock(&js_devdata->queue_mutex);
+ if (!(katom_evict->atom_flags & KBASE_KATOM_FLAG_FAIL_PREV) &&
+ katom_evict != start_katom)
+ return false;
- mutex_lock(&js_devdata->runpool_mutex);
- {
- is_runpool_full = check_is_runpool_full(kbdev, kctx);
- if (is_runpool_full != MALI_FALSE) {
- /* Evict jobs from the NEXT registers to free an AS asap */
- kbasep_js_runpool_evict_next_jobs(kbdev, kctx);
- }
- }
- mutex_unlock(&js_devdata->runpool_mutex);
- mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
- /* Fast-starting requires the jsctx_mutex to be dropped, because it works on multiple ctxs */
+ if (katom_evict->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
+ WARN_ON(katom_evict->gpu_rb_state !=
+ KBASE_ATOM_GPU_RB_RETURN_TO_JS);
+ WARN_ON(katom_evict->event_code != head_katom->event_code);
- if (is_runpool_full != MALI_FALSE) {
- /* Evict non-running contexts from the runpool */
- kbasep_js_runpool_attempt_fast_start_ctx(kbdev, NULL);
- }
- /* Try to schedule the context in */
- kbasep_js_try_schedule_head_ctx(kbdev);
+ return false;
+ }
- /* Wait for the context to be scheduled in */
- wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, kctx->jctx.sched_info.ctx.is_scheduled == MALI_TRUE);
- } else {
- /* Already scheduled in - We need to retain it to keep the corresponding address space */
- kbasep_js_runpool_retain_ctx(kbdev, kctx);
- mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ if (katom_evict->status == KBASE_JD_ATOM_STATE_HW_COMPLETED &&
+ katom_evict != head_katom)
+ return false;
+
+ /* Evict cross dependency if present */
+ if (x_dep && (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED)
+ && (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER))
+ list_add_tail(&x_dep->dep_item[0], evict_list);
+
+ /* If cross dependency is present and does not have a data dependency
+ * then unblock */
+ if (x_dep && (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED)
+ && !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER))
+ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED;
+
+ if (katom_evict != head_katom) {
+ rb->entries[idx & JSCTX_RB_MASK].atom_id =
+ KBASEP_ATOM_ID_INVALID;
+
+ katom_evict->event_code = head_katom->event_code;
+ katom_evict->atom_flags &=
+ ~KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED;
+
+ if (katom_evict->atom_flags & KBASE_KATOM_FLAG_HOLDING_CTX_REF)
+ kbase_jd_done(katom_evict, katom_evict->slot_nr, NULL,
+ 0);
+ else
+ kbase_jd_evict(kctx->kbdev, katom_evict);
+ }
+
+ return true;
+}
+
+/**
+ * kbase_js_evict_deps - Evict dependencies
+ * @kctx: Context pointer
+ * @head_katom: Pointer to the atom to evict
+ *
+ * Remove all post dependencies of an atom from the context ringbuffers.
+ *
+ * The original atom's event_code will be propogated to all dependent atoms.
+ *
+ * Context: Caller must hold both jctx and HW access locks
+ */
+static void kbase_js_evict_deps(struct kbase_context *kctx,
+ struct kbase_jd_atom *head_katom)
+{
+ struct list_head evict_list;
+
+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
+
+ INIT_LIST_HEAD(&evict_list);
+
+ list_add_tail(&head_katom->dep_item[0], &evict_list);
+
+ while (!list_empty(&evict_list)) {
+ struct kbase_jd_atom *start_katom;
+
+ start_katom = list_entry(evict_list.prev, struct kbase_jd_atom,
+ dep_item[0]);
+ list_del(evict_list.prev);
+
+ jsctx_rb_evict(kctx, start_katom, head_katom, &evict_list);
}
}
-void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
+/**
+ * kbase_js_compact - Compact JSCTX ringbuffers
+ * @kctx: Context pointer
+ *
+ * Compact the JSCTX ringbuffers, removing any NULL entries
+ *
+ * Context: Caller must hold both jctx and HW access locks
+ */
+static void kbase_js_compact(struct kbase_context *kctx)
+{
+ struct kbase_device *kbdev = kctx->kbdev;
+ int js;
+
+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex);
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++)
+ jsctx_rb_compact(kctx, js);
+}
+
+void kbase_js_complete_atom_wq(struct kbase_context *kctx,
+ struct kbase_jd_atom *katom)
{
struct kbasep_js_kctx_info *js_kctx_info;
+ struct kbasep_js_device_data *js_devdata;
+ struct kbase_device *kbdev;
+ unsigned long flags;
+ bool timer_sync = false;
+ int atom_slot;
+ bool context_idle = false;
+
+ kbdev = kctx->kbdev;
+ atom_slot = katom->slot_nr;
- KBASE_DEBUG_ASSERT(kctx != NULL);
js_kctx_info = &kctx->jctx.sched_info;
+ js_devdata = &kbdev->js_data;
- /* We don't need to use the address space anymore */
- mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
- js_kctx_info->ctx.flags &= (~KBASE_CTX_FLAG_PRIVILEGED);
- mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
- kbase_pm_release_l2_caches(kbdev);
+ mutex_lock(&js_devdata->runpool_mutex);
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- /* Release the context - it will be scheduled out if there is no pending job */
- kbasep_js_runpool_release_ctx(kbdev, kctx);
+ if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED) {
+ if (katom->event_code != BASE_JD_EVENT_DONE)
+ kbase_js_evict_deps(kctx, katom);
+
+ jsctx_rb_remove(kctx, katom);
+
+ context_idle = !atomic_dec_return(&kctx->atoms_pulled);
+ atomic_dec(&kctx->atoms_pulled_slot[atom_slot]);
+
+ if (!atomic_read(&kctx->atoms_pulled) && !kctx->slots_pullable)
+ atomic_dec(&kbdev->js_data.nr_contexts_runnable);
+
+ if (katom->event_code != BASE_JD_EVENT_DONE)
+ kbase_js_compact(kctx);
+ }
+
+ if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) &&
+ jsctx_rb_is_empty(kctx, atom_slot))
+ timer_sync |= kbase_js_ctx_list_remove(kctx->kbdev, kctx,
+ atom_slot);
+
+ /*
+ * If submission is disabled on this context (most likely due to an
+ * atom failure) and there are now no atoms left in the system then
+ * re-enable submission so that context can be scheduled again.
+ */
+ if (!kbasep_js_is_submit_allowed(js_devdata, kctx) &&
+ !atomic_read(&kctx->atoms_pulled) &&
+ !js_kctx_info->ctx.is_dying) {
+ int js;
+
+ kbasep_js_set_submit_allowed(js_devdata, kctx);
+
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
+ if (kbase_js_ctx_pullable(kctx, js, true))
+ timer_sync |= kbase_js_ctx_list_add_pullable(
+ kbdev, kctx, js);
+ }
+ } else if (katom->x_post_dep &&
+ kbasep_js_is_submit_allowed(js_devdata, kctx)) {
+ int js;
+
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
+ if (kbase_js_ctx_pullable(kctx, js, true))
+ timer_sync |= kbase_js_ctx_list_add_pullable(
+ kbdev, kctx, js);
+ }
+ }
+
+ if (context_idle)
+ kbase_jm_idle_ctx(kbdev, kctx);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ if (timer_sync)
+ kbase_backend_ctx_count_changed(kbdev);
+ mutex_unlock(&js_devdata->runpool_mutex);
+
+ if (context_idle)
+ kbase_pm_context_idle(kbdev);
}
-void kbasep_js_job_done_slot_irq(struct kbase_jd_atom *katom, int slot_nr,
- ktime_t *end_timestamp, kbasep_js_atom_done_code done_code)
+void kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp)
{
+ u64 microseconds_spent = 0;
struct kbase_device *kbdev;
+ struct kbase_context *kctx = katom->kctx;
union kbasep_js_policy *js_policy;
struct kbasep_js_device_data *js_devdata;
- mali_bool submit_retry_needed = MALI_TRUE; /* If we don't start jobs here, start them from the workqueue */
- ktime_t tick_diff;
- u64 microseconds_spent = 0u;
- struct kbase_context *parent_ctx;
-
- KBASE_DEBUG_ASSERT(katom);
- parent_ctx = katom->kctx;
- KBASE_DEBUG_ASSERT(parent_ctx);
- kbdev = parent_ctx->kbdev;
- KBASE_DEBUG_ASSERT(kbdev);
- js_devdata = &kbdev->js_data;
- js_policy = &kbdev->js_data.policy;
+ kbdev = kctx->kbdev;
- lockdep_assert_held(&js_devdata->runpool_irq.lock);
+ js_policy = &kbdev->js_data.policy;
+ js_devdata = &kbdev->js_data;
- /*
- * Release resources before submitting new jobs (bounds the refcount of
- * the resource to BASE_JM_SUBMIT_SLOTS)
- */
-#ifdef CONFIG_MALI_GATOR_SUPPORT
- kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_STOP, slot_nr), NULL, 0);
-#endif /* CONFIG_MALI_GATOR_SUPPORT */
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
- /* Check if submitted jobs no longer require the cycle counter to be enabled */
- kbasep_js_deref_permon_check_and_disable_cycle_counter(kbdev, katom);
+ katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED;
- /* Release the affinity from the slot - must happen before next submission to this slot */
- kbase_js_affinity_release_slot_cores(kbdev, slot_nr, katom->affinity);
- kbase_js_debug_log_current_affinities(kbdev);
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
+ kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_STOP,
+ katom->slot_nr), NULL, 0);
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_tl_nret_atom_lpu(
+ katom,
+ &kbdev->gpu_props.props.raw_props.js_features[
+ katom->slot_nr]);
+#endif
/* Calculate the job's time used */
if (end_timestamp != NULL) {
- /* Only calculating it for jobs that really run on the HW (e.g. removed
- * from next jobs never actually ran, so really did take zero time) */
- tick_diff = ktime_sub(*end_timestamp, katom->start_timestamp);
+ /* Only calculating it for jobs that really run on the HW (e.g.
+ * removed from next jobs never actually ran, so really did take
+ * zero time) */
+ ktime_t tick_diff = ktime_sub(*end_timestamp,
+ katom->start_timestamp);
microseconds_spent = ktime_to_ns(tick_diff);
+
do_div(microseconds_spent, 1000);
/* Round up time spent to the minimum timer resolution */
/* Log the result of the job (completion status, and time spent). */
kbasep_js_policy_log_job_result(js_policy, katom, microseconds_spent);
- /* Determine whether the parent context's timeslice is up */
- if (kbasep_js_policy_should_remove_ctx(js_policy, parent_ctx) != MALI_FALSE)
- kbasep_js_clear_submit_allowed(js_devdata, parent_ctx);
- if (done_code & KBASE_JS_ATOM_DONE_START_NEW_ATOMS) {
- /* Submit a new job (if there is one) to help keep the GPU's HEAD and NEXT registers full */
- KBASE_TRACE_ADD_SLOT(kbdev, JS_JOB_DONE_TRY_RUN_NEXT_JOB, parent_ctx, katom, katom->jc, slot_nr);
-
- submit_retry_needed = kbasep_js_try_run_next_job_on_slot_irq_nolock(kbdev, slot_nr, &kbdev->slot_submit_count_irq[slot_nr]);
- }
+ kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0);
- if (submit_retry_needed != MALI_FALSE || katom->event_code == BASE_JD_EVENT_STOPPED) {
- /* The extra condition on STOPPED jobs is needed because they may be
- * the only job present, but they won't get re-run until the JD work
- * queue activates. Crucially, work queues can run items out of order
- * e.g. on different CPUs, so being able to submit from the IRQ handler
- * is not a good indication that we don't need to run jobs; the
- * submitted job could be processed on the work-queue *before* the
- * stopped job, even though it was submitted after.
- *
- * Therefore, we must try to run it, otherwise it might not get run at
- * all after this. */
-
- KBASE_TRACE_ADD_SLOT(kbdev, JS_JOB_DONE_RETRY_NEEDED, parent_ctx, katom, katom->jc, slot_nr);
- kbasep_js_set_job_retry_submit_slot(katom, slot_nr);
- }
+ /* Unblock cross dependency if present */
+ if (katom->x_post_dep && (katom->event_code == BASE_JD_EVENT_DONE ||
+ !(katom->x_post_dep->atom_flags &
+ KBASE_KATOM_FLAG_FAIL_BLOCKER)))
+ katom->x_post_dep->atom_flags &=
+ ~KBASE_KATOM_FLAG_X_DEP_BLOCKED;
}
-void kbasep_js_suspend(struct kbase_device *kbdev)
+void kbase_js_sched(struct kbase_device *kbdev, int js_mask)
{
- unsigned long flags;
struct kbasep_js_device_data *js_devdata;
- int i;
- u16 retained = 0u;
- int nr_privileged_ctx = 0;
- KBASE_DEBUG_ASSERT(kbdev);
- KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev));
+ union kbasep_js_policy *js_policy;
+ bool timer_sync = false;
+
js_devdata = &kbdev->js_data;
+ js_policy = &js_devdata->policy;
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ down(&js_devdata->schedule_sem);
+ mutex_lock(&js_devdata->queue_mutex);
- /* Prevent all contexts from submitting */
- js_devdata->runpool_irq.submit_allowed = 0;
+ while (js_mask) {
+ int js;
- /* Retain each of the contexts, so we can cause it to leave even if it had
- * no refcount to begin with */
- for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) {
- struct kbasep_js_per_as_data *js_per_as_data = &js_devdata->runpool_irq.per_as_data[i];
- struct kbase_context *kctx = js_per_as_data->kctx;
+ js = ffs(js_mask) - 1;
- retained = retained << 1;
+ while (1) {
+ struct kbase_context *kctx;
+ unsigned long flags;
+ bool context_idle = false;
- if (kctx) {
- ++(js_per_as_data->as_busy_refcount);
- retained |= 1u;
- /* We can only cope with up to 1 privileged context - the
- * instrumented context. It'll be suspended by disabling
- * instrumentation */
- if (kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_PRIVILEGED)
- KBASE_DEBUG_ASSERT(++nr_privileged_ctx == 1);
+ kctx = kbase_js_ctx_list_pop_head(kbdev, js);
+
+ if (!kctx) {
+ js_mask &= ~(1 << js);
+ break; /* No contexts on pullable list */
+ }
+
+ if (!atomic_read(&kctx->atoms_pulled)) {
+ context_idle = true;
+
+ if (kbase_pm_context_active_handle_suspend(
+ kbdev,
+ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) {
+ /* Suspend pending - return context to
+ * queue and stop scheduling */
+ mutex_lock(
+ &kctx->jctx.sched_info.ctx.jsctx_mutex);
+ if (kbase_js_ctx_list_add_pullable_head(
+ kctx->kbdev, kctx, js))
+ kbase_js_sync_timers(kbdev);
+ mutex_unlock(
+ &kctx->jctx.sched_info.ctx.jsctx_mutex);
+ mutex_unlock(&js_devdata->queue_mutex);
+ up(&js_devdata->schedule_sem);
+ return;
+ }
+ }
+
+ if (!kbase_js_use_ctx(kbdev, kctx)) {
+ mutex_lock(
+ &kctx->jctx.sched_info.ctx.jsctx_mutex);
+ /* Context can not be used at this time */
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock,
+ flags);
+ if (kctx->as_pending ||
+ kbase_js_ctx_pullable(kctx, js, false)
+ || (kctx->jctx.sched_info.ctx.flags &
+ KBASE_CTX_FLAG_PRIVILEGED))
+ timer_sync |=
+ kbase_js_ctx_list_add_pullable_head(
+ kctx->kbdev, kctx, js);
+ else
+ timer_sync |=
+ kbase_js_ctx_list_add_unpullable(
+ kctx->kbdev, kctx, js);
+ spin_unlock_irqrestore(
+ &js_devdata->runpool_irq.lock, flags);
+ mutex_unlock(
+ &kctx->jctx.sched_info.ctx.jsctx_mutex);
+ if (context_idle)
+ kbase_pm_context_idle(kbdev);
+
+ /* No more jobs can be submitted on this slot */
+ js_mask &= ~(1 << js);
+ break;
+ }
+ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ kctx->pulled = false;
+
+ if (!kbase_jm_kick(kbdev, 1 << js))
+ /* No more jobs can be submitted on this slot */
+ js_mask &= ~(1 << js);
+
+ if (!kctx->pulled) {
+ /* Failed to pull jobs - push to head of list */
+ if (kbase_js_ctx_pullable(kctx, js, true))
+ timer_sync |=
+ kbase_js_ctx_list_add_pullable_head(
+ kctx->kbdev,
+ kctx, js);
+ else
+ timer_sync |=
+ kbase_js_ctx_list_add_unpullable(
+ kctx->kbdev,
+ kctx, js);
+
+ if (context_idle) {
+ kbase_jm_idle_ctx(kbdev, kctx);
+ spin_unlock_irqrestore(
+ &js_devdata->runpool_irq.lock,
+ flags);
+ kbase_pm_context_idle(kbdev);
+ } else {
+ spin_unlock_irqrestore(
+ &js_devdata->runpool_irq.lock,
+ flags);
+ }
+ mutex_unlock(
+ &kctx->jctx.sched_info.ctx.jsctx_mutex);
+
+ js_mask &= ~(1 << js);
+ break; /* Could not run atoms on this slot */
+ }
+
+ /* Push to back of list */
+ if (kbase_js_ctx_pullable(kctx, js, true))
+ timer_sync |= kbase_js_ctx_list_add_pullable(
+ kctx->kbdev, kctx, js);
+ else
+ timer_sync |= kbase_js_ctx_list_add_unpullable(
+ kctx->kbdev, kctx, js);
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
+ flags);
+ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
}
}
- CSTD_UNUSED(nr_privileged_ctx);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
- /* De-ref the previous retain to ensure each context gets pulled out
- * sometime later. */
- for (i = 0;
- i < BASE_MAX_NR_AS;
- ++i, retained = retained >> 1) {
- struct kbasep_js_per_as_data *js_per_as_data = &js_devdata->runpool_irq.per_as_data[i];
- struct kbase_context *kctx = js_per_as_data->kctx;
+ if (timer_sync)
+ kbase_js_sync_timers(kbdev);
- if (retained & 1u)
- kbasep_js_runpool_release_ctx(kbdev, kctx);
+ mutex_unlock(&js_devdata->queue_mutex);
+ up(&js_devdata->schedule_sem);
+}
+
+void kbase_js_zap_context(struct kbase_context *kctx)
+{
+ struct kbase_device *kbdev = kctx->kbdev;
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
+ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info;
+ int js;
+
+ /*
+ * Critical assumption: No more submission is possible outside of the
+ * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs)
+ * whilst the struct kbase_context is terminating.
+ */
+
+ /* First, atomically do the following:
+ * - mark the context as dying
+ * - try to evict it from the policy queue */
+ mutex_lock(&js_devdata->queue_mutex);
+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
+ js_kctx_info->ctx.is_dying = true;
+
+ dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx);
+
+ /*
+ * At this point we know:
+ * - If eviction succeeded, it was in the policy queue, but now no
+ * longer is
+ * - We must cancel the jobs here. No Power Manager active reference to
+ * release.
+ * - This happens asynchronously - kbase_jd_zap_context() will wait for
+ * those jobs to be killed.
+ * - If eviction failed, then it wasn't in the policy queue. It is one
+ * of the following:
+ * - a. it didn't have any jobs, and so is not in the Policy Queue or
+ * the Run Pool (not scheduled)
+ * - Hence, no more work required to cancel jobs. No Power Manager
+ * active reference to release.
+ * - b. it was in the middle of a scheduling transaction (and thus must
+ * have at least 1 job). This can happen from a syscall or a
+ * kernel thread. We still hold the jsctx_mutex, and so the thread
+ * must be waiting inside kbasep_js_try_schedule_head_ctx(),
+ * before checking whether the runpool is full. That thread will
+ * continue after we drop the mutex, and will notice the context
+ * is dying. It will rollback the transaction, killing all jobs at
+ * the same time. kbase_jd_zap_context() will wait for those jobs
+ * to be killed.
+ * - Hence, no more work required to cancel jobs, or to release the
+ * Power Manager active reference.
+ * - c. it is scheduled, and may or may not be running jobs
+ * - We must cause it to leave the runpool by stopping it from
+ * submitting any more jobs. When it finally does leave,
+ * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs
+ * (because it is dying), release the Power Manager active reference,
+ * and will not requeue the context in the policy queue.
+ * kbase_jd_zap_context() will wait for those jobs to be killed.
+ * - Hence, work required just to make it leave the runpool. Cancelling
+ * jobs and releasing the Power manager active reference will be
+ * handled when it leaves the runpool.
+ */
+ if (!js_kctx_info->ctx.is_scheduled) {
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
+ if (!list_empty(
+ &kctx->jctx.sched_info.ctx.ctx_list_entry[js]))
+ list_del_init(
+ &kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
+ }
+
+ /* The following events require us to kill off remaining jobs
+ * and update PM book-keeping:
+ * - we evicted it correctly (it must have jobs to be in the
+ * Policy Queue)
+ *
+ * These events need no action, but take this path anyway:
+ * - Case a: it didn't have any jobs, and was never in the Queue
+ * - Case b: scheduling transaction will be partially rolled-
+ * back (this already cancels the jobs)
+ */
+
+ KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u,
+ js_kctx_info->ctx.is_scheduled);
+
+ dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx);
+
+ /* Only cancel jobs when we evicted from the policy
+ * queue. No Power Manager active reference was held.
+ *
+ * Having is_dying set ensures that this kills, and
+ * doesn't requeue */
+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false);
+
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_unlock(&js_devdata->queue_mutex);
+ } else {
+ unsigned long flags;
+ bool was_retained;
+
+ /* Case c: didn't evict, but it is scheduled - it's in the Run
+ * Pool */
+ KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u,
+ js_kctx_info->ctx.is_scheduled);
+ dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx);
+
+ /* Disable the ctx from submitting any more jobs */
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ kbasep_js_clear_submit_allowed(js_devdata, kctx);
+
+ /* Retain and (later) release the context whilst it is is now
+ * disallowed from submitting jobs - ensures that someone
+ * somewhere will be removing the context later on */
+ was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
+
+ /* Since it's scheduled and we have the jsctx_mutex, it must be
+ * retained successfully */
+ KBASE_DEBUG_ASSERT(was_retained);
+
+ dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx);
+
+ /* Cancel any remaining running jobs for this kctx - if any.
+ * Submit is disallowed which takes effect immediately, so no
+ * more new jobs will appear after we do this. */
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++)
+ kbase_job_slot_hardstop(kctx, js, NULL);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
+ mutex_unlock(&js_devdata->queue_mutex);
+
+ dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)",
+ kctx);
+
+ kbasep_js_runpool_release_ctx(kbdev, kctx);
}
- /* Caller must wait for all Power Manager active references to be dropped */
+ KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u);
+
+ /* After this, you must wait on both the
+ * kbase_jd_context::zero_jobs_wait and the
+ * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs
+ * to be destroyed, and the context to be de-scheduled (if it was on the
+ * runpool).
+ *
+ * kbase_jd_zap_context() will do this. */
}
-void kbasep_js_resume(struct kbase_device *kbdev)
+static inline int trace_get_refcnt(struct kbase_device *kbdev,
+ struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
- int i;
+ int as_nr;
+ int refcnt = 0;
- KBASE_DEBUG_ASSERT(kbdev);
js_devdata = &kbdev->js_data;
- KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
+ as_nr = kctx->as_nr;
+ if (as_nr != KBASEP_AS_NR_INVALID) {
+ struct kbasep_js_per_as_data *js_per_as_data;
- /* Schedule in as many contexts as address spaces. This also starts atoms. */
- for (i = 0 ; i < kbdev->nr_hw_address_spaces; ++i)
- kbasep_js_try_schedule_head_ctx(kbdev);
+ js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
- /* JS Resume complete */
+ refcnt = js_per_as_data->as_busy_refcount;
+ }
+
+ return refcnt;
+}
+
+/**
+ * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context
+ * @kctx: Pointer to context.
+ * @callback: Pointer to function to call for each job.
+ *
+ * Call a function on all jobs belonging to a non-queued, non-running
+ * context, and detach the jobs from the context as it goes.
+ *
+ * Due to the locks that might be held at the time of the call, the callback
+ * may need to defer work on a workqueue to complete its actions (e.g. when
+ * cancelling jobs)
+ *
+ * Atoms will be removed from the queue, so this must only be called when
+ * cancelling jobs (which occurs as part of context destruction).
+ *
+ * The locking conditions on the caller are as follows:
+ * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex.
+ */
+static void kbase_js_foreach_ctx_job(struct kbase_context *kctx,
+ kbasep_js_policy_ctx_job_cb callback)
+{
+ struct kbase_device *kbdev;
+ struct kbasep_js_device_data *js_devdata;
+ unsigned long flags;
+ u32 js;
+
+ kbdev = kctx->kbdev;
+
+ js_devdata = &kbdev->js_data;
+
+ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+
+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL,
+ 0u, trace_get_refcnt(kbdev, kctx));
+
+ /* Invoke callback on jobs on each slot in turn */
+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++)
+ jsctx_rb_foreach(kctx, js, callback);
+
+ spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#ifndef _KBASE_JS_H_
#define _KBASE_JS_H_
-#include <malisw/mali_malisw.h>
-
#include "mali_kbase_js_defs.h"
#include "mali_kbase_js_policy.h"
#include "mali_kbase_defs.h"
* initialized before passing to the kbasep_js_devdata_init() function. This is
* to give efficient error path code.
*/
-mali_error kbasep_js_devdata_init(struct kbase_device * const kbdev);
+int kbasep_js_devdata_init(struct kbase_device * const kbdev);
/**
* @brief Halt the Job Scheduler.
* The struct kbase_context must be zero intitialized before passing to the
* kbase_js_init() function. This is to give efficient error path code.
*/
-mali_error kbasep_js_kctx_init(struct kbase_context * const kctx);
+int kbasep_js_kctx_init(struct kbase_context * const kctx);
/**
* @brief Terminate the Scheduling Component of a struct kbase_context on the Job Scheduler
*
* The Policy's Queue can be updated by this in the following ways:
* - In the above case that this is the first job on the context
- * - If the job is high priority and the context is not scheduled, then it
+ * - If the context is high priority and the context is not scheduled, then it
* could cause the Policy to schedule out a low-priority context, allowing
* this context to be scheduled in.
*
* If the context is already scheduled on the RunPool, then adding a job to it
* is guarenteed not to update the Policy Queue. And so, the caller is
* guarenteed to not need to try scheduling a context from the Run Pool - it
- * can safely assert that the result is MALI_FALSE.
+ * can safely assert that the result is false.
*
* It is a programming error to have more than U32_MAX jobs in flight at a time.
*
* obtained internally)
* - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally).
*
- * @return MALI_TRUE indicates that the Policy Queue was updated, and so the
+ * @return true indicates that the Policy Queue was updated, and so the
* caller will need to try scheduling a context onto the Run Pool.
- * @return MALI_FALSE indicates that no updates were made to the Policy Queue,
+ * @return false indicates that no updates were made to the Policy Queue,
* so no further action is required from the caller. This is \b always returned
* when the context is currently scheduled.
*/
-mali_bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom);
+bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom);
/**
* @brief Remove a job chain from the Job Scheduler, except for its 'retained state'.
* obtained internally)
* - it must \em not hold kbasep_js_device_data::runpool_mutex (as this could be
* obtained internally)
+ *
+ * @return true indicates that ctx attributes have changed and the caller
+ * should call kbase_js_sched_all() to try to run more jobs
+ * @return false otherwise
*/
-void kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom);
+bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev,
+ struct kbase_context *kctx,
+ struct kbase_jd_atom *katom);
/**
* @brief Refcount a context as being busy, preventing it from being scheduled
* - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
* it will be used internally.
*
- * @return value != MALI_FALSE if the retain succeeded, and the context will not be scheduled out.
- * @return MALI_FALSE if the retain failed (because the context is being/has been scheduled out).
+ * @return value != false if the retain succeeded, and the context will not be scheduled out.
+ * @return false if the retain failed (because the context is being/has been scheduled out).
*/
-mali_bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
+bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
/**
* @brief Refcount a context as being busy, preventing it from being scheduled
* The following locks must be held by the caller:
* - kbasep_js_device_data::runpool_irq::lock
*
- * @return value != MALI_FALSE if the retain succeeded, and the context will not be scheduled out.
- * @return MALI_FALSE if the retain failed (because the context is being/has been scheduled out).
+ * @return value != false if the retain succeeded, and the context will not be scheduled out.
+ * @return false if the retain failed (because the context is being/has been scheduled out).
*/
-mali_bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx);
+bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx);
/**
* @brief Lookup a context in the Run Pool based upon its current address space
*
* The following locking conditions are made on the caller:
* - it must \em not hold the kbasep_js_device_data::runpoool_irq::lock, because
- * it will be used internally.
+ * it will be used internally. If the runpool_irq::lock is already held, then
+ * the caller should use kbasep_js_runpool_lookup_ctx_nolock() instead.
*
* @return a valid struct kbase_context on success, which has been refcounted as being busy.
* @return NULL on failure, indicating that no context was found in \a as_nr
*/
struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr);
+/**
+ * kbasep_js_runpool_lookup_ctx_nolock - Lookup a context in the Run Pool based
+ * upon its current address space and ensure that is stays scheduled in.
+ * @kbdev: Device pointer
+ * @as_nr: Address space to lookup
+ *
+ * The context is refcounted as being busy to prevent it from scheduling
+ * out. It must be released with kbasep_js_runpool_release_ctx() when it is no
+ * longer required to stay scheduled in.
+ *
+ * Note: This function can safely be called from IRQ context.
+ *
+ * The following locking conditions are made on the caller:
+ * - it must the kbasep_js_device_data::runpoool_irq::lock.
+ *
+ * Return: a valid struct kbase_context on success, which has been refcounted as
+ * being busy.
+ * NULL on failure, indicating that no context was found in \a as_nr
+ */
+struct kbase_context *kbasep_js_runpool_lookup_ctx_nolock(
+ struct kbase_device *kbdev, int as_nr);
+
/**
* @brief Handling the requeuing/killing of a context that was evicted from the
* policy queue or runpool.
* - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be
* obtained internally)
*/
-void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, mali_bool has_pm_ref);
+void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, bool has_pm_ref);
/**
* @brief Release a refcount of a context being busy, allowing it to be
*/
void kbasep_js_runpool_release_ctx_and_katom_retained_state(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state);
-/**
- * @brief Try to submit the next job on a \b particular slot whilst in IRQ
- * context, and whilst the caller already holds the runpool IRQ spinlock.
- *
- * \a *submit_count will be checked against
- * KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ to see whether too many jobs have
- * been submitted. This is to prevent the IRQ handler looping over lots of GPU
- * NULL jobs, which may complete whilst the IRQ handler is still processing. \a
- * submit_count itself should point to kbase_device::slot_submit_count_irq[ \a js ],
- * which is initialized to zero on entry to the IRQ handler.
- *
- * The following locks must be held by the caller:
- * - kbasep_js_device_data::runpool_irq::lock
- *
- * @return truthful (i.e. != MALI_FALSE) if too many jobs were submitted from
- * IRQ. Therefore, this indicates that submission should be retried from a
- * work-queue, by using
- * kbasep_js_try_run_next_job_on_slot_nolock()/kbase_js_try_run_jobs_on_slot().
- * @return MALI_FALSE if submission had no problems: the GPU is either already
- * full of jobs in the HEAD and NEXT registers, or we were able to get enough
- * jobs from the Run Pool to fill the GPU's HEAD and NEXT registers.
- */
-mali_bool kbasep_js_try_run_next_job_on_slot_irq_nolock(struct kbase_device *kbdev, int js, s8 *submit_count);
-
-/**
- * @brief Try to submit the next job on a particular slot, outside of IRQ context
- *
- * This obtains the Job Slot lock for the duration of the call only.
- *
- * Unlike kbasep_js_try_run_next_job_on_slot_irq_nolock(), there is no limit on
- * submission, because eventually IRQ_THROTTLE will kick in to prevent us
- * getting stuck in a loop of submitting GPU NULL jobs. This is because the IRQ
- * handler will be delayed, and so this function will eventually fill up the
- * space in our software 'submitted' slot (kbase_jm_slot::submitted).
- *
- * In addition, there's no return value - we'll run the maintenence functions
- * on the Policy's Run Pool, but if there's nothing there after that, then the
- * Run Pool is truely empty, and so no more action need be taken.
- *
- * The following locking conditions are made on the caller:
- * - it must hold kbasep_js_device_data::runpool_mutex
- * - it must hold kbasep_js_device_data::runpool_irq::lock
- *
- * This must only be called whilst the GPU is powered - for example, when
- * kbdev->jsdata.nr_user_contexts_running > 0.
- *
- * @note The caller \em might be holding one of the
- * kbasep_js_kctx_info::ctx::jsctx_mutex locks.
- *
- */
-void kbasep_js_try_run_next_job_on_slot_nolock(struct kbase_device *kbdev, int js);
/**
- * @brief Try to submit the next job for each slot in the system, outside of IRQ context
- *
- * This will internally call kbasep_js_try_run_next_job_on_slot_nolock(), so similar
- * locking conditions on the caller are required.
- *
- * The following locking conditions are made on the caller:
- * - it must hold kbasep_js_device_data::runpool_mutex
- * - it must hold kbasep_js_device_data::runpool_irq::lock
+ * @brief Release the refcount of the context and allow further submission
+ * of the context after the dump on error in user space terminates.
*
- * @note The caller \em might be holding one of the
- * kbasep_js_kctx_info::ctx::jsctx_mutex locks.
+ * Before this function is called, when a fault happens the kernel should
+ * have disallowed the context from further submission of jobs and
+ * retained the context to avoid it from being removed. This function
+ * releases the refcount of the context and allow further submission of
+ * jobs.
*
+ * This function should only be called when "instr=2" during compile time.
*/
-void kbasep_js_try_run_next_job_nolock(struct kbase_device *kbdev);
+void kbasep_js_dump_fault_term(struct kbase_device *kbdev, struct kbase_context *kctx);
/**
- * @brief Try to schedule the next context onto the Run Pool
- *
- * This checks whether there's space in the Run Pool to accommodate a new
- * context. If so, it attempts to dequeue a context from the Policy Queue, and
- * submit this to the Run Pool.
- *
- * If the scheduling succeeds, then it also makes a call to
- * kbasep_js_try_run_next_job_nolock(), in case the new context has jobs
- * matching the job slot requirements, but no other currently scheduled context
- * has such jobs.
- *
- * Whilst attempting to obtain a context from the policy queue, or add a
- * context to the runpool, this function takes a Power Manager active
- * reference. If for any reason a context cannot be added to the runpool, any
- * reference obtained is released once the context is safely back in the policy
- * queue. If no context was available on the policy queue, any reference
- * obtained is released too.
- *
- * Only if the context gets placed in the runpool does the Power Manager active
- * reference stay held (and is effectively now owned by the
- * context/runpool). It is only released once the context is removed
- * completely, or added back to the policy queue
- * (e.g. kbasep_js_runpool_release_ctx(),
- * kbasep_js_runpool_requeue_or_kill_ctx(), etc)
- *
- * If any of these actions fail (Run Pool Full, Policy Queue empty, can't get
- * PM active reference due to a suspend, etc) then any actions taken are rolled
- * back and the function just returns normally.
- *
- * The following locking conditions are made on the caller:
- * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
- * it will be used internally.
- * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
- * obtained internally)
- * - it must \em not hold the kbase_device::as[n].transaction_mutex (as this will be obtained internally)
- * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally).
- * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will
- * be used internally.
- *
+ * @brief Variant of kbase_js_runpool_release_ctx() that assumes that
+ * kbasep_js_device_data::runpool_mutex and
+ * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not
+ * attempt to schedule new contexts.
*/
-void kbasep_js_try_schedule_head_ctx(struct kbase_device *kbdev);
+void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev,
+ struct kbase_context *kctx);
/**
* @brief Schedule in a privileged context
*/
void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
-/**
- * @brief Handle the Job Scheduler component for the IRQ of a job finishing
- *
- * This does the following:
- * -# Releases resources held by the atom
- * -# if \a end_timestamp != NULL, updates the runpool's notion of time spent by a running ctx
- * -# determines whether a context should be marked for scheduling out
- * -# examines done_code to determine whether to submit the next job on the slot
- * (picking from all ctxs in the runpool)
- *
- * In addition, if submission didn't happen (the submit-from-IRQ function
- * failed or done_code didn't specify to start new jobs), then this sets a
- * message on katom that submission needs to be retried from the worker thread.
- *
- * Normally, the time calculated from end_timestamp is rounded up to the
- * minimum time precision. Therefore, to ensure the job is recorded as not
- * spending any time, then set end_timestamp to NULL. For example, this is necessary when
- * evicting jobs from JS_HEAD_NEXT (because they didn't actually run).
- *
- * NOTE: It's possible to move the steps (2) and (3) (inc calculating job's time
- * used) into the worker (outside of IRQ context), but this may allow a context
- * to use up to twice as much timeslice as is allowed by the policy. For
- * policies that order by time spent, this is not a problem for overall
- * 'fairness', but can still increase latency between contexts.
- *
- * The following locking conditions are made on the caller:
- * - it must hold kbasep_js_device_data::runpoool_irq::lock
- */
-void kbasep_js_job_done_slot_irq(struct kbase_jd_atom *katom, int slot_nr,
- ktime_t *end_timestamp,
- kbasep_js_atom_done_code done_code);
-
/**
* @brief Try to submit the next job on each slot
*
*/
void kbase_js_try_run_jobs(struct kbase_device *kbdev);
-/**
- * @brief Try to submit the next job on a specfic slot
- *
- * The following locking conditions are made on the caller:
- *
- * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
- * obtained internally)
- * - it must \em not hold kbasep_js_device_data::runpool_irq::lock (as this
- * will be obtained internally)
- *
- */
-void kbase_js_try_run_jobs_on_slot(struct kbase_device *kbdev, int js);
-
-/**
- * @brief Handle releasing cores for power management and affinity management,
- * ensuring that cores are powered down and affinity tracking is updated.
- *
- * This must only be called on an atom that is not currently running, and has
- * not been re-queued onto the context (and so does not need locking)
- *
- * This function enters at the following @ref enum kbase_atom_coreref_state states:
- * - NO_CORES_REQUESTED
- * - WAITING_FOR_REQUESTED_CORES
- * - RECHECK_AFFINITY
- * - READY
- *
- * It transitions the above states back to NO_CORES_REQUESTED by the end of the
- * function call (possibly via intermediate states).
- *
- * No locks need be held by the caller, since this takes the necessary Power
- * Management locks itself. The runpool_irq.lock is not taken (the work that
- * requires it is handled by kbase_js_affinity_submit_to_blocked_slots() ).
- *
- * @note The corresponding kbasep_js_job_check_ref_cores() is private to the
- * Job Scheduler, and is called automatically when running the next job.
- */
-void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, struct kbase_jd_atom *katom);
-
/**
* @brief Suspend the job scheduler during a Power Management Suspend event.
*
*/
void kbasep_js_resume(struct kbase_device *kbdev);
+/**
+ * @brief Submit an atom to the job scheduler.
+ *
+ * The atom is enqueued on the context's ringbuffer. The caller must have
+ * ensured that all dependencies can be represented in the ringbuffer.
+ *
+ * Caller must hold jctx->lock
+ *
+ * @param[in] kctx Context pointer
+ * @param[in] atom Pointer to the atom to submit
+ *
+ * @return 0 if submit succeeded
+ * error code if the atom can not be submitted at this
+ * time, due to insufficient space in the ringbuffer, or dependencies
+ * that can not be represented.
+ */
+int kbase_js_dep_resolved_submit(struct kbase_context *kctx,
+ struct kbase_jd_atom *katom,
+ bool *enqueue_required);
+
+/**
+ * @brief Pull an atom from a context in the job scheduler for execution.
+ *
+ * The atom will not be removed from the ringbuffer at this stage.
+ *
+ * The HW access lock must be held when calling this function.
+ *
+ * @param[in] kctx Context to pull from
+ * @param[in] js Job slot to pull from
+ * @return Pointer to an atom, or NULL if there are no atoms for this
+ * slot that can be currently run.
+ */
+struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js);
+
+/**
+ * @brief Return an atom to the job scheduler ringbuffer.
+ *
+ * An atom is 'unpulled' if execution is stopped but intended to be returned to
+ * later. The most common reason for this is that the atom has been
+ * soft-stopped.
+ *
+ * Note that if multiple atoms are to be 'unpulled', they must be returned in
+ * the reverse order to which they were originally pulled. It is a programming
+ * error to return atoms in any other order.
+ *
+ * The HW access lock must be held when calling this function.
+ *
+ * @param[in] kctx Context pointer
+ * @param[in] atom Pointer to the atom to unpull
+ */
+void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom);
+
+/**
+ * @brief Complete an atom from jd_done_worker(), removing it from the job
+ * scheduler ringbuffer.
+ *
+ * If the atom failed then all dependee atoms marked for failure propagation
+ * will also fail.
+ *
+ * @param[in] kctx Context pointer
+ * @param[in] katom Pointer to the atom to complete
+ */
+void kbase_js_complete_atom_wq(struct kbase_context *kctx,
+ struct kbase_jd_atom *katom);
+
+/**
+ * @brief Complete an atom.
+ *
+ * Most of the work required to complete an atom will be performed by
+ * jd_done_worker().
+ *
+ * The HW access lock must be held when calling this function.
+ *
+ * @param[in] katom Pointer to the atom to complete
+ * @param[in] end_timestamp The time that the atom completed (may be NULL)
+ */
+void kbase_js_complete_atom(struct kbase_jd_atom *katom,
+ ktime_t *end_timestamp);
+
+/**
+ * @brief Submit atoms from all available contexts.
+ *
+ * This will attempt to submit as many jobs as possible to the provided job
+ * slots. It will exit when either all job slots are full, or all contexts have
+ * been used.
+ *
+ * @param[in] kbdev Device pointer
+ * @param[in] js_mask Mask of job slots to submit to
+ */
+void kbase_js_sched(struct kbase_device *kbdev, int js_mask);
+
+/**
+ * kbase_jd_zap_context - Attempt to deschedule a context that is being
+ * destroyed
+ * @kctx: Context pointer
+ *
+ * This will attempt to remove a context from any internal job scheduler queues
+ * and perform any other actions to ensure a context will not be submitted
+ * from.
+ *
+ * If the context is currently scheduled, then the caller must wait for all
+ * pending jobs to complete before taking any further action.
+ */
+void kbase_js_zap_context(struct kbase_context *kctx);
+
+/**
+ * @brief Validate an atom
+ *
+ * This will determine whether the atom can be scheduled onto the GPU. Atoms
+ * with invalid combinations of core requirements will be rejected.
+ *
+ * @param[in] kbdev Device pointer
+ * @param[in] katom Atom to validate
+ * @return true if atom is valid
+ * false otherwise
+ */
+bool kbase_js_is_atom_valid(struct kbase_device *kbdev,
+ struct kbase_jd_atom *katom);
/*
* Helpers follow
* The purpose of this abstraction is to hide the underlying data size, and wrap up
* the long repeated line of code.
*
- * As with any mali_bool, never test the return value with MALI_TRUE.
+ * As with any bool, never test the return value with true.
*
* The caller must hold kbasep_js_device_data::runpool_irq::lock.
*/
-static INLINE mali_bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx)
+static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx)
{
u16 test_bit;
/* Ensure context really is scheduled in */
KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
- KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled != MALI_FALSE);
+ KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled);
test_bit = (u16) (1u << kctx->as_nr);
- return (mali_bool) (js_devdata->runpool_irq.submit_allowed & test_bit);
+ return (bool) (js_devdata->runpool_irq.submit_allowed & test_bit);
}
/**
*
* The caller must hold kbasep_js_device_data::runpool_irq::lock.
*/
-static INLINE void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx)
+static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx)
{
u16 set_bit;
/* Ensure context really is scheduled in */
KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
- KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled != MALI_FALSE);
+ KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled);
set_bit = (u16) (1u << kctx->as_nr);
*
* The caller must hold kbasep_js_device_data::runpool_irq::lock.
*/
-static INLINE void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx)
+static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx)
{
u16 clear_bit;
u16 clear_mask;
/* Ensure context really is scheduled in */
KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
- KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled != MALI_FALSE);
+ KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled);
clear_bit = (u16) (1u << kctx->as_nr);
clear_mask = ~clear_bit;
/**
* @brief Manage the 'retry_submit_on_slot' part of a kbase_jd_atom
*/
-static INLINE void kbasep_js_clear_job_retry_submit(struct kbase_jd_atom *atom)
+static inline void kbasep_js_clear_job_retry_submit(struct kbase_jd_atom *atom)
{
atom->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID;
}
* submitted on some other slot, then call kbasep_js_clear_job_retry_submit()
* first to silence the ASSERT.
*/
-static INLINE void kbasep_js_set_job_retry_submit_slot(struct kbase_jd_atom *atom, int js)
+static inline void kbasep_js_set_job_retry_submit_slot(struct kbase_jd_atom *atom, int js)
{
KBASE_DEBUG_ASSERT(0 <= js && js <= BASE_JM_MAX_NR_SLOTS);
- KBASE_DEBUG_ASSERT(atom->retry_submit_on_slot == KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID
- || atom->retry_submit_on_slot == js);
+ KBASE_DEBUG_ASSERT((atom->retry_submit_on_slot ==
+ KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID)
+ || (atom->retry_submit_on_slot == js));
atom->retry_submit_on_slot = js;
}
* atom-related work to be done on releasing with
* kbasep_js_runpool_release_ctx_and_katom_retained_state()
*/
-static INLINE void kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state)
+static inline void kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state)
{
retained_state->event_code = BASE_JD_EVENT_NOT_STARTED;
retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID;
* Copy atom state that can be made available after jd_done_nolock() is called
* on that atom.
*/
-static INLINE void kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, const struct kbase_jd_atom *katom)
+static inline void kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, const struct kbase_jd_atom *katom)
{
retained_state->event_code = katom->event_code;
retained_state->core_req = katom->core_req;
retained_state->retry_submit_on_slot = katom->retry_submit_on_slot;
+ retained_state->sched_priority = katom->sched_priority;
retained_state->device_nr = katom->device_nr;
}
* - Being soft-stopped (and so, the atom should be resubmitted sometime later)
*
* @param[in] katom_retained_state the retained state of the atom to check
- * @return MALI_FALSE if the atom has not finished
- * @return !=MALI_FALSE if the atom has finished
+ * @return false if the atom has not finished
+ * @return !=false if the atom has finished
*/
-static INLINE mali_bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state)
+static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state)
{
- return (mali_bool) (katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT);
+ return (bool) (katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT);
}
/**
* code should just ignore it.
*
* @param[in] katom_retained_state the atom's retained state to check
- * @return MALI_FALSE if the retained state is invalid, and can be ignored
- * @return !=MALI_FALSE if the retained state is valid
+ * @return false if the retained state is invalid, and can be ignored
+ * @return !=false if the retained state is valid
*/
-static INLINE mali_bool kbasep_js_atom_retained_state_is_valid(const struct kbasep_js_atom_retained_state *katom_retained_state)
+static inline bool kbasep_js_atom_retained_state_is_valid(const struct kbasep_js_atom_retained_state *katom_retained_state)
{
- return (mali_bool) (katom_retained_state->core_req != KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID);
+ return (bool) (katom_retained_state->core_req != KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID);
}
-static INLINE mali_bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_atom_retained_state *katom_retained_state, int *res)
+static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_atom_retained_state *katom_retained_state, int *res)
{
int js = katom_retained_state->retry_submit_on_slot;
+
*res = js;
- return (mali_bool) (js >= 0);
+ return (bool) (js >= 0);
}
#if KBASE_DEBUG_DISABLE_ASSERTS == 0
* @return current refcount of the context if it is scheduled in. The refcount
* is not guarenteed to be kept constant.
*/
-static INLINE int kbasep_js_debug_check_ctx_refcount(struct kbase_device *kbdev, struct kbase_context *kctx)
+static inline int kbasep_js_debug_check_ctx_refcount(struct kbase_device *kbdev, struct kbase_context *kctx)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
* to be non-zero and unmodified by this function.
* @return NULL on failure, indicating that no context was found in \a as_nr
*/
-static INLINE struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr)
+static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
* e.g.: when you need the number of cycles to guarantee you won't wait for
* longer than 'us' time (you might have a shorter wait).
*/
-static INLINE u32 kbasep_js_convert_us_to_gpu_ticks_min_freq(struct kbase_device *kbdev, u32 us)
+static inline u32 kbasep_js_convert_us_to_gpu_ticks_min_freq(struct kbase_device *kbdev, u32 us)
{
u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_min;
+
KBASE_DEBUG_ASSERT(0 != gpu_freq);
return us * (gpu_freq / 1000);
}
* e.g.: When you need the number of cycles to guarantee you'll wait at least
* 'us' amount of time (but you might wait longer).
*/
-static INLINE u32 kbasep_js_convert_us_to_gpu_ticks_max_freq(struct kbase_device *kbdev, u32 us)
+static inline u32 kbasep_js_convert_us_to_gpu_ticks_max_freq(struct kbase_device *kbdev, u32 us)
{
u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_max;
+
KBASE_DEBUG_ASSERT(0 != gpu_freq);
return us * (u32) (gpu_freq / 1000);
}
* take (you guarantee that you won't wait any longer than this, but it may
* be shorter).
*/
-static INLINE u32 kbasep_js_convert_gpu_ticks_to_us_min_freq(struct kbase_device *kbdev, u32 ticks)
+static inline u32 kbasep_js_convert_gpu_ticks_to_us_min_freq(struct kbase_device *kbdev, u32 ticks)
{
u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_min;
+
KBASE_DEBUG_ASSERT(0 != gpu_freq);
return ticks / gpu_freq * 1000;
}
* e.g.: When you need to know the best-case wait for 'tick' cycles (you
* guarantee to be waiting for at least this long, but it may be longer).
*/
-static INLINE u32 kbasep_js_convert_gpu_ticks_to_us_max_freq(struct kbase_device *kbdev, u32 ticks)
+static inline u32 kbasep_js_convert_gpu_ticks_to_us_max_freq(struct kbase_device *kbdev, u32 ticks)
{
u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_max;
+
KBASE_DEBUG_ASSERT(0 != gpu_freq);
return ticks / gpu_freq * 1000;
}
+/*
+ * The following locking conditions are made on the caller:
+ * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
+ * - The caller must hold the kbasep_js_device_data::runpool_mutex
+ */
+static inline void kbase_js_runpool_inc_context_count(
+ struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_device_data *js_devdata;
+ struct kbasep_js_kctx_info *js_kctx_info;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(kctx != NULL);
+
+ js_devdata = &kbdev->js_data;
+ js_kctx_info = &kctx->jctx.sched_info;
+
+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
+ lockdep_assert_held(&js_devdata->runpool_mutex);
+
+ /* Track total contexts */
+ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX);
+ ++(js_devdata->nr_all_contexts_running);
+
+ if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) == 0) {
+ /* Track contexts that can submit jobs */
+ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running <
+ S8_MAX);
+ ++(js_devdata->nr_user_contexts_running);
+ }
+}
+
+/*
+ * The following locking conditions are made on the caller:
+ * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
+ * - The caller must hold the kbasep_js_device_data::runpool_mutex
+ */
+static inline void kbase_js_runpool_dec_context_count(
+ struct kbase_device *kbdev,
+ struct kbase_context *kctx)
+{
+ struct kbasep_js_device_data *js_devdata;
+ struct kbasep_js_kctx_info *js_kctx_info;
+
+ KBASE_DEBUG_ASSERT(kbdev != NULL);
+ KBASE_DEBUG_ASSERT(kctx != NULL);
+
+ js_devdata = &kbdev->js_data;
+ js_kctx_info = &kctx->jctx.sched_info;
+
+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
+ lockdep_assert_held(&js_devdata->runpool_mutex);
+
+ /* Track total contexts */
+ --(js_devdata->nr_all_contexts_running);
+ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0);
+
+ if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) == 0) {
+ /* Track contexts that can submit jobs */
+ --(js_devdata->nr_user_contexts_running);
+ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0);
+ }
+}
+
+
+/**
+ * @brief Submit atoms from all available contexts to all job slots.
+ *
+ * This will attempt to submit as many jobs as possible. It will exit when
+ * either all job slots are full, or all contexts have been used.
+ *
+ * @param[in] kbdev Device pointer
+ */
+static inline void kbase_js_sched_all(struct kbase_device *kbdev)
+{
+ kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1);
+}
+
+extern const int
+kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS];
+
+extern const base_jd_prio
+kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT];
+
+/**
+ * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio)
+ * to relative ordering
+ * @atom_prio: Priority ID to translate.
+ *
+ * Atom priority values for @ref base_jd_prio cannot be compared directly to
+ * find out which are higher or lower.
+ *
+ * This function will convert base_jd_prio values for successively lower
+ * priorities into a monotonically increasing sequence. That is, the lower the
+ * base_jd_prio priority, the higher the value produced by this function. This
+ * is in accordance with how the rest of the kernel treates priority.
+ *
+ * The mapping is 1:1 and the size of the valid input range is the same as the
+ * size of the valid output range, i.e.
+ * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS
+ *
+ * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions
+ *
+ * Return: On success: a value in the inclusive range
+ * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure:
+ * KBASE_JS_ATOM_SCHED_PRIO_INVALID
+ */
+static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio)
+{
+ if (atom_prio >= BASE_JD_NR_PRIO_LEVELS)
+ return KBASE_JS_ATOM_SCHED_PRIO_INVALID;
+
+ return kbasep_js_atom_priority_to_relative[atom_prio];
+}
+
+static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio)
+{
+ unsigned int prio_idx;
+
+ KBASE_DEBUG_ASSERT(0 <= sched_prio
+ && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT);
+
+ prio_idx = (unsigned int)sched_prio;
+
+ return kbasep_js_relative_priority_to_atom[prio_idx];
+}
+
/** @} *//* end group kbase_js */
/** @} *//* end group base_kbase_api */
/** @} *//* end group base_api */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_js_affinity.c
- * Base kernel affinity manager APIs
- */
-
-#include <mali_kbase.h>
-#include "mali_kbase_js_affinity.h"
-
-
-STATIC INLINE mali_bool affinity_job_uses_high_cores(struct kbase_device *kbdev, struct kbase_jd_atom *katom)
-{
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
- struct kbase_context *kctx;
- kbase_context_flags ctx_flags;
-
- kctx = katom->kctx;
- ctx_flags = kctx->jctx.sched_info.ctx.flags;
-
- /* In this HW Workaround, compute-only jobs/contexts use the high cores
- * during a core-split, all other contexts use the low cores. */
- return (mali_bool) ((katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) != 0 || (ctx_flags & KBASE_CTX_FLAG_HINT_ONLY_COMPUTE) != 0);
- }
- return MALI_FALSE;
-}
-
-/**
- * @brief Decide whether a split in core affinity is required across job slots
- *
- * The following locking conditions are made on the caller:
- * - it must hold kbasep_js_device_data::runpool_irq::lock
- *
- * @param kbdev The kbase device structure of the device
- * @return MALI_FALSE if a core split is not required
- * @return != MALI_FALSE if a core split is required.
- */
-STATIC INLINE mali_bool kbase_affinity_requires_split(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
- s8 nr_compute_ctxs = kbasep_js_ctx_attr_count_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_COMPUTE);
- s8 nr_noncompute_ctxs = kbasep_js_ctx_attr_count_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_NON_COMPUTE);
-
- /* In this case, a mix of Compute+Non-Compute determines whether a
- * core-split is required, to ensure jobs with different numbers of RMUs
- * don't use the same cores.
- *
- * When it's entirely compute, or entirely non-compute, then no split is
- * required.
- *
- * A context can be both Compute and Non-compute, in which case this will
- * correctly decide that a core-split is required. */
-
- return (mali_bool) (nr_compute_ctxs > 0 && nr_noncompute_ctxs > 0);
- }
- return MALI_FALSE;
-}
-
-mali_bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, int js)
-{
- /*
- * Here are the reasons for using job slot 2:
- * - BASE_HW_ISSUE_8987 (which is entirely used for that purpose)
- * - In absence of the above, then:
- * - Atoms with BASE_JD_REQ_COHERENT_GROUP
- * - But, only when there aren't contexts with
- * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, because the atoms that run on
- * all cores on slot 1 could be blocked by those using a coherent group
- * on slot 2
- * - And, only when you actually have 2 or more coregroups - if you only
- * have 1 coregroup, then having jobs for slot 2 implies they'd also be
- * for slot 1, meaning you'll get interference from them. Jobs able to
- * run on slot 2 could also block jobs that can only run on slot 1
- * (tiler jobs)
- */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
- return MALI_TRUE;
-
- if (js != 2)
- return MALI_TRUE;
-
- /* Only deal with js==2 now: */
- if (kbdev->gpu_props.num_core_groups > 1) {
- /* Only use slot 2 in the 2+ coregroup case */
- if (kbasep_js_ctx_attr_is_attr_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) == MALI_FALSE) {
- /* ...But only when we *don't* have atoms that run on all cores */
-
- /* No specific check for BASE_JD_REQ_COHERENT_GROUP atoms - the policy will sort that out */
- return MALI_TRUE;
- }
- }
-
- /* Above checks failed mean we shouldn't use slot 2 */
- return MALI_FALSE;
-}
-
-/*
- * As long as it has been decided to have a deeper modification of
- * what job scheduler, power manager and affinity manager will
- * implement, this function is just an intermediate step that
- * assumes:
- * - all working cores will be powered on when this is called.
- * - largest current configuration is 2 core groups.
- * - It has been decided not to have hardcoded values so the low
- * and high cores in a core split will be evently distributed.
- * - Odd combinations of core requirements have been filtered out
- * and do not get to this function (e.g. CS+T+NSS is not
- * supported here).
- * - This function is frequently called and can be optimized,
- * (see notes in loops), but as the functionallity will likely
- * be modified, optimization has not been addressed.
-*/
-mali_bool kbase_js_choose_affinity(u64 * const affinity, struct kbase_device *kbdev, struct kbase_jd_atom *katom, int js)
-{
- base_jd_core_req core_req = katom->core_req;
- unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
- u64 core_availability_mask;
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- core_availability_mask = kbase_pm_ca_get_core_mask(kbdev);
-
- /*
- * If no cores are currently available (core availability policy is
- * transitioning) then fail.
- */
- if (0 == core_availability_mask) {
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
- *affinity = 0;
- return MALI_FALSE;
- }
-
- KBASE_DEBUG_ASSERT(js >= 0);
-
- if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == BASE_JD_REQ_T) {
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
- /* Tiler only job, bit 0 needed to enable tiler but no shader cores required */
- *affinity = 1;
- return MALI_TRUE;
- }
-
- if (1 == kbdev->gpu_props.num_cores) {
- /* trivial case only one core, nothing to do */
- *affinity = core_availability_mask;
- } else if (kbase_affinity_requires_split(kbdev) == MALI_FALSE) {
- if ((core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) {
- if (js == 0 || num_core_groups == 1) {
- /* js[0] and single-core-group systems just get the first core group */
- *affinity = kbdev->gpu_props.props.coherency_info.group[0].core_mask & core_availability_mask;
- } else {
- /* js[1], js[2] use core groups 0, 1 for dual-core-group systems */
- u32 core_group_idx = ((u32) js) - 1;
-
- KBASE_DEBUG_ASSERT(core_group_idx < num_core_groups);
- *affinity = kbdev->gpu_props.props.coherency_info.group[core_group_idx].core_mask & core_availability_mask;
-
- /* If the job is specifically targeting core group 1 and the core
- * availability policy is keeping that core group off, then fail */
- if (*affinity == 0 && core_group_idx == 1 && kbdev->pm.cg1_disabled == MALI_TRUE)
- katom->event_code = BASE_JD_EVENT_PM_EVENT;
- }
- } else {
- /* All cores are available when no core split is required */
- *affinity = core_availability_mask;
- }
- } else {
- /* Core split required - divide cores in two non-overlapping groups */
- u64 low_bitmap, high_bitmap;
- int n_high_cores = kbdev->gpu_props.num_cores >> 1;
- KBASE_DEBUG_ASSERT(1 == num_core_groups);
- KBASE_DEBUG_ASSERT(0 != n_high_cores);
-
- /* compute the reserved high cores bitmap */
- high_bitmap = ~0;
- /* note: this can take a while, optimization desirable */
- while (n_high_cores != hweight32(high_bitmap & kbdev->shader_present_bitmap))
- high_bitmap = high_bitmap << 1;
-
- high_bitmap &= core_availability_mask;
- low_bitmap = core_availability_mask ^ high_bitmap;
-
- if (affinity_job_uses_high_cores(kbdev, katom))
- *affinity = high_bitmap;
- else
- *affinity = low_bitmap;
- }
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- /*
- * If no cores are currently available in the desired core group(s)
- * (core availability policy is transitioning) then fail.
- */
- if (*affinity == 0)
- return MALI_FALSE;
-
- /* Enable core 0 if tiler required */
- if (core_req & BASE_JD_REQ_T)
- *affinity = *affinity | 1;
-
- return MALI_TRUE;
-}
-
-STATIC INLINE mali_bool kbase_js_affinity_is_violating(struct kbase_device *kbdev, u64 *affinities)
-{
- /* This implementation checks whether the two slots involved in Generic thread creation
- * have intersecting affinity. This is due to micro-architectural issues where a job in
- * slot A targetting cores used by slot B could prevent the job in slot B from making
- * progress until the job in slot A has completed.
- *
- * @note It just so happens that this restriction also allows
- * BASE_HW_ISSUE_8987 to be worked around by placing on job slot 2 the
- * atoms from ctxs with KBASE_CTX_FLAG_HINT_ONLY_COMPUTE flag set
- */
- u64 affinity_set_left;
- u64 affinity_set_right;
- u64 intersection;
-
- KBASE_DEBUG_ASSERT(affinities != NULL);
-
- affinity_set_left = affinities[1];
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
- /* The left set also includes those on the Fragment slot when
- * we are using the HW workaround for BASE_HW_ISSUE_8987 */
- affinity_set_left |= affinities[0];
- }
-
- affinity_set_right = affinities[2];
-
- /* A violation occurs when any bit in the left_set is also in the right_set */
- intersection = affinity_set_left & affinity_set_right;
-
- return (mali_bool) (intersection != (u64) 0u);
-}
-
-mali_bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, u64 affinity)
-{
- struct kbasep_js_device_data *js_devdata;
- u64 new_affinities[BASE_JM_MAX_NR_SLOTS];
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
- js_devdata = &kbdev->js_data;
-
- memcpy(new_affinities, js_devdata->runpool_irq.slot_affinities, sizeof(js_devdata->runpool_irq.slot_affinities));
-
- new_affinities[js] |= affinity;
-
- return kbase_js_affinity_is_violating(kbdev, new_affinities);
-}
-
-void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, u64 affinity)
-{
- struct kbasep_js_device_data *js_devdata;
- u64 cores;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
- js_devdata = &kbdev->js_data;
-
- KBASE_DEBUG_ASSERT(kbase_js_affinity_would_violate(kbdev, js, affinity) == MALI_FALSE);
-
- cores = affinity;
- while (cores) {
- int bitnum = fls64(cores) - 1;
- u64 bit = 1ULL << bitnum;
- s8 cnt;
-
- KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum] < BASE_JM_SUBMIT_SLOTS);
-
- cnt = ++(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]);
-
- if (cnt == 1)
- js_devdata->runpool_irq.slot_affinities[js] |= bit;
-
- cores &= ~bit;
- }
-}
-
-void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, u64 affinity)
-{
- struct kbasep_js_device_data *js_devdata;
- u64 cores;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
- js_devdata = &kbdev->js_data;
-
- cores = affinity;
- while (cores) {
- int bitnum = fls64(cores) - 1;
- u64 bit = 1ULL << bitnum;
- s8 cnt;
-
- KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum] > 0);
-
- cnt = --(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]);
-
- if (0 == cnt)
- js_devdata->runpool_irq.slot_affinities[js] &= ~bit;
-
- cores &= ~bit;
- }
-
-}
-
-void kbase_js_affinity_slot_blocked_an_atom(struct kbase_device *kbdev, int js)
-{
- struct kbasep_js_device_data *js_devdata;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
- js_devdata = &kbdev->js_data;
-
- js_devdata->runpool_irq.slots_blocked_on_affinity |= 1u << js;
-}
-
-void kbase_js_affinity_submit_to_blocked_slots(struct kbase_device *kbdev)
-{
- struct kbasep_js_device_data *js_devdata;
- u16 slots;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
-
- KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running != 0);
-
- /* Must take a copy because submitting jobs will update this member. */
- slots = js_devdata->runpool_irq.slots_blocked_on_affinity;
-
- while (slots) {
- int bitnum = fls(slots) - 1;
-
- u16 bit = 1u << bitnum;
- slots &= ~bit;
-
- KBASE_TRACE_ADD_SLOT(kbdev, JS_AFFINITY_SUBMIT_TO_BLOCKED, NULL, NULL, 0u, bitnum);
-
- /* must update this before we submit, incase it's set again */
- js_devdata->runpool_irq.slots_blocked_on_affinity &= ~bit;
-
- kbasep_js_try_run_next_job_on_slot_nolock(kbdev, bitnum);
-
- /* Don't re-read slots_blocked_on_affinity after this - it could loop for a long time */
- }
-}
-
-#if KBASE_TRACE_ENABLE
-void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev)
-{
- struct kbasep_js_device_data *js_devdata;
- int slot_nr;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
-
- for (slot_nr = 0; slot_nr < 3; ++slot_nr)
- KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_AFFINITY_CURRENT, NULL, NULL, 0u, slot_nr, (u32) js_devdata->runpool_irq.slot_affinities[slot_nr]);
-}
-#endif /* KBASE_TRACE_ENABLE */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_js_affinity.h
- * Affinity Manager internal APIs.
- */
-
-#ifndef _KBASE_JS_AFFINITY_H_
-#define _KBASE_JS_AFFINITY_H_
-
-/**
- * @addtogroup base_api
- * @{
- */
-
-/**
- * @addtogroup base_kbase_api
- * @{
- */
-
-/**
- * @addtogroup kbase_js_affinity Affinity Manager internal APIs.
- * @{
- *
- */
-
-/**
- * @brief Decide whether it is possible to submit a job to a particular job slot in the current status
- *
- * Will check if submitting to the given job slot is allowed in the current
- * status. For example using job slot 2 while in soft-stoppable state and only
- * having 1 coregroup is not allowed by the policy. This function should be
- * called prior to submitting a job to a slot to make sure policy rules are not
- * violated.
- *
- * The following locking conditions are made on the caller:
- * - it must hold kbasep_js_device_data::runpool_irq::lock
- *
- * @param kbdev The kbase device structure of the device
- * @param js Job slot number to check for allowance
- */
-mali_bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, int js);
-
-/**
- * @brief Compute affinity for a given job.
- *
- * Currently assumes an all-on/all-off power management policy.
- * Also assumes there is at least one core with tiler available.
- *
- * Returns MALI_TRUE if a valid affinity was chosen, MALI_FALSE if
- * no cores were available.
- *
- * @param[out] affinity Affinity bitmap computed
- * @param kbdev The kbase device structure of the device
- * @param katom Job chain of which affinity is going to be found
- * @param js Slot the job chain is being submitted
-
- */
-mali_bool kbase_js_choose_affinity(u64 * const affinity, struct kbase_device *kbdev, struct kbase_jd_atom *katom, int js);
-
-/**
- * @brief Determine whether a proposed \a affinity on job slot \a js would
- * cause a violation of affinity restrictions.
- *
- * The following locks must be held by the caller:
- * - kbasep_js_device_data::runpool_irq::lock
- */
-mali_bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, u64 affinity);
-
-/**
- * @brief Affinity tracking: retain cores used by a slot
- *
- * The following locks must be held by the caller:
- * - kbasep_js_device_data::runpool_irq::lock
- */
-void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, u64 affinity);
-
-/**
- * @brief Affinity tracking: release cores used by a slot
- *
- * Cores \b must be released as soon as a job is dequeued from a slot's 'submit
- * slots', and before another job is submitted to those slots. Otherwise, the
- * refcount could exceed the maximum number submittable to a slot,
- * BASE_JM_SUBMIT_SLOTS.
- *
- * The following locks must be held by the caller:
- * - kbasep_js_device_data::runpool_irq::lock
- */
-void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, u64 affinity);
-
-/**
- * @brief Register a slot as blocking atoms due to affinity violations
- *
- * Once a slot has been registered, we must check after every atom completion
- * (including those on different slots) to see if the slot can be
- * unblocked. This is done by calling
- * kbase_js_affinity_submit_to_blocked_slots(), which will also deregister the
- * slot if it no long blocks atoms due to affinity violations.
- *
- * The following locks must be held by the caller:
- * - kbasep_js_device_data::runpool_irq::lock
- */
-void kbase_js_affinity_slot_blocked_an_atom(struct kbase_device *kbdev, int js);
-
-/**
- * @brief Submit to job slots that have registered that an atom was blocked on
- * the slot previously due to affinity violations.
- *
- * This submits to all slots registered by
- * kbase_js_affinity_slot_blocked_an_atom(). If submission succeeded, then the
- * slot is deregistered as having blocked atoms due to affinity
- * violations. Otherwise it stays registered, and the next atom to complete
- * must attempt to submit to the blocked slots again.
- *
- * This must only be called whilst the GPU is powered - for example, when
- * kbdev->jsdata.nr_user_contexts_running > 0.
- *
- * The following locking conditions are made on the caller:
- * - it must hold kbasep_js_device_data::runpool_mutex
- * - it must hold kbasep_js_device_data::runpool_irq::lock
- */
-void kbase_js_affinity_submit_to_blocked_slots(struct kbase_device *kbdev);
-
-/**
- * @brief Output to the Trace log the current tracked affinities on all slots
- */
-#if KBASE_TRACE_ENABLE
-void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev);
-#else /* KBASE_TRACE_ENABLE */
-static INLINE void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev)
-{
-}
-#endif /* KBASE_TRACE_ENABLE */
-
- /** @} *//* end group kbase_js_affinity */
- /** @} *//* end group base_kbase_api */
- /** @} *//* end group base_api */
-
-
-#endif /* _KBASE_JS_AFFINITY_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* - runpool_irq spinlock
* - ctx is scheduled on the runpool
*
- * @return MALI_TRUE indicates a change in ctx attributes state of the runpool.
+ * @return true indicates a change in ctx attributes state of the runpool.
* In this state, the scheduler might be able to submit more jobs than
* previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock()
* or similar is called sometime later.
- * @return MALI_FALSE indicates no change in ctx attributes state of the runpool.
+ * @return false indicates no change in ctx attributes state of the runpool.
*/
-STATIC mali_bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
+static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
- mali_bool runpool_state_changed = MALI_FALSE;
+ bool runpool_state_changed = false;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
BUG_ON(!mutex_is_locked(&js_kctx_info->ctx.jsctx_mutex));
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled != MALI_FALSE);
+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled != false);
- if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != MALI_FALSE) {
+ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) {
KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX);
++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]);
if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) {
/* First refcount indicates a state change */
- runpool_state_changed = MALI_TRUE;
+ runpool_state_changed = true;
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute);
}
}
* - runpool_irq spinlock
* - ctx is scheduled on the runpool
*
- * @return MALI_TRUE indicates a change in ctx attributes state of the runpool.
+ * @return true indicates a change in ctx attributes state of the runpool.
* In this state, the scheduler might be able to submit more jobs than
* previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock()
* or similar is called sometime later.
- * @return MALI_FALSE indicates no change in ctx attributes state of the runpool.
+ * @return false indicates no change in ctx attributes state of the runpool.
*/
-STATIC mali_bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
+static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
- mali_bool runpool_state_changed = MALI_FALSE;
+ bool runpool_state_changed = false;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
BUG_ON(!mutex_is_locked(&js_kctx_info->ctx.jsctx_mutex));
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled != MALI_FALSE);
+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled != false);
- if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != MALI_FALSE) {
+ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) {
KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0);
--(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]);
if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) {
/* Last de-refcount indicates a state change */
- runpool_state_changed = MALI_TRUE;
+ runpool_state_changed = true;
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute);
}
}
* - jsctx mutex
* - If the context is scheduled, then runpool_irq spinlock must also be held
*
- * @return MALI_TRUE indicates a change in ctx attributes state of the runpool.
+ * @return true indicates a change in ctx attributes state of the runpool.
* This may allow the scheduler to submit more jobs than previously.
- * @return MALI_FALSE indicates no change in ctx attributes state of the runpool.
+ * @return false indicates no change in ctx attributes state of the runpool.
*/
-STATIC mali_bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
+static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
{
struct kbasep_js_kctx_info *js_kctx_info;
- mali_bool runpool_state_changed = MALI_FALSE;
+ bool runpool_state_changed = false;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
js_kctx_info = &kctx->jctx.sched_info;
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
BUG_ON(!mutex_is_locked(&js_kctx_info->ctx.jsctx_mutex));
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX);
++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]);
- if (js_kctx_info->ctx.is_scheduled != MALI_FALSE && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
- lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
+ if (js_kctx_info->ctx.is_scheduled != false && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
/* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute);
runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute);
return runpool_state_changed;
}
-/**
- * @brief Release a certain attribute on a ctx, also releasign it from the runpool
+/*
+ * @brief Release a certain attribute on a ctx, also releasing it from the runpool
* if the context is scheduled.
*
* Requires:
* - jsctx mutex
* - If the context is scheduled, then runpool_irq spinlock must also be held
*
- * @return MALI_TRUE indicates a change in ctx attributes state of the runpool.
+ * @return true indicates a change in ctx attributes state of the runpool.
* This may allow the scheduler to submit more jobs than previously.
- * @return MALI_FALSE indicates no change in ctx attributes state of the runpool.
+ * @return false indicates no change in ctx attributes state of the runpool.
*/
-STATIC mali_bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
+static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
{
struct kbasep_js_kctx_info *js_kctx_info;
- mali_bool runpool_state_changed = MALI_FALSE;
+ bool runpool_state_changed = false;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
BUG_ON(!mutex_is_locked(&js_kctx_info->ctx.jsctx_mutex));
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0);
- if (js_kctx_info->ctx.is_scheduled != MALI_FALSE && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
+ if (js_kctx_info->ctx.is_scheduled != false && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
/* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */
runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute);
void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx)
{
struct kbasep_js_kctx_info *js_kctx_info;
- mali_bool runpool_state_changed = MALI_FALSE;
+ bool runpool_state_changed = false;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_kctx_info = &kctx->jctx.sched_info;
- if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) != MALI_FALSE) {
+ if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) != false) {
/* This context never submits, so don't track any scheduling attributes */
return;
}
/* Transfer attributes held in the context flags for contexts that have submit enabled */
- if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_HINT_ONLY_COMPUTE) != MALI_FALSE) {
+ if ((js_kctx_info->ctx.flags & KBASE_CTX_FLAG_HINT_ONLY_COMPUTE) != false) {
/* Compute context */
runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE);
}
/* The context should not have been scheduled yet, so ASSERT if this caused
* runpool state changes (note that other threads *can't* affect the value
* of runpool_state_changed, due to how it's calculated) */
- KBASE_DEBUG_ASSERT(runpool_state_changed == MALI_FALSE);
+ KBASE_DEBUG_ASSERT(runpool_state_changed == false);
CSTD_UNUSED(runpool_state_changed);
}
void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
{
- mali_bool runpool_state_changed;
+ bool runpool_state_changed;
int i;
/* Retain any existing attributes */
for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) {
- if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != MALI_FALSE) {
+ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) {
/* The context is being scheduled in, so update the runpool with the new attributes */
runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i);
}
}
-mali_bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
+bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
{
- mali_bool runpool_state_changed = MALI_FALSE;
+ bool runpool_state_changed = false;
int i;
/* Release any existing attributes */
for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) {
- if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != MALI_FALSE) {
+ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) {
/* The context is being scheduled out, so update the runpool on the removed attributes */
runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i);
}
void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
- mali_bool runpool_state_changed = MALI_FALSE;
+ bool runpool_state_changed = false;
base_jd_core_req core_req;
KBASE_DEBUG_ASSERT(katom);
CSTD_UNUSED(runpool_state_changed);
}
-mali_bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state)
+bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state)
{
- mali_bool runpool_state_changed = MALI_FALSE;
+ bool runpool_state_changed = false;
base_jd_core_req core_req;
KBASE_DEBUG_ASSERT(katom_retained_state);
core_req = katom_retained_state->core_req;
/* No-op for invalid atoms */
- if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == MALI_FALSE)
- return MALI_FALSE;
-
- if (core_req & BASE_JD_REQ_ONLY_COMPUTE) {
-#if KBASE_PM_EN
- unsigned long flags;
- int device_nr = (core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) ? katom_retained_state->device_nr : 0;
- KBASE_DEBUG_ASSERT(device_nr < 2);
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbasep_pm_record_job_status(kbdev);
- kbdev->pm.metrics.active_cl_ctx[device_nr]--;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-#endif
+ if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false)
+ return false;
+
+ if (core_req & BASE_JD_REQ_ONLY_COMPUTE)
runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE);
- } else {
-#if KBASE_PM_EN
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbasep_pm_record_job_status(kbdev);
- kbdev->pm.metrics.active_gl_ctx--;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-#endif
+ else
runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE);
- }
if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) {
/* Atom that can run on slot1 or slot2, and can use all cores */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* - runpool_irq spinlock
* - ctx->is_scheduled is true
*
- * @return MALI_TRUE indicates a change in ctx attributes state of the runpool.
+ * @return true indicates a change in ctx attributes state of the runpool.
* In this state, the scheduler might be able to submit more jobs than
* previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock()
* or similar is called sometime later.
- * @return MALI_FALSE indicates no change in ctx attributes state of the runpool.
+ * @return false indicates no change in ctx attributes state of the runpool.
*/
-mali_bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
+bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
/**
* Retain all attributes of an atom
*
* This is a no-op when \a katom_retained_state is invalid.
*
- * @return MALI_TRUE indicates a change in ctx attributes state of the runpool.
+ * @return true indicates a change in ctx attributes state of the runpool.
* In this state, the scheduler might be able to submit more jobs than
* previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock()
* or similar is called sometime later.
- * @return MALI_FALSE indicates no change in ctx attributes state of the runpool.
+ * @return false indicates no change in ctx attributes state of the runpool.
*/
-mali_bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state);
+bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state);
/**
* Requires:
* - runpool_irq spinlock
*/
-static INLINE s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute)
+static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute)
{
struct kbasep_js_device_data *js_devdata;
* Requires:
* - runpool_irq spinlock
*/
-static INLINE mali_bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute)
+static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute)
{
/* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */
- return (mali_bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute);
+ return (bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute);
}
/**
* Requires:
* - jsctx mutex
*/
-static INLINE mali_bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
+static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
{
struct kbasep_js_kctx_info *js_kctx_info;
js_kctx_info = &kctx->jctx.sched_info;
/* In general, attributes are 'on' when they have a refcount (which should never be < 0) */
- return (mali_bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]);
+ return (bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]);
}
/** @} *//* end group kbase_js */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/* Wrapper Interface - doxygen is elsewhere */
union kbasep_js_policy {
-#ifdef KBASE_JS_POLICY_AVAILABLE_FCFS
- struct kbasep_js_policy_fcfs fcfs;
-#endif
-#ifdef KBASE_JS_POLICY_AVAILABLE_CFS
struct kbasep_js_policy_cfs cfs;
-#endif
};
/* Wrapper Interface - doxygen is elsewhere */
union kbasep_js_policy_ctx_info {
-#ifdef KBASE_JS_POLICY_AVAILABLE_FCFS
- struct kbasep_js_policy_fcfs_ctx fcfs;
-#endif
-#ifdef KBASE_JS_POLICY_AVAILABLE_CFS
struct kbasep_js_policy_cfs_ctx cfs;
-#endif
};
/* Wrapper Interface - doxygen is elsewhere */
union kbasep_js_policy_job_info {
-#ifdef KBASE_JS_POLICY_AVAILABLE_FCFS
- struct kbasep_js_policy_fcfs_job fcfs;
-#endif
-#ifdef KBASE_JS_POLICY_AVAILABLE_CFS
struct kbasep_js_policy_cfs_job cfs;
-#endif
};
* When bit 'N' is set in this, it indicates whether the context bound to address space
* 'N' (per_as_data[N].kctx) is allowed to submit jobs.
*
- * It is placed here because it's much more memory efficient than having a mali_bool8 in
+ * It is placed here because it's much more memory efficient than having a u8 in
* struct kbasep_js_per_as_data to store this flag */
u16 submit_allowed;
/** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates
* that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */
u64 slot_affinities[BASE_JM_MAX_NR_SLOTS];
- /** Bitvector indicating which slots \em might have atoms blocked on
- * them because otherwise they'd violate affinity restrictions */
- u16 slots_blocked_on_affinity;
/** Refcount for each core owned by each slot. Used to generate the
* slot_affinities array of bitvectors
*
* submitted to a slot, and is de-refcounted immediately after a job
* finishes */
s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64];
+
+ /*
+ * true when GPU is put into secure mode
+ */
+ bool secure_mode;
} runpool_irq;
/**
*/
struct mutex queue_mutex;
+ /**
+ * Scheduling semaphore. This must be held when calling
+ * kbase_jm_kick()
+ */
+ struct semaphore schedule_sem;
+
+ /**
+ * List of contexts that can currently be pulled from
+ */
+ struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS];
+ /**
+ * List of contexts that can not currently be pulled from, but have
+ * jobs currently running.
+ */
+ struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS];
+
u16 as_free; /**< Bitpattern of free Address Spaces */
/** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */
* @note This is a write-once member, and so no locking is required to read */
base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS];
- u32 scheduling_tick_ns; /**< Value for KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS */
- u32 soft_stop_ticks; /**< Value for KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS */
- u32 soft_stop_ticks_cl; /**< Value for KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL */
- u32 hard_stop_ticks_ss; /**< Value for KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS */
- u32 hard_stop_ticks_cl; /**< Value for KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL */
- u32 hard_stop_ticks_nss; /**< Value for KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS */
- u32 gpu_reset_ticks_ss; /**< Value for KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS */
- u32 gpu_reset_ticks_cl; /**< Value for KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL */
- u32 gpu_reset_ticks_nss; /**< Value for KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS */
- u32 ctx_timeslice_ns; /**< Value for KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS */
+ u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */
+ u32 soft_stop_ticks; /*< Value for JS_SOFT_STOP_TICKS */
+ u32 soft_stop_ticks_cl; /*< Value for JS_SOFT_STOP_TICKS_CL */
+ u32 hard_stop_ticks_ss; /*< Value for JS_HARD_STOP_TICKS_SS */
+ u32 hard_stop_ticks_cl; /*< Value for JS_HARD_STOP_TICKS_CL */
+ u32 hard_stop_ticks_dumping; /*< Value for JS_HARD_STOP_TICKS_DUMPING */
+ u32 gpu_reset_ticks_ss; /*< Value for JS_RESET_TICKS_SS */
+ u32 gpu_reset_ticks_cl; /*< Value for JS_RESET_TICKS_CL */
+ u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */
+ u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */
u32 cfs_ctx_runtime_init_slices; /**< Value for DEFAULT_JS_CFS_CTX_RUNTIME_INIT_SLICES */
u32 cfs_ctx_runtime_min_slices; /**< Value for DEFAULT_JS_CFS_CTX_RUNTIME_MIN_SLICES */
#ifdef CONFIG_MALI_DEBUG
/* Support soft-stop on a single context */
- mali_bool softstop_always;
+ bool softstop_always;
#endif /* CONFIG_MALI_DEBUG */
/** The initalized-flag is placed at the end, to avoid cache-pollution (we should
* only be using this during init/term paths).
* @note This is a write-once member, and so no locking is required to read */
int init_status;
+
+ /* Number of contexts that can currently be pulled from */
+ u32 nr_contexts_pullable;
+
+ /* Number of contexts that can either be pulled from or are currently
+ * running */
+ atomic_t nr_contexts_runnable;
};
/**
*
* This is only ever updated whilst the jsctx_mutex is held.
*/
- mali_bool is_scheduled;
+ bool is_scheduled;
/**
* Wait queue to wait for is_scheduled state changes.
* */
wait_queue_head_t is_scheduled_wait;
- mali_bool is_dying; /**< Is the context in the process of being evicted? */
+ bool is_dying; /**< Is the context in the process of being evicted? */
+
+ /** Link implementing JS queues. Context can be present on one
+ * list per job slot
+ */
+ struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS];
+
+ atomic_t fault_count; /**< The no. of times the context is retained due to the fault job. */
} ctx;
/* The initalized-flag is placed at the end, to avoid cache-pollution (we should
enum base_jd_event_code event_code;
/** core requirements */
base_jd_core_req core_req;
+ /* priority */
+ int sched_priority;
/** Job Slot to retry submitting to if submission from IRQ handler failed */
int retry_submit_on_slot;
/* Core group atom was executed on */
*/
#define KBASEP_JS_TICK_RESOLUTION_US 1
-#endif /* _KBASE_JS_DEFS_H_ */
+/*
+ * Internal atom priority defines for kbase_jd_atom::sched_prio
+ */
+enum {
+ KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0,
+ KBASE_JS_ATOM_SCHED_PRIO_MED,
+ KBASE_JS_ATOM_SCHED_PRIO_LOW,
+ KBASE_JS_ATOM_SCHED_PRIO_COUNT,
+};
+
+/* Invalid priority for kbase_jd_atom::sched_prio */
+#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1
+
+/* Default priority in the case of contexts with no atoms, or being lenient
+ * about invalid priorities from userspace */
+#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED
/** @} *//* end group kbase_js */
/** @} *//* end group base_kbase_api */
/** @} *//* end group base_api */
+
+#endif /* _KBASE_JS_DEFS_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* The following diagram shows an example Policy that contains a Low Priority
* queue, and a Real-time (High Priority) Queue. The RT queue is examined
* before the LowP one on dequeuing from the head. The Low Priority Queue is
- * ordered by time, and the RT queue is ordered by RT-priority, and then by
- * time. In addition, it shows that the Job Scheduler Core will start a
+ * ordered by time, and the RT queue is ordered by time weighted by
+ * RT-priority. In addition, it shows that the Job Scheduler Core will start a
* Soft-Stop Timer (SS-Timer) when it dequeue's and submits a job. The
* Soft-Stop time is set by a global configuration value, and must be a value
* appropriate for the policy. For example, this could include "don't run a
*
* @section sec_kbase_js_policy_operation_prio Dealing with Priority
*
- * Priority applies both to a context as a whole, and to the jobs within a
- * context. The jobs specify a priority in the base_jd_atom::prio member, which
- * is relative to that of the context. A positive setting indicates a reduction
- * in priority, whereas a negative setting indicates a boost in priority. Of
- * course, the boost in priority should only be honoured when the originating
- * process has sufficient priviledges, and should be ignored for unpriviledged
- * processes. The meaning of the combined priority value is up to the policy
+ * Priority applies separately to a context as a whole, and to the jobs within
+ * a context. The jobs specify a priority in the base_jd_atom::prio member, but
+ * it is independent of the context priority. That is, it only affects
+ * scheduling of atoms within a context. Refer to @ref base_jd_prio for more
+ * details. The meaning of the context's priority value is up to the policy
* itself, and could be a logarithmic scale instead of a linear scale (e.g. the
* policy could implement an increase/decrease in priority by 1 results in an
* increase/decrease in \em proportion of time spent scheduled in by 25%, an
* It is up to the policy whether a boost in priority boosts the priority of
* the entire context (e.g. to such an extent where it may pre-empt other
* running contexts). If it chooses to do this, the Policy must make sure that
- * only the high-priority jobs are run, and that the context is scheduled out
- * once only low priority jobs remain. This ensures that the low priority jobs
- * within the context do not gain from the priority boost, yet they still get
- * scheduled correctly with respect to other low priority contexts.
+ * only jobs from high-priority contexts are run, and that the context is
+ * scheduled out once only jobs from low priority contexts remain. This ensures
+ * that the low priority contexts do not gain from the priority boost, yet they
+ * still get scheduled correctly with respect to other low priority contexts.
*
*
* @section sec_kbase_js_policy_operation_irq IRQ Path
* - If there is space in the completed job slots' HEAD/NEXT registers, run the next job:
* - kbasep_js_policy_dequeue_job() <em>in the context of the irq
* handler</em> with core_req set to that of the completing slot
- * - if this returned MALI_TRUE, submit the job to the completed slot.
+ * - if this returned true, submit the job to the completed slot.
* - This is repeated until kbasep_js_policy_dequeue_job() returns
- * MALI_FALSE, or the job slot has a job queued on both the HEAD and NEXT registers.
+ * false, or the job slot has a job queued on both the HEAD and NEXT registers.
* - If kbasep_js_policy_dequeue_job() returned false, submit some work to
* the work-queue to retry from outside of IRQ context (calling
* kbasep_js_policy_dequeue_job() from a work-queue).
* slot
* - if we got one, submit it to the job slot.
* - This is repeated until kbasep_js_policy_dequeue_job() returns
- * MALI_FALSE, or the job slot has a job queued on both the HEAD and NEXT registers.
+ * false, or the job slot has a job queued on both the HEAD and NEXT registers.
*
* The above case shows that we should attempt to run jobs in cases where a) a ctx
* has been added to the Run Pool, and b) new jobs have been added to a context
/**
* @brief Initialize the Job Scheduler Policy
*/
-mali_error kbasep_js_policy_init(struct kbase_device *kbdev);
+int kbasep_js_policy_init(struct kbase_device *kbdev);
/**
* @brief Terminate the Job Scheduler Policy
* This effectively initializes the union kbasep_js_policy_ctx_info structure within
* the struct kbase_context (itself located within the kctx->jctx.sched_info structure).
*/
-mali_error kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
+int kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx);
/**
* @brief Terminate resources associated with using a ctx in the Job Scheduler
*
* The caller will be holding kbasep_js_device_data::queue_mutex.
*
- * @return MALI_TRUE if a context was available, and *kctx_ptr points to
+ * @return true if a context was available, and *kctx_ptr points to
* the kctx dequeued.
- * @return MALI_FALSE if no contexts were available.
+ * @return false if no contexts were available.
*/
-mali_bool kbasep_js_policy_dequeue_head_ctx(union kbasep_js_policy *js_policy, struct kbase_context ** const kctx_ptr);
+bool kbasep_js_policy_dequeue_head_ctx(union kbasep_js_policy *js_policy, struct kbase_context ** const kctx_ptr);
/**
* @brief Evict a context from the Job Scheduler Policy Queue
*
* The caller will be holding kbasep_js_device_data::queue_mutex.
*
- * @return MALI_TRUE if the context was evicted from the Policy Queue
- * @return MALI_FALSE if the context was not found in the Policy Queue
+ * @return true if the context was evicted from the Policy Queue
+ * @return false if the context was not found in the Policy Queue
*/
-mali_bool kbasep_js_policy_try_evict_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx);
+bool kbasep_js_policy_try_evict_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx);
/**
* @brief Call a function on all jobs belonging to a non-queued, non-running
* context, optionally detaching the jobs from the context as it goes.
*
* At the time of the call, the context is guarenteed to be not-currently
- * scheduled on the Run Pool (is_scheduled == MALI_FALSE), and not present in
+ * scheduled on the Run Pool (is_scheduled == false), and not present in
* the Policy Queue. This is because one of the following functions was used
* recently on the context:
* - kbasep_js_policy_evict_ctx()
* - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex.
*/
void kbasep_js_policy_foreach_ctx_job(union kbasep_js_policy *js_policy, struct kbase_context *kctx,
- kbasep_js_policy_ctx_job_cb callback, mali_bool detach_jobs);
+ kbasep_js_policy_ctx_job_cb callback, bool detach_jobs);
/**
* @brief Add a context to the Job Scheduler Policy's Run Pool
* - the context has its submit_allowed flag set
* - kbasep_js_device_data::runpool_irq::per_as_data[kctx->as_nr] is valid
* - The refcount of the context is guarenteed to be zero.
- * - kbasep_js_kctx_info::ctx::is_scheduled will be MALI_TRUE.
+ * - kbasep_js_kctx_info::ctx::is_scheduled will be true.
*
* The locking conditions on the caller are as follows:
* - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex.
*
* @note This API is called from IRQ context.
*/
-mali_bool kbasep_js_policy_should_remove_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx);
+bool kbasep_js_policy_should_remove_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx);
/**
* @brief Synchronize with any timers acting upon the runpool
* cannot be held). Therefore, this function should only be seen as a heuristic
* guide as to whether \a new_ctx is higher priority than \a current_ctx
*/
-mali_bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx);
+bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx);
/** @} *//* end group kbase_js_policy_ctx */
*
* This structure is embedded in the struct kbase_jd_atom structure. It is used to:
* - track information needed for the policy to schedule the job (e.g. time
- * used, OS priority etc.)
+ * used, etc.)
* - link together jobs into a queue/buffer, so that a struct kbase_jd_atom can be
* obtained as the container of the policy job info. This allows the API to
* return what "the next job" should be.
- * - obtain other information already stored in the struct kbase_context for
- * scheduling purposes (e.g user-side relative priority)
*/
union kbasep_js_policy_job_info;
* The caller will not be holding any locks, and so this function will not
* modify any information in \a kctx or \a js_policy.
*
- * @return MALI_ERROR_NONE if initialization was correct.
+ * @return 0 if initialization was correct.
*/
-mali_error kbasep_js_policy_init_job(const union kbasep_js_policy *js_policy, const struct kbase_context *kctx, struct kbase_jd_atom *katom);
+int kbasep_js_policy_init_job(const union kbasep_js_policy *js_policy, const struct kbase_context *kctx, struct kbase_jd_atom *katom);
/**
* @brief Register context/policy-wide information for a job on the Job Scheduler Policy.
* this new job when the currently executing job (in the jobslot's HEAD
* register) has completed.
*
- * @return MALI_TRUE if a job was available, and *kctx_ptr points to
+ * @return true if a job was available, and *kctx_ptr points to
* the kctx dequeued.
- * @return MALI_FALSE if no jobs were available among all ctxs in the Run Pool.
+ * @return false if no jobs were available among all ctxs in the Run Pool.
*
* @note base_jd_core_req is currently a u8 - beware of type conversion.
*
* - kbasep_js_device_data::runpool_mutex will be held.
* - kbasep_js_kctx_info::ctx::jsctx_mutex. will be held
*/
-mali_bool kbasep_js_policy_dequeue_job(struct kbase_device *kbdev, int job_slot_idx, struct kbase_jd_atom ** const katom_ptr);
+bool kbasep_js_policy_dequeue_job(struct kbase_device *kbdev, int job_slot_idx, struct kbase_jd_atom ** const katom_ptr);
/**
* @brief Requeue a Job back into the the Job Scheduler Policy Run Pool
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
*/
#include <mali_kbase.h>
-#include <mali_kbase_jm.h>
#include <mali_kbase_js.h>
#include <mali_kbase_js_policy_cfs.h>
+#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
#include <linux/sched/rt.h>
#endif
*/
#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL)
-/** Fixed point constants used for runtime weight calculations */
+/* Fixed point constants used for runtime weight calculations */
#define WEIGHT_FIXEDPOINT_SHIFT 10
#define WEIGHT_TABLE_SIZE 40
#define WEIGHT_0_NICE (WEIGHT_TABLE_SIZE/2)
#define WEIGHT_0_VAL (1 << WEIGHT_FIXEDPOINT_SHIFT)
-#define LOOKUP_VARIANT_MASK ((1u<<KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS) - 1u)
-
#define PROCESS_PRIORITY_MIN (-20)
#define PROCESS_PRIORITY_MAX (19)
-/** Core requirements that all the variants support */
-#define JS_CORE_REQ_ALL_OTHERS \
- (BASE_JD_REQ_CF | BASE_JD_REQ_V | BASE_JD_REQ_PERMON | BASE_JD_REQ_EXTERNAL_RESOURCES | BASEP_JD_REQ_EVENT_NEVER)
-
-/** Context requirements the all the variants support */
-
-/* In HW issue 8987 workaround, restrict Compute-only contexts and Compute jobs onto job slot[2],
- * which will ensure their affinity does not intersect GLES jobs */
-#define JS_CTX_REQ_ALL_OTHERS_8987 \
- (KBASE_CTX_FLAG_PRIVILEGED)
-#define JS_CORE_REQ_COMPUTE_SLOT_8987 \
- (BASE_JD_REQ_CS)
-#define JS_CORE_REQ_ONLY_COMPUTE_SLOT_8987 \
- (BASE_JD_REQ_ONLY_COMPUTE)
-
-/* Otherwise, compute-only contexts/compute jobs can use any job slot */
-#define JS_CTX_REQ_ALL_OTHERS \
- (KBASE_CTX_FLAG_PRIVILEGED | KBASE_CTX_FLAG_HINT_ONLY_COMPUTE)
-#define JS_CORE_REQ_COMPUTE_SLOT \
- (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE)
-
-/* core_req variants are ordered by least restrictive first, so that our
- * algorithm in cached_variant_idx_init picks the least restrictive variant for
- * each job . Note that coherent_group requirement is added to all CS variants as the
- * selection of job-slot does not depend on the coherency requirement. */
-static const struct kbasep_atom_req core_req_variants[] = {
- {
- /* 0: Fragment variant */
- (JS_CORE_REQ_ALL_OTHERS | BASE_JD_REQ_FS | BASE_JD_REQ_FS_AFBC |
- BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS),
- 0},
- {
- /* 1: Compute variant, can use all coregroups */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT),
- (JS_CTX_REQ_ALL_OTHERS),
- 0},
- {
- /* 2: Compute variant, uses only coherent coregroups */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT | BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS),
- 0},
- {
- /* 3: Compute variant, might only use coherent coregroup, and must use tiling */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_T),
- (JS_CTX_REQ_ALL_OTHERS),
- 0},
-
- {
- /* 4: Unused */
- 0,
- 0,
- 0},
-
- {
- /* 5: Compute variant for specific-coherent-group targetting CoreGroup 0 */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS),
- 0 /* device_nr */
- },
- {
- /* 6: Compute variant for specific-coherent-group targetting CoreGroup 1 */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS),
- 1 /* device_nr */
- },
-
- /* Unused core_req variants, to bring the total up to a power of 2 */
- {
- /* 7 */
- 0,
- 0,
- 0},
-};
-
-static const struct kbasep_atom_req core_req_variants_8987[] = {
- {
- /* 0: Fragment variant */
- (JS_CORE_REQ_ALL_OTHERS | BASE_JD_REQ_FS | BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987),
- 0},
- {
- /* 1: Compute variant, can use all coregroups */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT_8987),
- (JS_CTX_REQ_ALL_OTHERS_8987),
- 0},
- {
- /* 2: Compute variant, uses only coherent coregroups */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987),
- 0},
- {
- /* 3: Compute variant, might only use coherent coregroup, and must use tiling */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_T),
- (JS_CTX_REQ_ALL_OTHERS_8987),
- 0},
-
- {
- /* 4: Variant guarenteed to support Compute contexts/atoms
- *
- * In the case of a context that's specified as 'Only Compute', it'll
- * not allow Tiler or Fragment atoms, and so those get rejected */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_ONLY_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987 | KBASE_CTX_FLAG_HINT_ONLY_COMPUTE),
- 0},
-
- {
- /* 5: Compute variant for specific-coherent-group targetting CoreGroup 0
- * Specifically, this only allows 'Only Compute' contexts/atoms */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_ONLY_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987 | KBASE_CTX_FLAG_HINT_ONLY_COMPUTE),
- 0 /* device_nr */
- },
- {
- /* 6: Compute variant for specific-coherent-group targetting CoreGroup 1
- * Specifically, this only allows 'Only Compute' contexts/atoms */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_ONLY_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987 | KBASE_CTX_FLAG_HINT_ONLY_COMPUTE),
- 1 /* device_nr */
- },
- /* Unused core_req variants, to bring the total up to a power of 2 */
- {
- /* 7 */
- 0,
- 0,
- 0},
-};
-
-#define CORE_REQ_VARIANT_FRAGMENT 0
-#define CORE_REQ_VARIANT_COMPUTE_ALL_CORES 1
-#define CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP 2
-#define CORE_REQ_VARIANT_COMPUTE_OR_TILING 3
-#define CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_0 5
-#define CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_1 6
-
-#define CORE_REQ_VARIANT_ONLY_COMPUTE_8987 4
-#define CORE_REQ_VARIANT_ONLY_COMPUTE_8987_SPECIFIC_COHERENT_0 5
-#define CORE_REQ_VARIANT_ONLY_COMPUTE_8987_SPECIFIC_COHERENT_1 6
-
-#define NUM_CORE_REQ_VARIANTS NELEMS(core_req_variants)
-#define NUM_CORE_REQ_VARIANTS_8987 NELEMS(core_req_variants_8987)
-
-/** Mappings between job slot and variant lists for Soft-Stoppable State */
-static const u32 variants_supported_ss_state[] = {
- /* js[0] uses Fragment only */
- (1u << CORE_REQ_VARIANT_FRAGMENT),
-
- /* js[1] uses: Compute-all-cores, Compute-only-coherent, Compute-or-Tiling,
- * compute-specific-coregroup-0 */
- (1u << CORE_REQ_VARIANT_COMPUTE_ALL_CORES)
- | (1u << CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP)
- | (1u << CORE_REQ_VARIANT_COMPUTE_OR_TILING)
- | (1u << CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_0),
-
- /* js[2] uses: Compute-only-coherent, compute-specific-coregroup-1 */
- (1u << CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP)
- | (1u << CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_1)
-};
-
-/** Mappings between job slot and variant lists for Soft-Stoppable State, when
- * we have atoms that can use all the cores (KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES)
- * and there's more than one coregroup */
-static const u32 variants_supported_ss_allcore_state[] = {
- /* js[0] uses Fragment only */
- (1u << CORE_REQ_VARIANT_FRAGMENT),
-
- /* js[1] uses: Compute-all-cores, Compute-only-coherent, Compute-or-Tiling,
- * compute-specific-coregroup-0, compute-specific-coregroup-1 */
- (1u << CORE_REQ_VARIANT_COMPUTE_ALL_CORES)
- | (1u << CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP)
- | (1u << CORE_REQ_VARIANT_COMPUTE_OR_TILING)
- | (1u << CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_0)
- | (1u << CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_1),
-
- /* js[2] not used */
- 0
-};
-
-/** Mappings between job slot and variant lists for Soft-Stoppable State for
- * BASE_HW_ISSUE_8987
- *
- * @note There is no 'allcores' variant of this, because this HW issue forces all
- * atoms with BASE_JD_CORE_REQ_SPECIFIC_COHERENT_GROUP to use slot 2 anyway -
- * hence regardless of whether a specific coregroup is targetted, those atoms
- * still make progress. */
-static const u32 variants_supported_ss_state_8987[] = {
- /* js[0] uses Fragment only */
- (1u << CORE_REQ_VARIANT_FRAGMENT),
-
- /* js[1] uses: Compute-all-cores, Compute-only-coherent, Compute-or-Tiling */
- (1u << CORE_REQ_VARIANT_COMPUTE_ALL_CORES)
- | (1u << CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP)
- | (1u << CORE_REQ_VARIANT_COMPUTE_OR_TILING),
-
- /* js[2] uses: All Only-compute atoms (including those targetting a
- * specific coregroup), and nothing else. This is because their affinity
- * must not intersect with non-only-compute atoms.
- *
- * As a side effect, this causes the 'device_nr' for atoms targetting a
- * specific coregroup to be ignored */
- (1u << CORE_REQ_VARIANT_ONLY_COMPUTE_8987)
- | (1u << CORE_REQ_VARIANT_ONLY_COMPUTE_8987_SPECIFIC_COHERENT_0)
- | (1u << CORE_REQ_VARIANT_ONLY_COMPUTE_8987_SPECIFIC_COHERENT_1)
-};
-
/* Defines for easy asserts 'is scheduled'/'is queued'/'is neither queued norscheduled' */
#define KBASEP_JS_CHECKFLAG_QUEUED (1u << 0) /**< Check the queued state */
#define KBASEP_JS_CHECKFLAG_SCHEDULED (1u << 1) /**< Check the scheduled state */
* Private Functions
*/
-/* Table autogenerated using util built from: midgard/scripts/gen_cfs_weight_of_prio.c */
+/* Table autogenerated using util built from: base/tools/gen_cfs_weight_of_prio/ */
/* weight = 1.25 */
static const int weight_of_priority[] = {
/* 16 */ 36388, 45485, 56856, 71070
};
-/**
- * @note There is nothing to stop the priority of the ctx containing \a
+/*
+ * Note: There is nothing to stop the priority of the ctx containing
* ctx_info changing during or immediately after this function is called
* (because its jsctx_mutex cannot be held during IRQ). Therefore, this
* function should only be seen as a heuristic guide as to the priority weight
* of the context.
*/
-STATIC u64 priority_weight(struct kbasep_js_policy_cfs_ctx *ctx_info, u64 time_us)
+static u64 priority_weight(struct kbasep_js_policy_cfs_ctx *ctx_info, u64 time_us)
{
u64 time_delta_us;
int priority;
- priority = ctx_info->process_priority + ctx_info->bag_priority;
+ priority = ctx_info->process_priority;
/* Adjust runtime_us using priority weight if required */
if (priority != 0 && time_us != 0) {
}
#if KBASE_TRACE_ENABLE
-STATIC int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
+static int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
int as_nr;
as_nr = kctx->as_nr;
if (as_nr != KBASEP_AS_NR_INVALID) {
struct kbasep_js_per_as_data *js_per_as_data;
+
js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
refcnt = js_per_as_data->as_busy_refcount;
return refcnt;
}
-STATIC INLINE int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
+static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
return refcnt;
}
#else /* KBASE_TRACE_ENABLE */
-STATIC int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
-{
- CSTD_UNUSED(kbdev);
- CSTD_UNUSED(kctx);
- return 0;
-}
-
-STATIC INLINE int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
+static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
{
CSTD_UNUSED(kbdev);
CSTD_UNUSED(kctx);
}
#endif /* KBASE_TRACE_ENABLE */
-#ifdef CONFIG_MALI_DEBUG
-STATIC void kbasep_js_debug_check(struct kbasep_js_policy_cfs *policy_info, struct kbase_context *kctx, kbasep_js_check check_flag)
-{
- /* This function uses the ternary operator and non-explicit comparisons,
- * because it makes for much shorter, easier to read code */
-
- if (check_flag & KBASEP_JS_CHECKFLAG_QUEUED) {
- mali_bool is_queued;
- mali_bool expect_queued;
-
- is_queued = (kbasep_list_member_of(
- &policy_info->ctx_queue_head,
- &kctx->jctx.sched_info.runpool.policy_ctx.cfs.list)) ?
- MALI_TRUE : MALI_FALSE;
-
- if (!is_queued)
- is_queued = (kbasep_list_member_of(&policy_info->ctx_rt_queue_head, &kctx->jctx.sched_info.runpool.policy_ctx.cfs.list)) ? MALI_TRUE : MALI_FALSE;
-
- expect_queued = (check_flag & KBASEP_JS_CHECKFLAG_IS_QUEUED) ? MALI_TRUE : MALI_FALSE;
-
- KBASE_DEBUG_ASSERT_MSG(expect_queued == is_queued, "Expected context %p to be %s but it was %s\n", kctx, (expect_queued) ? "queued" : "not queued", (is_queued) ? "queued" : "not queued");
-
- }
-
- if (check_flag & KBASEP_JS_CHECKFLAG_SCHEDULED) {
- mali_bool is_scheduled;
- mali_bool expect_scheduled;
-
- is_scheduled = (kbasep_list_member_of(
- &policy_info->scheduled_ctxs_head,
- &kctx->jctx.sched_info.runpool.policy_ctx.cfs.list)) ?
- MALI_TRUE : MALI_FALSE;
-
- expect_scheduled = (check_flag & KBASEP_JS_CHECKFLAG_IS_SCHEDULED) ? MALI_TRUE : MALI_FALSE;
- KBASE_DEBUG_ASSERT_MSG(expect_scheduled == is_scheduled, "Expected context %p to be %s but it was %s\n", kctx, (expect_scheduled) ? "scheduled" : "not scheduled", (is_scheduled) ? "scheduled" : "not scheduled");
- }
-
-}
-#else /* CONFIG_MALI_DEBUG */
-STATIC void kbasep_js_debug_check(struct kbasep_js_policy_cfs *policy_info, struct kbase_context *kctx, kbasep_js_check check_flag)
-{
- CSTD_UNUSED(policy_info);
- CSTD_UNUSED(kctx);
- CSTD_UNUSED(check_flag);
-}
-#endif /* CONFIG_MALI_DEBUG */
-
-STATIC INLINE void set_slot_to_variant_lookup(u32 *bit_array, u32 slot_idx, u32 variants_supported)
-{
- u32 overall_bit_idx = slot_idx * KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS;
- u32 word_idx = overall_bit_idx / 32;
- u32 bit_idx = overall_bit_idx % 32;
-
- KBASE_DEBUG_ASSERT(slot_idx < BASE_JM_MAX_NR_SLOTS);
- KBASE_DEBUG_ASSERT((variants_supported & ~LOOKUP_VARIANT_MASK) == 0);
-
- bit_array[word_idx] |= variants_supported << bit_idx;
-}
-
-STATIC INLINE u32 get_slot_to_variant_lookup(u32 *bit_array, u32 slot_idx)
-{
- u32 overall_bit_idx = slot_idx * KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS;
- u32 word_idx = overall_bit_idx / 32;
- u32 bit_idx = overall_bit_idx % 32;
-
- u32 res;
-
- KBASE_DEBUG_ASSERT(slot_idx < BASE_JM_MAX_NR_SLOTS);
-
- res = bit_array[word_idx] >> bit_idx;
- res &= LOOKUP_VARIANT_MASK;
-
- return res;
-}
-
-/* Check the core_req_variants: make sure that every job slot is satisifed by
- * one of the variants. This checks that cached_variant_idx_init will produce a
- * valid result for jobs that make maximum use of the job slots.
- *
- * @note The checks are limited to the job slots - this does not check that
- * every context requirement is covered (because some are intentionally not
- * supported, such as KBASE_CTX_FLAG_SUBMIT_DISABLED) */
-#ifdef CONFIG_MALI_DEBUG
-STATIC void debug_check_core_req_variants(struct kbase_device *kbdev, struct kbasep_js_policy_cfs *policy_info)
-{
- struct kbasep_js_device_data *js_devdata;
- u32 i;
- int j;
-
- js_devdata = &kbdev->js_data;
-
- for (j = 0; j < kbdev->gpu_props.num_job_slots; ++j) {
- base_jd_core_req job_core_req;
- mali_bool found = MALI_FALSE;
-
- job_core_req = js_devdata->js_reqs[j];
- for (i = 0; i < policy_info->num_core_req_variants; ++i) {
- base_jd_core_req var_core_req;
-
- var_core_req = policy_info->core_req_variants[i].core_req;
-
- if ((var_core_req & job_core_req) == job_core_req) {
- found = MALI_TRUE;
- break;
- }
- }
-
- /* Early-out on any failure */
- KBASE_DEBUG_ASSERT_MSG(found != MALI_FALSE, "Job slot %d features 0x%x not matched by core_req_variants. " "Rework core_req_variants and vairants_supported_<...>_state[] to match\n", j, job_core_req);
- }
-}
-#endif
-
-STATIC void build_core_req_variants(struct kbase_device *kbdev, struct kbasep_js_policy_cfs *policy_info)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(policy_info != NULL);
- CSTD_UNUSED(kbdev);
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
- KBASE_DEBUG_ASSERT(NUM_CORE_REQ_VARIANTS_8987 <= KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS);
-
- /* Assume a static set of variants */
- memcpy(policy_info->core_req_variants, core_req_variants_8987, sizeof(core_req_variants_8987));
-
- policy_info->num_core_req_variants = NUM_CORE_REQ_VARIANTS_8987;
- } else {
- KBASE_DEBUG_ASSERT(NUM_CORE_REQ_VARIANTS <= KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS);
-
- /* Assume a static set of variants */
- memcpy(policy_info->core_req_variants, core_req_variants, sizeof(core_req_variants));
-
- policy_info->num_core_req_variants = NUM_CORE_REQ_VARIANTS;
- }
-
- KBASE_DEBUG_CODE(debug_check_core_req_variants(kbdev, policy_info));
-}
-
-STATIC void build_slot_lookups(struct kbase_device *kbdev, struct kbasep_js_policy_cfs *policy_info)
-{
- u8 i;
- const u32 *variants_supported_ss_for_this_hw = variants_supported_ss_state;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(policy_info != NULL);
-
- KBASE_DEBUG_ASSERT(kbdev->gpu_props.num_job_slots <= NELEMS(variants_supported_ss_state));
- KBASE_DEBUG_ASSERT(kbdev->gpu_props.num_job_slots <= NELEMS(variants_supported_ss_allcore_state));
- KBASE_DEBUG_ASSERT(kbdev->gpu_props.num_job_slots <= NELEMS(variants_supported_ss_state_8987));
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
- variants_supported_ss_for_this_hw = variants_supported_ss_state_8987;
-
- /* Given the static set of variants, provide a static set of lookups */
- for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) {
- set_slot_to_variant_lookup(policy_info->slot_to_variant_lookup_ss_state, i, variants_supported_ss_for_this_hw[i]);
-
- set_slot_to_variant_lookup(policy_info->slot_to_variant_lookup_ss_allcore_state, i, variants_supported_ss_allcore_state[i]);
- }
-
-}
-
-STATIC mali_error cached_variant_idx_init(const struct kbasep_js_policy_cfs *policy_info, const struct kbase_context *kctx, struct kbase_jd_atom *atom)
-{
- struct kbasep_js_policy_cfs_job *job_info;
- u32 i;
- base_jd_core_req job_core_req;
- u32 job_device_nr;
- kbase_context_flags ctx_flags;
- const struct kbasep_js_kctx_info *js_kctx_info;
- const struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(policy_info != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
- KBASE_DEBUG_ASSERT(atom != NULL);
-
- kbdev = container_of(policy_info, const struct kbase_device, js_data.policy.cfs);
- job_info = &atom->sched_info.cfs;
- job_core_req = atom->core_req;
- job_device_nr = atom->device_nr;
- js_kctx_info = &kctx->jctx.sched_info;
- ctx_flags = js_kctx_info->ctx.flags;
-
- /* Initial check for atoms targetting a specific coregroup */
- if ((job_core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) != MALI_FALSE && job_device_nr >= kbdev->gpu_props.num_core_groups) {
- /* device_nr exceeds the number of coregroups - not allowed by
- * @ref struct base_jd_atom_v2 API contract */
- return MALI_ERROR_FUNCTION_FAILED;
- }
-
- /* Pick a core_req variant that matches us. Since they're ordered by least
- * restrictive first, it picks the least restrictive variant */
- for (i = 0; i < policy_info->num_core_req_variants; ++i) {
- base_jd_core_req var_core_req;
- kbase_context_flags var_ctx_req;
- u32 var_device_nr;
- var_core_req = policy_info->core_req_variants[i].core_req;
- var_ctx_req = policy_info->core_req_variants[i].ctx_req;
- var_device_nr = policy_info->core_req_variants[i].device_nr;
-
- if ((var_core_req & job_core_req) == job_core_req && (var_ctx_req & ctx_flags) == ctx_flags && ((var_core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) == MALI_FALSE || var_device_nr == job_device_nr)) {
- job_info->cached_variant_idx = i;
- return MALI_ERROR_NONE;
- }
- }
-
- /* Could not find a matching requirement, this should only be caused by an
- * attempt to attack the driver. */
- return MALI_ERROR_FUNCTION_FAILED;
-}
-
-STATIC mali_bool dequeue_job(struct kbase_device *kbdev,
- struct kbase_context *kctx,
- u32 variants_supported,
- struct kbase_jd_atom ** const katom_ptr,
- int job_slot_idx)
-{
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_policy_cfs_ctx *ctx_info;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(katom_ptr != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- js_devdata = &kbdev->js_data;
- policy_info = &js_devdata->policy.cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- /* Only submit jobs from contexts that are allowed */
- if (kbasep_js_is_submit_allowed(js_devdata, kctx) != MALI_FALSE) {
- /* Check each variant in turn */
- while (variants_supported != 0) {
- long variant_idx;
- struct list_head *job_list;
-
- variant_idx = ffs(variants_supported) - 1;
- job_list = &ctx_info->job_list_head[variant_idx];
-
- if (!list_empty(job_list)) {
- /* Found a context with a matching job */
- {
- struct kbase_jd_atom *front_atom =
- list_entry(job_list->next, struct kbase_jd_atom, sched_info.cfs.list);
-
- KBASE_TRACE_ADD_SLOT(kbdev, JS_POLICY_DEQUEUE_JOB, front_atom->kctx, front_atom, front_atom->jc, job_slot_idx);
- }
- *katom_ptr = list_entry(job_list->next, struct kbase_jd_atom, sched_info.cfs.list);
- list_del(job_list->next);
-
- (*katom_ptr)->sched_info.cfs.ticks = 0;
-
- /* Put this context at the back of the Run Pool */
- list_del(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
- list_add_tail(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list, &policy_info->scheduled_ctxs_head);
-
- return MALI_TRUE;
- }
-
- variants_supported &= ~(1u << variant_idx);
- }
- /* All variants checked by here */
- }
-
- /* The context does not have a matching job */
-
- return MALI_FALSE;
-}
-
-/**
- * Hold the runpool_irq spinlock for this
- */
-STATIC INLINE mali_bool timer_callback_should_run(struct kbase_device *kbdev)
-{
- struct kbasep_js_device_data *js_devdata;
- s8 nr_running_ctxs;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
-
- /* nr_user_contexts_running is updated with the runpool_mutex. However, the
- * locking in the caller gives us a barrier that ensures nr_user_contexts is
- * up-to-date for reading */
- nr_running_ctxs = js_devdata->nr_user_contexts_running;
-
-#ifdef CONFIG_MALI_DEBUG
- if (js_devdata->softstop_always && nr_running_ctxs > 0) {
- /* Debug support for allowing soft-stop on a single context */
- return MALI_TRUE;
- }
-#endif /* CONFIG_MALI_DEBUG */
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) {
- /* Timeouts would have to be 4x longer (due to micro-architectural design)
- * to support OpenCL conformance tests, so only run the timer when there's:
- * - 2 or more CL contexts
- * - 1 or more GLES contexts
- *
- * NOTE: We will treat a context that has both Compute and Non-Compute jobs
- * will be treated as an OpenCL context (hence, we don't check
- * KBASEP_JS_CTX_ATTR_NON_COMPUTE).
- */
- {
- s8 nr_compute_ctxs = kbasep_js_ctx_attr_count_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_COMPUTE);
- s8 nr_noncompute_ctxs = nr_running_ctxs - nr_compute_ctxs;
-
- return (mali_bool) (nr_compute_ctxs >= 2 || nr_noncompute_ctxs > 0);
- }
- } else {
- /* Run the timer callback whenever you have at least 1 context */
- return (mali_bool) (nr_running_ctxs > 0);
- }
-}
-
-static enum hrtimer_restart timer_callback(struct hrtimer *timer)
-{
- unsigned long flags;
- struct kbase_device *kbdev;
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_policy_cfs *policy_info;
- int s;
- mali_bool reset_needed = MALI_FALSE;
-
- KBASE_DEBUG_ASSERT(timer != NULL);
-
- policy_info = container_of(timer, struct kbasep_js_policy_cfs, scheduling_timer);
- kbdev = container_of(policy_info, struct kbase_device, js_data.policy.cfs);
- js_devdata = &kbdev->js_data;
-
- /* Loop through the slots */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) {
- struct kbase_jm_slot *slot = &kbdev->jm_slots[s];
- struct kbase_jd_atom *atom = NULL;
-
- if (kbasep_jm_nr_jobs_submitted(slot) > 0) {
- atom = kbasep_jm_peek_idx_submit_slot(slot, 0);
- KBASE_DEBUG_ASSERT(atom != NULL);
-
- if (kbasep_jm_is_dummy_workaround_job(kbdev, atom) != MALI_FALSE) {
- /* Prevent further use of the atom - never cause a soft-stop, hard-stop, or a GPU reset due to it. */
- atom = NULL;
- }
- }
-
- if (atom != NULL) {
- /* The current version of the model doesn't support Soft-Stop */
- if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) {
- u32 ticks = atom->sched_info.cfs.ticks++;
-
-#if !CINSTR_DUMPING_ENABLED
- u32 soft_stop_ticks, hard_stop_ticks, gpu_reset_ticks;
- if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
- soft_stop_ticks = js_devdata->soft_stop_ticks_cl;
- hard_stop_ticks = js_devdata->hard_stop_ticks_cl;
- gpu_reset_ticks = js_devdata->gpu_reset_ticks_cl;
- } else {
- soft_stop_ticks = js_devdata->soft_stop_ticks;
- hard_stop_ticks = js_devdata->hard_stop_ticks_ss;
- gpu_reset_ticks = js_devdata->gpu_reset_ticks_ss;
- }
-
- /* Job is Soft-Stoppable */
- if (ticks == soft_stop_ticks) {
- int disjoint_threshold =
- KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD;
- u32 softstop_flags = 0u;
- /* Job has been scheduled for at least js_devdata->soft_stop_ticks ticks.
- * Soft stop the slot so we can run other jobs.
- */
- dev_dbg(kbdev->dev, "Soft-stop");
-
-#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS
- /* nr_user_contexts_running is updated with the runpool_mutex,
- * but we can't take that here.
- *
- * However, if it's about to be increased then the new context
- * can't run any jobs until they take the runpool_irq lock, so
- * it's OK to observe the older value.
- *
- * Similarly, if it's about to be decreased, the last job from
- * another context has already finished, so it's not too bad
- * that we observe the older value and register a disjoint
- * event when we try soft-stopping */
- if (js_devdata->nr_user_contexts_running >= disjoint_threshold)
- softstop_flags |= JS_COMMAND_SW_CAUSES_DISJOINT;
- kbase_job_slot_softstop_swflags(kbdev,
- s, atom, softstop_flags);
-#endif
- } else if (ticks == hard_stop_ticks) {
- /* Job has been scheduled for at least js_devdata->hard_stop_ticks_ss ticks.
- * It should have been soft-stopped by now. Hard stop the slot.
- */
-#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
- dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", (unsigned long)ticks, (unsigned long)(js_devdata->scheduling_tick_ns / 1000000u));
- kbase_job_slot_hardstop(atom->kctx, s, atom);
-#endif
- } else if (ticks == gpu_reset_ticks) {
- /* Job has been scheduled for at least js_devdata->gpu_reset_ticks_ss ticks.
- * It should have left the GPU by now. Signal that the GPU needs to be reset.
- */
- reset_needed = MALI_TRUE;
- }
-#else /* !CINSTR_DUMPING_ENABLED */
- /* NOTE: During CINSTR_DUMPING_ENABLED, we use the alternate timeouts, which
- * makes the hard-stop and GPU reset timeout much longer. We also ensure that
- * we don't soft-stop at all. */
- if (ticks == js_devdata->soft_stop_ticks) {
- /* Job has been scheduled for at least js_devdata->soft_stop_ticks.
- * We do not soft-stop during CINSTR_DUMPING_ENABLED, however.
- */
- dev_dbg(kbdev->dev, "Soft-stop");
- } else if (ticks == js_devdata->hard_stop_ticks_nss) {
- /* Job has been scheduled for at least js_devdata->hard_stop_ticks_nss ticks.
- * Hard stop the slot.
- */
-#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
- dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", (unsigned long)ticks, (unsigned long)(js_devdata->scheduling_tick_ns / 1000000u));
- kbase_job_slot_hardstop(atom->kctx, s, atom);
-#endif
- } else if (ticks == js_devdata->gpu_reset_ticks_nss) {
- /* Job has been scheduled for at least js_devdata->gpu_reset_ticks_nss ticks.
- * It should have left the GPU by now. Signal that the GPU needs to be reset.
- */
- reset_needed = MALI_TRUE;
- }
-#endif /* !CINSTR_DUMPING_ENABLED */
- }
- }
- }
-#if KBASE_GPU_RESET_EN
- if (reset_needed) {
- dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS/NSS timeout hit). Issueing GPU soft-reset to resolve.");
-
- if (kbase_prepare_to_reset_gpu_locked(kbdev))
- kbase_reset_gpu_locked(kbdev);
- }
-#endif /* KBASE_GPU_RESET_EN */
- /* the timer is re-issued if there is contexts in the run-pool */
-
- if (timer_callback_should_run(kbdev) != MALI_FALSE) {
- hrtimer_start(&policy_info->scheduling_timer, HR_TIMER_DELAY_NSEC(js_devdata->scheduling_tick_ns), HRTIMER_MODE_REL);
- } else {
- KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_END, NULL, NULL, 0u, 0u);
- /* timer_running state is updated by kbasep_js_policy_runpool_timers_sync() */
- }
-
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
-
- return HRTIMER_NORESTART;
-}
/*
* Non-private functions
*/
-mali_error kbasep_js_policy_init(struct kbase_device *kbdev)
+int kbasep_js_policy_init(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_policy_cfs *policy_info;
js_devdata = &kbdev->js_data;
policy_info = &js_devdata->policy.cfs;
- INIT_LIST_HEAD(&policy_info->ctx_queue_head);
- INIT_LIST_HEAD(&policy_info->scheduled_ctxs_head);
- INIT_LIST_HEAD(&policy_info->ctx_rt_queue_head);
-
atomic64_set(&policy_info->least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
atomic64_set(&policy_info->rt_least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
- hrtimer_init(&policy_info->scheduling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- policy_info->scheduling_timer.function = timer_callback;
-
- policy_info->timer_running = MALI_FALSE;
policy_info->head_runtime_us = 0;
- /* Build up the core_req variants */
- build_core_req_variants(kbdev, policy_info);
- /* Build the slot to variant lookups */
- build_slot_lookups(kbdev, policy_info);
-
- return MALI_ERROR_NONE;
+ return 0;
}
void kbasep_js_policy_term(union kbasep_js_policy *js_policy)
{
- struct kbasep_js_policy_cfs *policy_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- policy_info = &js_policy->cfs;
-
- /* ASSERT that there are no contexts queued */
- KBASE_DEBUG_ASSERT(list_empty(&policy_info->ctx_queue_head));
- KBASE_DEBUG_ASSERT(KBASEP_JS_RUNTIME_EMPTY == atomic64_read(&policy_info->least_runtime_us));
-
- /* ASSERT that there are no contexts scheduled */
- KBASE_DEBUG_ASSERT(list_empty(&policy_info->scheduled_ctxs_head));
-
- /* ASSERT that there are no contexts queued */
- KBASE_DEBUG_ASSERT(list_empty(&policy_info->ctx_rt_queue_head));
- KBASE_DEBUG_ASSERT(KBASEP_JS_RUNTIME_EMPTY == atomic64_read(&policy_info->rt_least_runtime_us));
-
- hrtimer_cancel(&policy_info->scheduling_timer);
+ CSTD_UNUSED(js_policy);
}
-mali_error kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
+int kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_policy_cfs_ctx *ctx_info;
struct kbasep_js_policy_cfs *policy_info;
- u32 i;
int policy;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_INIT_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
- for (i = 0; i < policy_info->num_core_req_variants; ++i)
- INIT_LIST_HEAD(&ctx_info->job_list_head[i]);
-
policy = current->policy;
if (policy == SCHED_FIFO || policy == SCHED_RR) {
- ctx_info->process_rt_policy = MALI_TRUE;
+ ctx_info->process_rt_policy = true;
ctx_info->process_priority = (((MAX_RT_PRIO - 1) - current->rt_priority) / 5) - 20;
} else {
- ctx_info->process_rt_policy = MALI_FALSE;
+ ctx_info->process_rt_policy = false;
ctx_info->process_priority = (current->static_prio - MAX_RT_PRIO) - 20;
}
- ctx_info->bag_total_priority = 0;
- ctx_info->bag_total_nr_atoms = 0;
-
/* Initial runtime (relative to least-run context runtime)
*
* This uses the Policy Queue's most up-to-date head_runtime_us by using the
mutex_unlock(&js_devdata->queue_mutex);
- return MALI_ERROR_NONE;
+ return 0;
}
void kbasep_js_policy_term_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
{
struct kbasep_js_policy_cfs_ctx *ctx_info;
struct kbasep_js_policy_cfs *policy_info;
- u32 i;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- {
- struct kbase_device *kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_TERM_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
- }
-
- /* ASSERT that no jobs are present */
- for (i = 0; i < policy_info->num_core_req_variants; ++i)
- KBASE_DEBUG_ASSERT(list_empty(&ctx_info->job_list_head[i]));
-
- /* No work to do */
-}
-
-/*
- * Context Management
- */
-
-void kbasep_js_policy_enqueue_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbase_context *head_ctx;
- struct kbase_context *list_kctx = NULL;
- struct kbasep_js_device_data *js_devdata;
- struct list_head *queue_head;
- struct list_head *pos;
- struct kbase_device *kbdev;
- atomic64_t *least_runtime_us;
- u64 head_runtime;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
- kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- js_devdata = &kbdev->js_data;
-
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_ENQUEUE_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
-
- /* ASSERT about scheduled-ness/queued-ness */
- kbasep_js_debug_check(policy_info, kctx, KBASEP_JS_CHECK_NOTQUEUED);
-
- /* Clamp the runtime to prevent DoS attacks through "stored-up" runtime */
- if (policy_info->head_runtime_us > ctx_info->runtime_us + (u64) js_devdata->cfs_ctx_runtime_min_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u)) {
- /* No need to hold the the runpool_irq.lock here, because we're essentially
- * initializing the value, and the context is definitely not being updated in the
- * runpool at this point. The queue_mutex held by the caller ensures the memory
- * barrier. */
- ctx_info->runtime_us = policy_info->head_runtime_us - (u64) js_devdata->cfs_ctx_runtime_min_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u);
- }
-
- /* Find the position where the context should be enqueued */
- if (ctx_info->process_rt_policy) {
- queue_head = &policy_info->ctx_rt_queue_head;
- least_runtime_us = &policy_info->rt_least_runtime_us;
- } else {
- queue_head = &policy_info->ctx_queue_head;
- least_runtime_us = &policy_info->least_runtime_us;
- }
-
- if (list_empty(queue_head)) {
- list_add_tail(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list, queue_head);
- } else {
- list_for_each(pos, queue_head) {
- struct kbasep_js_policy_cfs_ctx *list_ctx_info;
-
- list_kctx = list_entry(pos, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- list_ctx_info = &list_kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- if ((kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_PRIVILEGED) != 0)
- break;
-
- if ((list_ctx_info->runtime_us > ctx_info->runtime_us) && ((list_kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_PRIVILEGED) == 0))
- break;
- }
-
- /* Add the context to the queue */
- list_add_tail(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list, &list_kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
- }
-
- /* Ensure least_runtime_us is up to date*/
- head_ctx = list_entry(queue_head->next, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- head_runtime = head_ctx->jctx.sched_info.runpool.policy_ctx.cfs.runtime_us;
- atomic64_set(least_runtime_us, head_runtime);
-}
-
-mali_bool kbasep_js_policy_dequeue_head_ctx(union kbasep_js_policy *js_policy, struct kbase_context ** const kctx_ptr)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbase_context *head_ctx;
- struct list_head *queue_head;
- atomic64_t *least_runtime_us;
- struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx_ptr != NULL);
-
- policy_info = &js_policy->cfs;
- kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
-
- /* attempt to dequeue from the 'realttime' queue first */
- if (list_empty(&policy_info->ctx_rt_queue_head)) {
- if (list_empty(&policy_info->ctx_queue_head)) {
- /* Nothing to dequeue */
- return MALI_FALSE;
- } else {
- queue_head = &policy_info->ctx_queue_head;
- least_runtime_us = &policy_info->least_runtime_us;
- }
- } else {
- queue_head = &policy_info->ctx_rt_queue_head;
- least_runtime_us = &policy_info->rt_least_runtime_us;
- }
-
- /* Contexts are dequeued from the front of the queue */
- *kctx_ptr = list_entry(queue_head->next, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- /* If dequeuing will empty the list, then set least_runtime_us prior to deletion */
- if (queue_head->next->next == queue_head)
- atomic64_set(least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
- list_del(queue_head->next);
-
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_DEQUEUE_HEAD_CTX, *kctx_ptr, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, *kctx_ptr));
-
- /* Update the head runtime */
- if (!list_empty(queue_head)) {
- u64 head_runtime;
-
- head_ctx = list_entry(queue_head->next, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
-
- /* No need to hold the the runpool_irq.lock here for reading - the
- * context is definitely not being updated in the runpool at this
- * point. The queue_mutex held by the caller ensures the memory barrier. */
- head_runtime = head_ctx->jctx.sched_info.runpool.policy_ctx.cfs.runtime_us;
-
- if (head_runtime > policy_info->head_runtime_us)
- policy_info->head_runtime_us = head_runtime;
-
- atomic64_set(least_runtime_us, head_runtime);
- }
-
- return MALI_TRUE;
-}
-
-mali_bool kbasep_js_policy_try_evict_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbasep_js_policy_cfs *policy_info;
- mali_bool is_present;
- struct list_head *queue_head;
- atomic64_t *least_runtime_us;
- struct list_head *qhead;
struct kbase_device *kbdev;
KBASE_DEBUG_ASSERT(js_policy != NULL);
policy_info = &js_policy->cfs;
ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
- kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
-
- if (ctx_info->process_rt_policy) {
- queue_head = &policy_info->ctx_rt_queue_head;
- least_runtime_us = &policy_info->rt_least_runtime_us;
- } else {
- queue_head = &policy_info->ctx_queue_head;
- least_runtime_us = &policy_info->least_runtime_us;
- }
-
- qhead = queue_head;
-
- is_present = kbasep_list_member_of(qhead, &kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
-
- KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, JS_POLICY_TRY_EVICT_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx), is_present);
-
- if (is_present != MALI_FALSE) {
- struct kbase_context *head_ctx;
-
- qhead = queue_head;
-
- /* If dequeuing will empty the list, then set least_runtime_us prior to deletion */
- if (queue_head->next->next == queue_head)
- atomic64_set(least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
-
- /* Remove the context */
- list_del(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
-
- qhead = queue_head;
- /* Update the head runtime */
- if (!list_empty(qhead)) {
- u64 head_runtime;
-
- head_ctx = list_entry(qhead->next, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
-
- /* No need to hold the the runpool_irq.lock here for reading - the
- * context is definitely not being updated in the runpool at this
- * point. The queue_mutex held by the caller ensures the memory barrier. */
- head_runtime = head_ctx->jctx.sched_info.runpool.policy_ctx.cfs.runtime_us;
-
- if (head_runtime > policy_info->head_runtime_us)
- policy_info->head_runtime_us = head_runtime;
-
- atomic64_set(least_runtime_us, head_runtime);
- }
- }
-
- return is_present;
-}
-
-void kbasep_js_policy_foreach_ctx_job(union kbasep_js_policy *js_policy, struct kbase_context *kctx,
- kbasep_js_policy_ctx_job_cb callback, mali_bool detach_jobs)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbase_device *kbdev;
- u32 i;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- policy_info = &js_policy->cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
-
- /* Invoke callback on jobs on each variant in turn */
- for (i = 0; i < policy_info->num_core_req_variants; ++i) {
- struct list_head *job_list;
- struct kbase_jd_atom *atom;
- struct kbase_jd_atom *tmp_iter;
- job_list = &ctx_info->job_list_head[i];
- /* Invoke callback on all kbase_jd_atoms in this list, optionally
- * removing them from the list */
- list_for_each_entry_safe(atom, tmp_iter, job_list, sched_info.cfs.list) {
- if (detach_jobs)
- list_del(&atom->sched_info.cfs.list);
- callback(kbdev, atom);
- }
- }
-
-}
-
-void kbasep_js_policy_runpool_add_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_device_data *js_devdata;
- struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
- js_devdata = container_of(js_policy, struct kbasep_js_device_data, policy);
-
- kbdev = kctx->kbdev;
-
- {
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_RUNPOOL_ADD_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx));
- }
-
- /* ASSERT about scheduled-ness/queued-ness */
- kbasep_js_debug_check(policy_info, kctx, KBASEP_JS_CHECK_NOTSCHEDULED);
-
- /* All enqueued contexts go to the back of the runpool */
- list_add_tail(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list, &policy_info->scheduled_ctxs_head);
-
- if (timer_callback_should_run(kbdev) != MALI_FALSE && policy_info->timer_running == MALI_FALSE) {
- hrtimer_start(&policy_info->scheduling_timer, HR_TIMER_DELAY_NSEC(js_devdata->scheduling_tick_ns), HRTIMER_MODE_REL);
-
- KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u);
- policy_info->timer_running = MALI_TRUE;
- }
-}
+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_TERM_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
-void kbasep_js_policy_runpool_remove_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs *policy_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
-
- {
- struct kbase_device *kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
-
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_RUNPOOL_REMOVE_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx));
- }
-
- /* ASSERT about scheduled-ness/queued-ness */
- kbasep_js_debug_check(policy_info, kctx, KBASEP_JS_CHECK_SCHEDULED);
-
- /* No searching or significant list maintenance required to remove this context */
- list_del(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
-}
-
-mali_bool kbasep_js_policy_should_remove_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_device_data *js_devdata;
- u64 least_runtime_us;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
- js_devdata = container_of(js_policy, struct kbasep_js_device_data, policy);
-
- if (ctx_info->process_rt_policy)
- least_runtime_us = atomic64_read(&policy_info->rt_least_runtime_us);
- else
- least_runtime_us = atomic64_read(&policy_info->least_runtime_us);
-
- if (KBASEP_JS_RUNTIME_EMPTY == least_runtime_us) {
- /* Queue is empty */
- return MALI_FALSE;
- }
-
- if ((least_runtime_us + priority_weight(ctx_info, (u64) (js_devdata->ctx_timeslice_ns / 1000u)))
- < ctx_info->runtime_us) {
- /* The context is scheduled out if it's not the least-run context anymore.
- * The "real" head runtime is used instead of the cached runtime so the current
- * context is not scheduled out when there is less contexts than address spaces.
- */
- return MALI_TRUE;
- }
-
- return MALI_FALSE;
-}
-
-void kbasep_js_policy_runpool_timers_sync(union kbasep_js_policy *js_policy)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbase_device *kbdev;
- struct kbasep_js_device_data *js_devdata;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
-
- policy_info = &js_policy->cfs;
- kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- js_devdata = &kbdev->js_data;
-
- if (!timer_callback_should_run(kbdev)) {
- unsigned long flags;
-
- /* If the timer is running now, synchronize with it by
- * locking/unlocking its spinlock, to ensure it's not using an old value
- * from timer_callback_should_run() */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
-
- /* From now on, return value of timer_callback_should_run() will also
- * cause the timer to not requeue itself. Its return value cannot
- * change, because it depends on variables updated with the
- * runpool_mutex held, which the caller of this must also hold */
- hrtimer_cancel(&policy_info->scheduling_timer);
-
- policy_info->timer_running = MALI_FALSE;
- }
+ /* No work to do */
}
/*
* Job Chain Management
*/
-mali_error kbasep_js_policy_init_job(const union kbasep_js_policy *js_policy, const struct kbase_context *kctx, struct kbase_jd_atom *katom)
-{
- const struct kbasep_js_policy_cfs *policy_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(katom != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
-
- /* Determine the job's index into the job list head, will return error if the
- * atom is malformed and so is reported. */
- return cached_variant_idx_init(policy_info, kctx, katom);
-}
-
-void kbasep_js_policy_register_job(union kbasep_js_policy *js_policy, struct kbase_context *kctx, struct kbase_jd_atom *katom)
-{
- struct kbasep_js_policy_cfs_ctx *ctx_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(katom != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- /* Adjust context priority to include the new job */
- ctx_info->bag_total_nr_atoms++;
- ctx_info->bag_total_priority += katom->nice_prio;
-
- /* Get average priority and convert to NICE range -20..19 */
- if (ctx_info->bag_total_nr_atoms)
- ctx_info->bag_priority = (ctx_info->bag_total_priority / ctx_info->bag_total_nr_atoms) - 20;
-}
-
-void kbasep_js_policy_deregister_job(union kbasep_js_policy *js_policy, struct kbase_context *kctx, struct kbase_jd_atom *katom)
-{
- struct kbasep_js_policy_cfs_ctx *ctx_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- CSTD_UNUSED(js_policy);
- KBASE_DEBUG_ASSERT(katom != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- /* Adjust context priority to no longer include removed job */
- KBASE_DEBUG_ASSERT(ctx_info->bag_total_nr_atoms > 0);
- ctx_info->bag_total_nr_atoms--;
- ctx_info->bag_total_priority -= katom->nice_prio;
- KBASE_DEBUG_ASSERT(ctx_info->bag_total_priority >= 0);
-
- /* Get average priority and convert to NICE range -20..19 */
- if (ctx_info->bag_total_nr_atoms)
- ctx_info->bag_priority = (ctx_info->bag_total_priority / ctx_info->bag_total_nr_atoms) - 20;
-}
-KBASE_EXPORT_TEST_API(kbasep_js_policy_deregister_job)
-
-mali_bool kbasep_js_policy_dequeue_job(struct kbase_device *kbdev,
- int job_slot_idx,
- struct kbase_jd_atom ** const katom_ptr)
-{
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_policy_cfs *policy_info;
- struct kbase_context *kctx;
- u32 variants_supported;
- struct list_head *pos;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(katom_ptr != NULL);
- KBASE_DEBUG_ASSERT(job_slot_idx < BASE_JM_MAX_NR_SLOTS);
-
- js_devdata = &kbdev->js_data;
- policy_info = &js_devdata->policy.cfs;
-
- /* Get the variants for this slot */
- if (kbdev->gpu_props.num_core_groups > 1 && kbasep_js_ctx_attr_is_attr_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) != MALI_FALSE) {
- /* SS-allcore state, and there's more than one coregroup */
- variants_supported = get_slot_to_variant_lookup(policy_info->slot_to_variant_lookup_ss_allcore_state, job_slot_idx);
- } else {
- /* SS-state */
- variants_supported = get_slot_to_variant_lookup(policy_info->slot_to_variant_lookup_ss_state, job_slot_idx);
- }
-
- /* First pass through the runpool we consider the realtime priority jobs */
- list_for_each(pos, &policy_info->scheduled_ctxs_head) {
- kctx = list_entry(pos, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- if (kctx->jctx.sched_info.runpool.policy_ctx.cfs.process_rt_policy) {
- if (dequeue_job(kbdev, kctx, variants_supported, katom_ptr, job_slot_idx)) {
- /* Realtime policy job matched */
- return MALI_TRUE;
- }
- }
- }
-
- /* Second pass through the runpool we consider the non-realtime priority jobs */
- list_for_each(pos, &policy_info->scheduled_ctxs_head) {
- kctx = list_entry(pos, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- if (kctx->jctx.sched_info.runpool.policy_ctx.cfs.process_rt_policy == MALI_FALSE) {
- if (dequeue_job(kbdev, kctx, variants_supported, katom_ptr, job_slot_idx)) {
- /* Non-realtime policy job matched */
- return MALI_TRUE;
- }
- }
- }
-
- /* By this point, no contexts had a matching job */
- return MALI_FALSE;
-}
-
-void kbasep_js_policy_enqueue_job(union kbasep_js_policy *js_policy, struct kbase_jd_atom *katom)
-{
- struct kbasep_js_policy_cfs_job *job_info;
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbase_context *parent_ctx;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(katom != NULL);
- parent_ctx = katom->kctx;
- KBASE_DEBUG_ASSERT(parent_ctx != NULL);
-
- job_info = &katom->sched_info.cfs;
- ctx_info = &parent_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- {
- struct kbase_device *kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- KBASE_TRACE_ADD(kbdev, JS_POLICY_ENQUEUE_JOB, katom->kctx, katom, katom->jc, 0);
- }
- list_add_tail(&katom->sched_info.cfs.list, &ctx_info->job_list_head[job_info->cached_variant_idx]);
-}
-
void kbasep_js_policy_log_job_result(union kbasep_js_policy *js_policy, struct kbase_jd_atom *katom, u64 time_spent_us)
{
struct kbasep_js_policy_cfs_ctx *ctx_info;
struct kbase_context *parent_ctx;
+
KBASE_DEBUG_ASSERT(js_policy != NULL);
KBASE_DEBUG_ASSERT(katom != NULL);
CSTD_UNUSED(js_policy);
katom->time_spent_us += time_spent_us;
}
-mali_bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx)
+bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx)
{
struct kbasep_js_policy_cfs_ctx *current_ctx_info;
struct kbasep_js_policy_cfs_ctx *new_ctx_info;
current_ctx_info = ¤t_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
new_ctx_info = &new_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
- if ((current_ctx_info->process_rt_policy == MALI_FALSE) && (new_ctx_info->process_rt_policy == MALI_TRUE))
- return MALI_TRUE;
+ if (!current_ctx_info->process_rt_policy && new_ctx_info->process_rt_policy)
+ return true;
- if ((current_ctx_info->process_rt_policy == new_ctx_info->process_rt_policy) && (current_ctx_info->bag_priority > new_ctx_info->bag_priority))
- return MALI_TRUE;
+ if (current_ctx_info->process_rt_policy ==
+ new_ctx_info->process_rt_policy)
+ return true;
- return MALI_FALSE;
+ return false;
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-/**
- * @file mali_kbase_js_policy_cfs.h
+/*
* Completely Fair Job Scheduler Policy structure definitions
*/
#ifndef _KBASE_JS_POLICY_CFS_H_
#define _KBASE_JS_POLICY_CFS_H_
-#define KBASE_JS_POLICY_AVAILABLE_CFS
-
-/** @addtogroup base_api
- * @{ */
-/** @addtogroup base_kbase_api
- * @{ */
-/** @addtogroup kbase_js_policy
- * @{ */
+#define KBASEP_JS_RUNTIME_EMPTY ((u64)-1)
/**
- * Internally, this policy keeps a few internal queues for different variants
- * of core requirements, which are used to decide how to schedule onto the
- * different job slots.
- *
- * Must be a power of 2 to keep the lookup math simple
+ * struct kbasep_js_policy_cfs - Data for the CFS policy
+ * @head_runtime_us: Number of microseconds the least-run context has been
+ * running for. The kbasep_js_device_data.queue_mutex must
+ * be held whilst updating this.
+ * Reads are possible without this mutex, but an older value
+ * might be read if no memory barries are issued beforehand.
+ * @least_runtime_us: Number of microseconds the least-run context in the
+ * context queue has been running for.
+ * -1 if context queue is empty.
+ * @rt_least_runtime_us: Number of microseconds the least-run context in the
+ * realtime (priority) context queue has been running for.
+ * -1 if realtime context queue is empty
*/
-#define KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS_LOG2 3
-#define KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS (1u << KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS_LOG2)
-
-/** Bits needed in the lookup to support all slots */
-#define KBASEP_JS_VARIANT_LOOKUP_BITS_NEEDED (BASE_JM_MAX_NR_SLOTS * KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS)
-/** Number of u32s needed in the lookup array to support all slots */
-#define KBASEP_JS_VARIANT_LOOKUP_WORDS_NEEDED ((KBASEP_JS_VARIANT_LOOKUP_BITS_NEEDED + 31) / 32)
-
-#define KBASEP_JS_RUNTIME_EMPTY ((u64)-1)
-
-typedef struct kbasep_js_policy_cfs {
- /** List of all contexts in the context queue. Hold
- * kbasep_js_device_data::queue_mutex whilst accessing. */
- struct list_head ctx_queue_head;
-
- /** List of all contexts in the realtime (priority) context queue */
- struct list_head ctx_rt_queue_head;
-
- /** List of scheduled contexts. Hold kbasep_jd_device_data::runpool_irq::lock
- * whilst accessing, which is a spinlock */
- struct list_head scheduled_ctxs_head;
-
- /** Number of valid elements in the core_req_variants member, and the
- * kbasep_js_policy_rr_ctx::job_list_head array */
- u32 num_core_req_variants;
-
- /** Variants of the core requirements */
- struct kbasep_atom_req core_req_variants[KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS];
-
- /* Lookups per job slot against which core_req_variants match it */
- u32 slot_to_variant_lookup_ss_state[KBASEP_JS_VARIANT_LOOKUP_WORDS_NEEDED];
- u32 slot_to_variant_lookup_ss_allcore_state[KBASEP_JS_VARIANT_LOOKUP_WORDS_NEEDED];
-
- /* The timer tick used for rescheduling jobs */
- struct hrtimer scheduling_timer;
-
- /* Is the timer running?
- *
- * The kbasep_js_device_data::runpool_mutex must be held whilst modifying this. */
- mali_bool timer_running;
-
- /* Number of us the least-run context has been running for
- *
- * The kbasep_js_device_data::queue_mutex must be held whilst updating this
- * Reads are possible without this mutex, but an older value might be read
- * if no memory barriers are issued beforehand. */
+struct kbasep_js_policy_cfs {
u64 head_runtime_us;
-
- /* Number of us the least-run context in the context queue has been running for.
- * -1 if context queue is empty. */
atomic64_t least_runtime_us;
-
- /* Number of us the least-run context in the realtime (priority) context queue
- * has been running for. -1 if realtime context queue is empty. */
atomic64_t rt_least_runtime_us;
-} kbasep_js_policy_cfs;
+};
/**
- * This policy contains a single linked list of all contexts.
+ * struct kbasep_js_policy_cfs_ctx - a single linked list of all contexts
+ * @runtime_us: microseconds this context has been running for
+ * @process_rt_policy: set if calling process policy scheme is a realtime
+ * scheduler and will use the priority queue. Non-mutable
+ * after ctx init
+ * @process_priority: calling process NICE priority, in the range -20..19
+ *
+ * &kbasep_js_device_data.runpool_irq.lock must be held when updating
+ * @runtime_us. Initializing will occur on context init and context enqueue
+ * (which can only occur in one thread at a time), but multi-thread access only
+ * occurs while the context is in the runpool.
+ *
+ * Reads are possible without the spinlock, but an older value might be read if
+ * no memory barries are issued beforehand.
*/
-typedef struct kbasep_js_policy_cfs_ctx {
- /** Link implementing the Policy's Queue, and Currently Scheduled list */
- struct list_head list;
-
- /** Job lists for use when in the Run Pool - only using
- * kbasep_js_policy_fcfs::num_unique_slots of them. We still need to track
- * the jobs when we're not in the runpool, so this member is accessed from
- * outside the policy queue (for the first job), inside the policy queue,
- * and inside the runpool.
- *
- * If the context is in the runpool, then this must only be accessed with
- * kbasep_js_device_data::runpool_irq::lock held
- *
- * Jobs are still added to this list even when the context is not in the
- * runpool. In that case, the kbasep_js_kctx_info::ctx::jsctx_mutex must be
- * held before accessing this. */
- struct list_head job_list_head[KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS];
-
- /** Number of us this context has been running for
- *
- * The kbasep_js_device_data::runpool_irq::lock (a spinlock) must be held
- * whilst updating this. Initializing will occur on context init and
- * context enqueue (which can only occur in one thread at a time), but
- * multi-thread access only occurs while the context is in the runpool.
- *
- * Reads are possible without this spinlock, but an older value might be read
- * if no memory barriers are issued beforehand */
+struct kbasep_js_policy_cfs_ctx {
u64 runtime_us;
-
- /* Calling process policy scheme is a realtime scheduler and will use the priority queue
- * Non-mutable after ctx init */
- mali_bool process_rt_policy;
- /* Calling process NICE priority */
+ bool process_rt_policy;
int process_priority;
- /* Average NICE priority of all atoms in bag:
- * Hold the kbasep_js_kctx_info::ctx::jsctx_mutex when accessing */
- int bag_priority;
- /* Total NICE priority of all atoms in bag
- * Hold the kbasep_js_kctx_info::ctx::jsctx_mutex when accessing */
- int bag_total_priority;
- /* Total number of atoms in the bag
- * Hold the kbasep_js_kctx_info::ctx::jsctx_mutex when accessing */
- int bag_total_nr_atoms;
-
-} kbasep_js_policy_cfs_ctx;
+};
/**
- * In this policy, each Job is part of at most one of the per_corereq lists
+ * struct kbasep_js_policy_cfs_job - per job information for CFS
+ * @ticks: number of ticks that this job has been executing for
+ *
+ * &kbasep_js_device_data.runpool_irq.lock must be held when accessing @ticks.
*/
-typedef struct kbasep_js_policy_cfs_job {
- struct list_head list; /**< Link implementing the Run Pool list/Jobs owned by the ctx */
- u32 cached_variant_idx; /**< Cached index of the list this should be entered into on re-queue */
-
- /** Number of ticks that this job has been executing for
- *
- * To access this, the kbasep_js_device_data::runpool_irq::lock must be held */
+struct kbasep_js_policy_cfs_job {
u32 ticks;
-} kbasep_js_policy_cfs_job;
-
- /** @} *//* end group kbase_js_policy */
- /** @} *//* end group base_kbase_api */
- /** @} *//* end group base_api */
+};
#endif
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <linux/module.h>
#include <linux/atomic.h>
-#if defined(MALI_KERNEL_TEST_API)
-#if (1 == MALI_KERNEL_TEST_API)
-#define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func);
+#if (defined(MALI_KERNEL_TEST_API) && (1 == MALI_KERNEL_TEST_API))
+ #define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func)
#else
-#define KBASE_EXPORT_TEST_API(func)
-#endif
-#else
-#define KBASE_EXPORT_TEST_API(func)
+ #define KBASE_EXPORT_TEST_API(func)
#endif
-#define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func);
+#define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func)
-#endif /* _KBASE_LINUX_H_ */
+#endif /* _KBASE_LINUX_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <linux/dma-buf.h>
#endif /* CONFIG_DMA_SHARED_BUFFER */
+#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/compat.h>
#include <mali_kbase_cache_policy.h>
#include <mali_kbase_hw.h>
#include <mali_kbase_gator.h>
+#include <mali_kbase_hwaccess_time.h>
+
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+#include <mali_kbase_tlstream.h>
+#endif
/**
* @brief Check the zone compatibility of two regions.
*/
-STATIC int kbase_region_tracker_match_zone(struct kbase_va_region *reg1, struct kbase_va_region *reg2)
+static int kbase_region_tracker_match_zone(struct kbase_va_region *reg1,
+ struct kbase_va_region *reg2)
{
- return ((reg1->flags & KBASE_REG_ZONE_MASK) == (reg2->flags & KBASE_REG_ZONE_MASK));
+ return ((reg1->flags & KBASE_REG_ZONE_MASK) ==
+ (reg2->flags & KBASE_REG_ZONE_MASK));
}
-KBASE_EXPORT_TEST_API(kbase_region_tracker_match_zone)
+KBASE_EXPORT_TEST_API(kbase_region_tracker_match_zone);
/* This function inserts a region into the tree. */
static void kbase_region_tracker_insert(struct kbase_context *kctx, struct kbase_va_region *new_reg)
}
/* Find region enclosing given address. */
-struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, mali_addr64 gpu_addr)
+struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr)
{
struct rb_node *rbnode;
struct kbase_va_region *reg;
return NULL;
}
-KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address)
+KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address);
/* Find region with given base address */
-struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, mali_addr64 gpu_addr)
+struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr)
{
u64 gpu_pfn = gpu_addr >> PAGE_SHIFT;
struct rb_node *rbnode;
rbnode = rbnode->rb_left;
else if (reg->start_pfn < gpu_pfn)
rbnode = rbnode->rb_right;
- else if (gpu_pfn == reg->start_pfn)
- return reg;
else
- rbnode = NULL;
+ return reg;
+
}
return NULL;
}
-KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address)
+KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address);
/* Find region meeting given requirements */
static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs(struct kbase_context *kctx, struct kbase_va_region *reg_reqs, size_t nr_pages, size_t align)
* region lock held. The associated memory is not released (see
* kbase_free_alloced_region). Internal use only.
*/
-STATIC mali_error kbase_remove_va_region(struct kbase_context *kctx, struct kbase_va_region *reg)
+static int kbase_remove_va_region(struct kbase_context *kctx, struct kbase_va_region *reg)
{
struct rb_node *rbprev;
struct kbase_va_region *prev = NULL;
int merged_front = 0;
int merged_back = 0;
- mali_error err = MALI_ERROR_NONE;
+ int err = 0;
/* Try to merge with the previous block first */
rbprev = rb_prev(&(reg->rblink));
free_reg = kbase_alloc_free_region(kctx, reg->start_pfn, reg->nr_pages, reg->flags & KBASE_REG_ZONE_MASK);
if (!free_reg) {
- err = MALI_ERROR_OUT_OF_MEMORY;
+ err = -ENOMEM;
goto out;
}
return err;
}
-KBASE_EXPORT_TEST_API(kbase_remove_va_region)
+KBASE_EXPORT_TEST_API(kbase_remove_va_region);
/**
* @brief Insert a VA region to the list, replacing the current at_reg.
*/
-static mali_error kbase_insert_va_region_nolock(struct kbase_context *kctx, struct kbase_va_region *new_reg, struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages)
+static int kbase_insert_va_region_nolock(struct kbase_context *kctx, struct kbase_va_region *new_reg, struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages)
{
- mali_error err = MALI_ERROR_NONE;
+ int err = 0;
/* Must be a free region */
KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0);
kbase_region_tracker_insert(kctx, new_front_reg);
kbase_region_tracker_insert(kctx, new_reg);
} else {
- err = MALI_ERROR_OUT_OF_MEMORY;
+ err = -ENOMEM;
}
}
/**
* @brief Add a VA region to the list.
*/
-mali_error kbase_add_va_region(struct kbase_context *kctx,
- struct kbase_va_region *reg, mali_addr64 addr,
+int kbase_add_va_region(struct kbase_context *kctx,
+ struct kbase_va_region *reg, u64 addr,
size_t nr_pages, size_t align)
{
struct kbase_va_region *tmp;
u64 gpu_pfn = addr >> PAGE_SHIFT;
- mali_error err = MALI_ERROR_NONE;
+ int err = 0;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(NULL != reg);
tmp = kbase_region_tracker_find_region_enclosing_range_free(kctx, gpu_pfn, nr_pages);
if (!tmp) {
dev_warn(dev, "Enclosing region not found: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages);
- err = MALI_ERROR_OUT_OF_GPU_MEMORY;
+ err = -ENOMEM;
goto exit;
}
dev_warn(dev, "Zone mismatch: %lu != %lu", tmp->flags & KBASE_REG_ZONE_MASK, reg->flags & KBASE_REG_ZONE_MASK);
dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages);
dev_warn(dev, "in function %s (%p, %p, 0x%llx, 0x%zx, 0x%zx)\n", __func__, kctx, reg, addr, nr_pages, align);
- err = MALI_ERROR_OUT_OF_GPU_MEMORY;
+ err = -ENOMEM;
goto exit;
}
err = kbase_insert_va_region_nolock(kctx, reg, tmp, gpu_pfn, nr_pages);
if (err) {
dev_warn(dev, "Failed to insert va region");
- err = MALI_ERROR_OUT_OF_GPU_MEMORY;
+ err = -ENOMEM;
goto exit;
}
tmp = kbase_region_tracker_find_region_meeting_reqs(kctx, reg, nr_pages, align);
if (!tmp) {
- err = MALI_ERROR_OUT_OF_GPU_MEMORY;
+ err = -ENOMEM;
goto exit;
}
start_pfn = (tmp->start_pfn + align - 1) & ~(align - 1);
return err;
}
-KBASE_EXPORT_TEST_API(kbase_add_va_region)
+KBASE_EXPORT_TEST_API(kbase_add_va_region);
/**
* @brief Initialize the internal region tracker data structure.
/**
* Initialize the region tracker data structure.
*/
-mali_error kbase_region_tracker_init(struct kbase_context *kctx)
+int kbase_region_tracker_init(struct kbase_context *kctx)
{
struct kbase_va_region *same_va_reg;
struct kbase_va_region *exec_reg = NULL;
#endif
if (kctx->kbdev->gpu_props.mmu.va_bits < same_va_bits)
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
/* all have SAME_VA */
same_va_reg = kbase_alloc_free_region(kctx, 1,
KBASE_REG_ZONE_SAME_VA);
if (!same_va_reg)
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
#ifdef CONFIG_64BIT
/* only 32-bit clients have the other two zones */
#endif
if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) {
kbase_free_alloced_region(same_va_reg);
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
/* If the current size of TMEM is out of range of the
* virtual address space addressable by the MMU then
if (!exec_reg) {
kbase_free_alloced_region(same_va_reg);
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
}
custom_va_reg = kbase_alloc_free_region(kctx,
if (!custom_va_reg) {
kbase_free_alloced_region(same_va_reg);
kbase_free_alloced_region(exec_reg);
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
}
#ifdef CONFIG_64BIT
}
kbase_region_tracker_ds_init(kctx, same_va_reg, exec_reg, custom_va_reg);
- return MALI_ERROR_NONE;
+ return 0;
}
-mali_error kbase_mem_init(struct kbase_device *kbdev)
+int kbase_mem_init(struct kbase_device *kbdev)
{
struct kbasep_mem_device *memdev;
atomic_set(&memdev->used_pages, 0);
/* nothing to do, zero-inited when struct kbase_device was created */
- return MALI_ERROR_NONE;
+ return 0;
}
void kbase_mem_halt(struct kbase_device *kbdev)
dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages);
}
-KBASE_EXPORT_TEST_API(kbase_mem_term)
+KBASE_EXPORT_TEST_API(kbase_mem_term);
-/**
- * @brief Wait for GPU write flush - only in use for BASE_HW_ISSUE_6367
- *
- * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush its write buffer.
- * @note If GPU resets occur then the counters are reset to zero, the delay may not be as expected.
- */
-#ifndef CONFIG_MALI_NO_MALI
-void kbase_wait_write_flush(struct kbase_context *kctx)
-{
- u32 base_count = 0;
-
- /* A suspend won't happen here, because we're in a syscall from a userspace thread */
- kbase_pm_context_active(kctx->kbdev);
- kbase_pm_request_gpu_cycle_counter(kctx->kbdev);
-
- while (MALI_TRUE) {
- u32 new_count;
-
- new_count = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
- /* First time around, just store the count. */
- if (base_count == 0) {
- base_count = new_count;
- continue;
- }
-
- /* No need to handle wrapping, unsigned maths works for this. */
- if ((new_count - base_count) > 1000)
- break;
- }
-
- kbase_pm_release_gpu_cycle_counter(kctx->kbdev);
- kbase_pm_context_idle(kctx->kbdev);
-}
-#endif /* CONFIG_MALI_NO_MALI */
/* zone argument should only contain zone related region flags */
KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0);
KBASE_DEBUG_ASSERT(nr_pages > 0);
- KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (UINT64_MAX / PAGE_SIZE)); /* 64-bit address range is the max */
+ /* 64-bit address range is the max */
+ KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE));
new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL);
- if (!new_reg) {
- dev_warn(kctx->kbdev->dev, "kzalloc failed");
+ if (!new_reg)
return NULL;
- }
- new_reg->alloc = NULL; /* no alloc bound yet */
+ new_reg->cpu_alloc = NULL; /* no alloc bound yet */
+ new_reg->gpu_alloc = NULL; /* no alloc bound yet */
new_reg->kctx = kctx;
new_reg->flags = zone | KBASE_REG_FREE;
return new_reg;
}
-KBASE_EXPORT_TEST_API(kbase_alloc_free_region)
+KBASE_EXPORT_TEST_API(kbase_alloc_free_region);
/**
* @brief Free a region object.
{
KBASE_DEBUG_ASSERT(NULL != reg);
if (!(reg->flags & KBASE_REG_FREE)) {
- kbase_mem_phy_alloc_put(reg->alloc);
+ kbase_mem_phy_alloc_put(reg->cpu_alloc);
+ kbase_mem_phy_alloc_put(reg->gpu_alloc);
/* To detect use-after-free in debug builds */
KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE);
}
kfree(reg);
}
-KBASE_EXPORT_TEST_API(kbase_free_alloced_region)
+KBASE_EXPORT_TEST_API(kbase_free_alloced_region);
void kbase_mmu_update(struct kbase_context *kctx)
{
- struct kbase_device *kbdev;
- struct kbase_as *as;
- struct kbase_mmu_setup *current_setup;
-
KBASE_DEBUG_ASSERT(NULL != kctx);
-
+ lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock);
/* ASSERT that the context has a valid as_nr, which is only the case
* when it's scheduled in.
*
* as_nr won't change because the caller has the runpool_irq lock */
KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
- kbdev = kctx->kbdev;
- as = &kbdev->as[kctx->as_nr];
- current_setup = &as->current_setup;
-
- /* Use GPU implementation-defined caching policy. */
- current_setup->memattr = kctx->mem_attrs;
-
- current_setup->transtab = (u64) kctx->pgd & ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_ADDR_SPACE_MASK);
- current_setup->transtab |= AS_TRANSTAB_READ_INNER |
- AS_TRANSTAB_ADRMODE_TABLE;
-
- /* Apply the address space setting */
- kbase_mmu_hw_configure(kbdev, as, kctx);
+ kctx->kbdev->mmu_mode->update(kctx);
}
-KBASE_EXPORT_TEST_API(kbase_mmu_update)
+KBASE_EXPORT_TEST_API(kbase_mmu_update);
void kbase_mmu_disable(struct kbase_context *kctx)
{
- struct kbase_device *kbdev;
- struct kbase_as *as;
- struct kbase_mmu_setup *current_setup;
-
KBASE_DEBUG_ASSERT(NULL != kctx);
/* ASSERT that the context has a valid as_nr, which is only the case
* when it's scheduled in.
* as_nr won't change because the caller has the runpool_irq lock */
KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
- kbdev = kctx->kbdev;
- as = &kbdev->as[kctx->as_nr];
- current_setup = &as->current_setup;
+ kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr);
+}
- current_setup->transtab = 0ULL;
+KBASE_EXPORT_TEST_API(kbase_mmu_disable);
- /* Apply the address space setting */
- kbase_mmu_hw_configure(kbdev, as, kctx);
+void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr)
+{
+ kbdev->mmu_mode->disable_as(kbdev, as_nr);
}
-KBASE_EXPORT_TEST_API(kbase_mmu_disable)
-
-mali_error kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, mali_addr64 addr, size_t nr_pages, size_t align)
+int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align)
{
- mali_error err;
+ int err;
size_t i = 0;
-
+ unsigned long mask = ~KBASE_REG_MEMATTR_MASK;
+#if defined(CONFIG_MALI_CACHE_COHERENT)
+ unsigned long attr =
+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA);
+#else
+ unsigned long attr =
+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC);
+#endif
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(NULL != reg);
err = kbase_add_va_region(kctx, reg, addr, nr_pages, align);
- if (MALI_ERROR_NONE != err)
+ if (err)
return err;
- if (reg->alloc->type == KBASE_MEM_TYPE_ALIAS) {
+ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) {
u64 stride;
+ struct kbase_mem_phy_alloc *alloc;
- stride = reg->alloc->imported.alias.stride;
- KBASE_DEBUG_ASSERT(reg->alloc->imported.alias.aliased);
- for (i = 0; i < reg->alloc->imported.alias.nents; i++) {
- if (reg->alloc->imported.alias.aliased[i].alloc) {
+ alloc = reg->gpu_alloc;
+ stride = alloc->imported.alias.stride;
+ KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased);
+ for (i = 0; i < alloc->imported.alias.nents; i++) {
+ if (alloc->imported.alias.aliased[i].alloc) {
err = kbase_mmu_insert_pages(kctx,
reg->start_pfn + (i * stride),
- reg->alloc->imported.alias.aliased[i].alloc->pages + reg->alloc->imported.alias.aliased[i].offset,
- reg->alloc->imported.alias.aliased[i].length,
+ alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset,
+ alloc->imported.alias.aliased[i].length,
reg->flags);
- if (MALI_ERROR_NONE != err)
+ if (err)
goto bad_insert;
- kbase_mem_phy_alloc_gpu_mapped(reg->alloc->imported.alias.aliased[i].alloc);
+ kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc);
} else {
err = kbase_mmu_insert_single_page(kctx,
- reg->start_pfn + i * stride,
- kctx->aliasing_sink_page,
- reg->alloc->imported.alias.aliased[i].length,
- (reg->flags & ~KBASE_REG_MEMATTR_MASK) | KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC)
- );
- if (MALI_ERROR_NONE != err)
+ reg->start_pfn + i * stride,
+ kctx->aliasing_sink_page,
+ alloc->imported.alias.aliased[i].length,
+ (reg->flags & mask) | attr);
+
+ if (err)
goto bad_insert;
}
}
} else {
err = kbase_mmu_insert_pages(kctx, reg->start_pfn,
- kbase_get_phy_pages(reg),
+ kbase_get_gpu_phy_pages(reg),
kbase_reg_current_backed_size(reg),
reg->flags);
- if (MALI_ERROR_NONE != err)
+ if (err)
goto bad_insert;
- kbase_mem_phy_alloc_gpu_mapped(reg->alloc);
+ kbase_mem_phy_alloc_gpu_mapped(reg->gpu_alloc);
}
return err;
bad_insert:
- if (reg->alloc->type == KBASE_MEM_TYPE_ALIAS) {
+ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) {
u64 stride;
- stride = reg->alloc->imported.alias.stride;
- KBASE_DEBUG_ASSERT(reg->alloc->imported.alias.aliased);
+ stride = reg->gpu_alloc->imported.alias.stride;
+ KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased);
while (i--)
- if (reg->alloc->imported.alias.aliased[i].alloc) {
- kbase_mmu_teardown_pages(kctx, reg->start_pfn + (i * stride), reg->alloc->imported.alias.aliased[i].length);
- kbase_mem_phy_alloc_gpu_unmapped(reg->alloc->imported.alias.aliased[i].alloc);
+ if (reg->gpu_alloc->imported.alias.aliased[i].alloc) {
+ kbase_mmu_teardown_pages(kctx, reg->start_pfn + (i * stride), reg->gpu_alloc->imported.alias.aliased[i].length);
+ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc);
}
}
return err;
}
-KBASE_EXPORT_TEST_API(kbase_gpu_mmap)
+KBASE_EXPORT_TEST_API(kbase_gpu_mmap);
-mali_error kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg)
+int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg)
{
- mali_error err;
+ int err;
if (reg->start_pfn == 0)
- return MALI_ERROR_NONE;
+ return 0;
- if (reg->alloc && reg->alloc->type == KBASE_MEM_TYPE_ALIAS) {
+ if (reg->gpu_alloc && reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) {
size_t i;
err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, reg->nr_pages);
- KBASE_DEBUG_ASSERT(reg->alloc->imported.alias.aliased);
- for (i = 0; i < reg->alloc->imported.alias.nents; i++)
- if (reg->alloc->imported.alias.aliased[i].alloc)
- kbase_mem_phy_alloc_gpu_unmapped(reg->alloc->imported.alias.aliased[i].alloc);
+ KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased);
+ for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++)
+ if (reg->gpu_alloc->imported.alias.aliased[i].alloc)
+ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc);
} else {
err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, kbase_reg_current_backed_size(reg));
- kbase_mem_phy_alloc_gpu_unmapped(reg->alloc);
+ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc);
}
- if (MALI_ERROR_NONE != err)
+ if (err)
return err;
err = kbase_remove_va_region(kctx, reg);
return err;
}
-STATIC struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping_of_region(const struct kbase_va_region *reg, unsigned long uaddr, size_t size)
+static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping_of_region(const struct kbase_va_region *reg, unsigned long uaddr, size_t size)
{
struct kbase_cpu_mapping *map;
struct list_head *pos;
KBASE_DEBUG_ASSERT(NULL != reg);
- KBASE_DEBUG_ASSERT(reg->alloc);
+ KBASE_DEBUG_ASSERT(reg->cpu_alloc);
if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */
return NULL;
- list_for_each(pos, ®->alloc->mappings) {
- map = list_entry(pos, kbase_cpu_mapping, mappings_list);
+ list_for_each(pos, ®->cpu_alloc->mappings) {
+ map = list_entry(pos, struct kbase_cpu_mapping, mappings_list);
if (map->vm_start <= uaddr && map->vm_end >= uaddr + size)
return map;
}
return NULL;
}
-KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_of_region)
+KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_of_region);
-mali_error kbasep_find_enclosing_cpu_mapping_offset(
- struct kbase_context *kctx, mali_addr64 gpu_addr,
- unsigned long uaddr, size_t size, mali_size64 *offset)
+int kbasep_find_enclosing_cpu_mapping_offset(
+ struct kbase_context *kctx, u64 gpu_addr,
+ unsigned long uaddr, size_t size, u64 * offset)
{
struct kbase_cpu_mapping *map = NULL;
const struct kbase_va_region *reg;
- mali_error err = MALI_ERROR_FUNCTION_FAILED;
+ int err = -EINVAL;
KBASE_DEBUG_ASSERT(kctx != NULL);
if (map) {
*offset = (uaddr - PTR_TO_U64(map->vm_start)) +
(map->page_off << PAGE_SHIFT);
- err = MALI_ERROR_NONE;
+ err = 0;
}
}
return err;
}
-KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset)
+KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset);
void kbase_sync_single(struct kbase_context *kctx,
- phys_addr_t pa, size_t size, kbase_sync_kmem_fn sync_fn)
+ phys_addr_t cpu_pa, phys_addr_t gpu_pa,
+ off_t offset, size_t size, enum kbase_sync_type sync_fn)
{
- struct page *p = pfn_to_page(PFN_DOWN(pa));
- off_t offset = pa & ~PAGE_MASK;
- dma_addr_t dma_addr;
+ struct page *cpu_page;
- BUG_ON(!p);
- BUG_ON(offset + size > PAGE_SIZE);
+ cpu_page = pfn_to_page(PFN_DOWN(cpu_pa));
- dma_addr = kbase_dma_addr(p) + offset;
+ if (likely(cpu_pa == gpu_pa)) {
+ dma_addr_t dma_addr;
- sync_fn(kctx->kbdev->dev, dma_addr, size, DMA_BIDIRECTIONAL);
+ BUG_ON(!cpu_page);
+ BUG_ON(offset + size > PAGE_SIZE);
+
+ dma_addr = kbase_dma_addr(cpu_page) + offset;
+ if (sync_fn == KBASE_SYNC_TO_CPU)
+ dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr,
+ size, DMA_BIDIRECTIONAL);
+ else if (sync_fn == KBASE_SYNC_TO_DEVICE)
+ dma_sync_single_for_device(kctx->kbdev->dev, dma_addr,
+ size, DMA_BIDIRECTIONAL);
+ } else {
+ void *src = NULL;
+ void *dst = NULL;
+ struct page *gpu_page;
+
+ if (WARN(!gpu_pa, "No GPU PA found for infinite cache op"))
+ return;
+
+ gpu_page = pfn_to_page(PFN_DOWN(gpu_pa));
+
+ if (sync_fn == KBASE_SYNC_TO_DEVICE) {
+ src = ((unsigned char *)kmap(cpu_page)) + offset;
+ dst = ((unsigned char *)kmap(gpu_page)) + offset;
+ } else if (sync_fn == KBASE_SYNC_TO_CPU) {
+ dma_sync_single_for_cpu(kctx->kbdev->dev,
+ kbase_dma_addr(gpu_page) + offset,
+ size, DMA_BIDIRECTIONAL);
+ src = ((unsigned char *)kmap(gpu_page)) + offset;
+ dst = ((unsigned char *)kmap(cpu_page)) + offset;
+ }
+ memcpy(dst, src, size);
+ kunmap(gpu_page);
+ kunmap(cpu_page);
+ if (sync_fn == KBASE_SYNC_TO_DEVICE)
+ dma_sync_single_for_device(kctx->kbdev->dev,
+ kbase_dma_addr(gpu_page) + offset,
+ size, DMA_BIDIRECTIONAL);
+ }
}
-static mali_error kbase_do_syncset(struct kbase_context *kctx, struct base_syncset *set, kbase_sync_kmem_fn sync_fn)
+static int kbase_do_syncset(struct kbase_context *kctx,
+ struct base_syncset *set, enum kbase_sync_type sync_fn)
{
- mali_error err = MALI_ERROR_NONE;
+ int err = 0;
struct basep_syncset *sset = &set->basep_sset;
struct kbase_va_region *reg;
struct kbase_cpu_mapping *map;
unsigned long start;
size_t size;
- phys_addr_t *pa;
+ phys_addr_t *cpu_pa;
+ phys_addr_t *gpu_pa;
u64 page_off, page_count;
u64 i;
- unsigned int offset;
+ off_t offset;
kbase_os_mem_map_lock(kctx);
kbase_gpu_vm_lock(kctx);
/* find the region where the virtual address is contained */
- reg = kbase_region_tracker_find_region_enclosing_address(kctx, sset->mem_handle);
+ reg = kbase_region_tracker_find_region_enclosing_address(kctx,
+ sset->mem_handle);
if (!reg) {
- dev_warn(kctx->kbdev->dev, "Can't find region at VA 0x%016llX", sset->mem_handle);
- err = MALI_ERROR_FUNCTION_FAILED;
+ dev_warn(kctx->kbdev->dev, "Can't find region at VA 0x%016llX",
+ sset->mem_handle);
+ err = -EINVAL;
goto out_unlock;
}
map = kbasep_find_enclosing_cpu_mapping_of_region(reg, start, size);
if (!map) {
- dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", start, sset->mem_handle);
- err = MALI_ERROR_FUNCTION_FAILED;
+ dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX",
+ start, sset->mem_handle);
+ err = -EINVAL;
goto out_unlock;
}
offset = start & (PAGE_SIZE - 1);
page_off = map->page_off + ((start - map->vm_start) >> PAGE_SHIFT);
page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
- pa = kbase_get_phy_pages(reg);
+ cpu_pa = kbase_get_cpu_phy_pages(reg);
+ gpu_pa = kbase_get_gpu_phy_pages(reg);
/* Sync first page */
- if (pa[page_off]) {
+ if (cpu_pa[page_off]) {
size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
- kbase_sync_single(kctx, pa[page_off] + offset, sz, sync_fn);
+ kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off],
+ offset, sz, sync_fn);
}
/* Sync middle pages (if any) */
for (i = 1; page_count > 2 && i < page_count - 1; i++) {
/* we grow upwards, so bail on first non-present page */
- if (!pa[page_off + i])
+ if (!cpu_pa[page_off + i])
break;
- kbase_sync_single(kctx, pa[page_off + i], PAGE_SIZE, sync_fn);
+ kbase_sync_single(kctx, cpu_pa[page_off + i],
+ gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn);
}
/* Sync last page (if any) */
- if (page_count > 1 && pa[page_off + page_count - 1]) {
+ if (page_count > 1 && cpu_pa[page_off + page_count - 1]) {
size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1;
- kbase_sync_single(kctx, pa[page_off + page_count - 1], sz,
- sync_fn);
+ kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1],
+ gpu_pa[page_off + page_count - 1], 0, sz,
+ sync_fn);
}
out_unlock:
return err;
}
-mali_error kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset)
+int kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset)
{
- mali_error err = MALI_ERROR_FUNCTION_FAILED;
+ int err = -EINVAL;
struct basep_syncset *sset;
KBASE_DEBUG_ASSERT(NULL != kctx);
switch (sset->type) {
case BASE_SYNCSET_OP_MSYNC:
- err = kbase_do_syncset(kctx, syncset, dma_sync_single_for_device);
+ err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_DEVICE);
break;
case BASE_SYNCSET_OP_CSYNC:
- err = kbase_do_syncset(kctx, syncset, dma_sync_single_for_cpu);
+ err = kbase_do_syncset(kctx, syncset, KBASE_SYNC_TO_CPU);
break;
default:
return err;
}
-KBASE_EXPORT_TEST_API(kbase_sync_now)
+KBASE_EXPORT_TEST_API(kbase_sync_now);
/* vm lock must be held */
-mali_error kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg)
+int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg)
{
- mali_error err;
+ int err;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(NULL != reg);
dev_warn(reg->kctx->kbdev->dev, "Could not unmap from the GPU...\n");
goto out;
}
-
+#ifndef CONFIG_MALI_NO_MALI
if (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) {
/* Wait for GPU to flush write buffer before freeing physical pages */
kbase_wait_write_flush(kctx);
}
-
+#endif
/* This will also free the physical pages */
kbase_free_alloced_region(reg);
return err;
}
-KBASE_EXPORT_TEST_API(kbase_mem_free_region)
+KBASE_EXPORT_TEST_API(kbase_mem_free_region);
/**
* @brief Free the region from the GPU and unregister it.
* This function implements the free operation on a memory segment.
* It will loudly fail if called with outstanding mappings.
*/
-mali_error kbase_mem_free(struct kbase_context *kctx, mali_addr64 gpu_addr)
+int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr)
{
- mali_error err = MALI_ERROR_NONE;
+ int err = 0;
struct kbase_va_region *reg;
KBASE_DEBUG_ASSERT(kctx != NULL);
if (0 == gpu_addr) {
dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
kbase_gpu_vm_lock(kctx);
reg = kctx->pending_regions[cookie];
if (!reg) {
- err = MALI_ERROR_FUNCTION_FAILED;
+ err = -EINVAL;
goto out_unlock;
}
if (!reg || (reg->flags & KBASE_REG_FREE)) {
dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX",
gpu_addr);
- err = MALI_ERROR_FUNCTION_FAILED;
+ err = -EINVAL;
goto out_unlock;
}
/* SAME_VA must be freed through munmap */
dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__,
gpu_addr);
- err = MALI_ERROR_FUNCTION_FAILED;
+ err = -EINVAL;
goto out_unlock;
}
return err;
}
-KBASE_EXPORT_TEST_API(kbase_mem_free)
+KBASE_EXPORT_TEST_API(kbase_mem_free);
void kbase_update_region_flags(struct kbase_va_region *reg, unsigned long flags)
{
if (0 == (flags & BASE_MEM_PROT_GPU_EX))
reg->flags |= KBASE_REG_GPU_NX;
- if (flags & BASE_MEM_COHERENT_LOCAL)
- reg->flags |= KBASE_REG_SHARE_IN;
- else if (flags & BASE_MEM_COHERENT_SYSTEM)
+ if (flags & BASE_MEM_COHERENT_SYSTEM ||
+ flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED)
reg->flags |= KBASE_REG_SHARE_BOTH;
+ else if (flags & BASE_MEM_COHERENT_LOCAL)
+ reg->flags |= KBASE_REG_SHARE_IN;
}
-KBASE_EXPORT_TEST_API(kbase_update_region_flags)
+KBASE_EXPORT_TEST_API(kbase_update_region_flags);
int kbase_alloc_phy_pages_helper(
struct kbase_mem_phy_alloc *alloc,
* allocation is visible to the OOM killer */
kbase_process_page_usage_inc(alloc->imported.kctx, nr_pages_requested);
- if (MALI_ERROR_NONE != kbase_mem_allocator_alloc(&alloc->imported.kctx->osalloc, nr_pages_requested, alloc->pages + alloc->nents))
+ if (kbase_mem_allocator_alloc(&alloc->imported.kctx->osalloc,
+ nr_pages_requested, alloc->pages + alloc->nents) != 0)
goto no_alloc;
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_aux_pagesalloc((s64)nr_pages_requested);
+#endif
+
alloc->nents += nr_pages_requested;
done:
return 0;
struct kbase_mem_phy_alloc *alloc,
size_t nr_pages_to_free)
{
- mali_bool syncback;
+ bool syncback;
phys_addr_t *start_free;
KBASE_DEBUG_ASSERT(alloc);
start_free = alloc->pages + alloc->nents - nr_pages_to_free;
- syncback = (alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED) ? MALI_TRUE : MALI_FALSE;
+ syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED;
kbase_mem_allocator_free(&alloc->imported.kctx->osalloc,
nr_pages_to_free,
kbase_atomic_sub_pages(nr_pages_to_free, &alloc->imported.kctx->used_pages);
kbase_atomic_sub_pages(nr_pages_to_free, &alloc->imported.kctx->kbdev->memdev.used_pages);
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_aux_pagesalloc(-(s64)nr_pages_to_free);
+#endif
+
return 0;
}
/* Prevent vsize*sizeof from wrapping around.
* For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail.
*/
- if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->alloc->pages)))
+ if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages)))
goto out_term;
KBASE_DEBUG_ASSERT(0 != vsize);
- if (MALI_ERROR_NONE != kbase_alloc_phy_pages_helper(reg->alloc, size))
+ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0)
goto out_term;
+ if (reg->cpu_alloc != reg->gpu_alloc) {
+ if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0)
+ goto out_rollback;
+ }
+
return 0;
- out_term:
+out_rollback:
+ kbase_free_phy_pages_helper(reg->cpu_alloc, size);
+out_term:
return -1;
}
-KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages)
+KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages);
-mali_bool kbase_check_alloc_flags(unsigned long flags)
+bool kbase_check_alloc_flags(unsigned long flags)
{
- /* Only known flags should be set. */
- if (flags & ~((1ul << BASE_MEM_FLAGS_NR_INPUT_BITS) - 1))
- return MALI_FALSE;
+ /* Only known input flags should be set. */
+ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK)
+ return false;
/* At least one flag should be set */
if (flags == 0)
- return MALI_FALSE;
+ return false;
/* Either the GPU or CPU must be reading from the allocated memory */
if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0)
- return MALI_FALSE;
+ return false;
/* Either the GPU or CPU must be writing to the allocated memory */
if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0)
- return MALI_FALSE;
+ return false;
/* GPU cannot be writing to GPU executable memory and cannot grow the memory on page fault. */
if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF)))
- return MALI_FALSE;
+ return false;
/* GPU should have at least read or write access otherwise there is no
reason for allocating. */
if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0)
- return MALI_FALSE;
+ return false;
+
+ return true;
+}
+
+bool kbase_check_import_flags(unsigned long flags)
+{
+ /* Only known input flags should be set. */
+ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK)
+ return false;
+
+ /* At least one flag should be set */
+ if (flags == 0)
+ return false;
+
+ /* Imported memory cannot be GPU executable */
+ if (flags & BASE_MEM_PROT_GPU_EX)
+ return false;
+
+ /* Imported memory cannot grow on page fault */
+ if (flags & BASE_MEM_GROW_ON_GPF)
+ return false;
+
+ /* GPU should have at least read or write access otherwise there is no
+ reason for importing. */
+ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0)
+ return false;
+
+ /* Secure memory cannot be read by the CPU */
+ if ((flags & BASE_MEM_SECURE) && (flags & BASE_MEM_PROT_CPU_RD))
+ return false;
- return MALI_TRUE;
+ return true;
}
/**
mutex_lock(&kctx->reg_lock);
}
-KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock)
+KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock);
/**
* @brief Release the per-context region list lock
mutex_unlock(&kctx->reg_lock);
}
-KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock)
+KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#error "Don't include this file directly, use mali_kbase.h instead"
#endif
-#include <malisw/mali_malisw.h>
#include <linux/kref.h>
#ifdef CONFIG_UMP
#include <mali_kbase_hw.h>
#include "mali_kbase_pm.h"
#include "mali_kbase_defs.h"
-#ifdef CONFIG_MALI_GATOR_SUPPORT
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
#include "mali_kbase_gator.h"
-#endif /*CONFIG_MALI_GATOR_SUPPORT*/
+#endif
/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */
/**
* A CPU mapping
*/
-typedef struct kbase_cpu_mapping {
+struct kbase_cpu_mapping {
struct list_head mappings_list;
struct kbase_mem_phy_alloc *alloc;
struct kbase_context *kctx;
int count;
unsigned long vm_start;
unsigned long vm_end;
-} kbase_cpu_mapping;
+};
enum kbase_memory_type {
KBASE_MEM_TYPE_NATIVE,
* shared with another region or client. CPU mappings are OK to exist when changing, as
* long as the tracked mappings objects are updated as part of the change.
*/
-struct kbase_mem_phy_alloc
-{
+struct kbase_mem_phy_alloc {
struct kref kref; /* number of users of this alloc */
atomic_t gpu_mappings;
size_t nents; /* 0..N */
- phys_addr_t * pages; /* N elements, only 0..nents are valid */
+ phys_addr_t *pages; /* N elements, only 0..nents are valid */
/* kbase_cpu_mappings */
struct list_head mappings;
} umm;
#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */
struct {
- mali_size64 stride;
+ u64 stride;
size_t nents;
struct kbase_aliased *aliased;
} alias;
}
}
-void kbase_mem_kref_free(struct kref * kref);
+void kbase_mem_kref_free(struct kref *kref);
-mali_error kbase_mem_init(struct kbase_device * kbdev);
-void kbase_mem_halt(struct kbase_device * kbdev);
-void kbase_mem_term(struct kbase_device * kbdev);
+int kbase_mem_init(struct kbase_device *kbdev);
+void kbase_mem_halt(struct kbase_device *kbdev);
+void kbase_mem_term(struct kbase_device *kbdev);
-static inline struct kbase_mem_phy_alloc * kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc * alloc)
+static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc)
{
kref_get(&alloc->kref);
return alloc;
}
-static inline struct kbase_mem_phy_alloc * kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc * alloc)
+static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc)
{
kref_put(&alloc->kref, kbase_mem_kref_free);
return NULL;
/**
* A GPU memory region, and attributes for CPU mappings.
*/
-typedef struct kbase_va_region {
+struct kbase_va_region {
struct rb_node rblink;
struct list_head link;
#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16)
#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16)
+#define KBASE_REG_SECURE (1ul << 19)
+
#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0)
/* only used with 32-bit clients */
/*
* On a 32bit platform, custom VA should be wired from (4GB + shader region)
- * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface
+ * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface
* limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference).
* So we put the default limit to the maximum possible on Linux and shrink
* it down, if required by the GPU, during initialization.
size_t extent; /* nr of pages alloc'd on PF */
- struct kbase_mem_phy_alloc * alloc; /* the one alloc object we mmap to the GPU and CPU when mapping this region */
+ struct kbase_mem_phy_alloc *cpu_alloc; /* the one alloc object we mmap to the CPU when mapping this region */
+ struct kbase_mem_phy_alloc *gpu_alloc; /* the one alloc object we mmap to the GPU when mapping this region */
/* non-NULL if this memory object is a kds_resource */
struct kds_resource *kds_res;
-} kbase_va_region;
+};
/* Common functions */
-static INLINE phys_addr_t *kbase_get_phy_pages(struct kbase_va_region *reg)
+static inline phys_addr_t *kbase_get_cpu_phy_pages(struct kbase_va_region *reg)
+{
+ KBASE_DEBUG_ASSERT(reg);
+ KBASE_DEBUG_ASSERT(reg->cpu_alloc);
+ KBASE_DEBUG_ASSERT(reg->gpu_alloc);
+ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents);
+
+ return reg->cpu_alloc->pages;
+}
+
+static inline phys_addr_t *kbase_get_gpu_phy_pages(struct kbase_va_region *reg)
{
KBASE_DEBUG_ASSERT(reg);
- KBASE_DEBUG_ASSERT(reg->alloc);
+ KBASE_DEBUG_ASSERT(reg->cpu_alloc);
+ KBASE_DEBUG_ASSERT(reg->gpu_alloc);
+ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents);
- return reg->alloc->pages;
+ return reg->gpu_alloc->pages;
}
-static INLINE size_t kbase_reg_current_backed_size(struct kbase_va_region * reg)
+static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg)
{
KBASE_DEBUG_ASSERT(reg);
/* if no alloc object the backed size naturally is 0 */
- if (reg->alloc)
- return reg->alloc->nents;
- else
+ if (!reg->cpu_alloc)
return 0;
+
+ KBASE_DEBUG_ASSERT(reg->cpu_alloc);
+ KBASE_DEBUG_ASSERT(reg->gpu_alloc);
+ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents);
+
+ return reg->cpu_alloc->nents;
}
#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */
-static INLINE struct kbase_mem_phy_alloc * kbase_alloc_create(size_t nr_pages, enum kbase_memory_type type)
+static inline struct kbase_mem_phy_alloc *kbase_alloc_create(size_t nr_pages, enum kbase_memory_type type)
{
struct kbase_mem_phy_alloc *alloc;
const size_t alloc_size =
kref_init(&alloc->kref);
atomic_set(&alloc->gpu_mappings, 0);
alloc->nents = 0;
- alloc->pages = (void*)(alloc + 1);
+ alloc->pages = (void *)(alloc + 1);
INIT_LIST_HEAD(&alloc->mappings);
alloc->type = type;
return alloc;
}
-static INLINE int kbase_reg_prepare_native(struct kbase_va_region * reg, struct kbase_context * kctx)
+static inline int kbase_reg_prepare_native(struct kbase_va_region *reg,
+ struct kbase_context *kctx)
{
KBASE_DEBUG_ASSERT(reg);
- KBASE_DEBUG_ASSERT(!reg->alloc);
+ KBASE_DEBUG_ASSERT(!reg->cpu_alloc);
+ KBASE_DEBUG_ASSERT(!reg->gpu_alloc);
KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE);
- reg->alloc = kbase_alloc_create(reg->nr_pages, KBASE_MEM_TYPE_NATIVE);
- if (IS_ERR(reg->alloc))
- return PTR_ERR(reg->alloc);
- else if (!reg->alloc)
+ reg->cpu_alloc = kbase_alloc_create(reg->nr_pages,
+ KBASE_MEM_TYPE_NATIVE);
+ if (IS_ERR(reg->cpu_alloc))
+ return PTR_ERR(reg->cpu_alloc);
+ else if (!reg->cpu_alloc)
return -ENOMEM;
- reg->alloc->imported.kctx = kctx;
+ reg->cpu_alloc->imported.kctx = kctx;
+ if (kctx->infinite_cache_active && (reg->flags & KBASE_REG_CPU_CACHED)) {
+ reg->gpu_alloc = kbase_alloc_create(reg->nr_pages,
+ KBASE_MEM_TYPE_NATIVE);
+ reg->gpu_alloc->imported.kctx = kctx;
+ } else {
+ reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
+ }
+
reg->flags &= ~KBASE_REG_FREE;
return 0;
}
static inline int kbase_atomic_add_pages(int num_pages, atomic_t *used_pages)
{
int new_val = atomic_add_return(num_pages, used_pages);
-#ifdef CONFIG_MALI_GATOR_SUPPORT
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
kbase_trace_mali_total_alloc_pages_change((long long int)new_val);
#endif
return new_val;
static inline int kbase_atomic_sub_pages(int num_pages, atomic_t *used_pages)
{
int new_val = atomic_sub_return(num_pages, used_pages);
-#ifdef CONFIG_MALI_GATOR_SUPPORT
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
kbase_trace_mali_total_alloc_pages_change((long long int)new_val);
#endif
return new_val;
* @param allocator Allocator object to initialize
* @param max_size Maximum number of pages to keep on the freelist.
* @param kbdev The kbase device this allocator is used with
- * @return MALI_ERROR_NONE on success, an error code indicating what failed on
+ * @return 0 on success, an error code indicating what failed on
* error.
*/
-mali_error kbase_mem_allocator_init(struct kbase_mem_allocator *allocator,
+int kbase_mem_allocator_init(struct kbase_mem_allocator *allocator,
unsigned int max_size,
struct kbase_device *kbdev);
* @param[in] allocator Allocator to obtain the memory from
* @param nr_pages Number of pages to allocate
* @param[out] pages Pointer to an array where the physical address of the allocated pages will be stored
- * @return MALI_ERROR_NONE if the pages were allocated, an error code indicating what failed on error
+ * @return 0 if the pages were allocated, an error code indicating what failed on error
*/
-mali_error kbase_mem_allocator_alloc(struct kbase_mem_allocator * allocator, size_t nr_pages, phys_addr_t *pages);
+int kbase_mem_allocator_alloc(struct kbase_mem_allocator *allocator, size_t nr_pages, phys_addr_t *pages);
/**
* @brief Free memory obtained for an OS based memory allocator.
* @param[in] allocator Allocator to free the memory back to
* @param nr_pages Number of pages to free
* @param[in] pages Pointer to an array holding the physical address of the paghes to free.
- * @param[in] sync_back MALI_TRUE case the memory should be synced back
+ * @param[in] sync_back true case the memory should be synced back
*/
-void kbase_mem_allocator_free(struct kbase_mem_allocator * allocator, size_t nr_pages, phys_addr_t *pages, mali_bool sync_back);
+void kbase_mem_allocator_free(struct kbase_mem_allocator *allocator, size_t nr_pages, phys_addr_t *pages, bool sync_back);
/**
* @brief Terminate an OS based memory allocator.
*
* @param[in] allocator Allocator to terminate
*/
-void kbase_mem_allocator_term(struct kbase_mem_allocator * allocator);
+void kbase_mem_allocator_term(struct kbase_mem_allocator *allocator);
-mali_error kbase_region_tracker_init(struct kbase_context *kctx);
+int kbase_region_tracker_init(struct kbase_context *kctx);
void kbase_region_tracker_term(struct kbase_context *kctx);
-struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, mali_addr64 gpu_addr);
+struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr);
/**
* @brief Check that a pointer is actually a valid region.
*
* Must be called with context lock held.
*/
-struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, mali_addr64 gpu_addr);
+struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr);
struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone);
void kbase_free_alloced_region(struct kbase_va_region *reg);
-mali_error kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, mali_addr64 addr, size_t nr_pages, size_t align);
+int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align);
-mali_error kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, mali_addr64 addr, size_t nr_pages, size_t align);
-mali_bool kbase_check_alloc_flags(unsigned long flags);
+int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align);
+bool kbase_check_alloc_flags(unsigned long flags);
+bool kbase_check_import_flags(unsigned long flags);
void kbase_update_region_flags(struct kbase_va_region *reg, unsigned long flags);
void kbase_gpu_vm_lock(struct kbase_context *kctx);
int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size);
-mali_error kbase_mmu_init(struct kbase_context *kctx);
+int kbase_mmu_init(struct kbase_context *kctx);
void kbase_mmu_term(struct kbase_context *kctx);
phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx);
void kbase_mmu_free_pgd(struct kbase_context *kctx);
-mali_error kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn,
+int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn,
phys_addr_t *phys, size_t nr,
unsigned long flags);
-mali_error kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn,
+int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn,
phys_addr_t phys, size_t nr,
unsigned long flags);
-mali_error kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr);
-mali_error kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t* phys, size_t nr, unsigned long flags);
+int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr);
+int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags);
/**
* @brief Register region and map it on the GPU.
*
* Call kbase_add_va_region() and map the region on the GPU.
*/
-mali_error kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, mali_addr64 addr, size_t nr_pages, size_t align);
+int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align);
/**
* @brief Remove the region from the GPU and unregister it.
*
* Must be called with context lock held.
*/
-mali_error kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg);
+int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg);
/**
* The caller has the following locking conditions:
*/
void kbase_mmu_disable(struct kbase_context *kctx);
+/**
+ * kbase_mmu_disable_as() - set the MMU in unmapped mode for an address space.
+ *
+ * @kbdev: Kbase device
+ * @as_nr: Number of the address space for which the MMU
+ * should be set in unmapped mode.
+ *
+ * The caller must hold kbdev->as[as_nr].transaction_mutex.
+ */
+void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr);
+
void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat);
/** Dump the MMU tables to a buffer
*/
void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages);
-mali_error kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset);
-void kbase_sync_single(struct kbase_context *kctx, phys_addr_t pa,
- size_t size, kbase_sync_kmem_fn sync_fn);
+int kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset);
+void kbase_sync_single(struct kbase_context *kctx, phys_addr_t cpu_pa,
+ phys_addr_t gpu_pa, off_t offset, size_t size,
+ enum kbase_sync_type sync_fn);
void kbase_pre_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr);
void kbase_post_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr);
* This function sets (extends with) requested attributes for given region
* of imported external memory
*
- * @param[in] kctx The kbase context which the tmem belongs to
+ * @param[in] kctx The kbase context which the tmem belongs to
* @param[in] gpu_addr The base address of the tmem region
* @param[in] attributes The attributes of tmem region to be set
*
- * @return MALI_ERROR_NONE on success. Any other value indicates failure.
+ * @return 0 on success. Any other value indicates failure.
*/
-mali_error kbase_tmem_set_attributes(struct kbase_context *kctx, mali_addr64 gpu_addr, u32 attributes);
+int kbase_tmem_set_attributes(struct kbase_context *kctx, u64 gpu_addr, u32 attributes);
/**
* Get attributes of imported tmem region
*
* This function retrieves the attributes of imported external memory
*
- * @param[in] kctx The kbase context which the tmem belongs to
+ * @param[in] kctx The kbase context which the tmem belongs to
* @param[in] gpu_addr The base address of the tmem region
* @param[out] attributes The actual attributes of tmem region
*
- * @return MALI_ERROR_NONE on success. Any other value indicates failure.
+ * @return 0 on success. Any other value indicates failure.
*/
-mali_error kbase_tmem_get_attributes(struct kbase_context *kctx, mali_addr64 gpu_addr, u32 * const attributes);
+int kbase_tmem_get_attributes(struct kbase_context *kctx, u64 gpu_addr, u32 *const attributes);
/* OS specific functions */
-mali_error kbase_mem_free(struct kbase_context *kctx, mali_addr64 gpu_addr);
-mali_error kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg);
+int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr);
+int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg);
void kbase_os_mem_map_lock(struct kbase_context *kctx);
void kbase_os_mem_map_unlock(struct kbase_context *kctx);
* OS specific call to updates the current memory allocation counters for the current process with
* the supplied delta.
*
- * @param[in] kctx The kbase context
+ * @param[in] kctx The kbase context
* @param[in] pages The desired delta to apply to the memory usage counters.
*/
* @param[in] pages The desired delta to apply to the memory usage counters.
*/
-static INLINE void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages)
+static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages)
{
kbasep_os_process_page_usage_update(kctx, pages);
}
* @param[in] pages The desired delta to apply to the memory usage counters.
*/
-static INLINE void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages)
+static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages)
{
kbasep_os_process_page_usage_update(kctx, 0 - pages);
}
* @param[out] offset The offset from the start of the allocation to the
* specified CPU virtual address.
*
- * @return MALI_ERROR_NONE if offset was obtained successfully. Error code
+ * @return 0 if offset was obtained successfully. Error code
* otherwise.
*/
-mali_error kbasep_find_enclosing_cpu_mapping_offset(struct kbase_context *kctx,
- mali_addr64 gpu_addr,
+int kbasep_find_enclosing_cpu_mapping_offset(struct kbase_context *kctx,
+ u64 gpu_addr,
unsigned long uaddr,
size_t size,
- mali_size64 *offset);
+ u64 *offset);
enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer);
void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom);
*
* @return 0 if all pages have been successfully allocated. Error code otherwise
*/
-int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc * alloc, size_t nr_pages_requested);
+int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_requested);
/**
* @brief Free physical pages.
* @param[in] alloc allocation object to free pages from
* @param[in] nr_pages_to_free number of physical pages to free
*/
-int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc * alloc, size_t nr_pages_to_free);
-
-#ifdef CONFIG_MALI_NO_MALI
-static inline void kbase_wait_write_flush(struct kbase_context *kctx)
-{
-}
-#else
-void kbase_wait_write_flush(struct kbase_context *kctx);
-#endif
+int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free);
static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr)
{
void kbase_mmu_interrupt_process(struct kbase_device *kbdev,
struct kbase_context *kctx, struct kbase_as *as);
+/**
+ * @brief Process a page fault.
+ *
+ * @param[in] data work_struct passed by queue_work()
+ */
+void page_fault_worker(struct work_struct *data);
+
+/**
+ * @brief Process a bus fault.
+ *
+ * @param[in] data work_struct passed by queue_work()
+ */
+void bus_fault_worker(struct work_struct *data);
+
+/**
+ * @brief Flush MMU workqueues.
+ *
+ * This function will cause any outstanding page or bus faults to be processed.
+ * It should be called prior to powering off the GPU.
+ *
+ * @param[in] kbdev Device pointer
+ */
+void kbase_flush_mmu_wqs(struct kbase_device *kbdev);
+
+/**
+ * kbase_sync_single_for_device - update physical memory and give GPU ownership
+ * @kbdev: Device pointer
+ * @handle: DMA address of region
+ * @size: Size of region to sync
+ * @dir: DMA data direction
+ */
+
+void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle,
+ size_t size, enum dma_data_direction dir);
+
+/**
+ * kbase_sync_single_for_cpu - update physical memory and give CPU ownership
+ * @kbdev: Device pointer
+ * @handle: DMA address of region
+ * @size: Size of region to sync
+ * @dir: DMA data direction
+ */
+
+void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle,
+ size_t size, enum dma_data_direction dir);
+
#endif /* _KBASE_MEM_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
void kbase_mem_lowlevel_term(struct kbase_device *kbdev)
{
- return;
}
static unsigned long kbase_mem_allocator_count(struct shrinker *s,
}
mutex_unlock(&allocator->free_list_lock);
return atomic_read(&allocator->free_list_size);
-
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0)
{
if (sc->nr_to_scan == 0)
return kbase_mem_allocator_count(s, sc);
- else
- return kbase_mem_allocator_scan(s, sc);
+
+ return kbase_mem_allocator_scan(s, sc);
}
#endif
-mali_error kbase_mem_allocator_init(struct kbase_mem_allocator *const allocator,
+int kbase_mem_allocator_init(struct kbase_mem_allocator *const allocator,
unsigned int max_size, struct kbase_device *kbdev)
{
KBASE_DEBUG_ASSERT(NULL != allocator);
register_shrinker(&allocator->free_list_reclaimer);
- return MALI_ERROR_NONE;
+ return 0;
}
-KBASE_EXPORT_TEST_API(kbase_mem_allocator_init)
+KBASE_EXPORT_TEST_API(kbase_mem_allocator_init);
void kbase_mem_allocator_term(struct kbase_mem_allocator *allocator)
{
mutex_unlock(&allocator->free_list_lock);
mutex_destroy(&allocator->free_list_lock);
}
-KBASE_EXPORT_TEST_API(kbase_mem_allocator_term)
+KBASE_EXPORT_TEST_API(kbase_mem_allocator_term);
-mali_error kbase_mem_allocator_alloc(struct kbase_mem_allocator *allocator, size_t nr_pages, phys_addr_t *pages)
+int kbase_mem_allocator_alloc(struct kbase_mem_allocator *allocator, size_t nr_pages, phys_addr_t *pages)
{
struct page *p;
- void *mp;
int i;
int num_from_free_list;
struct list_head from_free_list = LIST_HEAD_INIT(from_free_list);
mutex_lock(&allocator->free_list_lock);
num_from_free_list = MIN(nr_pages, atomic_read(&allocator->free_list_size));
atomic_sub(num_from_free_list, &allocator->free_list_size);
- for (i = 0; i < num_from_free_list; i++)
- {
+ for (i = 0; i < num_from_free_list; i++) {
BUG_ON(list_empty(&allocator->free_list_head));
p = list_first_entry(&allocator->free_list_head, struct page, lru);
list_move(&p->lru, &from_free_list);
}
if (i == nr_pages)
- return MALI_ERROR_NONE;
+ return 0;
#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
/* DMA cache sync fails for HIGHMEM before 3.5 on ARM */
- gfp = GFP_USER;
+ gfp = GFP_USER | __GFP_ZERO;
#else
- gfp = GFP_HIGHUSER;
+ gfp = GFP_HIGHUSER | __GFP_ZERO;
#endif
if (current->flags & PF_KTHREAD) {
/* If not all pages were sourced from the pool, request new ones. */
for (; i < nr_pages; i++) {
dma_addr_t dma_addr;
+
p = alloc_page(gfp);
if (NULL == p)
goto err_out_roll_back;
- mp = kmap(p);
- if (NULL == mp) {
- __free_page(p);
- goto err_out_roll_back;
- }
- memset(mp, 0x00, PAGE_SIZE); /* instead of __GFP_ZERO, so we can do cache maintenance */
- kunmap(p);
dma_addr = dma_map_page(allocator->kbdev->dev, p, 0, PAGE_SIZE,
- DMA_BIDIRECTIONAL);
+ DMA_BIDIRECTIONAL);
if (dma_mapping_error(allocator->kbdev->dev, dma_addr)) {
__free_page(p);
goto err_out_roll_back;
}
- SetPagePrivate(p);
kbase_set_dma_addr(p, dma_addr);
pages[i] = PFN_PHYS(page_to_pfn(p));
BUG_ON(dma_addr != pages[i]);
}
- return MALI_ERROR_NONE;
+ return 0;
err_out_roll_back:
while (i--) {
- struct page *p;
p = pfn_to_page(PFN_DOWN(pages[i]));
pages[i] = (phys_addr_t)0;
dma_unmap_page(allocator->kbdev->dev, kbase_dma_addr(p),
__free_page(p);
}
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
}
-KBASE_EXPORT_TEST_API(kbase_mem_allocator_alloc)
+KBASE_EXPORT_TEST_API(kbase_mem_allocator_alloc);
-void kbase_mem_allocator_free(struct kbase_mem_allocator *allocator, size_t nr_pages, phys_addr_t *pages, mali_bool sync_back)
+void kbase_mem_allocator_free(struct kbase_mem_allocator *allocator, size_t nr_pages, phys_addr_t *pages, bool sync_back)
{
int i = 0;
int page_count = 0;
atomic_add(page_count, &allocator->free_list_size);
mutex_unlock(&allocator->free_list_lock);
}
-KBASE_EXPORT_TEST_API(kbase_mem_allocator_free)
+KBASE_EXPORT_TEST_API(kbase_mem_allocator_free);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <linux/slab.h>
/* raw page handling */
-struct kbase_mem_allocator
-{
+struct kbase_mem_allocator {
struct kbase_device *kbdev;
atomic_t free_list_size;
unsigned int free_list_max_size;
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
dma_addr_t dma_addr;
#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
/* DMA cache sync fails for HIGHMEM before 3.5 on ARM */
- gfp = GFP_USER;
+ gfp = GFP_USER | __GFP_ZERO;
#else
- gfp = GFP_HIGHUSER;
+ gfp = GFP_HIGHUSER | __GFP_ZERO;
#endif
if (current->flags & PF_KTHREAD) {
{
phys_addr_t mem;
-#if defined(CONFIG_ARM) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
+#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) \
+ && LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
/* DMA cache sync fails for HIGHMEM before 3.5 on ARM */
mem = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ACCESSIBLE);
#else
{
}
-STATIC int kbase_mem_allocator_shrink(struct shrinker *s, struct shrink_control *sc)
+static int kbase_mem_allocator_shrink(struct shrinker *s, struct shrink_control *sc)
{
struct kbase_mem_allocator *allocator;
int i;
return atomic_read(&allocator->free_list_size);
}
-mali_error kbase_mem_allocator_init(struct kbase_mem_allocator * const allocator,
+int kbase_mem_allocator_init(struct kbase_mem_allocator * const allocator,
unsigned int max_size, struct kbase_device *kbdev)
{
KBASE_DEBUG_ASSERT(NULL != allocator);
register_shrinker(&allocator->free_list_reclaimer);
- return MALI_ERROR_NONE;
+ return 0;
}
void kbase_mem_allocator_term(struct kbase_mem_allocator *allocator)
}
-mali_error kbase_mem_allocator_alloc(struct kbase_mem_allocator *allocator, size_t nr_pages, phys_addr_t *pages)
+int kbase_mem_allocator_alloc(struct kbase_mem_allocator *allocator, size_t nr_pages, phys_addr_t *pages)
{
struct page *p;
- void *mp;
int i;
int num_from_free_list;
struct list_head from_free_list = LIST_HEAD_INIT(from_free_list);
i = 0;
/* Allocate as many pages from the pool of already allocated pages. */
- list_for_each_entry(p, &from_free_list, lru)
- {
+ list_for_each_entry(p, &from_free_list, lru) {
pages[i] = PFN_PHYS(page_to_pfn(p));
i++;
}
if (i == nr_pages)
- return MALI_ERROR_NONE;
+ return 0;
/* If not all pages were sourced from the pool, request new ones. */
for (; i < nr_pages; i++) {
if (NULL == p)
goto err_out_roll_back;
- mp = kmap(p);
- if (NULL == mp) {
- kbase_carveout_put_page(p, allocator);
- goto err_out_roll_back;
- }
- memset(mp, 0x00, PAGE_SIZE); /* instead of __GFP_ZERO, so we can
- do cache maintenance */
- dma_sync_single_for_device(allocator->kbdev->dev,
+ kbase_sync_single_for_device(allocator->kbdev,
kbase_dma_addr(p),
PAGE_SIZE,
DMA_BIDIRECTIONAL);
- kunmap(p);
+
pages[i] = PFN_PHYS(page_to_pfn(p));
}
- return MALI_ERROR_NONE;
+ return 0;
err_out_roll_back:
while (i--) {
kbase_carveout_put_page(p, allocator);
}
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
}
-void kbase_mem_allocator_free(struct kbase_mem_allocator *allocator, u32 nr_pages, phys_addr_t *pages, mali_bool sync_back)
+void kbase_mem_allocator_free(struct kbase_mem_allocator *allocator, u32 nr_pages, phys_addr_t *pages, bool sync_back)
{
int i = 0;
int page_count = 0;
* invalidations don't trample on memory.
*/
if (sync_back)
- dma_sync_single_for_cpu(allocator->kbdev->dev,
+ kbase_sync_single_for_cpu(allocator->kbdev,
kbase_dma_addr(p),
PAGE_SIZE,
DMA_BIDIRECTIONAL);
atomic_add(page_count, &allocator->free_list_size);
mutex_unlock(&allocator->free_list_lock);
}
-KBASE_EXPORT_TEST_API(kbase_mem_allocator_free)
+KBASE_EXPORT_TEST_API(kbase_mem_allocator_free);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase.h>
#include <mali_kbase_mem_linux.h>
#include <mali_kbase_config_defaults.h>
+#include <mali_kbase_hwaccess_time.h>
static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma);
static const struct vm_operations_struct kbase_vm_ops;
int cpu_va_bits;
struct kbase_va_region *reg;
struct device *dev;
+
KBASE_DEBUG_ASSERT(kctx);
KBASE_DEBUG_ASSERT(flags);
KBASE_DEBUG_ASSERT(gpu_va);
goto bad_size;
}
- if (va_pages > (UINT64_MAX / PAGE_SIZE))
+ if (va_pages > (U64_MAX / PAGE_SIZE))
/* 64-bit address range is the max */
goto bad_size;
goto bad_flags;
}
+ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
+ kctx->kbdev->system_coherency != COHERENCY_ACE) {
+ dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable");
+ goto bad_flags;
+ }
+ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
+ kctx->kbdev->system_coherency != COHERENCY_ACE) {
+ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */
+ *flags &= ~BASE_MEM_COHERENT_SYSTEM;
+ }
+
/* Limit GPU executable allocs to GPU PC size */
if ((*flags & BASE_MEM_PROT_GPU_EX) &&
(va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT)))
goto no_region;
}
- if (MALI_ERROR_NONE != kbase_reg_prepare_native(reg, kctx)) {
+ kbase_update_region_flags(reg, *flags);
+
+ if (kbase_reg_prepare_native(reg, kctx) != 0) {
dev_err(dev, "Failed to prepare region");
goto prepare_failed;
}
- kbase_update_region_flags(reg, *flags);
-
if (*flags & BASE_MEM_GROW_ON_GPF)
reg->extent = extent;
else
reg->extent = 0;
if (kbase_alloc_phy_pages(reg, va_pages, commit_pages)) {
- dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)",
+ dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)",
(unsigned long long)commit_pages,
(unsigned long long)va_pages);
goto no_mem;
reg->flags |= KBASE_REG_ALIGNED;
}
} else /* we control the VA */ {
- if (MALI_ERROR_NONE != kbase_gpu_mmap(kctx, reg, 0, va_pages, 1)) {
+ if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) {
dev_warn(dev, "Failed to map memory on GPU");
goto no_mmap;
}
no_cookie:
kbase_gpu_vm_unlock(kctx);
no_mem:
- kbase_mem_phy_alloc_put(reg->alloc);
+ kbase_mem_phy_alloc_put(reg->cpu_alloc);
+ kbase_mem_phy_alloc_put(reg->gpu_alloc);
prepare_failed:
kfree(reg);
no_region:
return NULL;
}
-mali_error kbase_mem_query(struct kbase_context *kctx, mali_addr64 gpu_addr, int query, u64 * const out)
+int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out)
{
struct kbase_va_region *reg;
- mali_error ret = MALI_ERROR_FUNCTION_FAILED;
+ int ret = -EINVAL;
KBASE_DEBUG_ASSERT(kctx);
KBASE_DEBUG_ASSERT(out);
switch (query) {
case KBASE_MEM_QUERY_COMMIT_SIZE:
- if (reg->alloc->type != KBASE_MEM_TYPE_ALIAS) {
+ if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) {
*out = kbase_reg_current_backed_size(reg);
} else {
size_t i;
struct kbase_aliased *aliased;
*out = 0;
- aliased = reg->alloc->imported.alias.aliased;
- for (i = 0; i < reg->alloc->imported.alias.nents; i++)
+ aliased = reg->cpu_alloc->imported.alias.aliased;
+ for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++)
*out += aliased[i].length;
}
break;
goto out_unlock;
}
- ret = MALI_ERROR_NONE;
+ ret = 0;
out_unlock:
kbase_gpu_vm_unlock(kctx);
return ret;
}
-mali_error kbase_mem_flags_change(struct kbase_context *kctx, mali_addr64 gpu_addr, unsigned int flags, unsigned int mask)
+int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask)
{
struct kbase_va_region *reg;
- mali_error ret = MALI_ERROR_FUNCTION_FAILED;
+ int ret = -EINVAL;
unsigned int real_flags = 0;
unsigned int prev_flags = 0;
KBASE_DEBUG_ASSERT(kctx);
if (!gpu_addr)
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
/* nuke other bits */
flags &= mask;
goto out_unlock;
/* limit to imported memory */
- if ((reg->alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) &&
- (reg->alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM))
+ if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) &&
+ (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM))
goto out_unlock;
/* no change? */
if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) {
- ret = MALI_ERROR_NONE;
+ ret = 0;
goto out_unlock;
}
reg->flags |= real_flags;
/* Currently supporting only imported memory */
- switch (reg->alloc->type) {
+ switch (reg->gpu_alloc->type) {
#ifdef CONFIG_UMP
- case KBASE_MEM_TYPE_IMPORTED_UMP:
- ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_phy_pages(reg), reg->alloc->nents, reg->flags);
- break;
+ case KBASE_MEM_TYPE_IMPORTED_UMP:
+ ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_cpu_phy_pages(reg), reg->gpu_alloc->nents, reg->flags);
+ break;
#endif
#ifdef CONFIG_DMA_SHARED_BUFFER
- case KBASE_MEM_TYPE_IMPORTED_UMM:
- /* Future use will use the new flags, existing mapping will NOT be updated
- * as memory should not be in use by the GPU when updating the flags.
- */
- ret = MALI_ERROR_NONE;
- WARN_ON(reg->alloc->imported.umm.current_mapping_usage_count);
- break;
+ case KBASE_MEM_TYPE_IMPORTED_UMM:
+ /* Future use will use the new flags, existing mapping will NOT be updated
+ * as memory should not be in use by the GPU when updating the flags.
+ */
+ ret = 0;
+ WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count);
+ break;
#endif
- default:
- break;
+ default:
+ break;
}
/* roll back on error, i.e. not UMP */
- if (ret != MALI_ERROR_NONE)
+ if (ret)
reg->flags = prev_flags;
out_unlock:
return ret;
}
-#define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_TOTAL_BITS)
+#define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS)
#ifdef CONFIG_UMP
static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags)
KBASE_DEBUG_ASSERT(va_pages);
KBASE_DEBUG_ASSERT(flags);
+ if (*flags & BASE_MEM_SECURE)
+ goto bad_flags;
+
umph = ump_dd_from_secure_id(id);
if (UMP_DD_INVALID_MEMORY_HANDLE == umph)
goto bad_id;
if (!*va_pages)
goto bad_size;
- if (*va_pages > (UINT64_MAX / PAGE_SIZE))
+ if (*va_pages > (U64_MAX / PAGE_SIZE))
/* 64-bit address range is the max */
goto bad_size;
/* we've got pages to map now, and support SAME_VA */
*flags |= KBASE_MEM_IMPORT_HAVE_PAGES;
- reg->alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP);
- if (IS_ERR_OR_NULL(reg->alloc))
+ reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP);
+ if (IS_ERR_OR_NULL(reg->gpu_alloc))
goto no_alloc_obj;
- reg->alloc->imported.ump_handle = umph;
+ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
+
+ reg->gpu_alloc->imported.ump_handle = umph;
reg->flags &= ~KBASE_REG_FREE;
reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */
for (i = 0; i < block_count; i++) {
for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) {
- reg->alloc->pages[page] = block_array[i].addr + (j << PAGE_SHIFT);
+ reg->gpu_alloc->pages[page] = block_array[i].addr + (j << PAGE_SHIFT);
page++;
}
}
- reg->alloc->nents = *va_pages;
+ reg->gpu_alloc->nents = *va_pages;
reg->extent = 0;
return reg;
bad_size:
ump_dd_release(umph);
bad_id:
+bad_flags:
return NULL;
}
#endif /* CONFIG_UMP */
if (!*va_pages)
goto bad_size;
- if (*va_pages > (UINT64_MAX / PAGE_SIZE))
+ if (*va_pages > (U64_MAX / PAGE_SIZE))
/* 64-bit address range is the max */
goto bad_size;
if (!reg)
goto no_region;
- reg->alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM);
- if (IS_ERR_OR_NULL(reg->alloc))
+ reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM);
+ if (IS_ERR_OR_NULL(reg->gpu_alloc))
goto no_alloc_obj;
+ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
+
/* No pages to map yet */
- reg->alloc->nents = 0;
+ reg->gpu_alloc->nents = 0;
reg->flags &= ~KBASE_REG_FREE;
reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */
if (*flags & BASE_MEM_PROT_GPU_RD)
reg->flags |= KBASE_REG_GPU_RD;
+ if (*flags & BASE_MEM_SECURE)
+ reg->flags |= KBASE_REG_SECURE;
+
/* no read or write permission given on import, only on run do we give the right permissions */
- reg->alloc->type = BASE_TMEM_IMPORT_TYPE_UMM;
- reg->alloc->imported.umm.sgt = NULL;
- reg->alloc->imported.umm.dma_buf = dma_buf;
- reg->alloc->imported.umm.dma_attachment = dma_attachment;
- reg->alloc->imported.umm.current_mapping_usage_count = 0;
+ reg->gpu_alloc->type = BASE_TMEM_IMPORT_TYPE_UMM;
+ reg->gpu_alloc->imported.umm.sgt = NULL;
+ reg->gpu_alloc->imported.umm.dma_buf = dma_buf;
+ reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment;
+ reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0;
reg->extent = 0;
return reg;
struct kbase_va_region *reg;
u64 gpu_va;
size_t i;
+ bool coherent;
KBASE_DEBUG_ASSERT(kctx);
KBASE_DEBUG_ASSERT(flags);
/* mask to only allowed flags */
*flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR |
- BASE_MEM_HINT_GPU_RD | BASE_MEM_HINT_GPU_WR |
- BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL);
+ BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL |
+ BASE_MEM_COHERENT_SYSTEM_REQUIRED);
if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) {
dev_warn(kctx->kbdev->dev,
(unsigned long long)*flags);
goto bad_flags;
}
+ coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 ||
+ (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0;
if (!stride)
goto bad_stride;
if (!nents)
goto bad_nents;
- if ((nents * stride) > (UINT64_MAX / PAGE_SIZE))
+ if ((nents * stride) > (U64_MAX / PAGE_SIZE))
/* 64-bit address range is the max */
goto bad_size;
goto no_reg;
/* zero-sized page array, as we don't need one/can support one */
- reg->alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS);
- if (IS_ERR_OR_NULL(reg->alloc))
+ reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS);
+ if (IS_ERR_OR_NULL(reg->gpu_alloc))
goto no_alloc_obj;
+ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
+
kbase_update_region_flags(reg, *flags);
- reg->alloc->imported.alias.nents = nents;
- reg->alloc->imported.alias.stride = stride;
- reg->alloc->imported.alias.aliased = vzalloc(sizeof(*reg->alloc->imported.alias.aliased) * nents);
- if (!reg->alloc->imported.alias.aliased)
+ reg->gpu_alloc->imported.alias.nents = nents;
+ reg->gpu_alloc->imported.alias.stride = stride;
+ reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents);
+ if (!reg->gpu_alloc->imported.alias.aliased)
goto no_aliased_array;
kbase_gpu_vm_lock(kctx);
if (ai[i].length > stride)
goto bad_handle; /* can't be larger than the
stride */
- reg->alloc->imported.alias.aliased[i].length = ai[i].length;
+ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
} else {
struct kbase_va_region *aliasing_reg;
struct kbase_mem_phy_alloc *alloc;
goto bad_handle; /* Not found */
if (aliasing_reg->flags & KBASE_REG_FREE)
goto bad_handle; /* Free region */
- if (!aliasing_reg->alloc)
+ if (!aliasing_reg->gpu_alloc)
goto bad_handle; /* No alloc */
- if (aliasing_reg->alloc->type != KBASE_MEM_TYPE_NATIVE)
+ if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE)
goto bad_handle; /* Not a native alloc */
+ if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0))
+ goto bad_handle;
+ /* Non-coherent memory cannot alias
+ coherent memory, and vice versa.*/
/* check size against stride */
if (!ai[i].length)
goto bad_handle; /* can't be larger than the
stride */
- alloc = aliasing_reg->alloc;
+ alloc = aliasing_reg->gpu_alloc;
/* check against the alloc's size */
if (ai[i].offset > alloc->nents)
if (ai[i].offset + ai[i].length > alloc->nents)
goto bad_handle; /* beyond end */
- reg->alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc);
- reg->alloc->imported.alias.aliased[i].length = ai[i].length;
- reg->alloc->imported.alias.aliased[i].offset = ai[i].offset;
+ reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc);
+ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length;
+ reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset;
}
}
#else
if (1) {
#endif
- if (MALI_ERROR_NONE != kbase_gpu_mmap(kctx, reg, 0,
- *num_pages, 1)) {
- dev_warn(kctx->kbdev->dev,
- "Failed to map memory on GPU");
+ if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) {
+ dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU");
goto no_mmap;
}
/* return real GPU VA */
bad_handle:
kbase_gpu_vm_unlock(kctx);
no_aliased_array:
- kbase_mem_phy_alloc_put(reg->alloc);
+ kbase_mem_phy_alloc_put(reg->cpu_alloc);
+ kbase_mem_phy_alloc_put(reg->gpu_alloc);
no_alloc_obj:
kfree(reg);
no_reg:
return 0;
}
-int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, int handle, mali_addr64 *gpu_va, u64 *va_pages, u64 *flags)
+int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, int handle, u64 *gpu_va, u64 *va_pages, u64 *flags)
{
struct kbase_va_region *reg;
*flags |= BASE_MEM_SAME_VA;
#endif
+ if (!kbase_check_import_flags(*flags)) {
+ dev_warn(kctx->kbdev->dev,
+ "kbase_mem_import called with bad flags (%llx)",
+ (unsigned long long)*flags);
+ goto bad_flags;
+ }
+
switch (type) {
#ifdef CONFIG_UMP
case BASE_MEM_IMPORT_TYPE_UMP:
} else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) {
/* we control the VA, mmap now to the GPU */
- if (MALI_ERROR_NONE != kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1))
+ if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0)
goto no_gpu_va;
/* return real GPU VA */
*gpu_va = reg->start_pfn << PAGE_SHIFT;
} else {
/* we control the VA, but nothing to mmap yet */
- if (MALI_ERROR_NONE != kbase_add_va_region(kctx, reg, 0, *va_pages, 1))
+ if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0)
goto no_gpu_va;
/* return real GPU VA */
*gpu_va = reg->start_pfn << PAGE_SHIFT;
}
/* clear out private flags */
- *flags &= ((1UL << BASE_MEM_FLAGS_NR_TOTAL_BITS) - 1);
+ *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1);
kbase_gpu_vm_unlock(kctx);
no_gpu_va:
no_cookie:
kbase_gpu_vm_unlock(kctx);
- kbase_mem_phy_alloc_put(reg->alloc);
+ kbase_mem_phy_alloc_put(reg->cpu_alloc);
+ kbase_mem_phy_alloc_put(reg->gpu_alloc);
kfree(reg);
no_reg:
+bad_flags:
*gpu_va = 0;
*va_pages = 0;
*flags = 0;
return err;
}
-int kbase_mem_commit(struct kbase_context *kctx, mali_addr64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason)
+int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason)
{
u64 old_pages;
u64 delta;
goto out_unlock;
}
- KBASE_DEBUG_ASSERT(reg->alloc);
+ KBASE_DEBUG_ASSERT(reg->cpu_alloc);
+ KBASE_DEBUG_ASSERT(reg->gpu_alloc);
- if (reg->alloc->type != KBASE_MEM_TYPE_NATIVE) {
+ if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) {
*failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
goto out_unlock;
}
}
/* can't be mapped more than once on the GPU */
- if (atomic_read(®->alloc->gpu_mappings) > 1) {
+ if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) {
*failure_reason = BASE_BACKING_THRESHOLD_ERROR_NOT_GROWABLE;
goto out_unlock;
}
- if (new_pages == reg->alloc->nents) {
+ if (new_pages == reg->gpu_alloc->nents) {
/* no change */
res = 0;
goto out_unlock;
}
- phy_pages = kbase_get_phy_pages(reg);
+ phy_pages = kbase_get_gpu_phy_pages(reg);
old_pages = kbase_reg_current_backed_size(reg);
if (new_pages > old_pages) {
/* growing */
- mali_error err;
+ int err;
+
delta = new_pages - old_pages;
/* Allocate some more pages */
- if (MALI_ERROR_NONE != kbase_alloc_phy_pages_helper(reg->alloc, delta)) {
+ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) {
*failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
goto out_unlock;
}
- err = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages, phy_pages + old_pages, delta, reg->flags);
- if (MALI_ERROR_NONE != err) {
- kbase_free_phy_pages_helper(reg->alloc, delta);
+ if (reg->cpu_alloc != reg->gpu_alloc) {
+ if (kbase_alloc_phy_pages_helper(
+ reg->gpu_alloc, delta) != 0) {
+ *failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
+ kbase_free_phy_pages_helper(reg->cpu_alloc,
+ delta);
+ goto out_unlock;
+ }
+ }
+ err = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages,
+ phy_pages + old_pages, delta, reg->flags);
+ if (err) {
+ kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
+ if (reg->cpu_alloc != reg->gpu_alloc)
+ kbase_free_phy_pages_helper(reg->gpu_alloc,
+ delta);
*failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
goto out_unlock;
}
} else {
/* shrinking */
struct kbase_cpu_mapping *mapping;
- mali_error err;
+ int err;
/* first, unmap from any mappings affected */
- list_for_each_entry(mapping, ®->alloc->mappings, mappings_list) {
+ list_for_each_entry(mapping, ®->cpu_alloc->mappings, mappings_list) {
unsigned long mapping_size = (mapping->vm_end - mapping->vm_start) >> PAGE_SHIFT;
/* is this mapping affected ?*/
/* Free some pages */
delta = old_pages - new_pages;
- err = kbase_mmu_teardown_pages(kctx, reg->start_pfn + new_pages, delta);
- if (MALI_ERROR_NONE != err) {
+ err = kbase_mmu_teardown_pages(kctx, reg->start_pfn + new_pages,
+ delta);
+ if (err) {
*failure_reason = BASE_BACKING_THRESHOLD_ERROR_OOM;
goto out_unlock;
}
-
+#ifndef CONFIG_MALI_NO_MALI
if (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) {
/* Wait for GPU to flush write buffer before freeing physical pages */
kbase_wait_write_flush(kctx);
}
-
- kbase_free_phy_pages_helper(reg->alloc, delta);
+#endif
+ kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
+ if (reg->cpu_alloc != reg->gpu_alloc)
+ kbase_free_phy_pages_helper(reg->gpu_alloc, delta);
}
res = 0;
return res;
}
-STATIC void kbase_cpu_vm_open(struct vm_area_struct *vma)
+static void kbase_cpu_vm_open(struct vm_area_struct *vma)
{
struct kbase_cpu_mapping *map = vma->vm_private_data;
map->count++;
}
-STATIC void kbase_cpu_vm_close(struct vm_area_struct *vma)
+static void kbase_cpu_vm_close(struct vm_area_struct *vma)
{
struct kbase_cpu_mapping *map = vma->vm_private_data;
kbase_gpu_vm_lock(map->kctx);
if (map->region) {
- KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA);
- kbase_mem_free_region(map->kctx, map->region);
+ KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) ==
+ KBASE_REG_ZONE_SAME_VA);
+ /* Avoid freeing memory on the process death which results in
+ * GPU Page Fault. Memory will be freed in kbase_destroy_context
+ */
+ if (!(current->flags & PF_EXITING))
+ kbase_mem_free_region(map->kctx, map->region);
}
list_del(&map->mappings_list);
kfree(map);
}
-KBASE_EXPORT_TEST_API(kbase_cpu_vm_close)
+KBASE_EXPORT_TEST_API(kbase_cpu_vm_close);
-STATIC int kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+static int kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
struct kbase_cpu_mapping *map = vma->vm_private_data;
pgoff_t rel_pgoff;
vma->vm_ops = &kbase_vm_ops;
vma->vm_private_data = map;
- page_array = kbase_get_phy_pages(reg);
+ page_array = kbase_get_cpu_phy_pages(reg);
if (!(reg->flags & KBASE_REG_CPU_CACHED) &&
(reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) {
} else {
map->vm_end = vma->vm_end;
}
- map->alloc = kbase_mem_phy_alloc_get(reg->alloc);
+ map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
map->count = 1; /* start with one ref */
if (reg->flags & KBASE_REG_CPU_CACHED)
goto out_no_region;
}
- new_reg->alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB);
- if (IS_ERR_OR_NULL(new_reg->alloc)) {
+ new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB);
+ if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
err = -ENOMEM;
- new_reg->alloc = NULL;
+ new_reg->cpu_alloc = NULL;
WARN_ON(1);
goto out_no_alloc;
}
- new_reg->alloc->imported.kctx = kctx;
+ new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
+
+ new_reg->cpu_alloc->imported.kctx = kctx;
new_reg->flags &= ~KBASE_REG_FREE;
new_reg->flags |= KBASE_REG_CPU_CACHED;
/* alloc now owns the tb */
owns_tb = 0;
- if (MALI_ERROR_NONE != kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1)) {
+ if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
err = -ENOMEM;
WARN_ON(1);
goto out_no_va_region;
goto out;
}
- new_reg->alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW);
- if (IS_ERR_OR_NULL(new_reg->alloc)) {
+ new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW);
+ if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) {
err = -ENOMEM;
- new_reg->alloc = NULL;
+ new_reg->cpu_alloc = NULL;
WARN_ON(1);
goto out_no_alloc;
}
+ new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc);
+
new_reg->flags &= ~KBASE_REG_FREE;
new_reg->flags |= KBASE_REG_CPU_CACHED;
- if (MALI_ERROR_NONE != kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1)) {
+ if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) {
err = -ENOMEM;
WARN_ON(1);
goto out_va_region;
/* adjust down nr_pages to what we have physically */
nr_pages = kbase_reg_current_backed_size(reg);
- if (MALI_ERROR_NONE != kbase_gpu_mmap(kctx, reg,
- vma->vm_start +
- aligned_offset,
- reg->nr_pages,
- 1)) {
+ if (kbase_gpu_mmap(kctx, reg,
+ vma->vm_start + aligned_offset,
+ reg->nr_pages, 1) != 0) {
dev_err(dev, "%s:%d\n", __FILE__, __LINE__);
/* Unable to map in GPU space. */
WARN_ON(1);
}
#ifdef CONFIG_DMA_SHARED_BUFFER
- if (reg->alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM)
+ if (reg->cpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM)
goto dma_map;
#endif /* CONFIG_DMA_SHARED_BUFFER */
/* limit what we map to the amount currently backed */
- if (reg->alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) {
- if ((vma->vm_pgoff - reg->start_pfn) >= reg->alloc->nents)
+ if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) {
+ if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents)
nr_pages = 0;
else
- nr_pages = reg->alloc->nents - (vma->vm_pgoff - reg->start_pfn);
+ nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn);
}
goto map;
#ifdef CONFIG_DMA_SHARED_BUFFER
dma_map:
- err = dma_buf_mmap(reg->alloc->imported.umm.dma_buf, vma, vma->vm_pgoff - reg->start_pfn);
+ err = dma_buf_mmap(reg->cpu_alloc->imported.umm.dma_buf, vma, vma->vm_pgoff - reg->start_pfn);
#endif /* CONFIG_DMA_SHARED_BUFFER */
out_unlock:
kbase_gpu_vm_unlock(kctx);
return err;
}
-KBASE_EXPORT_TEST_API(kbase_mmap)
+KBASE_EXPORT_TEST_API(kbase_mmap);
-void *kbase_vmap(struct kbase_context *kctx, mali_addr64 gpu_addr, size_t size,
+void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size,
struct kbase_vmap_struct *map)
{
struct kbase_va_region *reg;
void *cpu_addr = NULL;
pgprot_t prot;
size_t i;
+ bool sync_needed;
if (!size || !map)
return NULL;
if (page_index + page_count > kbase_reg_current_backed_size(reg))
goto out_unlock;
- page_array = kbase_get_phy_pages(reg);
+ page_array = kbase_get_cpu_phy_pages(reg);
if (!page_array)
goto out_unlock;
goto out_unlock;
map->gpu_addr = gpu_addr;
- map->alloc = kbase_mem_phy_alloc_get(reg->alloc);
- map->pages = &kbase_get_phy_pages(reg)[page_index];
+ map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
+ map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index];
+ map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
+ map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index];
map->addr = (void *)((uintptr_t)cpu_addr + offset);
map->size = size;
map->is_cached = (reg->flags & KBASE_REG_CPU_CACHED) != 0;
+ sync_needed = map->is_cached;
- if (map->is_cached) {
+ if (sync_needed) {
/* Sync first page */
size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
- phys_addr_t pa = map->pages[0] + offset;
+ phys_addr_t cpu_pa = map->cpu_pages[0];
+ phys_addr_t gpu_pa = map->gpu_pages[0];
- kbase_sync_single(kctx, pa, sz, dma_sync_single_for_cpu);
+ kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
+ KBASE_SYNC_TO_CPU);
/* Sync middle pages (if any) */
for (i = 1; page_count > 2 && i < page_count - 1; i++) {
- pa = map->pages[i];
- kbase_sync_single(kctx, pa, PAGE_SIZE, dma_sync_single_for_cpu);
+ cpu_pa = map->cpu_pages[i];
+ gpu_pa = map->gpu_pages[i];
+ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
+ KBASE_SYNC_TO_CPU);
}
/* Sync last page (if any) */
if (page_count > 1) {
- pa = map->pages[page_count - 1];
+ cpu_pa = map->cpu_pages[page_count - 1];
+ gpu_pa = map->gpu_pages[page_count - 1];
sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
- kbase_sync_single(kctx, pa, sz, dma_sync_single_for_cpu);
+ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
+ KBASE_SYNC_TO_CPU);
}
}
-
kbase_gpu_vm_unlock(kctx);
return map->addr;
void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map)
{
void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK);
-
+ bool sync_needed = map->is_cached;
vunmap(addr);
-
- if (map->is_cached) {
+ if (sync_needed) {
off_t offset = (uintptr_t)map->addr & ~PAGE_MASK;
size_t size = map->size;
size_t page_count = PFN_UP(offset + size);
/* Sync first page */
size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
- phys_addr_t pa = map->pages[0] + offset;
+ phys_addr_t cpu_pa = map->cpu_pages[0];
+ phys_addr_t gpu_pa = map->gpu_pages[0];
- kbase_sync_single(kctx, pa, sz, dma_sync_single_for_device);
+ kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz,
+ KBASE_SYNC_TO_DEVICE);
/* Sync middle pages (if any) */
for (i = 1; page_count > 2 && i < page_count - 1; i++) {
- pa = map->pages[i];
- kbase_sync_single(kctx, pa, PAGE_SIZE, dma_sync_single_for_device);
+ cpu_pa = map->cpu_pages[i];
+ gpu_pa = map->gpu_pages[i];
+ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE,
+ KBASE_SYNC_TO_DEVICE);
}
/* Sync last page (if any) */
if (page_count > 1) {
- pa = map->pages[page_count - 1];
+ cpu_pa = map->cpu_pages[page_count - 1];
+ gpu_pa = map->gpu_pages[page_count - 1];
sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
- kbase_sync_single(kctx, pa, sz, dma_sync_single_for_device);
+ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz,
+ KBASE_SYNC_TO_DEVICE);
}
}
-
map->gpu_addr = 0;
- map->alloc = kbase_mem_phy_alloc_put(map->alloc);
- map->pages = NULL;
+ map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc);
+ map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc);
+ map->cpu_pages = NULL;
+ map->gpu_pages = NULL;
map->addr = NULL;
map->size = 0;
map->is_cached = false;
reg->flags &= ~KBASE_REG_FREE;
kbase_update_region_flags(reg, flags);
- reg->alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW);
- if (IS_ERR_OR_NULL(reg->alloc))
+ reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW);
+ if (IS_ERR_OR_NULL(reg->cpu_alloc))
goto no_alloc;
- page_array = kbase_get_phy_pages(reg);
+ reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
+
+ page_array = kbase_get_cpu_phy_pages(reg);
for (i = 0; i < pages; i++)
page_array[i] = dma_pa + (i << PAGE_SHIFT);
- reg->alloc->nents = pages;
+ reg->cpu_alloc->nents = pages;
kbase_gpu_vm_lock(kctx);
res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1);
return va;
no_mmap:
- kbase_mem_phy_alloc_put(reg->alloc);
+ kbase_mem_phy_alloc_put(reg->cpu_alloc);
+ kbase_mem_phy_alloc_put(reg->gpu_alloc);
no_alloc:
kfree(reg);
no_reg:
void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle)
{
struct kbase_va_region *reg;
- mali_error err;
+ int err;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
DEFINE_DMA_ATTRS(attrs);
#endif
KBASE_DEBUG_ASSERT(reg);
err = kbase_gpu_munmap(kctx, reg);
kbase_gpu_vm_unlock(kctx);
- KBASE_DEBUG_ASSERT(err == MALI_ERROR_NONE);
+ KBASE_DEBUG_ASSERT(!err);
- kbase_mem_phy_alloc_put(reg->alloc);
+ kbase_mem_phy_alloc_put(reg->cpu_alloc);
+ kbase_mem_phy_alloc_put(reg->gpu_alloc);
kfree(reg);
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010, 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define _KBASE_MEM_LINUX_H_
/** A HWC dump mapping */
-typedef struct kbase_hwc_dma_mapping {
+struct kbase_hwc_dma_mapping {
void *cpu_va;
dma_addr_t dma_pa;
size_t size;
-} kbase_hwc_dma_mapping;
+};
-struct kbase_va_region * kbase_mem_alloc(struct kbase_context * kctx, u64 va_pages, u64 commit_pages, u64 extent, u64 * flags, u64 * gpu_va, u16 * va_alignment);
-mali_error kbase_mem_query(struct kbase_context *kctx, mali_addr64 gpu_addr, int query, u64 * const pages);
-int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, int handle, mali_addr64 * gpu_va, u64 * va_pages, u64 * flags);
-u64 kbase_mem_alias(struct kbase_context *kctx, u64* flags, u64 stride, u64 nents, struct base_mem_aliasing_info* ai, u64 * num_pages);
-mali_error kbase_mem_flags_change(struct kbase_context *kctx, mali_addr64 gpu_addr, unsigned int flags, unsigned int mask);
-int kbase_mem_commit(struct kbase_context * kctx, mali_addr64 gpu_addr, u64 new_pages, enum base_backing_threshold_status * failure_reason);
+struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, u64 *gpu_va, u16 *va_alignment);
+int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 *const pages);
+int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, int handle, u64 *gpu_va, u64 *va_pages, u64 *flags);
+u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages);
+int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask);
+int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages, enum base_backing_threshold_status *failure_reason);
int kbase_mmap(struct file *file, struct vm_area_struct *vma);
struct kbase_vmap_struct {
- mali_addr64 gpu_addr;
- struct kbase_mem_phy_alloc *alloc;
- phys_addr_t *pages;
+ u64 gpu_addr;
+ struct kbase_mem_phy_alloc *cpu_alloc;
+ struct kbase_mem_phy_alloc *gpu_alloc;
+ phys_addr_t *cpu_pages;
+ phys_addr_t *gpu_pages;
void *addr;
size_t size;
bool is_cached;
};
-void *kbase_vmap(struct kbase_context *kctx, mali_addr64 gpu_addr, size_t size,
+void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size,
struct kbase_vmap_struct *map);
void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map);
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-#include <mali_kbase.h>
-
-#include <linux/io.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/dma-mapping.h>
-#include <linux/mutex.h>
-#include <asm/cacheflush.h>
-
-void kbase_sync_to_memory(phys_addr_t paddr, void *vaddr, size_t sz)
-{
-#ifdef CONFIG_ARM
- __cpuc_flush_dcache_area(vaddr, sz);
- outer_flush_range(paddr, paddr + sz);
-#elif defined(CONFIG_ARM64)
- /* TODO (MID64-46): There's no other suitable cache flush function for ARM64 */
- flush_cache_all();
-#elif defined(CONFIG_X86)
- struct scatterlist scl = { 0, };
- sg_set_page(&scl, pfn_to_page(PFN_DOWN(paddr)), sz, paddr & (PAGE_SIZE - 1));
- dma_sync_sg_for_cpu(NULL, &scl, 1, DMA_TO_DEVICE);
- mb(); /* for outer_sync (if needed) */
-#else
-#error Implement cache maintenance for your architecture here
-#endif
-}
-
-void kbase_sync_to_cpu(phys_addr_t paddr, void *vaddr, size_t sz)
-{
-#ifdef CONFIG_ARM
- __cpuc_flush_dcache_area(vaddr, sz);
- outer_flush_range(paddr, paddr + sz);
-#elif defined(CONFIG_ARM64)
- /* TODO (MID64-46): There's no other suitable cache flush function for ARM64 */
- flush_cache_all();
-#elif defined(CONFIG_X86)
- struct scatterlist scl = { 0, };
- sg_set_page(&scl, pfn_to_page(PFN_DOWN(paddr)), sz, paddr & (PAGE_SIZE - 1));
- dma_sync_sg_for_cpu(NULL, &scl, 1, DMA_FROM_DEVICE);
-#else
-#error Implement cache maintenance for your architecture here
-#endif
-}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define KBASE_PHY_PAGES_POISON_VALUE 0xFD /** Value to fill the memory with when KBASE_PHY_PAGES_FLAG_POISON is set */
-/**
- * A pointer to a cache synchronization function, either dma_sync_single_for_cpu
- * or dma_sync_single_for_device.
- */
-typedef void (*kbase_sync_kmem_fn) (struct device *, dma_addr_t, size_t size,
- enum dma_data_direction);
-
+enum kbase_sync_type {
+ KBASE_SYNC_TO_CPU,
+ KBASE_SYNC_TO_DEVICE
+};
#endif /* _KBASE_LOWLEVEL_H */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
* File operations related to debugfs entry for mem_profile
*/
-STATIC int kbasep_mem_profile_debugfs_open(struct inode *in, struct file *file)
+static int kbasep_mem_profile_debugfs_open(struct inode *in, struct file *file)
{
return single_open(file, kbasep_mem_profile_seq_show, in->i_private);
}
.release = single_release,
};
-mali_error kbasep_mem_profile_debugfs_add(struct kbase_context *kctx)
+void kbasep_mem_profile_debugfs_add(struct kbase_context *kctx)
{
- char name[KBASEP_DEBUGFS_FNAME_SIZE_MAX];
-
KBASE_DEBUG_ASSERT(kctx != NULL);
spin_lock_init(&kctx->mem_profile_lock);
- scnprintf(name, KBASEP_DEBUGFS_FNAME_SIZE_MAX, "%d_%d", kctx->pid,
- kctx->id);
-
- kctx->mem_dentry = debugfs_create_file(name, S_IRUGO,
- kctx->kbdev->memory_profile_directory,
- kctx, &kbasep_mem_profile_debugfs_fops);
- if (IS_ERR(kctx->mem_dentry))
- goto error_out;
-
- return MALI_ERROR_NONE;
-
-error_out:
- return MALI_ERROR_FUNCTION_FAILED;
+ debugfs_create_file("mem_profile", S_IRUGO, kctx->kctx_dentry, kctx,
+ &kbasep_mem_profile_debugfs_fops);
}
void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx)
kfree(kctx->mem_profile_data);
kctx->mem_profile_data = NULL;
spin_unlock(&kctx->mem_profile_lock);
-
- if (IS_ERR(kctx->mem_dentry))
- return;
- debugfs_remove(kctx->mem_dentry);
}
#else /* CONFIG_DEBUG_FS */
-/**
- * @brief Stub function for when debugfs is disabled
- */
-mali_error kbasep_mem_profile_debugfs_add(struct kbase_context *ctx)
-{
- return MALI_ERROR_NONE;
-}
-
-/**
- * @brief Stub function for when debugfs is disabled
- */
-void kbasep_mem_profile_debugfs_remove(struct kbase_context *ctx)
-{
-}
-
/**
* @brief Stub function for when debugfs is disabled
*/
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/**
* @brief Add new entry to Mali memory profile debugfs
*/
-mali_error kbasep_mem_profile_debugfs_add(struct kbase_context *kctx);
+void kbasep_mem_profile_debugfs_add(struct kbase_context *kctx);
/**
* @brief Remove entry from Mali memory profile debugfs
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* The size of the buffer to accumulate the histogram report text in
* @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT
*/
-#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ( ( size_t ) ( 64 + ( ( 80 + ( 56 * 64 ) ) * 15 ) + 56 ) )
+#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t) (64 + ((80 + (56 * 64)) * 15) + 56))
#endif /*_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
*/
/* #define DEBUG 1 */
+#include <linux/kernel.h>
#include <linux/dma-mapping.h>
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
#include <mali_kbase_gator.h>
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+#include <mali_kbase_tlstream.h>
+#endif
#include <mali_kbase_debug.h>
#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
#include <mali_kbase_defs.h>
#include <mali_kbase_hw.h>
#include <mali_kbase_mmu_hw.h>
+#include <mali_kbase_hwaccess_jm.h>
#define KBASE_MMU_PAGE_ENTRIES 512
+/**
+ * kbase_mmu_sync_pgd - sync page directory to memory
+ * @dev: Device pointer.
+ * @handle: Address of DMA region.
+ * @size: Size of the region to sync.
+ *
+ * This should be called after each page directory update.
+ */
+
+static void kbase_mmu_sync_pgd(struct device *dev,
+ dma_addr_t handle, size_t size)
+{
+ dma_sync_single_for_device(dev, handle, size, DMA_TO_DEVICE);
+}
+
/*
* Definitions:
* - PGD: Page Directory.
struct kbase_as *as, const char *reason_str);
-/* Helper Function to perform assignment of page table entries, to ensure the use of
- * strd, which is required on LPAE systems.
- */
-
-static inline void page_table_entry_set(struct kbase_device *kbdev, u64 *pte, u64 phy)
-{
-#ifdef CONFIG_64BIT
- *pte = phy;
-#elif defined(CONFIG_ARM)
- /*
- *
- * In order to prevent the compiler keeping cached copies of memory, we have to explicitly
- * say that we have updated memory.
- *
- * Note: We could manually move the data ourselves into R0 and R1 by specifying
- * register variables that are explicitly given registers assignments, the down side of
- * this is that we have to assume cpu endianess. To avoid this we can use the ldrd to read the
- * data from memory into R0 and R1 which will respect the cpu endianess, we then use strd to
- * make the 64 bit assignment to the page table entry.
- *
- */
-
- asm volatile("ldrd r0, r1, [%[ptemp]]\n\t"
- "strd r0, r1, [%[pte]]\n\t"
- : "=m" (*pte)
- : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy)
- : "r0", "r1");
-#else
-#error "64-bit atomic write must be implemented for your architecture"
-#endif
-}
-
static size_t make_multiple(size_t minimum, size_t multiple)
{
size_t remainder = minimum % multiple;
+
if (remainder == 0)
return minimum;
- else
- return minimum + multiple - remainder;
+
+ return minimum + multiple - remainder;
}
-static void page_fault_worker(struct work_struct *data)
+void page_fault_worker(struct work_struct *data)
{
u64 fault_pfn;
- u32 fault_access;
+ u32 fault_status;
size_t new_pages;
size_t fault_rel_pfn;
struct kbase_as *faulting_as;
struct kbase_context *kctx;
struct kbase_device *kbdev;
struct kbase_va_region *region;
- mali_error err;
+ int err;
+ bool grown = false;
faulting_as = container_of(data, struct kbase_as, work_pagefault);
fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT;
if (kctx == NULL) {
/* Only handle this if not already suspended */
if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
- struct kbase_mmu_setup *current_setup = &faulting_as->current_setup;
-
/* Address space has no context, terminate the work */
/* AS transaction begin */
mutex_lock(&faulting_as->transaction_mutex);
- /* Switch to unmapped mode */
- current_setup->transtab &= ~(u64)MMU_TRANSTAB_ADRMODE_MASK;
- current_setup->transtab |= AS_TRANSTAB_ADRMODE_UNMAPPED;
-
- /* Apply new address space settings */
- kbase_mmu_hw_configure(kbdev, faulting_as, kctx);
+ kbase_mmu_disable_as(kbdev, as_no);
mutex_unlock(&faulting_as->transaction_mutex);
/* AS transaction end */
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_pm_context_idle(kbdev);
}
+ atomic_dec(&kbdev->faults_pending);
return;
}
KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev);
- kbase_gpu_vm_lock(kctx);
+ fault_status = faulting_as->fault_status;
+ switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) {
- region = kbase_region_tracker_find_region_enclosing_address(kctx, faulting_as->fault_addr);
- if (NULL == region || region->flags & KBASE_REG_FREE) {
- kbase_gpu_vm_unlock(kctx);
+ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT:
+ /* need to check against the region to handle this one */
+ break;
+
+ case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT:
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
- "Memory is not mapped on the GPU");
+ "Permission failure");
+ goto fault_done;
+
+ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT:
+ kbase_mmu_report_fault_and_kill(kctx, faulting_as,
+ "Tranlation table bus fault");
+ goto fault_done;
+
+ case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG:
+ /* nothing to do, but we don't expect this fault currently */
+ dev_warn(kbdev->dev, "Access flag unexpectedly set");
+ goto fault_done;
+
+
+ default:
+ kbase_mmu_report_fault_and_kill(kctx, faulting_as,
+ "Unknown fault code");
goto fault_done;
}
- fault_access = faulting_as->fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK;
- if (((fault_access == AS_FAULTSTATUS_ACCESS_TYPE_READ) &&
- !(region->flags & KBASE_REG_GPU_RD)) ||
- ((fault_access == AS_FAULTSTATUS_ACCESS_TYPE_WRITE) &&
- !(region->flags & KBASE_REG_GPU_WR)) ||
- ((fault_access == AS_FAULTSTATUS_ACCESS_TYPE_EX) &&
- (region->flags & KBASE_REG_GPU_NX))) {
- dev_warn(kbdev->dev, "Access permissions don't match: region->flags=0x%lx", region->flags);
+ /* so we have a translation fault, let's see if it is for growable
+ * memory */
+ kbase_gpu_vm_lock(kctx);
+
+ region = kbase_region_tracker_find_region_enclosing_address(kctx,
+ faulting_as->fault_addr);
+ if (!region || region->flags & KBASE_REG_FREE) {
kbase_gpu_vm_unlock(kctx);
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
- "Access permissions mismatch");
+ "Memory is not mapped on the GPU");
goto fault_done;
}
- if (!(region->flags & GROWABLE_FLAGS_REQUIRED)) {
+ if ((region->flags & GROWABLE_FLAGS_REQUIRED)
+ != GROWABLE_FLAGS_REQUIRED) {
kbase_gpu_vm_unlock(kctx);
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Memory is not growable");
* transaction (which should cause the other page fault to be
* raised again).
*/
- kbase_mmu_hw_do_operation(kbdev, faulting_as, 0, 0, 0,
+ kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
AS_COMMAND_UNLOCK, 1);
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
/* See comment [1] about UNLOCK usage */
- kbase_mmu_hw_do_operation(kbdev, faulting_as, 0, 0, 0,
+ kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0,
AS_COMMAND_UNLOCK, 1);
kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx,
KBASE_MMU_FAULT_TYPE_PAGE);
goto fault_done;
}
- if (MALI_ERROR_NONE == kbase_alloc_phy_pages_helper(region->alloc, new_pages)) {
+ if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) {
+ if (region->gpu_alloc != region->cpu_alloc) {
+ if (kbase_alloc_phy_pages_helper(
+ region->cpu_alloc, new_pages) == 0) {
+ grown = true;
+ } else {
+ kbase_free_phy_pages_helper(region->gpu_alloc,
+ new_pages);
+ }
+ } else {
+ grown = true;
+ }
+ }
+
+
+ if (grown) {
u32 op;
/* alloc success */
mutex_lock(&faulting_as->transaction_mutex);
/* set up the new pages */
- err = kbase_mmu_insert_pages(kctx, region->start_pfn + kbase_reg_current_backed_size(region) - new_pages, &kbase_get_phy_pages(region)[kbase_reg_current_backed_size(region) - new_pages], new_pages, region->flags);
- if (MALI_ERROR_NONE != err) {
+ err = kbase_mmu_insert_pages(kctx, region->start_pfn + kbase_reg_current_backed_size(region) - new_pages, &kbase_get_gpu_phy_pages(region)[kbase_reg_current_backed_size(region) - new_pages], new_pages, region->flags);
+ if (err) {
/* failed to insert pages, handle as a normal PF */
mutex_unlock(&faulting_as->transaction_mutex);
- kbase_free_phy_pages_helper(region->alloc, new_pages);
+ kbase_free_phy_pages_helper(region->gpu_alloc, new_pages);
+ if (region->gpu_alloc != region->cpu_alloc)
+ kbase_free_phy_pages_helper(region->cpu_alloc,
+ new_pages);
kbase_gpu_vm_unlock(kctx);
/* The locked VA region will be unlocked and the cache invalidated in here */
kbase_mmu_report_fault_and_kill(kctx, faulting_as,
"Page table update failure");
goto fault_done;
}
-#ifdef CONFIG_MALI_GATOR_SUPPORT
+#if defined(CONFIG_MALI_GATOR_SUPPORT)
kbase_trace_mali_page_fault_insert_pages(as_no, new_pages);
-#endif /* CONFIG_MALI_GATOR_SUPPORT */
+#endif
+#if defined(CONFIG_MALI_MIPE_ENABLED)
+ kbase_tlstream_aux_pagefault(
+ as_no,
+ atomic_read(&kctx->used_pages));
+#endif
/* flush L2 and unlock the VA (resumes the MMU) */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367))
* so release the ctx refcount
*/
kbasep_js_runpool_release_ctx(kbdev, kctx);
+
+ atomic_dec(&kbdev->faults_pending);
}
phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx)
kbase_atomic_add_pages(1, &kctx->used_pages);
kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages);
- if (MALI_ERROR_NONE != kbase_mem_allocator_alloc(kctx->pgd_allocator, 1, &pgd))
+ if (kbase_mem_allocator_alloc(kctx->pgd_allocator, 1, &pgd) != 0)
goto sub_pages;
p = pfn_to_page(PFN_DOWN(pgd));
kbase_process_page_usage_inc(kctx, 1);
for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++)
- page_table_entry_set(kctx->kbdev, &page[i], ENTRY_IS_INVAL);
+ kctx->kbdev->mmu_mode->entry_invalidate(&page[i]);
+
+ kbase_mmu_sync_pgd(kctx->kbdev->dev, kbase_dma_addr(p), PAGE_SIZE);
- /* Clean the full page */
- dma_sync_single_for_device(kctx->kbdev->dev,
- kbase_dma_addr(p),
- PAGE_SIZE,
- DMA_TO_DEVICE);
kunmap(pfn_to_page(PFN_DOWN(pgd)));
return pgd;
alloc_free:
- kbase_mem_allocator_free(kctx->pgd_allocator, 1, &pgd, MALI_FALSE);
+ kbase_mem_allocator_free(kctx->pgd_allocator, 1, &pgd, false);
sub_pages:
kbase_atomic_sub_pages(1, &kctx->used_pages);
kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
return 0;
}
-KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd)
-
-static phys_addr_t mmu_pte_to_phy_addr(u64 entry)
-{
- if (!(entry & 1))
- return 0;
-
- return entry & ~0xFFF;
-}
-
-static u64 mmu_phyaddr_to_pte(phys_addr_t phy)
-{
- return (phy & ~0xFFF) | ENTRY_IS_PTE;
-}
-
-static u64 mmu_phyaddr_to_ate(phys_addr_t phy, u64 flags)
-{
- return (phy & ~0xFFF) | (flags & ENTRY_FLAGS_MASK) | ENTRY_IS_ATE;
-}
+KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd);
/* Given PGD PFN for level N, return PGD PFN for level N+1 */
static phys_addr_t mmu_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level)
return 0;
}
- target_pgd = mmu_pte_to_phy_addr(page[vpfn]);
+ target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
if (!target_pgd) {
target_pgd = kbase_mmu_alloc_pgd(kctx);
return 0;
}
- page_table_entry_set(kctx->kbdev, &page[vpfn],
- mmu_phyaddr_to_pte(target_pgd));
+ kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd);
- dma_sync_single_for_device(kctx->kbdev->dev,
- kbase_dma_addr(p),
- PAGE_SIZE,
- DMA_TO_DEVICE);
+ kbase_mmu_sync_pgd(kctx->kbdev->dev,
+ kbase_dma_addr(p), PAGE_SIZE);
/* Rely on the caller to update the address space flags. */
}
/* kmap_atomic should NEVER fail */
KBASE_DEBUG_ASSERT(NULL != page);
- target_pgd = mmu_pte_to_phy_addr(page[vpfn]);
+ target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]);
/* As we are recovering from what has already been set up, we should have a target_pgd */
KBASE_DEBUG_ASSERT(0 != target_pgd);
-
kunmap_atomic(page);
return target_pgd;
}
{
phys_addr_t pgd;
u64 *pgd_page;
+ struct kbase_mmu_mode const *mmu_mode;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(0 != vpfn);
/* 64-bit address range is the max */
- KBASE_DEBUG_ASSERT(vpfn <= (UINT64_MAX / PAGE_SIZE));
+ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
lockdep_assert_held(&kctx->reg_lock);
+ mmu_mode = kctx->kbdev->mmu_mode;
+
while (nr) {
unsigned int i;
unsigned int index = vpfn & 0x1FF;
/* Invalidate the entries we added */
for (i = 0; i < count; i++)
- page_table_entry_set(kctx->kbdev, &pgd_page[index + i],
- ENTRY_IS_INVAL);
+ mmu_mode->entry_invalidate(&pgd_page[index + i]);
vpfn += count;
nr -= count;
- dma_sync_single_for_device(kctx->kbdev->dev,
+ kbase_mmu_sync_pgd(kctx->kbdev->dev,
kbase_dma_addr(p),
- PAGE_SIZE, DMA_TO_DEVICE);
- kunmap_atomic(pgd_page);
- }
-}
+ PAGE_SIZE);
-/**
- * Map KBASE_REG flags to MMU flags
- */
-static u64 kbase_mmu_get_mmu_flags(unsigned long flags)
-{
- u64 mmu_flags;
-
- /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */
- mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2;
-
- /* write perm if requested */
- mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0;
- /* read perm if requested */
- mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0;
- /* nx if requested */
- mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0;
-
- if (flags & KBASE_REG_SHARE_BOTH) {
- /* inner and outer shareable */
- mmu_flags |= SHARE_BOTH_BITS;
- } else if (flags & KBASE_REG_SHARE_IN) {
- /* inner shareable coherency */
- mmu_flags |= SHARE_INNER_BITS;
+ kunmap_atomic(pgd_page);
}
-
- return mmu_flags;
}
/*
* Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn'
*/
-mali_error kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn,
+int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn,
phys_addr_t phys, size_t nr,
unsigned long flags)
{
phys_addr_t pgd;
u64 *pgd_page;
- u64 pte_entry;
/* In case the insert_single_page only partially completes we need to be
* able to recover */
- mali_bool recover_required = MALI_FALSE;
+ bool recover_required = false;
u64 recover_vpfn = vpfn;
size_t recover_count = 0;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(0 != vpfn);
/* 64-bit address range is the max */
- KBASE_DEBUG_ASSERT(vpfn <= (UINT64_MAX / PAGE_SIZE));
+ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
lockdep_assert_held(&kctx->reg_lock);
- /* the one entry we'll populate everywhere */
- pte_entry = mmu_phyaddr_to_ate(phys, kbase_mmu_get_mmu_flags(flags));
-
while (nr) {
unsigned int i;
unsigned int index = vpfn & 0x1FF;
*/
pgd = mmu_get_bottom_pgd(kctx, vpfn);
if (!pgd) {
- dev_warn(kctx->kbdev->dev,
- "kbase_mmu_insert_pages: "
- "mmu_get_bottom_pgd failure\n");
+ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
if (recover_required) {
/* Invalidate the pages we have partially
* completed */
recover_vpfn,
recover_count);
}
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
p = pfn_to_page(PFN_DOWN(pgd));
pgd_page = kmap(p);
if (!pgd_page) {
- dev_warn(kctx->kbdev->dev,
- "kbase_mmu_insert_pages: "
- "kmap failure\n");
+ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
if (recover_required) {
/* Invalidate the pages we have partially
* completed */
recover_vpfn,
recover_count);
}
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
}
for (i = 0; i < count; i++) {
unsigned int ofs = index + i;
+
KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
- page_table_entry_set(kctx->kbdev, &pgd_page[ofs],
- pte_entry);
+ kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
+ phys, flags);
}
vpfn += count;
nr -= count;
- dma_sync_single_for_device(kctx->kbdev->dev,
+ kbase_mmu_sync_pgd(kctx->kbdev->dev,
kbase_dma_addr(p) +
(index * sizeof(u64)),
- count * sizeof(u64),
- DMA_TO_DEVICE);
-
+ count * sizeof(u64));
kunmap(p);
/* We have started modifying the page table.
* If further pages need inserting and fail we need to undo what
* has already taken place */
- recover_required = MALI_TRUE;
+ recover_required = true;
recover_count += count;
}
- return MALI_ERROR_NONE;
+ return 0;
}
/*
* Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn'
*/
-mali_error kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn,
+int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn,
phys_addr_t *phys, size_t nr,
unsigned long flags)
{
phys_addr_t pgd;
u64 *pgd_page;
- u64 mmu_flags = 0;
/* In case the insert_pages only partially completes we need to be able
* to recover */
- mali_bool recover_required = MALI_FALSE;
+ bool recover_required = false;
u64 recover_vpfn = vpfn;
size_t recover_count = 0;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(0 != vpfn);
/* 64-bit address range is the max */
- KBASE_DEBUG_ASSERT(vpfn <= (UINT64_MAX / PAGE_SIZE));
+ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
lockdep_assert_held(&kctx->reg_lock);
- mmu_flags = kbase_mmu_get_mmu_flags(flags);
-
while (nr) {
unsigned int i;
unsigned int index = vpfn & 0x1FF;
*/
pgd = mmu_get_bottom_pgd(kctx, vpfn);
if (!pgd) {
- dev_warn(kctx->kbdev->dev,
- "kbase_mmu_insert_pages: "
- "mmu_get_bottom_pgd failure\n");
+ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n");
if (recover_required) {
/* Invalidate the pages we have partially
* completed */
recover_vpfn,
recover_count);
}
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
p = pfn_to_page(PFN_DOWN(pgd));
pgd_page = kmap(p);
if (!pgd_page) {
- dev_warn(kctx->kbdev->dev,
- "kbase_mmu_insert_pages: "
- "kmap failure\n");
+ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n");
if (recover_required) {
/* Invalidate the pages we have partially
* completed */
recover_vpfn,
recover_count);
}
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
}
for (i = 0; i < count; i++) {
unsigned int ofs = index + i;
+
KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL));
- page_table_entry_set(kctx->kbdev, &pgd_page[ofs],
- mmu_phyaddr_to_ate(phys[i],
- mmu_flags)
- );
+ kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs],
+ phys[i], flags);
}
phys += count;
vpfn += count;
nr -= count;
- dma_sync_single_for_device(kctx->kbdev->dev,
+ kbase_mmu_sync_pgd(kctx->kbdev->dev,
kbase_dma_addr(p) +
(index * sizeof(u64)),
- count * sizeof(u64),
- DMA_TO_DEVICE);
+ count * sizeof(u64));
kunmap(p);
/* We have started modifying the page table. If further pages
* need inserting and fail we need to undo what has already
* taken place */
- recover_required = MALI_TRUE;
+ recover_required = true;
recover_count += count;
}
- return MALI_ERROR_NONE;
+ return 0;
}
-KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages)
+KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages);
/**
* This function is responsible for validating the MMU PTs
static void kbase_mmu_flush(struct kbase_context *kctx, u64 vpfn, size_t nr)
{
struct kbase_device *kbdev;
- mali_bool ctx_is_in_runpool;
+ bool ctx_is_in_runpool;
KBASE_DEBUG_ASSERT(NULL != kctx);
/* Second level check is to try to only do this when jobs are running. The refcount is
* a heuristic for this. */
if (kbdev->js_data.runpool_irq.per_as_data[kctx->as_nr].as_busy_refcount >= 2) {
- int ret;
- u32 op;
-
- /* AS transaction begin */
- mutex_lock(&kbdev->as[kctx->as_nr].transaction_mutex);
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367))
- op = AS_COMMAND_FLUSH;
- else
- op = AS_COMMAND_FLUSH_MEM;
-
- ret = kbase_mmu_hw_do_operation(kbdev,
+ if (!kbase_pm_context_active_handle_suspend(kbdev,
+ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
+ int ret;
+ u32 op;
+
+ /* AS transaction begin */
+ mutex_lock(&kbdev->as[
+ kctx->as_nr].transaction_mutex);
+
+ if (kbase_hw_has_issue(kbdev,
+ BASE_HW_ISSUE_6367))
+ op = AS_COMMAND_FLUSH;
+ else
+ op = AS_COMMAND_FLUSH_MEM;
+
+ ret = kbase_mmu_hw_do_operation(kbdev,
&kbdev->as[kctx->as_nr],
kctx, vpfn, nr,
op, 0);
#if KBASE_GPU_RESET_EN
- if (ret) {
- /* Flush failed to complete, assume the GPU has hung and perform a reset to recover */
- dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n");
- if (kbase_prepare_to_reset_gpu(kbdev))
- kbase_reset_gpu(kbdev);
- }
+ if (ret) {
+ /* Flush failed to complete, assume the
+ * GPU has hung and perform a reset to
+ * recover */
+ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n");
+ if (kbase_prepare_to_reset_gpu(kbdev))
+ kbase_reset_gpu(kbdev);
+ }
#endif /* KBASE_GPU_RESET_EN */
- mutex_unlock(&kbdev->as[kctx->as_nr].transaction_mutex);
- /* AS transaction end */
+ mutex_unlock(&kbdev->as[
+ kctx->as_nr].transaction_mutex);
+ /* AS transaction end */
+
+ kbase_pm_context_idle(kbdev);
+ }
}
kbasep_js_runpool_release_ctx(kbdev, kctx);
}
* already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
* information.
*/
-mali_error kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr)
+int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr)
{
phys_addr_t pgd;
u64 *pgd_page;
struct kbase_device *kbdev;
size_t requested_nr = nr;
+ struct kbase_mmu_mode const *mmu_mode;
KBASE_DEBUG_ASSERT(NULL != kctx);
beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr);
if (0 == nr) {
/* early out if nothing to do */
- return MALI_ERROR_NONE;
+ return 0;
}
kbdev = kctx->kbdev;
+ mmu_mode = kbdev->mmu_mode;
while (nr) {
unsigned int i;
unsigned int index = vpfn & 0x1FF;
unsigned int count = KBASE_MMU_PAGE_ENTRIES - index;
struct page *p;
+
if (count > nr)
count = nr;
pgd = mmu_get_bottom_pgd(kctx, vpfn);
if (!pgd) {
dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: mmu_get_bottom_pgd failure\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
p = pfn_to_page(PFN_DOWN(pgd));
pgd_page = kmap(p);
if (!pgd_page) {
dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: kmap failure\n");
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
}
- for (i = 0; i < count; i++) {
- page_table_entry_set(kctx->kbdev, &pgd_page[index + i], ENTRY_IS_INVAL);
- }
+ for (i = 0; i < count; i++)
+ mmu_mode->entry_invalidate(&pgd_page[index + i]);
vpfn += count;
nr -= count;
- dma_sync_single_for_device(kctx->kbdev->dev,
+ kbase_mmu_sync_pgd(kctx->kbdev->dev,
kbase_dma_addr(p) +
(index * sizeof(u64)),
- count * sizeof(u64),
- DMA_TO_DEVICE);
+ count * sizeof(u64));
kunmap(p);
}
kbase_mmu_flush(kctx, vpfn, requested_nr);
- return MALI_ERROR_NONE;
+ return 0;
}
-KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages)
+KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages);
/**
* Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'.
* already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more
* information.
*/
-mali_error kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t* phys, size_t nr, unsigned long flags)
+int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags)
{
phys_addr_t pgd;
- u64* pgd_page;
- u64 mmu_flags = 0;
+ u64 *pgd_page;
size_t requested_nr = nr;
+ struct kbase_mmu_mode const *mmu_mode;
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(0 != vpfn);
- KBASE_DEBUG_ASSERT(vpfn <= (UINT64_MAX / PAGE_SIZE));
+ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE));
lockdep_assert_held(&kctx->reg_lock);
- mmu_flags = kbase_mmu_get_mmu_flags(flags);
+ mmu_mode = kctx->kbdev->mmu_mode;
- dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags "\
- "on GPU PFN 0x%llx from phys %p, %zu pages",
+ dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages",
vpfn, phys, nr);
-
- while(nr) {
+ while (nr) {
unsigned int i;
unsigned int index = vpfn & 0x1FF;
size_t count = KBASE_MMU_PAGE_ENTRIES - index;
pgd = mmu_get_bottom_pgd(kctx, vpfn);
if (!pgd) {
dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd failure\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
p = pfn_to_page(PFN_DOWN(pgd));
pgd_page = kmap(p);
if (!pgd_page) {
dev_warn(kctx->kbdev->dev, "kmap failure\n");
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
}
- for (i = 0; i < count; i++) {
- page_table_entry_set(kctx->kbdev, &pgd_page[index + i], mmu_phyaddr_to_ate(phys[i], mmu_flags));
- }
+ for (i = 0; i < count; i++)
+ mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i],
+ flags);
phys += count;
vpfn += count;
nr -= count;
- dma_sync_single_for_device(kctx->kbdev->dev,
+ kbase_mmu_sync_pgd(kctx->kbdev->dev,
kbase_dma_addr(p) +
(index * sizeof(u64)),
- count * sizeof(u64),
- DMA_TO_DEVICE);
+ count * sizeof(u64));
kunmap(pfn_to_page(PFN_DOWN(pgd)));
}
kbase_mmu_flush(kctx, vpfn, requested_nr);
- return MALI_ERROR_NONE;
-}
-
-static int mmu_pte_is_valid(u64 pte)
-{
- return ((pte & 3) == ENTRY_IS_ATE);
+ return 0;
}
/* This is a debug feature only */
KBASE_DEBUG_ASSERT(NULL != page);
for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
- if (mmu_pte_is_valid(page[i]))
+ if (kctx->kbdev->mmu_mode->ate_is_valid(page[i]))
beenthere(kctx, "live pte %016lx", (unsigned long)page[i]);
}
kunmap_atomic(page);
phys_addr_t target_pgd;
u64 *pgd_page;
int i;
+ struct kbase_mmu_mode const *mmu_mode;
KBASE_DEBUG_ASSERT(NULL != kctx);
lockdep_assert_held(&kctx->reg_lock);
kunmap_atomic(pgd_page);
pgd_page = pgd_page_buffer;
+ mmu_mode = kctx->kbdev->mmu_mode;
+
for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
- target_pgd = mmu_pte_to_phy_addr(pgd_page[i]);
+ target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]);
if (target_pgd) {
if (level < 2) {
beenthere(kctx, "pte %lx level %d", (unsigned long)target_pgd, level + 1);
if (zap) {
- kbase_mem_allocator_free(kctx->pgd_allocator, 1, &target_pgd, MALI_TRUE);
+ kbase_mem_allocator_free(kctx->pgd_allocator, 1, &target_pgd, true);
kbase_process_page_usage_dec(kctx, 1);
kbase_atomic_sub_pages(1, &kctx->used_pages);
kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
}
}
-mali_error kbase_mmu_init(struct kbase_context *kctx)
+int kbase_mmu_init(struct kbase_context *kctx)
{
KBASE_DEBUG_ASSERT(NULL != kctx);
KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages);
/* Preallocate MMU depth of four pages for mmu_teardown_level to use */
kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL);
- kctx->mem_attrs = (AS_MEMATTR_IMPL_DEF_CACHE_POLICY <<
- (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) |
- (AS_MEMATTR_FORCE_TO_CACHE_ALL <<
- (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) |
- (AS_MEMATTR_WRITE_ALLOC <<
- (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) |
- 0; /* The other indices are unused for now */
-
if (NULL == kctx->mmu_teardown_pages)
- return MALI_ERROR_OUT_OF_MEMORY;
+ return -ENOMEM;
- return MALI_ERROR_NONE;
+ return 0;
}
void kbase_mmu_term(struct kbase_context *kctx)
mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, 1, kctx->mmu_teardown_pages);
beenthere(kctx, "pgd %lx", (unsigned long)kctx->pgd);
- kbase_mem_allocator_free(kctx->pgd_allocator, 1, &kctx->pgd, MALI_TRUE);
+ kbase_mem_allocator_free(kctx->pgd_allocator, 1, &kctx->pgd, true);
kbase_process_page_usage_dec(kctx, 1);
kbase_atomic_sub_pages(1, &kctx->used_pages);
kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages);
}
-KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd)
+KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd);
static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left)
{
int i;
size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64);
size_t dump_size;
+ struct kbase_mmu_mode const *mmu_mode;
KBASE_DEBUG_ASSERT(NULL != kctx);
lockdep_assert_held(&kctx->reg_lock);
+ mmu_mode = kctx->kbdev->mmu_mode;
+
pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd)));
if (!pgd_page) {
dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n");
}
for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) {
- if ((pgd_page[i] & ENTRY_IS_PTE) == ENTRY_IS_PTE) {
- target_pgd = mmu_pte_to_phy_addr(pgd_page[i]);
+ if (mmu_mode->pte_is_valid(pgd_page[i])) {
+ target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]);
dump_size = kbasep_mmu_dump_level(kctx, target_pgd, level + 1, buffer, size_left);
if (!dump_size) {
if (kaddr) {
u64 end_marker = 0xFFULL;
char *buffer = (char *)kaddr;
-
size_t size = kbasep_mmu_dump_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, &buffer, &size_left);
+
if (!size) {
vfree(kaddr);
return NULL;
return kaddr;
}
-KBASE_EXPORT_TEST_API(kbase_mmu_dump)
+KBASE_EXPORT_TEST_API(kbase_mmu_dump);
-static void bus_fault_worker(struct work_struct *data)
+void bus_fault_worker(struct work_struct *data)
{
struct kbase_as *faulting_as;
int as_no;
struct kbase_context *kctx;
struct kbase_device *kbdev;
#if KBASE_GPU_RESET_EN
- mali_bool reset_status = MALI_FALSE;
+ bool reset_status = false;
#endif /* KBASE_GPU_RESET_EN */
faulting_as = container_of(data, struct kbase_as, work_busfault);
#endif /* KBASE_GPU_RESET_EN */
/* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */
if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
- struct kbase_mmu_setup *current_setup = &faulting_as->current_setup;
/* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
/* AS transaction begin */
mutex_lock(&kbdev->as[as_no].transaction_mutex);
/* Set the MMU into unmapped mode */
- current_setup->transtab &= ~(u64)MMU_TRANSTAB_ADRMODE_MASK;
- current_setup->transtab |= AS_TRANSTAB_ADRMODE_UNMAPPED;
-
- /* Apply the new settings */
- kbase_mmu_hw_configure(kbdev, faulting_as, kctx);
+ kbase_mmu_disable_as(kbdev, as_no);
mutex_unlock(&kbdev->as[as_no].transaction_mutex);
/* AS transaction end */
/* By this point, the fault was handled in some way, so release the ctx refcount */
if (kctx != NULL)
kbasep_js_runpool_release_ctx(kbdev, kctx);
+
+ atomic_dec(&kbdev->faults_pending);
}
-const char *kbase_exception_name(u32 exception_code)
+const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code)
{
const char *e;
case 0xD8:
e = "ACCESS_FLAG";
break;
+ break;
default:
e = "UNKNOWN";
break;
return e;
}
+static const char *access_type_name(struct kbase_device *kbdev,
+ u32 fault_status)
+{
+ switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
+ return "UNKNOWN";
+ case AS_FAULTSTATUS_ACCESS_TYPE_READ:
+ return "READ";
+ case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
+ return "WRITE";
+ case AS_FAULTSTATUS_ACCESS_TYPE_EX:
+ return "EXECUTE";
+ default:
+ KBASE_DEBUG_ASSERT(0);
+ return NULL;
+ }
+}
+
/**
* The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on.
*/
int source_id;
int as_no;
struct kbase_device *kbdev;
- struct kbase_mmu_setup *current_setup;
struct kbasep_js_device_data *js_devdata;
#if KBASE_GPU_RESET_EN
- mali_bool reset_status = MALI_FALSE;
+ bool reset_status = false;
#endif
- static const char * const access_type_names[] = { "RESERVED", "EXECUTE", "READ", "WRITE" };
as_no = as->number;
kbdev = kctx->kbdev;
"decoded fault status: %s\n"
"exception type 0x%X: %s\n"
"access type 0x%X: %s\n"
- "source id 0x%X\n",
+ "source id 0x%X\n"
+ "pid: %d\n",
as_no, as->fault_addr,
reason_str,
as->fault_status,
(as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
- exception_type, kbase_exception_name(exception_type),
- access_type, access_type_names[access_type],
- source_id);
+ exception_type, kbase_exception_name(kbdev, exception_type),
+ access_type, access_type_name(kbdev, as->fault_status),
+ source_id,
+ kctx->pid);
/* hardware counters dump fault handling */
- if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && (kbdev->hwcnt.state == KBASE_INSTR_STATE_DUMPING)) {
+ if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) &&
+ (kbdev->hwcnt.backend.state ==
+ KBASE_INSTR_STATE_DUMPING)) {
unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
- if ((as->fault_addr >= kbdev->hwcnt.addr) && (as->fault_addr < (kbdev->hwcnt.addr + (num_core_groups * 2048))))
- kbdev->hwcnt.state = KBASE_INSTR_STATE_FAULT;
+
+ if ((as->fault_addr >= kbdev->hwcnt.addr) &&
+ (as->fault_addr < (kbdev->hwcnt.addr +
+ (num_core_groups * 2048))))
+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT;
}
/* Stop the kctx from submitting more jobs and cause it to be scheduled
/* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this
* context can appear in the job slots from this point on */
- kbase_job_kill_jobs_from_context(kctx);
+ kbase_backend_jm_kill_jobs_from_kctx(kctx);
/* AS transaction begin */
mutex_lock(&as->transaction_mutex);
#if KBASE_GPU_RESET_EN
}
#endif /* KBASE_GPU_RESET_EN */
/* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */
- current_setup = &as->current_setup;
-
- current_setup->transtab &= ~(u64)MMU_TRANSTAB_ADRMODE_MASK;
- current_setup->transtab |= AS_TRANSTAB_ADRMODE_UNMAPPED;
-
- /* Apply the new address space setting */
- kbase_mmu_hw_configure(kbdev, as, kctx);
+ kbase_mmu_disable_as(kbdev, as_no);
mutex_unlock(&as->transaction_mutex);
/* AS transaction end */
-
/* Clear down the fault */
kbase_mmu_hw_clear_fault(kbdev, as, kctx, KBASE_MMU_FAULT_TYPE_PAGE);
kbase_mmu_hw_enable_fault(kbdev, as, kctx, KBASE_MMU_FAULT_TYPE_PAGE);
hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL);
}
spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
-
}
enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer)
void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
struct kbase_as *as;
+
KBASE_DEBUG_ASSERT(kbdev);
KBASE_DEBUG_ASSERT(kctx);
KBASE_DEBUG_ASSERT(katom);
void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
- unsigned long flags;
+
+ lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
if (kctx == NULL) {
dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n",
* other thread that it failed
*/
if ((kbdev->hwcnt.kctx == kctx) &&
- (kbdev->hwcnt.state == KBASE_INSTR_STATE_DUMPING))
- kbdev->hwcnt.state = KBASE_INSTR_STATE_FAULT;
+ (kbdev->hwcnt.backend.state ==
+ KBASE_INSTR_STATE_DUMPING))
+ kbdev->hwcnt.backend.state =
+ KBASE_INSTR_STATE_FAULT;
/*
* Stop the kctx from submitting more jobs and cause it
* to be scheduled out/rescheduled when all references
* to it are released
*/
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
kbasep_js_clear_submit_allowed(js_devdata, kctx);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
- flags);
dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n",
- as->number, as->fault_addr);
+ as->number, as->fault_addr);
+
}
/*
*/
kbdev->kbase_group_error++;
KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault));
+ WARN_ON(work_pending(&as->work_busfault));
INIT_WORK(&as->work_busfault, bus_fault_worker);
queue_work(as->pf_wq, &as->work_busfault);
+ atomic_inc(&kbdev->faults_pending);
} else {
kbdev->kbase_group_error++;
KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault));
+ WARN_ON(work_pending(&as->work_pagefault));
INIT_WORK(&as->work_pagefault, page_fault_worker);
queue_work(as->pf_wq, &as->work_pagefault);
+ atomic_inc(&kbdev->faults_pending);
+ }
+}
+
+void kbase_flush_mmu_wqs(struct kbase_device *kbdev)
+{
+ int i;
+
+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
+ struct kbase_as *as = &kbdev->as[i];
+
+ flush_workqueue(as->pf_wq);
}
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-#include <linux/bitops.h>
-
-#include <mali_kbase.h>
-#include <mali_kbase_mem.h>
-#include <mali_kbase_mmu_hw.h>
-#include <mali_kbase_mmu_hw_direct.h>
-
-#if KBASE_MMU_HW_BACKEND
-
-static inline u64 lock_region(struct kbase_device *kbdev, u64 pfn,
- u32 num_pages)
-{
- u64 region;
-
- /* can't lock a zero sized range */
- KBASE_DEBUG_ASSERT(num_pages);
-
- region = pfn << PAGE_SHIFT;
- /*
- * fls returns (given the ASSERT above):
- * 1 .. 32
- *
- * 10 + fls(num_pages)
- * results in the range (11 .. 42)
- */
-
- /* gracefully handle num_pages being zero */
- if (0 == num_pages) {
- region |= 11;
- } else {
- u8 region_width;
-
- region_width = 10 + fls(num_pages);
- if (num_pages != (1ul << (region_width - 11))) {
- /* not pow2, so must go up to the next pow2 */
- region_width += 1;
- }
- KBASE_DEBUG_ASSERT(region_width <= KBASE_LOCK_REGION_MAX_SIZE);
- KBASE_DEBUG_ASSERT(region_width >= KBASE_LOCK_REGION_MIN_SIZE);
- region |= region_width;
- }
-
- return region;
-}
-
-static int wait_ready(struct kbase_device *kbdev,
- unsigned int as_nr, struct kbase_context *kctx)
-{
- unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
-
- /* Wait for the MMU status to indicate there is no active command. */
- while (--max_loops && kbase_reg_read(kbdev,
- MMU_AS_REG(as_nr, AS_STATUS),
- kctx) & AS_STATUS_AS_ACTIVE) {
- ;
- }
-
- if (max_loops == 0) {
- dev_err(kbdev->dev, "AS_ACTIVE bit stuck\n");
- return -1;
- }
-
- return 0;
-}
-
-static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd,
- struct kbase_context *kctx)
-{
- int status;
-
- /* write AS_COMMAND when MMU is ready to accept another command */
- status = wait_ready(kbdev, as_nr, kctx);
- if (status == 0)
- kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd, kctx);
-
- return status;
-}
-
-void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat)
-{
- const int num_as = 16;
- const int busfault_shift = MMU_PAGE_FAULT_FLAGS;
- const int pf_shift = 0;
- const unsigned long as_bit_mask = (1UL << num_as) - 1;
- unsigned long flags;
- u32 new_mask;
- u32 tmp;
-
- /* bus faults */
- u32 bf_bits = (irq_stat >> busfault_shift) & as_bit_mask;
- /* page faults (note: Ignore ASes with both pf and bf) */
- u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits;
-
- KBASE_DEBUG_ASSERT(NULL != kbdev);
-
- /* remember current mask */
- spin_lock_irqsave(&kbdev->mmu_mask_change, flags);
- new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL);
- /* mask interrupts for now */
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL);
- spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
-
- while (bf_bits | pf_bits) {
- struct kbase_as *as;
- int as_no;
- struct kbase_context *kctx;
-
- /*
- * the while logic ensures we have a bit set, no need to check
- * for not-found here
- */
- as_no = ffs(bf_bits | pf_bits) - 1;
- as = &kbdev->as[as_no];
-
- /*
- * Refcount the kctx ASAP - it shouldn't disappear anyway, since
- * Bus/Page faults _should_ only occur whilst jobs are running,
- * and a job causing the Bus/Page fault shouldn't complete until
- * the MMU is updated
- */
- kctx = kbasep_js_runpool_lookup_ctx(kbdev, as_no);
-
- /* find faulting address */
- as->fault_addr = kbase_reg_read(kbdev,
- MMU_AS_REG(as_no, AS_FAULTADDRESS_HI),
- kctx);
- as->fault_addr <<= 32;
- as->fault_addr |= kbase_reg_read(kbdev,
- MMU_AS_REG(as_no, AS_FAULTADDRESS_LO),
- kctx);
-
- /* record the fault status */
- as->fault_status = kbase_reg_read(kbdev,
- MMU_AS_REG(as_no, AS_FAULTSTATUS),
- kctx);
-
- /* find the fault type */
- as->fault_type = (bf_bits & (1 << as_no)) ?
- KBASE_MMU_FAULT_TYPE_BUS :
- KBASE_MMU_FAULT_TYPE_PAGE;
-
- if (kbase_as_has_bus_fault(as)) {
- /* Mark bus fault as handled.
- * Note that a bus fault is processed first in case
- * where both a bus fault and page fault occur.
- */
- bf_bits &= ~(1UL << as_no);
-
- /* remove the queued BF (and PF) from the mask */
- new_mask &= ~(MMU_BUS_ERROR(as_no) |
- MMU_PAGE_FAULT(as_no));
- } else {
- /* Mark page fault as handled */
- pf_bits &= ~(1UL << as_no);
-
- /* remove the queued PF from the mask */
- new_mask &= ~MMU_PAGE_FAULT(as_no);
- }
-
- /* Process the interrupt for this address space */
- kbase_mmu_interrupt_process(kbdev, kctx, as);
- }
-
- /* reenable interrupts */
- spin_lock_irqsave(&kbdev->mmu_mask_change, flags);
- tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL);
- new_mask |= tmp;
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask, NULL);
- spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
-}
-
-void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as,
- struct kbase_context *kctx)
-{
- struct kbase_mmu_setup *current_setup = &as->current_setup;
-
- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO),
- current_setup->transtab & 0xFFFFFFFFUL, kctx);
- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI),
- (current_setup->transtab >> 32) & 0xFFFFFFFFUL, kctx);
-
- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO),
- current_setup->memattr & 0xFFFFFFFFUL, kctx);
- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI),
- (current_setup->memattr >> 32) & 0xFFFFFFFFUL, kctx);
- write_cmd(kbdev, as->number, AS_COMMAND_UPDATE, kctx);
-}
-
-int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as,
- struct kbase_context *kctx, u64 vpfn, u32 nr, u32 op,
- unsigned int handling_irq)
-{
- int ret;
-
- if (op == AS_COMMAND_UNLOCK) {
- /* Unlock doesn't require a lock first */
- ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx);
- } else {
- u64 lock_addr = lock_region(kbdev, vpfn, nr);
-
- /* Lock the region that needs to be updated */
- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_LO),
- lock_addr & 0xFFFFFFFFUL, kctx);
- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_HI),
- (lock_addr >> 32) & 0xFFFFFFFFUL, kctx);
- write_cmd(kbdev, as->number, AS_COMMAND_LOCK, kctx);
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3285) &&
- handling_irq) {
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR),
- (1UL << as->number), NULL);
- write_cmd(kbdev, as->number, AS_COMMAND_LOCK, kctx);
- }
-
- /* Run the MMU operation */
- write_cmd(kbdev, as->number, op, kctx);
-
- /* Wait for the flush to complete */
- ret = wait_ready(kbdev, as->number, kctx);
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9630)) {
- /* Issue an UNLOCK command to ensure that valid page
- tables are re-read by the GPU after an update.
- Note that, the FLUSH command should perform all the
- actions necessary, however the bus logs show that if
- multiple page faults occur within an 8 page region
- the MMU does not always re-read the updated page
- table entries for later faults or is only partially
- read, it subsequently raises the page fault IRQ for
- the same addresses, the unlock ensures that the MMU
- cache is flushed, so updates can be re-read. As the
- region is now unlocked we need to issue 2 UNLOCK
- commands in order to flush the MMU/uTLB,
- see PRLAM-8812.
- */
- write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx);
- write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx);
- }
- }
-
- return ret;
-}
-
-void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as,
- struct kbase_context *kctx, enum kbase_mmu_fault_type type)
-{
- u32 pf_bf_mask;
-
- /* Clear the page (and bus fault IRQ as well in case one occurred) */
- pf_bf_mask = MMU_PAGE_FAULT(as->number);
- if (type == KBASE_MMU_FAULT_TYPE_BUS)
- pf_bf_mask |= MMU_BUS_ERROR(as->number);
-
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask, kctx);
-}
-
-void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as,
- struct kbase_context *kctx, enum kbase_mmu_fault_type type)
-{
- unsigned long flags;
- u32 irq_mask;
-
- /* Enable the page fault IRQ (and bus fault IRQ as well in case one
- * occurred) */
- spin_lock_irqsave(&kbdev->mmu_mask_change, flags);
-
- irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), kctx) |
- MMU_PAGE_FAULT(as->number);
-
- if (type == KBASE_MMU_FAULT_TYPE_BUS)
- irq_mask |= MMU_BUS_ERROR(as->number);
-
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask, kctx);
-
- spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags);
-}
-#endif
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * @file
- * Interface file for the direct implementation for MMU hardware access
- */
-
-/**
- * @page mali_kbase_mmu_hw_direct_page Direct MMU hardware interface
- *
- * @section mali_kbase_mmu_hw_direct_intro_sec Introduction
- * This module provides the interface(s) that are required by the direct
- * register access implementation of the MMU hardware interface
- * @ref mali_kbase_mmu_hw_page .
- */
-
-#ifndef _MALI_KBASE_MMU_HW_DIRECT_H_
-#define _MALI_KBASE_MMU_HW_DIRECT_H_
-
-#include <mali_kbase_defs.h>
-
-/**
- * @addtogroup mali_kbase_mmu_hw
- * @{
- */
-
-/**
- * @addtogroup mali_kbase_mmu_hw_direct Direct register access to MMU
- * @{
- */
-
-/** @brief Process an MMU interrupt.
- *
- * Process the MMU interrupt that was reported by the @ref kbase_device.
- *
- * @param[in] kbdev kbase context to clear the fault from.
- * @param[in] irq_stat Value of the MMU_IRQ_STATUS register
- */
-void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat);
-
-/** @} *//* end group mali_kbase_mmu_hw_direct */
-/** @} *//* end group mali_kbase_mmu_hw */
-
-#endif /* _MALI_KBASE_MMU_HW_DIRECT_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+#ifndef _MALI_KBASE_MMU_MODE_
+#define _MALI_KBASE_MMU_MODE_
+
+#include <linux/types.h>
+
+/* Forward declarations */
+struct kbase_context;
+struct kbase_device;
+struct kbase_as;
+
+struct kbase_mmu_mode {
+ void (*update)(struct kbase_context *kctx);
+ void (*disable_as)(struct kbase_device *kbdev, int as_nr);
+ phys_addr_t (*pte_to_phy_addr)(u64 entry);
+ int (*ate_is_valid)(u64 ate);
+ int (*pte_is_valid)(u64 pte);
+ void (*entry_set_ate)(u64 *entry, phys_addr_t phy, unsigned long flags);
+ void (*entry_set_pte)(u64 *entry, phys_addr_t phy);
+ void (*entry_invalidate)(u64 *entry);
+};
+
+struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void);
+struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void);
+
+#endif /* _MALI_KBASE_MMU_MODE_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+#include "mali_kbase_mmu_mode.h"
+
+#include "mali_kbase.h"
+#include "mali_midg_regmap.h"
+
+#define ENTRY_TYPE_MASK 3ULL
+#define ENTRY_IS_ATE 1ULL
+#define ENTRY_IS_INVAL 2ULL
+#define ENTRY_IS_PTE 3ULL
+
+#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */
+#define ENTRY_RD_BIT (1ULL << 6)
+#define ENTRY_WR_BIT (1ULL << 7)
+#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */
+#define ENTRY_ACCESS_BIT (1ULL << 10)
+#define ENTRY_NX_BIT (1ULL << 54)
+
+#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \
+ ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT)
+
+/* Helper Function to perform assignment of page table entries, to
+ * ensure the use of strd, which is required on LPAE systems.
+ */
+static inline void page_table_entry_set(u64 *pte, u64 phy)
+{
+#ifdef CONFIG_64BIT
+ *pte = phy;
+#elif defined(CONFIG_ARM)
+ /*
+ * In order to prevent the compiler keeping cached copies of
+ * memory, we have to explicitly say that we have updated
+ * memory.
+ *
+ * Note: We could manually move the data ourselves into R0 and
+ * R1 by specifying register variables that are explicitly
+ * given registers assignments, the down side of this is that
+ * we have to assume cpu endianness. To avoid this we can use
+ * the ldrd to read the data from memory into R0 and R1 which
+ * will respect the cpu endianness, we then use strd to make
+ * the 64 bit assignment to the page table entry.
+ */
+ asm volatile("ldrd r0, r1, [%[ptemp]]\n\t"
+ "strd r0, r1, [%[pte]]\n\t"
+ : "=m" (*pte)
+ : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy)
+ : "r0", "r1");
+#else
+#error "64-bit atomic write must be implemented for your architecture"
+#endif
+}
+
+static void mmu_update(struct kbase_context *kctx)
+{
+ struct kbase_device * const kbdev = kctx->kbdev;
+ struct kbase_as * const as = &kbdev->as[kctx->as_nr];
+ struct kbase_mmu_setup * const current_setup = &as->current_setup;
+
+ /* Set up the required caching policies at the correct indices
+ * in the memattr register. */
+ current_setup->memattr =
+ (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY <<
+ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) |
+ (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL <<
+ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) |
+ (AS_MEMATTR_LPAE_WRITE_ALLOC <<
+ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) |
+ 0; /* The other indices are unused for now */
+
+ current_setup->transtab = (u64)kctx->pgd &
+ ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK);
+
+ current_setup->transtab |= AS_TRANSTAB_LPAE_ADRMODE_TABLE;
+ current_setup->transtab |= AS_TRANSTAB_LPAE_READ_INNER;
+
+
+ /* Apply the address space setting */
+ kbase_mmu_hw_configure(kbdev, as, kctx);
+}
+
+static void mmu_disable_as(struct kbase_device *kbdev, int as_nr)
+{
+ struct kbase_as * const as = &kbdev->as[as_nr];
+ struct kbase_mmu_setup * const current_setup = &as->current_setup;
+
+ current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED;
+
+
+ /* Apply the address space setting */
+ kbase_mmu_hw_configure(kbdev, as, NULL);
+}
+
+static phys_addr_t pte_to_phy_addr(u64 entry)
+{
+ if (!(entry & 1))
+ return 0;
+
+ return entry & ~0xFFF;
+}
+
+static int ate_is_valid(u64 ate)
+{
+ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE);
+}
+
+static int pte_is_valid(u64 pte)
+{
+ return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE);
+}
+
+/*
+ * Map KBASE_REG flags to MMU flags
+ */
+static u64 get_mmu_flags(unsigned long flags)
+{
+ u64 mmu_flags;
+
+ /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */
+ mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2;
+
+ /* write perm if requested */
+ mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0;
+ /* read perm if requested */
+ mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0;
+ /* nx if requested */
+ mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0;
+
+ if (flags & KBASE_REG_SHARE_BOTH) {
+ /* inner and outer shareable */
+ mmu_flags |= SHARE_BOTH_BITS;
+ } else if (flags & KBASE_REG_SHARE_IN) {
+ /* inner shareable coherency */
+ mmu_flags |= SHARE_INNER_BITS;
+ }
+
+ return mmu_flags;
+}
+
+static void entry_set_ate(u64 *entry, phys_addr_t phy, unsigned long flags)
+{
+ page_table_entry_set(entry, (phy & ~0xFFF) |
+ get_mmu_flags(flags) |
+ ENTRY_IS_ATE);
+}
+
+static void entry_set_pte(u64 *entry, phys_addr_t phy)
+{
+ page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE);
+}
+
+static void entry_invalidate(u64 *entry)
+{
+ page_table_entry_set(entry, ENTRY_IS_INVAL);
+}
+
+static struct kbase_mmu_mode const lpae_mode = {
+ .update = mmu_update,
+ .disable_as = mmu_disable_as,
+ .pte_to_phy_addr = pte_to_phy_addr,
+ .ate_is_valid = ate_is_valid,
+ .pte_is_valid = pte_is_valid,
+ .entry_set_ate = entry_set_ate,
+ .entry_set_pte = entry_set_pte,
+ .entry_invalidate = entry_invalidate
+};
+
+struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void)
+{
+ return &lpae_mode;
+}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
int kbase_platform_fake_register(void)
{
struct kbase_platform_config *config;
- int attribute_count;
#ifndef CONFIG_OF
struct resource resources[PLATFORM_CONFIG_RESOURCE_COUNT];
#endif
return -ENODEV;
}
- attribute_count = kbasep_get_config_attribute_count(config->attributes);
-#ifdef CONFIG_MACH_MANTA
- err = platform_device_add_data(&exynos5_device_g3d, config->attributes, attribute_count * sizeof(config->attributes[0]));
- if (err)
- return err;
-#else
-
mali_device = platform_device_alloc("mali", 0);
if (mali_device == NULL)
return -ENOMEM;
}
#endif /* CONFIG_OF */
- err = platform_device_add_data(mali_device, config->attributes, attribute_count * sizeof(config->attributes[0]));
- if (err) {
- platform_device_unregister(mali_device);
- mali_device = NULL;
- return err;
- }
-
err = platform_device_add(mali_device);
if (err) {
platform_device_unregister(mali_device);
mali_device = NULL;
return err;
}
-#endif /* CONFIG_CONFIG_MACH_MANTA */
return 0;
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* @file mali_kbase_pm.c
* Base kernel power management APIs
*/
-
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
+#include <mali_kbase_config_defaults.h>
+#include <mali_kbase_instr.h>
#include <mali_kbase_pm.h>
-#if KBASE_PM_EN
-
-void kbase_pm_register_access_enable(struct kbase_device *kbdev)
-{
- struct kbase_pm_callback_conf *callbacks;
-
- callbacks = (struct kbase_pm_callback_conf *)kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS);
-
- if (callbacks)
- callbacks->power_on_callback(kbdev);
-}
-
-void kbase_pm_register_access_disable(struct kbase_device *kbdev)
-{
- struct kbase_pm_callback_conf *callbacks;
-
- callbacks = (struct kbase_pm_callback_conf *)kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS);
-
- if (callbacks)
- callbacks->power_off_callback(kbdev);
-}
-
-mali_error kbase_pm_init(struct kbase_device *kbdev)
-{
- mali_error ret = MALI_ERROR_NONE;
- struct kbase_pm_callback_conf *callbacks;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- mutex_init(&kbdev->pm.lock);
-
- kbdev->pm.gpu_powered = MALI_FALSE;
- kbdev->pm.suspending = MALI_FALSE;
-#ifdef CONFIG_MALI_DEBUG
- kbdev->pm.driver_ready_for_irqs = MALI_FALSE;
-#endif /* CONFIG_MALI_DEBUG */
- kbdev->pm.gpu_in_desired_state = MALI_TRUE;
- init_waitqueue_head(&kbdev->pm.gpu_in_desired_state_wait);
-
- callbacks = (struct kbase_pm_callback_conf *)kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS);
- if (callbacks) {
- kbdev->pm.callback_power_on = callbacks->power_on_callback;
- kbdev->pm.callback_power_off = callbacks->power_off_callback;
- kbdev->pm.callback_power_suspend =
- callbacks->power_suspend_callback;
- kbdev->pm.callback_power_resume =
- callbacks->power_resume_callback;
- kbdev->pm.callback_power_runtime_init = callbacks->power_runtime_init_callback;
- kbdev->pm.callback_power_runtime_term = callbacks->power_runtime_term_callback;
- kbdev->pm.callback_power_runtime_on = callbacks->power_runtime_on_callback;
- kbdev->pm.callback_power_runtime_off = callbacks->power_runtime_off_callback;
- } else {
- kbdev->pm.callback_power_on = NULL;
- kbdev->pm.callback_power_off = NULL;
- kbdev->pm.callback_power_suspend = NULL;
- kbdev->pm.callback_power_resume = NULL;
- kbdev->pm.callback_power_runtime_init = NULL;
- kbdev->pm.callback_power_runtime_term = NULL;
- kbdev->pm.callback_power_runtime_on = NULL;
- kbdev->pm.callback_power_runtime_off = NULL;
- }
-
- kbdev->pm.platform_dvfs_frequency = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_POWER_MANAGEMENT_DVFS_FREQ);
-
- /* Initialise the metrics subsystem */
- ret = kbasep_pm_metrics_init(kbdev);
- if (MALI_ERROR_NONE != ret)
- return ret;
-
- init_waitqueue_head(&kbdev->pm.l2_powered_wait);
- kbdev->pm.l2_powered = 0;
-
- init_waitqueue_head(&kbdev->pm.reset_done_wait);
- kbdev->pm.reset_done = MALI_FALSE;
-
- init_waitqueue_head(&kbdev->pm.zero_active_count_wait);
- kbdev->pm.active_count = 0;
-
- spin_lock_init(&kbdev->pm.power_change_lock);
- spin_lock_init(&kbdev->pm.gpu_cycle_counter_requests_lock);
- spin_lock_init(&kbdev->pm.gpu_powered_lock);
-
- if (MALI_ERROR_NONE != kbase_pm_ca_init(kbdev))
- goto workq_fail;
-
- if (MALI_ERROR_NONE != kbase_pm_policy_init(kbdev))
- goto pm_policy_fail;
-
- return MALI_ERROR_NONE;
-
-pm_policy_fail:
- kbase_pm_ca_term(kbdev);
-workq_fail:
- kbasep_pm_metrics_term(kbdev);
- return MALI_ERROR_FUNCTION_FAILED;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_init)
-
-void kbase_pm_do_poweron(struct kbase_device *kbdev, mali_bool is_resume)
-{
- lockdep_assert_held(&kbdev->pm.lock);
-
- /* Turn clocks and interrupts on - no-op if we haven't done a previous
- * kbase_pm_clock_off() */
- kbase_pm_clock_on(kbdev, is_resume);
-
- /* Update core status as required by the policy */
- KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START);
- kbase_pm_update_cores_state(kbdev);
- KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END);
-
- /* NOTE: We don't wait to reach the desired state, since running atoms
- * will wait for that state to be reached anyway */
-}
-
-void kbase_pm_do_poweroff(struct kbase_device *kbdev, mali_bool is_suspend)
+int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags)
{
- unsigned long flags;
- mali_bool cores_are_available;
-
- lockdep_assert_held(&kbdev->pm.lock);
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- /* Force all cores off */
- kbdev->pm.desired_shader_state = 0;
-
- /* Force all cores to be unavailable, in the situation where
- * transitions are in progress for some cores but not others,
- * and kbase_pm_check_transitions_nolock can not immediately
- * power off the cores */
- kbdev->shader_available_bitmap = 0;
- kbdev->tiler_available_bitmap = 0;
- kbdev->l2_available_bitmap = 0;
-
- KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START);
- cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
- KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END);
- /* Don't need 'cores_are_available', because we don't return anything */
- CSTD_UNUSED(cores_are_available);
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- /* NOTE: We won't wait to reach the core's desired state, even if we're
- * powering off the GPU itself too. It's safe to cut the power whilst
- * they're transitioning to off, because the cores should be idle and all
- * cache flushes should already have occurred */
-
- /* Consume any change-state events */
- kbase_timeline_pm_check_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
- /* Disable interrupts and turn the clock off */
- kbase_pm_clock_off(kbdev, is_suspend);
+ return kbase_hwaccess_pm_powerup(kbdev, flags);
}
-mali_error kbase_pm_powerup(struct kbase_device *kbdev)
+void kbase_pm_halt(struct kbase_device *kbdev)
{
- unsigned long flags;
- mali_error ret;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- mutex_lock(&kbdev->pm.lock);
-
- /* A suspend won't happen during startup/insmod */
- KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
-
- /* Power up the GPU, don't enable IRQs as we are not ready to receive them. */
- ret = kbase_pm_init_hw(kbdev, MALI_FALSE);
- if (ret != MALI_ERROR_NONE) {
- mutex_unlock(&kbdev->pm.lock);
- return ret;
- }
-
- kbasep_pm_read_present_cores(kbdev);
-
- kbdev->pm.debug_core_mask = kbdev->shader_present_bitmap;
-
- /* Pretend the GPU is active to prevent a power policy turning the GPU cores off */
- kbdev->pm.active_count = 1;
-
- spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
- /* Ensure cycle counter is off */
- kbdev->pm.gpu_cycle_counter_requests = 0;
- spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
-
- /* We are ready to receive IRQ's now as power policy is set up, so enable them now. */
-#ifdef CONFIG_MALI_DEBUG
- spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
- kbdev->pm.driver_ready_for_irqs = MALI_TRUE;
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
-#endif
- kbase_pm_enable_interrupts(kbdev);
-
- /* Turn on the GPU and any cores needed by the policy */
- kbase_pm_do_poweron(kbdev, MALI_FALSE);
- mutex_unlock(&kbdev->pm.lock);
-
- /* Idle the GPU and/or cores, if the policy wants it to */
- kbase_pm_context_idle(kbdev);
-
- return MALI_ERROR_NONE;
+ kbase_hwaccess_pm_halt(kbdev);
}
-KBASE_EXPORT_TEST_API(kbase_pm_powerup)
-
void kbase_pm_context_active(struct kbase_device *kbdev)
{
(void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE);
int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler)
{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
int c;
int old_count;
if (old_count == 0)
kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE);
+ mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
if (kbase_pm_is_suspending(kbdev)) {
switch (suspend_handler) {
/* FALLTHROUGH */
case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE:
mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
if (old_count == 0)
kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE);
return 1;
case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE:
/* FALLTHROUGH */
default:
- KBASE_DEBUG_ASSERT_MSG(MALI_FALSE, "unreachable");
+ KBASE_DEBUG_ASSERT_MSG(false, "unreachable");
break;
}
}
c = ++kbdev->pm.active_count;
KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c);
-
KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_ACTIVE, NULL, NULL, 0u, c);
/* Trace the event being handled */
if (old_count == 0)
kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE);
- if (c == 1) {
+ if (c == 1)
/* First context active: Power on the GPU and any cores requested by
* the policy */
- kbase_pm_update_active(kbdev);
-
- kbasep_pm_record_gpu_active(kbdev);
- }
+ kbase_hwaccess_pm_gpu_active(kbdev);
mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
return 0;
}
-KBASE_EXPORT_TEST_API(kbase_pm_context_active)
+KBASE_EXPORT_TEST_API(kbase_pm_context_active);
void kbase_pm_context_idle(struct kbase_device *kbdev)
{
+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
int c;
int old_count;
if (old_count == 0)
kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE);
+ mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
c = --kbdev->pm.active_count;
KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c);
-
KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_IDLE, NULL, NULL, 0u, c);
KBASE_DEBUG_ASSERT(c >= 0);
if (c == 0) {
/* Last context has gone idle */
- kbase_pm_update_active(kbdev);
-
- kbasep_pm_record_gpu_idle(kbdev);
+ kbase_hwaccess_pm_gpu_idle(kbdev);
/* Wake up anyone waiting for this to become 0 (e.g. suspend). The
* waiters must synchronize with us by locking the pm.lock after
}
mutex_unlock(&kbdev->pm.lock);
+ mutex_unlock(&js_devdata->runpool_mutex);
}
-KBASE_EXPORT_TEST_API(kbase_pm_context_idle)
-
-void kbase_pm_halt(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- mutex_lock(&kbdev->pm.lock);
- kbase_pm_cancel_deferred_poweroff(kbdev);
- kbase_pm_do_poweroff(kbdev, MALI_FALSE);
- mutex_unlock(&kbdev->pm.lock);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_halt)
-
-void kbase_pm_term(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
- KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests == 0);
-
- /* Free any resources the policy allocated */
- kbase_pm_policy_term(kbdev);
- kbase_pm_ca_term(kbdev);
-
- /* Shut down the metrics subsystem */
- kbasep_pm_metrics_term(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_term)
+KBASE_EXPORT_TEST_API(kbase_pm_context_idle);
void kbase_pm_suspend(struct kbase_device *kbdev)
{
- int nr_keep_gpu_powered_ctxs;
-
KBASE_DEBUG_ASSERT(kbdev);
mutex_lock(&kbdev->pm.lock);
KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
- kbdev->pm.suspending = MALI_TRUE;
+ kbdev->pm.suspending = true;
mutex_unlock(&kbdev->pm.lock);
/* From now on, the active count will drop towards zero. Sometimes, it'll
/* Suspend any counter collection that might be happening */
kbase_instr_hwcnt_suspend(kbdev);
- /* Cancel the keep_gpu_powered calls */
- for (nr_keep_gpu_powered_ctxs = atomic_read(&kbdev->keep_gpu_powered_count);
- nr_keep_gpu_powered_ctxs > 0;
- --nr_keep_gpu_powered_ctxs) {
- kbase_pm_context_idle(kbdev);
- }
-
/* Wait for the active count to reach zero. This is not the same as
* waiting for a power down, since not all policies power down when this
* reaches zero. */
/* NOTE: We synchronize with anything that was just finishing a
* kbase_pm_context_idle() call by locking the pm.lock below */
- /* Force power off the GPU and all cores (regardless of policy), only after
- * the PM active count reaches zero (otherwise, we risk turning it off
- * prematurely) */
- mutex_lock(&kbdev->pm.lock);
- kbase_pm_cancel_deferred_poweroff(kbdev);
- kbase_pm_do_poweroff(kbdev, MALI_TRUE);
- mutex_unlock(&kbdev->pm.lock);
+ kbase_hwaccess_pm_suspend(kbdev);
}
void kbase_pm_resume(struct kbase_device *kbdev)
{
- int nr_keep_gpu_powered_ctxs;
-
/* MUST happen before any pm_context_active calls occur */
- mutex_lock(&kbdev->pm.lock);
- kbdev->pm.suspending = MALI_FALSE;
- kbase_pm_do_poweron(kbdev, MALI_TRUE);
- mutex_unlock(&kbdev->pm.lock);
+ kbase_hwaccess_pm_resume(kbdev);
/* Initial active call, to power on the GPU/cores if needed */
kbase_pm_context_active(kbdev);
- /* Restore the keep_gpu_powered calls */
- for (nr_keep_gpu_powered_ctxs = atomic_read(&kbdev->keep_gpu_powered_count);
- nr_keep_gpu_powered_ctxs > 0;
- --nr_keep_gpu_powered_ctxs) {
- kbase_pm_context_active(kbdev);
- }
-
/* Re-enable instrumentation, if it was previously disabled */
kbase_instr_hwcnt_resume(kbdev);
* need it and the policy doesn't want it on */
kbase_pm_context_idle(kbdev);
}
-#endif /* KBASE_PM_EN */
+
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#ifndef _KBASE_PM_H_
#define _KBASE_PM_H_
-#include <mali_midg_regmap.h>
-#include <linux/atomic.h>
+#include "mali_kbase_hwaccess_pm.h"
-/* Forward definition - see mali_kbase.h */
-struct kbase_device;
+#define PM_ENABLE_IRQS 0x01
+#define PM_HW_ISSUES_DETECT 0x02
-#include "mali_kbase_pm_ca.h"
-#include "mali_kbase_pm_policy.h"
-
-#include "mali_kbase_pm_ca_fixed.h"
-#if !MALI_CUSTOMER_RELEASE
-#include "mali_kbase_pm_ca_random.h"
-#endif
-
-#include "mali_kbase_pm_always_on.h"
-#include "mali_kbase_pm_coarse_demand.h"
-#include "mali_kbase_pm_demand.h"
-#if !MALI_CUSTOMER_RELEASE
-#include "mali_kbase_pm_demand_always_powered.h"
-#include "mali_kbase_pm_fast_start.h"
-#endif
-
-/** The types of core in a GPU.
- *
- * These enumerated values are used in calls to:
- * - @ref kbase_pm_get_present_cores
- * - @ref kbase_pm_get_active_cores
- * - @ref kbase_pm_get_trans_cores
- * - @ref kbase_pm_get_ready_cores.
- *
- * They specify which type of core should be acted on. These values are set in
- * a manner that allows @ref core_type_to_reg function to be simpler and more
- * efficient.
- */
-enum kbase_pm_core_type {
- KBASE_PM_CORE_L3 = L3_PRESENT_LO, /**< The L3 cache */
- KBASE_PM_CORE_L2 = L2_PRESENT_LO, /**< The L2 cache */
- KBASE_PM_CORE_SHADER = SHADER_PRESENT_LO, /**< Shader cores */
- KBASE_PM_CORE_TILER = TILER_PRESENT_LO /**< Tiler cores */
-};
/** Initialize the power management framework.
*
*
* @param kbdev The kbase device structure for the device (must be a valid pointer)
*
- * @return MALI_ERROR_NONE if the power management framework was successfully initialized.
+ * @return 0 if the power management framework was successfully initialized.
*/
-mali_error kbase_pm_init(struct kbase_device *kbdev);
+int kbase_pm_init(struct kbase_device *kbdev);
/** Power up GPU after all modules have been initialized and interrupt handlers installed.
*
* @param kbdev The kbase device structure for the device (must be a valid pointer)
*
- * @return MALI_ERROR_NONE if powerup was successful.
+ * @param flags Flags to pass on to kbase_pm_init_hw
+ *
+ * @return 0 if powerup was successful.
*/
-mali_error kbase_pm_powerup(struct kbase_device *kbdev);
+int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags);
/**
* Halt the power management framework.
*/
void kbase_pm_term(struct kbase_device *kbdev);
-/** Metrics data collected for use by the power management framework.
- *
- */
-struct kbasep_pm_metrics_data {
- int vsync_hit;
- int utilisation;
- int util_gl_share;
- int util_cl_share[2]; /* 2 is a max number of core groups we can have */
- ktime_t time_period_start;
- u32 time_busy;
- u32 time_idle;
- u32 prev_busy;
- u32 prev_idle;
- mali_bool gpu_active;
- u32 busy_cl[2];
- u32 busy_gl;
- u32 active_cl_ctx[2];
- u32 active_gl_ctx;
-
- spinlock_t lock;
-
-#ifdef CONFIG_MALI_MIDGARD_DVFS
- struct hrtimer timer;
- mali_bool timer_active;
-#endif
-
- void *platform_data;
- struct kbase_device *kbdev;
-};
-
-/** Actions for DVFS.
- *
- * kbase_pm_get_dvfs_action will return one of these enumerated values to
- * describe the action that the DVFS system should take.
- */
-enum kbase_pm_dvfs_action {
- KBASE_PM_DVFS_NOP, /**< No change in clock frequency is requested */
- KBASE_PM_DVFS_CLOCK_UP, /**< The clock frequency should be increased if possible */
- KBASE_PM_DVFS_CLOCK_DOWN /**< The clock frequency should be decreased if possible */
-};
-
-union kbase_pm_policy_data {
- struct kbasep_pm_policy_always_on always_on;
- struct kbasep_pm_policy_coarse_demand coarse_demand;
- struct kbasep_pm_policy_demand demand;
-#if !MALI_CUSTOMER_RELEASE
- struct kbasep_pm_policy_demand_always_powered demand_always_powered;
- struct kbasep_pm_policy_fast_start fast_start;
-#endif
-};
-
-union kbase_pm_ca_policy_data {
- struct kbasep_pm_ca_policy_fixed fixed;
-#if !MALI_CUSTOMER_RELEASE
- struct kbasep_pm_ca_policy_random random;
-#endif
-};
-
-/** Data stored per device for power management.
- *
- * This structure contains data for the power management framework. There is one instance of this structure per device
- * in the system.
- */
-struct kbase_pm_device_data {
- /** The lock protecting Power Management structures accessed
- * outside of IRQ.
- *
- * This lock must also be held whenever the GPU is being powered on or off.
- */
- struct mutex lock;
-
- /** The policy that is currently actively controlling core availability.
- *
- * @note: During an IRQ, this can be NULL when the policy is being changed
- * with kbase_pm_ca_set_policy(). The change is protected under
- * kbase_device::pm::power_change_lock. Direct access to this from IRQ
- * context must therefore check for NULL. If NULL, then
- * kbase_pm_ca_set_policy() will re-issue the policy functions that would've
- * been done under IRQ.
- */
- const struct kbase_pm_ca_policy *ca_current_policy;
-
- /** The policy that is currently actively controlling the power state.
- *
- * @note: During an IRQ, this can be NULL when the policy is being changed
- * with kbase_pm_set_policy(). The change is protected under
- * kbase_device::pm::power_change_lock. Direct access to this from IRQ
- * context must therefore check for NULL. If NULL, then
- * kbase_pm_set_policy() will re-issue the policy functions that would've
- * been done under IRQ.
- */
- const struct kbase_pm_policy *pm_current_policy;
-
- /** Private data for current CA policy */
- union kbase_pm_ca_policy_data ca_policy_data;
-
- /** Private data for current PM policy */
- union kbase_pm_policy_data pm_policy_data;
-
- /** Flag indicating when core availability policy is transitioning cores.
- * The core availability policy must set this when a change in core availability
- * is occuring.
- *
- * power_change_lock must be held when accessing this. */
- mali_bool ca_in_transition;
-
- /** Waiting for reset and a queue to wait for changes */
- mali_bool reset_done;
- wait_queue_head_t reset_done_wait;
-
- /** Wait queue for whether the l2 cache has been powered as requested */
- wait_queue_head_t l2_powered_wait;
- /** State indicating whether all the l2 caches are powered.
- * Non-zero indicates they're *all* powered
- * Zero indicates that some (or all) are not powered */
- int l2_powered;
-
- /** The reference count of active contexts on this device. */
- int active_count;
- /** Flag indicating suspending/suspended */
- mali_bool suspending;
- /* Wait queue set when active_count == 0 */
- wait_queue_head_t zero_active_count_wait;
-
- /** The reference count of active gpu cycle counter users */
- int gpu_cycle_counter_requests;
- /** Lock to protect gpu_cycle_counter_requests */
- spinlock_t gpu_cycle_counter_requests_lock;
-
- /** A bit mask identifying the shader cores that the power policy would like to be on.
- * The current state of the cores may be different, but there should be transitions in progress that will
- * eventually achieve this state (assuming that the policy doesn't change its mind in the mean time.
- */
- u64 desired_shader_state;
- /** bit mask indicating which shader cores are currently in a power-on transition */
- u64 powering_on_shader_state;
- /** A bit mask identifying the tiler cores that the power policy would like to be on.
- * @see kbase_pm_device_data:desired_shader_state */
- u64 desired_tiler_state;
- /** bit mask indicating which tiler core are currently in a power-on transition */
- u64 powering_on_tiler_state;
-
- /** bit mask indicating which l2-caches are currently in a power-on transition */
- u64 powering_on_l2_state;
- /** bit mask indicating which l3-caches are currently in a power-on transition */
- u64 powering_on_l3_state;
-
- /** Lock protecting the power state of the device.
- *
- * This lock must be held when accessing the shader_available_bitmap, tiler_available_bitmap, l2_available_bitmap,
- * shader_inuse_bitmap and tiler_inuse_bitmap fields of kbase_device, and the ca_in_transition and shader_poweroff_pending
- * fields of kbase_pm_device_data. It is also held when the hardware power registers are being written to, to ensure
- * that two threads do not conflict over the power transitions that the hardware should make.
- */
- spinlock_t power_change_lock;
-
- /** This flag is set iff the GPU is powered as requested by the
- * desired_xxx_state variables */
- mali_bool gpu_in_desired_state;
- /* Wait queue set when gpu_in_desired_state != 0 */
- wait_queue_head_t gpu_in_desired_state_wait;
-
- /** Set to true when the GPU is powered and register accesses are possible, false otherwise */
- mali_bool gpu_powered;
-
- /** A bit mask identifying the available shader cores that are specified via sysfs */
- u64 debug_core_mask;
-
- /** Set to true when instrumentation is enabled, false otherwise */
- mali_bool instr_enabled;
-
- mali_bool cg1_disabled;
-
-#ifdef CONFIG_MALI_DEBUG
- /** Debug state indicating whether sufficient initialization of the driver
- * has occurred to handle IRQs */
- mali_bool driver_ready_for_irqs;
-#endif /* CONFIG_MALI_DEBUG */
-
- /** Spinlock that must be held when:
- * - writing gpu_powered
- * - accessing driver_ready_for_irqs (in CONFIG_MALI_DEBUG builds) */
- spinlock_t gpu_powered_lock;
-
- /** Time in milliseconds between each dvfs sample */
-
- u32 platform_dvfs_frequency;
-
- /** Structure to hold metrics for the GPU */
-
- struct kbasep_pm_metrics_data metrics;
-
- /** Set to the number of poweroff timer ticks until the GPU is powered off */
- int gpu_poweroff_pending;
-
- /** Set to the number of poweroff timer ticks until shaders are powered off */
- int shader_poweroff_pending_time;
-
- /** Timer for powering off GPU */
- struct hrtimer gpu_poweroff_timer;
-
- struct workqueue_struct *gpu_poweroff_wq;
-
- struct work_struct gpu_poweroff_work;
-
- /** Period of GPU poweroff timer */
- ktime_t gpu_poweroff_time;
-
- /** Bit mask of shaders to be powered off on next timer callback */
- u64 shader_poweroff_pending;
-
- /** Set to MALI_TRUE if the poweroff timer is currently running, MALI_FALSE otherwise */
- mali_bool poweroff_timer_needed;
-
- int poweroff_shader_ticks;
-
- int poweroff_gpu_ticks;
-
- /** Callback when the GPU needs to be turned on. See @ref kbase_pm_callback_conf
- *
- * @param kbdev The kbase device
- *
- * @return 1 if GPU state was lost, 0 otherwise
- */
- int (*callback_power_on)(struct kbase_device *kbdev);
-
- /** Callback when the GPU may be turned off. See @ref kbase_pm_callback_conf
- *
- * @param kbdev The kbase device
- */
- void (*callback_power_off)(struct kbase_device *kbdev);
-
- /** Callback when a suspend occurs and the GPU needs to be turned off.
- * See @ref kbase_pm_callback_conf
- *
- * @param kbdev The kbase device
- */
- void (*callback_power_suspend)(struct kbase_device *kbdev);
-
- /** Callback when a resume occurs and the GPU needs to be turned on.
- * See @ref kbase_pm_callback_conf
- *
- * @param kbdev The kbase device
- */
- void (*callback_power_resume)(struct kbase_device *kbdev);
-
- /** Callback for initializing the runtime power management.
- *
- * @param kbdev The kbase device
- *
- * @return MALI_ERROR_NONE on success, else error code
- */
- mali_error (*callback_power_runtime_init)(struct kbase_device *kbdev);
-
- /** Callback for terminating the runtime power management.
- *
- * @param kbdev The kbase device
- */
- void (*callback_power_runtime_term)(struct kbase_device *kbdev);
-
- /** Callback when the GPU needs to be turned on. See @ref kbase_pm_callback_conf
- *
- * @param kbdev The kbase device
- *
- * @return 1 if GPU state was lost, 0 otherwise
- */
- int (*callback_power_runtime_on)(struct kbase_device *kbdev);
-
- /** Callback when the GPU may be turned off. See @ref kbase_pm_callback_conf
- *
- * @param kbdev The kbase device
- */
- void (*callback_power_runtime_off)(struct kbase_device *kbdev);
-
-};
-
-/** The GPU is idle.
- *
- * The OS may choose to turn off idle devices
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_dev_idle(struct kbase_device *kbdev);
-
-/** The GPU is active.
- *
- * The OS should avoid opportunistically turning off the GPU while it is active
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_dev_activate(struct kbase_device *kbdev);
-
-/** Get details of the cores that are present in the device.
- *
- * This function can be called by the active power policy to return a bitmask of the cores (of a specified type)
- * present in the GPU device and also a count of the number of cores.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param type The type of core (see the @ref enum kbase_pm_core_type enumeration)
- *
- * @return The bit mask of cores present
- */
-u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, enum kbase_pm_core_type type);
-
-/** Get details of the cores that are currently active in the device.
- *
- * This function can be called by the active power policy to return a bitmask of the cores (of a specified type) that
- * are actively processing work (i.e. turned on *and* busy).
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param type The type of core (see the @ref enum kbase_pm_core_type enumeration)
- *
- * @return The bit mask of active cores
- */
-u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, enum kbase_pm_core_type type);
-
-/** Get details of the cores that are currently transitioning between power states.
- *
- * This function can be called by the active power policy to return a bitmask of the cores (of a specified type) that
- * are currently transitioning between power states.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param type The type of core (see the @ref enum kbase_pm_core_type enumeration)
- *
- * @return The bit mask of transitioning cores
- */
-u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, enum kbase_pm_core_type type);
-
-/** Get details of the cores that are currently powered and ready for jobs.
- *
- * This function can be called by the active power policy to return a bitmask of the cores (of a specified type) that
- * are powered and ready for jobs (they may or may not be currently executing jobs).
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param type The type of core (see the @ref enum kbase_pm_core_type enumeration)
- *
- * @return The bit mask of ready cores
- */
-u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, enum kbase_pm_core_type type);
-
-/** Turn the clock for the device on, and enable device interrupts.
- *
- * This function can be used by a power policy to turn the clock for the GPU on. It should be modified during
- * integration to perform the necessary actions to ensure that the GPU is fully powered and clocked.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param is_resume MALI_TRUE if clock on due to resume after suspend,
- * MALI_FALSE otherwise
- */
-void kbase_pm_clock_on(struct kbase_device *kbdev, mali_bool is_resume);
-
-/** Disable device interrupts, and turn the clock for the device off.
- *
- * This function can be used by a power policy to turn the clock for the GPU off. It should be modified during
- * integration to perform the necessary actions to turn the clock off (if this is possible in the integration).
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param is_suspend MALI_TRUE if clock off due to suspend, MALI_FALSE otherwise
- */
-void kbase_pm_clock_off(struct kbase_device *kbdev, mali_bool is_suspend);
-
-/** Enable interrupts on the device.
- *
- * Interrupts are also enabled after a call to kbase_pm_clock_on().
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_enable_interrupts(struct kbase_device *kbdev);
-
-/** Disable interrupts on the device.
- *
- * This prevents delivery of Power Management interrupts to the CPU so that
- * kbase_pm_check_transitions_nolock() will not be called from the IRQ handler
- * until @ref kbase_pm_enable_interrupts or kbase_pm_clock_on() is called.
- *
- * Interrupts are also disabled after a call to kbase_pm_clock_off().
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_disable_interrupts(struct kbase_device *kbdev);
-
-/** Initialize the hardware
- *
- * This function checks the GPU ID register to ensure that the GPU is supported by the driver and performs a reset on
- * the device so that it is in a known state before the device is used.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param enable_irqs When set to MALI_TRUE gpu irqs will be enabled after this call, else
- * they will be left disabled.
- *
- * @return MALI_ERROR_NONE if the device is supported and successfully reset.
- */
-mali_error kbase_pm_init_hw(struct kbase_device *kbdev, mali_bool enable_irqs);
-
-/** The GPU has been reset successfully.
- *
- * This function must be called by the GPU interrupt handler when the RESET_COMPLETED bit is set. It signals to the
- * power management initialization code that the GPU has been successfully reset.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_reset_done(struct kbase_device *kbdev);
-
/** Increment the count of active contexts.
*
* This function should be called when a context is about to submit a job. It informs the active power policy that the
*/
void kbase_pm_context_idle(struct kbase_device *kbdev);
-/** Check if there are any power transitions to make, and if so start them.
- *
- * This function will check the desired_xx_state members of struct kbase_pm_device_data and the actual status of the
- * hardware to see if any power transitions can be made at this time to make the hardware state closer to the state
- * desired by the power policy.
- *
- * The return value can be used to check whether all the desired cores are
- * available, and so whether it's worth submitting a job (e.g. from a Power
- * Management IRQ).
- *
- * Note that this still returns MALI_TRUE when desired_xx_state has no
- * cores. That is: of the no cores desired, none were <em>un</em>available. In
- * this case, the caller may still need to try submitting jobs. This is because
- * the Core Availability Policy might have taken us to an intermediate state
- * where no cores are powered, before powering on more cores (e.g. for core
- * rotation)
- *
- * The caller must hold kbase_device::pm::power_change_lock
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @return non-zero when all desired cores are available. That is,
- * it's worthwhile for the caller to submit a job.
- * @return MALI_FALSE otherwise
- */
-mali_bool kbase_pm_check_transitions_nolock(struct kbase_device *kbdev);
-
-/** Synchronous and locking variant of kbase_pm_check_transitions_nolock()
- *
- * On returning, the desired state at the time of the call will have been met.
- *
- * @note There is nothing to stop the core being switched off by calls to
- * kbase_pm_release_cores() or kbase_pm_unrequest_cores(). Therefore, the
- * caller must have already made a call to
- * kbase_pm_request_cores()/kbase_pm_request_cores_sync() previously.
- *
- * The usual use-case for this is to ensure cores are 'READY' after performing
- * a GPU Reset.
- *
- * Unlike kbase_pm_check_transitions_nolock(), the caller must not hold
- * kbase_device::pm::power_change_lock, because this function will take that
- * lock itself.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_check_transitions_sync(struct kbase_device *kbdev);
-
-/** Variant of kbase_pm_update_cores_state() where the caller must hold
- * kbase_device::pm::power_change_lock
- *
- * @param kbdev The kbase device structure for the device (must be a valid
- * pointer)
- */
-void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev);
-
-/** Update the desired state of shader cores from the Power Policy, and begin
- * any power transitions.
- *
- * This function will update the desired_xx_state members of
- * struct kbase_pm_device_data by calling into the current Power Policy. It will then
- * begin power transitions to make the hardware acheive the desired shader core
- * state.
- *
- * @param kbdev The kbase device structure for the device (must be a valid
- * pointer)
- */
-void kbase_pm_update_cores_state(struct kbase_device *kbdev);
-
-/** Cancel any pending requests to power off the GPU and/or shader cores.
- *
- * This should be called by any functions which directly power off the GPU.
- *
- * @param kbdev The kbase device structure for the device (must be a valid
- * pointer)
- */
-void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev);
-
-/** Read the bitmasks of present cores.
- *
- * This information is cached to avoid having to perform register reads whenever the information is required.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbasep_pm_read_present_cores(struct kbase_device *kbdev);
-
-/** Initialize the metrics gathering framework.
- *
- * This must be called before other metric gathering APIs are called.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- *
- * @return MALI_ERROR_NONE on success, MALI_ERROR_FUNCTION_FAILED on error
- */
-mali_error kbasep_pm_metrics_init(struct kbase_device *kbdev);
-
-/** Terminate the metrics gathering framework.
- *
- * This must be called when metric gathering is no longer required. It is an error to call any metrics gathering
- * function (other than kbasep_pm_metrics_init) after calling this function.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbasep_pm_metrics_term(struct kbase_device *kbdev);
-
-/** Record state of jobs currently active on GPU.
- *
- * This function record time spent executing jobs split per GL and CL
- * contexts, per core group (only CL jobs).
- *
- * @param kbdev The kbase device structure for the device
- * (must be a valid pointer)
- */
-void kbasep_pm_record_job_status(struct kbase_device *kbdev);
-
-/** Record that the GPU is active.
- *
- * This records that the GPU is now active. The previous GPU state must have been idle, the function will assert if
- * this is not true in a debug build.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbasep_pm_record_gpu_active(struct kbase_device *kbdev);
-
-/** Record that the GPU is idle.
- *
- * This records that the GPU is now idle. The previous GPU state must have been active, the function will assert if
- * this is not true in a debug build.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbasep_pm_record_gpu_idle(struct kbase_device *kbdev);
-
-/** Function to be called by the frame buffer driver to update the vsync metric.
- *
- * This function should be called by the frame buffer driver to update whether the system is hitting the vsync target
- * or not. buffer_updated should be true if the vsync corresponded with a new frame being displayed, otherwise it
- * should be false. This function does not need to be called every vsync, but only when the value of buffer_updated
- * differs from a previous call.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param buffer_updated True if the buffer has been updated on this VSync, false otherwise
- */
-void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated);
-
-/** Configure the frame buffer device to set the vsync callback.
- *
- * This function should do whatever is necessary for this integration to ensure that kbase_pm_report_vsync is
- * called appropriately.
- *
- * This function will need porting as part of the integration for a device.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_register_vsync_callback(struct kbase_device *kbdev);
-
-/** Free any resources that kbase_pm_register_vsync_callback allocated.
- *
- * This function should perform any cleanup required from the call to kbase_pm_register_vsync_callback.
- * No call backs should occur after this function has returned.
- *
- * This function will need porting as part of the integration for a device.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_unregister_vsync_callback(struct kbase_device *kbdev);
-
-/** Determine whether the DVFS system should change the clock speed of the GPU.
- *
- * This function should be called regularly by the DVFS system to check whether the clock speed of the GPU needs
- * updating. It will return one of three enumerated values of kbase_pm_dvfs_action:
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @retval KBASE_PM_DVFS_NOP The clock does not need changing
- * @retval KBASE_PM_DVFS_CLOCK_UP, The clock frequency should be increased if possible.
- * @retval KBASE_PM_DVFS_CLOCK_DOWN The clock frequency should be decreased if possible.
- */
-enum kbase_pm_dvfs_action kbase_pm_get_dvfs_action(struct kbase_device *kbdev);
-
-/** Mark that the GPU cycle counter is needed, if the caller is the first caller
- * then the GPU cycle counters will be enabled along with the l2 cache
- *
- * The GPU must be powered when calling this function (i.e. @ref kbase_pm_context_active must have been called).
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev);
-
-/** This is a version of the above function (@ref kbase_pm_request_gpu_cycle_counter) suitable for being
- * called when the l2 cache is known to be on and assured to be on until the subsequent call of
- * kbase_pm_release_gpu_cycle_counter such as when a job is submitted.
- * It does not sleep and can be called from atomic functions.
- *
- * The GPU must be powered when calling this function (i.e. @ref kbase_pm_context_active must have been called).
- * and the l2 cache must be powered on
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev);
-
-/** Mark that the GPU cycle counter is no longer in use, if the caller is the last
- * caller then the GPU cycle counters will be disabled. A request must have been made
- * before a call to this.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-
-void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev);
-
-/** Enables access to the GPU registers before power management has powered up the GPU
- * with kbase_pm_powerup().
- *
- * Access to registers should be done using kbase_os_reg_read/write() at this stage,
- * not kbase_reg_read/write().
- *
- * This results in the power management callbacks provided in the driver configuration
- * to get called to turn on power and/or clocks to the GPU.
- * See @ref kbase_pm_callback_conf.
- *
- * This should only be used before power management is powered up with kbase_pm_powerup()
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_register_access_enable(struct kbase_device *kbdev);
-
-/** Disables access to the GPU registers enabled earlier by a call to
- * kbase_pm_register_access_enable().
- *
- * This results in the power management callbacks provided in the driver configuration
- * to get called to turn off power and/or clocks to the GPU.
- * See @ref kbase_pm_callback_conf
- *
- * This should only be used before power management is powered up with kbase_pm_powerup()
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_register_access_disable(struct kbase_device *kbdev);
-
/**
* Suspend the GPU and prevent any further register accesses to it from Kernel
* threads.
*/
void kbase_pm_resume(struct kbase_device *kbdev);
-/* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline function */
-
/**
- * Check if the power management metrics collection is active.
- *
- * Note that this returns if the power management metrics collection was
- * active at the time of calling, it is possible that after the call the metrics
- * collection enable may have changed state.
+ * kbase_pm_vsync_callback - vsync callback
*
- * The caller must handle the consequence that the state may have changed.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @return MALI_TRUE if metrics collection was active else MALI_FALSE.
- */
-
-mali_bool kbase_pm_metrics_is_active(struct kbase_device *kbdev);
-
-/**
- * Power on the GPU, and any cores that are requested.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param is_resume MALI_TRUE if power on due to resume after suspend,
- * MALI_FALSE otherwise
- */
-void kbase_pm_do_poweron(struct kbase_device *kbdev, mali_bool is_resume);
-
-/**
- * Power off the GPU, and any cores that have been requested.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param is_suspend MALI_TRUE if power off due to suspend,
- * MALI_FALSE otherwise
- */
-void kbase_pm_do_poweroff(struct kbase_device *kbdev, mali_bool is_suspend);
-
-#ifdef CONFIG_PM_DEVFREQ
-void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev,
- unsigned long *total, unsigned long *busy);
-void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev);
-#endif
-
-#ifdef CONFIG_MALI_MIDGARD_DVFS
-
-/**
- * Function provided by platform specific code when DVFS is enabled to allow
- * the power management metrics system to report utilisation.
+ * @buffer_updated: 1 if a new frame was displayed, 0 otherwise
+ * @data: Pointer to the kbase device as returned by kbase_find_device()
*
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param utilisation The current calculated utilisation by the metrics system.
- * @param util_gl_share The current calculated gl share of utilisation.
- * @param util_cl_share The current calculated cl share of utilisation per core group.
- * @return Returns 0 on failure and non zero on success.
+ * Callback function used to notify the power management code that a vsync has
+ * occurred on the display.
*/
+void kbase_pm_vsync_callback(int buffer_updated, void *data);
-int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation,
- u32 util_gl_share, u32 util_cl_share[2]);
-#endif
#endif /* _KBASE_PM_H_ */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_pm_always_on.c
- * "Always on" power management policy
- */
-
-#include <mali_kbase.h>
-#include <mali_kbase_pm.h>
-
-#if KBASE_PM_EN
-static u64 always_on_get_core_mask(struct kbase_device *kbdev)
-{
- return kbdev->shader_present_bitmap;
-}
-
-static mali_bool always_on_get_core_active(struct kbase_device *kbdev)
-{
- return MALI_TRUE;
-}
-
-static void always_on_init(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
-static void always_on_term(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
-/** The @ref struct kbase_pm_policy structure for the demand power policy.
- *
- * This is the static structure that defines the demand power policy's callback and name.
- */
-const struct kbase_pm_policy kbase_pm_always_on_policy_ops = {
- "always_on", /* name */
- always_on_init, /* init */
- always_on_term, /* term */
- always_on_get_core_mask, /* get_core_mask */
- always_on_get_core_active, /* get_core_active */
- 0u, /* flags */
- KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */
-};
-
-KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops)
-#endif /* KBASE_PM_EN */
+++ /dev/null
-
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_pm_always_on.h
- * "Always on" power management policy
- */
-
-#ifndef MALI_KBASE_PM_ALWAYS_ON_H
-#define MALI_KBASE_PM_ALWAYS_ON_H
-
-/**
- * The "Always on" power management policy has the following
- * characteristics:
- * - When KBase indicates that the GPU will be powered up, but we don't yet
- * know which Job Chains are to be run:
- * - All Shader Cores are powered up, regardless of whether or not they will
- * be needed later.
- * - When KBase indicates that a set of Shader Cores are needed to submit the
- * currently queued Job Chains:
- * - All Shader Cores are kept powered, regardless of whether or not they will
- * be needed
- * - When KBase indicates that the GPU need not be powered:
- * - The Shader Cores are kept powered, regardless of whether or not they will
- * be needed. The GPU itself is also kept powered, even though it is not
- * needed.
- *
- * This policy is automatically overridden during system suspend: the desired
- * core state is ignored, and the cores are forced off regardless of what the
- * policy requests. After resuming from suspend, new changes to the desired
- * core state made by the policy are honored.
- *
- * @note:
- * - KBase indicates the GPU will be powered up when it has a User Process that
- * has just started to submit Job Chains.
- * - KBase indicates the GPU need not be powered when all the Job Chains from
- * User Processes have finished, and it is waiting for a User Process to
- * submit some more Job Chains.
- */
-
-/**
- * Private structure for policy instance data.
- *
- * This contains data that is private to the particular power policy that is active.
- */
-typedef struct kbasep_pm_policy_always_on {
- /** No state needed - just have a dummy variable here */
- int dummy;
-} kbasep_pm_policy_always_on;
-
-#endif /* MALI_KBASE_PM_ALWAYS_ON_H */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * @file mali_kbase_pm_ca.c
- * Base kernel core availability APIs
- */
-
-#include <mali_kbase.h>
-#include <mali_kbase_pm.h>
-#if KBASE_PM_EN
-extern const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops;
-#if !MALI_CUSTOMER_RELEASE
-extern const struct kbase_pm_ca_policy kbase_pm_ca_random_policy_ops;
-#endif
-
-static const struct kbase_pm_ca_policy *const policy_list[] = {
- &kbase_pm_ca_fixed_policy_ops,
-#if !MALI_CUSTOMER_RELEASE
- &kbase_pm_ca_random_policy_ops
-#endif
-};
-
-/** The number of policies available in the system.
- * This is derived from the number of functions listed in policy_get_functions.
- */
-#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
-
-mali_error kbase_pm_ca_init(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- kbdev->pm.ca_current_policy = policy_list[0];
-
- kbdev->pm.ca_current_policy->init(kbdev);
-
- return MALI_ERROR_NONE;
-}
-
-void kbase_pm_ca_term(struct kbase_device *kbdev)
-{
- kbdev->pm.ca_current_policy->term(kbdev);
-}
-
-int kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **list)
-{
- if (!list)
- return POLICY_COUNT;
-
- *list = policy_list;
-
- return POLICY_COUNT;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_ca_list_policies)
-
-const struct kbase_pm_ca_policy *kbase_pm_ca_get_policy(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- return kbdev->pm.ca_current_policy;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_ca_get_policy)
-
-void kbase_pm_ca_set_policy(struct kbase_device *kbdev, const struct kbase_pm_ca_policy *new_policy)
-{
- const struct kbase_pm_ca_policy *old_policy;
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(new_policy != NULL);
-
- KBASE_TRACE_ADD(kbdev, PM_CA_SET_POLICY, NULL, NULL, 0u, new_policy->id);
-
- /* During a policy change we pretend the GPU is active */
- /* A suspend won't happen here, because we're in a syscall from a userspace thread */
- kbase_pm_context_active(kbdev);
-
- mutex_lock(&kbdev->pm.lock);
-
- /* Remove the policy to prevent IRQ handlers from working on it */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- old_policy = kbdev->pm.ca_current_policy;
- kbdev->pm.ca_current_policy = NULL;
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- if (old_policy->term)
- old_policy->term(kbdev);
-
- if (new_policy->init)
- new_policy->init(kbdev);
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- kbdev->pm.ca_current_policy = new_policy;
-
- /* If any core power state changes were previously attempted, but couldn't
- * be made because the policy was changing (current_policy was NULL), then
- * re-try them here. */
- kbase_pm_update_cores_state_nolock(kbdev);
-
- kbdev->pm.ca_current_policy->update_core_status(kbdev, kbdev->shader_ready_bitmap, kbdev->shader_transitioning_bitmap);
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- mutex_unlock(&kbdev->pm.lock);
-
- /* Now the policy change is finished, we release our fake context active reference */
- kbase_pm_context_idle(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_ca_set_policy)
-
-u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev)
-{
- lockdep_assert_held(&kbdev->pm.power_change_lock);
-
- /* All cores must be enabled when instrumentation is in use */
- if (kbdev->pm.instr_enabled == MALI_TRUE)
- return kbdev->shader_present_bitmap & kbdev->pm.debug_core_mask;
-
- if (kbdev->pm.ca_current_policy == NULL)
- return kbdev->shader_present_bitmap & kbdev->pm.debug_core_mask;
-
- return kbdev->pm.ca_current_policy->get_core_mask(kbdev) & kbdev->pm.debug_core_mask;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask)
-
-void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, u64 cores_transitioning)
-{
- lockdep_assert_held(&kbdev->pm.power_change_lock);
-
- if (kbdev->pm.ca_current_policy != NULL)
- kbdev->pm.ca_current_policy->update_core_status(kbdev, cores_ready, cores_transitioning);
-}
-
-void kbase_pm_ca_instr_enable(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- kbdev->pm.instr_enabled = MALI_TRUE;
-
- kbase_pm_update_cores_state_nolock(kbdev);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-}
-
-void kbase_pm_ca_instr_disable(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- kbdev->pm.instr_enabled = MALI_FALSE;
-
- kbase_pm_update_cores_state_nolock(kbdev);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-}
-#endif /* KBASE_PM_EN */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * @file mali_kbase_pm_ca.h
- * Base kernel core availability APIs
- */
-
-#ifndef _KBASE_PM_CA_H_
-#define _KBASE_PM_CA_H_
-
-enum kbase_pm_ca_policy_id {
- KBASE_PM_CA_POLICY_ID_FIXED = 1,
- KBASE_PM_CA_POLICY_ID_RANDOM
-};
-
-typedef u32 kbase_pm_ca_policy_flags;
-
-/** Core availability policy structure.
- *
- * Each core availability policy exposes a (static) instance of this structure which contains function pointers to the
- * policy's methods.
- */
-struct kbase_pm_ca_policy {
- /** The name of this policy */
- char *name;
-
- /** Function called when the policy is selected
- *
- * This should initialize the kbdev->pm.ca_policy_data structure. It should not attempt
- * to make any changes to hardware state.
- *
- * It is undefined what state the cores are in when the function is called.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
- void (*init)(struct kbase_device *kbdev);
-
- /** Function called when the policy is unselected.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
- void (*term)(struct kbase_device *kbdev);
-
- /** Function called to get the current shader core availability mask
- *
- * When a change in core availability is occuring, the policy must set kbdev->pm.ca_in_transition
- * to MALI_TRUE. This is to indicate that reporting changes in power state cannot be optimized out,
- * even if kbdev->pm.desired_shader_state remains unchanged. This must be done by any functions
- * internal to the Core Availability Policy that change the return value of
- * kbase_pm_ca_policy::get_core_mask.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- *
- * @return The current core availability mask */
- u64 (*get_core_mask)(struct kbase_device *kbdev);
-
- /** Function called to update the current core status
- *
- * If none of the cores in core group 0 are ready or transitioning, then the policy must
- * ensure that the next call to get_core_mask does not return 0 for all cores in core group
- * 0. It is an error to disable core group 0 through the core availability policy.
- *
- * When a change in core availability has finished, the policy must set kbdev->pm.ca_in_transition
- * to MALI_FALSE. This is to indicate that changes in power state can once again be optimized out
- * when kbdev->pm.desired_shader_state is unchanged.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param cores_ready The mask of cores currently powered and ready to run jobs
- * @param cores_transitioning The mask of cores currently transitioning power state */
- void (*update_core_status)(struct kbase_device *kbdev, u64 cores_ready, u64 cores_transitioning);
-
- /** Field indicating flags for this policy */
- kbase_pm_ca_policy_flags flags;
-
- /** Field indicating an ID for this policy. This is not necessarily the
- * same as its index in the list returned by kbase_pm_list_policies().
- * It is used purely for debugging. */
- enum kbase_pm_ca_policy_id id;
-};
-
-/** Initialize core availability framework
- *
- * Must be called before calling any other core availability function
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- *
- * @return MALI_ERROR_NONE if the core availability framework was successfully initialized.
- */
-mali_error kbase_pm_ca_init(struct kbase_device *kbdev);
-
-/** Terminate core availability framework
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_ca_term(struct kbase_device *kbdev);
-
-/** Return mask of currently available shaders cores
- * Calls into the core availability policy
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- *
- * @return The bit mask of available cores
- */
-u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev);
-
-/** Update core availability policy with current core power status
- * Calls into the core availability policy
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param cores_ready The bit mask of cores ready for job submission
- * @param cores_transitioning The bit mask of cores that are transitioning power state
- */
-void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, u64 cores_transitioning);
-
-/** Enable override for instrumentation
- *
- * This overrides the output of the core availability policy, ensuring that all cores are available
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_ca_instr_enable(struct kbase_device *kbdev);
-
-/** Disable override for instrumentation
- *
- * This disables any previously enabled override, and resumes normal policy functionality
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_ca_instr_disable(struct kbase_device *kbdev);
-
-/** Get the current policy.
- * Returns the policy that is currently active.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- *
- * @return The current policy
- */
-const struct kbase_pm_ca_policy *kbase_pm_ca_get_policy(struct kbase_device *kbdev);
-
-/** Change the policy to the one specified.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param policy The policy to change to (valid pointer returned from @ref kbase_pm_ca_list_policies)
- */
-void kbase_pm_ca_set_policy(struct kbase_device *kbdev, const struct kbase_pm_ca_policy *policy);
-
-/** Retrieve a static list of the available policies.
- * @param[out] policies An array pointer to take the list of policies. This may be NULL.
- * The contents of this array must not be modified.
- *
- * @return The number of policies
- */
-int kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies);
-
-#endif /* _KBASE_PM_CA_H_ */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * @file mali_kbase_pm_ca_fixed.c
- * A power policy implementing fixed core availability
- */
-
-#include <mali_kbase.h>
-#include <mali_kbase_pm.h>
-#if KBASE_PM_EN
-static void fixed_init(struct kbase_device *kbdev)
-{
- kbdev->pm.ca_in_transition = MALI_FALSE;
-}
-
-static void fixed_term(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
-static u64 fixed_get_core_mask(struct kbase_device *kbdev)
-{
- return kbdev->shader_present_bitmap;
-}
-
-static void fixed_update_core_status(struct kbase_device *kbdev, u64 cores_ready, u64 cores_transitioning)
-{
- CSTD_UNUSED(kbdev);
- CSTD_UNUSED(cores_ready);
- CSTD_UNUSED(cores_transitioning);
-}
-
-/** The @ref struct kbase_pm_policy structure for the fixed power policy.
- *
- * This is the static structure that defines the fixed power policy's callback and name.
- */
-const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops = {
- "fixed", /* name */
- fixed_init, /* init */
- fixed_term, /* term */
- fixed_get_core_mask, /* get_core_mask */
- fixed_update_core_status, /* update_core_status */
- 0u, /* flags */
- KBASE_PM_CA_POLICY_ID_FIXED, /* id */
-};
-
-KBASE_EXPORT_TEST_API(kbase_pm_ca_fixed_policy_ops)
-#endif /* KBASE_PM_EN */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * @file mali_kbase_pm_ca_fixed.h
- * A power policy implementing fixed core availability
- */
-
-#ifndef MALI_KBASE_PM_CA_FIXED_H
-#define MALI_KBASE_PM_CA_FIXED_H
-
-/**
- * Private structure for policy instance data.
- *
- * This contains data that is private to the particular power policy that is active.
- */
-typedef struct kbasep_pm_ca_policy_fixed {
- /** No state needed - just have a dummy variable here */
- int dummy;
-} kbasep_pm_ca_policy_fixed;
-
-#endif /* MALI_KBASE_PM_CA_FIXED_H */
-
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_pm_coarse_demand.c
- * "Coarse Demand" power management policy
- */
-
-#include <mali_kbase.h>
-#include <mali_kbase_pm.h>
-
-#if KBASE_PM_EN
-static u64 coarse_demand_get_core_mask(struct kbase_device *kbdev)
-{
- if (kbdev->pm.active_count == 0)
- return 0;
-
- return kbdev->shader_present_bitmap;
-}
-
-static mali_bool coarse_demand_get_core_active(struct kbase_device *kbdev)
-{
- if (kbdev->pm.active_count == 0)
- return MALI_FALSE;
-
- return MALI_TRUE;
-}
-
-static void coarse_demand_init(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
-static void coarse_demand_term(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
-/** The @ref struct kbase_pm_policy structure for the demand power policy.
- *
- * This is the static structure that defines the demand power policy's callback and name.
- */
-const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = {
- "coarse_demand", /* name */
- coarse_demand_init, /* init */
- coarse_demand_term, /* term */
- coarse_demand_get_core_mask, /* get_core_mask */
- coarse_demand_get_core_active, /* get_core_active */
- 0u, /* flags */
- KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */
-};
-
-KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops)
-#endif /* KBASE_PM_EN */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_pm_coarse_demand.h
- * "Coarse Demand" power management policy
- */
-
-#ifndef MALI_KBASE_PM_COARSE_DEMAND_H
-#define MALI_KBASE_PM_COARSE_DEMAND_H
-
-/**
- * The "Coarse" demand power management policy has the following
- * characteristics:
- * - When KBase indicates that the GPU will be powered up, but we don't yet
- * know which Job Chains are to be run:
- * - All Shader Cores are powered up, regardless of whether or not they will
- * be needed later.
- * - When KBase indicates that a set of Shader Cores are needed to submit the
- * currently queued Job Chains:
- * - All Shader Cores are kept powered, regardless of whether or not they will
- * be needed
- * - When KBase indicates that the GPU need not be powered:
- * - The Shader Cores are powered off, and the GPU itself is powered off too.
- *
- * @note:
- * - KBase indicates the GPU will be powered up when it has a User Process that
- * has just started to submit Job Chains.
- * - KBase indicates the GPU need not be powered when all the Job Chains from
- * User Processes have finished, and it is waiting for a User Process to
- * submit some more Job Chains.
- */
-
-/**
- * Private structure for policy instance data.
- *
- * This contains data that is private to the particular power policy that is active.
- */
-typedef struct kbasep_pm_policy_coarse_demand {
- /** No state needed - just have a dummy variable here */
- int dummy;
-} kbasep_pm_policy_coarse_demand;
-
-#endif /* MALI_KBASE_PM_COARSE_DEMAND_H */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_pm_demand.c
- * A simple demand based power management policy
- */
-
-#include <mali_kbase.h>
-#include <mali_kbase_pm.h>
-
-#if KBASE_PM_EN
-
-static u64 demand_get_core_mask(struct kbase_device *kbdev)
-{
- u64 desired = kbdev->shader_needed_bitmap | kbdev->shader_inuse_bitmap;
-
- if (0 == kbdev->pm.active_count)
- return 0;
-
- return desired;
-}
-
-static mali_bool demand_get_core_active(struct kbase_device *kbdev)
-{
- if (0 == kbdev->pm.active_count)
- return MALI_FALSE;
-
- return MALI_TRUE;
-}
-
-static void demand_init(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
-static void demand_term(struct kbase_device *kbdev)
-{
- CSTD_UNUSED(kbdev);
-}
-
-/** The @ref struct kbase_pm_policy structure for the demand power policy.
- *
- * This is the static structure that defines the demand power policy's callback and name.
- */
-const struct kbase_pm_policy kbase_pm_demand_policy_ops = {
- "demand", /* name */
- demand_init, /* init */
- demand_term, /* term */
- demand_get_core_mask, /* get_core_mask */
- demand_get_core_active, /* get_core_active */
- 0u, /* flags */
- KBASE_PM_POLICY_ID_DEMAND, /* id */
-};
-
-KBASE_EXPORT_TEST_API(kbase_pm_demand_policy_ops)
-#endif /* KBASE_PM_EN */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_pm_demand.h
- * A simple demand based power management policy
- */
-
-#ifndef MALI_KBASE_PM_DEMAND_H
-#define MALI_KBASE_PM_DEMAND_H
-
-/**
- * The demand power management policy has the following characteristics:
- * - When KBase indicates that the GPU will be powered up, but we don't yet
- * know which Job Chains are to be run:
- * - The Shader Cores are not powered up
- * - When KBase indicates that a set of Shader Cores are needed to submit the
- * currently queued Job Chains:
- * - Only those Shader Cores are powered up
- * - When KBase indicates that the GPU need not be powered:
- * - The Shader Cores are powered off, and the GPU itself is powered off too.
- *
- * @note:
- * - KBase indicates the GPU will be powered up when it has a User Process that
- * has just started to submit Job Chains.
- * - KBase indicates the GPU need not be powered when all the Job Chains from
- * User Processes have finished, and it is waiting for a User Process to
- * submit some more Job Chains.
- */
-
-/**
- * Private structure for policy instance data.
- *
- * This contains data that is private to the particular power policy that is active.
- */
-typedef struct kbasep_pm_policy_demand {
- /** No state needed - just have a dummy variable here */
- int dummy;
-} kbasep_pm_policy_demand;
-
-#endif /* MALI_KBASE_PM_DEMAND_H */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_pm_driver.c
- * Base kernel Power Management hardware control
- */
-
-#include <mali_kbase.h>
-#include <mali_kbase_config_defaults.h>
-#include <mali_midg_regmap.h>
-#include <mali_kbase_gator.h>
-#include <mali_kbase_pm.h>
-#include <mali_kbase_config_defaults.h>
-
-#if KBASE_PM_EN
-
-#if MALI_MOCK_TEST
-#define MOCKABLE(function) function##_original
-#else
-#define MOCKABLE(function) function
-#endif /* MALI_MOCK_TEST */
-
-/** Actions that can be performed on a core.
- *
- * This enumeration is private to the file. Its values are set to allow @ref core_type_to_reg function,
- * which decodes this enumeration, to be simpler and more efficient.
- */
-enum kbasep_pm_action {
- ACTION_PRESENT = 0,
- ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO),
- ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO),
- ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO),
- ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO),
- ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO)
-};
-
-/** Decode a core type and action to a register.
- *
- * Given a core type (defined by @ref kbase_pm_core_type) and an action (defined by @ref kbasep_pm_action) this
- * function will return the register offset that will perform the action on the core type. The register returned is
- * the \c _LO register and an offset must be applied to use the \c _HI register.
- *
- * @param core_type The type of core
- * @param action The type of action
- *
- * @return The register offset of the \c _LO register that performs an action of type \c action on a core of type \c
- * core_type.
- */
-static u32 core_type_to_reg(enum kbase_pm_core_type core_type, enum kbasep_pm_action action)
-{
- return (u32)core_type + (u32)action;
-}
-
-/** Invokes an action on a core set
- *
- * This function performs the action given by \c action on a set of cores of a type given by \c core_type. It is a
- * static function used by @ref kbase_pm_transition_core_type
- *
- * @param kbdev The kbase device structure of the device
- * @param core_type The type of core that the action should be performed on
- * @param cores A bit mask of cores to perform the action on (low 32 bits)
- * @param action The action to perform on the cores
- */
-STATIC void kbase_pm_invoke(struct kbase_device *kbdev, enum kbase_pm_core_type core_type, u64 cores, enum kbasep_pm_action action)
-{
- u32 reg;
- u32 lo = cores & 0xFFFFFFFF;
- u32 hi = (cores >> 32) & 0xFFFFFFFF;
-
- lockdep_assert_held(&kbdev->pm.power_change_lock);
-
- reg = core_type_to_reg(core_type, action);
-
- KBASE_DEBUG_ASSERT(reg);
-#ifdef CONFIG_MALI_GATOR_SUPPORT
- if (cores) {
- if (action == ACTION_PWRON)
- kbase_trace_mali_pm_power_on(core_type, cores);
- else if (action == ACTION_PWROFF)
- kbase_trace_mali_pm_power_off(core_type, cores);
- }
-#endif /* CONFIG_MALI_GATOR_SUPPORT */
- /* Tracing */
- if (cores) {
- if (action == ACTION_PWRON)
- switch (core_type) {
- case KBASE_PM_CORE_SHADER:
- KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, lo);
- break;
- case KBASE_PM_CORE_TILER:
- KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, NULL, 0u, lo);
- break;
- case KBASE_PM_CORE_L2:
- KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, 0u, lo);
- break;
- default:
- /* L3 not handled */
- break;
- }
- else if (action == ACTION_PWROFF)
- switch (core_type) {
- case KBASE_PM_CORE_SHADER:
- KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, 0u, lo);
- break;
- case KBASE_PM_CORE_TILER:
- KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, NULL, 0u, lo);
- break;
- case KBASE_PM_CORE_L2:
- KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, 0u, lo);
- break;
- default:
- /* L3 not handled */
- break;
- }
- }
-
- if (lo != 0)
- kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL);
-
- if (hi != 0)
- kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL);
-}
-
-/** Get information about a core set
- *
- * This function gets information (chosen by \c action) about a set of cores of a type given by \c core_type. It is a
- * static function used by @ref kbase_pm_get_present_cores, @ref kbase_pm_get_active_cores, @ref
- * kbase_pm_get_trans_cores and @ref kbase_pm_get_ready_cores.
- *
- * @param kbdev The kbase device structure of the device
- * @param core_type The type of core that the should be queried
- * @param action The property of the cores to query
- *
- * @return A bit mask specifying the state of the cores
- */
-static u64 kbase_pm_get_state(struct kbase_device *kbdev, enum kbase_pm_core_type core_type, enum kbasep_pm_action action)
-{
- u32 reg;
- u32 lo, hi;
-
- reg = core_type_to_reg(core_type, action);
-
- KBASE_DEBUG_ASSERT(reg);
-
- lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL);
- hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL);
-
- return (((u64) hi) << 32) | ((u64) lo);
-}
-
-void kbasep_pm_read_present_cores(struct kbase_device *kbdev)
-{
- kbdev->shader_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_SHADER, ACTION_PRESENT);
- kbdev->tiler_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_TILER, ACTION_PRESENT);
- kbdev->l2_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_L2, ACTION_PRESENT);
- kbdev->l3_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_L3, ACTION_PRESENT);
-
- kbdev->shader_inuse_bitmap = 0;
- kbdev->shader_needed_bitmap = 0;
- kbdev->shader_available_bitmap = 0;
- kbdev->tiler_available_bitmap = 0;
- kbdev->l2_users_count = 0;
- kbdev->l2_available_bitmap = 0;
- kbdev->tiler_needed_cnt = 0;
- kbdev->tiler_inuse_cnt = 0;
-
- memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt));
-}
-
-KBASE_EXPORT_TEST_API(kbasep_pm_read_present_cores)
-
-/** Get the cores that are present
- */
-u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, enum kbase_pm_core_type type)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- switch (type) {
- case KBASE_PM_CORE_L3:
- return kbdev->l3_present_bitmap;
- break;
- case KBASE_PM_CORE_L2:
- return kbdev->l2_present_bitmap;
- break;
- case KBASE_PM_CORE_SHADER:
- return kbdev->shader_present_bitmap;
- break;
- case KBASE_PM_CORE_TILER:
- return kbdev->tiler_present_bitmap;
- break;
- }
- KBASE_DEBUG_ASSERT(0);
- return 0;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores)
-
-/** Get the cores that are "active" (busy processing work)
- */
-u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, enum kbase_pm_core_type type)
-{
- return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores)
-
-/** Get the cores that are transitioning between power states
- */
-u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, enum kbase_pm_core_type type)
-{
- return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores)
-/** Get the cores that are powered on
- */
-u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, enum kbase_pm_core_type type)
-{
- u64 result;
-
- result = kbase_pm_get_state(kbdev, type, ACTION_READY);
-
- switch (type) {
- case KBASE_PM_CORE_SHADER:
- KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, (u32) result);
- break;
- case KBASE_PM_CORE_TILER:
- KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, (u32) result);
- break;
- case KBASE_PM_CORE_L2:
- KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, (u32) result);
- break;
- default:
- /* NB: L3 not currently traced */
- break;
- }
-
- return result;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores)
-
-/** Perform power transitions for a particular core type.
- *
- * This function will perform any available power transitions to make the actual hardware state closer to the desired
- * state. If a core is currently transitioning then changes to the power state of that call cannot be made until the
- * transition has finished. Cores which are not present in the hardware are ignored if they are specified in the
- * desired_state bitmask, however the return value will always be 0 in this case.
- *
- * @param kbdev The kbase device
- * @param type The core type to perform transitions for
- * @param desired_state A bit mask of the desired state of the cores
- * @param in_use A bit mask of the cores that are currently running jobs.
- * These cores have to be kept powered up because there are jobs
- * running (or about to run) on them.
- * @param[out] available Receives a bit mask of the cores that the job scheduler can use to submit jobs to.
- * May be NULL if this is not needed.
- * @param[in,out] powering_on Bit mask to update with cores that are transitioning to a power-on state.
- *
- * @return MALI_TRUE if the desired state has been reached, MALI_FALSE otherwise
- */
-STATIC mali_bool kbase_pm_transition_core_type(struct kbase_device *kbdev, enum kbase_pm_core_type type, u64 desired_state,
- u64 in_use, u64 * const available, u64 *powering_on)
-{
- u64 present;
- u64 ready;
- u64 trans;
- u64 powerup;
- u64 powerdown;
- u64 powering_on_trans;
- u64 desired_state_in_use;
-
- lockdep_assert_held(&kbdev->pm.power_change_lock);
-
- /* Get current state */
- present = kbase_pm_get_present_cores(kbdev, type);
- trans = kbase_pm_get_trans_cores(kbdev, type);
- ready = kbase_pm_get_ready_cores(kbdev, type);
- /* mask off ready from trans in case transitions finished between the register reads */
- trans &= ~ready;
-
- powering_on_trans = trans & *powering_on;
- *powering_on = powering_on_trans;
-
- if (available != NULL)
- *available = (ready | powering_on_trans) & desired_state;
-
- /* Update desired state to include the in-use cores. These have to be kept powered up because there are jobs
- * running or about to run on these cores
- */
- desired_state_in_use = desired_state | in_use;
-
- /* Update state of whether l2 caches are powered */
- if (type == KBASE_PM_CORE_L2) {
- if ((ready == present) && (desired_state_in_use == ready) && (trans == 0)) {
- /* All are ready, none will be turned off, and none are transitioning */
- kbdev->pm.l2_powered = 1;
- if (kbdev->l2_users_count > 0) {
- /* Notify any registered l2 cache users (optimized out when no users waiting) */
- wake_up(&kbdev->pm.l2_powered_wait);
- }
- } else {
- kbdev->pm.l2_powered = 0;
- }
- }
-
- if (desired_state_in_use == ready && (trans == 0))
- return MALI_TRUE;
-
- /* Restrict the cores to those that are actually present */
- powerup = desired_state_in_use & present;
- powerdown = (~desired_state_in_use) & present;
-
- /* Restrict to cores that are not already in the desired state */
- powerup &= ~ready;
- powerdown &= ready;
-
- /* Don't transition any cores that are already transitioning, except for
- * Mali cores that support the following case:
- *
- * If the SHADER_PWRON or TILER_PWRON registers are written to turn on
- * a core that is currently transitioning to power off, then this is
- * remembered and the shader core is automatically powered up again once
- * the original transition completes. Once the automatic power on is
- * complete any job scheduled on the shader core should start.
- */
- powerdown &= ~trans;
-
- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS))
- if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type)
- trans = powering_on_trans; /* for exception cases, only mask off cores in power on transitions */
-
- powerup &= ~trans;
-
- /* Perform transitions if any */
- kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON);
- kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF);
-
- /* Recalculate cores transitioning on, and re-evaluate our state */
- powering_on_trans |= powerup;
- *powering_on = powering_on_trans;
- if (available != NULL)
- *available = (ready | powering_on_trans) & desired_state;
-
- return MALI_FALSE;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type)
-
-/** Determine which caches should be on for a particular core state.
- *
- * This function takes a bit mask of the present caches and the cores (or caches) that are attached to the caches that
- * will be powered. It then computes which caches should be turned on to allow the cores requested to be powered up.
- *
- * @param present The bit mask of present caches
- * @param cores_powered A bit mask of cores (or L2 caches) that are desired to be powered
- *
- * @return A bit mask of the caches that should be turned on
- */
-STATIC u64 get_desired_cache_status(u64 present, u64 cores_powered)
-{
- u64 desired = 0;
-
- while (present) {
- /* Find out which is the highest set bit */
- u64 bit = fls64(present) - 1;
- u64 bit_mask = 1ull << bit;
- /* Create a mask which has all bits from 'bit' upwards set */
-
- u64 mask = ~(bit_mask - 1);
-
- /* If there are any cores powered at this bit or above (that haven't previously been processed) then we need
- * this core on */
- if (cores_powered & mask)
- desired |= bit_mask;
-
- /* Remove bits from cores_powered and present */
- cores_powered &= ~mask;
- present &= ~bit_mask;
- }
-
- return desired;
-}
-
-KBASE_EXPORT_TEST_API(get_desired_cache_status)
-
-mali_bool MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev)
-{
- mali_bool cores_are_available = MALI_FALSE;
- mali_bool in_desired_state = MALI_TRUE;
- u64 desired_l2_state;
- u64 desired_l3_state;
- u64 cores_powered;
- u64 tiler_available_bitmap;
- u64 shader_available_bitmap;
- u64 shader_ready_bitmap;
- u64 shader_transitioning_bitmap;
- u64 l2_available_bitmap;
- u64 prev_l2_available_bitmap;
-
- KBASE_DEBUG_ASSERT(NULL != kbdev);
- lockdep_assert_held(&kbdev->pm.power_change_lock);
-
- spin_lock(&kbdev->pm.gpu_powered_lock);
- if (kbdev->pm.gpu_powered == MALI_FALSE) {
- spin_unlock(&kbdev->pm.gpu_powered_lock);
- if (kbdev->pm.desired_shader_state == 0 && kbdev->pm.desired_tiler_state == 0)
- return MALI_TRUE;
- return MALI_FALSE;
- }
-
- /* Trace that a change-state is being requested, and that it took
- * (effectively) no time to start it. This is useful for counting how many
- * state changes occurred, in a way that's backwards-compatible with
- * processing the trace data */
- kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
- kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
-
- /* If any cores are already powered then, we must keep the caches on */
- cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
-
- cores_powered |= kbdev->pm.desired_shader_state;
-
- /* If there are l2 cache users registered, keep all l2s powered even if all other cores are off. */
- if (kbdev->l2_users_count > 0)
- cores_powered |= kbdev->l2_present_bitmap;
-
- desired_l2_state = get_desired_cache_status(kbdev->l2_present_bitmap, cores_powered);
-
- /* If any l2 cache is on, then enable l2 #0, for use by job manager */
- if (0 != desired_l2_state) {
- desired_l2_state |= 1;
- /* Also enable tiler if l2 cache is powered */
- kbdev->pm.desired_tiler_state = kbdev->tiler_present_bitmap;
- } else {
- kbdev->pm.desired_tiler_state = 0;
- }
-
- desired_l3_state = get_desired_cache_status(kbdev->l3_present_bitmap, desired_l2_state);
- prev_l2_available_bitmap = kbdev->l2_available_bitmap;
- in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_L3, desired_l3_state, 0, NULL, &kbdev->pm.powering_on_l3_state);
- in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_L2, desired_l2_state, 0, &l2_available_bitmap, &kbdev->pm.powering_on_l2_state);
-
- if (kbdev->l2_available_bitmap != l2_available_bitmap) {
- KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap);
- }
-
- kbdev->l2_available_bitmap = l2_available_bitmap;
-
- if (in_desired_state) {
- in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_TILER, kbdev->pm.desired_tiler_state, 0, &tiler_available_bitmap, &kbdev->pm.powering_on_tiler_state);
- in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_SHADER, kbdev->pm.desired_shader_state, kbdev->shader_inuse_bitmap, &shader_available_bitmap, &kbdev->pm.powering_on_shader_state);
-
- if (kbdev->shader_available_bitmap != shader_available_bitmap) {
- KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, NULL, 0u, (u32) shader_available_bitmap);
- KBASE_TIMELINE_POWER_SHADER(kbdev, shader_available_bitmap);
- }
-
- kbdev->shader_available_bitmap = shader_available_bitmap;
-
- if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
- KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, NULL, 0u, (u32) tiler_available_bitmap);
- KBASE_TIMELINE_POWER_TILER(kbdev, tiler_available_bitmap);
- }
-
- kbdev->tiler_available_bitmap = tiler_available_bitmap;
-
- } else if ((l2_available_bitmap & kbdev->tiler_present_bitmap) != kbdev->tiler_present_bitmap) {
- tiler_available_bitmap = 0;
-
- if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
- KBASE_TIMELINE_POWER_TILER(kbdev, tiler_available_bitmap);
- }
-
- kbdev->tiler_available_bitmap = tiler_available_bitmap;
- }
-
- /* State updated for slow-path waiters */
- kbdev->pm.gpu_in_desired_state = in_desired_state;
-
- shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
- shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER);
-
- /* Determine whether the cores are now available (even if the set of
- * available cores is empty). Note that they can be available even if we've
- * not finished transitioning to the desired state */
- if ((kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state) == kbdev->pm.desired_shader_state &&
- (kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state) == kbdev->pm.desired_tiler_state) {
- cores_are_available = MALI_TRUE;
-
- KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u, (u32)(kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state));
- KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u, (u32)(kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state));
-
- /* Log timelining information about handling events that power up
- * cores, to match up either with immediate submission either because
- * cores already available, or from PM IRQ */
- if (!in_desired_state)
- kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
- }
-
- if (in_desired_state) {
- KBASE_DEBUG_ASSERT(cores_are_available);
-
-#ifdef CONFIG_MALI_GATOR_SUPPORT
- kbase_trace_mali_pm_status(KBASE_PM_CORE_L3, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L3));
- kbase_trace_mali_pm_status(KBASE_PM_CORE_L2, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2));
- kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER));
- kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER));
-#endif /* CONFIG_MALI_GATOR_SUPPORT */
-
- KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, kbdev->pm.gpu_in_desired_state, (u32)kbdev->pm.desired_shader_state);
- KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u, (u32)kbdev->pm.desired_tiler_state);
-
- /* Log timelining information for synchronous waiters */
- kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
- /* Wake slow-path waiters. Job scheduler does not use this. */
- KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0);
- wake_up(&kbdev->pm.gpu_in_desired_state_wait);
- }
-
- spin_unlock(&kbdev->pm.gpu_powered_lock);
-
- /* kbase_pm_ca_update_core_status can cause one-level recursion into
- * this function, so it must only be called once all changes to kbdev
- * have been committed, and after the gpu_powered_lock has been
- * dropped. */
- if (kbdev->shader_ready_bitmap != shader_ready_bitmap ||
- kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) {
- kbdev->shader_ready_bitmap = shader_ready_bitmap;
- kbdev->shader_transitioning_bitmap = shader_transitioning_bitmap;
-
- kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap, shader_transitioning_bitmap);
- }
-
- /* The core availability policy is not allowed to keep core group 0 turned off (unless it was changing the l2 power state) */
- if (!((shader_ready_bitmap | shader_transitioning_bitmap) & kbdev->gpu_props.props.coherency_info.group[0].core_mask) &&
- (prev_l2_available_bitmap == desired_l2_state) &&
- !(kbase_pm_ca_get_core_mask(kbdev) & kbdev->gpu_props.props.coherency_info.group[0].core_mask))
- BUG();
-
- /* The core availability policy is allowed to keep core group 1 off,
- * but all jobs specifically targeting CG1 must fail */
- if (!((shader_ready_bitmap | shader_transitioning_bitmap) & kbdev->gpu_props.props.coherency_info.group[1].core_mask) &&
- !(kbase_pm_ca_get_core_mask(kbdev) & kbdev->gpu_props.props.coherency_info.group[1].core_mask))
- kbdev->pm.cg1_disabled = MALI_TRUE;
- else
- kbdev->pm.cg1_disabled = MALI_FALSE;
-
- return cores_are_available;
-}
-KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock)
-
-void kbase_pm_check_transitions_sync(struct kbase_device *kbdev)
-{
- unsigned long flags;
- mali_bool cores_are_available;
- /* Force the transition to be checked and reported - the cores may be
- * 'available' (for job submission) but not fully powered up. */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
- /* Don't need 'cores_are_available', because we don't return anything */
- CSTD_UNUSED(cores_are_available);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- /* Wait for cores */
- wait_event(kbdev->pm.gpu_in_desired_state_wait, kbdev->pm.gpu_in_desired_state);
-
- /* Log timelining information that a change in state has completed */
- kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
-}
-KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync)
-
-void kbase_pm_enable_interrupts(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(NULL != kbdev);
- /*
- * Clear all interrupts,
- * and unmask them all.
- */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, NULL);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL, NULL);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, NULL);
- kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL);
-
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts)
-
-void kbase_pm_disable_interrupts(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(NULL != kbdev);
- /*
- * Mask all interrupts,
- * and clear them all.
- */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, NULL);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL);
- kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, NULL);
-
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL);
- kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts)
-
-/*
- * pmu layout:
- * 0x0000: PMU TAG (RO) (0xCAFECAFE)
- * 0x0004: PMU VERSION ID (RO) (0x00000000)
- * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE)
- */
-void kbase_pm_clock_on(struct kbase_device *kbdev, mali_bool is_resume)
-{
- mali_bool reset_required = is_resume;
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(NULL != kbdev);
- lockdep_assert_held(&kbdev->pm.lock);
-
- if (kbdev->pm.gpu_powered) {
- /* Already turned on */
- KBASE_DEBUG_ASSERT(!is_resume);
- return;
- }
-
- KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u);
-
- if (is_resume && kbdev->pm.callback_power_resume) {
- kbdev->pm.callback_power_resume(kbdev);
- } else if (kbdev->pm.callback_power_on) {
- kbdev->pm.callback_power_on(kbdev);
- /* If your platform properly keeps the GPU state you may use the return
- * value of the callback_power_on function to conditionally reset the
- * GPU on power up. Currently we are conservative and always reset the
- * GPU. */
- reset_required = MALI_TRUE;
- }
-
- spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
- kbdev->pm.gpu_powered = MALI_TRUE;
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
-
- if (reset_required) {
- /* GPU state was lost, reset GPU to ensure it is in a
- * consistent state */
- kbase_pm_init_hw(kbdev, MALI_TRUE);
- }
-
- /* Lastly, enable the interrupts */
- kbase_pm_enable_interrupts(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_clock_on)
-
-void kbase_pm_clock_off(struct kbase_device *kbdev, mali_bool is_suspend)
-{
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(NULL != kbdev);
- lockdep_assert_held(&kbdev->pm.lock);
-
- /* ASSERT that the cores should now be unavailable. No lock needed. */
- KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u);
-
- if (!kbdev->pm.gpu_powered) {
- /* Already turned off */
- if (is_suspend && kbdev->pm.callback_power_suspend)
- kbdev->pm.callback_power_suspend(kbdev);
- return;
- }
-
- KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u);
-
- /* Disable interrupts. This also clears any outstanding interrupts */
- kbase_pm_disable_interrupts(kbdev);
- /* Ensure that any IRQ handlers have finished */
- kbase_synchronize_irqs(kbdev);
-
- /* The GPU power may be turned off from this point */
- spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
- kbdev->pm.gpu_powered = MALI_FALSE;
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
-
- if (is_suspend && kbdev->pm.callback_power_suspend)
- kbdev->pm.callback_power_suspend(kbdev);
- else if (kbdev->pm.callback_power_off)
- kbdev->pm.callback_power_off(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_clock_off)
-
-struct kbasep_reset_timeout_data {
- struct hrtimer timer;
- mali_bool timed_out;
- struct kbase_device *kbdev;
-};
-
-void kbase_pm_reset_done(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- kbdev->pm.reset_done = MALI_TRUE;
- wake_up(&kbdev->pm.reset_done_wait);
-}
-
-/**
- * Wait for the RESET_COMPLETED IRQ to occur, then reset the waiting state.
- */
-STATIC void kbase_pm_wait_for_reset(struct kbase_device *kbdev)
-{
- lockdep_assert_held(&kbdev->pm.lock);
-
- wait_event(kbdev->pm.reset_done_wait, (kbdev->pm.reset_done));
- kbdev->pm.reset_done = MALI_FALSE;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_reset_done)
-
-static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
-{
- struct kbasep_reset_timeout_data *rtdata = container_of(timer, struct kbasep_reset_timeout_data, timer);
-
- rtdata->timed_out = 1;
-
- /* Set the wait queue to wake up kbase_pm_init_hw even though the reset hasn't completed */
- kbase_pm_reset_done(rtdata->kbdev);
-
- return HRTIMER_NORESTART;
-}
-
-static void kbase_pm_hw_issues(struct kbase_device *kbdev)
-{
- u32 value = 0;
-
- /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443.
- * and
- * needed due to MIDGLES-3539. See PRLAM-11035 */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) ||
- kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035))
- value |= SC_LS_PAUSEBUFFER_DISABLE;
-
- /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327. */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327))
- value |= SC_SDC_DISABLE_OQ_DISCARD;
-
- /* Enable alternative hardware counter selection if configured. */
- if (DEFAULT_ALTERNATIVE_HWC)
- value |= SC_ALT_COUNTERS;
-
- /* Use software control of forward pixel kill when needed. See MIDEUR-174. */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_2121))
- value |= SC_OVERRIDE_FWD_PIXEL_KILL;
-
- /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797))
- value |= SC_ENABLE_TEXGRD_FLAGS;
-
- if (value != 0)
- kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), value, NULL);
-
- /* Set tiler clock gate override if required */
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) {
- value = kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG), NULL);
- value |= TC_CLOCK_GATE_OVERRIDE;
- kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG), value, NULL);
- }
-
- /* Limit the GPU bus bandwidth if the platform needs this. */
- value = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL);
-
- /* Limit read ID width for AXI */
- value &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS);
- value |= (DEFAULT_ARID_LIMIT & 0x3) << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT;
-
- /* Limit write ID width for AXI */
- value &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES);
- value |= (DEFAULT_AWID_LIMIT & 0x3) << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT;
-
- kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), value, NULL);
-}
-
-mali_error kbase_pm_init_hw(struct kbase_device *kbdev, mali_bool enable_irqs)
-{
- unsigned long flags;
- struct kbasep_reset_timeout_data rtdata;
-
- KBASE_DEBUG_ASSERT(NULL != kbdev);
- lockdep_assert_held(&kbdev->pm.lock);
-
- /* Ensure the clock is on before attempting to access the hardware */
- if (!kbdev->pm.gpu_powered) {
- if (kbdev->pm.callback_power_on)
- kbdev->pm.callback_power_on(kbdev);
-
- spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
- kbdev->pm.gpu_powered = MALI_TRUE;
- spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
- }
-
- /* Ensure interrupts are off to begin with, this also clears any outstanding interrupts */
- kbase_pm_disable_interrupts(kbdev);
-
- /* Prepare for the soft-reset */
- kbdev->pm.reset_done = MALI_FALSE;
-
- /* The cores should be made unavailable due to the reset */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- if (kbdev->shader_available_bitmap != 0u)
- KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, NULL, 0u, (u32)0u);
- if (kbdev->tiler_available_bitmap != 0u)
- KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, NULL, 0u, (u32)0u);
- kbdev->shader_available_bitmap = 0u;
- kbdev->tiler_available_bitmap = 0u;
- kbdev->l2_available_bitmap = 0u;
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- /* Soft reset the GPU */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_SOFT_RESET, NULL);
-
- /* Unmask the reset complete interrupt only */
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED, NULL);
-
- /* Initialize a structure for tracking the status of the reset */
- rtdata.kbdev = kbdev;
- rtdata.timed_out = 0;
-
- /* Create a timer to use as a timeout on the reset */
- hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- rtdata.timer.function = kbasep_reset_timeout;
-
- hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL);
-
- /* Wait for the RESET_COMPLETED interrupt to be raised */
- kbase_pm_wait_for_reset(kbdev);
-
- if (rtdata.timed_out == 0) {
- /* GPU has been reset */
- hrtimer_cancel(&rtdata.timer);
- destroy_hrtimer_on_stack(&rtdata.timer);
- goto out;
- }
-
- /* No interrupt has been received - check if the RAWSTAT register says the reset has completed */
- if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & RESET_COMPLETED) {
- /* The interrupt is set in the RAWSTAT; this suggests that the interrupts are not getting to the CPU */
- dev_warn(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
- /* If interrupts aren't working we can't continue. */
- destroy_hrtimer_on_stack(&rtdata.timer);
- goto out;
- }
-
- /* The GPU doesn't seem to be responding to the reset so try a hard reset */
- dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", RESET_TIMEOUT);
- KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET, NULL);
-
- /* Restart the timer to wait for the hard reset to complete */
- rtdata.timed_out = 0;
-
- hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL);
-
- /* Wait for the RESET_COMPLETED interrupt to be raised */
- kbase_pm_wait_for_reset(kbdev);
-
- if (rtdata.timed_out == 0) {
- /* GPU has been reset */
- hrtimer_cancel(&rtdata.timer);
- destroy_hrtimer_on_stack(&rtdata.timer);
- goto out;
- }
-
- destroy_hrtimer_on_stack(&rtdata.timer);
-
- dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", RESET_TIMEOUT);
-
- /* The GPU still hasn't reset, give up */
- return MALI_ERROR_FUNCTION_FAILED;
-
- out:
-
- /* If cycle counter was in use-re enable it enable_irqs will only be false when called from kbase_pm_powerup */
- if (kbdev->pm.gpu_cycle_counter_requests && enable_irqs) {
- /* enable interrupts as the L2 may have to be powered on */
- kbase_pm_enable_interrupts(kbdev);
- kbase_pm_request_l2_caches(kbdev);
-
- /* Re-enable the counters if we need to */
- spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
- if (kbdev->pm.gpu_cycle_counter_requests)
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START, NULL);
- spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
-
- kbase_pm_release_l2_caches(kbdev);
- kbase_pm_disable_interrupts(kbdev);
- }
-
- if (enable_irqs)
- kbase_pm_enable_interrupts(kbdev);
-
- kbase_pm_hw_issues(kbdev);
-
- return MALI_ERROR_NONE;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_init_hw)
-
-/** Increase the count of cycle counter users and turn the cycle counters on if they were previously off
- *
- * this function is designed to be called by @ref kbase_pm_request_gpu_cycle_counter or
- * @ref kbase_pm_request_gpu_cycle_counter_l2_is_on only
- *
- * When this function is called the l2 cache must be on and the l2 cache users count must
- * have been incremented by a call to (@ref kbase_pm_request_l2_caches or @ref kbase_pm_request_l2_caches_l2_on)
- *
- * @param kbdev The kbase device structure of the device
- *
- */
-static void kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
-
- ++kbdev->pm.gpu_cycle_counter_requests;
-
- if (1 == kbdev->pm.gpu_cycle_counter_requests)
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START, NULL);
-
- spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
-}
-
-void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- KBASE_DEBUG_ASSERT(kbdev->pm.gpu_powered);
-
- KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests < INT_MAX);
-
- kbase_pm_request_l2_caches(kbdev);
-
- kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter)
-
-void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- KBASE_DEBUG_ASSERT(kbdev->pm.gpu_powered);
-
- KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests < INT_MAX);
-
- kbase_pm_request_l2_caches_l2_is_on(kbdev);
-
- kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on)
-
-void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
-
- KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests > 0);
-
- --kbdev->pm.gpu_cycle_counter_requests;
-
- if (0 == kbdev->pm.gpu_cycle_counter_requests)
- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
-
- spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
-
- kbase_pm_release_l2_caches(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter)
-#endif /* KBASE_PM_EN */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_pm_metrics.c
- * Metrics for power management
- */
-
-#include <mali_kbase.h>
-#include <mali_kbase_pm.h>
-#if KBASE_PM_EN
-/* When VSync is being hit aim for utilisation between 70-90% */
-#define KBASE_PM_VSYNC_MIN_UTILISATION 70
-#define KBASE_PM_VSYNC_MAX_UTILISATION 90
-/* Otherwise aim for 10-40% */
-#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10
-#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40
-
-/* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns
- This gives a maximum period between samples of 2^(32+8)/100 ns = slightly under 11s.
- Exceeding this will cause overflow */
-#define KBASE_PM_TIME_SHIFT 8
-
-/* Maximum time between sampling of utilization data, without resetting the
- * counters. */
-#define MALI_UTILIZATION_MAX_PERIOD 100000 /* ns = 100ms */
-
-#ifdef CONFIG_MALI_MIDGARD_DVFS
-static enum hrtimer_restart dvfs_callback(struct hrtimer *timer)
-{
- unsigned long flags;
- enum kbase_pm_dvfs_action action;
- struct kbasep_pm_metrics_data *metrics;
-
- KBASE_DEBUG_ASSERT(timer != NULL);
-
- metrics = container_of(timer, struct kbasep_pm_metrics_data, timer);
- action = kbase_pm_get_dvfs_action(metrics->kbdev);
-
- spin_lock_irqsave(&metrics->lock, flags);
-
- if (metrics->timer_active)
- hrtimer_start(timer,
- HR_TIMER_DELAY_MSEC(metrics->kbdev->pm.platform_dvfs_frequency),
- HRTIMER_MODE_REL);
-
- spin_unlock_irqrestore(&metrics->lock, flags);
-
- return HRTIMER_NORESTART;
-}
-#endif /* CONFIG_MALI_MIDGARD_DVFS */
-
-mali_error kbasep_pm_metrics_init(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- kbdev->pm.metrics.kbdev = kbdev;
- kbdev->pm.metrics.vsync_hit = 0;
- kbdev->pm.metrics.utilisation = 0;
- kbdev->pm.metrics.util_cl_share[0] = 0;
- kbdev->pm.metrics.util_cl_share[1] = 0;
- kbdev->pm.metrics.util_gl_share = 0;
-
- kbdev->pm.metrics.time_period_start = ktime_get();
- kbdev->pm.metrics.time_busy = 0;
- kbdev->pm.metrics.time_idle = 0;
- kbdev->pm.metrics.prev_busy = 0;
- kbdev->pm.metrics.prev_idle = 0;
- kbdev->pm.metrics.gpu_active = MALI_TRUE;
- kbdev->pm.metrics.active_cl_ctx[0] = 0;
- kbdev->pm.metrics.active_cl_ctx[1] = 0;
- kbdev->pm.metrics.active_gl_ctx = 0;
- kbdev->pm.metrics.busy_cl[0] = 0;
- kbdev->pm.metrics.busy_cl[1] = 0;
- kbdev->pm.metrics.busy_gl = 0;
-
- spin_lock_init(&kbdev->pm.metrics.lock);
-
-#ifdef CONFIG_MALI_MIDGARD_DVFS
- kbdev->pm.metrics.timer_active = MALI_TRUE;
- hrtimer_init(&kbdev->pm.metrics.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- kbdev->pm.metrics.timer.function = dvfs_callback;
-
- hrtimer_start(&kbdev->pm.metrics.timer, HR_TIMER_DELAY_MSEC(kbdev->pm.platform_dvfs_frequency), HRTIMER_MODE_REL);
-#endif /* CONFIG_MALI_MIDGARD_DVFS */
-
- kbase_pm_register_vsync_callback(kbdev);
-
- return MALI_ERROR_NONE;
-}
-
-KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init)
-
-void kbasep_pm_metrics_term(struct kbase_device *kbdev)
-{
-#ifdef CONFIG_MALI_MIDGARD_DVFS
- unsigned long flags;
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbdev->pm.metrics.timer_active = MALI_FALSE;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-
- hrtimer_cancel(&kbdev->pm.metrics.timer);
-#endif /* CONFIG_MALI_MIDGARD_DVFS */
-
- kbase_pm_unregister_vsync_callback(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term)
-
-/*caller needs to hold kbdev->pm.metrics.lock before calling this function*/
-void kbasep_pm_record_job_status(struct kbase_device *kbdev)
-{
- ktime_t now;
- ktime_t diff;
- u32 ns_time;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- now = ktime_get();
- diff = ktime_sub(now, kbdev->pm.metrics.time_period_start);
-
- ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT);
- kbdev->pm.metrics.time_busy += ns_time;
- kbdev->pm.metrics.busy_gl += ns_time * kbdev->pm.metrics.active_gl_ctx;
- kbdev->pm.metrics.busy_cl[0] += ns_time * kbdev->pm.metrics.active_cl_ctx[0];
- kbdev->pm.metrics.busy_cl[1] += ns_time * kbdev->pm.metrics.active_cl_ctx[1];
- kbdev->pm.metrics.time_period_start = now;
-}
-
-KBASE_EXPORT_TEST_API(kbasep_pm_record_job_status)
-
-void kbasep_pm_record_gpu_idle(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
-
- KBASE_DEBUG_ASSERT(kbdev->pm.metrics.gpu_active == MALI_TRUE);
-
- kbdev->pm.metrics.gpu_active = MALI_FALSE;
-
- kbasep_pm_record_job_status(kbdev);
-
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-}
-
-KBASE_EXPORT_TEST_API(kbasep_pm_record_gpu_idle)
-
-void kbasep_pm_record_gpu_active(struct kbase_device *kbdev)
-{
- unsigned long flags;
- ktime_t now;
- ktime_t diff;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
-
- KBASE_DEBUG_ASSERT(kbdev->pm.metrics.gpu_active == MALI_FALSE);
-
- kbdev->pm.metrics.gpu_active = MALI_TRUE;
-
- now = ktime_get();
- diff = ktime_sub(now, kbdev->pm.metrics.time_period_start);
-
- kbdev->pm.metrics.time_idle += (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT);
- kbdev->pm.metrics.time_period_start = now;
-
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-}
-
-KBASE_EXPORT_TEST_API(kbasep_pm_record_gpu_active)
-
-void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated)
-{
- unsigned long flags;
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbdev->pm.metrics.vsync_hit = buffer_updated;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_report_vsync)
-
-/*caller needs to hold kbdev->pm.metrics.lock before calling this function*/
-static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, ktime_t now)
-{
- ktime_t diff;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- diff = ktime_sub(now, kbdev->pm.metrics.time_period_start);
-
- if (kbdev->pm.metrics.gpu_active) {
- u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT);
- kbdev->pm.metrics.time_busy += ns_time;
- kbdev->pm.metrics.busy_cl[0] += ns_time * kbdev->pm.metrics.active_cl_ctx[0];
- kbdev->pm.metrics.busy_cl[1] += ns_time * kbdev->pm.metrics.active_cl_ctx[1];
- kbdev->pm.metrics.busy_gl += ns_time * kbdev->pm.metrics.active_gl_ctx;
- } else {
- kbdev->pm.metrics.time_idle += (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT);
- }
-}
-
-/* Caller needs to hold kbdev->pm.metrics.lock before calling this function. */
-static void kbase_pm_reset_dvfs_utilisation_unlocked(struct kbase_device *kbdev, ktime_t now)
-{
- /* Store previous value */
- kbdev->pm.metrics.prev_idle = kbdev->pm.metrics.time_idle;
- kbdev->pm.metrics.prev_busy = kbdev->pm.metrics.time_busy;
-
- /* Reset current values */
- kbdev->pm.metrics.time_period_start = now;
- kbdev->pm.metrics.time_idle = 0;
- kbdev->pm.metrics.time_busy = 0;
- kbdev->pm.metrics.busy_cl[0] = 0;
- kbdev->pm.metrics.busy_cl[1] = 0;
- kbdev->pm.metrics.busy_gl = 0;
-}
-
-void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, ktime_get());
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-}
-
-void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev,
- unsigned long *total_out, unsigned long *busy_out)
-{
- ktime_t now = ktime_get();
- unsigned long flags, busy, total;
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbase_pm_get_dvfs_utilisation_calc(kbdev, now);
-
- busy = kbdev->pm.metrics.time_busy;
- total = busy + kbdev->pm.metrics.time_idle;
-
- /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default
- * 100ms) */
- if (total >= MALI_UTILIZATION_MAX_PERIOD) {
- kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now);
- } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) {
- total += kbdev->pm.metrics.prev_idle +
- kbdev->pm.metrics.prev_busy;
- busy += kbdev->pm.metrics.prev_busy;
- }
-
- *total_out = total;
- *busy_out = busy;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-}
-
-#ifdef CONFIG_MALI_MIDGARD_DVFS
-
-/*caller needs to hold kbdev->pm.metrics.lock before calling this function*/
-int kbase_pm_get_dvfs_utilisation_old(struct kbase_device *kbdev, int *util_gl_share, int util_cl_share[2])
-{
- int utilisation;
- int busy;
- ktime_t now = ktime_get();
-
- kbase_pm_get_dvfs_utilisation_calc(kbdev, now);
-
- if (kbdev->pm.metrics.time_idle + kbdev->pm.metrics.time_busy == 0) {
- /* No data - so we return NOP */
- utilisation = -1;
- if (util_gl_share)
- *util_gl_share = -1;
- if (util_cl_share) {
- util_cl_share[0] = -1;
- util_cl_share[1] = -1;
- }
- goto out;
- }
-
- utilisation = (100 * kbdev->pm.metrics.time_busy) /
- (kbdev->pm.metrics.time_idle +
- kbdev->pm.metrics.time_busy);
-
- busy = kbdev->pm.metrics.busy_gl +
- kbdev->pm.metrics.busy_cl[0] +
- kbdev->pm.metrics.busy_cl[1];
-
- if (busy != 0) {
- if (util_gl_share)
- *util_gl_share =
- (100 * kbdev->pm.metrics.busy_gl) / busy;
- if (util_cl_share) {
- util_cl_share[0] =
- (100 * kbdev->pm.metrics.busy_cl[0]) / busy;
- util_cl_share[1] =
- (100 * kbdev->pm.metrics.busy_cl[1]) / busy;
- }
- } else {
- if (util_gl_share)
- *util_gl_share = -1;
- if (util_cl_share) {
- util_cl_share[0] = -1;
- util_cl_share[1] = -1;
- }
- }
-
-out:
- kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now);
-
- return utilisation;
-}
-
-enum kbase_pm_dvfs_action kbase_pm_get_dvfs_action(struct kbase_device *kbdev)
-{
- unsigned long flags;
- int utilisation, util_gl_share;
- int util_cl_share[2];
- enum kbase_pm_dvfs_action action;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
-
- utilisation = kbase_pm_get_dvfs_utilisation_old(kbdev, &util_gl_share, util_cl_share);
-
- if (utilisation < 0 || util_gl_share < 0 || util_cl_share[0] < 0 || util_cl_share[1] < 0) {
- action = KBASE_PM_DVFS_NOP;
- utilisation = 0;
- util_gl_share = 0;
- util_cl_share[0] = 0;
- util_cl_share[1] = 0;
- goto out;
- }
-
- if (kbdev->pm.metrics.vsync_hit) {
- /* VSync is being met */
- if (utilisation < KBASE_PM_VSYNC_MIN_UTILISATION)
- action = KBASE_PM_DVFS_CLOCK_DOWN;
- else if (utilisation > KBASE_PM_VSYNC_MAX_UTILISATION)
- action = KBASE_PM_DVFS_CLOCK_UP;
- else
- action = KBASE_PM_DVFS_NOP;
- } else {
- /* VSync is being missed */
- if (utilisation < KBASE_PM_NO_VSYNC_MIN_UTILISATION)
- action = KBASE_PM_DVFS_CLOCK_DOWN;
- else if (utilisation > KBASE_PM_NO_VSYNC_MAX_UTILISATION)
- action = KBASE_PM_DVFS_CLOCK_UP;
- else
- action = KBASE_PM_DVFS_NOP;
- }
-
- kbdev->pm.metrics.utilisation = utilisation;
- kbdev->pm.metrics.util_cl_share[0] = util_cl_share[0];
- kbdev->pm.metrics.util_cl_share[1] = util_cl_share[1];
- kbdev->pm.metrics.util_gl_share = util_gl_share;
-out:
-#ifdef CONFIG_MALI_MIDGARD_DVFS
- kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, util_cl_share);
-#endif /*CONFIG_MALI_MIDGARD_DVFS */
- kbdev->pm.metrics.time_idle = 0;
- kbdev->pm.metrics.time_busy = 0;
- kbdev->pm.metrics.busy_cl[0] = 0;
- kbdev->pm.metrics.busy_cl[1] = 0;
- kbdev->pm.metrics.busy_gl = 0;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-
- return action;
-}
-KBASE_EXPORT_TEST_API(kbase_pm_get_dvfs_action)
-
-mali_bool kbase_pm_metrics_is_active(struct kbase_device *kbdev)
-{
- mali_bool isactive;
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- isactive = (kbdev->pm.metrics.timer_active == MALI_TRUE);
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
-
- return isactive;
-}
-KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active)
-
-#endif /* CONFIG_MALI_MIDGARD_DVFS */
-
-#endif /* KBASE_PM_EN */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @file mali_kbase_pm_metrics_dummy.c
- * Dummy Metrics for power management.
- */
-
-#include <mali_kbase.h>
-#include <mali_kbase_pm.h>
-
-#if KBASE_PM_EN
-void kbase_pm_register_vsync_callback(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- /* no VSync metrics will be available */
- kbdev->pm.metrics.platform_data = NULL;
-}
-
-void kbase_pm_unregister_vsync_callback(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-}
-#endif /* KBASE_PM_EN */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * @file mali_kbase_pm_policy.c
- * Power policy API implementations
- */
-
-#include <mali_kbase.h>
-#include <mali_midg_regmap.h>
-#include <mali_kbase_gator.h>
-#include <mali_kbase_pm.h>
-
-#if KBASE_PM_EN
-
-extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops;
-extern const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops;
-extern const struct kbase_pm_policy kbase_pm_demand_policy_ops;
-
-#if !MALI_CUSTOMER_RELEASE
-extern const struct kbase_pm_policy kbase_pm_fast_start_policy_ops;
-extern const struct kbase_pm_policy kbase_pm_demand_always_powered_policy_ops;
-#endif
-
-static const struct kbase_pm_policy *const policy_list[] = {
-#ifdef CONFIG_MALI_NO_MALI
- &kbase_pm_always_on_policy_ops,
- &kbase_pm_demand_policy_ops,
- &kbase_pm_coarse_demand_policy_ops,
-#if !MALI_CUSTOMER_RELEASE
- &kbase_pm_demand_always_powered_policy_ops,
- &kbase_pm_fast_start_policy_ops,
-#endif
-#else /* CONFIG_MALI_NO_MALI */
- &kbase_pm_demand_policy_ops,
- &kbase_pm_always_on_policy_ops,
- &kbase_pm_coarse_demand_policy_ops,
-#if !MALI_CUSTOMER_RELEASE
- &kbase_pm_demand_always_powered_policy_ops,
- &kbase_pm_fast_start_policy_ops,
-#endif
-#endif /* CONFIG_MALI_NO_MALI */
-};
-
-/** The number of policies available in the system.
- * This is derived from the number of functions listed in policy_get_functions.
- */
-#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
-
-
-/* Function IDs for looking up Timeline Trace codes in kbase_pm_change_state_trace_code */
-enum kbase_pm_func_id {
- KBASE_PM_FUNC_ID_REQUEST_CORES_START,
- KBASE_PM_FUNC_ID_REQUEST_CORES_END,
- KBASE_PM_FUNC_ID_RELEASE_CORES_START,
- KBASE_PM_FUNC_ID_RELEASE_CORES_END,
- /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither
- * expect to hit it nor tend to hit it very much anyway. We can detect
- * whether we need more instrumentation by a difference between
- * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */
-
- /* Must be the last */
- KBASE_PM_FUNC_ID_COUNT
-};
-
-
-/* State changes during request/unrequest/release-ing cores */
-enum {
- KBASE_PM_CHANGE_STATE_SHADER = (1u << 0),
- KBASE_PM_CHANGE_STATE_TILER = (1u << 1),
-
- /* These two must be last */
- KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER|KBASE_PM_CHANGE_STATE_SHADER),
- KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1
-};
-typedef u32 kbase_pm_change_state;
-
-
-#ifdef CONFIG_MALI_TRACE_TIMELINE
-/* Timeline Trace code lookups for each function */
-static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT][KBASE_PM_CHANGE_STATE_COUNT] = {
- /* kbase_pm_request_cores */
- [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0,
- [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
- SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START,
- [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
- SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START,
- [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
- SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START,
-
- [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0,
- [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
- SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END,
- [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
- SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END,
- [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
- SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END,
-
- /* kbase_pm_release_cores */
- [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0,
- [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
- SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START,
- [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
- SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START,
- [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
- SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START,
-
- [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0,
- [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
- SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END,
- [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
- SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END,
- [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
- SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END
-};
-
-STATIC INLINE void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
- enum kbase_pm_func_id func_id,
- kbase_pm_change_state state)
-{
- int trace_code;
-
- KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT);
- KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) == state);
-
- trace_code = kbase_pm_change_state_trace_code[func_id][state];
- KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code);
-}
-
-#else /* CONFIG_MALI_TRACE_TIMELINE */
-STATIC INLINE void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
- enum kbase_pm_func_id func_id, kbase_pm_change_state state)
-{
-}
-
-#endif /* CONFIG_MALI_TRACE_TIMELINE */
-
-static enum hrtimer_restart kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer)
-{
- struct kbase_device *kbdev;
-
- kbdev = container_of(timer, struct kbase_device, pm.gpu_poweroff_timer);
-
- /* It is safe for this call to do nothing if the work item is already queued.
- * The worker function will read the must up-to-date state of kbdev->pm.gpu_poweroff_pending
- * under lock.
- *
- * If a state change occurs while the worker function is processing, this
- * call will succeed as a work item can be requeued once it has started
- * processing.
- */
- if (kbdev->pm.gpu_poweroff_pending)
- queue_work(kbdev->pm.gpu_poweroff_wq, &kbdev->pm.gpu_poweroff_work);
-
- if (kbdev->pm.shader_poweroff_pending) {
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- if (kbdev->pm.shader_poweroff_pending) {
- kbdev->pm.shader_poweroff_pending_time--;
-
- KBASE_DEBUG_ASSERT(kbdev->pm.shader_poweroff_pending_time >= 0);
-
- if (kbdev->pm.shader_poweroff_pending_time == 0) {
- u64 prev_shader_state = kbdev->pm.desired_shader_state;
-
- kbdev->pm.desired_shader_state &= ~kbdev->pm.shader_poweroff_pending;
- kbdev->pm.shader_poweroff_pending = 0;
-
- if (prev_shader_state != kbdev->pm.desired_shader_state ||
- kbdev->pm.ca_in_transition != MALI_FALSE) {
- mali_bool cores_are_available;
-
- KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START);
- cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
- KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END);
-
- /* Don't need 'cores_are_available', because we don't return anything */
- CSTD_UNUSED(cores_are_available);
- }
- }
- }
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
- }
-
- if (kbdev->pm.poweroff_timer_needed) {
- hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time);
-
- return HRTIMER_RESTART;
- }
-
- return HRTIMER_NORESTART;
-}
-
-static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data)
-{
- unsigned long flags;
- struct kbase_device *kbdev;
- mali_bool do_poweroff = MALI_FALSE;
-
- kbdev = container_of(data, struct kbase_device, pm.gpu_poweroff_work);
-
- mutex_lock(&kbdev->pm.lock);
-
- if (kbdev->pm.gpu_poweroff_pending == 0) {
- mutex_unlock(&kbdev->pm.lock);
- return;
- }
-
- kbdev->pm.gpu_poweroff_pending--;
-
- if (kbdev->pm.gpu_poweroff_pending > 0) {
- mutex_unlock(&kbdev->pm.lock);
- return;
- }
-
- KBASE_DEBUG_ASSERT(kbdev->pm.gpu_poweroff_pending == 0);
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- /* Only power off the GPU if a request is still pending */
- if (kbdev->pm.pm_current_policy->get_core_active(kbdev) == MALI_FALSE)
- do_poweroff = MALI_TRUE;
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- if (do_poweroff != MALI_FALSE) {
- kbdev->pm.poweroff_timer_needed = MALI_FALSE;
- /* Power off the GPU */
- kbase_pm_do_poweroff(kbdev, MALI_FALSE);
- hrtimer_cancel(&kbdev->pm.gpu_poweroff_timer);
- }
-
- mutex_unlock(&kbdev->pm.lock);
-}
-
-mali_error kbase_pm_policy_init(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- kbdev->pm.gpu_poweroff_wq = alloc_workqueue("kbase_pm_do_poweroff", WQ_HIGHPRI | WQ_UNBOUND, 1);
- if (NULL == kbdev->pm.gpu_poweroff_wq)
- return MALI_ERROR_OUT_OF_MEMORY;
- INIT_WORK(&kbdev->pm.gpu_poweroff_work, kbasep_pm_do_gpu_poweroff_wq);
-
- hrtimer_init(&kbdev->pm.gpu_poweroff_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- kbdev->pm.gpu_poweroff_timer.function = kbasep_pm_do_gpu_poweroff_callback;
-
- kbdev->pm.pm_current_policy = policy_list[0];
-
- kbdev->pm.pm_current_policy->init(kbdev);
-
- kbdev->pm.gpu_poweroff_time = HR_TIMER_DELAY_NSEC(kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS));
-
- kbdev->pm.poweroff_shader_ticks = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_SHADER);
- kbdev->pm.poweroff_gpu_ticks = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_GPU);
-
- return MALI_ERROR_NONE;
-}
-
-void kbase_pm_policy_term(struct kbase_device *kbdev)
-{
- kbdev->pm.pm_current_policy->term(kbdev);
-}
-
-void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- lockdep_assert_held(&kbdev->pm.lock);
-
- kbdev->pm.poweroff_timer_needed = MALI_FALSE;
- hrtimer_cancel(&kbdev->pm.gpu_poweroff_timer);
-
- /* If wq is already running but is held off by pm.lock, make sure it has no effect */
- kbdev->pm.gpu_poweroff_pending = 0;
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- kbdev->pm.shader_poweroff_pending = 0;
- kbdev->pm.shader_poweroff_pending_time = 0;
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-}
-
-void kbase_pm_update_active(struct kbase_device *kbdev)
-{
- unsigned long flags;
- mali_bool active;
-
- lockdep_assert_held(&kbdev->pm.lock);
-
- /* pm_current_policy will never be NULL while pm.lock is held */
- KBASE_DEBUG_ASSERT(kbdev->pm.pm_current_policy);
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- active = kbdev->pm.pm_current_policy->get_core_active(kbdev);
-
- if (active != MALI_FALSE) {
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- if (kbdev->pm.gpu_poweroff_pending) {
- /* Cancel any pending power off request */
- kbdev->pm.gpu_poweroff_pending = 0;
-
- /* If a request was pending then the GPU was still powered, so no need to continue */
- return;
- }
-
- if (!kbdev->pm.poweroff_timer_needed && !kbdev->pm.gpu_powered) {
- kbdev->pm.poweroff_timer_needed = MALI_TRUE;
- hrtimer_start(&kbdev->pm.gpu_poweroff_timer, kbdev->pm.gpu_poweroff_time, HRTIMER_MODE_REL);
- }
-
- /* Power on the GPU and any cores requested by the policy */
- kbase_pm_do_poweron(kbdev, MALI_FALSE);
- } else {
- /* It is an error for the power policy to power off the GPU
- * when there are contexts active */
- KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
-
- if (kbdev->pm.shader_poweroff_pending) {
- kbdev->pm.shader_poweroff_pending = 0;
- kbdev->pm.shader_poweroff_pending_time = 0;
- }
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
-
- /* Request power off */
- if (kbdev->pm.gpu_powered) {
- kbdev->pm.gpu_poweroff_pending = kbdev->pm.poweroff_gpu_ticks;
- if (!kbdev->pm.poweroff_timer_needed) {
- /* Start timer if not running (eg if power policy has been changed from always_on
- * to something else). This will ensure the GPU is actually powered off */
- kbdev->pm.poweroff_timer_needed = MALI_TRUE;
- hrtimer_start(&kbdev->pm.gpu_poweroff_timer, kbdev->pm.gpu_poweroff_time, HRTIMER_MODE_REL);
- }
- }
- }
-}
-
-void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
-{
- u64 desired_bitmap;
- mali_bool cores_are_available;
-
- lockdep_assert_held(&kbdev->pm.power_change_lock);
-
- if (kbdev->pm.pm_current_policy == NULL)
- return;
-
- desired_bitmap = kbdev->pm.pm_current_policy->get_core_mask(kbdev);
- desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);
-
- /* Enable core 0 if tiler required, regardless of core availability */
- if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
- desired_bitmap |= 1;
-
- if (kbdev->pm.desired_shader_state != desired_bitmap)
- KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, (u32)desired_bitmap);
-
- /* Are any cores being powered on? */
- if (~kbdev->pm.desired_shader_state & desired_bitmap ||
- kbdev->pm.ca_in_transition != MALI_FALSE) {
- /* Check if we are powering off any cores before updating shader state */
- if (kbdev->pm.desired_shader_state & ~desired_bitmap) {
- /* Start timer to power off cores */
- kbdev->pm.shader_poweroff_pending |= (kbdev->pm.desired_shader_state & ~desired_bitmap);
- kbdev->pm.shader_poweroff_pending_time = kbdev->pm.poweroff_shader_ticks;
- }
-
- kbdev->pm.desired_shader_state = desired_bitmap;
-
- /* If any cores are being powered on, transition immediately */
- cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
- } else if (kbdev->pm.desired_shader_state & ~desired_bitmap) {
- /* Start timer to power off cores */
- kbdev->pm.shader_poweroff_pending |= (kbdev->pm.desired_shader_state & ~desired_bitmap);
- kbdev->pm.shader_poweroff_pending_time = kbdev->pm.poweroff_shader_ticks;
- } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 && kbdev->pm.poweroff_timer_needed) {
- /* If power policy is keeping cores on despite there being no
- * active contexts then disable poweroff timer as it isn't
- * required.
- * Only reset poweroff_timer_needed if we're not in the middle
- * of the power off callback */
- kbdev->pm.poweroff_timer_needed = MALI_FALSE;
- hrtimer_try_to_cancel(&kbdev->pm.gpu_poweroff_timer);
- }
-
- /* Ensure timer does not power off wanted cores and make sure to power off unwanted cores */
- if (kbdev->pm.shader_poweroff_pending != 0) {
- kbdev->pm.shader_poweroff_pending &= ~(kbdev->pm.desired_shader_state & desired_bitmap);
- if (kbdev->pm.shader_poweroff_pending == 0)
- kbdev->pm.shader_poweroff_pending_time = 0;
- }
-
- /* Don't need 'cores_are_available', because we don't return anything */
- CSTD_UNUSED(cores_are_available);
-}
-
-void kbase_pm_update_cores_state(struct kbase_device *kbdev)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- kbase_pm_update_cores_state_nolock(kbdev);
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-}
-
-int kbase_pm_list_policies(const struct kbase_pm_policy * const **list)
-{
- if (!list)
- return POLICY_COUNT;
-
- *list = policy_list;
-
- return POLICY_COUNT;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_list_policies)
-
-const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- return kbdev->pm.pm_current_policy;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_get_policy)
-
-void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_policy *new_policy)
-{
- const struct kbase_pm_policy *old_policy;
- unsigned long flags;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(new_policy != NULL);
-
- KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
-
- /* During a policy change we pretend the GPU is active */
- /* A suspend won't happen here, because we're in a syscall from a userspace thread */
- kbase_pm_context_active(kbdev);
-
- mutex_lock(&kbdev->pm.lock);
-
- /* Remove the policy to prevent IRQ handlers from working on it */
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- old_policy = kbdev->pm.pm_current_policy;
- kbdev->pm.pm_current_policy = NULL;
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, old_policy->id);
- if (old_policy->term)
- old_policy->term(kbdev);
-
- KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, new_policy->id);
- if (new_policy->init)
- new_policy->init(kbdev);
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
- kbdev->pm.pm_current_policy = new_policy;
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- /* If any core power state changes were previously attempted, but couldn't
- * be made because the policy was changing (current_policy was NULL), then
- * re-try them here. */
- kbase_pm_update_active(kbdev);
- kbase_pm_update_cores_state(kbdev);
-
- mutex_unlock(&kbdev->pm.lock);
-
- /* Now the policy change is finished, we release our fake context active reference */
- kbase_pm_context_idle(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_set_policy)
-
-/** Check whether a state change has finished, and trace it as completed */
-STATIC void kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev)
-{
- if ((kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state) == kbdev->pm.desired_shader_state &&
- (kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state) == kbdev->pm.desired_tiler_state)
- kbase_timeline_pm_check_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
-}
-
-void kbase_pm_request_cores(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
-{
- unsigned long flags;
- u64 cores;
-
- kbase_pm_change_state change_gpu_state = 0u;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- cores = shader_cores;
- while (cores) {
- int bitnum = fls64(cores) - 1;
- u64 bit = 1ULL << bitnum;
-
- /* It should be almost impossible for this to overflow. It would require 2^32 atoms
- * to request a particular core, which would require 2^24 contexts to submit. This
- * would require an amount of memory that is impossible on a 32-bit system and
- * extremely unlikely on a 64-bit system. */
- int cnt = ++kbdev->shader_needed_cnt[bitnum];
-
- if (1 == cnt) {
- kbdev->shader_needed_bitmap |= bit;
- change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
- }
-
- cores &= ~bit;
- }
-
- if (tiler_required != MALI_FALSE) {
- ++kbdev->tiler_needed_cnt;
-
- KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
-
- /* For tiler jobs, we must make sure that core 0 is not turned off if it's already on.
- * However, it's safe for core 0 to be left off and turned on later whilst a tiler job
- * is running. Hence, we don't need to update the cores state immediately. Also,
- * attempts to turn off cores will always check the tiler_needed/inuse state first anyway.
- *
- * Finally, kbase_js_choose_affinity() ensures core 0 is always requested for tiler jobs
- * anyway. Hence when there's only a tiler job in the system, this will still cause
- * kbase_pm_update_cores_state_nolock() to be called.
- *
- * Note that we still need to keep track of tiler_needed/inuse_cnt, to ensure that
- * kbase_pm_update_cores_state_nolock() can override the core availability policy and
- * force core 0 to be powered when a tiler job is in the system. */
- }
-
- if (change_gpu_state) {
- KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
-
- kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_REQUEST_CORES_START, change_gpu_state);
- kbase_pm_update_cores_state_nolock(kbdev);
- kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_REQUEST_CORES_END, change_gpu_state);
- }
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_request_cores)
-
-void kbase_pm_unrequest_cores(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
-{
- unsigned long flags;
-
- kbase_pm_change_state change_gpu_state = 0u;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- while (shader_cores) {
- int bitnum = fls64(shader_cores) - 1;
- u64 bit = 1ULL << bitnum;
- int cnt;
-
- KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
-
- cnt = --kbdev->shader_needed_cnt[bitnum];
-
- if (0 == cnt) {
- kbdev->shader_needed_bitmap &= ~bit;
-
- change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
- }
-
- shader_cores &= ~bit;
- }
-
- if (tiler_required != MALI_FALSE) {
- KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
-
- --kbdev->tiler_needed_cnt;
-
- /* Whilst tiler jobs must not allow core 0 to be turned off, we don't need to make an
- * extra call to kbase_pm_update_cores_state_nolock() to ensure core 0 is turned off
- * when the last tiler job unrequests cores: kbase_js_choose_affinity() ensures core 0
- * was originally requested for tiler jobs. Hence when there's only a tiler job in the
- * system, this will still cause kbase_pm_update_cores_state_nolock() to be called. */
- }
-
- if (change_gpu_state) {
- KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
-
- kbase_pm_update_cores_state_nolock(kbdev);
-
- /* Trace that any state change effectively completes immediately -
- * no-one will wait on the state change */
- kbase_pm_trace_check_and_finish_state_change(kbdev);
- }
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores)
-
-enum kbase_pm_cores_ready kbase_pm_register_inuse_cores(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
-{
- unsigned long flags;
- u64 prev_shader_needed; /* Just for tracing */
- u64 prev_shader_inuse; /* Just for tracing */
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- prev_shader_needed = kbdev->shader_needed_bitmap;
- prev_shader_inuse = kbdev->shader_inuse_bitmap;
-
- /* If desired_shader_state does not contain the requested cores, then power
- * management is not attempting to powering those cores (most likely
- * due to core availability policy) and a new job affinity must be
- * chosen */
- if ((kbdev->pm.desired_shader_state & shader_cores) != shader_cores) {
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- return KBASE_NEW_AFFINITY;
- }
-
- if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
- (tiler_required != MALI_FALSE && !kbdev->tiler_available_bitmap)) {
- /* Trace ongoing core transition */
- kbase_timeline_pm_l2_transition_start(kbdev);
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
- return KBASE_CORES_NOT_READY;
- }
-
- /* If we started to trace a state change, then trace it has being finished
- * by now, at the very latest */
- kbase_pm_trace_check_and_finish_state_change(kbdev);
- /* Trace core transition done */
- kbase_timeline_pm_l2_transition_done(kbdev);
-
- while (shader_cores) {
- int bitnum = fls64(shader_cores) - 1;
- u64 bit = 1ULL << bitnum;
- int cnt;
-
- KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
-
- cnt = --kbdev->shader_needed_cnt[bitnum];
-
- if (0 == cnt)
- kbdev->shader_needed_bitmap &= ~bit;
-
- /* shader_inuse_cnt should not overflow because there can only be a
- * very limited number of jobs on the h/w at one time */
-
- kbdev->shader_inuse_cnt[bitnum]++;
- kbdev->shader_inuse_bitmap |= bit;
-
- shader_cores &= ~bit;
- }
-
- if (tiler_required != MALI_FALSE) {
- KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
-
- --kbdev->tiler_needed_cnt;
-
- kbdev->tiler_inuse_cnt++;
-
- KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
- }
-
- if (prev_shader_needed != kbdev->shader_needed_bitmap)
- KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
-
- if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
- KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL, NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-
- return KBASE_CORES_READY;
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores)
-
-void kbase_pm_release_cores(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
-{
- unsigned long flags;
- kbase_pm_change_state change_gpu_state = 0u;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- while (shader_cores) {
- int bitnum = fls64(shader_cores) - 1;
- u64 bit = 1ULL << bitnum;
- int cnt;
-
- KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);
-
- cnt = --kbdev->shader_inuse_cnt[bitnum];
-
- if (0 == cnt) {
- kbdev->shader_inuse_bitmap &= ~bit;
- change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
- }
-
- shader_cores &= ~bit;
- }
-
- if (tiler_required != MALI_FALSE) {
- KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);
-
- --kbdev->tiler_inuse_cnt;
-
- /* Whilst tiler jobs must not allow core 0 to be turned off, we don't need to make an
- * extra call to kbase_pm_update_cores_state_nolock() to ensure core 0 is turned off
- * when the last tiler job finishes: kbase_js_choose_affinity() ensures core 0 was
- * originally requested for tiler jobs. Hence when there's only a tiler job in the
- * system, this will still cause kbase_pm_update_cores_state_nolock() to be called */
- }
-
- if (change_gpu_state) {
- KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL, NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
-
- kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_RELEASE_CORES_START, change_gpu_state);
- kbase_pm_update_cores_state_nolock(kbdev);
- kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_RELEASE_CORES_END, change_gpu_state);
-
- /* Trace that any state change completed immediately */
- kbase_pm_trace_check_and_finish_state_change(kbdev);
- }
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_release_cores)
-
-void kbase_pm_request_cores_sync(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
-{
- kbase_pm_request_cores(kbdev, tiler_required, shader_cores);
-
- kbase_pm_check_transitions_sync(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync)
-
-void kbase_pm_request_l2_caches(struct kbase_device *kbdev)
-{
- unsigned long flags;
- u32 prior_l2_users_count;
-
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- prior_l2_users_count = kbdev->l2_users_count++;
-
- KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0);
-
- /* if the GPU is reset while the l2 is on, l2 will be off but prior_l2_users_count will be > 0
- * l2_available_bitmap will have been set to 0 though by kbase_pm_init_hw */
- if (!prior_l2_users_count || !kbdev->l2_available_bitmap)
- kbase_pm_check_transitions_nolock(kbdev);
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
- wait_event(kbdev->pm.l2_powered_wait, kbdev->pm.l2_powered == 1);
-
- /* Trace that any state change completed immediately */
- kbase_pm_trace_check_and_finish_state_change(kbdev);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches)
-
-void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev)
-{
- unsigned long flags;
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- kbdev->l2_users_count++;
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on)
-
-void kbase_pm_release_l2_caches(struct kbase_device *kbdev)
-{
- unsigned long flags;
- spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
-
- KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0);
-
- --kbdev->l2_users_count;
-
- if (!kbdev->l2_users_count) {
- kbase_pm_check_transitions_nolock(kbdev);
- /* Trace that any state change completed immediately */
- kbase_pm_trace_check_and_finish_state_change(kbdev);
- }
-
- spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
-}
-
-KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches)
-#endif /* KBASE_PM_EN */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * @file mali_kbase_pm_policy.h
- * Power policy API definitions
- */
-
-#ifndef _KBASE_PM_POLICY_H_
-#define _KBASE_PM_POLICY_H_
-
-/** List of policy IDs */
-enum kbase_pm_policy_id {
- KBASE_PM_POLICY_ID_DEMAND = 1,
- KBASE_PM_POLICY_ID_ALWAYS_ON,
- KBASE_PM_POLICY_ID_COARSE_DEMAND,
-#if !MALI_CUSTOMER_RELEASE
- KBASE_PM_POLICY_ID_DEMAND_ALWAYS_POWERED,
- KBASE_PM_POLICY_ID_FAST_START
-#endif
-};
-
-typedef u32 kbase_pm_policy_flags;
-
-/** Power policy structure.
- *
- * Each power policy exposes a (static) instance of this structure which contains function pointers to the
- * policy's methods.
- */
-typedef struct kbase_pm_policy {
- /** The name of this policy */
- char *name;
-
- /** Function called when the policy is selected
- *
- * This should initialize the kbdev->pm.pm_policy_data structure. It should not attempt
- * to make any changes to hardware state.
- *
- * It is undefined what state the cores are in when the function is called.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
- void (*init)(struct kbase_device *kbdev);
-
- /** Function called when the policy is unselected.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
- void (*term)(struct kbase_device *kbdev);
-
- /** Function called to get the current shader core mask
- *
- * The returned mask should meet or exceed (kbdev->shader_needed_bitmap | kbdev->shader_inuse_bitmap).
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- *
- * @return The mask of shader cores to be powered */
- u64 (*get_core_mask)(struct kbase_device *kbdev);
-
- /** Function called to get the current overall GPU power state
- *
- * This function should consider the state of kbdev->pm.active_count. If this count is greater than 0 then
- * there is at least one active context on the device and the GPU should be powered. If it is equal to 0
- * then there are no active contexts and the GPU could be powered off if desired.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- *
- * @return MALI_TRUE if the GPU should be powered, MALI_FALSE otherwise */
- mali_bool (*get_core_active) (struct kbase_device *kbdev);
-
- /** Field indicating flags for this policy */
- kbase_pm_policy_flags flags;
-
- /** Field indicating an ID for this policy. This is not necessarily the
- * same as its index in the list returned by kbase_pm_list_policies().
- * It is used purely for debugging. */
- enum kbase_pm_policy_id id;
-} kbase_pm_policy;
-
-/** Initialize power policy framework
- *
- * Must be called before calling any other policy function
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- *
- * @return MALI_ERROR_NONE if the power policy framework was successfully initialized.
- */
-mali_error kbase_pm_policy_init(struct kbase_device *kbdev);
-
-/** Terminate power policy framework
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_policy_term(struct kbase_device *kbdev);
-
-/** Update the active power state of the GPU
- * Calls into the current power policy
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_update_active(struct kbase_device *kbdev);
-
-/** Update the desired core state of the GPU
- * Calls into the current power policy
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_update_cores(struct kbase_device *kbdev);
-
-/** Get the current policy.
- * Returns the policy that is currently active.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- *
- * @return The current policy
- */
-const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev);
-
-/** Change the policy to the one specified.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- * @param policy The policy to change to (valid pointer returned from @ref kbase_pm_list_policies)
- */
-void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_policy *policy);
-
-/** Retrieve a static list of the available policies.
- * @param[out] policies An array pointer to take the list of policies. This may be NULL.
- * The contents of this array must not be modified.
- *
- * @return The number of policies
- */
-int kbase_pm_list_policies(const struct kbase_pm_policy * const **policies);
-
-
-enum kbase_pm_cores_ready {
- KBASE_CORES_NOT_READY = 0,
- KBASE_NEW_AFFINITY = 1,
- KBASE_CORES_READY = 2
-};
-
-
-/** Synchronous variant of kbase_pm_request_cores()
- *
- * When this function returns, the @a shader_cores will be in the READY state.
- *
- * This is safe variant of kbase_pm_check_transitions_sync(): it handles the
- * work of ensuring the requested cores will remain powered until a matching
- * call to kbase_pm_unrequest_cores()/kbase_pm_release_cores() (as appropriate)
- * is made.
- *
- * @param kbdev The kbase device structure for the device
- * @param tiler_required MALI_TRUE if the tiler is required, MALI_FALSE otherwise
- * @param shader_cores A bitmask of shader cores which are necessary for the job
- */
-
-void kbase_pm_request_cores_sync(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores);
-
-/** Mark one or more cores as being required for jobs to be submitted.
- *
- * This function is called by the job scheduler to mark one or more cores
- * as being required to submit jobs that are ready to run.
- *
- * The cores requested are reference counted and a subsequent call to @ref kbase_pm_register_inuse_cores or
- * @ref kbase_pm_unrequest_cores should be made to dereference the cores as being 'needed'.
- *
- * The active power policy will meet or exceed the requirements of the
- * requested cores in the system. Any core transitions needed will be begun
- * immediately, but they might not complete/the cores might not be available
- * until a Power Management IRQ.
- *
- * @param kbdev The kbase device structure for the device
- * @param tiler_required MALI_TRUE if the tiler is required, MALI_FALSE otherwise
- * @param shader_cores A bitmask of shader cores which are necessary for the job
- *
- * @return MALI_ERROR_NONE if the cores were successfully requested.
- */
-void kbase_pm_request_cores(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores);
-
-/** Unmark one or more cores as being required for jobs to be submitted.
- *
- * This function undoes the effect of @ref kbase_pm_request_cores. It should be used when a job is not
- * going to be submitted to the hardware (e.g. the job is cancelled before it is enqueued).
- *
- * The active power policy will meet or exceed the requirements of the
- * requested cores in the system. Any core transitions needed will be begun
- * immediately, but they might not complete until a Power Management IRQ.
- *
- * The policy may use this as an indication that it can power down cores.
- *
- * @param kbdev The kbase device structure for the device
- * @param tiler_required MALI_TRUE if the tiler is required, MALI_FALSE otherwise
- * @param shader_cores A bitmask of shader cores (as given to @ref kbase_pm_request_cores)
- */
-void kbase_pm_unrequest_cores(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores);
-
-/** Register a set of cores as in use by a job.
- *
- * This function should be called after @ref kbase_pm_request_cores when the job is about to be submitted to
- * the hardware. It will check that the necessary cores are available and if so update the 'needed' and 'inuse'
- * bitmasks to reflect that the job is now committed to being run.
- *
- * If the necessary cores are not currently available then the function will return MALI_FALSE and have no effect.
- *
- * @param kbdev The kbase device structure for the device
- * @param tiler_required MALI_TRUE if the tiler is required, MALI_FALSE otherwise
- * @param shader_cores A bitmask of shader cores (as given to @ref kbase_pm_request_cores)
- *
- * @return MALI_TRUE if the job can be submitted to the hardware or MALI_FALSE if the job is not ready to run.
- */
-enum kbase_pm_cores_ready kbase_pm_register_inuse_cores(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores);
-
-/** Release cores after a job has run.
- *
- * This function should be called when a job has finished running on the hardware. A call to @ref
- * kbase_pm_register_inuse_cores must have previously occurred. The reference counts of the specified cores will be
- * decremented which may cause the bitmask of 'inuse' cores to be reduced. The power policy may then turn off any
- * cores which are no longer 'inuse'.
- *
- * @param kbdev The kbase device structure for the device
- * @param tiler_required MALI_TRUE if the tiler is required, MALI_FALSE otherwise
- * @param shader_cores A bitmask of shader cores (as given to @ref kbase_pm_register_inuse_cores)
- */
-void kbase_pm_release_cores(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores);
-
-/** Request the use of l2 caches for all core groups, power up, wait and prevent the power manager from
- * powering down the l2 caches.
- *
- * This tells the power management that the caches should be powered up, and they
- * should remain powered, irrespective of the usage of shader cores. This does not
- * return until the l2 caches are powered up.
- *
- * The caller must call @ref kbase_pm_release_l2_caches when they are finished to
- * allow normal power management of the l2 caches to resume.
- *
- * This should only be used when power management is active.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_request_l2_caches(struct kbase_device *kbdev);
-
-/** Increment the count of l2 users but do not attempt to power on the l2
- * It is the callers responsibility to ensure that the l2 is already powered up
- * and to eventually call @ref kbase_pm_release_l2_caches
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev);
-
-/** Release the use of l2 caches for all core groups and allow the power manager to
- * power them down when necessary.
- *
- * This tells the power management that the caches can be powered down if necessary, with respect
- * to the usage of shader cores.
- *
- * The caller must have called @ref kbase_pm_request_l2_caches prior to a call to this.
- *
- * This should only be used when power management is active.
- *
- * @param kbdev The kbase device structure for the device (must be a valid pointer)
- */
-void kbase_pm_release_l2_caches(struct kbase_device *kbdev);
-
-#endif /* _KBASE_PM_POLICY_H_ */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-#include <mali_kbase.h>
-#include <mali_kbase_config_defaults.h>
-
-#include <linux/devfreq_cooling.h>
-#include <linux/power_actor.h>
-#include <linux/thermal.h>
-
-#include "mali_kbase_power_actor.h"
-
-
-static u32 mali_pa_get_req_power(struct power_actor *actor, struct thermal_zone_device *zone)
-{
- struct mali_power_actor *mali_actor = actor->data;
- struct kbase_device *kbdev = mali_actor->kbdev;
- unsigned long total_time, busy_time;
- unsigned long power, temperature;
- struct dev_pm_opp *opp;
- unsigned long voltage;
- unsigned long freq;
-
- kbase_pm_get_dvfs_utilisation(kbdev, &total_time, &busy_time);
-
- freq = clk_get_rate(kbdev->clock);
-
- rcu_read_lock();
- opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq);
- if (IS_ERR_OR_NULL(opp)) {
- rcu_read_unlock();
- return 0;
- }
-
- voltage = dev_pm_opp_get_voltage(opp) / 1000; /* mV */
- rcu_read_unlock();
-
- power = mali_actor->ops->get_dynamic_power(freq, voltage);
- power = (power * busy_time) / total_time;
-
- temperature = zone->temperature;
-
- /* Assume all cores are always powered */
- power += mali_actor->ops->get_static_power(voltage, temperature);
-
- dev_dbg(kbdev->dev, "get req power = %lu\n", power);
-
- return (u32)power;
-}
-
-static u32 mali_pa_get_max_power(struct power_actor *actor, struct thermal_zone_device *zone)
-{
- struct mali_power_actor *mali_actor = actor->data;
- struct kbase_device *kbdev = mali_actor->kbdev;
- struct dev_pm_opp *opp;
- unsigned long voltage, temperature;
- unsigned long freq = ULONG_MAX;
- u32 power;
-
- rcu_read_lock();
- opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq);
- if (IS_ERR_OR_NULL(opp)) {
- rcu_read_unlock();
- dev_err(kbdev->dev, "Failed to get OPP for max freq\n");
- return 0;
- }
- voltage = dev_pm_opp_get_voltage(opp) / 1000; /* mV */
- rcu_read_unlock();
-
- temperature = zone->temperature;
-
- power = mali_actor->ops->get_static_power(voltage, temperature)
- + mali_actor->ops->get_dynamic_power(freq, voltage);
-
- dev_dbg(kbdev->dev, "get max power = %u\n", power);
-
- return power;
-}
-
-static int mali_pa_set_power(struct power_actor *actor, struct thermal_zone_device *zone, u32 power)
-{
- struct mali_power_actor *mali_actor = actor->data;
- struct kbase_device *kbdev = mali_actor->kbdev;
- struct thermal_cooling_device *cdev;
- unsigned long total_time, busy_time;
- unsigned long freq, state;
- unsigned long static_power, normalized_power;
- unsigned long voltage, temperature;
- struct dev_pm_opp *opp;
- int err, i;
-
- dev_dbg(kbdev->dev, "Setting max power %u\n", power);
-
- kbase_pm_get_dvfs_utilisation(kbdev, &total_time, &busy_time);
-
- freq = clk_get_rate(kbdev->clock);
-
- rcu_read_lock();
- opp = dev_pm_opp_find_freq_exact(kbdev->dev, freq, true);
- if (IS_ERR_OR_NULL(opp)) {
- rcu_read_unlock();
- return -ENOENT;
- }
- voltage = dev_pm_opp_get_voltage(opp) / 1000; /* mV */
- rcu_read_unlock();
-
- temperature = zone->temperature;
-
- static_power = mali_actor->ops->get_static_power(voltage, temperature);
-
- if (power < static_power) {
- normalized_power = 0;
- } else {
- unsigned long dyn_power = power - static_power;
-
- if (!busy_time)
- normalized_power = dyn_power;
- else
- normalized_power = (dyn_power * total_time) / busy_time;
- }
-
- /* Find target frequency. Use the lowest OPP if allocated power is too
- * low. */
- freq = mali_actor->dyn_table[0].freq;
- for (i = 1; i < mali_actor->dyn_table_count; i++) {
- if (mali_actor->dyn_table[i].power > normalized_power)
- break;
- else
- freq = mali_actor->dyn_table[i].freq;
- }
-
- state = devfreq_cooling_get_level(kbdev->devfreq, freq);
- if (state == THERMAL_CSTATE_INVALID) {
- dev_err(kbdev->dev,
- "Failed to lookup cooling level for freq %ld\n", freq);
- return -EINVAL;
- }
-
- cdev = kbdev->devfreq_cooling->cdev;
- err = cdev->ops->set_cur_state(cdev, state);
-
- dev_dbg(kbdev->dev,
- "Max power set to %u using frequency %ld (cooling level %ld) (%d)\n",
- power, freq, state, err);
-
- return err;
-}
-
-static struct power_actor_ops mali_pa_ops = {
- .get_req_power = mali_pa_get_req_power,
- .get_max_power = mali_pa_get_max_power,
- .set_power = mali_pa_set_power,
-};
-
-int mali_pa_init(struct kbase_device *kbdev)
-{
- struct power_actor *actor;
- struct mali_power_actor *mali_actor;
- struct mali_pa_model_ops *callbacks;
- struct mali_pa_power_table *table;
- unsigned long freq;
- int i, num_opps;
-
- callbacks = (void *)kbasep_get_config_value(kbdev, kbdev->config_attributes,
- KBASE_CONFIG_ATTR_POWER_MODEL_CALLBACKS);
- if (!callbacks)
- return -ENODEV;
-
- mali_actor = kzalloc(sizeof(*mali_actor), GFP_KERNEL);
- if (!mali_actor)
- return -ENOMEM;
-
- mali_actor->ops = callbacks;
- mali_actor->kbdev = kbdev;
-
- rcu_read_lock();
- num_opps = dev_pm_opp_get_opp_count(kbdev->dev);
- rcu_read_unlock();
-
- table = kcalloc(num_opps, sizeof(table[0]), GFP_KERNEL);
- if (!table) {
- kfree(mali_actor);
- return -ENOMEM;
- }
-
- rcu_read_lock();
- for (i = 0, freq = 0; i < num_opps; i++, freq++) {
- unsigned long power_static, power_dyn, voltage;
- struct dev_pm_opp *opp;
-
- opp = dev_pm_opp_find_freq_ceil(kbdev->dev, &freq);
- if (IS_ERR(opp))
- break;
-
- voltage = dev_pm_opp_get_voltage(opp) / 1000; /* mV */
-
- table[i].freq = freq;
-
- power_dyn = callbacks->get_dynamic_power(freq, voltage);
- power_static = callbacks->get_static_power(voltage, 85000);
-
- dev_info(kbdev->dev, "Power table: %lu MHz @ %lu mV: %lu + %lu = %lu mW\n",
- freq / 1000000, voltage,
- power_dyn, power_static, power_dyn + power_static);
-
- table[i].power = power_dyn;
- }
- rcu_read_unlock();
-
- if (i != num_opps)
- dev_warn(kbdev->dev, "Power actor: Unable to enumerate all OPPs (%d != %d)\n",
- i, num_opps);
-
- mali_actor->dyn_table = table;
- mali_actor->dyn_table_count = i;
-
- /* Register power actor.
- * Set default actor weight to 1 (8-bit fixed point). */
- actor = power_actor_register(1 * 256, &mali_pa_ops, mali_actor);
- if (IS_ERR_OR_NULL(actor)) {
- kfree(mali_actor->dyn_table);
- kfree(mali_actor);
- return PTR_ERR(actor);
- }
-
- kbdev->power_actor = actor;
-
- dev_info(kbdev->dev, "Initalized power actor\n");
-
- return 0;
-}
-
-void mali_pa_term(struct kbase_device *kbdev)
-{
- struct mali_power_actor *mali_actor;
-
- if (kbdev->power_actor) {
- mali_actor = kbdev->power_actor->data;
-
- power_actor_unregister(kbdev->power_actor);
- kbdev->power_actor = NULL;
-
- kfree(mali_actor->dyn_table);
- kfree(mali_actor);
- }
-}
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-#ifndef _KBASE_POWER_ACTOR_H_
-#define _KBASE_POWER_ACTOR_H_
-
-#include <mali_kbase.h>
-
-#include <linux/pm_opp.h>
-
-/** struct mali_pa_model_ops - Function pointer for power model
- *
- * @get_static_power: Pointer to a function that returns the estimated static
- * power usage in mW, based on the input voltage in mV and
- * temperature in millidegrees Celsius.
- * @get_dynamic_power: Pointer to a function that returns the estimated dynamic power
- * usage in mW, based on the input voltage in mV and
- * frequency in Hz.
- */
-struct mali_pa_model_ops {
- unsigned long (*get_static_power)(unsigned long voltage,
- unsigned long temperature);
- unsigned long (*get_dynamic_power)(unsigned long freq,
- unsigned long voltage);
-};
-
-struct mali_pa_power_table {
- unsigned long freq;
- unsigned long power;
-};
-
-struct mali_power_actor {
- struct kbase_device *kbdev;
- struct mali_pa_model_ops *ops;
- struct mali_pa_power_table *dyn_table;
- int dyn_table_count;
-};
-
-int mali_pa_init(struct kbase_device *kbdev);
-void mali_pa_term(struct kbase_device *kbdev);
-
-
-#endif
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010, 2013 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase_config.h>
#include <mali_kbase.h>
#include <mali_kbase_mem.h>
+#include <mali_kbase_mem_linux.h>
#define JOB_NOT_STARTED 0
#define JOB_TYPE_MASK 0xfe
#endif
}
-struct kbasep_map_struct {
- mali_addr64 gpu_addr;
- struct kbase_mem_phy_alloc *alloc;
- struct page **pages;
- void *addr;
- size_t size;
- mali_bool is_cached;
-};
-
-static void *kbasep_map(struct kbase_context *kctx, mali_addr64 gpu_addr,
- size_t size, struct kbasep_map_struct *map)
-{
- struct kbase_va_region *region;
- unsigned long page_index;
- unsigned int offset = gpu_addr & ~PAGE_MASK;
- size_t page_count = PFN_UP(offset + size);
- phys_addr_t *page_array;
- struct page **pages;
- void *cpu_addr = NULL;
- pgprot_t prot;
- size_t i;
-
- if (!size || !map)
- return NULL;
-
- /* check if page_count calculation will wrap */
- if (size > ((size_t)-1 / PAGE_SIZE))
- return NULL;
-
- region = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr);
- if (!region || (region->flags & KBASE_REG_FREE))
- return NULL;
-
- page_index = (gpu_addr >> PAGE_SHIFT) - region->start_pfn;
-
- /* check if page_index + page_count will wrap */
- if (-1UL - page_count < page_index)
- return NULL;
-
- if (page_index + page_count > kbase_reg_current_backed_size(region))
- return NULL;
-
- page_array = kbase_get_phy_pages(region);
- if (!page_array)
- return NULL;
-
- pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL);
- if (!pages)
- return NULL;
-
- for (i = 0; i < page_count; i++)
- pages[i] = pfn_to_page(PFN_DOWN(page_array[page_index + i]));
-
- prot = PAGE_KERNEL;
- if (!(region->flags & KBASE_REG_CPU_CACHED)) {
- /* Map uncached */
- prot = pgprot_writecombine(prot);
- }
-
- cpu_addr = vmap(pages, page_count, VM_MAP, prot);
- if (!cpu_addr)
- goto vmap_failed;
-
- map->gpu_addr = gpu_addr;
- map->alloc = kbase_mem_phy_alloc_get(region->alloc);
- map->pages = pages;
- map->addr = (void *)((uintptr_t)cpu_addr + offset);
- map->size = size;
- map->is_cached = (region->flags & KBASE_REG_CPU_CACHED) != 0;
-
- if (map->is_cached) {
- /* Sync first page */
- size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
- phys_addr_t pa = page_to_phys(map->pages[0]) + offset;
-
- kbase_sync_single(kctx, pa, sz, dma_sync_single_for_cpu);
-
- /* Sync middle pages (if any) */
- for (i = 1; page_count > 2 && i < page_count - 1; i++) {
- pa = page_to_phys(map->pages[i]);
- kbase_sync_single(kctx, pa, PAGE_SIZE,
- dma_sync_single_for_cpu);
- }
-
- /* Sync last page (if any) */
- if (page_count > 1) {
- pa = page_to_phys(map->pages[page_count - 1]);
- sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
- kbase_sync_single(kctx, pa, sz,
- dma_sync_single_for_cpu);
- }
- }
-
- return map->addr;
-
-vmap_failed:
- kfree(pages);
-
- return NULL;
-}
-
-static void kbasep_unmap(struct kbase_context *kctx,
- struct kbasep_map_struct *map)
-{
- void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK);
-
- vunmap(addr);
-
- if (map->is_cached) {
- off_t offset = (uintptr_t)map->addr & ~PAGE_MASK;
- size_t size = map->size;
- size_t page_count = PFN_UP(offset + size);
- size_t i;
-
- /* Sync first page */
- size_t sz = MIN(((size_t) PAGE_SIZE - offset), size);
- phys_addr_t pa = page_to_phys(map->pages[0]) + offset;
-
- kbase_sync_single(kctx, pa, sz, dma_sync_single_for_device);
-
- /* Sync middle pages (if any) */
- for (i = 1; page_count > 2 && i < page_count - 1; i++) {
- pa = page_to_phys(map->pages[i]);
- kbase_sync_single(kctx, pa, PAGE_SIZE,
- dma_sync_single_for_device);
- }
-
- /* Sync last page (if any) */
- if (page_count > 1) {
- pa = page_to_phys(map->pages[page_count - 1]);
- sz = ((offset + size - 1) & ~PAGE_MASK) + 1;
- kbase_sync_single(kctx, pa, sz,
- dma_sync_single_for_device);
- }
- }
-
- kfree(map->pages);
-
- map->gpu_addr = 0;
- map->alloc = kbase_mem_phy_alloc_put(map->alloc);
- map->pages = NULL;
- map->addr = NULL;
- map->size = 0;
- map->is_cached = MALI_FALSE;
-}
-
-static mali_error kbasep_replay_reset_sfbd(struct kbase_context *kctx,
- mali_addr64 fbd_address, mali_addr64 tiler_heap_free,
+static int kbasep_replay_reset_sfbd(struct kbase_context *kctx,
+ u64 fbd_address, u64 tiler_heap_free,
u16 hierarchy_mask, u32 default_weight)
{
struct {
u32 padding[8];
u32 weights[FBD_HIERARCHY_WEIGHTS];
} *fbd_tiler;
- struct kbasep_map_struct map;
+ struct kbase_vmap_struct map;
dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address);
- fbd_tiler = kbasep_map(kctx, fbd_address + SFBD_TILER_OFFSET,
+ fbd_tiler = kbase_vmap(kctx, fbd_address + SFBD_TILER_OFFSET,
sizeof(*fbd_tiler), &map);
if (!fbd_tiler) {
dev_err(kctx->kbdev->dev, "kbasep_replay_reset_fbd: failed to map fbd\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
#ifdef CONFIG_MALI_DEBUG
dev_dbg(kctx->kbdev->dev, "heap_free_address=%llx flags=%x\n",
fbd_tiler->heap_free_address, fbd_tiler->flags);
- kbasep_unmap(kctx, &map);
+ kbase_vunmap(kctx, &map);
- return MALI_ERROR_NONE;
+ return 0;
}
-static mali_error kbasep_replay_reset_mfbd(struct kbase_context *kctx,
- mali_addr64 fbd_address, mali_addr64 tiler_heap_free,
+static int kbasep_replay_reset_mfbd(struct kbase_context *kctx,
+ u64 fbd_address, u64 tiler_heap_free,
u16 hierarchy_mask, u32 default_weight)
{
- struct kbasep_map_struct map;
+ struct kbase_vmap_struct map;
struct {
u32 padding_0;
u32 flags;
dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address);
- fbd_tiler = kbasep_map(kctx, fbd_address + MFBD_TILER_OFFSET,
+ fbd_tiler = kbase_vmap(kctx, fbd_address + MFBD_TILER_OFFSET,
sizeof(*fbd_tiler), &map);
if (!fbd_tiler) {
dev_err(kctx->kbdev->dev,
"kbasep_replay_reset_fbd: failed to map fbd\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
#ifdef CONFIG_MALI_DEBUG
fbd_tiler->heap_free_address = tiler_heap_free;
- kbasep_unmap(kctx, &map);
+ kbase_vunmap(kctx, &map);
- return MALI_ERROR_NONE;
+ return 0;
}
/**
* @param[in] hierarchy_mask The hierarchy mask to use
* @param[in] default_weight Default hierarchy weight to write when no other
* weight is given in the FBD
- * @param[in] job_64 MALI_TRUE if this job is using 64-bit
+ * @param[in] job_64 true if this job is using 64-bit
* descriptors
*
- * @return MALI_ERROR_NONE on success, error code on failure
+ * @return 0 on success, error code on failure
*/
-static mali_error kbasep_replay_reset_tiler_job(struct kbase_context *kctx,
- mali_addr64 job_header, mali_addr64 tiler_heap_free,
- u16 hierarchy_mask, u32 default_weight, mali_bool job_64)
+static int kbasep_replay_reset_tiler_job(struct kbase_context *kctx,
+ u64 job_header, u64 tiler_heap_free,
+ u16 hierarchy_mask, u32 default_weight, bool job_64)
{
- struct kbasep_map_struct map;
- mali_addr64 fbd_address;
+ struct kbase_vmap_struct map;
+ u64 fbd_address;
if (job_64) {
u64 *job_ext;
- job_ext = kbasep_map(kctx,
+ job_ext = kbase_vmap(kctx,
job_header + JOB_HEADER_64_FBD_OFFSET,
sizeof(*job_ext), &map);
if (!job_ext) {
dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
fbd_address = *job_ext;
- kbasep_unmap(kctx, &map);
+ kbase_vunmap(kctx, &map);
} else {
u32 *job_ext;
- job_ext = kbasep_map(kctx,
+ job_ext = kbase_vmap(kctx,
job_header + JOB_HEADER_32_FBD_OFFSET,
sizeof(*job_ext), &map);
if (!job_ext) {
dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
fbd_address = *job_ext;
- kbasep_unmap(kctx, &map);
+ kbase_vunmap(kctx, &map);
}
if (fbd_address & FBD_TYPE) {
* @param[in] hierarchy_mask The hierarchy mask to use
* @param[in] default_weight Default hierarchy weight to write when no other
* weight is given in the FBD
- * @param[in] first_in_chain MALI_TRUE if this job is the first in the chain
- * @param[in] fragment_chain MALI_TRUE if this job is in the fragment chain
+ * @param[in] first_in_chain true if this job is the first in the chain
+ * @param[in] fragment_chain true if this job is in the fragment chain
*
- * @return MALI_ERROR_NONE on success, error code on failure
+ * @return 0 on success, error code on failure
*/
-static mali_error kbasep_replay_reset_job(struct kbase_context *kctx,
- mali_addr64 *job_header, mali_addr64 prev_jc,
- mali_addr64 tiler_heap_free, u16 hierarchy_mask,
+static int kbasep_replay_reset_job(struct kbase_context *kctx,
+ u64 *job_header, u64 prev_jc,
+ u64 tiler_heap_free, u16 hierarchy_mask,
u32 default_weight, u16 hw_job_id_offset,
- mali_bool first_in_chain, mali_bool fragment_chain)
+ bool first_in_chain, bool fragment_chain)
{
struct job_head *job;
- mali_addr64 new_job_header;
- struct kbasep_map_struct map;
+ u64 new_job_header;
+ struct kbase_vmap_struct map;
- job = kbasep_map(kctx, *job_header, sizeof(*job), &map);
+ job = kbase_vmap(kctx, *job_header, sizeof(*job), &map);
if (!job) {
dev_err(kctx->kbdev->dev,
"kbasep_replay_parse_jc: failed to map jc\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
dump_job_head(kctx, "Job header:", job);
dump_job_head(kctx, "Updated to:", job);
if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_TILER) {
- mali_bool job_64 = (job->flags & JOB_FLAG_DESC_SIZE) != 0;
+ bool job_64 = (job->flags & JOB_FLAG_DESC_SIZE) != 0;
if (kbasep_replay_reset_tiler_job(kctx, *job_header,
tiler_heap_free, hierarchy_mask,
- default_weight, job_64) != MALI_ERROR_NONE)
+ default_weight, job_64) != 0)
goto out_unmap;
} else if ((job->flags & JOB_TYPE_MASK) == JOB_TYPE_FRAGMENT) {
fbd_address & FBD_POINTER_MASK,
tiler_heap_free,
hierarchy_mask,
- default_weight) != MALI_ERROR_NONE)
+ default_weight) != 0)
goto out_unmap;
} else {
if (kbasep_replay_reset_sfbd(kctx,
fbd_address & FBD_POINTER_MASK,
tiler_heap_free,
hierarchy_mask,
- default_weight) != MALI_ERROR_NONE)
+ default_weight) != 0)
goto out_unmap;
}
}
- kbasep_unmap(kctx, &map);
+ kbase_vunmap(kctx, &map);
*job_header = new_job_header;
- return MALI_ERROR_NONE;
+ return 0;
out_unmap:
- kbasep_unmap(kctx, &map);
- return MALI_ERROR_FUNCTION_FAILED;
+ kbase_vunmap(kctx, &map);
+ return -EINVAL;
}
/**
* @param[in] jc Job chain start address
* @param[out] hw_job_id Highest job ID in chain
*
- * @return MALI_ERROR_NONE on success, error code on failure
+ * @return 0 on success, error code on failure
*/
-static mali_error kbasep_replay_find_hw_job_id(struct kbase_context *kctx,
- mali_addr64 jc, u16 *hw_job_id)
+static int kbasep_replay_find_hw_job_id(struct kbase_context *kctx,
+ u64 jc, u16 *hw_job_id)
{
while (jc) {
struct job_head *job;
- struct kbasep_map_struct map;
+ struct kbase_vmap_struct map;
dev_dbg(kctx->kbdev->dev,
"kbasep_replay_find_hw_job_id: parsing jc=%llx\n", jc);
- job = kbasep_map(kctx, jc, sizeof(*job), &map);
+ job = kbase_vmap(kctx, jc, sizeof(*job), &map);
if (!job) {
dev_err(kctx->kbdev->dev, "failed to map jc\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
if (job->index > *hw_job_id)
else
jc = job->next._32;
- kbasep_unmap(kctx, &map);
+ kbase_vunmap(kctx, &map);
}
- return MALI_ERROR_NONE;
+ return 0;
}
/**
* @param[in] default_weight Default hierarchy weight to write when no other
* weight is given in the FBD
* @param[in] hw_job_id_offset Offset for HW job IDs
- * @param[in] fragment_chain MAIL_TRUE if this chain is the fragment chain
+ * @param[in] fragment_chain true if this chain is the fragment chain
*
- * @return MALI_ERROR_NONE on success, error code otherwise
+ * @return 0 on success, error code otherwise
*/
-static mali_error kbasep_replay_parse_jc(struct kbase_context *kctx,
- mali_addr64 jc, mali_addr64 prev_jc,
- mali_addr64 tiler_heap_free, u16 hierarchy_mask,
+static int kbasep_replay_parse_jc(struct kbase_context *kctx,
+ u64 jc, u64 prev_jc,
+ u64 tiler_heap_free, u16 hierarchy_mask,
u32 default_weight, u16 hw_job_id_offset,
- mali_bool fragment_chain)
+ bool fragment_chain)
{
- mali_bool first_in_chain = MALI_TRUE;
+ bool first_in_chain = true;
int nr_jobs = 0;
dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: jc=%llx hw_job_id=%x\n",
if (kbasep_replay_reset_job(kctx, &jc, prev_jc,
tiler_heap_free, hierarchy_mask,
default_weight, hw_job_id_offset,
- first_in_chain, fragment_chain) != MALI_ERROR_NONE)
- return MALI_ERROR_FUNCTION_FAILED;
+ first_in_chain, fragment_chain) != 0)
+ return -EINVAL;
- first_in_chain = MALI_FALSE;
+ first_in_chain = false;
nr_jobs++;
if (fragment_chain &&
nr_jobs >= BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT) {
dev_err(kctx->kbdev->dev,
"Exceeded maximum number of jobs in fragment chain\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
}
- return MALI_ERROR_NONE;
+ return 0;
}
/**
static void kbasep_replay_create_atom(struct kbase_context *kctx,
struct base_jd_atom_v2 *atom,
int atom_nr,
- int prio)
+ base_jd_prio prio)
{
atom->nr_extres = 0;
atom->extres_list.value = NULL;
atom->device_nr = 0;
- /* Convert priority back from NICE range */
- atom->prio = ((prio << 16) / ((20 << 16) / 128)) - 128;
+ atom->prio = prio;
atom->atom_number = atom_nr;
base_jd_atom_dep_set(&atom->pre_dep[0], 0 , BASE_JD_DEP_TYPE_INVALID);
* @param[out] f_atom Atom to use for fragment jobs
* @param[in] prio Priority of new atom (inherited from replay soft
* job)
- * @return MALI_ERROR_NONE on success, error code on failure
+ * @return 0 on success, error code on failure
*/
-static mali_error kbasep_replay_create_atoms(struct kbase_context *kctx,
+static int kbasep_replay_create_atoms(struct kbase_context *kctx,
struct base_jd_atom_v2 *t_atom,
- struct base_jd_atom_v2 *f_atom, int prio)
+ struct base_jd_atom_v2 *f_atom,
+ base_jd_prio prio)
{
int t_atom_nr, f_atom_nr;
t_atom_nr = kbasep_allocate_katom(kctx);
if (t_atom_nr < 0) {
dev_err(kctx->kbdev->dev, "Failed to allocate katom\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
f_atom_nr = kbasep_allocate_katom(kctx);
if (f_atom_nr < 0) {
dev_err(kctx->kbdev->dev, "Failed to allocate katom\n");
kbasep_release_katom(kctx, t_atom_nr);
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
kbasep_replay_create_atom(kctx, t_atom, t_atom_nr, prio);
base_jd_atom_dep_set(&f_atom->pre_dep[0], t_atom_nr , BASE_JD_DEP_TYPE_DATA);
- return MALI_ERROR_NONE;
+ return 0;
}
#ifdef CONFIG_MALI_DEBUG
static void payload_dump(struct kbase_context *kctx, base_jd_replay_payload *payload)
{
- mali_addr64 next;
+ u64 next;
dev_dbg(kctx->kbdev->dev, "Tiler jc list :\n");
next = payload->tiler_jc_list;
while (next) {
- struct kbasep_map_struct map;
+ struct kbase_vmap_struct map;
base_jd_replay_jc *jc_struct;
- jc_struct = kbasep_map(kctx, next, sizeof(*jc_struct), &map);
+ jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &map);
if (!jc_struct)
return;
next = jc_struct->next;
- kbasep_unmap(kctx, &map);
+ kbase_vunmap(kctx, &map);
}
}
#endif
* @param[in] replay_atom Replay soft job atom
* @param[in] t_atom Atom to use for tiler jobs
* @param[in] f_atom Atom to use for fragment jobs
- * @return MALI_ERROR_NONE on success, error code on failure
+ * @return 0 on success, error code on failure
*/
-static mali_error kbasep_replay_parse_payload(struct kbase_context *kctx,
+static int kbasep_replay_parse_payload(struct kbase_context *kctx,
struct kbase_jd_atom *replay_atom,
struct base_jd_atom_v2 *t_atom,
struct base_jd_atom_v2 *f_atom)
{
base_jd_replay_payload *payload;
- mali_addr64 next;
- mali_addr64 prev_jc = 0;
+ u64 next;
+ u64 prev_jc = 0;
u16 hw_job_id_offset = 0;
- mali_error ret = MALI_ERROR_FUNCTION_FAILED;
- struct kbasep_map_struct map;
+ int ret = -EINVAL;
+ struct kbase_vmap_struct map;
dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: replay_atom->jc = %llx sizeof(payload) = %zu\n",
replay_atom->jc, sizeof(payload));
- kbase_gpu_vm_lock(kctx);
-
- payload = kbasep_map(kctx, replay_atom->jc, sizeof(*payload), &map);
+ payload = kbase_vmap(kctx, replay_atom->jc, sizeof(*payload), &map);
if (!payload) {
- kbase_gpu_vm_unlock(kctx);
dev_err(kctx->kbdev->dev, "kbasep_replay_parse_payload: failed to map payload into kernel space\n");
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
#ifdef CONFIG_MALI_DEBUG
while (next) {
base_jd_replay_jc *jc_struct;
- struct kbasep_map_struct jc_map;
- mali_addr64 jc;
+ struct kbase_vmap_struct jc_map;
+ u64 jc;
- jc_struct = kbasep_map(kctx, next, sizeof(*jc_struct), &jc_map);
+ jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &jc_map);
if (!jc_struct) {
dev_err(kctx->kbdev->dev, "Failed to map jc struct\n");
if (next)
jc_struct->jc = 0;
- kbasep_unmap(kctx, &jc_map);
+ kbase_vunmap(kctx, &jc_map);
if (jc) {
u16 max_hw_job_id = 0;
if (kbasep_replay_find_hw_job_id(kctx, jc,
- &max_hw_job_id) != MALI_ERROR_NONE)
+ &max_hw_job_id) != 0)
goto out;
if (kbasep_replay_parse_jc(kctx, jc, prev_jc,
payload->tiler_heap_free,
payload->tiler_hierarchy_mask,
payload->hierarchy_default_weight,
- hw_job_id_offset, MALI_FALSE) !=
- MALI_ERROR_NONE) {
+ hw_job_id_offset, false) != 0) {
goto out;
}
payload->tiler_heap_free,
payload->fragment_hierarchy_mask,
payload->hierarchy_default_weight, 0,
- MALI_TRUE) != MALI_ERROR_NONE) {
+ true) != 0) {
goto out;
}
dev_dbg(kctx->kbdev->dev, "t_atom->jc=%llx f_atom->jc=%llx\n",
t_atom->jc, f_atom->jc);
- ret = MALI_ERROR_NONE;
+ ret = 0;
out:
- kbasep_unmap(kctx, &map);
-
- kbase_gpu_vm_unlock(kctx);
+ kbase_vunmap(kctx, &map);
return ret;
}
struct base_jd_atom_v2 t_atom, f_atom;
struct kbase_jd_atom *t_katom, *f_katom;
+ base_jd_prio atom_prio;
katom = container_of(data, struct kbase_jd_atom, work);
kctx = katom->kctx;
mutex_lock(&jctx->lock);
- if (kbasep_replay_create_atoms(kctx, &t_atom, &f_atom,
- katom->nice_prio) != MALI_ERROR_NONE) {
+ atom_prio = kbasep_js_sched_prio_to_atom_prio(katom->sched_priority);
+
+ if (kbasep_replay_create_atoms(
+ kctx, &t_atom, &f_atom, atom_prio) != 0) {
katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
goto out;
}
t_katom = &jctx->atoms[t_atom.atom_number];
f_katom = &jctx->atoms[f_atom.atom_number];
- if (kbasep_replay_parse_payload(kctx, katom, &t_atom, &f_atom) !=
- MALI_ERROR_NONE) {
+ if (kbasep_replay_parse_payload(kctx, katom, &t_atom, &f_atom) != 0) {
kbasep_release_katom(kctx, t_atom.atom_number);
kbasep_release_katom(kctx, f_atom.atom_number);
katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
katom->event_code = BASE_JD_EVENT_DONE;
out:
- if (katom->event_code != BASE_JD_EVENT_DONE)
+ if (katom->event_code != BASE_JD_EVENT_DONE) {
+ kbase_disjoint_state_down(kctx->kbdev);
+
need_to_try_schedule_context |= jd_done_nolock(katom);
+ }
if (need_to_try_schedule_context)
- kbasep_js_try_schedule_head_ctx(kctx->kbdev);
+ kbase_js_sched_all(kctx->kbdev);
+
mutex_unlock(&jctx->lock);
}
struct kbase_context *kctx = katom->kctx;
struct device *dev = kctx->kbdev->dev;
base_jd_replay_payload *payload;
- mali_addr64 job_header;
- mali_addr64 job_loop_detect;
+ u64 job_header;
+ u64 job_loop_detect;
struct job_head *job;
- struct kbasep_map_struct job_map;
- struct kbasep_map_struct map;
+ struct kbase_vmap_struct job_map;
+ struct kbase_vmap_struct map;
bool err = false;
/* Replay job if fault is of type BASE_JD_EVENT_JOB_WRITE_FAULT or
* to find out whether the source of exception is POLYGON_LIST. Replay
* is required if the source of fault is POLYGON_LIST.
*/
- kbase_gpu_vm_lock(kctx);
-
- payload = kbasep_map(kctx, katom->jc, sizeof(*payload), &map);
+ payload = kbase_vmap(kctx, katom->jc, sizeof(*payload), &map);
if (!payload) {
- kbase_gpu_vm_unlock(kctx);
dev_err(dev, "kbase_replay_fault_check: failed to map payload.\n");
return false;
}
payload->fragment_core_req);
#endif
/* Process fragment job chain */
- job_header = (mali_addr64) payload->fragment_jc;
+ job_header = (u64) payload->fragment_jc;
job_loop_detect = job_header;
while (job_header) {
- job = kbasep_map(kctx, job_header, sizeof(*job), &job_map);
+ job = kbase_vmap(kctx, job_header, sizeof(*job), &job_map);
if (!job) {
dev_err(dev, "failed to map jc\n");
/* unmap payload*/
- kbasep_unmap(kctx, &map);
- kbase_gpu_vm_unlock(kctx);
+ kbase_vunmap(kctx, &map);
return false;
}
if ((BASE_JD_EVENT_DATA_INVALID_FAULT == katom->event_code) &&
(JOB_POLYGON_LIST == JOB_SOURCE_ID(job->status))) {
err = true;
- kbasep_unmap(kctx, &job_map);
+ kbase_vunmap(kctx, &job_map);
break;
}
else
job_header = job->next._32;
- kbasep_unmap(kctx, &job_map);
+ kbase_vunmap(kctx, &job_map);
/* Job chain loop detected */
if (job_header == job_loop_detect)
}
/* unmap payload*/
- kbasep_unmap(kctx, &map);
- kbase_gpu_vm_unlock(kctx);
+ kbase_vunmap(kctx, &map);
return err;
}
}
/* Check job exception type and source before replaying. */
- if (false == kbase_replay_fault_check(katom)) {
+ if (!kbase_replay_fault_check(katom)) {
dev_dbg(kctx->kbdev->dev,
"Replay cancelled on event %x\n", katom->event_code);
/* katom->event_code is already set to the failure code of the
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase.h>
-static inline mali_bool kbasep_am_i_root(void)
+static inline bool kbasep_am_i_root(void)
{
#if KBASE_HWCNT_DUMP_BYPASS_ROOT
- return MALI_TRUE;
+ return true;
#else
/* Check if root */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
if (uid_eq(current_euid(), GLOBAL_ROOT_UID))
- return MALI_TRUE;
+ return true;
#else
if (current_euid() == 0)
- return MALI_TRUE;
+ return true;
#endif /*LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)*/
- return MALI_FALSE;
+ return false;
#endif /*KBASE_HWCNT_DUMP_BYPASS_ROOT*/
}
* kbase_security_has_capability - see mali_kbase_caps.h for description.
*/
-mali_bool kbase_security_has_capability(struct kbase_context *kctx, enum kbase_security_capability cap, u32 flags)
+bool kbase_security_has_capability(struct kbase_context *kctx, enum kbase_security_capability cap, u32 flags)
{
/* Assume failure */
- mali_bool access_allowed = MALI_FALSE;
- mali_bool audit = (KBASE_SEC_FLAG_AUDIT & flags) ? MALI_TRUE : MALI_FALSE;
+ bool access_allowed = false;
+ bool audit = KBASE_SEC_FLAG_AUDIT & flags;
KBASE_DEBUG_ASSERT(NULL != kctx);
CSTD_UNUSED(kctx);
}
/* Report problem if requested */
- if (MALI_FALSE == access_allowed) {
- if (MALI_FALSE != audit)
- dev_warn(kctx->kbdev->dev, "Security capability failure: %d, %p", cap, (void *)kctx);
- }
+ if (!access_allowed && audit)
+ dev_warn(kctx->kbdev->dev, "Security capability failure: %d, %p", cap, (void *)kctx);
return access_allowed;
}
-KBASE_EXPORT_TEST_API(kbase_security_has_capability)
+KBASE_EXPORT_TEST_API(kbase_security_has_capability);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* @param[in] cap The capability to check for.
* @param[in] flags Additional configuration information
* Such as whether to write an audit message or not.
- * @return MALI_TRUE if success (capability is allowed), MALI_FALSE otherwise.
+ * @return true if success (capability is allowed), false otherwise.
*/
-mali_bool kbase_security_has_capability(struct kbase_context *kctx, enum kbase_security_capability cap, u32 flags);
+bool kbase_security_has_capability(struct kbase_context *kctx, enum kbase_security_capability cap, u32 flags);
#endif /* _KBASE_SECURITY_H_ */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifdef CONFIG_ARM64
+
+#include <mali_kbase.h>
+#include <mali_kbase_smc.h>
+
+
+static noinline u32 invoke_smc_fid(u32 function_id, u64 arg0, u64 arg1,
+ u64 arg2, u64 *res0, u64 *res1, u64 *res2)
+{
+ /* 3 args and 3 returns are chosen arbitrarily,
+ see SMC calling convention for limits */
+ asm volatile(
+ "mov x0, %[fid]\n"
+ "mov x1, %[a0]\n"
+ "mov x2, %[a1]\n"
+ "mov x3, %[a2]\n"
+ "smc #0\n"
+ "str x0, [%[re0]]\n"
+ "str x1, [%[re1]]\n"
+ "str x2, [%[re2]]\n"
+ : [fid] "+r" (function_id), [a0] "+r" (arg0),
+ [a1] "+r" (arg1), [a2] "+r" (arg2)
+ : [re0] "r" (res0), [re1] "r" (res1), [re2] "r" (res2)
+ : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
+ "x8", "x9", "x10", "x11", "x12", "x13",
+ "x14", "x15", "x16", "x17");
+ return function_id;
+}
+
+void kbase_invoke_smc_fid(u32 fid)
+{
+ u64 res0, res1, res2;
+
+ /* Is fast call (bit 31 set) */
+ KBASE_DEBUG_ASSERT(fid & ~SMC_FAST_CALL);
+ /* bits 16-23 must be zero for fast calls */
+ KBASE_DEBUG_ASSERT((fid & (0xFF << 16)) == 0);
+
+ invoke_smc_fid(fid, 0, 0, 0, &res0, &res1, &res2);
+}
+
+void kbase_invoke_smc(u32 oen, u16 function_number, u64 arg0, u64 arg1,
+ u64 arg2, u64 *res0, u64 *res1, u64 *res2)
+{
+ u32 fid = 0;
+
+ /* Only the six bits allowed should be used. */
+ KBASE_DEBUG_ASSERT((oen & ~SMC_OEN_MASK) == 0);
+
+ fid |= SMC_FAST_CALL; /* Bit 31: Fast call */
+ /* Bit 30: 1=SMC64, 0=SMC32 */
+ fid |= oen; /* Bit 29:24: OEN */
+ /* Bit 23:16: Must be zero for fast calls */
+ fid |= (function_number); /* Bit 15:0: function number */
+
+ invoke_smc_fid(fid, arg0, arg1, arg2, res0, res1, res2);
+}
+
+#endif /* CONFIG_ARM64 */
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+
+
+#ifndef _KBASE_SMC_H_
+#define _KBASE_SMC_H_
+
+#ifdef CONFIG_ARM64
+
+#include <mali_kbase.h>
+
+#define SMC_FAST_CALL (1 << 31)
+
+#define SMC_OEN_OFFSET 24
+#define SMC_OEN_MASK (0x3F << SMC_OEN_OFFSET) /* 6 bits */
+#define SMC_OEN_SIP (2 << SMC_OEN_OFFSET)
+#define SMC_OEN_STD (4 << SMC_OEN_OFFSET)
+
+
+/**
+ * kbase_invoke_smc_fid - Does a secure monitor call with the given function_id
+ * @function_id: The SMC function to call, see SMC Calling convention.
+ */
+void kbase_invoke_smc_fid(u32 function_id);
+
+/**
+ * kbase_invoke_smc_fid - Does a secure monitor call with the given parameters.
+ * see SMC Calling Convention for details
+ * @oen: Owning Entity number (SIP, STD etc).
+ * @function_number: ID specifiy which function within the OEN.
+ * @arg0: argument 0 to pass in the SMC call.
+ * @arg1: argument 1 to pass in the SMC call.
+ * @arg2: argument 2 to pass in the SMC call.
+ * @res0: result 0 returned from the SMC call.
+ * @res1: result 1 returned from the SMC call.
+ * @res2: result 2 returned from the SMC call.
+ */
+void kbase_invoke_smc(u32 oen, u16 function_number, u64 arg0, u64 arg1,
+ u64 arg2, u64 *res0, u64 *res1, u64 *res2);
+
+#endif /* CONFIG_ARM64 */
+
+#endif /* _KBASE_SMC_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <linux/syscalls.h>
#include "mali_kbase_sync.h"
#endif
-
+#include <mali_kbase_hwaccess_time.h>
+#include <linux/version.h>
/* Mask to check cache alignment of data structures */
#define KBASE_CACHE_ALIGNMENT_MASK ((1<<L1_CACHE_SHIFT)-1)
struct base_dump_cpu_gpu_counters data;
u64 system_time;
u64 cycle_counter;
- mali_addr64 jc = katom->jc;
+ u64 jc = katom->jc;
struct kbase_context *kctx = katom->kctx;
int pm_active_err;
- u32 hi1, hi2;
-
memset(&data, 0, sizeof(data));
/* Take the PM active reference as late as possible - otherwise, it could
return pm_active_err;
}
- kbase_pm_request_gpu_cycle_counter(kctx->kbdev);
-
- /* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
- do {
- hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
- cycle_counter = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL);
- hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), NULL);
- cycle_counter |= (((u64) hi1) << 32);
- } while (hi1 != hi2);
-
- /* Read hi, lo, hi to ensure that overflow from lo to hi is handled correctly */
- do {
- hi1 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
- system_time = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_LO), NULL);
- hi2 = kbase_reg_read(kctx->kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), NULL);
- system_time |= (((u64) hi1) << 32);
- } while (hi1 != hi2);
-
- /* Record the CPU's idea of current time */
- getrawmonotonic(&ts);
-
- kbase_pm_release_gpu_cycle_counter(kctx->kbdev);
+ kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time,
+ &ts);
kbase_pm_context_idle(kctx->kbdev);
reg = kbase_region_tracker_find_region_enclosing_address(kctx, jc);
if (reg &&
(reg->flags & KBASE_REG_GPU_WR) &&
- reg->alloc && reg->alloc->pages)
- addr = reg->alloc->pages[pfn - reg->start_pfn];
+ reg->cpu_alloc && reg->cpu_alloc->pages)
+ addr = reg->cpu_alloc->pages[pfn - reg->start_pfn];
kbase_gpu_vm_unlock(kctx);
if (!addr)
if (!page)
return 0;
- dma_sync_single_for_cpu(katom->kctx->kbdev->dev,
+ kbase_sync_single_for_cpu(katom->kctx->kbdev,
kbase_dma_addr(pfn_to_page(PFN_DOWN(addr))) +
offset, sizeof(data),
DMA_BIDIRECTIONAL);
+
memcpy(page + offset, &data, sizeof(data));
- dma_sync_single_for_device(katom->kctx->kbdev->dev,
+
+ kbase_sync_single_for_device(katom->kctx->kbdev,
kbase_dma_addr(pfn_to_page(PFN_DOWN(addr))) +
offset, sizeof(data),
DMA_BIDIRECTIONAL);
+
kunmap(pfn_to_page(PFN_DOWN(addr)));
/* Atom was fine - mark it as done */
list_del(&katom->dep_item[0]);
kbase_finish_soft_job(katom);
if (jd_done_nolock(katom))
- kbasep_js_try_schedule_head_ctx(kctx->kbdev);
+ kbase_js_sched_all(kctx->kbdev);
mutex_unlock(&kctx->jctx.lock);
}
struct sync_pt *pt;
struct sync_timeline *timeline;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
if (!list_is_singular(&katom->fence->pt_list_head)) {
+#else
+ if (katom->fence->num_fences != 1) {
+#endif
/* Not exactly one item in the list - so it didn't (directly) come from us */
return BASE_JD_EVENT_JOB_CANCELLED;
}
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
pt = list_first_entry(&katom->fence->pt_list_head, struct sync_pt, pt_list);
- timeline = pt->parent;
+#else
+ pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base);
+#endif
+ timeline = sync_pt_parent(pt);
if (!kbase_sync_timeline_is_ours(timeline)) {
/* Fence has a sync_pt which isn't ours! */
/* Propagate the fence status to the atom.
* If negative then cancel this atom and its dependencies.
*/
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
if (fence->status < 0)
+#else
+ if (atomic_read(&fence->status) < 0)
+#endif
katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
/* To prevent a potential deadlock we schedule the work onto the job_done_wq workqueue
kbase_finish_soft_job(katom);
if (jd_done_nolock(katom))
- kbasep_js_try_schedule_head_ctx(katom->kctx->kbdev);
-
- return;
+ kbase_js_sched_all(katom->kctx->kbdev);
}
#endif /* CONFIG_SYNC */
}
}
-mali_error kbase_prepare_soft_job(struct kbase_jd_atom *katom)
+int kbase_prepare_soft_job(struct kbase_jd_atom *katom)
{
switch (katom->core_req & BASEP_JD_REQ_ATOM_TYPE) {
case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME:
{
if (0 != (katom->jc & KBASE_CACHE_ALIGNMENT_MASK))
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
break;
#ifdef CONFIG_SYNC
int fd;
if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence)))
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
fd = kbase_stream_create_fence(fence.basep.stream_fd);
if (fd < 0)
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
katom->fence = sync_fence_fdget(fd);
if (katom->fence == NULL) {
/* The only way the fence can be NULL is if userspace closed it for us.
* So we don't need to clear it up */
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
fence.basep.fd = fd;
if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) {
katom->fence = NULL;
sys_close(fd);
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
}
break;
struct base_fence fence;
if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence)))
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
/* Get a reference to the fence object */
katom->fence = sync_fence_fdget(fence.basep.fd);
if (katom->fence == NULL)
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
break;
#endif /* CONFIG_SYNC */
break;
default:
/* Unsupported soft-job */
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
}
- return MALI_ERROR_NONE;
+ return 0;
}
void kbase_finish_soft_job(struct kbase_jd_atom *katom)
break;
#ifdef CONFIG_SYNC
case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
+ /* If fence has not yet been signalled, do it now */
if (katom->fence) {
- /* The fence has not yet been signalled, so we do it now */
- kbase_fence_trigger(katom, katom->event_code == BASE_JD_EVENT_DONE ? 0 : -EFAULT);
+ kbase_fence_trigger(katom, katom->event_code ==
+ BASE_JD_EVENT_DONE ? 0 : -EFAULT);
sync_fence_put(katom->fence);
katom->fence = NULL;
}
struct kbase_jd_atom *tmp_iter;
struct kbase_jd_atom *katom_iter;
struct kbasep_js_device_data *js_devdata;
- mali_bool resched = MALI_FALSE;
+ bool resched = false;
KBASE_DEBUG_ASSERT(kbdev);
/* Move out the entire list */
mutex_lock(&js_devdata->runpool_mutex);
- list_splice_init(&js_devdata->suspended_soft_jobs_list, &local_suspended_soft_jobs);
+ list_splice_init(&js_devdata->suspended_soft_jobs_list,
+ &local_suspended_soft_jobs);
mutex_unlock(&js_devdata->runpool_mutex);
- /* Each atom must be detached from the list and ran separately - it could
- * be re-added to the old list, but this is unlikely */
- list_for_each_entry_safe(katom_iter, tmp_iter, &local_suspended_soft_jobs, dep_item[1])
- {
+ /*
+ * Each atom must be detached from the list and ran separately -
+ * it could be re-added to the old list, but this is unlikely
+ */
+ list_for_each_entry_safe(katom_iter, tmp_iter,
+ &local_suspended_soft_jobs, dep_item[1]) {
struct kbase_context *kctx = katom_iter->kctx;
mutex_lock(&kctx->jctx.lock);
resched |= jd_done_nolock(katom_iter);
} else {
/* The job has not completed */
- KBASE_DEBUG_ASSERT((katom_iter->core_req & BASEP_JD_REQ_ATOM_TYPE)
+ KBASE_DEBUG_ASSERT((katom_iter->core_req &
+ BASEP_JD_REQ_ATOM_TYPE)
!= BASE_JD_REQ_SOFT_REPLAY);
- list_add_tail(&katom_iter->dep_item[0], &kctx->waiting_soft_jobs);
+ list_add_tail(&katom_iter->dep_item[0],
+ &kctx->waiting_soft_jobs);
}
mutex_unlock(&kctx->jctx.lock);
}
if (resched)
- kbasep_js_try_schedule_head_ctx(kbdev);
+ kbase_js_sched_all(kbdev);
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-
-
-/**
- * @file mali_kbase_sync.c
- *
- */
-
#ifdef CONFIG_SYNC
+#include <linux/seq_file.h>
#include "sync.h"
#include <mali_kbase.h>
+#include <mali_kbase_sync.h>
struct mali_sync_timeline {
struct sync_timeline timeline;
{
struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
struct mali_sync_pt *new_mpt;
- struct sync_pt *new_pt = sync_pt_create(pt->parent, sizeof(struct mali_sync_pt));
+ struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), sizeof(struct mali_sync_pt));
if (!new_pt)
return NULL;
static int timeline_has_signaled(struct sync_pt *pt)
{
struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
- struct mali_sync_timeline *mtl = to_mali_sync_timeline(pt->parent);
+ struct mali_sync_timeline *mtl = to_mali_sync_timeline(sync_pt_parent(pt));
int result = mpt->result;
int diff = atomic_read(&mtl->signalled) - mpt->order;
if (diff >= 0)
- return result < 0 ? result : 1;
- else
- return 0;
+ return (result < 0) ? result : 1;
+
+ return 0;
}
static int timeline_compare(struct sync_pt *a, struct sync_pt *b)
int diff = ma->order - mb->order;
- if (diff < 0)
- return -1;
- else if (diff == 0)
+ if (diff == 0)
return 0;
- else
- return 1;
+
+ return (diff < 0) ? -1 : 1;
}
static void timeline_value_str(struct sync_timeline *timeline, char *str,
void kbase_sync_signal_pt(struct sync_pt *pt, int result)
{
struct mali_sync_pt *mpt = to_mali_sync_pt(pt);
- struct mali_sync_timeline *mtl = to_mali_sync_timeline(pt->parent);
+ struct mali_sync_timeline *mtl = to_mali_sync_timeline(sync_pt_parent(pt));
int signalled;
int diff;
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define MALI_KBASE_SYNC_H
#include "sync.h"
-#include <malisw/mali_malisw.h>
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
+/* For backwards compatiblility with kernels before 3.17. After 3.17
+ * sync_pt_parent is included in the kernel. */
+static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt)
+{
+ return pt->parent;
+}
+#endif
/*
* Create a stream object.
* - dup to add a ref
* - close to remove a ref
*/
-mali_error kbase_stream_create(const char *name, int *const out_fd);
+int kbase_stream_create(const char *name, int *const out_fd);
/*
* Create a fence in a stream object
* This function is only usable to catch unintentional user errors early,
* it does not stop malicious code changing the fd after this function returns.
*/
-mali_error kbase_fence_validate(int fd);
+int kbase_fence_validate(int fd);
/* Returns true if the specified timeline is allocated by Mali */
int kbase_sync_timeline_is_ours(struct sync_timeline *timeline);
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
.release = kbase_stream_close,
};
-mali_error kbase_stream_create(const char *name, int *const out_fd)
+int kbase_stream_create(const char *name, int *const out_fd)
{
struct sync_timeline *tl;
tl = kbase_sync_timeline_alloc(name);
if (!tl)
- return MALI_ERROR_FUNCTION_FAILED;
+ return -EINVAL;
*out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY | O_CLOEXEC);
if (*out_fd < 0) {
sync_timeline_destroy(tl);
- return MALI_ERROR_FUNCTION_FAILED;
- } else {
- return MALI_ERROR_NONE;
+ return -EINVAL;
}
+
+ return 0;
}
int kbase_stream_create_fence(int tl_fd)
return fd;
}
-mali_error kbase_fence_validate(int fd)
+int kbase_fence_validate(int fd)
{
struct sync_fence *fence;
fence = sync_fence_fdget(fd);
- if (NULL != fence) {
- sync_fence_put(fence);
- return MALI_ERROR_NONE;
- } else {
- return MALI_ERROR_FUNCTION_FAILED;
- }
+ if (!fence)
+ return -EINVAL;
+
+ sync_fence_put(fence);
+ return 0;
}
#endif /* CONFIG_SYNC */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include <linux/anon_inodes.h>
+#include <linux/atomic.h>
+#include <linux/file.h>
+#include <linux/mutex.h>
+#include <linux/poll.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/stringify.h>
+#include <linux/timer.h>
+#include <linux/wait.h>
+
+#include <mali_kbase.h>
+#include <mali_kbase_jm.h>
+#include <mali_kbase_tlstream.h>
+
+/*****************************************************************************/
+
+/* The version of timeline stream. */
+#define KBASEP_TLSTREAM_VERSION 1
+
+/* The maximum expected length of string in tracepoint descriptor. */
+#define STRLEN_MAX 64 /* bytes */
+
+/* The number of nanoseconds in a second. */
+#define NSECS_IN_SEC 1000000000ull /* ns */
+
+/* The number of nanoseconds to wait before autoflushing the stream. */
+#define AUTOFLUSH_TIMEOUT (2ull * NSECS_IN_SEC) /* ns */
+
+/* The period of autoflush checker execution in milliseconds. */
+#define AUTOFLUSH_INTERVAL 1000 /* ms */
+
+/* The maximum size of a single packet used by timeline. */
+#define PACKET_SIZE 2048 /* bytes */
+
+/* The number of packets used by one timeline stream. */
+#define PACKET_COUNT 16
+
+/* The number of bytes reserved for packet header.
+ * These value must be defined according to MIPE documentation. */
+#define PACKET_HEADER_SIZE 8 /* bytes */
+
+/* The number of bytes reserved for packet sequence number.
+ * These value must be defined according to MIPE documentation. */
+#define PACKET_NUMBER_SIZE 4 /* bytes */
+
+/* Packet header - first word.
+ * These values must be defined according to MIPE documentation. */
+#define PACKET_STREAMID_POS 0
+#define PACKET_STREAMID_LEN 8
+#define PACKET_RSVD1_POS (PACKET_STREAMID_POS + PACKET_STREAMID_LEN)
+#define PACKET_RSVD1_LEN 8
+#define PACKET_TYPE_POS (PACKET_RSVD1_POS + PACKET_RSVD1_LEN)
+#define PACKET_TYPE_LEN 3
+#define PACKET_CLASS_POS (PACKET_TYPE_POS + PACKET_TYPE_LEN)
+#define PACKET_CLASS_LEN 7
+#define PACKET_FAMILY_POS (PACKET_CLASS_POS + PACKET_CLASS_LEN)
+#define PACKET_FAMILY_LEN 6
+
+/* Packet header - second word
+ * These values must be defined according to MIPE documentation. */
+#define PACKET_LENGTH_POS 0
+#define PACKET_LENGTH_LEN 24
+#define PACKET_SEQBIT_POS (PACKET_LENGTH_POS + PACKET_LENGTH_LEN)
+#define PACKET_SEQBIT_LEN 1
+#define PACKET_RSVD2_POS (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN)
+#define PACKET_RSVD2_LEN 7
+
+/* Types of streams generated by timeline.
+ * Order is significant! Header streams must precede respective body streams. */
+enum tl_stream_type {
+ TL_STREAM_TYPE_OBJ_HEADER,
+ TL_STREAM_TYPE_OBJ_SUMMARY,
+ TL_STREAM_TYPE_OBJ,
+ TL_STREAM_TYPE_AUX_HEADER,
+ TL_STREAM_TYPE_AUX,
+
+ TL_STREAM_TYPE_COUNT
+};
+
+/* Timeline packet family ids.
+ * Values are significant! Check MIPE documentation. */
+enum tl_packet_family {
+ TL_PACKET_FAMILY_CTRL = 0, /* control packets */
+ TL_PACKET_FAMILY_TL = 1, /* timeline packets */
+
+ TL_PACKET_FAMILY_COUNT
+};
+
+/* Packet classes used in timeline streams.
+ * Values are significant! Check MIPE documentation. */
+enum tl_packet_class {
+ TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */
+ TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */
+};
+
+/* Packet types used in timeline streams.
+ * Values are significant! Check MIPE documentation. */
+enum tl_packet_type {
+ TL_PACKET_TYPE_HEADER = 0, /* stream's header/directory */
+ TL_PACKET_TYPE_BODY = 1, /* stream's body */
+ TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */
+};
+
+/* Message ids of trace events that are recorded in the timeline stream. */
+enum tl_msg_id {
+ /* Timeline object events. */
+ KBASE_TL_NEW_CTX,
+ KBASE_TL_NEW_GPU,
+ KBASE_TL_NEW_LPU,
+ KBASE_TL_NEW_ATOM,
+ KBASE_TL_DEL_CTX,
+ KBASE_TL_DEL_ATOM,
+ KBASE_TL_LIFELINK_LPU_GPU,
+ KBASE_TL_RET_GPU_CTX,
+ KBASE_TL_RET_ATOM_CTX,
+ KBASE_TL_RET_ATOM_LPU,
+ KBASE_TL_NRET_GPU_CTX,
+ KBASE_TL_NRET_ATOM_CTX,
+ KBASE_TL_NRET_ATOM_LPU,
+
+ /* Timeline non-object events. */
+ KBASE_AUX_PM_STATE,
+ KBASE_AUX_JOB_SOFTSTOP,
+ KBASE_AUX_PAGEFAULT,
+ KBASE_AUX_PAGESALLOC
+};
+
+/*****************************************************************************/
+
+/**
+ * struct tl_stream - timeline stream structure
+ * @lock: message order lock
+ * @buffer: array of buffers
+ * @wbi: write buffer index
+ * @rbi: read buffer index
+ * @numbered: if non-zero stream's packets are sequentially numbered
+ * @last_write_time: timestamp indicating last write
+ *
+ * This structure holds information needed to construct proper packets in the
+ * timeline stream. Each message in sequence must bear timestamp that is greater
+ * to one in previous message in the same stream. For this reason lock is held
+ * throughout the process of message creation. Each stream contains set of
+ * buffers. Each buffer will hold one MIPE packet. In case there is no free
+ * space required to store incoming message the oldest buffer is discarded.
+ * Each packet in timeline body stream has sequence number embedded (this value
+ * must increment monotonically and is used by packets receiver to discover
+ * buffer overflows.
+ */
+struct tl_stream {
+ spinlock_t lock;
+
+ struct {
+ atomic_t size; /* number of bytes in buffer */
+ char data[PACKET_SIZE]; /* buffer's data */
+ } buffer[PACKET_COUNT];
+
+ atomic_t wbi;
+ atomic_t rbi;
+
+ int numbered;
+ u64 last_write_time;
+};
+
+/**
+ * struct tp_desc - tracepoint message descriptor structure
+ * @id: tracepoint ID identifying message in stream
+ * @id_str: human readable version of tracepoint ID
+ * @name: tracepoint description
+ * @arg_types: tracepoint's arguments types declaration
+ * @arg_names: comma separated list of tracepoint's arguments names
+ */
+struct tp_desc {
+ u32 id;
+ const char *id_str;
+ const char *name;
+ const char *arg_types;
+ const char *arg_names;
+};
+
+/*****************************************************************************/
+
+/* Configuration of timeline streams generated by kernel.
+ * Kernel emit only streams containing either timeline object events or
+ * auxiliary events. All streams have stream id value of 1 (as opposed to user
+ * space streams that have value of 0). */
+static const struct {
+ enum tl_packet_family pkt_family;
+ enum tl_packet_class pkt_class;
+ enum tl_packet_type pkt_type;
+ unsigned int stream_id;
+} tl_stream_cfg[TL_STREAM_TYPE_COUNT] = {
+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_HEADER, 1},
+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_SUMMARY, 1},
+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY, 1},
+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_HEADER, 1},
+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_BODY, 1}
+};
+
+/* The timeline streams generated by kernel. */
+static struct tl_stream *tl_stream[TL_STREAM_TYPE_COUNT];
+
+/* Autoflush timer. */
+static struct timer_list autoflush_timer;
+
+/* If non-zero autoflush timer is active. */
+static atomic_t autoflush_timer_active;
+
+/* Reader lock. Only one reader is allowed to have access to the timeline
+ * streams at any given time. */
+static DEFINE_MUTEX(tl_reader_lock);
+
+/* Indicator of whether the timeline stream file descriptor is already used. */
+static atomic_t tlstream_busy = {0};
+
+/* Timeline stream event queue. */
+static DECLARE_WAIT_QUEUE_HEAD(tl_event_queue);
+
+/* The timeline stream file operations functions. */
+static ssize_t kbasep_tlstream_read(
+ struct file *filp,
+ char __user *buffer,
+ size_t size,
+ loff_t *f_pos);
+static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait);
+static int kbasep_tlstream_release(struct inode *inode, struct file *filp);
+
+/* The timeline stream file operations structure. */
+static const struct file_operations kbasep_tlstream_fops = {
+ .release = kbasep_tlstream_release,
+ .read = kbasep_tlstream_read,
+ .poll = kbasep_tlstream_poll,
+};
+
+/* Descriptors of timeline messages transmitted in object events stream. */
+static const struct tp_desc tp_desc_obj[] = {
+ {
+ KBASE_TL_NEW_CTX,
+ __stringify(KBASE_TL_NEW_CTX),
+ "object ctx is created",
+ "@pI",
+ "ctx,ctx_nr"
+ },
+ {
+ KBASE_TL_NEW_GPU,
+ __stringify(KBASE_TL_NEW_GPU),
+ "object gpu is created",
+ "@pII",
+ "gpu,gpu_id,core_count"
+ },
+ {
+ KBASE_TL_NEW_LPU,
+ __stringify(KBASE_TL_NEW_LPU),
+ "object lpu is created",
+ "@pII",
+ "lpu,lpu_nr,lpu_fn"
+ },
+ {
+ KBASE_TL_NEW_ATOM,
+ __stringify(KBASE_TL_NEW_ATOM),
+ "object atom is created",
+ "@pI",
+ "atom,atom_nr"
+ },
+ {
+ KBASE_TL_DEL_CTX,
+ __stringify(KBASE_TL_DEL_CTX),
+ "context is destroyed",
+ "@p",
+ "context"
+ },
+ {
+ KBASE_TL_DEL_ATOM,
+ __stringify(KBASE_TL_DEL_ATOM),
+ "atom is destroyed",
+ "@p",
+ "atom"
+ },
+ {
+ KBASE_TL_LIFELINK_LPU_GPU,
+ __stringify(KBASE_TL_LIFELINK_LPU_GPU),
+ "lpu is deleted with gpu",
+ "@pp",
+ "lpu,gpu"
+ },
+ {
+ KBASE_TL_RET_GPU_CTX,
+ __stringify(KBASE_TL_RET_GPU_CTX),
+ "gpu is retained by context",
+ "@pp",
+ "gpu,ctx"
+ },
+ {
+ KBASE_TL_RET_ATOM_CTX,
+ __stringify(KBASE_TL_RET_ATOM_CTX),
+ "atom is retained by context",
+ "@pp",
+ "atom,ctx"
+ },
+ {
+ KBASE_TL_RET_ATOM_LPU,
+ __stringify(KBASE_TL_RET_ATOM_LPU),
+ "atom is retained by lpu",
+ "@pp",
+ "atom,lpu"
+ },
+ {
+ KBASE_TL_NRET_GPU_CTX,
+ __stringify(KBASE_TL_NRET_GPU_CTX),
+ "gpu is released by context",
+ "@pp",
+ "gpu,ctx"
+ },
+ {
+ KBASE_TL_NRET_ATOM_CTX,
+ __stringify(KBASE_TL_NRET_ATOM_CTX),
+ "atom is released by context",
+ "@pp",
+ "atom,context"
+ },
+ {
+ KBASE_TL_NRET_ATOM_LPU,
+ __stringify(KBASE_TL_NRET_ATOM_LPU),
+ "atom is released by lpu",
+ "@pp",
+ "atom,lpu"
+ },
+};
+
+/* Descriptors of timeline messages transmitted in auxiliary events stream. */
+static const struct tp_desc tp_desc_aux[] = {
+ {
+ KBASE_AUX_PM_STATE,
+ __stringify(KBASE_AUX_PM_STATE),
+ "PM state",
+ "@IL",
+ "core_type,core_state_bitset"
+ },
+ {
+ KBASE_AUX_JOB_SOFTSTOP,
+ __stringify(KBASE_AUX_JOB_SOFTSTOP),
+ "Job soft stop",
+ "@I",
+ "tag_id"
+ },
+ {
+ KBASE_AUX_PAGEFAULT,
+ __stringify(KBASE_AUX_PAGEFAULT),
+ "Page fault",
+ "@II",
+ "as_id,page_cnt"
+ },
+ {
+ KBASE_AUX_PAGESALLOC,
+ __stringify(KBASE_AUX_PAGESALLOC),
+ "Total alloc pages change",
+ "@l",
+ "page_cnt_change"
+ }
+};
+
+#if MALI_UNIT_TEST
+/* Number of bytes read by user. */
+static atomic_t tlstream_bytes_collected = {0};
+
+/* Number of bytes generated by tracepoint messages. */
+static atomic_t tlstream_bytes_generated = {0};
+#endif /* MALI_UNIT_TEST */
+
+/*****************************************************************************/
+
+/**
+ * kbasep_tlstream_get_timestamp - return timestamp
+ *
+ * Function returns timestamp value based on raw monotonic timer. Value will
+ * wrap around zero in case of overflow.
+ * Return: timestamp value
+ */
+static u64 kbasep_tlstream_get_timestamp(void)
+{
+ struct timespec ts;
+ u64 timestamp;
+
+ getrawmonotonic(&ts);
+ timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
+ return timestamp;
+}
+
+/**
+ * kbasep_tlstream_write_bytes - write data to message buffer
+ * @buffer: buffer where data will be written
+ * @pos: position in the buffer where to place data
+ * @bytes: pointer to buffer holding data
+ * @len: length of data to be written
+ *
+ * Return: updated position in the buffer
+ */
+static size_t kbasep_tlstream_write_bytes(
+ char *buffer,
+ size_t pos,
+ const void *bytes,
+ size_t len)
+{
+ KBASE_DEBUG_ASSERT(buffer);
+ KBASE_DEBUG_ASSERT(bytes);
+
+ memcpy(&buffer[pos], bytes, len);
+
+ return pos + len;
+}
+
+/**
+ * kbasep_tlstream_write_string - write string to message buffer
+ * @buffer: buffer where data will be written
+ * @pos: position in the buffer where to place data
+ * @string: pointer to buffer holding the source string
+ * @max_write_size: number of bytes that can be stored in buffer
+ *
+ * Return: updated position in the buffer
+ */
+static size_t kbasep_tlstream_write_string(
+ char *buffer,
+ size_t pos,
+ const char *string,
+ size_t max_write_size)
+{
+ u32 string_len;
+
+ KBASE_DEBUG_ASSERT(buffer);
+ KBASE_DEBUG_ASSERT(string);
+ /* Timeline string consists of at least string length and nul
+ * terminator. */
+ KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char));
+ max_write_size -= sizeof(string_len);
+
+ string_len = strlcpy(
+ &buffer[pos + sizeof(string_len)],
+ string,
+ max_write_size);
+ string_len += sizeof(char);
+
+ /* Make sure that the source string fit into the buffer. */
+ KBASE_DEBUG_ASSERT(string_len <= max_write_size);
+
+ /* Update string length. */
+ memcpy(&buffer[pos], &string_len, sizeof(string_len));
+
+ return pos + sizeof(string_len) + string_len;
+}
+
+/**
+ * kbasep_tlstream_write_timestamp - write timestamp to message buffer
+ * @buffer: buffer where data will be written
+ * @pos: position in the buffer where to place data
+ *
+ * Return: updated position in the buffer
+ */
+static size_t kbasep_tlstream_write_timestamp(void *buffer, size_t pos)
+{
+ u64 timestamp = kbasep_tlstream_get_timestamp();
+
+ return kbasep_tlstream_write_bytes(
+ buffer, pos,
+ ×tamp, sizeof(timestamp));
+}
+
+/**
+ * kbasep_tlstream_put_bits - put bits in a word
+ * @word: pointer to the words being modified
+ * @value: value that shall be written to given position
+ * @bitpos: position where value shall be written (in bits)
+ * @bitlen: length of value (in bits)
+ */
+static void kbasep_tlstream_put_bits(
+ u32 *word,
+ u32 value,
+ unsigned int bitpos,
+ unsigned int bitlen)
+{
+ const u32 mask = ((1 << bitlen) - 1) << bitpos;
+
+ KBASE_DEBUG_ASSERT(word);
+ KBASE_DEBUG_ASSERT((0 != bitlen) && (32 >= bitlen));
+ KBASE_DEBUG_ASSERT((bitpos + bitlen) <= 32);
+
+ *word &= ~mask;
+ *word |= ((value << bitpos) & mask);
+}
+
+/**
+ * kbasep_tlstream_packet_header_setup - setup the packet header
+ * @buffer: pointer to the buffer
+ * @pkt_family: packet's family
+ * @pkt_type: packet's type
+ * @pkt_class: packet's class
+ * @stream_id: stream id
+ * @numbered: non-zero if this stream is numbered
+ *
+ * Function sets up immutable part of packet header in the given buffer.
+ */
+static void kbasep_tlstream_packet_header_setup(
+ char *buffer,
+ enum tl_packet_family pkt_family,
+ enum tl_packet_class pkt_class,
+ enum tl_packet_type pkt_type,
+ unsigned int stream_id,
+ int numbered)
+{
+ u32 word0 = 0;
+ u32 word1 = 0;
+
+ KBASE_DEBUG_ASSERT(buffer);
+ KBASE_DEBUG_ASSERT(pkt_family == TL_PACKET_FAMILY_TL);
+ KBASE_DEBUG_ASSERT(
+ (pkt_type == TL_PACKET_TYPE_HEADER) ||
+ (pkt_type == TL_PACKET_TYPE_SUMMARY) ||
+ (pkt_type == TL_PACKET_TYPE_BODY));
+ KBASE_DEBUG_ASSERT(
+ (pkt_class == TL_PACKET_CLASS_OBJ) ||
+ (pkt_class == TL_PACKET_CLASS_AUX));
+
+ kbasep_tlstream_put_bits(
+ &word0, pkt_family,
+ PACKET_FAMILY_POS, PACKET_FAMILY_LEN);
+ kbasep_tlstream_put_bits(
+ &word0, pkt_class,
+ PACKET_CLASS_POS, PACKET_CLASS_LEN);
+ kbasep_tlstream_put_bits(
+ &word0, pkt_type,
+ PACKET_TYPE_POS, PACKET_TYPE_LEN);
+ kbasep_tlstream_put_bits(
+ &word0, stream_id,
+ PACKET_STREAMID_POS, PACKET_STREAMID_LEN);
+
+ if (numbered)
+ kbasep_tlstream_put_bits(
+ &word1, 1,
+ PACKET_SEQBIT_POS, PACKET_SEQBIT_LEN);
+
+ memcpy(&buffer[0], &word0, sizeof(word0));
+ memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1));
+}
+
+/**
+ * kbasep_tlstream_packet_header_update - update the packet header
+ * @buffer: pointer to the buffer
+ * @data_size: amount of data carried in this packet
+ *
+ * Function updates mutable part of packet header in the given buffer.
+ * Note that value of data_size must not including size of the header.
+ */
+static void kbasep_tlstream_packet_header_update(
+ char *buffer,
+ size_t data_size)
+{
+ u32 word0;
+ u32 word1;
+
+ KBASE_DEBUG_ASSERT(buffer);
+ CSTD_UNUSED(word0);
+
+ memcpy(&word1, &buffer[sizeof(word0)], sizeof(word1));
+
+ kbasep_tlstream_put_bits(
+ &word1, data_size,
+ PACKET_LENGTH_POS, PACKET_LENGTH_LEN);
+
+ memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1));
+}
+
+/**
+ * kbasep_tlstream_packet_number_update - update the packet number
+ * @buffer: pointer to the buffer
+ * @counter: value of packet counter for this packet's stream
+ *
+ * Function updates packet number embedded within the packet placed in the
+ * given buffer.
+ */
+static void kbasep_tlstream_packet_number_update(char *buffer, u32 counter)
+{
+ KBASE_DEBUG_ASSERT(buffer);
+
+ memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter));
+}
+
+/**
+ * kbasep_timeline_stream_reset - reset stream
+ * @stream: pointer to the stream structure
+ *
+ * Function discards all pending messages and resets packet counters.
+ */
+static void kbasep_timeline_stream_reset(struct tl_stream *stream)
+{
+ unsigned int i;
+
+ for (i = 0; i < PACKET_COUNT; i++) {
+ if (stream->numbered)
+ atomic_set(
+ &stream->buffer[i].size,
+ PACKET_HEADER_SIZE +
+ PACKET_NUMBER_SIZE);
+ else
+ atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE);
+ }
+
+ atomic_set(&stream->wbi, 0);
+ atomic_set(&stream->rbi, 0);
+}
+
+/**
+ * kbasep_timeline_stream_init - initialize timeline stream
+ * @stream: pointer to the stream structure
+ * @stream_type: stream type
+ */
+static void kbasep_timeline_stream_init(
+ struct tl_stream *stream,
+ enum tl_stream_type stream_type)
+{
+ unsigned int i;
+
+ KBASE_DEBUG_ASSERT(stream);
+ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
+
+ spin_lock_init(&stream->lock);
+
+ /* All packets carrying tracepoints shall be numbered. */
+ if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type)
+ stream->numbered = 1;
+ else
+ stream->numbered = 0;
+
+ for (i = 0; i < PACKET_COUNT; i++)
+ kbasep_tlstream_packet_header_setup(
+ stream->buffer[i].data,
+ tl_stream_cfg[stream_type].pkt_family,
+ tl_stream_cfg[stream_type].pkt_class,
+ tl_stream_cfg[stream_type].pkt_type,
+ tl_stream_cfg[stream_type].stream_id,
+ stream->numbered);
+
+ kbasep_timeline_stream_reset(tl_stream[stream_type]);
+}
+
+/**
+ * kbasep_timeline_stream_term - terminate timeline stream
+ * @stream: pointer to the stream structure
+ */
+static void kbasep_timeline_stream_term(struct tl_stream *stream)
+{
+ KBASE_DEBUG_ASSERT(stream);
+}
+
+/**
+ * kbasep_tlstream_msgbuf_submit - submit packet to the user space
+ * @stream: pointer to the stream structure
+ * @wb_idx_raw: write buffer index
+ * @wb_size: length of data stored in current buffer
+ *
+ * Function updates currently written buffer with packet header. Then write
+ * index is incremented and buffer is handled to user space. Parameters
+ * of new buffer are returned using provided arguments.
+ *
+ * Return: length of data in new buffer
+ *
+ * Warning: User must update the stream structure with returned value.
+ */
+static size_t kbasep_tlstream_msgbuf_submit(
+ struct tl_stream *stream,
+ unsigned int wb_idx_raw,
+ unsigned int wb_size)
+{
+ unsigned int rb_idx_raw = atomic_read(&stream->rbi);
+ unsigned int wb_idx = wb_idx_raw % PACKET_COUNT;
+
+ kbasep_tlstream_packet_header_update(
+ stream->buffer[wb_idx].data,
+ wb_size - PACKET_HEADER_SIZE);
+
+ if (stream->numbered)
+ kbasep_tlstream_packet_number_update(
+ stream->buffer[wb_idx].data,
+ wb_idx_raw);
+
+ /* Increasing write buffer index will expose this packet to the reader.
+ * As stream->lock is not taken on reader side we must make sure memory
+ * is updated correctly before this will happen. */
+ smp_wmb();
+ wb_idx_raw++;
+ atomic_set(&stream->wbi, wb_idx_raw);
+
+ /* Inform user that packets are ready for reading. */
+ wake_up_interruptible(&tl_event_queue);
+
+ /* Detect and mark overflow in this stream. */
+ if (PACKET_COUNT == wb_idx_raw - rb_idx_raw) {
+ /* Reader side depends on this increment to correctly handle
+ * overflows. The value shall be updated only if it was not
+ * modified by the reader. The data holding buffer will not be
+ * updated before stream->lock is released, however size of the
+ * buffer will. Make sure this increment is globally visible
+ * before information about selected write buffer size. */
+ atomic_cmpxchg(&stream->rbi, rb_idx_raw, rb_idx_raw + 1);
+ }
+
+ wb_size = PACKET_HEADER_SIZE;
+ if (stream->numbered)
+ wb_size += PACKET_NUMBER_SIZE;
+
+ return wb_size;
+}
+
+/**
+ * kbasep_tlstream_msgbuf_acquire - lock selected stream and reserves buffer
+ * @stream_type: type of the stream that shall be locked
+ * @msg_size: message size
+ * @flags: pointer to store flags passed back on stream release
+ *
+ * Function will lock the stream and reserve the number of bytes requested
+ * in msg_size for the user.
+ *
+ * Return: pointer to the buffer where message can be stored
+ *
+ * Warning: Stream must be relased with kbasep_tlstream_msgbuf_release().
+ * Only atomic operations are allowed while stream is locked
+ * (i.e. do not use any operation that may sleep).
+ */
+static char *kbasep_tlstream_msgbuf_acquire(
+ enum tl_stream_type stream_type,
+ size_t msg_size,
+ unsigned long *flags)
+{
+ struct tl_stream *stream;
+ unsigned int wb_idx_raw;
+ unsigned int wb_idx;
+ size_t wb_size;
+
+ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
+ KBASE_DEBUG_ASSERT(
+ PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >=
+ msg_size);
+
+ stream = tl_stream[stream_type];
+
+ spin_lock_irqsave(&stream->lock, *flags);
+
+ wb_idx_raw = atomic_read(&stream->wbi);
+ wb_idx = wb_idx_raw % PACKET_COUNT;
+ wb_size = atomic_read(&stream->buffer[wb_idx].size);
+
+ /* Select next buffer if data will not fit into current one. */
+ if (PACKET_SIZE < wb_size + msg_size) {
+ wb_size = kbasep_tlstream_msgbuf_submit(
+ stream, wb_idx_raw, wb_size);
+ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT;
+ }
+
+ /* Reserve space in selected buffer. */
+ atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size);
+
+#if MALI_UNIT_TEST
+ atomic_add(msg_size, &tlstream_bytes_generated);
+#endif /* MALI_UNIT_TEST */
+
+ return &stream->buffer[wb_idx].data[wb_size];
+}
+
+/**
+ * kbasep_tlstream_msgbuf_release - unlock selected stream
+ * @stream_type: type of the stream that shall be locked
+ * @flags: value obtained during stream acquire
+ *
+ * Function releases stream that has been previously locked with a call to
+ * kbasep_tlstream_msgbuf_acquire().
+ */
+static void kbasep_tlstream_msgbuf_release(
+ enum tl_stream_type stream_type,
+ unsigned long flags)
+{
+ struct tl_stream *stream;
+
+ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
+
+ stream = tl_stream[stream_type];
+ stream->last_write_time = kbasep_tlstream_get_timestamp();
+
+ spin_unlock_irqrestore(&stream->lock, flags);
+}
+
+/*****************************************************************************/
+
+/**
+ * kbasep_tlstream_flush_stream - flush stream
+ * @stype: type of stream to be flushed
+ *
+ * Flush pending data in timeline stream.
+ */
+static void kbasep_tlstream_flush_stream(enum tl_stream_type stype)
+{
+ struct tl_stream *stream = tl_stream[stype];
+ unsigned long flags;
+ unsigned int wb_idx_raw;
+ unsigned int wb_idx;
+ size_t wb_size;
+ size_t min_size = PACKET_HEADER_SIZE;
+
+ if (stream->numbered)
+ min_size += PACKET_NUMBER_SIZE;
+
+ spin_lock_irqsave(&stream->lock, flags);
+
+ wb_idx_raw = atomic_read(&stream->wbi);
+ wb_idx = wb_idx_raw % PACKET_COUNT;
+ wb_size = atomic_read(&stream->buffer[wb_idx].size);
+
+ if (wb_size > min_size) {
+ wb_size = kbasep_tlstream_msgbuf_submit(
+ stream, wb_idx_raw, wb_size);
+ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT;
+ atomic_set(&stream->buffer[wb_idx].size, wb_size);
+ }
+ spin_unlock_irqrestore(&stream->lock, flags);
+}
+
+/**
+ * kbasep_tlstream_autoflush_timer_callback - autoflush timer callback
+ * @data: unused
+ *
+ * Timer is executed periodically to check if any of the stream contains
+ * buffer ready to be submitted to user space.
+ */
+static void kbasep_tlstream_autoflush_timer_callback(unsigned long data)
+{
+ u64 timestamp = kbasep_tlstream_get_timestamp();
+ enum tl_stream_type stype;
+ int rcode;
+
+ CSTD_UNUSED(data);
+
+ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) {
+ struct tl_stream *stream = tl_stream[stype];
+ unsigned long flags;
+ unsigned int wb_idx_raw;
+ unsigned int wb_idx;
+ size_t wb_size;
+ size_t min_size = PACKET_HEADER_SIZE;
+
+ if (stream->numbered)
+ min_size += PACKET_NUMBER_SIZE;
+
+ spin_lock_irqsave(&stream->lock, flags);
+
+ wb_idx_raw = atomic_read(&stream->wbi);
+ wb_idx = wb_idx_raw % PACKET_COUNT;
+ wb_size = atomic_read(&stream->buffer[wb_idx].size);
+
+ if (
+ (wb_size > min_size) &&
+ (
+ timestamp - stream->last_write_time >
+ AUTOFLUSH_TIMEOUT)) {
+
+ wb_size = kbasep_tlstream_msgbuf_submit(
+ stream, wb_idx_raw, wb_size);
+ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT;
+ atomic_set(&stream->buffer[wb_idx].size, wb_size);
+ }
+ spin_unlock_irqrestore(&stream->lock, flags);
+ }
+
+ if (atomic_read(&autoflush_timer_active))
+ rcode = mod_timer(
+ &autoflush_timer,
+ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL));
+ CSTD_UNUSED(rcode);
+}
+
+/**
+ * kbasep_tlstream_packet_pending - check timeline streams for pending packets
+ * @stype: pointer to variable where stream type will be placed
+ * @rb_idx_raw: pointer to variable where read buffer index will be placed
+ *
+ * Function checks all streams for pending packets. It will stop as soon as
+ * packet ready to be submitted to user space is detected. Variables under
+ * pointers, passed as the parameters to this function will be updated with
+ * values pointing to right stream and buffer.
+ *
+ * Return: non-zero if any of timeline streams has at last one packet ready
+ */
+static int kbasep_tlstream_packet_pending(
+ enum tl_stream_type *stype,
+ unsigned int *rb_idx_raw)
+{
+ int pending = 0;
+
+ KBASE_DEBUG_ASSERT(stype);
+ KBASE_DEBUG_ASSERT(rb_idx_raw);
+
+ for (
+ *stype = 0;
+ (*stype < TL_STREAM_TYPE_COUNT) && !pending;
+ (*stype)++) {
+ if (NULL != tl_stream[*stype]) {
+ *rb_idx_raw = atomic_read(&tl_stream[*stype]->rbi);
+ /* Read buffer index may be updated by writer in case of
+ * overflow. Read and write buffer indexes must be
+ * loaded in correct order. */
+ smp_rmb();
+ if (atomic_read(&tl_stream[*stype]->wbi) != *rb_idx_raw)
+ pending = 1;
+ }
+ }
+ (*stype)--;
+
+ return pending;
+}
+
+/**
+ * kbasep_tlstream_read - copy data from streams to buffer provided by user
+ * @filp: pointer to file structure (unused)
+ * @buffer: pointer to the buffer provided by user
+ * @size: maximum amount of data that can be stored in the buffer
+ * @f_pos: pointer to file offset (unused)
+ *
+ * Return: number of bytes stored in the buffer
+ */
+static ssize_t kbasep_tlstream_read(
+ struct file *filp,
+ char __user *buffer,
+ size_t size,
+ loff_t *f_pos)
+{
+ ssize_t copy_len = 0;
+
+ KBASE_DEBUG_ASSERT(filp);
+ KBASE_DEBUG_ASSERT(buffer);
+ KBASE_DEBUG_ASSERT(f_pos);
+ CSTD_UNUSED(filp);
+
+ if ((0 > *f_pos) || (PACKET_SIZE > size))
+ return -EINVAL;
+
+ mutex_lock(&tl_reader_lock);
+
+ while (copy_len < size) {
+ enum tl_stream_type stype;
+ unsigned int rb_idx_raw;
+ unsigned int rb_idx;
+ size_t rb_size;
+
+ /* If we don't have any data yet, wait for packet to be
+ * submitted. If we already read some packets and there is no
+ * packet pending return back to user. */
+ if (0 < copy_len) {
+ if (!kbasep_tlstream_packet_pending(
+ &stype,
+ &rb_idx_raw))
+ break;
+ } else {
+ if (wait_event_interruptible(
+ tl_event_queue,
+ kbasep_tlstream_packet_pending(
+ &stype,
+ &rb_idx_raw))) {
+ copy_len = -ERESTARTSYS;
+ break;
+ }
+ }
+
+ /* Check if this packet fits into the user buffer.
+ * If so copy its content. */
+ rb_idx = rb_idx_raw % PACKET_COUNT;
+ rb_size = atomic_read(&tl_stream[stype]->buffer[rb_idx].size);
+ if (rb_size > size - copy_len)
+ break;
+ if (copy_to_user(
+ &buffer[copy_len],
+ tl_stream[stype]->buffer[rb_idx].data,
+ rb_size)) {
+ copy_len = -EFAULT;
+ break;
+ }
+
+ /* Verify if there was no overflow in selected stream. Make sure
+ * that if incorrect size was used we will know about it. */
+ smp_rmb();
+ if (atomic_read(&tl_stream[stype]->rbi) == rb_idx_raw) {
+ copy_len += rb_size;
+ atomic_inc(&tl_stream[stype]->rbi);
+
+#if MALI_UNIT_TEST
+ atomic_add(rb_size, &tlstream_bytes_collected);
+#endif /* MALI_UNIT_TEST */
+ }
+ }
+
+ mutex_unlock(&tl_reader_lock);
+
+ return copy_len;
+}
+
+/**
+ * kbasep_tlstream_poll - poll timeline stream for packets
+ * @filp: pointer to file structure
+ * @wait: pointer to poll table
+ * Return: POLLIN if data can be read without blocking, otherwise zero
+ */
+static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait)
+{
+ enum tl_stream_type stream_type;
+ unsigned int rb_idx;
+
+ KBASE_DEBUG_ASSERT(filp);
+ KBASE_DEBUG_ASSERT(wait);
+
+ poll_wait(filp, &tl_event_queue, wait);
+ if (kbasep_tlstream_packet_pending(&stream_type, &rb_idx))
+ return POLLIN;
+ return 0;
+}
+
+/**
+ * kbasep_tlstream_release - release timeline stream descriptor
+ * @inode: pointer to inode structure
+ * @filp: pointer to file structure
+ *
+ * Return always return zero
+ */
+static int kbasep_tlstream_release(struct inode *inode, struct file *filp)
+{
+ KBASE_DEBUG_ASSERT(inode);
+ KBASE_DEBUG_ASSERT(filp);
+ CSTD_UNUSED(inode);
+ CSTD_UNUSED(filp);
+ atomic_set(&tlstream_busy, 0);
+ return 0;
+}
+
+/**
+ * kbasep_tlstream_timeline_header - prepare timeline header stream packet
+ * @stream_type: type of the stream that will carry header data
+ * @tp_desc: pointer to array with tracepoint descriptors
+ * @tp_count: number of descriptors in the given array
+ *
+ * Functions fills in information about tracepoints stored in body stream
+ * associated with this header stream.
+ */
+static void kbasep_tlstream_timeline_header(
+ enum tl_stream_type stream_type,
+ const struct tp_desc *tp_desc,
+ u32 tp_count)
+{
+ const u8 tv = KBASEP_TLSTREAM_VERSION; /* tlstream version */
+ const u8 ps = sizeof(void *); /* pointer size */
+ size_t msg_size = sizeof(tv) + sizeof(ps) + sizeof(tp_count);
+ char *buffer;
+ size_t pos = 0;
+ unsigned long flags;
+ unsigned int i;
+
+ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type);
+ KBASE_DEBUG_ASSERT(tp_desc);
+
+ /* Calculate the size of the timeline message. */
+ for (i = 0; i < tp_count; i++) {
+ msg_size += sizeof(tp_desc[i].id);
+ msg_size +=
+ strnlen(tp_desc[i].id_str, STRLEN_MAX) +
+ sizeof(char) + sizeof(u32);
+ msg_size +=
+ strnlen(tp_desc[i].name, STRLEN_MAX) +
+ sizeof(char) + sizeof(u32);
+ msg_size +=
+ strnlen(tp_desc[i].arg_types, STRLEN_MAX) +
+ sizeof(char) + sizeof(u32);
+ msg_size +=
+ strnlen(tp_desc[i].arg_names, STRLEN_MAX) +
+ sizeof(char) + sizeof(u32);
+ }
+
+ KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE >= msg_size);
+
+ buffer = kbasep_tlstream_msgbuf_acquire(stream_type, msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &tv, sizeof(tv));
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &ps, sizeof(ps));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &tp_count, sizeof(tp_count));
+
+ for (i = 0; i < tp_count; i++) {
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos,
+ &tp_desc[i].id, sizeof(tp_desc[i].id));
+ pos = kbasep_tlstream_write_string(
+ buffer, pos,
+ tp_desc[i].id_str, msg_size - pos);
+ pos = kbasep_tlstream_write_string(
+ buffer, pos,
+ tp_desc[i].name, msg_size - pos);
+ pos = kbasep_tlstream_write_string(
+ buffer, pos,
+ tp_desc[i].arg_types, msg_size - pos);
+ pos = kbasep_tlstream_write_string(
+ buffer, pos,
+ tp_desc[i].arg_names, msg_size - pos);
+ }
+
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(stream_type, flags);
+
+ /* We don't expect any more data to be read in this stream.
+ * As header stream must be read before its associated body stream,
+ * make this packet visible to the user straightaway. */
+ kbasep_tlstream_flush_stream(stream_type);
+}
+
+/*****************************************************************************/
+
+int kbase_tlstream_init(void)
+{
+ enum tl_stream_type i;
+ int rcode;
+
+ /* Prepare stream structures. */
+ for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) {
+ tl_stream[i] = kmalloc(sizeof(**tl_stream), GFP_KERNEL);
+ if (!tl_stream[i])
+ break;
+ kbasep_timeline_stream_init(tl_stream[i], i);
+ }
+ if (TL_STREAM_TYPE_COUNT > i) {
+ for (; i > 0; i--) {
+ kbasep_timeline_stream_term(tl_stream[i - 1]);
+ kfree(tl_stream[i - 1]);
+ }
+ return -ENOMEM;
+ }
+
+ /* Initialize autoflush timer. */
+ atomic_set(&autoflush_timer_active, 1);
+ setup_timer(&autoflush_timer,
+ kbasep_tlstream_autoflush_timer_callback,
+ 0);
+ rcode = mod_timer(
+ &autoflush_timer,
+ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL));
+ CSTD_UNUSED(rcode);
+
+ return 0;
+}
+
+void kbase_tlstream_term(void)
+{
+ enum tl_stream_type i;
+
+ atomic_set(&autoflush_timer_active, 0);
+ del_timer_sync(&autoflush_timer);
+
+ for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) {
+ kbasep_timeline_stream_term(tl_stream[i]);
+ kfree(tl_stream[i]);
+ }
+}
+
+int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd)
+{
+ if (0 == atomic_cmpxchg(&tlstream_busy, 0, 1)) {
+ *fd = anon_inode_getfd(
+ "[mali_tlstream]",
+ &kbasep_tlstream_fops,
+ kctx,
+ O_RDONLY | O_CLOEXEC);
+ if (0 > *fd) {
+ atomic_set(&tlstream_busy, 0);
+ return *fd;
+ }
+
+ /* Reset and initialize header streams. */
+ kbasep_timeline_stream_reset(
+ tl_stream[TL_STREAM_TYPE_OBJ_HEADER]);
+ kbasep_timeline_stream_reset(
+ tl_stream[TL_STREAM_TYPE_OBJ_SUMMARY]);
+ kbasep_timeline_stream_reset(
+ tl_stream[TL_STREAM_TYPE_AUX_HEADER]);
+ kbasep_tlstream_timeline_header(
+ TL_STREAM_TYPE_OBJ_HEADER,
+ tp_desc_obj,
+ ARRAY_SIZE(tp_desc_obj));
+ kbasep_tlstream_timeline_header(
+ TL_STREAM_TYPE_AUX_HEADER,
+ tp_desc_aux,
+ ARRAY_SIZE(tp_desc_aux));
+ } else {
+ *fd = -EBUSY;
+ }
+
+ return 0;
+}
+
+void kbase_tlstream_flush_streams(void)
+{
+ enum tl_stream_type stype;
+
+ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++)
+ kbasep_tlstream_flush_stream(stype);
+}
+
+void kbase_tlstream_reset_body_streams(void)
+{
+ kbasep_timeline_stream_reset(
+ tl_stream[TL_STREAM_TYPE_OBJ]);
+ kbasep_timeline_stream_reset(
+ tl_stream[TL_STREAM_TYPE_AUX]);
+}
+
+#if MALI_UNIT_TEST
+void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated)
+{
+ KBASE_DEBUG_ASSERT(bytes_collected);
+ KBASE_DEBUG_ASSERT(bytes_generated);
+ *bytes_collected = atomic_read(&tlstream_bytes_collected);
+ *bytes_generated = atomic_read(&tlstream_bytes_generated);
+}
+#endif /* MALI_UNIT_TEST */
+
+/*****************************************************************************/
+
+void kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr)
+{
+ const u32 msg_id = KBASE_TL_NEW_CTX;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ_SUMMARY,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &context, sizeof(context));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &nr, sizeof(nr));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
+}
+
+void kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count)
+{
+ const u32 msg_id = KBASE_TL_NEW_GPU;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(id) +
+ sizeof(core_count);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ_SUMMARY,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &gpu, sizeof(gpu));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &id, sizeof(id));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &core_count, sizeof(core_count));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
+}
+
+void kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn)
+{
+ const u32 msg_id = KBASE_TL_NEW_LPU;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(nr) +
+ sizeof(fn);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ_SUMMARY,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &lpu, sizeof(lpu));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &nr, sizeof(nr));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &fn, sizeof(fn));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
+}
+
+void kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu)
+{
+ const u32 msg_id = KBASE_TL_LIFELINK_LPU_GPU;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(gpu);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ_SUMMARY,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &lpu, sizeof(lpu));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &gpu, sizeof(gpu));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags);
+}
+
+/*****************************************************************************/
+
+void kbase_tlstream_tl_new_ctx(void *context, u32 nr)
+{
+ const u32 msg_id = KBASE_TL_NEW_CTX;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &context, sizeof(context));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &nr, sizeof(nr));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void kbase_tlstream_tl_new_atom(void *atom, u32 nr)
+{
+ const u32 msg_id = KBASE_TL_NEW_ATOM;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(nr);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom, sizeof(atom));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &nr, sizeof(nr));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void kbase_tlstream_tl_del_ctx(void *context)
+{
+ const u32 msg_id = KBASE_TL_DEL_CTX;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(context);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &context, sizeof(context));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void kbase_tlstream_tl_del_atom(void *atom)
+{
+ const u32 msg_id = KBASE_TL_DEL_ATOM;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(atom);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom, sizeof(atom));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void kbase_tlstream_tl_ret_gpu_ctx(void *gpu, void *context)
+{
+ const u32 msg_id = KBASE_TL_RET_GPU_CTX;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(context);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &gpu, sizeof(gpu));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &context, sizeof(context));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context)
+{
+ const u32 msg_id = KBASE_TL_RET_ATOM_CTX;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom, sizeof(atom));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &context, sizeof(context));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void kbase_tlstream_tl_ret_atom_lpu(void *atom, void *lpu)
+{
+ const u32 msg_id = KBASE_TL_RET_ATOM_LPU;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom, sizeof(atom));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &lpu, sizeof(lpu));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void kbase_tlstream_tl_nret_gpu_ctx(void *gpu, void *context)
+{
+ const u32 msg_id = KBASE_TL_NRET_GPU_CTX;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(context);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &gpu, sizeof(gpu));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &context, sizeof(context));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context)
+{
+ const u32 msg_id = KBASE_TL_NRET_ATOM_CTX;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom, sizeof(atom));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &context, sizeof(context));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+void kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu)
+{
+ const u32 msg_id = KBASE_TL_NRET_ATOM_LPU;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_OBJ,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &atom, sizeof(atom));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &lpu, sizeof(lpu));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags);
+}
+
+/*****************************************************************************/
+
+void kbase_tlstream_aux_pm_state(u32 core_type, u64 state)
+{
+ const u32 msg_id = KBASE_AUX_PM_STATE;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(core_type) +
+ sizeof(state);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_AUX,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &core_type, sizeof(core_type));
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &state, sizeof(state));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
+}
+
+void kbase_tlstream_aux_job_softstop(u32 js_id)
+{
+ const u32 msg_id = KBASE_AUX_JOB_SOFTSTOP;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(js_id);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_AUX,
+ msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &js_id, sizeof(js_id));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
+}
+
+void kbase_tlstream_aux_pagefault(u32 mmu_as, u32 page_count)
+{
+ const u32 msg_id = KBASE_AUX_PAGEFAULT;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(mmu_as) +
+ sizeof(page_count);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_AUX, msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &mmu_as, sizeof(mmu_as));
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos, &page_count, sizeof(page_count));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
+}
+
+void kbase_tlstream_aux_pagesalloc(s64 page_count_change)
+{
+ const u32 msg_id = KBASE_AUX_PAGESALLOC;
+ const size_t msg_size =
+ sizeof(msg_id) + sizeof(u64) + sizeof(page_count_change);
+ unsigned long flags;
+ char *buffer;
+ size_t pos = 0;
+
+ buffer = kbasep_tlstream_msgbuf_acquire(
+ TL_STREAM_TYPE_AUX, msg_size, &flags);
+ KBASE_DEBUG_ASSERT(buffer);
+
+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id));
+ pos = kbasep_tlstream_write_timestamp(buffer, pos);
+ pos = kbasep_tlstream_write_bytes(
+ buffer, pos,
+ &page_count_change, sizeof(page_count_change));
+ KBASE_DEBUG_ASSERT(msg_size == pos);
+
+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags);
+}
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#if !defined(_KBASE_TLSTREAM_H)
+#define _KBASE_TLSTREAM_H
+
+#include <mali_kbase.h>
+
+/*****************************************************************************/
+
+/**
+ * kbase_tlstream_init - initialize timeline infrastructure in kernel
+ * Return: zero on success, negative number on error
+ */
+int kbase_tlstream_init(void);
+
+/**
+ * kbase_tlstream_term - terminate timeline infrastructure in kernel
+ *
+ * Timeline need have to been previously enabled with kbase_tlstream_init().
+ */
+void kbase_tlstream_term(void);
+
+/**
+ * kbase_tlstream_acquire - acquire timeline stream file descriptor
+ * @kctx: kernel common context
+ * @fd: timeline stream file descriptor
+ *
+ * This descriptor is meant to be used by userspace timeline to gain access to
+ * kernel timeline stream. This stream is later broadcasted by user space to the
+ * timeline client.
+ * Only one entity can own the descriptor at any given time. Descriptor shall be
+ * closed if unused. If descriptor cannot be obtained (i.e. when it is already
+ * being used) argument fd will contain negative value.
+ *
+ * Return: zero on success (this does not necessarily mean that stream
+ * descriptor could be returned), negative number on error
+ */
+int kbase_tlstream_acquire(struct kbase_context *kctx, int *fd);
+
+/**
+ * kbase_tlstream_flush_streams - flush timeline streams.
+ *
+ * Function will flush pending data in all timeline streams.
+ */
+void kbase_tlstream_flush_streams(void);
+
+/**
+ * kbase_tlstream_reset_body_streams - reset timeline body streams.
+ *
+ * Function will discard pending data in all timeline body streams.
+ */
+void kbase_tlstream_reset_body_streams(void);
+
+#if MALI_UNIT_TEST
+/**
+ * kbase_tlstream_test - start timeline stream data generator
+ * @tpw_count: number of trace point writers in each context
+ * @msg_delay: time delay in milliseconds between trace points written by one
+ * writer
+ * @msg_count: number of trace points written by one writer
+ * @aux_msg: if non-zero aux messages will be included
+ *
+ * This test starts a requested number of asynchronous writers in both IRQ and
+ * thread context. Each writer will generate required number of test
+ * tracepoints (tracepoints with embedded information about writer that
+ * should be verified by user space reader). Tracepoints will be emitted in
+ * all timeline body streams. If aux_msg is non-zero writer will also
+ * generate not testable tracepoints (tracepoints without information about
+ * writer). These tracepoints are used to check correctness of remaining
+ * timeline message generating functions. Writer will wait requested time
+ * between generating another set of messages. This call blocks until all
+ * writers finish.
+ */
+void kbase_tlstream_test(
+ unsigned int tpw_count,
+ unsigned int msg_delay,
+ unsigned int msg_count,
+ int aux_msg);
+
+/**
+ * kbase_tlstream_stats - read timeline stream statistics
+ * @bytes_collected: will hold number of bytes read by the user
+ * @bytes_generated: will hold number of bytes generated by trace points
+ */
+void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated);
+#endif /* MALI_UNIT_TEST */
+
+/*****************************************************************************/
+
+/**
+ * kbase_tlstream_tl_summary_new_ctx - create context object in timeline
+ * summary
+ * @context: name of the context object
+ * @nr: context number
+ *
+ * Function emits a timeline message informing about context creation. Context
+ * is created with context number (its attribute), that can be used to link
+ * kbase context with userspace context.
+ * This message is directed to timeline summary stream.
+ */
+void kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr);
+
+/**
+ * kbase_tlstream_tl_summary_new_gpu - create GPU object in timeline summary
+ * @gpu: name of the GPU object
+ * @id: id value of this GPU
+ * @core_count: number of cores this GPU hosts
+ *
+ * Function emits a timeline message informing about GPU creation. GPU is
+ * created with two attributes: id and core count.
+ * This message is directed to timeline summary stream.
+ */
+void kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count);
+
+/**
+ * kbase_tlstream_tl_summary_new_lpu - create LPU object in timeline summary
+ * @lpu: name of the Logical Processing Unit object
+ * @nr: sequential number assigned to this LPU
+ * @fn: property describing this LPU's functional abilities
+ *
+ * Function emits a timeline message informing about LPU creation. LPU is
+ * created with two attributes: number linking this LPU with GPU's job slot
+ * and function bearing information about this LPU abilities.
+ * This message is directed to timeline summary stream.
+ */
+void kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn);
+
+/**
+ * kbase_tlstream_tl_summary_lifelink_lpu_gpu - lifelink LPU object to GPU
+ * @lpu: name of the Logical Processing Unit object
+ * @gpu: name of the GPU object
+ *
+ * Function emits a timeline message informing that LPU object shall be deleted
+ * along with GPU object.
+ * This message is directed to timeline summary stream.
+ */
+void kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu);
+
+/**
+ * kbase_tlstream_tl_new_ctx - create context object in timeline
+ * @context: name of the context object
+ * @nr: context number
+ *
+ * Function emits a timeline message informing about context creation. Context
+ * is created with context number (its attribute), that can be used to link
+ * kbase context with userspace context.
+ */
+void kbase_tlstream_tl_new_ctx(void *context, u32 nr);
+
+/**
+ * kbase_tlstream_tl_new_atom - create atom object in timeline
+ * @atom: name of the atom object
+ * @nr: sequential number assigned to this atom
+ *
+ * Function emits a timeline message informing about atom creation. Atom is
+ * created with atom number (its attribute) that links it with actual work
+ * bucket id understood by hardware.
+ */
+void kbase_tlstream_tl_new_atom(void *atom, u32 nr);
+
+/**
+ * kbase_tlstream_tl_del_ctx - destroy context object in timeline
+ * @context: name of the context object
+ *
+ * Function emits a timeline message informing that context object ceased to
+ * exist.
+ */
+void kbase_tlstream_tl_del_ctx(void *context);
+
+/**
+ * kbase_tlstream_tl_del_atom - destroy atom object in timeline
+ * @atom: name of the atom object
+ *
+ * Function emits a timeline message informing that atom object ceased to
+ * exist.
+ */
+void kbase_tlstream_tl_del_atom(void *atom);
+
+/**
+ * kbase_tlstream_tl_ret_gpu_ctx - retain GPU by context
+ * @gpu: name of the GPU object
+ * @context: name of the context object
+ *
+ * Function emits a timeline message informing that GPU object is being held
+ * by context and must not be deleted unless it is released.
+ */
+void kbase_tlstream_tl_ret_gpu_ctx(void *gpu, void *context);
+
+/**
+ * kbase_tlstream_tl_ret_atom_ctx - retain atom by context
+ * @atom: name of the atom object
+ * @context: name of the context object
+ *
+ * Function emits a timeline message informing that atom object is being held
+ * by context and must not be deleted unless it is released.
+ */
+void kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context);
+
+/**
+ * kbase_tlstream_tl_ret_atom_lpu - retain atom by LPU
+ * @atom: name of the atom object
+ * @lpu: name of the Logical Processing Unit object
+ *
+ * Function emits a timeline message informing that atom object is being held
+ * by LPU and must not be deleted unless it is released.
+ */
+void kbase_tlstream_tl_ret_atom_lpu(void *atom, void *lpu);
+
+/**
+ * kbase_tlstream_tl_nret_gpu_ctx - release GPU by context
+ * @gpu: name of the GPU object
+ * @context: name of the context object
+ *
+ * Function emits a timeline message informing that GPU object is being released
+ * by context.
+ */
+void kbase_tlstream_tl_nret_gpu_ctx(void *gpu, void *context);
+
+/**
+ * kbase_tlstream_tl_nret_atom_ctx - release atom by context
+ * @atom: name of the atom object
+ * @context: name of the context object
+ *
+ * Function emits a timeline message informing that atom object is being
+ * released by context.
+ */
+void kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context);
+
+/**
+ * kbase_tlstream_tl_nret_atom_lpu - release atom by LPU
+ * @atom: name of the atom object
+ * @lpu: name of the Logical Processing Unit object
+ *
+ * Function emits a timeline message informing that atom object is being
+ * released by LPU.
+ */
+void kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu);
+
+/**
+ * kbase_tlstream_aux_pm_state - timeline message: power management state
+ * @core_type: core type (shader, tiler, l2 cache, l3 cache)
+ * @state: 64bits bitmask reporting power state of the cores (1-ON, 0-OFF)
+ */
+void kbase_tlstream_aux_pm_state(u32 core_type, u64 state);
+
+/**
+ * kbase_tlstream_aux_job_softstop - soft job stop occurred
+ * @js_id: job slot id
+ */
+void kbase_tlstream_aux_job_softstop(u32 js_id);
+
+/**
+ * kbase_tlstream_aux_pagefault - timeline message: MMU page fault event
+ * resulting in new pages being mapped
+ * @mmu_as: MMU address space number
+ * @page_count: number of currently used pages
+ */
+void kbase_tlstream_aux_pagefault(u32 mmu_as, u32 page_count);
+
+/**
+ * kbase_tlstream_aux_pagesalloc - timeline message: total number of allocated
+ * pages is changed
+ * @page_count_change: number of pages to be added or subtracted (according to
+ * the sign)
+ */
+void kbase_tlstream_aux_pagesalloc(s64 page_count_change);
+
+#endif /* _KBASE_TLSTREAM_H */
+
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase.h>
#include <mali_kbase_jm.h>
+#include <mali_kbase_hwaccess_jm.h>
#define CREATE_TRACE_POINTS
#include <linux/debugfs.h>
#include <linux/seq_file.h>
-struct kbase_trace_timeline_desc
-{
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atoms_in_flight);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atom);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_active);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_action);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_power_active);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_l2_power_active);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_event);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_slot_atom);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_checktrans);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_context_active);
+
+struct kbase_trace_timeline_desc {
char *enum_str;
char *desc;
char *format;
char *format_desc;
};
-struct kbase_trace_timeline_desc kbase_trace_timeline_desc_table[] =
-{
+static struct kbase_trace_timeline_desc kbase_trace_timeline_desc_table[] = {
#define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) { #enum_val, desc, format, format_desc }
#include "mali_kbase_trace_timeline_defs.h"
#undef KBASE_TIMELINE_TRACE_CODE
#define KBASE_NR_TRACE_CODES ARRAY_SIZE(kbase_trace_timeline_desc_table)
-STATIC void *kbasep_trace_timeline_seq_start(struct seq_file *s, loff_t *pos)
+static void *kbasep_trace_timeline_seq_start(struct seq_file *s, loff_t *pos)
{
if (*pos >= KBASE_NR_TRACE_CODES)
return NULL;
return &kbase_trace_timeline_desc_table[*pos];
}
-STATIC void kbasep_trace_timeline_seq_stop(struct seq_file *s, void *data)
+static void kbasep_trace_timeline_seq_stop(struct seq_file *s, void *data)
{
}
-STATIC void *kbasep_trace_timeline_seq_next(struct seq_file *s, void *data, loff_t *pos)
+static void *kbasep_trace_timeline_seq_next(struct seq_file *s, void *data, loff_t *pos)
{
(*pos)++;
return &kbase_trace_timeline_desc_table[*pos];
}
-STATIC int kbasep_trace_timeline_seq_show(struct seq_file *s, void *data)
+static int kbasep_trace_timeline_seq_show(struct seq_file *s, void *data)
{
struct kbase_trace_timeline_desc *trace_desc = data;
.show = kbasep_trace_timeline_seq_show,
};
-STATIC int kbasep_trace_timeline_debugfs_open(struct inode *inode, struct file *file)
+static int kbasep_trace_timeline_debugfs_open(struct inode *inode, struct file *file)
{
return seq_open(file, &kbasep_trace_timeline_seq_ops);
}
.release = seq_release_private,
};
-mali_error kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev)
+void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev)
{
- kbdev->timeline.dentry = debugfs_create_file("mali_timeline_defs",
+ debugfs_create_file("mali_timeline_defs",
S_IRUGO, kbdev->mali_debugfs_directory, NULL,
&kbasep_trace_timeline_debugfs_fops);
- if (IS_ERR(kbdev->timeline.dentry))
- return MALI_ERROR_FUNCTION_FAILED;
-
- return MALI_ERROR_NONE;
-}
-
-void kbasep_trace_timeline_debugfs_term(struct kbase_device *kbdev)
-{
- debugfs_remove(kbdev->timeline.dentry);
}
void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx,
- struct kbase_jd_atom *katom, int js)
+ struct kbase_jd_atom *katom, int js)
{
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 1);
} else {
base_atom_id atom_number = kbase_jd_atom_id(kctx, katom);
+
KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 1);
KBASE_TIMELINE_JOB_START(kctx, js, atom_number);
}
}
void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx,
- struct kbase_jd_atom *katom, int js,
- kbasep_js_atom_done_code done_code)
+ struct kbase_jd_atom *katom, int js,
+ kbasep_js_atom_done_code done_code)
{
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
} else {
/* Job finished in JS_HEAD */
base_atom_id atom_number = kbase_jd_atom_id(kctx, katom);
+
KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 0);
KBASE_TIMELINE_JOB_STOP(kctx, js, atom_number);
+
/* see if we need to trace the job in JS_NEXT moving to JS_HEAD */
- if (kbdev->timeline.slot_atoms_submitted[js] > 1) {
- /* Tag events with next_katom's kctx */
- struct kbase_jm_slot *slot = &kbdev->jm_slots[js];
+ if (kbase_backend_nr_atoms_submitted(kbdev, js)) {
struct kbase_jd_atom *next_katom;
struct kbase_context *next_kctx;
- KBASE_DEBUG_ASSERT(kbasep_jm_nr_jobs_submitted(slot) > 0);
/* Peek the next atom - note that the atom in JS_HEAD will already
* have been dequeued */
- next_katom = kbasep_jm_peek_idx_submit_slot(slot, 0);
+ next_katom = kbase_backend_inspect_head(kbdev, js);
+ WARN_ON(!next_katom);
next_kctx = next_katom->kctx;
KBASE_TIMELINE_JOB_START_NEXT(next_kctx, js, 0);
KBASE_TIMELINE_JOB_START_HEAD(next_kctx, js, 1);
{
lockdep_assert_held(&kbdev->pm.power_change_lock);
/* Simply log the start of the transition */
- kbdev->timeline.l2_transitioning = MALI_TRUE;
+ kbdev->timeline.l2_transitioning = true;
KBASE_TIMELINE_POWERING_L2(kbdev);
}
{
lockdep_assert_held(&kbdev->pm.power_change_lock);
/* Simply log the end of the transition */
- if (MALI_FALSE != kbdev->timeline.l2_transitioning)
- {
- kbdev->timeline.l2_transitioning = MALI_FALSE;
+ if (kbdev->timeline.l2_transitioning) {
+ kbdev->timeline.l2_transitioning = false;
KBASE_TIMELINE_POWERED_L2(kbdev);
}
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#ifdef CONFIG_MALI_TRACE_TIMELINE
-typedef enum
-{
+enum kbase_trace_timeline_code {
#define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) enum_val
#include "mali_kbase_trace_timeline_defs.h"
#undef KBASE_TIMELINE_TRACE_CODE
-} kbase_trace_timeline_code;
+};
/** Initialize Timeline DebugFS entries */
-mali_error kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev);
-/** Terminate Timeline DebugFS entries */
-void kbasep_trace_timeline_debugfs_term(struct kbase_device *kbdev);
+void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev);
/* mali_timeline.h defines kernel tracepoints used by the KBASE_TIMELINE
* functions.
/* Trace number of atoms in flight for kctx (atoms either not completed, or in
process of being returned to user */
-#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_atoms_in_flight(ts.tv_sec, ts.tv_nsec, \
- (int)kctx->timeline.owner_tgid, \
- count); \
+#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_atoms_in_flight(ts.tv_sec, ts.tv_nsec, \
+ (int)kctx->timeline.owner_tgid, \
+ count); \
} while (0)
/* Trace atom_id being Ready to Run */
-#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_atom(ts.tv_sec, ts.tv_nsec, \
- CTX_FLOW_ATOM_READY, \
- (int)kctx->timeline.owner_tgid, \
- atom_id); \
+#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_atom(ts.tv_sec, ts.tv_nsec, \
+ CTX_FLOW_ATOM_READY, \
+ (int)kctx->timeline.owner_tgid, \
+ atom_id); \
} while (0)
/* Trace number of atoms submitted to job slot js
*
* This is because this is more useful, as we can use it to calculate general
* utilization easily and accurately */
-#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_gpu_slot_active(ts.tv_sec, ts.tv_nsec, \
- SW_SET_GPU_SLOT_ACTIVE, \
- (int)kctx->timeline.owner_tgid, \
- js, count); \
+#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_gpu_slot_active(ts.tv_sec, ts.tv_nsec, \
+ SW_SET_GPU_SLOT_ACTIVE, \
+ (int)kctx->timeline.owner_tgid, \
+ js, count); \
} while (0)
/* Trace atoms present in JS_NEXT */
-#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \
- SW_SET_GPU_SLOT_NEXT, \
- (int)kctx->timeline.owner_tgid, \
- js, count); \
+#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \
+ SW_SET_GPU_SLOT_NEXT, \
+ (int)kctx->timeline.owner_tgid, \
+ js, count); \
} while (0)
/* Trace atoms present in JS_HEAD */
-#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \
- SW_SET_GPU_SLOT_HEAD, \
- (int)kctx->timeline.owner_tgid, \
- js, count); \
+#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \
+ SW_SET_GPU_SLOT_HEAD, \
+ (int)kctx->timeline.owner_tgid, \
+ js, count); \
} while (0)
/* Trace that a soft stop/evict from next is being attempted on a slot */
#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \
- SW_SET_GPU_SLOT_STOPPING, \
- (kctx) ? (int)kctx->timeline.owner_tgid : 0, \
- js, count); \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \
+ SW_SET_GPU_SLOT_STOPPING, \
+ (kctx) ? (int)kctx->timeline.owner_tgid : 0, \
+ js, count); \
} while (0)
/* Trace state of overall GPU power */
-#define KBASE_TIMELINE_GPU_POWER(kbdev, active) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \
- SW_SET_GPU_POWER_ACTIVE, active); \
+#define KBASE_TIMELINE_GPU_POWER(kbdev, active) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \
+ SW_SET_GPU_POWER_ACTIVE, active); \
} while (0)
/* Trace state of tiler power */
-#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \
- SW_SET_GPU_POWER_TILER_ACTIVE, \
- hweight64(bitmap)); \
+#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \
+ SW_SET_GPU_POWER_TILER_ACTIVE, \
+ hweight64(bitmap)); \
} while (0)
/* Trace number of shaders currently powered */
-#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \
- SW_SET_GPU_POWER_SHADER_ACTIVE, \
- hweight64(bitmap)); \
+#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \
+ SW_SET_GPU_POWER_SHADER_ACTIVE, \
+ hweight64(bitmap)); \
} while (0)
/* Trace state of L2 power */
-#define KBASE_TIMELINE_POWER_L2(kbdev, bitmap) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \
- SW_SET_GPU_POWER_L2_ACTIVE, \
- hweight64(bitmap)); \
+#define KBASE_TIMELINE_POWER_L2(kbdev, bitmap) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \
+ SW_SET_GPU_POWER_L2_ACTIVE, \
+ hweight64(bitmap)); \
} while (0)
/* Trace state of L2 cache*/
-#define KBASE_TIMELINE_POWERING_L2(kbdev) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \
- SW_FLOW_GPU_POWER_L2_POWERING, \
- 1); \
+#define KBASE_TIMELINE_POWERING_L2(kbdev) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \
+ SW_FLOW_GPU_POWER_L2_POWERING, \
+ 1); \
} while (0)
-#define KBASE_TIMELINE_POWERED_L2(kbdev) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \
- SW_FLOW_GPU_POWER_L2_ACTIVE, \
- 1); \
+#define KBASE_TIMELINE_POWERED_L2(kbdev) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \
+ SW_FLOW_GPU_POWER_L2_ACTIVE, \
+ 1); \
} while (0)
/* Trace kbase_pm_send_event message send */
-#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) \
- do \
- { \
+#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) \
+ do { \
struct timespec ts; \
- getnstimeofday(&ts); \
+ getrawmonotonic(&ts); \
trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \
- SW_FLOW_PM_SEND_EVENT, \
- event_type, pm_event_id); \
+ SW_FLOW_PM_SEND_EVENT, \
+ event_type, pm_event_id); \
} while (0)
/* Trace kbase_pm_worker message receive */
-#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \
- SW_FLOW_PM_HANDLE_EVENT, \
- event_type, pm_event_id); \
+#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \
+ SW_FLOW_PM_HANDLE_EVENT, \
+ event_type, pm_event_id); \
} while (0)
/* Trace atom_id starting in JS_HEAD */
-#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \
- HW_START_GPU_JOB_CHAIN_SW_APPROX, \
- (int)kctx->timeline.owner_tgid, \
- js, _consumerof_atom_number); \
+#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \
+ HW_START_GPU_JOB_CHAIN_SW_APPROX, \
+ (int)kctx->timeline.owner_tgid, \
+ js, _consumerof_atom_number); \
} while (0)
/* Trace atom_id stopping on JS_HEAD */
#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \
- HW_STOP_GPU_JOB_CHAIN_SW_APPROX, \
- (int)kctx->timeline.owner_tgid, \
- js, _producerof_atom_number_completed); \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \
+ HW_STOP_GPU_JOB_CHAIN_SW_APPROX, \
+ (int)kctx->timeline.owner_tgid, \
+ js, _producerof_atom_number_completed); \
} while (0)
/** Trace beginning/end of a call to kbase_pm_check_transitions_nolock from a
* certin caller */
-#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_pm_checktrans(ts.tv_sec, ts.tv_nsec, \
- trace_code, \
- 1); \
+#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_pm_checktrans(ts.tv_sec, ts.tv_nsec, \
+ trace_code, 1); \
} while (0)
/* Trace number of contexts active */
-#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) \
- do \
- { \
- struct timespec ts; \
- getnstimeofday(&ts); \
- trace_mali_timeline_context_active(ts.tv_sec, ts.tv_nsec, \
- count); \
+#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) \
+ do { \
+ struct timespec ts; \
+ getrawmonotonic(&ts); \
+ trace_mali_timeline_context_active(ts.tv_sec, ts.tv_nsec, \
+ count); \
} while (0)
* The caller must be holding kbasep_js_device_data::runpool_irq::lock
*/
void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx,
- struct kbase_jd_atom *katom, int js);
+ struct kbase_jd_atom *katom, int js);
/**
* Trace that an atom has done on a job slot
* The caller must be holding kbasep_js_device_data::runpool_irq::lock
*/
void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx,
- struct kbase_jd_atom *katom, int js,
- kbasep_js_atom_done_code done_code);
+ struct kbase_jd_atom *katom, int js,
+ kbasep_js_atom_done_code done_code);
/** Trace a pm event starting */
void kbase_timeline_pm_send_event(struct kbase_device *kbdev,
- enum kbase_timeline_pm_event event_sent);
+ enum kbase_timeline_pm_event event_sent);
/** Trace a pm event finishing */
void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event);
#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) CSTD_NOP()
-static INLINE void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx,
- struct kbase_jd_atom *katom, int js)
+static inline void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx,
+ struct kbase_jd_atom *katom, int js)
{
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
}
-static INLINE void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx,
- struct kbase_jd_atom *katom, int js,
- kbasep_js_atom_done_code done_code)
+static inline void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx,
+ struct kbase_jd_atom *katom, int js,
+ kbasep_js_atom_done_code done_code)
{
lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
}
-static INLINE void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent)
+static inline void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent)
{
}
-static INLINE void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event)
+static inline void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event)
{
}
-static INLINE void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event)
+static inline void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event)
{
}
-static INLINE void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev)
+static inline void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev)
{
-
}
-static INLINE void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev)
+static inline void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev)
{
-
}
#endif /* CONFIG_MALI_TRACE_TIMELINE */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2008-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define _KBASE_UKU_H_
#include "mali_uk.h"
-#include <malisw/mali_malisw.h>
#include "mali_base_kernel.h"
/* This file needs to support being included from kernel and userside (which use different defines) */
-#if defined(CONFIG_MALI_ERROR_INJECT)
+#if defined(CONFIG_MALI_ERROR_INJECT) || MALI_ERROR_INJECT_ON
#define SUPPORT_MALI_ERROR_INJECT
-#elif defined(MALI_ERROR_INJECT)
-#if MALI_ERROR_INJECT
-#define SUPPORT_MALI_ERROR_INJECT
-#endif
-#endif
+#endif /* defined(CONFIG_MALI_ERROR_INJECT) || MALI_ERROR_INJECT_ON */
#if defined(CONFIG_MALI_NO_MALI)
#define SUPPORT_MALI_NO_MALI
#elif defined(MALI_NO_MALI)
#endif
#if defined(SUPPORT_MALI_NO_MALI) || defined(SUPPORT_MALI_ERROR_INJECT)
-#include "mali_kbase_model_dummy.h"
+#include "backend/gpu/mali_kbase_model_dummy.h"
#endif
#include "mali_kbase_gpuprops_types.h"
-#define BASE_UK_VERSION_MAJOR 8
+#define BASE_UK_VERSION_MAJOR 9
#define BASE_UK_VERSION_MINOR 0
struct kbase_uk_mem_alloc {
struct kbase_uk_mem_free {
union uk_header header;
/* IN */
- mali_addr64 gpu_addr;
+ u64 gpu_addr;
/* OUT */
};
/* IN/OUT */
u64 flags;
/* OUT */
- mali_addr64 gpu_va;
+ u64 gpu_va;
u64 va_pages;
};
struct kbase_uk_mem_flags_change {
union uk_header header;
/* IN */
- mali_addr64 gpu_va;
+ u64 gpu_va;
u64 flags;
u64 mask;
};
union uk_header header;
};
+struct kbase_uk_dump_fault_term {
+ union uk_header header;
+};
+
struct kbase_uk_sync_now {
union uk_header header;
union uk_header header;
/* IN */
- mali_addr64 dump_buffer;
+ u64 dump_buffer;
u32 jm_bm;
u32 shader_bm;
u32 tiler_bm;
- u32 l3_cache_bm;
+ u32 unused_1; /* keep for backwards compatibility */
u32 mmu_l2_bm;
u32 padding;
/* OUT */
* is removed. Removal of KBASE_FUNC_CPU_PROPS_REG_DUMP is part of having
* the function for reading cpu properties moved from base to osu.
*/
-#define BASE_CPU_PROPERTY_FLAG_LITTLE_ENDIAN F_BIT_0
+#define BASE_CPU_PROPERTY_FLAG_LITTLE_ENDIAN ((u32)0x00000001)
struct base_cpu_id_props {
/**
* CPU ID
struct kbase_uk_mem_query {
union uk_header header;
/* IN */
- mali_addr64 gpu_addr;
+ u64 gpu_addr;
#define KBASE_MEM_QUERY_COMMIT_SIZE 1
#define KBASE_MEM_QUERY_VA_SIZE 2
#define KBASE_MEM_QUERY_FLAGS 3
/* OUT */
u64 value;
};
-
+
struct kbase_uk_mem_commit {
union uk_header header;
/* IN */
- mali_addr64 gpu_addr;
+ u64 gpu_addr;
u64 pages;
/* OUT */
u32 result_subcode;
struct kbase_uk_find_cpu_offset {
union uk_header header;
/* IN */
- mali_addr64 gpu_addr;
+ u64 gpu_addr;
u64 cpu_addr;
u64 size;
/* OUT */
- mali_size64 offset;
+ u64 offset;
};
#define KBASE_GET_VERSION_BUFFER_SIZE 64
#if MALI_UNIT_TEST
#define TEST_ADDR_COUNT 4
#define KBASE_TEST_BUFFER_SIZE 128
-typedef struct kbase_exported_test_data {
- mali_addr64 test_addr[TEST_ADDR_COUNT]; /**< memory address */
+struct kbase_exported_test_data {
+ u64 test_addr[TEST_ADDR_COUNT]; /**< memory address */
u32 test_addr_pages[TEST_ADDR_COUNT]; /**< memory size in pages */
union kbase_pointer kctx; /**< base context created by process */
union kbase_pointer mm; /**< pointer to process address space */
u8 buffer1[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */
u8 buffer2[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */
-} kbase_exported_test_data;
+};
struct kbase_uk_set_test_data {
union uk_header header;
u32 padding;
};
-struct kbase_uk_keep_gpu_powered {
- union uk_header header;
- u32 enabled;
- u32 padding;
-};
-
struct kbase_uk_profiling_controls {
union uk_header header;
u32 profiling_controls[FBDUMP_CONTROL_MAX];
union kbase_pointer buf;
};
+struct kbase_uk_context_id {
+ union uk_header header;
+ /* OUT */
+ int id;
+};
+
+#if (defined(MALI_KTLSTREAM_ENABLED) && MALI_KTLSTREAM_ENABLED) || \
+ defined(CONFIG_MALI_MIPE_ENABLED)
+/**
+ * struct kbase_uk_tlstream_acquire - User/Kernel space data exchange structure
+ * @header: UK structure header
+ * @fd: timeline stream file descriptor
+ *
+ * This structure is used used when performing a call to acquire kernel side
+ * timeline stream file descriptor.
+ */
+struct kbase_uk_tlstream_acquire {
+ union uk_header header;
+ /* IN */
+ /* OUT */
+ s32 fd;
+};
+
+/**
+ * struct kbase_uk_tlstream_flush - User/Kernel space data exchange structure
+ * @header: UK structure header
+ *
+ * This structure is used when performing a call to flush kernel side
+ * timeline streams.
+ */
+struct kbase_uk_tlstream_flush {
+ union uk_header header;
+ /* IN */
+ /* OUT */
+};
+
+#if MALI_UNIT_TEST
+/**
+ * struct kbase_uk_tlstream_acquire - User/Kernel space data exchange structure
+ * @header: UK structure header
+ * @tpw_count: number of trace point writers in each context
+ * @msg_delay: time delay between tracepoints from one writer in milliseconds
+ * @msg_count: number of trace points written by one writer
+ * @aux_msg: if non-zero aux messages will be included
+ *
+ * This structure is used when performing a call to start timeline stream test
+ * embedded in kernel.
+ */
+struct kbase_uk_tlstream_test {
+ union uk_header header;
+ /* IN */
+ u32 tpw_count;
+ u32 msg_delay;
+ u32 msg_count;
+ u32 aux_msg;
+ /* OUT */
+};
+
+/**
+ * struct kbase_uk_tlstream_acquire - User/Kernel space data exchange structure
+ * @header: UK structure header
+ * @bytes_collected: number of bytes read by user
+ * @bytes_generated: number of bytes generated by tracepoints
+ *
+ * This structure is used when performing a call to obtain timeline stream
+ * statistics.
+ */
+struct kbase_uk_tlstream_stats {
+ union uk_header header; /**< UK structure header. */
+ /* IN */
+ /* OUT */
+ u32 bytes_collected;
+ u32 bytes_generated;
+};
+#endif /* MALI_UNIT_TEST */
+#endif /* MALI_KTLSTREAM_ENABLED */
+
enum kbase_uk_function_id {
KBASE_FUNC_MEM_ALLOC = (UK_FUNC_ID + 0),
KBASE_FUNC_MEM_IMPORT = (UK_FUNC_ID + 1),
KBASE_FUNC_INJECT_ERROR = (UK_FUNC_ID + 20),
KBASE_FUNC_MODEL_CONTROL = (UK_FUNC_ID + 21),
- KBASE_FUNC_KEEP_GPU_POWERED = (UK_FUNC_ID + 22),
-
KBASE_FUNC_FENCE_VALIDATE = (UK_FUNC_ID + 23),
KBASE_FUNC_STREAM_CREATE = (UK_FUNC_ID + 24),
KBASE_FUNC_GET_PROFILING_CONTROLS = (UK_FUNC_ID + 25),
KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD = (UK_FUNC_ID + 27),
KBASE_FUNC_JOB_SUBMIT = (UK_FUNC_ID + 28),
- KBASE_FUNC_DISJOINT_QUERY = (UK_FUNC_ID + 29)
+ KBASE_FUNC_DISJOINT_QUERY = (UK_FUNC_ID + 29),
+
+ KBASE_FUNC_DUMP_FAULT_TERM = (UK_FUNC_ID + 30),
+
+ KBASE_FUNC_GET_CONTEXT_ID = (UK_FUNC_ID + 31),
+#if (defined(MALI_KTLSTREAM_ENABLED) && MALI_KTLSTREAM_ENABLED) || \
+ defined(CONFIG_MALI_MIPE_ENABLED)
+ KBASE_FUNC_TLSTREAM_ACQUIRE = (UK_FUNC_ID + 32),
+#if MALI_UNIT_TEST
+ KBASE_FUNC_TLSTREAM_TEST = (UK_FUNC_ID + 33),
+ KBASE_FUNC_TLSTREAM_STATS = (UK_FUNC_ID + 34),
+#endif /* MALI_UNIT_TEST */
+ KBASE_FUNC_TLSTREAM_FLUSH = (UK_FUNC_ID + 35),
+#endif /* MALI_KTLSTREAM_ENABLED */
+
+ KBASE_FUNC_MAX
};
#endif /* _KBASE_UKU_H_ */
+
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase.h>
-mali_bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry)
+bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry)
{
struct list_head *pos = base->next;
while (pos != base) {
if (pos == entry)
- return MALI_TRUE;
+ return true;
pos = pos->next;
}
- return MALI_FALSE;
+ return false;
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* @param base The head of the list to be tested
* @param entry The list entry to be tested
*
- * @return MALI_TRUE if entry is a member of base
- * MALI_FALSE otherwise
+ * @return true if entry is a member of base
+ * false otherwise
*/
-mali_bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry);
+bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry);
#endif /* _KBASE_UTILITY_H */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include <mali_kbase.h>
+#include <mali_kbase_mem_linux.h>
+
+#define NR_CNT_BLOCKS_PER_GROUP 8
+#define NR_CNT_PER_BLOCK 64
+#define NR_BYTES_PER_CNT 4
+#define NR_BYTES_PER_HDR 16
+#define PRFCNT_EN_MASK_OFFSET 0x8
+
+struct kbase_vinstr_context {
+ struct kbase_device *kbdev;
+ struct kbase_context *kctx;
+ struct kbase_vmap_struct vmap;
+ struct mutex lock;
+ u64 gpu_va;
+ void *cpu_va;
+ size_t dump_size;
+ u32 nclients;
+ struct list_head clients;
+};
+
+struct kbase_vinstr_client {
+ bool kernel;
+ void *dump_buffer;
+ u32 bitmap[4];
+ void *accum_buffer;
+ size_t dump_size;
+ struct list_head list;
+};
+
+static int map_kernel_dump_buffer(struct kbase_vinstr_context *ctx)
+{
+ struct kbase_va_region *reg;
+ struct kbase_context *kctx = ctx->kctx;
+ u64 flags, nr_pages;
+ u16 va_align = 0;
+
+ flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR;
+ ctx->dump_size = kbase_vinstr_dump_size(ctx);
+ nr_pages = PFN_UP(ctx->dump_size);
+
+ reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags,
+ &ctx->gpu_va, &va_align);
+ if (!reg)
+ return -ENOMEM;
+
+ ctx->cpu_va = kbase_vmap(kctx, ctx->gpu_va, ctx->dump_size, &ctx->vmap);
+ if (!ctx->cpu_va) {
+ kbase_mem_free(kctx, ctx->gpu_va);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void unmap_kernel_dump_buffer(struct kbase_vinstr_context *ctx)
+{
+ struct kbase_context *kctx = ctx->kctx;
+
+ kbase_vunmap(kctx, &ctx->vmap);
+ kbase_mem_free(kctx, ctx->gpu_va);
+}
+
+static int map_client_accum_buffer(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli)
+{
+ cli->dump_size = kbase_vinstr_dump_size(ctx);
+ cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL);
+ return !cli->accum_buffer ? -ENOMEM : 0;
+}
+
+static void unmap_client_accum_buffer(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli)
+{
+ kfree(cli->accum_buffer);
+}
+
+static int create_vinstr_kctx(struct kbase_vinstr_context *ctx)
+{
+ struct kbase_uk_hwcnt_setup setup;
+ int err;
+
+ ctx->kctx = kbase_create_context(ctx->kbdev, true);
+ if (!ctx->kctx)
+ return -ENOMEM;
+
+ /* Map the master kernel dump buffer. The HW dumps the counters
+ * into this memory region. */
+ err = map_kernel_dump_buffer(ctx);
+ if (err) {
+ kbase_destroy_context(ctx->kctx);
+ return err;
+ }
+
+ setup.dump_buffer = ctx->gpu_va;
+ /* The GPU requires us to disable the prfcnt collection block for
+ * reprogramming it. This introduces jitter and disrupts existing
+ * clients. Therefore we enable all of them. */
+ setup.jm_bm = 0xffffffff;
+ setup.tiler_bm = 0xffffffff;
+ setup.shader_bm = 0xffffffff;
+ setup.mmu_l2_bm = 0xffffffff;
+
+ err = kbase_instr_hwcnt_enable(ctx->kctx, &setup);
+ if (err) {
+ unmap_kernel_dump_buffer(ctx);
+ kbase_destroy_context(ctx->kctx);
+ return err;
+ }
+
+ return 0;
+}
+
+static void destroy_vinstr_kctx(struct kbase_vinstr_context *ctx)
+{
+ kbase_instr_hwcnt_disable(ctx->kctx);
+ unmap_kernel_dump_buffer(ctx);
+ kbase_destroy_context(ctx->kctx);
+ ctx->kctx = NULL;
+}
+
+struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev)
+{
+ struct kbase_vinstr_context *ctx;
+
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return NULL;
+ INIT_LIST_HEAD(&ctx->clients);
+ mutex_init(&ctx->lock);
+ ctx->kbdev = kbdev;
+ return ctx;
+}
+
+void kbase_vinstr_term(struct kbase_vinstr_context *ctx)
+{
+ struct kbase_vinstr_client *cli;
+
+ while (!list_empty(&ctx->clients)) {
+ cli = list_first_entry(&ctx->clients,
+ struct kbase_vinstr_client, list);
+ list_del(&cli->list);
+ unmap_client_accum_buffer(ctx, cli);
+ kfree(cli);
+ ctx->nclients--;
+ }
+ if (ctx->kctx)
+ destroy_vinstr_kctx(ctx);
+ kfree(ctx);
+}
+
+struct kbase_vinstr_client *kbase_vinstr_attach_client(struct kbase_vinstr_context *ctx,
+ bool kernel, u64 dump_buffer, u32 bitmap[4])
+{
+ struct kbase_vinstr_client *cli;
+
+ cli = kmalloc(sizeof(*cli), GFP_KERNEL);
+ if (!cli)
+ return NULL;
+
+ cli->kernel = kernel;
+ cli->dump_buffer = (void *)(uintptr_t)dump_buffer;
+ cli->bitmap[SHADER_HWCNT_BM] = bitmap[SHADER_HWCNT_BM];
+ cli->bitmap[TILER_HWCNT_BM] = bitmap[TILER_HWCNT_BM];
+ cli->bitmap[MMU_L2_HWCNT_BM] = bitmap[MMU_L2_HWCNT_BM];
+ cli->bitmap[JM_HWCNT_BM] = bitmap[JM_HWCNT_BM];
+
+ mutex_lock(&ctx->lock);
+ /* If this is the first client, create the vinstr kbase
+ * context. This context is permanently resident until the
+ * last client exits. */
+ if (!ctx->nclients) {
+ if (create_vinstr_kctx(ctx) < 0) {
+ kfree(cli);
+ mutex_unlock(&ctx->lock);
+ return NULL;
+ }
+ }
+
+ /* The GPU resets the counter block every time there is a request
+ * to dump it. We need a per client kernel buffer for accumulating
+ * the counters. */
+ if (map_client_accum_buffer(ctx, cli) < 0) {
+ kfree(cli);
+ if (!ctx->nclients)
+ destroy_vinstr_kctx(ctx);
+ mutex_unlock(&ctx->lock);
+ return NULL;
+ }
+
+ ctx->nclients++;
+ list_add(&cli->list, &ctx->clients);
+ mutex_unlock(&ctx->lock);
+
+ return cli;
+}
+
+void kbase_vinstr_detach_client(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli)
+{
+ struct kbase_vinstr_client *iter, *tmp;
+
+ mutex_lock(&ctx->lock);
+ list_for_each_entry_safe(iter, tmp, &ctx->clients, list) {
+ if (iter == cli) {
+ list_del(&iter->list);
+ unmap_client_accum_buffer(ctx, cli);
+ kfree(iter);
+ ctx->nclients--;
+ if (!ctx->nclients)
+ destroy_vinstr_kctx(ctx);
+ break;
+ }
+ }
+ mutex_unlock(&ctx->lock);
+}
+
+size_t kbase_vinstr_dump_size(struct kbase_vinstr_context *ctx)
+{
+ struct kbase_device *kbdev = ctx->kctx->kbdev;
+ size_t dump_size;
+
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) {
+ u32 nr_cg;
+
+ nr_cg = kbdev->gpu_props.num_core_groups;
+ dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP *
+ NR_CNT_PER_BLOCK *
+ NR_BYTES_PER_CNT;
+ } else {
+ /* assume v5 for now */
+ u32 nr_l2, nr_sc;
+
+ nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices;
+ nr_sc = kbdev->gpu_props.props.coherency_info.group[0].num_cores;
+ /* JM and tiler counter blocks are always present */
+ dump_size = (2 + nr_l2 + nr_sc) *
+ NR_CNT_PER_BLOCK *
+ NR_BYTES_PER_CNT;
+ }
+ return dump_size;
+}
+
+/* Accumulate counters in the dump buffer */
+static void accum_dump_buffer(void *dst, void *src, size_t dump_size)
+{
+ size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
+ u32 *d = dst;
+ u32 *s = src;
+ size_t i, j;
+
+ for (i = 0; i < dump_size; i += block_size) {
+ /* skip over the header block */
+ d += NR_BYTES_PER_HDR / sizeof(u32);
+ s += NR_BYTES_PER_HDR / sizeof(u32);
+ for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) {
+ /* saturate result if addition would result in wraparound */
+ if (U32_MAX - *d < *s)
+ *d = U32_MAX;
+ else
+ *d += *s;
+ d++;
+ s++;
+ }
+ }
+}
+
+/* This is the Midgard v4 patch function. It copies the headers for each
+ * of the defined blocks from the master kernel buffer and then patches up
+ * the performance counter enable mask for each of the blocks to exclude
+ * counters that were not requested by the client. */
+static void patch_dump_buffer_hdr_v4(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli)
+{
+ u32 *mask;
+ u8 *dst = cli->accum_buffer;
+ u8 *src = ctx->cpu_va;
+ u32 nr_cg = ctx->kctx->kbdev->gpu_props.num_core_groups;
+ size_t i, group_size, group;
+ enum {
+ SC0_BASE = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
+ SC1_BASE = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
+ SC2_BASE = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
+ SC3_BASE = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
+ TILER_BASE = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
+ MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
+ JM_BASE = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT
+ };
+
+ group_size = NR_CNT_BLOCKS_PER_GROUP *
+ NR_CNT_PER_BLOCK *
+ NR_BYTES_PER_CNT;
+ for (i = 0; i < nr_cg; i++) {
+ group = i * group_size;
+ /* copy shader core headers */
+ memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE],
+ NR_BYTES_PER_HDR);
+ memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE],
+ NR_BYTES_PER_HDR);
+ memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE],
+ NR_BYTES_PER_HDR);
+ memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE],
+ NR_BYTES_PER_HDR);
+
+ /* copy tiler header */
+ memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE],
+ NR_BYTES_PER_HDR);
+
+ /* copy mmu header */
+ memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE],
+ NR_BYTES_PER_HDR);
+
+ /* copy job manager header */
+ memcpy(&dst[group + JM_BASE], &src[group + JM_BASE],
+ NR_BYTES_PER_HDR);
+
+ /* patch the shader core enable mask */
+ mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[SHADER_HWCNT_BM];
+ mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[SHADER_HWCNT_BM];
+ mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[SHADER_HWCNT_BM];
+ mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[SHADER_HWCNT_BM];
+
+ /* patch the tiler core enable mask */
+ mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[TILER_HWCNT_BM];
+
+ /* patch the mmu core enable mask */
+ mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
+
+ /* patch the job manager enable mask */
+ mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[JM_HWCNT_BM];
+ }
+}
+
+/* This is the Midgard v5 patch function. It copies the headers for each
+ * of the defined blocks from the master kernel buffer and then patches up
+ * the performance counter enable mask for each of the blocks to exclude
+ * counters that were not requested by the client. */
+static void patch_dump_buffer_hdr_v5(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli)
+{
+ struct kbase_device *kbdev = ctx->kctx->kbdev;
+ u32 i, nr_l2, nr_sc;
+ u32 *mask;
+ u8 *dst = cli->accum_buffer;
+ u8 *src = ctx->cpu_va;
+ size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
+
+ /* copy and patch job manager header */
+ memcpy(dst, src, NR_BYTES_PER_HDR);
+ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[JM_HWCNT_BM];
+ dst += block_size;
+ src += block_size;
+
+ /* copy and patch tiler header */
+ memcpy(dst, src, NR_BYTES_PER_HDR);
+ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[TILER_HWCNT_BM];
+ dst += block_size;
+ src += block_size;
+
+ /* copy and patch MMU/L2C headers */
+ nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices;
+ for (i = 0; i < nr_l2; i++) {
+ memcpy(dst, src, NR_BYTES_PER_HDR);
+ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
+ dst += block_size;
+ src += block_size;
+ }
+
+ /* copy and patch shader core headers */
+ nr_sc = kbdev->gpu_props.props.coherency_info.group[0].num_cores;
+ for (i = 0; i < nr_sc; i++) {
+ memcpy(dst, src, NR_BYTES_PER_HDR);
+ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
+ *mask &= cli->bitmap[SHADER_HWCNT_BM];
+ dst += block_size;
+ src += block_size;
+ }
+}
+
+static void accum_clients(struct kbase_vinstr_context *ctx)
+{
+ struct kbase_vinstr_client *iter;
+ int v4;
+
+ v4 = kbase_hw_has_feature(ctx->kbdev, BASE_HW_FEATURE_V4);
+ list_for_each_entry(iter, &ctx->clients, list) {
+ if (v4)
+ patch_dump_buffer_hdr_v4(ctx, iter);
+ else
+ patch_dump_buffer_hdr_v5(ctx, iter);
+ accum_dump_buffer(iter->accum_buffer, ctx->cpu_va,
+ iter->dump_size);
+ }
+}
+
+int kbase_vinstr_dump(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli)
+{
+ int err = 0;
+
+ if (!cli)
+ return -EINVAL;
+
+ mutex_lock(&ctx->lock);
+ err = kbase_instr_hwcnt_request_dump(ctx->kctx);
+ if (err)
+ goto out;
+
+ err = kbase_instr_hwcnt_wait_for_dump(ctx->kctx);
+ if (err)
+ goto out;
+
+ accum_clients(ctx);
+
+ if (!cli->kernel) {
+ if (copy_to_user((void __user *)cli->dump_buffer,
+ cli->accum_buffer, cli->dump_size)) {
+ err = -EFAULT;
+ goto out;
+ }
+ } else {
+ memcpy(cli->dump_buffer, cli->accum_buffer, cli->dump_size);
+ }
+
+ memset(cli->accum_buffer, 0, cli->dump_size);
+out:
+ mutex_unlock(&ctx->lock);
+ return err;
+}
+
+int kbase_vinstr_clear(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli)
+{
+ int err = 0;
+
+ if (!cli)
+ return -EINVAL;
+
+ mutex_lock(&ctx->lock);
+ err = kbase_instr_hwcnt_request_dump(ctx->kctx);
+ if (err)
+ goto out;
+
+ err = kbase_instr_hwcnt_wait_for_dump(ctx->kctx);
+ if (err)
+ goto out;
+
+ err = kbase_instr_hwcnt_clear(ctx->kctx);
+ if (err)
+ goto out;
+
+ accum_clients(ctx);
+
+ memset(cli->accum_buffer, 0, cli->dump_size);
+out:
+ mutex_unlock(&ctx->lock);
+ return err;
+}
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifndef _KBASE_VINSTR_H_
+#define _KBASE_VINSTR_H_
+
+enum {
+ SHADER_HWCNT_BM,
+ TILER_HWCNT_BM,
+ MMU_L2_HWCNT_BM,
+ JM_HWCNT_BM
+};
+
+struct kbase_vinstr_context;
+struct kbase_vinstr_client;
+
+/**
+ * kbase_vinstr_init() - Initialize the vinstr core
+ * @kbdev: Kbase device
+ *
+ * Return: A pointer to the vinstr context on success or NULL on failure
+ */
+struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev);
+
+/**
+ * kbase_vinstr_term() - Terminate the vinstr core
+ * @ctx: Vinstr context
+ */
+void kbase_vinstr_term(struct kbase_vinstr_context *ctx);
+
+/**
+ * kbase_vinstr_attach_client - Attach a client to the vinstr core
+ * @ctx: Vinstr context
+ * @kernel: True if this client is a kernel-side client, false
+ * otherwise
+ * @dump_buffer: Client's dump buffer
+ * @bitmap: Bitmaps describing which counters should be enabled
+ *
+ * Return: A vinstr opaque client handle or NULL or failure
+ */
+struct kbase_vinstr_client *kbase_vinstr_attach_client(struct kbase_vinstr_context *ctx,
+ bool kernel,
+ u64 dump_buffer,
+ u32 bitmap[4]);
+
+/**
+ * kbase_vinstr_detach_client - Detach a client from the vinstr core
+ * @ctx: Vinstr context
+ * @cli: Pointer to vinstr client
+ */
+void kbase_vinstr_detach_client(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli);
+
+/**
+ * kbase_vinstr_dump_size - Get the size of the dump buffer
+ * @ctx: Vinstr context
+ *
+ * This is only useful for kernel-side clients to know how much
+ * memory they need to allocate to receive the performance counter
+ * memory block.
+ *
+ * Return: Returns the size of the client side buffer
+ */
+size_t kbase_vinstr_dump_size(struct kbase_vinstr_context *ctx);
+
+/**
+ * kbase_vinstr_dump - Performs a synchronous hardware counter dump for a given
+ * kbase context
+ * @ctx: Vinstr context
+ * @cli: Pointer to vinstr client
+ *
+ * Return: 0 on success
+ */
+int kbase_vinstr_dump(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli);
+
+/**
+ * kbase_vinstr_clear - Performs a reset of the hardware counters for a given
+ * kbase context
+ * @ctx: Vinstr context
+ * @cli: Pointer to vinstr client
+ *
+ * Return: 0 on success
+ */
+int kbase_vinstr_clear(struct kbase_vinstr_context *ctx,
+ struct kbase_vinstr_client *cli);
+
+#endif /* _KBASE_VINSTR_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* mali_job_slots_event - called from mali_kbase_core_linux.c
* @event_id: ORed together bitfields representing a type of event, made with the GATOR_MAKE_EVENT() macro.
*/
-TRACE_EVENT(mali_job_slots_event, TP_PROTO(unsigned int event_id, unsigned int tgid, unsigned int pid, unsigned char job_id), TP_ARGS(event_id, tgid, pid, job_id), TP_STRUCT__entry(__field(unsigned int, event_id)
- __field(unsigned int, tgid)
- __field(unsigned int, pid)
- __field(unsigned char, job_id)
- ), TP_fast_assign(__entry->event_id = event_id; __entry->tgid = tgid; __entry->pid = pid; __entry->job_id = job_id;), TP_printk("event=%u tgid=%u pid=%u job_id=%u", __entry->event_id, __entry->tgid, __entry->pid, __entry->job_id)
- );
+TRACE_EVENT(mali_job_slots_event,
+ TP_PROTO(unsigned int event_id, unsigned int tgid, unsigned int pid,
+ unsigned char job_id),
+ TP_ARGS(event_id, tgid, pid, job_id),
+ TP_STRUCT__entry(
+ __field(unsigned int, event_id)
+ __field(unsigned int, tgid)
+ __field(unsigned int, pid)
+ __field(unsigned char, job_id)
+ ),
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ __entry->tgid = tgid;
+ __entry->pid = pid;
+ __entry->job_id = job_id;
+ ),
+ TP_printk("event=%u tgid=%u pid=%u job_id=%u",
+ __entry->event_id, __entry->tgid, __entry->pid, __entry->job_id)
+);
/**
* mali_pm_status - Called by mali_kbase_pm_driver.c
- * @event_id: core type (shader, tiler, l2 cache, l3 cache)
+ * @event_id: core type (shader, tiler, l2 cache)
* @value: 64bits bitmask reporting either power status of the cores (1-ON, 0-OFF)
*/
-TRACE_EVENT(mali_pm_status, TP_PROTO(unsigned int event_id, unsigned long long value), TP_ARGS(event_id, value), TP_STRUCT__entry(__field(unsigned int, event_id)
- __field(unsigned long long, value)
- ), TP_fast_assign(__entry->event_id = event_id;), TP_printk("event %u = %llu", __entry->event_id, __entry->value)
- );
+TRACE_EVENT(mali_pm_status,
+ TP_PROTO(unsigned int event_id, unsigned long long value),
+ TP_ARGS(event_id, value),
+ TP_STRUCT__entry(
+ __field(unsigned int, event_id)
+ __field(unsigned long long, value)
+ ),
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ __entry->value = value;
+ ),
+ TP_printk("event %u = %llu", __entry->event_id, __entry->value)
+);
/**
* mali_pm_power_on - Called by mali_kbase_pm_driver.c
- * @event_id: core type (shader, tiler, l2 cache, l3 cache)
+ * @event_id: core type (shader, tiler, l2 cache)
* @value: 64bits bitmask reporting the cores to power up
*/
-TRACE_EVENT(mali_pm_power_on, TP_PROTO(unsigned int event_id, unsigned long long value), TP_ARGS(event_id, value), TP_STRUCT__entry(__field(unsigned int, event_id)
- __field(unsigned long long, value)
- ), TP_fast_assign(__entry->event_id = event_id;), TP_printk("event %u = %llu", __entry->event_id, __entry->value)
- );
+TRACE_EVENT(mali_pm_power_on,
+ TP_PROTO(unsigned int event_id, unsigned long long value),
+ TP_ARGS(event_id, value),
+ TP_STRUCT__entry(
+ __field(unsigned int, event_id)
+ __field(unsigned long long, value)
+ ),
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ __entry->value = value;
+ ),
+ TP_printk("event %u = %llu", __entry->event_id, __entry->value)
+);
/**
* mali_pm_power_off - Called by mali_kbase_pm_driver.c
- * @event_id: core type (shader, tiler, l2 cache, l3 cache)
+ * @event_id: core type (shader, tiler, l2 cache)
* @value: 64bits bitmask reporting the cores to power down
*/
-TRACE_EVENT(mali_pm_power_off, TP_PROTO(unsigned int event_id, unsigned long long value), TP_ARGS(event_id, value), TP_STRUCT__entry(__field(unsigned int, event_id)
- __field(unsigned long long, value)
- ), TP_fast_assign(__entry->event_id = event_id;), TP_printk("event %u = %llu", __entry->event_id, __entry->value)
- );
+TRACE_EVENT(mali_pm_power_off,
+ TP_PROTO(unsigned int event_id, unsigned long long value),
+ TP_ARGS(event_id, value),
+ TP_STRUCT__entry(
+ __field(unsigned int, event_id)
+ __field(unsigned long long, value)
+ ),
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ __entry->value = value;
+ ),
+ TP_printk("event %u = %llu", __entry->event_id, __entry->value)
+);
/**
* mali_page_fault_insert_pages - Called by page_fault_worker()
* @event_id: MMU address space number.
* @value: number of newly allocated pages
*/
-TRACE_EVENT(mali_page_fault_insert_pages, TP_PROTO(int event_id, unsigned long value), TP_ARGS(event_id, value), TP_STRUCT__entry(__field(int, event_id)
- __field(unsigned long, value)
- ), TP_fast_assign(__entry->event_id = event_id;), TP_printk("event %d = %lu", __entry->event_id, __entry->value)
- );
+TRACE_EVENT(mali_page_fault_insert_pages,
+ TP_PROTO(int event_id, unsigned long value),
+ TP_ARGS(event_id, value),
+ TP_STRUCT__entry(
+ __field(int, event_id)
+ __field(unsigned long, value)
+ ),
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ __entry->value = value;
+ ),
+ TP_printk("event %d = %lu", __entry->event_id, __entry->value)
+);
/**
* mali_mmu_as_in_use - Called by assign_and_activate_kctx_addr_space()
* it reports that a certain MMU address space is in use now.
* @event_id: MMU address space number.
*/
-TRACE_EVENT(mali_mmu_as_in_use, TP_PROTO(int event_id), TP_ARGS(event_id), TP_STRUCT__entry(__field(int, event_id)
- ), TP_fast_assign(__entry->event_id = event_id;), TP_printk("event=%d", __entry->event_id)
- );
+TRACE_EVENT(mali_mmu_as_in_use,
+ TP_PROTO(int event_id),
+ TP_ARGS(event_id),
+ TP_STRUCT__entry(
+ __field(int, event_id)
+ ),
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ ),
+ TP_printk("event=%d", __entry->event_id)
+);
/**
* mali_mmu_as_released - Called by kbasep_js_runpool_release_ctx_internal()
* it reports that a certain MMU address space has been released now.
* @event_id: MMU address space number.
*/
-TRACE_EVENT(mali_mmu_as_released, TP_PROTO(int event_id), TP_ARGS(event_id), TP_STRUCT__entry(__field(int, event_id)
- ), TP_fast_assign(__entry->event_id = event_id;), TP_printk("event=%d", __entry->event_id)
- );
+TRACE_EVENT(mali_mmu_as_released,
+ TP_PROTO(int event_id),
+ TP_ARGS(event_id),
+ TP_STRUCT__entry(
+ __field(int, event_id)
+ ),
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ ),
+ TP_printk("event=%d", __entry->event_id)
+);
/**
* mali_total_alloc_pages_change - Called by kbase_atomic_add_pages()
* it reports that the total number of allocated pages is changed.
* @event_id: number of pages to be added or subtracted (according to the sign).
*/
-TRACE_EVENT(mali_total_alloc_pages_change, TP_PROTO(long long int event_id), TP_ARGS(event_id), TP_STRUCT__entry(__field(long long int, event_id)
- ), TP_fast_assign(__entry->event_id = event_id;), TP_printk("event=%lld", __entry->event_id)
- );
+TRACE_EVENT(mali_total_alloc_pages_change,
+ TP_PROTO(long long int event_id),
+ TP_ARGS(event_id),
+ TP_STRUCT__entry(
+ __field(long long int, event_id)
+ ),
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ ),
+ TP_printk("event=%lld", __entry->event_id)
+);
/**
* mali_sw_counter - not currently used
* @event_id: counter id
*/
-TRACE_EVENT(mali_sw_counter, TP_PROTO(unsigned int event_id, signed long long value), TP_ARGS(event_id, value), TP_STRUCT__entry(__field(int, event_id)
- __field(long long, value)
- ), TP_fast_assign(__entry->event_id = event_id;), TP_printk("event %d = %lld", __entry->event_id, __entry->value)
- );
+TRACE_EVENT(mali_sw_counter,
+ TP_PROTO(unsigned int event_id, signed long long value),
+ TP_ARGS(event_id, value),
+ TP_STRUCT__entry(
+ __field(int, event_id)
+ __field(long long, value)
+ ),
+ TP_fast_assign(
+ __entry->event_id = event_id;
+ __entry->value = value;
+ ),
+ TP_printk("event %d = %lld", __entry->event_id, __entry->value)
+);
#endif /* _TRACE_MALI_H */
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * Kernel-wide include for common macros and types.
+ */
+
+#ifndef _MALISW_H_
+#define _MALISW_H_
+
+#include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0)
+#define U8_MAX ((u8)~0U)
+#define S8_MAX ((s8)(U8_MAX>>1))
+#define S8_MIN ((s8)(-S8_MAX - 1))
+#define U16_MAX ((u16)~0U)
+#define S16_MAX ((s16)(U16_MAX>>1))
+#define S16_MIN ((s16)(-S16_MAX - 1))
+#define U32_MAX ((u32)~0U)
+#define S32_MAX ((s32)(U32_MAX>>1))
+#define S32_MIN ((s32)(-S32_MAX - 1))
+#define U64_MAX ((u64)~0ULL)
+#define S64_MAX ((s64)(U64_MAX>>1))
+#define S64_MIN ((s64)(-S64_MAX - 1))
+#endif /* LINUX_VERSION_CODE */
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0)
+#define SIZE_MAX (~(size_t)0)
+#endif /* LINUX_VERSION_CODE */
+
+/**
+ * MIN - Return the lesser of two values.
+ *
+ * As a macro it may evaluate its arguments more than once.
+ * Refer to MAX macro for more details
+ */
+#define MIN(x, y) ((x) < (y) ? (x) : (y))
+
+/**
+ * MAX - Return the greater of two values.
+ *
+ * As a macro it may evaluate its arguments more than once.
+ * If called on the same two arguments as MIN it is guaranteed to return
+ * the one that MIN didn't return. This is significant for types where not
+ * all values are comparable e.g. NaNs in floating-point types. But if you want
+ * to retrieve the min and max of two values, consider using a conditional swap
+ * instead.
+ */
+#define MAX(x, y) ((x) < (y) ? (y) : (x))
+
+/**
+ * @hideinitializer
+ * Function-like macro for suppressing unused variable warnings. Where possible
+ * such variables should be removed; this macro is present for cases where we
+ * much support API backwards compatibility.
+ */
+#define CSTD_UNUSED(x) ((void)(x))
+
+/**
+ * @hideinitializer
+ * Function-like macro for use where "no behavior" is desired. This is useful
+ * when compile time macros turn a function-like macro in to a no-op, but
+ * where having no statement is otherwise invalid.
+ */
+#define CSTD_NOP(...) ((void)#__VA_ARGS__)
+
+/**
+ * Function-like macro for converting a pointer in to a u64 for storing into
+ * an external data structure. This is commonly used when pairing a 32-bit
+ * CPU with a 64-bit peripheral, such as a Midgard GPU. C's type promotion
+ * is complex and a straight cast does not work reliably as pointers are
+ * often considered as signed.
+ */
+#define PTR_TO_U64(x) ((uint64_t)((uintptr_t)(x)))
+
+/**
+ * @hideinitializer
+ * Function-like macro for stringizing a single level macro.
+ * @code
+ * #define MY_MACRO 32
+ * CSTD_STR1( MY_MACRO )
+ * > "MY_MACRO"
+ * @endcode
+ */
+#define CSTD_STR1(x) #x
+
+/**
+ * @hideinitializer
+ * Function-like macro for stringizing a macro's value. This should not be used
+ * if the macro is defined in a way which may have no value; use the
+ * alternative @c CSTD_STR2N macro should be used instead.
+ * @code
+ * #define MY_MACRO 32
+ * CSTD_STR2( MY_MACRO )
+ * > "32"
+ * @endcode
+ */
+#define CSTD_STR2(x) CSTD_STR1(x)
+
+/**
+ * Specify an assertion value which is evaluated at compile time. Recommended
+ * usage is specification of a @c static @c INLINE function containing all of
+ * the assertions thus:
+ *
+ * @code
+ * static INLINE [module]_compile_time_assertions( void )
+ * {
+ * COMPILE_TIME_ASSERT( sizeof(uintptr_t) == sizeof(intptr_t) );
+ * }
+ * @endcode
+ *
+ * @note Use @c static not @c STATIC. We never want to turn off this @c static
+ * specification for testing purposes.
+ */
+#define CSTD_COMPILE_TIME_ASSERT(expr) \
+ do { switch (0) { case 0: case (expr):; } } while (false)
+
+#endif /* _MALISW_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r))
#define GPU_ID 0x000 /* (RO) GPU and revision identifier */
#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */
-#define L3_FEATURES 0x008 /* (RO) Level 3 cache features */
+#define SUSPEND_SIZE 0x008 /* (RO) Fixed-function suspend buffer
+ size */
#define TILER_FEATURES 0x00C /* (RO) Tiler Features */
#define MEM_FEATURES 0x010 /* (RO) Memory system features */
#define MMU_FEATURES 0x014 /* (RO) MMU features */
#define GPU_COMMAND 0x030 /* (WO) */
#define GPU_STATUS 0x034 /* (RO) */
+
#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */
-#define GROUPS_L3_COHERENT (1 << 1) /* Cores groups are l3 coherent */
#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */
#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */
#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable flags for Job Manager */
#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable flags for shader cores */
#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable flags for tiler */
-#define PRFCNT_L3_CACHE_EN 0x078 /* (RW) Performance counter enable flags for L3 cache */
#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable flags for MMU/L2 cache */
#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */
#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */
#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */
-#define L3_PRESENT_LO 0x130 /* (RO) Level 3 cache present bitmap, low word */
-#define L3_PRESENT_HI 0x134 /* (RO) Level 3 cache present bitmap, high word */
#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */
#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */
#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */
#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */
-#define L3_READY_LO 0x170 /* (RO) Level 3 cache ready bitmap, low word */
-#define L3_READY_HI 0x174 /* (RO) Level 3 cache ready bitmap, high word */
#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */
#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */
#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */
#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */
-#define L3_PWRON_LO 0x1B0 /* (WO) Level 3 cache power on bitmap, low word */
-#define L3_PWRON_HI 0x1B4 /* (WO) Level 3 cache power on bitmap, high word */
-
#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */
#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */
#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */
#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */
-#define L3_PWROFF_LO 0x1F0 /* (WO) Level 3 cache power off bitmap, low word */
-#define L3_PWROFF_HI 0x1F4 /* (WO) Level 3 cache power off bitmap, high word */
-
#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */
#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */
#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */
#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */
-#define L3_PWRTRANS_LO 0x230 /* (RO) Level 3 cache power transition bitmap, low word */
-#define L3_PWRTRANS_HI 0x234 /* (RO) Level 3 cache power transition bitmap, high word */
-
#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */
#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */
#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */
#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */
-#define L3_PWRACTIVE_LO 0x270 /* (RO) Level 3 cache active bitmap, low word */
-#define L3_PWRACTIVE_HI 0x274 /* (RO) Level 3 cache active bitmap, high word */
#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration settings (Implementation specific register) */
#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration settings (Implementation specific register) */
#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */
+
#define MEMORY_MANAGEMENT_BASE 0x2000
#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r))
#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */
#define AS_STATUS 0x28 /* (RO) Status flags for address space n */
+
+
/* End Register Offsets */
/*
#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS))
/*
- * Begin MMU TRANSTAB register values
+ * Begin LPAE MMU TRANSTAB register values
*/
-#define AS_TRANSTAB_ADDR_SPACE_MASK 0xfffff000
-#define AS_TRANSTAB_ADRMODE_UNMAPPED (0u << 0)
-#define AS_TRANSTAB_ADRMODE_IDENTITY (1u << 1)
-#define AS_TRANSTAB_ADRMODE_TABLE (3u << 0)
-#define AS_TRANSTAB_READ_INNER (1u << 2)
-#define AS_TRANSTAB_SHARE_OUTER (1u << 4)
+#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffff000
+#define AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED (0u << 0)
+#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY (1u << 1)
+#define AS_TRANSTAB_LPAE_ADRMODE_TABLE (3u << 0)
+#define AS_TRANSTAB_LPAE_READ_INNER (1u << 2)
+#define AS_TRANSTAB_LPAE_SHARE_OUTER (1u << 4)
+
+#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x00000003
-#define MMU_TRANSTAB_ADRMODE_MASK 0x00000003
/*
* Begin MMU STATUS register values
*/
#define AS_STATUS_AS_ACTIVE 0x01
-#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3<<8)
-#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1<<8)
-#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2<<8)
-#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3<<8)
+#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3)
+#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3)
+#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3)
+#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3)
+#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3)
+
+
+#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3<<8)
+#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1<<8)
+#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2<<8)
+#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3<<8)
+
/*
* Begin Command Values
#define PRFCNT_CONFIG_MODE_TILE 2 /* The performance counters are enabled, and are written out each time a tile finishes rendering. */
/* AS<n>_MEMATTR values: */
+
/* Use GPU implementation-defined caching policy. */
-#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x48
+#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull
/* The attribute set to force all resources to be cached. */
-#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x4F
+#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full
/* Inner write-alloc cache setup, no outer caching */
-#define AS_MEMATTR_WRITE_ALLOC 0x4D
-/* symbol for default MEMATTR to use */
+#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull
+/* Set to implementation defined, outer caching */
+#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull
+/* Set to write back memory, outer caching */
+#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull
+
+/* Symbol for default MEMATTR to use */
#define AS_MEMATTR_INDEX_DEFAULT 0
+
/* HW implementation defined caching */
#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0
/* Force cache on */
#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1
-/* Write-alloc inner */
+/* Write-alloc */
#define AS_MEMATTR_INDEX_WRITE_ALLOC 2
+/* Outer coherent, inner implementation defined policy */
+#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3
+/* Outer coherent, write alloc inner */
+#define AS_MEMATTR_INDEX_OUTER_WA 4
/* GPU_ID register */
#define GPU_ID_VERSION_STATUS_SHIFT 0
#define GPU_ID_PI_T62X 0x0620
#define GPU_ID_PI_T76X 0x0750
#define GPU_ID_PI_T72X 0x0720
-#ifdef MALI_INCLUDE_TFRX
#define GPU_ID_PI_TFRX 0x0880
-#endif /* MALI_INCLUDE_TFRX */
#define GPU_ID_PI_T86X 0x0860
+#define GPU_ID_PI_T82X 0x0820
+#define GPU_ID_PI_T83X 0x0830
/* Values for GPU_ID_VERSION_STATUS field for PRODUCT_ID GPU_ID_PI_T60X */
#define GPU_ID_S_15DEV0 0x1
/* End THREAD_* registers */
+/* COHERENCY_* values*/
+#define COHERENCY_ACE_LITE 0
+#define COHERENCY_ACE 1
+#define COHERENCY_NONE 0xFFFF
+#define COHERENCY_FEATURE_BIT(x) (1 << (x))
+/* End COHERENCY_* values */
+
/* SHADER_CONFIG register */
#define SC_ALT_COUNTERS (1ul << 3)
/* End TILER_CONFIG register */
-#endif /* _MIDGARD_REGMAP_H_ */
+#endif /* _MIDGARD_REGMAP_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
),
TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type,
- (int)__entry->ts_sec,
- (int)__entry->ts_nsec,
- __entry->tgid,
- __entry->atom_id,
- __entry->atom_id)
+ (int)__entry->ts_sec,
+ (int)__entry->ts_nsec,
+ __entry->tgid,
+ __entry->atom_id,
+ __entry->atom_id)
);
TRACE_EVENT(mali_timeline_gpu_slot_active,
),
TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type,
- (int)__entry->ts_sec,
- (int)__entry->ts_nsec,
- __entry->tgid,
- __entry->js,
- __entry->count)
+ (int)__entry->ts_sec,
+ (int)__entry->ts_nsec,
+ __entry->tgid,
+ __entry->js,
+ __entry->count)
);
TRACE_EVENT(mali_timeline_gpu_slot_action,
),
TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type,
- (int)__entry->ts_sec,
- (int)__entry->ts_nsec,
- __entry->tgid,
- __entry->js,
- __entry->count)
+ (int)__entry->ts_sec,
+ (int)__entry->ts_nsec,
+ __entry->tgid,
+ __entry->js,
+ __entry->count)
);
TRACE_EVENT(mali_timeline_gpu_power_active,
),
TP_printk("%i,%i.%.9i,0,%i", __entry->event_type,
- (int)__entry->ts_sec,
- (int)__entry->ts_nsec,
- __entry->active)
+ (int)__entry->ts_sec,
+ (int)__entry->ts_nsec,
+ __entry->active)
);
),
TP_printk("%i,%i.%.9i,0,%i", __entry->event_type,
- (int)__entry->ts_sec,
- (int)__entry->ts_nsec,
- __entry->state)
+ (int)__entry->ts_sec,
+ (int)__entry->ts_nsec,
+ __entry->state)
);
TRACE_EVENT(mali_timeline_pm_event,
),
TP_printk("%i,%i.%.9i,0,%i,%u", __entry->event_type,
- (int)__entry->ts_sec,
- (int)__entry->ts_nsec,
- __entry->pm_event_type, __entry->pm_event_id)
+ (int)__entry->ts_sec,
+ (int)__entry->ts_nsec,
+ __entry->pm_event_type, __entry->pm_event_id)
);
),
TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type,
- (int)__entry->ts_sec,
- (int)__entry->ts_nsec,
- __entry->tgid,
- __entry->js,
- __entry->atom_id)
+ (int)__entry->ts_sec,
+ (int)__entry->ts_nsec,
+ __entry->tgid,
+ __entry->js,
+ __entry->atom_id)
);
TRACE_EVENT(mali_timeline_pm_checktrans,
),
TP_printk("%i,%i.%.9i,0,%i", __entry->trans_code,
- (int)__entry->ts_sec,
- (int)__entry->ts_nsec,
- __entry->trans_id)
+ (int)__entry->ts_sec,
+ (int)__entry->ts_nsec,
+ __entry->trans_id)
);
),
TP_printk("%i,%i.%.9i,0,%i", SW_SET_CONTEXT_ACTIVE,
- (int)__entry->ts_sec,
- (int)__entry->ts_nsec,
- __entry->count)
+ (int)__entry->ts_sec,
+ (int)__entry->ts_nsec,
+ __entry->count)
);
-#endif /* _MALI_TIMELINE_H */
+#endif /* _MALI_TIMELINE_H */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010, 2012-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
extern "C" {
#endif /* __cplusplus */
-#include <malisw/mali_stdtypes.h>
-
/**
* @addtogroup base_api
* @{
* identifying the UK function to be called (see uk_func). When the UKK client
* receives this header and executed the requested UK function, it will use
* the same header to store the result of the function in the form of a
- * mali_error return code. The size of this structure is such that the
+ * int return code. The size of this structure is such that the
* first member of the payload following the header can be accessed efficiently
* on a 32 and 64-bit kernel and the structure has the same size regardless
* of a 32 or 64-bit kernel. The uk_kernel_size_type type should be defined
*/
u32 id;
/**
- * The mali_error return code returned by the called UK function.
+ * The int return code returned by the called UK function.
* See the specification of the particular UK function you are
* calling for the meaning of the error codes returned. All
- * UK functions return MALI_ERROR_NONE on success.
+ * UK functions return 0 on success.
*/
u32 ret;
/*
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-/**
- * @addtogroup malisw
- * @{
- */
-
-/* ============================================================================
- Description
-============================================================================ */
-/**
- * @defgroup arm_cstd_coding_standard ARM C standard types and constants
- * The common files are a set of standard headers which are used by all parts
- * of this development, describing types, and generic constants.
- *
- * Files in group:
- * - arm_cstd.h
- * - arm_cstd_compilers.h
- * - arm_cstd_types.h
- * - arm_cstd_types_rvct.h
- * - arm_cstd_types_gcc.h
- * - arm_cstd_types_msvc.h
- * - arm_cstd_pack_push.h
- * - arm_cstd_pack_pop.h
- */
-
-/**
- * @addtogroup arm_cstd_coding_standard
- * @{
- */
-
-#ifndef _ARM_CSTD_
-#define _ARM_CSTD_
-
-/* ============================================================================
- Import standard C99 types
-============================================================================ */
-#include "arm_cstd_compilers.h"
-#include "arm_cstd_types.h"
-
-/* ============================================================================
- Min and Max Values
-============================================================================ */
-#if !defined(INT8_MAX)
- #define INT8_MAX ((int8_t) 0x7F)
-#endif
-#if !defined(INT8_MIN)
- #define INT8_MIN (-INT8_MAX - 1)
-#endif
-
-#if !defined(INT16_MAX)
- #define INT16_MAX ((int16_t)0x7FFF)
-#endif
-#if !defined(INT16_MIN)
- #define INT16_MIN (-INT16_MAX - 1)
-#endif
-
-#if !defined(INT32_MAX)
- #define INT32_MAX ((int32_t)0x7FFFFFFF)
-#endif
-#if !defined(INT32_MIN)
- #define INT32_MIN (-INT32_MAX - 1)
-#endif
-
-#if !defined(INT64_MAX)
- #define INT64_MAX ((int64_t)0x7FFFFFFFFFFFFFFFLL)
-#endif
-#if !defined(INT64_MIN)
- #define INT64_MIN (-INT64_MAX - 1)
-#endif
-
-#if !defined(UINT8_MAX)
- #define UINT8_MAX ((uint8_t) 0xFF)
-#endif
-
-#if !defined(UINT16_MAX)
- #define UINT16_MAX ((uint16_t)0xFFFF)
-#endif
-
-#if !defined(UINT32_MAX)
- #define UINT32_MAX ((uint32_t)0xFFFFFFFF)
-#endif
-
-#if !defined(UINT64_MAX)
- #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL)
-#endif
-
-/* fallbacks if limits.h wasn't available */
-#if !defined(UCHAR_MAX)
- #define UCHAR_MAX ((unsigned char)~0U)
-#endif
-
-#if !defined(SCHAR_MAX)
- #define SCHAR_MAX ((signed char)(UCHAR_MAX >> 1))
-#endif
-#if !defined(SCHAR_MIN)
- #define SCHAR_MIN ((signed char)(-SCHAR_MAX - 1))
-#endif
-
-#if !defined(USHRT_MAX)
- #define USHRT_MAX ((unsigned short)~0U)
-#endif
-
-#if !defined(SHRT_MAX)
- #define SHRT_MAX ((signed short)(USHRT_MAX >> 1))
-#endif
-#if !defined(SHRT_MIN)
- #define SHRT_MIN ((signed short)(-SHRT_MAX - 1))
-#endif
-
-#if !defined(UINT_MAX)
- #define UINT_MAX ((unsigned int)~0U)
-#endif
-
-#if !defined(INT_MAX)
- #define INT_MAX ((signed int)(UINT_MAX >> 1))
-#endif
-#if !defined(INT_MIN)
- #define INT_MIN ((signed int)(-INT_MAX - 1))
-#endif
-
-#if !defined(ULONG_MAX)
- #define ULONG_MAX ((unsigned long)~0UL)
-#endif
-
-#if !defined(LONG_MAX)
- #define LONG_MAX ((signed long)(ULONG_MAX >> 1))
-#endif
-#if !defined(LONG_MIN)
- #define LONG_MIN ((signed long)(-LONG_MAX - 1))
-#endif
-
-#if !defined(ULLONG_MAX)
- #define ULLONG_MAX ((unsigned long long)~0ULL)
-#endif
-
-#if !defined(LLONG_MAX)
- #define LLONG_MAX ((signed long long)(ULLONG_MAX >> 1))
-#endif
-#if !defined(LLONG_MIN)
- #define LLONG_MIN ((signed long long)(-LLONG_MAX - 1))
-#endif
-
-#if !defined(SIZE_MAX)
- #if 1 == CSTD_CPU_32BIT
- #define SIZE_MAX UINT32_MAX
- #elif 1 == CSTD_CPU_64BIT
- #define SIZE_MAX UINT64_MAX
- #endif
-#endif
-
-/* ============================================================================
- Keywords
-============================================================================ */
-/* Portable keywords. */
-
-#if !defined(CONST)
-/**
- * @hideinitializer
- * Variable is a C @c const, which can be made non-const for testing purposes.
- */
- #define CONST const
-#endif
-
-#if !defined(STATIC)
-/**
- * @hideinitializer
- * Variable is a C @c static, which can be made non-static for testing
- * purposes.
- */
- #define STATIC static
-#endif
-
-/**
- * Specifies a function as being exported outside of a logical module.
- */
-#define PUBLIC
-
-/**
- * @def PROTECTED
- * Specifies a a function which is internal to an logical module, but which
- * should not be used outside of that module. This cannot be enforced by the
- * compiler, as a module is typically more than one translation unit.
- */
-#define PROTECTED
-
-/**
- * Specifies a function as being internal to a translation unit. Private
- * functions would typically be declared as STATIC, unless they are being
- * exported for unit test purposes.
- */
-#define PRIVATE STATIC
-
-/**
- * Specify an assertion value which is evaluated at compile time. Recommended
- * usage is specification of a @c static @c INLINE function containing all of
- * the assertions thus:
- *
- * @code
- * static INLINE [module]_compile_time_assertions( void )
- * {
- * COMPILE_TIME_ASSERT( sizeof(uintptr_t) == sizeof(intptr_t) );
- * }
- * @endcode
- *
- * @note Use @c static not @c STATIC. We never want to turn off this @c static
- * specification for testing purposes.
- */
-#define CSTD_COMPILE_TIME_ASSERT( expr ) \
- do { switch(0){case 0: case (expr):;} } while( FALSE )
-
-/**
- * @hideinitializer
- * @deprecated Prefered form is @c CSTD_UNUSED
- * Function-like macro for suppressing unused variable warnings. Where possible
- * such variables should be removed; this macro is present for cases where we
- * much support API backwards compatibility.
- */
-#define UNUSED( x ) ((void)(x))
-
-/**
- * @hideinitializer
- * Function-like macro for suppressing unused variable warnings. Where possible
- * such variables should be removed; this macro is present for cases where we
- * much support API backwards compatibility.
- */
-#define CSTD_UNUSED( x ) ((void)(x))
-
-/**
- * @hideinitializer
- * Function-like macro for use where "no behavior" is desired. This is useful
- * when compile time macros turn a function-like macro in to a no-op, but
- * where having no statement is otherwise invalid.
- */
-#define CSTD_NOP( ... ) ((void)#__VA_ARGS__)
-
-/**
- * @hideinitializer
- * Function-like macro for converting a pointer in to a u64 for storing into
- * an external data structure. This is commonly used when pairing a 32-bit
- * CPU with a 64-bit peripheral, such as a Midgard GPU. C's type promotion
- * is complex and a straight cast does not work reliably as pointers are
- * often considered as signed.
- */
-#define CSTD_PTR_TO_U64( x ) ((uint64_t)((uintptr_t)(x)))
-
-/**
- * @hideinitializer
- * Function-like macro for stringizing a single level macro.
- * @code
- * #define MY_MACRO 32
- * CSTD_STR1( MY_MACRO )
- * > "MY_MACRO"
- * @endcode
- */
-#define CSTD_STR1( x ) #x
-
-/**
- * @hideinitializer
- * Function-like macro for stringizing a macro's value. This should not be used
- * if the macro is defined in a way which may have no value; use the
- * alternative @c CSTD_STR2N macro should be used instead.
- * @code
- * #define MY_MACRO 32
- * CSTD_STR2( MY_MACRO )
- * > "32"
- * @endcode
- */
-#define CSTD_STR2( x ) CSTD_STR1( x )
-
-/**
- * @hideinitializer
- * Utility function for stripping the first character off a string.
- */
-static INLINE char* arm_cstd_strstrip( char * string )
-{
- return ++string;
-}
-
-/**
- * @hideinitializer
- * Function-like macro for stringizing a single level macro where the macro
- * itself may not have a value. Parameter @c a should be set to any single
- * character which is then stripped by the macro via an inline function. This
- * should only be used via the @c CSTD_STR2N macro; for printing a single
- * macro only the @c CSTD_STR1 macro is a better alternative.
- *
- * This macro requires run-time code to handle the case where the macro has
- * no value (you can't concat empty strings in the preprocessor).
- */
-#define CSTD_STR1N( a, x ) arm_cstd_strstrip( CSTD_STR1( a##x ) )
-
-/**
- * @hideinitializer
- * Function-like macro for stringizing a two level macro where the macro itself
- * may not have a value.
- * @code
- * #define MY_MACRO 32
- * CSTD_STR2N( MY_MACRO )
- * > "32"
- *
- * #define MY_MACRO 32
- * CSTD_STR2N( MY_MACRO )
- * > "32"
- * @endcode
- */
-#define CSTD_STR2N( x ) CSTD_STR1N( _, x )
-
-/* ============================================================================
- Validate portability constructs
-============================================================================ */
-static INLINE void arm_cstd_compile_time_assertions( void )
-{
- CSTD_COMPILE_TIME_ASSERT( sizeof(uint8_t) == 1 );
- CSTD_COMPILE_TIME_ASSERT( sizeof(int8_t) == 1 );
- CSTD_COMPILE_TIME_ASSERT( sizeof(uint16_t) == 2 );
- CSTD_COMPILE_TIME_ASSERT( sizeof(int16_t) == 2 );
- CSTD_COMPILE_TIME_ASSERT( sizeof(uint32_t) == 4 );
- CSTD_COMPILE_TIME_ASSERT( sizeof(int32_t) == 4 );
- CSTD_COMPILE_TIME_ASSERT( sizeof(uint64_t) == 8 );
- CSTD_COMPILE_TIME_ASSERT( sizeof(int64_t) == 8 );
- CSTD_COMPILE_TIME_ASSERT( sizeof(intptr_t) == sizeof(uintptr_t) );
-
- CSTD_COMPILE_TIME_ASSERT( 1 == TRUE );
- CSTD_COMPILE_TIME_ASSERT( 0 == FALSE );
-
-#if 1 == CSTD_CPU_32BIT
- CSTD_COMPILE_TIME_ASSERT( sizeof(uintptr_t) == 4 );
-#elif 1 == CSTD_CPU_64BIT
- CSTD_COMPILE_TIME_ASSERT( sizeof(uintptr_t) == 8 );
-#endif
-
-}
-
-/* ============================================================================
- Useful function-like macro
-============================================================================ */
-/**
- * @brief Return the lesser of two values.
- * As a macro it may evaluate its arguments more than once.
- * @see CSTD_MAX
- */
-#define CSTD_MIN( x, y ) ((x) < (y) ? (x) : (y))
-
-/**
- * @brief Return the greater of two values.
- * As a macro it may evaluate its arguments more than once.
- * If called on the same two arguments as CSTD_MIN it is guaranteed to return
- * the one that CSTD_MIN didn't return. This is significant for types where not
- * all values are comparable e.g. NaNs in floating-point types. But if you want
- * to retrieve the min and max of two values, consider using a conditional swap
- * instead.
- */
-#define CSTD_MAX( x, y ) ((x) < (y) ? (y) : (x))
-
-/**
- * @brief Clamp value @c x to within @c min and @c max inclusive.
- */
-#define CSTD_CLAMP( x, min, max ) ((x)<(min) ? (min):((x)>(max) ? (max):(x)))
-
-/**
- * Flag a cast as a reinterpretation, usually of a pointer type.
- */
-#define CSTD_REINTERPRET_CAST(type) (type)
-
-/**
- * Flag a cast as casting away const, usually of a pointer type.
- */
-#define CSTD_CONST_CAST(type) (type)
-
-/**
- * Flag a cast as a (potentially complex) value conversion, usually of a
- * numerical type.
- */
-#define CSTD_STATIC_CAST(type) (type)
-
-/* ============================================================================
- Useful bit constants
-============================================================================ */
-/**
- * @cond arm_cstd_utilities
- */
-
-/* Common bit constant values, useful in embedded programming. */
-#define F_BIT_0 ((uint32_t)0x00000001)
-#define F_BIT_1 ((uint32_t)0x00000002)
-#define F_BIT_2 ((uint32_t)0x00000004)
-#define F_BIT_3 ((uint32_t)0x00000008)
-#define F_BIT_4 ((uint32_t)0x00000010)
-#define F_BIT_5 ((uint32_t)0x00000020)
-#define F_BIT_6 ((uint32_t)0x00000040)
-#define F_BIT_7 ((uint32_t)0x00000080)
-#define F_BIT_8 ((uint32_t)0x00000100)
-#define F_BIT_9 ((uint32_t)0x00000200)
-#define F_BIT_10 ((uint32_t)0x00000400)
-#define F_BIT_11 ((uint32_t)0x00000800)
-#define F_BIT_12 ((uint32_t)0x00001000)
-#define F_BIT_13 ((uint32_t)0x00002000)
-#define F_BIT_14 ((uint32_t)0x00004000)
-#define F_BIT_15 ((uint32_t)0x00008000)
-#define F_BIT_16 ((uint32_t)0x00010000)
-#define F_BIT_17 ((uint32_t)0x00020000)
-#define F_BIT_18 ((uint32_t)0x00040000)
-#define F_BIT_19 ((uint32_t)0x00080000)
-#define F_BIT_20 ((uint32_t)0x00100000)
-#define F_BIT_21 ((uint32_t)0x00200000)
-#define F_BIT_22 ((uint32_t)0x00400000)
-#define F_BIT_23 ((uint32_t)0x00800000)
-#define F_BIT_24 ((uint32_t)0x01000000)
-#define F_BIT_25 ((uint32_t)0x02000000)
-#define F_BIT_26 ((uint32_t)0x04000000)
-#define F_BIT_27 ((uint32_t)0x08000000)
-#define F_BIT_28 ((uint32_t)0x10000000)
-#define F_BIT_29 ((uint32_t)0x20000000)
-#define F_BIT_30 ((uint32_t)0x40000000)
-#define F_BIT_31 ((uint32_t)0x80000000)
-
-/* Common 2^n size values, useful in embedded programming. */
-#define C_SIZE_1B ((uint32_t)0x00000001)
-#define C_SIZE_2B ((uint32_t)0x00000002)
-#define C_SIZE_4B ((uint32_t)0x00000004)
-#define C_SIZE_8B ((uint32_t)0x00000008)
-#define C_SIZE_16B ((uint32_t)0x00000010)
-#define C_SIZE_32B ((uint32_t)0x00000020)
-#define C_SIZE_64B ((uint32_t)0x00000040)
-#define C_SIZE_128B ((uint32_t)0x00000080)
-#define C_SIZE_256B ((uint32_t)0x00000100)
-#define C_SIZE_512B ((uint32_t)0x00000200)
-#define C_SIZE_1KB ((uint32_t)0x00000400)
-#define C_SIZE_2KB ((uint32_t)0x00000800)
-#define C_SIZE_4KB ((uint32_t)0x00001000)
-#define C_SIZE_8KB ((uint32_t)0x00002000)
-#define C_SIZE_16KB ((uint32_t)0x00004000)
-#define C_SIZE_32KB ((uint32_t)0x00008000)
-#define C_SIZE_64KB ((uint32_t)0x00010000)
-#define C_SIZE_128KB ((uint32_t)0x00020000)
-#define C_SIZE_256KB ((uint32_t)0x00040000)
-#define C_SIZE_512KB ((uint32_t)0x00080000)
-#define C_SIZE_1MB ((uint32_t)0x00100000)
-#define C_SIZE_2MB ((uint32_t)0x00200000)
-#define C_SIZE_4MB ((uint32_t)0x00400000)
-#define C_SIZE_8MB ((uint32_t)0x00800000)
-#define C_SIZE_16MB ((uint32_t)0x01000000)
-#define C_SIZE_32MB ((uint32_t)0x02000000)
-#define C_SIZE_64MB ((uint32_t)0x04000000)
-#define C_SIZE_128MB ((uint32_t)0x08000000)
-#define C_SIZE_256MB ((uint32_t)0x10000000)
-#define C_SIZE_512MB ((uint32_t)0x20000000)
-#define C_SIZE_1GB ((uint32_t)0x40000000)
-#define C_SIZE_2GB ((uint32_t)0x80000000)
-
-/**
- * @endcond
- */
-
-/**
- * @}
- */
-
-/**
- * @}
- */
-
-#endif /* End (_ARM_CSTD_) */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-#ifndef _ARM_CSTD_COMPILERS_H_
-#define _ARM_CSTD_COMPILERS_H_
-
-/* ============================================================================
- Document default definitions - assuming nothing set at this point.
-============================================================================ */
-/**
- * @addtogroup arm_cstd_coding_standard
- * @{
- */
-
-/**
- * @hideinitializer
- * Defined with value of 1 if toolchain is Microsoft Visual Studio, 0
- * otherwise.
- */
-#define CSTD_TOOLCHAIN_MSVC 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if toolchain is the GNU Compiler Collection, 0
- * otherwise.
- */
-#define CSTD_TOOLCHAIN_GCC 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if toolchain is ARM RealView Compiler Tools, 0
- * otherwise. Note - if running RVCT in GCC mode this define will be set to 0;
- * @c CSTD_TOOLCHAIN_GCC and @c CSTD_TOOLCHAIN_RVCT_GCC_MODE will both be
- * defined as 1.
- */
-#define CSTD_TOOLCHAIN_RVCT 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if toolchain is ARM RealView Compiler Tools running
- * in GCC mode, 0 otherwise.
- */
-#define CSTD_TOOLCHAIN_RVCT_GCC_MODE 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if processor is an x86 32-bit machine, 0 otherwise.
- */
-#define CSTD_CPU_X86_32 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if processor is an x86-64 (AMD64) machine, 0
- * otherwise.
- */
-#define CSTD_CPU_X86_64 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if processor is an ARM machine, 0 otherwise.
- */
-#define CSTD_CPU_ARM 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if processor is an AARCH64 machine, 0 otherwise.
- */
-#define CSTD_CPU_AARCH64 0
-
-
-/**
- * @hideinitializer
- * Defined with value of 1 if processor is a MIPS machine, 0 otherwise.
- */
-#define CSTD_CPU_MIPS 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if CPU is 32-bit, 0 otherwise.
- */
-#define CSTD_CPU_32BIT 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if CPU is 64-bit, 0 otherwise.
- */
-#define CSTD_CPU_64BIT 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if processor configured as big-endian, 0 if it
- * is little-endian.
- */
-#define CSTD_CPU_BIG_ENDIAN 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a version of Windows, 0 if
- * it is not.
- */
-#define CSTD_OS_WINDOWS 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a 32-bit version of Windows,
- * 0 if it is not.
- */
-#define CSTD_OS_WIN32 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a 64-bit version of Windows,
- * 0 if it is not.
- */
-#define CSTD_OS_WIN64 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is Linux, 0 if it is not.
- */
-#define CSTD_OS_LINUX 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if we are compiling Linux kernel code, 0 otherwise.
- */
-#define CSTD_OS_LINUX_KERNEL 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a 32-bit version of Linux,
- * 0 if it is not.
- */
-#define CSTD_OS_LINUX32 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a 64-bit version of Linux,
- * 0 if it is not.
- */
-#define CSTD_OS_LINUX64 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is Android, 0 if it is not.
- */
-#define CSTD_OS_ANDROID 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if we are compiling Android kernel code, 0 otherwise.
- */
-#define CSTD_OS_ANDROID_KERNEL 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a 32-bit version of Android,
- * 0 if it is not.
- */
-#define CSTD_OS_ANDROID32 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a 64-bit version of Android,
- * 0 if it is not.
- */
-#define CSTD_OS_ANDROID64 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a version of Apple OS,
- * 0 if it is not.
- */
-#define CSTD_OS_APPLEOS 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a 32-bit version of Apple OS,
- * 0 if it is not.
- */
-#define CSTD_OS_APPLEOS32 0
-
-/**
- * @hideinitializer
- * Defined with value of 1 if operating system is a 64-bit version of Apple OS,
- * 0 if it is not.
- */
-#define CSTD_OS_APPLEOS64 0
-
-/**
- * @def CSTD_OS_SYMBIAN
- * @hideinitializer
- * Defined with value of 1 if operating system is Symbian, 0 if it is not.
- */
-#define CSTD_OS_SYMBIAN 0
-
-/**
- * @def CSTD_OS_NONE
- * @hideinitializer
- * Defined with value of 1 if there is no operating system (bare metal), 0
- * otherwise
- */
-#define CSTD_OS_NONE 0
-
-/* ============================================================================
- Determine the compiler in use
-============================================================================ */
-
-/* Default empty definitions of compiler-specific option enable/disable. This will be overridden
- * if applicable by preprocessor defines below. */
-#define CSTD_PUSH_WARNING_GCC_WADDRESS
-#define CSTD_POP_WARNING_GCC_WADDRESS
-
-#if defined(_MSC_VER)
- #undef CSTD_TOOLCHAIN_MSVC
- #define CSTD_TOOLCHAIN_MSVC 1
-
-#elif defined(__GNUC__)
- #undef CSTD_TOOLCHAIN_GCC
- #define CSTD_TOOLCHAIN_GCC 1
-
- /* Detect RVCT pretending to be GCC. */
- #if defined(__ARMCC_VERSION)
- #undef CSTD_TOOLCHAIN_RVCT_GCC_MODE
- #define CSTD_TOOLCHAIN_RVCT_GCC_MODE 1
- #endif
-
- #if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6 && MALI_GCC_WORKAROUND_MIDCOM_4598 == 0)
- /* As a workaround to MIDCOM-4598 (GCC internal defect), these pragmas are not compiled if the GCC version
- * is within a certain range, or if a #define is enabled by the build system. For more, see a comment
- * in the build system also referring to the MIDCOM issue mentioned, where the environment is updated
- * for the GNU toolchain. */
- #undef CSTD_PUSH_WARNING_GCC_WADDRESS
- #define CSTD_PUSH_WARNING_GCC_WADDRESS \
- do\
- {\
- _Pragma("GCC diagnostic push")\
- _Pragma("GCC diagnostic ignored \"-Waddress\"")\
- }while(MALI_FALSE)
-
- #undef CSTD_POP_WARNING_GCC_WADDRESS
- #define CSTD_POP_WARNING_GCC_WADDRESS \
- do\
- {\
- _Pragma("GCC diagnostic pop")\
- }while(MALI_FALSE)
- #endif
-
-#elif defined(__ARMCC_VERSION)
- #undef CSTD_TOOLCHAIN_RVCT
- #define CSTD_TOOLCHAIN_RVCT 1
-
-#else
- #warning "Unsupported or unknown toolchain"
-
-#endif
-
-/* ============================================================================
- Determine the processor
-============================================================================ */
-#if 1 == CSTD_TOOLCHAIN_MSVC
- #if defined(_M_IX86)
- #undef CSTD_CPU_X86_32
- #define CSTD_CPU_X86_32 1
-
- #elif defined(_M_X64) || defined(_M_AMD64)
- #undef CSTD_CPU_X86_64
- #define CSTD_CPU_X86_64 1
-
- #elif defined(_M_ARM)
- #undef CSTD_CPU_ARM
- #define CSTD_CPU_ARM 1
-
- #elif defined(_M_MIPS)
- #undef CSTD_CPU_MIPS
- #define CSTD_CPU_MIPS 1
-
- #else
- #warning "Unsupported or unknown host CPU for MSVC tools"
-
- #endif
-
-#elif 1 == CSTD_TOOLCHAIN_GCC
- #if defined(__amd64__)
- #undef CSTD_CPU_X86_64
- #define CSTD_CPU_X86_64 1
-
- #elif defined(__i386__)
- #undef CSTD_CPU_X86_32
- #define CSTD_CPU_X86_32 1
-
- #elif defined(__arm__)
- #undef CSTD_CPU_ARM
- #define CSTD_CPU_ARM 1
-
- #elif defined(__aarch64__)
- #undef CSTD_CPU_AARCH64
- #define CSTD_CPU_AARCH64 1
-
- #elif defined(__mips__)
- #undef CSTD_CPU_MIPS
- #define CSTD_CPU_MIPS 1
-
- #else
- #warning "Unsupported or unknown host CPU for GCC tools"
-
- #endif
-
-#elif 1 == CSTD_TOOLCHAIN_RVCT
- #undef CSTD_CPU_ARM
- #define CSTD_CPU_ARM 1
-
-#else
- #warning "Unsupported or unknown toolchain"
-
-#endif
-
-/* ============================================================================
- Determine the Processor Endianness
-============================================================================ */
-
-#if ((1 == CSTD_CPU_X86_32) || (1 == CSTD_CPU_X86_64))
- /* Note: x86 and x86-64 are always little endian, so leave at default. */
-
-#elif 1 == CSTD_CPU_AARCH64
- /* No big endian support? */
-
-#elif 1 == CSTD_TOOLCHAIN_RVCT
- #if defined(__BIG_ENDIAN)
- #undef CSTD_ENDIAN_BIG
- #define CSTD_ENDIAN_BIG 1
- #endif
-
-#elif ((1 == CSTD_TOOLCHAIN_GCC) && (1 == CSTD_CPU_ARM))
- #if defined(__ARMEB__)
- #undef CSTD_ENDIAN_BIG
- #define CSTD_ENDIAN_BIG 1
- #endif
-
-#elif ((1 == CSTD_TOOLCHAIN_GCC) && (1 == CSTD_CPU_MIPS))
- #if defined(__MIPSEB__)
- #undef CSTD_ENDIAN_BIG
- #define CSTD_ENDIAN_BIG 1
- #endif
-
-#elif 1 == CSTD_TOOLCHAIN_MSVC
- /* Note: Microsoft only support little endian, so leave at default. */
-
-#else
- #warning "Unsupported or unknown CPU"
-
-#endif
-
-/* ============================================================================
- Determine the operating system and addressing width
-============================================================================ */
-#if 1 == CSTD_TOOLCHAIN_MSVC
- #if defined(_WIN32) && !defined(_WIN64)
- #undef CSTD_OS_WINDOWS
- #define CSTD_OS_WINDOWS 1
- #undef CSTD_OS_WIN32
- #define CSTD_OS_WIN32 1
- #undef CSTD_CPU_32BIT
- #define CSTD_CPU_32BIT 1
-
- #elif defined(_WIN32) && defined(_WIN64)
- #undef CSTD_OS_WINDOWS
- #define CSTD_OS_WINDOWS 1
- #undef CSTD_OS_WIN64
- #define CSTD_OS_WIN64 1
- #undef CSTD_CPU_64BIT
- #define CSTD_CPU_64BIT 1
-
- #else
- #warning "Unsupported or unknown host OS for MSVC tools"
-
- #endif
-
-#elif 1 == CSTD_TOOLCHAIN_GCC
- #if defined(_WIN32) && defined(_WIN64)
- #undef CSTD_OS_WINDOWS
- #define CSTD_OS_WINDOWS 1
- #undef CSTD_OS_WIN64
- #define CSTD_OS_WIN64 1
- #undef CSTD_CPU_64BIT
- #define CSTD_CPU_64BIT 1
-
- #elif defined(_WIN32) && !defined(_WIN64)
- #undef CSTD_OS_WINDOWS
- #define CSTD_OS_WINDOWS 1
- #undef CSTD_OS_WIN32
- #define CSTD_OS_WIN32 1
- #undef CSTD_CPU_32BIT
- #define CSTD_CPU_32BIT 1
-
- #elif defined(ANDROID)
- #undef CSTD_OS_ANDROID
- #define CSTD_OS_ANDROID 1
-
- #if defined(__KERNEL__)
- #undef CSTD_OS_ANDROID_KERNEL
- #define CSTD_OS_ANDROID_KERNEL 1
- #endif
-
- #if defined(__LP64__) || defined(_LP64)
- #undef CSTD_OS_ANDROID64
- #define CSTD_OS_ANDROID64 1
- #undef CSTD_CPU_64BIT
- #define CSTD_CPU_64BIT 1
- #else
- #undef CSTD_OS_ANDROID32
- #define CSTD_OS_ANDROID32 1
- #undef CSTD_CPU_32BIT
- #define CSTD_CPU_32BIT 1
- #endif
-
- #elif defined(__KERNEL__) || defined(__linux)
- #undef CSTD_OS_LINUX
- #define CSTD_OS_LINUX 1
-
- #if defined(__KERNEL__)
- #undef CSTD_OS_LINUX_KERNEL
- #define CSTD_OS_LINUX_KERNEL 1
- #endif
-
- #if defined(__LP64__) || defined(_LP64)
- #undef CSTD_OS_LINUX64
- #define CSTD_OS_LINUX64 1
- #undef CSTD_CPU_64BIT
- #define CSTD_CPU_64BIT 1
- #else
- #undef CSTD_OS_LINUX32
- #define CSTD_OS_LINUX32 1
- #undef CSTD_CPU_32BIT
- #define CSTD_CPU_32BIT 1
- #endif
-
- #elif defined(__APPLE__)
- #undef CSTD_OS_APPLEOS
- #define CSTD_OS_APPLEOS 1
-
- #if defined(__LP64__) || defined(_LP64)
- #undef CSTD_OS_APPLEOS64
- #define CSTD_OS_APPLEOS64 1
- #undef CSTD_CPU_64BIT
- #define CSTD_CPU_64BIT 1
- #else
- #undef CSTD_OS_APPLEOS32
- #define CSTD_OS_APPLEOS32 1
- #undef CSTD_CPU_32BIT
- #define CSTD_CPU_32BIT 1
- #endif
-
- #elif defined(__SYMBIAN32__)
- #undef CSTD_OS_SYMBIAN
- #define CSTD_OS_SYMBIAN 1
- #undef CSTD_CPU_32BIT
- #define CSTD_CPU_32BIT 1
-
- #else
- #undef CSTD_OS_NONE
- #define CSTD_OS_NONE 1
- #undef CSTD_CPU_32BIT
- #define CSTD_CPU_32BIT 1
-
-#endif
-
-#elif 1 == CSTD_TOOLCHAIN_RVCT
-
- #if defined(ANDROID)
- #undef CSTD_OS_ANDROID
- #undef CSTD_OS_ANDROID32
- #define CSTD_OS_ANDROID 1
- #define CSTD_OS_ANDROID32 1
-
- #elif defined(__linux)
- #undef CSTD_OS_LINUX
- #undef CSTD_OS_LINUX32
- #define CSTD_OS_LINUX 1
- #define CSTD_OS_LINUX32 1
-
- #elif defined(__SYMBIAN32__)
- #undef CSTD_OS_SYMBIAN
- #define CSTD_OS_SYMBIAN 1
-
- #else
- #undef CSTD_OS_NONE
- #define CSTD_OS_NONE 1
-
-#endif
-
-#else
- #warning "Unsupported or unknown host OS"
-
-#endif
-
-/* ============================================================================
- Determine the correct linker symbol Import and Export Macros
-============================================================================ */
-/**
- * @defgroup arm_cstd_linkage_specifiers Linkage Specifiers
- * @{
- *
- * This set of macros contain system-dependent linkage specifiers which
- * determine the visibility of symbols across DLL boundaries. A header for a
- * particular DLL should define a set of local macros derived from these,
- * and should not use these macros to decorate functions directly as there may
- * be multiple DLLs being used.
- *
- * These DLL library local macros should be (with appropriate library prefix)
- * <tt>[MY_LIBRARY]_API</tt>, <tt>[MY_LIBRARY]_IMPL</tt>, and
- * <tt>[MY_LIBRARY]_LOCAL</tt>.
- *
- * - <tt>[MY_LIBRARY]_API</tt> should be use to decorate the function
- * declarations in the header. It should be defined as either
- * @c CSTD_LINK_IMPORT or @c CSTD_LINK_EXPORT, depending whether the
- * current situation is a compile of the DLL itself (use export) or a
- * compile of an external user of the DLL (use import).
- * - <tt>[MY_LIBRARY]_IMPL</tt> should be defined as @c CSTD_LINK_IMPL
- * and should be used to decorate the definition of functions in the C
- * file.
- * - <tt>[MY_LIBRARY]_LOCAL</tt> should be used to decorate function
- * declarations which are exported across translation units within the
- * DLL, but which are not exported outside of the DLL boundary.
- *
- * Functions which are @c static in either a C file or in a header file do not
- * need any form of linkage decoration, and should therefore have no linkage
- * macro applied to them.
- */
-
-/**
- * @def CSTD_LINK_IMPORT
- * Specifies a function as being imported to a translation unit across a DLL
- * boundary.
- */
-
-/**
- * @def CSTD_LINK_EXPORT
- * Specifies a function as being exported across a DLL boundary by a
- * translation unit.
- */
-
-/**
- * @def CSTD_LINK_IMPL
- * Specifies a function which will be exported across a DLL boundary as
- * being implemented by a translation unit.
- */
-
-/**
- * @def CSTD_LINK_LOCAL
- * Specifies a function which is internal to a DLL, and which should not be
- * exported outside of it.
- */
-
-/**
- * @}
- */
-
-#if 1 == CSTD_OS_LINUX
- #define CSTD_LINK_IMPORT __attribute__((visibility("default")))
- #define CSTD_LINK_EXPORT __attribute__((visibility("default")))
- #define CSTD_LINK_IMPL __attribute__((visibility("default")))
- #define CSTD_LINK_LOCAL __attribute__((visibility("hidden")))
-
-#elif 1 == CSTD_OS_WINDOWS
- #define CSTD_LINK_IMPORT __declspec(dllimport)
- #define CSTD_LINK_EXPORT __declspec(dllexport)
- #define CSTD_LINK_IMPL __declspec(dllexport)
- #define CSTD_LINK_LOCAL
-
-#elif 1 == CSTD_OS_SYMBIAN
- #define CSTD_LINK_IMPORT IMPORT_C
- #define CSTD_LINK_EXPORT IMPORT_C
- #define CSTD_LINK_IMPL EXPORT_C
- #define CSTD_LINK_LOCAL
-
-#elif 1 == CSTD_OS_APPLEOS
- #define CSTD_LINK_IMPORT __attribute__((visibility("default")))
- #define CSTD_LINK_EXPORT __attribute__((visibility("default")))
- #define CSTD_LINK_IMPL __attribute__((visibility("default")))
- #define CSTD_LINK_LOCAL __attribute__((visibility("hidden")))
-
-#elif 1 == CSTD_OS_ANDROID
- #define CSTD_LINK_IMPORT __attribute__((visibility("default")))
- #define CSTD_LINK_EXPORT __attribute__((visibility("default")))
- #define CSTD_LINK_IMPL __attribute__((visibility("default")))
- #define CSTD_LINK_LOCAL __attribute__((visibility("hidden")))
-
-#else /* CSTD_OS_NONE */
- #define CSTD_LINK_IMPORT
- #define CSTD_LINK_EXPORT
- #define CSTD_LINK_IMPL
- #define CSTD_LINK_LOCAL
-
-#endif
-
-/**
- * @}
- */
-
-#endif /* End (_ARM_CSTD_COMPILERS_H_) */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-#ifndef _ARM_CSTD_PACK_POP_H_
-#define _ARM_CSTD_PACK_POP_H_
-
-#if 1 == CSTD_TOOLCHAIN_MSVC
- #include <poppack.h>
-#endif
-
-#endif /* End (_ARM_CSTD_PACK_POP_H_) */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-#ifndef _ARM_CSTD_PACK_PUSH_H_
-#define _ARM_CSTD_PACK_PUSH_H_
-
-#if 1 == CSTD_TOOLCHAIN_MSVC
- #include <pshpack1.h>
-#endif
-
-#endif /* End (_ARM_CSTD_PACK_PUSH_H_) */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-#ifndef _ARM_CSTD_TYPES_H_
-#define _ARM_CSTD_TYPES_H_
-
-#if 1 == CSTD_TOOLCHAIN_MSVC
- #include "arm_cstd_types_msvc.h"
-#elif 1 == CSTD_TOOLCHAIN_GCC
- #include "arm_cstd_types_gcc.h"
-#elif 1 == CSTD_TOOLCHAIN_RVCT
- #include "arm_cstd_types_rvct.h"
-#else
- #error "Toolchain not recognized"
-#endif
-
-#endif /* End (_ARM_CSTD_TYPES_H_) */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-#ifndef _ARM_CSTD_TYPES_GCC_H_
-#define _ARM_CSTD_TYPES_GCC_H_
-
-/* ============================================================================
- Type definitions
-============================================================================ */
-/* All modern versions of GCC support stdint outside of C99 Mode. */
-/* However, Linux kernel limits what headers are available! */
-#if 1 == CSTD_OS_LINUX_KERNEL
- #include <linux/kernel.h>
- #include <linux/types.h>
- #include <linux/stddef.h>
- #include <linux/version.h>
-
- /* Fix up any types which CSTD provdes but which Linux is missing. */
- /* Note Linux assumes pointers are "long", so this is safe. */
- #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)
- typedef unsigned long uintptr_t;
- #endif
- typedef long intptr_t;
-
-#else
- #include <stdint.h>
- #include <stddef.h>
- #include <limits.h>
-#endif
-
-typedef uint32_t bool_t;
-
-#if !defined(TRUE)
- #define TRUE ((bool_t)1)
-#endif
-
-#if !defined(FALSE)
- #define FALSE ((bool_t)0)
-#endif
-
-/* ============================================================================
- Keywords
-============================================================================ */
-/* Doxygen documentation for these is in the RVCT header. */
-#define ASM __asm__
-
-#define INLINE __inline__
-
-#define FORCE_INLINE __attribute__((__always_inline__)) __inline__
-
-#define NEVER_INLINE __attribute__((__noinline__))
-
-#define PURE __attribute__((__pure__))
-
-#define PACKED __attribute__((__packed__))
-
-/* GCC does not support pointers to UNALIGNED data, so we do not define it to
- * force a compile error if this macro is used. */
-
-#define RESTRICT __restrict__
-
-/* RVCT in GCC mode does not support the CHECK_RESULT attribute. */
-#if 0 == CSTD_TOOLCHAIN_RVCT_GCC_MODE
- #define CHECK_RESULT __attribute__((__warn_unused_result__))
-#else
- #define CHECK_RESULT
-#endif
-
-/* RVCT in GCC mode does not support the __func__ name outside of C99. */
-#if (0 == CSTD_TOOLCHAIN_RVCT_GCC_MODE)
- #define CSTD_FUNC __func__
-#else
- #define CSTD_FUNC __FUNCTION__
-#endif
-
-#endif /* End (_ARM_CSTD_TYPES_GCC_H_) */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-#ifndef _ARM_CSTD_TYPES_RVCT_H_
-#define _ARM_CSTD_TYPES_RVCT_H_
-
-/* ============================================================================
- Type definitions
-============================================================================ */
-#include <stddef.h>
-#include <limits.h>
-
-#if 199901L <= __STDC_VERSION__
- #include <inttypes.h>
-#else
- typedef unsigned char uint8_t;
- typedef signed char int8_t;
- typedef unsigned short uint16_t;
- typedef signed short int16_t;
- typedef unsigned int uint32_t;
- typedef signed int int32_t;
- typedef unsigned __int64 uint64_t;
- typedef signed __int64 int64_t;
- typedef ptrdiff_t intptr_t;
- typedef size_t uintptr_t;
-#endif
-
-typedef uint32_t bool_t;
-
-#if !defined(TRUE)
- #define TRUE ((bool_t)1)
-#endif
-
-#if !defined(FALSE)
- #define FALSE ((bool_t)0)
-#endif
-
-/* ============================================================================
- Keywords
-============================================================================ */
-/**
- * @addtogroup arm_cstd_coding_standard
- * @{
- */
-
-/**
- * @def ASM
- * @hideinitializer
- * Mark an assembler block. Such blocks are often compiler specific, so often
- * need to be surrounded in appropriate @c ifdef and @c endif blocks
- * using the relevant @c CSTD_TOOLCHAIN macro.
- */
-#define ASM __asm
-
-/**
- * @def INLINE
- * @hideinitializer
- * Mark a definition as something which should be inlined. This is not always
- * possible on a given compiler, and may be disabled at lower optimization
- * levels.
- */
-#define INLINE __inline
-
-/**
- * @def FORCE_INLINE
- * @hideinitializer
- * Mark a definition as something which should be inlined. This provides a much
- * stronger hint to the compiler than @c INLINE, and if supported should always
- * result in an inlined function being emitted. If not supported this falls
- * back to using the @c INLINE definition.
- */
-#define FORCE_INLINE __forceinline
-
-/**
- * @def NEVER_INLINE
- * @hideinitializer
- * Mark a definition as something which should not be inlined. This provides a
- * stronger hint to the compiler than the function should not be inlined,
- * bypassing any heuristic rules the compiler normally applies. If not
- * supported by a toolchain this falls back to being an empty macro.
- */
-#define NEVER_INLINE __declspec(noinline)
-
-/**
- * @def PURE
- * @hideinitializer
- * Denotes that a function's return is only dependent on its inputs, enabling
- * more efficient optimizations. Falls back to an empty macro if not supported.
- */
-#define PURE __pure
-
-/**
- * @def PACKED
- * @hideinitializer
- * Denotes that a structure should be stored in a packed form. This macro must
- * be used in conjunction with the @c arm_cstd_pack_* headers for portability:
- *
- * @code
- * #include <cstd/arm_cstd_pack_push.h>
- *
- * struct PACKED myStruct {
- * ...
- * };
- *
- * #include <cstd/arm_cstd_pack_pop.h>
- * PACKED
- * @endcode
- */
-#define PACKED __packed
-
-/**
- * @def UNALIGNED
- * @hideinitializer
- * Denotes that a pointer points to a buffer with lower alignment than the
- * natural alignment required by the C standard. This should only be used
- * in extreme cases, as the emitted code is normally more efficient if memory
- * is aligned.
- *
- * @warning This is \b NON-PORTABLE. The GNU tools are anti-unaligned pointers
- * and have no support for such a construction.
- */
-#define UNALIGNED __packed
-
-/**
- * @def RESTRICT
- * @hideinitializer
- * Denotes that a pointer does not overlap with any other points currently in
- * scope, increasing the range of optimizations which can be performed by the
- * compiler.
- *
- * @warning Specification of @c RESTRICT is a contract between the programmer
- * and the compiler. If you place @c RESTICT on buffers which do actually
- * overlap the behavior is undefined, and likely to vary at different
- * optimization levels.!
- */
-#define RESTRICT __restrict
-
-/**
- * @def CHECK_RESULT
- * @hideinitializer
- * Function attribute which causes a warning to be emitted if the compiler's
- * return value is not used by the caller. Compiles to an empty macro if
- * there is no supported mechanism for this check in the underlying compiler.
- *
- * @note At the time of writing this is only supported by GCC. RVCT does not
- * support this attribute, even in GCC mode, so engineers are encouraged to
- * compile their code using GCC even if primarily working with another
- * compiler.
- *
- * @code
- * CHECK_RESULT int my_func( void );
- * @endcode
- */
-#define CHECK_RESULT
-
-/**
- * @def CSTD_FUNC
- * Specify the @c CSTD_FUNC macro, a portable construct containing the name of
- * the current function. On most compilers it is illegal to use this macro
- * outside of a function scope. If not supported by the compiler we define
- * @c CSTD_FUNC as an empty string.
- *
- * @warning Due to the implementation of this on most modern compilers this
- * expands to a magically defined "static const" variable, not a constant
- * string. This makes injecting @c CSTD_FUNC directly in to compile-time
- * strings impossible, so if you want to make the function name part of a
- * larger string you must use a printf-like function with a @c @%s template
- * which is populated with @c CSTD_FUNC
- */
-#define CSTD_FUNC __FUNCTION__
-
-/**
- * @}
- */
-
-#endif /* End (_ARM_CSTD_TYPES_RVCT_H_) */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-#ifndef _MALISW_H_
-#define _MALISW_H_
-
-/**
- * @file mali_malisw.h
- * Driver-wide include for common macros and types.
- */
-
-/**
- * @defgroup malisw Mali software definitions and types
- * @{
- */
-
-#include <stddef.h>
-
-#include "mali_stdtypes.h"
-
-/** @brief Gets the container object when given a pointer to a member of an object. */
-#define CONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type,member)))
-
-/** @brief Gets the number of elements of type s in a fixed length array of s */
-#define NELEMS(s) (sizeof(s)/sizeof((s)[0]))
-
-/**
- * @brief The lesser of two values.
- * May evaluate its arguments more than once.
- * @see CSTD_MIN
- */
-#define MIN(x,y) CSTD_MIN(x,y)
-
-/**
- * @brief The greater of two values.
- * May evaluate its arguments more than once.
- * @see CSTD_MAX
- */
-#define MAX(x,y) CSTD_MAX(x,y)
-
-/**
- * @brief Clamp value x to within min and max inclusive
- * May evaluate its arguments more than once.
- * @see CSTD_CLAMP
- */
-#define CLAMP( x, min, max ) CSTD_CLAMP( x, min, max )
-
-/**
- * @brief Convert a pointer into a u64 for storing in a data structure.
- * This is commonly used when pairing a 32-bit CPU with a 64-bit peripheral,
- * such as a Midgard GPU. C's type promotion is complex and a straight cast
- * does not work reliably as pointers are often considered as signed.
- */
-#define PTR_TO_U64( x ) CSTD_PTR_TO_U64( x )
-
-/**
- * @name Mali library linkage specifiers
- * These directly map to the cstd versions described in detail here: @ref arm_cstd_linkage_specifiers
- * @{
- */
-#define MALI_IMPORT CSTD_LINK_IMPORT
-#define MALI_EXPORT CSTD_LINK_EXPORT
-#define MALI_IMPL CSTD_LINK_IMPL
-#if defined(CONFIG_MALI_DEBUG) || !MALI_CUSTOMER_RELEASE
-#define MALI_LOCAL CSTD_LINK_EXPORT
-#else
-#define MALI_LOCAL CSTD_LINK_LOCAL
-#endif
-
-/** @brief Decorate exported function prototypes.
- *
- * The file containing the implementation of the function should define this to be MALI_EXPORT before including
- * malisw/mali_malisw.h.
- */
-#ifndef MALI_API
-#define MALI_API MALI_IMPORT
-#endif
-/** @} */
-
-/** @name Testable static functions
- * @{
- *
- * These macros can be used to allow functions to be static in release builds but exported from a shared library in unit
- * test builds, allowing them to be tested or used to assist testing.
- *
- * Example mali_foo_bar.c containing the function to test:
- *
- * @code
- * #define MALI_API MALI_EXPORT
- *
- * #include <malisw/mali_malisw.h>
- * #include "mali_foo_testable_statics.h"
- *
- * MALI_TESTABLE_STATIC_IMPL void my_func()
- * {
- * //Implementation
- * }
- * @endcode
- *
- * Example mali_foo_testable_statics.h:
- *
- * @code
- * #if 1 == MALI_UNIT_TEST
- * #include <malisw/mali_malisw.h>
- *
- * MALI_TESTABLE_STATIC_API void my_func();
- *
- * #endif
- * @endcode
- *
- * Example mali_foo_tests.c:
- *
- * @code
- * #include <foo/src/mali_foo_testable_statics.h>
- *
- * void my_test_func()
- * {
- * my_func();
- * }
- * @endcode
- */
-
-/** @brief Decorate testable static function implementations.
- *
- * A header file containing a MALI_TESTABLE_STATIC_API-decorated prototype for each static function will be required
- * when MALI_UNIT_TEST == 1 in order to link the function from the test.
- */
-#if 1 == MALI_UNIT_TEST
-#define MALI_TESTABLE_STATIC_IMPL MALI_IMPL
-#else
-#define MALI_TESTABLE_STATIC_IMPL static
-#endif
-
-/** @brief Decorate testable static function prototypes.
- *
- * @note Prototypes should @em only be declared when MALI_UNIT_TEST == 1
- */
-#define MALI_TESTABLE_STATIC_API MALI_API
-/** @} */
-
-/** @name Testable local functions
- * @{
- *
- * These macros can be used to allow functions to be local to a shared library in release builds but be exported in unit
- * test builds, allowing them to be tested or used to assist testing.
- *
- * Example mali_foo_bar.c containing the function to test:
- *
- * @code
- * #define MALI_API MALI_EXPORT
- *
- * #include <malisw/mali_malisw.h>
- * #include "mali_foo_bar.h"
- *
- * MALI_TESTABLE_LOCAL_IMPL void my_func()
- * {
- * //Implementation
- * }
- * @endcode
- *
- * Example mali_foo_bar.h:
- *
- * @code
- * #include <malisw/mali_malisw.h>
- *
- * MALI_TESTABLE_LOCAL_API void my_func();
- *
- * @endcode
- *
- * Example mali_foo_tests.c:
- *
- * @code
- * #include <foo/src/mali_foo_bar.h>
- *
- * void my_test_func()
- * {
- * my_func();
- * }
- * @endcode
- */
-
-/** @brief Decorate testable local function implementations.
- *
- * This can be used to have a function normally local to the shared library except in debug builds where it will be
- * exported.
- */
-#ifdef CONFIG_MALI_DEBUG
-#define MALI_TESTABLE_LOCAL_IMPL MALI_IMPL
-#else
-#define MALI_TESTABLE_LOCAL_IMPL MALI_LOCAL
-#endif /* CONFIG_MALI_DEBUG */
-
-/** @brief Decorate testable local function prototypes.
- *
- * This can be used to have a function normally local to the shared library except in debug builds where it will be
- * exported.
- */
-#ifdef CONFIG_MALI_DEBUG
-#define MALI_TESTABLE_LOCAL_API MALI_API
-#else
-#define MALI_TESTABLE_LOCAL_API MALI_LOCAL
-#endif /* CONFIG_MALI_DEBUG */
-/** @} */
-
-/**
- * Flag a cast as a reinterpretation, usually of a pointer type.
- * @see CSTD_REINTERPRET_CAST
- */
-#define REINTERPRET_CAST(type) CSTD_REINTERPRET_CAST(type)
-
-/**
- * Flag a cast as casting away const, usually of a pointer type.
- * @see CSTD_CONST_CAST
- */
-#define CONST_CAST(type) (type) CSTD_CONST_CAST(type)
-
-/**
- * Flag a cast as a (potentially complex) value conversion, usually of a numerical type.
- * @see CSTD_STATIC_CAST
- */
-#define STATIC_CAST(type) (type) CSTD_STATIC_CAST(type)
-
-
-/** @} */
-
-#endif /* _MALISW_H_ */
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-
-
-#ifndef _MALISW_STDTYPES_H_
-#define _MALISW_STDTYPES_H_
-
-/**
- * @file mali_stdtypes.h
- * This file defines the standard types used by the Mali codebase.
- */
-
-/**
- * @addtogroup malisw
- * @{
- */
-
-/**
- * @defgroup malisw_stdtypes Mali software standard types
- *
- * Basic driver-wide types.
- */
-
-/**
- * @addtogroup malisw_stdtypes
- * @{
- */
-
-#include "arm_cstd/arm_cstd.h"
-
-/**
- * @name Scalar types.
- * These are the scalar types used within the mali driver.
- * @{
- */
-/* Note: if compiling the Linux kernel then avoid redefining these. */
-#if 0 == CSTD_OS_LINUX_KERNEL
- typedef uint64_t u64;
- typedef uint32_t u32;
- typedef uint16_t u16;
- typedef uint8_t u8;
-
- typedef int64_t s64;
- typedef int32_t s32;
- typedef int16_t s16;
- typedef int8_t s8;
-#endif
-
-typedef double f64;
-typedef float f32;
-typedef u16 f16;
-
-typedef u32 mali_fixed16_16;
-/* @} */
-
-/**
- * @name Boolean types.
- * The intended use is for bool8 to be used when storing boolean values in
- * structures, casting to mali_bool to be used in code sections.
- * @{
- */
-typedef bool_t mali_bool;
-typedef u8 mali_bool8;
-
-#define MALI_FALSE FALSE
-#define MALI_TRUE TRUE
-/* @} */
-
-/**
- * @name Integer bounding values
- * Maximum and minimum values for integer types
- * @{
- */
-#ifndef U64_MAX
-#define U64_MAX UINT64_MAX
-#endif
-
-#ifndef U32_MAX
-#define U32_MAX UINT32_MAX
-#endif
-
-#ifndef U16_MAX
-#define U16_MAX UINT16_MAX
-#endif
-
-#ifndef U8_MAX
-#define U8_MAX UINT8_MAX
-#endif
-
-#ifndef S64_MAX
-#define S64_MAX INT64_MAX
-#endif
-
-#ifndef S64_MIN
-#define S64_MIN INT64_MIN
-#endif
-
-#ifndef S32_MAX
-#define S32_MAX INT32_MAX
-#endif
-
-#ifndef S32_MIN
-#define S32_MIN INT32_MIN
-#endif
-
-#ifndef S16_MAX
-#define S16_MAX INT16_MAX
-#endif
-
-#ifndef S16_MIN
-#define S16_MIN INT16_MIN
-#endif
-
-#ifndef S8_MAX
-#define S8_MAX INT8_MAX
-#endif
-
-#ifndef S8_MIN
-#define S8_MIN INT8_MIN
-#endif
-
-/* @} */
-
-/**
- * @name GPU address types
- * Types for integers which hold a GPU pointer or GPU pointer offsets.
- * @{
- */
-typedef u64 mali_addr64;
-typedef u32 mali_addr32;
-typedef u64 mali_size64;
-typedef s64 mali_offset64;
-/* 32 bit offsets and sizes are always for native types and so use ptrdiff_t and size_t respectively */
-/* @} */
-
-/**
- * @name Mali error types
- * @brief The common error type for the mali drivers
- * The mali_error type, all driver error handling should be of this type unless
- * it must deal with a specific APIs error type.
- * @{
- */
-typedef enum
-{
- /**
- * @brief Common Mali errors for the entire driver
- * MALI_ERROR_NONE is guaranteed to be 0.
- * @{
- */
- MALI_ERROR_NONE = 0,
- MALI_ERROR_OUT_OF_GPU_MEMORY,
- MALI_ERROR_OUT_OF_MEMORY,
- MALI_ERROR_FUNCTION_FAILED,
- /* @} */
- /**
- * @brief Mali errors for Client APIs to pass to EGL when creating EGLImages
- * These errors must only be returned to EGL from one of the Client APIs as part of the
- * (clientapi)_egl_image_interface.h
- * @{
- */
- MALI_ERROR_EGLP_BAD_ACCESS,
- MALI_ERROR_EGLP_BAD_PARAMETER,
- /* @} */
- /**
- * @brief Mali errors for the MCL module.
- * These errors must only be used within the private components of the OpenCL implementation that report
- * directly to API functions for cases where errors cannot be detected in the entrypoints file. They must
- * not be passed between driver components.
- * These are errors in the mali error space specifically for the MCL module, hence the MCLP prefix.
- * @{
- */
- MALI_ERROR_MCLP_DEVICE_NOT_FOUND,
- MALI_ERROR_MCLP_DEVICE_NOT_AVAILABLE,
- MALI_ERROR_MCLP_COMPILER_NOT_AVAILABLE,
- MALI_ERROR_MCLP_MEM_OBJECT_ALLOCATION_FAILURE,
- MALI_ERROR_MCLP_PROFILING_INFO_NOT_AVAILABLE,
- MALI_ERROR_MCLP_MEM_COPY_OVERLAP,
- MALI_ERROR_MCLP_IMAGE_FORMAT_MISMATCH,
- MALI_ERROR_MCLP_IMAGE_FORMAT_NOT_SUPPORTED,
- MALI_ERROR_MCLP_BUILD_PROGRAM_FAILURE,
- MALI_ERROR_MCLP_MAP_FAILURE,
- MALI_ERROR_MCLP_MISALIGNED_SUB_BUFFER_OFFSET,
- MALI_ERROR_MCLP_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST,
- MALI_ERROR_MCLP_INVALID_VALUE,
- MALI_ERROR_MCLP_INVALID_DEVICE_TYPE,
- MALI_ERROR_MCLP_INVALID_PLATFORM,
- MALI_ERROR_MCLP_INVALID_DEVICE,
- MALI_ERROR_MCLP_INVALID_CONTEXT,
- MALI_ERROR_MCLP_INVALID_QUEUE_PROPERTIES,
- MALI_ERROR_MCLP_INVALID_COMMAND_QUEUE,
- MALI_ERROR_MCLP_INVALID_HOST_PTR,
- MALI_ERROR_MCLP_INVALID_MEM_OBJECT,
- MALI_ERROR_MCLP_INVALID_IMAGE_FORMAT_DESCRIPTOR,
- MALI_ERROR_MCLP_INVALID_IMAGE_SIZE,
- MALI_ERROR_MCLP_INVALID_SAMPLER,
- MALI_ERROR_MCLP_INVALID_BINARY,
- MALI_ERROR_MCLP_INVALID_BUILD_OPTIONS,
- MALI_ERROR_MCLP_INVALID_PROGRAM,
- MALI_ERROR_MCLP_INVALID_PROGRAM_EXECUTABLE,
- MALI_ERROR_MCLP_INVALID_KERNEL_NAME,
- MALI_ERROR_MCLP_INVALID_KERNEL_DEFINITION,
- MALI_ERROR_MCLP_INVALID_KERNEL,
- MALI_ERROR_MCLP_INVALID_ARG_INDEX,
- MALI_ERROR_MCLP_INVALID_ARG_VALUE,
- MALI_ERROR_MCLP_INVALID_ARG_SIZE,
- MALI_ERROR_MCLP_INVALID_KERNEL_ARGS,
- MALI_ERROR_MCLP_INVALID_WORK_DIMENSION,
- MALI_ERROR_MCLP_INVALID_WORK_GROUP_SIZE,
- MALI_ERROR_MCLP_INVALID_WORK_ITEM_SIZE,
- MALI_ERROR_MCLP_INVALID_GLOBAL_OFFSET,
- MALI_ERROR_MCLP_INVALID_EVENT_WAIT_LIST,
- MALI_ERROR_MCLP_INVALID_EVENT,
- MALI_ERROR_MCLP_INVALID_OPERATION,
- MALI_ERROR_MCLP_INVALID_GL_OBJECT,
- MALI_ERROR_MCLP_INVALID_BUFFER_SIZE,
- MALI_ERROR_MCLP_INVALID_MIP_LEVEL,
- MALI_ERROR_MCLP_INVALID_GLOBAL_WORK_SIZE,
- MALI_ERROR_MCLP_INVALID_GL_SHAREGROUP_REFERENCE_KHR,
- MALI_ERROR_MCLP_INVALID_EGL_OBJECT,
- /* @} */
- /**
- * @brief Mali errors for the BASE module
- * These errors must only be used within the private components of the Base implementation. They will not
- * passed to other modules by the base driver.
- * These are errors in the mali error space specifically for the BASE module, hence the BASEP prefix.
- * @{
- */
- MALI_ERROR_BASEP_INVALID_FUNCTION,
- /* @} */
- /** A dependency exists upon a resource that the client application wants to modify, so the driver must either
- * create a copy of the resource (if possible) or block until the dependency has been satisfied.
- */
- MALI_ERROR_RESOURCE_IN_USE,
-
- /**
- * @brief A stride value was too big.
- *
- * A surface descriptor can store strides of up to 2<sup>31</sup>-1 bytes but strides greater than
- * 2<sup>28</sup>-1 bytes cannot be expressed in bits without overflow.
- */
- MALI_ERROR_STRIDE_TOO_BIG
-
-} mali_error;
-/* @} */
-
-/* @} */
-
-/* @} */
-
-#endif /* _MALISW_STDTYPES_H_ */
--- /dev/null
+#
+# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved.
+#
+# This program is free software and is provided to you under the terms of the
+# GNU General Public License version 2 as published by the Free Software
+# Foundation, and any use by you of this program is subject to the terms
+# of such GNU licence.
+#
+# A copy of the licence is included with the program, and can also be obtained
+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+#
+#
+
+
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+int kbase_platform_early_init(void)
+{
+ /* Nothing needed at this stage */
+ return 0;
+}
+
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * Maximum frequency GPU will be clocked at. Given in kHz.
+ * This must be specified as there is no default value.
+ *
+ * Attached value: number in kHz
+ * Default value: NA
+ */
+#define GPU_FREQ_KHZ_MAX (5000)
+/**
+ * Minimum frequency GPU will be clocked at. Given in kHz.
+ * This must be specified as there is no default value.
+ *
+ * Attached value: number in kHz
+ * Default value: NA
+ */
+#define GPU_FREQ_KHZ_MIN (5000)
+
+/**
+ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock
+ *
+ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_cpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define CPU_SPEED_FUNC (NULL)
+
+/**
+ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock
+ *
+ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_gpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define GPU_SPEED_FUNC (NULL)
+
+/**
+ * Power management configuration
+ *
+ * Attached value: pointer to @ref kbase_pm_callback_conf
+ * Default value: See @ref kbase_pm_callback_conf
+ */
+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks)
+
+/**
+ * Platform specific configuration functions
+ *
+ * Attached value: pointer to @ref kbase_platform_funcs_conf
+ * Default value: See @ref kbase_platform_funcs_conf
+ */
+#define PLATFORM_FUNCS (NULL)
+
+/** Power model for IPA
+ *
+ * Attached value: pointer to @ref mali_pa_model_ops
+ */
+#define POWER_MODEL_CALLBACKS (NULL)
+
+extern struct kbase_pm_callback_conf pm_callbacks;
+
+/**
+ * Secure mode switch
+ *
+ * Attached value: pointer to @ref kbase_secure_ops
+ */
+#define SECURE_CALLBACKS (NULL)
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#include <mali_kbase.h>
+#include <mali_kbase_defs.h>
+#include <linux/pm_runtime.h>
+#include <linux/suspend.h>
+
+static int pm_callback_power_on(struct kbase_device *kbdev)
+{
+ int ret;
+
+ dev_dbg(kbdev->dev, "pm_callback_power_on %p\n",
+ (void *)kbdev->dev->pm_domain);
+
+ ret = pm_runtime_get_sync(kbdev->dev);
+
+ dev_dbg(kbdev->dev, "pm_runtime_get returned %d\n", ret);
+
+ return 1;
+}
+
+static void pm_callback_power_off(struct kbase_device *kbdev)
+{
+ dev_dbg(kbdev->dev, "pm_callback_power_off\n");
+
+ pm_runtime_put_autosuspend(kbdev->dev);
+}
+
+int kbase_device_runtime_init(struct kbase_device *kbdev)
+{
+ dev_dbg(kbdev->dev, "kbase_device_runtime_init\n");
+ pm_runtime_enable(kbdev->dev);
+#ifdef CONFIG_MALI_MIDGARD_DEBUG_SYS
+ {
+ int err = kbase_platform_create_sysfs_file(kbdev->dev);
+
+ if (err)
+ return err;
+ }
+#endif /* CONFIG_MALI_MIDGARD_DEBUG_SYS */
+ return 0;
+}
+
+void kbase_device_runtime_disable(struct kbase_device *kbdev)
+{
+ dev_dbg(kbdev->dev, "kbase_device_runtime_disable\n");
+ pm_runtime_disable(kbdev->dev);
+}
+
+static int pm_callback_runtime_on(struct kbase_device *kbdev)
+{
+ dev_dbg(kbdev->dev, "pm_callback_runtime_on\n");
+
+ return 0;
+}
+
+static void pm_callback_runtime_off(struct kbase_device *kbdev)
+{
+ dev_dbg(kbdev->dev, "pm_callback_runtime_off\n");
+}
+
+static void pm_callback_resume(struct kbase_device *kbdev)
+{
+ int ret = pm_callback_runtime_on(kbdev);
+
+ WARN_ON(ret);
+}
+
+static void pm_callback_suspend(struct kbase_device *kbdev)
+{
+ pm_callback_runtime_off(kbdev);
+}
+
+struct kbase_pm_callback_conf pm_callbacks = {
+ .power_on_callback = pm_callback_power_on,
+ .power_off_callback = pm_callback_power_off,
+ .power_suspend_callback = pm_callback_suspend,
+ .power_resume_callback = pm_callback_resume,
+#ifdef CONFIG_PM_RUNTIME
+ .power_runtime_init_callback = kbase_device_runtime_init,
+ .power_runtime_term_callback = kbase_device_runtime_disable,
+ .power_runtime_on_callback = pm_callback_runtime_on,
+ .power_runtime_off_callback = pm_callback_runtime_off,
+#else /* CONFIG_PM_RUNTIME */
+ .power_runtime_init_callback = NULL,
+ .power_runtime_term_callback = NULL,
+ .power_runtime_on_callback = NULL,
+ .power_runtime_off_callback = NULL,
+#endif /* CONFIG_PM_RUNTIME */
+};
+
+
#
-# (C) COPYRIGHT 2013 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <linux/module.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
-#include <linux/pm_opp.h>
#include <linux/scpi_protocol.h>
+#include <linux/version.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
+#include <linux/pm_opp.h>
+#else /* Linux >= 3.13 */
+/* In 3.13 the OPP include header file, types, and functions were all
+ * renamed. Use the old filename for the include, and define the new names to
+ * the old, when an old kernel is detected.
+ */
+#include <linux/opp.h>
+#define dev_pm_opp_add opp_add
+#endif /* Linux >= 3.13 */
static int init_juno_opps_from_scpi(struct device *dev)
for (i = 0; i < sopp->count; i++) {
struct scpi_opp_entry *e = &sopp->opp[i];
+
dev_info(dev, "Mali OPP from SCPI: %u Hz @ %u mV\n",
e->freq_hz, e->volt_mv);
np = of_find_node_by_name(NULL, "gpu");
if (!np) {
- printk(KERN_ERR "Failed to find DT entry for Mali\n");
+ pr_err("Failed to find DT entry for Mali\n");
return -EFAULT;
}
pdev = of_find_device_by_node(np);
if (!pdev) {
- printk(KERN_ERR "Failed to find device for Mali\n");
+ pr_err("Failed to find device for Mali\n");
of_node_put(np);
return -EFAULT;
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <linux/ioport.h>
+#ifdef CONFIG_DEVFREQ_THERMAL
+#include <linux/devfreq_cooling.h>
+#endif
+#include <linux/thermal.h>
#include <mali_kbase.h>
#include <mali_kbase_defs.h>
#include <mali_kbase_config.h>
-#include "../mali_kbase_power_actor.h"
-
/* Versatile Express (VE) Juno Development Platform */
#define HARD_RESET_AT_POWER_OFF 0
#endif
}
-static struct kbase_pm_callback_conf pm_callbacks = {
+struct kbase_pm_callback_conf pm_callbacks = {
.power_on_callback = pm_callback_power_on,
.power_off_callback = pm_callback_power_off,
.power_suspend_callback = NULL,
.power_resume_callback = NULL
};
-static unsigned long juno_model_static_power(unsigned long voltage, unsigned long temperature)
+#ifdef CONFIG_DEVFREQ_THERMAL
+
+#define FALLBACK_STATIC_TEMPERATURE 55000
+
+static unsigned long juno_model_static_power(unsigned long voltage)
{
- /* Calculate power, corrected for voltage.
- * Shifts are done to avoid overflow. */
+ struct thermal_zone_device *tz;
+ unsigned long temperature, temp;
+ unsigned long temp_squared, temp_cubed, temp_scaling_factor;
const unsigned long coefficient = (410UL << 20) / (729000000UL >> 10);
const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10;
+ tz = thermal_zone_get_zone_by_name("gpu");
+ if (IS_ERR(tz)) {
+ pr_warn_ratelimited("Error getting gpu thermal zone (%ld), not yet ready?\n",
+ PTR_ERR(tz));
+ temperature = FALLBACK_STATIC_TEMPERATURE;
+ } else {
+ int ret;
+
+ ret = tz->ops->get_temp(tz, &temperature);
+ if (ret) {
+ pr_warn_ratelimited("Error reading temperature for gpu thermal zone: %d\n",
+ ret);
+ temperature = FALLBACK_STATIC_TEMPERATURE;
+ }
+ }
+
/* Calculate the temperature scaling factor. To be applied to the
- * voltage scaled power. */
- const unsigned long temp = temperature / 1000;
- const unsigned long temp_squared = temp * temp;
- const unsigned long temp_cubed = temp_squared * temp;
- const unsigned long temp_scaling_factor =
+ * voltage scaled power.
+ */
+ temp = temperature / 1000;
+ temp_squared = temp * temp;
+ temp_cubed = temp_squared * temp;
+ temp_scaling_factor =
(2 * temp_cubed)
- (80 * temp_squared)
+ (4700 * temp)
+ 32000;
- return (((coefficient * voltage_cubed) >> 20) * temp_scaling_factor) / 1000000;
+ return (((coefficient * voltage_cubed) >> 20)
+ * temp_scaling_factor)
+ / 1000000;
}
static unsigned long juno_model_dynamic_power(unsigned long freq,
return (coefficient * v2 * f_mhz) / 1000000; /* mW */
}
-static struct mali_pa_model_ops juno_model_ops = {
+struct devfreq_cooling_ops juno_model_ops = {
.get_static_power = juno_model_static_power,
.get_dynamic_power = juno_model_dynamic_power,
};
-static struct kbase_attribute config_attributes[] = {
- { KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS, 500 },
- { KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS, ((uintptr_t)&pm_callbacks) },
- { KBASE_CONFIG_ATTR_POWER_MODEL_CALLBACKS, ((uintptr_t)&juno_model_ops) },
+#endif /* CONFIG_DEVFREQ_THERMAL */
+
+static int juno_secure_mode_enable(struct kbase_device *kbdev)
+{
+ /* TODO: enable secure mode */
+ /*dev_err(kbdev->dev, "SWITCHING TO SECURE\n");*/
+ return 0; /* all ok */
+}
+
+static int juno_secure_mode_disable(struct kbase_device *kbdev)
+{
+ /* TODO: Turn off secure mode and reset GPU */
+ /*dev_err(kbdev->dev, "SWITCHING TO NON-SECURE\n");*/
+ return 0; /* all ok */
+}
- { KBASE_CONFIG_ATTR_END, 0 }
+struct kbase_secure_ops juno_secure_ops = {
+ .secure_mode_enable = juno_secure_mode_enable,
+ .secure_mode_disable = juno_secure_mode_disable,
};
static struct kbase_platform_config versatile_platform_config = {
- .attributes = config_attributes,
#ifndef CONFIG_OF
.io_resources = &io_resources
#endif
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Default value: NA
*/
#define GPU_FREQ_KHZ_MIN 600000
+
+/**
+ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock
+ *
+ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_cpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define CPU_SPEED_FUNC (&kbase_cpuprops_get_default_clock_speed)
+
+/**
+ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock
+ *
+ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_gpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define GPU_SPEED_FUNC (NULL)
+
+/**
+ * Power management configuration
+ *
+ * Attached value: pointer to @ref kbase_pm_callback_conf
+ * Default value: See @ref kbase_pm_callback_conf
+ */
+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks)
+
+/**
+ * Platform specific configuration functions
+ *
+ * Attached value: pointer to @ref kbase_platform_funcs_conf
+ * Default value: See @ref kbase_platform_funcs_conf
+ */
+#define PLATFORM_FUNCS (NULL)
+
+/** Power model for IPA
+ *
+ * Attached value: pointer to @ref mali_pa_model_ops
+ */
+#ifdef CONFIG_DEVFREQ_THERMAL
+#define POWER_MODEL_CALLBACKS (&juno_model_ops)
+#else
+#define POWER_MODEL_CALLBACKS (NULL)
+#endif
+
+/**
+ * Secure mode switch
+ *
+ * Attached value: pointer to @ref kbase_secure_ops
+ */
+#define SECURE_CALLBACKS (&juno_secure_ops)
+
+extern struct kbase_pm_callback_conf pm_callbacks;
+#ifdef CONFIG_DEVFREQ_THERMAL
+extern struct devfreq_cooling_ops juno_model_ops;
+#endif
+extern struct kbase_secure_ops juno_secure_ops;
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2013 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+#ifdef CONFIG_MALI_PLATFORM_FAKE
+
+/**
+ * kbase_platform_fake_register - Entry point for fake platform registration
+ *
+ * This function is called early on in the initialization during execution of
+ * kbase_driver_init.
+ *
+ * Return: 0 to indicate success, non-zero for failure.
+ */
+int kbase_platform_fake_register(void);
+
+/**
+ * kbase_platform_fake_unregister - Entry point for fake platform unregistration
+ *
+ * This function is called in the termination during execution of
+ * kbase_driver_exit.
+ */
+void kbase_platform_fake_unregister(void);
+
+#endif /* CONFIG_MALI_PLATFORM_FAKE */
/* --------------------------------------------------------------------------------------------------------\r
* File: custom_log.h \r
*\r
- * Desc: ChenZhen Æ«ºÃµÄ log Êä³öµÄ¶¨ÖÆʵÏÖ. \r
+ * Desc: ChenZhen 偏好的 log 输出的定制实现. \r
*\r
* -----------------------------------------------------------------------------------\r
- * < Ï°Óï ºÍ ËõÂÔÓï > : \r
+ * < 习语 和 缩略语 > : \r
*\r
* -----------------------------------------------------------------------------------\r
* Usage: \r
* ---------------------------------------------------------------------------------------------------------\r
*/\r
\r
-/** ÈôÏÂÁÐ macro ÓỶ¨Òå, ²Å ʹÄÜ log Êä³ö. */\r
+/** 若下列 macro 有被定义, 才 使能 log 输出. */\r
// #define ENABLE_DEBUG_LOG\r
\r
-/** .! : ÈôÐèҪȫ¾ÖµØ¹Ø±Õ D log, ¿ÉÒÔʹÄÜÏÂÃæµÄ´úÂë. */\r
+/** .! : 若需要全局地关闭 D log, 可以使能下面的代码. */\r
/*\r
#undef ENABLE_DEBUG_LOG\r
#warning "custom debug log is disabled globally!"\r
\r
/*-------------------------------------------------------*/\r
\r
-/** ʹÓà D(), ÒÔÊ®½øÖƵÄÐÎʽ´òÓ¡±äÁ¿ 'var' µÄ value. */\r
+/** 使用 D(), 以十进制的形式打印变量 'var' 的 value. */\r
#define D_DEC(var) D(#var " = %d.", var);\r
\r
#define E_DEC(var) E(#var " = %d.", var);\r
\r
-/** ʹÓà D(), ÒÔÊ®Áù½øÖƵÄÐÎʽ´òÓ¡±äÁ¿ 'var' µÄ value. */\r
+/** 使用 D(), 以十六进制的形式打印变量 'var' 的 value. */\r
#define D_HEX(var) D(#var " = 0x%x.", var);\r
\r
#define E_HEX(var) E(#var " = 0x%x.", var);\r
\r
-/** ʹÓà D(), ÒÔÊ®Áù½øÖƵÄÐÎʽ ´òÓ¡Ö¸ÕëÀàÐͱäÁ¿ 'ptr' µÄ value. */\r
+/** 使用 D(), 以十六进制的形式 打印指针类型变量 'ptr' 的 value. */\r
#define D_PTR(ptr) D(#ptr " = %p.", ptr);\r
\r
#define E_PTR(ptr) E(#ptr " = %p.", ptr);\r
\r
-/** ʹÓà D(), ´òÓ¡ char ×Ö´®. */\r
+/** 使用 D(), 打印 char 字串. */\r
#define D_STR(pStr) \\r
{\\r
if ( NULL == pStr )\\r
\r
#ifdef ENABLE_DEBUG_LOG\r
/**\r
- * log ´Ó 'pStart' µØÖ·¿ªÊ¼µÄ 'len' ¸ö×Ö½ÚµÄÊý¾Ý. \r
+ * log 从 'pStart' 地址开始的 'len' 个字节的数据. \r
*/\r
#define D_MEM(pStart, len) \\r
{\\r
/*-------------------------------------------------------*/\r
\r
/**\r
- * µ÷Óú¯Êý, ²¢¼ì²é·µ»ØÖµ, ¸ù¾Ý·µ»ØÖµ¾ö¶¨ÊÇ·ñÌøתµ½Ö¸¶¨µÄ´íÎó´¦Àí´úÂë. \r
+ * 调用函数, 并检查返回值, 根据返回值决定是否跳转到指定的错误处理代码. \r
* @param functionCall\r
- * ¶ÔÌض¨º¯ÊýµÄµ÷ÓÃ, ¸Ãº¯ÊýµÄ·µ»ØÖµ±ØÐëÊÇ ±íÕ÷ ³É¹¦ or err µÄ ÕûÐÍÊý. \r
- * ÕâÀï, ±»µ÷Óú¯Êý "±ØÐë" ÊDZ»¶¨ÒåΪ "·µ»Ø 0 ±íʾ²Ù×÷³É¹¦". \r
+ * 对特定函数的调用, 该函数的返回值必须是 表征 成功 or err 的 整型数. \r
+ * 这里, 被调用函数 "必须" 是被定义为 "返回 0 表示操作成功". \r
* @param result\r
- * ÓÃÓڼǼº¯Êý·µ»ØµÄ error code µÄ ÕûÐͱäÁ¿, ͨ³£ÊÇ "ret" or "result" µÈ.\r
+ * 用于记录函数返回的 error code 的 整型变量, 通常是 "ret" or "result" 等.\r
* @param label\r
- * Èôº¯Êý·µ»Ø´íÎó, ³ÌÐò½«ÒªÌøתµ½µÄ ´íÎó´¦Àí´¦µÄ ±êºÅ, ͨ³£¾ÍÊÇ "EXIT". \r
+ * 若函数返回错误, 程序将要跳转到的 错误处理处的 标号, 通常就是 "EXIT". \r
*/\r
#define CHECK_FUNC_CALL(functionCall, result, label) \\r
{\\r
}\r
\r
/**\r
- * ÔÚÌض¨Ìõ¼þÏÂ, Åж¨ error ·¢Éú, ¶Ô±äÁ¿ 'retVar' ÉèÖà 'errCode', \r
- * Log Êä³ö¶ÔÓ¦µÄ Error Caution, È»ºóÌøת 'label' Ö¸¶¨µÄ´úÂë´¦Ö´ÐÐ. \r
+ * 在特定条件下, 判定 error 发生, 对变量 'retVar' 设置 'errCode', \r
+ * Log 输出对应的 Error Caution, 然后跳转 'label' 指定的代码处执行. \r
* @param msg\r
- * ´¿×Ö´®ÐÎʽµÄÌáʾÐÅÏ¢. \r
+ * 纯字串形式的提示信息. \r
* @param retVar\r
- * ±êʶº¯ÊýÖ´ÐÐ״̬»òÕß½á¹ûµÄ±äÁ¿, ½«±»ÉèÖþßÌåµÄ Error Code. \r
- * ͨ³£ÊÇ 'ret' or 'result'. \r
+ * 标识函数执行状态或者结果的变量, 将被设置具体的 Error Code. \r
+ * 通常是 'ret' or 'result'. \r
* @param errCode\r
- * ±íÕ÷Ìض¨ error µÄ³£Êý±êʶ, ͨ³£ÊÇ ºêµÄÐÎ̬. \r
+ * 表征特定 error 的常数标识, 通常是 宏的形态. \r
* @param label\r
- * ³ÌÐò½«ÒªÌøתµ½µÄ´íÎó´¦Àí´úÂëµÄ±êºÅ, ͨ³£¾ÍÊÇ 'EXIT'. \r
+ * 程序将要跳转到的错误处理代码的标号, 通常就是 'EXIT'. \r
* @param args...\r
- * ¶ÔÓ¦ 'msgFmt' ʵ²ÎÖÐ '%s', '%d', ... µÈ ת»»ËµÃ÷·û µÄ¾ßÌå¿É±ä³¤Êµ²Î. \r
+ * 对应 'msgFmt' 实参中 '%s', '%d', ... 等 转换说明符 的具体可变长实参. \r
*/\r
#define SET_ERROR_AND_JUMP(msgFmt, retVar, errCode, label, args...) \\r
{\\r
--- /dev/null
+/*
+ *
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * A copy of the licence is included with the program, and can also be obtained
+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ */
+
+
+
+/**
+ * Maximum frequency
+ * GPU will be clocked at.
+ * Given in kHz.
+ * This must be specified
+ * as there is no default value.
+ *
+ * Attached value: number in kHz
+ * Default value: NA
+ */
+#define GPU_FREQ_KHZ_MAX (5000)
+
+/**
+ * Minimum frequency GPU will be clocked at.
+ * Given in kHz.
+ * This must be specified
+ * as there is no default value.
+ *
+ * Attached value: number in kHz
+ * Default value: NA
+ */
+#define GPU_FREQ_KHZ_MIN (5000)
+
+/**
+ * CPU_SPEED_FUNC
+ * - A pointer to a function that calculates the CPU clock
+ *
+ * CPU clock speed of the platform is in MHz
+ * - see kbase_cpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_cpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define CPU_SPEED_FUNC (NULL)
+
+/**
+ * GPU_SPEED_FUNC
+ * - A pointer to a function
+ * that calculates the GPU clock
+ *
+ * GPU clock speed of the platform in MHz
+ * - see kbase_gpu_clk_speed_func for the function prototype.
+ *
+ * Attached value: A kbase_gpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define GPU_SPEED_FUNC (NULL)
+
+/**
+ * Power management configuration
+ *
+ * Attached value:
+ * pointer to @ref kbase_pm_callback_conf
+ * Default value:
+ * See @ref kbase_pm_callback_conf
+ */
+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks)
+extern struct kbase_pm_callback_conf pm_callbacks;
+
+/**
+ * Platform specific configuration functions
+ *
+ * Attached value:
+ * pointer to @ref kbase_platform_funcs_conf
+ * Default value:
+ * See @ref kbase_platform_funcs_conf
+ */
+#define PLATFORM_FUNCS (&platform_funcs)
+extern struct kbase_platform_funcs_conf platform_funcs;
+
+/** Power model for IPA
+ *
+ * Attached value: pointer to @ref mali_pa_model_ops
+ */
+#define POWER_MODEL_CALLBACKS (NULL)
+
+
+/**
+ * Secure mode switch
+ *
+ * Attached value: pointer to @ref kbase_secure_ops
+ */
+#define SECURE_CALLBACKS (NULL)
+
*
*/
-
-
-
+// #define ENABLE_DEBUG_LOG
+#include "custom_log.h"
#include <linux/ioport.h>
#include <mali_kbase.h>
.end = 0xFC010000 + (4096 * 5) - 1}
};
#endif
+
int get_cpu_clock_speed(u32 *cpu_clock)
{
#if 0
/*
rk3288 hardware specific initialization
*/
-mali_bool kbase_platform_rk_init(struct kbase_device *kbdev)
+int kbase_platform_rk_init(struct kbase_device *kbdev)
{
if(MALI_ERROR_NONE == kbase_platform_init(kbdev))
{
if (register_pm_notifier(&mali_pm_nb)) {
- return MALI_FALSE;
+ E("fail to register pm_notifier.");
+ return -1;
}
pr_info("%s,register_reboot_notifier\n",__func__);
register_reboot_notifier(&mali_reboot_notifier);
- return MALI_TRUE;
+ return 0;
}
- return MALI_FALSE;
+
+ E("fail to init platform.");
+ return -1;
}
/*
kbase_platform_term(kbdev);
}
-kbase_platform_funcs_conf platform_funcs = {
+struct kbase_platform_funcs_conf platform_funcs =
+{
.platform_init_func = &kbase_platform_rk_init,
.platform_term_func = &kbase_platform_rk_term,
};
pm_schedule_suspend(dev, RUNTIME_PM_DELAY_TIME);
}
-mali_error kbase_device_runtime_init(struct kbase_device *kbdev)
+int kbase_device_runtime_init(struct kbase_device *kbdev)
{
pm_suspend_ignore_children(kbdev->dev, true);
pm_runtime_enable(kbdev->dev);
#endif
}
-static kbase_pm_callback_conf pm_callbacks = {
+struct kbase_pm_callback_conf pm_callbacks = {
.power_on_callback = pm_callback_power_on,
.power_off_callback = pm_callback_power_off,
#ifdef CONFIG_PM_RUNTIME
};
#endif
-
-/* Please keep table config_attributes in sync with config_attributes_hw_issue_8408 */
-static kbase_attribute config_attributes[] = {
-#ifdef CONFIG_UMP
- {
- KBASE_CONFIG_ATTR_UMP_DEVICE,
- KBASE_VE_UMP_DEVICE},
-#endif /* CONFIG_UMP */
-#ifdef CONFIG_MALI_MIDGARD_RT_PM
- {
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
- (uintptr_t)&pm_callbacks},
-#endif
- {
- KBASE_CONFIG_ATTR_PLATFORM_FUNCS,
- (uintptr_t) &platform_funcs},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- KBASE_VE_JS_RESET_TIMEOUT_MS},
- {
- KBASE_CONFIG_ATTR_END,
- 0}
-};
-
-static kbase_platform_config rk_platform_config = {
- .attributes = config_attributes,
-#ifndef CONFIG_OF
- .io_resources = &io_resources
-#endif
-};
-#if 1
-kbase_platform_config *kbase_get_platform_config(void)
-{
- return &rk_platform_config;
-}
-#endif
int kbase_platform_early_init(void)
{
/* Nothing needed at this stage */
static DECLARE_WORK(mali_dvfs_work, mali_dvfs_event_proc);
-int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation,
- u32 util_gl_share_no_use,u32 util_cl_share_no_use[2])
+int kbase_platform_dvfs_event(struct kbase_device *kbdev,
+ u32 utilisation,
+ u32 util_gl_share_no_use,
+ u32 util_cl_share_no_use[2] )
{
unsigned long flags;
struct rk_context *platform;
spin_lock_irqsave(&mali_dvfs_spinlock, flags);
if (platform->time_tick < MALI_DVFS_UP_TIME_INTERVAL) {
platform->time_tick++;
- platform->time_busy += kbdev->pm.metrics.time_busy;
- platform->time_idle += kbdev->pm.metrics.time_idle;
+ platform->time_busy += kbdev->pm.backend.metrics.time_busy;
+
+ platform->time_idle += kbdev->pm.backend.metrics.time_idle;
} else {
- platform->time_busy = kbdev->pm.metrics.time_busy;
- platform->time_idle = kbdev->pm.metrics.time_idle;
+ platform->time_busy = kbdev->pm.backend.metrics.time_busy;
+ platform->time_idle = kbdev->pm.backend.metrics.time_idle;
platform->time_tick = 0;
}
int enable;
kbdev = mali_dvfs_status_current.kbdev;
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- enable = kbdev->pm.metrics.timer_active;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+ enable = kbdev->pm.backend.metrics.timer_active;
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
return enable;
}
mutex_lock(&mali_enable_clock_lock);
- if (enable != kbdev->pm.metrics.timer_active) {
+ if (enable != kbdev->pm.backend.metrics.timer_active) {
if (enable) {
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbdev->pm.metrics.timer_active = MALI_TRUE;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
- hrtimer_start(&kbdev->pm.metrics.timer,
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+ kbdev->pm.backend.metrics.timer_active = MALI_TRUE;
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+ hrtimer_start(&kbdev->pm.backend.metrics.timer,
HR_TIMER_DELAY_MSEC(KBASE_PM_DVFS_FREQUENCY),
HRTIMER_MODE_REL);
} else {
- spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
- kbdev->pm.metrics.timer_active = MALI_FALSE;
- spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
- hrtimer_cancel(&kbdev->pm.metrics.timer);
+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags);
+ kbdev->pm.backend.metrics.timer_active = MALI_FALSE;
+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags);
+ hrtimer_cancel(&kbdev->pm.backend.metrics.timer);
}
}
spinlock_t gpu_in_touch_lock;
#endif
};
+
+/*-------------------------------------------------------*/
+
+typedef unsigned long mali_bool;
+
+#ifndef MALI_TRUE
+#define MALI_TRUE (1)
+#endif
+
+#ifndef MALI_FALSE
+#define MALI_FALSE (0)
+#endif
+
+/*-------------------------------------------------------*/
+
+/**
+ * @name Mali error types
+ * @brief The common error type for the mali drivers
+ * The mali_error type, all driver error handling should be of this type unless
+ * it must deal with a specific APIs error type.
+ * @{
+ *
+ * .R : r6p0-02rel0 已经不再 头文件中提供对 mali_error 的定义.
+ */
+typedef enum
+{
+ /**
+ * @brief Common Mali errors for the entire driver
+ * MALI_ERROR_NONE is guaranteed to be 0.
+ * @{
+ */
+ MALI_ERROR_NONE = 0,
+ MALI_ERROR_OUT_OF_GPU_MEMORY,
+ MALI_ERROR_OUT_OF_MEMORY,
+ MALI_ERROR_FUNCTION_FAILED,
+ /* @} */
+ /**
+ * @brief Mali errors for Client APIs to pass to EGL when creating EGLImages
+ * These errors must only be returned to EGL from one of the Client APIs as part of the
+ * (clientapi)_egl_image_interface.h
+ * @{
+ */
+ MALI_ERROR_EGLP_BAD_ACCESS,
+ MALI_ERROR_EGLP_BAD_PARAMETER,
+ /* @} */
+ /**
+ * @brief Mali errors for the MCL module.
+ * These errors must only be used within the private components of the OpenCL implementation that report
+ * directly to API functions for cases where errors cannot be detected in the entrypoints file. They must
+ * not be passed between driver components.
+ * These are errors in the mali error space specifically for the MCL module, hence the MCLP prefix.
+ * @{
+ */
+ MALI_ERROR_MCLP_DEVICE_NOT_FOUND,
+ MALI_ERROR_MCLP_DEVICE_NOT_AVAILABLE,
+ MALI_ERROR_MCLP_COMPILER_NOT_AVAILABLE,
+ MALI_ERROR_MCLP_MEM_OBJECT_ALLOCATION_FAILURE,
+ MALI_ERROR_MCLP_PROFILING_INFO_NOT_AVAILABLE,
+ MALI_ERROR_MCLP_MEM_COPY_OVERLAP,
+ MALI_ERROR_MCLP_IMAGE_FORMAT_MISMATCH,
+ MALI_ERROR_MCLP_IMAGE_FORMAT_NOT_SUPPORTED,
+ MALI_ERROR_MCLP_BUILD_PROGRAM_FAILURE,
+ MALI_ERROR_MCLP_MAP_FAILURE,
+ MALI_ERROR_MCLP_MISALIGNED_SUB_BUFFER_OFFSET,
+ MALI_ERROR_MCLP_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST,
+ MALI_ERROR_MCLP_INVALID_VALUE,
+ MALI_ERROR_MCLP_INVALID_DEVICE_TYPE,
+ MALI_ERROR_MCLP_INVALID_PLATFORM,
+ MALI_ERROR_MCLP_INVALID_DEVICE,
+ MALI_ERROR_MCLP_INVALID_CONTEXT,
+ MALI_ERROR_MCLP_INVALID_QUEUE_PROPERTIES,
+ MALI_ERROR_MCLP_INVALID_COMMAND_QUEUE,
+ MALI_ERROR_MCLP_INVALID_HOST_PTR,
+ MALI_ERROR_MCLP_INVALID_MEM_OBJECT,
+ MALI_ERROR_MCLP_INVALID_IMAGE_FORMAT_DESCRIPTOR,
+ MALI_ERROR_MCLP_INVALID_IMAGE_SIZE,
+ MALI_ERROR_MCLP_INVALID_SAMPLER,
+ MALI_ERROR_MCLP_INVALID_BINARY,
+ MALI_ERROR_MCLP_INVALID_BUILD_OPTIONS,
+ MALI_ERROR_MCLP_INVALID_PROGRAM,
+ MALI_ERROR_MCLP_INVALID_PROGRAM_EXECUTABLE,
+ MALI_ERROR_MCLP_INVALID_KERNEL_NAME,
+ MALI_ERROR_MCLP_INVALID_KERNEL_DEFINITION,
+ MALI_ERROR_MCLP_INVALID_KERNEL,
+ MALI_ERROR_MCLP_INVALID_ARG_INDEX,
+ MALI_ERROR_MCLP_INVALID_ARG_VALUE,
+ MALI_ERROR_MCLP_INVALID_ARG_SIZE,
+ MALI_ERROR_MCLP_INVALID_KERNEL_ARGS,
+ MALI_ERROR_MCLP_INVALID_WORK_DIMENSION,
+ MALI_ERROR_MCLP_INVALID_WORK_GROUP_SIZE,
+ MALI_ERROR_MCLP_INVALID_WORK_ITEM_SIZE,
+ MALI_ERROR_MCLP_INVALID_GLOBAL_OFFSET,
+ MALI_ERROR_MCLP_INVALID_EVENT_WAIT_LIST,
+ MALI_ERROR_MCLP_INVALID_EVENT,
+ MALI_ERROR_MCLP_INVALID_OPERATION,
+ MALI_ERROR_MCLP_INVALID_GL_OBJECT,
+ MALI_ERROR_MCLP_INVALID_BUFFER_SIZE,
+ MALI_ERROR_MCLP_INVALID_MIP_LEVEL,
+ MALI_ERROR_MCLP_INVALID_GLOBAL_WORK_SIZE,
+ MALI_ERROR_MCLP_INVALID_GL_SHAREGROUP_REFERENCE_KHR,
+ MALI_ERROR_MCLP_INVALID_EGL_OBJECT,
+ /* @} */
+ /**
+ * @brief Mali errors for the BASE module
+ * These errors must only be used within the private components of the Base implementation. They will not
+ * passed to other modules by the base driver.
+ * These are errors in the mali error space specifically for the BASE module, hence the BASEP prefix.
+ * @{
+ */
+ MALI_ERROR_BASEP_INVALID_FUNCTION,
+ /* @} */
+ /** A dependency exists upon a resource that the client application wants to modify, so the driver must either
+ * create a copy of the resource (if possible) or block until the dependency has been satisfied.
+ */
+ MALI_ERROR_RESOURCE_IN_USE,
+
+ /**
+ * @brief A stride value was too big.
+ *
+ * A surface descriptor can store strides of up to 2<sup>31</sup>-1 bytes but strides greater than
+ * 2<sup>28</sup>-1 bytes cannot be expressed in bits without overflow.
+ */
+ MALI_ERROR_STRIDE_TOO_BIG
+
+} mali_error;
+
+
int mali_dvfs_clk_set(struct dvfs_node * node,unsigned long rate);
/* All things that are needed for the Linux port. */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
+#include "mali_kbase_cpu_vexpress.h"
+
/**
* Maximum frequency GPU will be clocked at. Given in kHz.
* This must be specified as there is no default value.
* Attached value: number in kHz
* Default value: NA
*/
-#define GPU_FREQ_KHZ_MAX 5000
+#define GPU_FREQ_KHZ_MAX (5000)
/**
* Minimum frequency GPU will be clocked at. Given in kHz.
* This must be specified as there is no default value.
* Attached value: number in kHz
* Default value: NA
*/
-#define GPU_FREQ_KHZ_MIN 5000
+#define GPU_FREQ_KHZ_MIN (5000)
+
+/**
+ * Values used for determining the GPU frequency based on the LogicTile type
+ * Used by the function kbase_get_platform_logic_tile_type
+ */
+#define VE_VIRTEX6_GPU_FREQ_MIN 5000
+#define VE_VIRTEX6_GPU_FREQ_MAX 5000
+#define VE_VIRTEX7_GPU_FREQ_MIN 40000
+#define VE_VIRTEX7_GPU_FREQ_MAX 40000
+
+/**
+ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock
+ *
+ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_cpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed)
+
+/**
+ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock
+ *
+ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_gpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define GPU_SPEED_FUNC (NULL)
+
+/**
+ * Power management configuration
+ *
+ * Attached value: pointer to @ref kbase_pm_callback_conf
+ * Default value: See @ref kbase_pm_callback_conf
+ */
+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks)
+
+/**
+ * Platform specific configuration functions
+ *
+ * Attached value: pointer to @ref kbase_platform_funcs_conf
+ * Default value: See @ref kbase_platform_funcs_conf
+ */
+#define PLATFORM_FUNCS (NULL)
+
+/** Power model for IPA
+ *
+ * Attached value: pointer to @ref mali_pa_model_ops
+ */
+#define POWER_MODEL_CALLBACKS (NULL)
+
+/**
+ * Secure mode switch
+ *
+ * Attached value: pointer to @ref kbase_secure_ops
+ */
+#define SECURE_CALLBACKS (NULL)
+
+extern struct kbase_pm_callback_conf pm_callbacks;
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase_defs.h>
#include <mali_kbase_config.h>
#include "mali_kbase_cpu_vexpress.h"
-
-/* Versatile Express (VE) configuration defaults shared between config_attributes[]
- * and config_attributes_hw_issue_8408[]. Settings are not shared for
- * JS_HARD_STOP_TICKS_SS and JS_RESET_TICKS_SS.
- */
-#define KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG 15000000u /* 15ms, an agressive tick for testing purposes. This will reduce performance significantly */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG 1 /* between 15ms and 30ms before soft-stop a job */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_CL_DEBUG 1 /* between 15ms and 30ms before soft-stop a CL job */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_DEBUG 333 /* 5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_8401_DEBUG 2000 /* 30s before hard-stop, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) - for issue 8401 */
-#define KBASE_VE_JS_HARD_STOP_TICKS_CL_DEBUG 166 /* 2.5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG 100000 /* 1500s (25mins) before NSS hard-stop */
-#define KBASE_VE_JS_RESET_TICKS_SS_DEBUG 500 /* 45s before resetting GPU, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) */
-#define KBASE_VE_JS_RESET_TICKS_SS_8401_DEBUG 3000 /* 7.5s before resetting GPU - for issue 8401 */
-#define KBASE_VE_JS_RESET_TICKS_CL_DEBUG 500 /* 45s before resetting GPU */
-#define KBASE_VE_JS_RESET_TICKS_NSS_DEBUG 100166 /* 1502s before resetting GPU */
-
-#define KBASE_VE_JS_SCHEDULING_TICK_NS 1250000000u /* 1.25s */
-#define KBASE_VE_JS_SOFT_STOP_TICKS 2 /* 2.5s before soft-stop a job */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_CL 1 /* 1.25s before soft-stop a CL job */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS 4 /* 5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_8401 24 /* 30s before hard-stop, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) - for issue 8401 */
-#define KBASE_VE_JS_HARD_STOP_TICKS_CL 2 /* 2.5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_NSS 1200 /* 1500s before NSS hard-stop */
-#define KBASE_VE_JS_RESET_TICKS_SS 6 /* 7.5s before resetting GPU */
-#define KBASE_VE_JS_RESET_TICKS_SS_8401 36 /* 45s before resetting GPU, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) - for issue 8401 */
-#define KBASE_VE_JS_RESET_TICKS_CL 3 /* 7.5s before resetting GPU */
-#define KBASE_VE_JS_RESET_TICKS_NSS 1201 /* 1502s before resetting GPU */
-
-#define KBASE_VE_JS_RESET_TIMEOUT_MS 3000 /* 3s before cancelling stuck jobs */
-#define KBASE_VE_JS_CTX_TIMESLICE_NS 1000000 /* 1ms - an agressive timeslice for testing purposes (causes lots of scheduling out for >4 ctxs) */
-#define KBASE_VE_POWER_MANAGEMENT_CALLBACKS ((uintptr_t)&pm_callbacks)
-#define KBASE_VE_CPU_SPEED_FUNC ((uintptr_t)&kbase_get_vexpress_cpu_clock_speed)
+#include "mali_kbase_config_platform.h"
#define HARD_RESET_AT_POWER_OFF 0
#ifndef CONFIG_OF
-static kbase_io_resources io_resources = {
+static struct kbase_io_resources io_resources = {
.job_irq_number = 68,
.mmu_irq_number = 69,
.gpu_irq_number = 70,
#endif
}
-static struct kbase_pm_callback_conf pm_callbacks = {
+struct kbase_pm_callback_conf pm_callbacks = {
.power_on_callback = pm_callback_power_on,
.power_off_callback = pm_callback_power_off,
.power_suspend_callback = NULL,
.power_resume_callback = NULL
};
-/* Please keep table config_attributes in sync with config_attributes_hw_issue_8408 */
-static struct kbase_attribute config_attributes[] = {
-#ifdef CONFIG_MALI_DEBUG
-/* Use more aggressive scheduling timeouts in debug builds for testing purposes */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL,
- KBASE_VE_JS_SOFT_STOP_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL,
- KBASE_VE_JS_HARD_STOP_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL,
- KBASE_VE_JS_RESET_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS_DEBUG},
-#else /* CONFIG_MALI_DEBUG */
-/* In release builds same as the defaults but scaled for 5MHz FPGA */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL,
- KBASE_VE_JS_SOFT_STOP_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL,
- KBASE_VE_JS_HARD_STOP_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL,
- KBASE_VE_JS_RESET_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS},
-#endif /* CONFIG_MALI_DEBUG */
- {
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- KBASE_VE_JS_RESET_TIMEOUT_MS},
-
- {
- KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS,
- KBASE_VE_JS_CTX_TIMESLICE_NS},
-
- {
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
- KBASE_VE_POWER_MANAGEMENT_CALLBACKS},
-
- {
- KBASE_CONFIG_ATTR_CPU_SPEED_FUNC,
- KBASE_VE_CPU_SPEED_FUNC},
-
- {
- KBASE_CONFIG_ATTR_END,
- 0}
-};
-
-/* as config_attributes array above except with different settings for
- * JS_HARD_STOP_TICKS_SS, JS_RESET_TICKS_SS that
- * are needed for BASE_HW_ISSUE_8408.
- */
-struct kbase_attribute config_attributes_hw_issue_8408[] = {
-#ifdef CONFIG_MALI_DEBUG
-/* Use more aggressive scheduling timeouts in debug builds for testing purposes */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_8401_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_8401_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS_DEBUG},
-#else /* CONFIG_MALI_DEBUG */
-/* In release builds same as the defaults but scaled for 5MHz FPGA */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_8401},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_8401},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS},
-#endif /* CONFIG_MALI_DEBUG */
- {
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- KBASE_VE_JS_RESET_TIMEOUT_MS},
-
- {
- KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS,
- KBASE_VE_JS_CTX_TIMESLICE_NS},
-
- {
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
- KBASE_VE_POWER_MANAGEMENT_CALLBACKS},
-
- {
- KBASE_CONFIG_ATTR_CPU_SPEED_FUNC,
- KBASE_VE_CPU_SPEED_FUNC},
-
- {
- KBASE_CONFIG_ATTR_END,
- 0}
-};
-
static struct kbase_platform_config versatile_platform_config = {
- .attributes = config_attributes,
#ifndef CONFIG_OF
.io_resources = &io_resources
#endif
return &versatile_platform_config;
}
+
int kbase_platform_early_init(void)
{
/* Nothing needed at this stage */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
-
-
#include <linux/io.h>
#include <mali_kbase.h>
#include "mali_kbase_cpu_vexpress.h"
-#define HZ_IN_MHZ (1000000)
+#define HZ_IN_MHZ (1000000)
#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000)
-#define MOTHERBOARD_SYS_CFG_START (0x10000000)
-#define SYS_CFGDATA_OFFSET (0x000000A0)
-#define SYS_CFGCTRL_OFFSET (0x000000A4)
-#define SYS_CFGSTAT_OFFSET (0x000000A8)
-
-#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31)
-#define READ_REG_BIT_VALUE (0 << 30)
-#define DCC_DEFAULT_BIT_VALUE (0 << 26)
-#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20)
-#define SITE_DEFAULT_BIT_VALUE (1 << 16)
-#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12)
-#define DEVICE_DEFAULT_BIT_VALUE (2 << 0)
-#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0)
-#define SYS_CFG_ERROR_BIT_VALUE (1 << 1)
-
-#define FEED_REG_BIT_MASK (0x0F)
+#define MOTHERBOARD_SYS_CFG_START (0x10000000)
+#define SYS_CFGDATA_OFFSET (0x000000A0)
+#define SYS_CFGCTRL_OFFSET (0x000000A4)
+#define SYS_CFGSTAT_OFFSET (0x000000A8)
+
+#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31)
+#define READ_REG_BIT_VALUE (0 << 30)
+#define DCC_DEFAULT_BIT_VALUE (0 << 26)
+#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20)
+#define SITE_DEFAULT_BIT_VALUE (1 << 16)
+#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12)
+#define DEVICE_DEFAULT_BIT_VALUE (2 << 0)
+#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0)
+#define SYS_CFG_ERROR_BIT_VALUE (1 << 1)
+
+#define FEED_REG_BIT_MASK (0x0F)
#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03)
#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07)
#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B)
#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F)
#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13)
-#define IS_SINGLE_BIT_SET(val, pos) (val&(1<<pos))
+/* the following three values used for reading
+ * HBI value of the LogicTile daughterboard */
+#define VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 (0x10000000)
+#define VE_SYS_PROC_ID1_OFFSET (0x00000088)
+#define VE_LOGIC_TILE_HBI_MASK (0x00000FFF)
-#define CPU_CLOCK_SPEED_UNDEFINED 0
+#define IS_SINGLE_BIT_SET(val, pos) (val&(1<<pos))
+
+#define CPU_CLOCK_SPEED_UNDEFINED (0)
static u32 cpu_clock_speed = CPU_CLOCK_SPEED_UNDEFINED;
static DEFINE_RAW_SPINLOCK(syscfg_lock);
/**
- * kbase_get_vendor_specific_cpu_clock_speed
- * @brief Retrieves the CPU clock speed.
- * The implementation is platform specific.
- * @param[out] cpu_clock - the value of CPU clock speed in MHz
- * @return 0 on success, 1 otherwise
+ * kbase_get_vendor_specific_cpu_clock_speed -Retrieves the CPU clock speed
+ * @cpu_clock - the value of CPU clock speed in MHz
+ *
+ * Returns 0 on success, error code otherwise.
+ *
+ * The implementation is platform specific.
*/
int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock)
{
-
-
- if (CPU_CLOCK_SPEED_UNDEFINED != cpu_clock_speed)
- {
+ int err = 0;
+ u32 reg_val = 0;
+ u32 osc2_value = 0;
+ u32 pa_divide = 0;
+ u32 pb_divide = 0;
+ u32 pc_divide = 0;
+ void __iomem *syscfg_reg = NULL;
+ void __iomem *scc_reg = NULL;
+
+ if (CPU_CLOCK_SPEED_UNDEFINED != cpu_clock_speed) {
*cpu_clock = cpu_clock_speed;
return 0;
}
- else
- {
- int result = 0;
- u32 reg_val = 0;
- u32 osc2_value = 0;
- u32 pa_divide = 0;
- u32 pb_divide = 0;
- u32 pc_divide = 0;
- void __iomem *pSysCfgReg = NULL;
- void __iomem *pSCCReg = NULL;
-
- /* Init the value case something goes wrong */
- *cpu_clock = 0;
-
- /* Map CPU register into virtual memory */
- pSysCfgReg = ioremap(MOTHERBOARD_SYS_CFG_START, 0x1000);
- if (pSysCfgReg == NULL) {
- result = 1;
-
- goto pSysCfgReg_map_failed;
- }
-
- pSCCReg = ioremap(CORETILE_EXPRESS_A9X4_SCC_START, 0x1000);
- if (pSCCReg == NULL) {
- result = 1;
-
- goto pSCCReg_map_failed;
- }
-
- raw_spin_lock(&syscfg_lock);
-
- /*Read SYS regs - OSC2 */
- reg_val = readl(pSysCfgReg + SYS_CFGCTRL_OFFSET);
-
- /*Verify if there is no other undergoing request */
- if (!(reg_val & SYS_CFGCTRL_START_BIT_VALUE)) {
- /*Reset the CGFGSTAT reg */
- writel(0, (pSysCfgReg + SYS_CFGSTAT_OFFSET));
-
- writel(SYS_CFGCTRL_START_BIT_VALUE | READ_REG_BIT_VALUE | DCC_DEFAULT_BIT_VALUE | SYS_CFG_OSC_FUNC_BIT_VALUE | SITE_DEFAULT_BIT_VALUE | BOARD_STACK_POS_DEFAULT_BIT_VALUE | DEVICE_DEFAULT_BIT_VALUE, (pSysCfgReg + SYS_CFGCTRL_OFFSET));
- /* Wait for the transaction to complete */
- while (!(readl(pSysCfgReg + SYS_CFGSTAT_OFFSET) & SYS_CFG_COMPLETE_BIT_VALUE))
- ;
- /* Read SYS_CFGSTAT Register to get the status of submitted transaction */
- reg_val = readl(pSysCfgReg + SYS_CFGSTAT_OFFSET);
-
- /*------------------------------------------------------------------------------------------*/
- /* Check for possible errors */
- if (reg_val & SYS_CFG_ERROR_BIT_VALUE) {
- /* Error while setting register */
- result = 1;
- } else {
- osc2_value = readl(pSysCfgReg + SYS_CFGDATA_OFFSET);
- /* Read the SCC CFGRW0 register */
- reg_val = readl(pSCCReg);
-
- /*
- Select the appropriate feed:
- CFGRW0[0] - CLKOB
- CFGRW0[1] - CLKOC
- CFGRW0[2] - FACLK (CLK)B FROM AXICLK PLL)
- */
- /* Calculate the FCLK */
- if (IS_SINGLE_BIT_SET(reg_val, 0)) { /*CFGRW0[0] - CLKOB */
- /* CFGRW0[6:3] */
- pa_divide = ((reg_val & (FEED_REG_BIT_MASK << FCLK_PA_DIVIDE_BIT_SHIFT)) >> FCLK_PA_DIVIDE_BIT_SHIFT);
- /* CFGRW0[10:7] */
- pb_divide = ((reg_val & (FEED_REG_BIT_MASK << FCLK_PB_DIVIDE_BIT_SHIFT)) >> FCLK_PB_DIVIDE_BIT_SHIFT);
- *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1);
- } else {
- if (IS_SINGLE_BIT_SET(reg_val, 1)) { /*CFGRW0[1] - CLKOC */
- /* CFGRW0[6:3] */
- pa_divide = ((reg_val & (FEED_REG_BIT_MASK << FCLK_PA_DIVIDE_BIT_SHIFT)) >> FCLK_PA_DIVIDE_BIT_SHIFT);
- /* CFGRW0[14:11] */
- pc_divide = ((reg_val & (FEED_REG_BIT_MASK << FCLK_PC_DIVIDE_BIT_SHIFT)) >> FCLK_PC_DIVIDE_BIT_SHIFT);
- *cpu_clock = osc2_value * (pa_divide + 1) / (pc_divide + 1);
- } else if (IS_SINGLE_BIT_SET(reg_val, 2)) { /*CFGRW0[2] - FACLK */
- /* CFGRW0[18:15] */
- pa_divide = ((reg_val & (FEED_REG_BIT_MASK << AXICLK_PA_DIVIDE_BIT_SHIFT)) >> AXICLK_PA_DIVIDE_BIT_SHIFT);
- /* CFGRW0[22:19] */
- pb_divide = ((reg_val & (FEED_REG_BIT_MASK << AXICLK_PB_DIVIDE_BIT_SHIFT)) >> AXICLK_PB_DIVIDE_BIT_SHIFT);
- *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1);
- } else {
- result = 1;
- }
- }
- }
- } else {
- result = 1;
- }
- raw_spin_unlock(&syscfg_lock);
- /* Convert result expressed in Hz to Mhz units. */
- *cpu_clock /= HZ_IN_MHZ;
- if (!result)
- {
- cpu_clock_speed = *cpu_clock;
- }
-
- /* Unmap memory */
- iounmap(pSCCReg);
-
- pSCCReg_map_failed:
- iounmap(pSysCfgReg);
-
- pSysCfgReg_map_failed:
-
- return result;
+
+ /* Init the value in case something goes wrong */
+ *cpu_clock = 0;
+
+ /* Map CPU register into virtual memory */
+ syscfg_reg = ioremap(MOTHERBOARD_SYS_CFG_START, 0x1000);
+ if (syscfg_reg == NULL) {
+ err = -EIO;
+ goto syscfg_reg_map_failed;
+ }
+
+ scc_reg = ioremap(CORETILE_EXPRESS_A9X4_SCC_START, 0x1000);
+ if (scc_reg == NULL) {
+ err = -EIO;
+ goto scc_reg_map_failed;
+ }
+
+ raw_spin_lock(&syscfg_lock);
+
+ /* Read SYS regs - OSC2 */
+ reg_val = readl(syscfg_reg + SYS_CFGCTRL_OFFSET);
+
+ /* Check if there is any other undergoing request */
+ if (reg_val & SYS_CFGCTRL_START_BIT_VALUE) {
+ err = -EBUSY;
+ goto ongoing_request;
+ }
+ /* Reset the CGFGSTAT reg */
+ writel(0, (syscfg_reg + SYS_CFGSTAT_OFFSET));
+
+ writel(SYS_CFGCTRL_START_BIT_VALUE | READ_REG_BIT_VALUE |
+ DCC_DEFAULT_BIT_VALUE |
+ SYS_CFG_OSC_FUNC_BIT_VALUE |
+ SITE_DEFAULT_BIT_VALUE |
+ BOARD_STACK_POS_DEFAULT_BIT_VALUE |
+ DEVICE_DEFAULT_BIT_VALUE,
+ (syscfg_reg + SYS_CFGCTRL_OFFSET));
+ /* Wait for the transaction to complete */
+ while (!(readl(syscfg_reg + SYS_CFGSTAT_OFFSET) &
+ SYS_CFG_COMPLETE_BIT_VALUE))
+ ;
+ /* Read SYS_CFGSTAT Register to get the status of submitted
+ * transaction */
+ reg_val = readl(syscfg_reg + SYS_CFGSTAT_OFFSET);
+
+ if (reg_val & SYS_CFG_ERROR_BIT_VALUE) {
+ /* Error while setting register */
+ err = -EIO;
+ goto set_reg_error;
+ }
+
+ osc2_value = readl(syscfg_reg + SYS_CFGDATA_OFFSET);
+ /* Read the SCC CFGRW0 register */
+ reg_val = readl(scc_reg);
+
+ /*
+ * Select the appropriate feed:
+ * CFGRW0[0] - CLKOB
+ * CFGRW0[1] - CLKOC
+ * CFGRW0[2] - FACLK (CLK)B FROM AXICLK PLL)
+ */
+ /* Calculate the FCLK */
+ if (IS_SINGLE_BIT_SET(reg_val, 0)) {
+ /* CFGRW0[0] - CLKOB */
+ /* CFGRW0[6:3] */
+ pa_divide = ((reg_val & (FEED_REG_BIT_MASK <<
+ FCLK_PA_DIVIDE_BIT_SHIFT)) >>
+ FCLK_PA_DIVIDE_BIT_SHIFT);
+ /* CFGRW0[10:7] */
+ pb_divide = ((reg_val & (FEED_REG_BIT_MASK <<
+ FCLK_PB_DIVIDE_BIT_SHIFT)) >>
+ FCLK_PB_DIVIDE_BIT_SHIFT);
+ *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1);
+ } else if (IS_SINGLE_BIT_SET(reg_val, 1)) {
+ /* CFGRW0[1] - CLKOC */
+ /* CFGRW0[6:3] */
+ pa_divide = ((reg_val & (FEED_REG_BIT_MASK <<
+ FCLK_PA_DIVIDE_BIT_SHIFT)) >>
+ FCLK_PA_DIVIDE_BIT_SHIFT);
+ /* CFGRW0[14:11] */
+ pc_divide = ((reg_val & (FEED_REG_BIT_MASK <<
+ FCLK_PC_DIVIDE_BIT_SHIFT)) >>
+ FCLK_PC_DIVIDE_BIT_SHIFT);
+ *cpu_clock = osc2_value * (pa_divide + 1) / (pc_divide + 1);
+ } else if (IS_SINGLE_BIT_SET(reg_val, 2)) {
+ /* CFGRW0[2] - FACLK */
+ /* CFGRW0[18:15] */
+ pa_divide = ((reg_val & (FEED_REG_BIT_MASK <<
+ AXICLK_PA_DIVIDE_BIT_SHIFT)) >>
+ AXICLK_PA_DIVIDE_BIT_SHIFT);
+ /* CFGRW0[22:19] */
+ pb_divide = ((reg_val & (FEED_REG_BIT_MASK <<
+ AXICLK_PB_DIVIDE_BIT_SHIFT)) >>
+ AXICLK_PB_DIVIDE_BIT_SHIFT);
+ *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1);
+ } else {
+ err = -EIO;
}
+
+set_reg_error:
+ongoing_request:
+ raw_spin_unlock(&syscfg_lock);
+ *cpu_clock /= HZ_IN_MHZ;
+
+ if (!err)
+ cpu_clock_speed = *cpu_clock;
+
+ iounmap(scc_reg);
+
+scc_reg_map_failed:
+ iounmap(syscfg_reg);
+
+syscfg_reg_map_failed:
+
+ return err;
+}
+
+u32 kbase_get_platform_logic_tile_type(void)
+{
+ void __iomem *syscfg_reg = NULL;
+ u32 sys_procid1 = 0;
+
+ syscfg_reg = ioremap(VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 + VE_SYS_PROC_ID1_OFFSET, 4);
+
+ sys_procid1 = (NULL != syscfg_reg) ? readl(syscfg_reg) : 0;
+
+ return sys_procid1 & VE_LOGIC_TILE_HBI_MASK;
}
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define _KBASE_CPU_VEXPRESS_H_
/**
- * Versatile Express implementation of @ref kbase_cpuprops_clock_speed_function.
+ * Versatile Express implementation of @ref kbase_cpu_clk_speed_func.
*/
int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock);
+/**
+ * kbase_get_platform_logic_tile_type - determines which LogicTile type
+ * is used by Versatile Express
+ *
+ * When platform_config build parameter is specified as vexpress, i.e.,
+ * platform_config=vexpress, GPU frequency may vary dependent on the
+ * particular platform. The GPU frequency depends on the LogicTile type.
+ *
+ * This function is called by kbase_common_device_init to determine
+ * which LogicTile type is used by the platform by reading the HBI value
+ * of the daughterboard which holds the LogicTile:
+ *
+ * 0x192 HBI0192 Virtex-5
+ * 0x217 HBI0217 Virtex-6
+ * 0x247 HBI0247 Virtex-7
+ *
+ * Return: HBI value of the logic tile daughterboard, zero if not accessible
+ */
+u32 kbase_get_platform_logic_tile_type(void);
+
#endif /* _KBASE_CPU_VEXPRESS_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Default value: NA
*/
#define GPU_FREQ_KHZ_MIN 5000
+
+/**
+ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock
+ *
+ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_cpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define CPU_SPEED_FUNC (&kbase_cpuprops_get_default_clock_speed)
+
+/**
+ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock
+ *
+ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_gpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define GPU_SPEED_FUNC (NULL)
+
+/**
+ * Power management configuration
+ *
+ * Attached value: pointer to @ref kbase_pm_callback_conf
+ * Default value: See @ref kbase_pm_callback_conf
+ */
+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks)
+
+/**
+ * Platform specific configuration functions
+ *
+ * Attached value: pointer to @ref kbase_platform_funcs_conf
+ * Default value: See @ref kbase_platform_funcs_conf
+ */
+#define PLATFORM_FUNCS (NULL)
+
+/** Power model for IPA
+ *
+ * Attached value: pointer to @ref mali_pa_model_ops
+ */
+#define POWER_MODEL_CALLBACKS (NULL)
+
+/**
+ * Secure mode switch
+ *
+ * Attached value: pointer to @ref kbase_secure_ops
+ */
+#define SECURE_CALLBACKS (NULL)
+
+extern struct kbase_pm_callback_conf pm_callbacks;
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase_defs.h>
#include <mali_kbase_config.h>
-/* Versatile Express (VE) configuration defaults shared between config_attributes[]
- * and config_attributes_hw_issue_8408[]. Settings are not shared for
- * JS_HARD_STOP_TICKS_SS and JS_RESET_TICKS_SS.
- */
-#define KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG 3000000000u /* 3s */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG 10 /* 30s */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_CL_DEBUG 5 /* 15s */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_DEBUG 20 /* 60s */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_8401_DEBUG 120 /* 360s */
-#define KBASE_VE_JS_HARD_STOP_TICKS_CL_DEBUG 10 /* 30s */
-#define KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG 6000 /* 18000s */
-#define KBASE_VE_JS_RESET_TICKS_SS_DEBUG 30 /* 180s */
-#define KBASE_VE_JS_RESET_TICKS_SS_8401_DEBUG 180 /* 540s */
-#define KBASE_VE_JS_RESET_TICKS_CL_DEBUG 30 /* 180s */
-#define KBASE_VE_JS_RESET_TICKS_NSS_DEBUG 6010 /* 18030s*/
-
-#define KBASE_VE_JS_SCHEDULING_TICK_NS 3000000000u /* 3s */
-#define KBASE_VE_JS_SOFT_STOP_TICKS 10 /* 30s */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_CL 5 /* 15s */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS 20 /* 60s */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_8401 120 /* 360s */
-#define KBASE_VE_JS_HARD_STOP_TICKS_CL 10 /* 30s */
-#define KBASE_VE_JS_HARD_STOP_TICKS_NSS 6000 /* 18000s */
-#define KBASE_VE_JS_RESET_TICKS_SS 30 /* 180s */
-#define KBASE_VE_JS_RESET_TICKS_SS_8401 180 /* 540s */
-#define KBASE_VE_JS_RESET_TICKS_CL 30 /* 180s */
-#define KBASE_VE_JS_RESET_TICKS_NSS 6010 /* 18030s*/
-
-#define KBASE_VE_JS_RESET_TIMEOUT_MS 3000 /* 3s before cancelling stuck jobs */
-#define KBASE_VE_JS_CTX_TIMESLICE_NS 1000000 /* 1ms - an agressive timeslice for testing purposes (causes lots of scheduling out for >4 ctxs) */
-#define KBASE_VE_POWER_MANAGEMENT_CALLBACKS ((uintptr_t)&pm_callbacks)
-
#define HARD_RESET_AT_POWER_OFF 0
#ifndef CONFIG_OF
#endif
}
-static struct kbase_pm_callback_conf pm_callbacks = {
+struct kbase_pm_callback_conf pm_callbacks = {
.power_on_callback = pm_callback_power_on,
.power_off_callback = pm_callback_power_off,
.power_suspend_callback = NULL,
.power_resume_callback = NULL
};
-/* Please keep table config_attributes in sync with config_attributes_hw_issue_8408 */
-static struct kbase_attribute config_attributes[] = {
-#ifdef CONFIG_MALI_DEBUG
-/* Use more aggressive scheduling timeouts in debug builds for testing purposes */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL,
- KBASE_VE_JS_SOFT_STOP_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL,
- KBASE_VE_JS_HARD_STOP_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL,
- KBASE_VE_JS_RESET_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS_DEBUG},
-#else /* CONFIG_MALI_DEBUG */
-/* In release builds same as the defaults but scaled for 5MHz FPGA */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL,
- KBASE_VE_JS_SOFT_STOP_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL,
- KBASE_VE_JS_HARD_STOP_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL,
- KBASE_VE_JS_RESET_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS},
-#endif /* CONFIG_MALI_DEBUG */
- {
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- KBASE_VE_JS_RESET_TIMEOUT_MS},
-
- {
- KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS,
- KBASE_VE_JS_CTX_TIMESLICE_NS},
-
- {
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
- KBASE_VE_POWER_MANAGEMENT_CALLBACKS},
-
- {
- KBASE_CONFIG_ATTR_END,
- 0}
-};
-
-/* as config_attributes array above except with different settings for
- * JS_HARD_STOP_TICKS_SS, JS_RESET_TICKS_SS that
- * are needed for BASE_HW_ISSUE_8408.
- */
-struct kbase_attribute config_attributes_hw_issue_8408[] = {
-#ifdef CONFIG_MALI_DEBUG
-/* Use more aggressive scheduling timeouts in debug builds for testing purposes */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_8401_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_8401_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS_DEBUG},
-#else /* CONFIG_MALI_DEBUG */
-/* In release builds same as the defaults but scaled for 5MHz FPGA */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_8401},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_8401},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS},
-#endif /* CONFIG_MALI_DEBUG */
- {
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- KBASE_VE_JS_RESET_TIMEOUT_MS},
-
- {
- KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS,
- KBASE_VE_JS_CTX_TIMESLICE_NS},
-
- {
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
- KBASE_VE_POWER_MANAGEMENT_CALLBACKS},
-
- {
- KBASE_CONFIG_ATTR_END,
- 0}
-};
-
static struct kbase_platform_config versatile_platform_config = {
- .attributes = config_attributes,
#ifndef CONFIG_OF
.io_resources = &io_resources
#endif
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
+#include "mali_kbase_cpu_vexpress.h"
+
/**
* Maximum frequency GPU will be clocked at. Given in kHz.
* This must be specified as there is no default value.
* Default value: NA
*/
#define GPU_FREQ_KHZ_MIN 10000
+
+/**
+ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock
+ *
+ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_cpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed)
+
+/**
+ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock
+ *
+ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func
+ * for the function prototype.
+ *
+ * Attached value: A kbase_gpu_clk_speed_func.
+ * Default Value: NA
+ */
+#define GPU_SPEED_FUNC (NULL)
+
+/**
+ * Power management configuration
+ *
+ * Attached value: pointer to @ref kbase_pm_callback_conf
+ * Default value: See @ref kbase_pm_callback_conf
+ */
+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks)
+
+/**
+ * Platform specific configuration functions
+ *
+ * Attached value: pointer to @ref kbase_platform_funcs_conf
+ * Default value: See @ref kbase_platform_funcs_conf
+ */
+#define PLATFORM_FUNCS (NULL)
+
+/** Power model for IPA
+ *
+ * Attached value: pointer to @ref mali_pa_model_ops
+ */
+#define POWER_MODEL_CALLBACKS (NULL)
+
+/**
+ * Secure mode switch
+ *
+ * Attached value: pointer to @ref kbase_secure_ops
+ */
+#define SECURE_CALLBACKS (NULL)
+
+extern struct kbase_pm_callback_conf pm_callbacks;
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#include <mali_kbase_config.h>
#include "mali_kbase_cpu_vexpress.h"
-/* Versatile Express (VE) configuration defaults shared between config_attributes[]
- * and config_attributes_hw_issue_8408[]. Settings are not shared for
- * JS_HARD_STOP_TICKS_SS and JS_RESET_TICKS_SS.
- */
-#define KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG 15000000u /* 15ms, an agressive tick for testing purposes. This will reduce performance significantly */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG 1 /* between 15ms and 30ms before soft-stop a job */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_CL_DEBUG 1 /* between 15ms and 30ms before soft-stop a CL job */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_DEBUG 333 /* 5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_8401_DEBUG 2000 /* 30s before hard-stop, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) - for issue 8401 */
-#define KBASE_VE_JS_HARD_STOP_TICKS_CL_DEBUG 166 /* 2.5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG 100000 /* 1500s (25mins) before NSS hard-stop */
-#define KBASE_VE_JS_RESET_TICKS_SS_DEBUG 500 /* 45s before resetting GPU, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) */
-#define KBASE_VE_JS_RESET_TICKS_SS_8401_DEBUG 3000 /* 7.5s before resetting GPU - for issue 8401 */
-#define KBASE_VE_JS_RESET_TICKS_CL_DEBUG 500 /* 45s before resetting GPU */
-#define KBASE_VE_JS_RESET_TICKS_NSS_DEBUG 100166 /* 1502s before resetting GPU */
-
-#define KBASE_VE_JS_SCHEDULING_TICK_NS 1250000000u /* 1.25s */
-#define KBASE_VE_JS_SOFT_STOP_TICKS 2 /* 2.5s before soft-stop a job */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_CL 1 /* 1.25s before soft-stop a CL job */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS 4 /* 5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_8401 24 /* 30s before hard-stop, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) - for issue 8401 */
-#define KBASE_VE_JS_HARD_STOP_TICKS_CL 2 /* 2.5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_NSS 1200 /* 1500s before NSS hard-stop */
-#define KBASE_VE_JS_RESET_TICKS_SS 6 /* 7.5s before resetting GPU */
-#define KBASE_VE_JS_RESET_TICKS_SS_8401 36 /* 45s before resetting GPU, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) - for issue 8401 */
-#define KBASE_VE_JS_RESET_TICKS_CL 3 /* 3.75s before resetting GPU */
-#define KBASE_VE_JS_RESET_TICKS_NSS 1201 /* 1502s before resetting GPU */
-
-#define KBASE_VE_JS_RESET_TIMEOUT_MS 3000 /* 3s before cancelling stuck jobs */
-#define KBASE_VE_JS_CTX_TIMESLICE_NS 1000000 /* 1ms - an agressive timeslice for testing purposes (causes lots of scheduling out for >4 ctxs) */
-#define KBASE_VE_POWER_MANAGEMENT_CALLBACKS ((uintptr_t)&pm_callbacks)
-#define KBASE_VE_CPU_SPEED_FUNC ((uintptr_t)&kbase_get_vexpress_cpu_clock_speed)
-
#define HARD_RESET_AT_POWER_OFF 0
#ifndef CONFIG_OF
#endif
}
-static struct kbase_pm_callback_conf pm_callbacks = {
+struct kbase_pm_callback_conf pm_callbacks = {
.power_on_callback = pm_callback_power_on,
.power_off_callback = pm_callback_power_off,
.power_suspend_callback = NULL,
.power_resume_callback = NULL
};
-/* Please keep table config_attributes in sync with config_attributes_hw_issue_8408 */
-static struct kbase_attribute config_attributes[] = {
-#ifdef CONFIG_MALI_DEBUG
-/* Use more aggressive scheduling timeouts in debug builds for testing purposes */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL,
- KBASE_VE_JS_SOFT_STOP_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL,
- KBASE_VE_JS_HARD_STOP_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL,
- KBASE_VE_JS_RESET_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS_DEBUG},
-#else /* CONFIG_MALI_DEBUG */
-/* In release builds same as the defaults but scaled for 5MHz FPGA */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL,
- KBASE_VE_JS_SOFT_STOP_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL,
- KBASE_VE_JS_HARD_STOP_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL,
- KBASE_VE_JS_RESET_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS},
-#endif /* CONFIG_MALI_DEBUG */
- {
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- KBASE_VE_JS_RESET_TIMEOUT_MS},
-
- {
- KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS,
- KBASE_VE_JS_CTX_TIMESLICE_NS},
-
- {
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
- KBASE_VE_POWER_MANAGEMENT_CALLBACKS},
-
- {
- KBASE_CONFIG_ATTR_CPU_SPEED_FUNC,
- KBASE_VE_CPU_SPEED_FUNC},
-
- {
- KBASE_CONFIG_ATTR_END,
- 0}
-};
-
-/* as config_attributes array above except with different settings for
- * JS_HARD_STOP_TICKS_SS, JS_RESET_TICKS_SS that
- * are needed for BASE_HW_ISSUE_8408.
- */
-struct kbase_attribute config_attributes_hw_issue_8408[] = {
-#ifdef CONFIG_MALI_DEBUG
-/* Use more aggressive scheduling timeouts in debug builds for testing purposes */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_8401_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_8401_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS_DEBUG},
-#else /* CONFIG_MALI_DEBUG */
-/* In release builds same as the defaults but scaled for 5MHz FPGA */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_8401},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_8401},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS},
-#endif /* CONFIG_MALI_DEBUG */
- {
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- KBASE_VE_JS_RESET_TIMEOUT_MS},
-
- {
- KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS,
- KBASE_VE_JS_CTX_TIMESLICE_NS},
-
- {
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
- KBASE_VE_POWER_MANAGEMENT_CALLBACKS},
-
- {
- KBASE_CONFIG_ATTR_CPU_SPEED_FUNC,
- KBASE_VE_CPU_SPEED_FUNC},
-
- {
- KBASE_CONFIG_ATTR_END,
- 0}
-};
-
static struct kbase_platform_config versatile_platform_config = {
- .attributes = config_attributes,
#ifndef CONFIG_OF
.io_resources = &io_resources
#endif
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
#define _KBASE_CPU_VEXPRESS_H_
/**
- * Versatile Express implementation of @ref kbase_cpuprops_clock_speed_function.
+ * Versatile Express implementation of @ref kbase_cpu_clk_speed_func.
*/
int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock);
+++ /dev/null
-#
-# (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
-#
-# This program is free software and is provided to you under the terms of the
-# GNU General Public License version 2 as published by the Free Software
-# Foundation, and any use by you of this program is subject to the terms
-# of such GNU licence.
-#
-# A copy of the licence is included with the program, and can also be obtained
-# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-# Boston, MA 02110-1301, USA.
-#
-#
-
-
-obj-y += mali_kbase_config_vexpress.o
-obj-y += mali_kbase_cpu_vexpress.o
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-/**
- * Maximum frequency GPU will be clocked at. Given in kHz.
- * This must be specified as there is no default value.
- *
- * Attached value: number in kHz
- * Default value: NA
- */
-#define GPU_FREQ_KHZ_MAX 40000
-/**
- * Minimum frequency GPU will be clocked at. Given in kHz.
- * This must be specified as there is no default value.
- *
- * Attached value: number in kHz
- * Default value: NA
- */
-#define GPU_FREQ_KHZ_MIN 40000
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-#include <linux/ioport.h>
-#include <mali_kbase.h>
-#include <mali_kbase_defs.h>
-#include <mali_kbase_config.h>
-
-#include "mali_kbase_cpu_vexpress.h"
-
-/* Versatile Express (VE) configuration defaults shared between config_attributes[]
- * and config_attributes_hw_issue_8408[]. Settings are not shared for
- * JS_HARD_STOP_TICKS_SS and JS_RESET_TICKS_SS.
- */
-#define KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG 15000000u /* 15ms, an agressive tick for testing purposes. This will reduce performance significantly */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG 1 /* between 15ms and 30ms before soft-stop a job */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_CL_DEBUG 1 /* between 15ms and 30ms before soft-stop a CL job */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_DEBUG 333 /* 5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_8401_DEBUG 2000 /* 30s before hard-stop, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) - for issue 8401 */
-#define KBASE_VE_JS_HARD_STOP_TICKS_CL_DEBUG 166 /* 2.5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG 100000 /* 1500s (25mins) before NSS hard-stop */
-#define KBASE_VE_JS_RESET_TICKS_SS_DEBUG 500 /* 45s before resetting GPU, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) */
-#define KBASE_VE_JS_RESET_TICKS_SS_8401_DEBUG 3000 /* 7.5s before resetting GPU - for issue 8401 */
-#define KBASE_VE_JS_RESET_TICKS_CL_DEBUG 500 /* 45s before resetting GPU */
-#define KBASE_VE_JS_RESET_TICKS_NSS_DEBUG 100166 /* 1502s before resetting GPU */
-
-#define KBASE_VE_JS_SCHEDULING_TICK_NS 1250000000u /* 1.25s */
-#define KBASE_VE_JS_SOFT_STOP_TICKS 2 /* 2.5s before soft-stop a job */
-#define KBASE_VE_JS_SOFT_STOP_TICKS_CL 1 /* 1.25s before soft-stop a CL job */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS 4 /* 5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_SS_8401 24 /* 30s before hard-stop, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) - for issue 8401 */
-#define KBASE_VE_JS_HARD_STOP_TICKS_CL 2 /* 2.5s before hard-stop */
-#define KBASE_VE_JS_HARD_STOP_TICKS_NSS 1200 /* 1500s before NSS hard-stop */
-#define KBASE_VE_JS_RESET_TICKS_SS 6 /* 7.5s before resetting GPU */
-#define KBASE_VE_JS_RESET_TICKS_SS_8401 36 /* 45s before resetting GPU, for a certain GLES2 test at 128x128 (bound by combined vertex+tiler job) - for issue 8401 */
-#define KBASE_VE_JS_RESET_TICKS_CL 3 /* 3.75s before resetting GPU */
-#define KBASE_VE_JS_RESET_TICKS_NSS 1201 /* 1502s before resetting GPU */
-
-#define KBASE_VE_JS_RESET_TIMEOUT_MS 3000 /* 3s before cancelling stuck jobs */
-#define KBASE_VE_JS_CTX_TIMESLICE_NS 1000000 /* 1ms - an agressive timeslice for testing purposes (causes lots of scheduling out for >4 ctxs) */
-#define KBASE_VE_POWER_MANAGEMENT_CALLBACKS ((uintptr_t)&pm_callbacks)
-#define KBASE_VE_CPU_SPEED_FUNC ((uintptr_t)&kbase_get_vexpress_cpu_clock_speed)
-
-#define HARD_RESET_AT_POWER_OFF 0
-
-#ifndef CONFIG_OF
-static kbase_io_resources io_resources = {
- .job_irq_number = 68,
- .mmu_irq_number = 69,
- .gpu_irq_number = 70,
- .io_memory_region = {
- .start = 0xFC010000,
- .end = 0xFC010000 + (4096 * 4) - 1
- }
-
-};
-#endif /* CONFIG_OF */
-
-static int pm_callback_power_on(struct kbase_device *kbdev)
-{
- /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */
- return 1;
-}
-
-static void pm_callback_power_off(struct kbase_device *kbdev)
-{
-#if HARD_RESET_AT_POWER_OFF
- /* Cause a GPU hard reset to test whether we have actually idled the GPU
- * and that we properly reconfigure the GPU on power up.
- * Usually this would be dangerous, but if the GPU is working correctly it should
- * be completely safe as the GPU should not be active at this point.
- * However this is disabled normally because it will most likely interfere with
- * bus logging etc.
- */
- KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
- kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET);
-#endif
-}
-
-static struct kbase_pm_callback_conf pm_callbacks = {
- .power_on_callback = pm_callback_power_on,
- .power_off_callback = pm_callback_power_off,
- .power_suspend_callback = NULL,
- .power_resume_callback = NULL
-};
-
-/* Please keep table config_attributes in sync with config_attributes_hw_issue_8408 */
-static struct kbase_attribute config_attributes[] = {
-#ifdef CONFIG_MALI_DEBUG
-/* Use more aggressive scheduling timeouts in debug builds for testing purposes */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL,
- KBASE_VE_JS_SOFT_STOP_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL,
- KBASE_VE_JS_HARD_STOP_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL,
- KBASE_VE_JS_RESET_TICKS_CL_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS_DEBUG},
-#else /* CONFIG_MALI_DEBUG */
-/* In release builds same as the defaults but scaled for 5MHz FPGA */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS_CL,
- KBASE_VE_JS_SOFT_STOP_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_CL,
- KBASE_VE_JS_HARD_STOP_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_CL,
- KBASE_VE_JS_RESET_TICKS_CL},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS},
-#endif /* CONFIG_MALI_DEBUG */
- {
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- KBASE_VE_JS_RESET_TIMEOUT_MS},
-
- {
- KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS,
- KBASE_VE_JS_CTX_TIMESLICE_NS},
-
- {
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
- KBASE_VE_POWER_MANAGEMENT_CALLBACKS},
-
- {
- KBASE_CONFIG_ATTR_CPU_SPEED_FUNC,
- KBASE_VE_CPU_SPEED_FUNC},
-
- {
- KBASE_CONFIG_ATTR_END,
- 0}
-};
-
-/* as config_attributes array above except with different settings for
- * JS_HARD_STOP_TICKS_SS, JS_RESET_TICKS_SS that
- * are needed for BASE_HW_ISSUE_8408.
- */
-struct kbase_attribute config_attributes_hw_issue_8408[] = {
-#ifdef CONFIG_MALI_DEBUG
-/* Use more aggressive scheduling timeouts in debug builds for testing purposes */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_8401_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_8401_DEBUG},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS_DEBUG},
-#else /* CONFIG_MALI_DEBUG */
-/* In release builds same as the defaults but scaled for 5MHz FPGA */
- {
- KBASE_CONFIG_ATTR_JS_SCHEDULING_TICK_NS,
- KBASE_VE_JS_SCHEDULING_TICK_NS},
-
- {
- KBASE_CONFIG_ATTR_JS_SOFT_STOP_TICKS,
- KBASE_VE_JS_SOFT_STOP_TICKS},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_SS,
- KBASE_VE_JS_HARD_STOP_TICKS_SS_8401},
-
- {
- KBASE_CONFIG_ATTR_JS_HARD_STOP_TICKS_NSS,
- KBASE_VE_JS_HARD_STOP_TICKS_NSS},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS,
- KBASE_VE_JS_RESET_TICKS_SS_8401},
-
- {
- KBASE_CONFIG_ATTR_JS_RESET_TICKS_NSS,
- KBASE_VE_JS_RESET_TICKS_NSS},
-#endif /* CONFIG_MALI_DEBUG */
- {
- KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
- KBASE_VE_JS_RESET_TIMEOUT_MS},
-
- {
- KBASE_CONFIG_ATTR_JS_CTX_TIMESLICE_NS,
- KBASE_VE_JS_CTX_TIMESLICE_NS},
-
- {
- KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
- KBASE_VE_POWER_MANAGEMENT_CALLBACKS},
-
- {
- KBASE_CONFIG_ATTR_CPU_SPEED_FUNC,
- KBASE_VE_CPU_SPEED_FUNC},
-
- {
- KBASE_CONFIG_ATTR_END,
- 0}
-};
-
-static struct kbase_platform_config virtex7_platform_config = {
- .attributes = config_attributes,
-#ifndef CONFIG_OF
- .io_resources = &io_resources
-#endif
-};
-
-struct kbase_platform_config *kbase_get_platform_config(void)
-{
- return &virtex7_platform_config;
-}
-
-int kbase_platform_early_init(void)
-{
- /* Nothing needed at this stage */
- return 0;
-}
-
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-#include <linux/io.h>
-#include <mali_kbase.h>
-#include "mali_kbase_cpu_vexpress.h"
-
-#define HZ_IN_MHZ (1000000)
-
-#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000)
-#define MOTHERBOARD_SYS_CFG_START (0x10000000)
-#define SYS_CFGDATA_OFFSET (0x000000A0)
-#define SYS_CFGCTRL_OFFSET (0x000000A4)
-#define SYS_CFGSTAT_OFFSET (0x000000A8)
-
-#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31)
-#define READ_REG_BIT_VALUE (0 << 30)
-#define DCC_DEFAULT_BIT_VALUE (0 << 26)
-#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20)
-#define SITE_DEFAULT_BIT_VALUE (1 << 16)
-#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12)
-#define DEVICE_DEFAULT_BIT_VALUE (2 << 0)
-#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0)
-#define SYS_CFG_ERROR_BIT_VALUE (1 << 1)
-
-#define FEED_REG_BIT_MASK (0x0F)
-#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03)
-#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07)
-#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B)
-#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F)
-#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13)
-
-#define IS_SINGLE_BIT_SET(val, pos) (val&(1<<pos))
-
-#define CPU_CLOCK_SPEED_UNDEFINED 0
-
-static u32 cpu_clock_speed = CPU_CLOCK_SPEED_UNDEFINED;
-
-static DEFINE_RAW_SPINLOCK(syscfg_lock);
-/**
- * kbase_get_vendor_specific_cpu_clock_speed
- * @brief Retrieves the CPU clock speed.
- * The implementation is platform specific.
- * @param[out] cpu_clock - the value of CPU clock speed in MHz
- * @return 0 on success, 1 otherwise
-*/
-int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock)
-{
-
-
- if (CPU_CLOCK_SPEED_UNDEFINED != cpu_clock_speed)
- {
- *cpu_clock = cpu_clock_speed;
- return 0;
- }
- else
- {
- int result = 0;
- u32 reg_val = 0;
- u32 osc2_value = 0;
- u32 pa_divide = 0;
- u32 pb_divide = 0;
- u32 pc_divide = 0;
- void *volatile pSysCfgReg = 0;
- void *volatile pSCCReg = 0;
-
- /* Init the value case something goes wrong */
- *cpu_clock = 0;
-
- /* Map CPU register into virtual memory */
- pSysCfgReg = ioremap(MOTHERBOARD_SYS_CFG_START, 0x1000);
- if (pSysCfgReg == NULL) {
- result = 1;
-
- goto pSysCfgReg_map_failed;
- }
-
- pSCCReg = ioremap(CORETILE_EXPRESS_A9X4_SCC_START, 0x1000);
- if (pSCCReg == NULL) {
- result = 1;
-
- goto pSCCReg_map_failed;
- }
-
- raw_spin_lock(&syscfg_lock);
-
- /*Read SYS regs - OSC2 */
- reg_val = readl(pSysCfgReg + SYS_CFGCTRL_OFFSET);
-
- /*Verify if there is no other undergoing request */
- if (!(reg_val & SYS_CFGCTRL_START_BIT_VALUE)) {
- /*Reset the CGFGSTAT reg */
- writel(0, (pSysCfgReg + SYS_CFGSTAT_OFFSET));
-
- writel(SYS_CFGCTRL_START_BIT_VALUE | READ_REG_BIT_VALUE | DCC_DEFAULT_BIT_VALUE | SYS_CFG_OSC_FUNC_BIT_VALUE | SITE_DEFAULT_BIT_VALUE | BOARD_STACK_POS_DEFAULT_BIT_VALUE | DEVICE_DEFAULT_BIT_VALUE, (pSysCfgReg + SYS_CFGCTRL_OFFSET));
- /* Wait for the transaction to complete */
- while (!(readl(pSysCfgReg + SYS_CFGSTAT_OFFSET) & SYS_CFG_COMPLETE_BIT_VALUE))
- ;
- /* Read SYS_CFGSTAT Register to get the status of submitted transaction */
- reg_val = readl(pSysCfgReg + SYS_CFGSTAT_OFFSET);
-
- /*------------------------------------------------------------------------------------------*/
- /* Check for possible errors */
- if (reg_val & SYS_CFG_ERROR_BIT_VALUE) {
- /* Error while setting register */
- result = 1;
- } else {
- osc2_value = readl(pSysCfgReg + SYS_CFGDATA_OFFSET);
- /* Read the SCC CFGRW0 register */
- reg_val = readl(pSCCReg);
-
- /*
- Select the appropriate feed:
- CFGRW0[0] - CLKOB
- CFGRW0[1] - CLKOC
- CFGRW0[2] - FACLK (CLK)B FROM AXICLK PLL)
- */
- /* Calculate the FCLK */
- if (IS_SINGLE_BIT_SET(reg_val, 0)) { /*CFGRW0[0] - CLKOB */
- /* CFGRW0[6:3] */
- pa_divide = ((reg_val & (FEED_REG_BIT_MASK << FCLK_PA_DIVIDE_BIT_SHIFT)) >> FCLK_PA_DIVIDE_BIT_SHIFT);
- /* CFGRW0[10:7] */
- pb_divide = ((reg_val & (FEED_REG_BIT_MASK << FCLK_PB_DIVIDE_BIT_SHIFT)) >> FCLK_PB_DIVIDE_BIT_SHIFT);
- *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1);
- } else {
- if (IS_SINGLE_BIT_SET(reg_val, 1)) { /*CFGRW0[1] - CLKOC */
- /* CFGRW0[6:3] */
- pa_divide = ((reg_val & (FEED_REG_BIT_MASK << FCLK_PA_DIVIDE_BIT_SHIFT)) >> FCLK_PA_DIVIDE_BIT_SHIFT);
- /* CFGRW0[14:11] */
- pc_divide = ((reg_val & (FEED_REG_BIT_MASK << FCLK_PC_DIVIDE_BIT_SHIFT)) >> FCLK_PC_DIVIDE_BIT_SHIFT);
- *cpu_clock = osc2_value * (pa_divide + 1) / (pc_divide + 1);
- } else if (IS_SINGLE_BIT_SET(reg_val, 2)) { /*CFGRW0[2] - FACLK */
- /* CFGRW0[18:15] */
- pa_divide = ((reg_val & (FEED_REG_BIT_MASK << AXICLK_PA_DIVIDE_BIT_SHIFT)) >> AXICLK_PA_DIVIDE_BIT_SHIFT);
- /* CFGRW0[22:19] */
- pb_divide = ((reg_val & (FEED_REG_BIT_MASK << AXICLK_PB_DIVIDE_BIT_SHIFT)) >> AXICLK_PB_DIVIDE_BIT_SHIFT);
- *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1);
- } else {
- result = 1;
- }
- }
- }
- } else {
- result = 1;
- }
- raw_spin_unlock(&syscfg_lock);
- /* Convert result expressed in Hz to Mhz units. */
- *cpu_clock /= HZ_IN_MHZ;
- if (!result)
- {
- cpu_clock_speed = *cpu_clock;
- }
-
- /* Unmap memory */
- iounmap(pSCCReg);
-
- pSCCReg_map_failed:
- iounmap(pSysCfgReg);
-
- pSysCfgReg_map_failed:
-
- return result;
- }
-}
+++ /dev/null
-/*
- *
- * (C) COPYRIGHT ARM Limited. All rights reserved.
- *
- * This program is free software and is provided to you under the terms of the
- * GNU General Public License version 2 as published by the Free Software
- * Foundation, and any use by you of this program is subject to the terms
- * of such GNU licence.
- *
- * A copy of the licence is included with the program, and can also be obtained
- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- */
-
-
-
-#ifndef _KBASE_CPU_VEXPRESS_H_
-#define _KBASE_CPU_VEXPRESS_H_
-
-/**
- * Versatile Express implementation of @ref kbase_cpuprops_clock_speed_function.
- */
-int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock);
-
-#endif /* _KBASE_CPU_VEXPRESS_H_ */
/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010, 2012-2014 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Internal OS specific data structure associated with each UKK session. Part
* of a ukk_session object.
*/
-typedef struct ukkp_session
-{
+typedef struct ukkp_session {
int dummy; /**< No internal OS specific data at this time */
} ukkp_session;
#
-# (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved.
+# (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
# Fake platform is a transient solution for GPL drivers running in kernel that does not provide configuration via platform data.
# For such kernels fake_platform_device should be set to 1. For kernels providing platform data fake_platform_device should be set to 0.
-fake_platform_device = 1
+if env['platform_config']=='devicetree':
+ fake_platform_device = 0
+else:
+ fake_platform_device = 1
# Source files required for kbase.
kbase_src = [Glob('#kernel/drivers/gpu/arm/midgard/*.c'),
Glob('#kernel/drivers/gpu/arm/midgard/*.c'),
+ Glob('#kernel/drivers/gpu/arm/midgard/backend/*/*.c'),
+ Glob('#kernel/drivers/gpu/arm/midgard/backend/*/*.h'),
Glob('#kernel/drivers/gpu/arm/midgard/platform/%s/*.c' % (env['platform_config'])),
Glob('#kernel/drivers/gpu/arm/midgard/*.h'),
Glob('#kernel/drivers/gpu/arm/midgard/*.h'),
if re.search(search_term, line):
REALVIEW_PBX = 1
break
- if REALVIEW_PBX == 1 and (env['platform_config'] == 'vexpress' or env['platform_config'] == 'vexpress_virtex7_40mhz' or env['platform_config'] == 'vexpress_6xvirtex7_10mhz'):
+ if REALVIEW_PBX == 1 and (env['platform_config'] == 'vexpress' or env['platform_config'] == 'vexpress_6xvirtex7_10mhz'):
sys.stderr.write("WARNING: Building for a PBX kernel but with platform_config=vexpress*\n")
# if the file platform config file is in the tpip directory then use that, otherwise use the default config directory
if Glob('#kernel/drivers/gpu/arm/midgard/config/tpip/*%s.c' % (env['platform_config'])):
else:
env['kernel_test'] = 0
- makeAction=Action("cd ${SOURCE.dir} && make PLATFORM=${platform} MALI_ERROR_INJECT_ON=${error_inject} MALI_ANDROID=${android} MALI_KERNEL_TEST_API=${kernel_test} MALI_UNIT_TEST=${unit} MALI_RELEASE_NAME=\"${mali_release_name}\" MALI_MOCK_TEST=%s MALI_CUSTOMER_RELEASE=${release} MALI_INSTRUMENTATION_LEVEL=${instr} MALI_COVERAGE=${coverage} %s && cp mali_kbase.ko $STATIC_LIB_PATH/mali_kbase.ko" % (mock_test, env.kernel_get_config_defines(fake_platform_device)), '$MAKECOMSTR')
+ makeAction=Action("cd ${SOURCE.dir} && make -j%d PLATFORM=${platform} MALI_ERROR_INJECT_ON=${error_inject} MALI_ANDROID=${android} MALI_KERNEL_TEST_API=${kernel_test} MALI_UNIT_TEST=${unit} MALI_RELEASE_NAME=\"${mali_release_name}\" MALI_MOCK_TEST=%s MALI_CUSTOMER_RELEASE=${release} MALI_INSTRUMENTATION_LEVEL=${instr} MALI_COVERAGE=${coverage} %s && cp mali_kbase.ko $STATIC_LIB_PATH/mali_kbase.ko" % (GetOption('num_jobs'), mock_test, env.kernel_get_config_defines(fake_platform_device)), '$MAKECOMSTR')
cmd = env.Command('$STATIC_LIB_PATH/mali_kbase.ko', kbase_src, [makeAction])
# Add a dependency on kds.ko.
if int(env['ump']) == 1:
env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/ump.ko')
+# need Module.symvers from kutf.ko build
+if env['unit'] == '1':
+ env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/kutf.ko')
+
# Until we fathom out how the invoke the Linux build system to clean, we can use Clean
# to remove generated files.
patterns = ['*.mod.c', '*.o', '*.ko', '*.a', '.*.cmd', 'modules.order', '.tmp_versions', 'Module.symvers']
Clean(cmd, Glob('#kernel/drivers/gpu/arm/midgard/internal/*/%s' % p))
env.ProgTarget('kbase', cmd)
-env.AppendUnique(BASE=['cutils_list'])
+env.AppendUnique(BASE=['cutils_linked_list'])
projects / firefly-linux-kernel-4.4.55.git / commitdiff