From: chenzhen Date: Tue, 12 Jan 2016 08:58:39 +0000 (+0800) Subject: MALI: rockchip: upgrade utgard DDK to r6p0-01rel0 X-Git-Tag: firefly_0821_release~2268 X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=3446f414bca7ee607174a9e7b0c5d31a14750baf;p=firefly-linux-kernel-4.4.55.git MALI: rockchip: upgrade utgard DDK to r6p0-01rel0 Conflicts: drivers/gpu/arm/mali400/mali/common/mali_control_timer.c drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h Change-Id: I13d02e836efcebd0dd2367ce138aac258dacda24 Signed-off-by: chenzhen --- diff --git a/drivers/gpu/arm/mali400/mali/Kbuild b/drivers/gpu/arm/mali400/mali/Kbuild index dafd6fedf1db..8b3e08edb743 100755 --- a/drivers/gpu/arm/mali400/mali/Kbuild +++ b/drivers/gpu/arm/mali400/mali/Kbuild @@ -71,11 +71,20 @@ mali-y += \ linux/mali_osk_mali.o \ linux/mali_osk_notification.o \ linux/mali_osk_time.o \ - linux/mali_osk_timers.o + linux/mali_osk_timers.o \ + linux/mali_osk_bitmap.o mali-y += linux/mali_memory.o linux/mali_memory_os_alloc.o mali-y += linux/mali_memory_external.o mali-y += linux/mali_memory_block_alloc.o +mali-y += linux/mali_memory_swap_alloc.o + +mali-y += \ + linux/mali_memory_manager.o \ + linux/mali_memory_virtual.o \ + linux/mali_memory_util.o \ + linux/mali_memory_cow.o \ + linux/mali_memory_defer_bind.o mali-y += \ linux/mali_ukk_mem.o \ @@ -89,7 +98,6 @@ mali-y += \ mali-y += \ common/mali_kernel_core.o \ linux/mali_kernel_linux.o \ - common/mali_kernel_descriptor_mapping.o \ common/mali_session.o \ linux/mali_device_pause_resume.o \ common/mali_kernel_vsync.o \ @@ -164,7 +172,7 @@ ccflags-$(CONFIG_MALI400_UMP) += -I$(src)/../../ump/include/ump ccflags-$(CONFIG_MALI400_DEBUG) += -DDEBUG # Use our defines when compiling -ccflags-y += -I$(src) -I$(src)/include -I$(src)/common -I$(src)/linux -I$(src)/platform +ccflags-y += -I$(src) -I$(src)/include -I$(src)/common -I$(src)/linux -I$(src)/platform -Wno-date-time # Get subversion revision number, fall back to only ${MALI_RELEASE_NAME} if no svn info is available MALI_RELEASE_NAME=$(shell cat $(src)/.version 2> /dev/null) diff --git a/drivers/gpu/arm/mali400/mali/Kconfig b/drivers/gpu/arm/mali400/mali/Kconfig index fe3f8b0f8c60..f199123e6989 100755 --- a/drivers/gpu/arm/mali400/mali/Kconfig +++ b/drivers/gpu/arm/mali400/mali/Kconfig @@ -1,6 +1,6 @@ config MALI400 tristate "Mali-300/400/450 support" - depends on ARM + depends on ARM || ARM64 select DMA_SHARED_BUFFER ---help--- This enables support for the ARM Mali-300, Mali-400, and Mali-450 @@ -15,6 +15,12 @@ config MALI450 ---help--- This enables support for Mali-450 specific features. +config MALI470 + bool "Enable Mali-470 support" + depends on MALI400 + ---help--- + This enables support for Mali-470 specific features. + config MALI400_DEBUG bool "Enable debug in Mali driver" depends on MALI400 diff --git a/drivers/gpu/arm/mali400/mali/Makefile b/drivers/gpu/arm/mali400/mali/Makefile index 412e9ce95776..44c7bb83981a 100755 --- a/drivers/gpu/arm/mali400/mali/Makefile +++ b/drivers/gpu/arm/mali400/mali/Makefile @@ -1,5 +1,5 @@ # -# Copyright (C) 2010-2014 ARM Limited. All rights reserved. +# Copyright (C) 2010-2015 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the GNU General Public License version 2 # as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -16,6 +16,7 @@ MALI_HEATMAPS_ENABLED ?= 0 MALI_DMA_BUF_MAP_ON_ATTACH ?= 1 MALI_PMU_PARALLEL_POWER_UP ?= 0 USING_DT ?= 0 +MALI_MEM_SWAP_TRACKING ?= 0 # The Makefile sets up "arch" based on the CONFIG, creates the version info # string and the __malidrv_build_info.c file, and then call the Linux build @@ -102,9 +103,11 @@ endif # Set up build config export CONFIG_MALI400=m export CONFIG_MALI450=y +export CONFIG_MALI470=y export EXTRA_DEFINES += -DCONFIG_MALI400=1 export EXTRA_DEFINES += -DCONFIG_MALI450=1 +export EXTRA_DEFINES += -DCONFIG_MALI470=1 ifneq ($(MALI_PLATFORM),) export EXTRA_DEFINES += -DMALI_FAKE_PLATFORM_DEVICE=1 @@ -170,6 +173,10 @@ ifeq ($(MALI_SKIP_JOBS),1) EXTRA_DEFINES += -DPROFILING_SKIP_PP_JOBS=1 -DPROFILING_SKIP_GP_JOBS=1 endif +ifeq ($(MALI_MEM_SWAP_TRACKING),1) +EXTRA_DEFINES += -DMALI_MEM_SWAP_TRACKING=1 +endif + all: $(UMP_SYMVERS_FILE) $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) modules @rm $(FILES_PREFIX)__malidrv_build_info.c $(FILES_PREFIX)__malidrv_build_info.o diff --git a/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c index 2ffa1d36660e..136db61ace4a 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h index 049a89cb1c16..efce44142ee9 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c index 990e7ab5ede2..2d1f579692ca 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2012, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2012, 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -43,9 +43,10 @@ static void mali_control_timer_callback(void *arg) if (mali_utilization_enabled()) { struct mali_gpu_utilization_data *util_data = NULL; u64 time_period = 0; + mali_bool need_add_timer = MALI_TRUE; /* Calculate gpu utilization */ - util_data = mali_utilization_calculate(&period_start_time, &time_period); + util_data = mali_utilization_calculate(&period_start_time, &time_period, &need_add_timer); if (util_data) { #if defined(CONFIG_MALI_DVFS) @@ -53,10 +54,12 @@ static void mali_control_timer_callback(void *arg) #else mali_utilization_platform_realize(util_data); #endif - } if (MALI_TRUE == timer_running) - mali_control_timer_mod(mali_control_timeout); + if (MALI_TRUE == need_add_timer) { + mali_control_timer_mod(mali_control_timeout); + } + } } } @@ -94,6 +97,8 @@ void mali_control_timer_term(void) mali_bool mali_control_timer_resume(u64 time_now) { + mali_utilization_data_assert_locked(); + if (timer_running != MALI_TRUE) { timer_running = MALI_TRUE; @@ -107,6 +112,14 @@ mali_bool mali_control_timer_resume(u64 time_now) return MALI_FALSE; } +void mali_control_timer_pause(void) +{ + mali_utilization_data_assert_locked(); + if (timer_running == MALI_TRUE) { + timer_running = MALI_FALSE; + } +} + void mali_control_timer_suspend(mali_bool suspend) { mali_utilization_data_lock(); diff --git a/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h index 73cc77f75ce7..f829174de58e 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2012, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2012, 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -20,6 +20,7 @@ void mali_control_timer_term(void); mali_bool mali_control_timer_resume(u64 time_now); void mali_control_timer_suspend(mali_bool suspend); +void mali_control_timer_pause(void); void mali_control_timer_add(u32 timeout); diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c index 41c4ac2ae8ae..efe1ab3ab5cc 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h index d5436d3d57f9..6b068884bd49 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c index 7e6c1145b601..12ba069ec13b 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2012, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2012, 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h index 7d127153a3a4..55e4b354c1a9 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2012, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2012, 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_executor.c b/drivers/gpu/arm/mali400/mali/common/mali_executor.c index 4def9ae7cee6..d16a47b695e1 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_executor.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_executor.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -121,8 +121,6 @@ static _mali_osk_wait_queue_t *executor_notify_core_change_wait_queue = NULL; /* * ---------- Forward declaration of static functions ---------- */ -static void mali_executor_lock(void); -static void mali_executor_unlock(void); static mali_bool mali_executor_is_suspended(void *data); static mali_bool mali_executor_is_working(void); static void mali_executor_disable_empty_virtual(void); @@ -131,10 +129,9 @@ static mali_bool mali_executor_has_virtual_group(void); static mali_bool mali_executor_virtual_group_is_usable(void); static void mali_executor_schedule(void); static void mali_executor_wq_schedule(void *arg); -static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job); +static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job, u32 added_size); static void mali_executor_complete_group(struct mali_group *group, mali_bool success, - mali_bool release_jobs, struct mali_gp_job **gp_job_done, struct mali_pp_job **pp_job_done); static void mali_executor_change_state_pp_physical(struct mali_group *group, @@ -449,8 +446,7 @@ void mali_executor_zap_all_active(struct mali_session_data *session) if (MALI_FALSE == ret) { struct mali_gp_job *gp_job = NULL; - mali_executor_complete_group(gp_group, MALI_FALSE, - MALI_TRUE, &gp_job, NULL); + mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL); MALI_DEBUG_ASSERT_POINTER(gp_job); @@ -464,8 +460,7 @@ void mali_executor_zap_all_active(struct mali_session_data *session) if (MALI_FALSE == ret) { struct mali_pp_job *pp_job = NULL; - mali_executor_complete_group(virtual_group, MALI_FALSE, - MALI_TRUE, NULL, &pp_job); + mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job); if (NULL != pp_job) { /* PP job completed, make sure it is freed */ @@ -483,8 +478,7 @@ void mali_executor_zap_all_active(struct mali_session_data *session) if (MALI_FALSE == ret) { struct mali_pp_job *pp_job = NULL; - mali_executor_complete_group(group, MALI_FALSE, - MALI_TRUE, NULL, &pp_job); + mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job); if (NULL != pp_job) { /* PP job completed, free it */ @@ -580,24 +574,11 @@ _mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group, return _MALI_OSK_ERR_OK; } } else if (MALI_INTERRUPT_RESULT_OOM == int_result) { - struct mali_gp_job *job = mali_group_get_running_gp_job(group); - - /* PLBU out of mem */ - MALI_DEBUG_PRINT(3, ("Executor: PLBU needs more heap memory\n")); - -#if defined(CONFIG_MALI400_PROFILING) - /* Give group a chance to generate a SUSPEND event */ - mali_group_oom(group); -#endif - - /* - * no need to hold interrupt raised while - * waiting for more memory. - */ - mali_executor_send_gp_oom_to_user(job); mali_executor_unlock(); + mali_group_schedule_oom_work_handler(group); + return _MALI_OSK_ERR_OK; } @@ -623,11 +604,14 @@ _mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group, struct mali_gp_job *job; mali_bool success; + if (MALI_TRUE == time_out) { + mali_group_dump_status(group); + } + success = (int_result != MALI_INTERRUPT_RESULT_ERROR) ? MALI_TRUE : MALI_FALSE; - mali_executor_complete_group(group, success, - MALI_TRUE, &job, NULL); + mali_executor_complete_group(group, success, &job, NULL); mali_executor_unlock(); @@ -700,9 +684,6 @@ _mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group, } #else MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result); - if (!mali_group_has_timed_out(group)) { - MALI_DEBUG_ASSERT(!mali_group_pp_is_active(group)); - } #endif /*Add voltage scan function*/ @@ -729,11 +710,14 @@ _mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group, struct mali_pp_job *job = NULL; mali_bool success; + if (MALI_TRUE == time_out) { + mali_group_dump_status(group); + } + success = (int_result == MALI_INTERRUPT_RESULT_SUCCESS) ? MALI_TRUE : MALI_FALSE; - mali_executor_complete_group(group, success, - MALI_TRUE, NULL, &job); + mali_executor_complete_group(group, success, NULL, &job); mali_executor_unlock(); @@ -815,10 +799,10 @@ _mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group, group->mmu->hw_core.description)); MALI_DEBUG_PRINT(3, ("Executor: MMU rawstat = 0x%08X, MMU status = 0x%08X\n", mali_mmu_get_rawstat(group->mmu), status)); + mali_mmu_pagedir_diag(mali_session_get_page_directory(group->session), fault_address); #endif - mali_executor_complete_group(group, MALI_FALSE, - MALI_TRUE, &gp_job, &pp_job); + mali_executor_complete_group(group, MALI_FALSE, &gp_job, &pp_job); mali_executor_unlock(); @@ -841,6 +825,59 @@ _mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group, return _MALI_OSK_ERR_OK; } +void mali_executor_group_oom(struct mali_group *group) +{ + struct mali_gp_job *job = NULL; + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + mali_executor_lock(); + + job = mali_group_get_running_gp_job(group); + + MALI_DEBUG_ASSERT_POINTER(job); + +#if defined(CONFIG_MALI400_PROFILING) + /* Give group a chance to generate a SUSPEND event */ + mali_group_oom(group); +#endif + + mali_gp_job_set_current_heap_addr(job, mali_gp_read_plbu_alloc_start_addr(group->gp_core)); + + mali_executor_unlock(); + + if (_MALI_OSK_ERR_OK == mali_mem_add_mem_size(job->session, job->heap_base_addr, job->heap_grow_size)) { + _mali_osk_notification_t *new_notification = NULL; + + new_notification = _mali_osk_notification_create( + _MALI_NOTIFICATION_GP_STALLED, + sizeof(_mali_uk_gp_job_suspended_s)); + + /* resume job with new heap, + * This will also re-enable interrupts + */ + mali_executor_lock(); + + mali_executor_send_gp_oom_to_user(job, job->heap_grow_size); + + if (NULL != new_notification) { + + mali_gp_job_set_oom_notification(job, new_notification); + + mali_group_resume_gp_with_new_heap(group, mali_gp_job_get_id(job), + job->heap_current_addr, + job->heap_current_addr + job->heap_grow_size); + } + mali_executor_unlock(); + } else { + mali_executor_lock(); + mali_executor_send_gp_oom_to_user(job, 0); + mali_executor_unlock(); + } + +} + void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups) { u32 i; @@ -950,11 +987,20 @@ void mali_executor_group_power_down(struct mali_group *groups[], MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups\n", num_groups)); for (i = 0; i < num_groups; i++) { - /* Groups must be either disabled or inactive */ + /* Groups must be either disabled or inactive. while for virtual group, + * it maybe in empty state, because when we meet pm_runtime_suspend, + * virtual group could be powered off, and before we acquire mali_executor_lock, + * we must release mali_pm_state_lock, if there is a new physical job was queued, + * all of physical groups in virtual group could be pulled out, so we only can + * powered down an empty virtual group. Those physical groups will be powered + * up in following pm_runtime_resume callback function. + */ MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(groups[i], EXEC_STATE_DISABLED) || mali_executor_group_is_in_state(groups[i], - EXEC_STATE_INACTIVE)); + EXEC_STATE_INACTIVE) || + mali_executor_group_is_in_state(groups[i], + EXEC_STATE_EMPTY)); MALI_DEBUG_PRINT(3, ("Executor: powering down group %s\n", mali_group_core_description(groups[i]))); @@ -985,8 +1031,7 @@ void mali_executor_abort_session(struct mali_session_data *session) if (EXEC_STATE_WORKING == gp_group_state) { struct mali_gp_job *gp_job = NULL; - mali_executor_complete_group(gp_group, MALI_FALSE, - MALI_TRUE, &gp_job, NULL); + mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL); MALI_DEBUG_ASSERT_POINTER(gp_job); @@ -1004,8 +1049,7 @@ void mali_executor_abort_session(struct mali_session_data *session) && mali_group_get_session(virtual_group) == session) { struct mali_pp_job *pp_job = NULL; - mali_executor_complete_group(virtual_group, MALI_FALSE, - MALI_FALSE, NULL, &pp_job); + mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job); if (NULL != pp_job) { /* PP job completed, make sure it is freed */ @@ -1020,8 +1064,7 @@ void mali_executor_abort_session(struct mali_session_data *session) if (mali_group_get_session(group) == session) { struct mali_pp_job *pp_job = NULL; - mali_executor_complete_group(group, MALI_FALSE, - MALI_FALSE, NULL, &pp_job); + mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job); if (NULL != pp_job) { /* PP job completed, make sure it is freed */ @@ -1303,6 +1346,9 @@ _mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s args->arguments[0], args->arguments[1]); + job->heap_base_addr = args->arguments[0]; + job->heap_current_addr = args->arguments[0]; + mali_executor_unlock(); return _MALI_OSK_ERR_OK; } else { @@ -1330,8 +1376,7 @@ _mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s /* Correct job is still running */ struct mali_gp_job *job_done = NULL; - mali_executor_complete_group(gp_group, MALI_FALSE, - MALI_TRUE, &job_done, NULL); + mali_executor_complete_group(gp_group, MALI_FALSE, &job_done, NULL); /* The same job should have completed */ MALI_DEBUG_ASSERT(job_done == job); @@ -1350,13 +1395,13 @@ _mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s * ---------- Implementation of static functions ---------- */ -static void mali_executor_lock(void) +void mali_executor_lock(void) { _mali_osk_spinlock_irq_lock(mali_executor_lock_obj); MALI_DEBUG_PRINT(5, ("Executor: lock taken\n")); } -static void mali_executor_unlock(void) +void mali_executor_unlock(void) { MALI_DEBUG_PRINT(5, ("Executor: Releasing lock\n")); _mali_osk_spinlock_irq_unlock(mali_executor_lock_obj); @@ -1456,23 +1501,23 @@ static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group) static mali_bool mali_executor_has_virtual_group(void) { -#if defined(CONFIG_MALI450) +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) return (NULL != virtual_group) ? MALI_TRUE : MALI_FALSE; #else return MALI_FALSE; -#endif /* defined(CONFIG_MALI450) */ +#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ } static mali_bool mali_executor_virtual_group_is_usable(void) { -#if defined(CONFIG_MALI450) +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); - return (EXEC_STATE_INACTIVE == virtual_group_state || - EXEC_STATE_IDLE == virtual_group_state) ? + return ((EXEC_STATE_INACTIVE == virtual_group_state || + EXEC_STATE_IDLE == virtual_group_state) && (virtual_group->state != MALI_GROUP_STATE_ACTIVATION_PENDING)) ? MALI_TRUE : MALI_FALSE; #else return MALI_FALSE; -#endif /* defined(CONFIG_MALI450) */ +#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ } static mali_bool mali_executor_tackle_gp_bound(void) @@ -1669,7 +1714,7 @@ static void mali_executor_schedule(void) MALI_DEBUG_ASSERT_POINTER(job); MALI_DEBUG_ASSERT(sub_job <= MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS); - + /* Put job + group on list of jobs to start later on */ groups_to_start[num_jobs_to_start] = group; @@ -1693,7 +1738,20 @@ static void mali_executor_schedule(void) } } - /* 3. Activate virtual group, if needed */ + + /* 3. Deactivate idle pp group , must put deactive here before active vitual group + * for cover case first only has physical job in normal queue but group inactive, + * so delay the job start go to active group, when group activated, + * call scheduler again, but now if we get high queue virtual job, + * we will do nothing in schedule cause executor schedule stop + */ + + if (MALI_TRUE == mali_executor_deactivate_list_idle(deactivate_idle_group + && (!mali_timeline_has_physical_pp_job()))) { + trigger_pm_update = MALI_TRUE; + } + + /* 4. Activate virtual group, if needed */ if (EXEC_STATE_INACTIVE == virtual_group_state && 0 < mali_scheduler_job_next_is_virtual()) { @@ -1707,20 +1765,13 @@ static void mali_executor_schedule(void) } } - /* 4. To power up group asap, we trigger pm update here. */ + /* 5. To power up group asap, we trigger pm update here. */ if (MALI_TRUE == trigger_pm_update) { trigger_pm_update = MALI_FALSE; mali_pm_update_async(); } - /* 5. Deactivate idle pp group */ - - if (MALI_TRUE == mali_executor_deactivate_list_idle(deactivate_idle_group - && (!mali_timeline_has_physical_pp_job()))) { - trigger_pm_update = MALI_TRUE; - } - /* 6. Assign jobs to idle virtual group (or deactivate if no job) */ if (EXEC_STATE_IDLE == virtual_group_state) { @@ -1793,7 +1844,7 @@ static void mali_executor_wq_schedule(void *arg) mali_executor_unlock(); } -static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job) +static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job, u32 added_size) { _mali_uk_gp_job_suspended_s *jobres; _mali_osk_notification_t *notification; @@ -1809,13 +1860,12 @@ static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job) jobres = (_mali_uk_gp_job_suspended_s *)notification->result_buffer; jobres->user_job_ptr = mali_gp_job_get_user_id(job); jobres->cookie = gp_returned_cookie; - + jobres->heap_added_size = added_size; mali_session_send_notification(mali_gp_job_get_session(job), notification); } static struct mali_gp_job *mali_executor_complete_gp(struct mali_group *group, - mali_bool success, - mali_bool release_jobs) + mali_bool success) { struct mali_gp_job *job; @@ -1829,20 +1879,17 @@ static struct mali_gp_job *mali_executor_complete_gp(struct mali_group *group, /* Core is now ready to go into idle list */ gp_group_state = EXEC_STATE_IDLE; - if (release_jobs) { - /* This will potentially queue more GP and PP jobs */ - mali_timeline_tracker_release(&job->tracker); + /* This will potentially queue more GP and PP jobs */ + mali_timeline_tracker_release(&job->tracker); - /* Signal PP job */ - mali_gp_job_signal_pp_tracker(job, success); - } + /* Signal PP job */ + mali_gp_job_signal_pp_tracker(job, success); return job; } static struct mali_pp_job *mali_executor_complete_pp(struct mali_group *group, - mali_bool success, - mali_bool release_jobs) + mali_bool success) { struct mali_pp_job *job; u32 sub_job; @@ -1871,7 +1918,7 @@ static struct mali_pp_job *mali_executor_complete_pp(struct mali_group *group, mali_pp_job_mark_sub_job_completed(job, success); job_is_done = mali_pp_job_is_complete(job); - if (job_is_done && release_jobs) { + if (job_is_done) { /* This will potentially queue more GP and PP jobs */ mali_timeline_tracker_release(&job->tracker); } @@ -1881,7 +1928,6 @@ static struct mali_pp_job *mali_executor_complete_pp(struct mali_group *group, static void mali_executor_complete_group(struct mali_group *group, mali_bool success, - mali_bool release_jobs, struct mali_gp_job **gp_job_done, struct mali_pp_job **pp_job_done) { @@ -1892,13 +1938,11 @@ static void mali_executor_complete_group(struct mali_group *group, mali_bool pp_job_is_done = MALI_TRUE; if (NULL != gp_core) { - gp_job = mali_executor_complete_gp(group, - success, release_jobs); + gp_job = mali_executor_complete_gp(group, success); } else { MALI_DEBUG_ASSERT_POINTER(pp_core); MALI_IGNORE(pp_core); - pp_job = mali_executor_complete_pp(group, - success, release_jobs); + pp_job = mali_executor_complete_pp(group, success); pp_job_is_done = mali_pp_job_is_complete(pp_job); } @@ -2151,7 +2195,7 @@ static void mali_executor_notify_core_change(u32 num_cores) { mali_bool done = MALI_FALSE; - if (mali_is_mali450()) { + if (mali_is_mali450() || mali_is_mali470()) { return; } @@ -2253,7 +2297,7 @@ static void mali_executor_wq_notify_core_change(void *arg) { MALI_IGNORE(arg); - if (mali_is_mali450()) { + if (mali_is_mali450() || mali_is_mali470()) { return; } @@ -2292,7 +2336,6 @@ static void mali_executor_core_scale(unsigned int target_core_nr) { int current_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; int target_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; - mali_bool update_global_core_scaling_mask = MALI_FALSE; int i; MALI_DEBUG_ASSERT(0 < target_core_nr); @@ -2355,7 +2398,6 @@ static void mali_executor_core_scale(unsigned int target_core_nr) struct mali_pm_domain *domain; if (num_physical_pp_cores_enabled >= target_core_nr) { - update_global_core_scaling_mask = MALI_TRUE; break; } @@ -2385,11 +2427,9 @@ static void mali_executor_core_scale(unsigned int target_core_nr) * Here, we may still have some pp cores not been enabled because of some * pp cores need to be disabled are still in working state. */ - if (update_global_core_scaling_mask) { - for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { - if (0 < target_core_scaling_mask[i]) { - core_scaling_delay_up_mask[i] = target_core_scaling_mask[i]; - } + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (0 < target_core_scaling_mask[i]) { + core_scaling_delay_up_mask[i] = target_core_scaling_mask[i]; } } @@ -2530,3 +2570,85 @@ static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_gr return trigger_pm_update; } + +void mali_executor_running_status_print(void) +{ + struct mali_group *group = NULL; + struct mali_group *temp = NULL; + + MALI_PRINT(("GP running job: %p\n", gp_group->gp_running_job)); + if ((gp_group->gp_core) && (gp_group->is_working)) { + mali_group_dump_status(gp_group); + } + MALI_PRINT(("Physical PP groups in WORKING state (count = %u):\n", group_list_working_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) { + MALI_PRINT(("PP running job: %p, subjob %d \n", group->pp_running_job, group->pp_running_sub_job)); + mali_group_dump_status(group); + } + MALI_PRINT(("Physical PP groups in INACTIVE state (count = %u):\n", group_list_inactive_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { + MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); + MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); + } + MALI_PRINT(("Physical PP groups in IDLE state (count = %u):\n", group_list_idle_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { + MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); + MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); + } + MALI_PRINT(("Physical PP groups in DISABLED state (count = %u):\n", group_list_disabled_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { + MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); + MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); + } + + if (mali_executor_has_virtual_group()) { + MALI_PRINT(("Virtual group running job: %p\n", virtual_group->pp_running_job)); + MALI_PRINT(("Virtual group status: %d\n", virtual_group_state)); + MALI_PRINT(("Virtual group->status: %d\n", virtual_group->state)); + MALI_PRINT(("\tSW power: %s\n", virtual_group->power_is_on ? "On" : "Off")); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &virtual_group->group_list, + struct mali_group, group_list) { + int i = 0; + MALI_PRINT(("\tchild group(%s) running job: %p\n", group->pp_core->hw_core.description, group->pp_running_job)); + MALI_PRINT(("\tchild group(%s)->status: %d\n", group->pp_core->hw_core.description, group->state)); + MALI_PRINT(("\tchild group(%s) SW power: %s\n", group->pp_core->hw_core.description, group->power_is_on ? "On" : "Off")); + if (group->pm_domain) { + MALI_PRINT(("\tPower domain: id %u\n", mali_pm_domain_get_id(group->pm_domain))); + MALI_PRINT(("\tMask:0x%04x \n", mali_pm_domain_get_mask(group->pm_domain))); + MALI_PRINT(("\tUse-count:%u \n", mali_pm_domain_get_use_count(group->pm_domain))); + MALI_PRINT(("\tCurrent power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_current_mask()) ? "On" : "Off")); + MALI_PRINT(("\tWanted power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_wanted_mask()) ? "On" : "Off")); + } + + for (i = 0; i < 2; i++) { + if (NULL != group->l2_cache_core[i]) { + struct mali_pm_domain *domain; + domain = mali_l2_cache_get_pm_domain(group->l2_cache_core[i]); + MALI_PRINT(("\t L2(index %d) group SW power: %s\n", i, group->l2_cache_core[i]->power_is_on ? "On" : "Off")); + if (domain) { + MALI_PRINT(("\tL2 Power domain: id %u\n", mali_pm_domain_get_id(domain))); + MALI_PRINT(("\tL2 Mask:0x%04x \n", mali_pm_domain_get_mask(domain))); + MALI_PRINT(("\tL2 Use-count:%u \n", mali_pm_domain_get_use_count(domain))); + MALI_PRINT(("\tL2 Current power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_current_mask()) ? "On" : "Off")); + MALI_PRINT(("\tL2 Wanted power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_wanted_mask()) ? "On" : "Off")); + } + } + } + } + if (EXEC_STATE_WORKING == virtual_group_state) { + mali_group_dump_status(virtual_group); + } + } +} + +void mali_executor_status_dump(void) +{ + mali_executor_lock(); + mali_scheduler_lock(); + + /* print schedule queue status */ + mali_scheduler_gp_pp_job_queue_print(); + + mali_scheduler_unlock(); + mali_executor_unlock(); +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_executor.h b/drivers/gpu/arm/mali400/mali/common/mali_executor.h index 48d5ffa9aa4e..a756d3f40c2a 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_executor.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_executor.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2012, 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -59,6 +59,7 @@ _mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group, mali_bo _mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group, mali_bool in_upper_half); _mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group, mali_bool in_upper_half); +void mali_executor_group_oom(struct mali_group *group); void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups); void mali_executor_group_power_down(struct mali_group *groups[], u32 num_groups); @@ -96,4 +97,8 @@ MALI_STATIC_INLINE mali_bool mali_executor_hint_is_enabled(mali_executor_hint hi return mali_executor_hints[hint]; } +void mali_executor_running_status_print(void); +void mali_executor_status_dump(void); +void mali_executor_lock(void); +void mali_executor_unlock(void); #endif /* __MALI_EXECUTOR_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp.c b/drivers/gpu/arm/mali400/mali/common/mali_gp.c index 49c4dde2ecdd..a690781837e9 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_gp.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_gp.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -116,7 +116,7 @@ _mali_osk_errcode_t mali_gp_stop_bus_wait(struct mali_gp_core *core) void mali_gp_hard_reset(struct mali_gp_core *core) { - const u32 reset_wait_target_register = MALIGP2_REG_ADDR_MGMT_WRITE_BOUND_LOW; + const u32 reset_wait_target_register = MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_LIMIT; const u32 reset_invalid_value = 0xC0FFE000; const u32 reset_check_value = 0xC01A0000; const u32 reset_default_value = 0; @@ -296,7 +296,7 @@ static void mali_gp_irq_probe_trigger(void *data) struct mali_gp_core *core = (struct mali_gp_core *)data; mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); - mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT, MALIGP2_REG_VAL_CMD_FORCE_HANG); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT, MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR); _mali_osk_mem_barrier(); } @@ -306,8 +306,8 @@ static _mali_osk_errcode_t mali_gp_irq_probe_ack(void *data) u32 irq_readout; irq_readout = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_STAT); - if (MALIGP2_REG_VAL_IRQ_FORCE_HANG & irq_readout) { - mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_FORCE_HANG); + if (MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR & irq_readout) { + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR); _mali_osk_mem_barrier(); return _MALI_OSK_ERR_OK; } @@ -340,6 +340,7 @@ void mali_gp_update_performance_counters(struct mali_gp_core *core, struct mali_ #if defined(CONFIG_MALI400_PROFILING) _mali_osk_profiling_report_hw_counter(COUNTER_VP_0_C0, val0); + _mali_osk_profiling_record_global_counters(COUNTER_VP_0_C0, val0); #endif } @@ -350,6 +351,7 @@ void mali_gp_update_performance_counters(struct mali_gp_core *core, struct mali_ #if defined(CONFIG_MALI400_PROFILING) _mali_osk_profiling_report_hw_counter(COUNTER_VP_0_C1, val1); + _mali_osk_profiling_record_global_counters(COUNTER_VP_0_C1, val1); #endif } } diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp.h b/drivers/gpu/arm/mali400/mali/common/mali_gp.h index 93a7365b1a54..8d5f69c23229 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_gp.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_gp.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -61,7 +61,7 @@ MALI_STATIC_INLINE const char *mali_gp_core_description(struct mali_gp_core *cor MALI_STATIC_INLINE enum mali_interrupt_result mali_gp_get_interrupt_result(struct mali_gp_core *core) { u32 stat_used = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_STAT) & - MALIGP2_REG_VAL_IRQ_MASK_USED; + MALIGP2_REG_VAL_IRQ_MASK_USED; if (0 == stat_used) { return MALI_INTERRUPT_RESULT_NONE; diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c index b34ad9141edb..adc30a3408a8 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -12,35 +12,86 @@ #include "mali_osk.h" #include "mali_osk_list.h" #include "mali_uk_types.h" +#include "mali_memory_virtual.h" +#include "mali_memory_defer_bind.h" static u32 gp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ static u32 gp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ +static void _mali_gp_del_varying_allocations(struct mali_gp_job *job); + + +static int _mali_gp_add_varying_allocations(struct mali_session_data *session, + struct mali_gp_job *job, + u32 *alloc, + u32 num) +{ + int i = 0; + struct mali_gp_allocation_node *alloc_node; + mali_mem_allocation *mali_alloc = NULL; + struct mali_vma_node *mali_vma_node = NULL; + + for (i = 0 ; i < num ; i++) { + MALI_DEBUG_ASSERT(alloc[i]); + alloc_node = _mali_osk_calloc(1, sizeof(struct mali_gp_allocation_node)); + if (alloc_node) { + INIT_LIST_HEAD(&alloc_node->node); + /* find mali allocation structure by vaddress*/ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, alloc[i], 0); + + if (likely(mali_vma_node)) { + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(alloc[i] == mali_vma_node->vm_node.start); + } else { + MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,can't find allocation %d by address =0x%x, num=%d\n", i, alloc[i], num)); + MALI_DEBUG_ASSERT(0); + } + alloc_node->alloc = mali_alloc; + /* add to gp job varying alloc list*/ + list_move(&alloc_node->node, &job->varying_alloc); + } else + goto fail; + } + + return 0; +fail: + MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,failed to alloc memory!\n")); + _mali_gp_del_varying_allocations(job); + return -1; +} + + +static void _mali_gp_del_varying_allocations(struct mali_gp_job *job) +{ + struct mali_gp_allocation_node *alloc_node, *tmp_node; + + list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) { + list_del(&alloc_node->node); + kfree(alloc_node); + } + INIT_LIST_HEAD(&job->varying_alloc); +} struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker) { struct mali_gp_job *job; u32 perf_counter_flag; + u32 __user *memory_list = NULL; + struct mali_gp_allocation_node *alloc_node, *tmp_node; - job = _mali_osk_malloc(sizeof(struct mali_gp_job)); + job = _mali_osk_calloc(1, sizeof(struct mali_gp_job)); if (NULL != job) { job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_FINISHED, sizeof(_mali_uk_gp_job_finished_s)); if (NULL == job->finished_notification) { - _mali_osk_free(job); - return NULL; + goto fail3; } job->oom_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_STALLED, sizeof(_mali_uk_gp_job_suspended_s)); if (NULL == job->oom_notification) { - _mali_osk_notification_delete(job->finished_notification); - _mali_osk_free(job); - return NULL; + goto fail2; } if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) { - _mali_osk_notification_delete(job->finished_notification); - _mali_osk_notification_delete(job->oom_notification); - _mali_osk_free(job); - return NULL; + goto fail1; } perf_counter_flag = mali_gp_job_get_perf_counter_flag(job); @@ -56,12 +107,71 @@ struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_ _mali_osk_list_init(&job->list); job->session = session; job->id = id; + job->heap_base_addr = job->uargs.frame_registers[4]; job->heap_current_addr = job->uargs.frame_registers[4]; + job->heap_grow_size = job->uargs.heap_grow_size; job->perf_counter_value0 = 0; job->perf_counter_value1 = 0; job->pid = _mali_osk_get_pid(); job->tid = _mali_osk_get_tid(); + + INIT_LIST_HEAD(&job->varying_alloc); + INIT_LIST_HEAD(&job->vary_todo); + job->dmem = NULL; + /* add varying allocation list*/ + if (uargs->varying_alloc_num) { + /* copy varying list from user space*/ + job->varying_list = _mali_osk_calloc(1, sizeof(u32) * uargs->varying_alloc_num); + if (!job->varying_list) { + MALI_PRINT_ERROR(("Mali GP job: allocate varying_list failed varying_alloc_num = %d !\n", uargs->varying_alloc_num)); + goto fail1; + } + + memory_list = (u32 __user *)(uintptr_t)uargs->varying_alloc_list; + + if (0 != _mali_osk_copy_from_user(job->varying_list, memory_list, sizeof(u32)*uargs->varying_alloc_num)) { + MALI_PRINT_ERROR(("Mali GP job: Failed to copy varying list from user space!\n")); + goto fail; + } + + if (unlikely(_mali_gp_add_varying_allocations(session, job, job->varying_list, + uargs->varying_alloc_num))) { + MALI_PRINT_ERROR(("Mali GP job: _mali_gp_add_varying_allocations failed!\n")); + goto fail; + } + + /* do preparetion for each allocation */ + list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) { + if (unlikely(_MALI_OSK_ERR_OK != mali_mem_defer_bind_allocation_prepare(alloc_node->alloc, &job->vary_todo))) { + MALI_PRINT_ERROR(("Mali GP job: mali_mem_defer_bind_allocation_prepare failed!\n")); + goto fail; + } + } + + _mali_gp_del_varying_allocations(job); + + /* bind varying here, to avoid memory latency issue. */ + { + struct mali_defer_mem_block dmem_block; + + INIT_LIST_HEAD(&dmem_block.free_pages); + atomic_set(&dmem_block.num_free_pages, 0); + + if (mali_mem_prepare_mem_for_job(job, &dmem_block)) { + MALI_PRINT_ERROR(("Mali GP job: mali_mem_prepare_mem_for_job failed!\n")); + goto fail; + } + if (_MALI_OSK_ERR_OK != mali_mem_defer_bind(job->uargs.varying_memsize / _MALI_OSK_MALI_PAGE_SIZE, job, &dmem_block)) { + MALI_PRINT_ERROR(("gp job create, mali_mem_defer_bind failed! GP %x fail!", job)); + goto fail; + } + } + + if (uargs->varying_memsize > MALI_UK_BIG_VARYING_SIZE) { + job->big_job = 1; + } + } job->pp_tracker = pp_tracker; if (NULL != job->pp_tracker) { /* Take a reference on PP job's tracker that will be released when the GP @@ -73,16 +183,50 @@ struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_ mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence)); return job; + } else { + MALI_PRINT_ERROR(("Mali GP job: _mali_osk_calloc failed!\n")); + return NULL; } + +fail: + _mali_osk_free(job->varying_list); + /* Handle allocate fail here, free all varying node */ + { + struct mali_backend_bind_list *bkn, *bkn_tmp; + list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) { + list_del(&bkn->node); + _mali_osk_free(bkn); + } + } +fail1: + _mali_osk_notification_delete(job->oom_notification); +fail2: + _mali_osk_notification_delete(job->finished_notification); +fail3: + _mali_osk_free(job); return NULL; } void mali_gp_job_delete(struct mali_gp_job *job) { + struct mali_backend_bind_list *bkn, *bkn_tmp; MALI_DEBUG_ASSERT_POINTER(job); MALI_DEBUG_ASSERT(NULL == job->pp_tracker); MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); + _mali_osk_free(job->varying_list); + + /* Handle allocate fail here, free all varying node */ + list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) { + list_del(&bkn->node); + _mali_osk_free(bkn); + } + + if (!list_empty(&job->vary_todo)) { + MALI_DEBUG_ASSERT(0); + } + + mali_mem_defer_dmem_free(job); /* de-allocate the pre-allocated oom notifications */ if (NULL != job->oom_notification) { diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h index 51c66eaa8b6a..f249439c7155 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -21,6 +21,7 @@ #include "mali_executor.h" #include "mali_timeline.h" +struct mali_defer_mem; /** * This structure represents a GP job * @@ -64,9 +65,25 @@ struct mali_gp_job { * returning job to user. Hold executor lock when setting, * no lock needed when reading */ + u32 heap_base_addr; /** < Holds the base mali addr of mem handle which is used for new heap*/ u32 heap_current_addr; /**< Holds the current HEAP address when the job has completed */ + u32 heap_grow_size; /** < Holds the HEAP grow size when HEAP oom */ u32 perf_counter_value0; /**< Value of performance counter 0 (to be returned to user space) */ u32 perf_counter_value1; /**< Value of performance counter 1 (to be returned to user space) */ + struct mali_defer_mem *dmem; /** < used for defer bind to store dmem info */ + struct list_head varying_alloc; /**< hold the list of varying allocations */ + u32 bind_flag; /** < flag for deferbind*/ + u32 *varying_list; /**< varying memory list need to to defer bind*/ + struct list_head vary_todo; /**< list of backend list need to do defer bind*/ + u32 big_job; /** < if the gp job have large varying output and may take long time*/ +}; + +#define MALI_DEFER_BIND_MEMORY_PREPARED (0x1 << 0) +#define MALI_DEFER_BIND_MEMORY_BINDED (0x1 << 2) + +struct mali_gp_allocation_node { + struct list_head node; + mali_mem_allocation *alloc; }; struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker); diff --git a/drivers/gpu/arm/mali400/mali/common/mali_group.c b/drivers/gpu/arm/mali400/mali/common/mali_group.c index f31cb3e47f4c..b4cd3a1fcd8c 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_group.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_group.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -44,6 +44,7 @@ static void mali_group_bottom_half_mmu(void *data); static void mali_group_bottom_half_gp(void *data); static void mali_group_bottom_half_pp(void *data); static void mali_group_timeout(void *data); +static void mali_group_out_of_memory(void *data); static void mali_group_reset_pp(struct mali_group *group); static void mali_group_reset_mmu(struct mali_group *group); @@ -195,6 +196,11 @@ _mali_osk_errcode_t mali_group_add_gp_core(struct mali_group *group, struct mali if (NULL == group->bottom_half_work_gp) { return _MALI_OSK_ERR_FAULT; } + + group->oom_work_handler = _mali_osk_wq_create_work(mali_group_out_of_memory, group); + if (NULL == group->oom_work_handler) { + _mali_osk_wq_delete_work(group->bottom_half_work_gp); + } return _MALI_OSK_ERR_OK; } @@ -205,6 +211,10 @@ void mali_group_remove_gp_core(struct mali_group *group) if (NULL != group->bottom_half_work_gp) { _mali_osk_wq_delete_work(group->bottom_half_work_gp); } + + if (NULL != group->oom_work_handler) { + _mali_osk_wq_delete_work(group->oom_work_handler); + } } _mali_osk_errcode_t mali_group_add_pp_core(struct mali_group *group, struct mali_pp_core *pp_core) @@ -460,7 +470,7 @@ void mali_group_power_up(struct mali_group *group) group->power_is_on = MALI_TRUE; if (MALI_FALSE == mali_group_is_virtual(group) - && MALI_FALSE == mali_group_is_in_virtual(group)) { + && MALI_FALSE == mali_group_is_in_virtual(group)) { mali_group_reset(group); } @@ -520,6 +530,71 @@ MALI_DEBUG_CODE(static void mali_group_print_virtual(struct mali_group *vgroup) } }) +static void mali_group_dump_core_status(struct mali_group *group) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT(NULL != group->gp_core || (NULL != group->pp_core && !mali_group_is_virtual(group))); + + if (NULL != group->gp_core) { + MALI_PRINT(("Dump Group %s\n", group->gp_core->hw_core.description)); + + for (i = 0; i < 0xA8; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->gp_core->hw_core, i), + mali_hw_core_register_read(&group->gp_core->hw_core, i + 4), + mali_hw_core_register_read(&group->gp_core->hw_core, i + 8), + mali_hw_core_register_read(&group->gp_core->hw_core, i + 12))); + } + + + } else { + MALI_PRINT(("Dump Group %s\n", group->pp_core->hw_core.description)); + + for (i = 0; i < 0x5c; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 4), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 8), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 12))); + } + + /* Ignore some minor registers */ + for (i = 0x1000; i < 0x1068; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 4), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 8), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 12))); + } + } + + MALI_PRINT(("Dump Group MMU\n")); + for (i = 0; i < 0x24; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->mmu->hw_core, i), + mali_hw_core_register_read(&group->mmu->hw_core, i + 4), + mali_hw_core_register_read(&group->mmu->hw_core, i + 8), + mali_hw_core_register_read(&group->mmu->hw_core, i + 12))); + } +} + + +/** + * @Dump group status + */ +void mali_group_dump_status(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + + if (mali_group_is_virtual(group)) { + struct mali_group *group_c; + struct mali_group *temp; + _MALI_OSK_LIST_FOREACHENTRY(group_c, temp, &group->group_list, struct mali_group, group_list) { + mali_group_dump_core_status(group_c); + } + } else { + mali_group_dump_core_status(group); + } +} + /** * @brief Add child group to virtual group parent */ @@ -1353,6 +1428,15 @@ _mali_osk_errcode_t mali_group_upper_half_mmu(void *data) MALI_DEBUG_ASSERT_POINTER(group); MALI_DEBUG_ASSERT_POINTER(group->mmu); +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif if (NULL != group->gp_core) { _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | @@ -1370,9 +1454,23 @@ _mali_osk_errcode_t mali_group_upper_half_mmu(void *data) mali_pp_core_get_id(group->pp_core)), mali_mmu_get_rawstat(group->mmu), 0); } +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif ret = mali_executor_interrupt_mmu(group, MALI_TRUE); +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + if (NULL != group->gp_core) { _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | @@ -1389,6 +1487,10 @@ _mali_osk_errcode_t mali_group_upper_half_mmu(void *data) mali_pp_core_get_id(group->pp_core)), mali_mmu_get_rawstat(group->mmu), 0); } +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif return ret; } @@ -1447,6 +1549,15 @@ _mali_osk_errcode_t mali_group_upper_half_gp(void *data) MALI_DEBUG_ASSERT_POINTER(group->gp_core); MALI_DEBUG_ASSERT_POINTER(group->mmu); +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, @@ -1457,16 +1568,31 @@ _mali_osk_errcode_t mali_group_upper_half_gp(void *data) MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n", mali_gp_get_rawstat(group->gp_core), mali_group_core_description(group))); - +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif ret = mali_executor_interrupt_gp(group, MALI_TRUE); +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, 0, 0, /* No pid and tid for interrupt handler */ MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), mali_gp_get_rawstat(group->gp_core), 0); - +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif return ret; } @@ -1504,6 +1630,16 @@ _mali_osk_errcode_t mali_group_upper_half_pp(void *data) MALI_DEBUG_ASSERT_POINTER(group->pp_core); MALI_DEBUG_ASSERT_POINTER(group->mmu); +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, @@ -1515,9 +1651,22 @@ _mali_osk_errcode_t mali_group_upper_half_pp(void *data) MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n", mali_pp_get_rawstat(group->pp_core), mali_group_core_description(group))); +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif ret = mali_executor_interrupt_pp(group, MALI_TRUE); +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, @@ -1525,7 +1674,10 @@ _mali_osk_errcode_t mali_group_upper_half_pp(void *data) MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( mali_pp_core_get_id(group->pp_core)), mali_pp_get_rawstat(group->pp_core), 0); - +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif return ret; } @@ -1573,6 +1725,17 @@ static void mali_group_timeout(void *data) } } +static void mali_group_out_of_memory(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + mali_executor_group_oom(group); +} + mali_bool mali_group_zap_session(struct mali_group *group, struct mali_session_data *session) { diff --git a/drivers/gpu/arm/mali400/mali/common/mali_group.h b/drivers/gpu/arm/mali400/mali/common/mali_group.h index 49f841061b59..705605ec1d9f 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_group.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_group.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -84,6 +84,7 @@ struct mali_group { _mali_osk_wq_work_t *bottom_half_work_gp; _mali_osk_wq_work_t *bottom_half_work_pp; + _mali_osk_wq_work_t *oom_work_handler; _mali_osk_timer_t *timeout_timer; }; @@ -96,6 +97,8 @@ struct mali_group *mali_group_create(struct mali_l2_cache_core *core, struct mali_bcast_unit *bcast, u32 domain_index); +void mali_group_dump_status(struct mali_group *group); + void mali_group_delete(struct mali_group *group); _mali_osk_errcode_t mali_group_add_mmu_core(struct mali_group *group, @@ -126,7 +129,7 @@ MALI_STATIC_INLINE mali_bool mali_group_is_virtual(struct mali_group *group) { MALI_DEBUG_ASSERT_POINTER(group); -#if defined(CONFIG_MALI450) +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) return (NULL != group->dlbu_core); #else return MALI_FALSE; @@ -140,7 +143,7 @@ MALI_STATIC_INLINE mali_bool mali_group_is_in_virtual(struct mali_group *group) MALI_DEBUG_ASSERT_POINTER(group); MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); -#if defined(CONFIG_MALI450) +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) return (NULL != group->parent_group) ? MALI_TRUE : MALI_FALSE; #else return MALI_FALSE; @@ -409,6 +412,13 @@ MALI_STATIC_INLINE void mali_group_schedule_bottom_half_gp(struct mali_group *gr _mali_osk_wq_schedule_work(group->bottom_half_work_gp); } +MALI_STATIC_INLINE void mali_group_schedule_oom_work_handler(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + _mali_osk_wq_schedule_work(group->oom_work_handler); +} + MALI_STATIC_INLINE void mali_group_schedule_bottom_half_pp(struct mali_group *group) { MALI_DEBUG_ASSERT_POINTER(group); diff --git a/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c index f90620899f63..c90cf38d8516 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h index 47899fde6dcb..ac2ffbedf308 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -26,7 +26,7 @@ struct mali_hw_core { const char *description; /**< Name of unit (as specified in device configuration) */ }; -#define MALI_REG_POLL_COUNT_FAST 1000 +#define MALI_REG_POLL_COUNT_FAST 1000000 #define MALI_REG_POLL_COUNT_SLOW 1000000 /* diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h index f6e14f7810b4..990cf3ac78e0 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c index 4f0e0b76cdcf..6dca53af2e47 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -41,6 +41,7 @@ #endif #include "mali_control_timer.h" #include "mali_dvfs_policy.h" +#include #define MALI_SHARED_MEMORY_DEFAULT_SIZE 0xffffffff @@ -74,6 +75,7 @@ static u32 global_gpu_major_version = 0; static u32 global_gpu_minor_version = 0; mali_bool mali_gpu_class_is_mali450 = MALI_FALSE; +mali_bool mali_gpu_class_is_mali470 = MALI_FALSE; static _mali_osk_errcode_t mali_set_global_gpu_base_address(void) { @@ -154,6 +156,10 @@ static _mali_osk_errcode_t mali_parse_product_info(void) global_product_id = _MALI_PRODUCT_ID_MALI450; MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-450 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); break; + case MALI470_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI470; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-470 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + break; default: MALI_DEBUG_PRINT(2, ("Found unknown Mali GPU (r%up%u)\n", global_gpu_major_version, global_gpu_minor_version)); return _MALI_OSK_ERR_FAULT; @@ -276,6 +282,20 @@ static _mali_osk_errcode_t mali_parse_config_l2_cache(void) return _MALI_OSK_ERR_FAULT; } } + } else if (mali_is_mali470()) { + _mali_osk_resource_t l2c1_resource; + + /* Make cluster for L2C1 */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI470_OFFSET_L2_CACHE1, &l2c1_resource)) { + MALI_DEBUG_PRINT(3, ("Creating Mali-470 L2 cache 1\n")); + l2_cache = mali_create_l2_cache_core(&l2c1_resource, MALI_DOMAIN_INDEX_L21); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } else { + MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for L2C1\n")); + return _MALI_OSK_ERR_FAULT; + } } return _MALI_OSK_ERR_OK; @@ -443,7 +463,7 @@ static _mali_osk_errcode_t mali_parse_config_groups(void) _mali_osk_errcode_t resource_dlbu_found; _mali_osk_errcode_t resource_bcast_found; - if (!(mali_is_mali400() || mali_is_mali450())) { + if (!(mali_is_mali400() || mali_is_mali450() || mali_is_mali470())) { /* No known HW core */ return _MALI_OSK_ERR_FAULT; } @@ -486,7 +506,7 @@ static _mali_osk_errcode_t mali_parse_config_groups(void) resource_pp_mmu_found[7] = _mali_osk_resource_find(MALI_OFFSET_PP7_MMU, &(resource_pp_mmu[7])); - if (mali_is_mali450()) { + if (mali_is_mali450() || mali_is_mali470()) { resource_bcast_found = _mali_osk_resource_find(MALI_OFFSET_BCAST, &resource_bcast); resource_dlbu_found = _mali_osk_resource_find(MALI_OFFSET_DLBU, &resource_dlbu); resource_pp_mmu_bcast_found = _mali_osk_resource_find(MALI_OFFSET_PP_BCAST_MMU, &resource_pp_mmu_bcast); @@ -496,7 +516,7 @@ static _mali_osk_errcode_t mali_parse_config_groups(void) _MALI_OSK_ERR_OK != resource_dlbu_found || _MALI_OSK_ERR_OK != resource_pp_mmu_bcast_found || _MALI_OSK_ERR_OK != resource_pp_bcast_found) { - /* Missing mandatory core(s) for Mali-450 */ + /* Missing mandatory core(s) for Mali-450 or Mali-470 */ MALI_DEBUG_PRINT(2, ("Missing mandatory resources, Mali-450 needs DLBU, Broadcast unit, virtual PP core and virtual MMU\n")); return _MALI_OSK_ERR_FAULT; } @@ -555,7 +575,7 @@ static _mali_osk_errcode_t mali_parse_config_groups(void) } } - if (mali_is_mali450()) { + if (mali_is_mali450() || mali_is_mali470()) { _mali_osk_errcode_t err = mali_create_virtual_group(&resource_pp_mmu_bcast, &resource_pp_bcast, &resource_dlbu, &resource_bcast); if (_MALI_OSK_ERR_OK != err) { return err; @@ -622,7 +642,7 @@ static _mali_osk_errcode_t mali_parse_config_memory(void) } if (MALI_SHARED_MEMORY_DEFAULT_SIZE == mali_shared_mem_size && - 0 != data.shared_mem_size) { + 0 != data.shared_mem_size) { mali_shared_mem_size = data.shared_mem_size; } } @@ -684,14 +704,16 @@ static _mali_osk_errcode_t mali_parse_config_memory(void) static void mali_detect_gpu_class(void) { - if (_mali_osk_l2_resource_count() > 1) { + if (_mali_osk_identify_gpu_resource() == 0x450) mali_gpu_class_is_mali450 = MALI_TRUE; - } + + if (_mali_osk_identify_gpu_resource() == 0x470) + mali_gpu_class_is_mali470 = MALI_TRUE; } static _mali_osk_errcode_t mali_init_hw_reset(void) { -#if defined(CONFIG_MALI450) +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) _mali_osk_resource_t resource_bcast; /* Ensure broadcast unit is in a good state before we start creating @@ -707,7 +729,7 @@ static _mali_osk_errcode_t mali_init_hw_reset(void) } mali_bcast_unit_delete(bcast_core); } -#endif /* CONFIG_MALI450 */ +#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ return _MALI_OSK_ERR_OK; } @@ -825,7 +847,7 @@ _mali_osk_errcode_t mali_initialize_subsystems(void) return err; } - if (mali_is_mali450()) { + if (mali_is_mali450() || mali_is_mali470()) { err = mali_dlbu_initialize(); if (_MALI_OSK_ERR_OK != err) { mali_pm_init_end(); @@ -903,7 +925,7 @@ void mali_terminate_subsystems(void) mali_delete_l2_cache_cores(); mali_mmu_terminate(); - if (mali_is_mali450()) { + if (mali_is_mali450() || mali_is_mali470()) { mali_dlbu_terminate(); } @@ -974,7 +996,7 @@ _mali_osk_errcode_t _mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s * args->version = _MALI_UK_API_VERSION; /* report our version */ /* success regardless of being compatible or not */ - return _MALI_OSK_ERR_OK;; + return _MALI_OSK_ERR_OK; } _mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s *args) @@ -995,7 +1017,7 @@ _mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notificati if (NULL == queue) { MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n")); args->type = _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS; - return _MALI_OSK_ERR_OK;; + return _MALI_OSK_ERR_OK; } /* receive a notification, might sleep */ @@ -1011,7 +1033,7 @@ _mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notificati /* finished with the notification */ _mali_osk_notification_delete(notification); - return _MALI_OSK_ERR_OK;; /* all ok */ + return _MALI_OSK_ERR_OK; /* all ok */ } _mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *args) @@ -1030,7 +1052,7 @@ _mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *ar /* if the queue does not exist we're currently shutting down */ if (NULL == queue) { MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n")); - return _MALI_OSK_ERR_OK;; + return _MALI_OSK_ERR_OK; } notification = _mali_osk_notification_create(args->type, 0); @@ -1041,9 +1063,28 @@ _mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *ar _mali_osk_notification_queue_send(queue, notification); - return _MALI_OSK_ERR_OK;; /* all ok */ + return _MALI_OSK_ERR_OK; /* all ok */ +} + +_mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args) +{ + wait_queue_head_t *queue; + + /* check input */ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + queue = mali_session_get_wait_queue(); + + /* check pending big job number, might sleep if larger than MAX allowed number */ + if (wait_event_interruptible(*queue, MALI_MAX_PENDING_BIG_JOB > mali_scheduler_job_gp_big_job_count())) { + return _MALI_OSK_ERR_RESTARTSYSCALL; + } + + return _MALI_OSK_ERR_OK; /* all ok */ } + _mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args) { struct mali_session_data *session; @@ -1058,7 +1099,7 @@ _mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priori MALI_DEBUG_PRINT(2, ("Session 0x%08X with pid %d was granted higher priority.\n", session, _mali_osk_get_pid())); } - return _MALI_OSK_ERR_OK;; + return _MALI_OSK_ERR_OK; } _mali_osk_errcode_t _mali_ukk_open(void **context) @@ -1075,22 +1116,17 @@ _mali_osk_errcode_t _mali_ukk_open(void **context) /* create a response queue for this session */ session->ioctl_queue = _mali_osk_notification_queue_init(); if (NULL == session->ioctl_queue) { - _mali_osk_free(session); - MALI_ERROR(_MALI_OSK_ERR_NOMEM); + goto err; } session->page_directory = mali_mmu_pagedir_alloc(); if (NULL == session->page_directory) { - _mali_osk_notification_queue_term(session->ioctl_queue); - _mali_osk_free(session); - MALI_ERROR(_MALI_OSK_ERR_NOMEM); + goto err_mmu; } if (_MALI_OSK_ERR_OK != mali_mmu_pagedir_map(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE)) { MALI_PRINT_ERROR(("Failed to map DLBU page into session\n")); - _mali_osk_notification_queue_term(session->ioctl_queue); - _mali_osk_free(session); - MALI_ERROR(_MALI_OSK_ERR_NOMEM); + goto err_mmu; } if (0 != mali_dlbu_phys_addr) { @@ -1099,31 +1135,19 @@ _mali_osk_errcode_t _mali_ukk_open(void **context) } if (_MALI_OSK_ERR_OK != mali_memory_session_begin(session)) { - mali_mmu_pagedir_free(session->page_directory); - _mali_osk_notification_queue_term(session->ioctl_queue); - _mali_osk_free(session); - MALI_ERROR(_MALI_OSK_ERR_NOMEM); + goto err_session; } /* Create soft system. */ session->soft_job_system = mali_soft_job_system_create(session); if (NULL == session->soft_job_system) { - mali_memory_session_end(session); - mali_mmu_pagedir_free(session->page_directory); - _mali_osk_notification_queue_term(session->ioctl_queue); - _mali_osk_free(session); - MALI_ERROR(_MALI_OSK_ERR_NOMEM); + goto err_soft; } /* Create timeline system. */ session->timeline_system = mali_timeline_system_create(session); if (NULL == session->timeline_system) { - mali_soft_job_system_destroy(session->soft_job_system); - mali_memory_session_end(session); - mali_mmu_pagedir_free(session->page_directory); - _mali_osk_notification_queue_term(session->ioctl_queue); - _mali_osk_free(session); - MALI_ERROR(_MALI_OSK_ERR_NOMEM); + goto err_time_line; } #if defined(CONFIG_MALI_DVFS) @@ -1142,15 +1166,31 @@ _mali_osk_errcode_t _mali_ukk_open(void **context) session->pid = _mali_osk_get_pid(); session->comm = _mali_osk_get_comm(); - session->max_mali_mem_allocated = 0; - _mali_osk_memset(session->mali_mem_array, 0, sizeof(size_t) * MALI_MEM_TYPE_MAX); + session->max_mali_mem_allocated_size = 0; + for (i = 0; i < MALI_MEM_TYPE_MAX; i ++) { + atomic_set(&session->mali_mem_array[i], 0); + } + atomic_set(&session->mali_mem_allocated_pages, 0); *context = (void *)session; /* Add session to the list of all sessions. */ mali_session_add(session); MALI_DEBUG_PRINT(3, ("Session started\n")); - return _MALI_OSK_ERR_OK;; + return _MALI_OSK_ERR_OK; + +err_time_line: + mali_soft_job_system_destroy(session->soft_job_system); +err_soft: + mali_memory_session_end(session); +err_session: + mali_mmu_pagedir_free(session->page_directory); +err_mmu: + _mali_osk_notification_queue_term(session->ioctl_queue); +err: + _mali_osk_free(session); + MALI_ERROR(_MALI_OSK_ERR_NOMEM); + } #if defined(DEBUG) @@ -1234,7 +1274,12 @@ _mali_osk_errcode_t _mali_ukk_close(void **context) _mali_osk_atomic_term(&session->number_of_window_jobs); #endif +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_stop_sampling(session->pid); +#endif + /* Free session data structures */ + mali_mmu_pagedir_unmap(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE); mali_mmu_pagedir_free(session->page_directory); _mali_osk_notification_queue_term(session->ioctl_queue); _mali_osk_free(session); diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h index 88dc80ed1349..8cdbc5af3205 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -19,9 +19,11 @@ typedef enum { _MALI_PRODUCT_ID_MALI300, _MALI_PRODUCT_ID_MALI400, _MALI_PRODUCT_ID_MALI450, + _MALI_PRODUCT_ID_MALI470, } _mali_product_id_t; extern mali_bool mali_gpu_class_is_mali450; +extern mali_bool mali_gpu_class_is_mali470; _mali_osk_errcode_t mali_initialize_subsystems(void); @@ -35,22 +37,21 @@ u32 mali_kernel_core_get_gpu_minor_version(void); u32 _mali_kernel_core_dump_state(char *buf, u32 size); +MALI_STATIC_INLINE mali_bool mali_is_mali470(void) +{ + return mali_gpu_class_is_mali470; +} + MALI_STATIC_INLINE mali_bool mali_is_mali450(void) { -#if defined(CONFIG_MALI450) return mali_gpu_class_is_mali450; -#else - return MALI_FALSE; -#endif } MALI_STATIC_INLINE mali_bool mali_is_mali400(void) { -#if !defined(CONFIG_MALI450) + if (mali_gpu_class_is_mali450 || mali_gpu_class_is_mali470) + return MALI_FALSE; + return MALI_TRUE; -#else - return !mali_gpu_class_is_mali450; -#endif } - #endif /* __MALI_KERNEL_CORE_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_descriptor_mapping.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_descriptor_mapping.c deleted file mode 100755 index ce33451baed7..000000000000 --- a/drivers/gpu/arm/mali400/mali/common/mali_kernel_descriptor_mapping.c +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the GNU General Public License version 2 - * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. - * - * A copy of the licence is included with the program, and can also be obtained from Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#include "mali_kernel_common.h" -#include "mali_kernel_descriptor_mapping.h" -#include "mali_osk.h" -#include "mali_osk_bitops.h" -#include "mali_memory_types.h" -#include "mali_session.h" - -#define MALI_PAD_INT(x) (((x) + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1)) - -/** - * Allocate a descriptor table capable of holding 'count' mappings - * @param count Number of mappings in the table - * @return Pointer to a new table, NULL on error - */ -static mali_descriptor_table *descriptor_table_alloc(int count); - -/** - * Free a descriptor table - * @param table The table to free - */ -static void descriptor_table_free(mali_descriptor_table *table); - -mali_descriptor_mapping *mali_descriptor_mapping_create(int init_entries, int max_entries) -{ - mali_descriptor_mapping *map = _mali_osk_calloc(1, sizeof(mali_descriptor_mapping)); - - init_entries = MALI_PAD_INT(init_entries); - max_entries = MALI_PAD_INT(max_entries); - - if (NULL != map) { - map->table = descriptor_table_alloc(init_entries); - if (NULL != map->table) { - map->lock = _mali_osk_mutex_rw_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP); - if (NULL != map->lock) { - _mali_osk_set_nonatomic_bit(0, map->table->usage); /* reserve bit 0 to prevent NULL/zero logic to kick in */ - map->max_nr_mappings_allowed = max_entries; - map->current_nr_mappings = init_entries; - return map; - } - descriptor_table_free(map->table); - } - _mali_osk_free(map); - } - return NULL; -} - -void mali_descriptor_mapping_destroy(mali_descriptor_mapping *map) -{ - descriptor_table_free(map->table); - _mali_osk_mutex_rw_term(map->lock); - _mali_osk_free(map); -} - -_mali_osk_errcode_t mali_descriptor_mapping_allocate_mapping(mali_descriptor_mapping *map, void *target, int *odescriptor) -{ - _mali_osk_errcode_t err = _MALI_OSK_ERR_FAULT; - int new_descriptor; - mali_mem_allocation *descriptor; - struct mali_session_data *session; - - MALI_DEBUG_ASSERT_POINTER(map); - MALI_DEBUG_ASSERT_POINTER(odescriptor); - MALI_DEBUG_ASSERT_POINTER(target); - - _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); - new_descriptor = _mali_osk_find_first_zero_bit(map->table->usage, map->current_nr_mappings); - if (new_descriptor == map->current_nr_mappings) { - /* no free descriptor, try to expand the table */ - mali_descriptor_table *new_table, * old_table; - if (map->current_nr_mappings >= map->max_nr_mappings_allowed) goto unlock_and_exit; - - map->current_nr_mappings += BITS_PER_LONG; - new_table = descriptor_table_alloc(map->current_nr_mappings); - if (NULL == new_table) goto unlock_and_exit; - - old_table = map->table; - _mali_osk_memcpy(new_table->usage, old_table->usage, (sizeof(unsigned long)*map->current_nr_mappings) / BITS_PER_LONG); - _mali_osk_memcpy(new_table->mappings, old_table->mappings, map->current_nr_mappings * sizeof(void *)); - map->table = new_table; - descriptor_table_free(old_table); - } - - /* we have found a valid descriptor, set the value and usage bit */ - _mali_osk_set_nonatomic_bit(new_descriptor, map->table->usage); - map->table->mappings[new_descriptor] = target; - *odescriptor = new_descriptor; - - /* To calculate the mali mem usage for the session */ - descriptor = (mali_mem_allocation *)target; - session = descriptor->session; - - MALI_DEBUG_ASSERT_POINTER(session); - - session->mali_mem_array[descriptor->type] += descriptor->size; - if ((MALI_MEM_OS == descriptor->type || MALI_MEM_BLOCK == descriptor->type) && - (session->mali_mem_array[MALI_MEM_OS] + session->mali_mem_array[MALI_MEM_BLOCK] > session->max_mali_mem_allocated)) { - session->max_mali_mem_allocated = session->mali_mem_array[MALI_MEM_OS] + session->mali_mem_array[MALI_MEM_BLOCK]; - } - err = _MALI_OSK_ERR_OK; - -unlock_and_exit: - _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); - MALI_ERROR(err); -} - -void mali_descriptor_mapping_call_for_each(mali_descriptor_mapping *map, void (*callback)(int, void *)) -{ - int i; - - MALI_DEBUG_ASSERT_POINTER(map); - MALI_DEBUG_ASSERT_POINTER(callback); - - _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); - /* id 0 is skipped as it's an reserved ID not mapping to anything */ - for (i = 1; i < map->current_nr_mappings; ++i) { - if (_mali_osk_test_bit(i, map->table->usage)) { - callback(i, map->table->mappings[i]); - } - } - _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); -} - -_mali_osk_errcode_t mali_descriptor_mapping_get(mali_descriptor_mapping *map, int descriptor, void **target) -{ - _mali_osk_errcode_t result = _MALI_OSK_ERR_FAULT; - MALI_DEBUG_ASSERT_POINTER(map); - _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); - if ((descriptor >= 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { - *target = map->table->mappings[descriptor]; - result = _MALI_OSK_ERR_OK; - } else *target = NULL; - _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); - MALI_ERROR(result); -} - -_mali_osk_errcode_t mali_descriptor_mapping_set(mali_descriptor_mapping *map, int descriptor, void *target) -{ - _mali_osk_errcode_t result = _MALI_OSK_ERR_FAULT; - _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); - if ((descriptor >= 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { - map->table->mappings[descriptor] = target; - result = _MALI_OSK_ERR_OK; - } - _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); - MALI_ERROR(result); -} - -void *mali_descriptor_mapping_free(mali_descriptor_mapping *map, int descriptor) -{ - void *old_value = NULL; - mali_mem_allocation *tmp_descriptor; - struct mali_session_data *session; - - _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); - if ((descriptor >= 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { - old_value = map->table->mappings[descriptor]; - map->table->mappings[descriptor] = NULL; - _mali_osk_clear_nonatomic_bit(descriptor, map->table->usage); - } - if (NULL != old_value) { - tmp_descriptor = (mali_mem_allocation *)old_value; - session = tmp_descriptor->session; - - MALI_DEBUG_ASSERT_POINTER(session); - - MALI_DEBUG_ASSERT(session->mali_mem_array[tmp_descriptor->type] >= tmp_descriptor->size); - - session->mali_mem_array[tmp_descriptor->type] -= tmp_descriptor->size; - } - _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); - - return old_value; -} - -static mali_descriptor_table *descriptor_table_alloc(int count) -{ - mali_descriptor_table *table; - - table = _mali_osk_calloc(1, sizeof(mali_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG) + (sizeof(void *) * count)); - - if (NULL != table) { - table->usage = (u32 *)((u8 *)table + sizeof(mali_descriptor_table)); - table->mappings = (void **)((u8 *)table + sizeof(mali_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG)); - } - - return table; -} - -static void descriptor_table_free(mali_descriptor_table *table) -{ - _mali_osk_free(table); -} \ No newline at end of file diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_descriptor_mapping.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_descriptor_mapping.h deleted file mode 100755 index 90ac9babe8d5..000000000000 --- a/drivers/gpu/arm/mali400/mali/common/mali_kernel_descriptor_mapping.h +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the GNU General Public License version 2 - * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. - * - * A copy of the licence is included with the program, and can also be obtained from Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -/** - * @file mali_kernel_descriptor_mapping.h - */ - -#ifndef __MALI_KERNEL_DESCRIPTOR_MAPPING_H__ -#define __MALI_KERNEL_DESCRIPTOR_MAPPING_H__ - -#include "mali_osk.h" - -struct mali_session_data; - -/** - * The actual descriptor mapping table, never directly accessed by clients - */ -typedef struct mali_descriptor_table { - u32 *usage; /**< Pointer to bitpattern indicating if a descriptor is valid/used or not */ - void **mappings; /**< Array of the pointers the descriptors map to */ -} mali_descriptor_table; - -/** - * The descriptor mapping object - * Provides a separate namespace where we can map an integer to a pointer - */ -typedef struct mali_descriptor_mapping { - _mali_osk_mutex_rw_t *lock; /**< Lock protecting access to the mapping object */ - int max_nr_mappings_allowed; /**< Max number of mappings to support in this namespace */ - int current_nr_mappings; /**< Current number of possible mappings */ - mali_descriptor_table *table; /**< Pointer to the current mapping table */ -} mali_descriptor_mapping; - -/** - * Create a descriptor mapping object - * Create a descriptor mapping capable of holding init_entries growable to max_entries - * @param init_entries Number of entries to preallocate memory for - * @param max_entries Number of entries to max support - * @return Pointer to a descriptor mapping object, NULL on failure - */ -mali_descriptor_mapping *mali_descriptor_mapping_create(int init_entries, int max_entries); - -/** - * Destroy a descriptor mapping object - * @param map The map to free - */ -void mali_descriptor_mapping_destroy(mali_descriptor_mapping *map); - -/** - * Allocate a new mapping entry (descriptor ID) - * Allocates a new entry in the map. - * @param map The map to allocate a new entry in - * @param target The value to map to - * @return The descriptor allocated, a negative value on error - */ -_mali_osk_errcode_t mali_descriptor_mapping_allocate_mapping(mali_descriptor_mapping *map, void *target, int *descriptor); - -/** - * Get the value mapped to by a descriptor ID - * @param map The map to lookup the descriptor id in - * @param descriptor The descriptor ID to lookup - * @param target Pointer to a pointer which will receive the stored value - * @return 0 on successful lookup, negative on error - */ -_mali_osk_errcode_t mali_descriptor_mapping_get(mali_descriptor_mapping *map, int descriptor, void **target); - -/** - * Set the value mapped to by a descriptor ID - * @param map The map to lookup the descriptor id in - * @param descriptor The descriptor ID to lookup - * @param target Pointer to replace the current value with - * @return 0 on successful lookup, negative on error - */ -_mali_osk_errcode_t mali_descriptor_mapping_set(mali_descriptor_mapping *map, int descriptor, void *target); - -/** - * Call the specified callback function for each descriptor in map. - * Entire function is mutex protected. - * @param map The map to do callbacks for - * @param callback A callback function which will be calle for each entry in map - */ -void mali_descriptor_mapping_call_for_each(mali_descriptor_mapping *map, void (*callback)(int, void *)); - -/** - * Free the descriptor ID - * For the descriptor to be reused it has to be freed - * @param map The map to free the descriptor from - * @param descriptor The descriptor ID to free - * - * @return old value of descriptor mapping - */ -void *mali_descriptor_mapping_free(mali_descriptor_mapping *map, int descriptor); - -#endif /* __MALI_KERNEL_DESCRIPTOR_MAPPING_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c index 8c6da7a15a4f..63b941742249 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -45,7 +45,7 @@ void (*mali_utilization_callback)(struct mali_gpu_utilization_data *data) = NULL static u32 mali_control_first_timeout = 100; static struct mali_gpu_utilization_data mali_util_data = {0, }; -struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period) +struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period, mali_bool *need_add_timer) { u64 time_now; u32 leading_zeroes; @@ -65,6 +65,7 @@ struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u6 *time_period = time_now - *start_time; if (accumulated_work_time_gpu == 0 && work_start_time_gpu == 0) { + mali_control_timer_pause(); /* * No work done for this period * - No need to reschedule timer @@ -80,8 +81,7 @@ struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u6 mali_utilization_data_unlock(); - /* Stop add timer until the next job submited */ - mali_control_timer_suspend(MALI_FALSE); + *need_add_timer = MALI_FALSE; mali_executor_hint_disable(MALI_EXECUTOR_HINT_GP_BOUND); @@ -172,6 +172,8 @@ struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u6 mali_utilization_data_unlock(); + *need_add_timer = MALI_TRUE; + MALI_DEBUG_PRINT(4, ("last_utilization_gpu = %d \n", last_utilization_gpu)); MALI_DEBUG_PRINT(4, ("last_utilization_gp = %d \n", last_utilization_gp)); MALI_DEBUG_PRINT(4, ("last_utilization_pp = %d \n", last_utilization_pp)); @@ -417,6 +419,11 @@ void mali_utilization_data_unlock(void) _mali_osk_spinlock_irq_unlock(utilization_data_lock); } +void mali_utilization_data_assert_locked(void) +{ + MALI_DEBUG_ASSERT_LOCK_HELD(utilization_data_lock); +} + u32 _mali_ukk_utilization_gp_pp(void) { return last_utilization_gpu; diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h index 1a22dc38e151..3c20b1983762 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -54,7 +54,7 @@ void mali_utilization_pp_end(void); /** * Should be called to calcution the GPU utilization */ -struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period); +struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period, mali_bool *need_add_timer); _mali_osk_spinlock_irq_t *mali_utilization_get_lock(void); @@ -64,6 +64,8 @@ void mali_utilization_data_lock(void); void mali_utilization_data_unlock(void); +void mali_utilization_data_assert_locked(void); + void mali_utilization_reset(void); diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c index d51e049e6174..2eed4c88cf56 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c index baaa6f5892a3..494ba789cd08 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -68,7 +68,7 @@ typedef enum mali_l2_cache_status { #define MALI400_L2_MAX_READS_NOT_SET -1 static struct mali_l2_cache_core * - mali_global_l2s[MALI_MAX_NUMBER_OF_L2_CACHE_CORES] = { NULL, }; + mali_global_l2s[MALI_MAX_NUMBER_OF_L2_CACHE_CORES] = { NULL, }; static u32 mali_global_num_l2s = 0; int mali_l2_max_reads = MALI400_L2_MAX_READS_NOT_SET; @@ -203,7 +203,8 @@ void mali_l2_cache_power_up(struct mali_l2_cache_core *cache) mali_l2_cache_reset(cache); - MALI_DEBUG_ASSERT(MALI_FALSE == cache->power_is_on); + if ((1 << MALI_DOMAIN_INDEX_DUMMY) != cache->pm_domain->pmu_mask) + MALI_DEBUG_ASSERT(MALI_FALSE == cache->power_is_on); cache->power_is_on = MALI_TRUE; mali_l2_cache_unlock(cache); diff --git a/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h index ebfb1f8abaf2..6dc8ec22d6de 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c index 17e0b9d5c323..0ab35094004b 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h index f0a76d1c8bc9..267720625d87 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu.c b/drivers/gpu/arm/mali400/mali/common/mali_mmu.c index 585e803fa38d..b975c1468d67 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_mmu.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu.h b/drivers/gpu/arm/mali400/mali/common/mali_mmu.h index 14dd085df090..101c968bd45d 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_mmu.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c index 3e0d9d40ad24..126fd77ec9c9 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -120,8 +120,8 @@ _mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u3 _mali_osk_errcode_t err; mali_io_address pde_mapping; mali_dma_addr pde_phys; - int i; - + int i, page_count; + u32 start_address; if (last_pde < first_pde) return _MALI_OSK_ERR_INVALID_ARGS; @@ -144,9 +144,20 @@ _mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u3 pde_phys | MALI_MMU_FLAGS_PRESENT); MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]); - pagedir->page_entries_usage_count[i] = 1; + } + + if (first_pde == last_pde) { + pagedir->page_entries_usage_count[i] += size / MALI_MMU_PAGE_SIZE; + } else if (i == first_pde) { + start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; + page_count = (start_address + MALI_MMU_VIRTUAL_PAGE_SIZE - mali_address) / MALI_MMU_PAGE_SIZE; + pagedir->page_entries_usage_count[i] += page_count; + } else if (i == last_pde) { + start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; + page_count = (mali_address + size - start_address) / MALI_MMU_PAGE_SIZE; + pagedir->page_entries_usage_count[i] += page_count; } else { - pagedir->page_entries_usage_count[i]++; + pagedir->page_entries_usage_count[i] = 1024; } } _mali_osk_write_mem_barrier(); @@ -198,7 +209,7 @@ _mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset; } - pagedir->page_entries_usage_count[i]--; + pagedir->page_entries_usage_count[i] -= size_in_pde / MALI_MMU_PAGE_SIZE; /* If entire page table is unused, free it */ if (0 == pagedir->page_entries_usage_count[i]) { diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h index a53fee966b5a..561fb60b9803 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk.h b/drivers/gpu/arm/mali400/mali/common/mali_osk.h index 184564e549be..4c9e57cba18f 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_osk.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -1323,6 +1323,58 @@ void _mali_osk_pm_dev_barrier(void); /** @} */ /* end group _mali_osk_miscellaneous */ +/** @defgroup _mali_osk_bitmap OSK Bitmap + * @{ */ + +/** @brief Allocate a unique number from the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @return An unique existence in the bitmap object. + */ +u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap); + +/** @brief Free a interger to the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @param obj An number allocated from bitmap object. + */ +void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj); + +/** @brief Allocate continuous number from the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @return start number of the continuous number block. + */ +u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt); + +/** @brief Free a block of continuous number block to the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @param obj Start number. + * @param cnt The size of the continuous number block. + */ +void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt); + +/** @brief Available count could be used to allocate in the given bitmap object. + * + */ +u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap); + +/** @brief Initialize an bitmap object.. + * + * @param bitmap An poiter of uninitialized bitmap object. + * @param num Size of thei bitmap object and decide the memory size allocated. + * @param reserve start number used to allocate. + */ +int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve); + +/** @brief Free the given bitmap object. + * + * @param bitmap Initialized bitmap object. + */ +void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap); +/** @} */ /* end group _mali_osk_bitmap */ + /** @} */ /* end group osuapi */ /** @} */ /* end group uddapi */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h index 525b944e9852..c1709f94c883 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h index 2cd190e7d497..22d22446d7a0 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h index efef034818fd..230dbecc2040 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -53,11 +53,15 @@ _mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res) */ uintptr_t _mali_osk_resource_base_address(void); -/** @brief Find the number of L2 cache cores. +/** @brief Find the specific GPU resource. + * + * @return value + * 0x400 if Mali 400 specific GPU resource identified + * 0x450 if Mali 450 specific GPU resource identified + * 0x470 if Mali 470 specific GPU resource identified * - * @return return the number of l2 cache cores we find in device resources. */ -u32 _mali_osk_l2_resource_count(void); +u32 _mali_osk_identify_gpu_resource(void); /** @brief Retrieve the Mali GPU specific data * diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h index 6463177a4dfe..10f4dc552b03 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -35,6 +35,11 @@ _mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start); */ void _mali_osk_profiling_term(void); +/** + * Stop the profile sampling operation. + */ +void _mali_osk_profiling_stop_sampling(u32 pid); + /** * Start recording profiling data * @@ -55,10 +60,8 @@ _mali_osk_errcode_t _mali_osk_profiling_start(u32 *limit); * @param data2 Third data parameter, depending on event_id specified. * @param data3 Fourth data parameter, depending on event_id specified. * @param data4 Fifth data parameter, depending on event_id specified. - * @return _MALI_OSK_ERR_OK on success, otherwise failure. */ -/* Call Linux tracepoint directly */ -#define _mali_osk_profiling_add_event(event_id, data0, data1, data2, data3, data4) trace_mali_timeline_event((event_id), (data0), (data1), (data2), (data3), (data4)) +void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4); /** * Report a hardware counter event. @@ -77,6 +80,8 @@ _mali_osk_errcode_t _mali_osk_profiling_start(u32 *limit); */ void _mali_osk_profiling_report_sw_counters(u32 *counters); +void _mali_osk_profiling_record_global_counters(int counter_id, u32 value); + /** * Stop recording profiling data * diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h index 899c570a6e0a..b65ad29e16c0 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -64,6 +64,9 @@ typedef unsigned long mali_bool; #define MALI_HW_CORE_NO_COUNTER ((u32)-1) + +#define MALI_S32_MAX 0x7fffffff + /** * @brief OSK Error codes * @@ -335,11 +338,11 @@ typedef struct _mali_io_address *mali_io_address; */ /** Mali Page Order, as log to base 2 of the Page size. @see _MALI_OSK_MALI_PAGE_SIZE */ -#define _MALI_OSK_MALI_PAGE_ORDER ((u32)12) +#define _MALI_OSK_MALI_PAGE_ORDER PAGE_SHIFT /** Mali Page Size, in bytes. */ -#define _MALI_OSK_MALI_PAGE_SIZE (((u32)1) << (_MALI_OSK_MALI_PAGE_ORDER)) +#define _MALI_OSK_MALI_PAGE_SIZE PAGE_SIZE /** Mali Page Mask, which masks off the offset within a page */ -#define _MALI_OSK_MALI_PAGE_MASK (~((((u32)1) << (_MALI_OSK_MALI_PAGE_ORDER)) - ((u32)1))) +#define _MALI_OSK_MALI_PAGE_MASK PAGE_MASK /** @} */ /* end of group _MALI_OSK_MALI_PAGE*/ /** @brief flags for mapping a user-accessible memory range @@ -449,6 +452,18 @@ typedef struct _mali_osk_wait_queue_t_struct _mali_osk_wait_queue_t; */ typedef struct seq_file _mali_osk_print_ctx; +#define _MALI_OSK_BITMAP_INVALIDATE_INDEX -1 + +typedef struct _mali_osk_bitmap { + u32 reserve; + u32 last; + u32 max; + u32 avail; + _mali_osk_spinlock_t *lock; + unsigned long *table; +} _mali_osk_bitmap_t; + + #ifdef __cplusplus } #endif diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm.c b/drivers/gpu/arm/mali400/mali/common/mali_pm.c index 65f268fb5481..dbd94d310741 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pm.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -709,7 +709,7 @@ static void mali_pm_update_sync_internal(void) struct mali_group *groups_up[MALI_MAX_NUMBER_OF_GROUPS]; u32 num_groups_up = 0; struct mali_l2_cache_core * - l2_up[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; + l2_up[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; u32 num_l2_up = 0; u32 i; @@ -779,7 +779,7 @@ static void mali_pm_update_sync_internal(void) struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS]; u32 num_groups_down = 0; struct mali_l2_cache_core * - l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; + l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; u32 num_l2_down = 0; u32 i; @@ -883,7 +883,7 @@ static mali_bool mali_pm_common_suspend(void) struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS]; u32 num_groups_down = 0; struct mali_l2_cache_core * - l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; + l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; u32 num_l2_down = 0; u32 i; @@ -974,6 +974,8 @@ static void mali_pm_set_default_pm_domain_config(void) domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 2; } else if (mali_is_mali450()) { domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 1; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 0; } } @@ -983,6 +985,8 @@ static void mali_pm_set_default_pm_domain_config(void) domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 3; } else if (mali_is_mali450()) { domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 2; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 1; } } @@ -992,6 +996,8 @@ static void mali_pm_set_default_pm_domain_config(void) domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 4; } else if (mali_is_mali450()) { domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 2; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 1; } } @@ -1001,6 +1007,8 @@ static void mali_pm_set_default_pm_domain_config(void) domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 5; } else if (mali_is_mali450()) { domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 2; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 1; } } @@ -1028,24 +1036,29 @@ static void mali_pm_set_default_pm_domain_config(void) /* L2gp/L2PP0/L2PP4 */ if (mali_is_mali400()) { if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( - MALI400_OFFSET_L2_CACHE0, NULL)) { - domain_config[MALI_DOMAIN_INDEX_L20] = 0x01 << 1; + MALI400_OFFSET_L2_CACHE0, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L20] = 0x01 << 1; } } else if (mali_is_mali450()) { if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( - MALI450_OFFSET_L2_CACHE0, NULL)) { + MALI450_OFFSET_L2_CACHE0, NULL)) { domain_config[MALI_DOMAIN_INDEX_L20] = 0x01 << 0; } if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( - MALI450_OFFSET_L2_CACHE1, NULL)) { + MALI450_OFFSET_L2_CACHE1, NULL)) { domain_config[MALI_DOMAIN_INDEX_L21] = 0x01 << 1; } if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( - MALI450_OFFSET_L2_CACHE2, NULL)) { + MALI450_OFFSET_L2_CACHE2, NULL)) { domain_config[MALI_DOMAIN_INDEX_L22] = 0x01 << 3; } + } else if (mali_is_mali470()) { + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI470_OFFSET_L2_CACHE1, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L21] = 0x01 << 0; + } } } @@ -1069,14 +1082,21 @@ static void mali_pm_set_pmu_domain_config(void) for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS - 1; i++) { if (0 != domain_config[i]) { + MALI_DEBUG_PRINT(2, ("Using customer pmu config:\n")); break; } } if (MALI_MAX_NUMBER_OF_DOMAINS - 1 == i) { + MALI_DEBUG_PRINT(2, ("Using hw detect pmu config:\n")); mali_pm_set_default_pm_domain_config(); } + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS - 1; i++) { + if (domain_config[i]) { + MALI_DEBUG_PRINT(2, ("domain_config[%d] = 0x%x \n", i, domain_config[i])); + } + } /* Can't override dummy domain mask */ domain_config[MALI_DOMAIN_INDEX_DUMMY] = 1 << MALI_DOMAIN_INDEX_DUMMY; @@ -1330,3 +1350,13 @@ void mali_pm_get_best_power_cost_mask(int num_requested, int *dst) _mali_osk_memcpy(dst, mali_pm_domain_power_cost_result[num_requested], MALI_MAX_NUMBER_OF_DOMAINS * sizeof(int)); } + +u32 mali_pm_get_current_mask(void) +{ + return pd_mask_current; +} + +u32 mali_pm_get_wanted_mask(void) +{ + return pd_mask_wanted; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm.h b/drivers/gpu/arm/mali400/mali/common/mali_pm.h index c6eae927045f..d72c732e698d 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pm.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -86,4 +86,6 @@ void mali_pm_get_best_power_cost_mask(int num_requested, int *dst); const char *mali_pm_mask_to_string(u32 mask); #endif +u32 mali_pm_get_current_mask(void); +u32 mali_pm_get_wanted_mask(void); #endif /* __MALI_PM_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c index 5bb5df3741dc..dbf985e6d37b 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h index 74d77c78134b..aceb3449359a 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pmu.c b/drivers/gpu/arm/mali400/mali/common/mali_pmu.c index 93dca0abaa56..2a3008a6dd83 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pmu.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_pmu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -143,6 +143,8 @@ _mali_osk_errcode_t mali_pmu_power_down(struct mali_pmu_core *pmu, u32 mask) */ MALI_DEBUG_ASSERT(0 == (stat & mask)); + mask &= ~(0x1 << MALI_DOMAIN_INDEX_DUMMY); + if (0 == mask || 0 == ((~stat) & mask)) return _MALI_OSK_ERR_OK; mali_hw_core_register_write(&pmu->hw_core, @@ -153,7 +155,7 @@ _mali_osk_errcode_t mali_pmu_power_down(struct mali_pmu_core *pmu, u32 mask) * powered off by our power down command, because the HW will simply * not generate an interrupt in this case. */ - if (mali_is_mali450() || pmu->registered_cores_mask != (mask | stat)) { + if (mali_is_mali450() || mali_is_mali470() || pmu->registered_cores_mask != (mask | stat)) { err = mali_pmu_wait_for_command_finish(pmu); if (_MALI_OSK_ERR_OK != err) { return err; @@ -195,6 +197,8 @@ _mali_osk_errcode_t mali_pmu_power_up(struct mali_pmu_core *pmu, u32 mask) stat = mali_hw_core_register_read(&pmu->hw_core, PMU_REG_ADDR_MGMT_STATUS); stat &= pmu->registered_cores_mask; + + mask &= ~(0x1 << MALI_DOMAIN_INDEX_DUMMY); if (0 == mask || 0 == (stat & mask)) return _MALI_OSK_ERR_OK; /* diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pmu.h b/drivers/gpu/arm/mali400/mali/common/mali_pmu.h index 69dac7f8aeb5..5ca78795f535 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pmu.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_pmu.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp.c b/drivers/gpu/arm/mali400/mali/common/mali_pp.c index 65b80250c554..68bfd50bf9ae 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pp.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_pp.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -43,7 +43,7 @@ struct mali_pp_core *mali_pp_create(const _mali_osk_resource_t *resource, struct return NULL; } - core = _mali_osk_malloc(sizeof(struct mali_pp_core)); + core = _mali_osk_calloc(1, sizeof(struct mali_pp_core)); if (NULL != core) { core->core_id = mali_global_num_pp_cores; core->bcast_id = bcast_id; @@ -202,6 +202,7 @@ static const u32 mali_perf_cnt_enable_reset_value = 0; _mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core) { /* Bus must be stopped before calling this function */ + const u32 reset_wait_target_register = MALI200_REG_ADDR_MGMT_PERF_CNT_0_LIMIT; const u32 reset_invalid_value = 0xC0FFE000; const u32 reset_check_value = 0xC01A0000; int i; @@ -210,7 +211,7 @@ _mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core) MALI_DEBUG_PRINT(2, ("Mali PP: Hard reset of core %s\n", core->hw_core.description)); /* Set register to a bogus value. The register will be used to detect when reset is complete */ - mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_WRITE_BOUNDARY_LOW, reset_invalid_value); + mali_hw_core_register_write_relaxed(&core->hw_core, reset_wait_target_register, reset_invalid_value); mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_NONE); /* Force core to reset */ @@ -218,8 +219,8 @@ _mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core) /* Wait for reset to be complete */ for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { - mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_WRITE_BOUNDARY_LOW, reset_check_value); - if (reset_check_value == mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_WRITE_BOUNDARY_LOW)) { + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_check_value); + if (reset_check_value == mali_hw_core_register_read(&core->hw_core, reset_wait_target_register)) { break; } } @@ -228,7 +229,7 @@ _mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core) MALI_PRINT_ERROR(("Mali PP: The hard reset loop didn't work, unable to recover\n")); } - mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_WRITE_BOUNDARY_LOW, 0x00000000); /* set it back to the default */ + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, 0x00000000); /* set it back to the default */ /* Re-enable interrupts */ mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_MASK_ALL); mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); @@ -411,7 +412,7 @@ static void mali_pp_irq_probe_trigger(void *data) { struct mali_pp_core *core = (struct mali_pp_core *)data; mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); - mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_FORCE_HANG); + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_BUS_ERROR); _mali_osk_mem_barrier(); } @@ -421,8 +422,8 @@ static _mali_osk_errcode_t mali_pp_irq_probe_ack(void *data) u32 irq_readout; irq_readout = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_STATUS); - if (MALI200_REG_VAL_IRQ_FORCE_HANG & irq_readout) { - mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_FORCE_HANG); + if (MALI200_REG_VAL_IRQ_BUS_ERROR & irq_readout) { + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_BUS_ERROR); _mali_osk_mem_barrier(); return _MALI_OSK_ERR_OK; } @@ -473,6 +474,7 @@ void mali_pp_update_performance_counters(struct mali_pp_core *parent, struct mal #if defined(CONFIG_MALI400_PROFILING) _mali_osk_profiling_report_hw_counter(counter_index, val0); + _mali_osk_profiling_record_global_counters(counter_index, val0); #endif } @@ -482,6 +484,7 @@ void mali_pp_update_performance_counters(struct mali_pp_core *parent, struct mal #if defined(CONFIG_MALI400_PROFILING) _mali_osk_profiling_report_hw_counter(counter_index + 1, val1); + _mali_osk_profiling_record_global_counters(counter_index + 1, val1); #endif } } diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp.h b/drivers/gpu/arm/mali400/mali/common/mali_pp.h index cc73e8653c15..45712a30e831 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pp.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_pp.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c index 3fc3f73cabbf..5528360841af 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -18,6 +18,8 @@ #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) #include "linux/mali_memory_dma_buf.h" #endif +#include "mali_memory_swap_alloc.h" +#include "mali_scheduler.h" static u32 pp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ static u32 pp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ @@ -43,6 +45,10 @@ struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, job = _mali_osk_calloc(1, sizeof(struct mali_pp_job)); if (NULL != job) { + + _mali_osk_list_init(&job->list); + _mali_osk_list_init(&job->session_fb_lookup_list); + if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) { goto fail; } @@ -77,7 +83,6 @@ struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, } } - _mali_osk_list_init(&job->list); job->session = session; job->id = id; @@ -87,17 +92,20 @@ struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, _mali_osk_atomic_init(&job->sub_jobs_completed, 0); _mali_osk_atomic_init(&job->sub_job_errors, 0); + job->swap_status = MALI_NO_SWAP_IN; + job->user_notification = MALI_FALSE; + job->num_pp_cores_in_virtual = 0; + + if (job->uargs.num_memory_cookies > session->allocation_mgr.mali_allocation_num) { + MALI_PRINT_ERROR(("Mali PP job: The number of memory cookies is invalid !\n")); + goto fail; + } if (job->uargs.num_memory_cookies > 0) { u32 size; u32 __user *memory_cookies = (u32 __user *)(uintptr_t)job->uargs.memory_cookies; - if (job->uargs.num_memory_cookies > session->descriptor_mapping->current_nr_mappings) { - MALI_PRINT_ERROR(("Mali PP job: Too many memory cookies specified in job object\n")); - goto fail; - } - - size = sizeof(*memory_cookies) * job->uargs.num_memory_cookies; + size = sizeof(*memory_cookies) * (job->uargs.num_memory_cookies); job->memory_cookies = _mali_osk_malloc(size); if (NULL == job->memory_cookies) { @@ -109,17 +117,6 @@ struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size)); goto fail; } - -#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) - if (0 < job->uargs.num_memory_cookies) { - job->dma_bufs = _mali_osk_calloc(job->uargs.num_memory_cookies, - sizeof(struct mali_dma_buf_attachment *)); - if (NULL == job->dma_bufs) { - MALI_PRINT_ERROR(("Mali PP job: Failed to allocate dma_bufs array!\n")); - goto fail; - } - } -#endif } if (_MALI_OSK_ERR_OK != mali_pp_job_check(job)) { @@ -130,6 +127,8 @@ struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_PP, NULL, job); mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence)); + mali_mem_swap_in_pages(job); + return job; } @@ -147,24 +146,27 @@ void mali_pp_job_delete(struct mali_pp_job *job) MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->session_fb_lookup_list)); - if (NULL != job->finished_notification) { - _mali_osk_notification_delete(job->finished_notification); - } - + if (NULL != job->memory_cookies) { #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) - /* Unmap buffers attached to job */ - if (0 < job->uargs.num_memory_cookies) { + /* Unmap buffers attached to job */ mali_dma_buf_unmap_job(job); - if (NULL != job->dma_bufs) { - _mali_osk_free(job->dma_bufs); +#endif + if (MALI_NO_SWAP_IN != job->swap_status) { + mali_mem_swap_out_pages(job); } - } -#endif /* CONFIG_DMA_SHARED_BUFFER */ - if (NULL != job->memory_cookies) { _mali_osk_free(job->memory_cookies); } + if (job->user_notification) { + mali_scheduler_return_pp_job_to_user(job, + job->num_pp_cores_in_virtual); + } + + if (NULL != job->finished_notification) { + _mali_osk_notification_delete(job->finished_notification); + } + _mali_osk_atomic_term(&job->sub_jobs_completed); _mali_osk_atomic_term(&job->sub_job_errors); diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h index e9e4783717f3..7b9d2efa3019 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -26,6 +26,12 @@ #include "linux/mali_memory_dma_buf.h" #endif +typedef enum pp_job_status { + MALI_NO_SWAP_IN, + MALI_SWAP_IN_FAIL, + MALI_SWAP_IN_SUCC, +} pp_job_status; + /** * This structure represents a PP job, including all sub jobs. * @@ -55,6 +61,10 @@ struct mali_pp_job { u32 perf_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS]; /**< Per sub job counters src1 */ u32 sub_jobs_num; /**< Number of subjobs; set to 1 for Mali-450 if DLBU is used, otherwise equals number of PP cores */ + pp_job_status swap_status; /**< Used to track each PP job swap status, if fail, we need to drop them in scheduler part */ + mali_bool user_notification; /**< When we deferred delete PP job, we need to judge if we need to send job finish notification to user space */ + u32 num_pp_cores_in_virtual; /**< How many PP cores we have when job finished */ + /* * These members are used by both scheduler and executor. * They are "protected" by atomic operations. @@ -68,9 +78,6 @@ struct mali_pp_job { * No lock is thus needed for these. */ u32 *memory_cookies; /**< Memory cookies attached to job */ -#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) - struct mali_dma_buf_attachment **dma_bufs; /**< Array of DMA-bufs used by job */ -#endif /* * These members are used by the scheduler, @@ -172,7 +179,7 @@ MALI_STATIC_INLINE u32 *mali_pp_job_get_dlbu_registers(struct mali_pp_job *job) MALI_STATIC_INLINE mali_bool mali_pp_job_is_virtual(struct mali_pp_job *job) { -#if defined(CONFIG_MALI450) +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) MALI_DEBUG_ASSERT_POINTER(job); return (0 == job->uargs.num_cores) ? MALI_TRUE : MALI_FALSE; #else @@ -419,32 +426,6 @@ MALI_STATIC_INLINE mali_bool mali_pp_job_needs_dma_buf_mapping(struct mali_pp_jo return MALI_FALSE; } -#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) -MALI_STATIC_INLINE u32 mali_pp_job_num_dma_bufs(struct mali_pp_job *job) -{ - MALI_DEBUG_ASSERT_POINTER(job); - return job->uargs.num_memory_cookies; -} - -MALI_STATIC_INLINE struct mali_dma_buf_attachment *mali_pp_job_get_dma_buf( - struct mali_pp_job *job, u32 index) -{ - MALI_DEBUG_ASSERT_POINTER(job); - MALI_DEBUG_ASSERT(index < job->uargs.num_memory_cookies); - MALI_DEBUG_ASSERT_POINTER(job->dma_bufs); - return job->dma_bufs[index]; -} - -MALI_STATIC_INLINE void mali_pp_job_set_dma_buf(struct mali_pp_job *job, - u32 index, struct mali_dma_buf_attachment *mem) -{ - MALI_DEBUG_ASSERT_POINTER(job); - MALI_DEBUG_ASSERT(index < job->uargs.num_memory_cookies); - MALI_DEBUG_ASSERT_POINTER(job->dma_bufs); - job->dma_bufs[index] = mem; -} -#endif - MALI_STATIC_INLINE void mali_pp_job_mark_sub_job_started(struct mali_pp_job *job, u32 sub_job) { MALI_DEBUG_ASSERT_POINTER(job); diff --git a/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c index a180fcd71e8d..5547159db94c 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -18,6 +18,9 @@ #include "mali_pp_job.h" #include "mali_executor.h" #include "mali_group.h" +#include +#include + #if defined(CONFIG_DMA_SHARED_BUFFER) #include "mali_memory_dma_buf.h" @@ -32,16 +35,16 @@ */ /* - * If dma_buf with map on demand is used, we defer job deletion and job queue + * If dma_buf with map on demand is used, we defer job queue * if in atomic context, since both might sleep. */ #if defined(CONFIG_DMA_SHARED_BUFFER) #if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) -#define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE 1 #define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE 1 #endif #endif + /* * ---------- global variables (exported due to inline functions) ---------- */ @@ -61,11 +64,9 @@ _mali_osk_atomic_t mali_job_cache_order_autonumber; * ---------- static variables ---------- */ -#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) -static _mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL; -static _mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL; +_mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL; +_mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL; static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_deletion_queue); -#endif #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) static _mali_osk_wq_work_t *scheduler_wq_pp_job_queue = NULL; @@ -87,13 +88,9 @@ static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job); static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job, mali_bool success); -static void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, - u32 num_cores_in_virtual); -#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job); -static void mali_scheduler_do_pp_job_delete(void *arg); -#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */ +void mali_scheduler_do_pp_job_delete(void *arg); #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job); @@ -112,10 +109,12 @@ _mali_osk_errcode_t mali_scheduler_initialize(void) _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.normal_pri); _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.high_pri); job_queue_gp.depth = 0; + job_queue_gp.big_job_num = 0; _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.normal_pri); _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.high_pri); job_queue_pp.depth = 0; + job_queue_pp.big_job_num = 0; mali_scheduler_lock_obj = _mali_osk_spinlock_irq_init( _MALI_OSK_LOCKFLAG_ORDERED, @@ -124,7 +123,6 @@ _mali_osk_errcode_t mali_scheduler_initialize(void) mali_scheduler_terminate(); } -#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) scheduler_wq_pp_job_delete = _mali_osk_wq_create_work( mali_scheduler_do_pp_job_delete, NULL); if (NULL == scheduler_wq_pp_job_delete) { @@ -139,7 +137,6 @@ _mali_osk_errcode_t mali_scheduler_initialize(void) mali_scheduler_terminate(); return _MALI_OSK_ERR_FAULT; } -#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */ #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) scheduler_wq_pp_job_queue = _mali_osk_wq_create_work( @@ -175,7 +172,6 @@ void mali_scheduler_terminate(void) } #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ -#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) if (NULL != scheduler_pp_job_delete_lock) { _mali_osk_spinlock_irq_term(scheduler_pp_job_delete_lock); scheduler_pp_job_delete_lock = NULL; @@ -185,7 +181,6 @@ void mali_scheduler_terminate(void) _mali_osk_wq_delete_work(scheduler_wq_pp_job_delete); scheduler_wq_pp_job_delete = NULL; } -#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */ if (NULL != mali_scheduler_lock_obj) { _mali_osk_spinlock_irq_term(mali_scheduler_lock_obj); @@ -282,6 +277,7 @@ struct mali_gp_job *mali_scheduler_job_gp_get(void) MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); MALI_DEBUG_ASSERT(0 < job_queue_gp.depth); + MALI_DEBUG_ASSERT(job_queue_gp.big_job_num <= job_queue_gp.depth); if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) { queue = &job_queue_gp.high_pri; @@ -296,7 +292,14 @@ struct mali_gp_job *mali_scheduler_job_gp_get(void) mali_gp_job_list_remove(job); job_queue_gp.depth--; - + if (job->big_job) { + job_queue_gp.big_job_num --; + if (job_queue_gp.big_job_num < MALI_MAX_PENDING_BIG_JOB) { + /* wake up process */ + wait_queue_head_t *queue = mali_session_get_wait_queue(); + wake_up(queue); + } + } return job; } @@ -541,10 +544,8 @@ void mali_scheduler_complete_pp_job(struct mali_pp_job *job, mali_bool user_notification, mali_bool dequeued) { - if (user_notification) { - mali_scheduler_return_pp_job_to_user(job, - num_cores_in_virtual); - } + job->user_notification = user_notification; + job->num_pp_cores_in_virtual = num_cores_in_virtual; if (dequeued) { #if defined(CONFIG_MALI_DVFS) @@ -562,16 +563,8 @@ void mali_scheduler_complete_pp_job(struct mali_pp_job *job, } } -#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) - /* - * The deletion of the job object (releasing sync refs etc) - * must be done in a different context - */ + /* With ZRAM feature enabled, all pp jobs will be force to use deferred delete. */ mali_scheduler_deferred_pp_job_delete(job); -#else - /* no use cases need this in this configuration */ - mali_pp_job_delete(job); -#endif } void mali_scheduler_abort_session(struct mali_session_data *session) @@ -597,6 +590,7 @@ void mali_scheduler_abort_session(struct mali_session_data *session) if (mali_gp_job_get_session(gp_job) == session) { mali_gp_job_list_move(gp_job, &removed_jobs_gp); job_queue_gp.depth--; + job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0; } } @@ -606,6 +600,7 @@ void mali_scheduler_abort_session(struct mali_session_data *session) if (mali_gp_job_get_session(gp_job) == session) { mali_gp_job_list_move(gp_job, &removed_jobs_gp); job_queue_gp.depth--; + job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0; } } @@ -621,9 +616,13 @@ void mali_scheduler_abort_session(struct mali_session_data *session) pp_job); mali_pp_job_mark_unstarted_failed(pp_job); - if (mali_pp_job_is_complete(pp_job)) { - mali_pp_job_list_move(pp_job, - &removed_jobs_pp); + if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) { + if (mali_pp_job_is_complete(pp_job)) { + mali_pp_job_list_move(pp_job, + &removed_jobs_pp); + } else { + mali_pp_job_list_remove(pp_job); + } } } } @@ -640,9 +639,13 @@ void mali_scheduler_abort_session(struct mali_session_data *session) pp_job); mali_pp_job_mark_unstarted_failed(pp_job); - if (mali_pp_job_is_complete(pp_job)) { - mali_pp_job_list_move(pp_job, - &removed_jobs_pp); + if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) { + if (mali_pp_job_is_complete(pp_job)) { + mali_pp_job_list_move(pp_job, + &removed_jobs_pp); + } else { + mali_pp_job_list_remove(pp_job); + } } } } @@ -961,6 +964,7 @@ static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job) } job_queue_gp.depth += 1; + job_queue_gp.big_job_num += (job->big_job) ? 1 : 0; /* Add job to queue (mali_gp_job_queue_add find correct place). */ mali_gp_job_list_add(job, queue); @@ -1020,6 +1024,10 @@ static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job) MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while session is aborting.\n", mali_pp_job_get_id(job), job)); return MALI_FALSE; /* job not queued */ + } else if (unlikely(MALI_SWAP_IN_FAIL == job->swap_status)) { + MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while swap in failed.\n", + mali_pp_job_get_id(job), job)); + return MALI_FALSE; } mali_pp_job_set_cache_order(job, mali_scheduler_get_new_cache_order()); @@ -1098,6 +1106,8 @@ static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job, jobres = notification->result_buffer; MALI_DEBUG_ASSERT_POINTER(jobres); + jobres->pending_big_job_num = mali_scheduler_job_gp_big_job_count(); + jobres->user_job_ptr = mali_gp_job_get_user_id(job); if (MALI_TRUE == success) { jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS; @@ -1111,7 +1121,7 @@ static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job, mali_session_send_notification(session, notification); } -static void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, +void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, u32 num_cores_in_virtual) { u32 i; @@ -1162,8 +1172,6 @@ static void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, mali_session_send_notification(session, notification); } -#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) - static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job) { MALI_DEBUG_ASSERT_POINTER(job); @@ -1175,7 +1183,7 @@ static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job) _mali_osk_wq_schedule_work(scheduler_wq_pp_job_delete); } -static void mali_scheduler_do_pp_job_delete(void *arg) +void mali_scheduler_do_pp_job_delete(void *arg) { _MALI_OSK_LIST_HEAD_STATIC_INIT(list); struct mali_pp_job *job; @@ -1194,14 +1202,12 @@ static void mali_scheduler_do_pp_job_delete(void *arg) _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, struct mali_pp_job, list) { - _mali_osk_list_delinit(&job->list); + mali_pp_job_delete(job); /* delete the job object itself */ } } -#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_DELETE) */ - #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job) @@ -1266,8 +1272,8 @@ static void mali_scheduler_do_pp_job_queue(void *arg) /* unlock scheduler in this uncommon case */ mali_scheduler_unlock(); - mali_timeline_tracker_release( - mali_pp_job_get_tracker(job)); + schedule_mask |= mali_timeline_tracker_release( + mali_pp_job_get_tracker(job)); /* Notify user space and close the job object */ mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, @@ -1284,3 +1290,65 @@ static void mali_scheduler_do_pp_job_queue(void *arg) } #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + +void mali_scheduler_gp_pp_job_queue_print(void) +{ + struct mali_gp_job *gp_job = NULL; + struct mali_gp_job *tmp_gp_job = NULL; + struct mali_pp_job *pp_job = NULL; + struct mali_pp_job *tmp_pp_job = NULL; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj); + + /* dump job queup status */ + if ((0 == job_queue_gp.depth) && (0 == job_queue_pp.depth)) { + MALI_PRINT(("No GP&PP job in the job queue.\n")); + return; + } + + MALI_PRINT(("Total (%d) GP job in the job queue.\n", job_queue_gp.depth)); + if (job_queue_gp.depth > 0) { + if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.high_pri, + struct mali_gp_job, list) { + MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job high_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid)); + } + } + + if (!_mali_osk_list_empty(&job_queue_gp.normal_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.normal_pri, + struct mali_gp_job, list) { + MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job normal_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid)); + } + } + } + + MALI_PRINT(("Total (%d) PP job in the job queue.\n", job_queue_pp.depth)); + if (job_queue_pp.depth > 0) { + if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.high_pri, + struct mali_pp_job, list) { + if (mali_pp_job_is_virtual(pp_job)) { + MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } else { + MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } + } + } + + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.normal_pri, + struct mali_pp_job, list) { + if (mali_pp_job_is_virtual(pp_job)) { + MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } else { + MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } + } + } + } + + /* dump group running job status */ + mali_executor_running_status_print(); +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h index d6ed28987ffb..f24cf42b8a79 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -20,6 +20,7 @@ struct mali_scheduler_job_queue { _MALI_OSK_LIST_HEAD(normal_pri); /* Queued jobs with normal priority */ _MALI_OSK_LIST_HEAD(high_pri); /* Queued jobs with high priority */ u32 depth; /* Depth of combined queues. */ + u32 big_job_num; }; extern _mali_osk_spinlock_irq_t *mali_scheduler_lock_obj; @@ -54,6 +55,10 @@ MALI_STATIC_INLINE u32 mali_scheduler_job_gp_count(void) { return job_queue_gp.depth; } +MALI_STATIC_INLINE u32 mali_scheduler_job_gp_big_job_count(void) +{ + return job_queue_gp.big_job_num; +} u32 mali_scheduler_job_physical_head_count(void); @@ -113,8 +118,13 @@ void mali_scheduler_complete_pp_job(struct mali_pp_job *job, void mali_scheduler_abort_session(struct mali_session_data *session); +void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, + u32 num_cores_in_virtual); + #if MALI_STATE_TRACKING u32 mali_scheduler_dump_state(char *buf, u32 size); #endif +void mali_scheduler_gp_pp_job_queue_print(void); + #endif /* __MALI_SCHEDULER_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h b/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h index d07cf4a93653..f862961d146e 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_session.c b/drivers/gpu/arm/mali400/mali/common/mali_session.c index 821ad979c69d..e0a2805b13d3 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_session.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_session.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -12,15 +12,21 @@ #include "mali_osk_list.h" #include "mali_session.h" #include "mali_ukk.h" +#ifdef MALI_MEM_SWAP_TRACKING +#include "mali_memory_swap_alloc.h" +#endif _MALI_OSK_LIST_HEAD(mali_sessions); static u32 mali_session_count = 0; _mali_osk_spinlock_irq_t *mali_sessions_lock = NULL; +wait_queue_head_t pending_queue; _mali_osk_errcode_t mali_session_initialize(void) { _MALI_OSK_INIT_LIST_HEAD(&mali_sessions); + /* init wait queue for big varying job */ + init_waitqueue_head(&pending_queue); mali_sessions_lock = _mali_osk_spinlock_irq_init( _MALI_OSK_LOCKFLAG_ORDERED, @@ -61,6 +67,11 @@ u32 mali_session_get_count(void) return mali_session_count; } +wait_queue_head_t *mali_session_get_wait_queue(void) +{ + return &pending_queue; +} + /* * Get the max completed window jobs from all active session, * which will be used in window render frame per sec calculate @@ -93,18 +104,41 @@ void mali_session_memory_tracking(_mali_osk_print_ctx *print_ctx) struct mali_session_data *session, *tmp; u32 mali_mem_usage; u32 total_mali_mem_size; +#ifdef MALI_MEM_SWAP_TRACKING + u32 swap_pool_size; + u32 swap_unlock_size; +#endif MALI_DEBUG_ASSERT_POINTER(print_ctx); mali_session_lock(); MALI_SESSION_FOREACH(session, tmp, link) { - _mali_osk_ctxprintf(print_ctx, " %-25s %-10u %-10u %-15u %-15u %-10u %-10u\n", +#ifdef MALI_MEM_SWAP_TRACKING + _mali_osk_ctxprintf(print_ctx, " %-25s %-10u %-10u %-15u %-15u %-10u %-10u %-10u\n", session->comm, session->pid, - session->mali_mem_array[MALI_MEM_OS] + session->mali_mem_array[MALI_MEM_BLOCK], session->max_mali_mem_allocated, - session->mali_mem_array[MALI_MEM_EXTERNAL], session->mali_mem_array[MALI_MEM_UMP], - session->mali_mem_array[MALI_MEM_DMA_BUF]); + (atomic_read(&session->mali_mem_allocated_pages)) * _MALI_OSK_MALI_PAGE_SIZE, + session->max_mali_mem_allocated_size, + (atomic_read(&session->mali_mem_array[MALI_MEM_EXTERNAL])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_UMP])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_DMA_BUF])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_SWAP])) * _MALI_OSK_MALI_PAGE_SIZE + ); +#else + _mali_osk_ctxprintf(print_ctx, " %-25s %-10u %-10u %-15u %-15u %-10u %-10u \n", + session->comm, session->pid, + (atomic_read(&session->mali_mem_allocated_pages)) * _MALI_OSK_MALI_PAGE_SIZE, + session->max_mali_mem_allocated_size, + (atomic_read(&session->mali_mem_array[MALI_MEM_EXTERNAL])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_UMP])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_DMA_BUF])) * _MALI_OSK_MALI_PAGE_SIZE + ); +#endif } mali_session_unlock(); mali_mem_usage = _mali_ukk_report_memory_usage(); total_mali_mem_size = _mali_ukk_report_total_memory_size(); _mali_osk_ctxprintf(print_ctx, "Mali mem usage: %u\nMali mem limit: %u\n", mali_mem_usage, total_mali_mem_size); -} \ No newline at end of file +#ifdef MALI_MEM_SWAP_TRACKING + mali_mem_swap_tracking(&swap_pool_size, &swap_unlock_size); + _mali_osk_ctxprintf(print_ctx, "Mali swap mem pool : %u\nMali swap mem unlock: %u\n", swap_pool_size, swap_unlock_size); +#endif +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_session.h b/drivers/gpu/arm/mali400/mali/common/mali_session.h index f027b3f99fbf..6791b2b5f110 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_session.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_session.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -12,10 +12,10 @@ #define __MALI_SESSION_H__ #include "mali_mmu_page_directory.h" -#include "mali_kernel_descriptor_mapping.h" #include "mali_osk.h" #include "mali_osk_list.h" #include "mali_memory_types.h" +#include "mali_memory_manager.h" struct mali_timeline_system; struct mali_soft_system; @@ -23,14 +23,16 @@ struct mali_soft_system; /* Number of frame builder job lists per session. */ #define MALI_PP_JOB_FB_LOOKUP_LIST_SIZE 16 #define MALI_PP_JOB_FB_LOOKUP_LIST_MASK (MALI_PP_JOB_FB_LOOKUP_LIST_SIZE - 1) +/*Max pending big job allowed in kernel*/ +#define MALI_MAX_PENDING_BIG_JOB (2) struct mali_session_data { _mali_osk_notification_queue_t *ioctl_queue; _mali_osk_mutex_t *memory_lock; /**< Lock protecting the vm manipulation */ - mali_descriptor_mapping *descriptor_mapping; /**< Mapping between userspace descriptors and our pointers */ +#if 0 _mali_osk_list_t memory_head; /**< Track all the memory allocated in this session, for freeing on abnormal termination */ - +#endif struct mali_page_directory *page_directory; /**< MMU page directory for this session */ _MALI_OSK_LIST_HEAD(link); /**< Link for list of all sessions */ @@ -49,8 +51,11 @@ struct mali_session_data { mali_bool use_high_priority_job_queue; /**< If MALI_TRUE, jobs added from this session will use the high priority job queues. */ u32 pid; char *comm; - size_t mali_mem_array[MALI_MEM_TYPE_MAX]; /**< The array to record all mali mem types' usage for this session. */ - size_t max_mali_mem_allocated; /**< The past max mali memory usage for this session. */ + atomic_t mali_mem_array[MALI_MEM_TYPE_MAX]; /**< The array to record mem types' usage for this session. */ + atomic_t mali_mem_allocated_pages; /** The current allocated mali memory pages, which include mali os memory and mali dedicated memory.*/ + size_t max_mali_mem_allocated_size; /**< The past max mali memory allocated size, which include mali os memory and mali dedicated memory. */ + /* Added for new memroy system */ + struct mali_allocation_manager allocation_mgr; }; _mali_osk_errcode_t mali_session_initialize(void); @@ -74,6 +79,7 @@ MALI_STATIC_INLINE void mali_session_unlock(void) void mali_session_add(struct mali_session_data *session); void mali_session_remove(struct mali_session_data *session); u32 mali_session_get_count(void); +wait_queue_head_t *mali_session_get_wait_queue(void); #define MALI_SESSION_FOREACH(session, tmp, link) \ _MALI_OSK_LIST_FOREACHENTRY(session, tmp, &mali_sessions, struct mali_session_data, link) diff --git a/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c index e8ad737bfbac..36ac982e1df0 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -289,8 +289,10 @@ static void mali_soft_job_send_activated_notification(struct mali_soft_job *job) job->activated_notification = NULL; } -void mali_soft_job_system_activate_job(struct mali_soft_job *job) +mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job) { + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + MALI_DEBUG_ASSERT_POINTER(job); MALI_DEBUG_ASSERT_POINTER(job->system); MALI_DEBUG_ASSERT_POINTER(job->system->session); @@ -307,7 +309,7 @@ void mali_soft_job_system_activate_job(struct mali_soft_job *job) /* Since we are in shutdown, we can ignore the scheduling bitmask. */ mali_timeline_tracker_release(&job->tracker); mali_soft_job_destroy(job); - return; + return schedule_mask; } /* Send activated notification. */ @@ -318,15 +320,12 @@ void mali_soft_job_system_activate_job(struct mali_soft_job *job) /* If job type is self signaled, release tracker, move soft job to free list, and scheduler at once */ if (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == job->type) { - mali_scheduler_mask schedule_mask; - MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); job->state = MALI_SOFT_JOB_STATE_SIGNALED; mali_soft_job_system_unlock(job->system); - schedule_mask = mali_timeline_tracker_release(&job->tracker); - mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); + schedule_mask |= mali_timeline_tracker_release(&job->tracker); mali_soft_job_destroy(job); } else { @@ -334,6 +333,8 @@ void mali_soft_job_system_activate_job(struct mali_soft_job *job) mali_soft_job_system_unlock(job->system); } + + return schedule_mask; } mali_scheduler_mask mali_soft_job_system_timeout_job(struct mali_soft_job *job) diff --git a/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h index 77711b1bd3ed..f35394e60384 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -165,8 +165,9 @@ _mali_osk_errcode_t mali_soft_job_system_signal_job(struct mali_soft_job_system * Used by the Timeline system to activate a soft job. * * @param job The soft job that is being activated. + * @return A scheduling bitmask. */ -void mali_soft_job_system_activate_job(struct mali_soft_job *job); +mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job); /** * Used by the Timeline system to timeout a soft job. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c index de076abfc65b..178abaf43ba1 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h index f252ab46a007..6a62df850b2f 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline.c index a1c01f188b76..5a767b39d56e 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_timeline.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -115,6 +115,11 @@ static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, str mali_spinlock_reentrant_signal(system->spinlock, tid); + /* + * Older versions of Linux, before 3.5, doesn't support fput() in interrupt + * context. For those older kernels, allocate a list object and put the + * fence object on that and defer the call to sync_fence_put() to a workqueue. + */ #if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) { struct mali_deferred_fence_put_entry *obj; @@ -243,7 +248,9 @@ static void mali_timeline_destroy(struct mali_timeline *timeline) _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work); } +#ifndef CONFIG_SYNC _mali_osk_free(timeline); +#endif } } @@ -301,11 +308,19 @@ static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *s return NULL; } + timeline->destroyed = MALI_FALSE; + timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name); if (NULL == timeline->sync_tl) { mali_timeline_destroy(timeline); return NULL; } + + timeline->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM); + if (NULL == timeline->spinlock) { + mali_timeline_destroy(timeline); + return NULL; + } } #endif /* defined(CONFIG_SYNC) */ @@ -642,19 +657,18 @@ static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_t _mali_osk_atomic_dec(&gp_tracker_count); break; case MALI_TIMELINE_TRACKER_PP: - schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job); - if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) { _mali_osk_atomic_dec(&virt_pp_tracker_count); } else { _mali_osk_atomic_dec(&phy_pp_tracker_count); } + schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job); break; case MALI_TIMELINE_TRACKER_SOFT: timeline = tracker->timeline; MALI_DEBUG_ASSERT_POINTER(timeline); - mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job); + schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job); /* Start a soft timer to make sure the soft job be released in a limited time */ mali_spinlock_reentrant_wait(system->spinlock, tid); @@ -898,6 +912,9 @@ void mali_timeline_system_destroy(struct mali_timeline_system *system) { u32 i; struct mali_timeline_waiter *waiter, *next; +#if defined(CONFIG_SYNC) + u32 tid = _mali_osk_get_tid(); +#endif MALI_DEBUG_ASSERT_POINTER(system); MALI_DEBUG_ASSERT_POINTER(system->session); @@ -905,6 +922,7 @@ void mali_timeline_system_destroy(struct mali_timeline_system *system) MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n")); if (NULL != system) { + /* There should be no waiters left on this queue. */ if (NULL != system->wait_queue) { _mali_osk_wait_queue_term(system->wait_queue); @@ -923,6 +941,14 @@ void mali_timeline_system_destroy(struct mali_timeline_system *system) if (NULL != system->signaled_sync_tl) { sync_timeline_destroy(system->signaled_sync_tl); } + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + if ((NULL != system->timelines[i]) && (NULL != system->timelines[i]->spinlock)) { + mali_spinlock_reentrant_wait(system->timelines[i]->spinlock, tid); + system->timelines[i]->destroyed = MALI_TRUE; + mali_spinlock_reentrant_signal(system->timelines[i]->spinlock, tid); + } + } #endif /* defined(CONFIG_SYNC) */ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { @@ -930,6 +956,7 @@ void mali_timeline_system_destroy(struct mali_timeline_system *system) mali_timeline_destroy(system->timelines[i]); } } + if (NULL != system->spinlock) { mali_spinlock_reentrant_term(system->spinlock); } @@ -1468,6 +1495,62 @@ void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_os } } +#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) +void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker) +{ + const char *tracker_state = "IWAF"; + char state_char = 'I'; + char tracker_type[32] = {0}; + + MALI_DEBUG_ASSERT_POINTER(tracker); + + state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)); + _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type)); + +#if defined(CONFIG_SYNC) + if (0 != tracker->trigger_ref_count) { + MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); + } else { + MALI_PRINT(("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); + } +#else + if (0 != tracker->trigger_ref_count) { + MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->job)); + } else { + MALI_PRINT(("TL: %s %u %c job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->job)); + } +#endif +} + +void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline) +{ + struct mali_timeline_tracker *tracker = NULL; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + tracker = timeline->tracker_tail; + while (NULL != tracker) { + mali_timeline_debug_direct_print_tracker(tracker); + tracker = tracker->timeline_next; + } +} + +#endif + void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx) { int i; diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline.h index cd43465cd5e0..58d83839f4fe 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_timeline.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -18,6 +18,7 @@ #include "mali_spinlock_reentrant.h" #include "mali_sync.h" #include "mali_scheduler_types.h" +#include /** * Soft job timeout. @@ -140,6 +141,8 @@ struct mali_timeline { #if defined(CONFIG_SYNC) struct sync_timeline *sync_tl; /**< Sync timeline that corresponds to this timeline. */ + mali_bool destroyed; + struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ #endif /* defined(CONFIG_SYNC) */ /* The following fields are used to time out soft job trackers. */ @@ -515,6 +518,11 @@ void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _m */ void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx); +#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) +void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker); +void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline); +#endif + /** * Print debug information about timeline system. * diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c index dc22ccfa76e8..3c58928dd3a2 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -102,7 +102,7 @@ static mali_bool mali_timeline_fence_wait_check_status(struct mali_timeline_syst sync_fence = sync_fence_fdget(fence->sync_fd); if (likely(NULL != sync_fence)) { #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) - if( 0 == sync_fence->status) { + if (0 == sync_fence->status) { #else if (0 == atomic_read(&sync_fence->status)) { #endif diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h index 393a71d44e0c..f5440ab6fc6d 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c index ebfa569b0552..73843f07c990 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h index 54b92d5184b8..29a3822457e9 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_ukk.h b/drivers/gpu/arm/mali400/mali/common/mali_ukk.h index f819d50b3ef2..597685a53f3b 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_ukk.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_ukk.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -271,6 +271,11 @@ _mali_osk_errcode_t _mali_ukk_get_user_setting(_mali_uk_get_user_setting_s *args */ _mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args); +/** @brief Make process sleep if the pending big job in kernel >= MALI_MAX_PENDING_BIG_JOB + * + */ +_mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args); + /** @} */ /* end group _mali_uk_core */ @@ -350,31 +355,6 @@ _mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s */ _mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args); -/** @brief Map a physically contiguous range of memory into Mali - * @param args see _mali_uk_map_external_mem_s in mali_utgard_uk_types.h - * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. - */ -_mali_osk_errcode_t _mali_ukk_map_external_mem(_mali_uk_map_external_mem_s *args); - -/** @brief Unmap a physically contiguous range of memory from Mali - * @param args see _mali_uk_unmap_external_mem_s in mali_utgard_uk_types.h - * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. - */ -_mali_osk_errcode_t _mali_ukk_unmap_external_mem(_mali_uk_unmap_external_mem_s *args); - -#if defined(CONFIG_MALI400_UMP) -/** @brief Map UMP memory into Mali - * @param args see _mali_uk_attach_ump_mem_s in mali_utgard_uk_types.h - * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. - */ -_mali_osk_errcode_t _mali_ukk_attach_ump_mem(_mali_uk_attach_ump_mem_s *args); -/** @brief Unmap UMP memory from Mali - * @param args see _mali_uk_release_ump_mem_s in mali_utgard_uk_types.h - * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. - */ -_mali_osk_errcode_t _mali_ukk_release_ump_mem(_mali_uk_release_ump_mem_s *args); -#endif /* CONFIG_MALI400_UMP */ - /** @} */ /* end group _mali_uk_memory */ @@ -509,12 +489,17 @@ _mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s */ _mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args); -/** @brief Return the total memory usage +/** @brief Get profiling stream fd. * - * @param args see _mali_uk_profiling_memory_usage_get_s in "mali_utgard_uk_types.h" - * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + * @param args see _mali_uk_profiling_stream_fd_get_s in "mali_utgard_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args); + +/** @brief Profiling control set. + * + * @param args see _mali_uk_profiling_control_set_s in "mali_utgard_uk_types.h" */ -_mali_osk_errcode_t _mali_ukk_profiling_memory_usage_get(_mali_uk_profiling_memory_usage_get_s *args); +_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args); /** @} */ /* end group _mali_uk_profiling */ #endif diff --git a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c index b5d65216ae90..54e1580fad1a 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c +++ b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h index 824e3e1782c5..0732c3e56e2a 100755 --- a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h +++ b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h index c7ccbf6322d8..4ea02fe65cb5 100755 --- a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -50,6 +50,7 @@ #define MALI450_OFFSET_L2_CACHE0 MALI_OFFSET_L2_RESOURCE1 #define MALI450_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 #define MALI450_OFFSET_L2_CACHE2 MALI_OFFSET_L2_RESOURCE2 +#define MALI470_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 #define MALI_OFFSET_BCAST 0x13000 #define MALI_OFFSET_DLBU 0x14000 @@ -112,7 +113,7 @@ MALI_GPU_RESOURCES_MALI400_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -/* Mali-450 */ + /* Mali-450 */ #define MALI_GPU_RESOURCES_MALI450_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ @@ -207,6 +208,65 @@ MALI_GPU_RESOURCES_MALI450_MP8(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + /* Mali - 470 */ +#define MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + #define MALI_GPU_RESOURCE_L2(addr) \ { \ .name = "Mali_L2", \ @@ -228,7 +288,7 @@ .start = gp_irq, \ .end = gp_irq, \ }, \ - + #define MALI_GPU_RESOURCE_GP_WITH_MMU(gp_addr, gp_irq, gp_mmu_addr, gp_mmu_irq) \ { \ .name = "Mali_GP", \ @@ -268,7 +328,7 @@ .start = pp_irq, \ .end = pp_irq, \ }, \ - + #define MALI_GPU_RESOURCE_PP_WITH_MMU(id, pp_addr, pp_irq, pp_mmu_addr, pp_mmu_irq) \ { \ .name = "Mali_PP" #id, \ @@ -363,85 +423,85 @@ .end = pp_mmu_bcast_addr + 0x100, \ }, -struct mali_gpu_utilization_data { - unsigned int utilization_gpu; /* Utilization for GP and all PP cores combined, 0 = no utilization, 256 = full utilization */ - unsigned int utilization_gp; /* Utilization for GP core only, 0 = no utilization, 256 = full utilization */ - unsigned int utilization_pp; /* Utilization for all PP cores combined, 0 = no utilization, 256 = full utilization */ -}; - -struct mali_gpu_clk_item { - unsigned int clock; /* unit(MHz) */ - unsigned int vol; -}; - -struct mali_gpu_clock { - struct mali_gpu_clk_item *item; - unsigned int num_of_steps; -}; - -struct mali_gpu_device_data { - /* Shared GPU memory */ - unsigned long shared_mem_size; - - /* - * Mali PMU switch delay. - * Only needed if the power gates are connected to the PMU in a high fanout - * network. This value is the number of Mali clock cycles it takes to - * enable the power gates and turn on the power mesh. - * This value will have no effect if a daisy chain implementation is used. + struct mali_gpu_utilization_data { + unsigned int utilization_gpu; /* Utilization for GP and all PP cores combined, 0 = no utilization, 256 = full utilization */ + unsigned int utilization_gp; /* Utilization for GP core only, 0 = no utilization, 256 = full utilization */ + unsigned int utilization_pp; /* Utilization for all PP cores combined, 0 = no utilization, 256 = full utilization */ + }; + + struct mali_gpu_clk_item { + unsigned int clock; /* unit(MHz) */ + unsigned int vol; + }; + + struct mali_gpu_clock { + struct mali_gpu_clk_item *item; + unsigned int num_of_steps; + }; + + struct mali_gpu_device_data { + /* Shared GPU memory */ + unsigned long shared_mem_size; + + /* + * Mali PMU switch delay. + * Only needed if the power gates are connected to the PMU in a high fanout + * network. This value is the number of Mali clock cycles it takes to + * enable the power gates and turn on the power mesh. + * This value will have no effect if a daisy chain implementation is used. + */ + u32 pmu_switch_delay; + + /* Mali Dynamic power domain configuration in sequence from 0-11 + * GP PP0 PP1 PP2 PP3 PP4 PP5 PP6 PP7, L2$0 L2$1 L2$2 + */ + u16 pmu_domain_config[12]; + + /* Dedicated GPU memory range (physical). */ + unsigned long dedicated_mem_start; + unsigned long dedicated_mem_size; + + /* Frame buffer memory to be accessible by Mali GPU (physical) */ + unsigned long fb_start; + unsigned long fb_size; + + /* Max runtime [ms] for jobs */ + int max_job_runtime; + + /* Report GPU utilization and related control in this interval (specified in ms) */ + unsigned long control_interval; + + /* Function that will receive periodic GPU utilization numbers */ + void (*utilization_callback)(struct mali_gpu_utilization_data *data); + + /* Fuction that platform callback for freq setting, needed when CONFIG_MALI_DVFS enabled */ + int (*set_freq)(int setting_clock_step); + /* Function that platfrom report it's clock info which driver can set, needed when CONFIG_MALI_DVFS enabled */ + void (*get_clock_info)(struct mali_gpu_clock **data); + /* Function that get the current clock info, needed when CONFIG_MALI_DVFS enabled */ + int (*get_freq)(void); + }; + + /** + * Pause the scheduling and power state changes of Mali device driver. + * mali_dev_resume() must always be called as soon as possible after this function + * in order to resume normal operation of the Mali driver. */ - u32 pmu_switch_delay; + void mali_dev_pause(void); - /* Mali Dynamic power domain configuration in sequence from 0-11 - * GP PP0 PP1 PP2 PP3 PP4 PP5 PP6 PP7, L2$0 L2$1 L2$2 + /** + * Resume scheduling and allow power changes in Mali device driver. + * This must always be called after mali_dev_pause(). */ - u16 pmu_domain_config[12]; - - /* Dedicated GPU memory range (physical). */ - unsigned long dedicated_mem_start; - unsigned long dedicated_mem_size; - - /* Frame buffer memory to be accessible by Mali GPU (physical) */ - unsigned long fb_start; - unsigned long fb_size; - - /* Max runtime [ms] for jobs */ - int max_job_runtime; - - /* Report GPU utilization and related control in this interval (specified in ms) */ - unsigned long control_interval; - - /* Function that will receive periodic GPU utilization numbers */ - void (*utilization_callback)(struct mali_gpu_utilization_data *data); - - /* Fuction that platform callback for freq setting, needed when CONFIG_MALI_DVFS enabled */ - int (*set_freq)(int setting_clock_step); - /* Function that platfrom report it's clock info which driver can set, needed when CONFIG_MALI_DVFS enabled */ - void (*get_clock_info)(struct mali_gpu_clock **data); - /* Function that get the current clock info, needed when CONFIG_MALI_DVFS enabled */ - int (*get_freq)(void); -}; - -/** - * Pause the scheduling and power state changes of Mali device driver. - * mali_dev_resume() must always be called as soon as possible after this function - * in order to resume normal operation of the Mali driver. - */ -void mali_dev_pause(void); - -/** - * Resume scheduling and allow power changes in Mali device driver. - * This must always be called after mali_dev_pause(). - */ -void mali_dev_resume(void); - -/** @brief Set the desired number of PP cores to use. - * - * The internal Mali PMU will be used, if present, to physically power off the PP cores. - * - * @param num_cores The number of desired cores - * @return 0 on success, otherwise error. -EINVAL means an invalid number of cores was specified. - */ -int mali_perf_set_num_pp_cores(unsigned int num_cores); + void mali_dev_resume(void); + + /** @brief Set the desired number of PP cores to use. + * + * The internal Mali PMU will be used, if present, to physically power off the PP cores. + * + * @param num_cores The number of desired cores + * @return 0 on success, otherwise error. -EINVAL means an invalid number of cores was specified. + */ + int mali_perf_set_num_pp_cores(unsigned int num_cores); #endif diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_counters.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_counters.h deleted file mode 100755 index b3caaa121c00..000000000000 --- a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_counters.h +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the GNU General Public License version 2 - * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. - * - * A copy of the licence is included with the program, and can also be obtained from Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#ifndef _MALI_UTGARD_COUNTERS_H_ -#define _MALI_UTGARD_COUNTERS_H_ - -typedef struct { - void *unused; -} mali_cinstr_counter_info; - -typedef enum { - MALI_CINSTR_COUNTER_SOURCE_EGL = 0, - MALI_CINSTR_COUNTER_SOURCE_OPENGLES = 1000, - MALI_CINSTR_COUNTER_SOURCE_OPENVG = 2000, - MALI_CINSTR_COUNTER_SOURCE_GP = 3000, - MALI_CINSTR_COUNTER_SOURCE_PP = 4000, -} cinstr_counter_source; - -#define MALI_CINSTR_EGL_FIRST_COUNTER MALI_CINSTR_COUNTER_SOURCE_EGL -#define MALI_CINSTR_EGL_LAST_COUNTER (MALI_CINSTR_COUNTER_SOURCE_EGL + 999) - -#define MALI_CINSTR_GLES_FIRST_COUNTER MALI_CINSTR_COUNTER_SOURCE_OPENGLES -#define MALI_CINSTR_GLES_LAST_COUNTER (MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 999) - -#define MALI_CINSTR_VG_FIRST_COUNTER MALI_CINSTR_COUNTER_SOURCE_OPENVG -#define MALI_CINSTR_VG_LAST_COUNTER (MALI_CINSTR_COUNTER_SOURCE_OPENVG + 999) - -#define MALI_CINSTR_GP_FIRST_COUNTER MALI_CINSTR_COUNTER_SOURCE_GP -#define MALI_CINSTR_GP_LAST_COUNTER (MALI_CINSTR_COUNTER_SOURCE_GP + 999) - -#define MALI_CINSTR_PP_FIRST_COUNTER MALI_CINSTR_COUNTER_SOURCE_PP -#define MALI_CINSTR_PP_LAST_COUNTER (MALI_CINSTR_COUNTER_SOURCE_PP + 999) - - -typedef enum { - /* EGL counters */ - - MALI_CINSTR_EGL_BLIT_TIME = MALI_CINSTR_COUNTER_SOURCE_EGL + 0, - - /* Last counter in the EGL set */ - MALI_CINSTR_EGL_MAX_COUNTER = MALI_CINSTR_COUNTER_SOURCE_EGL + 1, - - /* GLES counters */ - - MALI_CINSTR_GLES_DRAW_ELEMENTS_CALLS = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 0, - MALI_CINSTR_GLES_DRAW_ELEMENTS_NUM_INDICES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 1, - MALI_CINSTR_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 2, - MALI_CINSTR_GLES_DRAW_ARRAYS_CALLS = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 3, - MALI_CINSTR_GLES_DRAW_ARRAYS_NUM_TRANSFORMED = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 4, - MALI_CINSTR_GLES_DRAW_POINTS = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 5, - MALI_CINSTR_GLES_DRAW_LINES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 6, - MALI_CINSTR_GLES_DRAW_LINE_LOOP = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 7, - MALI_CINSTR_GLES_DRAW_LINE_STRIP = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 8, - MALI_CINSTR_GLES_DRAW_TRIANGLES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 9, - MALI_CINSTR_GLES_DRAW_TRIANGLE_STRIP = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 10, - MALI_CINSTR_GLES_DRAW_TRIANGLE_FAN = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 11, - MALI_CINSTR_GLES_NON_VBO_DATA_COPY_TIME = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 12, - MALI_CINSTR_GLES_UNIFORM_BYTES_COPIED_TO_MALI = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 13, - MALI_CINSTR_GLES_UPLOAD_TEXTURE_TIME = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 14, - MALI_CINSTR_GLES_UPLOAD_VBO_TIME = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 15, - MALI_CINSTR_GLES_NUM_FLUSHES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 16, - MALI_CINSTR_GLES_NUM_VSHADERS_GENERATED = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 17, - MALI_CINSTR_GLES_NUM_FSHADERS_GENERATED = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 18, - MALI_CINSTR_GLES_VSHADER_GEN_TIME = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 19, - MALI_CINSTR_GLES_FSHADER_GEN_TIME = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 20, - MALI_CINSTR_GLES_INPUT_TRIANGLES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 21, - MALI_CINSTR_GLES_VXCACHE_HIT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 22, - MALI_CINSTR_GLES_VXCACHE_MISS = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 23, - MALI_CINSTR_GLES_VXCACHE_COLLISION = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 24, - MALI_CINSTR_GLES_CULLED_TRIANGLES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 25, - MALI_CINSTR_GLES_CULLED_LINES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 26, - MALI_CINSTR_GLES_BACKFACE_TRIANGLES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 27, - MALI_CINSTR_GLES_GBCLIP_TRIANGLES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 28, - MALI_CINSTR_GLES_GBCLIP_LINES = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 29, - MALI_CINSTR_GLES_TRIANGLES_DRAWN = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 30, - MALI_CINSTR_GLES_DRAWCALL_TIME = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 31, - MALI_CINSTR_GLES_TRIANGLES_COUNT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 32, - MALI_CINSTR_GLES_INDEPENDENT_TRIANGLES_COUNT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 33, - MALI_CINSTR_GLES_STRIP_TRIANGLES_COUNT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 34, - MALI_CINSTR_GLES_FAN_TRIANGLES_COUNT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 35, - MALI_CINSTR_GLES_LINES_COUNT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 36, - MALI_CINSTR_GLES_INDEPENDENT_LINES_COUNT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 37, - MALI_CINSTR_GLES_STRIP_LINES_COUNT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 38, - MALI_CINSTR_GLES_LOOP_LINES_COUNT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 39, - MALI_CINSTR_GLES_POINTS_COUNT = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 40, - - /* Last counter in the GLES set */ - MALI_CINSTR_GLES_MAX_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENGLES + 41, - - /* OpenVG counters */ - - MALI_CINSTR_VG_MASK_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 0, - MALI_CINSTR_VG_CLEAR_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 1, - MALI_CINSTR_VG_APPEND_PATH_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 2, - MALI_CINSTR_VG_APPEND_PATH_DATA_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 3, - MALI_CINSTR_VG_MODIFY_PATH_COORDS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 4, - MALI_CINSTR_VG_TRANSFORM_PATH_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 5, - MALI_CINSTR_VG_INTERPOLATE_PATH_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 6, - MALI_CINSTR_VG_PATH_LENGTH_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 7, - MALI_CINSTR_VG_POINT_ALONG_PATH_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 8, - MALI_CINSTR_VG_PATH_BOUNDS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 9, - MALI_CINSTR_VG_PATH_TRANSFORMED_BOUNDS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 10, - MALI_CINSTR_VG_DRAW_PATH_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 11, - MALI_CINSTR_VG_CLEAR_IMAGE_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 12, - MALI_CINSTR_VG_IMAGE_SUB_DATA_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 13, - MALI_CINSTR_VG_GET_IMAGE_SUB_DATA_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 14, - MALI_CINSTR_VG_COPY_IMAGE_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 15, - MALI_CINSTR_VG_DRAW_IMAGE_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 16, - MALI_CINSTR_VG_SET_PIXELS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 17, - MALI_CINSTR_VG_WRITE_PIXELS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 18, - MALI_CINSTR_VG_GET_PIXELS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 19, - MALI_CINSTR_VG_READ_PIXELS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 20, - MALI_CINSTR_VG_COPY_PIXELS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 21, - MALI_CINSTR_VG_COLOR_MATRIX_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 22, - MALI_CINSTR_VG_CONVOLVE_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 23, - MALI_CINSTR_VG_SEPARABLE_CONVOLVE_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 24, - MALI_CINSTR_VG_GAUSSIAN_BLUR_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 25, - MALI_CINSTR_VG_LOOKUP_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 26, - MALI_CINSTR_VG_LOOKUP_SINGLE_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 27, - MALI_CINSTR_VG_CONTEXT_CREATE_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 28, - MALI_CINSTR_VG_STROKED_CUBICS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 29, - MALI_CINSTR_VG_STROKED_QUADS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 30, - MALI_CINSTR_VG_STROKED_ARCS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 31, - MALI_CINSTR_VG_STROKED_LINES_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 32, - MALI_CINSTR_VG_FILLED_CUBICS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 33, - MALI_CINSTR_VG_FILLED_QUADS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 34, - MALI_CINSTR_VG_FILLED_ARCS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 35, - MALI_CINSTR_VG_FILLED_LINES_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 36, - MALI_CINSTR_VG_DRAW_PATH_CALLS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 37, - MALI_CINSTR_VG_TRIANGLES_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 38, - MALI_CINSTR_VG_VERTICES_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 39, - MALI_CINSTR_VG_INDICES_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 40, - MALI_CINSTR_VG_FILLED_PATHS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 41, - MALI_CINSTR_VG_STROKED_PATHS_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 42, - MALI_CINSTR_VG_FILL_EXTRACT_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 43, - MALI_CINSTR_VG_DRAW_FILLED_PATH_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 44, - MALI_CINSTR_VG_STROKE_EXTRACT_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 45, - MALI_CINSTR_VG_DRAW_STROKED_PATH_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 46, - MALI_CINSTR_VG_DRAW_PAINT_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 47, - MALI_CINSTR_VG_DATA_STRUCTURES_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 48, - MALI_CINSTR_VG_MEM_PATH_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 49, - MALI_CINSTR_VG_RSW_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 50, - - /* Last counter in the VG set */ - MALI_CINSTR_VG_MAX_COUNTER = MALI_CINSTR_COUNTER_SOURCE_OPENVG + 51, - - /* Mali GP counters */ - - MALI_CINSTR_GP_DEPRECATED_0 = MALI_CINSTR_COUNTER_SOURCE_GP + 0, - MALI_CINSTR_GP_ACTIVE_CYCLES_GP = MALI_CINSTR_COUNTER_SOURCE_GP + 1, - MALI_CINSTR_GP_ACTIVE_CYCLES_VERTEX_SHADER = MALI_CINSTR_COUNTER_SOURCE_GP + 2, - MALI_CINSTR_GP_ACTIVE_CYCLES_VERTEX_STORER = MALI_CINSTR_COUNTER_SOURCE_GP + 3, - MALI_CINSTR_GP_ACTIVE_CYCLES_VERTEX_LOADER = MALI_CINSTR_COUNTER_SOURCE_GP + 4, - MALI_CINSTR_GP_CYCLES_VERTEX_LOADER_WAITING_FOR_VERTEX_SHADER = MALI_CINSTR_COUNTER_SOURCE_GP + 5, - MALI_CINSTR_GP_NUMBER_OF_WORDS_READ = MALI_CINSTR_COUNTER_SOURCE_GP + 6, - MALI_CINSTR_GP_NUMBER_OF_WORDS_WRITTEN = MALI_CINSTR_COUNTER_SOURCE_GP + 7, - MALI_CINSTR_GP_NUMBER_OF_READ_BURSTS = MALI_CINSTR_COUNTER_SOURCE_GP + 8, - MALI_CINSTR_GP_NUMBER_OF_WRITE_BURSTS = MALI_CINSTR_COUNTER_SOURCE_GP + 9, - MALI_CINSTR_GP_NUMBER_OF_VERTICES_PROCESSED = MALI_CINSTR_COUNTER_SOURCE_GP + 10, - MALI_CINSTR_GP_NUMBER_OF_VERTICES_FETCHED = MALI_CINSTR_COUNTER_SOURCE_GP + 11, - MALI_CINSTR_GP_NUMBER_OF_PRIMITIVES_FETCHED = MALI_CINSTR_COUNTER_SOURCE_GP + 12, - MALI_CINSTR_GP_RESERVED_13 = MALI_CINSTR_COUNTER_SOURCE_GP + 13, - MALI_CINSTR_GP_NUMBER_OF_BACKFACE_CULLINGS_DONE = MALI_CINSTR_COUNTER_SOURCE_GP + 14, - MALI_CINSTR_GP_NUMBER_OF_COMMANDS_WRITTEN_TO_TILES = MALI_CINSTR_COUNTER_SOURCE_GP + 15, - MALI_CINSTR_GP_NUMBER_OF_MEMORY_BLOCKS_ALLOCATED = MALI_CINSTR_COUNTER_SOURCE_GP + 16, - MALI_CINSTR_GP_RESERVED_17 = MALI_CINSTR_COUNTER_SOURCE_GP + 17, - MALI_CINSTR_GP_RESERVED_18 = MALI_CINSTR_COUNTER_SOURCE_GP + 18, - MALI_CINSTR_GP_NUMBER_OF_VERTEX_LOADER_CACHE_MISSES = MALI_CINSTR_COUNTER_SOURCE_GP + 19, - MALI_CINSTR_GP_RESERVED_20 = MALI_CINSTR_COUNTER_SOURCE_GP + 20, - MALI_CINSTR_GP_RESERVED_21 = MALI_CINSTR_COUNTER_SOURCE_GP + 21, - MALI_CINSTR_GP_ACTIVE_CYCLES_VERTEX_SHADER_COMMAND_PROCESSOR = MALI_CINSTR_COUNTER_SOURCE_GP + 22, - MALI_CINSTR_GP_ACTIVE_CYCLES_PLBU_COMMAND_PROCESSOR = MALI_CINSTR_COUNTER_SOURCE_GP + 23, - MALI_CINSTR_GP_ACTIVE_CYCLES_PLBU_LIST_WRITER = MALI_CINSTR_COUNTER_SOURCE_GP + 24, - MALI_CINSTR_GP_ACTIVE_CYCLES_THROUGH_THE_PREPARE_LIST_COMMANDS = MALI_CINSTR_COUNTER_SOURCE_GP + 25, - MALI_CINSTR_GP_RESERVED_26 = MALI_CINSTR_COUNTER_SOURCE_GP + 26, - MALI_CINSTR_GP_ACTIVE_CYCLES_PRIMITIVE_ASSEMBLY = MALI_CINSTR_COUNTER_SOURCE_GP + 27, - MALI_CINSTR_GP_ACTIVE_CYCLES_PLBU_VERTEX_FETCHER = MALI_CINSTR_COUNTER_SOURCE_GP + 28, - MALI_CINSTR_GP_RESERVED_29 = MALI_CINSTR_COUNTER_SOURCE_GP + 29, - MALI_CINSTR_GP_ACTIVE_CYCLES_BOUNDINGBOX_AND_COMMAND_GENERATOR = MALI_CINSTR_COUNTER_SOURCE_GP + 30, - MALI_CINSTR_GP_RESERVED_31 = MALI_CINSTR_COUNTER_SOURCE_GP + 31, - MALI_CINSTR_GP_ACTIVE_CYCLES_SCISSOR_TILE_ITERATOR = MALI_CINSTR_COUNTER_SOURCE_GP + 32, - MALI_CINSTR_GP_ACTIVE_CYCLES_PLBU_TILE_ITERATOR = MALI_CINSTR_COUNTER_SOURCE_GP + 33, - MALI_CINSTR_GP_JOB_COUNT = MALI_CINSTR_COUNTER_SOURCE_GP + 900, - - /* Mali PP counters */ - - MALI_CINSTR_PP_ACTIVE_CLOCK_CYCLES_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 0, - MALI_CINSTR_PP_TOTAL_CLOCK_CYCLES_COUNT_REMOVED = MALI_CINSTR_COUNTER_SOURCE_PP + 1, - MALI_CINSTR_PP_TOTAL_BUS_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 2, - MALI_CINSTR_PP_TOTAL_BUS_WRITES = MALI_CINSTR_COUNTER_SOURCE_PP + 3, - MALI_CINSTR_PP_BUS_READ_REQUEST_CYCLES_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 4, - MALI_CINSTR_PP_BUS_WRITE_REQUEST_CYCLES_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 5, - MALI_CINSTR_PP_BUS_READ_TRANSACTIONS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 6, - MALI_CINSTR_PP_BUS_WRITE_TRANSACTIONS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 7, - MALI_CINSTR_PP_RESERVED_08 = MALI_CINSTR_COUNTER_SOURCE_PP + 8, - MALI_CINSTR_PP_TILE_WRITEBACK_WRITES = MALI_CINSTR_COUNTER_SOURCE_PP + 9, - MALI_CINSTR_PP_STORE_UNIT_WRITES = MALI_CINSTR_COUNTER_SOURCE_PP + 10, - MALI_CINSTR_PP_RESERVED_11 = MALI_CINSTR_COUNTER_SOURCE_PP + 11, - MALI_CINSTR_PP_PALETTE_CACHE_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 12, - MALI_CINSTR_PP_TEXTURE_CACHE_UNCOMPRESSED_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 13, - MALI_CINSTR_PP_POLYGON_LIST_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 14, - MALI_CINSTR_PP_RSW_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 15, - MALI_CINSTR_PP_VERTEX_CACHE_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 16, - MALI_CINSTR_PP_UNIFORM_REMAPPING_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 17, - MALI_CINSTR_PP_PROGRAM_CACHE_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 18, - MALI_CINSTR_PP_VARYING_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 19, - MALI_CINSTR_PP_TEXTURE_DESCRIPTORS_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 20, - MALI_CINSTR_PP_TEXTURE_DESCRIPTORS_REMAPPING_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 21, - MALI_CINSTR_PP_TEXTURE_CACHE_COMPRESSED_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 22, - MALI_CINSTR_PP_LOAD_UNIT_READS = MALI_CINSTR_COUNTER_SOURCE_PP + 23, - MALI_CINSTR_PP_POLYGON_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 24, - MALI_CINSTR_PP_PIXEL_RECTANGLE_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 25, - MALI_CINSTR_PP_LINES_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 26, - MALI_CINSTR_PP_POINTS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 27, - MALI_CINSTR_PP_STALL_CYCLES_POLYGON_LIST_READER = MALI_CINSTR_COUNTER_SOURCE_PP + 28, - MALI_CINSTR_PP_STALL_CYCLES_TRIANGLE_SETUP = MALI_CINSTR_COUNTER_SOURCE_PP + 29, - MALI_CINSTR_PP_QUAD_RASTERIZED_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 30, - MALI_CINSTR_PP_FRAGMENT_RASTERIZED_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 31, - MALI_CINSTR_PP_FRAGMENT_REJECTED_FRAGMENT_KILL_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 32, - MALI_CINSTR_PP_FRAGMENT_REJECTED_FWD_FRAGMENT_KILL_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 33, - MALI_CINSTR_PP_FRAGMENT_PASSED_ZSTENCIL_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 34, - MALI_CINSTR_PP_PATCHES_REJECTED_EARLY_Z_STENCIL_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 35, - MALI_CINSTR_PP_PATCHES_EVALUATED = MALI_CINSTR_COUNTER_SOURCE_PP + 36, - MALI_CINSTR_PP_INSTRUCTION_COMPLETED_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 37, - MALI_CINSTR_PP_INSTRUCTION_FAILED_RENDEZVOUS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 38, - MALI_CINSTR_PP_INSTRUCTION_FAILED_VARYING_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 39, - MALI_CINSTR_PP_INSTRUCTION_FAILED_TEXTURE_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 40, - MALI_CINSTR_PP_INSTRUCTION_FAILED_LOAD_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 41, - MALI_CINSTR_PP_INSTRUCTION_FAILED_TILE_READ_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 42, - MALI_CINSTR_PP_INSTRUCTION_FAILED_STORE_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 43, - MALI_CINSTR_PP_RENDEZVOUS_BREAKAGE_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 44, - MALI_CINSTR_PP_PIPELINE_BUBBLES_CYCLE_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 45, - MALI_CINSTR_PP_TEXTURE_MAPPER_MULTIPASS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 46, - MALI_CINSTR_PP_TEXTURE_MAPPER_CYCLE_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 47, - MALI_CINSTR_PP_VERTEX_CACHE_HIT_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 48, - MALI_CINSTR_PP_VERTEX_CACHE_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 49, - MALI_CINSTR_PP_VARYING_CACHE_HIT_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 50, - MALI_CINSTR_PP_VARYING_CACHE_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 51, - MALI_CINSTR_PP_VARYING_CACHE_CONFLICT_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 52, - MALI_CINSTR_PP_TEXTURE_CACHE_HIT_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 53, - MALI_CINSTR_PP_TEXTURE_CACHE_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 54, - MALI_CINSTR_PP_TEXTURE_CACHE_CONFLICT_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 55, - MALI_CINSTR_PP_PALETTE_CACHE_HIT_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 56, /* Mali 200 only */ - MALI_CINSTR_PP_PALETTE_CACHE_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 57, /* Mali 200 only */ - MALI_CINSTR_PP_COMPRESSED_TEXTURE_CACHE_HIT_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 56, /* Mali 400 class only */ - MALI_CINSTR_PP_COMPRESSED_TEXTURE_CACHE_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 57, /* Mali 400 class only */ - MALI_CINSTR_PP_LOAD_STORE_CACHE_HIT_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 58, - MALI_CINSTR_PP_LOAD_STORE_CACHE_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 59, - MALI_CINSTR_PP_PROGRAM_CACHE_HIT_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 60, - MALI_CINSTR_PP_PROGRAM_CACHE_MISS_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 61, - MALI_CINSTR_PP_JOB_COUNT = MALI_CINSTR_COUNTER_SOURCE_PP + 900, -} cinstr_counters_m200_t; - -#endif /*_MALI_UTGARD_COUNTERS_H_*/ diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h index d280dfdf640d..e8b96e271218 100755 --- a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h @@ -1,11 +1,21 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. * * A copy of the licence is included with the program, and can also be obtained from Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. */ #ifndef __MALI_UTGARD_IOCTL_H__ @@ -52,14 +62,16 @@ extern "C" { #define MALI_IOC_TIMELINE_CREATE_SYNC_FENCE _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, _mali_uk_timeline_create_sync_fence_s) #define MALI_IOC_SOFT_JOB_START _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_START, _mali_uk_soft_job_start_s) #define MALI_IOC_SOFT_JOB_SIGNAL _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_SIGNAL, _mali_uk_soft_job_signal_s) +#define MALI_IOC_PENDING_SUBMIT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_PENDING_SUBMIT, _mali_uk_pending_submit_s) -#define MALI_IOC_MEM_MAP_EXT _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_MAP_EXT_MEM, _mali_uk_map_external_mem_s) -#define MALI_IOC_MEM_UNMAP_EXT _IOW (MALI_IOC_MEMORY_BASE, _MALI_UK_UNMAP_EXT_MEM, _mali_uk_unmap_external_mem_s) -#define MALI_IOC_MEM_ATTACH_DMA_BUF _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_ATTACH_DMA_BUF, _mali_uk_attach_dma_buf_s) -#define MALI_IOC_MEM_RELEASE_DMA_BUF _IOW(MALI_IOC_MEMORY_BASE, _MALI_UK_RELEASE_DMA_BUF, _mali_uk_release_dma_buf_s) +#define MALI_IOC_MEM_ALLOC _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_ALLOC_MEM, _mali_uk_alloc_mem_s) +#define MALI_IOC_MEM_FREE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_FREE_MEM, _mali_uk_free_mem_s) +#define MALI_IOC_MEM_BIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_BIND_MEM, _mali_uk_bind_mem_s) +#define MALI_IOC_MEM_UNBIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_UNBIND_MEM, _mali_uk_unbind_mem_s) +#define MALI_IOC_MEM_COW _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MEM, _mali_uk_cow_mem_s) +#define MALI_IOC_MEM_COW_MODIFY_RANGE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MODIFY_RANGE, _mali_uk_cow_modify_range_s) +#define MALI_IOC_MEM_RESIZE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_RESIZE_MEM, _mali_uk_mem_resize_s) #define MALI_IOC_MEM_DMA_BUF_GET_SIZE _IOR(MALI_IOC_MEMORY_BASE, _MALI_UK_DMA_BUF_GET_SIZE, _mali_uk_dma_buf_get_size_s) -#define MALI_IOC_MEM_ATTACH_UMP _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_ATTACH_UMP_MEM, _mali_uk_attach_ump_mem_s) -#define MALI_IOC_MEM_RELEASE_UMP _IOW(MALI_IOC_MEMORY_BASE, _MALI_UK_RELEASE_UMP_MEM, _mali_uk_release_ump_mem_s) #define MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE _IOR (MALI_IOC_MEMORY_BASE, _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, _mali_uk_query_mmu_page_table_dump_size_s) #define MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_DUMP_MMU_PAGE_TABLE, _mali_uk_dump_mmu_page_table_s) #define MALI_IOC_MEM_WRITE_SAFE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_MEM_WRITE_SAFE, _mali_uk_mem_write_safe_s) @@ -78,6 +90,8 @@ extern "C" { #define MALI_IOC_PROFILING_ADD_EVENT _IOWR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_ADD_EVENT, _mali_uk_profiling_add_event_s) #define MALI_IOC_PROFILING_REPORT_SW_COUNTERS _IOW (MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_REPORT_SW_COUNTERS, _mali_uk_sw_counters_report_s) #define MALI_IOC_PROFILING_MEMORY_USAGE_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_MEMORY_USAGE_GET, _mali_uk_profiling_memory_usage_get_s) +#define MALI_IOC_PROFILING_STREAM_FD_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_STREAM_FD_GET, _mali_uk_profiling_stream_fd_get_s) +#define MALI_IOC_PROILING_CONTROL_SET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_CONTROL_SET, _mali_uk_profiling_control_set_s) #define MALI_IOC_VSYNC_EVENT_REPORT _IOW (MALI_IOC_VSYNC_BASE, _MALI_UK_VSYNC_EVENT_REPORT, _mali_uk_vsync_event_report_s) diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h index 40858b800ce0..279bf8ee38a4 100755 --- a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h @@ -1,11 +1,21 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. * * A copy of the licence is included with the program, and can also be obtained from Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. */ #ifndef _MALI_UTGARD_PROFILING_EVENTS_H_ @@ -131,6 +141,7 @@ typedef enum { MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_JOBS_WAIT = 46, /* USED */ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOFRAMES_WAIT = 47, /* USED */ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOJOBS_WAIT = 48, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_SUBMIT_LIMITER_WAIT = 49, /* USED */ } cinstr_profiling_event_reason_suspend_resume_sw_t; /** diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h index 20a630ffd0df..b922187a0b88 100755 --- a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h @@ -1,11 +1,21 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. * * A copy of the licence is included with the program, and can also be obtained from Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. */ #ifndef __MALI_UTGARD_PROFILING_GATOR_API_H__ @@ -21,6 +31,39 @@ extern "C" { #define MAX_NUM_FP_CORES 8 #define MAX_NUM_VP_CORES 1 +#define _MALI_SPCIAL_COUNTER_DESCRIPTIONS \ + { \ + "Filmstrip_cnt0", \ + "Frequency", \ + "Voltage", \ + "vertex", \ + "fragment", \ + "Total_alloc_pages", \ + }; + +#define _MALI_MEM_COUTNER_DESCRIPTIONS \ + { \ + "untyped_memory", \ + "vertex_index_buffer", \ + "texture_buffer", \ + "varying_buffer", \ + "render_target", \ + "pbuffer_buffer", \ + "plbu_heap", \ + "pointer_array_buffer", \ + "slave_tilelist", \ + "untyped_gp_cmdlist", \ + "polygon_cmdlist", \ + "texture_descriptor", \ + "render_state_word", \ + "shader", \ + "stream_buffer", \ + "fragment_stack", \ + "uniform", \ + "untyped_frame_pool", \ + "untyped_surface", \ + }; + /** The list of events supported by the Mali DDK. */ typedef enum { /* Vertex processor activity */ @@ -116,8 +159,41 @@ typedef enum { COUNTER_GLES_STRIP_LINES_COUNT, COUNTER_GLES_LOOP_LINES_COUNT, + /* Special counter */ + /* Framebuffer capture pseudo-counter */ COUNTER_FILMSTRIP, + COUNTER_FREQUENCY, + COUNTER_VOLTAGE, + COUNTER_VP_ACTIVITY, + COUNTER_FP_ACTIVITY, + COUNTER_TOTAL_ALLOC_PAGES, + + /* Memory usage counter */ + COUNTER_MEM_UNTYPED, + COUNTER_MEM_VB_IB, + COUNTER_MEM_TEXTURE, + COUNTER_MEM_VARYING, + COUNTER_MEM_RT, + COUNTER_MEM_PBUFFER, + /* memory usages for gp command */ + COUNTER_MEM_PLBU_HEAP, + COUNTER_MEM_POINTER_ARRAY, + COUNTER_MEM_SLAVE_TILELIST, + COUNTER_MEM_UNTYPE_GP_CMDLIST, + /* memory usages for polygon list command */ + COUNTER_MEM_POLYGON_CMDLIST, + /* memory usages for pp command */ + COUNTER_MEM_TD, + COUNTER_MEM_RSW, + /* other memory usages */ + COUNTER_MEM_SHADER, + COUNTER_MEM_STREAMS, + COUNTER_MEM_FRAGMENT_STACK, + COUNTER_MEM_UNIFORM, + /* Special mem usage, which is used for mem pool allocation */ + COUNTER_MEM_UNTYPE_MEM_POOL, + COUNTER_MEM_UNTYPE_SURFACE, NUMBER_OF_EVENTS } _mali_osk_counter_id; @@ -132,7 +208,34 @@ typedef enum { #define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT #define FIRST_SPECIAL_COUNTER COUNTER_FILMSTRIP -#define LAST_SPECIAL_COUNTER COUNTER_FILMSTRIP +#define LAST_SPECIAL_COUNTER COUNTER_TOTAL_ALLOC_PAGES + +#define FIRST_MEM_COUNTER COUNTER_MEM_UNTYPED +#define LAST_MEM_COUNTER COUNTER_MEM_UNTYPE_SURFACE + +#define MALI_PROFILING_MEM_COUNTERS_NUM (LAST_MEM_COUNTER - FIRST_MEM_COUNTER + 1) +#define MALI_PROFILING_SPECIAL_COUNTERS_NUM (LAST_SPECIAL_COUNTER - FIRST_SPECIAL_COUNTER + 1) +#define MALI_PROFILING_SW_COUNTERS_NUM (LAST_SW_COUNTER - FIRST_SW_COUNTER + 1) + +/** + * Define the stream header type for porfiling stream. + */ +#define STREAM_HEADER_FRAMEBUFFER 0x05 /* The stream packet header type for framebuffer dumping. */ +#define STREAM_HEADER_COUNTER_VALUE 0x09 /* The stream packet header type for hw/sw/memory counter sampling. */ +#define STREAM_HEADER_CORE_ACTIVITY 0x0a /* The stream packet header type for activity counter sampling. */ +#define STREAM_HEADER_SIZE 5 + +/** + * Define the packet header type of profiling control packet. + */ +#define PACKET_HEADER_ERROR 0x80 /* The response packet header type if error. */ +#define PACKET_HEADER_ACK 0x81 /* The response packet header type if OK. */ +#define PACKET_HEADER_COUNTERS_REQUEST 0x82 /* The control packet header type to request counter information from ddk. */ +#define PACKET_HEADER_COUNTERS_ACK 0x83 /* The response packet header type to send out counter information. */ +#define PACKET_HEADER_COUNTERS_ENABLE 0x84 /* The control packet header type to enable counters. */ +#define PACKET_HEADER_START_CAPTURE_VALUE 0x85 /* The control packet header type to start capture values. */ + +#define PACKET_HEADER_SIZE 5 /** * Structure to pass performance counter data of a Mali core @@ -170,6 +273,19 @@ typedef struct _mali_profiling_mali_version { u32 num_of_vp_cores; } _mali_profiling_mali_version; +/** + * Structure to define the mali profiling counter struct. + */ +typedef struct mali_profiling_counter { + char counter_name[40]; + u32 counter_id; + u32 counter_event; + u32 prev_counter_value; + u32 current_counter_value; + u32 key; + int enabled; +} mali_profiling_counter; + /* * List of possible actions to be controlled by Streamline. * The following numbers are used by gator to control the frame buffer dumping and s/w counter reporting. @@ -179,6 +295,8 @@ typedef struct _mali_profiling_mali_version { #define FBDUMP_CONTROL_RATE (2) #define SW_COUNTER_ENABLE (3) #define FBDUMP_CONTROL_RESIZE_FACTOR (4) +#define MEM_COUNTER_ENABLE (5) +#define ANNOTATE_PROFILING_ENABLE (6) void _mali_profiling_control(u32 action, u32 value); diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h index fd55d5f920b4..b1be0f0b4d48 100755 --- a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h @@ -1,11 +1,21 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. * * A copy of the licence is included with the program, and can also be obtained from Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. */ /** @@ -26,6 +36,8 @@ extern "C" { #define MALI_UK_TIMELINE_SOFT 2 #define MALI_UK_TIMELINE_MAX 3 +#define MALI_UK_BIG_VARYING_SIZE (1024*1024*2) + typedef struct { u32 points[MALI_UK_TIMELINE_MAX]; s32 sync_fd; @@ -81,23 +93,23 @@ typedef enum { _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, /**< _mali_ukk_timeline_create_sync_fence() */ _MALI_UK_SOFT_JOB_START, /**< _mali_ukk_soft_job_start() */ _MALI_UK_SOFT_JOB_SIGNAL, /**< _mali_ukk_soft_job_signal() */ - _MALI_GET_RK_KO_VERSION, /* rk_ext */ + _MALI_UK_PENDING_SUBMIT, /**< _mali_ukk_pending_submit() */ + + _MALI_GET_RK_KO_VERSION, /* rk_ext */ _MALI_UK_GET_MALI_VERSION_IN_RK30, + /** Memory functions */ - _MALI_UK_INIT_MEM = 0, /**< _mali_ukk_init_mem() */ - _MALI_UK_TERM_MEM, /**< _mali_ukk_term_mem() */ - _MALI_UK_MAP_MEM, /**< _mali_ukk_mem_mmap() */ - _MALI_UK_UNMAP_MEM, /**< _mali_ukk_mem_munmap() */ + _MALI_UK_ALLOC_MEM = 0, /**< _mali_ukk_alloc_mem() */ + _MALI_UK_FREE_MEM, /**< _mali_ukk_free_mem() */ + _MALI_UK_BIND_MEM, /**< _mali_ukk_mem_bind() */ + _MALI_UK_UNBIND_MEM, /**< _mali_ukk_mem_unbind() */ + _MALI_UK_COW_MEM, /**< _mali_ukk_mem_cow() */ + _MALI_UK_COW_MODIFY_RANGE, /**< _mali_ukk_mem_cow_modify_range() */ + _MALI_UK_RESIZE_MEM, /**<._mali_ukk_mem_resize() */ _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, /**< _mali_ukk_mem_get_mmu_page_table_dump_size() */ _MALI_UK_DUMP_MMU_PAGE_TABLE, /**< _mali_ukk_mem_dump_mmu_page_table() */ - _MALI_UK_ATTACH_DMA_BUF, /**< _mali_ukk_attach_dma_buf() */ - _MALI_UK_RELEASE_DMA_BUF, /**< _mali_ukk_release_dma_buf() */ _MALI_UK_DMA_BUF_GET_SIZE, /**< _mali_ukk_dma_buf_get_size() */ - _MALI_UK_ATTACH_UMP_MEM, /**< _mali_ukk_attach_ump_mem() */ - _MALI_UK_RELEASE_UMP_MEM, /**< _mali_ukk_release_ump_mem() */ - _MALI_UK_MAP_EXT_MEM, /**< _mali_uku_map_external_mem() */ - _MALI_UK_UNMAP_EXT_MEM, /**< _mali_uku_unmap_external_mem() */ _MALI_UK_MEM_WRITE_SAFE, /**< _mali_uku_mem_write_safe() */ /** Common functions for each core */ @@ -126,6 +138,8 @@ typedef enum { _MALI_UK_PROFILING_ADD_EVENT = 0, /**< __mali_uku_profiling_add_event() */ _MALI_UK_PROFILING_REPORT_SW_COUNTERS,/**< __mali_uku_profiling_report_sw_counters() */ _MALI_UK_PROFILING_MEMORY_USAGE_GET, /**< __mali_uku_profiling_memory_usage_get() */ + _MALI_UK_PROFILING_STREAM_FD_GET, /** < __mali_uku_profiling_stream_fd_get() */ + _MALI_UK_PROFILING_CONTROL_SET, /** < __mali_uku_profiling_control_set() */ /** VSYNC reporting fuctions */ _MALI_UK_VSYNC_EVENT_REPORT = 0, /**< _mali_ukk_vsync_event_report() */ @@ -261,6 +275,7 @@ typedef struct { u64 user_job_ptr; /**< [in] identifier for the job in user space, a @c mali_gp_job_info* */ u32 priority; /**< [in] job priority. A lower number means higher priority */ u32 frame_registers[MALIGP2_NUM_REGS_FRAME]; /**< [in] core specific registers associated with this job */ + u32 heap_grow_size; /** <[in] the grow size of the plbu heap when out of memory */ u32 perf_counter_flag; /**< [in] bitmask indicating which performance counters to enable, see \ref _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE and related macro definitions */ u32 perf_counter_src0; /**< [in] source id for performance counter 0 (see ARM DDI0415A, Table 3-60) */ u32 perf_counter_src1; /**< [in] source id for performance counter 1 (see ARM DDI0415A, Table 3-60) */ @@ -268,6 +283,9 @@ typedef struct { u32 flush_id; /**< [in] flush id within the originating frame builder */ _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ u64 timeline_point_ptr; /**< [in,out] pointer to u32: location where point on gp timeline for this job will be written */ + u32 varying_memsize; /** < [in] size of varying memory to use deffer bind*/ + u32 varying_alloc_num; + u64 varying_alloc_list; /** < [in] memory hanlde list of varying buffer to use deffer bind */ } _mali_uk_gp_start_job_s; #define _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE (1<<0) /**< Enable performance counter SRC0 for a job */ @@ -282,11 +300,13 @@ typedef struct { u32 heap_current_addr; /**< [out] value of the GP PLB PL heap start address register */ u32 perf_counter0; /**< [out] value of performance counter 0 (see ARM DDI0415A) */ u32 perf_counter1; /**< [out] value of performance counter 1 (see ARM DDI0415A) */ + u32 pending_big_job_num; } _mali_uk_gp_job_finished_s; typedef struct { u64 user_job_ptr; /**< [out] identifier for the job in user space */ u32 cookie; /**< [out] identifier for the core in kernel space on which the job stalled */ + u32 heap_added_size; } _mali_uk_gp_job_suspended_s; /** @} */ /* end group _mali_uk_gp */ @@ -444,6 +464,18 @@ typedef struct { /** @} */ /* end group _mali_uk_soft_job */ +typedef struct { + u32 counter_id; + u32 key; + int enable; +} _mali_uk_annotate_profiling_mem_counter_s; + +typedef struct { + u32 sampling_rate; + int enable; +} _mali_uk_annotate_profiling_enable_s; + + /** @addtogroup _mali_uk_core U/K Core * @{ */ @@ -478,6 +510,9 @@ typedef enum { _MALI_NOTIFICATION_GP_FINISHED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x10, _MALI_NOTIFICATION_GP_STALLED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x20, + /** Profiling notifications */ + _MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x10, + _MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x20, } _mali_uk_notification_type; /** to assist in splitting up 32-bit notification value in subsystem and id value */ @@ -582,6 +617,8 @@ typedef struct { _mali_uk_pp_job_finished_s pp_job_finished; /**< [out] Notification data for _MALI_NOTIFICATION_PP_FINISHED notification type */ _mali_uk_settings_changed_s setting_changed;/**< [out] Notification data for _MALI_NOTIFICAATION_SETTINGS_CHANGED notification type */ _mali_uk_soft_job_activated_s soft_job_activated; /**< [out] Notification data for _MALI_NOTIFICATION_SOFT_ACTIVATED notification type */ + _mali_uk_annotate_profiling_mem_counter_s profiling_mem_counter; + _mali_uk_annotate_profiling_enable_s profiling_enable; } data; } _mali_uk_wait_for_notification_s; @@ -629,7 +666,7 @@ typedef struct { * The 16bit integer is stored twice in a 32bit integer * For example, for version 1 the value would be 0x00010001 */ -#define _MALI_API_VERSION 600 +#define _MALI_API_VERSION 800 #define _MALI_UK_API_VERSION _MAKE_VERSION_ID(_MALI_API_VERSION) /** @@ -720,40 +757,117 @@ typedef struct { u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ } _mali_uk_request_high_priority_s; +/** @brief Arguments for _mali_ukk_pending_submit() */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ +} _mali_uk_pending_submit_s; + /** @} */ /* end group _mali_uk_core */ /** @defgroup _mali_uk_memory U/K Memory * @{ */ -/** Flag for _mali_uk_map_external_mem_s, _mali_uk_attach_ump_mem_s and _mali_uk_attach_dma_buf_s */ -#define _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE (1<<0) +#define _MALI_MEMORY_ALLOCATE_RESIZEABLE (1<<4) /* BUFFER can trim dow/grow*/ +#define _MALI_MEMORY_ALLOCATE_NO_BIND_GPU (1<<5) /*Not map to GPU when allocate, must call bind later*/ +#define _MALI_MEMORY_ALLOCATE_SWAPPABLE (1<<6) /* Allocate swappale memory. */ +#define _MALI_MEMORY_ALLOCATE_DEFER_BIND (1<<7) /*Not map to GPU when allocate, must call bind later*/ + typedef struct { - u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ - u32 phys_addr; /**< [in] physical address */ - u32 size; /**< [in] size */ - u32 mali_address; /**< [in] mali address to map the physical memory to */ - u32 rights; /**< [in] rights necessary for accessing memory */ - u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ - u32 cookie; /**< [out] identifier for mapped memory object in kernel space */ -} _mali_uk_map_external_mem_s; + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 gpu_vaddr; /**< [in] GPU virtual address */ + u32 vsize; /**< [in] vitrual size of the allocation */ + u32 psize; /**< [in] physical size of the allocation */ + u32 flags; + u64 backend_handle; /**< [out] backend handle */ + struct { + /* buffer types*/ + /* CPU read/write info*/ + } buffer_info; +} _mali_uk_alloc_mem_s; + typedef struct { u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ - u32 cookie; /**< [out] identifier for mapped memory object in kernel space */ -} _mali_uk_unmap_external_mem_s; + u32 gpu_vaddr; /**< [in] use as handle to free allocation */ + u32 free_pages_nr; /** < [out] record the number of free pages */ +} _mali_uk_free_mem_s; + + +#define _MALI_MEMORY_BIND_BACKEND_UMP (1<<8) +#define _MALI_MEMORY_BIND_BACKEND_DMA_BUF (1<<9) +#define _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY (1<<10) +#define _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY (1<<11) +#define _MALI_MEMORY_BIND_BACKEND_EXT_COW (1<<12) +#define _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION (1<<13) + + +#define _MALI_MEMORY_BIND_BACKEND_MASK (_MALI_MEMORY_BIND_BACKEND_UMP| \ + _MALI_MEMORY_BIND_BACKEND_DMA_BUF |\ + _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY |\ + _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY |\ + _MALI_MEMORY_BIND_BACKEND_EXT_COW |\ + _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION) + + +#define _MALI_MEMORY_GPU_READ_ALLOCATE (1<<16) + -/** @note This is identical to _mali_uk_map_external_mem_s above, however phys_addr is replaced by memory descriptor */ typedef struct { - u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ - u32 mem_fd; /**< [in] Memory descriptor */ - u32 size; /**< [in] size */ - u32 mali_address; /**< [in] mali address to map the physical memory to */ - u32 rights; /**< [in] rights necessary for accessing memory */ - u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ - u32 cookie; /**< [out] identifier for mapped memory object in kernel space */ -} _mali_uk_attach_dma_buf_s; + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 vaddr; /**< [in] mali address to map the physical memory to */ + u32 size; /**< [in] size */ + u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ + u32 padding; /** padding for 32/64 struct alignment */ + union { + struct { + u32 secure_id; /**< [in] secure id */ + u32 rights; /**< [in] rights necessary for accessing memory */ + u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ + } bind_ump; + struct { + u32 mem_fd; /**< [in] Memory descriptor */ + u32 rights; /**< [in] rights necessary for accessing memory */ + u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ + } bind_dma_buf; + struct { + /**/ + } bind_mali_memory; + struct { + u32 phys_addr; /**< [in] physical address */ + u32 rights; /**< [in] rights necessary for accessing memory */ + u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ + } bind_ext_memory; + } mem_union; +} _mali_uk_bind_mem_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ + u32 vaddr; /**< [in] identifier for mapped memory object in kernel space */ +} _mali_uk_unbind_mem_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 target_handle; /**< [in] handle of allocation need to do COW */ + u32 target_offset; /**< [in] offset in target allocation to do COW(for support COW a memory allocated from memory_bank, PAGE_SIZE align)*/ + u32 target_size; /**< [in] size of target allocation to do COW (for support memory bank, PAGE_SIZE align)(in byte) */ + u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation (PAGE_SIZE align)*/ + u32 range_size; /**< [in] re allocate size (PAGE_SIZE align)*/ + u32 vaddr; /**< [in] mali address for the new allocaiton */ + u32 backend_handle; /**< [out] backend handle */ + u32 flags; +} _mali_uk_cow_mem_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation */ + u32 size; /**< [in] re allocate size*/ + u32 vaddr; /**< [in] mali address for the new allocaiton */ + s32 change_pages_nr; /**< [out] record the page number change for cow operation */ +} _mali_uk_cow_modify_range_s; + typedef struct { u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ @@ -761,26 +875,15 @@ typedef struct { u32 size; /**< [out] size */ } _mali_uk_dma_buf_get_size_s; -typedef struct { - u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ - u64 cookie; /**< [in] identifier for mapped memory object in kernel space */ -} _mali_uk_release_dma_buf_s; +/** Flag for _mali_uk_map_external_mem_s, _mali_uk_attach_ump_mem_s and _mali_uk_attach_dma_buf_s */ +#define _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE (1<<0) -/** @note This is identical to _mali_uk_map_external_mem_s above, however phys_addr is replaced by secure_id */ -typedef struct { - u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ - u32 secure_id; /**< [in] secure id */ - u32 size; /**< [in] size */ - u32 mali_address; /**< [in] mali address to map the physical memory to */ - u32 rights; /**< [in] rights necessary for accessing memory */ - u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ - u32 cookie; /**< [out] identifier for mapped memory object in kernel space */ -} _mali_uk_attach_ump_mem_s; typedef struct { - u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ - u32 cookie; /**< [in] identifier for mapped memory object in kernel space */ -} _mali_uk_release_ump_mem_s; + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 vaddr; /* the buffer to do resize*/ + u32 psize; /* wanted physical size of this memory */ +} _mali_uk_mem_resize_s; /** * @brief Arguments for _mali_uk[uk]_mem_write_safe() @@ -876,6 +979,8 @@ typedef struct { typedef struct { u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ u32 memory_usage; /**< [out] total memory usage */ + u32 vaddr; /**< [in] mali address for the cow allocaiton */ + s32 change_pages_nr; /**< [out] record the page number change for cow operation */ } _mali_uk_profiling_memory_usage_get_s; @@ -913,7 +1018,7 @@ typedef struct { void *mapping; /**< [out] Returns user-space virtual address for the mapping */ u32 size; /**< [in] Size of the requested mapping */ u32 phys_addr; /**< [in] Physical address - could be offset, depending on caller+callee convention */ - u32 cookie; /**< [out] Returns a cookie for use in munmap calls */ + mali_bool writeable; } _mali_uk_mem_mmap_s; /** @brief Arguments to _mali_ukk_mem_munmap() @@ -930,7 +1035,6 @@ typedef struct { u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ void *mapping; /**< [in] The mapping returned from mmap call */ u32 size; /**< [in] The size passed to mmap call */ - u32 cookie; /**< [in] Cookie from mmap call */ } _mali_uk_mem_munmap_s; /** @} */ /* end group _mali_uk_memory */ @@ -1000,6 +1104,19 @@ typedef struct { /** @} */ /* end group uddapi */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + s32 stream_fd; /**< [in] The profiling kernel base stream fd handle */ +} _mali_uk_profiling_stream_fd_get_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 control_packet_data; /**< [in] the control packet data for control settings */ + u32 control_packet_size; /**< [in] The control packet size */ + u64 response_packet_data; /** < [out] The response packet data */ + u32 response_packet_size; /** < [in,out] The response packet data */ +} _mali_uk_profiling_control_set_s; + #ifdef __cplusplus } #endif diff --git a/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h b/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h index 9442bd3b469c..c88c992e859b 100755 --- a/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h +++ b/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c b/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c index 7dfa387625d9..37076a2eaf67 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c index c32b24c05bf3..ec481ba2797c 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -42,12 +42,15 @@ #include "mali_kernel_license.h" #include "mali_memory.h" #include "mali_memory_dma_buf.h" +#include "mali_memory_manager.h" +#include "mali_memory_swap_alloc.h" #if defined(CONFIG_MALI400_INTERNAL_PROFILING) #include "mali_profiling_internal.h" #endif #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) #include "mali_osk_profiling.h" #include "mali_dvfs_policy.h" + static int is_first_resume = 1; /*Store the clk and vol for boot/insmod and mali_resume*/ static struct mali_gpu_clk_item mali_gpu_clk[2]; @@ -106,6 +109,10 @@ extern int mali_max_pp_cores_group_2; module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH); MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only)."); +extern unsigned int mali_mem_swap_out_threshold_value; +module_param(mali_mem_swap_out_threshold_value, uint, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_mem_swap_out_threshold_value, "Threshold value used to limit how much swappable memory cached in Mali driver."); + #if defined(CONFIG_MALI_DVFS) /** the max fps the same as display vsync default 60, can set by module insert parameter */ extern int mali_max_system_fps; @@ -207,7 +214,7 @@ static struct of_device_id base_dt_ids[] = { {.compatible = "arm,mali400"}, /*-------------------------------------------------------*/ {.compatible = "arm,mali-450"}, - {.compatible = "arm,mali-utgard"}, + {.compatible = "arm,mali-470"}, {}, }; @@ -407,7 +414,7 @@ int mali_module_init(void) MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err)); #ifdef MALI_FAKE_PLATFORM_DEVICE #ifndef CONFIG_MALI_DT - mali_platform_device_unregister(); + mali_platform_device_unregister(); #endif #endif mali_platform_device = NULL; @@ -427,11 +434,11 @@ int mali_module_init(void) /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/ mali_get_current_gpu_clk_item(&mali_gpu_clk[0]); _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | - MALI_PROFILING_EVENT_CHANNEL_GPU | - MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, - mali_gpu_clk[0].clock, - mali_gpu_clk[0].vol / 1000, - 0, 0, 0); + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + mali_gpu_clk[0].clock, + mali_gpu_clk[0].vol / 1000, + 0, 0, 0); #endif MALI_PRINT(("Mali device driver loaded\n")); @@ -670,6 +677,8 @@ static int mali_open(struct inode *inode, struct file *filp) /* link in our session data */ filp->private_data = (void *)session_data; + filp->f_mapping = mali_mem_swap_get_global_swap_file()->f_mapping; + return 0; } @@ -781,6 +790,11 @@ static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg); break; + case MALI_IOC_PENDING_SUBMIT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pending_submit_s), sizeof(u64))); + err = pending_submit_wrapper(session_data, (_mali_uk_pending_submit_s __user *)arg); + break; + #if defined(CONFIG_MALI400_PROFILING) case MALI_IOC_PROFILING_ADD_EVENT: BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64))); @@ -792,93 +806,88 @@ static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg); break; - - case MALI_IOC_PROFILING_MEMORY_USAGE_GET: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64))); - err = profiling_memory_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg); + case MALI_IOC_PROFILING_STREAM_FD_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_stream_fd_get_s), sizeof(u64))); + err = profiling_get_stream_fd_wrapper(session_data, (_mali_uk_profiling_stream_fd_get_s __user *)arg); break; + case MALI_IOC_PROILING_CONTROL_SET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_control_set_s), sizeof(u64))); + err = profiling_control_set_wrapper(session_data, (_mali_uk_profiling_control_set_s __user *)arg); + break; #else case MALI_IOC_PROFILING_ADD_EVENT: /* FALL-THROUGH */ case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */ - case MALI_IOC_PROFILING_MEMORY_USAGE_GET: /* FALL-THROUGH */ MALI_DEBUG_PRINT(2, ("Profiling not supported\n")); err = -ENOTTY; break; - #endif - case MALI_IOC_MEM_WRITE_SAFE: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64))); - err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg); + case MALI_IOC_PROFILING_MEMORY_USAGE_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64))); + err = mem_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg); break; - case MALI_IOC_MEM_MAP_EXT: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_map_external_mem_s), sizeof(u64))); - err = mem_map_ext_wrapper(session_data, (_mali_uk_map_external_mem_s __user *)arg); + case MALI_IOC_MEM_ALLOC: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_alloc_mem_s), sizeof(u64))); + err = mem_alloc_wrapper(session_data, (_mali_uk_alloc_mem_s __user *)arg); break; - case MALI_IOC_MEM_UNMAP_EXT: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unmap_external_mem_s), sizeof(u64))); - err = mem_unmap_ext_wrapper(session_data, (_mali_uk_unmap_external_mem_s __user *)arg); + case MALI_IOC_MEM_FREE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_free_mem_s), sizeof(u64))); + err = mem_free_wrapper(session_data, (_mali_uk_free_mem_s __user *)arg); break; - case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64))); - err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg); + case MALI_IOC_MEM_BIND: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_bind_mem_s), sizeof(u64))); + err = mem_bind_wrapper(session_data, (_mali_uk_bind_mem_s __user *)arg); break; - case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64))); - err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg); + case MALI_IOC_MEM_UNBIND: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unbind_mem_s), sizeof(u64))); + err = mem_unbind_wrapper(session_data, (_mali_uk_unbind_mem_s __user *)arg); break; -#if defined(CONFIG_MALI400_UMP) - - case MALI_IOC_MEM_ATTACH_UMP: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_attach_ump_mem_s), sizeof(u64))); - err = mem_attach_ump_wrapper(session_data, (_mali_uk_attach_ump_mem_s __user *)arg); + case MALI_IOC_MEM_COW: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_mem_s), sizeof(u64))); + err = mem_cow_wrapper(session_data, (_mali_uk_cow_mem_s __user *)arg); break; - case MALI_IOC_MEM_RELEASE_UMP: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_release_ump_mem_s), sizeof(u64))); - err = mem_release_ump_wrapper(session_data, (_mali_uk_release_ump_mem_s __user *)arg); + case MALI_IOC_MEM_COW_MODIFY_RANGE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_modify_range_s), sizeof(u64))); + err = mem_cow_modify_range_wrapper(session_data, (_mali_uk_cow_modify_range_s __user *)arg); break; -#else + case MALI_IOC_MEM_RESIZE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_resize_s), sizeof(u64))); + err = mem_resize_mem_wrapper(session_data, (_mali_uk_mem_resize_s __user *)arg); + break; - case MALI_IOC_MEM_ATTACH_UMP: - case MALI_IOC_MEM_RELEASE_UMP: /* FALL-THROUGH */ - MALI_DEBUG_PRINT(2, ("UMP not supported\n")); - err = -ENOTTY; + case MALI_IOC_MEM_WRITE_SAFE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64))); + err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg); break; -#endif -#ifdef CONFIG_DMA_SHARED_BUFFER - case MALI_IOC_MEM_ATTACH_DMA_BUF: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_attach_dma_buf_s), sizeof(u64))); - err = mali_attach_dma_buf(session_data, (_mali_uk_attach_dma_buf_s __user *)arg); + case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64))); + err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg); break; - case MALI_IOC_MEM_RELEASE_DMA_BUF: - BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_release_dma_buf_s), sizeof(u64))); - err = mali_release_dma_buf(session_data, (_mali_uk_release_dma_buf_s __user *)arg); + case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64))); + err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg); break; case MALI_IOC_MEM_DMA_BUF_GET_SIZE: +#ifdef CONFIG_DMA_SHARED_BUFFER BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64))); err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg); - break; #else - - case MALI_IOC_MEM_ATTACH_DMA_BUF: /* FALL-THROUGH */ - case MALI_IOC_MEM_RELEASE_DMA_BUF: /* FALL-THROUGH */ - case MALI_IOC_MEM_DMA_BUF_GET_SIZE: /* FALL-THROUGH */ MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n")); err = -ENOTTY; - break; #endif + break; case MALI_IOC_PP_START_JOB: BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64))); diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h index ee7444bad124..f6042712d1e3 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -16,6 +16,8 @@ extern "C" { #endif #include /* character device definitions */ +#include +#include #include "mali_kernel_license.h" #include "mali_osk_types.h" diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c index 44a57703f000..4f06ca14bf48 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -526,13 +526,13 @@ static const struct file_operations power_always_on_fops = { static ssize_t power_power_events_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { - if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_SUSPEND], strlen(mali_power_events[_MALI_DEVICE_SUSPEND])-1)) { + if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_SUSPEND], strlen(mali_power_events[_MALI_DEVICE_SUSPEND]) - 1)) { mali_pm_os_suspend(MALI_TRUE); - } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_RESUME], strlen(mali_power_events[_MALI_DEVICE_RESUME])-1)) { + } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_RESUME], strlen(mali_power_events[_MALI_DEVICE_RESUME]) - 1)) { mali_pm_os_resume(); - } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_PAUSE], strlen(mali_power_events[_MALI_DEVICE_DVFS_PAUSE])-1)) { + } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_PAUSE], strlen(mali_power_events[_MALI_DEVICE_DVFS_PAUSE]) - 1)) { mali_dev_pause(); - } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_RESUME], strlen(mali_power_events[_MALI_DEVICE_DVFS_RESUME])-1)) { + } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_RESUME], strlen(mali_power_events[_MALI_DEVICE_DVFS_RESUME]) - 1)) { mali_dev_resume(); } *ppos += cnt; @@ -880,10 +880,17 @@ static const struct file_operations profiling_events_human_readable_fops = { static int memory_debugfs_show(struct seq_file *s, void *private_data) { - seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s\n"\ - "==============================================================================================================\n", +#ifdef MALI_MEM_SWAP_TRACKING + seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s %-10s \n"\ + "=================================================================================================================================\n", + "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", + "external_mem", "ump_mem", "dma_mem", "swap_mem"); +#else + seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s \n"\ + "========================================================================================================================\n", "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", "external_mem", "ump_mem", "dma_mem"); +#endif mali_session_memory_tracking(s); return 0; } @@ -1136,6 +1143,9 @@ static ssize_t version_read(struct file *filp, char __user *buf, size_t count, l case _MALI_PRODUCT_ID_MALI450: r = snprintf(buffer, 64, "Mali-450 MP\n"); break; + case _MALI_PRODUCT_ID_MALI470: + r = snprintf(buffer, 64, "Mali-470 MP\n"); + break; case _MALI_PRODUCT_ID_UNKNOWN: return -EINVAL; break; diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h index 2e9c9a5d8485..a36a0cea9972 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h b/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h index c4186130008c..c6cd2bfb7217 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory.c index 3dab8b025099..c45f6ee25887 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -16,136 +16,142 @@ #include #include #include +#include #include "mali_osk.h" -#include "mali_osk_mali.h" -#include "mali_kernel_linux.h" -#include "mali_scheduler.h" #include "mali_executor.h" -#include "mali_kernel_descriptor_mapping.h" #include "mali_memory.h" -#include "mali_memory_dma_buf.h" #include "mali_memory_os_alloc.h" #include "mali_memory_block_alloc.h" +#include "mali_memory_util.h" +#include "mali_memory_virtual.h" +#include "mali_memory_manager.h" +#include "mali_memory_cow.h" +#include "mali_memory_swap_alloc.h" +#include "mali_memory_defer_bind.h" extern unsigned int mali_dedicated_mem_size; extern unsigned int mali_shared_mem_size; -/* session->memory_lock must be held when calling this function */ -static void mali_mem_release(mali_mem_allocation *descriptor) -{ - MALI_DEBUG_ASSERT_POINTER(descriptor); - MALI_DEBUG_ASSERT_LOCK_HELD(descriptor->session->memory_lock); - - MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); - - switch (descriptor->type) { - case MALI_MEM_OS: - mali_mem_os_release(descriptor); - break; - case MALI_MEM_DMA_BUF: -#if defined(CONFIG_DMA_SHARED_BUFFER) - mali_mem_dma_buf_release(descriptor); -#endif - break; - case MALI_MEM_UMP: -#if defined(CONFIG_MALI400_UMP) - mali_mem_ump_release(descriptor); -#endif - break; - case MALI_MEM_EXTERNAL: - mali_mem_external_release(descriptor); - break; - case MALI_MEM_BLOCK: - mali_mem_block_release(descriptor); - break; - default: - MALI_DEBUG_PRINT(1, ("mem type %d is not in the mali_mem_type enum.\n", descriptor->type)); - break; - } -} +#define MALI_VM_NUM_FAULT_PREFETCH (0x8) static void mali_mem_vma_open(struct vm_area_struct *vma) { - mali_mem_allocation *descriptor = (mali_mem_allocation *)vma->vm_private_data; + mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; MALI_DEBUG_PRINT(4, ("Open called on vma %p\n", vma)); - descriptor->cpu_mapping.ref++; - + /* If need to share the allocation, add ref_count here */ + mali_allocation_ref(alloc); return; } - static void mali_mem_vma_close(struct vm_area_struct *vma) { - mali_mem_allocation *descriptor; - struct mali_session_data *session; - mali_mem_virt_cpu_mapping *mapping; - - MALI_DEBUG_PRINT(3, ("Close called on vma %p\n", vma)); - - descriptor = (mali_mem_allocation *)vma->vm_private_data; - BUG_ON(!descriptor); - - MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); - - mapping = &descriptor->cpu_mapping; - BUG_ON(0 == mapping->ref); - - mapping->ref--; - if (0 != mapping->ref) { - MALI_DEBUG_PRINT(3, ("Ignoring this close, %d references still exists\n", mapping->ref)); - return; - } - - session = descriptor->session; - - mali_descriptor_mapping_free(session->descriptor_mapping, descriptor->id); - - _mali_osk_mutex_wait(session->memory_lock); - mali_mem_release(descriptor); - _mali_osk_mutex_signal(session->memory_lock); + /* If need to share the allocation, unref ref_count here */ + mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; - mali_mem_descriptor_destroy(descriptor); + mali_allocation_unref(&alloc); + vma->vm_private_data = NULL; } -static int mali_kernel_memory_cpu_page_fault_handler(struct vm_area_struct *vma, struct vm_fault *vmf) +static int mali_mem_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { - void __user *address; - mali_mem_allocation *descriptor; - - address = vmf->virtual_address; - descriptor = (mali_mem_allocation *)vma->vm_private_data; - - MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); - - /* - * We always fail the call since all memory is pre-faulted when assigned to the process. - * Only the Mali cores can use page faults to extend buffers. - */ + mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; + mali_mem_backend *mem_bkend = NULL; + int ret; + int prefetch_num = MALI_VM_NUM_FAULT_PREFETCH; + + unsigned long address = (unsigned long)vmf->virtual_address; + MALI_DEBUG_ASSERT(alloc->backend_handle); + MALI_DEBUG_ASSERT((unsigned long)alloc->cpu_mapping.addr <= address); + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { + MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); + mutex_unlock(&mali_idr_mutex); + return VM_FAULT_SIGBUS; + } + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(mem_bkend->type == alloc->type); + + if ((mem_bkend->type == MALI_MEM_COW && (MALI_MEM_BACKEND_FLAG_SWAP_COWED != + (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) && + (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE)) { + /*check if use page fault to do COW*/ + MALI_DEBUG_PRINT(4, ("mali_vma_fault: do cow allocate on demand!, address=0x%x\n", address)); + mutex_lock(&mem_bkend->mutex); + ret = mali_mem_cow_allocate_on_demand(mem_bkend, + (address - vma->vm_start) / PAGE_SIZE); + mutex_unlock(&mem_bkend->mutex); + + if (ret != _MALI_OSK_ERR_OK) { + return VM_FAULT_OOM; + } + prefetch_num = 1; - MALI_DEBUG_PRINT(1, ("Page-fault in Mali memory region caused by the CPU.\n")); - MALI_DEBUG_PRINT(1, ("Tried to access %p (process local virtual address) which is not currently mapped to any Mali memory.\n", (void *)address)); + /* handle COW modified range cpu mapping + we zap the mapping in cow_modify_range, it will trigger page fault + when CPU access it, so here we map it to CPU*/ + mutex_lock(&mem_bkend->mutex); + ret = mali_mem_cow_cpu_map_pages_locked(mem_bkend, vma, address, prefetch_num); + mutex_unlock(&mem_bkend->mutex); - MALI_IGNORE(address); - MALI_IGNORE(descriptor); + if (unlikely(ret != _MALI_OSK_ERR_OK)) { + return VM_FAULT_SIGBUS; + } + } else if ((mem_bkend->type == MALI_MEM_SWAP) || + (mem_bkend->type == MALI_MEM_COW && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + u32 offset_in_bkend = (address - vma->vm_start) / PAGE_SIZE; + int ret = _MALI_OSK_ERR_OK; + + mutex_lock(&mem_bkend->mutex); + if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE) { + ret = mali_mem_swap_cow_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); + } else { + ret = mali_mem_swap_allocate_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); + } + mutex_unlock(&mem_bkend->mutex); - return VM_FAULT_SIGBUS; + if (ret != _MALI_OSK_ERR_OK) { + MALI_DEBUG_PRINT(2, ("Mali swap memory page fault process failed, address=0x%x\n", address)); + return VM_FAULT_OOM; + } else { + return VM_FAULT_LOCKED; + } + } else { + MALI_DEBUG_ASSERT(0); + /*NOT support yet*/ + } + return VM_FAULT_NOPAGE; } static struct vm_operations_struct mali_kernel_vm_ops = { .open = mali_mem_vma_open, .close = mali_mem_vma_close, - .fault = mali_kernel_memory_cpu_page_fault_handler + .fault = mali_mem_vma_fault, }; -/** @note munmap handler is done by vma close handler */ + +/** @ map mali allocation to CPU address +* +* Supported backend types: +* --MALI_MEM_OS +* -- need to add COW? + *Not supported backend types: +* -_MALI_MEMORY_BIND_BACKEND_UMP +* -_MALI_MEMORY_BIND_BACKEND_DMA_BUF +* -_MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY +* +*/ int mali_mmap(struct file *filp, struct vm_area_struct *vma) { struct mali_session_data *session; - mali_mem_allocation *descriptor; - u32 size = vma->vm_end - vma->vm_start; + mali_mem_allocation *mali_alloc = NULL; u32 mali_addr = vma->vm_pgoff << PAGE_SHIFT; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_backend *mem_bkend = NULL; + int ret = -EFAULT; session = (struct mali_session_data *)filp->private_data; if (NULL == session) { @@ -157,85 +163,111 @@ int mali_mmap(struct file *filp, struct vm_area_struct *vma) (unsigned int)vma->vm_start, (unsigned int)(vma->vm_pgoff << PAGE_SHIFT), (unsigned int)(vma->vm_end - vma->vm_start), vma->vm_flags)); - /* Set some bits which indicate that, the memory is IO memory, meaning - * that no paging is to be performed and the memory should not be - * included in crash dumps. And that the memory is reserved, meaning - * that it's present and can never be paged out (see also previous - * entry) - */ - vma->vm_flags |= VM_IO; - vma->vm_flags |= VM_DONTCOPY; - vma->vm_flags |= VM_PFNMAP; -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) - vma->vm_flags |= VM_RESERVED; -#else - vma->vm_flags |= VM_DONTDUMP; - vma->vm_flags |= VM_DONTEXPAND; -#endif - - vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); - vma->vm_ops = &mali_kernel_vm_ops; /* Operations used on any memory system */ - - descriptor = mali_mem_block_alloc(mali_addr, size, vma, session); - if (NULL == descriptor) { - descriptor = mali_mem_os_alloc(mali_addr, size, vma, session); - if (NULL == descriptor) { - MALI_DEBUG_PRINT(3, ("MMAP failed\n")); - return -ENOMEM; + /* Operations used on any memory system */ + /* do not need to anything in vm open/close now */ + + /* find mali allocation structure by vaddress*/ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + if (likely(mali_vma_node)) { + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); + if (unlikely(mali_addr != mali_vma_node->vm_node.start)) { + /* only allow to use start address for mmap */ + MALI_DEBUG_PRINT(1, ("mali_addr != mali_vma_node->vm_node.start\n")); + return -EFAULT; } + } else { + MALI_DEBUG_ASSERT(NULL == mali_vma_node); + return -EFAULT; } - MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); - - vma->vm_private_data = (void *)descriptor; + mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; - /* Put on descriptor map */ - if (_MALI_OSK_ERR_OK != mali_descriptor_mapping_allocate_mapping(session->descriptor_mapping, descriptor, &descriptor->id)) { - _mali_osk_mutex_wait(session->memory_lock); - if (MALI_MEM_OS == descriptor->type) { - mali_mem_os_release(descriptor); - } else if (MALI_MEM_BLOCK == descriptor->type) { - mali_mem_block_release(descriptor); - } - _mali_osk_mutex_signal(session->memory_lock); + if (mali_alloc->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { + MALI_DEBUG_PRINT(1, ("ERROR : trying to access varying memory by CPU!\n")); return -EFAULT; } - return 0; -} - - -/* Prepare memory descriptor */ -mali_mem_allocation *mali_mem_descriptor_create(struct mali_session_data *session, mali_mem_type type) -{ - mali_mem_allocation *descriptor; + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + if (!(mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle))) { + MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); + mutex_unlock(&mali_idr_mutex); + return -EFAULT; + } + mutex_unlock(&mali_idr_mutex); + + if (!(MALI_MEM_SWAP == mali_alloc->type || + (MALI_MEM_COW == mali_alloc->type && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { + /* Set some bits which indicate that, the memory is IO memory, meaning + * that no paging is to be performed and the memory should not be + * included in crash dumps. And that the memory is reserved, meaning + * that it's present and can never be paged out (see also previous + * entry) + */ + vma->vm_flags |= VM_IO; + vma->vm_flags |= VM_DONTCOPY; + vma->vm_flags |= VM_PFNMAP; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) + vma->vm_flags |= VM_RESERVED; +#else + vma->vm_flags |= VM_DONTDUMP; + vma->vm_flags |= VM_DONTEXPAND; +#endif + } else if (MALI_MEM_SWAP == mali_alloc->type) { + vma->vm_pgoff = mem_bkend->start_idx; + } - descriptor = (mali_mem_allocation *)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL); - if (NULL == descriptor) { - MALI_DEBUG_PRINT(3, ("mali_ukk_mem_mmap: descriptor was NULL\n")); - return NULL; + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + vma->vm_ops = &mali_kernel_vm_ops; + + mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; + + /* If it's a copy-on-write mapping, map to read only */ + if (!(vma->vm_flags & VM_WRITE)) { + MALI_DEBUG_PRINT(4, ("mmap allocation with read only !\n")); + /* add VM_WRITE for do_page_fault will check this when a write fault */ + vma->vm_flags |= VM_WRITE | VM_READ; + vma->vm_page_prot = PAGE_READONLY; + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE; + goto out; } - MALI_DEBUG_CODE(descriptor->magic = MALI_MEM_ALLOCATION_VALID_MAGIC); + if (mem_bkend->type == MALI_MEM_OS) { + ret = mali_mem_os_cpu_map(mem_bkend, vma); + } else if (mem_bkend->type == MALI_MEM_COW && + (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + ret = mali_mem_cow_cpu_map(mem_bkend, vma); + } else if (mem_bkend->type == MALI_MEM_BLOCK) { + ret = mali_mem_block_cpu_map(mem_bkend, vma); + } else if ((mem_bkend->type == MALI_MEM_SWAP) || (mem_bkend->type == MALI_MEM_COW && + (MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { + /*For swappable memory, CPU page table will be created by page fault handler. */ + ret = 0; + } else { + /* Not support yet*/ + MALI_DEBUG_ASSERT(0); + } - descriptor->flags = 0; - descriptor->type = type; - descriptor->session = session; + if (ret != 0) { + MALI_DEBUG_PRINT(1, ("ret != 0\n")); + return -EFAULT; + } +out: + MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == mali_alloc->magic); - return descriptor; -} + vma->vm_private_data = (void *)mali_alloc; + mali_alloc->cpu_mapping.vma = vma; -void mali_mem_descriptor_destroy(mali_mem_allocation *descriptor) -{ - MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); - MALI_DEBUG_CODE(descriptor->magic = MALI_MEM_ALLOCATION_FREED_MAGIC); + mali_allocation_ref(mali_alloc); - kfree(descriptor); + return 0; } _mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor) { - u32 size = descriptor->size; + u32 size = descriptor->psize; struct mali_session_data *session = descriptor->session; MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); @@ -246,23 +278,35 @@ _mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor) size += MALI_MMU_PAGE_SIZE; } - return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_mapping.addr, size); + return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start, size); } -void mali_mem_mali_map_free(mali_mem_allocation *descriptor) +_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size) { - u32 size = descriptor->size; + u32 old_size = descriptor->psize; struct mali_session_data *session = descriptor->session; MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + new_size += MALI_MMU_PAGE_SIZE; + } + + if (new_size > old_size) { + MALI_DEBUG_ASSERT(new_size <= descriptor->mali_vma_node.vm_node.size); + return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start + old_size, new_size - old_size); + } + return _MALI_OSK_ERR_OK; +} + +void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags) +{ + if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { size += MALI_MMU_PAGE_SIZE; } /* Umap and flush L2 */ - mali_mmu_pagedir_unmap(session->page_directory, descriptor->mali_mapping.addr, descriptor->size); - + mali_mmu_pagedir_unmap(session->page_directory, vaddr, size); mali_executor_zap_all_active(session); } @@ -270,7 +314,10 @@ u32 _mali_ukk_report_memory_usage(void) { u32 sum = 0; - sum += mali_mem_block_allocator_stat(); + if (MALI_TRUE == mali_memory_have_dedicated_memory()) { + sum += mali_mem_block_allocator_stat(); + } + sum += mali_mem_os_stat(); return sum; @@ -292,45 +339,19 @@ _mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session_ { MALI_DEBUG_PRINT(5, ("Memory session begin\n")); - /* Create descriptor mapping table */ - session_data->descriptor_mapping = mali_descriptor_mapping_create(MALI_MEM_DESCRIPTORS_INIT, MALI_MEM_DESCRIPTORS_MAX); - - if (NULL == session_data->descriptor_mapping) { - MALI_ERROR(_MALI_OSK_ERR_NOMEM); - } - session_data->memory_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_MEM_SESSION); if (NULL == session_data->memory_lock) { - mali_descriptor_mapping_destroy(session_data->descriptor_mapping); - _mali_osk_free(session_data); MALI_ERROR(_MALI_OSK_ERR_FAULT); } + mali_memory_manager_init(&session_data->allocation_mgr); + MALI_DEBUG_PRINT(5, ("MMU session begin: success\n")); MALI_SUCCESS; } -/** @brief Callback function that releases memory - * - * session->memory_lock must be held when calling this function. - */ -static void descriptor_table_cleanup_callback(int descriptor_id, void *map_target) -{ - mali_mem_allocation *descriptor; - - descriptor = (mali_mem_allocation *)map_target; - - MALI_DEBUG_ASSERT_LOCK_HELD(descriptor->session->memory_lock); - - MALI_DEBUG_PRINT(3, ("Cleanup of descriptor %d mapping to 0x%x in descriptor table\n", descriptor_id, map_target)); - MALI_DEBUG_ASSERT(descriptor); - - mali_mem_release(descriptor); - mali_mem_descriptor_destroy(descriptor); -} - void mali_memory_session_end(struct mali_session_data *session) { MALI_DEBUG_PRINT(3, ("MMU session end\n")); @@ -339,18 +360,10 @@ void mali_memory_session_end(struct mali_session_data *session) MALI_DEBUG_PRINT(1, ("No session data found during session end\n")); return; } - - /* Lock the session so we can modify the memory list */ - _mali_osk_mutex_wait(session->memory_lock); - - /* Free all allocations still in the descriptor map, and terminate the map */ - if (NULL != session->descriptor_mapping) { - mali_descriptor_mapping_call_for_each(session->descriptor_mapping, descriptor_table_cleanup_callback); - mali_descriptor_mapping_destroy(session->descriptor_mapping); - session->descriptor_mapping = NULL; - } - - _mali_osk_mutex_signal(session->memory_lock); + /* free allocation */ + mali_free_session_allocations(session); + /* do some check in unint*/ + mali_memory_manager_uninit(&session->allocation_mgr); /* Free the lock */ _mali_osk_mutex_term(session->memory_lock); @@ -360,11 +373,138 @@ void mali_memory_session_end(struct mali_session_data *session) _mali_osk_errcode_t mali_memory_initialize(void) { - return mali_mem_os_init(); + _mali_osk_errcode_t err; + + idr_init(&mali_backend_idr); + mutex_init(&mali_idr_mutex); + + err = mali_mem_swap_init(); + if (err != _MALI_OSK_ERR_OK) { + return err; + } + err = mali_mem_os_init(); + if (_MALI_OSK_ERR_OK == err) { + err = mali_mem_defer_bind_manager_init(); + } + + return err; } void mali_memory_terminate(void) { + mali_mem_swap_term(); + mali_mem_defer_bind_manager_destory(); mali_mem_os_term(); - mali_mem_block_allocator_destroy(NULL); + if (mali_memory_have_dedicated_memory()) { + mali_mem_block_allocator_destroy(); + } +} + + +struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type) +{ + mali_page_node *page_node = NULL; + + page_node = kzalloc(sizeof(mali_page_node), GFP_KERNEL); + MALI_DEBUG_ASSERT(NULL != page_node); + + if (page_node) { + page_node->type = type; + INIT_LIST_HEAD(&page_node->list); + } + + return page_node; +} + +void _mali_page_node_ref(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + /* add ref to this page */ + get_page(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + mali_mem_block_add_ref(node); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + atomic_inc(&node->swap_it->ref_count); + } else + MALI_DEBUG_ASSERT(0); +} + +void _mali_page_node_unref(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + /* unref to this page */ + put_page(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + mali_mem_block_dec_ref(node); + } else + MALI_DEBUG_ASSERT(0); +} + + +void _mali_page_node_add_page(struct mali_page_node *node, struct page *page) +{ + MALI_DEBUG_ASSERT(MALI_PAGE_NODE_OS == node->type); + node->page = page; +} + + +void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item) +{ + MALI_DEBUG_ASSERT(MALI_PAGE_NODE_SWAP == node->type); + node->swap_it = item; +} + +void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item) +{ + MALI_DEBUG_ASSERT(MALI_PAGE_NODE_BLOCK == node->type); + node->blk_it = item; +} + + +int _mali_page_node_get_ref_count(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + /* get ref count of this page */ + return page_count(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + return mali_mem_block_get_ref_count(node); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return atomic_read(&node->swap_it->ref_count); + } else { + MALI_DEBUG_ASSERT(0); + } + return -1; +} + + +dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + return page_private(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + return _mali_blk_item_get_phy_addr(node->blk_it); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return node->swap_it->dma_addr; + } else { + MALI_DEBUG_ASSERT(0); + } + return 0; } + + +unsigned long _mali_page_node_get_pfn(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + return page_to_pfn(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + /* get phy addr for BLOCK page*/ + return _mali_blk_item_get_pfn(node->blk_it); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return page_to_pfn(node->swap_it->page); + } else { + MALI_DEBUG_ASSERT(0); + } + return 0; +} + + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory.h index dcdc953f2bd1..3140c6c98d2c 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -58,22 +58,6 @@ mali_mmu_release_table_page(mali_dma_addr phys, void *virt) */ int mali_mmap(struct file *filp, struct vm_area_struct *vma); -/** @brief Allocate and initialize a Mali memory descriptor - * - * @param session Pointer to the session allocating the descriptor - * @param type Type of memory the descriptor will represent - */ -mali_mem_allocation *mali_mem_descriptor_create(struct mali_session_data *session, mali_mem_type type); - -/** @brief Destroy a Mali memory descriptor - * - * This function will only free the descriptor itself, and not the memory it - * represents. - * - * @param descriptor Pointer to the descriptor to destroy - */ -void mali_mem_descriptor_destroy(mali_mem_allocation *descriptor); - /** @brief Start a new memory session * * Called when a process opens the Mali device node. @@ -103,6 +87,18 @@ void mali_memory_session_end(struct mali_session_data *session); */ _mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor); +/** @brief Resize Mali page tables for mapping + * + * This function will Resize the Mali page tables for mapping the memory + * described by \a descriptor. + * + * Page tables will be reference counted and allocated, if not yet present. + * + * @param descriptor Pointer to the memory descriptor to the mapping + * @param new_size The new size of descriptor + */ +_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size); + /** @brief Free Mali page tables for mapping * * This function will unmap pages from Mali memory and free the page tables @@ -112,7 +108,7 @@ _mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor); * * @param descriptor Pointer to the memory descriptor to unmap */ -void mali_mem_mali_map_free(mali_mem_allocation *descriptor); +void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags); /** @brief Parse resource and prepare the OS memory allocator * @@ -130,7 +126,18 @@ _mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size); _mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); -void mali_mem_ump_release(mali_mem_allocation *descriptor); -void mali_mem_external_release(mali_mem_allocation *descriptor); +struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type); + +void _mali_page_node_ref(struct mali_page_node *node); +void _mali_page_node_unref(struct mali_page_node *node); +void _mali_page_node_add_page(struct mali_page_node *node, struct page *page); + +void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item); + +void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item); + +int _mali_page_node_get_ref_count(struct mali_page_node *node); +dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node); +unsigned long _mali_page_node_get_pfn(struct mali_page_node *node); #endif /* __MALI_MEMORY_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c index a7698a403733..453ddda3080f 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -7,42 +7,65 @@ * A copy of the licence is included with the program, and can also be obtained from Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ + #include "mali_kernel_common.h" #include "mali_memory.h" #include "mali_memory_block_alloc.h" #include "mali_osk.h" #include -#define MALI_BLOCK_SIZE (256UL * 1024UL) /* 256 kB, remember to keep the ()s */ -struct block_info { - struct block_info *next; -}; -typedef struct block_info block_info; +static mali_block_allocator *mali_mem_block_gobal_allocator = NULL; + +unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item) +{ + return (item->phy_addr & ~(MALI_BLOCK_REF_MASK)); +} -typedef struct block_allocator { - struct mutex mutex; - block_info *all_blocks; - block_info *first_free; - u32 base; - u32 cpu_usage_adjust; - u32 num_blocks; - u32 free_blocks; -} block_allocator; +unsigned long _mali_blk_item_get_pfn(mali_block_item *item) +{ + return (item->phy_addr / MALI_BLOCK_SIZE); +} + + +u32 mali_mem_block_get_ref_count(mali_page_node *node) +{ + MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); + return (node->blk_it->phy_addr & MALI_BLOCK_REF_MASK); +} + -static block_allocator *mali_mem_block_gobal_allocator = NULL; +/* Increase the refence count +* It not atomic, so it need to get sp_lock before call this function +*/ -MALI_STATIC_INLINE u32 get_phys(block_allocator *info, block_info *block) +u32 mali_mem_block_add_ref(mali_page_node *node) { - return info->base + ((block - info->all_blocks) * MALI_BLOCK_SIZE); + MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); + MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) < MALI_BLOCK_MAX_REF_COUNT); + return (node->blk_it->phy_addr++ & MALI_BLOCK_REF_MASK); } -static mali_mem_allocator *mali_mem_block_allocator_create(u32 base_address, u32 cpu_usage_adjust, u32 size) +/* Decase the refence count +* It not atomic, so it need to get sp_lock before call this function +*/ +u32 mali_mem_block_dec_ref(mali_page_node *node) { - block_allocator *info; + MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); + MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) > 0); + return (node->blk_it->phy_addr-- & MALI_BLOCK_REF_MASK); +} + + +static mali_block_allocator *mali_mem_block_allocator_create(u32 base_address, u32 size) +{ + mali_block_allocator *info; u32 usable_size; u32 num_blocks; + mali_page_node *m_node; + mali_block_item *mali_blk_items = NULL; + int i = 0; usable_size = size & ~(MALI_BLOCK_SIZE - 1); MALI_DEBUG_PRINT(3, ("Mali block allocator create for region starting at 0x%08X length 0x%08X\n", base_address, size)); @@ -55,246 +78,253 @@ static mali_mem_allocator *mali_mem_block_allocator_create(u32 base_address, u32 return NULL; } - info = _mali_osk_malloc(sizeof(block_allocator)); + info = _mali_osk_calloc(1, sizeof(mali_block_allocator)); if (NULL != info) { - mutex_init(&info->mutex); - info->all_blocks = _mali_osk_malloc(sizeof(block_info) * num_blocks); - if (NULL != info->all_blocks) { - u32 i; - info->first_free = NULL; - info->num_blocks = num_blocks; - info->free_blocks = num_blocks; - - info->base = base_address; - info->cpu_usage_adjust = cpu_usage_adjust; - - for (i = 0; i < num_blocks; i++) { - info->all_blocks[i].next = info->first_free; - info->first_free = &info->all_blocks[i]; + INIT_LIST_HEAD(&info->free); + spin_lock_init(&info->sp_lock); + info->total_num = num_blocks; + mali_blk_items = _mali_osk_calloc(1, sizeof(mali_block_item) * num_blocks); + + if (mali_blk_items) { + info->items = mali_blk_items; + /* add blocks(4k size) to free list*/ + for (i = 0 ; i < num_blocks ; i++) { + /* add block information*/ + mali_blk_items[i].phy_addr = base_address + (i * MALI_BLOCK_SIZE); + /* add to free list */ + m_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); + if (m_node == NULL) + goto fail; + _mali_page_node_add_block_item(m_node, &(mali_blk_items[i])); + list_add_tail(&m_node->list, &info->free); + atomic_add(1, &info->free_num); } - - return (mali_mem_allocator *)info; + return info; } - _mali_osk_free(info); } - +fail: + mali_mem_block_allocator_destroy(); return NULL; } -void mali_mem_block_allocator_destroy(mali_mem_allocator *allocator) +void mali_mem_block_allocator_destroy(void) { - block_allocator *info = (block_allocator *)allocator; + struct mali_page_node *m_page, *m_tmp; + mali_block_allocator *info = mali_mem_block_gobal_allocator; + MALI_DEBUG_ASSERT_POINTER(info); + MALI_DEBUG_PRINT(4, ("Memory block destroy !\n")); - info = mali_mem_block_gobal_allocator; - if (NULL == info) return; + if (NULL == info) + return; - MALI_DEBUG_ASSERT_POINTER(info); + list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + list_del(&m_page->list); + kfree(m_page); + } - _mali_osk_free(info->all_blocks); + _mali_osk_free(info->items); _mali_osk_free(info); } -static void mali_mem_block_mali_map(mali_mem_allocation *descriptor, u32 phys, u32 virt, u32 size) +u32 mali_mem_block_release(mali_mem_backend *mem_bkend) { - struct mali_page_directory *pagedir = descriptor->session->page_directory; - u32 prop = descriptor->mali_mapping.properties; - u32 offset = 0; - - while (size) { - mali_mmu_pagedir_update(pagedir, virt + offset, phys + offset, MALI_MMU_PAGE_SIZE, prop); - - size -= MALI_MMU_PAGE_SIZE; - offset += MALI_MMU_PAGE_SIZE; - } + mali_mem_allocation *alloc = mem_bkend->mali_allocation; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); + + /* Unmap the memory from the mali virtual address space. */ + mali_mem_block_mali_unmap(alloc); + mutex_lock(&mem_bkend->mutex); + free_pages_nr = mali_mem_block_free(&mem_bkend->block_mem); + mutex_unlock(&mem_bkend->mutex); + return free_pages_nr; } -static int mali_mem_block_cpu_map(mali_mem_allocation *descriptor, struct vm_area_struct *vma, u32 mali_phys, u32 mapping_offset, u32 size, u32 cpu_usage_adjust) -{ - u32 virt = vma->vm_start + mapping_offset; - u32 cpu_phys = mali_phys + cpu_usage_adjust; - u32 offset = 0; - int ret; - while (size) { - ret = vm_insert_pfn(vma, virt + offset, __phys_to_pfn(cpu_phys + offset)); +int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size) +{ + struct mali_page_node *m_page, *m_tmp; + size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; + mali_block_allocator *info = mali_mem_block_gobal_allocator; + MALI_DEBUG_ASSERT_POINTER(info); - if (unlikely(ret)) { - MALI_DEBUG_PRINT(1, ("Block allocator: Failed to insert pfn into vma\n")); - return 1; + MALI_DEBUG_PRINT(4, ("BLOCK Mem: Allocate size = 0x%x\n", size)); + /*do some init */ + INIT_LIST_HEAD(&block_mem->pfns); + + spin_lock(&info->sp_lock); + /*check if have enough space*/ + if (atomic_read(&info->free_num) > page_count) { + list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { + if (page_count > 0) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(m_page) == 0); + list_move(&m_page->list, &block_mem->pfns); + block_mem->count++; + atomic_dec(&info->free_num); + _mali_page_node_ref(m_page); + } else { + break; + } + page_count--; } - - size -= MALI_MMU_PAGE_SIZE; - offset += MALI_MMU_PAGE_SIZE; + } else { + /* can't allocate from BLOCK memory*/ + spin_unlock(&info->sp_lock); + return -1; } + spin_unlock(&info->sp_lock); return 0; } -mali_mem_allocation *mali_mem_block_alloc(u32 mali_addr, u32 size, struct vm_area_struct *vma, struct mali_session_data *session) +u32 mali_mem_block_free(mali_mem_block_mem *block_mem) { - _mali_osk_errcode_t err; - mali_mem_allocation *descriptor; - block_allocator *info; - u32 left; - block_info *last_allocated = NULL; - block_allocator_allocation *ret_allocation; - u32 offset = 0; - - size = ALIGN(size, MALI_BLOCK_SIZE); - - info = mali_mem_block_gobal_allocator; - if (NULL == info) return NULL; + u32 free_pages_nr = 0; - left = size; - MALI_DEBUG_ASSERT(0 != left); - - descriptor = mali_mem_descriptor_create(session, MALI_MEM_BLOCK); - if (NULL == descriptor) { - return NULL; - } + free_pages_nr = mali_mem_block_free_list(&block_mem->pfns); + MALI_DEBUG_PRINT(4, ("BLOCK Mem free : allocated size = 0x%x, free size = 0x%x\n", block_mem->count * _MALI_OSK_MALI_PAGE_SIZE, + free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); + block_mem->count = 0; + MALI_DEBUG_ASSERT(list_empty(&block_mem->pfns)); - descriptor->mali_mapping.addr = mali_addr; - descriptor->size = size; - descriptor->cpu_mapping.addr = (void __user *)vma->vm_start; - descriptor->cpu_mapping.ref = 1; - - if (VM_SHARED == (VM_SHARED & vma->vm_flags)) { - descriptor->mali_mapping.properties = MALI_MMU_FLAGS_DEFAULT; - } else { - /* Cached Mali memory mapping */ - descriptor->mali_mapping.properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE; - vma->vm_flags |= VM_SHARED; - } - - ret_allocation = &descriptor->block_mem.mem; - - ret_allocation->mapping_length = 0; - - _mali_osk_mutex_wait(session->memory_lock); - mutex_lock(&info->mutex); + return free_pages_nr; +} - if (left > (info->free_blocks * MALI_BLOCK_SIZE)) { - MALI_DEBUG_PRINT(2, ("Mali block allocator: not enough free blocks to service allocation (%u)\n", left)); - mutex_unlock(&info->mutex); - _mali_osk_mutex_signal(session->memory_lock); - mali_mem_descriptor_destroy(descriptor); - return NULL; - } - err = mali_mem_mali_map_prepare(descriptor); - if (_MALI_OSK_ERR_OK != err) { - mutex_unlock(&info->mutex); - _mali_osk_mutex_signal(session->memory_lock); - mali_mem_descriptor_destroy(descriptor); - return NULL; +u32 mali_mem_block_free_list(struct list_head *list) +{ + struct mali_page_node *m_page, *m_tmp; + mali_block_allocator *info = mali_mem_block_gobal_allocator; + u32 free_pages_nr = 0; + + if (info) { + spin_lock(&info->sp_lock); + list_for_each_entry_safe(m_page, m_tmp , list, list) { + if (1 == _mali_page_node_get_ref_count(m_page)) { + free_pages_nr++; + } + mali_mem_block_free_node(m_page); + } + spin_unlock(&info->sp_lock); } + return free_pages_nr; +} - while ((left > 0) && (info->first_free)) { - block_info *block; - u32 phys_addr; - u32 current_mapping_size; - - block = info->first_free; - info->first_free = info->first_free->next; - block->next = last_allocated; - last_allocated = block; - - phys_addr = get_phys(info, block); - - if (MALI_BLOCK_SIZE < left) { - current_mapping_size = MALI_BLOCK_SIZE; +/* free the node,*/ +void mali_mem_block_free_node(struct mali_page_node *node) +{ + mali_block_allocator *info = mali_mem_block_gobal_allocator; + + /* only handle BLOCK node */ + if (node->type == MALI_PAGE_NODE_BLOCK && info) { + /*Need to make this atomic?*/ + if (1 == _mali_page_node_get_ref_count(node)) { + /*Move to free list*/ + _mali_page_node_unref(node); + list_move_tail(&node->list, &info->free); + atomic_add(1, &info->free_num); } else { - current_mapping_size = left; + _mali_page_node_unref(node); + list_del(&node->list); + kfree(node); } + } +} - mali_mem_block_mali_map(descriptor, phys_addr, mali_addr + offset, current_mapping_size); - if (mali_mem_block_cpu_map(descriptor, vma, phys_addr, offset, current_mapping_size, info->cpu_usage_adjust)) { - /* release all memory back to the pool */ - while (last_allocated) { - /* This relinks every block we've just allocated back into the free-list */ - block = last_allocated->next; - last_allocated->next = info->first_free; - info->first_free = last_allocated; - last_allocated = block; - } - - mutex_unlock(&info->mutex); - _mali_osk_mutex_signal(session->memory_lock); - - mali_mem_mali_map_free(descriptor); - mali_mem_descriptor_destroy(descriptor); +/* unref the node, but not free it */ +_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node) +{ + mali_block_allocator *info = mali_mem_block_gobal_allocator; + mali_page_node *new_node; + + /* only handle BLOCK node */ + if (node->type == MALI_PAGE_NODE_BLOCK && info) { + /*Need to make this atomic?*/ + if (1 == _mali_page_node_get_ref_count(node)) { + /* allocate a new node, Add to free list, keep the old node*/ + _mali_page_node_unref(node); + new_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); + if (new_node) { + memcpy(new_node, node, sizeof(mali_page_node)); + list_add(&new_node->list, &info->free); + atomic_add(1, &info->free_num); + } else + return _MALI_OSK_ERR_FAULT; - return NULL; + } else { + _mali_page_node_unref(node); } - - left -= current_mapping_size; - offset += current_mapping_size; - ret_allocation->mapping_length += current_mapping_size; - - --info->free_blocks; } + return _MALI_OSK_ERR_OK; +} - mutex_unlock(&info->mutex); - _mali_osk_mutex_signal(session->memory_lock); - - MALI_DEBUG_ASSERT(0 == left); - /* Record all the information about this allocation */ - ret_allocation->last_allocated = last_allocated; - ret_allocation->info = info; +int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props) +{ + struct mali_page_directory *pagedir = session->page_directory; + struct mali_page_node *m_page; + dma_addr_t phys; + u32 virt = vaddr; + u32 prop = props; + + list_for_each_entry(m_page, &block_mem->pfns, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + phys = _mali_page_node_get_dma_addr(m_page); +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); + virt += MALI_MMU_PAGE_SIZE; + } - return descriptor; + return 0; } -void mali_mem_block_release(mali_mem_allocation *descriptor) +void mali_mem_block_mali_unmap(mali_mem_allocation *alloc) { - block_allocator *info = descriptor->block_mem.mem.info; - block_info *block, *next; - block_allocator_allocation *allocation = &descriptor->block_mem.mem; - - MALI_DEBUG_ASSERT(MALI_MEM_BLOCK == descriptor->type); - - block = allocation->last_allocated; - - MALI_DEBUG_ASSERT_POINTER(block); - - /* unmap */ - mali_mem_mali_map_free(descriptor); - - mutex_lock(&info->mutex); + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} - while (block) { - MALI_DEBUG_ASSERT(!((block < info->all_blocks) || (block > (info->all_blocks + info->num_blocks)))); - next = block->next; +int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) +{ + int ret; + mali_mem_block_mem *block_mem = &mem_bkend->block_mem; + unsigned long addr = vma->vm_start; + struct mali_page_node *m_page; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); - /* relink into free-list */ - block->next = info->first_free; - info->first_free = block; + list_for_each_entry(m_page, &block_mem->pfns, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + ret = vm_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); - /* advance the loop */ - block = next; + if (unlikely(0 != ret)) { + return -EFAULT; + } + addr += _MALI_OSK_MALI_PAGE_SIZE; - ++info->free_blocks; } - mutex_unlock(&info->mutex); + return 0; } -u32 mali_mem_block_allocator_stat(void) -{ - block_allocator *info = (block_allocator *)mali_mem_block_gobal_allocator; - - if (NULL == info) return 0; - - MALI_DEBUG_ASSERT_POINTER(info); - - return (info->num_blocks - info->free_blocks) * MALI_BLOCK_SIZE; -} _mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size) { - mali_mem_allocator *allocator; + mali_block_allocator *allocator; /* Do the low level linux operation first */ @@ -305,7 +335,7 @@ _mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 si } /* Create generic block allocator object to handle it */ - allocator = mali_mem_block_allocator_create(start, 0 /* cpu_usage_adjust */, size); + allocator = mali_mem_block_allocator_create(start, size); if (NULL == allocator) { MALI_DEBUG_PRINT(1, ("Memory bank registration failed\n")); @@ -313,7 +343,20 @@ _mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 si MALI_ERROR(_MALI_OSK_ERR_FAULT); } - mali_mem_block_gobal_allocator = (block_allocator *)allocator; + mali_mem_block_gobal_allocator = (mali_block_allocator *)allocator; return _MALI_OSK_ERR_OK; } + +mali_bool mali_memory_have_dedicated_memory(void) +{ + return mali_mem_block_gobal_allocator ? MALI_TRUE : MALI_FALSE; +} + +u32 mali_mem_block_allocator_stat(void) +{ + mali_block_allocator *allocator = mali_mem_block_gobal_allocator; + MALI_DEBUG_ASSERT_POINTER(allocator); + + return (allocator->total_num - atomic_read(&allocator->free_num)) * _MALI_OSK_MALI_PAGE_SIZE; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h index 43b13e42f37e..129434de67df 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -13,17 +13,46 @@ #include "mali_session.h" #include "mali_memory.h" +#include #include "mali_memory_types.h" -typedef struct mali_mem_allocator mali_mem_allocator; - -mali_mem_allocator *mali_block_allocator_create(u32 base_address, u32 cpu_usage_adjust, u32 size); -void mali_mem_block_allocator_destroy(mali_mem_allocator *allocator); - -mali_mem_allocation *mali_mem_block_alloc(u32 mali_addr, u32 size, struct vm_area_struct *vma, struct mali_session_data *session); -void mali_mem_block_release(mali_mem_allocation *descriptor); - +#define MALI_BLOCK_SIZE (PAGE_SIZE) /* 4 kB, manage BLOCK memory as page size */ +#define MALI_BLOCK_REF_MASK (0xFFF) +#define MALI_BLOCK_MAX_REF_COUNT (0xFFF) + + + +typedef struct mali_block_allocator { + /* + * In free list, each node's ref_count is 0, + * ref_count added when allocated or referenced in COW + */ + mali_block_item *items; /* information for each block item*/ + struct list_head free; /*free list of mali_memory_node*/ + spinlock_t sp_lock; /*lock for reference count & free list opertion*/ + u32 total_num; /* Number of total pages*/ + atomic_t free_num; /*number of free pages*/ +} mali_block_allocator; + +unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item); +unsigned long _mali_blk_item_get_pfn(mali_block_item *item); +u32 mali_mem_block_get_ref_count(mali_page_node *node); +u32 mali_mem_block_add_ref(mali_page_node *node); +u32 mali_mem_block_dec_ref(mali_page_node *node); +u32 mali_mem_block_release(mali_mem_backend *mem_bkend); +int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size); +int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props); +void mali_mem_block_mali_unmap(mali_mem_allocation *alloc); + +int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); +_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); +mali_bool mali_memory_have_dedicated_memory(void); +u32 mali_mem_block_free(mali_mem_block_mem *block_mem); +u32 mali_mem_block_free_list(struct list_head *list); +void mali_mem_block_free_node(struct mali_page_node *node); +void mali_mem_block_allocator_destroy(void); +_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node); u32 mali_mem_block_allocator_stat(void); #endif /* __MALI_BLOCK_ALLOCATOR_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c new file mode 100644 index 000000000000..bcd0f8713771 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c @@ -0,0 +1,754 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_ARM +#include +#endif +#include + +#include "mali_memory.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_kernel_linux.h" +#include "mali_memory_cow.h" +#include "mali_memory_block_alloc.h" +#include "mali_memory_swap_alloc.h" + +/** +* allocate pages for COW backend and flush cache +*/ +static struct page *mali_mem_cow_alloc_page(void) + +{ + mali_mem_os_mem os_mem; + struct mali_page_node *node; + struct page *new_page; + + int ret = 0; + /* allocate pages from os mem */ + ret = mali_mem_os_alloc_pages(&os_mem, _MALI_OSK_MALI_PAGE_SIZE); + + if (ret) { + return NULL; + } + + MALI_DEBUG_ASSERT(1 == os_mem.count); + + node = _MALI_OSK_CONTAINER_OF(os_mem.pages.next, struct mali_page_node, list); + new_page = node->page; + node->page = NULL; + list_del(&node->list); + kfree(node); + + return new_page; +} + + +static struct list_head *_mali_memory_cow_get_node_list(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size) +{ + MALI_DEBUG_ASSERT(MALI_MEM_OS == target_bk->type || MALI_MEM_COW == target_bk->type || + MALI_MEM_BLOCK == target_bk->type || MALI_MEM_SWAP == target_bk->type); + + if (MALI_MEM_OS == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->os_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->os_mem.count); + return &target_bk->os_mem.pages; + } else if (MALI_MEM_COW == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->cow_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->cow_mem.count); + return &target_bk->cow_mem.pages; + } else if (MALI_MEM_BLOCK == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->block_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->block_mem.count); + return &target_bk->block_mem.pfns; + } else if (MALI_MEM_SWAP == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->swap_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->swap_mem.count); + return &target_bk->swap_mem.pages; + } + + return NULL; +} + +/** +* Do COW for os memory - support do COW for memory from bank memory +* The range_start/size can be zero, which means it will call cow_modify_range +* latter. +* This function allocate new pages for COW backend from os mem for a modified range +* It will keep the page which not in the modified range and Add ref to it +* +* @target_bk - target allocation's backend(the allocation need to do COW) +* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) +* @target_size - size of target allocation to do COW (for support memory bank) +* @backend -COW backend +* @range_start - offset of modified range (4K align) +* @range_size - size of modified range +*/ +_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + mali_mem_cow *cow = &backend->cow_mem; + struct mali_page_node *m_page, *m_tmp, *page_node; + int target_page = 0; + struct page *new_page; + struct list_head *pages = NULL; + + pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); + + if (NULL == pages) { + MALI_DEBUG_ASSERT(0); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT(0 == cow->count); + + INIT_LIST_HEAD(&cow->pages); + mutex_lock(&target_bk->mutex); + list_for_each_entry_safe(m_page, m_tmp, pages, list) { + /* add page from (target_offset,target_offset+size) to cow backend */ + if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && + (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { + + /* allocate a new page node, alway use OS memory for COW */ + page_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); + + if (NULL == page_node) { + mutex_unlock(&target_bk->mutex); + goto error; + } + + INIT_LIST_HEAD(&page_node->list); + + /* check if in the modified range*/ + if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && + (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { + /* need to allocate a new page */ + /* To simplify the case, All COW memory is allocated from os memory ?*/ + new_page = mali_mem_cow_alloc_page(); + + if (NULL == new_page) { + kfree(page_node); + mutex_unlock(&target_bk->mutex); + goto error; + } + + _mali_page_node_add_page(page_node, new_page); + } else { + /*Add Block memory case*/ + if (m_page->type != MALI_PAGE_NODE_BLOCK) { + _mali_page_node_add_page(page_node, m_page->page); + } else { + page_node->type = MALI_PAGE_NODE_BLOCK; + _mali_page_node_add_block_item(page_node, m_page->blk_it); + } + + /* add ref to this page */ + _mali_page_node_ref(m_page); + } + + /* add it to COW backend page list */ + list_add_tail(&page_node->list, &cow->pages); + cow->count++; + } + target_page++; + } + mutex_unlock(&target_bk->mutex); + return _MALI_OSK_ERR_OK; +error: + mali_mem_cow_release(backend, MALI_FALSE); + return _MALI_OSK_ERR_FAULT; +} + +_mali_osk_errcode_t mali_memory_cow_swap_memory(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + mali_mem_cow *cow = &backend->cow_mem; + struct mali_page_node *m_page, *m_tmp, *page_node; + int target_page = 0; + struct mali_swap_item *swap_item; + struct list_head *pages = NULL; + + pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); + if (NULL == pages) { + MALI_DEBUG_ASSERT(0); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT(0 == cow->count); + + INIT_LIST_HEAD(&cow->pages); + mutex_lock(&target_bk->mutex); + + backend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; + + list_for_each_entry_safe(m_page, m_tmp, pages, list) { + /* add page from (target_offset,target_offset+size) to cow backend */ + if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && + (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { + + /* allocate a new page node, use swap memory for COW memory swap cowed flag. */ + page_node = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); + + if (NULL == page_node) { + mutex_unlock(&target_bk->mutex); + goto error; + } + + /* check if in the modified range*/ + if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && + (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { + /* need to allocate a new page */ + /* To simplify the case, All COW memory is allocated from os memory ?*/ + swap_item = mali_mem_swap_alloc_swap_item(); + + if (NULL == swap_item) { + kfree(page_node); + mutex_unlock(&target_bk->mutex); + goto error; + } + + swap_item->idx = mali_mem_swap_idx_alloc(); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { + MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW.\n")); + kfree(page_node); + kfree(swap_item); + mutex_unlock(&target_bk->mutex); + goto error; + } + + _mali_page_node_add_swap_item(page_node, swap_item); + } else { + _mali_page_node_add_swap_item(page_node, m_page->swap_it); + + /* add ref to this page */ + _mali_page_node_ref(m_page); + } + + list_add_tail(&page_node->list, &cow->pages); + cow->count++; + } + target_page++; + } + mutex_unlock(&target_bk->mutex); + + return _MALI_OSK_ERR_OK; +error: + mali_mem_swap_release(backend, MALI_FALSE); + return _MALI_OSK_ERR_FAULT; + +} + + +_mali_osk_errcode_t _mali_mem_put_page_node(mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + return mali_mem_os_put_page(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + return mali_mem_block_unref_node(node); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return _mali_mem_swap_put_page_node(node); + } else + MALI_DEBUG_ASSERT(0); + return _MALI_OSK_ERR_FAULT; +} + + +/** +* Modify a range of a exist COW backend +* @backend -COW backend +* @range_start - offset of modified range (4K align) +* @range_size - size of modified range(in byte) +*/ +_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + mali_mem_allocation *alloc = NULL; + mali_mem_cow *cow = &backend->cow_mem; + struct mali_page_node *m_page, *m_tmp; + LIST_HEAD(pages); + struct page *new_page; + u32 count = 0; + s32 change_pages_nr = 0; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; + + if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + + alloc = backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); + MALI_DEBUG_ASSERT(((range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE) <= cow->count); + + mutex_lock(&backend->mutex); + + /* free pages*/ + list_for_each_entry_safe(m_page, m_tmp, &cow->pages, list) { + + /* check if in the modified range*/ + if ((count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && + (count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { + if (MALI_PAGE_NODE_SWAP != m_page->type) { + new_page = mali_mem_cow_alloc_page(); + + if (NULL == new_page) { + goto error; + } + if (1 != _mali_page_node_get_ref_count(m_page)) + change_pages_nr++; + /* unref old page*/ + if (_mali_mem_put_page_node(m_page)) { + __free_page(new_page); + goto error; + } + /* add new page*/ + /* always use OS for COW*/ + m_page->type = MALI_PAGE_NODE_OS; + _mali_page_node_add_page(m_page, new_page); + } else { + struct mali_swap_item *swap_item; + + swap_item = mali_mem_swap_alloc_swap_item(); + + if (NULL == swap_item) { + goto error; + } + + swap_item->idx = mali_mem_swap_idx_alloc(); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { + MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW modify range.\n")); + kfree(swap_item); + goto error; + } + + if (1 != _mali_page_node_get_ref_count(m_page)) { + change_pages_nr++; + } + + if (_mali_mem_put_page_node(m_page)) { + mali_mem_swap_free_swap_item(swap_item); + goto error; + } + + _mali_page_node_add_swap_item(m_page, swap_item); + } + } + count++; + } + cow->change_pages_nr = change_pages_nr; + + MALI_DEBUG_ASSERT(MALI_MEM_COW == alloc->type); + + /* ZAP cpu mapping(modified range), and do cpu mapping here if need */ + if (NULL != alloc->cpu_mapping.vma) { + MALI_DEBUG_ASSERT(0 != alloc->backend_handle); + MALI_DEBUG_ASSERT(NULL != alloc->cpu_mapping.vma); + MALI_DEBUG_ASSERT(alloc->cpu_mapping.vma->vm_end - alloc->cpu_mapping.vma->vm_start >= range_size); + + if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { + zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); + + ret = mali_mem_cow_cpu_map_pages_locked(backend, alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size / _MALI_OSK_MALI_PAGE_SIZE); + + if (unlikely(ret != _MALI_OSK_ERR_OK)) { + MALI_DEBUG_PRINT(2, ("mali_memory_cow_modify_range: cpu mapping failed !\n")); + ret = _MALI_OSK_ERR_FAULT; + } + } else { + /* used to trigger page fault for swappable cowed memory. */ + alloc->cpu_mapping.vma->vm_flags |= VM_PFNMAP; + alloc->cpu_mapping.vma->vm_flags |= VM_MIXEDMAP; + + zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); + /* delete this flag to let swappble is ummapped regard to stauct page not page frame. */ + alloc->cpu_mapping.vma->vm_flags &= ~VM_PFNMAP; + alloc->cpu_mapping.vma->vm_flags &= ~VM_MIXEDMAP; + } + } + +error: + mutex_unlock(&backend->mutex); + return ret; + +} + + +/** +* Allocate pages for COW backend +* @alloc -allocation for COW allocation +* @target_bk - target allocation's backend(the allocation need to do COW) +* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) +* @target_size - size of target allocation to do COW (for support memory bank)(in byte) +* @backend -COW backend +* @range_start - offset of modified range (4K align) +* @range_size - size of modified range(in byte) +*/ +_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + struct mali_session_data *session = backend->mali_allocation->session; + + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); + + /* size & offset must be a multiple of the system page size */ + if (target_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (target_offset % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + + /* check backend type */ + MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); + + switch (target_bk->type) { + case MALI_MEM_OS: + case MALI_MEM_BLOCK: + return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + break; + case MALI_MEM_COW: + if (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { + return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + } else { + return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + } + break; + case MALI_MEM_SWAP: + return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + break; + case MALI_MEM_EXTERNAL: + /*NOT support yet*/ + MALI_DEBUG_ASSERT(0); + break; + case MALI_MEM_DMA_BUF: + /*NOT support yet*/ + MALI_DEBUG_ASSERT(0); + break; + case MALI_MEM_UMP: + /*NOT support yet*/ + MALI_DEBUG_ASSERT(0); + break; + default: + /*Not support yet*/ + MALI_DEBUG_ASSERT(0); + break; + } + return _MALI_OSK_ERR_OK; +} + + +/** +* Map COW backend memory to mali +* Support OS/BLOCK for mali_page_node +*/ +int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size) +{ + mali_mem_allocation *cow_alloc; + struct mali_page_node *m_page; + struct mali_session_data *session; + struct mali_page_directory *pagedir; + u32 virt, start; + + cow_alloc = mem_bkend->mali_allocation; + virt = cow_alloc->mali_vma_node.vm_node.start; + start = virt; + + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT_POINTER(cow_alloc); + + session = cow_alloc->session; + pagedir = session->page_directory; + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); + list_for_each_entry(m_page, &mem_bkend->cow_mem.pages, list) { + if ((virt - start >= range_start) && (virt - start < range_start + range_size)) { + dma_addr_t phys = _mali_page_node_get_dma_addr(m_page); +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, + MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + } + virt += MALI_MMU_PAGE_SIZE; + } + return 0; +} + +/** +* Map COW backend to cpu +* support OS/BLOCK memory +*/ +int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) +{ + mali_mem_cow *cow = &mem_bkend->cow_mem; + struct mali_page_node *m_page; + int ret; + unsigned long addr = vma->vm_start; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); + + list_for_each_entry(m_page, &cow->pages, list) { + /* We should use vm_insert_page, but it does a dcache + * flush which makes it way slower than remap_pfn_range or vm_insert_pfn. + ret = vm_insert_page(vma, addr, page); + */ + ret = vm_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); + + if (unlikely(0 != ret)) { + return ret; + } + addr += _MALI_OSK_MALI_PAGE_SIZE; + } + + return 0; +} + +/** +* Map some pages(COW backend) to CPU vma@vaddr +*@ mem_bkend - COW backend +*@ vma +*@ vaddr -start CPU vaddr mapped to +*@ num - max number of pages to map to CPU vaddr +*/ +_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, + struct vm_area_struct *vma, + unsigned long vaddr, + int num) +{ + mali_mem_cow *cow = &mem_bkend->cow_mem; + struct mali_page_node *m_page; + int ret; + int offset; + int count ; + unsigned long vstart = vma->vm_start; + count = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); + MALI_DEBUG_ASSERT(0 == vaddr % _MALI_OSK_MALI_PAGE_SIZE); + MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); + offset = (vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; + + list_for_each_entry(m_page, &cow->pages, list) { + if ((count >= offset) && (count < offset + num)) { + ret = vm_insert_pfn(vma, vaddr, _mali_page_node_get_pfn(m_page)); + + if (unlikely(0 != ret)) { + if (count == offset) { + return _MALI_OSK_ERR_FAULT; + } else { + /* ret is EBUSY when page isn't in modify range, but now it's OK*/ + return _MALI_OSK_ERR_OK; + } + } + vaddr += _MALI_OSK_MALI_PAGE_SIZE; + } + count++; + } + return _MALI_OSK_ERR_OK; +} + +/** +* Release COW backend memory +* free it directly(put_page--unref page), not put into pool +*/ +u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) +{ + mali_mem_allocation *alloc; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + alloc = mem_bkend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)) { + /* Unmap the memory from the mali virtual address space. */ + if (MALI_TRUE == is_mali_mapped) + mali_mem_os_mali_unmap(alloc); + /* free cow backend list*/ + free_pages_nr = mali_mem_os_free(&mem_bkend->cow_mem.pages, mem_bkend->cow_mem.count, MALI_TRUE); + free_pages_nr += mali_mem_block_free_list(&mem_bkend->cow_mem.pages); + + MALI_DEBUG_ASSERT(list_empty(&mem_bkend->cow_mem.pages)); + } else { + free_pages_nr = mali_mem_swap_release(mem_bkend, is_mali_mapped); + } + + + MALI_DEBUG_PRINT(4, ("COW Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->cow_mem.count * _MALI_OSK_MALI_PAGE_SIZE, + free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); + + mem_bkend->cow_mem.count = 0; + return free_pages_nr; +} + + +/* Dst node could os node or swap node. */ +void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node) +{ + void *dst, *src; + struct page *dst_page; + dma_addr_t dma_addr; + + MALI_DEBUG_ASSERT(src_node != NULL); + MALI_DEBUG_ASSERT(dst_node != NULL); + MALI_DEBUG_ASSERT(dst_node->type == MALI_PAGE_NODE_OS + || dst_node->type == MALI_PAGE_NODE_SWAP); + + if (dst_node->type == MALI_PAGE_NODE_OS) { + dst_page = dst_node->page; + } else { + dst_page = dst_node->swap_it->page; + } + + dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(dst_node), + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + /* map it , and copy the content*/ + dst = kmap_atomic(dst_page); + + if (src_node->type == MALI_PAGE_NODE_OS || + src_node->type == MALI_PAGE_NODE_SWAP) { + struct page *src_page; + + if (src_node->type == MALI_PAGE_NODE_OS) { + src_page = src_node->page; + } else { + src_page = src_node->swap_it->page; + } + + /* Clear and invaliate cache */ + /* In ARM architecture, speculative read may pull stale data into L1 cache + * for kernel linear mapping page table. DMA_BIDIRECTIONAL could + * invalidate the L1 cache so that following read get the latest data + */ + dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(src_node), + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + src = kmap_atomic(src_page); + memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); + kunmap_atomic(src); + dma_addr = dma_map_page(&mali_platform_device->dev, src_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + if (src_node->type == MALI_PAGE_NODE_SWAP) { + src_node->swap_it->dma_addr = dma_addr; + } + } else if (src_node->type == MALI_PAGE_NODE_BLOCK) { + /* + * use ioremap to map src for BLOCK memory + */ + src = ioremap_nocache(_mali_page_node_get_dma_addr(src_node), _MALI_OSK_MALI_PAGE_SIZE); + memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); + iounmap(src); + } + kunmap_atomic(dst); + dma_addr = dma_map_page(&mali_platform_device->dev, dst_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + if (dst_node->type == MALI_PAGE_NODE_SWAP) { + dst_node->swap_it->dma_addr = dma_addr; + } +} + + +/* +* allocate page on demand when CPU access it, +* THis used in page fault handler +*/ +_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page) +{ + struct page *new_page = NULL; + struct mali_page_node *new_node = NULL; + int i = 0; + struct mali_page_node *m_page, *found_node = NULL; + struct mali_session_data *session = NULL; + mali_mem_cow *cow = &mem_bkend->cow_mem; + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT(offset_page < mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE); + MALI_DEBUG_PRINT(4, ("mali_mem_cow_allocate_on_demand !, offset_page =0x%x\n", offset_page)); + + /* allocate new page here */ + new_page = mali_mem_cow_alloc_page(); + if (!new_page) + return _MALI_OSK_ERR_NOMEM; + + new_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); + if (!new_node) { + __free_page(new_page); + return _MALI_OSK_ERR_NOMEM; + } + + /* find the page in backend*/ + list_for_each_entry(m_page, &cow->pages, list) { + if (i == offset_page) { + found_node = m_page; + break; + } + i++; + } + MALI_DEBUG_ASSERT(found_node); + if (NULL == found_node) { + __free_page(new_page); + kfree(new_node); + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + _mali_page_node_add_page(new_node, new_page); + + /* Copy the src page's content to new page */ + _mali_mem_cow_copy_page(found_node, new_node); + + MALI_DEBUG_ASSERT_POINTER(mem_bkend->mali_allocation); + session = mem_bkend->mali_allocation->session; + MALI_DEBUG_ASSERT_POINTER(session); + if (1 != _mali_page_node_get_ref_count(found_node)) { + atomic_add(1, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + mem_bkend->cow_mem.change_pages_nr++; + } + if (_mali_mem_put_page_node(found_node)) { + __free_page(new_page); + kfree(new_node); + return _MALI_OSK_ERR_NOMEM; + } + + list_replace(&found_node->list, &new_node->list); + + kfree(found_node); + + /* map to GPU side*/ + _mali_osk_mutex_wait(session->memory_lock); + mali_mem_cow_mali_map(mem_bkend, offset_page * _MALI_OSK_MALI_PAGE_SIZE, _MALI_OSK_MALI_PAGE_SIZE); + _mali_osk_mutex_signal(session->memory_lock); + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h new file mode 100644 index 000000000000..c16ec7633c0a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_COW_H__ +#define __MALI_MEMORY_COW_H__ + +#include "mali_osk.h" +#include "mali_session.h" +#include "mali_memory_types.h" + +int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); +_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, + struct vm_area_struct *vma, + unsigned long vaddr, + int num); + +_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size); + +_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, + u32 range_start, + u32 range_size); + +_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size); + +void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node); + +int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size); +u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); +_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page); +#endif + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c new file mode 100644 index 000000000000..e51902f1bf65 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_ARM +#include +#endif +#include + +#include "mali_memory.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_kernel_linux.h" +#include "mali_memory_defer_bind.h" +#include "mali_executor.h" +#include "mali_osk.h" +#include "mali_scheduler.h" +#include "mali_gp_job.h" + +mali_defer_bind_manager *mali_dmem_man = NULL; + +static u32 mali_dmem_get_gp_varying_size(struct mali_gp_job *gp_job) +{ + return gp_job->uargs.varying_memsize / _MALI_OSK_MALI_PAGE_SIZE; +} + +_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void) +{ + mali_dmem_man = _mali_osk_calloc(1, sizeof(struct mali_defer_bind_manager)); + if (!mali_dmem_man) + return _MALI_OSK_ERR_NOMEM; + + atomic_set(&mali_dmem_man->num_used_pages, 0); + atomic_set(&mali_dmem_man->num_dmem, 0); + + return _MALI_OSK_ERR_OK; +} + + +void mali_mem_defer_bind_manager_destory(void) +{ + if (mali_dmem_man) { + MALI_DEBUG_ASSERT(0 == atomic_read(&mali_dmem_man->num_dmem)); + kfree(mali_dmem_man); + } + mali_dmem_man = NULL; +} + + +/*allocate pages from OS memory*/ +_mali_osk_errcode_t mali_mem_defer_alloc_mem(u32 require, struct mali_session_data *session, mali_defer_mem_block *dblock) +{ + int retval = 0; + u32 num_pages = require; + mali_mem_os_mem os_mem; + + retval = mali_mem_os_alloc_pages(&os_mem, num_pages * _MALI_OSK_MALI_PAGE_SIZE); + + /* add to free pages list */ + if (0 == retval) { + MALI_DEBUG_PRINT(4, ("mali_mem_defer_alloc_mem ,,*** pages allocate = 0x%x \n", num_pages)); + list_splice(&os_mem.pages, &dblock->free_pages); + atomic_add(os_mem.count, &dblock->num_free_pages); + atomic_add(os_mem.count, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + return _MALI_OSK_ERR_OK; + } else + return _MALI_OSK_ERR_FAULT; +} + +_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock) +{ + u32 require_page; + + if (!next_gp_job) + return _MALI_OSK_ERR_FAULT; + + require_page = mali_dmem_get_gp_varying_size(next_gp_job); + + MALI_DEBUG_PRINT(4, ("mali_mem_defer_prepare_mem_work, require alloc page 0x%x\n", + require_page)); + /* allocate more pages from OS */ + if (_MALI_OSK_ERR_OK != mali_mem_defer_alloc_mem(require_page, next_gp_job->session, dblock)) { + MALI_DEBUG_PRINT(1, ("ERROR##mali_mem_defer_prepare_mem_work, allocate page failed!!")); + return _MALI_OSK_ERR_NOMEM; + } + + next_gp_job->bind_flag = MALI_DEFER_BIND_MEMORY_PREPARED; + + return _MALI_OSK_ERR_OK; +} + + +/* do preparetion for allocation before defer bind */ +_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list) +{ + mali_mem_backend *mem_bkend = NULL; + struct mali_backend_bind_list *bk_list = _mali_osk_calloc(1, sizeof(struct mali_backend_bind_list)); + if (NULL == bk_list) + return _MALI_OSK_ERR_FAULT; + + INIT_LIST_HEAD(&bk_list->node); + /* Get backend memory */ + mutex_lock(&mali_idr_mutex); + if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { + MALI_DEBUG_PRINT(1, ("Can't find memory backend in defer bind!\n")); + mutex_unlock(&mali_idr_mutex); + kfree(bk_list); + return _MALI_OSK_ERR_FAULT; + } + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_PRINT(4, ("bind_allocation_prepare:: allocation =%x vaddr=0x%x!\n", alloc, alloc->mali_vma_node.vm_node.start)); + + INIT_LIST_HEAD(&mem_bkend->os_mem.pages); + + bk_list->bkend = mem_bkend; + bk_list->vaddr = alloc->mali_vma_node.vm_node.start; + bk_list->session = alloc->session; + bk_list->page_num = mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); + + /* add to job to do list */ + list_add(&bk_list->node, list); + + return _MALI_OSK_ERR_OK; +} + + + +/* bind phyiscal memory to allocation +This function will be called in IRQ handler*/ +static _mali_osk_errcode_t mali_mem_defer_bind_allocation(struct mali_backend_bind_list *bk_node, + struct list_head *pages) +{ + struct mali_session_data *session = bk_node->session; + mali_mem_backend *mem_bkend = bk_node->bkend; + MALI_DEBUG_PRINT(4, ("mali_mem_defer_bind_allocation, bind bkend = %x page num=0x%x vaddr=%x session=%x\n", mem_bkend, bk_node->page_num, bk_node->vaddr, session)); + + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); + list_splice(pages, &mem_bkend->os_mem.pages); + mem_bkend->os_mem.count = bk_node->page_num; + + if (mem_bkend->type == MALI_MEM_OS) { + mali_mem_os_mali_map(&mem_bkend->os_mem, session, bk_node->vaddr, 0, + mem_bkend->os_mem.count, MALI_MMU_FLAGS_DEFAULT); + } + smp_wmb(); + bk_node->flag = MALI_DEFER_BIND_MEMORY_BINDED; + mem_bkend->flags &= ~MALI_MEM_BACKEND_FLAG_NOT_BINDED; + mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_BINDED; + return _MALI_OSK_ERR_OK; +} + + +static struct list_head *mali_mem_defer_get_free_page_list(u32 count, struct list_head *pages, mali_defer_mem_block *dblock) +{ + int i = 0; + struct mali_page_node *m_page, *m_tmp; + + if (atomic_read(&dblock->num_free_pages) < count) { + return NULL; + } else { + list_for_each_entry_safe(m_page, m_tmp, &dblock->free_pages, list) { + if (i < count) { + list_move_tail(&m_page->list, pages); + } else { + break; + } + i++; + } + MALI_DEBUG_ASSERT(i == count); + atomic_sub(count, &dblock->num_free_pages); + return pages; + } +} + + +/* called in job start IOCTL to bind physical memory for each allocations +@ bk_list backend list to do defer bind +@ pages page list to do this bind +@ count number of pages +*/ +_mali_osk_errcode_t mali_mem_defer_bind(u32 count, struct mali_gp_job *gp, + struct mali_defer_mem_block *dmem_block) +{ + struct mali_defer_mem *dmem = NULL; + struct mali_backend_bind_list *bkn, *bkn_tmp; + LIST_HEAD(pages); + + MALI_DEBUG_PRINT(4, ("#BIND: GP job=%x## \n", gp)); + dmem = (mali_defer_mem *)_mali_osk_calloc(1, sizeof(struct mali_defer_mem)); + if (dmem) { + INIT_LIST_HEAD(&dmem->node); + gp->dmem = dmem; + } else { + return _MALI_OSK_ERR_NOMEM; + } + + atomic_add(1, &mali_dmem_man->num_dmem); + /* for each bk_list backend, do bind */ + list_for_each_entry_safe(bkn, bkn_tmp , &gp->vary_todo, node) { + INIT_LIST_HEAD(&pages); + if (likely(mali_mem_defer_get_free_page_list(bkn->page_num, &pages, dmem_block))) { + list_del(&bkn->node); + mali_mem_defer_bind_allocation(bkn, &pages); + _mali_osk_free(bkn); + } else { + /* not enough memory will not happen */ + MALI_DEBUG_PRINT(1, ("#BIND: NOT enough memory when binded !!## \n")); + MALI_DEBUG_ASSERT(0); + } + } + + if (!list_empty(&gp->vary_todo)) { + MALI_DEBUG_ASSERT(0); + } + + dmem->flag = MALI_DEFER_BIND_MEMORY_BINDED; + + return _MALI_OSK_ERR_OK; +} + +void mali_mem_defer_dmem_free(struct mali_gp_job *gp) +{ + if (gp->dmem) { + atomic_dec(&mali_dmem_man->num_dmem); + _mali_osk_free(gp->dmem); + } +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h new file mode 100644 index 000000000000..ef67540434f5 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#ifndef __MALI_MEMORY_DEFER_BIND_H_ +#define __MALI_MEMORY_DEFER_BIND_H_ + + +#include "mali_osk.h" +#include "mali_session.h" + +#include +#include +#include +#include +#include + + +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" +#include "mali_uk_types.h" + +struct mali_gp_job; + +typedef struct mali_defer_mem { + struct list_head node; /*dlist node in bind manager */ + u32 flag; +} mali_defer_mem; + + +typedef struct mali_defer_mem_block { + struct list_head free_pages; /* page pool */ + atomic_t num_free_pages; +} mali_defer_mem_block; + +/* varying memory list need to bind */ +typedef struct mali_backend_bind_list { + struct list_head node; + struct mali_mem_backend *bkend; + u32 vaddr; + u32 page_num; + struct mali_session_data *session; + u32 flag; +} mali_backend_bind_lists; + + +typedef struct mali_defer_bind_manager { + atomic_t num_used_pages; + atomic_t num_dmem; +} mali_defer_bind_manager; + +_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void); +void mali_mem_defer_bind_manager_destory(void); +_mali_osk_errcode_t mali_mem_defer_bind(u32 count, struct mali_gp_job *gp, + struct mali_defer_mem_block *dmem_block); +_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list); +_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock); +void mali_mem_defer_dmem_free(struct mali_gp_job *gp); + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c index 97f28ad63e95..2fa5028d8342 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -26,84 +26,67 @@ #include "mali_memory.h" #include "mali_memory_dma_buf.h" - +#include "mali_memory_virtual.h" #include "mali_pp_job.h" -static void mali_dma_buf_unmap(struct mali_dma_buf_attachment *mem); - -struct mali_dma_buf_attachment { - struct dma_buf *buf; - struct dma_buf_attachment *attachment; - struct sg_table *sgt; - struct mali_session_data *session; - int map_ref; - struct mutex map_lock; - mali_bool is_mapped; - wait_queue_head_t wait_queue; -}; - -static void mali_dma_buf_release(struct mali_dma_buf_attachment *mem) -{ - MALI_DEBUG_PRINT(3, ("Mali DMA-buf: release attachment %p\n", mem)); - - MALI_DEBUG_ASSERT_POINTER(mem); - MALI_DEBUG_ASSERT_POINTER(mem->attachment); - MALI_DEBUG_ASSERT_POINTER(mem->buf); - -#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) - /* We mapped implicitly on attach, so we need to unmap on release */ - mali_dma_buf_unmap(mem); -#endif - - /* Wait for buffer to become unmapped */ - wait_event(mem->wait_queue, !mem->is_mapped); - MALI_DEBUG_ASSERT(!mem->is_mapped); - - dma_buf_detach(mem->buf, mem->attachment); - dma_buf_put(mem->buf); - - _mali_osk_free(mem); -} - -void mali_mem_dma_buf_release(mali_mem_allocation *descriptor) -{ - struct mali_dma_buf_attachment *mem = descriptor->dma_buf.attachment; - - mali_dma_buf_release(mem); -} - /* * Map DMA buf attachment \a mem into \a session at virtual address \a virt. */ -static int mali_dma_buf_map(struct mali_dma_buf_attachment *mem, struct mali_session_data *session, u32 virt, u32 flags) +static int mali_dma_buf_map(mali_mem_backend *mem_backend) { + mali_mem_allocation *alloc; + struct mali_dma_buf_attachment *mem; + struct mali_session_data *session; struct mali_page_directory *pagedir; + _mali_osk_errcode_t err; struct scatterlist *sg; + u32 virt, flags; int i; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + mem = mem_backend->dma_buf.attachment; MALI_DEBUG_ASSERT_POINTER(mem); + + session = alloc->session; MALI_DEBUG_ASSERT_POINTER(session); MALI_DEBUG_ASSERT(mem->session == session); - mutex_lock(&mem->map_lock); + virt = alloc->mali_vma_node.vm_node.start; + flags = alloc->flags; + mali_session_memory_lock(session); mem->map_ref++; MALI_DEBUG_PRINT(5, ("Mali DMA-buf: map attachment %p, new map_ref = %d\n", mem, mem->map_ref)); if (1 == mem->map_ref) { + /* First reference taken, so we need to map the dma buf */ MALI_DEBUG_ASSERT(!mem->is_mapped); - pagedir = mali_session_get_page_directory(session); - MALI_DEBUG_ASSERT_POINTER(pagedir); - mem->sgt = dma_buf_map_attachment(mem->attachment, DMA_BIDIRECTIONAL); if (IS_ERR_OR_NULL(mem->sgt)) { MALI_DEBUG_PRINT_ERROR(("Failed to map dma-buf attachment\n")); + mem->map_ref--; + mali_session_memory_unlock(session); return -EFAULT; } + err = mali_mem_mali_map_prepare(alloc); + if (_MALI_OSK_ERR_OK != err) { + MALI_DEBUG_PRINT(1, ("Mapping of DMA memory failed\n")); + mem->map_ref--; + mali_session_memory_unlock(session); + return -ENOMEM; + } + + pagedir = mali_session_get_page_directory(session); + MALI_DEBUG_ASSERT_POINTER(pagedir); + for_each_sg(mem->sgt->sgl, sg, mem->sgt->nents, i) { u32 size = sg_dma_len(sg); dma_addr_t phys = sg_dma_address(sg); @@ -126,38 +109,39 @@ static int mali_dma_buf_map(struct mali_dma_buf_attachment *mem, struct mali_ses } mem->is_mapped = MALI_TRUE; - mutex_unlock(&mem->map_lock); - + mali_session_memory_unlock(session); /* Wake up any thread waiting for buffer to become mapped */ wake_up_all(&mem->wait_queue); } else { MALI_DEBUG_ASSERT(mem->is_mapped); - mutex_unlock(&mem->map_lock); + mali_session_memory_unlock(session); } return 0; } -static void mali_dma_buf_unmap(struct mali_dma_buf_attachment *mem) +static void mali_dma_buf_unmap(mali_mem_allocation *alloc, struct mali_dma_buf_attachment *mem) { + MALI_DEBUG_ASSERT_POINTER(alloc); MALI_DEBUG_ASSERT_POINTER(mem); MALI_DEBUG_ASSERT_POINTER(mem->attachment); MALI_DEBUG_ASSERT_POINTER(mem->buf); + MALI_DEBUG_ASSERT_POINTER(alloc->session); - mutex_lock(&mem->map_lock); - + mali_session_memory_lock(alloc->session); mem->map_ref--; MALI_DEBUG_PRINT(5, ("Mali DMA-buf: unmap attachment %p, new map_ref = %d\n", mem, mem->map_ref)); if (0 == mem->map_ref) { dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL); - + if (MALI_TRUE == mem->is_mapped) { + mali_mem_mali_map_free(alloc->session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + } mem->is_mapped = MALI_FALSE; } - - mutex_unlock(&mem->map_lock); - + mali_session_memory_unlock(alloc->session); /* Wake up any thread waiting for buffer to become unmapped */ wake_up_all(&mem->wait_queue); } @@ -165,13 +149,15 @@ static void mali_dma_buf_unmap(struct mali_dma_buf_attachment *mem) #if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) int mali_dma_buf_map_job(struct mali_pp_job *job) { - mali_mem_allocation *descriptor; struct mali_dma_buf_attachment *mem; _mali_osk_errcode_t err; int i; int ret = 0; u32 num_memory_cookies; struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; MALI_DEBUG_ASSERT_POINTER(job); @@ -181,104 +167,92 @@ int mali_dma_buf_map_job(struct mali_pp_job *job) MALI_DEBUG_ASSERT_POINTER(session); - mali_session_memory_lock(session); - for (i = 0; i < num_memory_cookies; i++) { - u32 cookie = mali_pp_job_get_memory_cookie(job, i); - - if (0 == cookie) { - /* 0 is not a valid cookie */ - MALI_DEBUG_ASSERT(NULL == - mali_pp_job_get_dma_buf(job, i)); - continue; - } - - MALI_DEBUG_ASSERT(0 < cookie); - - err = mali_descriptor_mapping_get( - mali_pp_job_get_session(job)->descriptor_mapping, - cookie, (void **)&descriptor); - - if (_MALI_OSK_ERR_OK != err) { - MALI_DEBUG_PRINT_ERROR(("Mali DMA-buf: Failed to get descriptor for cookie %d\n", cookie)); - ret = -EFAULT; - MALI_DEBUG_ASSERT(NULL == - mali_pp_job_get_dma_buf(job, i)); + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + if (MALI_MEM_DMA_BUF != mali_alloc->type) { continue; } - if (MALI_MEM_DMA_BUF != descriptor->type) { - /* Not a DMA-buf */ - MALI_DEBUG_ASSERT(NULL == - mali_pp_job_get_dma_buf(job, i)); - continue; - } + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); - mem = descriptor->dma_buf.attachment; + mem = mem_bkend->dma_buf.attachment; MALI_DEBUG_ASSERT_POINTER(mem); MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); - err = mali_dma_buf_map(mem, mem->session, descriptor->mali_mapping.addr, descriptor->flags); + err = mali_dma_buf_map(mem_bkend); if (0 != err) { - MALI_DEBUG_PRINT_ERROR(("Mali DMA-buf: Failed to map dma-buf for cookie %d at mali address %x\b", - cookie, descriptor->mali_mapping.addr)); + MALI_DEBUG_PRINT_ERROR(("Mali DMA-buf: Failed to map dma-buf for mali address %x\n", mali_addr)); ret = -EFAULT; - MALI_DEBUG_ASSERT(NULL == - mali_pp_job_get_dma_buf(job, i)); continue; } - - /* Add mem to list of DMA-bufs mapped for this job */ - mali_pp_job_set_dma_buf(job, i, mem); } - - mali_session_memory_unlock(session); - return ret; } void mali_dma_buf_unmap_job(struct mali_pp_job *job) { - u32 i; - u32 num_dma_bufs = mali_pp_job_num_dma_bufs(job); + struct mali_dma_buf_attachment *mem; + int i; + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + + session = mali_pp_job_get_session(job); - for (i = 0; i < num_dma_bufs; i++) { - struct mali_dma_buf_attachment *mem; + MALI_DEBUG_ASSERT_POINTER(session); - mem = mali_pp_job_get_dma_buf(job, i); - if (NULL != mem) { - mali_dma_buf_unmap(mem); - mali_pp_job_set_dma_buf(job, i, NULL); + for (i = 0; i < num_memory_cookies; i++) { + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + if (MALI_MEM_DMA_BUF != mali_alloc->type) { + continue; } + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + mem = mem_bkend->dma_buf.attachment; + + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); + mali_dma_buf_unmap(mem_bkend->mali_allocation, mem); } } #endif /* !CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH */ -int mali_attach_dma_buf(struct mali_session_data *session, _mali_uk_attach_dma_buf_s __user *user_arg) +int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *user_arg) { - struct dma_buf *buf; - struct mali_dma_buf_attachment *mem; - _mali_uk_attach_dma_buf_s args; - mali_mem_allocation *descriptor; - int md; + _mali_uk_dma_buf_get_size_s args; int fd; + struct dma_buf *buf; - /* Get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ - if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_attach_dma_buf_s))) { + /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ + if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_dma_buf_get_size_s))) { return -EFAULT; } - if (args.mali_address & ~PAGE_MASK) { - MALI_DEBUG_PRINT_ERROR(("Requested address (0x%08x) is not page aligned\n", args.mali_address)); - return -EINVAL; - } - - if (args.mali_address >= args.mali_address + args.size) { - MALI_DEBUG_PRINT_ERROR(("Requested address and size (0x%08x + 0x%08x) is too big\n", args.mali_address, args.size)); - return -EINVAL; - } - + /* Do DMA-BUF stuff */ fd = args.mem_fd; buf = dma_buf_get(fd); @@ -287,169 +261,109 @@ int mali_attach_dma_buf(struct mali_session_data *session, _mali_uk_attach_dma_b return PTR_RET(buf); } - /* Currently, mapping of the full buffer are supported. */ - if (args.size != buf->size) { - MALI_DEBUG_PRINT_ERROR(("dma-buf size doesn't match mapping size.\n")); - dma_buf_put(buf); - return -EINVAL; - } - - mem = _mali_osk_calloc(1, sizeof(struct mali_dma_buf_attachment)); - if (NULL == mem) { - MALI_DEBUG_PRINT_ERROR(("Failed to allocate dma-buf tracing struct\n")); + if (0 != put_user(buf->size, &user_arg->size)) { dma_buf_put(buf); - return -ENOMEM; - } - - mem->buf = buf; - mem->session = session; - mem->map_ref = 0; - mutex_init(&mem->map_lock); - init_waitqueue_head(&mem->wait_queue); - - mem->attachment = dma_buf_attach(mem->buf, &mali_platform_device->dev); - if (NULL == mem->attachment) { - MALI_DEBUG_PRINT_ERROR(("Failed to attach to dma-buf %d\n", fd)); - dma_buf_put(mem->buf); - _mali_osk_free(mem); return -EFAULT; } - /* Set up Mali memory descriptor */ - descriptor = mali_mem_descriptor_create(session, MALI_MEM_DMA_BUF); - if (NULL == descriptor) { - MALI_DEBUG_PRINT_ERROR(("Failed to allocate descriptor dma-buf %d\n", fd)); - mali_dma_buf_release(mem); - return -ENOMEM; - } - - descriptor->size = args.size; - descriptor->mali_mapping.addr = args.mali_address; + dma_buf_put(buf); - descriptor->dma_buf.attachment = mem; + return 0; +} - descriptor->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; - if (args.flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { - descriptor->flags = MALI_MEM_FLAG_MALI_GUARD_PAGE; - } +_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + int fd, u32 flags) +{ + struct dma_buf *buf; + struct mali_dma_buf_attachment *dma_mem; + struct mali_session_data *session = alloc->session; - mali_session_memory_lock(session); + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT_POINTER(alloc); - /* Map dma-buf into this session's page tables */ - if (_MALI_OSK_ERR_OK != mali_mem_mali_map_prepare(descriptor)) { - mali_session_memory_unlock(session); - MALI_DEBUG_PRINT_ERROR(("Failed to map dma-buf on Mali\n")); - mali_mem_descriptor_destroy(descriptor); - mali_dma_buf_release(mem); - return -ENOMEM; + /* get dma buffer */ + buf = dma_buf_get(fd); + if (IS_ERR_OR_NULL(buf)) { + return _MALI_OSK_ERR_FAULT; } -#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) - /* Map memory into session's Mali virtual address space. */ - - if (0 != mali_dma_buf_map(mem, session, descriptor->mali_mapping.addr, descriptor->flags)) { - mali_mem_mali_map_free(descriptor); - mali_session_memory_unlock(session); - - MALI_DEBUG_PRINT_ERROR(("Failed to map dma-buf %d into Mali address space\n", fd)); - mali_mem_descriptor_destroy(descriptor); - mali_dma_buf_release(mem); - return -ENOMEM; + /* Currently, mapping of the full buffer are supported. */ + if (alloc->psize != buf->size) { + goto failed_alloc_mem; } -#endif - - mali_session_memory_unlock(session); - - /* Get descriptor mapping for memory. */ - if (_MALI_OSK_ERR_OK != mali_descriptor_mapping_allocate_mapping(session->descriptor_mapping, descriptor, &md)) { - mali_session_memory_lock(session); - mali_mem_mali_map_free(descriptor); - mali_session_memory_unlock(session); - - MALI_DEBUG_PRINT_ERROR(("Failed to create descriptor mapping for dma-buf %d\n", fd)); - mali_mem_descriptor_destroy(descriptor); - mali_dma_buf_release(mem); - return -EFAULT; + dma_mem = _mali_osk_calloc(1, sizeof(struct mali_dma_buf_attachment)); + if (NULL == dma_mem) { + goto failed_alloc_mem; } - /* Return stuff to user space */ - if (0 != put_user(md, &user_arg->cookie)) { - mali_session_memory_lock(session); - mali_mem_mali_map_free(descriptor); - mali_session_memory_unlock(session); + dma_mem->buf = buf; + dma_mem->session = session; + dma_mem->map_ref = 0; + init_waitqueue_head(&dma_mem->wait_queue); - MALI_DEBUG_PRINT_ERROR(("Failed to return descriptor to user space for dma-buf %d\n", fd)); - mali_descriptor_mapping_free(session->descriptor_mapping, md); - mali_dma_buf_release(mem); - return -EFAULT; + dma_mem->attachment = dma_buf_attach(dma_mem->buf, &mali_platform_device->dev); + if (NULL == dma_mem->attachment) { + goto failed_dma_attach; } - return 0; -} - -int mali_release_dma_buf(struct mali_session_data *session, _mali_uk_release_dma_buf_s __user *user_arg) -{ - int ret = 0; - _mali_uk_release_dma_buf_s args; - mali_mem_allocation *descriptor; + mem_backend->dma_buf.attachment = dma_mem; - /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ - if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_release_dma_buf_s))) { - return -EFAULT; + alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; + if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; } - MALI_DEBUG_PRINT(3, ("Mali DMA-buf: release descriptor cookie %ld\n", args.cookie)); - - mali_session_memory_lock(session); - - descriptor = mali_descriptor_mapping_free(session->descriptor_mapping, (u32)args.cookie); - - if (NULL != descriptor) { - MALI_DEBUG_PRINT(3, ("Mali DMA-buf: Releasing dma-buf at mali address %x\n", descriptor->mali_mapping.addr)); - mali_mem_mali_map_free(descriptor); - - mali_dma_buf_release(descriptor->dma_buf.attachment); - - mali_mem_descriptor_destroy(descriptor); - } else { - MALI_DEBUG_PRINT_ERROR(("Invalid memory descriptor %ld used to release dma-buf\n", args.cookie)); - ret = -EINVAL; +#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) + /* Map memory into session's Mali virtual address space. */ + if (0 != mali_dma_buf_map(mem_backend)) { + goto Failed_dma_map; } +#endif - mali_session_memory_unlock(session); + return _MALI_OSK_ERR_OK; - /* Return the error that _mali_ukk_map_external_ump_mem produced */ - return ret; +#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +Failed_dma_map: + mali_dma_buf_unmap(alloc, dma_mem); +#endif + /* Wait for buffer to become unmapped */ + wait_event(dma_mem->wait_queue, !dma_mem->is_mapped); + MALI_DEBUG_ASSERT(!dma_mem->is_mapped); + dma_buf_detach(dma_mem->buf, dma_mem->attachment); +failed_dma_attach: + _mali_osk_free(dma_mem); +failed_alloc_mem: + dma_buf_put(buf); + return _MALI_OSK_ERR_FAULT; } -int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *user_arg) +void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend) { - _mali_uk_dma_buf_get_size_s args; - int fd; - struct dma_buf *buf; - - /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ - if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_dma_buf_get_size_s))) { - return -EFAULT; - } - - /* Do DMA-BUF stuff */ - fd = args.mem_fd; + struct mali_dma_buf_attachment *mem; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_DMA_BUF == mem_backend->type); - buf = dma_buf_get(fd); - if (IS_ERR_OR_NULL(buf)) { - MALI_DEBUG_PRINT_ERROR(("Failed to get dma-buf from fd: %d\n", fd)); - return PTR_RET(buf); - } + mem = mem_backend->dma_buf.attachment; + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT_POINTER(mem->attachment); + MALI_DEBUG_ASSERT_POINTER(mem->buf); + MALI_DEBUG_PRINT(3, ("Mali DMA-buf: release attachment %p\n", mem)); - if (0 != put_user(buf->size, &user_arg->size)) { - dma_buf_put(buf); - return -EFAULT; - } +#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) + MALI_DEBUG_ASSERT_POINTER(mem_backend->mali_allocation); + /* We mapped implicitly on attach, so we need to unmap on release */ + mali_dma_buf_unmap(mem_backend->mali_allocation, mem); +#endif + /* Wait for buffer to become unmapped */ + wait_event(mem->wait_queue, !mem->is_mapped); + MALI_DEBUG_ASSERT(!mem->is_mapped); - dma_buf_put(buf); + dma_buf_detach(mem->buf, mem->attachment); + dma_buf_put(mem->buf); - return 0; + _mali_osk_free(mem); } diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h index 4465985ac06e..859d3849e6b3 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -22,12 +22,24 @@ extern "C" { struct mali_pp_job; struct mali_dma_buf_attachment; +struct mali_dma_buf_attachment { + struct dma_buf *buf; + struct dma_buf_attachment *attachment; + struct sg_table *sgt; + struct mali_session_data *session; + int map_ref; + struct mutex map_lock; + mali_bool is_mapped; + wait_queue_head_t wait_queue; +}; -int mali_attach_dma_buf(struct mali_session_data *session, _mali_uk_attach_dma_buf_s __user *arg); -int mali_release_dma_buf(struct mali_session_data *session, _mali_uk_release_dma_buf_s __user *arg); int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *arg); -void mali_mem_dma_buf_release(mali_mem_allocation *descriptor); +void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend); + +_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + int fd, u32 flags); #if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) int mali_dma_buf_map_job(struct mali_pp_job *job); diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c index 3b67d6c2344e..dcc9a7590885 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -12,114 +12,78 @@ #include "mali_osk.h" #include "mali_ukk.h" #include "mali_memory.h" -#include "mali_kernel_descriptor_mapping.h" #include "mali_mem_validation.h" #include "mali_uk_types.h" -void mali_mem_external_release(mali_mem_allocation *descriptor) +void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend) { - MALI_DEBUG_ASSERT(MALI_MEM_EXTERNAL == descriptor->type); - - mali_mem_mali_map_free(descriptor); + mali_mem_allocation *alloc; + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT(MALI_MEM_EXTERNAL == mem_backend->type); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); } -_mali_osk_errcode_t _mali_ukk_map_external_mem(_mali_uk_map_external_mem_s *args) +_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + u32 phys_addr, + u32 flag) { struct mali_session_data *session; - mali_mem_allocation *descriptor; - int md; _mali_osk_errcode_t err; - - MALI_DEBUG_ASSERT_POINTER(args); - - session = (struct mali_session_data *)(uintptr_t)args->ctx; + u32 virt, phys, size; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT_POINTER(alloc); + size = alloc->psize; + session = (struct mali_session_data *)(uintptr_t)alloc->session; MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); /* check arguments */ /* NULL might be a valid Mali address */ - if (! args->size) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (!size) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); /* size must be a multiple of the system page size */ - if (args->size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); - - MALI_DEBUG_PRINT(3, - ("Requested to map physical memory 0x%x-0x%x into virtual memory 0x%x\n", - args->phys_addr, (args->phys_addr + args->size - 1), - args->mali_address)); + if (size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); /* Validate the mali physical range */ - if (_MALI_OSK_ERR_OK != mali_mem_validation_check(args->phys_addr, args->size)) { + if (_MALI_OSK_ERR_OK != mali_mem_validation_check(phys_addr, size)) { return _MALI_OSK_ERR_FAULT; } - descriptor = mali_mem_descriptor_create(session, MALI_MEM_EXTERNAL); - if (NULL == descriptor) MALI_ERROR(_MALI_OSK_ERR_NOMEM); - - descriptor->mali_mapping.addr = args->mali_address; - descriptor->size = args->size; - - if (args->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { - descriptor->flags = MALI_MEM_FLAG_MALI_GUARD_PAGE; + if (flag & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; } - _mali_osk_mutex_wait(session->memory_lock); - { - u32 virt = descriptor->mali_mapping.addr; - u32 phys = args->phys_addr; - u32 size = args->size; + mali_session_memory_lock(session); - err = mali_mem_mali_map_prepare(descriptor); - if (_MALI_OSK_ERR_OK != err) { - _mali_osk_mutex_signal(session->memory_lock); - mali_mem_descriptor_destroy(descriptor); - return _MALI_OSK_ERR_NOMEM; - } + virt = alloc->mali_vma_node.vm_node.start; + phys = phys_addr; - mali_mmu_pagedir_update(session->page_directory, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); - - if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { - mali_mmu_pagedir_update(session->page_directory, virt + size, phys, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); - } + err = mali_mem_mali_map_prepare(alloc); + if (_MALI_OSK_ERR_OK != err) { + mali_session_memory_unlock(session); + return _MALI_OSK_ERR_NOMEM; } - _mali_osk_mutex_signal(session->memory_lock); - if (_MALI_OSK_ERR_OK != mali_descriptor_mapping_allocate_mapping(session->descriptor_mapping, descriptor, &md)) { - _mali_osk_mutex_wait(session->memory_lock); - mali_mem_external_release(descriptor); - _mali_osk_mutex_signal(session->memory_lock); - mali_mem_descriptor_destroy(descriptor); - MALI_ERROR(_MALI_OSK_ERR_FAULT); - } + mali_mmu_pagedir_update(session->page_directory, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); - args->cookie = md; - - MALI_SUCCESS; -} - -_mali_osk_errcode_t _mali_ukk_unmap_external_mem(_mali_uk_unmap_external_mem_s *args) -{ - mali_mem_allocation *descriptor; - void *old_value; - struct mali_session_data *session; - - MALI_DEBUG_ASSERT_POINTER(args); - - session = (struct mali_session_data *)(uintptr_t)args->ctx; - MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); - - if (_MALI_OSK_ERR_OK != mali_descriptor_mapping_get(session->descriptor_mapping, args->cookie, (void **)&descriptor)) { - MALI_DEBUG_PRINT(1, ("Invalid memory descriptor %d used to unmap external memory\n", args->cookie)); - MALI_ERROR(_MALI_OSK_ERR_FAULT); - } - - old_value = mali_descriptor_mapping_free(session->descriptor_mapping, args->cookie); - - if (NULL != old_value) { - _mali_osk_mutex_wait(session->memory_lock); - mali_mem_external_release(descriptor); - _mali_osk_mutex_signal(session->memory_lock); - mali_mem_descriptor_destroy(descriptor); + if (alloc->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + mali_mmu_pagedir_update(session->page_directory, virt + size, phys, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); } + MALI_DEBUG_PRINT(3, + ("Requested to map physical memory 0x%x-0x%x into virtual memory 0x%x\n", + phys_addr, (phys_addr + size - 1), + virt)); + mali_session_memory_unlock(session); MALI_SUCCESS; } + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h new file mode 100644 index 000000000000..645580b51fc9 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h @@ -0,0 +1,29 @@ + +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_EXTERNAL_H__ +#define __MALI_MEMORY_EXTERNAL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + u32 phys_addr, + u32 flag); +void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c new file mode 100644 index 000000000000..55e2c092d597 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c @@ -0,0 +1,965 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include +#endif +#include + +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_linux.h" +#include "mali_scheduler.h" +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include "mali_memory_dma_buf.h" +#endif +#if defined(CONFIG_MALI400_UMP) +#include "mali_memory_ump.h" +#endif +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" +#include "mali_memory_util.h" +#include "mali_memory_external.h" +#include "mali_memory_cow.h" +#include "mali_memory_block_alloc.h" +#include "mali_ukk.h" +#include "mali_memory_swap_alloc.h" + +/* +* New memory system interface +*/ + +/*inti idr for backend memory */ +struct idr mali_backend_idr; +struct mutex mali_idr_mutex; + +/* init allocation manager */ +int mali_memory_manager_init(struct mali_allocation_manager *mgr) +{ + /* init Locks */ + rwlock_init(&mgr->vm_lock); + mutex_init(&mgr->list_mutex); + + /* init link */ + INIT_LIST_HEAD(&mgr->head); + + /* init RB tree */ + mgr->allocation_mgr_rb = RB_ROOT; + mgr->mali_allocation_num = 0; + return 0; +} + +/* Deinit allocation manager +* Do some check for debug +*/ +void mali_memory_manager_uninit(struct mali_allocation_manager *mgr) +{ + /* check RB tree is empty */ + MALI_DEBUG_ASSERT(((void *)(mgr->allocation_mgr_rb.rb_node) == (void *)rb_last(&mgr->allocation_mgr_rb))); + /* check allocation List */ + MALI_DEBUG_ASSERT(list_empty(&mgr->head)); +} + +/* Prepare memory descriptor */ +static mali_mem_allocation *mali_mem_allocation_struct_create(struct mali_session_data *session) +{ + mali_mem_allocation *mali_allocation; + + /* Allocate memory */ + mali_allocation = (mali_mem_allocation *)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL); + if (NULL == mali_allocation) { + MALI_DEBUG_PRINT(1, ("mali_mem_allocation_struct_create: descriptor was NULL\n")); + return NULL; + } + + MALI_DEBUG_CODE(mali_allocation->magic = MALI_MEM_ALLOCATION_VALID_MAGIC); + + /* do init */ + mali_allocation->flags = 0; + mali_allocation->session = session; + + INIT_LIST_HEAD(&mali_allocation->list); + _mali_osk_atomic_init(&mali_allocation->mem_alloc_refcount, 1); + + /** + *add to session list + */ + mutex_lock(&session->allocation_mgr.list_mutex); + list_add_tail(&mali_allocation->list, &session->allocation_mgr.head); + session->allocation_mgr.mali_allocation_num++; + mutex_unlock(&session->allocation_mgr.list_mutex); + + return mali_allocation; +} + +void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc) +{ + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT_POINTER(alloc->session); + mutex_lock(&alloc->session->allocation_mgr.list_mutex); + list_del(&alloc->list); + alloc->session->allocation_mgr.mali_allocation_num--; + mutex_unlock(&alloc->session->allocation_mgr.list_mutex); + + kfree(alloc); +} + +int mali_mem_backend_struct_create(mali_mem_backend **backend, u32 psize) +{ + mali_mem_backend *mem_backend = NULL; + s32 ret = -ENOSPC; + s32 index = -1; + *backend = (mali_mem_backend *)kzalloc(sizeof(mali_mem_backend), GFP_KERNEL); + if (NULL == *backend) { + MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: backend descriptor was NULL\n")); + return -1; + } + mem_backend = *backend; + mem_backend->size = psize; + mutex_init(&mem_backend->mutex); + INIT_LIST_HEAD(&mem_backend->list); + mem_backend->using_count = 0; + + + /* link backend with id */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0) +again: + if (!idr_pre_get(&mali_backend_idr, GFP_KERNEL)) { + kfree(mem_backend); + return -ENOMEM; + } + mutex_lock(&mali_idr_mutex); + ret = idr_get_new_above(&mali_backend_idr, mem_backend, 1, &index); + mutex_unlock(&mali_idr_mutex); + + if (-ENOSPC == ret) { + kfree(mem_backend); + return -ENOSPC; + } + if (-EAGAIN == ret) + goto again; +#else + mutex_lock(&mali_idr_mutex); + ret = idr_alloc(&mali_backend_idr, mem_backend, 1, MALI_S32_MAX, GFP_KERNEL); + mutex_unlock(&mali_idr_mutex); + index = ret; + if (ret < 0) { + MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: Can't allocate idr for backend! \n")); + kfree(mem_backend); + return -ENOSPC; + } +#endif + return index; +} + + +static void mali_mem_backend_struct_destory(mali_mem_backend **backend, s32 backend_handle) +{ + mali_mem_backend *mem_backend = *backend; + + mutex_lock(&mali_idr_mutex); + idr_remove(&mali_backend_idr, backend_handle); + mutex_unlock(&mali_idr_mutex); + kfree(mem_backend); + *backend = NULL; +} + +mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address) +{ + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_backend *mem_bkend = NULL; + mali_mem_allocation *mali_alloc = NULL; + MALI_DEBUG_ASSERT_POINTER(session); + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_address, 0); + if (NULL == mali_vma_node) { + MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_search:vma node was NULL\n")); + return NULL; + } + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + return mem_bkend; +} + +static _mali_osk_errcode_t mali_mem_resize(struct mali_session_data *session, mali_mem_backend *mem_backend, u32 physical_size) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + int retval = 0; + mali_mem_allocation *mali_allocation = NULL; + mali_mem_os_mem tmp_os_mem; + s32 change_page_count; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); + MALI_DEBUG_ASSERT(0 == physical_size % MALI_MMU_PAGE_SIZE); + + mali_allocation = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(mali_allocation); + + MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE & mali_allocation->flags); + MALI_DEBUG_ASSERT(MALI_MEM_OS == mali_allocation->type); + + mutex_lock(&mem_backend->mutex); + + /* Do resize*/ + if (physical_size > mem_backend->size) { + u32 add_size = physical_size - mem_backend->size; + + MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); + + /* Allocate new pages from os mem */ + retval = mali_mem_os_alloc_pages(&tmp_os_mem, add_size); + + if (retval) { + if (-ENOMEM == retval) { + ret = _MALI_OSK_ERR_NOMEM; + } else { + ret = _MALI_OSK_ERR_FAULT; + } + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory allocation failed !\n")); + goto failed_alloc_memory; + } + + MALI_DEBUG_ASSERT(tmp_os_mem.count == add_size / MALI_MMU_PAGE_SIZE); + + /* Resize the memory of the backend */ + ret = mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); + + if (ret) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory resizing failed !\n")); + goto failed_resize_pages; + } + + /*Resize cpu mapping */ + if (NULL != mali_allocation->cpu_mapping.vma) { + ret = mali_mem_os_resize_cpu_map_locked(mem_backend, mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + mem_backend->size, add_size); + if (unlikely(ret != _MALI_OSK_ERR_OK)) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: cpu mapping failed !\n")); + goto failed_cpu_map; + } + } + + /* Resize mali mapping */ + _mali_osk_mutex_wait(session->memory_lock); + ret = mali_mem_mali_map_resize(mali_allocation, physical_size); + + if (ret) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_resize: mali map resize fail !\n")); + goto failed_gpu_map; + } + + ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, mali_allocation->mali_vma_node.vm_node.start, + mali_allocation->psize / MALI_MMU_PAGE_SIZE, add_size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); + if (ret) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: mali mapping failed !\n")); + goto failed_gpu_map; + } + + _mali_osk_mutex_signal(session->memory_lock); + } else { + u32 dec_size, page_count; + u32 vaddr = 0; + INIT_LIST_HEAD(&tmp_os_mem.pages); + tmp_os_mem.count = 0; + + dec_size = mem_backend->size - physical_size; + MALI_DEBUG_ASSERT(0 == dec_size % MALI_MMU_PAGE_SIZE); + + page_count = dec_size / MALI_MMU_PAGE_SIZE; + vaddr = mali_allocation->mali_vma_node.vm_node.start + physical_size; + + /* Resize the memory of the backend */ + ret = mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, physical_size / MALI_MMU_PAGE_SIZE, page_count); + + if (ret) { + MALI_DEBUG_PRINT(4, ("_mali_ukk_mem_resize: mali map resize failed!\n")); + goto failed_resize_pages; + } + + /* Resize mali map */ + _mali_osk_mutex_wait(session->memory_lock); + mali_mem_mali_map_free(session, dec_size, vaddr, mali_allocation->flags); + _mali_osk_mutex_signal(session->memory_lock); + + /* Zap cpu mapping */ + if (0 != mali_allocation->cpu_mapping.addr) { + MALI_DEBUG_ASSERT(NULL != mali_allocation->cpu_mapping.vma); + zap_vma_ptes(mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + physical_size, dec_size); + } + + /* Free those extra pages */ + mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); + } + + /* Resize memory allocation and memory backend */ + change_page_count = (s32)(physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE; + mali_allocation->psize = physical_size; + mem_backend->size = physical_size; + mutex_unlock(&mem_backend->mutex); + + if (change_page_count > 0) { + atomic_add(change_page_count, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + + } else { + atomic_sub((s32)(-change_page_count), &session->mali_mem_allocated_pages); + } + + return _MALI_OSK_ERR_OK; + +failed_gpu_map: + _mali_osk_mutex_signal(session->memory_lock); +failed_cpu_map: + if (physical_size > mem_backend->size) { + mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, mem_backend->size / MALI_MMU_PAGE_SIZE, + (physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE); + } else { + mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); + } +failed_resize_pages: + if (0 != tmp_os_mem.count) + mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); +failed_alloc_memory: + + mutex_unlock(&mem_backend->mutex); + return ret; +} + + +/* Set GPU MMU properties */ +static void _mali_memory_gpu_map_property_set(u32 *properties, u32 flags) +{ + if (_MALI_MEMORY_GPU_READ_ALLOCATE & flags) { + *properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE; + } else { + *properties = MALI_MMU_FLAGS_DEFAULT; + } +} + +_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size) +{ + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_allocation *mali_allocation = NULL; + u32 new_physical_size; + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); + + /* Get the memory backend that need to be resize. */ + mem_backend = mali_mem_backend_struct_search(session, mali_addr); + + if (NULL == mem_backend) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); + return ret; + } + + mali_allocation = mem_backend->mali_allocation; + + MALI_DEBUG_ASSERT_POINTER(mali_allocation); + + new_physical_size = add_size + mem_backend->size; + + if (new_physical_size > (mali_allocation->mali_vma_node.vm_node.size)) + return ret; + + MALI_DEBUG_ASSERT(new_physical_size != mem_backend->size); + + ret = mali_mem_resize(session, mem_backend, new_physical_size); + + return ret; +} + +/** +* function@_mali_ukk_mem_allocate - allocate mali memory +*/ +_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + int retval = 0; + mali_mem_allocation *mali_allocation = NULL; + struct mali_vma_node *mali_vma_node = NULL; + + MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_allocate, vaddr=0x%x, size =0x%x! \n", args->gpu_vaddr, args->psize)); + + /* Check if the address is allocated + */ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->gpu_vaddr, 0); + + if (unlikely(mali_vma_node)) { + MALI_DEBUG_ASSERT(0); + return _MALI_OSK_ERR_FAULT; + } + /** + *create mali memory allocation + */ + + mali_allocation = mali_mem_allocation_struct_create(session); + + if (mali_allocation == NULL) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_allocate: Failed to create allocation struct! \n")); + return _MALI_OSK_ERR_NOMEM; + } + mali_allocation->psize = args->psize; + mali_allocation->vsize = args->vsize; + + /* MALI_MEM_OS if need to support mem resize, + * or MALI_MEM_BLOCK if have dedicated memory, + * or MALI_MEM_OS, + * or MALI_MEM_SWAP. + */ + if (args->flags & _MALI_MEMORY_ALLOCATE_SWAPPABLE) { + mali_allocation->type = MALI_MEM_SWAP; + } else if (args->flags & _MALI_MEMORY_ALLOCATE_RESIZEABLE) { + mali_allocation->type = MALI_MEM_OS; + mali_allocation->flags |= MALI_MEM_FLAG_CAN_RESIZE; + } else if (MALI_TRUE == mali_memory_have_dedicated_memory()) { + mali_allocation->type = MALI_MEM_BLOCK; + } else { + mali_allocation->type = MALI_MEM_OS; + } + + /** + *add allocation node to RB tree for index + */ + mali_allocation->mali_vma_node.vm_node.start = args->gpu_vaddr; + mali_allocation->mali_vma_node.vm_node.size = args->vsize; + + mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); + + mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, args->psize); + if (mali_allocation->backend_handle < 0) { + ret = _MALI_OSK_ERR_NOMEM; + MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); + goto failed_alloc_backend; + } + + + mem_backend->mali_allocation = mali_allocation; + mem_backend->type = mali_allocation->type; + + mali_allocation->mali_mapping.addr = args->gpu_vaddr; + + /* set gpu mmu propery */ + _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); + /* do prepare for MALI mapping */ + if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { + _mali_osk_mutex_wait(session->memory_lock); + + ret = mali_mem_mali_map_prepare(mali_allocation); + if (0 != ret) { + _mali_osk_mutex_signal(session->memory_lock); + goto failed_prepare_map; + } + _mali_osk_mutex_signal(session->memory_lock); + } + + if (mali_allocation->psize == 0) { + mem_backend->os_mem.count = 0; + INIT_LIST_HEAD(&mem_backend->os_mem.pages); + goto done; + } + + if (args->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { + mali_allocation->flags |= _MALI_MEMORY_ALLOCATE_DEFER_BIND; + mem_backend->flags |= MALI_MEM_BACKEND_FLAG_NOT_BINDED; + /* init for defer bind backend*/ + mem_backend->os_mem.count = 0; + INIT_LIST_HEAD(&mem_backend->os_mem.pages); + + goto done; + } + /** + *allocate physical memory + */ + if (likely(mali_allocation->psize > 0)) { + + if (mem_backend->type == MALI_MEM_OS) { + retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); + } else if (mem_backend->type == MALI_MEM_BLOCK) { + /* try to allocated from BLOCK memory first, then try OS memory if failed.*/ + if (mali_mem_block_alloc(&mem_backend->block_mem, mem_backend->size)) { + retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); + mem_backend->type = MALI_MEM_OS; + mali_allocation->type = MALI_MEM_OS; + } + } else if (MALI_MEM_SWAP == mem_backend->type) { + retval = mali_mem_swap_alloc_pages(&mem_backend->swap_mem, mali_allocation->mali_vma_node.vm_node.size, &mem_backend->start_idx); + } else { + /* ONLY support mem_os type */ + MALI_DEBUG_ASSERT(0); + } + + if (retval) { + ret = _MALI_OSK_ERR_NOMEM; + MALI_DEBUG_PRINT(1, (" can't allocate enough pages! \n")); + goto failed_alloc_pages; + } + } + + /** + *map to GPU side + */ + if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { + _mali_osk_mutex_wait(session->memory_lock); + /* Map on Mali */ + + if (mem_backend->type == MALI_MEM_OS) { + ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, args->gpu_vaddr, 0, + mem_backend->size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); + + } else if (mem_backend->type == MALI_MEM_BLOCK) { + mali_mem_block_mali_map(&mem_backend->block_mem, session, args->gpu_vaddr, + mali_allocation->mali_mapping.properties); + } else if (mem_backend->type == MALI_MEM_SWAP) { + ret = mali_mem_swap_mali_map(&mem_backend->swap_mem, session, args->gpu_vaddr, + mali_allocation->mali_mapping.properties); + } else { /* unsupport type */ + MALI_DEBUG_ASSERT(0); + } + + _mali_osk_mutex_signal(session->memory_lock); + } +done: + if (MALI_MEM_OS == mem_backend->type) { + atomic_add(mem_backend->os_mem.count, &session->mali_mem_allocated_pages); + } else if (MALI_MEM_BLOCK == mem_backend->type) { + atomic_add(mem_backend->block_mem.count, &session->mali_mem_allocated_pages); + } else { + MALI_DEBUG_ASSERT(MALI_MEM_SWAP == mem_backend->type); + atomic_add(mem_backend->swap_mem.count, &session->mali_mem_allocated_pages); + atomic_add(mem_backend->swap_mem.count, &session->mali_mem_array[mem_backend->type]); + } + + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + return _MALI_OSK_ERR_OK; + +failed_alloc_pages: + mali_mem_mali_map_free(session, mali_allocation->psize, mali_allocation->mali_vma_node.vm_node.start, mali_allocation->flags); +failed_prepare_map: + mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); +failed_alloc_backend: + + mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); + mali_mem_allocation_struct_destory(mali_allocation); + + return ret; +} + + +_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + u32 vaddr = args->gpu_vaddr; + mali_mem_allocation *mali_alloc = NULL; + struct mali_vma_node *mali_vma_node = NULL; + + /* find mali allocation structure by vaddress*/ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, vaddr, 0); + if (NULL == mali_vma_node) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_free: invalid addr: 0x%x\n", vaddr)); + return _MALI_OSK_ERR_INVALID_ARGS; + } + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + + if (mali_alloc) + /* check ref_count */ + args->free_pages_nr = mali_allocation_unref(&mali_alloc); + + return _MALI_OSK_ERR_OK; +} + + +/** +* Function _mali_ukk_mem_bind -- bind a external memory to a new GPU address +* It will allocate a new mem allocation and bind external memory to it. +* Supported backend type are: +* _MALI_MEMORY_BIND_BACKEND_UMP +* _MALI_MEMORY_BIND_BACKEND_DMA_BUF +* _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY +* CPU access is not supported yet +*/ +_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_allocation *mali_allocation = NULL; + MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_bind, vaddr=0x%x, size =0x%x! \n", args->vaddr, args->size)); + + /** + * allocate mali allocation. + */ + mali_allocation = mali_mem_allocation_struct_create(session); + + if (mali_allocation == NULL) { + return _MALI_OSK_ERR_NOMEM; + } + mali_allocation->psize = args->size; + mali_allocation->vsize = args->size; + mali_allocation->mali_mapping.addr = args->vaddr; + + /* add allocation node to RB tree for index */ + mali_allocation->mali_vma_node.vm_node.start = args->vaddr; + mali_allocation->mali_vma_node.vm_node.size = args->size; + mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); + + /* allocate backend*/ + if (mali_allocation->psize > 0) { + mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); + if (mali_allocation->backend_handle < 0) { + goto Failed_alloc_backend; + } + + } else { + goto Failed_alloc_backend; + } + + mem_backend->size = mali_allocation->psize; + mem_backend->mali_allocation = mali_allocation; + + switch (args->flags & _MALI_MEMORY_BIND_BACKEND_MASK) { + case _MALI_MEMORY_BIND_BACKEND_UMP: +#if defined(CONFIG_MALI400_UMP) + mali_allocation->type = MALI_MEM_UMP; + mem_backend->type = MALI_MEM_UMP; + ret = mali_mem_bind_ump_buf(mali_allocation, mem_backend, + args->mem_union.bind_ump.secure_id, args->mem_union.bind_ump.flags); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Bind ump buf failed\n")); + goto Failed_bind_backend; + } +#else + MALI_DEBUG_PRINT(1, ("UMP not supported\n")); + goto Failed_bind_backend; +#endif + break; + case _MALI_MEMORY_BIND_BACKEND_DMA_BUF: +#if defined(CONFIG_DMA_SHARED_BUFFER) + mali_allocation->type = MALI_MEM_DMA_BUF; + mem_backend->type = MALI_MEM_DMA_BUF; + ret = mali_mem_bind_dma_buf(mali_allocation, mem_backend, + args->mem_union.bind_dma_buf.mem_fd, args->mem_union.bind_dma_buf.flags); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Bind dma buf failed\n")); + goto Failed_bind_backend; + } +#else + MALI_DEBUG_PRINT(1, ("DMA not supported\n")); + goto Failed_bind_backend; +#endif + break; + case _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY: + /* not allowed */ + MALI_DEBUG_ASSERT(0); + break; + + case _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY: + mali_allocation->type = MALI_MEM_EXTERNAL; + mem_backend->type = MALI_MEM_EXTERNAL; + ret = mali_mem_bind_ext_buf(mali_allocation, mem_backend, args->mem_union.bind_ext_memory.phys_addr, + args->mem_union.bind_ext_memory.flags); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Bind external buf failed\n")); + goto Failed_bind_backend; + } + break; + + case _MALI_MEMORY_BIND_BACKEND_EXT_COW: + /* not allowed */ + MALI_DEBUG_ASSERT(0); + break; + + default: + MALI_DEBUG_ASSERT(0); + break; + } + MALI_DEBUG_ASSERT(0 == mem_backend->size % MALI_MMU_PAGE_SIZE); + atomic_add(mem_backend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_backend->type]); + return _MALI_OSK_ERR_OK; + +Failed_bind_backend: + mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); + +Failed_alloc_backend: + mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); + mali_mem_allocation_struct_destory(mali_allocation); + + MALI_DEBUG_PRINT(1, (" _mali_ukk_mem_bind, return ERROR! \n")); + return ret; +} + + +/* +* Function _mali_ukk_mem_unbind -- unbind a external memory to a new GPU address +* This function unbind the backend memory and free the allocation +* no ref_count for this type of memory +*/ +_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args) +{ + /**/ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + mali_mem_allocation *mali_allocation = NULL; + struct mali_vma_node *mali_vma_node = NULL; + u32 mali_addr = args->vaddr; + MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_unbind, vaddr=0x%x! \n", args->vaddr)); + + /* find the allocation by vaddr */ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + if (likely(mali_vma_node)) { + MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); + mali_allocation = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + } else { + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + return _MALI_OSK_ERR_INVALID_ARGS; + } + + if (NULL != mali_allocation) + /* check ref_count */ + mali_allocation_unref(&mali_allocation); + return _MALI_OSK_ERR_OK; +} + +/* +* Function _mali_ukk_mem_cow -- COW for an allocation +* This function allocate new pages for a range (range, range+size) of allocation +* And Map it(keep use the not in range pages from target allocation ) to an GPU vaddr +*/ +_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_backend *target_backend = NULL; + mali_mem_backend *mem_backend = NULL; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_allocation = NULL; + + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + /* Get the target backend for cow */ + target_backend = mali_mem_backend_struct_search(session, args->target_handle); + + if (NULL == target_backend || 0 == target_backend->size) { + MALI_DEBUG_ASSERT_POINTER(target_backend); + MALI_DEBUG_ASSERT(0 != target_backend->size); + return ret; + } + + /*Cow not support resized mem */ + MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE != (MALI_MEM_FLAG_CAN_RESIZE & target_backend->mali_allocation->flags)); + + /* Check if the new mali address is allocated */ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->vaddr, 0); + + if (unlikely(mali_vma_node)) { + MALI_DEBUG_ASSERT(0); + return ret; + } + + /* create new alloction for COW*/ + mali_allocation = mali_mem_allocation_struct_create(session); + if (mali_allocation == NULL) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to create allocation struct!\n")); + return _MALI_OSK_ERR_NOMEM; + } + mali_allocation->psize = args->target_size; + mali_allocation->vsize = args->target_size; + mali_allocation->type = MALI_MEM_COW; + + /*add allocation node to RB tree for index*/ + mali_allocation->mali_vma_node.vm_node.start = args->vaddr; + mali_allocation->mali_vma_node.vm_node.size = mali_allocation->vsize; + mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); + + /* create new backend for COW memory */ + mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); + if (mali_allocation->backend_handle < 0) { + ret = _MALI_OSK_ERR_NOMEM; + MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); + goto failed_alloc_backend; + } + mem_backend->mali_allocation = mali_allocation; + mem_backend->type = mali_allocation->type; + + if (target_backend->type == MALI_MEM_SWAP || + (MALI_MEM_COW == target_backend->type && (MALI_MEM_BACKEND_FLAG_SWAP_COWED & target_backend->flags))) { + mem_backend->flags |= MALI_MEM_BACKEND_FLAG_SWAP_COWED; + /** + * CoWed swap backends couldn't be mapped as non-linear vma, because if one + * vma is set with flag VM_NONLINEAR, the vma->vm_private_data will be used by kernel, + * while in mali driver, we use this variable to store the pointer of mali_allocation, so there + * is a conflict. + * To resolve this problem, we have to do some fake things, we reserved about 64MB + * space from index 0, there isn't really page's index will be set from 0 to (64MB>>PAGE_SHIFT_NUM), + * and all of CoWed swap memory backends' start_idx will be assigned with 0, and these + * backends will be mapped as linear and will add to priority tree of global swap file, while + * these vmas will never be found by using normal page->index, these pages in those vma + * also couldn't be swapped out. + */ + mem_backend->start_idx = 0; + } + + /* Add the target backend's cow count, also allocate new pages for COW backend from os mem + *for a modified range and keep the page which not in the modified range and Add ref to it + */ + MALI_DEBUG_PRINT(3, ("Cow mapping: target_addr: 0x%x; cow_addr: 0x%x, size: %u\n", target_backend->mali_allocation->mali_vma_node.vm_node.start, + mali_allocation->mali_vma_node.vm_node.start, mali_allocation->mali_vma_node.vm_node.size)); + + ret = mali_memory_do_cow(target_backend, args->target_offset, args->target_size, mem_backend, args->range_start, args->range_size); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to cow!\n")); + goto failed_do_cow; + } + + /** + *map to GPU side + */ + mali_allocation->mali_mapping.addr = args->vaddr; + /* set gpu mmu propery */ + _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); + + _mali_osk_mutex_wait(session->memory_lock); + /* Map on Mali */ + ret = mali_mem_mali_map_prepare(mali_allocation); + if (0 != ret) { + MALI_DEBUG_PRINT(1, (" prepare map fail! \n")); + goto failed_gpu_map; + } + + if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { + mali_mem_cow_mali_map(mem_backend, 0, mem_backend->size); + } + + _mali_osk_mutex_signal(session->memory_lock); + + mutex_lock(&target_backend->mutex); + target_backend->flags |= MALI_MEM_BACKEND_FLAG_COWED; + mutex_unlock(&target_backend->mutex); + + atomic_add(args->range_size / MALI_MMU_PAGE_SIZE, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + return _MALI_OSK_ERR_OK; + +failed_gpu_map: + _mali_osk_mutex_signal(session->memory_lock); + mali_mem_cow_release(mem_backend, MALI_FALSE); + mem_backend->cow_mem.count = 0; +failed_do_cow: + mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); +failed_alloc_backend: + mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); + mali_mem_allocation_struct_destory(mali_allocation); + + return ret; +} + +_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_backend *mem_backend = NULL; + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + + MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_cow_modify_range called! \n")); + /* Get the backend that need to be modified. */ + mem_backend = mali_mem_backend_struct_search(session, args->vaddr); + + if (NULL == mem_backend || 0 == mem_backend->size) { + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(0 != mem_backend->size); + return ret; + } + + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_backend->type); + + ret = mali_memory_cow_modify_range(mem_backend, args->range_start, args->size); + args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; + if (_MALI_OSK_ERR_OK != ret) + return ret; + _mali_osk_mutex_wait(session->memory_lock); + if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { + mali_mem_cow_mali_map(mem_backend, args->range_start, args->size); + } + _mali_osk_mutex_signal(session->memory_lock); + + atomic_add(args->change_pages_nr, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + + return _MALI_OSK_ERR_OK; +} + + +_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args) +{ + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); + MALI_DEBUG_ASSERT(0 == args->psize % MALI_MMU_PAGE_SIZE); + + /* Get the memory backend that need to be resize. */ + mem_backend = mali_mem_backend_struct_search(session, args->vaddr); + + if (NULL == mem_backend) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); + return ret; + } + + MALI_DEBUG_ASSERT(args->psize != mem_backend->size); + + ret = mali_mem_resize(session, mem_backend, args->psize); + + return ret; +} + +_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args) +{ + args->memory_usage = _mali_ukk_report_memory_usage(); + if (0 != args->vaddr) { + mali_mem_backend *mem_backend = NULL; + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + /* Get the backend that need to be modified. */ + mem_backend = mali_mem_backend_struct_search(session, args->vaddr); + if (NULL == mem_backend) { + MALI_DEBUG_ASSERT_POINTER(mem_backend); + return _MALI_OSK_ERR_FAULT; + } + + if (MALI_MEM_COW == mem_backend->type) + args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; + } + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h new file mode 100644 index 000000000000..c454b9354676 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_MANAGER_H__ +#define __MALI_MEMORY_MANAGER_H__ + +#include "mali_osk.h" +#include +#include +#include +#include +#include +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" +#include "mali_uk_types.h" + +struct mali_allocation_manager { + rwlock_t vm_lock; + struct rb_root allocation_mgr_rb; + struct list_head head; + struct mutex list_mutex; + u32 mali_allocation_num; +}; + +extern struct idr mali_backend_idr; +extern struct mutex mali_idr_mutex; + +int mali_memory_manager_init(struct mali_allocation_manager *mgr); +void mali_memory_manager_uninit(struct mali_allocation_manager *mgr); + +void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc); +_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size); +mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address); +_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args); +_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args); +_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args); + +#endif + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c index 6636132968f4..1a6cc0649421 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -47,18 +47,7 @@ static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct #endif static void mali_mem_os_trim_pool(struct work_struct *work); -static struct mali_mem_os_allocator { - spinlock_t pool_lock; - struct list_head pool_pages; - size_t pool_count; - - atomic_t allocated_pages; - size_t allocation_limit; - - struct shrinker shrinker; - struct delayed_work timed_shrinker; - struct workqueue_struct *wq; -} mali_mem_os_allocator = { +struct mali_mem_os_allocator mali_mem_os_allocator = { .pool_lock = __SPIN_LOCK_UNLOCKED(pool_lock), .pool_pages = LIST_HEAD_INIT(mali_mem_os_allocator.pool_pages), .pool_count = 0, @@ -82,43 +71,109 @@ static struct mali_mem_os_allocator { #endif }; -static void mali_mem_os_free(mali_mem_allocation *descriptor) +u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag) { LIST_HEAD(pages); - - MALI_DEBUG_ASSERT(MALI_MEM_OS == descriptor->type); - - atomic_sub(descriptor->os_mem.count, &mali_mem_os_allocator.allocated_pages); + struct mali_page_node *m_page, *m_tmp; + u32 free_pages_nr = 0; + + if (MALI_TRUE == cow_flag) { + list_for_each_entry_safe(m_page, m_tmp, os_pages, list) { + /*only handle OS node here */ + if (m_page->type == MALI_PAGE_NODE_OS) { + if (1 == _mali_page_node_get_ref_count(m_page)) { + list_move(&m_page->list, &pages); + atomic_sub(1, &mali_mem_os_allocator.allocated_pages); + free_pages_nr ++; + } else { + _mali_page_node_unref(m_page); + m_page->page = NULL; + list_del(&m_page->list); + kfree(m_page); + } + } + } + } else { + list_cut_position(&pages, os_pages, os_pages->prev); + atomic_sub(pages_count, &mali_mem_os_allocator.allocated_pages); + free_pages_nr = pages_count; + } /* Put pages on pool. */ - list_cut_position(&pages, &descriptor->os_mem.pages, descriptor->os_mem.pages.prev); - spin_lock(&mali_mem_os_allocator.pool_lock); - list_splice(&pages, &mali_mem_os_allocator.pool_pages); - mali_mem_os_allocator.pool_count += descriptor->os_mem.count; - + mali_mem_os_allocator.pool_count += free_pages_nr; spin_unlock(&mali_mem_os_allocator.pool_lock); if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { MALI_DEBUG_PRINT(5, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count)); queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES); } + return free_pages_nr; } -static int mali_mem_os_alloc_pages(mali_mem_allocation *descriptor, u32 size) +/** +* put page without put it into page pool +*/ +_mali_osk_errcode_t mali_mem_os_put_page(struct page *page) { - struct page *new_page, *tmp; - LIST_HEAD(pages); + MALI_DEBUG_ASSERT_POINTER(page); + if (1 == page_count(page)) { + atomic_sub(1, &mali_mem_os_allocator.allocated_pages); + dma_unmap_page(&mali_platform_device->dev, page_private(page), + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + ClearPagePrivate(page); + } + put_page(page); + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count) +{ + struct mali_page_node *m_page, *m_tmp; + u32 i = 0; + + MALI_DEBUG_ASSERT_POINTER(mem_from); + MALI_DEBUG_ASSERT_POINTER(mem_to); + + if (mem_from->count < start_page + page_count) { + return _MALI_OSK_ERR_INVALID_ARGS; + } + + list_for_each_entry_safe(m_page, m_tmp, &mem_from->pages, list) { + if (i >= start_page && i < start_page + page_count) { + list_move_tail(&m_page->list, &mem_to->pages); + mem_from->count--; + mem_to->count++; + } + i++; + } + + return _MALI_OSK_ERR_OK; +} + + +int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size) +{ + struct page *new_page; + LIST_HEAD(pages_list); size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; size_t remaining = page_count; + struct mali_page_node *m_page, *m_tmp; u32 i; - MALI_DEBUG_ASSERT_POINTER(descriptor); - MALI_DEBUG_ASSERT(MALI_MEM_OS == descriptor->type); + MALI_DEBUG_ASSERT_POINTER(os_mem); + + if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { + MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", + size, + atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, + mali_mem_os_allocator.allocation_limit)); + return -ENOMEM; + } - INIT_LIST_HEAD(&descriptor->os_mem.pages); - descriptor->os_mem.count = page_count; + INIT_LIST_HEAD(&os_mem->pages); + os_mem->count = page_count; /* Grab pages from pool. */ { @@ -127,7 +182,7 @@ static int mali_mem_os_alloc_pages(mali_mem_allocation *descriptor, u32 size) pool_pages = min(remaining, mali_mem_os_allocator.pool_count); for (i = pool_pages; i > 0; i--) { BUG_ON(list_empty(&mali_mem_os_allocator.pool_pages)); - list_move(mali_mem_os_allocator.pool_pages.next, &pages); + list_move(mali_mem_os_allocator.pool_pages.next, &pages_list); } mali_mem_os_allocator.pool_count -= pool_pages; remaining -= pool_pages; @@ -136,26 +191,32 @@ static int mali_mem_os_alloc_pages(mali_mem_allocation *descriptor, u32 size) /* Process pages from pool. */ i = 0; - list_for_each_entry_safe(new_page, tmp, &pages, lru) { - BUG_ON(NULL == new_page); + list_for_each_entry_safe(m_page, m_tmp, &pages_list, list) { + BUG_ON(NULL == m_page); - list_move_tail(&new_page->lru, &descriptor->os_mem.pages); + list_move_tail(&m_page->list, &os_mem->pages); } /* Allocate new pages, if needed. */ for (i = 0; i < remaining; i++) { dma_addr_t dma_addr; - gfp_t flags = __GFP_ZERO | __GFP_REPEAT | __GFP_NOWARN | __GFP_COLD; + gfp_t flags = __GFP_ZERO | __GFP_NORETRY | __GFP_NOWARN | __GFP_COLD; int err; #if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) flags |= GFP_HIGHUSER; #else - /* After 3.15.0 kernel use ZONE_DMA replace ZONE_DMA32 */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +#ifdef CONFIG_ZONE_DMA32 flags |= GFP_DMA32; #else +#ifdef CONFIG_ZONE_DMA flags |= GFP_DMA; +#else + /* arm64 utgard only work on < 4G, but the kernel + * didn't provide method to allocte memory < 4G + */ + MALI_DEBUG_ASSERT(0); +#endif #endif #endif @@ -163,9 +224,9 @@ static int mali_mem_os_alloc_pages(mali_mem_allocation *descriptor, u32 size) if (unlikely(NULL == new_page)) { /* Calculate the number of pages actually allocated, and free them. */ - descriptor->os_mem.count = (page_count - remaining) + i; - atomic_add(descriptor->os_mem.count, &mali_mem_os_allocator.allocated_pages); - mali_mem_os_free(descriptor); + os_mem->count = (page_count - remaining) + i; + atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); + mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); return -ENOMEM; } @@ -178,9 +239,9 @@ static int mali_mem_os_alloc_pages(mali_mem_allocation *descriptor, u32 size) MALI_DEBUG_PRINT_ERROR(("OS Mem: Failed to DMA map page %p: %u", new_page, err)); __free_page(new_page); - descriptor->os_mem.count = (page_count - remaining) + i; - atomic_add(descriptor->os_mem.count, &mali_mem_os_allocator.allocated_pages); - mali_mem_os_free(descriptor); + os_mem->count = (page_count - remaining) + i; + atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); + mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); return -EFAULT; } @@ -188,7 +249,21 @@ static int mali_mem_os_alloc_pages(mali_mem_allocation *descriptor, u32 size) SetPagePrivate(new_page); set_page_private(new_page, dma_addr); - list_add_tail(&new_page->lru, &descriptor->os_mem.pages); + m_page = _mali_page_node_allocate(MALI_PAGE_NODE_OS); + if (unlikely(NULL == m_page)) { + MALI_PRINT_ERROR(("OS Mem: Can't allocate mali_page node! \n")); + dma_unmap_page(&mali_platform_device->dev, page_private(new_page), + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + ClearPagePrivate(new_page); + __free_page(new_page); + os_mem->count = (page_count - remaining) + i; + atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); + mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); + return -EFAULT; + } + m_page->page = new_page; + + list_add_tail(&m_page->list, &os_mem->pages); } atomic_add(page_count, &mali_mem_os_allocator.allocated_pages); @@ -201,54 +276,94 @@ static int mali_mem_os_alloc_pages(mali_mem_allocation *descriptor, u32 size) return 0; } -static int mali_mem_os_mali_map(mali_mem_allocation *descriptor, struct mali_session_data *session) + +_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props) { struct mali_page_directory *pagedir = session->page_directory; - struct page *page; - _mali_osk_errcode_t err; - u32 virt = descriptor->mali_mapping.addr; - u32 prop = descriptor->mali_mapping.properties; + struct mali_page_node *m_page; + u32 virt; + u32 prop = props; - MALI_DEBUG_ASSERT(MALI_MEM_OS == descriptor->type); + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(os_mem); - err = mali_mem_mali_map_prepare(descriptor); - if (_MALI_OSK_ERR_OK != err) { - return -ENOMEM; - } + MALI_DEBUG_ASSERT(start_page <= os_mem->count); + MALI_DEBUG_ASSERT((start_page + mapping_pgae_num) <= os_mem->count); + + if ((start_page + mapping_pgae_num) == os_mem->count) { + + virt = vaddr + MALI_MMU_PAGE_SIZE * (start_page + mapping_pgae_num); - list_for_each_entry(page, &descriptor->os_mem.pages, lru) { - dma_addr_t phys = page_private(page); + list_for_each_entry_reverse(m_page, &os_mem->pages, list) { + virt -= MALI_MMU_PAGE_SIZE; + if (mapping_pgae_num > 0) { + dma_addr_t phys = page_private(m_page->page); #if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) - /* Verify that the "physical" address is 32-bit and - * usable for Mali, when on a system with bus addresses - * wider than 32-bit. */ - MALI_DEBUG_ASSERT(0 == (phys >> 32)); + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (phys >> 32)); #endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); + } else { + break; + } + mapping_pgae_num--; + } - mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); - virt += MALI_MMU_PAGE_SIZE; - } + } else { + u32 i = 0; + virt = vaddr; + list_for_each_entry(m_page, &os_mem->pages, list) { - return 0; + if (i >= start_page) { + dma_addr_t phys = page_private(m_page->page); + +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); + } + i++; + virt += MALI_MMU_PAGE_SIZE; + } + } + return _MALI_OSK_ERR_OK; } -static void mali_mem_os_mali_unmap(struct mali_session_data *session, mali_mem_allocation *descriptor) + +void mali_mem_os_mali_unmap(mali_mem_allocation *alloc) { - mali_mem_mali_map_free(descriptor); + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); } -static int mali_mem_os_cpu_map(mali_mem_allocation *descriptor, struct vm_area_struct *vma) +int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) { + mali_mem_os_mem *os_mem = &mem_bkend->os_mem; + struct mali_page_node *m_page; struct page *page; int ret; unsigned long addr = vma->vm_start; + MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); - list_for_each_entry(page, &descriptor->os_mem.pages, lru) { + list_for_each_entry(m_page, &os_mem->pages, list) { /* We should use vm_insert_page, but it does a dcache * flush which makes it way slower than remap_pfn_range or vm_insert_pfn. ret = vm_insert_page(vma, addr, page); */ + page = m_page->page; ret = vm_insert_pfn(vma, addr, page_to_pfn(page)); if (unlikely(0 != ret)) { @@ -260,70 +375,103 @@ static int mali_mem_os_cpu_map(mali_mem_allocation *descriptor, struct vm_area_s return 0; } -mali_mem_allocation *mali_mem_os_alloc(u32 mali_addr, u32 size, struct vm_area_struct *vma, struct mali_session_data *session) +_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size) { - mali_mem_allocation *descriptor; - int err; - - if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { - MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", - size, - atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, - mali_mem_os_allocator.allocation_limit)); - return NULL; - } - - descriptor = mali_mem_descriptor_create(session, MALI_MEM_OS); - if (NULL == descriptor) return NULL; - - descriptor->mali_mapping.addr = mali_addr; - descriptor->size = size; - descriptor->cpu_mapping.addr = (void __user *)vma->vm_start; - descriptor->cpu_mapping.ref = 1; + mali_mem_os_mem *os_mem = &mem_bkend->os_mem; + struct mali_page_node *m_page; + int ret; + int offset; + int mapping_page_num; + int count ; + + unsigned long vstart = vma->vm_start; + count = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); + MALI_DEBUG_ASSERT(0 == start_vaddr % _MALI_OSK_MALI_PAGE_SIZE); + MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); + offset = (start_vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; + MALI_DEBUG_ASSERT(offset <= os_mem->count); + mapping_page_num = mappig_size / _MALI_OSK_MALI_PAGE_SIZE; + MALI_DEBUG_ASSERT((offset + mapping_page_num) <= os_mem->count); + + if ((offset + mapping_page_num) == os_mem->count) { + + unsigned long vm_end = start_vaddr + mappig_size; + + list_for_each_entry_reverse(m_page, &os_mem->pages, list) { + + vm_end -= _MALI_OSK_MALI_PAGE_SIZE; + if (mapping_page_num > 0) { + ret = vm_insert_pfn(vma, vm_end, page_to_pfn(m_page->page)); + + if (unlikely(0 != ret)) { + /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ + if (-EBUSY == ret) { + break; + } else { + MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, offset is %d,page_count is %d\n", + ret, offset + mapping_page_num, os_mem->count)); + } + return _MALI_OSK_ERR_FAULT; + } + } else { + break; + } + mapping_page_num--; - if (VM_SHARED == (VM_SHARED & vma->vm_flags)) { - descriptor->mali_mapping.properties = MALI_MMU_FLAGS_DEFAULT; + } } else { - /* Cached Mali memory mapping */ - descriptor->mali_mapping.properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE; - vma->vm_flags |= VM_SHARED; - } - - err = mali_mem_os_alloc_pages(descriptor, size); /* Allocate pages */ - if (0 != err) goto alloc_failed; - - /* Take session memory lock */ - _mali_osk_mutex_wait(session->memory_lock); - - err = mali_mem_os_mali_map(descriptor, session); /* Map on Mali */ - if (0 != err) goto mali_map_failed; - err = mali_mem_os_cpu_map(descriptor, vma); /* Map on CPU */ - if (0 != err) goto cpu_map_failed; - - _mali_osk_mutex_signal(session->memory_lock); - return descriptor; - -cpu_map_failed: - mali_mem_os_mali_unmap(session, descriptor); -mali_map_failed: - _mali_osk_mutex_signal(session->memory_lock); - mali_mem_os_free(descriptor); -alloc_failed: - mali_mem_descriptor_destroy(descriptor); - MALI_DEBUG_PRINT(2, ("OS allocator: Failed to allocate memory (%d)\n", err)); - return NULL; + list_for_each_entry(m_page, &os_mem->pages, list) { + if (count >= offset) { + + ret = vm_insert_pfn(vma, vstart, page_to_pfn(m_page->page)); + + if (unlikely(0 != ret)) { + /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ + if (-EBUSY == ret) { + break; + } else { + MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, count is %d, offset is %d,page_count is %d\n", + ret, count, offset, os_mem->count)); + } + return _MALI_OSK_ERR_FAULT; + } + } + count++; + vstart += _MALI_OSK_MALI_PAGE_SIZE; + } + } + return _MALI_OSK_ERR_OK; } -void mali_mem_os_release(mali_mem_allocation *descriptor) +u32 mali_mem_os_release(mali_mem_backend *mem_bkend) { - struct mali_session_data *session = descriptor->session; - /* Unmap the memory from the mali virtual address space. */ - mali_mem_os_mali_unmap(session, descriptor); + mali_mem_allocation *alloc; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); + + alloc = mem_bkend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + /* Unmap the memory from the mali virtual address space. */ + mali_mem_os_mali_unmap(alloc); + mutex_lock(&mem_bkend->mutex); /* Free pages */ - mali_mem_os_free(descriptor); + if (MALI_MEM_BACKEND_FLAG_COWED & mem_bkend->flags) { + free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_TRUE); + } else { + free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_FALSE); + } + mutex_unlock(&mem_bkend->mutex); + + MALI_DEBUG_PRINT(4, ("OS Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->os_mem.count * _MALI_OSK_MALI_PAGE_SIZE, + free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); + + mem_bkend->os_mem.count = 0; + return free_pages_nr; } @@ -406,16 +554,20 @@ void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt) } } -static void mali_mem_os_free_page(struct page *page) +void mali_mem_os_free_page_node(struct mali_page_node *m_page) { - BUG_ON(page_count(page) != 1); - - dma_unmap_page(&mali_platform_device->dev, page_private(page), - _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); - - ClearPagePrivate(page); + struct page *page = m_page->page; + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_OS); + if (1 == page_count(page)) { + dma_unmap_page(&mali_platform_device->dev, page_private(page), + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + ClearPagePrivate(page); + } __free_page(page); + m_page->page = NULL; + list_del(&m_page->list); + kfree(m_page); } /* The maximum number of page table pool pages to free in one go. */ @@ -498,7 +650,7 @@ static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink #endif /* Linux < 3.12.0 */ #endif /* Linux < 3.0.0 */ { - struct page *page, *tmp; + struct mali_page_node *m_page, *m_tmp; unsigned long flags; struct list_head *le, pages; #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) @@ -532,8 +684,8 @@ static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le); spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags); - list_for_each_entry_safe(page, tmp, &pages, lru) { - mali_mem_os_free_page(page); + list_for_each_entry_safe(m_page, m_tmp, &pages, list) { + mali_mem_os_free_page_node(m_page); } if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) { @@ -551,7 +703,7 @@ static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink static void mali_mem_os_trim_pool(struct work_struct *data) { - struct page *page, *tmp; + struct mali_page_node *m_page, *m_tmp; struct list_head *le; LIST_HEAD(pages); size_t nr_to_free; @@ -578,8 +730,8 @@ static void mali_mem_os_trim_pool(struct work_struct *data) } spin_unlock(&mali_mem_os_allocator.pool_lock); - list_for_each_entry_safe(page, tmp, &pages, lru) { - mali_mem_os_free_page(page); + list_for_each_entry_safe(m_page, m_tmp, &pages, list) { + mali_mem_os_free_page_node(m_page); } /* Release some pages from page table page pool */ @@ -609,8 +761,7 @@ _mali_osk_errcode_t mali_mem_os_init(void) void mali_mem_os_term(void) { - struct page *page, *tmp; - + struct mali_page_node *m_page, *m_tmp; unregister_shrinker(&mali_mem_os_allocator.shrinker); cancel_delayed_work_sync(&mali_mem_os_allocator.timed_shrinker); @@ -620,8 +771,8 @@ void mali_mem_os_term(void) } spin_lock(&mali_mem_os_allocator.pool_lock); - list_for_each_entry_safe(page, tmp, &mali_mem_os_allocator.pool_pages, lru) { - mali_mem_os_free_page(page); + list_for_each_entry_safe(m_page, m_tmp, &mali_mem_os_allocator.pool_pages, list) { + mali_mem_os_free_page_node(m_page); --mali_mem_os_allocator.pool_count; } diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h index 8df7e99c33c2..f9ead166c455 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -12,36 +12,43 @@ #define __MALI_MEMORY_OS_ALLOC_H__ #include "mali_osk.h" -#include "mali_session.h" - #include "mali_memory_types.h" -/* OS memory allocator */ -/** @brief Allocate memory from OS - * - * This function will create a descriptor, allocate pages and map these on the CPU and Mali. - * - * @param mali_addr Mali virtual address to use for Mali mapping - * @param size Size to allocate - * @param vma Pointer to vma for CPU mapping - * @param session Pointer to session doing the allocation - */ -mali_mem_allocation *mali_mem_os_alloc(u32 mali_addr, u32 size, struct vm_area_struct *vma, struct mali_session_data *session); /** @brief Release Mali OS memory * * The session memory_lock must be held when calling this function. * - * @param descriptor Pointer to the descriptor to release + * @param mem_bkend Pointer to the mali_mem_backend to release */ -void mali_mem_os_release(mali_mem_allocation *descriptor); +u32 mali_mem_os_release(mali_mem_backend *mem_bkend); _mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping); void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt); _mali_osk_errcode_t mali_mem_os_init(void); + void mali_mem_os_term(void); + u32 mali_mem_os_stat(void); +void mali_mem_os_free_page_node(struct mali_page_node *m_page); + +int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size); + +u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag); + +_mali_osk_errcode_t mali_mem_os_put_page(struct page *page); + +_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count); + +_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props); + +void mali_mem_os_mali_unmap(mali_mem_allocation *alloc); + +int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); + +_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size); + #endif /* __MALI_MEMORY_OS_ALLOC_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c new file mode 100644 index 000000000000..a46eb198c98f --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c @@ -0,0 +1,942 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_memory.h" +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" +#include "mali_memory_cow.h" +#include "mali_ukk.h" +#include "mali_kernel_utilization.h" +#include "mali_memory_swap_alloc.h" + + +static struct _mali_osk_bitmap idx_mgr; +static struct file *global_swap_file; +static struct address_space *global_swap_space; +static _mali_osk_wq_work_t *mali_mem_swap_out_workq = NULL; +static u32 mem_backend_swapped_pool_size; +#ifdef MALI_MEM_SWAP_TRACKING +static u32 mem_backend_swapped_unlock_size; +#endif +/* Lock order: mem_backend_swapped_pool_lock > each memory backend's mutex lock. + * This lock used to protect mem_backend_swapped_pool_size and mem_backend_swapped_pool. */ +static struct mutex mem_backend_swapped_pool_lock; +static struct list_head mem_backend_swapped_pool; + +extern struct mali_mem_os_allocator mali_mem_os_allocator; + +#define MALI_SWAP_LOW_MEM_DEFAULT_VALUE (60*1024*1024) +#define MALI_SWAP_INVALIDATE_MALI_ADDRESS (0) /* Used to mark the given memory cookie is invalidate. */ +#define MALI_SWAP_GLOBAL_SWAP_FILE_SIZE (0xFFFFFFFF) +#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX ((MALI_SWAP_GLOBAL_SWAP_FILE_SIZE) >> PAGE_CACHE_SHIFT) +#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE (1 << 15) /* Reserved for CoW nonlinear swap backend memory, the space size is 128MB. */ + +unsigned int mali_mem_swap_out_threshold_value = MALI_SWAP_LOW_MEM_DEFAULT_VALUE; + +/** + * We have two situations to do shrinking things, one is we met low GPU utilization which shows GPU needn't touch too + * swappable backends in short time, and the other one is we add new swappable backends, the total pool size exceed + * the threshold value of the swapped pool size. + */ +typedef enum { + MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION = 100, + MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS = 257, +} _mali_mem_swap_pool_shrink_type_t; + +static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg); + +_mali_osk_errcode_t mali_mem_swap_init(void) +{ + gfp_t flags = __GFP_NORETRY | __GFP_NOWARN; + + if (_MALI_OSK_ERR_OK != _mali_osk_bitmap_init(&idx_mgr, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE)) { + return _MALI_OSK_ERR_NOMEM; + } + + global_swap_file = shmem_file_setup("mali_swap", MALI_SWAP_GLOBAL_SWAP_FILE_SIZE, VM_NORESERVE); + if (IS_ERR(global_swap_file)) { + _mali_osk_bitmap_term(&idx_mgr); + return _MALI_OSK_ERR_NOMEM; + } + + global_swap_space = global_swap_file->f_path.dentry->d_inode->i_mapping; + + mali_mem_swap_out_workq = _mali_osk_wq_create_work(mali_mem_swap_swapped_bkend_pool_check_for_low_utilization, NULL); + if (NULL == mali_mem_swap_out_workq) { + _mali_osk_bitmap_term(&idx_mgr); + fput(global_swap_file); + return _MALI_OSK_ERR_NOMEM; + } + +#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) + flags |= GFP_HIGHUSER; +#else +#ifdef CONFIG_ZONE_DMA32 + flags |= GFP_DMA32; +#else +#ifdef CONFIG_ZONE_DMA + flags |= GFP_DMA; +#else + /* arm64 utgard only work on < 4G, but the kernel + * didn't provide method to allocte memory < 4G + */ + MALI_DEBUG_ASSERT(0); +#endif +#endif +#endif + + /* When we use shmem_read_mapping_page to allocate/swap-in, it will + * use these flags to allocate new page if need.*/ + mapping_set_gfp_mask(global_swap_space, flags); + + mem_backend_swapped_pool_size = 0; +#ifdef MALI_MEM_SWAP_TRACKING + mem_backend_swapped_unlock_size = 0; +#endif + mutex_init(&mem_backend_swapped_pool_lock); + INIT_LIST_HEAD(&mem_backend_swapped_pool); + + MALI_DEBUG_PRINT(2, ("Mali SWAP: Swap out threshold vaule is %uM\n", mali_mem_swap_out_threshold_value >> 20)); + + return _MALI_OSK_ERR_OK; +} + +void mali_mem_swap_term(void) +{ + _mali_osk_bitmap_term(&idx_mgr); + + fput(global_swap_file); + + _mali_osk_wq_delete_work(mali_mem_swap_out_workq); + + MALI_DEBUG_ASSERT(list_empty(&mem_backend_swapped_pool)); + MALI_DEBUG_ASSERT(0 == mem_backend_swapped_pool_size); + + return; +} + +struct file *mali_mem_swap_get_global_swap_file(void) +{ + return global_swap_file; +} + +/* Judge if swappable backend in swapped pool. */ +static mali_bool mali_memory_swap_backend_in_swapped_pool(mali_mem_backend *mem_bkend) +{ + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + + return !list_empty(&mem_bkend->list); +} + +void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend) +{ + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + + mutex_lock(&mem_backend_swapped_pool_lock); + mutex_lock(&mem_bkend->mutex); + + if (MALI_FALSE == mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { + mutex_unlock(&mem_bkend->mutex); + mutex_unlock(&mem_backend_swapped_pool_lock); + return; + } + + MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); + + list_del_init(&mem_bkend->list); + + mutex_unlock(&mem_bkend->mutex); + + mem_backend_swapped_pool_size -= mem_bkend->size; + + mutex_unlock(&mem_backend_swapped_pool_lock); +} + +static void mali_mem_swap_out_page_node(mali_page_node *page_node) +{ + MALI_DEBUG_ASSERT(page_node); + + dma_unmap_page(&mali_platform_device->dev, page_node->swap_it->dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + set_page_dirty(page_node->swap_it->page); + page_cache_release(page_node->swap_it->page); +} + +void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend) +{ + mali_page_node *m_page; + + MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); + + if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN)) { + return; + } + + mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; + + list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { + mali_mem_swap_out_page_node(m_page); + } + + return; +} + +static void mali_mem_swap_unlock_partial_locked_mem_backend(mali_mem_backend *mem_bkend, mali_page_node *page_node) +{ + mali_page_node *m_page; + + MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); + + list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { + if (m_page == page_node) { + break; + } + mali_mem_swap_out_page_node(m_page); + } +} + +static void mali_mem_swap_swapped_bkend_pool_shrink(_mali_mem_swap_pool_shrink_type_t shrink_type) +{ + mali_mem_backend *bkend, *tmp_bkend; + long system_free_size; + u32 last_gpu_utilization, gpu_utilization_threshold_value, temp_swap_out_threshold_value; + + MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_backend_swapped_pool_lock)); + + if (MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION == shrink_type) { + /** + * When we met that system memory is very low and Mali locked swappable memory size is less than + * threshold value, and at the same time, GPU load is very low and don't need high performance, + * at this condition, we can unlock more swap memory backend from swapped backends pool. + */ + gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION; + temp_swap_out_threshold_value = (mali_mem_swap_out_threshold_value >> 2); + } else { + /* When we add swappable memory backends to swapped pool, we need to think that we couldn't + * hold too much swappable backends in Mali driver, and also we need considering performance. + * So there is a balance for swapping out memory backend, we should follow the following conditions: + * 1. Total memory size in global mem backend swapped pool is more than the defined threshold value. + * 2. System level free memory size is less than the defined threshold value. + * 3. Please note that GPU utilization problem isn't considered in this condition. + */ + gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS; + temp_swap_out_threshold_value = mali_mem_swap_out_threshold_value; + } + + /* Get system free pages number. */ + system_free_size = global_page_state(NR_FREE_PAGES) * PAGE_SIZE; + last_gpu_utilization = _mali_ukk_utilization_gp_pp(); + + if ((last_gpu_utilization < gpu_utilization_threshold_value) + && (system_free_size < mali_mem_swap_out_threshold_value) + && (mem_backend_swapped_pool_size > temp_swap_out_threshold_value)) { + list_for_each_entry_safe(bkend, tmp_bkend, &mem_backend_swapped_pool, list) { + if (mem_backend_swapped_pool_size <= temp_swap_out_threshold_value) { + break; + } + + mutex_lock(&bkend->mutex); + + /* check if backend is in use. */ + if (0 < bkend->using_count) { + mutex_unlock(&bkend->mutex); + continue; + } + + mali_mem_swap_unlock_single_mem_backend(bkend); + list_del_init(&bkend->list); + mem_backend_swapped_pool_size -= bkend->size; +#ifdef MALI_MEM_SWAP_TRACKING + mem_backend_swapped_unlock_size += bkend->size; +#endif + mutex_unlock(&bkend->mutex); + } + } + + return; +} + +static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg) +{ + MALI_IGNORE(arg); + + mutex_lock(&mem_backend_swapped_pool_lock); + + mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION); + + mutex_unlock(&mem_backend_swapped_pool_lock); +} + +/** + * After PP job finished, we add all of swappable memory backend used by this PP + * job to the tail of the global swapped pool, and if the total size of swappable memory is more than threshold + * value, we also need to shrink the swapped pool start from the head of the list. + */ +void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend) +{ + mutex_lock(&mem_backend_swapped_pool_lock); + mutex_lock(&mem_bkend->mutex); + + if (mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { + MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); + + list_del_init(&mem_bkend->list); + list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); + mutex_unlock(&mem_bkend->mutex); + mutex_unlock(&mem_backend_swapped_pool_lock); + return; + } + + list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); + + mutex_unlock(&mem_bkend->mutex); + mem_backend_swapped_pool_size += mem_bkend->size; + + mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS); + + mutex_unlock(&mem_backend_swapped_pool_lock); + return; +} + + +u32 mali_mem_swap_idx_alloc(void) +{ + return _mali_osk_bitmap_alloc(&idx_mgr); +} + +void mali_mem_swap_idx_free(u32 idx) +{ + _mali_osk_bitmap_free(&idx_mgr, idx); +} + +static u32 mali_mem_swap_idx_range_alloc(u32 count) +{ + u32 index; + + index = _mali_osk_bitmap_alloc_range(&idx_mgr, count); + + return index; +} + +static void mali_mem_swap_idx_range_free(u32 idx, int num) +{ + _mali_osk_bitmap_free_range(&idx_mgr, idx, num); +} + +struct mali_swap_item *mali_mem_swap_alloc_swap_item(void) +{ + mali_swap_item *swap_item; + + swap_item = kzalloc(sizeof(mali_swap_item), GFP_KERNEL); + + if (NULL == swap_item) { + return NULL; + } + + atomic_set(&swap_item->ref_count, 1); + swap_item->page = NULL; + atomic_add(1, &mali_mem_os_allocator.allocated_pages); + + return swap_item; +} + +void mali_mem_swap_free_swap_item(mali_swap_item *swap_item) +{ + struct inode *file_node; + long long start, end; + + /* If this swap item is shared, we just reduce the reference counter. */ + if (0 == atomic_dec_return(&swap_item->ref_count)) { + file_node = global_swap_file->f_path.dentry->d_inode; + start = swap_item->idx; + start = start << 12; + end = start + PAGE_SIZE; + + shmem_truncate_range(file_node, start, (end - 1)); + + mali_mem_swap_idx_free(swap_item->idx); + + atomic_sub(1, &mali_mem_os_allocator.allocated_pages); + + kfree(swap_item); + } +} + +/* Used to allocate new swap item for new memory allocation and cow page for write. */ +struct mali_page_node *_mali_mem_swap_page_node_allocate(void) +{ + struct mali_page_node *m_page; + + m_page = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); + + if (NULL == m_page) { + return NULL; + } + + m_page->swap_it = mali_mem_swap_alloc_swap_item(); + + if (NULL == m_page->swap_it) { + kfree(m_page); + return NULL; + } + + return m_page; +} + +_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page) +{ + + mali_mem_swap_free_swap_item(m_page->swap_it); + + return _MALI_OSK_ERR_OK; +} + +void _mali_mem_swap_page_node_free(struct mali_page_node *m_page) +{ + _mali_mem_swap_put_page_node(m_page); + + kfree(m_page); + + return; +} + +u32 mali_mem_swap_free(mali_mem_swap *swap_mem) +{ + struct mali_page_node *m_page, *m_tmp; + u32 free_pages_nr = 0; + + MALI_DEBUG_ASSERT_POINTER(swap_mem); + + list_for_each_entry_safe(m_page, m_tmp, &swap_mem->pages, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); + + /* free the page node and release the swap item, if the ref count is 1, + * then need also free the swap item. */ + list_del(&m_page->list); + if (1 == _mali_page_node_get_ref_count(m_page)) { + free_pages_nr++; + } + + _mali_mem_swap_page_node_free(m_page); + } + + return free_pages_nr; +} + +static u32 mali_mem_swap_cow_free(mali_mem_cow *cow_mem) +{ + struct mali_page_node *m_page, *m_tmp; + u32 free_pages_nr = 0; + + MALI_DEBUG_ASSERT_POINTER(cow_mem); + + list_for_each_entry_safe(m_page, m_tmp, &cow_mem->pages, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); + + /* free the page node and release the swap item, if the ref count is 1, + * then need also free the swap item. */ + list_del(&m_page->list); + if (1 == _mali_page_node_get_ref_count(m_page)) { + free_pages_nr++; + } + + _mali_mem_swap_page_node_free(m_page); + } + + return free_pages_nr; +} + +u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) +{ + mali_mem_allocation *alloc; + u32 free_pages_nr = 0; + + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + alloc = mem_bkend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + if (is_mali_mapped) { + mali_mem_swap_mali_unmap(alloc); + } + + mali_memory_swap_list_backend_delete(mem_bkend); + + mutex_lock(&mem_bkend->mutex); + /* To make sure the given memory backend was unlocked from Mali side, + * and then free this memory block. */ + mali_mem_swap_unlock_single_mem_backend(mem_bkend); + mutex_unlock(&mem_bkend->mutex); + + if (MALI_MEM_SWAP == mem_bkend->type) { + free_pages_nr = mali_mem_swap_free(&mem_bkend->swap_mem); + } else { + free_pages_nr = mali_mem_swap_cow_free(&mem_bkend->cow_mem); + } + + return free_pages_nr; +} + +mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node) +{ + MALI_DEBUG_ASSERT(NULL != page_node); + + page_node->swap_it->page = shmem_read_mapping_page(global_swap_space, page_node->swap_it->idx); + + if (IS_ERR(page_node->swap_it->page)) { + MALI_DEBUG_PRINT_ERROR(("SWAP Mem: failed to swap in page with index: %d.\n", page_node->swap_it->idx)); + return MALI_FALSE; + } + + /* Ensure page is flushed from CPU caches. */ + page_node->swap_it->dma_addr = dma_map_page(&mali_platform_device->dev, page_node->swap_it->page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + return MALI_TRUE; +} + +int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx) +{ + size_t page_count = PAGE_ALIGN(size) / PAGE_SIZE; + struct mali_page_node *m_page; + long system_free_size; + u32 i, index; + mali_bool ret; + + MALI_DEBUG_ASSERT(NULL != swap_mem); + MALI_DEBUG_ASSERT(NULL != bkend_idx); + MALI_DEBUG_ASSERT(page_count <= MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE); + + if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { + MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", + size, + atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, + mali_mem_os_allocator.allocation_limit)); + return _MALI_OSK_ERR_NOMEM; + } + + INIT_LIST_HEAD(&swap_mem->pages); + swap_mem->count = page_count; + index = mali_mem_swap_idx_range_alloc(page_count); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == index) { + MALI_PRINT_ERROR(("Mali Swap: Failed to allocate continuous index for swappable Mali memory.")); + return _MALI_OSK_ERR_FAULT; + } + + for (i = 0; i < page_count; i++) { + m_page = _mali_mem_swap_page_node_allocate(); + + if (NULL == m_page) { + MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Failed to allocate mali page node.")); + swap_mem->count = i; + + mali_mem_swap_free(swap_mem); + mali_mem_swap_idx_range_free(index + i, page_count - i); + return _MALI_OSK_ERR_FAULT; + } + + m_page->swap_it->idx = index + i; + + ret = mali_mem_swap_in_page_node(m_page); + + if (MALI_FALSE == ret) { + MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Allocate new page from SHMEM file failed.")); + _mali_mem_swap_page_node_free(m_page); + mali_mem_swap_idx_range_free(index + i + 1, page_count - i - 1); + + swap_mem->count = i; + mali_mem_swap_free(swap_mem); + return _MALI_OSK_ERR_NOMEM; + } + + list_add_tail(&m_page->list, &swap_mem->pages); + } + + system_free_size = global_page_state(NR_FREE_PAGES) * PAGE_SIZE; + + if ((system_free_size < mali_mem_swap_out_threshold_value) + && (mem_backend_swapped_pool_size > (mali_mem_swap_out_threshold_value >> 2)) + && mali_utilization_enabled()) { + _mali_osk_wq_schedule_work(mali_mem_swap_out_workq); + } + + *bkend_idx = index; + return 0; +} + +void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + + +/* Insert these pages from shmem to mali page table*/ +_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props) +{ + struct mali_page_directory *pagedir = session->page_directory; + struct mali_page_node *m_page; + dma_addr_t phys; + u32 virt = vaddr; + u32 prop = props; + + list_for_each_entry(m_page, &swap_mem->pages, list) { + MALI_DEBUG_ASSERT(NULL != m_page->swap_it->page); + phys = m_page->swap_it->dma_addr; + + mali_mmu_pagedir_update(pagedir, virt, phys, MALI_MMU_PAGE_SIZE, prop); + virt += MALI_MMU_PAGE_SIZE; + } + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_in_pages(struct mali_pp_job *job) +{ + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + struct mali_page_node *m_page; + mali_bool swap_in_success = MALI_TRUE; + int i; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + for (i = 0; i < num_memory_cookies; i++) { + + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + if (NULL == mali_vma_node) { + job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; + swap_in_success = MALI_FALSE; + MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); + continue; + } + + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + + if (MALI_MEM_SWAP != mali_alloc->type && + MALI_MEM_COW != mali_alloc->type) { + continue; + } + + /* Get backend memory & Map on GPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + /* We neednot hold backend's lock here, race safe.*/ + if ((MALI_MEM_COW == mem_bkend->type) && + (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + continue; + } + + mutex_lock(&mem_bkend->mutex); + + /* When swap_in_success is MALI_FALSE, it means this job has memory backend that could not be swapped in, + * and it will be aborted in mali scheduler, so here, we just mark those memory cookies which + * should not be swapped out when delete job to invalide */ + if (MALI_FALSE == swap_in_success) { + job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; + mutex_unlock(&mem_bkend->mutex); + continue; + } + + /* Before swap in, checking if this memory backend has been swapped in by the latest flushed jobs. */ + ++mem_bkend->using_count; + + if (1 < mem_bkend->using_count) { + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); + mutex_unlock(&mem_bkend->mutex); + continue; + } + + if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)) { + mutex_unlock(&mem_bkend->mutex); + continue; + } + + + list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { + if (MALI_FALSE == mali_mem_swap_in_page_node(m_page)) { + /* Don't have enough memory to swap in page, so release pages have already been swapped + * in and then mark this pp job to be fail. */ + mali_mem_swap_unlock_partial_locked_mem_backend(mem_bkend, m_page); + swap_in_success = MALI_FALSE; + break; + } + } + + if (swap_in_success) { +#ifdef MALI_MEM_SWAP_TRACKING + mem_backend_swapped_unlock_size -= mem_bkend->size; +#endif + _mali_osk_mutex_wait(session->memory_lock); + mali_mem_swap_mali_map(&mem_bkend->swap_mem, session, mali_alloc->mali_mapping.addr, mali_alloc->mali_mapping.properties); + _mali_osk_mutex_signal(session->memory_lock); + + /* Remove the unlock flag from mem backend flags, mark this backend has been swapped in. */ + mem_bkend->flags &= ~(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN); + mutex_unlock(&mem_bkend->mutex); + } else { + --mem_bkend->using_count; + /* Marking that this backend is not swapped in, need not to be processed anymore. */ + job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; + mutex_unlock(&mem_bkend->mutex); + } + } + + job->swap_status = swap_in_success ? MALI_SWAP_IN_SUCC : MALI_SWAP_IN_FAIL; + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_out_pages(struct mali_pp_job *job) +{ + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + int i; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + + for (i = 0; i < num_memory_cookies; i++) { + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + + if (MALI_SWAP_INVALIDATE_MALI_ADDRESS == mali_addr) { + continue; + } + + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + + if (NULL == mali_vma_node) { + MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); + continue; + } + + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + + if (MALI_MEM_SWAP != mali_alloc->type && + MALI_MEM_COW != mali_alloc->type) { + continue; + } + + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + /* We neednot hold backend's lock here, race safe.*/ + if ((MALI_MEM_COW == mem_bkend->type) && + (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + continue; + } + + mutex_lock(&mem_bkend->mutex); + + MALI_DEBUG_ASSERT(0 < mem_bkend->using_count); + + /* Reducing the using_count of mem backend means less pp job are using this memory backend, + * if this count get to zero, it means no pp job is using it now, could put it to swap out list. */ + --mem_bkend->using_count; + + if (0 < mem_bkend->using_count) { + mutex_unlock(&mem_bkend->mutex); + continue; + } + mutex_unlock(&mem_bkend->mutex); + + mali_memory_swap_list_backend_add(mem_bkend); + } + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) +{ + struct mali_page_node *m_page, *found_node = NULL; + struct page *found_page; + mali_mem_swap *swap = NULL; + mali_mem_cow *cow = NULL; + dma_addr_t dma_addr; + u32 i = 0; + + if (MALI_MEM_SWAP == mem_bkend->type) { + swap = &mem_bkend->swap_mem; + list_for_each_entry(m_page, &swap->pages, list) { + if (i == offset) { + found_node = m_page; + break; + } + i++; + } + } else { + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)); + + cow = &mem_bkend->cow_mem; + list_for_each_entry(m_page, &cow->pages, list) { + if (i == offset) { + found_node = m_page; + break; + } + i++; + } + } + + if (NULL == found_node) { + return _MALI_OSK_ERR_FAULT; + } + + found_page = shmem_read_mapping_page(global_swap_space, found_node->swap_it->idx); + + if (!IS_ERR(found_page)) { + lock_page(found_page); + dma_addr = dma_map_page(&mali_platform_device->dev, found_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + dma_unmap_page(&mali_platform_device->dev, dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + *pagep = found_page; + } else { + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) +{ + struct mali_page_node *m_page, *found_node = NULL, *new_node = NULL; + mali_mem_cow *cow = NULL; + u32 i = 0; + + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)); + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); + MALI_DEBUG_ASSERT(!mali_memory_swap_backend_in_swapped_pool(mem_bkend)); + + cow = &mem_bkend->cow_mem; + list_for_each_entry(m_page, &cow->pages, list) { + if (i == offset) { + found_node = m_page; + break; + } + i++; + } + + if (NULL == found_node) { + return _MALI_OSK_ERR_FAULT; + } + + new_node = _mali_mem_swap_page_node_allocate(); + + if (NULL == new_node) { + return _MALI_OSK_ERR_FAULT; + } + + new_node->swap_it->idx = mali_mem_swap_idx_alloc(); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == new_node->swap_it->idx) { + MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW on demand.\n")); + kfree(new_node->swap_it); + kfree(new_node); + return _MALI_OSK_ERR_FAULT; + } + + if (MALI_FALSE == mali_mem_swap_in_page_node(new_node)) { + _mali_mem_swap_page_node_free(new_node); + return _MALI_OSK_ERR_FAULT; + } + + /* swap in found node for copy in kernel. */ + if (MALI_FALSE == mali_mem_swap_in_page_node(found_node)) { + mali_mem_swap_out_page_node(new_node); + _mali_mem_swap_page_node_free(new_node); + return _MALI_OSK_ERR_FAULT; + } + + _mali_mem_cow_copy_page(found_node, new_node); + + list_replace(&found_node->list, &new_node->list); + + if (1 != _mali_page_node_get_ref_count(found_node)) { + atomic_add(1, &mem_bkend->mali_allocation->session->mali_mem_allocated_pages); + if (atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > mem_bkend->mali_allocation->session->max_mali_mem_allocated_size) { + mem_bkend->mali_allocation->session->max_mali_mem_allocated_size = atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + mem_bkend->cow_mem.change_pages_nr++; + } + + mali_mem_swap_out_page_node(found_node); + _mali_mem_swap_page_node_free(found_node); + + /* When swap in the new page node, we have called dma_map_page for this page.\n */ + dma_unmap_page(&mali_platform_device->dev, new_node->swap_it->dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + lock_page(new_node->swap_it->page); + + *pagep = new_node->swap_it->page; + + return _MALI_OSK_ERR_OK; +} + +#ifdef MALI_MEM_SWAP_TRACKING +void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size) +{ + *swap_pool_size = mem_backend_swapped_pool_size; + *unlock_size = mem_backend_swapped_unlock_size; +} +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h new file mode 100644 index 000000000000..a393ecce3a00 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_SWAP_ALLOC_H__ +#define __MALI_MEMORY_SWAP_ALLOC_H__ + +#include "mali_osk.h" +#include "mali_session.h" + +#include "mali_memory_types.h" +#include "mali_pp_job.h" + +/** + * Initialize memory swapping module. + */ +_mali_osk_errcode_t mali_mem_swap_init(void); + +void mali_mem_swap_term(void); + +/** + * Return global share memory file to other modules. + */ +struct file *mali_mem_swap_get_global_swap_file(void); + +/** + * Unlock the given memory backend and pages in it could be swapped out by kernel. + */ +void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend); + +/** + * Remove the given memory backend from global swap list. + */ +void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend); + +/** + * Add the given memory backend to global swap list. + */ +void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend); + +/** + * Allocate 1 index from bitmap used as page index in global swap file. + */ +u32 mali_mem_swap_idx_alloc(void); + +void mali_mem_swap_idx_free(u32 idx); + +/** + * Allocate a new swap item without page index. + */ +struct mali_swap_item *mali_mem_swap_alloc_swap_item(void); + +/** + * Free a swap item, truncate the corresponding space in page cache and free index of page. + */ +void mali_mem_swap_free_swap_item(mali_swap_item *swap_item); + +/** + * Allocate a page node with swap item. + */ +struct mali_page_node *_mali_mem_swap_page_node_allocate(void); + +/** + * Reduce the reference count of given page node and if return 0, just free this page node. + */ +_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page); + +void _mali_mem_swap_page_node_free(struct mali_page_node *m_page); + +/** + * Free a swappable memory backend. + */ +u32 mali_mem_swap_free(mali_mem_swap *swap_mem); + +/** + * Ummap and free. + */ +u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); + +/** + * Read in a page from global swap file with the pre-allcated page index. + */ +mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node); + +int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx); + +_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props); + +void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc); + +/** + * When pp job created, we need swap in all of memory backend needed by this pp job. + */ +int mali_mem_swap_in_pages(struct mali_pp_job *job); + +/** + * Put all of memory backends used this pp job to the global swap list. + */ +int mali_mem_swap_out_pages(struct mali_pp_job *job); + +/** + * This will be called in page fault to process CPU read&write. + */ +int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) ; + +/** + * Used to process cow on demand for swappable memory backend. + */ +int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep); + +#ifdef MALI_MEM_SWAP_TRACKING +void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size); +#endif +#endif /* __MALI_MEMORY_SWAP_ALLOC_H__ */ + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h index 3ab3acbdfa3a..82af8fed66c9 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -11,6 +11,8 @@ #ifndef __MALI_MEMORY_TYPES_H__ #define __MALI_MEMORY_TYPES_H__ +#include + #if defined(CONFIG_MALI400_UMP) #include "ump_kernel_interface.h" #endif @@ -20,12 +22,49 @@ typedef u32 mali_address_t; typedef enum mali_mem_type { MALI_MEM_OS, MALI_MEM_EXTERNAL, + MALI_MEM_SWAP, MALI_MEM_DMA_BUF, MALI_MEM_UMP, MALI_MEM_BLOCK, + MALI_MEM_COW, MALI_MEM_TYPE_MAX, } mali_mem_type; +typedef struct mali_block_item { + /* for block type, the block_phy is alway page size align + * so use low 12bit used for ref_cout. + */ + unsigned long phy_addr; +} mali_block_item; + +/** + * idx is used to locate the given page in the address space of swap file. + * ref_count is used to mark how many memory backends are using this item. + */ +typedef struct mali_swap_item { + u32 idx; + atomic_t ref_count; + struct page *page; + dma_addr_t dma_addr; +} mali_swap_item; + +typedef enum mali_page_node_type { + MALI_PAGE_NODE_OS, + MALI_PAGE_NODE_BLOCK, + MALI_PAGE_NODE_SWAP, +} mali_page_node_type; + +typedef struct mali_page_node { + struct list_head list; + union { + struct page *page; + mali_block_item *blk_it; /*pointer to block item*/ + mali_swap_item *swap_it; + }; + + u32 type; +} mali_page_node; + typedef struct mali_mem_os_mem { struct list_head pages; u32 count; @@ -56,7 +95,8 @@ typedef struct block_allocator_allocation { } block_allocator_allocation; typedef struct mali_mem_block_mem { - block_allocator_allocation mem; + struct list_head pfns; + u32 count; } mali_mem_block_mem; typedef struct mali_mem_virt_mali_mapping { @@ -66,22 +106,83 @@ typedef struct mali_mem_virt_mali_mapping { typedef struct mali_mem_virt_cpu_mapping { void __user *addr; - u32 ref; + struct vm_area_struct *vma; } mali_mem_virt_cpu_mapping; #define MALI_MEM_ALLOCATION_VALID_MAGIC 0xdeda110c #define MALI_MEM_ALLOCATION_FREED_MAGIC 0x10101010 +typedef struct mali_mm_node { + /* MALI GPU vaddr start, use u32 for mmu only support 32bit address*/ + uint32_t start; /* GPU vaddr */ + uint32_t size; /* GPU allocation virtual size */ + unsigned allocated : 1; +} mali_mm_node; + +typedef struct mali_vma_node { + struct mali_mm_node vm_node; + struct rb_node vm_rb; +} mali_vma_node; + + typedef struct mali_mem_allocation { MALI_DEBUG_CODE(u32 magic); mali_mem_type type; /**< Type of memory */ - int id; /**< ID in the descriptor map for this allocation */ - - u32 size; /**< Size of the allocation */ u32 flags; /**< Flags for this allocation */ struct mali_session_data *session; /**< Pointer to session that owns the allocation */ + mali_mem_virt_cpu_mapping cpu_mapping; /**< CPU mapping */ + mali_mem_virt_mali_mapping mali_mapping; /**< Mali mapping */ + + /* add for new memory system */ + struct mali_vma_node mali_vma_node; + u32 vsize; /* virtual size*/ + u32 psize; /* physical backend memory size*/ + struct list_head list; + s32 backend_handle; /* idr for mem_backend */ + _mali_osk_atomic_t mem_alloc_refcount; +} mali_mem_allocation; + +struct mali_mem_os_allocator { + spinlock_t pool_lock; + struct list_head pool_pages; + size_t pool_count; + + atomic_t allocated_pages; + size_t allocation_limit; + + struct shrinker shrinker; + struct delayed_work timed_shrinker; + struct workqueue_struct *wq; +}; + +/* COW backend memory type */ +typedef struct mali_mem_cow { + struct list_head pages; /**< all pages for this cow backend allocation, + including new allocated pages for modified range*/ + u32 count; /**< number of pages */ + s32 change_pages_nr; +} mali_mem_cow; + +typedef struct mali_mem_swap { + struct list_head pages; + u32 count; +} mali_mem_swap; + +#define MALI_MEM_BACKEND_FLAG_COWED (0x1) /* COW has happen on this backend */ +#define MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE (0x2) /* This is an COW backend, mapped as not allowed cpu to write */ +#define MALI_MEM_BACKEND_FLAG_SWAP_COWED (0x4) /* Mark the given backend is cowed from swappable memory. */ +/* Mark this backend is not swapped_in in MALI driver, and before using it, + * we should swap it in and set up corresponding page table. */ +#define MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN (0x8) +#define MALI_MEM_BACKEND_FLAG_NOT_BINDED (0x1 << 5) /* this backend it not back with physical memory, used for defer bind */ +#define MALI_MEM_BACKEND_FLAG_BINDED (0x1 << 6) /* this backend it back with physical memory, used for defer bind */ + +typedef struct mali_mem_backend { + mali_mem_type type; /**< Type of backend memory */ + u32 flags; /**< Flags for this allocation */ + u32 size; /* Union selected by type. */ union { mali_mem_os_mem os_mem; /**< MALI_MEM_OS */ @@ -89,13 +190,19 @@ typedef struct mali_mem_allocation { mali_mem_dma_buf dma_buf; /**< MALI_MEM_DMA_BUF */ mali_mem_ump ump_mem; /**< MALI_MEM_UMP */ mali_mem_block_mem block_mem; /**< MALI_MEM_BLOCK */ + mali_mem_cow cow_mem; + mali_mem_swap swap_mem; }; + mali_mem_allocation *mali_allocation; + struct mutex mutex; + mali_mem_type cow_type; - mali_mem_virt_cpu_mapping cpu_mapping; /**< CPU mapping */ - mali_mem_virt_mali_mapping mali_mapping; /**< Mali mapping */ -} mali_mem_allocation; + struct list_head list; /**< Used to link swappable memory backend to the global swappable list */ + int using_count; /**< Mark how many PP jobs are using this memory backend */ + u32 start_idx; /**< If the correspondign vma of this backend is linear, this value will be used to set vma->vm_pgoff */ +} mali_mem_backend; -#define MALI_MEM_FLAG_MALI_GUARD_PAGE (1 << 0) +#define MALI_MEM_FLAG_MALI_GUARD_PAGE (_MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) #define MALI_MEM_FLAG_DONT_CPU_MAP (1 << 1) - +#define MALI_MEM_FLAG_CAN_RESIZE (_MALI_MEMORY_ALLOCATE_RESIZEABLE) #endif /* __MALI_MEMORY_TYPES__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c index 92e654a3de5b..bacadc85e47e 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -13,27 +13,31 @@ #include "mali_kernel_common.h" #include "mali_session.h" #include "mali_kernel_linux.h" - #include "mali_memory.h" - #include "ump_kernel_interface.h" -static int mali_ump_map(struct mali_session_data *session, mali_mem_allocation *descriptor) +static int mali_mem_ump_map(mali_mem_backend *mem_backend) { ump_dd_handle ump_mem; + mali_mem_allocation *alloc; + struct mali_session_data *session; u32 nr_blocks; u32 i; ump_dd_physical_block *ump_blocks; struct mali_page_directory *pagedir; u32 offset = 0; - u32 prop; _mali_osk_errcode_t err; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); + + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + session = alloc->session; MALI_DEBUG_ASSERT_POINTER(session); - MALI_DEBUG_ASSERT_POINTER(descriptor); - MALI_DEBUG_ASSERT(MALI_MEM_UMP == descriptor->type); - ump_mem = descriptor->ump_mem.handle; + ump_mem = mem_backend->ump_mem.handle; MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); nr_blocks = ump_dd_phys_block_count_get(ump_mem); @@ -53,161 +57,98 @@ static int mali_ump_map(struct mali_session_data *session, mali_mem_allocation * } pagedir = session->page_directory; - prop = descriptor->mali_mapping.properties; - err = mali_mem_mali_map_prepare(descriptor); + mali_session_memory_lock(session); + + err = mali_mem_mali_map_prepare(alloc); if (_MALI_OSK_ERR_OK != err) { MALI_DEBUG_PRINT(1, ("Mapping of UMP memory failed\n")); _mali_osk_free(ump_blocks); + mali_session_memory_unlock(session); return -ENOMEM; } for (i = 0; i < nr_blocks; ++i) { - u32 virt = descriptor->mali_mapping.addr + offset; + u32 virt = alloc->mali_vma_node.vm_node.start + offset; MALI_DEBUG_PRINT(7, ("Mapping in 0x%08x size %d\n", ump_blocks[i].addr , ump_blocks[i].size)); mali_mmu_pagedir_update(pagedir, virt, ump_blocks[i].addr, - ump_blocks[i].size, prop); + ump_blocks[i].size, MALI_MMU_FLAGS_DEFAULT); offset += ump_blocks[i].size; } - if (descriptor->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { - u32 virt = descriptor->mali_mapping.addr + offset; + if (alloc->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + u32 virt = alloc->mali_vma_node.vm_node.start + offset; /* Map in an extra virtual guard page at the end of the VMA */ MALI_DEBUG_PRINT(6, ("Mapping in extra guard page\n")); - mali_mmu_pagedir_update(pagedir, virt, ump_blocks[0].addr, _MALI_OSK_MALI_PAGE_SIZE, prop); + mali_mmu_pagedir_update(pagedir, virt, ump_blocks[0].addr, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); offset += _MALI_OSK_MALI_PAGE_SIZE; } - + mali_session_memory_unlock(session); _mali_osk_free(ump_blocks); - return 0; } -void mali_ump_unmap(struct mali_session_data *session, mali_mem_allocation *descriptor) +static void mali_mem_ump_unmap(mali_mem_allocation *alloc) { - ump_dd_handle ump_mem; - struct mali_page_directory *pagedir; - - ump_mem = descriptor->ump_mem.handle; - pagedir = session->page_directory; - - MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); - - mali_mem_mali_map_free(descriptor); - - ump_dd_reference_release(ump_mem); - return; + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); } -_mali_osk_errcode_t _mali_ukk_attach_ump_mem(_mali_uk_attach_ump_mem_s *args) +int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags) { ump_dd_handle ump_mem; - struct mali_session_data *session; - mali_mem_allocation *descriptor; - int md, ret; - - MALI_DEBUG_ASSERT_POINTER(args); - MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); - - session = (struct mali_session_data *)(uintptr_t)args->ctx; - - /* check arguments */ - /* NULL might be a valid Mali address */ - if (!args->size) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); - - /* size must be a multiple of the system page size */ - if (args->size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + int ret; + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); MALI_DEBUG_PRINT(3, ("Requested to map ump memory with secure id %d into virtual memory 0x%08X, size 0x%08X\n", - args->secure_id, args->mali_address, args->size)); - - ump_mem = ump_dd_handle_create_from_secure_id((int)args->secure_id); + secure_id, alloc->mali_vma_node.vm_node.start, alloc->mali_vma_node.vm_node.size)); + ump_mem = ump_dd_handle_create_from_secure_id(secure_id); if (UMP_DD_HANDLE_INVALID == ump_mem) MALI_ERROR(_MALI_OSK_ERR_FAULT); - - descriptor = mali_mem_descriptor_create(session, MALI_MEM_UMP); - if (NULL == descriptor) { - ump_dd_reference_release(ump_mem); - MALI_ERROR(_MALI_OSK_ERR_NOMEM); + alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; + if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; } - descriptor->ump_mem.handle = ump_mem; - descriptor->mali_mapping.addr = args->mali_address; - descriptor->size = args->size; - descriptor->mali_mapping.properties = MALI_MMU_FLAGS_DEFAULT; - descriptor->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; + mem_backend->ump_mem.handle = ump_mem; - if (args->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { - descriptor->flags = MALI_MEM_FLAG_MALI_GUARD_PAGE; - } - - _mali_osk_mutex_wait(session->memory_lock); - - ret = mali_ump_map(session, descriptor); + ret = mali_mem_ump_map(mem_backend); if (0 != ret) { - _mali_osk_mutex_signal(session->memory_lock); - ump_dd_reference_release(ump_mem); - mali_mem_descriptor_destroy(descriptor); - MALI_ERROR(_MALI_OSK_ERR_NOMEM); - } - - _mali_osk_mutex_signal(session->memory_lock); - - - if (_MALI_OSK_ERR_OK != mali_descriptor_mapping_allocate_mapping(session->descriptor_mapping, descriptor, &md)) { ump_dd_reference_release(ump_mem); - mali_mem_descriptor_destroy(descriptor); - MALI_ERROR(_MALI_OSK_ERR_FAULT); + return _MALI_OSK_ERR_FAULT; } - - args->cookie = md; - - MALI_DEBUG_PRINT(5, ("Returning from UMP attach\n")); - - MALI_SUCCESS; + MALI_DEBUG_PRINT(3, ("Returning from UMP bind\n")); + return _MALI_OSK_ERR_OK; } -void mali_mem_ump_release(mali_mem_allocation *descriptor) +void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend) { - struct mali_session_data *session = descriptor->session; - - MALI_DEBUG_ASSERT(MALI_MEM_UMP == descriptor->type); + ump_dd_handle ump_mem; + mali_mem_allocation *alloc; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); + ump_mem = mem_backend->ump_mem.handle; + MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); - mali_ump_unmap(session, descriptor); + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + mali_mem_ump_unmap(alloc); + ump_dd_reference_release(ump_mem); } -_mali_osk_errcode_t _mali_ukk_release_ump_mem(_mali_uk_release_ump_mem_s *args) -{ - mali_mem_allocation *descriptor; - struct mali_session_data *session; - - MALI_DEBUG_ASSERT_POINTER(args); - MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); - - session = (struct mali_session_data *)(uintptr_t)args->ctx; - - if (_MALI_OSK_ERR_OK != mali_descriptor_mapping_get(session->descriptor_mapping, args->cookie, (void **)&descriptor)) { - MALI_DEBUG_PRINT(1, ("Invalid memory descriptor %d used to release ump memory\n", args->cookie)); - MALI_ERROR(_MALI_OSK_ERR_FAULT); - } - - descriptor = mali_descriptor_mapping_free(session->descriptor_mapping, args->cookie); - - if (NULL != descriptor) { - _mali_osk_mutex_wait(session->memory_lock); - mali_mem_ump_release(descriptor); - _mali_osk_mutex_signal(session->memory_lock); - - mali_mem_descriptor_destroy(descriptor); - } - - MALI_SUCCESS; -} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h new file mode 100644 index 000000000000..4adb70df884e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_UMP_BUF_H__ +#define __MALI_MEMORY_UMP_BUF_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_memory.h" + +int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags); +void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_MEMORY_DMA_BUF_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c new file mode 100644 index 000000000000..bcca43c09f0b --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_linux.h" +#include "mali_scheduler.h" + +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include "mali_memory_dma_buf.h" +#endif +#if defined(CONFIG_MALI400_UMP) +#include "mali_memory_ump.h" +#endif +#include "mali_memory_external.h" +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" +#include "mali_memory_cow.h" +#include "mali_memory_block_alloc.h" +#include "mali_memory_swap_alloc.h" + + + +/** +*function @_mali_free_allocation_mem - free a memory allocation +*/ +static u32 _mali_free_allocation_mem(mali_mem_allocation *mali_alloc) +{ + mali_mem_backend *mem_bkend = NULL; + u32 free_pages_nr = 0; + + struct mali_session_data *session = mali_alloc->session; + MALI_DEBUG_PRINT(4, (" _mali_free_allocation_mem, psize =0x%x! \n", mali_alloc->psize)); + if (0 == mali_alloc->psize) + goto out; + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + switch (mem_bkend->type) { + case MALI_MEM_OS: + free_pages_nr = mali_mem_os_release(mem_bkend); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + break; + case MALI_MEM_UMP: +#if defined(CONFIG_MALI400_UMP) + mali_mem_unbind_ump_buf(mem_bkend); + atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); +#else + MALI_DEBUG_PRINT(2, ("UMP not supported\n")); +#endif + break; + case MALI_MEM_DMA_BUF: +#if defined(CONFIG_DMA_SHARED_BUFFER) + mali_mem_unbind_dma_buf(mem_bkend); + atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); +#else + MALI_DEBUG_PRINT(2, ("DMA not supported\n")); +#endif + break; + case MALI_MEM_EXTERNAL: + mali_mem_unbind_ext_buf(mem_bkend); + atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); + break; + + case MALI_MEM_BLOCK: + free_pages_nr = mali_mem_block_release(mem_bkend); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + break; + + case MALI_MEM_COW: + if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { + free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); + } else { + free_pages_nr = mali_mem_cow_release(mem_bkend, MALI_TRUE); + } + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + break; + case MALI_MEM_SWAP: + free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + atomic_sub(free_pages_nr, &session->mali_mem_array[mem_bkend->type]); + break; + default: + MALI_DEBUG_PRINT(1, ("mem type %d is not in the mali_mem_type enum.\n", mem_bkend->type)); + break; + } + + /*Remove backend memory idex */ + mutex_lock(&mali_idr_mutex); + idr_remove(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + kfree(mem_bkend); +out: + /* remove memory allocation */ + mali_vma_offset_remove(&session->allocation_mgr, &mali_alloc->mali_vma_node); + mali_mem_allocation_struct_destory(mali_alloc); + return free_pages_nr; +} + +/** +* ref_count for allocation +*/ +u32 mali_allocation_unref(struct mali_mem_allocation **alloc) +{ + u32 free_pages_nr = 0; + mali_mem_allocation *mali_alloc = *alloc; + *alloc = NULL; + if (0 == _mali_osk_atomic_dec_return(&mali_alloc->mem_alloc_refcount)) { + free_pages_nr = _mali_free_allocation_mem(mali_alloc); + } + return free_pages_nr; +} + +void mali_allocation_ref(struct mali_mem_allocation *alloc) +{ + _mali_osk_atomic_inc(&alloc->mem_alloc_refcount); +} + +void mali_free_session_allocations(struct mali_session_data *session) +{ + struct mali_mem_allocation *entry, *next; + + MALI_DEBUG_PRINT(4, (" mali_free_session_allocations! \n")); + + list_for_each_entry_safe(entry, next, &session->allocation_mgr.head, list) { + mali_allocation_unref(&entry); + } +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h new file mode 100644 index 000000000000..0d686979d7a8 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_UTIL_H__ +#define __MALI_MEMORY_UTIL_H__ + +u32 mali_allocation_unref(struct mali_mem_allocation **alloc); + +void mali_allocation_ref(struct mali_mem_allocation *alloc); + +void mali_free_session_allocations(struct mali_session_data *session); + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c new file mode 100644 index 000000000000..35f8f90a444d --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_linux.h" +#include "mali_scheduler.h" +#include "mali_memory_os_alloc.h" +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" + + +/** +*internal helper to link node into the rb-tree +*/ +static inline void _mali_vma_offset_add_rb(struct mali_allocation_manager *mgr, + struct mali_vma_node *node) +{ + struct rb_node **iter = &mgr->allocation_mgr_rb.rb_node; + struct rb_node *parent = NULL; + struct mali_vma_node *iter_node; + + while (likely(*iter)) { + parent = *iter; + iter_node = rb_entry(*iter, struct mali_vma_node, vm_rb); + + if (node->vm_node.start < iter_node->vm_node.start) + iter = &(*iter)->rb_left; + else if (node->vm_node.start > iter_node->vm_node.start) + iter = &(*iter)->rb_right; + else + MALI_DEBUG_ASSERT(0); + } + + rb_link_node(&node->vm_rb, parent, iter); + rb_insert_color(&node->vm_rb, &mgr->allocation_mgr_rb); +} + +/** + * mali_vma_offset_add() - Add offset node to RB Tree + */ +int mali_vma_offset_add(struct mali_allocation_manager *mgr, + struct mali_vma_node *node) +{ + int ret = 0; + write_lock(&mgr->vm_lock); + + if (node->vm_node.allocated) { + goto out; + } + + _mali_vma_offset_add_rb(mgr, node); + /* set to allocated */ + node->vm_node.allocated = 1; + +out: + write_unlock(&mgr->vm_lock); + return ret; +} + +/** + * mali_vma_offset_remove() - Remove offset node from RB tree + */ +void mali_vma_offset_remove(struct mali_allocation_manager *mgr, + struct mali_vma_node *node) +{ + write_lock(&mgr->vm_lock); + + if (node->vm_node.allocated) { + rb_erase(&node->vm_rb, &mgr->allocation_mgr_rb); + memset(&node->vm_node, 0, sizeof(node->vm_node)); + } + write_unlock(&mgr->vm_lock); +} + +/** +* mali_vma_offset_search - Search the node in RB tree +*/ +struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, + unsigned long start, unsigned long pages) +{ + struct mali_vma_node *node, *best; + struct rb_node *iter; + unsigned long offset; + read_lock(&mgr->vm_lock); + + iter = mgr->allocation_mgr_rb.rb_node; + best = NULL; + + while (likely(iter)) { + node = rb_entry(iter, struct mali_vma_node, vm_rb); + offset = node->vm_node.start; + if (start >= offset) { + iter = iter->rb_right; + best = node; + if (start == offset) + break; + } else { + iter = iter->rb_left; + } + } + + if (best) { + offset = best->vm_node.start + best->vm_node.size; + if (offset <= start + pages) + best = NULL; + } + read_unlock(&mgr->vm_lock); + + return best; +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h new file mode 100644 index 000000000000..3e2d48f44e28 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#ifndef __MALI_GPU_VMEM_H__ +#define __MALI_GPU_VMEM_H__ + +#include "mali_osk.h" +#include "mali_session.h" +#include +#include +#include +#include +#include +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" +#include "mali_memory_manager.h" + + + +int mali_vma_offset_add(struct mali_allocation_manager *mgr, + struct mali_vma_node *node); + +void mali_vma_offset_remove(struct mali_allocation_manager *mgr, + struct mali_vma_node *node); + +struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, + unsigned long start, unsigned long pages); + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c index 069bfe966384..ba630e2730cf 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c new file mode 100644 index 000000000000..01ca38235b20 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_bitmap.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include +#include +#include +#include +#include +#include "common/mali_kernel_common.h" +#include "mali_osk_types.h" +#include "mali_osk.h" + +u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap) +{ + u32 obj; + + MALI_DEBUG_ASSERT_POINTER(bitmap); + + _mali_osk_spinlock_lock(bitmap->lock); + + obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->reserve); + + if (obj < bitmap->max) { + set_bit(obj, bitmap->table); + } else { + obj = -1; + } + + if (obj != -1) + --bitmap->avail; + _mali_osk_spinlock_unlock(bitmap->lock); + + return obj; +} + +void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + _mali_osk_bitmap_free_range(bitmap, obj, 1); +} + +u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt) +{ + u32 obj; + + MALI_DEBUG_ASSERT_POINTER(bitmap); + + if (0 >= cnt) { + return -1; + } + + if (1 == cnt) { + return _mali_osk_bitmap_alloc(bitmap); + } + + _mali_osk_spinlock_lock(bitmap->lock); + obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, + bitmap->last, cnt, 0); + + if (obj >= bitmap->max) { + obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, + bitmap->reserve, cnt, 0); + } + + if (obj < bitmap->max) { + bitmap_set(bitmap->table, obj, cnt); + + bitmap->last = (obj + cnt); + if (bitmap->last >= bitmap->max) { + bitmap->last = bitmap->reserve; + } + } else { + obj = -1; + } + + if (obj != -1) { + bitmap->avail -= cnt; + } + + _mali_osk_spinlock_unlock(bitmap->lock); + + return obj; +} + +u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + return bitmap->avail; +} + +void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + _mali_osk_spinlock_lock(bitmap->lock); + bitmap_clear(bitmap->table, obj, cnt); + bitmap->last = min(bitmap->last, obj); + + bitmap->avail += cnt; + _mali_osk_spinlock_unlock(bitmap->lock); +} + +int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + MALI_DEBUG_ASSERT(reserve <= num); + + bitmap->reserve = reserve; + bitmap->last = reserve; + bitmap->max = num; + bitmap->avail = num - reserve; + bitmap->lock = _mali_osk_spinlock_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); + if (!bitmap->lock) { + return _MALI_OSK_ERR_NOMEM; + } + bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * + sizeof(long), GFP_KERNEL); + if (!bitmap->table) { + _mali_osk_spinlock_term(bitmap->lock); + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + if (NULL != bitmap->lock) { + _mali_osk_spinlock_term(bitmap->lock); + } + + if (NULL != bitmap->table) { + kfree(bitmap->table); + } +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c index 8d24222a9e9c..539832d9125a 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -153,7 +153,7 @@ _mali_osk_irq_t *_mali_osk_irq_init(u32 irqnum, _mali_osk_irq_uhandler_t uhandle #if defined(DEBUG) /* Verify that the configured interrupt settings are working */ if (_MALI_OSK_ERR_OK != test_interrupt(irqnum, trigger_func, ack_func, probe_data, description)) { - MALI_DEBUG_PRINT(2, ("Test of IRQ handler for core '%s' failed\n", description)); + MALI_DEBUG_PRINT(2, ("Test of IRQ(%d) handler for core '%s' failed\n", irqnum, description)); kfree(irq_object); return NULL; } diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c index b6252120962b..50c0a9d23819 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h index f09cf170957e..aa32a81e7496 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c index df3c29b7edae..c14e872a7d63 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c index 7cb7e472c231..1a6d257baa5a 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -56,59 +56,80 @@ #define MALI_OSK_RESOURCE_L2_LOCATION_START 20 #define MALI_OSK_RESOURCE_l2_LOCATION_END 22 +/** + * DMA unit location. + */ +#define MALI_OSK_RESOURCE_DMA_LOCATION 26 + static _mali_osk_resource_t mali_osk_resource_bank[MALI_OSK_MAX_RESOURCE_NUMBER] = { -/*-------------------------------------------------------*/ -/* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */ -// {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "IRQGP",}, -{.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "Mali_GP_IRQ",}, -// {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "IRQGPMMU",}, -{.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "Mali_GP_MMU_IRQ",}, -// {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "IRQPP0",}, -{.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "Mali_PP0_IRQ",}, -// {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "IRQPPMMU0",}, -{.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "Mali_PP0_MMU_IRQ",}, -// {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "IRQPP1",}, -{.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "Mali_PP1_IRQ",}, -// {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "IRQPPMMU1",}, -{.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "Mali_PP1_MMU_IRQ",}, -/*-------------------------------------------------------*/ -{.description = "Mali_PP2", .base = MALI_OFFSET_PP2, .irq_name = "IRQPP2",}, -{.description = "Mali_PP2_MMU", .base = MALI_OFFSET_PP2_MMU, .irq_name = "IRQPPMMU2",}, -{.description = "Mali_PP3", .base = MALI_OFFSET_PP3, .irq_name = "IRQPP3",}, -{.description = "Mali_PP3_MMU", .base = MALI_OFFSET_PP3_MMU, .irq_name = "IRQPPMMU3",}, -{.description = "Mali_PP4", .base = MALI_OFFSET_PP4, .irq_name = "IRQPP4",}, -{.description = "Mali_PP4_MMU", .base = MALI_OFFSET_PP4_MMU, .irq_name = "IRQPPMMU4",}, -{.description = "Mali_PP5", .base = MALI_OFFSET_PP5, .irq_name = "IRQPP5",}, -{.description = "Mali_PP5_MMU", .base = MALI_OFFSET_PP5_MMU, .irq_name = "IRQPPMMU5",}, -{.description = "Mali_PP6", .base = MALI_OFFSET_PP6, .irq_name = "IRQPP6",}, -{.description = "Mali_PP6_MMU", .base = MALI_OFFSET_PP6_MMU, .irq_name = "IRQPPMMU6",}, -{.description = "Mali_PP7", .base = MALI_OFFSET_PP7, .irq_name = "IRQPP7",}, -{.description = "Mali_PP7_MMU", .base = MALI_OFFSET_PP7_MMU, .irq_name = "IRQPPMMU",}, -{.description = "Mali_PP_Broadcast", .base = MALI_OFFSET_PP_BCAST, .irq_name = "IRQPP",}, -{.description = "Mali_PMU", .base = MALI_OFFSET_PMU, .irq_name = "IRQPMU",}, -{.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE0,}, -{.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE1,}, -{.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE2,}, -{.description = "Mali_PP_MMU_Broadcast", .base = MALI_OFFSET_PP_BCAST_MMU,}, -{.description = "Mali_Broadcast", .base = MALI_OFFSET_BCAST,}, -{.description = "Mali_DLBU", .base = MALI_OFFSET_DLBU,}, -{.description = "Mali_DMA", .base = MALI_OFFSET_DMA,}, + /*-------------------------------------------------------*/ + /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */ + /* {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "IRQGP",}, */ + {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "Mali_GP_IRQ",}, + /* {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "IRQGPMMU",}, */ + {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "Mali_GP_MMU_IRQ",}, + /* {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "IRQPP0",}, */ + {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "Mali_PP0_IRQ",}, + /* {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "IRQPPMMU0",}, */ + {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "Mali_PP0_MMU_IRQ",}, + /* {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "IRQPP1",}, */ + {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "Mali_PP1_IRQ",}, + /* {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "IRQPPMMU1",}, */ + {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "Mali_PP1_MMU_IRQ",}, + /*-------------------------------------------------------*/ + {.description = "Mali_PP2", .base = MALI_OFFSET_PP2, .irq_name = "IRQPP2",}, + {.description = "Mali_PP2_MMU", .base = MALI_OFFSET_PP2_MMU, .irq_name = "IRQPPMMU2",}, + {.description = "Mali_PP3", .base = MALI_OFFSET_PP3, .irq_name = "IRQPP3",}, + {.description = "Mali_PP3_MMU", .base = MALI_OFFSET_PP3_MMU, .irq_name = "IRQPPMMU3",}, + {.description = "Mali_PP4", .base = MALI_OFFSET_PP4, .irq_name = "IRQPP4",}, + {.description = "Mali_PP4_MMU", .base = MALI_OFFSET_PP4_MMU, .irq_name = "IRQPPMMU4",}, + {.description = "Mali_PP5", .base = MALI_OFFSET_PP5, .irq_name = "IRQPP5",}, + {.description = "Mali_PP5_MMU", .base = MALI_OFFSET_PP5_MMU, .irq_name = "IRQPPMMU5",}, + {.description = "Mali_PP6", .base = MALI_OFFSET_PP6, .irq_name = "IRQPP6",}, + {.description = "Mali_PP6_MMU", .base = MALI_OFFSET_PP6_MMU, .irq_name = "IRQPPMMU6",}, + {.description = "Mali_PP7", .base = MALI_OFFSET_PP7, .irq_name = "IRQPP7",}, + {.description = "Mali_PP7_MMU", .base = MALI_OFFSET_PP7_MMU, .irq_name = "IRQPPMMU",}, + {.description = "Mali_PP_Broadcast", .base = MALI_OFFSET_PP_BCAST, .irq_name = "IRQPP",}, + {.description = "Mali_PMU", .base = MALI_OFFSET_PMU, .irq_name = "IRQPMU",}, + {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE0,}, + {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE1,}, + {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE2,}, + {.description = "Mali_PP_MMU_Broadcast", .base = MALI_OFFSET_PP_BCAST_MMU,}, + {.description = "Mali_Broadcast", .base = MALI_OFFSET_BCAST,}, + {.description = "Mali_DLBU", .base = MALI_OFFSET_DLBU,}, + {.description = "Mali_DMA", .base = MALI_OFFSET_DMA,}, }; +static int _mali_osk_get_compatible_name(const char **out_string) +{ + struct device_node *node = mali_platform_device->dev.of_node; + + MALI_DEBUG_ASSERT(NULL != node); + + return of_property_read_string(node, "compatible", out_string); +} + _mali_osk_errcode_t _mali_osk_resource_initialize(void) { - mali_bool mali_is_450 = MALI_FALSE; + mali_bool mali_is_450 = MALI_FALSE, mali_is_470 = MALI_FALSE; int i, pp_core_num = 0, l2_core_num = 0; struct resource *res; + const char *compatible_name = NULL; + + if (0 == _mali_osk_get_compatible_name(&compatible_name)) { + if (0 == strncmp(compatible_name, "arm,mali-450", strlen("arm,mali-450"))) { + mali_is_450 = MALI_TRUE; + MALI_DEBUG_PRINT(2, ("mali-450 device tree detected.")); + } else if (0 == strncmp(compatible_name, "arm,mali-470", strlen("arm,mali-470"))) { + mali_is_470 = MALI_TRUE; + MALI_DEBUG_PRINT(2, ("mali-470 device tree detected.")); + } + } for (i = 0; i < MALI_OSK_RESOURCE_WITH_IRQ_NUMBER; i++) { res = platform_get_resource_byname(mali_platform_device, IORESOURCE_IRQ, mali_osk_resource_bank[i].irq_name); if (res) { mali_osk_resource_bank[i].irq = res->start; - if (0 == strncmp("Mali_PP_Broadcast", mali_osk_resource_bank[i].description, - strlen(mali_osk_resource_bank[i].description))) { - mali_is_450 = MALI_TRUE; - } } else { mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; } @@ -130,14 +151,15 @@ _mali_osk_errcode_t _mali_osk_resource_initialize(void) /** * we can caculate the number of l2 cache core according the number of pp core number - * and device type(mali400/mali450). + * and device type(mali400/mali450/mali470). */ - if (mali_is_450 && 4 < pp_core_num) { - l2_core_num = 3; - } else if (mali_is_450 && 4 >= pp_core_num) { - l2_core_num = 2; - } else { - l2_core_num = 1; + l2_core_num = 1; + if (mali_is_450) { + if (pp_core_num > 4) { + l2_core_num = 3; + } else if (pp_core_num <= 4) { + l2_core_num = 2; + } } for (i = MALI_OSK_RESOURCE_l2_LOCATION_END; i > MALI_OSK_RESOURCE_L2_LOCATION_START + l2_core_num - 1; i--) { @@ -145,12 +167,15 @@ _mali_osk_errcode_t _mali_osk_resource_initialize(void) } /* If device is not mali-450 type, we have to remove related resource from resource bank. */ - if (!mali_is_450) { + if (!(mali_is_450 || mali_is_470)) { for (i = MALI_OSK_RESOURCE_l2_LOCATION_END + 1; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) { mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; } } + if (mali_is_470) + mali_osk_resource_bank[MALI_OSK_RESOURCE_DMA_LOCATION].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; + return _MALI_OSK_ERR_OK; } @@ -207,13 +232,15 @@ void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_si int length = 0, i = 0; u32 u; + MALI_DEBUG_PRINT(2, ("Get pmu config from device tree configuration.\n")); + MALI_DEBUG_ASSERT(NULL != node); if (!of_get_property(node, "pmu_domain_config", &length)) { return; } - if (array_size != length/sizeof(u32)) { + if (array_size != length / sizeof(u32)) { MALI_PRINT_ERROR(("Wrong pmu domain config in device tree.")); return; } @@ -300,6 +327,7 @@ void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_si { _mali_osk_device_data data = { 0, }; + MALI_DEBUG_PRINT(2, ("Get pmu config from platform device data.\n")); if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { /* Copy the custom customer power domain config */ _mali_osk_memcpy(domain_config_array, data.pmu_domain_config, sizeof(data.pmu_domain_config)); @@ -343,22 +371,18 @@ _mali_osk_errcode_t _mali_osk_device_data_get(_mali_osk_device_data *data) return _MALI_OSK_ERR_ITEM_NOT_FOUND; } -u32 _mali_osk_l2_resource_count(void) +u32 _mali_osk_identify_gpu_resource(void) { - u32 l2_core_num = 0; - - if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_L2_RESOURCE0, NULL)) - l2_core_num++; - if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_L2_RESOURCE1, NULL)) - l2_core_num++; - - if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_L2_RESOURCE2, NULL)) - l2_core_num++; + /* Mali 450 */ + return 0x450; - MALI_DEBUG_ASSERT(0 < l2_core_num); + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_DLBU, NULL)) + /* Mali 470 */ + return 0x470; - return l2_core_num; + /* Mali 400 */ + return 0x400; } mali_bool _mali_osk_shared_interrupts(void) diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c index 02741b0fa532..085ce76f7665 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c index 27dea0e604d3..390e613e186d 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c index 474d2b6d6fa3..0a619e3fc27e 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c index 8a1e786e4d97..e66fe83f3557 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c index bf8f2d343da6..21180d33fe75 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c index 8351cbd3d08b..cc09748e7316 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -7,8 +7,12 @@ * A copy of the licence is included with the program, and can also be obtained from Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - +#include #include +#include +#include +#include +#include #include #include "mali_kernel_common.h" @@ -22,19 +26,768 @@ #include "mali_l2_cache.h" #include "mali_user_settings_db.h" #include "mali_executor.h" +#include "mali_memory_manager.h" + +#define MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE 100 +#define MALI_PROFILING_STREAM_HOLD_TIME 1000000 /*1 ms */ + +#define MALI_PROFILING_STREAM_BUFFER_SIZE (1 << 12) +#define MALI_PROFILING_STREAM_BUFFER_NUM 100 + +/** + * Define the mali profiling stream struct. + */ +typedef struct mali_profiling_stream { + u8 data[MALI_PROFILING_STREAM_BUFFER_SIZE]; + u32 used_size; + struct list_head list; +} mali_profiling_stream; + +typedef struct mali_profiling_stream_list { + spinlock_t spin_lock; + struct list_head free_list; + struct list_head queue_list; +} mali_profiling_stream_list; + +static const char mali_name[] = "4xx"; +static const char utgard_setup_version[] = "ANNOTATE_SETUP 1\n"; + +static u32 profiling_sample_rate = 0; +static u32 first_sw_counter_index = 0; + +static mali_bool l2_cache_counter_if_enabled = MALI_FALSE; +static u32 num_counters_enabled = 0; +static u32 mem_counters_enabled = 0; + +static _mali_osk_atomic_t stream_fd_if_used; + +static wait_queue_head_t stream_fd_wait_queue; +static mali_profiling_counter *global_mali_profiling_counters = NULL; +static u32 num_global_mali_profiling_counters = 0; + +static mali_profiling_stream_list *global_mali_stream_list = NULL; +static mali_profiling_stream *mali_counter_stream = NULL; +static mali_profiling_stream *mali_core_activity_stream = NULL; +static u64 mali_core_activity_stream_dequeue_time = 0; +static spinlock_t mali_activity_lock; +static u32 mali_activity_cores_num = 0; +static struct hrtimer profiling_sampling_timer; + +const char *_mali_mem_counter_descriptions[] = _MALI_MEM_COUTNER_DESCRIPTIONS; +const char *_mali_special_counter_descriptions[] = _MALI_SPCIAL_COUNTER_DESCRIPTIONS; + +static u32 current_profiling_pid = 0; + +static void _mali_profiling_stream_list_destory(mali_profiling_stream_list *profiling_stream_list) +{ + mali_profiling_stream *profiling_stream, *tmp_profiling_stream; + MALI_DEBUG_ASSERT_POINTER(profiling_stream_list); + + list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->free_list, list) { + list_del(&profiling_stream->list); + kfree(profiling_stream); + } + + list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->queue_list, list) { + list_del(&profiling_stream->list); + kfree(profiling_stream); + } + + kfree(profiling_stream_list); +} + +static void _mali_profiling_global_stream_list_free(void) +{ + mali_profiling_stream *profiling_stream, *tmp_profiling_stream; + unsigned long irq_flags; + + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &global_mali_stream_list->queue_list, list) { + profiling_stream->used_size = 0; + list_move(&profiling_stream->list, &global_mali_stream_list->free_list); + } + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); +} + +static _mali_osk_errcode_t _mali_profiling_global_stream_list_dequeue(struct list_head *stream_list, mali_profiling_stream **new_mali_profiling_stream) +{ + unsigned long irq_flags; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + MALI_DEBUG_ASSERT_POINTER(stream_list); + + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + + if (!list_empty(stream_list)) { + *new_mali_profiling_stream = list_entry(stream_list->next, mali_profiling_stream, list); + list_del_init(&(*new_mali_profiling_stream)->list); + } else { + ret = _MALI_OSK_ERR_NOMEM; + } + + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); + + return ret; +} + +static void _mali_profiling_global_stream_list_queue(struct list_head *stream_list, mali_profiling_stream *current_mali_profiling_stream) +{ + unsigned long irq_flags; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + MALI_DEBUG_ASSERT_POINTER(stream_list); + + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + list_add_tail(¤t_mali_profiling_stream->list, stream_list); + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); +} + +static mali_bool _mali_profiling_global_stream_queue_list_if_empty(void) +{ + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + return list_empty(&global_mali_stream_list->queue_list); +} + +static u32 _mali_profiling_global_stream_queue_list_next_size(void) +{ + unsigned long irq_flags; + u32 size = 0; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + if (!list_empty(&global_mali_stream_list->queue_list)) { + mali_profiling_stream *next_mali_profiling_stream = + list_entry(global_mali_stream_list->queue_list.next, mali_profiling_stream, list); + size = next_mali_profiling_stream->used_size; + } + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); + return size; +} + +/* The mali profiling stream file operations functions. */ +static ssize_t _mali_profiling_stream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos); + +static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait); + +static int _mali_profiling_stream_release(struct inode *inode, struct file *filp); + +/* The timeline stream file operations structure. */ +static const struct file_operations mali_profiling_stream_fops = { + .release = _mali_profiling_stream_release, + .read = _mali_profiling_stream_read, + .poll = _mali_profiling_stream_poll, +}; + +static ssize_t _mali_profiling_stream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos) +{ + u32 copy_len = 0; + mali_profiling_stream *current_mali_profiling_stream; + u32 used_size; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + + while (!_mali_profiling_global_stream_queue_list_if_empty()) { + used_size = _mali_profiling_global_stream_queue_list_next_size(); + if (used_size <= ((u32)size - copy_len)) { + current_mali_profiling_stream = NULL; + _mali_profiling_global_stream_list_dequeue(&global_mali_stream_list->queue_list, + ¤t_mali_profiling_stream); + MALI_DEBUG_ASSERT_POINTER(current_mali_profiling_stream); + if (copy_to_user(&buffer[copy_len], current_mali_profiling_stream->data, current_mali_profiling_stream->used_size)) { + current_mali_profiling_stream->used_size = 0; + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); + return -EFAULT; + } + copy_len += current_mali_profiling_stream->used_size; + current_mali_profiling_stream->used_size = 0; + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); + } else { + break; + } + } + return (ssize_t)copy_len; +} + +static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait) +{ + poll_wait(filp, &stream_fd_wait_queue, wait); + if (!_mali_profiling_global_stream_queue_list_if_empty()) + return POLLIN; + return 0; +} + +static int _mali_profiling_stream_release(struct inode *inode, struct file *filp) +{ + _mali_osk_atomic_init(&stream_fd_if_used, 0); + return 0; +} + +/* The funs for control packet and stream data.*/ +static void _mali_profiling_set_packet_size(unsigned char *const buf, const u32 size) +{ + u32 i; + + for (i = 0; i < sizeof(size); ++i) + buf[i] = (size >> 8 * i) & 0xFF; +} + +static u32 _mali_profiling_get_packet_size(unsigned char *const buf) +{ + u32 i; + u32 size = 0; + for (i = 0; i < sizeof(size); ++i) + size |= (u32)buf[i] << 8 * i; + return size; +} + +static u32 _mali_profiling_read_packet_int(unsigned char *const buf, u32 *const pos, u32 const packet_size) +{ + u64 int_value = 0; + u8 shift = 0; + u8 byte_value = ~0; + + while ((byte_value & 0x80) != 0) { + MALI_DEBUG_ASSERT((*pos) < packet_size); + byte_value = buf[*pos]; + *pos += 1; + int_value |= (u32)(byte_value & 0x7f) << shift; + shift += 7; + } + + if (shift < 8 * sizeof(int_value) && (byte_value & 0x40) != 0) { + int_value |= -(1 << shift); + } + + return int_value; +} + +static u32 _mali_profiling_pack_int(u8 *const buf, u32 const buf_size, u32 const pos, s32 value) +{ + u32 add_bytes = 0; + int more = 1; + while (more) { + /* low order 7 bits of val */ + char byte_value = value & 0x7f; + value >>= 7; + + if ((value == 0 && (byte_value & 0x40) == 0) || (value == -1 && (byte_value & 0x40) != 0)) { + more = 0; + } else { + byte_value |= 0x80; + } + + MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size); + buf[pos + add_bytes] = byte_value; + add_bytes++; + } + + return add_bytes; +} + +static int _mali_profiling_pack_long(uint8_t *const buf, u32 const buf_size, u32 const pos, s64 val) +{ + int add_bytes = 0; + int more = 1; + while (more) { + /* low order 7 bits of x */ + char byte_value = val & 0x7f; + val >>= 7; + + if ((val == 0 && (byte_value & 0x40) == 0) || (val == -1 && (byte_value & 0x40) != 0)) { + more = 0; + } else { + byte_value |= 0x80; + } + + MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size); + buf[pos + add_bytes] = byte_value; + add_bytes++; + } + + return add_bytes; +} + +static void _mali_profiling_stream_add_counter(mali_profiling_stream *profiling_stream, s64 current_time, u32 key, u32 counter_value) +{ + u32 add_size = STREAM_HEADER_SIZE; + MALI_DEBUG_ASSERT_POINTER(profiling_stream); + MALI_DEBUG_ASSERT((profiling_stream->used_size) < MALI_PROFILING_STREAM_BUFFER_SIZE); + + profiling_stream->data[profiling_stream->used_size] = STREAM_HEADER_COUNTER_VALUE; + + add_size += _mali_profiling_pack_long(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, current_time); + add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, (s32)0); + add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, (s32)key); + add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, (s32)counter_value); + + _mali_profiling_set_packet_size(profiling_stream->data + profiling_stream->used_size + 1, + add_size - STREAM_HEADER_SIZE); + + profiling_stream->used_size += add_size; +} + +/* The callback function for sampling timer.*/ +static enum hrtimer_restart _mali_profiling_sampling_counters(struct hrtimer *timer) +{ + u32 counter_index; + s64 current_time; + MALI_DEBUG_ASSERT_POINTER(global_mali_profiling_counters); + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + + MALI_DEBUG_ASSERT(NULL == mali_counter_stream); + if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( + &global_mali_stream_list->free_list, &mali_counter_stream)) { + + MALI_DEBUG_ASSERT_POINTER(mali_counter_stream); + MALI_DEBUG_ASSERT(0 == mali_counter_stream->used_size); + + /* Capture l2 cache counter values if enabled */ + if (MALI_TRUE == l2_cache_counter_if_enabled) { + int i, j = 0; + _mali_profiling_l2_counter_values l2_counters_values; + _mali_profiling_get_l2_counters(&l2_counters_values); + + for (i = COUNTER_L2_0_C0; i <= COUNTER_L2_2_C1; i++) { + if (0 == (j % 2)) + _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value0); + else + _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value1); + j++; + } + } + + current_time = (s64)_mali_osk_boot_time_get_ns(); + + /* Add all enabled counter values into stream */ + for (counter_index = 0; counter_index < num_global_mali_profiling_counters; counter_index++) { + /* No need to sample these couners here. */ + if (global_mali_profiling_counters[counter_index].enabled) { + if ((global_mali_profiling_counters[counter_index].counter_id >= FIRST_MEM_COUNTER && + global_mali_profiling_counters[counter_index].counter_id <= LAST_MEM_COUNTER) + || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_VP_ACTIVITY) + || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FP_ACTIVITY) + || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FILMSTRIP)) { + + continue; + } + + if (global_mali_profiling_counters[counter_index].counter_id >= COUNTER_L2_0_C0 && + global_mali_profiling_counters[counter_index].counter_id <= COUNTER_L2_2_C1) { + + u32 prev_val = global_mali_profiling_counters[counter_index].prev_counter_value; + + _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, + global_mali_profiling_counters[counter_index].current_counter_value - prev_val); + + prev_val = global_mali_profiling_counters[counter_index].current_counter_value; + + global_mali_profiling_counters[counter_index].prev_counter_value = prev_val; + } else { + + if (global_mali_profiling_counters[counter_index].counter_id == COUNTER_TOTAL_ALLOC_PAGES) { + u32 total_alloc_mem = _mali_ukk_report_memory_usage(); + global_mali_profiling_counters[counter_index].current_counter_value = total_alloc_mem / _MALI_OSK_MALI_PAGE_SIZE; + } + _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, + global_mali_profiling_counters[counter_index].current_counter_value); + if (global_mali_profiling_counters[counter_index].counter_id < FIRST_SPECIAL_COUNTER) + global_mali_profiling_counters[counter_index].current_counter_value = 0; + } + } + } + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_counter_stream); + mali_counter_stream = NULL; + } else { + MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); + } + + wake_up_interruptible(&stream_fd_wait_queue); + + /*Enable the sampling timer again*/ + if (0 != num_counters_enabled && 0 != profiling_sample_rate) { + hrtimer_forward_now(&profiling_sampling_timer, ns_to_ktime(profiling_sample_rate)); + return HRTIMER_RESTART; + } + return HRTIMER_NORESTART; +} + +static void _mali_profiling_sampling_core_activity_switch(int counter_id, int core, u32 activity, u32 pid) +{ + unsigned long irq_flags; + + spin_lock_irqsave(&mali_activity_lock, irq_flags); + if (activity == 0) + mali_activity_cores_num--; + else + mali_activity_cores_num++; + spin_unlock_irqrestore(&mali_activity_lock, irq_flags); + + if (NULL != global_mali_profiling_counters) { + int i ; + for (i = 0; i < num_global_mali_profiling_counters; i++) { + if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { + u64 current_time = _mali_osk_boot_time_get_ns(); + u32 add_size = STREAM_HEADER_SIZE; + + if (NULL != mali_core_activity_stream) { + if ((mali_core_activity_stream_dequeue_time + MALI_PROFILING_STREAM_HOLD_TIME < current_time) || + (MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE > MALI_PROFILING_STREAM_BUFFER_SIZE + - mali_core_activity_stream->used_size)) { + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); + mali_core_activity_stream = NULL; + wake_up_interruptible(&stream_fd_wait_queue); + } + } + + if (NULL == mali_core_activity_stream) { + if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( + &global_mali_stream_list->free_list, &mali_core_activity_stream)) { + mali_core_activity_stream_dequeue_time = current_time; + } else { + MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); + wake_up_interruptible(&stream_fd_wait_queue); + break; + } + + } + + mali_core_activity_stream->data[mali_core_activity_stream->used_size] = STREAM_HEADER_CORE_ACTIVITY; + + add_size += _mali_profiling_pack_long(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s64)current_time); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, core); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s32)global_mali_profiling_counters[i].key); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, activity); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, pid); + + _mali_profiling_set_packet_size(mali_core_activity_stream->data + mali_core_activity_stream->used_size + 1, + add_size - STREAM_HEADER_SIZE); + + mali_core_activity_stream->used_size += add_size; + + if (0 == mali_activity_cores_num) { + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); + mali_core_activity_stream = NULL; + wake_up_interruptible(&stream_fd_wait_queue); + } + + break; + } + } + } +} + +static mali_bool _mali_profiling_global_counters_init(void) +{ + int core_id, counter_index, counter_number, counter_id; + u32 num_l2_cache_cores; + u32 num_pp_cores; + u32 num_gp_cores = 1; + + MALI_DEBUG_ASSERT(NULL == global_mali_profiling_counters); + num_pp_cores = mali_pp_get_glob_num_pp_cores(); + num_l2_cache_cores = mali_l2_cache_core_get_glob_num_l2_cores(); + + num_global_mali_profiling_counters = 3 * (num_gp_cores + num_pp_cores) + 2 * num_l2_cache_cores + + MALI_PROFILING_SW_COUNTERS_NUM + + MALI_PROFILING_SPECIAL_COUNTERS_NUM + + MALI_PROFILING_MEM_COUNTERS_NUM; + global_mali_profiling_counters = _mali_osk_calloc(num_global_mali_profiling_counters, sizeof(mali_profiling_counter)); + + if (NULL == global_mali_profiling_counters) + return MALI_FALSE; + + counter_index = 0; + /*Vertex processor counters */ + for (core_id = 0; core_id < num_gp_cores; core_id ++) { + global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_VP_0 + core_id; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_active", mali_name, core_id); + + for (counter_number = 0; counter_number < 2; counter_number++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = COUNTER_VP_0_C0 + (2 * core_id) + counter_number; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_cnt%d", mali_name, core_id, counter_number); + } + } + + /* Fragment processors' counters */ + for (core_id = 0; core_id < num_pp_cores; core_id++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_FP_0 + core_id; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_active", mali_name, core_id); + + for (counter_number = 0; counter_number < 2; counter_number++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = COUNTER_FP_0_C0 + (2 * core_id) + counter_number; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_cnt%d", mali_name, core_id, counter_number); + } + } + + /* L2 Cache counters */ + for (core_id = 0; core_id < num_l2_cache_cores; core_id++) { + for (counter_number = 0; counter_number < 2; counter_number++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = COUNTER_L2_0_C0 + (2 * core_id) + counter_number; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_L2_%d_cnt%d", mali_name, core_id, counter_number); + } + } + + /* Now set up the software counter entries */ + for (counter_id = FIRST_SW_COUNTER; counter_id <= LAST_SW_COUNTER; counter_id++) { + counter_index++; + + if (0 == first_sw_counter_index) + first_sw_counter_index = counter_index; + + global_mali_profiling_counters[counter_index].counter_id = counter_id; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_SW_%d", mali_name, counter_id - FIRST_SW_COUNTER); + } + + /* Now set up the special counter entries */ + for (counter_id = FIRST_SPECIAL_COUNTER; counter_id <= LAST_SPECIAL_COUNTER; counter_id++) { + + counter_index++; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", + mali_name, _mali_special_counter_descriptions[counter_id - FIRST_SPECIAL_COUNTER]); + + global_mali_profiling_counters[counter_index].counter_id = counter_id; + } + + /* Now set up the mem counter entries*/ + for (counter_id = FIRST_MEM_COUNTER; counter_id <= LAST_MEM_COUNTER; counter_id++) { + + counter_index++; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", + mali_name, _mali_mem_counter_descriptions[counter_id - FIRST_MEM_COUNTER]); + + global_mali_profiling_counters[counter_index].counter_id = counter_id; + } + + MALI_DEBUG_ASSERT((counter_index + 1) == num_global_mali_profiling_counters); + + return MALI_TRUE; +} + +void _mali_profiling_notification_mem_counter(struct mali_session_data *session, u32 counter_id, u32 key, int enable) +{ + + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL != session) { + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + + queue = session->ioctl_queue; + MALI_DEBUG_ASSERT(NULL != queue); + + notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER, + sizeof(_mali_uk_annotate_profiling_mem_counter_s)); + + if (NULL != notification) { + _mali_uk_annotate_profiling_mem_counter_s *data = notification->result_buffer; + data->counter_id = counter_id; + data->key = key; + data->enable = enable; + + _mali_osk_notification_queue_send(queue, notification); + } else { + MALI_PRINT_ERROR(("Failed to create notification object!\n")); + } + } else { + MALI_PRINT_ERROR(("Failed to find the right session!\n")); + } +} + +void _mali_profiling_notification_enable(struct mali_session_data *session, u32 sampling_rate, int enable) +{ + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL != session) { + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + + queue = session->ioctl_queue; + MALI_DEBUG_ASSERT(NULL != queue); + + notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE, + sizeof(_mali_uk_annotate_profiling_enable_s)); + + if (NULL != notification) { + _mali_uk_annotate_profiling_enable_s *data = notification->result_buffer; + data->sampling_rate = sampling_rate; + data->enable = enable; + + _mali_osk_notification_queue_send(queue, notification); + } else { + MALI_PRINT_ERROR(("Failed to create notification object!\n")); + } + } else { + MALI_PRINT_ERROR(("Failed to find the right session!\n")); + } +} + _mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start) { + int i; + mali_profiling_stream *new_mali_profiling_stream = NULL; + mali_profiling_stream_list *new_mali_profiling_stream_list = NULL; if (MALI_TRUE == auto_start) { mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE); } + /*Init the global_mali_stream_list*/ + MALI_DEBUG_ASSERT(NULL == global_mali_stream_list); + new_mali_profiling_stream_list = (mali_profiling_stream_list *)kmalloc(sizeof(mali_profiling_stream_list), GFP_KERNEL); + + if (NULL == new_mali_profiling_stream_list) { + return _MALI_OSK_ERR_NOMEM; + } + + spin_lock_init(&new_mali_profiling_stream_list->spin_lock); + INIT_LIST_HEAD(&new_mali_profiling_stream_list->free_list); + INIT_LIST_HEAD(&new_mali_profiling_stream_list->queue_list); + + spin_lock_init(&mali_activity_lock); + mali_activity_cores_num = 0; + + for (i = 0; i < MALI_PROFILING_STREAM_BUFFER_NUM; i++) { + new_mali_profiling_stream = (mali_profiling_stream *)kmalloc(sizeof(mali_profiling_stream), GFP_KERNEL); + if (NULL == new_mali_profiling_stream) { + _mali_profiling_stream_list_destory(new_mali_profiling_stream_list); + return _MALI_OSK_ERR_NOMEM; + } + + INIT_LIST_HEAD(&new_mali_profiling_stream->list); + new_mali_profiling_stream->used_size = 0; + list_add_tail(&new_mali_profiling_stream->list, &new_mali_profiling_stream_list->free_list); + + } + + _mali_osk_atomic_init(&stream_fd_if_used, 0); + init_waitqueue_head(&stream_fd_wait_queue); + + hrtimer_init(&profiling_sampling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + + profiling_sampling_timer.function = _mali_profiling_sampling_counters; + + global_mali_stream_list = new_mali_profiling_stream_list; + return _MALI_OSK_ERR_OK; } void _mali_osk_profiling_term(void) { - /* Nothing to do */ + if (0 != profiling_sample_rate) { + hrtimer_cancel(&profiling_sampling_timer); + profiling_sample_rate = 0; + } + _mali_osk_atomic_term(&stream_fd_if_used); + + if (NULL != global_mali_profiling_counters) { + _mali_osk_free(global_mali_profiling_counters); + global_mali_profiling_counters = NULL; + num_global_mali_profiling_counters = 0; + } + + if (NULL != global_mali_stream_list) { + _mali_profiling_stream_list_destory(global_mali_stream_list); + global_mali_stream_list = NULL; + } + +} + +void _mali_osk_profiling_stop_sampling(u32 pid) +{ + if (pid == current_profiling_pid) { + + int i; + /* Reset all counter states when closing connection.*/ + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); + global_mali_profiling_counters[i].enabled = 0; + global_mali_profiling_counters[i].prev_counter_value = 0; + global_mali_profiling_counters[i].current_counter_value = 0; + } + l2_cache_counter_if_enabled = MALI_FALSE; + num_counters_enabled = 0; + mem_counters_enabled = 0; + _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); + _mali_profiling_control(SW_COUNTER_ENABLE, 0); + /* Delete sampling timer when closing connection. */ + if (0 != profiling_sample_rate) { + hrtimer_cancel(&profiling_sampling_timer); + profiling_sample_rate = 0; + } + current_profiling_pid = 0; + } +} + +void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4) +{ + /*Record the freq & volt to global_mali_profiling_counters here. */ + if (0 != profiling_sample_rate) { + u32 channel; + u32 state; + channel = (event_id >> 16) & 0xFF; + state = ((event_id >> 24) & 0xF) << 24; + + switch (state) { + case MALI_PROFILING_EVENT_TYPE_SINGLE: + if ((MALI_PROFILING_EVENT_CHANNEL_GPU >> 16) == channel) { + u32 reason = (event_id & 0xFFFF); + if (MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE == reason) { + _mali_osk_profiling_record_global_counters(COUNTER_FREQUENCY, data0); + _mali_osk_profiling_record_global_counters(COUNTER_VOLTAGE, data1); + } + } + break; + case MALI_PROFILING_EVENT_TYPE_START: + if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { + _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 1, data1); + } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && + (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { + u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); + _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 1, data1); + } + break; + case MALI_PROFILING_EVENT_TYPE_STOP: + if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { + _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 0, 0); + } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && + (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { + u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); + _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 0, 0); + } + break; + default: + break; + } + } + trace_mali_timeline_event(event_id, data0, data1, data2, data3, data4); } void _mali_osk_profiling_report_sw_counters(u32 *counters) @@ -42,9 +795,17 @@ void _mali_osk_profiling_report_sw_counters(u32 *counters) trace_mali_sw_counters(_mali_osk_get_pid(), _mali_osk_get_tid(), NULL, counters); } -void _mali_osk_profiling_memory_usage_get(u32 *memory_usage) +void _mali_osk_profiling_record_global_counters(int counter_id, u32 value) { - *memory_usage = _mali_ukk_report_memory_usage(); + if (NULL != global_mali_profiling_counters) { + int i ; + for (i = 0; i < num_global_mali_profiling_counters; i++) { + if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { + global_mali_profiling_counters[i].current_counter_value = value; + break; + } + } + } } _mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args) @@ -61,12 +822,271 @@ _mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s * _mali_osk_profiling_report_sw_counters(counters); + if (NULL != global_mali_profiling_counters) { + int i; + for (i = 0; i < MALI_PROFILING_SW_COUNTERS_NUM; i ++) { + if (global_mali_profiling_counters[first_sw_counter_index + i].enabled) { + global_mali_profiling_counters[first_sw_counter_index + i].current_counter_value = *(counters + i); + } + } + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + MALI_DEBUG_ASSERT_POINTER(session); + + if (1 == _mali_osk_atomic_inc_return(&stream_fd_if_used)) { + + s32 fd = anon_inode_getfd("[mali_profiling_stream]", &mali_profiling_stream_fops, + session, + O_RDONLY | O_CLOEXEC); + + args->stream_fd = fd; + if (0 > fd) { + _mali_osk_atomic_dec(&stream_fd_if_used); + return _MALI_OSK_ERR_FAULT; + } + args->stream_fd = fd; + } else { + _mali_osk_atomic_dec(&stream_fd_if_used); + args->stream_fd = -1; + return _MALI_OSK_ERR_BUSY; + } + return _MALI_OSK_ERR_OK; } -_mali_osk_errcode_t _mali_ukk_profiling_memory_usage_get(_mali_uk_profiling_memory_usage_get_s *args) +_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args) { - _mali_osk_profiling_memory_usage_get(&args->memory_usage); + u32 control_packet_size; + u32 output_buffer_size; + + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL == global_mali_profiling_counters && MALI_FALSE == _mali_profiling_global_counters_init()) { + MALI_PRINT_ERROR(("Failed to create global_mali_profiling_counters.\n")); + return _MALI_OSK_ERR_FAULT; + } + + control_packet_size = args->control_packet_size; + output_buffer_size = args->response_packet_size; + + if (0 != control_packet_size) { + u8 control_type; + u8 *control_packet_data; + u8 *response_packet_data; + u32 version_length = sizeof(utgard_setup_version) - 1; + + control_packet_data = (u8 *)(uintptr_t)args->control_packet_data; + MALI_DEBUG_ASSERT_POINTER(control_packet_data); + response_packet_data = (u8 *)(uintptr_t)args->response_packet_data; + MALI_DEBUG_ASSERT_POINTER(response_packet_data); + + /*Decide if need to ignore Utgard setup version.*/ + if (control_packet_size >= version_length) { + if (0 == memcmp(control_packet_data, utgard_setup_version, version_length)) { + if (control_packet_size == version_length) { + args->response_packet_size = 0; + return _MALI_OSK_ERR_OK; + } else { + control_packet_data += version_length; + control_packet_size -= version_length; + } + } + } + + current_profiling_pid = _mali_osk_get_pid(); + + control_type = control_packet_data[0]; + switch (control_type) { + case PACKET_HEADER_COUNTERS_REQUEST: { + int i; + + if (PACKET_HEADER_SIZE > control_packet_size || + control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { + MALI_PRINT_ERROR(("Wrong control packet size, type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); + return _MALI_OSK_ERR_FAULT; + } + + /* Send supported counters */ + *response_packet_data = PACKET_HEADER_COUNTERS_ACK; + args->response_packet_size = PACKET_HEADER_SIZE; + + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + u32 name_size = strlen(global_mali_profiling_counters[i].counter_name); + + if ((args->response_packet_size + name_size + 1) > output_buffer_size) { + MALI_PRINT_ERROR(("Response packet data is too large..\n")); + return _MALI_OSK_ERR_FAULT; + } + + memcpy(response_packet_data + args->response_packet_size, + global_mali_profiling_counters[i].counter_name, name_size + 1); + + args->response_packet_size += (name_size + 1); + + if (global_mali_profiling_counters[i].counter_id == COUNTER_VP_ACTIVITY) { + args->response_packet_size += _mali_profiling_pack_int(response_packet_data, + output_buffer_size, args->response_packet_size, (s32)1); + } else if (global_mali_profiling_counters[i].counter_id == COUNTER_FP_ACTIVITY) { + args->response_packet_size += _mali_profiling_pack_int(response_packet_data, + output_buffer_size, args->response_packet_size, (s32)mali_pp_get_glob_num_pp_cores()); + } else { + args->response_packet_size += _mali_profiling_pack_int(response_packet_data, + output_buffer_size, args->response_packet_size, (s32) - 1); + } + } + + _mali_profiling_set_packet_size(response_packet_data + 1, args->response_packet_size); + break; + } + + case PACKET_HEADER_COUNTERS_ENABLE: { + int i; + u32 request_pos = PACKET_HEADER_SIZE; + mali_bool sw_counter_if_enabled = MALI_FALSE; + + if (PACKET_HEADER_SIZE > control_packet_size || + control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { + MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); + return _MALI_OSK_ERR_FAULT; + } + + /* Init all counter states before enable requested counters.*/ + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); + global_mali_profiling_counters[i].enabled = 0; + global_mali_profiling_counters[i].prev_counter_value = 0; + global_mali_profiling_counters[i].current_counter_value = 0; + + if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && + global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { + _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, 0, 0); + } + } + + l2_cache_counter_if_enabled = MALI_FALSE; + num_counters_enabled = 0; + mem_counters_enabled = 0; + _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); + _mali_profiling_control(SW_COUNTER_ENABLE, 0); + _mali_profiling_notification_enable(session, 0, 0); + + /* Enable requested counters */ + while (request_pos < control_packet_size) { + u32 begin = request_pos; + u32 event; + u32 key; + + while (request_pos < control_packet_size && control_packet_data[request_pos] != '\0') { + ++request_pos; + } + + ++request_pos; + event = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); + key = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); + + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + u32 name_size = strlen((char *)(control_packet_data + begin)); + if (strncmp(global_mali_profiling_counters[i].counter_name, (char *)(control_packet_data + begin), name_size) == 0) { + if (!sw_counter_if_enabled && (FIRST_SW_COUNTER <= global_mali_profiling_counters[i].counter_id + && global_mali_profiling_counters[i].counter_id <= LAST_SW_COUNTER)) { + sw_counter_if_enabled = MALI_TRUE; + _mali_profiling_control(SW_COUNTER_ENABLE, 1); + } + + if (COUNTER_FILMSTRIP == global_mali_profiling_counters[i].counter_id) { + _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 1); + _mali_profiling_control(FBDUMP_CONTROL_RATE, event & 0xff); + _mali_profiling_control(FBDUMP_CONTROL_RESIZE_FACTOR, (event >> 8) & 0xff); + } + + if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && + global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { + _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, + key, 1); + mem_counters_enabled++; + } + + global_mali_profiling_counters[i].counter_event = event; + global_mali_profiling_counters[i].key = key; + global_mali_profiling_counters[i].enabled = 1; + + _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, + global_mali_profiling_counters[i].counter_event); + num_counters_enabled++; + break; + } + } + + if (i == num_global_mali_profiling_counters) { + MALI_PRINT_ERROR(("Counter name does not match for type %u.\n", control_type)); + return _MALI_OSK_ERR_FAULT; + } + } + + if (PACKET_HEADER_SIZE <= output_buffer_size) { + *response_packet_data = PACKET_HEADER_ACK; + _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); + args->response_packet_size = PACKET_HEADER_SIZE; + } else { + return _MALI_OSK_ERR_FAULT; + } + + break; + } + + case PACKET_HEADER_START_CAPTURE_VALUE: { + u32 live_rate; + u32 request_pos = PACKET_HEADER_SIZE; + + if (PACKET_HEADER_SIZE > control_packet_size || + control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { + MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); + return _MALI_OSK_ERR_FAULT; + } + + /* Read samping rate in nanoseconds and live rate, start capture.*/ + profiling_sample_rate = _mali_profiling_read_packet_int(control_packet_data, + &request_pos, control_packet_size); + + live_rate = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); + + if (PACKET_HEADER_SIZE <= output_buffer_size) { + *response_packet_data = PACKET_HEADER_ACK; + _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); + args->response_packet_size = PACKET_HEADER_SIZE; + } else { + return _MALI_OSK_ERR_FAULT; + } + + if (0 != num_counters_enabled && 0 != profiling_sample_rate) { + _mali_profiling_global_stream_list_free(); + if (mem_counters_enabled > 0) { + _mali_profiling_notification_enable(session, profiling_sample_rate, 1); + } + hrtimer_start(&profiling_sampling_timer, + ktime_set(profiling_sample_rate / 1000000000, profiling_sample_rate % 1000000000), + HRTIMER_MODE_REL_PINNED); + } + + break; + } + default: + MALI_PRINT_ERROR(("Unsupported profiling packet header type %u.\n", control_type)); + args->response_packet_size = 0; + return _MALI_OSK_ERR_FAULT; + } + } else { + _mali_osk_profiling_stop_sampling(current_profiling_pid); + _mali_profiling_notification_enable(session, 0, 0); + } + return _MALI_OSK_ERR_OK; } @@ -157,6 +1177,7 @@ int _mali_profiling_set_event(u32 counter_id, s32 event_id) u32 counter_src = (counter_id - COUNTER_L2_0_C0) & 1; mali_l2_cache_core_set_counter_src(l2_cache_core, counter_src, event_id); + l2_cache_counter_if_enabled = MALI_TRUE; } } else { return 0; /* Failure, unknown event */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h b/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h index 001650ce63c7..db034a5b3c70 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c index 83c438100a38..4deaa101e48f 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c index 8b7c40ee5c00..6bbaee749d64 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c index d14b7d5f19c3..15d5ce250eb1 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c index 64af192e48a4..2c34c91a7922 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c b/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c index c8706d28983d..61ff5c8fdca8 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h index e41323c51e66..0b90e8c5cf26 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2012, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h index bb2a37b80425..c98d127366ba 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c index 83ace8fb9d62..12aef4194ff5 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -104,6 +104,7 @@ _mali_osk_errcode_t _mali_internal_profiling_start(u32 *limit) new_profile_entries = _mali_osk_valloc(*limit * sizeof(mali_profiling_entry)); if (NULL == new_profile_entries) { + _mali_osk_mutex_signal(lock); _mali_osk_vfree(new_profile_entries); return _MALI_OSK_ERR_NOMEM; } diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h index db6380df5c3b..1c6f4da691d2 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_sync.c b/drivers/gpu/arm/mali400/mali/linux/mali_sync.c index 072231a44f04..68868ca5b6bd 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_sync.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_sync.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -13,6 +13,7 @@ #include "mali_osk.h" #include "mali_kernel_common.h" #include "mali_timeline.h" +#include "mali_executor.h" #include #include @@ -120,6 +121,24 @@ static void timeline_free_pt(struct sync_pt *pt) static void timeline_release(struct sync_timeline *sync_timeline) { + struct mali_sync_timeline_container *mali_sync_tl = NULL; + struct mali_timeline *mali_tl = NULL; + + MALI_DEBUG_ASSERT_POINTER(sync_timeline); + + mali_sync_tl = to_mali_sync_tl_container(sync_timeline); + MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); + + mali_tl = mali_sync_tl->timeline; + + /* always signaled timeline didn't have mali container */ + if (mali_tl) { + if (NULL != mali_tl->spinlock) { + mali_spinlock_reentrant_term(mali_tl->spinlock); + } + _mali_osk_free(mali_tl); + } + module_put(THIS_MODULE); } @@ -143,6 +162,42 @@ static void timeline_print_pt(struct seq_file *s, struct sync_pt *sync_pt) } } +static void timeline_print_obj(struct seq_file *s, struct sync_timeline *sync_tl) +{ + struct mali_sync_timeline_container *mali_sync_tl = NULL; + struct mali_timeline *mali_tl = NULL; + + MALI_DEBUG_ASSERT_POINTER(sync_tl); + + mali_sync_tl = to_mali_sync_tl_container(sync_tl); + MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); + + mali_tl = mali_sync_tl->timeline; + + if (NULL != mali_tl) { + seq_printf(s, "oldest (%u) ", mali_tl->point_oldest); + seq_printf(s, "next (%u)", mali_tl->point_next); + seq_printf(s, "\n"); + +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + { + u32 tid = _mali_osk_get_tid(); + struct mali_timeline_system *system = mali_tl->system; + + mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); + if (!mali_tl->destroyed) { + mali_spinlock_reentrant_wait(system->spinlock, tid); + mali_timeline_debug_print_timeline(mali_tl, s); + mali_spinlock_reentrant_signal(system->spinlock, tid); + } + mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); + + /* dump job queue status and group running status */ + mali_executor_status_dump(); + } +#endif + } +} #else static void timeline_pt_value_str(struct sync_pt *pt, char *str, int size) { @@ -165,22 +220,43 @@ static void timeline_pt_value_str(struct sync_pt *pt, char *str, int size) static void timeline_value_str(struct sync_timeline *timeline, char *str, int size) { - struct mali_sync_timeline_container *mali_sync_tl; + struct mali_sync_timeline_container *mali_sync_tl = NULL; + struct mali_timeline *mali_tl = NULL; MALI_DEBUG_ASSERT_POINTER(timeline); - MALI_DEBUG_ASSERT_POINTER(str); mali_sync_tl = to_mali_sync_tl_container(timeline); - MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); - if (NULL != mali_sync_tl->timeline) { - _mali_osk_snprintf(str, size, "oldest (%u) next (%u)\n", mali_sync_tl->timeline->point_oldest, - mali_sync_tl->timeline->point_next); + mali_tl = mali_sync_tl->timeline; + + if (NULL != mali_tl) { + _mali_osk_snprintf(str, size, "oldest (%u) ", mali_tl->point_oldest); + _mali_osk_snprintf(str, size, "next (%u)", mali_tl->point_next); + _mali_osk_snprintf(str, size, "\n"); + +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + { + u32 tid = _mali_osk_get_tid(); + struct mali_timeline_system *system = mali_tl->system; + + mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); + if (!mali_tl->destroyed) { + mali_spinlock_reentrant_wait(system->spinlock, tid); + mali_timeline_debug_direct_print_timeline(mali_tl); + mali_spinlock_reentrant_signal(system->spinlock, tid); + } + mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); + + /* dump job queue status and group running status */ + mali_executor_status_dump(); + } +#endif } } #endif + static struct sync_timeline_ops mali_timeline_ops = { .driver_name = "Mali", .dup = timeline_dup, @@ -190,6 +266,7 @@ static struct sync_timeline_ops mali_timeline_ops = { .release_obj = timeline_release, #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) .print_pt = timeline_print_pt, + .print_obj = timeline_print_obj, #else .pt_value_str = timeline_pt_value_str, .timeline_value_str = timeline_value_str, @@ -217,12 +294,6 @@ struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, return sync_tl; } -mali_bool mali_sync_timeline_is_ours(struct sync_timeline *sync_tl) -{ - MALI_DEBUG_ASSERT_POINTER(sync_tl); - return (sync_tl->ops == &mali_timeline_ops) ? MALI_TRUE : MALI_FALSE; -} - s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence) { s32 fd = -1; diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_sync.h b/drivers/gpu/arm/mali400/mali/linux/mali_sync.h index 2dd6f570a42b..0c541ff9a2e0 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_sync.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_sync.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -41,14 +41,6 @@ struct mali_timeline; */ struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name); -/** - * Check if sync timeline belongs to Mali. - * - * @param sync_tl Sync timeline to check. - * @return MALI_TRUE if sync timeline belongs to Mali, MALI_FALSE if not. - */ -mali_bool mali_sync_timeline_is_ours(struct sync_timeline *sync_tl); - /** * Creates a file descriptor representing the sync fence. Will release sync fence if allocation of * file descriptor fails. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h b/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h index c7e2ef58f381..1884cdbba424 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2012, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c index e8f27201321d..4ca85a8273e6 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -155,3 +155,17 @@ int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_ return map_errcode(err); } + +int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs) +{ + _mali_uk_pending_submit_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_pending_submit(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c index a8b5163b99c5..d4144c0f5e48 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c index 6edf9daa76be..ca1cba0585b3 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -16,159 +16,216 @@ #include "mali_session.h" #include "mali_ukk_wrappers.h" -int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs) +int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs) { - _mali_uk_mem_write_safe_s kargs; + _mali_uk_alloc_mem_s kargs; _mali_osk_errcode_t err; MALI_CHECK_NON_NULL(uargs, -EINVAL); MALI_CHECK_NON_NULL(session_data, -EINVAL); - if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_write_safe_s))) { + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_alloc_mem_s))) { return -EFAULT; } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_allocate(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { + return -EFAULT; + } + + return 0; +} + +int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs) +{ + _mali_uk_free_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_free_mem_s))) { + return -EFAULT; + } kargs.ctx = (uintptr_t)session_data; - /* Check if we can access the buffers */ - if (!access_ok(VERIFY_WRITE, kargs.dest, kargs.size) - || !access_ok(VERIFY_READ, kargs.src, kargs.size)) { - return -EINVAL; + err = _mali_ukk_mem_free(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); } - /* Check if size wraps */ - if ((kargs.size + kargs.dest) <= kargs.dest - || (kargs.size + kargs.src) <= kargs.src) { - return -EINVAL; + if (0 != put_user(kargs.free_pages_nr, &uargs->free_pages_nr)) { + return -EFAULT; } - err = _mali_ukk_mem_write_safe(&kargs); + return 0; +} + +int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs) +{ + _mali_uk_bind_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_bind_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_bind(&kargs); + if (_MALI_OSK_ERR_OK != err) { return map_errcode(err); } - if (0 != put_user(kargs.size, &uargs->size)) { + return 0; +} + +int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs) +{ + _mali_uk_unbind_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_unbind_mem_s))) { return -EFAULT; } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_unbind(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } return 0; } -int mem_map_ext_wrapper(struct mali_session_data *session_data, _mali_uk_map_external_mem_s __user *argument) + +int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs) { - _mali_uk_map_external_mem_s uk_args; - _mali_osk_errcode_t err_code; + _mali_uk_cow_mem_s kargs; + _mali_osk_errcode_t err; - /* validate input */ - /* the session_data pointer was validated by caller */ - MALI_CHECK_NON_NULL(argument, -EINVAL); + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); - /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ - if (0 != copy_from_user(&uk_args, (void __user *)argument, sizeof(_mali_uk_map_external_mem_s))) { + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_mem_s))) { return -EFAULT; } + kargs.ctx = (uintptr_t)session_data; - uk_args.ctx = (uintptr_t)session_data; - err_code = _mali_ukk_map_external_mem(&uk_args); + err = _mali_ukk_mem_cow(&kargs); - if (0 != put_user(uk_args.cookie, &argument->cookie)) { - if (_MALI_OSK_ERR_OK == err_code) { - /* Rollback */ - _mali_uk_unmap_external_mem_s uk_args_unmap; + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } - uk_args_unmap.ctx = (uintptr_t)session_data; - uk_args_unmap.cookie = uk_args.cookie; - err_code = _mali_ukk_unmap_external_mem(&uk_args_unmap); - if (_MALI_OSK_ERR_OK != err_code) { - MALI_DEBUG_PRINT(4, ("reverting _mali_ukk_unmap_external_mem, as a result of failing put_user(), failed\n")); - } - } + if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { return -EFAULT; } - /* Return the error that _mali_ukk_free_big_block produced */ - return map_errcode(err_code); + return 0; } -int mem_unmap_ext_wrapper(struct mali_session_data *session_data, _mali_uk_unmap_external_mem_s __user *argument) +int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs) { - _mali_uk_unmap_external_mem_s uk_args; - _mali_osk_errcode_t err_code; + _mali_uk_cow_modify_range_s kargs; + _mali_osk_errcode_t err; - /* validate input */ - /* the session_data pointer was validated by caller */ - MALI_CHECK_NON_NULL(argument, -EINVAL); + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); - /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ - if (0 != copy_from_user(&uk_args, (void __user *)argument, sizeof(_mali_uk_unmap_external_mem_s))) { + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_modify_range_s))) { return -EFAULT; } + kargs.ctx = (uintptr_t)session_data; - uk_args.ctx = (uintptr_t)session_data; - err_code = _mali_ukk_unmap_external_mem(&uk_args); + err = _mali_ukk_mem_cow_modify_range(&kargs); - /* Return the error that _mali_ukk_free_big_block produced */ - return map_errcode(err_code); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.change_pages_nr, &uargs->change_pages_nr)) { + return -EFAULT; + } + return 0; } -#if defined(CONFIG_MALI400_UMP) -int mem_release_ump_wrapper(struct mali_session_data *session_data, _mali_uk_release_ump_mem_s __user *argument) + +int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs) { - _mali_uk_release_ump_mem_s uk_args; - _mali_osk_errcode_t err_code; + _mali_uk_mem_resize_s kargs; + _mali_osk_errcode_t err; - /* validate input */ - /* the session_data pointer was validated by caller */ - MALI_CHECK_NON_NULL(argument, -EINVAL); + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); - /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ - if (0 != copy_from_user(&uk_args, (void __user *)argument, sizeof(_mali_uk_release_ump_mem_s))) { + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_resize_s))) { return -EFAULT; } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_resize(&kargs); - uk_args.ctx = (uintptr_t)session_data; - err_code = _mali_ukk_release_ump_mem(&uk_args); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } - /* Return the error that _mali_ukk_free_big_block produced */ - return map_errcode(err_code); + return 0; } -int mem_attach_ump_wrapper(struct mali_session_data *session_data, _mali_uk_attach_ump_mem_s __user *argument) +int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs) { - _mali_uk_attach_ump_mem_s uk_args; - _mali_osk_errcode_t err_code; + _mali_uk_mem_write_safe_s kargs; + _mali_osk_errcode_t err; - /* validate input */ - /* the session_data pointer was validated by caller */ - MALI_CHECK_NON_NULL(argument, -EINVAL); + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); - /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ - if (0 != copy_from_user(&uk_args, (void __user *)argument, sizeof(_mali_uk_attach_ump_mem_s))) { + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_write_safe_s))) { return -EFAULT; } - uk_args.ctx = (uintptr_t)session_data; - err_code = _mali_ukk_attach_ump_mem(&uk_args); + kargs.ctx = (uintptr_t)session_data; + + /* Check if we can access the buffers */ + if (!access_ok(VERIFY_WRITE, kargs.dest, kargs.size) + || !access_ok(VERIFY_READ, kargs.src, kargs.size)) { + return -EINVAL; + } + + /* Check if size wraps */ + if ((kargs.size + kargs.dest) <= kargs.dest + || (kargs.size + kargs.src) <= kargs.src) { + return -EINVAL; + } - if (0 != put_user(uk_args.cookie, &argument->cookie)) { - if (_MALI_OSK_ERR_OK == err_code) { - /* Rollback */ - _mali_uk_release_ump_mem_s uk_args_unmap; + err = _mali_ukk_mem_write_safe(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } - uk_args_unmap.ctx = (uintptr_t)session_data; - uk_args_unmap.cookie = uk_args.cookie; - err_code = _mali_ukk_release_ump_mem(&uk_args_unmap); - if (_MALI_OSK_ERR_OK != err_code) { - MALI_DEBUG_PRINT(4, ("reverting _mali_ukk_attach_mem, as a result of failing put_user(), failed\n")); - } - } + if (0 != put_user(kargs.size, &uargs->size)) { return -EFAULT; } - /* Return the error that _mali_ukk_map_external_ump_mem produced */ - return map_errcode(err_code); + return 0; } -#endif /* CONFIG_MALI400_UMP */ + + int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs) { @@ -247,3 +304,30 @@ err_exit: if (buffer) _mali_osk_vfree(buffer); return rc; } + +int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs) +{ + _mali_osk_errcode_t err; + _mali_uk_profiling_memory_usage_get_s kargs; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_mem_usage_get(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { + return -EFAULT; + } + + return 0; +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c index 7053ff36a59b..4c1c381cb90f 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c index e341f633bda4..e84544dee07d 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -37,28 +37,6 @@ int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk return 0; } -int profiling_memory_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs) -{ - _mali_osk_errcode_t err; - _mali_uk_profiling_memory_usage_get_s kargs; - - MALI_CHECK_NON_NULL(uargs, -EINVAL); - MALI_CHECK_NON_NULL(session_data, -EINVAL); - - kargs.ctx = (uintptr_t)session_data; - err = _mali_ukk_profiling_memory_usage_get(&kargs); - if (_MALI_OSK_ERR_OK != err) { - return map_errcode(err); - } - - kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ - if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { - return -EFAULT; - } - - return 0; -} - int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs) { _mali_uk_sw_counters_report_s kargs; @@ -103,3 +81,97 @@ int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, return 0; } + +int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs) +{ + _mali_uk_profiling_stream_fd_get_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_profiling_stream_fd_get(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { + return -EFAULT; + } + + return 0; +} + +int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs) +{ + _mali_uk_profiling_control_set_s kargs; + _mali_osk_errcode_t err; + u8 *kernel_control_data = NULL; + u8 *kernel_response_data = NULL; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != get_user(kargs.control_packet_size, &uargs->control_packet_size)) return -EFAULT; + if (0 != get_user(kargs.response_packet_size, &uargs->response_packet_size)) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + + if (0 != kargs.control_packet_size) { + + kernel_control_data = _mali_osk_calloc(1, kargs.control_packet_size); + if (NULL == kernel_control_data) { + return -ENOMEM; + } + + MALI_DEBUG_ASSERT(0 != kargs.response_packet_size); + + kernel_response_data = _mali_osk_calloc(1, kargs.response_packet_size); + if (NULL == kernel_response_data) { + _mali_osk_free(kernel_control_data); + return -ENOMEM; + } + + kargs.control_packet_data = (uintptr_t)kernel_control_data; + kargs.response_packet_data = (uintptr_t)kernel_response_data; + + if (0 != copy_from_user((void *)(uintptr_t)kernel_control_data, (void *)(uintptr_t)uargs->control_packet_data, kargs.control_packet_size)) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return -EFAULT; + } + + err = _mali_ukk_profiling_control_set(&kargs); + if (_MALI_OSK_ERR_OK != err) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return map_errcode(err); + } + + if (0 != kargs.response_packet_size && 0 != copy_to_user(((void *)(uintptr_t)uargs->response_packet_data), ((void *)(uintptr_t)kargs.response_packet_data), kargs.response_packet_size)) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return -EFAULT; + } + + if (0 != put_user(kargs.response_packet_size, &uargs->response_packet_size)) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return -EFAULT; + } + + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + } else { + + err = _mali_ukk_profiling_control_set(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + } + return 0; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c index c6b67c5f37f5..11c70060e489 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c index c8abbe4a1092..484d4041c869 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c index b0fa93bcf220..487c2478df4d 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2014 ARM Limited. All rights reserved. + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h index 21b66774a156..504ed4be4123 100755 --- a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -29,6 +29,7 @@ int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_ int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs); int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs); int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs); +int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs); /* rk_ext : 从对 r5p0-01rel0 集成开始, 不再使用. */ #if 0 @@ -37,23 +38,23 @@ int get_mali_version_in_rk30_wrapper(struct mali_session_data *session_data, _ma int get_rk_ko_version_wrapper(struct mali_session_data *session_data, _mali_rk_ko_version_s __user *uargs); #endif +int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs); +int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs); +int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs); +int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs); +int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs); +int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs); +int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs); int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs); -int mem_map_ext_wrapper(struct mali_session_data *session_data, _mali_uk_map_external_mem_s __user *argument); -int mem_unmap_ext_wrapper(struct mali_session_data *session_data, _mali_uk_unmap_external_mem_s __user *argument); int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs); int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs); +int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs); int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs); int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs); int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs); int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs); int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs); - -#if defined(CONFIG_MALI400_UMP) -int mem_attach_ump_wrapper(struct mali_session_data *session_data, _mali_uk_attach_ump_mem_s __user *argument); -int mem_release_ump_wrapper(struct mali_session_data *session_data, _mali_uk_release_ump_mem_s __user *argument); -#endif - int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs); int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs); int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs); @@ -66,7 +67,8 @@ int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs); int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs); -int profiling_memory_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs); +int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs); +int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs); int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs); diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm.c b/drivers/gpu/arm/mali400/mali/platform/arm/arm.c index 32476834f025..41ad63c0793b 100755 --- a/drivers/gpu/arm/mali400/mali/platform/arm/arm.c +++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -49,6 +49,22 @@ static struct resource mali_gpu_resources_m450_mp6[] = { MALI_GPU_RESOURCES_MALI450_MP6_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) }; +static struct resource mali_gpu_resources_m470_mp4[] = { + MALI_GPU_RESOURCES_MALI470_MP4_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp3[] = { + MALI_GPU_RESOURCES_MALI470_MP3_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp2[] = { + MALI_GPU_RESOURCES_MALI470_MP2_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp1[] = { + MALI_GPU_RESOURCES_MALI470_MP1_PMU(0x6F040000, 200, 200, 200, 200, 200) +}; + #else static struct resource mali_gpu_resources_m450_mp8[] = { MALI_GPU_RESOURCES_MALI450_MP8_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) @@ -61,6 +77,10 @@ static struct resource mali_gpu_resources_m450_mp6[] = { static struct resource mali_gpu_resources_m450_mp4[] = { MALI_GPU_RESOURCES_MALI450_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) }; + +static struct resource mali_gpu_resources_m470_mp4[] = { + MALI_GPU_RESOURCES_MALI470_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) +}; #endif /* CONFIG_ARM64 */ #elif defined(CONFIG_ARCH_REALVIEW) @@ -94,8 +114,11 @@ static struct mali_gpu_device_data mali_gpu_data = { .dedicated_mem_size = 0x10000000, /* 256MB */ #endif #if defined(CONFIG_ARM64) - .fb_start = 0x5f000000, - .fb_size = 0x91000000, + /* Some framebuffer drivers get the framebuffer dynamically, such as through GEM, + * in which the memory resource can't be predicted in advance. + */ + .fb_start = 0x0, + .fb_size = 0xFFFFF000, #else .fb_start = 0xe0000000, .fb_size = 0x01000000, @@ -116,9 +139,6 @@ static struct platform_device mali_gpu_device = { .dev.coherent_dma_mask = DMA_BIT_MASK(32), .dev.platform_data = &mali_gpu_data, -#if defined(CONFIG_ARM64) - .dev.archdata.dma_ops = &noncoherent_swiotlb_dma_ops, -#endif }; int mali_platform_device_register(void) @@ -135,11 +155,32 @@ int mali_platform_device_register(void) #if defined(CONFIG_ARCH_VEXPRESS) #if defined(CONFIG_ARM64) - if (mali_read_phys(0x6F000000) == 0x40601450) { + mali_gpu_device.dev.archdata.dma_ops = dma_ops; + if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); num_pp_cores = 6; mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp6); mali_gpu_device.resource = mali_gpu_resources_m450_mp6; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); + mali_gpu_device.resource = mali_gpu_resources_m470_mp4; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); + num_pp_cores = 3; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp3); + mali_gpu_device.resource = mali_gpu_resources_m470_mp3; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); + num_pp_cores = 2; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp2); + mali_gpu_device.resource = mali_gpu_resources_m470_mp2; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); + num_pp_cores = 1; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp1); + mali_gpu_device.resource = mali_gpu_resources_m470_mp1; } #else if (mali_read_phys(0xFC000000) == 0x00000450) { @@ -157,6 +198,11 @@ int mali_platform_device_register(void) num_pp_cores = 4; mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp4); mali_gpu_device.resource = mali_gpu_resources_m450_mp4; + } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); + mali_gpu_device.resource = mali_gpu_resources_m470_mp4; } #endif /* CONFIG_ARM64 */ @@ -230,7 +276,7 @@ static void mali_platform_device_release(struct device *device) #else /* CONFIG_MALI_DT */ int mali_platform_device_init(struct platform_device *device) { - int num_pp_cores; + int num_pp_cores = 0; int err = -1; #if defined(CONFIG_ARCH_REALVIEW) u32 m400_gp_version; @@ -240,43 +286,70 @@ int mali_platform_device_init(struct platform_device *device) #if defined(CONFIG_ARCH_VEXPRESS) #if defined(CONFIG_ARM64) - if (mali_read_phys(0x6F000000) == 0x40601450) { - MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); - num_pp_cores = 6; - } + if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); + num_pp_cores = 6; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); + num_pp_cores = 3; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); + num_pp_cores = 2; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); + num_pp_cores = 1; + } #else - if (mali_read_phys(0xFC000000) == 0x00000450) { - MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); - num_pp_cores = 8; - } else if (mali_read_phys(0xFC000000) == 0x40400450) { - MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); - num_pp_cores = 4; - } + if (mali_read_phys(0xFC000000) == 0x00000450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); + num_pp_cores = 8; + } else if (mali_read_phys(0xFC000000) == 0x40400450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); + num_pp_cores = 4; + } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + } #endif #elif defined(CONFIG_ARCH_REALVIEW) - m400_gp_version = mali_read_phys(0xC000006C); - if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { - MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); + m400_gp_version = mali_read_phys(0xC000006C); + if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { + MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); + num_pp_cores = 1; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { + u32 fpga_fw_version = mali_read_phys(0xC0010000); + if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { + /* Mali-400 MP1 r1p0 or r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); num_pp_cores = 1; mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ - } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { - u32 fpga_fw_version = mali_read_phys(0xC0010000); - if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { - /* Mali-400 MP1 r1p0 or r1p1 */ - MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); - num_pp_cores = 1; - mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ - } else if (fpga_fw_version == 0x130C000F) { - /* Mali-400 MP2 r1p1 */ - MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); - num_pp_cores = 2; - mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ - } + } else if (fpga_fw_version == 0x130C000F) { + /* Mali-400 MP2 r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); + num_pp_cores = 2; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ } + } #endif + /* After kernel 3.15 device tree will default set dev + * related parameters in of_platform_device_create_pdata. + * But kernel changes from version to version, + * For example 3.10 didn't include device->dev.dma_mask parameter setting, + * if we didn't include here will cause dma_mapping error, + * but in kernel 3.15 it include device->dev.dma_mask parameter setting, + * so it's better to set must need paramter by DDK itself. + */ + if (!device->dev.dma_mask) + device->dev.dma_mask = &device->dev.coherent_dma_mask; + device->dev.archdata.dma_ops = dma_ops; + err = platform_device_add_data(device, &mali_gpu_data, sizeof(mali_gpu_data)); if (0 == err) { diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c index f846163a7604..2c24742eb4de 100755 --- a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c +++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h index 9e984b840f77..325b5b1c6894 100755 --- a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h +++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h b/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h index 22806db1dd18..e76f9926f5c0 100755 --- a/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h +++ b/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -25,12 +25,11 @@ enum mali200_mgmt_reg { MALI200_REG_ADDR_MGMT_INT_MASK = 0x1028, MALI200_REG_ADDR_MGMT_INT_STATUS = 0x102c, - MALI200_REG_ADDR_MGMT_WRITE_BOUNDARY_LOW = 0x1044, - MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS = 0x1050, MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x1080, MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x1084, + MALI200_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x1088, MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x108c, MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x10a0, @@ -125,6 +124,8 @@ enum mali200_wb_unit_regs { #define MALI300_PP_PRODUCT_ID 0xCE07 #define MALI400_PP_PRODUCT_ID 0xCD07 #define MALI450_PP_PRODUCT_ID 0xCF07 +#define MALI470_PP_PRODUCT_ID 0xCF08 + #endif /* _MALI200_REGS_H_ */ diff --git a/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h b/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h index 7c9b5c80872f..9c101f9ddd22 100755 --- a/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h +++ b/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -31,13 +31,13 @@ typedef enum { MALIGP2_REG_ADDR_MGMT_INT_CLEAR = 0x28, MALIGP2_REG_ADDR_MGMT_INT_MASK = 0x2C, MALIGP2_REG_ADDR_MGMT_INT_STAT = 0x30, - MALIGP2_REG_ADDR_MGMT_WRITE_BOUND_LOW = 0x34, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x3C, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x40, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x44, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x48, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x4C, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x50, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x54, MALIGP2_REG_ADDR_MGMT_STATUS = 0x68, MALIGP2_REG_ADDR_MGMT_VERSION = 0x6C, MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR_READ = 0x80, diff --git a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c index d1c3e9968397..a486e2f7684f 100755 --- a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c +++ b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h index f0bd599f7c8d..65f3ab274c09 100755 --- a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h +++ b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c index d1c3e9968397..a486e2f7684f 100755 --- a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c +++ b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h index f7de0eb6a3ae..8ba47060828f 100755 --- a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h +++ b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/Makefile b/drivers/gpu/arm/mali400/ump/Makefile index e2ad9495a880..f24bc6c82a0f 100755 --- a/drivers/gpu/arm/mali400/ump/Makefile +++ b/drivers/gpu/arm/mali400/ump/Makefile @@ -1,5 +1,5 @@ # -# Copyright (C) 2010-2012, 2014 ARM Limited. All rights reserved. +# Copyright (C) 2010-2012, 2014-2015 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the GNU General Public License version 2 # as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/Makefile.common b/drivers/gpu/arm/mali400/ump/Makefile.common index 9bd2583c5180..6b3b8d1efde4 100755 --- a/drivers/gpu/arm/mali400/ump/Makefile.common +++ b/drivers/gpu/arm/mali400/ump/Makefile.common @@ -1,5 +1,5 @@ # -# Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. +# Copyright (C) 2010-2011, 2013, 2015 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the GNU General Public License version 2 # as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/arch-default/config.h b/drivers/gpu/arm/mali400/ump/arch-default/config.h index 54d29dfc9ded..cfd2b34b5334 100755 --- a/drivers/gpu/arm/mali400/ump/arch-default/config.h +++ b/drivers/gpu/arm/mali400/ump/arch-default/config.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012, 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h b/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h index 99679cd13fbf..c2c56cf1c009 100755 --- a/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h +++ b/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c index 398781dbebfe..21dde81ef419 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c index 3a9dfe8664b8..ca410a8916c0 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h index efc6c3f68c6e..2efe1ed1e789 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c index c89324e608ce..45e2bb4f7bd7 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h index 160e20ec56a7..60d691739c08 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h index 4fa4bda70c18..056244ebe162 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c index aa08c17a917f..37a682ac9ba4 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h index 7718956175ca..098ce1412738 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_osk.h b/drivers/gpu/arm/mali400/ump/common/ump_osk.h index 5759ddb719be..6fe73662c6ae 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_osk.h +++ b/drivers/gpu/arm/mali400/ump/common/ump_osk.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h b/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h index 48b588f8e2b7..d91b0707d936 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h +++ b/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/common/ump_ukk.h b/drivers/gpu/arm/mali400/ump/common/ump_ukk.h index da7917a893ca..ad097e8b209d 100755 --- a/drivers/gpu/arm/mali400/ump/common/ump_ukk.h +++ b/drivers/gpu/arm/mali400/ump/common/ump_ukk.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h b/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h index 567d803b312e..f562e5699e5f 100755 --- a/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h +++ b/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h b/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h index 239e8ab33db5..2a640e35c384 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h @@ -1,11 +1,21 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. * * A copy of the licence is included with the program, and can also be obtained from Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. */ #ifndef __UMP_IOCTL_H__ diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c index c99caa6dff54..77a5b21564c0 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h index c071b77b2cab..41dae7988c49 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c index 32ea8b0b070a..9f329e9524a1 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h index 5a5a4a315555..283dde793835 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c index 86a8132e3b23..763b6109e486 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h index 9ac3cb7a35d8..22666573bec7 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c index 95bcb95602a0..8fed5c38508a 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h index 7f89ef827b6f..d304157b1ffb 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2011, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c b/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c index b0915435e51b..3baa2ac4aa23 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c index 04934234ee73..9fae2953308a 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c index 1772120644d7..0d81fbe326b0 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c index 0f6829dfb67b..bd985d3afb72 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c index ed14987cb7bb..8350d3e0b75b 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h index c88b6665abbf..d0cdbb5281ff 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c index 49d58d797506..2616ebf2c81f 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h index e9110b7b153e..05000804af0a 100755 --- a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/umplock/Makefile b/drivers/gpu/arm/mali400/umplock/Makefile index 62593fc913bf..8b5a6d9be821 100755 --- a/drivers/gpu/arm/mali400/umplock/Makefile +++ b/drivers/gpu/arm/mali400/umplock/Makefile @@ -1,5 +1,5 @@ # -# Copyright (C) 2012, 2014 ARM Limited. All rights reserved. +# Copyright (C) 2012, 2015 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the GNU General Public License version 2 # as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. diff --git a/drivers/gpu/arm/mali400/umplock/umplock_driver.c b/drivers/gpu/arm/mali400/umplock/umplock_driver.c index 825f16fa8617..f117177fcc3e 100755 --- a/drivers/gpu/arm/mali400/umplock/umplock_driver.c +++ b/drivers/gpu/arm/mali400/umplock/umplock_driver.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. @@ -29,6 +29,7 @@ typedef struct lock_cmd_priv { typedef struct lock_ref { int ref_count; u32 pid; + u32 down_count; } _lock_ref; typedef struct umplock_item { @@ -141,6 +142,7 @@ static int do_umplock_create_locked(_lock_cmd_priv *lock_cmd) if (ref_index < MAX_PIDS) { device.items[i_index].references[ref_index].pid = lock_cmd->pid; device.items[i_index].references[ref_index].ref_count = 0; + device.items[i_index].references[ref_index].down_count = 0; } else { PERROR("whoops, item ran out of available reference slots\n"); return -EINVAL; @@ -155,6 +157,7 @@ static int do_umplock_create_locked(_lock_cmd_priv *lock_cmd) device.items[i_index].usage = lock_item->usage; device.items[i_index].references[0].pid = lock_cmd->pid; device.items[i_index].references[0].ref_count = 0; + device.items[i_index].references[0].down_count = 0; sema_init(&device.items[i_index].item_lock, 1); } else { PERROR("whoops, ran out of available slots\n"); @@ -207,12 +210,14 @@ static int do_umplock_process(_lock_cmd_priv *lock_cmd) return 0; } + device.items[i_index].references[ref_index].down_count++; mutex_unlock(&device.item_list_lock); if (down_interruptible(&device.items[i_index].item_lock)) { /*wait up without hold the umplock. restore previous state and return*/ mutex_lock(&device.item_list_lock); device.items[i_index].references[ref_index].ref_count--; device.items[i_index].id_ref_count--; + device.items[i_index].references[ref_index].down_count--; if (0 == device.items[i_index].references[ref_index].ref_count) { device.items[i_index].references[ref_index].pid = 0; if (0 == device.items[i_index].id_ref_count) { @@ -237,7 +242,7 @@ static int do_umplock_process(_lock_cmd_priv *lock_cmd) static int do_umplock_release(_lock_cmd_priv *lock_cmd) { - int ret, i_index, ref_index; + int ret, i_index, ref_index, call_up; _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; mutex_lock(&device.item_list_lock); @@ -281,6 +286,11 @@ static int do_umplock_release(_lock_cmd_priv *lock_cmd) device.items[i_index].id_ref_count--; PDEBUG(1, "unlock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); + call_up = 0; + if (device.items[i_index].references[ref_index].down_count > 1) { + call_up = 1; + device.items[i_index].references[ref_index].down_count--; + } if (0 == device.items[i_index].references[ref_index].ref_count) { device.items[i_index].references[ref_index].pid = 0; if (0 == device.items[i_index].id_ref_count) { @@ -288,6 +298,10 @@ static int do_umplock_release(_lock_cmd_priv *lock_cmd) device.items[i_index].secure_id = 0; } device.items[i_index].owner = 0; + call_up = 1; + } + if (call_up) { + PDEBUG(1, "call up, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); up(&device.items[i_index].item_lock); } mutex_unlock(&device.item_list_lock); diff --git a/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h b/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h index 9ac5b7f25a6e..f7d8176a7ed9 100755 --- a/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h +++ b/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h @@ -1,11 +1,21 @@ /* - * Copyright (C) 2012-2014 ARM Limited. All rights reserved. + * Copyright (C) 2012-2013, 2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. * * A copy of the licence is included with the program, and can also be obtained from Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. */ #ifndef __UMPLOCK_IOCTL_H__