amdkfd: Add binding/unbinding calls to amd_iommu driver

This patch adds the functions to bind and unbind pasid
from a device through the amd_iommu driver.

The unbind function is called when the mm_struct of the
process is released.

The bind function is not called here because it is called
only in the IOCTLs which are not yet implemented at this
stage of the patchset.

Signed-off-by: Oded Gabbay <oded.gabbay@amd.com>

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 71a03f7b004936c14b37fd97eb17a2863b4d0605..4b1cced9b2b71917a8279fa8155784def7765019 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -100,6 +100,65 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev)
        return kfd;
 }
 
+static bool device_iommu_pasid_init(struct kfd_dev *kfd)
+{
+       const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
+                                       AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
+                                       AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
+
+       struct amd_iommu_device_info iommu_info;
+       unsigned int pasid_limit;
+       int err;
+
+       err = amd_iommu_device_info(kfd->pdev, &iommu_info);
+       if (err < 0) {
+               dev_err(kfd_device,
+                       "error getting iommu info. is the iommu enabled?\n");
+               return false;
+       }
+
+       if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
+               dev_err(kfd_device, "error required iommu flags ats(%i), pri(%i), pasid(%i)\n",
+                      (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
+                      (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
+                      (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) != 0);
+               return false;
+       }
+
+       pasid_limit = min_t(unsigned int,
+                       (unsigned int)1 << kfd->device_info->max_pasid_bits,
+                       iommu_info.max_pasids);
+       /*
+        * last pasid is used for kernel queues doorbells
+        * in the future the last pasid might be used for a kernel thread.
+        */
+       pasid_limit = min_t(unsigned int,
+                               pasid_limit,
+                               kfd->doorbell_process_limit - 1);
+
+       err = amd_iommu_init_device(kfd->pdev, pasid_limit);
+       if (err < 0) {
+               dev_err(kfd_device, "error initializing iommu device\n");
+               return false;
+       }
+
+       if (!kfd_set_pasid_limit(pasid_limit)) {
+               dev_err(kfd_device, "error setting pasid limit\n");
+               amd_iommu_free_device(kfd->pdev);
+               return false;
+       }
+
+       return true;
+}
+
+static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
+{
+       struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
+
+       if (dev)
+               kfd_unbind_process_from_device(dev, pasid);
+}
+
 bool kgd2kfd_device_init(struct kfd_dev *kfd,
                         const struct kgd2kfd_shared_resources *gpu_resources)
 {
@@ -131,6 +190,14 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
                goto kfd_topology_add_device_error;
        }
 
+       if (!device_iommu_pasid_init(kfd)) {
+               dev_err(kfd_device,
+                       "Error initializing iommuv2 for device (%x:%x)\n",
+                       kfd->pdev->vendor, kfd->pdev->device);
+               goto device_iommu_pasid_error;
+       }
+       amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
+                                               iommu_pasid_shutdown_callback);
 
        kfd->init_complete = true;
        dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor,
@@ -138,6 +205,8 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 
        goto out;
 
+device_iommu_pasid_error:
+       kfd_topology_remove_device(kfd);
 kfd_topology_add_device_error:
        kfd2kgd->fini_sa_manager(kfd->kgd);
        dev_err(kfd_device,
@@ -149,7 +218,10 @@ out:
 
 void kgd2kfd_device_exit(struct kfd_dev *kfd)
 {
-       kfd_topology_remove_device(kfd);
+       if (kfd->init_complete) {
+               amd_iommu_free_device(kfd->pdev);
+               kfd_topology_remove_device(kfd);
+       }
 
        kfree(kfd);
 }
@@ -157,12 +229,28 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
 void kgd2kfd_suspend(struct kfd_dev *kfd)
 {
        BUG_ON(kfd == NULL);
+
+       if (kfd->init_complete)
+               amd_iommu_free_device(kfd->pdev);
 }
 
 int kgd2kfd_resume(struct kfd_dev *kfd)
 {
+       unsigned int pasid_limit;
+       int err;
+
        BUG_ON(kfd == NULL);
 
+       pasid_limit = kfd_get_pasid_limit();
+
+       if (kfd->init_complete) {
+               err = amd_iommu_init_device(kfd->pdev, pasid_limit);
+               if (err < 0)
+                       return -ENXIO;
+               amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
+                                               iommu_pasid_shutdown_callback);
+       }
+
        return 0;
 }
 

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 77d15dbebb0c1518b6639bc84d28e95c8318e1c7..431a09bbab6240847eb32fa3cd5729c52a9f09d4 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -194,6 +194,7 @@ void kfd_process_destroy_wq(void);
 struct kfd_process *kfd_create_process(const struct task_struct *);
 struct kfd_process *kfd_get_process(const struct task_struct *);
 
+void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid);
 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
                                                        struct kfd_process *p,
                                                        int create_pdd);

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 5596f698cc11e96ff05e8a0f61735dcaec09205b..5084794695fba069d25cbdd6ca1511c86b4d51d0 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -24,6 +24,7 @@
 #include <linux/log2.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
+#include <linux/amd-iommu.h>
 #include <linux/notifier.h>
 struct mm_struct;
 
@@ -163,6 +164,7 @@ static void kfd_process_wq_release(struct work_struct *work)
 
        list_for_each_entry_safe(pdd, temp, &p->per_device_data,
                                                        per_device_list) {
+               amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
                list_del(&pdd->per_device_list);
 
                kfree(pdd);
@@ -316,6 +318,7 @@ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
                                                        struct kfd_process *p)
 {
        struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1);
+       int err;
 
        if (pdd == NULL)
                return ERR_PTR(-ENOMEM);
@@ -323,6 +326,15 @@ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
        if (pdd->bound)
                return pdd;
 
+       err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
+       if (err < 0)
+               return ERR_PTR(err);
+
+       if (err < 0) {
+               amd_iommu_unbind_pasid(dev->pdev, p->pasid);
+               return ERR_PTR(err);
+       }
+
        pdd->bound = true;
 
        return pdd;