2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/amd-iommu.h>
24 #include <linux/bsearch.h>
25 #include <linux/pci.h>
26 #include <linux/slab.h>
28 #include "kfd_device_queue_manager.h"
30 #define MQD_SIZE_ALIGNED 768
32 static const struct kfd_device_info kaveri_device_info = {
34 .ih_ring_entry_size = 4 * sizeof(uint32_t),
35 .mqd_size_aligned = MQD_SIZE_ALIGNED
40 const struct kfd_device_info *device_info;
43 /* Please keep this sorted by increasing device id. */
44 static const struct kfd_deviceid supported_devices[] = {
45 { 0x1304, &kaveri_device_info }, /* Kaveri */
46 { 0x1305, &kaveri_device_info }, /* Kaveri */
47 { 0x1306, &kaveri_device_info }, /* Kaveri */
48 { 0x1307, &kaveri_device_info }, /* Kaveri */
49 { 0x1309, &kaveri_device_info }, /* Kaveri */
50 { 0x130A, &kaveri_device_info }, /* Kaveri */
51 { 0x130B, &kaveri_device_info }, /* Kaveri */
52 { 0x130C, &kaveri_device_info }, /* Kaveri */
53 { 0x130D, &kaveri_device_info }, /* Kaveri */
54 { 0x130E, &kaveri_device_info }, /* Kaveri */
55 { 0x130F, &kaveri_device_info }, /* Kaveri */
56 { 0x1310, &kaveri_device_info }, /* Kaveri */
57 { 0x1311, &kaveri_device_info }, /* Kaveri */
58 { 0x1312, &kaveri_device_info }, /* Kaveri */
59 { 0x1313, &kaveri_device_info }, /* Kaveri */
60 { 0x1315, &kaveri_device_info }, /* Kaveri */
61 { 0x1316, &kaveri_device_info }, /* Kaveri */
62 { 0x1317, &kaveri_device_info }, /* Kaveri */
63 { 0x1318, &kaveri_device_info }, /* Kaveri */
64 { 0x131B, &kaveri_device_info }, /* Kaveri */
65 { 0x131C, &kaveri_device_info }, /* Kaveri */
66 { 0x131D, &kaveri_device_info }, /* Kaveri */
69 static const struct kfd_device_info *lookup_device_info(unsigned short did)
73 for (i = 0; i < ARRAY_SIZE(supported_devices); i++) {
74 if (supported_devices[i].did == did) {
75 BUG_ON(supported_devices[i].device_info == NULL);
76 return supported_devices[i].device_info;
83 struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev)
87 const struct kfd_device_info *device_info =
88 lookup_device_info(pdev->device);
93 kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
98 kfd->device_info = device_info;
100 kfd->init_complete = false;
105 static bool device_iommu_pasid_init(struct kfd_dev *kfd)
107 const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
108 AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
109 AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
111 struct amd_iommu_device_info iommu_info;
112 unsigned int pasid_limit;
115 err = amd_iommu_device_info(kfd->pdev, &iommu_info);
118 "error getting iommu info. is the iommu enabled?\n");
122 if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
123 dev_err(kfd_device, "error required iommu flags ats(%i), pri(%i), pasid(%i)\n",
124 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
125 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
126 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) != 0);
130 pasid_limit = min_t(unsigned int,
131 (unsigned int)1 << kfd->device_info->max_pasid_bits,
132 iommu_info.max_pasids);
134 * last pasid is used for kernel queues doorbells
135 * in the future the last pasid might be used for a kernel thread.
137 pasid_limit = min_t(unsigned int,
139 kfd->doorbell_process_limit - 1);
141 err = amd_iommu_init_device(kfd->pdev, pasid_limit);
143 dev_err(kfd_device, "error initializing iommu device\n");
147 if (!kfd_set_pasid_limit(pasid_limit)) {
148 dev_err(kfd_device, "error setting pasid limit\n");
149 amd_iommu_free_device(kfd->pdev);
156 static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
158 struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
161 kfd_unbind_process_from_device(dev, pasid);
164 bool kgd2kfd_device_init(struct kfd_dev *kfd,
165 const struct kgd2kfd_shared_resources *gpu_resources)
169 kfd->shared_resources = *gpu_resources;
171 /* calculate max size of mqds needed for queues */
172 size = max_num_of_processes *
173 max_num_of_queues_per_process *
174 kfd->device_info->mqd_size_aligned;
176 /* add another 512KB for all other allocations on gart */
179 if (kfd2kgd->init_sa_manager(kfd->kgd, size)) {
181 "Error initializing sa manager for device (%x:%x)\n",
182 kfd->pdev->vendor, kfd->pdev->device);
186 kfd_doorbell_init(kfd);
188 if (kfd_topology_add_device(kfd) != 0) {
190 "Error adding device (%x:%x) to topology\n",
191 kfd->pdev->vendor, kfd->pdev->device);
192 goto kfd_topology_add_device_error;
195 if (kfd_interrupt_init(kfd)) {
197 "Error initializing interrupts for device (%x:%x)\n",
198 kfd->pdev->vendor, kfd->pdev->device);
199 goto kfd_interrupt_error;
202 if (!device_iommu_pasid_init(kfd)) {
204 "Error initializing iommuv2 for device (%x:%x)\n",
205 kfd->pdev->vendor, kfd->pdev->device);
206 goto device_iommu_pasid_error;
208 amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
209 iommu_pasid_shutdown_callback);
211 kfd->dqm = device_queue_manager_init(kfd);
214 "Error initializing queue manager for device (%x:%x)\n",
215 kfd->pdev->vendor, kfd->pdev->device);
216 goto device_queue_manager_error;
219 if (kfd->dqm->start(kfd->dqm) != 0) {
221 "Error starting queuen manager for device (%x:%x)\n",
222 kfd->pdev->vendor, kfd->pdev->device);
223 goto dqm_start_error;
226 kfd->init_complete = true;
227 dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor,
230 pr_debug("kfd: Starting kfd with the following scheduling policy %d\n",
236 device_queue_manager_uninit(kfd->dqm);
237 device_queue_manager_error:
238 amd_iommu_free_device(kfd->pdev);
239 device_iommu_pasid_error:
240 kfd_interrupt_exit(kfd);
242 kfd_topology_remove_device(kfd);
243 kfd_topology_add_device_error:
244 kfd2kgd->fini_sa_manager(kfd->kgd);
246 "device (%x:%x) NOT added due to errors\n",
247 kfd->pdev->vendor, kfd->pdev->device);
249 return kfd->init_complete;
252 void kgd2kfd_device_exit(struct kfd_dev *kfd)
254 if (kfd->init_complete) {
255 device_queue_manager_uninit(kfd->dqm);
256 amd_iommu_free_device(kfd->pdev);
257 kfd_interrupt_exit(kfd);
258 kfd_topology_remove_device(kfd);
264 void kgd2kfd_suspend(struct kfd_dev *kfd)
268 if (kfd->init_complete) {
269 kfd->dqm->stop(kfd->dqm);
270 amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
271 amd_iommu_free_device(kfd->pdev);
275 int kgd2kfd_resume(struct kfd_dev *kfd)
277 unsigned int pasid_limit;
282 pasid_limit = kfd_get_pasid_limit();
284 if (kfd->init_complete) {
285 err = amd_iommu_init_device(kfd->pdev, pasid_limit);
288 amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
289 iommu_pasid_shutdown_callback);
290 kfd->dqm->start(kfd->dqm);
296 /* This is called directly from KGD at ISR. */
297 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
299 if (kfd->init_complete) {
300 spin_lock(&kfd->interrupt_lock);
302 if (kfd->interrupts_active
303 && enqueue_ih_ring_entry(kfd, ih_ring_entry))
304 schedule_work(&kfd->interrupt_work);
306 spin_unlock(&kfd->interrupt_lock);