2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2007 - 2009 Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
19 #include <linux/kernel.h>
20 #include <linux/pci.h>
21 #include <linux/smp.h>
22 #include <linux/interrupt.h>
23 #include <linux/dca.h>
25 /* either a kernel change is needed, or we need something like this in kernel */
28 #undef cpu_physical_id
29 #define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
33 #include "registers.h"
37 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
38 * contain the bit number of the APIC ID to map into the DCA tag. If the valid
39 * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
41 #define DCA_TAG_MAP_VALID 0x80
43 #define DCA3_TAG_MAP_BIT_TO_INV 0x80
44 #define DCA3_TAG_MAP_BIT_TO_SEL 0x40
45 #define DCA3_TAG_MAP_LITERAL_VAL 0x1
47 #define DCA_TAG_MAP_MASK 0xDF
49 /* expected tag map bytes for I/OAT ver.2 */
50 #define DCA2_TAG_MAP_BYTE0 0x80
51 #define DCA2_TAG_MAP_BYTE1 0x0
52 #define DCA2_TAG_MAP_BYTE2 0x81
53 #define DCA2_TAG_MAP_BYTE3 0x82
54 #define DCA2_TAG_MAP_BYTE4 0x82
56 /* verify if tag map matches expected values */
57 static inline int dca2_tag_map_valid(u8 *tag_map)
59 return ((tag_map[0] == DCA2_TAG_MAP_BYTE0) &&
60 (tag_map[1] == DCA2_TAG_MAP_BYTE1) &&
61 (tag_map[2] == DCA2_TAG_MAP_BYTE2) &&
62 (tag_map[3] == DCA2_TAG_MAP_BYTE3) &&
63 (tag_map[4] == DCA2_TAG_MAP_BYTE4));
67 * "Legacy" DCA systems do not implement the DCA register set in the
68 * I/OAT device. Software needs direct support for their tag mappings.
71 #define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x))
72 #define IOAT_TAG_MAP_LEN 8
74 /* pack PCI B/D/F into a u16 */
75 static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
77 return (pci->bus->number << 8) | pci->devfn;
80 static int dca_enabled_in_bios(struct pci_dev *pdev)
82 /* CPUID level 9 returns DCA configuration */
83 /* Bit 0 indicates DCA enabled by the BIOS */
84 unsigned long cpuid_level_9;
87 cpuid_level_9 = cpuid_eax(9);
88 res = test_bit(0, &cpuid_level_9);
90 dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n");
95 int system_has_dca_enabled(struct pci_dev *pdev)
97 if (boot_cpu_has(X86_FEATURE_DCA))
98 return dca_enabled_in_bios(pdev);
100 dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
104 struct ioat_dca_slot {
105 struct pci_dev *pdev; /* requester device */
106 u16 rid; /* requester id, as used by IOAT */
109 #define IOAT_DCA_MAX_REQ 6
110 #define IOAT3_DCA_MAX_REQ 2
112 struct ioat_dca_priv {
113 void __iomem *iobase;
114 void __iomem *dca_base;
117 u8 tag_map[IOAT_TAG_MAP_LEN];
118 struct ioat_dca_slot req_slots[0];
121 static u8 ioat_dca_get_tag(struct dca_provider *dca,
125 struct ioat_dca_priv *ioatdca = dca_priv(dca);
126 int i, apic_id, bit, value;
130 apic_id = cpu_physical_id(cpu);
132 for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
133 entry = ioatdca->tag_map[i];
134 if (entry & DCA_TAG_MAP_VALID) {
135 bit = entry & ~DCA_TAG_MAP_VALID;
136 value = (apic_id & (1 << bit)) ? 1 : 0;
138 value = entry ? 1 : 0;
145 static int ioat_dca_dev_managed(struct dca_provider *dca,
148 struct ioat_dca_priv *ioatdca = dca_priv(dca);
149 struct pci_dev *pdev;
152 pdev = to_pci_dev(dev);
153 for (i = 0; i < ioatdca->max_requesters; i++) {
154 if (ioatdca->req_slots[i].pdev == pdev)
160 static int ioat2_dca_add_requester(struct dca_provider *dca, struct device *dev)
162 struct ioat_dca_priv *ioatdca = dca_priv(dca);
163 struct pci_dev *pdev;
166 u16 global_req_table;
168 /* This implementation only supports PCI-Express */
169 if (!dev_is_pci(dev))
171 pdev = to_pci_dev(dev);
172 id = dcaid_from_pcidev(pdev);
174 if (ioatdca->requester_count == ioatdca->max_requesters)
177 for (i = 0; i < ioatdca->max_requesters; i++) {
178 if (ioatdca->req_slots[i].pdev == NULL) {
179 /* found an empty slot */
180 ioatdca->requester_count++;
181 ioatdca->req_slots[i].pdev = pdev;
182 ioatdca->req_slots[i].rid = id;
184 readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
185 writel(id | IOAT_DCA_GREQID_VALID,
186 ioatdca->iobase + global_req_table + (i * 4));
190 /* Error, ioatdma->requester_count is out of whack */
194 static int ioat2_dca_remove_requester(struct dca_provider *dca,
197 struct ioat_dca_priv *ioatdca = dca_priv(dca);
198 struct pci_dev *pdev;
200 u16 global_req_table;
202 /* This implementation only supports PCI-Express */
203 if (!dev_is_pci(dev))
205 pdev = to_pci_dev(dev);
207 for (i = 0; i < ioatdca->max_requesters; i++) {
208 if (ioatdca->req_slots[i].pdev == pdev) {
210 readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
211 writel(0, ioatdca->iobase + global_req_table + (i * 4));
212 ioatdca->req_slots[i].pdev = NULL;
213 ioatdca->req_slots[i].rid = 0;
214 ioatdca->requester_count--;
221 static u8 ioat2_dca_get_tag(struct dca_provider *dca,
227 tag = ioat_dca_get_tag(dca, dev, cpu);
232 static struct dca_ops ioat2_dca_ops = {
233 .add_requester = ioat2_dca_add_requester,
234 .remove_requester = ioat2_dca_remove_requester,
235 .get_tag = ioat2_dca_get_tag,
236 .dev_managed = ioat_dca_dev_managed,
239 static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset)
243 u16 global_req_table;
245 global_req_table = readw(iobase + dca_offset + IOAT_DCA_GREQID_OFFSET);
246 if (global_req_table == 0)
249 req = readl(iobase + global_req_table + (slots * sizeof(u32)));
251 } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
256 struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
258 struct dca_provider *dca;
259 struct ioat_dca_priv *ioatdca;
269 if (!system_has_dca_enabled(pdev))
272 dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
276 slots = ioat2_dca_count_dca_slots(iobase, dca_offset);
280 dca = alloc_dca_provider(&ioat2_dca_ops,
282 + (sizeof(struct ioat_dca_slot) * slots));
286 ioatdca = dca_priv(dca);
287 ioatdca->iobase = iobase;
288 ioatdca->dca_base = iobase + dca_offset;
289 ioatdca->max_requesters = slots;
291 /* some bios might not know to turn these on */
292 csi_fsb_control = readw(ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
293 if ((csi_fsb_control & IOAT_FSB_CAP_ENABLE_PREFETCH) == 0) {
294 csi_fsb_control |= IOAT_FSB_CAP_ENABLE_PREFETCH;
295 writew(csi_fsb_control,
296 ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
298 pcie_control = readw(ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
299 if ((pcie_control & IOAT_PCI_CAP_ENABLE_MEMWR) == 0) {
300 pcie_control |= IOAT_PCI_CAP_ENABLE_MEMWR;
302 ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
306 /* TODO version, compatibility and configuration checks */
308 /* copy out the APIC to DCA tag map */
309 tag_map = readl(ioatdca->dca_base + IOAT_APICID_TAG_MAP_OFFSET);
310 for (i = 0; i < 5; i++) {
311 bit = (tag_map >> (4 * i)) & 0x0f;
313 ioatdca->tag_map[i] = bit | DCA_TAG_MAP_VALID;
315 ioatdca->tag_map[i] = 0;
318 if (!dca2_tag_map_valid(ioatdca->tag_map)) {
319 WARN_TAINT_ONCE(1, TAINT_FIRMWARE_WORKAROUND,
320 "%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n",
321 dev_driver_string(&pdev->dev),
322 dev_name(&pdev->dev));
323 free_dca_provider(dca);
327 err = register_dca_provider(dca, &pdev->dev);
329 free_dca_provider(dca);
336 static int ioat3_dca_add_requester(struct dca_provider *dca, struct device *dev)
338 struct ioat_dca_priv *ioatdca = dca_priv(dca);
339 struct pci_dev *pdev;
342 u16 global_req_table;
344 /* This implementation only supports PCI-Express */
345 if (!dev_is_pci(dev))
347 pdev = to_pci_dev(dev);
348 id = dcaid_from_pcidev(pdev);
350 if (ioatdca->requester_count == ioatdca->max_requesters)
353 for (i = 0; i < ioatdca->max_requesters; i++) {
354 if (ioatdca->req_slots[i].pdev == NULL) {
355 /* found an empty slot */
356 ioatdca->requester_count++;
357 ioatdca->req_slots[i].pdev = pdev;
358 ioatdca->req_slots[i].rid = id;
360 readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
361 writel(id | IOAT_DCA_GREQID_VALID,
362 ioatdca->iobase + global_req_table + (i * 4));
366 /* Error, ioatdma->requester_count is out of whack */
370 static int ioat3_dca_remove_requester(struct dca_provider *dca,
373 struct ioat_dca_priv *ioatdca = dca_priv(dca);
374 struct pci_dev *pdev;
376 u16 global_req_table;
378 /* This implementation only supports PCI-Express */
379 if (!dev_is_pci(dev))
381 pdev = to_pci_dev(dev);
383 for (i = 0; i < ioatdca->max_requesters; i++) {
384 if (ioatdca->req_slots[i].pdev == pdev) {
386 readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
387 writel(0, ioatdca->iobase + global_req_table + (i * 4));
388 ioatdca->req_slots[i].pdev = NULL;
389 ioatdca->req_slots[i].rid = 0;
390 ioatdca->requester_count--;
397 static u8 ioat3_dca_get_tag(struct dca_provider *dca,
403 struct ioat_dca_priv *ioatdca = dca_priv(dca);
404 int i, apic_id, bit, value;
408 apic_id = cpu_physical_id(cpu);
410 for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
411 entry = ioatdca->tag_map[i];
412 if (entry & DCA3_TAG_MAP_BIT_TO_SEL) {
414 ~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV);
415 value = (apic_id & (1 << bit)) ? 1 : 0;
416 } else if (entry & DCA3_TAG_MAP_BIT_TO_INV) {
417 bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV;
418 value = (apic_id & (1 << bit)) ? 0 : 1;
420 value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0;
428 static struct dca_ops ioat3_dca_ops = {
429 .add_requester = ioat3_dca_add_requester,
430 .remove_requester = ioat3_dca_remove_requester,
431 .get_tag = ioat3_dca_get_tag,
432 .dev_managed = ioat_dca_dev_managed,
435 static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
439 u16 global_req_table;
441 global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET);
442 if (global_req_table == 0)
446 req = readl(iobase + global_req_table + (slots * sizeof(u32)));
448 } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
453 static inline int dca3_tag_map_invalid(u8 *tag_map)
456 * If the tag map is not programmed by the BIOS the default is:
457 * 0x80 0x80 0x80 0x80 0x80 0x00 0x00 0x00
459 * This an invalid map and will result in only 2 possible tags
460 * 0x1F and 0x00. 0x00 is an invalid DCA tag so we know that
461 * this entire definition is invalid.
463 return ((tag_map[0] == DCA_TAG_MAP_VALID) &&
464 (tag_map[1] == DCA_TAG_MAP_VALID) &&
465 (tag_map[2] == DCA_TAG_MAP_VALID) &&
466 (tag_map[3] == DCA_TAG_MAP_VALID) &&
467 (tag_map[4] == DCA_TAG_MAP_VALID));
470 struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
472 struct dca_provider *dca;
473 struct ioat_dca_priv *ioatdca;
490 if (!system_has_dca_enabled(pdev))
493 dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
497 slots = ioat3_dca_count_dca_slots(iobase, dca_offset);
501 dca = alloc_dca_provider(&ioat3_dca_ops,
503 + (sizeof(struct ioat_dca_slot) * slots));
507 ioatdca = dca_priv(dca);
508 ioatdca->iobase = iobase;
509 ioatdca->dca_base = iobase + dca_offset;
510 ioatdca->max_requesters = slots;
512 /* some bios might not know to turn these on */
513 csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
514 if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) {
515 csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH;
516 writew(csi_fsb_control,
517 ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
519 pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
520 if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) {
521 pcie_control |= IOAT3_PCI_CONTROL_MEMWR;
523 ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
527 /* TODO version, compatibility and configuration checks */
529 /* copy out the APIC to DCA tag map */
531 readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW);
533 readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH);
534 for (i = 0; i < 8; i++) {
535 bit = tag_map.full >> (8 * i);
536 ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK;
539 if (dca3_tag_map_invalid(ioatdca->tag_map)) {
540 WARN_TAINT_ONCE(1, TAINT_FIRMWARE_WORKAROUND,
541 "%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n",
542 dev_driver_string(&pdev->dev),
543 dev_name(&pdev->dev));
544 free_dca_provider(dca);
548 err = register_dca_provider(dca, &pdev->dev);
550 free_dca_provider(dca);