2 * Copyright 2012 Tilera Corporation. All Rights Reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
15 #include <linux/kernel.h>
16 #include <linux/mmzone.h>
17 #include <linux/pci.h>
18 #include <linux/delay.h>
19 #include <linux/string.h>
20 #include <linux/init.h>
21 #include <linux/capability.h>
22 #include <linux/sched.h>
23 #include <linux/errno.h>
24 #include <linux/irq.h>
25 #include <linux/msi.h>
27 #include <linux/uaccess.h>
28 #include <linux/ctype.h>
30 #include <asm/processor.h>
31 #include <asm/sections.h>
32 #include <asm/byteorder.h>
34 #include <gxio/iorpc_globals.h>
35 #include <gxio/kiorpc.h>
36 #include <gxio/trio.h>
37 #include <gxio/iorpc_trio.h>
38 #include <hv/drv_trio_intf.h>
43 * This file containes the routines to search for PCI buses,
44 * enumerate the buses, and configure any attached devices.
47 #define DEBUG_PCI_CFG 0
50 #define TRACE_CFG_WR(size, val, bus, dev, func, offset) \
51 pr_info("CFG WR %d-byte VAL %#x to bus %d dev %d func %d addr %u\n", \
52 size, val, bus, dev, func, offset & 0xFFF);
53 #define TRACE_CFG_RD(size, val, bus, dev, func, offset) \
54 pr_info("CFG RD %d-byte VAL %#x from bus %d dev %d func %d addr %u\n", \
55 size, val, bus, dev, func, offset & 0xFFF);
57 #define TRACE_CFG_WR(...)
58 #define TRACE_CFG_RD(...)
61 static int pci_probe = 1;
63 /* Information on the PCIe RC ports configuration. */
64 static int pcie_rc[TILEGX_NUM_TRIO][TILEGX_TRIO_PCIES];
67 * On some platforms with one or more Gx endpoint ports, we need to
68 * delay the PCIe RC port probe for a few seconds to work around
69 * a HW PCIe link-training bug. The exact delay is specified with
70 * a kernel boot argument in the form of "pcie_rc_delay=T,P,S",
71 * where T is the TRIO instance number, P is the port number and S is
72 * the delay in seconds. If the argument is specified, but the delay is
73 * not provided, the value will be DEFAULT_RC_DELAY.
75 static int rc_delay[TILEGX_NUM_TRIO][TILEGX_TRIO_PCIES];
77 /* Default number of seconds that the PCIe RC port probe can be delayed. */
78 #define DEFAULT_RC_DELAY 10
80 /* The PCI I/O space size in each PCI domain. */
81 #define IO_SPACE_SIZE 0x10000
83 /* Provide shorter versions of some very long constant names. */
84 #define AUTO_CONFIG_RC \
85 TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_RC
86 #define AUTO_CONFIG_RC_G1 \
87 TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_RC_G1
88 #define AUTO_CONFIG_EP \
89 TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_ENDPOINT
90 #define AUTO_CONFIG_EP_G1 \
91 TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_ENDPOINT_G1
93 /* Array of the PCIe ports configuration info obtained from the BIB. */
94 struct pcie_trio_ports_property pcie_ports[TILEGX_NUM_TRIO];
96 /* Number of configured TRIO instances. */
99 /* All drivers share the TRIO contexts defined here. */
100 gxio_trio_context_t trio_contexts[TILEGX_NUM_TRIO];
102 /* Pointer to an array of PCIe RC controllers. */
103 struct pci_controller pci_controllers[TILEGX_NUM_TRIO * TILEGX_TRIO_PCIES];
104 int num_rc_controllers;
106 static struct pci_ops tile_cfg_ops;
108 /* Mask of CPUs that should receive PCIe interrupts. */
109 static struct cpumask intr_cpus_map;
111 /* We don't need to worry about the alignment of resources. */
112 resource_size_t pcibios_align_resource(void *data, const struct resource *res,
113 resource_size_t size,
114 resource_size_t align)
118 EXPORT_SYMBOL(pcibios_align_resource);
121 * Pick a CPU to receive and handle the PCIe interrupts, based on the IRQ #.
122 * For now, we simply send interrupts to non-dataplane CPUs.
123 * We may implement methods to allow user to specify the target CPUs,
124 * e.g. via boot arguments.
126 static int tile_irq_cpu(int irq)
132 count = cpumask_weight(&intr_cpus_map);
133 if (unlikely(count == 0)) {
134 pr_warning("intr_cpus_map empty, interrupts will be"
135 " delievered to dataplane tiles\n");
136 return irq % (smp_height * smp_width);
140 for_each_cpu(cpu, &intr_cpus_map) {
147 /* Open a file descriptor to the TRIO shim. */
148 static int tile_pcie_open(int trio_index)
150 gxio_trio_context_t *context = &trio_contexts[trio_index];
154 /* This opens a file descriptor to the TRIO shim. */
155 ret = gxio_trio_init(context, trio_index);
157 goto gxio_trio_init_failure;
159 /* Allocate an ASID for the kernel. */
160 ret = gxio_trio_alloc_asids(context, 1, 0, 0);
162 pr_err("PCI: ASID alloc failure on TRIO %d, give up\n",
164 goto asid_alloc_failure;
169 #ifdef USE_SHARED_PCIE_CONFIG_REGION
171 * Alloc a PIO region for config access, shared by all MACs per TRIO.
172 * This shouldn't fail since the kernel is supposed to the first
173 * client of the TRIO's PIO regions.
175 ret = gxio_trio_alloc_pio_regions(context, 1, 0, 0);
177 pr_err("PCI: CFG PIO alloc failure on TRIO %d, give up\n",
179 goto pio_alloc_failure;
182 context->pio_cfg_index = ret;
185 * For PIO CFG, the bus_address_hi parameter is 0. The mac parameter
186 * is also 0 because it is specified in PIO_REGION_SETUP_CFG_ADDR.
188 ret = gxio_trio_init_pio_region_aux(context, context->pio_cfg_index,
189 0, 0, HV_TRIO_PIO_FLAG_CONFIG_SPACE);
191 pr_err("PCI: CFG PIO init failure on TRIO %d, give up\n",
193 goto pio_alloc_failure;
197 /* Get the properties of the PCIe ports on this TRIO instance. */
198 ret = gxio_trio_get_port_property(context, &pcie_ports[trio_index]);
200 pr_err("PCI: PCIE_GET_PORT_PROPERTY failure, error %d,"
201 " on TRIO %d\n", ret, trio_index);
202 goto get_port_property_failure;
205 context->mmio_base_mac =
206 iorpc_ioremap(context->fd, 0, HV_TRIO_CONFIG_IOREMAP_SIZE);
207 if (context->mmio_base_mac == NULL) {
208 pr_err("PCI: TRIO config space mapping failure, error %d,"
209 " on TRIO %d\n", ret, trio_index);
212 goto trio_mmio_mapping_failure;
215 /* Check the port strap state which will override the BIB setting. */
216 for (mac = 0; mac < TILEGX_TRIO_PCIES; mac++) {
217 TRIO_PCIE_INTFC_PORT_CONFIG_t port_config;
218 unsigned int reg_offset;
220 /* Ignore ports that are not specified in the BIB. */
221 if (!pcie_ports[trio_index].ports[mac].allow_rc &&
222 !pcie_ports[trio_index].ports[mac].allow_ep)
226 (TRIO_PCIE_INTFC_PORT_CONFIG <<
227 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
228 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
229 TRIO_CFG_REGION_ADDR__INTFC_SHIFT) |
230 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
233 __gxio_mmio_read(context->mmio_base_mac + reg_offset);
235 if (port_config.strap_state != AUTO_CONFIG_RC &&
236 port_config.strap_state != AUTO_CONFIG_RC_G1) {
238 * If this is really intended to be an EP port, record
239 * it so that the endpoint driver will know about it.
241 if (port_config.strap_state == AUTO_CONFIG_EP ||
242 port_config.strap_state == AUTO_CONFIG_EP_G1)
243 pcie_ports[trio_index].ports[mac].allow_ep = 1;
249 trio_mmio_mapping_failure:
250 get_port_property_failure:
252 #ifdef USE_SHARED_PCIE_CONFIG_REGION
255 hv_dev_close(context->fd);
256 gxio_trio_init_failure:
262 static int __init tile_trio_init(void)
266 /* We loop over all the TRIO shims. */
267 for (i = 0; i < TILEGX_NUM_TRIO; i++) {
268 if (tile_pcie_open(i) < 0)
275 postcore_initcall(tile_trio_init);
277 static void tilegx_legacy_irq_ack(struct irq_data *d)
279 __insn_mtspr(SPR_IPI_EVENT_RESET_K, 1UL << d->irq);
282 static void tilegx_legacy_irq_mask(struct irq_data *d)
284 __insn_mtspr(SPR_IPI_MASK_SET_K, 1UL << d->irq);
287 static void tilegx_legacy_irq_unmask(struct irq_data *d)
289 __insn_mtspr(SPR_IPI_MASK_RESET_K, 1UL << d->irq);
292 static struct irq_chip tilegx_legacy_irq_chip = {
293 .name = "tilegx_legacy_irq",
294 .irq_ack = tilegx_legacy_irq_ack,
295 .irq_mask = tilegx_legacy_irq_mask,
296 .irq_unmask = tilegx_legacy_irq_unmask,
298 /* TBD: support set_affinity. */
302 * This is a wrapper function of the kernel level-trigger interrupt
303 * handler handle_level_irq() for PCI legacy interrupts. The TRIO
304 * is configured such that only INTx Assert interrupts are proxied
305 * to Linux which just calls handle_level_irq() after clearing the
306 * MAC INTx Assert status bit associated with this interrupt.
308 static void trio_handle_level_irq(unsigned int irq, struct irq_desc *desc)
310 struct pci_controller *controller = irq_desc_get_handler_data(desc);
311 gxio_trio_context_t *trio_context = controller->trio;
312 uint64_t intx = (uint64_t)irq_desc_get_chip_data(desc);
313 int mac = controller->mac;
314 unsigned int reg_offset;
317 handle_level_irq(irq, desc);
320 * Clear the INTx Level status, otherwise future interrupts are
323 reg_offset = (TRIO_PCIE_INTFC_MAC_INT_STS <<
324 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
325 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
326 TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
327 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
329 level_mask = TRIO_PCIE_INTFC_MAC_INT_STS__INT_LEVEL_MASK << intx;
331 __gxio_mmio_write(trio_context->mmio_base_mac + reg_offset, level_mask);
335 * Create kernel irqs and set up the handlers for the legacy interrupts.
336 * Also some minimum initialization for the MSI support.
338 static int tile_init_irqs(struct pci_controller *controller)
345 cpumask_copy(&intr_cpus_map, cpu_online_mask);
348 for (i = 0; i < 4; i++) {
349 gxio_trio_context_t *context = controller->trio;
352 /* Ask the kernel to allocate an IRQ. */
355 pr_err("PCI: no free irq vectors, failed for %d\n", i);
359 controller->irq_intx_table[i] = irq;
361 /* Distribute the 4 IRQs to different tiles. */
362 cpu = tile_irq_cpu(irq);
364 /* Configure the TRIO intr binding for this IRQ. */
365 result = gxio_trio_config_legacy_intr(context, cpu_x(cpu),
366 cpu_y(cpu), KERNEL_PL,
367 irq, controller->mac, i);
369 pr_err("PCI: MAC intx config failed for %d\n", i);
374 /* Register the IRQ handler with the kernel. */
375 irq_set_chip_and_handler(irq, &tilegx_legacy_irq_chip,
376 trio_handle_level_irq);
377 irq_set_chip_data(irq, (void *)(uint64_t)i);
378 irq_set_handler_data(irq, controller);
384 for (j = 0; j < i; j++)
385 destroy_irq(controller->irq_intx_table[j]);
391 * Return 1 if the port is strapped to operate in RC mode.
394 strapped_for_rc(gxio_trio_context_t *trio_context, int mac)
396 TRIO_PCIE_INTFC_PORT_CONFIG_t port_config;
397 unsigned int reg_offset;
399 /* Check the port configuration. */
401 (TRIO_PCIE_INTFC_PORT_CONFIG <<
402 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
403 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
404 TRIO_CFG_REGION_ADDR__INTFC_SHIFT) |
405 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
407 __gxio_mmio_read(trio_context->mmio_base_mac + reg_offset);
409 if (port_config.strap_state == AUTO_CONFIG_RC ||
410 port_config.strap_state == AUTO_CONFIG_RC_G1)
417 * Find valid controllers and fill in pci_controller structs for each
420 * Return the number of controllers discovered.
422 int __init tile_pci_init(void)
428 pr_info("PCI: disabled by boot argument\n");
432 pr_info("PCI: Searching for controllers...\n");
434 if (num_trio_shims == 0 || sim_is_simulator())
438 * Now determine which PCIe ports are configured to operate in RC
439 * mode. There is a differece in the port configuration capability
440 * between the Gx36 and Gx72 devices.
442 * The Gx36 has configuration capability for each of the 3 PCIe
443 * interfaces (disable, auto endpoint, auto RC, etc.).
444 * On the Gx72, you can only select one of the 3 PCIe interfaces per
445 * TRIO to train automatically. Further, the allowable training modes
446 * are reduced to four options (auto endpoint, auto RC, stream x1,
449 * For Gx36 ports, it must be allowed to be in RC mode by the
450 * Board Information Block, and the hardware strapping pins must be
453 * For Gx72 ports, the port will operate in RC mode if either of the
455 * 1. It is allowed to be in RC mode by the Board Information Block,
456 * and the BIB doesn't allow the EP mode.
457 * 2. It is allowed to be in either the RC or the EP mode by the BIB,
458 * and the hardware strapping pin is set to RC mode.
460 for (i = 0; i < TILEGX_NUM_TRIO; i++) {
461 gxio_trio_context_t *context = &trio_contexts[i];
466 for (j = 0; j < TILEGX_TRIO_PCIES; j++) {
469 if (pcie_ports[i].is_gx72 &&
470 pcie_ports[i].ports[j].allow_rc) {
471 if (!pcie_ports[i].ports[j].allow_ep ||
472 strapped_for_rc(context, j))
474 } else if (pcie_ports[i].ports[j].allow_rc &&
475 strapped_for_rc(context, j)) {
480 num_rc_controllers++;
485 /* Return if no PCIe ports are configured to operate in RC mode. */
486 if (num_rc_controllers == 0)
489 /* Set the TRIO pointer and MAC index for each PCIe RC port. */
490 for (i = 0; i < TILEGX_NUM_TRIO; i++) {
491 for (j = 0; j < TILEGX_TRIO_PCIES; j++) {
493 pci_controllers[ctl_index].trio =
495 pci_controllers[ctl_index].mac = j;
496 pci_controllers[ctl_index].trio_index = i;
498 if (ctl_index == num_rc_controllers)
505 /* Configure each PCIe RC port. */
506 for (i = 0; i < num_rc_controllers; i++) {
508 /* Configure the PCIe MAC to run in RC mode. */
509 struct pci_controller *controller = &pci_controllers[i];
511 controller->index = i;
512 controller->ops = &tile_cfg_ops;
514 controller->io_space.start = PCIBIOS_MIN_IO +
516 controller->io_space.end = controller->io_space.start +
518 BUG_ON(controller->io_space.end > IO_SPACE_LIMIT);
519 controller->io_space.flags = IORESOURCE_IO;
520 snprintf(controller->io_space_name,
521 sizeof(controller->io_space_name),
522 "PCI I/O domain %d", i);
523 controller->io_space.name = controller->io_space_name;
526 * The PCI memory resource is located above the PA space.
527 * For every host bridge, the BAR window or the MMIO aperture
528 * is in range [3GB, 4GB - 1] of a 4GB space beyond the
531 controller->mem_offset = TILE_PCI_MEM_START +
532 (i * TILE_PCI_BAR_WINDOW_TOP);
533 controller->mem_space.start = controller->mem_offset +
534 TILE_PCI_BAR_WINDOW_TOP - TILE_PCI_BAR_WINDOW_SIZE;
535 controller->mem_space.end = controller->mem_offset +
536 TILE_PCI_BAR_WINDOW_TOP - 1;
537 controller->mem_space.flags = IORESOURCE_MEM;
538 snprintf(controller->mem_space_name,
539 sizeof(controller->mem_space_name),
540 "PCI mem domain %d", i);
541 controller->mem_space.name = controller->mem_space_name;
544 return num_rc_controllers;
548 * (pin - 1) converts from the PCI standard's [1:4] convention to
549 * a normal [0:3] range.
551 static int tile_map_irq(const struct pci_dev *dev, u8 device, u8 pin)
553 struct pci_controller *controller =
554 (struct pci_controller *)dev->sysdata;
555 return controller->irq_intx_table[pin - 1];
558 static void fixup_read_and_payload_sizes(struct pci_controller *controller)
560 gxio_trio_context_t *trio_context = controller->trio;
561 struct pci_bus *root_bus = controller->root_bus;
562 TRIO_PCIE_RC_DEVICE_CONTROL_t dev_control;
563 TRIO_PCIE_RC_DEVICE_CAP_t rc_dev_cap;
564 unsigned int reg_offset;
565 struct pci_bus *child;
569 mac = controller->mac;
571 /* Set our max read request size to be 4KB. */
573 (TRIO_PCIE_RC_DEVICE_CONTROL <<
574 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
575 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
576 TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
577 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
579 dev_control.word = __gxio_mmio_read32(trio_context->mmio_base_mac +
581 dev_control.max_read_req_sz = 5;
582 __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset,
586 * Set the max payload size supported by this Gx PCIe MAC.
587 * Though Gx PCIe supports Max Payload Size of up to 1024 bytes,
588 * experiments have shown that setting MPS to 256 yields the
592 (TRIO_PCIE_RC_DEVICE_CAP <<
593 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
594 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
595 TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
596 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
598 rc_dev_cap.word = __gxio_mmio_read32(trio_context->mmio_base_mac +
600 rc_dev_cap.mps_sup = 1;
601 __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset,
604 /* Configure PCI Express MPS setting. */
605 list_for_each_entry(child, &root_bus->children, node) {
606 struct pci_dev *self = child->self;
610 pcie_bus_configure_settings(child, self->pcie_mpss);
614 * Set the mac_config register in trio based on the MPS/MRS of the link.
617 (TRIO_PCIE_RC_DEVICE_CONTROL <<
618 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
619 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
620 TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
621 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
623 dev_control.word = __gxio_mmio_read32(trio_context->mmio_base_mac +
626 err = gxio_trio_set_mps_mrs(trio_context,
627 dev_control.max_payload_size,
628 dev_control.max_read_req_sz,
631 pr_err("PCI: PCIE_CONFIGURE_MAC_MPS_MRS failure, "
632 "MAC %d on TRIO %d\n",
633 mac, controller->trio_index);
637 static int setup_pcie_rc_delay(char *str)
639 unsigned long delay = 0;
640 unsigned long trio_index;
643 if (str == NULL || !isdigit(*str))
645 trio_index = simple_strtoul(str, (char **)&str, 10);
646 if (trio_index >= TILEGX_NUM_TRIO)
655 mac = simple_strtoul(str, (char **)&str, 10);
656 if (mac >= TILEGX_TRIO_PCIES)
666 delay = simple_strtoul(str, (char **)&str, 10);
669 rc_delay[trio_index][mac] = delay ? : DEFAULT_RC_DELAY;
672 early_param("pcie_rc_delay", setup_pcie_rc_delay);
674 /* PCI initialization entry point, called by subsys_initcall. */
675 int __init pcibios_init(void)
677 resource_size_t offset;
678 LIST_HEAD(resources);
684 if (num_rc_controllers == 0)
688 * Delay a bit in case devices aren't ready. Some devices are
689 * known to require at least 20ms here, but we use a more
690 * conservative value.
694 /* Scan all of the recorded PCI controllers. */
695 for (next_busno = 0, i = 0; i < num_rc_controllers; i++) {
696 struct pci_controller *controller = &pci_controllers[i];
697 gxio_trio_context_t *trio_context = controller->trio;
698 TRIO_PCIE_INTFC_PORT_STATUS_t port_status;
699 TRIO_PCIE_INTFC_TX_FIFO_CTL_t tx_fifo_ctl;
701 unsigned int reg_offset;
702 unsigned int class_code_revision;
707 if (trio_context->fd < 0)
710 trio_index = controller->trio_index;
711 mac = controller->mac;
714 * Check for PCIe link-up status to decide if we need
715 * to force the link to come up.
718 (TRIO_PCIE_INTFC_PORT_STATUS <<
719 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
720 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
721 TRIO_CFG_REGION_ADDR__INTFC_SHIFT) |
722 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
725 __gxio_mmio_read(trio_context->mmio_base_mac +
727 if (!port_status.dl_up) {
728 if (rc_delay[trio_index][mac]) {
729 pr_info("Delaying PCIe RC TRIO init %d sec"
730 " on MAC %d on TRIO %d\n",
731 rc_delay[trio_index][mac], mac,
733 msleep(rc_delay[trio_index][mac] * 1000);
735 ret = gxio_trio_force_rc_link_up(trio_context, mac);
737 pr_err("PCI: PCIE_FORCE_LINK_UP failure, "
738 "MAC %d on TRIO %d\n", mac, trio_index);
741 pr_info("PCI: Found PCI controller #%d on TRIO %d MAC %d\n", i,
742 trio_index, controller->mac);
744 /* Delay the bus probe if needed. */
745 if (rc_delay[trio_index][mac]) {
746 pr_info("Delaying PCIe RC bus enumerating %d sec"
747 " on MAC %d on TRIO %d\n",
748 rc_delay[trio_index][mac], mac,
750 msleep(rc_delay[trio_index][mac] * 1000);
753 * Wait a bit here because some EP devices
754 * take longer to come up.
759 /* Check for PCIe link-up status again. */
761 __gxio_mmio_read(trio_context->mmio_base_mac +
763 if (!port_status.dl_up) {
764 if (pcie_ports[trio_index].ports[mac].removable) {
765 pr_info("PCI: link is down, MAC %d on TRIO %d\n",
767 pr_info("This is expected if no PCIe card"
768 " is connected to this link\n");
770 pr_err("PCI: link is down, MAC %d on TRIO %d\n",
776 * Ensure that the link can come out of L1 power down state.
777 * Strictly speaking, this is needed only in the case of
778 * heavy RC-initiated DMAs.
781 (TRIO_PCIE_INTFC_TX_FIFO_CTL <<
782 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
783 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
784 TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
785 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
787 __gxio_mmio_read(trio_context->mmio_base_mac +
789 tx_fifo_ctl.min_p_credits = 0;
790 __gxio_mmio_write(trio_context->mmio_base_mac + reg_offset,
794 * Change the device ID so that Linux bus crawl doesn't confuse
795 * the internal bridge with any Tilera endpoints.
798 (TRIO_PCIE_RC_DEVICE_ID_VEN_ID <<
799 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
800 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
801 TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
802 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
804 __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset,
805 (TILERA_GX36_RC_DEV_ID <<
806 TRIO_PCIE_RC_DEVICE_ID_VEN_ID__DEV_ID_SHIFT) |
809 /* Set the internal P2P bridge class code. */
811 (TRIO_PCIE_RC_REVISION_ID <<
812 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
813 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
814 TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
815 (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
817 class_code_revision =
818 __gxio_mmio_read32(trio_context->mmio_base_mac +
820 class_code_revision = (class_code_revision & 0xff) |
821 (PCI_CLASS_BRIDGE_PCI << 16);
823 __gxio_mmio_write32(trio_context->mmio_base_mac +
824 reg_offset, class_code_revision);
826 #ifdef USE_SHARED_PCIE_CONFIG_REGION
828 /* Map in the MMIO space for the PIO region. */
829 offset = HV_TRIO_PIO_OFFSET(trio_context->pio_cfg_index) |
830 (((unsigned long long)mac) <<
831 TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT);
835 /* Alloc a PIO region for PCI config access per MAC. */
836 ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0);
838 pr_err("PCI: PCI CFG PIO alloc failure for mac %d "
839 "on TRIO %d, give up\n", mac, trio_index);
844 trio_context->pio_cfg_index[mac] = ret;
846 /* For PIO CFG, the bus_address_hi parameter is 0. */
847 ret = gxio_trio_init_pio_region_aux(trio_context,
848 trio_context->pio_cfg_index[mac],
849 mac, 0, HV_TRIO_PIO_FLAG_CONFIG_SPACE);
851 pr_err("PCI: PCI CFG PIO init failure for mac %d "
852 "on TRIO %d, give up\n", mac, trio_index);
857 offset = HV_TRIO_PIO_OFFSET(trio_context->pio_cfg_index[mac]) |
858 (((unsigned long long)mac) <<
859 TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT);
864 * To save VMALLOC space, we take advantage of the fact that
865 * bit 29 in the PIO CFG address format is reserved 0. With
866 * TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT being 30,
867 * this cuts VMALLOC space usage from 1GB to 512MB per mac.
869 trio_context->mmio_base_pio_cfg[mac] =
870 iorpc_ioremap(trio_context->fd, offset, (1UL <<
871 (TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT - 1)));
872 if (trio_context->mmio_base_pio_cfg[mac] == NULL) {
873 pr_err("PCI: PIO map failure for mac %d on TRIO %d\n",
879 /* Initialize the PCIe interrupts. */
880 if (tile_init_irqs(controller)) {
881 pr_err("PCI: IRQs init failure for mac %d on TRIO %d\n",
888 * The PCI memory resource is located above the PA space.
889 * The memory range for the PCI root bus should not overlap
890 * with the physical RAM.
892 pci_add_resource_offset(&resources, &controller->mem_space,
893 controller->mem_offset);
894 pci_add_resource(&resources, &controller->io_space);
895 controller->first_busno = next_busno;
896 bus = pci_scan_root_bus(NULL, next_busno, controller->ops,
897 controller, &resources);
898 controller->root_bus = bus;
899 next_busno = bus->busn_res.end + 1;
902 /* Do machine dependent PCI interrupt routing */
903 pci_fixup_irqs(pci_common_swizzle, tile_map_irq);
906 * This comes from the generic Linux PCI driver.
908 * It allocates all of the resources (I/O memory, etc)
909 * associated with the devices read in above.
911 pci_assign_unassigned_resources();
913 /* Record the I/O resources in the PCI controller structure. */
914 for (i = 0; i < num_rc_controllers; i++) {
915 struct pci_controller *controller = &pci_controllers[i];
916 gxio_trio_context_t *trio_context = controller->trio;
917 struct pci_bus *root_bus = pci_controllers[i].root_bus;
922 * Skip controllers that are not properly initialized or
925 if (root_bus == NULL)
928 /* Configure the max_payload_size values for this domain. */
929 fixup_read_and_payload_sizes(controller);
931 /* Alloc a PIO region for PCI memory access for each RC port. */
932 ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0);
934 pr_err("PCI: MEM PIO alloc failure on TRIO %d mac %d, "
935 "give up\n", controller->trio_index,
941 controller->pio_mem_index = ret;
944 * For PIO MEM, the bus_address_hi parameter is hard-coded 0
945 * because we always assign 32-bit PCI bus BAR ranges.
947 ret = gxio_trio_init_pio_region_aux(trio_context,
948 controller->pio_mem_index,
953 pr_err("PCI: MEM PIO init failure on TRIO %d mac %d, "
954 "give up\n", controller->trio_index,
960 #ifdef CONFIG_TILE_PCI_IO
962 * Alloc a PIO region for PCI I/O space access for each RC port.
964 ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0);
966 pr_err("PCI: I/O PIO alloc failure on TRIO %d mac %d, "
967 "give up\n", controller->trio_index,
973 controller->pio_io_index = ret;
976 * For PIO IO, the bus_address_hi parameter is hard-coded 0
977 * because PCI I/O address space is 32-bit.
979 ret = gxio_trio_init_pio_region_aux(trio_context,
980 controller->pio_io_index,
983 HV_TRIO_PIO_FLAG_IO_SPACE);
985 pr_err("PCI: I/O PIO init failure on TRIO %d mac %d, "
986 "give up\n", controller->trio_index,
994 * Configure a Mem-Map region for each memory controller so
995 * that Linux can map all of its PA space to the PCI bus.
996 * Use the IOMMU to handle hash-for-home memory.
998 for_each_online_node(j) {
999 unsigned long start_pfn = node_start_pfn[j];
1000 unsigned long end_pfn = node_end_pfn[j];
1001 unsigned long nr_pages = end_pfn - start_pfn;
1003 ret = gxio_trio_alloc_memory_maps(trio_context, 1, 0,
1006 pr_err("PCI: Mem-Map alloc failure on TRIO %d "
1007 "mac %d for MC %d, give up\n",
1008 controller->trio_index,
1009 controller->mac, j);
1011 goto alloc_mem_map_failed;
1014 controller->mem_maps[j] = ret;
1017 * Initialize the Mem-Map and the I/O MMU so that all
1018 * the physical memory can be accessed by the endpoint
1019 * devices. The base bus address is set to the base CPA
1020 * of this memory controller plus an offset (see pci.h).
1021 * The region's base VA is set to the base CPA. The
1022 * I/O MMU table essentially translates the CPA to
1023 * the real PA. Implicitly, for node 0, we create
1024 * a separate Mem-Map region that serves as the inbound
1025 * window for legacy 32-bit devices. This is a direct
1026 * map of the low 4GB CPA space.
1028 ret = gxio_trio_init_memory_map_mmu_aux(trio_context,
1029 controller->mem_maps[j],
1030 start_pfn << PAGE_SHIFT,
1031 nr_pages << PAGE_SHIFT,
1034 (start_pfn << PAGE_SHIFT) +
1035 TILE_PCI_MEM_MAP_BASE_OFFSET,
1037 GXIO_TRIO_ORDER_MODE_UNORDERED);
1039 pr_err("PCI: Mem-Map init failure on TRIO %d "
1040 "mac %d for MC %d, give up\n",
1041 controller->trio_index,
1042 controller->mac, j);
1044 goto alloc_mem_map_failed;
1048 alloc_mem_map_failed:
1055 subsys_initcall(pcibios_init);
1057 /* No bus fixups needed. */
1058 void pcibios_fixup_bus(struct pci_bus *bus)
1062 /* Process any "pci=" kernel boot arguments. */
1063 char *pcibios_setup(char *str)
1065 if (!strcmp(str, "off")) {
1073 * Enable memory address decoding, as appropriate, for the
1074 * device described by the 'dev' struct.
1076 * This is called from the generic PCI layer, and can be called
1077 * for bridges or endpoints.
1079 int pcibios_enable_device(struct pci_dev *dev, int mask)
1081 return pci_enable_resources(dev, mask);
1084 /* Called for each device after PCI setup is done. */
1085 static void pcibios_fixup_final(struct pci_dev *pdev)
1087 set_dma_ops(&pdev->dev, gx_pci_dma_map_ops);
1088 set_dma_offset(&pdev->dev, TILE_PCI_MEM_MAP_BASE_OFFSET);
1089 pdev->dev.archdata.max_direct_dma_addr =
1090 TILE_PCI_MAX_DIRECT_DMA_ADDRESS;
1092 DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, pcibios_fixup_final);
1094 /* Map a PCI MMIO bus address into VA space. */
1095 void __iomem *ioremap(resource_size_t phys_addr, unsigned long size)
1097 struct pci_controller *controller = NULL;
1098 resource_size_t bar_start;
1099 resource_size_t bar_end;
1100 resource_size_t offset;
1101 resource_size_t start;
1102 resource_size_t end;
1107 end = phys_addr + size - 1;
1110 * By searching phys_addr in each controller's mem_space, we can
1111 * determine the controller that should accept the PCI memory access.
1113 for (i = 0; i < num_rc_controllers; i++) {
1115 * Skip controllers that are not properly initialized or
1118 if (pci_controllers[i].root_bus == NULL)
1121 bar_start = pci_controllers[i].mem_space.start;
1122 bar_end = pci_controllers[i].mem_space.end;
1124 if ((start >= bar_start) && (end <= bar_end)) {
1125 controller = &pci_controllers[i];
1130 if (controller == NULL)
1133 trio_fd = controller->trio->fd;
1135 /* Convert the resource start to the bus address offset. */
1136 start = phys_addr - controller->mem_offset;
1138 offset = HV_TRIO_PIO_OFFSET(controller->pio_mem_index) + start;
1140 /* We need to keep the PCI bus address's in-page offset in the VA. */
1141 return iorpc_ioremap(trio_fd, offset, size) +
1142 (start & (PAGE_SIZE - 1));
1144 EXPORT_SYMBOL(ioremap);
1146 #ifdef CONFIG_TILE_PCI_IO
1147 /* Map a PCI I/O address into VA space. */
1148 void __iomem *ioport_map(unsigned long port, unsigned int size)
1150 struct pci_controller *controller = NULL;
1151 resource_size_t bar_start;
1152 resource_size_t bar_end;
1153 resource_size_t offset;
1154 resource_size_t start;
1155 resource_size_t end;
1160 end = port + size - 1;
1163 * By searching the port in each controller's io_space, we can
1164 * determine the controller that should accept the PCI I/O access.
1166 for (i = 0; i < num_rc_controllers; i++) {
1168 * Skip controllers that are not properly initialized or
1171 if (pci_controllers[i].root_bus == NULL)
1174 bar_start = pci_controllers[i].io_space.start;
1175 bar_end = pci_controllers[i].io_space.end;
1177 if ((start >= bar_start) && (end <= bar_end)) {
1178 controller = &pci_controllers[i];
1183 if (controller == NULL)
1186 trio_fd = controller->trio->fd;
1188 /* Convert the resource start to the bus address offset. */
1189 port -= controller->io_space.start;
1191 offset = HV_TRIO_PIO_OFFSET(controller->pio_io_index) + port;
1193 /* We need to keep the PCI bus address's in-page offset in the VA. */
1194 return iorpc_ioremap(trio_fd, offset, size) + (port & (PAGE_SIZE - 1));
1196 EXPORT_SYMBOL(ioport_map);
1198 void ioport_unmap(void __iomem *addr)
1202 EXPORT_SYMBOL(ioport_unmap);
1205 void pci_iounmap(struct pci_dev *dev, void __iomem *addr)
1209 EXPORT_SYMBOL(pci_iounmap);
1211 /****************************************************************
1213 * Tile PCI config space read/write routines
1215 ****************************************************************/
1218 * These are the normal read and write ops
1219 * These are expanded with macros from pci_bus_read_config_byte() etc.
1221 * devfn is the combined PCI device & function.
1223 * offset is in bytes, from the start of config space for the
1224 * specified bus & device.
1226 static int tile_cfg_read(struct pci_bus *bus, unsigned int devfn, int offset,
1229 struct pci_controller *controller = bus->sysdata;
1230 gxio_trio_context_t *trio_context = controller->trio;
1231 int busnum = bus->number & 0xff;
1232 int device = PCI_SLOT(devfn);
1233 int function = PCI_FUNC(devfn);
1234 int config_type = 1;
1235 TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR_t cfg_addr;
1239 * Map all accesses to the local device on root bus into the
1240 * MMIO space of the MAC. Accesses to the downstream devices
1241 * go to the PIO space.
1243 if (pci_is_root_bus(bus)) {
1246 * This is the internal downstream P2P bridge,
1249 unsigned int reg_offset;
1251 reg_offset = ((offset & 0xFFF) <<
1252 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
1253 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_PROTECTED
1254 << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
1256 TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
1258 mmio_addr = trio_context->mmio_base_mac + reg_offset;
1264 * We fake an empty device for (device > 0),
1265 * since there is only one device on bus 0.
1267 goto invalid_device;
1272 * Accesses to the directly attached device have to be
1273 * sent as type-0 configs.
1275 if (busnum == (controller->first_busno + 1)) {
1277 * There is only one device off of our built-in P2P bridge.
1280 goto invalid_device;
1286 cfg_addr.reg_addr = (offset & 0xFFF);
1287 cfg_addr.fn = function;
1288 cfg_addr.dev = device;
1289 cfg_addr.bus = busnum;
1290 cfg_addr.type = config_type;
1293 * Note that we don't set the mac field in cfg_addr because the
1294 * mapping is per port.
1296 mmio_addr = trio_context->mmio_base_pio_cfg[controller->mac] +
1303 *val = __gxio_mmio_read32(mmio_addr);
1307 *val = __gxio_mmio_read16(mmio_addr);
1311 *val = __gxio_mmio_read8(mmio_addr);
1315 return PCIBIOS_FUNC_NOT_SUPPORTED;
1318 TRACE_CFG_RD(size, *val, busnum, device, function, offset);
1338 return PCIBIOS_FUNC_NOT_SUPPORTED;
1346 * See tile_cfg_read() for relevent comments.
1347 * Note that "val" is the value to write, not a pointer to that value.
1349 static int tile_cfg_write(struct pci_bus *bus, unsigned int devfn, int offset,
1352 struct pci_controller *controller = bus->sysdata;
1353 gxio_trio_context_t *trio_context = controller->trio;
1354 int busnum = bus->number & 0xff;
1355 int device = PCI_SLOT(devfn);
1356 int function = PCI_FUNC(devfn);
1357 int config_type = 1;
1358 TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR_t cfg_addr;
1360 u32 val_32 = (u32)val;
1361 u16 val_16 = (u16)val;
1365 * Map all accesses to the local device on root bus into the
1366 * MMIO space of the MAC. Accesses to the downstream devices
1367 * go to the PIO space.
1369 if (pci_is_root_bus(bus)) {
1372 * This is the internal downstream P2P bridge,
1375 unsigned int reg_offset;
1377 reg_offset = ((offset & 0xFFF) <<
1378 TRIO_CFG_REGION_ADDR__REG_SHIFT) |
1379 (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_PROTECTED
1380 << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
1382 TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
1384 mmio_addr = trio_context->mmio_base_mac + reg_offset;
1390 * We fake an empty device for (device > 0),
1391 * since there is only one device on bus 0.
1393 goto invalid_device;
1398 * Accesses to the directly attached device have to be
1399 * sent as type-0 configs.
1401 if (busnum == (controller->first_busno + 1)) {
1403 * There is only one device off of our built-in P2P bridge.
1406 goto invalid_device;
1412 cfg_addr.reg_addr = (offset & 0xFFF);
1413 cfg_addr.fn = function;
1414 cfg_addr.dev = device;
1415 cfg_addr.bus = busnum;
1416 cfg_addr.type = config_type;
1419 * Note that we don't set the mac field in cfg_addr because the
1420 * mapping is per port.
1422 mmio_addr = trio_context->mmio_base_pio_cfg[controller->mac] +
1429 __gxio_mmio_write32(mmio_addr, val_32);
1430 TRACE_CFG_WR(size, val_32, busnum, device, function, offset);
1434 __gxio_mmio_write16(mmio_addr, val_16);
1435 TRACE_CFG_WR(size, val_16, busnum, device, function, offset);
1439 __gxio_mmio_write8(mmio_addr, val_8);
1440 TRACE_CFG_WR(size, val_8, busnum, device, function, offset);
1444 return PCIBIOS_FUNC_NOT_SUPPORTED;
1453 static struct pci_ops tile_cfg_ops = {
1454 .read = tile_cfg_read,
1455 .write = tile_cfg_write,
1459 /* MSI support starts here. */
1460 static unsigned int tilegx_msi_startup(struct irq_data *d)
1468 static void tilegx_msi_ack(struct irq_data *d)
1470 __insn_mtspr(SPR_IPI_EVENT_RESET_K, 1UL << d->irq);
1473 static void tilegx_msi_mask(struct irq_data *d)
1476 __insn_mtspr(SPR_IPI_MASK_SET_K, 1UL << d->irq);
1479 static void tilegx_msi_unmask(struct irq_data *d)
1481 __insn_mtspr(SPR_IPI_MASK_RESET_K, 1UL << d->irq);
1485 static struct irq_chip tilegx_msi_chip = {
1486 .name = "tilegx_msi",
1487 .irq_startup = tilegx_msi_startup,
1488 .irq_ack = tilegx_msi_ack,
1489 .irq_mask = tilegx_msi_mask,
1490 .irq_unmask = tilegx_msi_unmask,
1492 /* TBD: support set_affinity. */
1495 int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc)
1497 struct pci_controller *controller;
1498 gxio_trio_context_t *trio_context;
1501 uint64_t mem_map_base;
1502 uint64_t mem_map_limit;
1514 * Since we use a 64-bit Mem-Map to accept the MSI write, we fail
1515 * devices that are not capable of generating a 64-bit message address.
1516 * These devices will fall back to using the legacy interrupts.
1517 * Most PCIe endpoint devices do support 64-bit message addressing.
1519 if (desc->msi_attrib.is_64 == 0) {
1520 dev_printk(KERN_INFO, &pdev->dev,
1521 "64-bit MSI message address not supported, "
1522 "falling back to legacy interrupts.\n");
1528 default_irq = desc->msi_attrib.default_irq;
1529 controller = irq_get_handler_data(default_irq);
1531 BUG_ON(!controller);
1533 trio_context = controller->trio;
1536 * Allocate a scatter-queue that will accept the MSI write and
1537 * trigger the TILE-side interrupts. We use the scatter-queue regions
1538 * before the mem map regions, because the latter are needed by more
1541 mem_map = gxio_trio_alloc_scatter_queues(trio_context, 1, 0, 0);
1543 TRIO_MAP_SQ_DOORBELL_FMT_t doorbell_template = {{
1548 mem_map += TRIO_NUM_MAP_MEM_REGIONS;
1549 mem_map_base = MEM_MAP_INTR_REGIONS_BASE +
1550 mem_map * MEM_MAP_INTR_REGION_SIZE;
1551 mem_map_limit = mem_map_base + MEM_MAP_INTR_REGION_SIZE - 1;
1553 msi_addr = mem_map_base + MEM_MAP_INTR_REGION_SIZE - 8;
1554 msg.data = (unsigned int)doorbell_template.word;
1556 /* SQ regions are out, allocate from map mem regions. */
1557 mem_map = gxio_trio_alloc_memory_maps(trio_context, 1, 0, 0);
1559 dev_printk(KERN_INFO, &pdev->dev,
1560 "%s Mem-Map alloc failure. "
1561 "Failed to initialize MSI interrupts. "
1562 "Falling back to legacy interrupts.\n",
1563 desc->msi_attrib.is_msix ? "MSI-X" : "MSI");
1565 goto msi_mem_map_alloc_failure;
1568 mem_map_base = MEM_MAP_INTR_REGIONS_BASE +
1569 mem_map * MEM_MAP_INTR_REGION_SIZE;
1570 mem_map_limit = mem_map_base + MEM_MAP_INTR_REGION_SIZE - 1;
1572 msi_addr = mem_map_base + TRIO_MAP_MEM_REG_INT3 -
1573 TRIO_MAP_MEM_REG_INT0;
1578 /* We try to distribute different IRQs to different tiles. */
1579 cpu = tile_irq_cpu(irq);
1582 * Now call up to the HV to configure the MSI interrupt and
1583 * set up the IPI binding.
1585 ret = gxio_trio_config_msi_intr(trio_context, cpu_x(cpu), cpu_y(cpu),
1586 KERNEL_PL, irq, controller->mac,
1587 mem_map, mem_map_base, mem_map_limit,
1588 trio_context->asid);
1590 dev_printk(KERN_INFO, &pdev->dev, "HV MSI config failed.\n");
1592 goto hv_msi_config_failure;
1595 irq_set_msi_desc(irq, desc);
1597 msg.address_hi = msi_addr >> 32;
1598 msg.address_lo = msi_addr & 0xffffffff;
1600 write_msi_msg(irq, &msg);
1601 irq_set_chip_and_handler(irq, &tilegx_msi_chip, handle_level_irq);
1602 irq_set_handler_data(irq, controller);
1606 hv_msi_config_failure:
1608 msi_mem_map_alloc_failure:
1614 void arch_teardown_msi_irq(unsigned int irq)