/* * Copyright 2010 Ben Herrenschmidt, IBM Corporation * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ #define DEBUG #include <linux/kernel.h> #include <linux/pci.h> #include <linux/delay.h> #include <linux/string.h> #include <linux/init.h> #include <linux/bootmem.h> #include <linux/irq.h> #include <linux/interrupt.h> #include <linux/debugfs.h> #include <asm/sections.h> #include <asm/io.h> #include <asm/prom.h> #include <asm/pci-bridge.h> #include <asm/machdep.h> #include <asm/ppc-pci.h> #include <asm/iommu.h> #include <asm/io-workarounds.h> #include <asm/debug.h> #include "wsp.h" #include "wsp_pci.h" #include "msi.h" /* Max number of TVTs for one table. Only 32-bit tables can use * multiple TVTs and so the max currently supported is thus 8 * since only 2G of DMA space is supported */ #define MAX_TABLE_TVT_COUNT 8 struct wsp_dma_table { struct list_head link; struct iommu_table table; struct wsp_phb *phb; struct page *tces[MAX_TABLE_TVT_COUNT]; }; /* We support DMA regions from 0...2G in 32bit space (no support for * 64-bit DMA just yet). Each device gets a separate TCE table (TVT * entry) with validation enabled (though not supported by SimiCS * just yet). * * To simplify things, we divide this 2G space into N regions based * on the constant below which could be turned into a tunable eventually * * We then assign dynamically those regions to devices as they show up. * * We use a bitmap as an allocator for these. * * Tables are allocated/created dynamically as devices are discovered, * multiple TVT entries are used if needed * * When 64-bit DMA support is added we should simply use a separate set * of larger regions (the HW supports 64 TVT entries). We can * additionally create a bypass region in 64-bit space for performances * though that would have a cost in term of security. * * If you set NUM_DMA32_REGIONS to 1, then a single table is shared * for all devices and bus/dev/fn validation is disabled * * Note that a DMA32 region cannot be smaller than 256M so the max * supported here for now is 8. We don't yet support sharing regions * between multiple devices so the max number of devices supported * is MAX_TABLE_TVT_COUNT. */ #define NUM_DMA32_REGIONS 1 struct wsp_phb { struct pci_controller *hose; /* Lock controlling access to the list of dma tables. * It does -not- protect against dma_* operations on * those tables, those should be stopped before an entry * is removed from the list. * * The lock is also used for error handling operations */ spinlock_t lock; struct list_head dma_tables; unsigned long dma32_map; unsigned long dma32_base; unsigned int dma32_num_regions; unsigned long dma32_region_size; /* Debugfs stuff */ struct dentry *ddir; struct list_head all; }; static LIST_HEAD(wsp_phbs); //#define cfg_debug(fmt...) pr_debug(fmt) #define cfg_debug(fmt...) static int wsp_pcie_read_config(struct pci_bus *bus, unsigned int devfn, int offset, int len, u32 *val) { struct pci_controller *hose; int suboff; u64 addr; hose = pci_bus_to_host(bus); if (hose == NULL) return PCIBIOS_DEVICE_NOT_FOUND; if (offset >= 0x1000) return PCIBIOS_BAD_REGISTER_NUMBER; addr = PCIE_REG_CA_ENABLE | ((u64)bus->number) << PCIE_REG_CA_BUS_SHIFT | ((u64)devfn) << PCIE_REG_CA_FUNC_SHIFT | ((u64)offset & ~3) << PCIE_REG_CA_REG_SHIFT; suboff = offset & 3; /* * Note: the caller has already checked that offset is * suitably aligned and that len is 1, 2 or 4. */ switch (len) { case 1: addr |= (0x8ul >> suboff) << PCIE_REG_CA_BE_SHIFT; out_be64(hose->cfg_data + PCIE_REG_CONFIG_ADDRESS, addr); *val = (in_le32(hose->cfg_data + PCIE_REG_CONFIG_DATA) >> (suboff << 3)) & 0xff; cfg_debug("read 1 %02x:%02x:%02x + %02x/%x addr=0x%llx val=%02x\n", bus->number, devfn >> 3, devfn & 7, offset, suboff, addr, *val); break; case 2: addr |= (0xcul >> suboff) << PCIE_REG_CA_BE_SHIFT; out_be64(hose->cfg_data + PCIE_REG_CONFIG_ADDRESS, addr); *val = (in_le32(hose->cfg_data + PCIE_REG_CONFIG_DATA) >> (suboff << 3)) & 0xffff; cfg_debug("read 2 %02x:%02x:%02x + %02x/%x addr=0x%llx val=%04x\n", bus->number, devfn >> 3, devfn & 7, offset, suboff, addr, *val); break; default: addr |= 0xful << PCIE_REG_CA_BE_SHIFT; out_be64(hose->cfg_data + PCIE_REG_CONFIG_ADDRESS, addr); *val = in_le32(hose->cfg_data + PCIE_REG_CONFIG_DATA); cfg_debug("read 4 %02x:%02x:%02x + %02x/%x addr=0x%llx val=%08x\n", bus->number, devfn >> 3, devfn & 7, offset, suboff, addr, *val); break; } return PCIBIOS_SUCCESSFUL; } static int wsp_pcie_write_config(struct pci_bus *bus, unsigned int devfn, int offset, int len, u32 val) { struct pci_controller *hose; int suboff; u64 addr; hose = pci_bus_to_host(bus); if (hose == NULL) return PCIBIOS_DEVICE_NOT_FOUND; if (offset >= 0x1000) return PCIBIOS_BAD_REGISTER_NUMBER; addr = PCIE_REG_CA_ENABLE | ((u64)bus->number) << PCIE_REG_CA_BUS_SHIFT | ((u64)devfn) << PCIE_REG_CA_FUNC_SHIFT | ((u64)offset & ~3) << PCIE_REG_CA_REG_SHIFT; suboff = offset & 3; /* * Note: the caller has already checked that offset is * suitably aligned and that len is 1, 2 or 4. */ switch (len) { case 1: addr |= (0x8ul >> suboff) << PCIE_REG_CA_BE_SHIFT; val <<= suboff << 3; out_be64(hose->cfg_data + PCIE_REG_CONFIG_ADDRESS, addr); out_le32(hose->cfg_data + PCIE_REG_CONFIG_DATA, val); cfg_debug("write 1 %02x:%02x:%02x + %02x/%x addr=0x%llx val=%02x\n", bus->number, devfn >> 3, devfn & 7, offset, suboff, addr, val); break; case 2: addr |= (0xcul >> suboff) << PCIE_REG_CA_BE_SHIFT; val <<= suboff << 3; out_be64(hose->cfg_data + PCIE_REG_CONFIG_ADDRESS, addr); out_le32(hose->cfg_data + PCIE_REG_CONFIG_DATA, val); cfg_debug("write 2 %02x:%02x:%02x + %02x/%x addr=0x%llx val=%04x\n", bus->number, devfn >> 3, devfn & 7, offset, suboff, addr, val); break; default: addr |= 0xful << PCIE_REG_CA_BE_SHIFT; out_be64(hose->cfg_data + PCIE_REG_CONFIG_ADDRESS, addr); out_le32(hose->cfg_data + PCIE_REG_CONFIG_DATA, val); cfg_debug("write 4 %02x:%02x:%02x + %02x/%x addr=0x%llx val=%08x\n", bus->number, devfn >> 3, devfn & 7, offset, suboff, addr, val); break; } return PCIBIOS_SUCCESSFUL; } static struct pci_ops wsp_pcie_pci_ops = { .read = wsp_pcie_read_config, .write = wsp_pcie_write_config, }; #define TCE_SHIFT 12 #define TCE_PAGE_SIZE (1 << TCE_SHIFT) #define TCE_PCI_WRITE 0x2 /* write from PCI allowed */ #define TCE_PCI_READ 0x1 /* read from PCI allowed */ #define TCE_RPN_MASK 0x3fffffffffful /* 42-bit RPN (4K pages) */ #define TCE_RPN_SHIFT 12 //#define dma_debug(fmt...) pr_debug(fmt) #define dma_debug(fmt...) static int tce_build_wsp(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, struct dma_attrs *attrs) { struct wsp_dma_table *ptbl = container_of(tbl, struct wsp_dma_table, table); u64 proto_tce; u64 *tcep; u64 rpn; proto_tce = TCE_PCI_READ; #ifdef CONFIG_WSP_DD1_WORKAROUND_DD1_TCE_BUGS proto_tce |= TCE_PCI_WRITE; #else if (direction != DMA_TO_DEVICE) proto_tce |= TCE_PCI_WRITE; #endif /* XXX Make this faster by factoring out the page address for * within a TCE table */ while (npages--) { /* We don't use it->base as the table can be scattered */ tcep = (u64 *)page_address(ptbl->tces[index >> 16]); tcep += (index & 0xffff); /* can't move this out since we might cross LMB boundary */ rpn = __pa(uaddr) >> TCE_SHIFT; *tcep = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT; dma_debug("[DMA] TCE %p set to 0x%016llx (dma addr: 0x%lx)\n", tcep, *tcep, (tbl->it_offset + index) << IOMMU_PAGE_SHIFT_4K); uaddr += TCE_PAGE_SIZE; index++; } return 0; } static void tce_free_wsp(struct iommu_table *tbl, long index, long npages) { struct wsp_dma_table *ptbl = container_of(tbl, struct wsp_dma_table, table); #ifndef CONFIG_WSP_DD1_WORKAROUND_DD1_TCE_BUGS struct pci_controller *hose = ptbl->phb->hose; #endif u64 *tcep; /* XXX Make this faster by factoring out the page address for * within a TCE table. Also use line-kill option to kill multiple * TCEs at once */ while (npages--) { /* We don't use it->base as the table can be scattered */ tcep = (u64 *)page_address(ptbl->tces[index >> 16]); tcep += (index & 0xffff); dma_debug("[DMA] TCE %p cleared\n", tcep); *tcep = 0; #ifndef CONFIG_WSP_DD1_WORKAROUND_DD1_TCE_BUGS /* Don't write there since it would pollute other MMIO accesses */ out_be64(hose->cfg_data + PCIE_REG_TCE_KILL, PCIE_REG_TCEKILL_SINGLE | PCIE_REG_TCEKILL_PS_4K | (__pa(tcep) & PCIE_REG_TCEKILL_ADDR_MASK)); #endif index++; } } static struct wsp_dma_table *wsp_pci_create_dma32_table(struct wsp_phb *phb, unsigned int region, struct pci_dev *validate) { struct pci_controller *hose = phb->hose; unsigned long size = phb->dma32_region_size; unsigned long addr = phb->dma32_region_size * region + phb->dma32_base; struct wsp_dma_table *tbl; int tvts_per_table, i, tvt, nid; unsigned long flags; nid = of_node_to_nid(phb->hose->dn); /* Calculate how many TVTs are needed */ tvts_per_table = size / 0x10000000; if (tvts_per_table == 0) tvts_per_table = 1; /* Calculate the base TVT index. We know all tables have the same * size so we just do a simple multiply here */ tvt = region * tvts_per_table; pr_debug(" Region : %d\n", region); pr_debug(" DMA range : 0x%08lx..0x%08lx\n", addr, addr + size - 1); pr_debug(" Number of TVTs : %d\n", tvts_per_table); pr_debug(" Base TVT : %d\n", tvt); pr_debug(" Node : %d\n", nid); tbl = kzalloc_node(sizeof(struct wsp_dma_table), GFP_KERNEL, nid); if (!tbl) return ERR_PTR(-ENOMEM); tbl->phb = phb; /* Create as many TVTs as needed, each represents 256M at most */ for (i = 0; i < tvts_per_table; i++) { u64 tvt_data1, tvt_data0; /* Allocate table. We use a 4K TCE size for now always so * one table is always 8 * (258M / 4K) == 512K */ tbl->tces[i] = alloc_pages_node(nid, GFP_KERNEL, get_order(0x80000)); if (tbl->tces[i] == NULL) goto fail; memset(page_address(tbl->tces[i]), 0, 0x80000); pr_debug(" TCE table %d at : %p\n", i, page_address(tbl->tces[i])); /* Table size. We currently set it to be the whole 256M region */ tvt_data0 = 2ull << IODA_TVT0_TCE_TABLE_SIZE_SHIFT; /* IO page size set to 4K */ tvt_data1 = 1ull << IODA_TVT1_IO_PAGE_SIZE_SHIFT; /* Shift in the address */ tvt_data0 |= __pa(page_address(tbl->tces[i])) << IODA_TVT0_TTA_SHIFT; /* Validation stuff. We only validate fully bus/dev/fn for now * one day maybe we can group devices but that isn't the case * at the moment */ if (validate) { tvt_data0 |= IODA_TVT0_BUSNUM_VALID_MASK; tvt_data0 |= validate->bus->number; tvt_data1 |= IODA_TVT1_DEVNUM_VALID; tvt_data1 |= ((u64)PCI_SLOT(validate->devfn)) << IODA_TVT1_DEVNUM_VALUE_SHIFT; tvt_data1 |= IODA_TVT1_FUNCNUM_VALID; tvt_data1 |= ((u64)PCI_FUNC(validate->devfn)) << IODA_TVT1_FUNCNUM_VALUE_SHIFT; } /* XX PE number is always 0 for now */ /* Program the values using the PHB lock */ spin_lock_irqsave(&phb->lock, flags); out_be64(hose->cfg_data + PCIE_REG_IODA_ADDR, (tvt + i) | PCIE_REG_IODA_AD_TBL_TVT); out_be64(hose->cfg_data + PCIE_REG_IODA_DATA1, tvt_data1); out_be64(hose->cfg_data + PCIE_REG_IODA_DATA0, tvt_data0); spin_unlock_irqrestore(&phb->lock, flags); } /* Init bits and pieces */ tbl->table.it_blocksize = 16; tbl->table.it_page_shift = IOMMU_PAGE_SHIFT_4K; tbl->table.it_offset = addr >> tbl->table.it_page_shift; tbl->table.it_size = size >> tbl->table.it_page_shift; /* * It's already blank but we clear it anyway. * Consider an aditiona interface that makes cleaing optional */ iommu_init_table(&tbl->table, nid); list_add(&tbl->link, &phb->dma_tables); return tbl; fail: pr_debug(" Failed to allocate a 256M TCE table !\n"); for (i = 0; i < tvts_per_table; i++) if (tbl->tces[i]) __free_pages(tbl->tces[i], get_order(0x80000)); kfree(tbl); return ERR_PTR(-ENOMEM); } static void wsp_pci_dma_dev_setup(struct pci_dev *pdev) { struct dev_archdata *archdata = &pdev->dev.archdata; struct pci_controller *hose = pci_bus_to_host(pdev->bus); struct wsp_phb *phb = hose->private_data; struct wsp_dma_table *table = NULL; unsigned long flags; int i; /* Don't assign an iommu table to a bridge */ if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) return; pr_debug("%s: Setting up DMA...\n", pci_name(pdev)); spin_lock_irqsave(&phb->lock, flags); /* If only one region, check if it already exist */ if (phb->dma32_num_regions == 1) { spin_unlock_irqrestore(&phb->lock, flags); if (list_empty(&phb->dma_tables)) table = wsp_pci_create_dma32_table(phb, 0, NULL); else table = list_first_entry(&phb->dma_tables, struct wsp_dma_table, link); } else { /* else find a free region */ for (i = 0; i < phb->dma32_num_regions && !table; i++) { if (__test_and_set_bit(i, &phb->dma32_map)) continue; spin_unlock_irqrestore(&phb->lock, flags); table = wsp_pci_create_dma32_table(phb, i, pdev); } } /* Check if we got an error */ if (IS_ERR(table)) { pr_err("%s: Failed to create DMA table, err %ld !\n", pci_name(pdev), PTR_ERR(table)); return; } /* Or a valid table */ if (table) { pr_info("%s: Setup iommu: 32-bit DMA region 0x%08lx..0x%08lx\n", pci_name(pdev), table->table.it_offset << IOMMU_PAGE_SHIFT_4K, (table->table.it_offset << IOMMU_PAGE_SHIFT_4K) + phb->dma32_region_size - 1); archdata->dma_data.iommu_table_base = &table->table; return; } /* Or no room */ spin_unlock_irqrestore(&phb->lock, flags); pr_err("%s: Out of DMA space !\n", pci_name(pdev)); } static void __init wsp_pcie_configure_hw(struct pci_controller *hose) { u64 val; int i; #define DUMP_REG(x) \ pr_debug("%-30s : 0x%016llx\n", #x, in_be64(hose->cfg_data + x)) /* * Some WSP variants has a bogus class code by default in the PCI-E * root complex's built-in P2P bridge */ val = in_be64(hose->cfg_data + PCIE_REG_SYS_CFG1); pr_debug("PCI-E SYS_CFG1 : 0x%llx\n", val); out_be64(hose->cfg_data + PCIE_REG_SYS_CFG1, (val & ~PCIE_REG_SYS_CFG1_CLASS_CODE) | (PCI_CLASS_BRIDGE_PCI << 8)); pr_debug("PCI-E SYS_CFG1 : 0x%llx\n", in_be64(hose->cfg_data + PCIE_REG_SYS_CFG1)); #ifdef CONFIG_WSP_DD1_WORKAROUND_DD1_TCE_BUGS /* XXX Disable TCE caching, it doesn't work on DD1 */ out_be64(hose->cfg_data + 0xe50, in_be64(hose->cfg_data + 0xe50) | (3ull << 62)); printk("PCI-E DEBUG CONTROL 5 = 0x%llx\n", in_be64(hose->cfg_data + 0xe50)); #endif /* Configure M32A and IO. IO is hard wired to be 1M for now */ out_be64(hose->cfg_data + PCIE_REG_IO_BASE_ADDR, hose->io_base_phys); out_be64(hose->cfg_data + PCIE_REG_IO_BASE_MASK, (~(hose->io_resource.end - hose->io_resource.start)) & 0x3fffffff000ul); out_be64(hose->cfg_data + PCIE_REG_IO_START_ADDR, 0 | 1); out_be64(hose->cfg_data + PCIE_REG_M32A_BASE_ADDR, hose->mem_resources[0].start); printk("Want to write to M32A_BASE_MASK : 0x%llx\n", (~(hose->mem_resources[0].end - hose->mem_resources[0].start)) & 0x3ffffff0000ul); out_be64(hose->cfg_data + PCIE_REG_M32A_BASE_MASK, (~(hose->mem_resources[0].end - hose->mem_resources[0].start)) & 0x3ffffff0000ul); out_be64(hose->cfg_data + PCIE_REG_M32A_START_ADDR, (hose->mem_resources[0].start - hose->mem_offset[0]) | 1); /* Clear all TVT entries * * XX Might get TVT count from device-tree */ for (i = 0; i < IODA_TVT_COUNT; i++) { out_be64(hose->cfg_data + PCIE_REG_IODA_ADDR, PCIE_REG_IODA_AD_TBL_TVT | i); out_be64(hose->cfg_data + PCIE_REG_IODA_DATA1, 0); out_be64(hose->cfg_data + PCIE_REG_IODA_DATA0, 0); } /* Kill the TCE cache */ out_be64(hose->cfg_data + PCIE_REG_PHB_CONFIG, in_be64(hose->cfg_data + PCIE_REG_PHB_CONFIG) | PCIE_REG_PHBC_64B_TCE_EN); /* Enable 32 & 64-bit MSIs, IO space and M32A */ val = PCIE_REG_PHBC_32BIT_MSI_EN | PCIE_REG_PHBC_IO_EN | PCIE_REG_PHBC_64BIT_MSI_EN | PCIE_REG_PHBC_M32A_EN; if (iommu_is_off) val |= PCIE_REG_PHBC_DMA_XLATE_BYPASS; pr_debug("Will write config: 0x%llx\n", val); out_be64(hose->cfg_data + PCIE_REG_PHB_CONFIG, val); /* Enable error reporting */ out_be64(hose->cfg_data + 0xe00, in_be64(hose->cfg_data + 0xe00) | 0x0008000000000000ull); /* Mask an error that's generated when doing config space probe * * XXX Maybe we should only mask it around config space cycles... that or * ignore it when we know we had a config space cycle recently ? */ out_be64(hose->cfg_data + PCIE_REG_DMA_ERR_STATUS_MASK, 0x8000000000000000ull); out_be64(hose->cfg_data + PCIE_REG_DMA_ERR1_STATUS_MASK, 0x8000000000000000ull); /* Enable UTL errors, for now, all of them got to UTL irq 1 * * We similarily mask one UTL error caused apparently during normal * probing. We also mask the link up error */ out_be64(hose->cfg_data + PCIE_UTL_SYS_BUS_AGENT_ERR_SEV, 0); out_be64(hose->cfg_data + PCIE_UTL_RC_ERR_SEVERITY, 0); out_be64(hose->cfg_data + PCIE_UTL_PCIE_PORT_ERROR_SEV, 0); out_be64(hose->cfg_data + PCIE_UTL_SYS_BUS_AGENT_IRQ_EN, 0xffffffff00000000ull); out_be64(hose->cfg_data + PCIE_UTL_PCIE_PORT_IRQ_EN, 0xff5fffff00000000ull); out_be64(hose->cfg_data + PCIE_UTL_EP_ERR_IRQ_EN, 0xffffffff00000000ull); DUMP_REG(PCIE_REG_IO_BASE_ADDR); DUMP_REG(PCIE_REG_IO_BASE_MASK); DUMP_REG(PCIE_REG_IO_START_ADDR); DUMP_REG(PCIE_REG_M32A_BASE_ADDR); DUMP_REG(PCIE_REG_M32A_BASE_MASK); DUMP_REG(PCIE_REG_M32A_START_ADDR); DUMP_REG(PCIE_REG_M32B_BASE_ADDR); DUMP_REG(PCIE_REG_M32B_BASE_MASK); DUMP_REG(PCIE_REG_M32B_START_ADDR); DUMP_REG(PCIE_REG_M64_BASE_ADDR); DUMP_REG(PCIE_REG_M64_BASE_MASK); DUMP_REG(PCIE_REG_M64_START_ADDR); DUMP_REG(PCIE_REG_PHB_CONFIG); } static void wsp_pci_wait_io_idle(struct wsp_phb *phb, unsigned long port) { u64 val; int i; for (i = 0; i < 10000; i++) { val = in_be64(phb->hose->cfg_data + 0xe08); if ((val & 0x1900000000000000ull) == 0x0100000000000000ull) return; udelay(1); } pr_warning("PCI IO timeout on domain %d port 0x%lx\n", phb->hose->global_number, port); } #define DEF_PCI_AC_RET_pio(name, ret, at, al, aa) \ static ret wsp_pci_##name at \ { \ struct iowa_bus *bus; \ struct wsp_phb *phb; \ unsigned long flags; \ ret rval; \ bus = iowa_pio_find_bus(aa); \ WARN_ON(!bus); \ phb = bus->private; \ spin_lock_irqsave(&phb->lock, flags); \ wsp_pci_wait_io_idle(phb, aa); \ rval = __do_##name al; \ spin_unlock_irqrestore(&phb->lock, flags); \ return rval; \ } #define DEF_PCI_AC_NORET_pio(name, at, al, aa) \ static void wsp_pci_##name at \ { \ struct iowa_bus *bus; \ struct wsp_phb *phb; \ unsigned long flags; \ bus = iowa_pio_find_bus(aa); \ WARN_ON(!bus); \ phb = bus->private; \ spin_lock_irqsave(&phb->lock, flags); \ wsp_pci_wait_io_idle(phb, aa); \ __do_##name al; \ spin_unlock_irqrestore(&phb->lock, flags); \ } #define DEF_PCI_AC_RET_mem(name, ret, at, al, aa) #define DEF_PCI_AC_NORET_mem(name, at, al, aa) #define DEF_PCI_AC_RET(name, ret, at, al, space, aa) \ DEF_PCI_AC_RET_##space(name, ret, at, al, aa) #define DEF_PCI_AC_NORET(name, at, al, space, aa) \ DEF_PCI_AC_NORET_##space(name, at, al, aa) \ #include <asm/io-defs.h> #undef DEF_PCI_AC_RET #undef DEF_PCI_AC_NORET static struct ppc_pci_io wsp_pci_iops = { .inb = wsp_pci_inb, .inw = wsp_pci_inw, .inl = wsp_pci_inl, .outb = wsp_pci_outb, .outw = wsp_pci_outw, .outl = wsp_pci_outl, .insb = wsp_pci_insb, .insw = wsp_pci_insw, .insl = wsp_pci_insl, .outsb = wsp_pci_outsb, .outsw = wsp_pci_outsw, .outsl = wsp_pci_outsl, }; static int __init wsp_setup_one_phb(struct device_node *np) { struct pci_controller *hose; struct wsp_phb *phb; pr_info("PCI: Setting up PCIe host bridge 0x%s\n", np->full_name); phb = zalloc_maybe_bootmem(sizeof(struct wsp_phb), GFP_KERNEL); if (!phb) return -ENOMEM; hose = pcibios_alloc_controller(np); if (!hose) { /* Can't really free the phb */ return -ENOMEM; } hose->private_data = phb; phb->hose = hose; INIT_LIST_HEAD(&phb->dma_tables); spin_lock_init(&phb->lock); /* XXX Use bus-range property ? */ hose->first_busno = 0; hose->last_busno = 0xff; /* We use cfg_data as the address for the whole bridge MMIO space */ hose->cfg_data = of_iomap(hose->dn, 0); pr_debug("PCIe registers mapped at 0x%p\n", hose->cfg_data); /* Get the ranges of the device-tree */ pci_process_bridge_OF_ranges(hose, np, 0); /* XXX Force re-assigning of everything for now */ pci_add_flags(PCI_REASSIGN_ALL_BUS | PCI_REASSIGN_ALL_RSRC | PCI_ENABLE_PROC_DOMAINS); /* Calculate how the TCE space is divided */ phb->dma32_base = 0; phb->dma32_num_regions = NUM_DMA32_REGIONS; if (phb->dma32_num_regions > MAX_TABLE_TVT_COUNT) { pr_warning("IOMMU: Clamped to %d DMA32 regions\n", MAX_TABLE_TVT_COUNT); phb->dma32_num_regions = MAX_TABLE_TVT_COUNT; } phb->dma32_region_size = 0x80000000 / phb->dma32_num_regions; BUG_ON(!is_power_of_2(phb->dma32_region_size)); /* Setup config ops */ hose->ops = &wsp_pcie_pci_ops; /* Configure the HW */ wsp_pcie_configure_hw(hose); /* Instanciate IO workarounds */ iowa_register_bus(hose, &wsp_pci_iops, NULL, phb); #ifdef CONFIG_PCI_MSI wsp_setup_phb_msi(hose); #endif /* Add to global list */ list_add(&phb->all, &wsp_phbs); return 0; } void __init wsp_setup_pci(void) { struct device_node *np; int rc; /* Find host bridges */ for_each_compatible_node(np, "pciex", PCIE_COMPATIBLE) { rc = wsp_setup_one_phb(np); if (rc) pr_err("Failed to setup PCIe bridge %s, rc=%d\n", np->full_name, rc); } /* Establish device-tree linkage */ pci_devs_phb_init(); /* Set DMA ops to use TCEs */ if (iommu_is_off) { pr_info("PCI-E: Disabled TCEs, using direct DMA\n"); set_pci_dma_ops(&dma_direct_ops); } else { ppc_md.pci_dma_dev_setup = wsp_pci_dma_dev_setup; ppc_md.tce_build = tce_build_wsp; ppc_md.tce_free = tce_free_wsp; set_pci_dma_ops(&dma_iommu_ops); } } #define err_debug(fmt...) pr_debug(fmt) //#define err_debug(fmt...) static int __init wsp_pci_get_err_irq_no_dt(struct device_node *np) { const u32 *prop; int hw_irq; /* Ok, no interrupts property, let's try to find our child P2P */ np = of_get_next_child(np, NULL); if (np == NULL) return 0; /* Grab it's interrupt map */ prop = of_get_property(np, "interrupt-map", NULL); if (prop == NULL) return 0; /* Grab one of the interrupts in there, keep the low 4 bits */ hw_irq = prop[5] & 0xf; /* 0..4 for PHB 0 and 5..9 for PHB 1 */ if (hw_irq < 5) hw_irq = 4; else hw_irq = 9; hw_irq |= prop[5] & ~0xf; err_debug("PCI: Using 0x%x as error IRQ for %s\n", hw_irq, np->parent->full_name); return irq_create_mapping(NULL, hw_irq); } static const struct { u32 offset; const char *name; } wsp_pci_regs[] = { #define DREG(x) { PCIE_REG_##x, #x } #define DUTL(x) { PCIE_UTL_##x, "UTL_" #x } /* Architected registers except CONFIG_ and IODA * to avoid side effects */ DREG(DMA_CHAN_STATUS), DREG(CPU_LOADSTORE_STATUS), DREG(LOCK0), DREG(LOCK1), DREG(PHB_CONFIG), DREG(IO_BASE_ADDR), DREG(IO_BASE_MASK), DREG(IO_START_ADDR), DREG(M32A_BASE_ADDR), DREG(M32A_BASE_MASK), DREG(M32A_START_ADDR), DREG(M32B_BASE_ADDR), DREG(M32B_BASE_MASK), DREG(M32B_START_ADDR), DREG(M64_BASE_ADDR), DREG(M64_BASE_MASK), DREG(M64_START_ADDR), DREG(TCE_KILL), DREG(LOCK2), DREG(PHB_GEN_CAP), DREG(PHB_TCE_CAP), DREG(PHB_IRQ_CAP), DREG(PHB_EEH_CAP), DREG(PAPR_ERR_INJ_CONTROL), DREG(PAPR_ERR_INJ_ADDR), DREG(PAPR_ERR_INJ_MASK), /* UTL core regs */ DUTL(SYS_BUS_CONTROL), DUTL(STATUS), DUTL(SYS_BUS_AGENT_STATUS), DUTL(SYS_BUS_AGENT_ERR_SEV), DUTL(SYS_BUS_AGENT_IRQ_EN), DUTL(SYS_BUS_BURST_SZ_CONF), DUTL(REVISION_ID), DUTL(OUT_POST_HDR_BUF_ALLOC), DUTL(OUT_POST_DAT_BUF_ALLOC), DUTL(IN_POST_HDR_BUF_ALLOC), DUTL(IN_POST_DAT_BUF_ALLOC), DUTL(OUT_NP_BUF_ALLOC), DUTL(IN_NP_BUF_ALLOC), DUTL(PCIE_TAGS_ALLOC), DUTL(GBIF_READ_TAGS_ALLOC), DUTL(PCIE_PORT_CONTROL), DUTL(PCIE_PORT_STATUS), DUTL(PCIE_PORT_ERROR_SEV), DUTL(PCIE_PORT_IRQ_EN), DUTL(RC_STATUS), DUTL(RC_ERR_SEVERITY), DUTL(RC_IRQ_EN), DUTL(EP_STATUS), DUTL(EP_ERR_SEVERITY), DUTL(EP_ERR_IRQ_EN), DUTL(PCI_PM_CTRL1), DUTL(PCI_PM_CTRL2), /* PCIe stack regs */ DREG(SYSTEM_CONFIG1), DREG(SYSTEM_CONFIG2), DREG(EP_SYSTEM_CONFIG), DREG(EP_FLR), DREG(EP_BAR_CONFIG), DREG(LINK_CONFIG), DREG(PM_CONFIG), DREG(DLP_CONTROL), DREG(DLP_STATUS), DREG(ERR_REPORT_CONTROL), DREG(SLOT_CONTROL1), DREG(SLOT_CONTROL2), DREG(UTL_CONFIG), DREG(BUFFERS_CONFIG), DREG(ERROR_INJECT), DREG(SRIOV_CONFIG), DREG(PF0_SRIOV_STATUS), DREG(PF1_SRIOV_STATUS), DREG(PORT_NUMBER), DREG(POR_SYSTEM_CONFIG), /* Internal logic regs */ DREG(PHB_VERSION), DREG(RESET), DREG(PHB_CONTROL), DREG(PHB_TIMEOUT_CONTROL1), DREG(PHB_QUIESCE_DMA), DREG(PHB_DMA_READ_TAG_ACTV), DREG(PHB_TCE_READ_TAG_ACTV), /* FIR registers */ DREG(LEM_FIR_ACCUM), DREG(LEM_FIR_AND_MASK), DREG(LEM_FIR_OR_MASK), DREG(LEM_ACTION0), DREG(LEM_ACTION1), DREG(LEM_ERROR_MASK), DREG(LEM_ERROR_AND_MASK), DREG(LEM_ERROR_OR_MASK), /* Error traps registers */ DREG(PHB_ERR_STATUS), DREG(PHB_ERR_STATUS), DREG(PHB_ERR1_STATUS), DREG(PHB_ERR_INJECT), DREG(PHB_ERR_LEM_ENABLE), DREG(PHB_ERR_IRQ_ENABLE), DREG(PHB_ERR_FREEZE_ENABLE), DREG(PHB_ERR_SIDE_ENABLE), DREG(PHB_ERR_LOG_0), DREG(PHB_ERR_LOG_1), DREG(PHB_ERR_STATUS_MASK), DREG(PHB_ERR1_STATUS_MASK), DREG(MMIO_ERR_STATUS), DREG(MMIO_ERR1_STATUS), DREG(MMIO_ERR_INJECT), DREG(MMIO_ERR_LEM_ENABLE), DREG(MMIO_ERR_IRQ_ENABLE), DREG(MMIO_ERR_FREEZE_ENABLE), DREG(MMIO_ERR_SIDE_ENABLE), DREG(MMIO_ERR_LOG_0), DREG(MMIO_ERR_LOG_1), DREG(MMIO_ERR_STATUS_MASK), DREG(MMIO_ERR1_STATUS_MASK), DREG(DMA_ERR_STATUS), DREG(DMA_ERR1_STATUS), DREG(DMA_ERR_INJECT), DREG(DMA_ERR_LEM_ENABLE), DREG(DMA_ERR_IRQ_ENABLE), DREG(DMA_ERR_FREEZE_ENABLE), DREG(DMA_ERR_SIDE_ENABLE), DREG(DMA_ERR_LOG_0), DREG(DMA_ERR_LOG_1), DREG(DMA_ERR_STATUS_MASK), DREG(DMA_ERR1_STATUS_MASK), /* Debug and Trace registers */ DREG(PHB_DEBUG_CONTROL0), DREG(PHB_DEBUG_STATUS0), DREG(PHB_DEBUG_CONTROL1), DREG(PHB_DEBUG_STATUS1), DREG(PHB_DEBUG_CONTROL2), DREG(PHB_DEBUG_STATUS2), DREG(PHB_DEBUG_CONTROL3), DREG(PHB_DEBUG_STATUS3), DREG(PHB_DEBUG_CONTROL4), DREG(PHB_DEBUG_STATUS4), DREG(PHB_DEBUG_CONTROL5), DREG(PHB_DEBUG_STATUS5), /* Don't seem to exist ... DREG(PHB_DEBUG_CONTROL6), DREG(PHB_DEBUG_STATUS6), */ }; static int wsp_pci_regs_show(struct seq_file *m, void *private) { struct wsp_phb *phb = m->private; struct pci_controller *hose = phb->hose; int i; for (i = 0; i < ARRAY_SIZE(wsp_pci_regs); i++) { /* Skip write-only regs */ if (wsp_pci_regs[i].offset == 0xc08 || wsp_pci_regs[i].offset == 0xc10 || wsp_pci_regs[i].offset == 0xc38 || wsp_pci_regs[i].offset == 0xc40) continue; seq_printf(m, "0x%03x: 0x%016llx %s\n", wsp_pci_regs[i].offset, in_be64(hose->cfg_data + wsp_pci_regs[i].offset), wsp_pci_regs[i].name); } return 0; } static int wsp_pci_regs_open(struct inode *inode, struct file *file) { return single_open(file, wsp_pci_regs_show, inode->i_private); } static const struct file_operations wsp_pci_regs_fops = { .open = wsp_pci_regs_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; static int wsp_pci_reg_set(void *data, u64 val) { out_be64((void __iomem *)data, val); return 0; } static int wsp_pci_reg_get(void *data, u64 *val) { *val = in_be64((void __iomem *)data); return 0; } DEFINE_SIMPLE_ATTRIBUTE(wsp_pci_reg_fops, wsp_pci_reg_get, wsp_pci_reg_set, "0x%llx\n"); static irqreturn_t wsp_pci_err_irq(int irq, void *dev_id) { struct wsp_phb *phb = dev_id; struct pci_controller *hose = phb->hose; irqreturn_t handled = IRQ_NONE; struct wsp_pcie_err_log_data ed; pr_err("PCI: Error interrupt on %s (PHB %d)\n", hose->dn->full_name, hose->global_number); again: memset(&ed, 0, sizeof(ed)); /* Read and clear UTL errors */ ed.utl_sys_err = in_be64(hose->cfg_data + PCIE_UTL_SYS_BUS_AGENT_STATUS); if (ed.utl_sys_err) out_be64(hose->cfg_data + PCIE_UTL_SYS_BUS_AGENT_STATUS, ed.utl_sys_err); ed.utl_port_err = in_be64(hose->cfg_data + PCIE_UTL_PCIE_PORT_STATUS); if (ed.utl_port_err) out_be64(hose->cfg_data + PCIE_UTL_PCIE_PORT_STATUS, ed.utl_port_err); ed.utl_rc_err = in_be64(hose->cfg_data + PCIE_UTL_RC_STATUS); if (ed.utl_rc_err) out_be64(hose->cfg_data + PCIE_UTL_RC_STATUS, ed.utl_rc_err); /* Read and clear main trap errors */ ed.phb_err = in_be64(hose->cfg_data + PCIE_REG_PHB_ERR_STATUS); if (ed.phb_err) { ed.phb_err1 = in_be64(hose->cfg_data + PCIE_REG_PHB_ERR1_STATUS); ed.phb_log0 = in_be64(hose->cfg_data + PCIE_REG_PHB_ERR_LOG_0); ed.phb_log1 = in_be64(hose->cfg_data + PCIE_REG_PHB_ERR_LOG_1); out_be64(hose->cfg_data + PCIE_REG_PHB_ERR1_STATUS, 0); out_be64(hose->cfg_data + PCIE_REG_PHB_ERR_STATUS, 0); } ed.mmio_err = in_be64(hose->cfg_data + PCIE_REG_MMIO_ERR_STATUS); if (ed.mmio_err) { ed.mmio_err1 = in_be64(hose->cfg_data + PCIE_REG_MMIO_ERR1_STATUS); ed.mmio_log0 = in_be64(hose->cfg_data + PCIE_REG_MMIO_ERR_LOG_0); ed.mmio_log1 = in_be64(hose->cfg_data + PCIE_REG_MMIO_ERR_LOG_1); out_be64(hose->cfg_data + PCIE_REG_MMIO_ERR1_STATUS, 0); out_be64(hose->cfg_data + PCIE_REG_MMIO_ERR_STATUS, 0); } ed.dma_err = in_be64(hose->cfg_data + PCIE_REG_DMA_ERR_STATUS); if (ed.dma_err) { ed.dma_err1 = in_be64(hose->cfg_data + PCIE_REG_DMA_ERR1_STATUS); ed.dma_log0 = in_be64(hose->cfg_data + PCIE_REG_DMA_ERR_LOG_0); ed.dma_log1 = in_be64(hose->cfg_data + PCIE_REG_DMA_ERR_LOG_1); out_be64(hose->cfg_data + PCIE_REG_DMA_ERR1_STATUS, 0); out_be64(hose->cfg_data + PCIE_REG_DMA_ERR_STATUS, 0); } /* Now print things out */ if (ed.phb_err) { pr_err(" PHB Error Status : 0x%016llx\n", ed.phb_err); pr_err(" PHB First Error Status: 0x%016llx\n", ed.phb_err1); pr_err(" PHB Error Log 0 : 0x%016llx\n", ed.phb_log0); pr_err(" PHB Error Log 1 : 0x%016llx\n", ed.phb_log1); } if (ed.mmio_err) { pr_err(" MMIO Error Status : 0x%016llx\n", ed.mmio_err); pr_err(" MMIO First Error Status: 0x%016llx\n", ed.mmio_err1); pr_err(" MMIO Error Log 0 : 0x%016llx\n", ed.mmio_log0); pr_err(" MMIO Error Log 1 : 0x%016llx\n", ed.mmio_log1); } if (ed.dma_err) { pr_err(" DMA Error Status : 0x%016llx\n", ed.dma_err); pr_err(" DMA First Error Status: 0x%016llx\n", ed.dma_err1); pr_err(" DMA Error Log 0 : 0x%016llx\n", ed.dma_log0); pr_err(" DMA Error Log 1 : 0x%016llx\n", ed.dma_log1); } if (ed.utl_sys_err) pr_err(" UTL Sys Error Status : 0x%016llx\n", ed.utl_sys_err); if (ed.utl_port_err) pr_err(" UTL Port Error Status : 0x%016llx\n", ed.utl_port_err); if (ed.utl_rc_err) pr_err(" UTL RC Error Status : 0x%016llx\n", ed.utl_rc_err); /* Interrupts are caused by the error traps. If we had any error there * we loop again in case the UTL buffered some new stuff between * going there and going to the traps */ if (ed.dma_err || ed.mmio_err || ed.phb_err) { handled = IRQ_HANDLED; goto again; } return handled; } static void __init wsp_setup_pci_err_reporting(struct wsp_phb *phb) { struct pci_controller *hose = phb->hose; int err_irq, i, rc; char fname[16]; /* Create a debugfs file for that PHB */ sprintf(fname, "phb%d", phb->hose->global_number); phb->ddir = debugfs_create_dir(fname, powerpc_debugfs_root); /* Some useful debug output */ if (phb->ddir) { struct dentry *d = debugfs_create_dir("regs", phb->ddir); char tmp[64]; for (i = 0; i < ARRAY_SIZE(wsp_pci_regs); i++) { sprintf(tmp, "%03x_%s", wsp_pci_regs[i].offset, wsp_pci_regs[i].name); debugfs_create_file(tmp, 0600, d, hose->cfg_data + wsp_pci_regs[i].offset, &wsp_pci_reg_fops); } debugfs_create_file("all_regs", 0600, phb->ddir, phb, &wsp_pci_regs_fops); } /* Find the IRQ number for that PHB */ err_irq = irq_of_parse_and_map(hose->dn, 0); if (err_irq == 0) /* XXX Error IRQ lacking from device-tree */ err_irq = wsp_pci_get_err_irq_no_dt(hose->dn); if (err_irq == 0) { pr_err("PCI: Failed to fetch error interrupt for %s\n", hose->dn->full_name); return; } /* Request it */ rc = request_irq(err_irq, wsp_pci_err_irq, 0, "wsp_pci error", phb); if (rc) { pr_err("PCI: Failed to request interrupt for %s\n", hose->dn->full_name); } /* Enable interrupts for all errors for now */ out_be64(hose->cfg_data + PCIE_REG_PHB_ERR_IRQ_ENABLE, 0xffffffffffffffffull); out_be64(hose->cfg_data + PCIE_REG_MMIO_ERR_IRQ_ENABLE, 0xffffffffffffffffull); out_be64(hose->cfg_data + PCIE_REG_DMA_ERR_IRQ_ENABLE, 0xffffffffffffffffull); } /* * This is called later to hookup with the error interrupt */ static int __init wsp_setup_pci_late(void) { struct wsp_phb *phb; list_for_each_entry(phb, &wsp_phbs, all) wsp_setup_pci_err_reporting(phb); return 0; } arch_initcall(wsp_setup_pci_late);