From b88214ce4d7064992452765028bd50702414f15f Mon Sep 17 00:00:00 2001 From: Noa Osherovich Date: Tue, 15 Nov 2016 09:59:58 +0200 Subject: PCI: Convert broken INTx masking quirks from HEADER to FINAL Convert all quirk_broken_intx_masking() quirks from HEADER to FINAL. The quirk sets dev->broken_intx_masking, which is only used by pci_intx_mask_supported(), which is not needed until after FINAL quirks have been run. [bhelgaas: changelog] Signed-off-by: Noa Osherovich Signed-off-by: Bjorn Helgaas Reviewed-by: Gavin Shan --- drivers/pci/quirks.c | 72 ++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) (limited to 'drivers/pci') diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index c232729f5b1b..85048fdf2474 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3146,53 +3146,53 @@ static void quirk_broken_intx_masking(struct pci_dev *dev) { dev->broken_intx_masking = 1; } -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_CHELSIO, 0x0030, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(0x1814, 0x0601, /* Ralink RT2800 802.11n PCI */ - quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x0030, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(0x1814, 0x0601, /* Ralink RT2800 802.11n PCI */ + quirk_broken_intx_masking); /* * Realtek RTL8169 PCI Gigabit Ethernet Controller (rev 10) * Subsystem: Realtek RTL8169/8110 Family PCI Gigabit Ethernet NIC * * RTL8110SC - Fails under PCI device assignment using DisINTx masking. */ -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_REALTEK, 0x8169, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, PCI_ANY_ID, - quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_REALTEK, 0x8169, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_MELLANOX, PCI_ANY_ID, + quirk_broken_intx_masking); /* * Intel i40e (XL710/X710) 10/20/40GbE NICs all have broken INTx masking, * DisINTx can be set but the interrupt status bit is non-functional. */ -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1572, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1574, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1580, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1581, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1583, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1584, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1585, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1586, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1587, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1588, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1589, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x37d0, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x37d1, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x37d2, - quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1572, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1574, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1580, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1581, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1583, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1584, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1585, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1586, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1587, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1588, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1589, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d0, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d1, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d2, + quirk_broken_intx_masking); static void quirk_no_bus_reset(struct pci_dev *dev) { -- cgit v1.2.3 From d76d2fe05fd93673d184af77255bbbc63780f4ea Mon Sep 17 00:00:00 2001 From: Noa Osherovich Date: Tue, 15 Nov 2016 09:59:59 +0200 Subject: PCI: Convert Mellanox broken INTx quirks to be for listed devices only Change Mellanox's broken_intx_masking() quirk from an "all Mellanox devices" to a quirk for listed devices only. [bhelgaas: remove #defines, reorder to keep other quirks together] Signed-off-by: Noa Osherovich Signed-off-by: Bjorn Helgaas Reviewed-by: Or Gerlitz Reviewed-by: Gavin Shan --- drivers/pci/quirks.c | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) (limited to 'drivers/pci') diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 85048fdf2474..f0515b78a4a6 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3137,8 +3137,9 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b5, quirk_remove_d3_delay); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b7, quirk_remove_d3_delay); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x2298, quirk_remove_d3_delay); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x229c, quirk_remove_d3_delay); + /* - * Some devices may pass our check in pci_intx_mask_supported if + * Some devices may pass our check in pci_intx_mask_supported() if * PCI_COMMAND_INTX_DISABLE works though they actually do not properly * support this feature. */ @@ -3150,6 +3151,7 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x0030, quirk_broken_intx_masking); DECLARE_PCI_FIXUP_FINAL(0x1814, 0x0601, /* Ralink RT2800 802.11n PCI */ quirk_broken_intx_masking); + /* * Realtek RTL8169 PCI Gigabit Ethernet Controller (rev 10) * Subsystem: Realtek RTL8169/8110 Family PCI Gigabit Ethernet NIC @@ -3158,8 +3160,6 @@ DECLARE_PCI_FIXUP_FINAL(0x1814, 0x0601, /* Ralink RT2800 802.11n PCI */ */ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_REALTEK, 0x8169, quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_MELLANOX, PCI_ANY_ID, - quirk_broken_intx_masking); /* * Intel i40e (XL710/X710) 10/20/40GbE NICs all have broken INTx masking, @@ -3194,6 +3194,40 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d1, DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d2, quirk_broken_intx_masking); +static u16 mellanox_broken_intx_devs[] = { + PCI_DEVICE_ID_MELLANOX_HERMON_SDR, + PCI_DEVICE_ID_MELLANOX_HERMON_DDR, + PCI_DEVICE_ID_MELLANOX_HERMON_QDR, + PCI_DEVICE_ID_MELLANOX_HERMON_DDR_GEN2, + PCI_DEVICE_ID_MELLANOX_HERMON_QDR_GEN2, + PCI_DEVICE_ID_MELLANOX_HERMON_EN, + PCI_DEVICE_ID_MELLANOX_HERMON_EN_GEN2, + PCI_DEVICE_ID_MELLANOX_CONNECTX_EN, + PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_T_GEN2, + PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_GEN2, + PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_5_GEN2, + PCI_DEVICE_ID_MELLANOX_CONNECTX2, + PCI_DEVICE_ID_MELLANOX_CONNECTX3, + PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO, + PCI_DEVICE_ID_MELLANOX_CONNECTIB, + PCI_DEVICE_ID_MELLANOX_CONNECTX4, + PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX, +}; + +static void mellanox_check_broken_intx_masking(struct pci_dev *pdev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(mellanox_broken_intx_devs); i++) { + if (pdev->device == mellanox_broken_intx_devs[i]) { + pdev->broken_intx_masking = 1; + return; + } + } +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_MELLANOX, PCI_ANY_ID, + mellanox_check_broken_intx_masking); + static void quirk_no_bus_reset(struct pci_dev *dev) { dev->dev_flags |= PCI_DEV_FLAGS_NO_BUS_RESET; -- cgit v1.2.3 From 1600f62534b7b3da7978b43b52231a54c24df287 Mon Sep 17 00:00:00 2001 From: Noa Osherovich Date: Tue, 15 Nov 2016 10:00:00 +0200 Subject: PCI: Support INTx masking on ConnectX-4 with firmware x.14.1100+ Mellanox devices were marked as having INTx masking ability broken. As a result, the VFIO driver fails to start when more than one device function is passed-through to a VM if both have the same INTx pin. Prior to Connect-IB, Mellanox devices exposed to the operating system one PCI function per all ports. Starting from Connect-IB, the devices are function-per-port. When passing the second function to a VM, VFIO will fail to start. Exclude ConnectX-4, ConnectX4-Lx and Connect-IB from the list of Mellanox devices marked as having broken INTx masking: - ConnectX-4 and ConnectX4-LX firmware version is checked. If INTx masking is supported, we unmark the broken INTx masking. - Connect-IB does not support INTx currently so will not cause any problem. [bhelgaas: call pci_disable_device() always, after iounmap()] Fixes: 11e42532ada3 ("PCI: Assume all Mellanox devices have broken INTx masking") Signed-off-by: Noa Osherovich Signed-off-by: Bjorn Helgaas Reviewed-by: Or Gerlitz Reviewed-by: Gavin Shan --- drivers/pci/quirks.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 3 deletions(-) (limited to 'drivers/pci') diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index f0515b78a4a6..93b03128bec0 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3209,13 +3209,25 @@ static u16 mellanox_broken_intx_devs[] = { PCI_DEVICE_ID_MELLANOX_CONNECTX2, PCI_DEVICE_ID_MELLANOX_CONNECTX3, PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO, - PCI_DEVICE_ID_MELLANOX_CONNECTIB, - PCI_DEVICE_ID_MELLANOX_CONNECTX4, - PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX, }; +#define CONNECTX_4_CURR_MAX_MINOR 99 +#define CONNECTX_4_INTX_SUPPORT_MINOR 14 + +/* + * Check ConnectX-4/LX FW version to see if it supports legacy interrupts. + * If so, don't mark it as broken. + * FW minor > 99 means older FW version format and no INTx masking support. + * FW minor < 14 means new FW version format and no INTx masking support. + */ static void mellanox_check_broken_intx_masking(struct pci_dev *pdev) { + __be32 __iomem *fw_ver; + u16 fw_major; + u16 fw_minor; + u16 fw_subminor; + u32 fw_maj_min; + u32 fw_sub_min; int i; for (i = 0; i < ARRAY_SIZE(mellanox_broken_intx_devs); i++) { @@ -3224,6 +3236,47 @@ static void mellanox_check_broken_intx_masking(struct pci_dev *pdev) return; } } + + /* Getting here means Connect-IB cards and up. Connect-IB has no INTx + * support so shouldn't be checked further + */ + if (pdev->device == PCI_DEVICE_ID_MELLANOX_CONNECTIB) + return; + + if (pdev->device != PCI_DEVICE_ID_MELLANOX_CONNECTX4 && + pdev->device != PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) + return; + + /* For ConnectX-4 and ConnectX-4LX, need to check FW support */ + if (pci_enable_device_mem(pdev)) { + dev_warn(&pdev->dev, "Can't enable device memory\n"); + return; + } + + fw_ver = ioremap(pci_resource_start(pdev, 0), 4); + if (!fw_ver) { + dev_warn(&pdev->dev, "Can't map ConnectX-4 initialization segment\n"); + goto out; + } + + /* Reading from resource space should be 32b aligned */ + fw_maj_min = ioread32be(fw_ver); + fw_sub_min = ioread32be(fw_ver + 1); + fw_major = fw_maj_min & 0xffff; + fw_minor = fw_maj_min >> 16; + fw_subminor = fw_sub_min & 0xffff; + if (fw_minor > CONNECTX_4_CURR_MAX_MINOR || + fw_minor < CONNECTX_4_INTX_SUPPORT_MINOR) { + dev_warn(&pdev->dev, "ConnectX-4: FW %u.%u.%u doesn't support INTx masking, disabling. Please upgrade FW to %d.14.1100 and up for INTx support\n", + fw_major, fw_minor, fw_subminor, pdev->device == + PCI_DEVICE_ID_MELLANOX_CONNECTX4 ? 12 : 14); + pdev->broken_intx_masking = 1; + } + + iounmap(fw_ver); + +out: + pci_disable_device(pdev); } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_MELLANOX, PCI_ANY_ID, mellanox_check_broken_intx_masking); -- cgit v1.2.3 From f40ec3c748c6912f6266c56a7f7992de61b255ed Mon Sep 17 00:00:00 2001 From: Gavin Shan Date: Wed, 26 Oct 2016 12:15:35 +1100 Subject: PCI: Do any VF BAR updates before enabling the BARs Previously we enabled VFs and enable their memory space before calling pcibios_sriov_enable(). But pcibios_sriov_enable() may update the VF BARs: for example, on PPC PowerNV we may change them to manage the association of VFs to PEs. Because 64-bit BARs cannot be updated atomically, it's unsafe to update them while they're enabled. The half-updated state may conflict with other devices in the system. Call pcibios_sriov_enable() before enabling the VFs so any BAR updates happen while the VF BARs are disabled. [bhelgaas: changelog] Tested-by: Carol Soto Signed-off-by: Gavin Shan Signed-off-by: Bjorn Helgaas --- drivers/pci/iov.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'drivers/pci') diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index e30f05c8517f..d41ec29be60b 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -306,13 +306,6 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn) return rc; } - pci_iov_set_numvfs(dev, nr_virtfn); - iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; - pci_cfg_access_lock(dev); - pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); - msleep(100); - pci_cfg_access_unlock(dev); - iov->initial_VFs = initial; if (nr_virtfn < initial) initial = nr_virtfn; @@ -323,6 +316,13 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn) goto err_pcibios; } + pci_iov_set_numvfs(dev, nr_virtfn); + iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; + pci_cfg_access_lock(dev); + pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); + msleep(100); + pci_cfg_access_unlock(dev); + for (i = 0; i < initial; i++) { rc = pci_iov_add_virtfn(dev, i, 0); if (rc) -- cgit v1.2.3 From 63880b230a4af502c56dde3d4588634c70c66006 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Mon, 28 Nov 2016 11:19:27 -0600 Subject: PCI: Ignore BAR updates on virtual functions VF BARs are read-only zero, so updating VF BARs will not have any effect. See the SR-IOV spec r1.1, sec 3.4.1.11. We already ignore these updates because of 70675e0b6a1a ("PCI: Don't try to restore VF BARs"); this merely restructures it slightly to make it easier to split updates for standard and SR-IOV BARs. Signed-off-by: Bjorn Helgaas Reviewed-by: Gavin Shan --- drivers/pci/pci.c | 4 ---- drivers/pci/setup-res.c | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'drivers/pci') diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index ba34907538f6..631eac2bed78 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -564,10 +564,6 @@ static void pci_restore_bars(struct pci_dev *dev) { int i; - /* Per SR-IOV spec 3.4.1.11, VF BARs are RO zero */ - if (dev->is_virtfn) - return; - for (i = 0; i < PCI_BRIDGE_RESOURCES; i++) pci_update_resource(dev, i); } diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index 66c4d8f42233..d2a32d88d2ae 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -36,10 +36,9 @@ void pci_update_resource(struct pci_dev *dev, int resno) enum pci_bar_type type; struct resource *res = dev->resource + resno; - if (dev->is_virtfn) { - dev_warn(&dev->dev, "can't update VF BAR%d\n", resno); + /* Per SR-IOV spec 3.4.1.11, VF BARs are RO zero */ + if (dev->is_virtfn) return; - } /* * Ignore resources for unimplemented BARs and unused resource slots -- cgit v1.2.3 From 45d004f4afefdd8d79916ee6d97a9ecd94bb1ffe Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 29 Nov 2016 08:14:47 -0600 Subject: PCI: Update BARs using property bits appropriate for type The BAR property bits (0-3 for memory BARs, 0-1 for I/O BARs) are supposed to be read-only, but we do save them in res->flags and include them when updating the BAR. Mask the I/O property bits with ~PCI_BASE_ADDRESS_IO_MASK (0x3) instead of PCI_REGION_FLAG_MASK (0xf) to make it obvious that we can't corrupt bits 2-3 of I/O addresses. Use PCI_ROM_ADDRESS_MASK for ROM BARs. This means we'll only check the top 21 bits (instead of the 28 bits we used to check) of a ROM BAR to see if the update was successful. Signed-off-by: Bjorn Helgaas --- drivers/pci/setup-res.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'drivers/pci') diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index d2a32d88d2ae..53bc0638cac6 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -59,12 +59,17 @@ void pci_update_resource(struct pci_dev *dev, int resno) return; pcibios_resource_to_bus(dev->bus, ®ion, res); + new = region.start; - new = region.start | (res->flags & PCI_REGION_FLAG_MASK); - if (res->flags & IORESOURCE_IO) + if (res->flags & IORESOURCE_IO) { mask = (u32)PCI_BASE_ADDRESS_IO_MASK; - else + new |= res->flags & ~PCI_BASE_ADDRESS_IO_MASK; + } else if (resno == PCI_ROM_RESOURCE) { + mask = (u32)PCI_ROM_ADDRESS_MASK; + } else { mask = (u32)PCI_BASE_ADDRESS_MEM_MASK; + new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK; + } reg = pci_resource_bar(dev, resno, &type); if (!reg) -- cgit v1.2.3 From 6ffa2489c51da77564a0881a73765ea2169f955d Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Mon, 28 Nov 2016 09:15:52 -0600 Subject: PCI: Separate VF BAR updates from standard BAR updates Previously pci_update_resource() used the same code path for updating standard BARs and VF BARs in SR-IOV capabilities. Split the VF BAR update into a new pci_iov_update_resource() internal interface, which makes it simpler to compute the BAR address (we can get rid of pci_resource_bar() and pci_iov_resource_bar()). This patch: - Renames pci_update_resource() to pci_std_update_resource(), - Adds pci_iov_update_resource(), - Makes pci_update_resource() a wrapper that calls the appropriate one, No functional change intended. Signed-off-by: Bjorn Helgaas Reviewed-by: Gavin Shan --- drivers/pci/iov.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ drivers/pci/pci.h | 1 + drivers/pci/setup-res.c | 13 +++++++++++-- 3 files changed, 62 insertions(+), 2 deletions(-) (limited to 'drivers/pci') diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index d41ec29be60b..aa499543473f 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -571,6 +571,56 @@ int pci_iov_resource_bar(struct pci_dev *dev, int resno) 4 * (resno - PCI_IOV_RESOURCES); } +/** + * pci_iov_update_resource - update a VF BAR + * @dev: the PCI device + * @resno: the resource number + * + * Update a VF BAR in the SR-IOV capability of a PF. + */ +void pci_iov_update_resource(struct pci_dev *dev, int resno) +{ + struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL; + struct resource *res = dev->resource + resno; + int vf_bar = resno - PCI_IOV_RESOURCES; + struct pci_bus_region region; + u32 new; + int reg; + + /* + * The generic pci_restore_bars() path calls this for all devices, + * including VFs and non-SR-IOV devices. If this is not a PF, we + * have nothing to do. + */ + if (!iov) + return; + + /* + * Ignore unimplemented BARs, unused resource slots for 64-bit + * BARs, and non-movable resources, e.g., those described via + * Enhanced Allocation. + */ + if (!res->flags) + return; + + if (res->flags & IORESOURCE_UNSET) + return; + + if (res->flags & IORESOURCE_PCI_FIXED) + return; + + pcibios_resource_to_bus(dev->bus, ®ion, res); + new = region.start; + new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK; + + reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar; + pci_write_config_dword(dev, reg, new); + if (res->flags & IORESOURCE_MEM_64) { + new = region.start >> 16 >> 16; + pci_write_config_dword(dev, reg + 4, new); + } +} + resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev, int resno) { diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 451856210e18..5bfcb922f7f7 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -290,6 +290,7 @@ static inline void pci_restore_ats_state(struct pci_dev *dev) int pci_iov_init(struct pci_dev *dev); void pci_iov_release(struct pci_dev *dev); int pci_iov_resource_bar(struct pci_dev *dev, int resno); +void pci_iov_update_resource(struct pci_dev *dev, int resno); resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno); void pci_restore_iov_state(struct pci_dev *dev); int pci_iov_bus_range(struct pci_bus *bus); diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index 53bc0638cac6..5ddeb6737f99 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -25,8 +25,7 @@ #include #include "pci.h" - -void pci_update_resource(struct pci_dev *dev, int resno) +static void pci_std_update_resource(struct pci_dev *dev, int resno) { struct pci_bus_region region; bool disable; @@ -114,6 +113,16 @@ void pci_update_resource(struct pci_dev *dev, int resno) pci_write_config_word(dev, PCI_COMMAND, cmd); } +void pci_update_resource(struct pci_dev *dev, int resno) +{ + if (resno <= PCI_ROM_RESOURCE) + pci_std_update_resource(dev, resno); +#ifdef CONFIG_PCI_IOV + else if (resno >= PCI_IOV_RESOURCES && resno <= PCI_IOV_RESOURCE_END) + pci_iov_update_resource(dev, resno); +#endif +} + int pci_claim_resource(struct pci_dev *dev, int resource) { struct resource *res = &dev->resource[resource]; -- cgit v1.2.3 From 546ba9f8f22f71b0202b6ba8967be5cc6dae4e21 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Mon, 28 Nov 2016 16:43:06 -0600 Subject: PCI: Don't update VF BARs while VF memory space is enabled If we update a VF BAR while it's enabled, there are two potential problems: 1) Any driver that's using the VF has a cached BAR value that is stale after the update, and 2) We can't update 64-bit BARs atomically, so the intermediate state (new lower dword with old upper dword) may conflict with another device, and an access by a driver unrelated to the VF may cause a bus error. Warn about attempts to update VF BARs while they are enabled. This is a programming error, so use dev_WARN() to get a backtrace. Signed-off-by: Bjorn Helgaas Reviewed-by: Gavin Shan --- drivers/pci/iov.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/pci') diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index aa499543473f..2480b3879182 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -584,6 +584,7 @@ void pci_iov_update_resource(struct pci_dev *dev, int resno) struct resource *res = dev->resource + resno; int vf_bar = resno - PCI_IOV_RESOURCES; struct pci_bus_region region; + u16 cmd; u32 new; int reg; @@ -595,6 +596,13 @@ void pci_iov_update_resource(struct pci_dev *dev, int resno) if (!iov) return; + pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd); + if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) { + dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n", + vf_bar, res); + return; + } + /* * Ignore unimplemented BARs, unused resource slots for 64-bit * BARs, and non-movable resources, e.g., those described via -- cgit v1.2.3 From 286c2378aaccc7343ebf17ec6cd86567659caf70 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Mon, 28 Nov 2016 16:51:19 -0600 Subject: PCI: Remove pci_resource_bar() and pci_iov_resource_bar() pci_std_update_resource() only deals with standard BARs, so we don't have to worry about the complications of VF BARs in an SR-IOV capability. Compute the BAR address inline and remove pci_resource_bar(). That makes pci_iov_resource_bar() unused, so remove that as well. Signed-off-by: Bjorn Helgaas Reviewed-by: Gavin Shan --- drivers/pci/iov.c | 18 ------------------ drivers/pci/pci.c | 30 ------------------------------ drivers/pci/pci.h | 6 ------ drivers/pci/setup-res.c | 13 +++++++------ 4 files changed, 7 insertions(+), 60 deletions(-) (limited to 'drivers/pci') diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 2480b3879182..47227820406d 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -553,24 +553,6 @@ void pci_iov_release(struct pci_dev *dev) sriov_release(dev); } -/** - * pci_iov_resource_bar - get position of the SR-IOV BAR - * @dev: the PCI device - * @resno: the resource number - * - * Returns position of the BAR encapsulated in the SR-IOV capability. - */ -int pci_iov_resource_bar(struct pci_dev *dev, int resno) -{ - if (resno < PCI_IOV_RESOURCES || resno > PCI_IOV_RESOURCE_END) - return 0; - - BUG_ON(!dev->is_physfn); - - return dev->sriov->pos + PCI_SRIOV_BAR + - 4 * (resno - PCI_IOV_RESOURCES); -} - /** * pci_iov_update_resource - update a VF BAR * @dev: the PCI device diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 631eac2bed78..ec3f16d13307 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4827,36 +4827,6 @@ int pci_select_bars(struct pci_dev *dev, unsigned long flags) } EXPORT_SYMBOL(pci_select_bars); -/** - * pci_resource_bar - get position of the BAR associated with a resource - * @dev: the PCI device - * @resno: the resource number - * @type: the BAR type to be filled in - * - * Returns BAR position in config space, or 0 if the BAR is invalid. - */ -int pci_resource_bar(struct pci_dev *dev, int resno, enum pci_bar_type *type) -{ - int reg; - - if (resno < PCI_ROM_RESOURCE) { - *type = pci_bar_unknown; - return PCI_BASE_ADDRESS_0 + 4 * resno; - } else if (resno == PCI_ROM_RESOURCE) { - *type = pci_bar_mem32; - return dev->rom_base_reg; - } else if (resno < PCI_BRIDGE_RESOURCES) { - /* device specific resource */ - *type = pci_bar_unknown; - reg = pci_iov_resource_bar(dev, resno); - if (reg) - return reg; - } - - dev_err(&dev->dev, "BAR %d: invalid resource\n", resno); - return 0; -} - /* Some architectures require additional programming to enable VGA */ static arch_set_vga_state_t arch_set_vga_state; diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 5bfcb922f7f7..a5d37f6a9fb5 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -245,7 +245,6 @@ bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl, int pci_setup_device(struct pci_dev *dev); int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, struct resource *res, unsigned int reg); -int pci_resource_bar(struct pci_dev *dev, int resno, enum pci_bar_type *type); void pci_configure_ari(struct pci_dev *dev); void __pci_bus_size_bridges(struct pci_bus *bus, struct list_head *realloc_head); @@ -289,7 +288,6 @@ static inline void pci_restore_ats_state(struct pci_dev *dev) #ifdef CONFIG_PCI_IOV int pci_iov_init(struct pci_dev *dev); void pci_iov_release(struct pci_dev *dev); -int pci_iov_resource_bar(struct pci_dev *dev, int resno); void pci_iov_update_resource(struct pci_dev *dev, int resno); resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno); void pci_restore_iov_state(struct pci_dev *dev); @@ -304,10 +302,6 @@ static inline void pci_iov_release(struct pci_dev *dev) { } -static inline int pci_iov_resource_bar(struct pci_dev *dev, int resno) -{ - return 0; -} static inline void pci_restore_iov_state(struct pci_dev *dev) { } diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index 5ddeb6737f99..99c9e32775ee 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -32,7 +32,6 @@ static void pci_std_update_resource(struct pci_dev *dev, int resno) u16 cmd; u32 new, check, mask; int reg; - enum pci_bar_type type; struct resource *res = dev->resource + resno; /* Per SR-IOV spec 3.4.1.11, VF BARs are RO zero */ @@ -70,14 +69,16 @@ static void pci_std_update_resource(struct pci_dev *dev, int resno) new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK; } - reg = pci_resource_bar(dev, resno, &type); - if (!reg) - return; - if (type != pci_bar_unknown) { + if (resno < PCI_ROM_RESOURCE) { + reg = PCI_BASE_ADDRESS_0 + 4 * resno; + } else if (resno == PCI_ROM_RESOURCE) { if (!(res->flags & IORESOURCE_ROM_ENABLE)) return; + + reg = dev->rom_base_reg; new |= PCI_ROM_ADDRESS_ENABLE; - } + } else + return; /* * We can't update a 64-bit BAR atomically, so when possible, -- cgit v1.2.3 From 7a6d312b50e63f598f5b5914c4fd21878ac2b595 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Mon, 28 Nov 2016 17:21:02 -0600 Subject: PCI: Decouple IORESOURCE_ROM_ENABLE and PCI_ROM_ADDRESS_ENABLE Remove the assumption that IORESOURCE_ROM_ENABLE == PCI_ROM_ADDRESS_ENABLE. PCI_ROM_ADDRESS_ENABLE is the ROM enable bit defined by the PCI spec, so if we're reading or writing a BAR register value, that's what we should use. IORESOURCE_ROM_ENABLE is a corresponding bit in struct resource flags. Signed-off-by: Bjorn Helgaas Reviewed-by: Gavin Shan --- drivers/pci/probe.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/pci') diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index ab002671fa60..cf7670e81979 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -227,7 +227,8 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, mask64 = (u32)PCI_BASE_ADDRESS_MEM_MASK; } } else { - res->flags |= (l & IORESOURCE_ROM_ENABLE); + if (l & PCI_ROM_ADDRESS_ENABLE) + res->flags |= IORESOURCE_ROM_ENABLE; l64 = l & PCI_ROM_ADDRESS_MASK; sz64 = sz & PCI_ROM_ADDRESS_MASK; mask64 = (u32)PCI_ROM_ADDRESS_MASK; -- cgit v1.2.3 From 0b457dde3cf8b7c76a60f8e960f21bbd4abdc416 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Mon, 28 Nov 2016 16:17:41 -0600 Subject: PCI: Add comments about ROM BAR updating pci_update_resource() updates a hardware BAR so its address matches the kernel's struct resource UNLESS it's a disabled ROM BAR. We only update those when we enable the ROM. It's not obvious from the code why ROM BARs should be handled specially. Apparently there are Matrox devices with defective ROM BARs that read as zero when disabled. That means that if pci_enable_rom() reads the disabled BAR, sets PCI_ROM_ADDRESS_ENABLE (without re-inserting the address), and writes it back, it would enable the ROM at address zero. Add comments and references to explain why we can't make the code look more rational. The code changes are from 755528c860b0 ("Ignore disabled ROM resources at setup") and 8085ce084c0f ("[PATCH] Fix PCI ROM mapping"). Link: https://lkml.org/lkml/2005/8/30/138 Signed-off-by: Bjorn Helgaas Reviewed-by: Gavin Shan --- drivers/pci/rom.c | 5 +++++ drivers/pci/setup-res.c | 6 ++++++ 2 files changed, 11 insertions(+) (limited to 'drivers/pci') diff --git a/drivers/pci/rom.c b/drivers/pci/rom.c index 06663d391b39..b6edb187d160 100644 --- a/drivers/pci/rom.c +++ b/drivers/pci/rom.c @@ -35,6 +35,11 @@ int pci_enable_rom(struct pci_dev *pdev) if (res->flags & IORESOURCE_ROM_SHADOW) return 0; + /* + * Ideally pci_update_resource() would update the ROM BAR address, + * and we would only set the enable bit here. But apparently some + * devices have buggy ROM BARs that read as zero when disabled. + */ pcibios_resource_to_bus(pdev->bus, ®ion, res); pci_read_config_dword(pdev, pdev->rom_base_reg, &rom_addr); rom_addr &= ~PCI_ROM_ADDRESS_MASK; diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index 99c9e32775ee..045427336e11 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -72,6 +72,12 @@ static void pci_std_update_resource(struct pci_dev *dev, int resno) if (resno < PCI_ROM_RESOURCE) { reg = PCI_BASE_ADDRESS_0 + 4 * resno; } else if (resno == PCI_ROM_RESOURCE) { + + /* + * Apparently some Matrox devices have ROM BARs that read + * as zero when disabled, so don't update ROM BARs unless + * they're enabled. See https://lkml.org/lkml/2005/8/30/138. + */ if (!(res->flags & IORESOURCE_ROM_ENABLE)) return; -- cgit v1.2.3