/* * Intel MIC Platform Software Stack (MPSS) * * Copyright(c) 2013 Intel Corporation. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, version 2, as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * The full GNU General Public License is included in this distribution in * the file called "COPYING". * * Intel MIC Host driver. * */ #include <linux/pci.h> #include <linux/interrupt.h> #include "../common/mic_dev.h" #include "mic_device.h" /* * mic_invoke_callback - Invoke callback functions registered for * the corresponding source id. * * @mdev: pointer to the mic_device instance * @idx: The interrupt source id. * * Returns none. */ static inline void mic_invoke_callback(struct mic_device *mdev, int idx) { struct mic_intr_cb *intr_cb; struct pci_dev *pdev = container_of(mdev->sdev->parent, struct pci_dev, dev); spin_lock(&mdev->irq_info.mic_intr_lock); list_for_each_entry(intr_cb, &mdev->irq_info.cb_list[idx], list) if (intr_cb->func) intr_cb->func(pdev->irq, intr_cb->data); spin_unlock(&mdev->irq_info.mic_intr_lock); } /** * mic_interrupt - Generic interrupt handler for * MSI and INTx based interrupts. */ static irqreturn_t mic_interrupt(int irq, void *dev) { struct mic_device *mdev = dev; struct mic_intr_info *info = mdev->intr_info; u32 mask; int i; mask = mdev->ops->ack_interrupt(mdev); if (!mask) return IRQ_NONE; for (i = info->intr_start_idx[MIC_INTR_DB]; i < info->intr_len[MIC_INTR_DB]; i++) if (mask & BIT(i)) mic_invoke_callback(mdev, i); return IRQ_HANDLED; } /* Return the interrupt offset from the index. Index is 0 based. */ static u16 mic_map_src_to_offset(struct mic_device *mdev, int intr_src, enum mic_intr_type type) { if (type >= MIC_NUM_INTR_TYPES) return MIC_NUM_OFFSETS; if (intr_src >= mdev->intr_info->intr_len[type]) return MIC_NUM_OFFSETS; return mdev->intr_info->intr_start_idx[type] + intr_src; } /* Return next available msix_entry. */ static struct msix_entry *mic_get_available_vector(struct mic_device *mdev) { int i; struct mic_irq_info *info = &mdev->irq_info; for (i = 0; i < info->num_vectors; i++) if (!info->mic_msi_map[i]) return &info->msix_entries[i]; return NULL; } /** * mic_register_intr_callback - Register a callback handler for the * given source id. * * @mdev: pointer to the mic_device instance * @idx: The source id to be registered. * @func: The function to be called when the source id receives * the interrupt. * @data: Private data of the requester. * Return the callback structure that was registered or an * appropriate error on failure. */ static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev, u8 idx, irqreturn_t (*func) (int irq, void *dev), void *data) { struct mic_intr_cb *intr_cb; unsigned long flags; int rc; intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL); if (!intr_cb) return ERR_PTR(-ENOMEM); intr_cb->func = func; intr_cb->data = data; intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida, 0, 0, GFP_KERNEL); if (intr_cb->cb_id < 0) { rc = intr_cb->cb_id; goto ida_fail; } spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]); spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); return intr_cb; ida_fail: kfree(intr_cb); return ERR_PTR(rc); } /** * mic_unregister_intr_callback - Unregister the callback handler * identified by its callback id. * * @mdev: pointer to the mic_device instance * @idx: The callback structure id to be unregistered. * Return the source id that was unregistered or MIC_NUM_OFFSETS if no * such callback handler was found. */ static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx) { struct list_head *pos, *tmp; struct mic_intr_cb *intr_cb; unsigned long flags; int i; for (i = 0; i < MIC_NUM_OFFSETS; i++) { spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) { intr_cb = list_entry(pos, struct mic_intr_cb, list); if (intr_cb->cb_id == idx) { list_del(pos); ida_simple_remove(&mdev->irq_info.cb_ida, intr_cb->cb_id); kfree(intr_cb); spin_unlock_irqrestore( &mdev->irq_info.mic_intr_lock, flags); return i; } } spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); } return MIC_NUM_OFFSETS; } /** * mic_setup_msix - Initializes MSIx interrupts. * * @mdev: pointer to mic_device instance * * * RETURNS: An appropriate -ERRNO error value on error, or zero for success. */ static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev) { int rc, i; int entry_size = sizeof(*mdev->irq_info.msix_entries); mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX, entry_size, GFP_KERNEL); if (!mdev->irq_info.msix_entries) { rc = -ENOMEM; goto err_nomem1; } for (i = 0; i < MIC_MIN_MSIX; i++) mdev->irq_info.msix_entries[i].entry = i; rc = pci_enable_msix(pdev, mdev->irq_info.msix_entries, MIC_MIN_MSIX); if (rc) { dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc); goto err_enable_msix; } mdev->irq_info.num_vectors = MIC_MIN_MSIX; mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) * mdev->irq_info.num_vectors), GFP_KERNEL); if (!mdev->irq_info.mic_msi_map) { rc = -ENOMEM; goto err_nomem2; } dev_dbg(mdev->sdev->parent, "%d MSIx irqs setup\n", mdev->irq_info.num_vectors); return 0; err_nomem2: pci_disable_msix(pdev); err_enable_msix: kfree(mdev->irq_info.msix_entries); err_nomem1: mdev->irq_info.num_vectors = 0; return rc; } /** * mic_setup_callbacks - Initialize data structures needed * to handle callbacks. * * @mdev: pointer to mic_device instance */ static int mic_setup_callbacks(struct mic_device *mdev) { int i; mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS, sizeof(*mdev->irq_info.cb_list), GFP_KERNEL); if (!mdev->irq_info.cb_list) return -ENOMEM; for (i = 0; i < MIC_NUM_OFFSETS; i++) INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]); ida_init(&mdev->irq_info.cb_ida); spin_lock_init(&mdev->irq_info.mic_intr_lock); return 0; } /** * mic_release_callbacks - Uninitialize data structures needed * to handle callbacks. * * @mdev: pointer to mic_device instance */ static void mic_release_callbacks(struct mic_device *mdev) { unsigned long flags; struct list_head *pos, *tmp; struct mic_intr_cb *intr_cb; int i; for (i = 0; i < MIC_NUM_OFFSETS; i++) { spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); if (list_empty(&mdev->irq_info.cb_list[i])) { spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); break; } list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) { intr_cb = list_entry(pos, struct mic_intr_cb, list); list_del(pos); ida_simple_remove(&mdev->irq_info.cb_ida, intr_cb->cb_id); kfree(intr_cb); } spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); } ida_destroy(&mdev->irq_info.cb_ida); kfree(mdev->irq_info.cb_list); } /** * mic_setup_msi - Initializes MSI interrupts. * * @mdev: pointer to mic_device instance * @pdev: PCI device structure * * RETURNS: An appropriate -ERRNO error value on error, or zero for success. */ static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev) { int rc; rc = pci_enable_msi(pdev); if (rc) { dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc); return rc; } mdev->irq_info.num_vectors = 1; mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) * mdev->irq_info.num_vectors), GFP_KERNEL); if (!mdev->irq_info.mic_msi_map) { rc = -ENOMEM; goto err_nomem1; } rc = mic_setup_callbacks(mdev); if (rc) { dev_err(&pdev->dev, "Error setting up callbacks\n"); goto err_nomem2; } rc = request_irq(pdev->irq, mic_interrupt, 0 , "mic-msi", mdev); if (rc) { dev_err(&pdev->dev, "Error allocating MSI interrupt\n"); goto err_irq_req_fail; } dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors); return 0; err_irq_req_fail: mic_release_callbacks(mdev); err_nomem2: kfree(mdev->irq_info.mic_msi_map); err_nomem1: pci_disable_msi(pdev); mdev->irq_info.num_vectors = 0; return rc; } /** * mic_setup_intx - Initializes legacy interrupts. * * @mdev: pointer to mic_device instance * @pdev: PCI device structure * * RETURNS: An appropriate -ERRNO error value on error, or zero for success. */ static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev) { int rc; pci_msi_off(pdev); /* Enable intx */ pci_intx(pdev, 1); rc = mic_setup_callbacks(mdev); if (rc) { dev_err(&pdev->dev, "Error setting up callbacks\n"); goto err_nomem; } rc = request_irq(pdev->irq, mic_interrupt, IRQF_SHARED, "mic-intx", mdev); if (rc) goto err; dev_dbg(&pdev->dev, "intx irq setup\n"); return 0; err: mic_release_callbacks(mdev); err_nomem: return rc; } /** * mic_next_db - Retrieve the next doorbell interrupt source id. * The id is picked sequentially from the available pool of * doorlbell ids. * * @mdev: pointer to the mic_device instance. * * Returns the next doorbell interrupt source. */ int mic_next_db(struct mic_device *mdev) { int next_db; next_db = mdev->irq_info.next_avail_src % mdev->intr_info->intr_len[MIC_INTR_DB]; mdev->irq_info.next_avail_src++; return next_db; } #define COOKIE_ID_SHIFT 16 #define GET_ENTRY(cookie) ((cookie) & 0xFFFF) #define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT) #define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT) /** * mic_request_irq - request an irq. mic_mutex needs * to be held before calling this function. * * @mdev: pointer to mic_device instance * @func: The callback function that handles the interrupt. * The function needs to call ack_interrupts * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts. * @name: The ASCII name of the callee requesting the irq. * @data: private data that is returned back when calling the * function handler. * @intr_src: The source id of the requester. Its the doorbell id * for Doorbell interrupts and DMA channel id for DMA interrupts. * @type: The type of interrupt. Values defined in mic_intr_type * * returns: The cookie that is transparent to the caller. Passed * back when calling mic_free_irq. An appropriate error code * is returned on failure. Caller needs to use IS_ERR(return_val) * to check for failure and PTR_ERR(return_val) to obtained the * error code. * */ struct mic_irq *mic_request_irq(struct mic_device *mdev, irqreturn_t (*func)(int irq, void *dev), const char *name, void *data, int intr_src, enum mic_intr_type type) { u16 offset; int rc = 0; struct msix_entry *msix = NULL; unsigned long cookie = 0; u16 entry; struct mic_intr_cb *intr_cb; struct pci_dev *pdev = container_of(mdev->sdev->parent, struct pci_dev, dev); offset = mic_map_src_to_offset(mdev, intr_src, type); if (offset >= MIC_NUM_OFFSETS) { dev_err(mdev->sdev->parent, "Error mapping index %d to a valid source id.\n", intr_src); rc = -EINVAL; goto err; } if (mdev->irq_info.num_vectors > 1) { msix = mic_get_available_vector(mdev); if (!msix) { dev_err(mdev->sdev->parent, "No MSIx vectors available for use.\n"); rc = -ENOSPC; goto err; } rc = request_irq(msix->vector, func, 0, name, data); if (rc) { dev_dbg(mdev->sdev->parent, "request irq failed rc = %d\n", rc); goto err; } entry = msix->entry; mdev->irq_info.mic_msi_map[entry] |= BIT(offset); mdev->intr_ops->program_msi_to_src_map(mdev, entry, offset, true); cookie = MK_COOKIE(entry, offset); dev_dbg(mdev->sdev->parent, "irq: %d assigned for src: %d\n", msix->vector, intr_src); } else { intr_cb = mic_register_intr_callback(mdev, offset, func, data); if (IS_ERR(intr_cb)) { dev_err(mdev->sdev->parent, "No available callback entries for use\n"); rc = PTR_ERR(intr_cb); goto err; } entry = 0; if (pci_dev_msi_enabled(pdev)) { mdev->irq_info.mic_msi_map[entry] |= (1 << offset); mdev->intr_ops->program_msi_to_src_map(mdev, entry, offset, true); } cookie = MK_COOKIE(entry, intr_cb->cb_id); dev_dbg(mdev->sdev->parent, "callback %d registered for src: %d\n", intr_cb->cb_id, intr_src); } return (struct mic_irq *)cookie; err: return ERR_PTR(rc); } /** * mic_free_irq - free irq. mic_mutex * needs to be held before calling this function. * * @mdev: pointer to mic_device instance * @cookie: cookie obtained during a successful call to mic_request_irq * @data: private data specified by the calling function during the * mic_request_irq * * returns: none. */ void mic_free_irq(struct mic_device *mdev, struct mic_irq *cookie, void *data) { u32 offset; u32 entry; u8 src_id; unsigned int irq; struct pci_dev *pdev = container_of(mdev->sdev->parent, struct pci_dev, dev); entry = GET_ENTRY((unsigned long)cookie); offset = GET_OFFSET((unsigned long)cookie); if (mdev->irq_info.num_vectors > 1) { if (entry >= mdev->irq_info.num_vectors) { dev_warn(mdev->sdev->parent, "entry %d should be < num_irq %d\n", entry, mdev->irq_info.num_vectors); return; } irq = mdev->irq_info.msix_entries[entry].vector; free_irq(irq, data); mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset)); mdev->intr_ops->program_msi_to_src_map(mdev, entry, offset, false); dev_dbg(mdev->sdev->parent, "irq: %d freed\n", irq); } else { irq = pdev->irq; src_id = mic_unregister_intr_callback(mdev, offset); if (src_id >= MIC_NUM_OFFSETS) { dev_warn(mdev->sdev->parent, "Error unregistering callback\n"); return; } if (pci_dev_msi_enabled(pdev)) { mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id)); mdev->intr_ops->program_msi_to_src_map(mdev, entry, src_id, false); } dev_dbg(mdev->sdev->parent, "callback %d unregistered for src: %d\n", offset, src_id); } } /** * mic_setup_interrupts - Initializes interrupts. * * @mdev: pointer to mic_device instance * @pdev: PCI device structure * * RETURNS: An appropriate -ERRNO error value on error, or zero for success. */ int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev) { int rc; rc = mic_setup_msix(mdev, pdev); if (!rc) goto done; rc = mic_setup_msi(mdev, pdev); if (!rc) goto done; rc = mic_setup_intx(mdev, pdev); if (rc) { dev_err(mdev->sdev->parent, "no usable interrupts\n"); return rc; } done: mdev->intr_ops->enable_interrupts(mdev); return 0; } /** * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts * * @mdev: pointer to mic_device instance * @pdev: PCI device structure * * returns none. */ void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev) { int i; mdev->intr_ops->disable_interrupts(mdev); if (mdev->irq_info.num_vectors > 1) { for (i = 0; i < mdev->irq_info.num_vectors; i++) { if (mdev->irq_info.mic_msi_map[i]) dev_warn(&pdev->dev, "irq %d may still be in use.\n", mdev->irq_info.msix_entries[i].vector); } kfree(mdev->irq_info.mic_msi_map); kfree(mdev->irq_info.msix_entries); pci_disable_msix(pdev); } else { if (pci_dev_msi_enabled(pdev)) { free_irq(pdev->irq, mdev); kfree(mdev->irq_info.mic_msi_map); pci_disable_msi(pdev); } else { free_irq(pdev->irq, mdev); } mic_release_callbacks(mdev); } } /** * mic_intr_restore - Restore MIC interrupt registers. * * @mdev: pointer to mic_device instance. * * Restore the interrupt registers to values previously * stored in the SW data structures. mic_mutex needs to * be held before calling this function. * * returns None. */ void mic_intr_restore(struct mic_device *mdev) { int entry, offset; struct pci_dev *pdev = container_of(mdev->sdev->parent, struct pci_dev, dev); if (!pci_dev_msi_enabled(pdev)) return; for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) { for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) { if (mdev->irq_info.mic_msi_map[entry] & BIT(offset)) mdev->intr_ops->program_msi_to_src_map(mdev, entry, offset, true); } } }