概述
本文讲述一个开源的PCIe设备驱动，通过这个例子可以基本上理解所有的PCIe设备驱动。后续也会做关于Linux各类驱动的文章。
通过前面的学习，我们知道PCIe设备访问之前需要先做枚举。一般来说，PCI设备的枚举操作不需要我们来做，BIOS或者系统初始化时已经做好了，当系统枚举完所有设备之后，PCI设备就会添加进系统，在Linux下使用 “lspci” 就能看到系统扫描到的所有PCI设备，我们只需要关注PCI设备driver的实现就好了。
在Linux源码中随便找了一个开源代码，tsi721（一款PCIe转RapidIO芯片）的一些源码，基本上一个普通的PCIE设备驱动模型都是这样的，其中在加上一些设备独有的处理流程。
那么PCIe驱动的入口在哪呢？
当系统枚举到的PCI设备的vendor id和device id与driver中的id匹配上之后，就会调用driver中的probe函数。
static const struct pci_device_id tsi721_pci_tbl[] = { { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) }, { 0, } /* terminate list */};MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);static struct pci_driver tsi721_driver = { .name = "tsi721", .id_table = tsi721_pci_tbl, .probe = tsi721_probe, .remove = tsi721_remove, .shutdown = tsi721_shutdown,};module_pci_driver(tsi721_driver);PCIe设备驱动不是通过name匹配，而是通过id去匹配的，当驱动与设备匹配上之后，就会执行tsi721_probe函数，一般的PCIe驱动基本上会用到以下步骤：
驱动加载流程
pci_enable_device(pdev); 使能pci设备
pci_resource_len(pdev, BAR_0);获取pci设备的资源大小，这是枚举时得到的值
pci_request_regions(pdev, DRV_NAME); 申请pci设备资源
pci_ioremap_bar(pdev, BAR_0); 映射虚拟内存
pci_set_master(pdev); 设置pci设备为master，master模式才能主动发起数据传输
pci_enable_msi(pdev); 使能MSI中断
request_irq(priv->pdev->irq, tsi721_irqhandler,(priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,DRV_NAME, (void *)priv); 注册中断处理函数
初始化流程中关于PCIe的就基本上完成了，其余还有关于具体业务的初始化，下面可以结合源码看看。
static int tsi721_probe(struct pci_dev *pdev, const struct pci_device_id *id){ struct tsi721_device *priv; int err; priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL); if (!priv) { err = -ENOMEM; goto err_exit; } err = pci_enable_device(pdev); // 使能pci设备 if (err) { tsi_err(&pdev->dev, "Failed to enable PCI device"); goto err_clean; } priv->pdev = pdev;#ifdef DEBUG { int i; for (i = 0; i < PCI_STD_NUM_BARS; i++) { tsi_debug(INIT, &pdev->dev, "res%d %pR", i, &pdev->resource[i]); } }#endif /* * Verify BAR configuration */ /* BAR_0 (registers) must be 512KB+ in 32-bit address space */ if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) || pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 || pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) {// 获取pci设备bar0的资源大小，这是枚举时得到的值 tsi_err(&pdev->dev, "Missing or misconfigured CSR BAR0"); err = -ENODEV; goto err_disable_pdev; } /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */ if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) || pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 || pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) { tsi_err(&pdev->dev, "Missing or misconfigured Doorbell BAR1"); err = -ENODEV; goto err_disable_pdev; } /* * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address * space. * NOTE: BAR_2 and BAR_4 are not used by this version of driver. * It may be a good idea to keep them disabled using HW configuration * to save PCI memory space. */ priv->p2r_bar[0].size = priv->p2r_bar[1].size = 0; if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64) { if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_PREFETCH) tsi_debug(INIT, &pdev->dev, "Prefetchable OBW BAR2 will not be used"); else { priv->p2r_bar[0].base = pci_resource_start(pdev, BAR_2);// 获取bar2的起始地址 priv->p2r_bar[0].size = pci_resource_len(pdev, BAR_2);// 获取bar2的起始长度 } } if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64) { if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_PREFETCH) tsi_debug(INIT, &pdev->dev, "Prefetchable OBW BAR4 will not be used"); else { priv->p2r_bar[1].base = pci_resource_start(pdev, BAR_4);// 获取bar4的起始地址 priv->p2r_bar[1].size = pci_resource_len(pdev, BAR_4);// 获取bar4的起始长度 } } err = pci_request_regions(pdev, DRV_NAME); // 申请pci设备资源 if (err) { tsi_err(&pdev->dev, "Unable to obtain PCI resources"); goto err_disable_pdev; } pci_set_master(pdev);// 设置pci设备为master模式 priv->regs = pci_ioremap_bar(pdev, BAR_0); // 映射虚拟内存 if (!priv->regs) { tsi_err(&pdev->dev, "Unable to map device registers space"); err = -ENOMEM; goto err_free_res; } priv->odb_base = pci_ioremap_bar(pdev, BAR_1);// 映射虚拟内存 if (!priv->odb_base) { tsi_err(&pdev->dev, "Unable to map outbound doorbells space"); err = -ENOMEM; goto err_unmap_bars; } /* Configure DMA attributes. */ if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); // 设置dma掩码 if (err) { tsi_err(&pdev->dev, "Unable to set DMA mask"); goto err_unmap_bars; } if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) tsi_info(&pdev->dev, "Unable to set consistent DMA mask"); } else { err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) tsi_info(&pdev->dev, "Unable to set consistent DMA mask"); } BUG_ON(!pci_is_pcie(pdev)); /* Clear "no snoop" and "relaxed ordering" bits. */ pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN, 0); /* Override PCIe Maximum Read Request Size setting if requested */ if (pcie_mrrs >= 0) { if (pcie_mrrs <= 5) pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_READRQ, pcie_mrrs << 12); else tsi_info(&pdev->dev, "Invalid MRRS override value %d", pcie_mrrs); } /* Set PCIe completion timeout to 1-10ms */ pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL2, PCI_EXP_DEVCTL2_COMP_TIMEOUT, 0x2); /* * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block */ pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01); pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL, TSI721_MSIXTBL_OFFSET); pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA, TSI721_MSIXPBA_OFFSET); pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0); /* End of FIXUP */ tsi721_disable_ints(priv); tsi721_init_pc2sr_mapping(priv); tsi721_init_sr2pc_mapping(priv); if (tsi721_bdma_maint_init(priv)) { tsi_err(&pdev->dev, "BDMA initialization failed"); err = -ENOMEM; goto err_unmap_bars; } err = tsi721_doorbell_init(priv); if (err) goto err_free_bdma; tsi721_port_write_init(priv); err = tsi721_messages_init(priv); if (err) goto err_free_consistent; err = tsi721_setup_mport(priv); if (err) goto err_free_consistent; pci_set_drvdata(pdev, priv); tsi721_interrupts_init(priv); return 0;err_free_consistent: tsi721_port_write_free(priv); tsi721_doorbell_free(priv);err_free_bdma: tsi721_bdma_maint_free(priv);err_unmap_bars: if (priv->regs) iounmap(priv->regs); if (priv->odb_base) iounmap(priv->odb_base);err_free_res: pci_release_regions(pdev); pci_clear_master(pdev);err_disable_pdev: pci_disable_device(pdev);err_clean: kfree(priv);err_exit: return err;}tsi721_probe调用tsi721_setup_mport(priv)，如下所示，和pcie相关的就是msi 中断的使能，中断函数的注册，其他就是芯片具体的业务相关的初始化。
static int tsi721_setup_mport(struct tsi721_device *priv){struct pci_dev *pdev = priv->pdev;int err = 0;struct rio_mport *mport = &priv->mport;err = rio_mport_initialize(mport);if (err)return err;mport->ops = &tsi721_rio_ops;mport->index = 0;mport->sys_size = 0; /* small system */mport->priv = (void *)priv;mport->phys_efptr = 0x100;mport->phys_rmap = 1;mport->dev.parent = &pdev->dev;mport->dev.release = tsi721_mport_release;INIT_LIST_HEAD(&mport->dbells);rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);snprintf(mport->name, RIO_MAX_MPORT_NAME, "%s(%s)", dev_driver_string(&pdev->dev), dev_name(&pdev->dev));/* Hook up interrupt handler */#ifdef CONFIG_PCI_MSIif (!tsi721_enable_msix(priv))// 使能MSI-X中断priv->flags |= TSI721_USING_MSIX;else if (!pci_enable_msi(pdev))// 使能MSI中断priv->flags |= TSI721_USING_MSI;elsetsi_debug(MPORT, &pdev->dev, "MSI/MSI-X is not available. Using legacy INTx.");#endif /* CONFIG_PCI_MSI */err = tsi721_request_irq(priv); // 中断注册，展开即为 request_irqif (err) {tsi_err(&pdev->dev, "Unable to get PCI IRQ %02X (err=0x%x)",pdev->irq, err);return err;}#ifdef CONFIG_RAPIDIO_DMA_ENGINEerr = tsi721_register_dma(priv);if (err)goto err_exit;#endif/* Enable SRIO link */iowrite32(ioread32(priv->regs + TSI721_DEVCTL) | TSI721_DEVCTL_SRBOOT_CMPL, priv->regs + TSI721_DEVCTL);if (mport->host_deviceid >= 0)iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER | RIO_PORT_GEN_DISCOVERED, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));elseiowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));err = rio_register_mport(mport);if (err) {tsi721_unregister_dma(priv);goto err_exit;}return 0;err_exit:tsi721_free_irq(priv);return err;}驱动加载初始化的代码就结束了。
驱动卸载流程
当驱动卸载时会调用到tsi721_remove函数，其流程基本上就是probe的逆向操作。
free_irq(priv->pdev->irq, (void *)priv); 释放中断iounmap(priv->regs); 地址解映射pci_disable_msi(priv->pdev);失能msi中断pci_release_regions(pdev);释放申请的资源pci_clear_master(pdev); 清除设备master属性pci_disable_device(pdev);失能pci设备下面一起看看源码
static void tsi721_remove(struct pci_dev *pdev){ struct tsi721_device *priv = pci_get_drvdata(pdev); tsi_debug(EXIT, &pdev->dev, "enter"); tsi721_disable_ints(priv); tsi721_free_irq(priv);// 释放中断 flush_scheduled_work(); rio_unregister_mport(&priv->mport); tsi721_unregister_dma(priv); tsi721_bdma_maint_free(priv); tsi721_doorbell_free(priv); tsi721_port_write_free(priv); tsi721_close_sr2pc_mapping(priv); if (priv->regs) iounmap(priv->regs);// 地址解映射 if (priv->odb_base) iounmap(priv->odb_base);#ifdef CONFIG_PCI_MSI if (priv->flags & TSI721_USING_MSIX) pci_disable_msix(priv->pdev);// 失能msi-x中断 else if (priv->flags & TSI721_USING_MSI) pci_disable_msi(priv->pdev);// 失能msi中断#endif pci_release_regions(pdev);// 释放申请的资源 pci_clear_master(pdev);// 清除设备master属性 pci_disable_device(pdev);// 失能pci设备 pci_set_drvdata(pdev, NULL); kfree(priv); tsi_debug(EXIT, &pdev->dev, "exit");}至此关于PCIe设备驱动的卸载就完成了。
以上就是一个PCIe设备的基本驱动框架。