PCIe devices in Linux are implemented as character devices, wich expose hardware functionality via a device node under the /dev directory. User applications interact with PCIe hardware using standard file I/O operations on this node.

Creating the Character Device Node

Folow these steps to register a character device and create its device node:

/* 1. Allocate dynamic device number */
ret = alloc_chrdev_region(&pcie_demo_data.dev_num, 0, 1, "pcie_demo");

/* 2. Initialize character device structure */
pcie_demo_data.cdev.owner = THIS_MODULE;
cdev_init(&pcie_demo_data.cdev, &pcie_dev_fops);

/* 3. Add character device to kernel */
cdev_add(&pcie_demo_data.cdev, pcie_demo_data.dev_num, 1);

/* 4. Create device class for udev auto node creation */
pcie_demo_data.dev_class = class_create(THIS_MODULE, "pcie_demo");
if (IS_ERR(pcie_demo_data.dev_class)) {
    return PTR_ERR(pcie_demo_data.dev_class);
}

/* 5. Create device node in /dev */
pcie_demo_data.device = device_create(pcie_demo_data.dev_class, NULL, pcie_demo_data.dev_num, NULL, "pcie_demo");
if (IS_ERR(pcie_demo_data.device)) {
    return PTR_ERR(pcie_demo_data.device);
}

We first define an empty device operation struct as a placeholder:

/* Device file operations structure */
static struct file_operations pcie_dev_fops = {
    .owner = THIS_MODULE,
};

Add the creation code above to your driver's initialization entry, and add corresponding cleanup logic to the driver exit function:

static void __exit pcie_demo_exit(void)
{
	if(pcie_demo_data.pci_dev != NULL) {
		cdev_del(&pcie_demo_data.cdev);
		unregister_chrdev_region(pcie_demo_data.dev_num, 1);

		device_destroy(pcie_demo_data.dev_class, pcie_demo_data.dev_num);
		class_destroy(pcie_demo_data.dev_class);
	}
	
	pci_unregister_driver(&pcie_demo_driver);
}

After compiling and loading the driver module, you can see the newly created pcie_demo device in /dev.

Implementing Device File Operations

Next, we add implementations for open, release, read, and write operations, plus the full PCIe driver framework that interacts with a sample factorial computation PCIe peripheral:

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/fs.h>
#include <linux/wait.h>
#include <linux/atomic.h>
#include <linux/io.h>

#define PCIE_DEMO_VENDOR_ID     0x1234
#define PCIE_DEMO_DEVICE_ID     0x11e8
#define PCIE_DEMO_REVISION_ID   0x10

/* PCIe device private data structure */
typedef struct {
    dev_t dev_num;
    struct cdev cdev;
    struct class *dev_class;
    struct device *device;
    struct pci_dev *pci_dev;
    void __iomem *bar0_base;
    atomic_t calc_in_progress;
    wait_queue_head_t calc_wait_queue;
} pcie_dev_data_t;

static pcie_dev_data_t pcie_demo_data;

/* PCIe device ID match table */
static struct pci_device_id pcie_demo_ids[] = {
    { PCI_DEVICE(PCIE_DEMO_VENDOR_ID, PCIE_DEMO_DEVICE_ID), },
    { 0 , }
};
MODULE_DEVICE_TABLE(pci, pcie_demo_ids);

/* Interrupt handler for computation completion */
static irqreturn_t pcie_demo_irq_handler(int irq, void *dev_private)
{
    pcie_dev_data_t *dev_data = (pcie_dev_data_t *)dev_private;
    uint32_t irq_status;

    /* Read interrupt status from BAR0 */
    irq_status = ioread32(dev_data->bar0_base + 0x24);
    /* Clear pending interrupt */
    iowrite32(irq_status, dev_data->bar0_base + 0x64);

    /* Verify interrupt clearing */
    irq_status = ioread32(dev_data->bar0_base + 0x24);
    if(irq_status != 0){
        dev_err(&dev_data->pci_dev->dev, "Failed to clear interrupt\n");
        return IRQ_NONE;
    }

    /* Wake up waiting read process */
    atomic_set(&dev_data->calc_in_progress, 0);
    wake_up_interruptible(&dev_data->calc_wait_queue);

    return IRQ_HANDLED;
}

/* Open device file */
static int pcie_dev_open(struct inode *inode, struct file *filp)
{
    init_waitqueue_head(&pcie_demo_data.calc_wait_queue);
    dev_info(pcie_demo_data.pci_dev, "Device file opened\n");
    return 0;
}

/* Close/release device file */
static int pcie_dev_release(struct inode *inode, struct file *filp)
{
    dev_info(pcie_demo_data.pci_dev, "Device file closed\n");
    return 0;
}

/* Write data to device: accept input value for factorial computation */
static ssize_t pcie_dev_write(struct file *filp, const char __user *buf, size_t count, loff_t *offset)
{
    int ret;
    uint32_t input;
    uint8_t tmp_buf[4] = {0};

    ret = copy_from_user(tmp_buf, buf, count);
    if(ret < 0) {
        dev_err(&pcie_demo_data.pci_dev->dev, "Copy from user failed\n");
        return -EFAULT;
    }

    /* Assemble 32-bit input value */
    input = tmp_buf[0] | (tmp_buf[1] << 8) | (tmp_buf[2] << 16) | (tmp_buf[3] << 24);
    /* Start computation on hardware */
    iowrite32(input, pcie_demo_data.bar0_base + 0x08);
    atomic_set(&pcie_demo_data.calc_in_progress, 1);

    return count;
}

/* Read data from device: get computation result after interrupt */
static ssize_t pcie_dev_read(struct file *filp, char __user *buf, size_t count, loff_t *offset)
{
    int ret;
    uint32_t result;

    /* Wait for computation completion interrupt */
    ret = wait_event_interruptible(pcie_demo_data.calc_wait_queue, 
        0 == atomic_read(&pcie_demo_data.calc_in_progress));
    if(ret)
        return ret;

    /* Read result from hardware BAR0 */
    result = ioread32(pcie_demo_data.bar0_base + 0x08);
    /* Copy result back to user space */
    ret = copy_to_user(buf, &result, sizeof(uint32_t));

    return sizeof(uint32_t);
}

/* Device operations struct */
static struct file_operations pcie_dev_fops = {
    .owner      = THIS_MODULE,
    .open       = pcie_dev_open,
    .release    = pcie_dev_release,
    .read       = pcie_dev_read,
    .write      = pcie_dev_write,
};

/* PCIe probe: run when device is matched */
static int pcie_demo_probe(struct pci_dev *dev, const struct pci_device_id *id)
{
    int ret;
    int bar = 0;
    resource_size_t bar_len;

    ret = pci_enable_device(dev);
    if(ret) {
        dev_err(&dev->dev, "Failed to enable PCIe device\n");
        return ret;
    }

    /* Map BAR0 region to kernel virtual address */
    bar_len = pci_resource_len(dev, bar);
    pcie_demo_data.bar0_base = pci_iomap(dev, bar, bar_len);
    pcie_demo_data.pci_dev = dev;

    /* Register interrupt handler */
    ret = request_irq(dev->irq, pcie_demo_irq_handler, IRQF_SHARED, "pcie_demo", &pcie_demo_data);
    if(ret) {
        dev_err(&dev->dev, "Failed to request IRQ\n");
        return ret;
    }

    /* Enable hardware interrupt */
    iowrite32(0x80, pcie_demo_data.bar0_base + 0x20);

    return 0;
}

/* PCIe remove: run when driver is unloaded */
static void pcie_demo_remove(struct pci_dev *dev)
{
    /* Disable hardware interrupt */
    iowrite32(0x01, pcie_demo_data.bar0_base + 0x20);

    free_irq(dev->irq, &pcie_demo_data);
    pci_iounmap(dev, pcie_demo_data.bar0_base);
    pci_disable_device(dev);
}

/* PCIe driver structure */
static struct pci_driver pcie_demo_driver = {
    .name       = "pcie_demo",
    .id_table   = pcie_demo_ids,
    .probe      = pcie_demo_probe,
    .remove     = pcie_demo_remove,
};

/* Driver initialization entry */
static int __init pcie_demo_init(void)
{
    int ret = pci_register_driver(&pcie_demo_driver);

    if(pcie_demo_data.pci_dev == NULL){
        pr_err("pcie_demo: Failed to probe PCIe device\n");
        return ret;
    }

    /* Create character device node */
    ret = alloc_chrdev_region(&pcie_demo_data.dev_num, 0, 1, "pcie_demo");
    if(ret != 0)
        return ret;

    pcie_demo_data.cdev.owner = THIS_MODULE;
    cdev_init(&pcie_demo_data.cdev, &pcie_dev_fops);
    cdev_add(&pcie_demo_data.cdev, pcie_demo_data.dev_num, 1);

    pcie_demo_data.dev_class = class_create(THIS_MODULE, "pcie_demo");
    if (IS_ERR(pcie_demo_data.dev_class)) {
        unregister_chrdev_region(pcie_demo_data.dev_num, 1);
        return PTR_ERR(pcie_demo_data.dev_class);
    }

    pcie_demo_data.device = device_create(pcie_demo_data.dev_class, NULL, pcie_demo_data.dev_num, NULL, "pcie_demo");
    if (IS_ERR(pcie_demo_data.device)) {
        class_destroy(pcie_demo_data.dev_class);
        unregister_chrdev_region(pcie_demo_data.dev_num, 1);
        return PTR_ERR(pcie_demo_data.device);
    }

    return 0;
}

static void __exit pcie_demo_exit(void)
{
    if(pcie_demo_data.pci_dev != NULL) {
        cdev_del(&pcie_demo_data.cdev);                     
        unregister_chrdev_region(pcie_demo_data.dev_num, 1);

        device_destroy(pcie_demo_data.dev_class, pcie_demo_data.dev_num);
        class_destroy(pcie_demo_data.dev_class);
    }
    
    pci_unregister_driver(&pcie_demo_driver);
}

module_init(pcie_demo_init);
module_exit(pcie_demo_exit);
MODULE_LICENSE("GPL");
MODULE_INFO(intree, "Y");

Writing User Space Test Program

Create a user space test application test_pcie.c to interact with the driver:

#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

int main(int argc, char *argv[])
{
    int fd;
    uint32_t input = 6;
    uint32_t result;
    char *dev_path = "/dev/pcie_demo";

    /* Open device node */
    fd = open(dev_path, O_RDWR);
    if(fd < 0){
        perror("Failed to open device node");
        return EXIT_FAILURE;
    }

    /* Send input value to hardware */
    if(write(fd, &input, sizeof(uint32_t)) != sizeof(uint32_t)){
        perror("Failed to write to device");
        close(fd);
        return EXIT_FAILURE;
    }

    /* Read computation result */
    if(read(fd, &result, sizeof(uint32_t)) != sizeof(uint32_t)){
        perror("Failed to read from device");
        close(fd);
        return EXIT_FAILURE;
    }

    printf("Factorial of %d = %d\n", input, result);

    close(fd);
    return EXIT_SUCCESS;
}

Testing the Driver

Compile and load the kernel driver, then compile the test application with the following command:

gcc test_pcie.c -o pcie_test

Run test application, you will get the output Factorial of 6 = 720, which matches the expected result of 6! = 6 * 5 * 4 * 3 * 2 * 1 = 720.