以下是一个基本的qemu使用virtio访问spdk的代码案例:
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdbool.h>
#include <inttypes.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <signal.h>
#include <rte_config.h>
#include <rte_mempool.h>
#include <rte_malloc.h>
#include <rte_ethdev.h>
#include <rte_lcore.h>
#include <rte_cycles.h>
#include <rte_eal.h>
#include <rte_common.h>
#include <rte_ring.h>
#include <rte_mbuf.h>
#define VIRTIO_PCI_VENDOR_ID 0x1af4
#define VIRTIO_PCI_DEVICE_ID_NET 0x1000
#define SPDK_RING_SIZE 1024
#define SPDK_MEMPOOL_CACHE_SIZE 256
struct spdk_virtio_dev {
struct virtio_dev vdev;
int lcore_id; /**< The id of the core that is used to manage this device. */
struct rte_ring *rx_ring; /**< Packet Rx queue. */
struct rte_mempool *tx_pool; /**< Packet Tx pool. */
};
/* Virtio device initialization function */
static void spdk_virtio_dev_init(struct spdk_virtio_dev *dev, int lcore_id)
{
dev->lcore_id = lcore_id;
dev->vdev.vtpci_ops = &virtio_pci_ops;
dev->vdev.notify_ops = &virtio_user_notify_ops;
dev->vdev.is_hw = false;
dev->vdev.mrg_rxbuf = true;
dev->rx_ring = rte_ring_create("spdk_virtio_rx_ring", SPDK_RING_SIZE,
rte_socket_id(), RING_F_SP_ENQ|RING_F_SC_DEQ);
if (dev->rx_ring == NULL) {
fprintf(stderr, "Error: Cannot create rx ring: %s\n", strerror(errno));
exit(1);
}
dev->tx_pool = rte_pktmbuf_pool_create("spdk_virtio_tx_pool", 65535 * 128,
SPDK_MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
rte_socket_id());
if (dev->tx_pool == NULL) {
fprintf(stderr, "Error: Cannot create tx pool: %s\n", strerror(errno));
exit(1);
}
dev->vdev.backend_data = dev;
}
/* Virtio device driver notification function */
static void spdk_virtio_dev_notify(void *_dev, struct virtio_vq_info *vq)
{
struct spdk_virtio_dev *dev = _dev;
uint16_t avail_idx, used_idx;
struct rte_mbuf *pkt_bufs[SPDK_RING_SIZE];
struct virtio_net_hdr_mrg_rxbuf *hdr;
int i, nb_rx;
/* Receive packets from the RX queue */
nb_rx = rte_ring_dequeue_burst(dev->rx_ring, (void **)pkt_bufs, SPDK_RING_SIZE, NULL);
/* Transmit packets to the TX queue */
for (i = 0; i < nb_rx; i++) {
hdr = rte_pktmbuf_mtod(pkt_bufs[i], struct virtio_net_hdr_mrg_rxbuf *);
hdr->num_buffers = 2;
hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
used_idx = vq->vq_nentries - vq->vq_avail_idx;
if (virtio_enqueue_pkt(dev->vdev.vqs[vq->vq_queue_index], dev->tx_pool,
&used_idx, pkt_bufs[i], 2) < 0) {
rte_pktmbuf_free(pkt_bufs[i]);
}
}
/* Process completed packets from the RX queue */
while (1) {
avail_idx = vq->vq_avail_idx;
used_idx = vq->vq_nentries - vq->vq_used_idx;
if (used_idx == 0) {
break;
}
for (i = 0; i < used_idx; i++) {
struct rte_mbuf *pkt_buf;
uint32_t len;
pkt_buf = vq->vq_desc[avail_idx].cookie;
len = vq->vq_desc[avail_idx].len;
rte_pktmbuf_trim(pkt_buf, len);
/* Add packet to the RX queue */
if (rte_ring_enqueue(dev->rx_ring, pkt_buf) < 0) {
rte_pktmbuf_free(pkt_buf);
}
avail_idx++;
if (avail_idx >= vq->vq_nentries) {
avail_idx = 0;
}
}
virtio_dequeue_pkt(dev->vdev.vqs[vq->vq_queue_index], used_idx);
vq->vq_avail_idx = avail_idx;
}
}
int main(int argc, char **argv)
{
int lcore_id, ret;
struct spdk_virtio_dev dev;
/* Initialize the EAL */
ret = rte_eal_init(argc, argv);
if (ret < 0) {
fprintf(stderr, "Error: Cannot initialize EAL\n");
return 1;
}
/* Initialize the virtio device */
spdk_virtio_dev_init(&dev, rte_lcore_id());
/* Register the device with the SPDK vhost driver */
ret = rte_vhost_driver_register(NULL, "spdk_virtio");
if (ret != 0) {
fprintf(stderr, "Error: Cannot register vhost driver: %s\n", strerror(-ret));
exit(1);
}
ret = rte_vhost_driver_attach_device("spdk_virtio", "/dev/vhost-net", &dev.vdev);
if (ret != 0) {
fprintf(stderr, "Error: Cannot attach device: %s\n", strerror(-ret));
exit(1);
}
/* Start the vhost driver */
lcore_id = rte_get_next_lcore(rte_lcore_id(), 0, 0);
rte_eal_remote_launch(rte_vhost_driver_start, "spdk_virtio", lcore_id);
/* Wait for the vhost driver to stop */
rte_eal_wait_lcore(lcore_id);
/* Detach the device from the vhost driver */
rte_vhost_driver_detach_device("/dev/vhost-net", &dev.vdev);
/* Unregister the vhost driver */
rte_vhost_driver_unregister("spdk_virtio");
return 0;
}
此代码示例使用SPDK虚拟主机驱动程序将virtio设备注册到vhost-net内核模块中。 虚拟设备在第一次和第二次设备初始化之间共享数据,因此需要考虑这种情况。
在此示例中,当收到来自客户端的数据包时,将其添加到RX队列中进行后续处理。 从TX队列中发送数据包时,将其添加到virtio网络标头中,设置标志并调用virtio_enqueue_pkt函数。
收到来自主机的完成通知时,使用virtio_dequeue_pkt函数处理已完成的数据包。 在此函数中,将数据包放入RX队列以进行进一步处理。