linux NVMe RDMA的代码案例-DPDK中文网

以下是一个简单的Linux NVMe RDMA代码案例，用于连接到远程NVMe设备并进行读写操作：

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <infiniband/verbs.h>
#define BUFFER_SIZE 4096
int main() {
    int nvme_fd;
    struct ibv_context *context;
    struct ibv_pd *pd;
    struct ibv_cq *cq;
    struct ibv_qp *qp;
    struct ibv_mr *mr;
    void *buffer;
    // 打开NVMe设备文件
    nvme_fd = open("/dev/nvme0n1", O_RDWR | O_DIRECT);
    if (nvme_fd == -1) {
        perror("Failed to open NVMe device file");
        return -1;
    }
    // 创建InfiniBand上下文
    context = ibv_open_device(NULL);
    if (context == NULL) {
        perror("Failed to open InfiniBand device");
        close(nvme_fd);
        return -1;
    }
   // 分配Protection Domain（PD）
   pd = ibv_alloc_pd(context);
   if (pd == NULL) {
       perror("Failed to allocate Protection Domain");
       ibv_close_device(context);
       close(nvme_fd);
       return -1;
   }
   // 创建完成队列（Completion Queue）
   cq = ibv_create_cq(context, 2, NULL, NULL, 0);
   if (cq == NULL) {
       perror("Failed to create Completion Queue");
       ibv_dealloc_pd(pd);
       ibv_close_device(context);
       close(nvme_fd);
       return -1;
   }
   // 创建队列对（Queue Pair）
   struct ibv_qp_init_attr qp_init_attr;
   memset(&qp_init_attr, 0, sizeof(qp_init_attr));
   qp_init_attr.qp_type = IBV_QPT_RC;
   qp_init_attr.send_cq = cq;
   qp_init_attr.recv_cq = cq;
   qp_init_attr.cap.max_send_wr = 2;
   qp_init_attr.cap.max_send_sge = 1;
   qp_init_attr.cap.max_recv_wr = 2;
   qp_init_attr.cap.max_recv_sge = 1;
   qp = ibv_create_qp(pd, &qp_init_attr);
   if (qp == NULL) {
       perror("Failed to create Queue Pair");
       ibv_destroy_cq(cq);
       ibv_dealloc_pd(pd);
       ibv_close_device(context);
       close(nvme_fd);
       return -1;
   }
    // 注册内存缓冲区
    buffer = malloc(BUFFER_SIZE);
    if (buffer == NULL) {
        perror("Failed to allocate memory for buffer");
        ibv_destroy_qp(qp);
        ibv_destroy_cq(cq);
        ibv_dealloc_pd(pd);
        ibv_close_device(context);
        close(nvme_fd);
        return -1;
    }
    memset(buffer, 0, BUFFER_SIZE);
    mr = ibv_reg_mr(pd, buffer, BUFFER_SIZE, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE);
    if (mr == NULL) {
        perror("Failed to register memory region");
        free(buffer); 
        ibv_destroy_qp(qp);
        ibv_destroy_cq(cq);  
        ibv_dealloc_pd(pd);    
	   	ibv_close_device(context);     
	  	close(nvme_fd);      
	  	return -1;  
	}    
	// 进行RDMA操作，例如读取数据到缓冲区
    struct ibv_sge sge;
    memset(&sge, 0, sizeof(sge));
    sge.addr = (uintptr_t)buffer;
    sge.length = BUFFER_SIZE;
    sge.lkey = mr->lkey;
	struct ibv_send_wr send_wr;
	memset(&send_wr, 0, sizeof(send_wr));
	send_wr.wr_id = 1;
	send_wr.sg_list = &sge;
	send_wr.num_sge = 1;
	send_wr.opcode = IBV_WR_RDMA_READ;
	send_wr.send_flags = IBV_SEND_SIGNALED;
	struct ibv_send_wr *bad_send_wr;
	if (ibv_post_send(qp, &send_wr, &bad_send_wr) != 0) {
	    perror("Failed to post RDMA READ request");
        ibv_dereg_mr(mr);
        free(buffer); 
        ibv_destroy_qp(qp);
        ibv_destroy_cq(cq);  
        ibv_dealloc_pd(pd);    
	   	ibv_close_device(context);     
	  	close(nvme_fd);      
	  	return -1;  
	}
    // 等待RDMA完成
    struct ibv_wc wc[2];
	if(ibv_poll_cq(cq, 2, wc) <= 0){
	   	perror("Failed to poll completion queue");
	}
	else{
	  if(wc[0].status != IBV_WC_SUCCESS){
		  perror("Error in WR completion status");
	  }
	  else{
	  	 printf("RDMA Read completed successfully!\n");
         // 在buffer中可以找到从NVMe读取的数据
	  }
   }
   // 清理资源
   ibv_dereg_mr(mr);
   free(buffer); 
   ibv_destroy_qp(qp);
   ibv_destroy_cq(cq);  
   ibv_dealloc_pd(pd);    
   ibv_close_device(context);     
   close(nvme_fd);      
   
   return 0;
}