以下是一个简单的Linux NVMe RDMA代码案例,用于连接到远程NVMe设备并进行读写操作:
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <infiniband/verbs.h>
#define BUFFER_SIZE 4096
int main() {
int nvme_fd;
struct ibv_context *context;
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_qp *qp;
struct ibv_mr *mr;
void *buffer;
// 打开NVMe设备文件
nvme_fd = open("/dev/nvme0n1", O_RDWR | O_DIRECT);
if (nvme_fd == -1) {
perror("Failed to open NVMe device file");
return -1;
}
// 创建InfiniBand上下文
context = ibv_open_device(NULL);
if (context == NULL) {
perror("Failed to open InfiniBand device");
close(nvme_fd);
return -1;
}
// 分配Protection Domain(PD)
pd = ibv_alloc_pd(context);
if (pd == NULL) {
perror("Failed to allocate Protection Domain");
ibv_close_device(context);
close(nvme_fd);
return -1;
}
// 创建完成队列(Completion Queue)
cq = ibv_create_cq(context, 2, NULL, NULL, 0);
if (cq == NULL) {
perror("Failed to create Completion Queue");
ibv_dealloc_pd(pd);
ibv_close_device(context);
close(nvme_fd);
return -1;
}
// 创建队列对(Queue Pair)
struct ibv_qp_init_attr qp_init_attr;
memset(&qp_init_attr, 0, sizeof(qp_init_attr));
qp_init_attr.qp_type = IBV_QPT_RC;
qp_init_attr.send_cq = cq;
qp_init_attr.recv_cq = cq;
qp_init_attr.cap.max_send_wr = 2;
qp_init_attr.cap.max_send_sge = 1;
qp_init_attr.cap.max_recv_wr = 2;
qp_init_attr.cap.max_recv_sge = 1;
qp = ibv_create_qp(pd, &qp_init_attr);
if (qp == NULL) {
perror("Failed to create Queue Pair");
ibv_destroy_cq(cq);
ibv_dealloc_pd(pd);
ibv_close_device(context);
close(nvme_fd);
return -1;
}
// 注册内存缓冲区
buffer = malloc(BUFFER_SIZE);
if (buffer == NULL) {
perror("Failed to allocate memory for buffer");
ibv_destroy_qp(qp);
ibv_destroy_cq(cq);
ibv_dealloc_pd(pd);
ibv_close_device(context);
close(nvme_fd);
return -1;
}
memset(buffer, 0, BUFFER_SIZE);
mr = ibv_reg_mr(pd, buffer, BUFFER_SIZE, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE);
if (mr == NULL) {
perror("Failed to register memory region");
free(buffer);
ibv_destroy_qp(qp);
ibv_destroy_cq(cq);
ibv_dealloc_pd(pd);
ibv_close_device(context);
close(nvme_fd);
return -1;
}
// 进行RDMA操作,例如读取数据到缓冲区
struct ibv_sge sge;
memset(&sge, 0, sizeof(sge));
sge.addr = (uintptr_t)buffer;
sge.length = BUFFER_SIZE;
sge.lkey = mr->lkey;
struct ibv_send_wr send_wr;
memset(&send_wr, 0, sizeof(send_wr));
send_wr.wr_id = 1;
send_wr.sg_list = &sge;
send_wr.num_sge = 1;
send_wr.opcode = IBV_WR_RDMA_READ;
send_wr.send_flags = IBV_SEND_SIGNALED;
struct ibv_send_wr *bad_send_wr;
if (ibv_post_send(qp, &send_wr, &bad_send_wr) != 0) {
perror("Failed to post RDMA READ request");
ibv_dereg_mr(mr);
free(buffer);
ibv_destroy_qp(qp);
ibv_destroy_cq(cq);
ibv_dealloc_pd(pd);
ibv_close_device(context);
close(nvme_fd);
return -1;
}
// 等待RDMA完成
struct ibv_wc wc[2];
if(ibv_poll_cq(cq, 2, wc) <= 0){
perror("Failed to poll completion queue");
}
else{
if(wc[0].status != IBV_WC_SUCCESS){
perror("Error in WR completion status");
}
else{
printf("RDMA Read completed successfully!\n");
// 在buffer中可以找到从NVMe读取的数据
}
}
// 清理资源
ibv_dereg_mr(mr);
free(buffer);
ibv_destroy_qp(qp);
ibv_destroy_cq(cq);
ibv_dealloc_pd(pd);
ibv_close_device(context);
close(nvme_fd);
return 0;
}
这只是一个简单示例,并且假设你已经正确配置了RDMA环境并有权限执行NVMe操作。在实际使用中,你可能需要根据具体情况进行修改。
内容由零声教学AI助手提供,问题来源于学员提问