krping模块是一个内核可加载模块，它实现了客户机/服务器ping/pong程序，这个模块仅仅为了测试内核rdma的API（单边的READ和Write；双边的SEND和RECEIVE）。该模块允许通过一个名为/proc/krping的/proc条目建立连接并运行ping/pong测试。这种简单的机制允许并发地启动多个内核线程，并避免了对用户空间应用程序的需要。
krping的项目链接，目前更新到的内核版本为5.5：
https://github.com/larrystevenwise/krping

krping的主要流程如下图所示：

主要是以下3步：
1.在开始数据传输前,client会先将ping data写入到事先准备好的start buff中
2. server会主动发起rdma read将ping data从start buff中读取到自己的rdma buff中
3.将buff中的ping data rdma write 到 client的rdma buff中. 至此一次完整的rdma ping-pong数据传输就完成了

工作过程如下图所示：

结合代码来说：
1.client 通过 send-recv 模式 将 start buf (client会写入原始的ping data) 的地址, 发放给server进行rdma read的rkey, 每次传递给对端的payload大小

static void krping_format_send(struct krping_cb *cb, u64 buf)
{struct krping_rdma_info *info = &cb->send_buf;u32 rkey;if (!cb->server || cb->wlat || cb->rlat || cb->bw) {//发放给server进行rdma read的rkeyrkey = krping_rdma_rkey(cb, buf, !cb->server_invalidate);//start buf (client会写入原始的ping data) 的地址info->buf = htonll(buf);info->rkey = htonl(rkey);//传递给对端的payload大小info->size = htonl(cb->size);DEBUG_LOG("RDMA addr %llx rkey %x len %d\n",(unsigned long long)buf, rkey, cb->size);}
}//post_send
ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr);

2.server 在recv操作的completion里, server获取到client传递过来的 start buf的总线地址(DMA分配的) , 还有操作start buf 的rkey, 以及start buf的大小

P("read from recvq\n");
ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr);
if (ret) {printk(KERN_ERR PFX "ib_post_recv failed: %d\n", ret);goto err2;
}krping_cq_event_handler:case IB_WC_RECV:DEBUG_LOG("recv completion\n");cb->stats.recv_bytes += sizeof(cb->recv_buf);cb->stats.recv_msgs++;if (cb->wlat || cb->rlat || cb->bw) {P("call server_recv\n");ret = server_recv(cb, &wc);} else {P("call client recv\n");ret = cb->server ? server_recv(cb, &wc) :client_recv(cb, &wc);}if (ret) {printk(KERN_ERR PFX "recv wc error: %d\n", ret);goto error;}ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr);if (ret) {printk(KERN_ERR PFX "post recv error: %d\n", ret);goto error;}wake_up_interruptible(&cb->sem);break;static int server_recv(struct krping_cb *cb, struct ib_wc *wc)
{if (wc->byte_len != sizeof(cb->recv_buf)) {printk(KERN_ERR PFX "Received bogus data, size %d\n", wc->byte_len);return -1;}cb->remote_rkey = ntohl(cb->recv_buf.rkey);cb->remote_addr = ntohll(cb->recv_buf.buf);cb->remote_len  = ntohl(cb->recv_buf.size);DEBUG_LOG("Received rkey %x addr %llx len %d from peer\n",cb->remote_rkey, (unsigned long long)cb->remote_addr, cb->remote_len);if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE)cb->state = RDMA_READ_ADV;elsecb->state = RDMA_WRITE_ADV;return 0;
}

3.server 构造rdma读请求, 开始从client的start buf中读取ping data

P("setup rdma_sq_wr and reset rkey\n");
cb->rdma_sq_wr.rkey = cb->remote_rkey; //设置rkey
cb->rdma_sq_wr.remote_addr = cb->remote_addr; //设置为client的start buff地址
cb->rdma_sq_wr.wr.sg_list->length = cb->remote_len; //
cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, !cb->read_inv); //设置rdma_sgl的lkey
cb->rdma_sq_wr.wr.next = NULL; /* 发起一个INV Read */
if (cb->read_inv) {P("send read with inv cmd\n");cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV;
} else {P("send read cmd\n");cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_READ;//链上一个opcode=local inv类型的WRcb->rdma_sq_wr.wr.next = &inv;memset(&inv, 0, sizeof inv);inv.opcode = IB_WR_LOCAL_INV;inv.ex.invalidate_rkey = cb->reg_mr->rkey;inv.send_flags = IB_SEND_FENCE;
}//post send wr list到qp
ret = ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr);
if (ret) {printk(KERN_ERR PFX "post send error %d\n", ret);break;
}
cb->rdma_sq_wr.wr.next = NULL;wait_event_interruptible(cb->sem, cb->state >= RDMA_READ_COMPLETE);

4.server 收到 rdma read 的WC之后, 就会主动调度等待队列里的进程

case IB_WC_RDMA_READ:DEBUG_LOG("rdma read completion\n");cb->stats.read_bytes += cb->rdma_sq_wr.wr.sg_list->length;cb->stats.read_msgs++;cb->state = RDMA_READ_COMPLETE;wake_up_interruptible(&cb->sem);break;

5.server从client rdma读取到ping data后, 会再次通过send-recv的方式发送一个go-ahead消息给client, 通知阻塞的client继续工作

//等待wc
ret = wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_COMPLETE);
if (cb->state != RDMA_WRITE_COMPLETE) {printk(KERN_ERR PFX "wait for RDMA_WRITE_COMPLETE state %d\n",cb->state);break;
}//重置状态为connected
cb->state = CONNECTED;if (cb->server && cb->server_invalidate) {cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey;cb->sq_wr.opcode = IB_WR_SEND_WITH_INV;DEBUG_LOG("send-w-inv rkey 0x%x\n", cb->remote_rkey);
} //post send to qp
ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr);
if (ret) {printk(KERN_ERR PFX "post send error %d\n", ret);break;
}
DEBUG_LOG("server posted go ahead\n");

6.client 收到 go-ahead消息的 WC

case IB_WC_RECV:DEBUG_LOG("recv completion\n");cb->stats.recv_bytes += sizeof(cb->recv_buf);cb->stats.recv_msgs++;if (cb->wlat || cb->rlat || cb->bw) {P("call server_recv\n");ret = server_recv(cb, &wc);} else {P("call client recv\n");ret = cb->server ? server_recv(cb, &wc) :client_recv(cb, &wc);}if (ret) {printk(KERN_ERR PFX "recv wc error: %d\n", ret);goto error;}//ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr);if (ret) {printk(KERN_ERR PFX "post recv error: %d\n", ret);goto error;}wake_up_interruptible(&cb->sem);break;static int client_recv(struct krping_cb *cb, struct ib_wc *wc)
{if (wc->byte_len != sizeof(cb->recv_buf)) {printk(KERN_ERR PFX "Received bogus data, size %d\n", wc->byte_len);return -1;}if (cb->state == RDMA_READ_ADV)cb->state = RDMA_WRITE_ADV;elsecb->state = RDMA_WRITE_COMPLETE;return 0;
}

krping的使用方法：

Server:

# modprobe rdma_krping
# echo "server,addr=192.168.69.127,port=9999" >/proc/krping

client:

# modprobe rdma_krping
# echo "client,addr=192.168.69.127,port=9999,count=100" >/proc/krping

参考资料
github上的krping项目：https://github.com/larrystevenwise/krping
其他博客：https://www.cnblogs.com/dennis-wong/p/16954934.html