dpdk18.11 收发包流程分析

pci probe

RTE_PMD_REGISTER_PCI(net_ixgbe, rte_ixgbe_pmd); 宏注册了net_ixgbe driver到pci bus

rte_ixgbe_pmd 的定义如下

static struct rte_pci_driver rte_ixgbe_pmd = {.id_table = pci_id_ixgbe_map,.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,.probe = eth_ixgbe_pci_probe,.remove = eth_ixgbe_pci_remove,
};

当扫描到到的device 能和pci_id_ixgbe_map 里的id 匹配上时就表示这个device和net_ixgbe 这个driver相匹配，接着便会调用.probe = eth_ixgbe_pci_probe指定的probe函数,这里我们关注下probe 函数对收发报函数的设置

static int
->eth_ixgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,struct rte_pci_device *)->retval = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name,sizeof(struct ixgbe_adapter),eth_dev_pci_specific_init, pci_dev,eth_ixgbe_dev_init, NULL);->eth_ixgbe_dev_init(struct rte_eth_dev *, void *init_params )

eth_ixgbe_dev_init 中会完成driver的初始化，其中就包括了调用 ixgbe_set_tx_function、ixgbe_set_rx_function根据情况设置对应的收发函数

static int
eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
{/*省略*/eth_dev->dev_ops = &ixgbe_eth_dev_ops;eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;eth_dev->tx_pkt_prepare = &ixgbe_prep_pkts;/*省略*/if (rte_eal_process_type() != RTE_PROC_PRIMARY) {struct ixgbe_tx_queue *txq;/* TX queue function in primary, set by last queue initialized* Tx queue may not initialized by primary process*/if (eth_dev->data->tx_queues) {txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1];ixgbe_set_tx_function(eth_dev, txq);} else {/* Use default TX function if we get here */PMD_INIT_LOG(NOTICE, "No TX queues configured yet. ""Using default TX function.");}ixgbe_set_rx_function(eth_dev);return 0;	//对于从核只会执行到这里}/*省略*/return 0;
}

设置收发函数

tx函数设置
- 发送队列offloads为0时
  - 未启用向量 tx_pkt_burst = ixgbe_xmit_pkts_simple
  - 启用向量 dev->tx_pkt_burst = ixgbe_xmit_pkts_vec;
- 发送队列offloads非0
  - dev->tx_pkt_burst = ixgbe_xmit_pkts
rx函数设置
- if dev->data->lro设置时
  - adapter->rx_bulk_alloc_allowed 设置，允许bulk时dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
  - dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
- else if dev->data->scattered_rx 设置时
  - dev->data->scattered_rx 设置，支持向量时 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec;
  - adapter->rx_bulk_alloc_allowed 设置支持bulk时 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
  - dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
- else ifadapter->rx_vec_allowed 设置时
  - dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
- else if adapter->rx_bulk_alloc_allowed 设置时
  - dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
- else dev->rx_pkt_burst = ixgbe_recv_pkts;

启动网卡及配置收发队列

下面是以L2FWD为例，总结了下dpdk对网卡配置、收发队列配置的API。以下代码做了省略，删除了返回判断代码

在这里插入图片描述

int
main(int argc, char **argv)
{   struct lcore_queue_conf *qconf;int ret;uint16_t nb_ports;uint16_t nb_ports_available = 0;uint16_t portid, last_port;unsigned lcore_id, rx_lcore_id;unsigned nb_ports_in_mask = 0;unsigned int nb_lcores = 0;unsigned int nb_mbufs;/* init EAL */ret = rte_eal_init(argc, argv);if (ret < 0)rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");argc -= ret;argv += ret;force_quit = false;signal(SIGINT, signal_handler);signal(SIGTERM, signal_handler);/* 解析传入的参数 (after the EAL ones) */ret = l2fwd_parse_args(argc, argv);if (ret < 0)rte_exit(EXIT_FAILURE, "Invalid L2FWD arguments\n");printf("MAC updating %s\n", mac_updating ? "enabled" : "disabled");nb_ports = rte_eth_dev_count_avail();if (nb_ports == 0)rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");/* check port mask to possible port mask */if (l2fwd_enabled_port_mask & ~((1 << nb_ports) - 1))rte_exit(EXIT_FAILURE, "Invalid portmask; possible (0x%x)\n",(1 << nb_ports) - 1);/*计算所需的mbuf个数*/nb_mbufs = RTE_MAX(nb_ports * (nb_rxd + nb_txd + MAX_PKT_BURST +nb_lcores * MEMPOOL_CACHE_SIZE), 8192U);/* 创建内存池 */l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", nb_mbufs,MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE,rte_socket_id());/* Initialise each port */RTE_ETH_FOREACH_DEV(portid) {struct rte_eth_rxconf rxq_conf;struct rte_eth_txconf txq_conf;struct rte_eth_conf local_port_conf = port_conf;struct rte_eth_dev_info dev_info;/* skip ports that are not enabled */if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) {printf("Skipping disabled port %u\n", portid);continue;}nb_ports_available++;/* init port */printf("Initializing port %u... ", portid);fflush(stdout);rte_eth_dev_info_get(portid, &dev_info);if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)local_port_conf.txmode.offloads |=DEV_TX_OFFLOAD_MBUF_FAST_FREE;ret = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,&nb_txd);rte_eth_macaddr_get(portid,&l2fwd_ports_eth_addr[portid]);/* init one RX queue */fflush(stdout);rxq_conf = dev_info.default_rxconf;rxq_conf.offloads = local_port_conf.rxmode.offloads;ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,rte_eth_dev_socket_id(portid),&rxq_conf,l2fwd_pktmbuf_pool);if (ret < 0)rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n",ret, portid);/* init one TX queue on each port */fflush(stdout);txq_conf = dev_info.default_txconf;txq_conf.offloads = local_port_conf.txmode.offloads;ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,rte_eth_dev_socket_id(portid),&txq_conf);/* Initialize TX buffers */tx_buffer[portid] = rte_zmalloc_socket("tx_buffer",RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0,rte_eth_dev_socket_id(portid));if (tx_buffer[portid] == NULL)rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n",portid);rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST);/*设置错误统计回调函数*/ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[portid],rte_eth_tx_buffer_count_callback,&port_statistics[portid].dropped);/* Start device */ret = rte_eth_dev_start(portid);rte_eth_promiscuous_enable(portid);}if (!nb_ports_available) {rte_exit(EXIT_FAILURE,"All available ports are disabled. Please set portmask.\n");}check_all_ports_link_status(l2fwd_enabled_port_mask);ret = 0;/* launch per-lcore init on every lcore */rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, NULL, CALL_MASTER);RTE_LCORE_FOREACH_SLAVE(lcore_id) {if (rte_eal_wait_lcore(lcore_id) < 0) {ret = -1;break;}}RTE_ETH_FOREACH_DEV(portid) {if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)continue;printf("Closing port %d...", portid);rte_eth_dev_stop(portid);rte_eth_dev_close(portid);printf(" Done\n");}printf("Bye...\n");return ret;
}