Linux系统TCP连接性能

一、测试环境

debian11 64bit

设置系统网络限制

sudo vi /etc/sysctl.conffs.file-max=10485760net.ipv4.ip_local_port_range=1024 65535#net.ipv4.tcp_tw_recycle=1net.ipv4.tcp_tw_reuse=1net.ipv4.tcp_timestamps=1net.core.rmem_default=209715200net.core.wmem_default=209715200net.core.rmem_max=209715200net.core.wmem_max=209715200
sudo /sbin/sysctl -p

设置文件描述符

sudo vi /etc/security/limits.conf root soft nofile 65535root hard nofile 65535
sudo vi /etc/profileulimit -SHn 65535sudo reboot

二、测试代码

服务端代码: tcpserver.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <time.h>
#include <sys/time.h>
#include <pthread.h>#define CONNECT_NUM 65535static int fd_count = 0;
static int fd_list[CONNECT_NUM] = {0};
pthread_mutex_t	connect_mutex;void* connect_thread(void* arg){uint16_t port = (uint16_t)arg;int sockfd = socket(AF_INET,SOCK_STREAM,0);if(sockfd < 0){printf("socket error!\n");exit(-1);}struct sockaddr_in my_addr;memset(&my_addr, 0, sizeof(my_addr));my_addr.sin_family=AF_INET;my_addr.sin_port=htons(port);my_addr.sin_addr.s_addr=htonl(INADDR_ANY);printf("Binding server to port %d\n",port);int err_log = bind(sockfd,(struct sockaddr *)&my_addr,sizeof(my_addr));if(err_log!=0){printf("bind error!\n");close(sockfd);exit(-1);}err_log = listen(sockfd, 1000);if(err_log != 0){perror("listen");close(sockfd);		exit(-1);}printf("listen client @port=%d...\n",port);while(1){int count=0;char recv_buf[128] = "";struct sockaddr_in client_addr;char cli_ip[INET_ADDRSTRLEN] = "";socklen_t cliaddr_len = sizeof(client_addr);int connfd = accept(sockfd, (struct sockaddr*)&client_addr, &cliaddr_len);       if(connfd < 0){perror("accept");continue;}inet_ntop(AF_INET, &client_addr.sin_addr, cli_ip, INET_ADDRSTRLEN);recv(connfd, recv_buf, sizeof(recv_buf), 0);if(strcmp(recv_buf, "ok") != 0){printf("recv error by %s\n", recv_buf);}pthread_mutex_lock(&connect_mutex);fd_list[fd_count] = connfd;fd_count++;pthread_mutex_unlock(&connect_mutex);printf("thread%d ip:%s, port:%d, connfd:%d, fd_count:%d\n",port, cli_ip, ntohs(client_addr.sin_port), connfd, fd_count);}close(sockfd);return NULL;}
int main(int argc, char *argv[]){pthread_t pid1, pid2;uint16_t port1 = 10000;uint16_t port2 = 10001;pthread_mutex_init(&connect_mutex, NULL);pthread_create(&pid1, NULL, connect_thread, (void*)port1);pthread_create(&pid2, NULL, connect_thread, (void*)port2);pthread_join(pid1, NULL);pthread_join(pid2, NULL);return 0;
}

客户端代码: tcpclient.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <time.h>
#include <sys/time.h>
#include <pthread.h>static int connect_total = 1;
typedef struct {pthread_t pid;uint16_t port;uint32_t duration;
}th_info_t;
uint64_t clocktime_now() {struct timespec ts;clock_gettime(CLOCK_MONOTONIC,&ts);return ts.tv_sec * 1000000 + ( ts.tv_nsec / 1000 );
}void* connect_thread(void* arg){char server_ip[16] = "192.168.1.108";int connect_num = (connect_total+1)/2;int connect_idx = 0;th_info_t* th_info = (th_info_t*)arg;printf("thread%d connect_num:%d\n", th_info->port, connect_num);uint64_t start_time = clocktime_now();while(connect_idx < connect_num){connect_idx++;uint64_t start = clocktime_now();int sockfd = socket(AF_INET, SOCK_STREAM, 0);if(sockfd < 0){printf("sockfd error!\n");exit(-1);}struct sockaddr_in dest_addr;memset(&dest_addr, 0, sizeof(dest_addr));dest_addr.sin_family = AF_INET;dest_addr.sin_port = htons(th_info->port);inet_pton(AF_INET, server_ip, &dest_addr.sin_addr);int err_log = connect(sockfd, (struct sockaddr*)&dest_addr, sizeof(dest_addr));      // 主动连接服务器if(err_log != 0){perror("connect");close(sockfd);exit(-1);}int ret = send(sockfd, "ok", strlen("ok"), 0);printf("thread%d connect fd:%d count:%d, ts:%dus\n", th_info->port, sockfd, connect_idx, clocktime_now()-start);}th_info->duration = (clocktime_now()-start_time)/1000;return NULL;
}
int main(int argc, char *argv[]){th_info_t th_info1, th_info2;th_info1.port = 10000;th_info2.port = 10001;if(argc >= 2){connect_total = atoi(argv[1]);}pthread_create(&th_info1.pid, NULL, connect_thread, (void*)&th_info1);pthread_create(&th_info2.pid, NULL, connect_thread, (void*)&th_info2);pthread_join(th_info1.pid, NULL);pthread_join(th_info2.pid, NULL);printf("thread%d connect success duration:%dms\n", th_info1.port, th_info1.duration);printf("thread%d connect success duration:%dms\n", th_info2.port, th_info2.duration);sleep(10);return 0;
}

编译脚本: make-linux.sh

#!/bin/bash
gcc -o server tcpserver.c -lpthread
gcc -o client tcpclient.c -lpthread

三、编译测试

将服务代码、客户端代码、编译脚本放入同一个目录
./make-linux.sh
测试
- 第一次测试：客户端建立40000连接，平均到10000和10001端口
  结果：每次连接完成20us(微妙)左右
- 第二次测试：客户端建立33000连接，只连接到10000端口
  结果：前32255个连接，每个连接20us(微妙)左右；后745个连接，每个连接完成3400us(微妙)左右
结论
- 单个socket的连接数超过最大值32255后，每个连接的完成耗时变大
- 为什么是32255？net.ipv4.ip_local_port_range设置端口范围 (65535-1024)/2=3225.5 ，至于为什么是超过范围量的二分之一变慢（改变范围，测出的结果），还没找到答案