修改k8s kube-proxy转发为ipvs

查看kube-proxy目前使用的转发模式
a. 通过查看kube-proxy Pod日志来确定

[root@k8s-master ~]# kubectl -n kube-system get pod -o wide | grep kube-proxy
kube-proxy-bt2lf                     1/1     Running   0          3m26s   192.168.44.148   k8s-master   <none>           <none>
[root@k8s-master ~]# kubectl -n kube-system logs kube-proxy-bt2lf
I0426 04:28:51.345337       1 server_others.go:69] "Using iptables proxy"		# 可以看出kube-proxy转发模式为iptables 
I0426 04:28:51.354294       1 node.go:141] Successfully retrieved node IP: 192.168.44.148
I0426 04:28:51.355978       1 conntrack.go:52] "Setting nf_conntrack_max" nfConntrackMax=131072
I0426 04:28:51.385423       1 server.go:632] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4"
I0426 04:28:51.387180       1 server_others.go:152] "Using iptables Proxier"
I0426 04:28:51.387205       1 server_others.go:421] "Detect-local-mode set to ClusterCIDR, but no cluster CIDR for family" ipFamily="IPv6"
I0426 04:28:51.387212       1 server_others.go:438] "Defaulting to no-op detect-local"
I0426 04:28:51.387279       1 proxier.go:251] "Setting route_localnet=1 to allow node-ports on localhost; to change this either disable iptables.localhostNodePorts (--iptables-localhost-nodeports) or set nodePortAddresses (--nodeport-addresses) to filter loopback addresses"
I0426 04:28:51.387658       1 server.go:846] "Version info" version="v1.28.0"
I0426 04:28:51.387670       1 server.go:848] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0426 04:28:51.389066       1 config.go:188] "Starting service config controller"
I0426 04:28:51.389113       1 shared_informer.go:311] Waiting for caches to sync for service config
I0426 04:28:51.389136       1 config.go:97] "Starting endpoint slice config controller"
I0426 04:28:51.389140       1 shared_informer.go:311] Waiting for caches to sync for endpoint slice config
I0426 04:28:51.389746       1 config.go:315] "Starting node config controller"
I0426 04:28:51.389754       1 shared_informer.go:311] Waiting for caches to sync for node config
I0426 04:28:51.491116       1 shared_informer.go:318] Caches are synced for node config
I0426 04:28:51.491122       1 shared_informer.go:318] Caches are synced for service config
I0426 04:28:51.491134       1 shared_informer.go:318] Caches are synced for endpoint slice config

b. 通过查看kube-proxy 的ConfigMap查看

[root@k8s-master ~]# kubectl -n kube-system get configmap | grep kube-proxy
kube-proxy                                             2      151m
[root@k8s-master ~]# kubectl -n kube-system get configmap -o yaml | grep  mode -C 1metricsBindAddress: ""mode: ""			# 为空默认就是iptables， 如果是ipvs，此处就会配置成ipvsnodePortAddresses: null
--extraArgs:authorization-mode: Node,RBACtimeoutForControlPlane: 4m0s
--authorization:mode: Webhookwebhook:

修改kube-proxy转发模式为 ipvs
2.1、首先所有节点需要安装ipvs软件（master和node节点）

# 所有节点安装yum -y install ipvsadm ipset

# 加载内核模块，所有节点执行
cat > /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOFchmod 755 /etc/sysconfig/modules/ipvs.modules
source /etc/sysconfig/modules/ipvs.modules

# 查看内核模块是否被加载
[root@k8s-master ~]# lsmod | grep ip_vs
ip_vs_sh               12688  0 
ip_vs_wrr              12697  0 
ip_vs_rr               12600  8 
ip_vs                 145458  14 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack          139264  10 ip_vs,nf_nat,nf_nat_ipv4,nf_nat_ipv6,xt_conntrack,nf_nat_masquerade_ipv4,nf_nat_masquerade_ipv6,nf_conntrack_netlink,nf_conntrack_ipv4,nf_conntrack_ipv6
libcrc32c              12644  4 xfs,ip_vs,nf_nat,nf_conntrack

2.2、修改kube-proxy 的configmap配置，将 mode: "" 改成 mode: "ipvs"

[root@k8s-master ~]# kubectl -n kube-system edit configmap kube-proxy
configmap/kube-proxy edited
[root@k8s-master ~]# kubectl -n kube-system get configmap -o yaml | grep  mode -C 1metricsBindAddress: ""mode: "ipvs"		# 现在转发模式变成了ipvsnodePortAddresses: null
--extraArgs:authorization-mode: Node,RBACtimeoutForControlPlane: 4m0s
--authorization:mode: Webhookwebhook:

2.3、现在只需要重新发布DaemonSet kube-proxy 即可

kube-proxy Pod是由DaemonSet 控制器管理。所以会在每个节点上都部署一个 kube-proxy Pod，重新发布 DaemonSet 会将所有节点的 kube-proxy Pod更新。

# 查看 DaemonSet 
[root@k8s-master ~]# kubectl -n kube-system get DaemonSet 
NAME         DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR            AGE
kube-proxy   1         1         1       1            1           kubernetes.io/os=linux   166m
# 重新发布 daemonSet kube-proxy
[root@k8s-master ~]# kubectl -n kube-system rollout restart daemonSet kube-proxy
daemonset.apps/kube-proxy restarted

查看现在kube-proxy Pod使用的转发模式

[root@k8s-master ~]# kubectl -n kube-system logs kube-proxy-qhmw2
I0426 05:33:18.817382       1 node.go:141] Successfully retrieved node IP: 192.168.44.148
I0426 05:33:18.820228       1 conntrack.go:52] "Setting nf_conntrack_max" nfConntrackMax=131072
I0426 05:33:18.869417       1 server.go:632] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4"
I0426 05:33:18.880584       1 server_others.go:218] "Using ipvs Proxier"		# 可以看出现在使用的转发模式是 ipvs
I0426 05:33:18.880629       1 server_others.go:421] "Detect-local-mode set to ClusterCIDR, but no cluster CIDR for family" ipFamily="IPv6"
I0426 05:33:18.880650       1 server_others.go:438] "Defaulting to no-op detect-local"
E0426 05:33:18.880922       1 proxier.go:354] "Can't set sysctl, kernel version doesn't satisfy minimum version requirements" sysctl="net/ipv4/vs/conn_reuse_mode" minimumKernelVersion="4.1"
I0426 05:33:18.881005       1 proxier.go:408] "IPVS scheduler not specified, use rr by default"
E0426 05:33:18.881105       1 proxier.go:354] "Can't set sysctl, kernel version doesn't satisfy minimum version requirements" sysctl="net/ipv4/vs/conn_reuse_mode" minimumKernelVersion="4.1"
I0426 05:33:18.881160       1 proxier.go:408] "IPVS scheduler not specified, use rr by default"
I0426 05:33:18.881179       1 ipset.go:116] "Ipset name truncated" ipSetName="KUBE-6-LOAD-BALANCER-SOURCE-CIDR" truncatedName="KUBE-6-LOAD-BALANCER-SOURCE-CID"
I0426 05:33:18.881190       1 ipset.go:116] "Ipset name truncated" ipSetName="KUBE-6-NODE-PORT-LOCAL-SCTP-HASH" truncatedName="KUBE-6-NODE-PORT-LOCAL-SCTP-HAS"
I0426 05:33:18.881651       1 server.go:846] "Version info" version="v1.28.0"
I0426 05:33:18.881661       1 server.go:848] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0426 05:33:18.882958       1 config.go:188] "Starting service config controller"
I0426 05:33:18.883031       1 shared_informer.go:311] Waiting for caches to sync for service config
I0426 05:33:18.883080       1 config.go:97] "Starting endpoint slice config controller"
I0426 05:33:18.883085       1 shared_informer.go:311] Waiting for caches to sync for endpoint slice config
I0426 05:33:18.884445       1 config.go:315] "Starting node config controller"
I0426 05:33:18.884471       1 shared_informer.go:311] Waiting for caches to sync for node config
I0426 05:33:18.984240       1 shared_informer.go:318] Caches are synced for endpoint slice config
I0426 05:33:18.984331       1 shared_informer.go:318] Caches are synced for service config
I0426 05:33:18.984497       1 shared_informer.go:318] Caches are synced for node confi

# 也可以通过 ipvsadm -Ln 查看 ipvs hash表来查看是否能成功转发流量，观察一下 ipvs hash表
# 查看node IP ： 192.168.44.148
[root@k8s-master ~]# kubectl get node -o wide
NAME         STATUS   ROLES           AGE    VERSION   INTERNAL-IP      EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION           CONTAINER-RUNTIM
k8s-master   Ready    control-plane   3h5m   v1.28.2   192.168.44.148   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   containerd://1.7.15
# 查看Pod IP : 10.244.0.4 
[root@k8s-master ~]# kubectl get pod -o wide
NAME    READY   STATUS    RESTARTS   AGE    IP           NODE         NOMINATED NODE   READINESS GATES
nginx   1/1     Running   0          153m   10.244.0.4   k8s-master   <none>           <none>
# 查看service IP：10.104.68.184  可以看出 service 模式为NodePort，外部端口为 30305
[root@k8s-master ~]# kubectl get svc
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        3h3m
nginx        NodePort    10.104.68.184   <none>        80:30305/TCP   153m
# 查看ipvs hash表
[root@k8s-master ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.44.148:30305 rr-> 10.244.0.4:80                Masq    1      0          0         
TCP  10.96.0.1:443 rr-> 192.168.44.148:6443          Masq    1      0          0         
TCP  10.96.0.10:53 rr-> 10.244.0.2:53                Masq    1      0          0         -> 10.244.0.3:53                Masq    1      0          0         
TCP  10.96.0.10:9153 rr-> 10.244.0.2:9153              Masq    1      0          0         -> 10.244.0.3:9153              Masq    1      0          0         
TCP  10.104.68.184:80 rr-> 10.244.0.4:80                Masq    1      0          0         
TCP  10.244.0.0:30305 rr-> 10.244.0.4:80                Masq    1      0          0         
TCP  10.244.0.1:30305 rr-> 10.244.0.4:80                Masq    1      0          0         
UDP  10.96.0.10:53 rr-> 10.244.0.2:53                Masq    1      0          0         -> 10.244.0.3:53                Masq    1      0

参考： https://blog.csdn.net/ss810540895/article/details/127264891
https://blog.csdn.net/LONG_Yi_1994/article/details/131459025
https://blog.csdn.net/qq_36807862/article/details/106068871