Kubeadm高可用

1、环境准备

2、所有节点安装docker

3、所有节点安装kubeadm，kubelet和kubectl

4、部署 master 节点

高可用多 master 节点部署

master01（2C/4G，cpu核心数要求大于2）       192.168.80.20
master02（2C/4G，cpu核心数要求大于2）       192.168.80.14
master03（2C/4G，cpu核心数要求大于2）       192.168.80.15

1、环境准备

//所有节点，关闭防火墙规则，关闭selinux，关闭swap交换
systemctl stop firewalld
systemctl disable firewalld
setenforce 0
iptables -F
swapoff -a #交换分区必须要关闭
#加载 ip_vs 模块
for i in $(ls /usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs|grep -o "^[^.]*");do echo $i; /sbin/modinfo -F filename $i >/dev/null 2>&1 && /sbin/modprobe $i;done

//修改主机名
hostnamectl set-hostname master01
hostnamectl set-hostname master02
hostnamectl set-hostname master03

//所有节点修改hosts文件
vim /etc/hosts
192.168.80.20 master01
192.168.80.14 master02
192.168.80.15 master03

//调整内核参数
cat > /etc/sysctl.d/kubernetes.conf << EOF
#开启网桥模式，可将网桥的流量传递给iptables链
net.bridge.bridge-nf-call-ip6tables=1
net.bridge.bridge-nf-call-iptables=1
#关闭ipv6协议
net.ipv6.conf.all.disable_ipv6=1
net.ipv4.ip_forward=1
EOF

//生效参数
sysctl --system

2、所有节点安装docker

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce docker-ce-cli containerd.io

mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
"registry-mirrors": ["https://6ijb8ubo.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
}
}
EOF
#使用Systemd管理的Cgroup来进行资源控制与管理，因为相对Cgroupfs而言，Systemd限制CPU、内存等资源更加简单和成熟。
#日志使用json-file格式类型存储，大小为100M，保存在/var/log/containers目录下，方便ELK等日志系统收集和管理日志。

systemctl daemon-reload
systemctl restart docker.service
systemctl enable docker.service

3、所有节点安装kubeadm，kubelet和kubectl

//定义kubernetes源
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum install -y kubelet-1.15.1 kubeadm-1.15.1 kubectl-1.15.1

//开机自启kubelet
systemctl enable kubelet.service
#K8S通过kubeadm安装出来以后都是以Pod方式存在，即底层是以容器方式运行，所以kubelet必须设置开机自启

4、部署 master 节点

//上传 kubeadm-basic.images.tar.gz、start.keep.tar.gz、haproxy.tar、keepalived.tar、flannel.tar 压缩包文件至 /opt 目录
cd /opt
tar zxvf kubeadm-basic.images.tar.gz
tar zxvf start.keep.tar.gz -C /

//编写自动导入镜像的脚本
vim load-images.sh
#!/bin/bash
mv /opt/*.tar /opt/kubeadm-basic.images/
cd /opt/kubeadm-basic.images
for i in $(ls /opt/kubeadm-basic.images)
do
docker load -i $i
done

bash load-images.sh

//配置并启动 haproxy 和 keepalived
cd /data/lb/
vim etc/haproxy.cfg
...... #在末尾设置集群节点，这里为了防止haproxy负载转发到空节点上，先开启一个当前节点
49 server rancher01 192.168.80.20:6443
50 #server rancher02 192.168.80.14:6443
51 #server rancher03 192.168.80.15:6443

vim start-haproxy.sh
#!/bin/bash
MasterIP1=192.168.80.20       #指定 master01 的IP地址
MasterIP2=192.168.80.14       #指定 master02 的IP地址
MasterIP3=192.168.80.15       #指定 master03 的IP地址
MasterPort=6443

docker run -d --restart=always --name HAProxy-K8S -p 6444:6444 \
-e MasterIP1=$MasterIP1 \
-e MasterIP2=$MasterIP2 \
-e MasterIP3=$MasterIP3 \
-e MasterPort=$MasterPort \
-v /data/lb/etc/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg \
wise2c/haproxy-k8s

vim start-keepalived.sh
#!/bin/bash
VIRTUAL_IP=192.168.80.100 #设置集群 VIP 地址
INTERFACE=ens33 #指定网卡名称
NETMASK_BIT=24
CHECK_PORT=6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18

docker run -itd --restart=always --name=Keepalived-K8S \
--net=host --cap-add=NET_ADMIN \
-e VIRTUAL_IP=$VIRTUAL_IP \
-e INTERFACE=$INTERFACE \
-e CHECK_PORT=$CHECK_PORT \
-e RID=$RID \
-e VRID=$VRID \
-e NETMASK_BIT=$NETMASK_BIT \
-e MCAST_GROUP=$MCAST_GROUP \
wise2c/keepalived-k8s

bash start-haproxy.sh
netstat -natp | grep 6444
tcp 0 0 0.0.0.0:6444 0.0.0.0:* LISTEN 3987/docker-proxy

bash start-keepalived.sh
ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:01:00:b4 brd ff:ff:ff:ff:ff:ff
inet 192.168.80.20/24 brd 192.168.80.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.80.100/24 scope global secondary ens33
valid_lft forever preferred_lft forever

//复制镜像和脚本到其它 master 节点
cd /opt
scp -r kubeadm-basic.images load-images.sh root@master02:/opt
scp -r kubeadm-basic.images load-images.sh root@master03:/opt
scp -r /data root@master02:/
scp -r /data root@master03:/

//在其它 master 节点上执行脚本
bash /opt/load-images.sh
bash /data/lb/start-haproxy.sh
bash /data/lb/start-keepalived.sh

//初始化 master01 节点
kubeadm config print init-defaults > /opt/kubeadm-config.yaml

cd /opt/
vim kubeadm-config.yaml
......
11 localAPIEndpoint:
12 advertiseAddress: 192.168.80.20       #指定当前master节点的IP地址
13 bindPort: 6443
......
25 clusterName: kubernetes
26 controlPlaneEndpoint: "192.168.80.100:6444"   #指定集群 VIP 地址
27 controllerManager: {}
......
35 kubernetesVersion: v1.15.1               #指定kubernetes版本号
36 networking:
37 dnsDomain: cluster.local
38 podSubnet: "10.244.0.0/16"               #指定pod网段，10.244.0.0/16用于匹配flannel默认网段
39 serviceSubnet: 10.96.0.0/16           #指定service网段
40 scheduler: {}
--- #末尾再添加以下内容
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs                                   #把默认的service调度方式改为ipvs模式

kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs | tee kubeadm-init.log
#--experimental-upload-certs 参数可以在后续执行加入节点时自动分发证书文件，k8sV1.16版本开始替换为 --upload-certs
#tee kubeadm-init.log 用以输出日志

提示：
......
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:
#此命令是用于在其它 master 节点上执行加入群集
kubeadm join 192.168.80.100:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:b7c84262895d9926c2011d02d234bfb882f97e4774431a0fa20cfc7fa92cec52 \
--control-plane --certificate-key 3ae9868e44b9b2cebcf36a22d7e29b897e6c22bdfe381e8caf9ee2d565575ab1

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:
#此命令是用于在其它 node 节点上执行加入群集
kubeadm join 192.168.80.100:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:b7c84262895d9926c2011d02d234bfb882f97e4774431a0fa20cfc7fa92cec52

//按照提示进行配置
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

//在其它 master 节点上执行 kubeadm join 命令加入群集，并按照提示进行配置
kubeadm join 192.168.80.100:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:b7c84262895d9926c2011d02d234bfb882f97e4774431a0fa20cfc7fa92cec52 \
--control-plane --certificate-key 3ae9868e44b9b2cebcf36a22d7e29b897e6c22bdfe381e8caf9ee2d565575ab1

mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

//在所有 master 节点上修改 haproxy 配置，开启所有节点，并进行重启
vim /data/lb/etc/haproxy.cfg
......
49 server rancher01 192.168.80.20:6443
50 server rancher02 192.168.80.14:6443
51 server rancher03 192.168.80.15:6443

docker ps -a
4521fa71d127 wise2c/haproxy-k8s "/docker-entrypoint.…" 22 minutes ago Up 22 minutes 0.0.0.0:6444->6444/tcp HAProxy-K8S

docker rm -f HAProxy-K8S && bash /data/lb/start-haproxy.sh

//在 master01 节点上传 kube-flannel.yml 文件到 /opt 目录，并创建 flannel 资源
cd /opt
kubectl apply -f kube-flannel.yml

kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-5c98db65d4-mknsx 1/1 Running 0 30m
coredns-5c98db65d4-w8tsr 1/1 Running 0 30m
etcd-master01 1/1 Running 0 29m
etcd-master02 1/1 Running 0 23m
etcd-master03 1/1 Running 0 18m
kube-apiserver-master01 1/1 Running 0 29m
kube-apiserver-master02 1/1 Running 0 23m
kube-apiserver-master03 1/1 Running 0 18m
kube-controller-manager-master01 1/1 Running 1 29m
kube-controller-manager-master02 1/1 Running 0 23m
kube-controller-manager-master03 1/1 Running 0 18m
kube-flannel-ds-amd64-f74hm 1/1 Running 0 54s
kube-flannel-ds-amd64-krv4x 1/1 Running 0 54s
kube-flannel-ds-amd64-x2x8v 1/1 Running 0 54s
kube-proxy-7qhtm 1/1 Running 0 23m
kube-proxy-d589b 1/1 Running 0 30m
kube-proxy-xdzk5 1/1 Running 0 18m
kube-scheduler-master01 1/1 Running 1 29m
kube-scheduler-master02 1/1 Running 0 23m
kube-scheduler-master03 1/1 Running 0 18m

kubectl get nodes
NAME STATUS ROLES AGE VERSION
master01 Ready master 31m v1.15.1
master02 Ready master 23m v1.15.1
master03 Ready master 18m v1.15.1

//在所有 master 节点上修改 K8S 集群配置文件
vim ~/.kube/config
......
#把连接地址修改成各自的节点IP:6443，以防某个master节点故障后，kubectl命令执行被阻塞
5 server: https://192.168.80.20:6443
......

//etcd 集群状态查看
kubectl -n kube-system exec etcd-master01 -- etcdctl \
--endpoints=https://192.168.80.20:2379 \
--ca-file=/etc/kubernetes/pki/etcd/ca.crt \
--cert-file=/etc/kubernetes/pki/etcd/server.crt \
--key-file=/etc/kubernetes/pki/etcd/server.key cluster-health

member 8d213672f975eae is healthy: got healthy result from https://192.168.80.14:2379
member 72de4f571ed38892 is healthy: got healthy result from https://192.168.80.15:2379
member 78af6498bc1a0b41 is healthy: got healthy result from https://192.168.80.20:2379
cluster is healthy

kubectl get endpoints kube-controller-manager --namespace=kube-system -o yaml
apiVersion: v1
kind: Endpoints
metadata:
annotations:
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master01_465eb264-5617-4599-9d99-4e0ec77c419a","leaseDurationSeconds":15,"acquireTime":"2021-06-09T11:46:06Z","renewTime":"2021-06-09T12:19:22Z","leaderTransitions":1}'
creationTimestamp: "2021-06-09T11:37:46Z"
name: kube-controller-manager
namespace: kube-system
resourceVersion: "4455"
selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager
uid: 28a7a017-3f29-4754-8e7d-c4a73e10c8e4

kubectl get endpoints kube-scheduler --namespace=kube-system -o yaml
apiVersion: v1
kind: Endpoints
metadata:
annotations:
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master01_a947a608-3209-43c1-80f6-bfa28f0ff5d8","leaseDurationSeconds":15,"acquireTime":"2021-06-09T11:46:05Z","renewTime":"2021-06-09T12:19:32Z","leaderTransitions":1}'
creationTimestamp: "2021-06-09T11:37:44Z"
name: kube-scheduler
namespace: kube-system
resourceVersion: "4468"
selfLink: /api/v1/namespaces/kube-system/endpoints/kube-scheduler
uid: 68450192-2999-4a97-ac41-2d9058edc7f9