一、前期系统环境准备
准备3台主机：硬盘50G cpu2个 内存2G
1、3台主机同时配置
1）关闭防火墙与selinux、NetworkManager
[root@k8s-master ~]# systemctl stop firewalld[root@k8s-master ~]# systemctl disable firewalldRemoved symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.[root@k8s-master ~]# setenforce 0[root@k8s-master ~]# sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux[root@k8s-master ~]# sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config[root@k8s-master ~]# systemctl disable --now NetworkManager
2）配置yum源
[root@k8s-master yum.repos.d]# lsCentOS-Base.repo epel.repo docker-ce.repo epel-testing.repo kubernetes.repo[root@k8s-master ~]# yum clean all && yum makecache
3）配置主机映射
[root@k8s-master ~]# yum -y install vim[root@k8s-master ~]# vim /etc/hosts10.0.0.66 k8s-master10.0.0.77 k8s-node0110.0.0.88 k8s-node02
4）配置主机间免密登录
[root@k8s-master ~]# ssh-keygen [root@k8s-master ~]# ssh-copy-id 10.0.0.77[root@k8s-master ~]# ssh-copy-id 10.0.0.88
5）安装必备工具
[root@k8s-master ~]# yum install wget jq psmisc net-tools telnet yum-utils device-mapper-persistent-data lvm2 git tree -y
6）关闭swap 分区
[root@k8s-master ~]# swapoff -a && sysctl -w vm.swappiness=0[root@k8s-master ~]# sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
7）同步时间
[root@k8s-master ~]# yum -y install ntpdate[root@k8s-master ~]# ntpdate time2.aliyun.com4 Sep 10:08:59 ntpdate[1897]: adjust time server 203.107.6.88 offset 0.007780 sec[root@k8s-master ~]# which ntpdate/usr/sbin/ntpdate[root@k8s-master ~]# crontab -e* 5 * * * /usr/sbin/ntpdate time2.aliyun.com[root@k8s-master ~]# crontab -l* 5 * * * /usr/sbin/ntpdate time2.aliyun.com
8）配置 limit
[root@k8s-master ~]# ulimit -SHn 65535 [root@k8s-master ~]# vim /etc/security/limits.conf # 末尾添加如下内容* soft nofile 65536* hard nofile 131072* soft nproc 65535* hard nproc 655350* soft memlock unlimited* hard memlock unlimited
2、只有master主机配置
1）安装 k8s ⾼可⽤性 Git 仓库并重启
# 在/root/⽬录下克隆⼀个名为k8s-ha-install.git的 Git仓库[root@k8s-master ~]# cd /root/ ; git clone https://gitee.com/dukuan/k8s-ha-install.git[root@k8s-master ~]# lsanaconda-ks.cfg k8s-ha-install# 后续配置功能性pod的yaml文件[root@k8s-master k8s-ha-install]# tree -L 2.├── calico.yaml├── krm.yaml├── LICENSE├── metrics-server-0.3.7│ └── components.yaml├── metrics-server-3.6.1│ ├── aggregated-metrics-reader.yaml│ ├── auth-delegator.yaml│ ├── auth-reader.yaml│ ├── metrics-apiservice.yaml│ ├── metrics-server-deployment.yaml│ ├── metrics-server-service.yaml│ └── resource-reader.yaml└── README.md2 directories, 12 files
二、配置内核模块
1、3台主机同时配置
使用该工具可以同时操作多个主机
1）配置ipvs模块
[root@k8s-master ~]# yum install ipvsadm ipset sysstat conntrack libseccomp -y[root@k8s-master ~]# modprobe -- ip_vs[root@k8s-master ~]# modprobe -- ip_vs_rr[root@k8s-master ~]# modprobe -- ip_vs_wrr[root@k8s-master ~]# modprobe -- ip_vs_sh[root@k8s-master ~]# modprobe -- nf_conntrack# 在系统启动时加载下列 IPVS 和相关功能所需的模块[root@k8s-master ~]# vim /etc/modules-load.d/ipvs.configip_vsip_vs_lcip_vs_wlcip_vs_rrip_vs_wrrip_vs_lblcip_vs_lblcrip_vs_dhip_vs_ship_vs_foip_vs_nqip_vs_sedip_vs_ftpip_vs_shnf_conntrackip_tablesip_setxt_setipt_setipt_rpfilteript_REJECTipip# 加载系统内核参数并应用它们[root@k8s-master ~]# sysctl --system# 开机⾃启systemd默认提供的⽹络管理服务[root@k8s-master ~]# systemctl enable systemd-modules-load.service[root@k8s-master ~]# systemctl start systemd-modules-load.service# 在已加载的内核模块列表中查找与 ip_vs（IP Virtual Server，IP 虚拟服务器）和 nf_conntrack（Netfilter Connection Tracking，网络过滤器连接跟踪）相关的模块信息[root@k8s-master ~]# lsmod | grep -e ip_vs -e nf_conntrackip_vs_sh 12688 0 ip_vs_wrr 12697 0 ip_vs 141432 4 ip_vs_sh,ip_vs_wrrnf_conntrack 133053 1 ip_vslibcrc32c 12644 3 xfs,ip_vs,nf_conntrack
2）配置k8s内核
[root@k8s-master ~]# vim /etc/sysctl.d/k8s.confnet.bridge.bridge-nf-call-iptables = 1net.bridge.bridge-nf-call-ip6tables = 1fs.may_detach_mounts = 1net.ipv4.conf.all.route_localnet = 1vm.overcommit_memory=1vm.panic_on_oom=0fs.inotify.max_user_watches=89100fs.file-max=52706963fs.nr_open=52706963net.netfilter.nf_conntrack_max=2310720net.ipv4.tcp_keepalive_time = 600net.ipv4.tcp_keepalive_probes = 3net.ipv4.tcp_keepalive_intvl =15net.ipv4.tcp_max_tw_buckets = 36000net.ipv4.tcp_tw_reuse = 1net.ipv4.tcp_max_orphans = 327680net.ipv4.tcp_orphan_retries = 3net.ipv4.tcp_syncookies = 1net.ipv4.tcp_max_syn_backlog = 16384net.ipv4.ip_conntrack_max = 65536net.ipv4.tcp_max_syn_backlog = 16384net.ipv4.tcp_timestamps = 0# 保存后，所有节点重启，保证重启后内核依然加载[root@k8s-master ~]# reboot
三、基本组件安装
1、3台主机同时配置
1）安装 Containerd
# 卸载之前的containerd[root@k8s-master ~]# yum remove -y podman runc containerd# 安装Docker和containerd[root@k8s-master ~]# yum install containerd.io docker-ce dockerce-cli -y[root@k8s-master ~]# yum list installed | grep dockercontainerd.io.x86_64 1.6.33-3.1.el7 @docker-ce-stabledocker-buildx-plugin.x86_64 0.14.1-1.el7 @docker-ce-stabledocker-ce.x86_64 3:26.1.4-1.el7 @docker-ce-stabledocker-ce-cli.x86_64 1:26.1.4-1.el7 @docker-ce-stabledocker-ce-rootless-extras.x86_6426.1.4-1.el7 @docker-ce-stabledocker-compose-plugin.x86_64 2.27.1-1.el7 @docker-ce-stable
2）配置 Containerd 所需模块
[root@k8s-master ~]# cat <<EOF | sudo tee /etc/modules-load.d/containerd.confoverlaybr_netfilterEOF[root@k8s-master ~]# modprobe -- overlay[root@k8s-master ~]# modprobe -- br_netfilter
3）配置 Containerd 所需内核
[root@k8s-master ~]# cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.confnet.bridge.bridge-nf-call-iptables = 1net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-ip6tables = 1 EOF[root@k8s-master ~]# sysctl --system
4）Containerd 配置⽂件
[root@k8s-master ~]# mkdir -p /etc/containerd# 读取containerd的配置并保存到/etc/containerd/config.toml[root@k8s-master ~]# containerd config default | tee /etc/containerd/config.toml[root@k8s-master ~]# vim /etc/containerd/config.toml # 找到第63行修改为sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9"# 找到containerd.runtimes.runc.options模块，添加SystemdCgroup = true，如果已经存在则直接修改（在第127行）# 添加sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9"（第128行）# 加载systemctl控制脚本[root@k8s-master ~]# systemctl daemon-reload# 启动containerd并设置开机启动[root@k8s-master ~]# systemctl enable --now containerdCreated symlink from /etc/systemd/system/multi-user.target.wants/containerd.service to /usr/lib/systemd/system/containerd.service.
5）配置 crictl 客户端连接的运⾏位置
# 配置容器运⾏环境的crictl.yml⽂件[root@k8s-master ~]# cat <<EOF | sudo tee /etc/crictl.yamlruntime-endpoint: unix:///run/containerd/containerd.sockimage-endpoint: unix:///run/containerd/containerd.socktimeout: 10 debug: false EOF
6）安装 Kubernetes 组件
# 安装 Kubeadm、Kubelet 和 Kubectl[root@k8s-master ~]# yum install kubeadm-1.28* kubelet-1.28* kubectl-1.28* -y[root@k8s-master ~]# systemctl daemon-reload[root@k8s-master ~]# systemctl enable --now kubeletCreated symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.[root@k8s-master ~]# yum list installed | grep kubecri-tools.x86_64 1.26.0-0 @kubernetes kubeadm.x86_64 1.28.2-0 @kubernetes kubectl.x86_64 1.28.2-0 @kubernetes kubelet.x86_64 1.28.2-0 @kubernetes kubernetes-cni.x86_64 1.2.0-0 @kubernetes
问题解决：kubelet启动失败
# 查看日志[root@k8s-master ~]# vim /var/log/messages# 配置文件未生成，重新安装kubelet
# 问题解决：[root@k8s-master ~]# yum -y remove kubelet[root@k8s-master ~]# yum -y install kubelet-1.28*[root@k8s-master ~]# systemctl start kubelet[root@k8s-master ~]# systemctl status kubeletActive: active (running) since 三 2024-09-11 14:25:57 CST; 3s ago# 由于kubeadm依赖kubelet所以卸载前者时后者也卸载了，需要重新安装[root@k8s-master ~]# yum -y install kubeadm-1.28*# 查看kubelet端口是否启动[root@k8s-master ~]# netstat -lntup | grep kubetcp 0 0 127.0.0.1:10248 0.0.0.0:* LISTEN 2392/kubelet tcp6 0 0 :::10250 :::* LISTEN 2392/kubelet tcp6 0 0 :::10255 :::* LISTEN 2392/kubelet
2、只有master主机配置（Kubernetes 集群初始化）
1）Kubeadm 配置⽂件
[root@k8s-master ~]# vim kubeadm-config.yaml # 粘贴文件内容并修改文件# 修改第12行、24行、29行的ip地址为自己本机的ip地址piVersion: kubeadm.k8s.io/v1beta3bootstrapTokens:- groups:- system:bootstrappers:kubeadm:default-node-tokentoken: 7t2weq.bjbawausm0jaxuryttl: 24h0m0susages:- signing- authenticationkind: InitConfigurationlocalAPIEndpoint:advertiseAddress: 10.0.0.66bindPort: 6443nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockname: k8s-mastertaints:- effect: NoSchedulekey: node-role.kubernetes.io/control-plane---apiServer:certSANs:- 10.0.0.66timeoutForControlPlane: 4m0sapiVersion: kubeadm.k8s.io/v1beta3certificatesDir: /etc/kubernetes/pkiclusterName: kubernetescontrolPlaneEndpoint: 10.0.0.66:6443controllerManager: {}etcd:local:dataDir: /var/lib/etcdimageRepository: registry.cn-hangzhou.aliyuncs.com/google_containerskind: ClusterConfigurationkubernetesVersion: v1.28.2networking:dnsDomain: cluster.localpodSubnet: 172.16.0.0/16serviceSubnet: 10.96.0.0/16scheduler: {}# 将旧的kubeadm配置⽂件转换为新的格式[root@k8s-master ~]# kubeadm config migrate --old-config kubeadm-config.yaml --new-config new.yaml
2）下载组件镜像
# 通过新的配置⽂件new.yaml从指定的阿⾥云仓库拉取kubernetes组件镜像[root@k8s-master ~]# kubeadm config images pull --config /root/new.yaml
3）集群初始化
[root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs# 根据提示信息完成配置[root@k8s-master ~]# mkdir -p $HOME/.kube[root@k8s-master ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config[root@k8s-master ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config# 将node结点加入集群的信息保存到一个文件中，以便使用[root@k8s-master ~]# vim k8s.txtkubeadm join 10.0.0.66:6443 --token 7t2weq.bjbawausm0jaxury \--discovery-token-ca-cert-hash sha256:f3ac431e03dae7f972728eb71eef1828264d42ec20a163893c812a2a0289cf99
问题解决：集群初始化失败
# 端口18258正被kubelet使用，初始化会自动启动kubelet，所以手动关闭kubelet服务[root@k8s-master ~]# systemctl stop kubelet# 修改ip_forward文件内容[root@k8s-master ~]# echo 1 > /proc/sys/net/ipv4/ip_forward[root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs
# 错误信息显示本机内存不够，cpu数量不够，我们现在将本机内存提到4个G，cpu数量提到4个# 注意要关闭本主机然后进行修改主机配置的操作[root@k8s-master ~]# echo 1 > /proc/sys/net/ipv4/ip_forward[root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs
# 检查kubelet为运行状态[root@master ~]# systemctl status kubeletActive: active (running) since 五 2024-09-06 17:33:30 CST; 5min ago# 可能是配置文件的地址没有改，所以找不到主机，所以超时[root@k8s-master ~]# vim new.yaml# 修改第12行、24行、29行的ip地址为自己本机的ip地址# 初始化重置[root@k8s-master ~]# kubeadm reset -f ; ipvsadm --clear ; rm -rf ~/.kube[root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs
4）加载环境变量
[root@k8s-master ~]# vim /root/.bashrcexport KUBECONFIG=/etc/kubernetes/admin.conf[root@k8s-master ~]# source /root/.bashrc
5）查看组件容器状态
| 状态名称 | 中文 | 说明 |
|---|---|---|
| pending | 挂起 | 当前pod没有工作 |
| running | 运行中 | 当前pod正常工作 |
| containercreating | 正在创建容器 | 正在创建容器 |
[root@k8s-master ~]# kubectl get po -ANAMESPACE NAME READY STATUS RESTARTS AGEkube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 52mkube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 52mkube-system etcd-k8s-master 1/1 Running 0 52mkube-system kube-apiserver-k8s-master 1/1 Running 0 52mkube-system kube-controller-manager-k8s-master 1/1 Running 0 52mkube-system kube-proxy-9r6st 1/1 Running 0 52mkube-system kube-proxy-lx5wz 1/1 Running 0 22mkube-system kube-proxy-xmk6s 1/1 Running 0 25mkube-system kube-scheduler-k8s-master 1/1 Running 0 52m
6）查看集群信息
[root@k8s-master ~]# kubectl get nodeNAME STATUS ROLES AGE VERSIONk8s-master NotReady control-plane 25s v1.28.2
7）Token 过期处理
Token 过期后⽣成新的 token：
kubeadm token create --print-join-command
Master 需要⽣成 --certificate-key：
kubeadm init phase upload-certs --upload-certs
3、node结点执行
1）加入集群
[root@k8s-node01 ~]# kubeadm join 10.0.0.66:6443 --token 7t2weq.bjbawausm0jaxury \> --discovery-token-ca-cert-hash sha256:f3ac431e03dae7f972728eb71eef1828264d42ec20a163893c812a2a0289cf99
问题解决：加入集群失败
# 端口被占用，手动停止kubelet，加入集群的过程中会自动启动[root@k8s-node01 ~]# systemctl stop kubeletWarning: kubelet.service changed on disk. Run 'systemctl daemon-reload' to reload units.# 修改ip_forward文件[root@k8s-node01 ~]# echo 1 > /proc/sys/net/ipv4/ip_forward[root@k8s-node01 ~]# kubeadm join 10.0.0.66:6443 --token 7t2weq.bjbawausm0jaxury --discovery-token-ca-cert-hash sha256:f3ac431e03dae7f972728eb71eef1828264d42ec20a163893c812a2a0289cf99
4、master主机执行（Calico 组件安装）
1）查看集群状态与容器状态
[root@k8s-master ~]# kubectl get nodeNAME STATUS ROLES AGE VERSIONk8s-master NotReady control-plane 31m v1.28.2k8s-node01 NotReady <none> 4m4s v1.28.2k8s-node02 NotReady <none> 57s v1.28.2[root@k8s-master ~]# kubectl get po -ANAMESPACE NAME READY STATUS RESTARTS AGEkube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 52mkube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 52mkube-system etcd-k8s-master 1/1 Running 0 52mkube-system kube-apiserver-k8s-master 1/1 Running 0 52mkube-system kube-controller-manager-k8s-master 1/1 Running 0 52mkube-system kube-proxy-9r6st 1/1 Running 0 52mkube-system kube-proxy-lx5wz 1/1 Running 0 22mkube-system kube-proxy-xmk6s 1/1 Running 0 25mkube-system kube-scheduler-k8s-master 1/1 Running 0 52m[root@k8s-master ~]# kubectl get po -AowideNAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATESkube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 53m <none> <none> <none> <none>kube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 53m <none> <none> <none> <none>kube-system etcd-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none>kube-system kube-apiserver-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none>kube-system kube-controller-manager-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none>kube-system kube-proxy-9r6st 1/1 Running 0 53m 10.0.0.66 k8s-master <none> <none>kube-system kube-proxy-lx5wz 1/1 Running 0 23m 10.0.0.88 k8s-node02 <none> <none>kube-system kube-proxy-xmk6s 1/1 Running 0 26m 10.0.0.77 k8s-node01 <none> <none>kube-system kube-scheduler-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none>
2）部署calico的pod
# 找到配置文件calico[root@k8s-master ~]# cd k8s-ha-install/# 切换 git 分⽀[root@k8s-master k8s-ha-install]# git checkout manual-installation-v1.28.x分支 manual-installation-v1.28.x 设置为跟踪来自 origin 的远程分支 manual-installation-v1.28.x。切换到一个新分支 'manual-installation-v1.28.x'# 修改 Pod ⽹段[root@k8s-master k8s-ha-install]# lsbootstrap CoreDNS dashboard metrics-server README.mdcalico csi-hostpath kubeadm-metrics-server pki snapshotter[root@k8s-master k8s-ha-install]# cd calico/[root@k8s-master calico]# lscalico.yaml[root@k8s-master calico]# vim /etc/kubernetes/manifests/kube-controller-manager.yaml# 获取已定义的Pod⽹段[root@k8s-master calico]# POD_SUBNET=`cat /etc/kubernetes/manifests/kube-controller-manager.yaml | grep cluster-cidr= | awk -F= '{print $NF}'` [root@k8s-master calico]# echo $POD_SUBNET172.16.0.0/16# 修改配置文件，将文件中的POD_CIDR替换成172.16.0.0/16[root@k8s-master calico]# sed -i "s#POD_CIDR#${POD_SUBNET}#g" calico.yaml# 创建pod[root@k8s-master calico]# kubectl apply -f calico.yaml
3）查看容器状态
[root@k8s-master calico]# kubectl get po -ANAMESPACE NAME READY STATUS RESTARTS AGEkube-system calico-kube-controllers-6d48795585-v5d7x 0/1 Pending 0 69skube-system calico-node-747k8 0/1 Init:0/3 0 69skube-system calico-node-7klq9 0/1 Init:0/3 0 69skube-system calico-node-j9b44 0/1 Init:0/3 0 69skube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 104mkube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 104mkube-system etcd-k8s-master 1/1 Running 0 104mkube-system kube-apiserver-k8s-master 1/1 Running 0 104mkube-system kube-controller-manager-k8s-master 1/1 Running 1 (7m42s ago) 7m27skube-system kube-proxy-9r6st 1/1 Running 0 104mkube-system kube-proxy-lx5wz 1/1 Running 0 74mkube-system kube-proxy-xmk6s 1/1 Running 0 77mkube-system kube-scheduler-k8s-master 1/1 Running 0 104m
