参考:
- https://kubernetes.io/zh-cn/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/
- https://kubernetes.io/zh-cn/docs/tasks/tls/certificate-rotation/
查看证书
查看 kubelet是否支持证书自动轮换,默认轮换的证书位于目录 /var/lib/kubelet/pki
[root@kube-master01 ~]# systemctl status kubelet
● kubelet.service - kubelet: The Kubernetes Node AgentLoaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled)Drop-In: /usr/lib/systemd/system/kubelet.service.d└─10-kubeadm.confActive: active (running) since Tue 2023-07-25 14:57:05 CST; 18min agoDocs: https://kubernetes.io/docs/Main PID: 2983 (kubelet)Tasks: 13 (limit: 49016)Memory: 55.3MCGroup: /system.slice/kubelet.service└─2983 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/[root@kube-master01 ~]# cat /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS[root@kube-master01 ~]# cat /var/lib/kubelet/config.yaml |grep rotate
rotateCertificates: true
查看集群指定证书位置
[root@kube-master01 ~]# kubeadm config print init-defaults
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:- system:bootstrappers:kubeadm:default-node-tokentoken: abcdef.0123456789abcdefttl: 24h0m0susages:- signing- authentication
kind: InitConfiguration
localAPIEndpoint:advertiseAddress: 1.2.3.4bindPort: 6443
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresentname: nodetaints: null
---
apiServer:timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:local:dataDir: /var/lib/etcd
imageRepository: registry.k8s.io
kind: ClusterConfiguration
kubernetesVersion: 1.27.0
networking:dnsDomain: cluster.localserviceSubnet: 10.96.0.0/12
scheduler: {}
[root@kube-master01 ~]# kubeadm config print join-defaults
apiVersion: kubeadm.k8s.io/v1beta3
caCertPath: /etc/kubernetes/pki/ca.crt
discovery:bootstrapToken:apiServerEndpoint: kube-apiserver:6443token: abcdef.0123456789abcdefunsafeSkipCAVerification: truetimeout: 5m0stlsBootstrapToken: abcdef.0123456789abcdef
kind: JoinConfiguration
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresentname: kube-master01taints: null
备份
mkdir certs-old
cp -rp /etc/kubernetes certs-old
查看
[root@kube-master01 ~]# ls -l certs-old/
total 0
drwxr-xr-x 5 root root 136 Jul 20 12:04 kubernetes
[root@kube-master01 ~]# ls -l certs-old/.kube/
total 8
drwxr-x--- 4 root root 35 Jul 18 17:11 cache
-rw------- 1 root root 5641 Jul 18 17:10 config
[root@kube-master01 ~]# ls -l certs-old/kubernetes/
total 32
-rw------- 1 root root 5637 Jul 20 12:05 admin.conf
-rw------- 1 root root 5669 Jul 20 12:05 controller-manager.conf
-rw------- 1 root root 1989 Jul 18 17:09 kubelet.conf
drwxr-xr-x 2 root root 113 Jul 20 02:57 manifests
drwxr-xr-x 3 root root 4096 Jul 18 17:08 pki
-rw------- 1 root root 5621 Jul 20 12:05 scheduler.conf
drwx------ 5 root root 145 Jul 20 12:05 tmp
[root@kube-master01 ~]# ls -l certs-old/kubernetes/pki/
total 56
-rw-r--r-- 1 root root 1294 Jul 20 12:04 apiserver.crt
-rw-r--r-- 1 root root 1155 Jul 20 12:04 apiserver-etcd-client.crt
-rw------- 1 root root 1679 Jul 20 12:04 apiserver-etcd-client.key
-rw------- 1 root root 1679 Jul 20 12:04 apiserver.key
-rw-r--r-- 1 root root 1164 Jul 20 12:04 apiserver-kubelet-client.crt
-rw------- 1 root root 1675 Jul 20 12:04 apiserver-kubelet-client.key
-rw-r--r-- 1 root root 1099 Jul 18 17:08 ca.crt
-rw------- 1 root root 1679 Jul 18 17:08 ca.key
drwxr-xr-x 2 root root 162 Jul 18 17:08 etcd
-rw-r--r-- 1 root root 1115 Jul 18 17:08 front-proxy-ca.crt
-rw------- 1 root root 1679 Jul 18 17:08 front-proxy-ca.key
-rw-r--r-- 1 root root 1119 Jul 20 12:04 front-proxy-client.crt
-rw------- 1 root root 1675 Jul 20 12:04 front-proxy-client.key
-rw------- 1 root root 1679 Jul 18 17:08 sa.key
-rw------- 1 root root 451 Jul 18 17:08 sa.pub$ ls certs-old/kubernetes/pki/
etcd.yaml kube-apiserver.yaml kube-controller-manager.yaml kube-scheduler.yaml[root@kube-master01 ~]# kubectl -n kube-system get cm kubeadm-config -o yaml
apiVersion: v1
data:ClusterConfiguration: |apiServer:extraArgs:authorization-mode: Node,RBACtimeoutForControlPlane: 4m0sapiVersion: kubeadm.k8s.io/v1beta3certificatesDir: /etc/kubernetes/pkiclusterName: kubernetescontrollerManager: {}dns: {}etcd:local:dataDir: /var/lib/etcdimageRepository: registry.k8s.iokind: ClusterConfigurationkubernetesVersion: v1.27.4networking:dnsDomain: cluster.localpodSubnet: 10.96.0.0/12serviceSubnet: 10.96.0.0/12scheduler: {}
kind: ConfigMap
metadata:creationTimestamp: "2023-07-18T09:09:00Z"name: kubeadm-confignamespace: kube-systemresourceVersion: "232092"uid: 5117897d-42fa-4b9d-b89d-f8a5e702e1b9
备份 kubelet
[root@kube-master01 ~]# ls -l /var/lib/kubelet/
config.yaml device-plugins/ memory_manager_state plugins/ pod-resources/
cpu_manager_state kubeadm-flags.env pki/ plugins_registry/ pods/
[root@kube-master01 ~]# cat /var/lib/kubelet/kubeadm-flags.env
KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=registry.k8s.io/pause:3.9"
[root@kube-master01 ~]# ls -l /var/lib/kubelet/pki/
total 12
-rw------- 1 root root 2830 Jul 18 17:08 kubelet-client-2023-07-18-17-08-53.pem
lrwxrwxrwx 1 root root 59 Jul 18 17:08 kubelet-client-current.pem -> /var/lib/kubelet/pki/kubelet-client-2023-07-18-17-08-53.pem
-rw-r--r-- 1 root root 2287 Jul 18 17:08 kubelet.crt
-rw------- 1 root root 1675 Jul 18 17:08 kubelet.key
更新
[root@kube-master01 ~]# kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf Jul 19, 2024 04:05 UTC 359d ca no
apiserver Jul 19, 2024 04:04 UTC 359d ca no
apiserver-etcd-client Jul 19, 2024 04:04 UTC 359d etcd-ca no
apiserver-kubelet-client Jul 19, 2024 04:04 UTC 359d ca no
controller-manager.conf Jul 19, 2024 04:05 UTC 359d ca no
etcd-healthcheck-client Jul 17, 2024 09:08 UTC 358d etcd-ca no
etcd-peer Jul 17, 2024 09:08 UTC 358d etcd-ca no
etcd-server Jul 17, 2024 09:08 UTC 358d etcd-ca no
front-proxy-client Jul 19, 2024 04:04 UTC 359d front-proxy-ca no
scheduler.conf Jul 19, 2024 04:05 UTC 359d ca no CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca Jul 15, 2033 09:08 UTC 9y no
etcd-ca Jul 15, 2033 09:08 UTC 9y no
front-proxy-ca Jul 15, 2033 09:08 UTC 9y no [root@kube-master01 ~]# kubeadm certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewedDone renewing certificates. You must restart the kube-apiserver, kube-controller-manager, kube-scheduler and etcd, so that they can use the new certificates.[root@kube-master01 ~]# kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf Jul 24, 2024 06:52 UTC 364d ca no
apiserver Jul 24, 2024 06:52 UTC 364d ca no
apiserver-etcd-client Jul 24, 2024 06:52 UTC 364d etcd-ca no
apiserver-kubelet-client Jul 24, 2024 06:52 UTC 364d ca no
controller-manager.conf Jul 24, 2024 06:52 UTC 364d ca no
etcd-healthcheck-client Jul 24, 2024 06:52 UTC 364d etcd-ca no
etcd-peer Jul 24, 2024 06:52 UTC 364d etcd-ca no
etcd-server Jul 24, 2024 06:52 UTC 364d etcd-ca no
front-proxy-client Jul 24, 2024 06:52 UTC 364d front-proxy-ca no
scheduler.conf Jul 24, 2024 06:52 UTC 364d ca no CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca Jul 15, 2033 09:08 UTC 9y no
etcd-ca Jul 15, 2033 09:08 UTC 9y no
front-proxy-ca Jul 15, 2033 09:08 UTC 9y no [root@kube-master01 ~]# ls -l /etc/kubernetes/
total 32
-rw------- 1 root root 5641 Jul 25 14:52 admin.conf
-rw------- 1 root root 5673 Jul 25 14:52 controller-manager.conf
-rw------- 1 root root 1989 Jul 18 17:09 kubelet.conf
drwxr-xr-x 2 root root 113 Jul 20 02:57 manifests
drwxr-xr-x 3 root root 4096 Jul 18 17:08 pki
-rw------- 1 root root 5617 Jul 25 14:52 scheduler.conf
drwx------ 5 root root 145 Jul 20 12:05 tmp
[root@kube-master01 ~]# ls -l /etc/kubernetes/pki/
total 56
-rw-r--r-- 1 root root 1294 Jul 25 14:52 apiserver.crt
-rw-r--r-- 1 root root 1155 Jul 25 14:52 apiserver-etcd-client.crt
-rw------- 1 root root 1675 Jul 25 14:52 apiserver-etcd-client.key
-rw------- 1 root root 1679 Jul 25 14:52 apiserver.key
-rw-r--r-- 1 root root 1164 Jul 25 14:52 apiserver-kubelet-client.crt
-rw------- 1 root root 1679 Jul 25 14:52 apiserver-kubelet-client.key
-rw-r--r-- 1 root root 1099 Jul 18 17:08 ca.crt
-rw------- 1 root root 1679 Jul 18 17:08 ca.key
drwxr-xr-x 2 root root 162 Jul 18 17:08 etcd
-rw-r--r-- 1 root root 1115 Jul 18 17:08 front-proxy-ca.crt
-rw------- 1 root root 1679 Jul 18 17:08 front-proxy-ca.key
-rw-r--r-- 1 root root 1119 Jul 25 14:52 front-proxy-client.crt
-rw------- 1 root root 1679 Jul 25 14:52 front-proxy-client.key
-rw------- 1 root root 1679 Jul 18 17:08 sa.key
-rw------- 1 root root 451 Jul 18 17:08 sa.pub
重启 控制平台pod
[root@kube-master01 manifests]# mv /etc/kubernetes/manifests/* /root/certs-old/etc-kubernetes-manifests/
[root@kube-master01 manifests]# crictl ps
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
5ede0cf8bb2d2 ead0a4a53df89 2 hours ago Running coredns 2 f5f9fc235b4f7 coredns-5d78c9869d-l6qsw
64591e9c84957 ead0a4a53df89 2 hours ago Running coredns 2 f7fd104b43a70 coredns-5d78c9869d-d6tjw
ae281e1b09574 6848d7eda0341 2 hours ago Running kube-proxy 2 ee26bf2b947f1 kube-proxy-862db#等待20秒
[root@kube-master01 manifests]# mv /root/certs-old/etc-kubernetes-manifests/* .
[root@kube-master01 manifests]# crictl ps
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
c79467f61b16f e7972205b6614 1 second ago Running kube-apiserver 0 054aa5b7aa334 kube-apiserver-kube-master01
1575c51f03ec5 f466468864b7a 1 second ago Running kube-controller-manager 0 dd7da3afef78a kube-controller-manager-kube-master01
e900e9e36238d 98ef2570f3cde 1 second ago Running kube-scheduler 0 28e22a70ffe19 kube-scheduler-kube-master01
9102a026ffc03 86b6af7dd652c 1 second ago Running etcd 0 cf60901f178f9 etcd-kube-master01
5ede0cf8bb2d2 ead0a4a53df89 2 hours ago Running coredns 2 f5f9fc235b4f7 coredns-5d78c9869d-l6qsw
64591e9c84957 ead0a4a53df89 2 hours ago Running coredns 2 f7fd104b43a70 coredns-5d78c9869d-d6tjw
ae281e1b09574 6848d7eda0341 2 hours ago Running kube-proxy 2 ee26bf2b947f1 kube-proxy-862db
[root@kube-master01 manifests]# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-5d78c9869d-d6tjw 1/1 Running 2 (140m ago) 5d5h
kube-system coredns-5d78c9869d-l6qsw 1/1 Running 2 (140m ago) 5d5h
kube-system etcd-kube-master01 1/1 Running 0 127m
kube-system kube-apiserver-kube-master01 1/1 Running 0 126m
kube-system kube-controller-manager-kube-master01 1/1 Running 0 20m
kube-system kube-proxy-862db 1/1 Running 2 (140m ago) 5d4h
kube-system kube-proxy-gnpjp 1/1 Running 1 (5h5m ago) 5d4h
kube-system kube-proxy-k7vpf 1/1 Running 1 (5h5m ago) 5d4h
kube-system kube-scheduler-kube-master01 1/1 Running 0 5d4h
不知道为啥我设置了kubelet 证书每过半个小时更新一次,kubelet直到第二天才开始自动轮训替换。