阿里云ACK的etcd证书过期手工升级操作

一.问题现象

阿里云ACK的etcd证书过期，通过图形化界面升级提示升级失败，考虑通过脚本的方式升级ETCD相关的证书。由于在前期做类似的升级ETCD证书失败导致整个集群业务出现访问异常，所有在升级之前做好对应的备份操作是很有必要的

二.前期准备

1.由于升级操作是在master节点操作，涉及的证书文件变更都在master节点，所以针对阿里云ACK的3台master节点做快照。

2.针对阿里云ACK的3台master节点涉及的证书目录做备份，必要情况下备份到其他服务器上。

3.针对阿里云ACK的etcd数据库做备份操作

date;
CACERT="/var/lib/etcd/cert/ca.pem"
CERT="/var/lib/etcd/cert/etcd-server.pem"
EKY="/var/lib/etcd/cert/etcd-server-key.pem"
ENDPOINTS="172.16.0.87:2379"#etcdctl snapshot save /data/etcd_backup_dir/etcd-snapshot-`date +%Y%m%d`.dbETCDCTL_API=3 etcdctl \
--cacert="${CACERT}" --cert="${CERT}" --key="${EKY}" \
--endpoints=${ENDPOINTS} \
snapshot save /data/etcd_backup_dir/etcd-snapshot-`date +%Y%m%d`.db

备份ETCD的数据目录

cp -r /var/lib/etcd /var/lib/etcd-`date +%Y%m%d`.bak

4.针对阿里云ACK的所有部署资源的yaml文件做备份，涉及service deploy ingress configmap secret job cronjob daemonset statefulset pvc等，备份脚本如下：

#!/bin/bash
#define variable
BACKUP_PATH=/data/k8s-backup
BACKUP_PATH_DATA=$BACKUP_PATH/yaml/`date +%Y%m%d%H%M%S`
BACKUP_PATH_LOG=$BACKUP_PATH/log
BACKUP_LOG_FILE=$BACKUP_PATH_LOG/k8s-backup-`date +%Y%m%d%H%M%S`.log
# base function
function printlog(){echo "`date +'%Y-%m-%d %H:%M:%S'` $1"echo "`date +'%Y-%m-%d %H:%M:%S'` $1" >> $BACKUP_LOG_FILE 2>&1 
}
function printlogonly(){echo "`date +'%Y-%m-%d %H:%M:%S'` $1" >> $BACKUP_LOG_FILE 2>&1 
}
# set K8s type（此处可根据集群资源自行修改）
CONFIG_TYPE="service deploy ingress configmap secret job cronjob daemonset statefulset pvc"
# make dir
mkdir -p $BACKUP_PATH_DATA
mkdir -p $BACKUP_PATH_LOG
cd $BACKUP_PATH_DATA
# set namespace list
ns_list=`kubectl get ns | awk '{print $1}' | grep -v NAME`
if [ $# -ge 1 ]; then
ns_list="$@"
fi
# define counters
COUNT_NS=0
COUNT_ITEM_IN_NS=0
COUNT_ITEM_IN_TYPE=0
COUNT_ITEM_ALL=0
# print hint
printlog "Backup kubernetes config in namespaces: ${ns_list}"
printlog "Backup kubernetes config for [type: ${CONFIG_TYPE}]."
printlog "If you want to read the record of backup, please input command ' tail -100f ${BACKUP_LOG_FILE} '"
# ask and answer
message="This will backup resources of kubernetes cluster to yaml files."
printlog ${message}ls# loop for namespaces
for ns in $ns_list;
do
COUNT_NS=`expr $COUNT_NS + 1`
printlog "Backup No.${COUNT_NS} namespace [namespace: ${ns}]."
COUNT_ITEM_IN_NS=0## loop for types
for type in $CONFIG_TYPE; 
do
printlogonly "Backup type [namespace: ${ns}, type: ${type}]."
item_list=`kubectl -n $ns get $type | awk '{print $1}' | grep -v NAME | grep -v "No "`
COUNT_ITEM_IN_TYPE=0## loop for items
for item in $item_list; 
do 
file_name=$BACKUP_PATH_DATA/${ns}_${type}_${item}.yaml
printlogonly "Backup kubernetes config yaml [namespace: ${ns}, type: ${type}, item: ${item}] to file: ${file_name}"
kubectl -n $ns get $type $item -o yaml > $file_name
COUNT_ITEM_IN_NS=`expr $COUNT_ITEM_IN_NS + 1`
COUNT_ITEM_IN_TYPE=`expr $COUNT_ITEM_IN_TYPE + 1`
COUNT_ITEM_ALL=`expr $COUNT_ITEM_ALL + 1`
printlogonly "Backup No.$COUNT_ITEM_ALL file done."
done;done;
printlogonly "Backup $COUNT_ITEM_IN_TYPE files in [namespace: ${ns}, type: ${type}]."printlog "Backup ${COUNT_ITEM_IN_NS} files done in [namespace: ${ns}]."
done;# show stats
printlog "Backup ${COUNT_ITEM_ALL} yaml files in all."
printlog "kubernetes Backup completed, all done."
exit 0

三.升级证书操作步骤

1.确认集群Master节点之间配置了root用户的免密登录。

在Master上通过SSH方式登录其他任意Master节点，如果提示输入密码，请您按如下方式配置Master节点之间的免密登录。

1. ssh-keygen -t rsa  # 生成密钥。
2. ssh-copy-id -i ~/.ssh/id_rsa.pub $(internal-ip)     # 使用ssh-copy-id工具传输公钥到其他所有Master节点，$（internal-ip）为其他Master节点的内网IP。

2.分别复制以下脚本内容，保存并命名为restart-apiserver.sh和rotate-etcd.sh，然后将两者保存到同一个文件夹下

restart-apiserver.sh

 #! /bin/bashfunction restart_apiserver() {apiserverID=$(/usr/bin/docker ps | grep kube-apiserver | grep -v NAME | awk '{print $1}')/usr/bin/docker stop $apiserverIDrm -rf /root/kube-apiserver.yamlmv /etc/kubernetes/manifests/kube-apiserver.yaml /root/kube-apiserver.yamlwhile true; doNUM=$(docker ps | grep kube-apiserver | wc -l)if [[ $NUM == 0 ]]; thenbreakfisleep 1done/usr/bin/docker ps -a | grep kube-apiserver | awk '{print $1}' | xargs docker stop/usr/bin/docker ps -a | grep kube-apiserver | awk '{print $1}' | xargs docker rmmv /root/kube-apiserver.yaml /etc/kubernetes/manifests/kube-apiserver.yamlwhile true; doNUM=$(docker ps | grep kube-apiserver | grep -v pause | wc -l)if [[ $NUM == 1 ]]; thenbreakfisleep 1donek8s::wait_apiserver_ready
}k8s::wait_apiserver_ready() {set -efor i in $(seq 180); doif kubectl get po &>/dev/null; thenreturn 0elseecho "wait apiserver to be ready, retry ${i}th after 1s"sleep 1fidoneecho "failed to wait apiserver to be ready"return 1
}function restart_container() {crictl pods | grep kube-apiserver | awk '{print $1}' | xargs -I '{}' crictl stopp {} || true
}if [[ -f /usr/bin/docker ]]; thenrestart_apiserver
elserestart_containerk8s::wait_apiserver_ready
fi
echo "API Server restarted"

rotate-etcd.sh

#!/bin/bashset -eo pipefaildir=/tmp/etcdcert
KUBE_CERT_PATH=/etc/kubernetes/pki
ETCD_CERT_DIR=/var/lib/etcd/cert
ETCD_HOSTS=""
function get_etcdhosts() {name1=$(ls $ETCD_CERT_DIR | grep name | grep name-1.pem | sed 's/-name-1.pem//g')name2=$(ls $ETCD_CERT_DIR | grep name | grep name-2.pem | sed 's/-name-2.pem//g')name3=$(ls $ETCD_CERT_DIR | grep name | grep name-3.pem | sed 's/-name-3.pem//g')echo "hosts: " $name1 $name2 $name3#  ETCD_HOSTS=($name1 $name2 $name3)ETCD_HOSTS=$name1" "$name2" "$name3
}function gencerts() {echo "generate ssl cert ..."rm -rf $dirmkdir -p "$dir"hosts=$(echo $ETCD_HOSTS | tr -s " " ",")echo "-----generate ca"echo '{"CN":"CA","key":{"algo":"rsa","size":2048}, "ca": {"expiry": "438000h"}}' |cfssl gencert -initca - | cfssljson -bare $dir/ca -echo '{"signing":{"default":{"expiry":"438000h","usages":["signing","key encipherment","server auth","client auth"]}}}' >$dir/ca-config.jsonecho "-----generate etcdserver"export ADDRESS=$hosts,ext1.example.com,coreos1.local,coreos1export NAME=etcd-serverecho '{"CN":"'$NAME'","hosts":[""],"key":{"algo":"rsa","size":2048}}' |cfssl gencert -config=$dir/ca-config.json -ca=$dir/ca.pem -ca-key=$dir/ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $dir/$NAMEexport ADDRESS=export NAME=etcd-clientecho '{"CN":"'$NAME'","hosts":[""],"key":{"algo":"rsa","size":2048}}' |cfssl gencert -config=$dir/ca-config.json -ca=$dir/ca.pem -ca-key=$dir/ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $dir/$NAME# gen peer-caecho "-----generate peer certificates"echo '{"CN":"Peer-CA","key":{"algo":"rsa","size":2048}, "ca": {"expiry": "438000h"}}' | cfssl gencert -initca - | cfssljson -bare $dir/peer-ca -echo '{"signing":{"default":{"expiry":"438000h","usages":["signing","key encipherment","server auth","client auth"]}}}' >$dir/peer-ca-config.jsoni=0for host in $ETCD_HOSTS; do((i = i + 1))export MEMBER=${host}-name-$iecho '{"CN":"'${MEMBER}'","hosts":[""],"key":{"algo":"rsa","size":2048}}' |cfssl gencert -ca=$dir/peer-ca.pem -ca-key=$dir/peer-ca-key.pem -config=$dir/peer-ca-config.json -profile=peer \-hostname="$hosts,${MEMBER}.local,${MEMBER}" - | cfssljson -bare $dir/${MEMBER}#-hostname="$host,${MEMBER}.local,${MEMBER}" - | cfssljson -bare $dir/${MEMBER}done## backupTIMESTAMP=$(date "+%Y%m%d%H%M%S")\cp -r $ETCD_CERT_DIR $ETCD_CERT_DIR_$TIMESTAMP\cp -r $KUBE_CERT_PATH/etcd $KUBE_CERT_PATH/etcd_$TIMESTAMP# 制作bundle cacat $KUBE_CERT_PATH/etcd/ca.pem >>$dir/bundle_ca.pemcat $ETCD_CERT_DIR/ca.pem >>$dir/bundle_ca.pemcat $dir/ca.pem >>$dir/bundle_ca.pem# 制作bundle peer-cacat $ETCD_CERT_DIR/peer-ca.pem >$dir/bundle_peer-ca.pemcat $dir/peer-ca.pem >>$dir/bundle_peer-ca.pem# chownchown -R etcd:etcd $dirchmod 0644 $dir/*
}function rotate_etcd_ca() {# Update certs on etcd nodes.for ADDR in $ETCD_HOSTS; doTIMESTAMP=$(date "+%Y%m%d%H%M%S")ssh -o StrictHostKeyChecking=no root@$ADDR cp -r $ETCD_CERT_DIR $ETCD_CERT_DIR_$TIMESTAMPecho "update etcd CA on node $ADDR"scp -o StrictHostKeyChecking=no $dir/bundle_ca.pem root@$ADDR:$ETCD_CERT_DIR/ca.pemscp -o StrictHostKeyChecking=no $dir/bundle_ca.pem root@$ADDR:$KUBE_CERT_PATH/etcd/ca.pemscp -o StrictHostKeyChecking=no $dir/etcd-client.pem root@$ADDR:$KUBE_CERT_PATH/etcd/etcd-client.pemscp -o StrictHostKeyChecking=no $dir/etcd-client-key.pem root@$ADDR:$KUBE_CERT_PATH/etcd/etcd-client-key.pemscp -o StrictHostKeyChecking=no $dir/bundle_peer-ca.pem root@$ADDR:$ETCD_CERT_DIR/peer-ca.pemssh -o StrictHostKeyChecking=no root@$ADDR chown -R etcd:etcd $ETCD_CERT_DIRssh -o StrictHostKeyChecking=no root@$ADDR chmod 0644 $ETCD_CERT_DIR/*echo "restart etcd on node $ADDR"ssh -o StrictHostKeyChecking=no root@$ADDR systemctl restart etcdecho "etcd on node $ADDR restarted"ssh -o StrictHostKeyChecking=no root@$ADDR /usr/bin/bash /tmp/restart-apiserver.shecho "apiserver on node $ADDR restarted"sleep 10done
}function rotate_etcd_certs() {for ADDR in $ETCD_HOSTS; doecho "update etcd peer certs on node $ADDR"scp -o StrictHostKeyChecking=no \$dir/{peer-ca-key.pem,etcd-server.pem,etcd-server-key.pem,etcd-client.pem,etcd-client-key.pem,ca-key.pem,*-name*.pem} root@$ADDR:$ETCD_CERT_DIR/ssh -o StrictHostKeyChecking=no root@$ADDR chown -R etcd:etcd $ETCD_CERT_DIRssh -o StrictHostKeyChecking=no root@$ADDR \chmod 0400 $ETCD_CERT_DIR/{peer-ca-key.pem,etcd-server.pem,etcd-server-key.pem,etcd-client.pem,etcd-client-key.pem,ca-key.pem,*-name*.pem}echo "restart etcd on node $ADDR"ssh -o StrictHostKeyChecking=no root@$ADDR systemctl restart etcdecho "etcd on node $ADDR restarted"sleep 10done
}function recover_etcd_ca() {# Update certs on etcd nodes.for ADDR in $ETCD_HOSTS; doecho "replace etcd CA on node $ADDR"scp -o StrictHostKeyChecking=no $dir/ca.pem root@$ADDR:$ETCD_CERT_DIR/ca.pemscp -o StrictHostKeyChecking=no $dir/ca.pem root@$ADDR:$KUBE_CERT_PATH/etcd/ca.pemscp -o StrictHostKeyChecking=no $dir/peer-ca.pem root@$ADDR:$ETCD_CERT_DIR/peer-ca.pemscp -o StrictHostKeyChecking=no $dir/ca.pem root@$ADDR:$KUBE_CERT_PATH/etcd/ca.pemssh -o StrictHostKeyChecking=no root@$ADDR chown -R etcd:etcd $ETCD_CERT_DIRecho "restart apiserver on node $ADDR"ssh -o StrictHostKeyChecking=no root@$ADDR bash /tmp/restart-apiserver.shecho "apiserver on node $ADDR restarted"echo "restart etcd on node $ADDR"sleep 5ssh -o StrictHostKeyChecking=no root@$ADDR systemctl restart etcdecho "etcd on node $ADDR restarted"sleep 5done
}function renew_k8s_certs() {# 更新K8s证书，根据集群Region替换下面cn-hangzhou的默认镜像地域。for ADDR in $ETCD_HOSTS; doecho "renew k8s components cert on node $ADDR"#compatible containerdset +essh -o StrictHostKeyChecking=no root@$ADDR docker run --privileged=true  -v /:/alicoud-k8s-host --pid host --net host \registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0 /renew/upgrade-k8s.sh --role masterssh -o StrictHostKeyChecking=no root@$ADDR ctr image pull registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0ssh -o StrictHostKeyChecking=no root@$ADDR ctr run --privileged=true --mount type=bind,src=/,dst=/alicoud-k8s-host,options=rbind:rw \--net-host registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0 cert-rotate /renew/upgrade-k8s.sh --role masterset -eecho "finished renew k8s components cert on $ADDR"sleep 5done
}function generate_cm() {echo "generate status configmap"cat <<-"EOF" > /tmp/ack-rotate-etcd-ca-cm.yaml.tpl
apiVersion: v1
kind: ConfigMap
metadata:name: ack-rotate-etcd-statusnamespace: kube-system
data:status: "success"hosts: "$hosts"
EOFsed -e "s#\$hosts#$ETCD_HOSTS#" /tmp/ack-rotate-etcd-ca-cm.yaml.tpl | kubectl apply -f -
}get_etcdhosts
echo "${ETCD_HOSTS[@]}"echo "---renew k8s components certs---"
renew_k8s_certs
echo "---end to renew k8s components certs---"# Update certs on etcd nodes.
for ADDR in $ETCD_HOSTS; doscp -o StrictHostKeyChecking=no restart-apiserver.sh root@$ADDR:/tmp/restart-apiserver.shssh -o StrictHostKeyChecking=no root@$ADDR chmod +x /tmp/restart-apiserver.sh
donegencerts
echo "---rotate etcd ca and etcd client ca---"
rotate_etcd_ca
echo "---end to rotate etcd ca and etcd client ca---"
echo
echo "---rotate etcd peer and certs---"
rotate_etcd_certs
echo "---end to rotate etcd peer and certs---"echo
echo "---replace etcd ca---"
recover_etcd_ca
echo "---end to replace etcd ca---"generate_cm
echo "etcd CA and certs have succesfully rotated!"

提前下载镜像文件 registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0

镜像里面的/renew/upgrade-k8s.sh

#!/bin/sh
set -xeif [ -d "/alicoud-k8s-host" ]; thenrm -rf /alicoud-k8s-host/usr/local/k8s-upgrademkdir -p /alicoud-k8s-host/usr/local/k8s-upgradecp -r /renew/* /alicoud-k8s-host/usr/local/k8s-upgradels -l /alicoud-k8s-host/usr/local/k8s-upgradechmod  -R +x /alicoud-k8s-host/usr/local/k8s-upgrade/chroot /alicoud-k8s-host /usr/local/k8s-upgrade/rotate.sh "$@"
fi

rotate.sh

#!/bin/sh
set -xeif [ -d "/alicoud-k8s-host" ]; thenrm -rf /alicoud-k8s-host/usr/local/k8s-upgrademkdir -p /alicoud-k8s-host/usr/local/k8s-upgradecp -r /renew/* /alicoud-k8s-host/usr/local/k8s-upgradels -l /alicoud-k8s-host/usr/local/k8s-upgradechmod  -R +x /alicoud-k8s-host/usr/local/k8s-upgrade/chroot /alicoud-k8s-host /usr/local/k8s-upgrade/rotate.sh "$@"
fi[root@master01 ~]# docker run -it registry.cn-hangzhou.aliyuncs.com/acs/etcd-rotate:v2.0.0 cat /renew/rotate.sh
#!/usr/bin/env bashset -e -xpublic::common::log() {echo $(date +"[%Y%m%d %H:%M:%S]: ") $1
}function retry() {local n=0local try=$1local cmd="${@:2}"[[ $# -le 1 ]] && {echo "Usage $0 <retry_number> <Command>"}set +euntil[[ $n -ge $try ]]do$cmd && break || {echo "Command Fail.."((n++))echo "retry $n :: [$cmd]"sleep 2}doneset -e
}public::upgrade::backupmaster() {local backup_dir=/etc/kubeadm/backup-rotate-$(date +%F)if [ ! -f $backup_dir/kubelet.conf ]; thenmkdir -p $backup_dir $backup_dir/kubelet $backup_dir/etcdcp -rf /etc/kubernetes/ $backup_dir/cp /etc/kubeadm/kubeadm.cfg $backup_dir/cp /etc/systemd/system/kubelet.service.d/10-kubeadm.conf $backup_dircp /etc/kubernetes/kubelet.conf $backup_dircp -rf /var/lib/kubelet/pki/* $backup_dir/kubeletcp -rf /var/lib/etcd/cert/* $backup_dir/etcdelsepublic::common::log "master configuration is already backup, skip."fi
}public::upgrade::backupnode() {public::common::log "Begin the node backup working."local backup_dir=/etc/kubeadm/backup-rotate-$(date +%F)if [ ! -f $backup_dir/10-kubeadm.conf ]; thenmkdir -p $backup_dir $backup_dir/kubeletcp -rf /etc/kubernetes/ $backup_dir/cp /etc/kubernetes/kubelet.conf $backup_dircp /etc/systemd/system/kubelet.service.d/10-kubeadm.conf $backup_dircp -rf /var/lib/kubelet/pki/* $backup_dir/kubeletelsepublic::common::log "node configuration is already backup, skip."fi
}public::main::master-rotate() {ls -l /usr/localpwdlocal backup_dir=/etc/kubeadm/backup-rotate-$(date +%F)echo "mode is $MODE"if ! grep "rotate-certificates" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf; then./usr/local/k8s-upgrade/cert-rotate -mode=$MODE -role=master -nodeip=$NODE_IP -auto-rotate=false >$backup_dir/renew.logelse./usr/local/k8s-upgrade/cert-rotate -mode=$MODE -role=master -nodeip=$NODE_IP >$backup_dir/renew.logfiif [[ "$MODE" = "etcd" ]]; thenpublic::common::log "Successful update cert on $(hostname)"exit 0fisleep 1#renew the dashboard certsif [ -d /etc/kubernetes/pki/dashboard ]; thencp -rf /etc/kubernetes/pki/apiserver.crt /etc/kubernetes/pki/dashboard/dashboard.crtcp -rf /etc/kubernetes/pki/apiserver.key /etc/kubernetes/pki/dashboard/dashboard.keycp -rf /etc/kubernetes/pki/ca.crt /etc/kubernetes/pki/dashboard/dashboard-ca.crtcat /etc/kubernetes/pki/client-ca.crt >>/etc/kubernetes/pki/dashboard/dashboard-ca.crtif [ -f /etc/kubernetes/pki/user-ca.crt ]; thencat /etc/kubernetes/pki/user-ca.crt >>/etc/kubernetes/pki/dashboard/dashboard-ca.crtficp -rf /etc/kubernetes/pki/dashboard/dashboard-ca.crt /etc/kubernetes/pki/apiserver-ca.crtelsecp -rf /etc/kubernetes/pki/ca.crt /etc/kubernetes/pki/apiserver-ca.crtcat /etc/kubernetes/pki/client-ca.crt >>/etc/kubernetes/pki/apiserver-ca.crtif [ -f /etc/kubernetes/pki/user-ca.crt ]; thencat /etc/kubernetes/pki/user-ca.crt >>/etc/kubernetes/pki/apiserver-ca.crtfifi# /etc/kubernetes/manifests pod can not be pull up automatically. use this to workaround.set +edocker ps | grep kube-controller-manager | awk '{print $1}' | xargs -I '{}' docker restart {} || truecrictl pods | grep kube-controller-manager | awk '{print $1}' | xargs -I '{}' crictl stopp {} || trueset -esleep 1#restart kubeletservice kubelet restartsleep 1# /etc/kubernetes/manifests pod can not be pull up automatically. use this to workaround.set +edocker ps | grep kube-apiserver | awk '{print $1}' | xargs -I '{}' docker restart {} || truecrictl pods | grep kube-apiserver | awk '{print $1}' | xargs -I '{}' crictl stopp {} || truesleep 1docker ps | grep kube-scheduler | awk '{print $1}' | xargs -I '{}' docker restart {} || truecrictl pods | grep kube-scheduler | awk '{print $1}' | xargs -I '{}' crictl stopp {} || trueset -epublic::common::log "Successful update cert on $(hostname)"
}public::main::node-rotate() {if [ -f /etc/kubernetes/pki/apiserver.crt ]; thenpublic::common::log "Skip node rotate on master node"exit 0filocal backup_dir=/etc/kubeadm/backup-rotate-$(date +%F)if ! grep "rotate-certificates" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf; then./usr/local/k8s-upgrade/cert-rotate -mode=$MODE -role=worker -auto-rotate=false -key=$KEY >$backup_dir/renew.logelse./usr/local/k8s-upgrade/cert-rotate -mode=$MODE -role=worker -key=$KEY >$backup_dir/renew.logfisleep 1#restart kubeletservice kubelet restartsleep 1public::common::log "Successful update cert on $(hostname)"
}public::main::master() {public::upgrade::backupmasterpublic::main::master-rotate
}public::main::node() {public::upgrade::backupnodepublic::main::node-rotate
}main() {#use renew mode in defaultexport MODE=renewwhile[[ $# -gt 0 ]]dokey="$1"case $key in--role)export ROLE=$2shift;;--mode)export MODE=$2shift;;--nodeip)export NODE_IP=$2shift;;--rootkey)export KEY=$2shift;;*)public::common::log "unkonw option [$key]"exit 1;;esacshiftdonemkdir -p /etc/kubeadm/backup-rotate-$(date +%F)#public::upgrade::backup######################################################case $ROLE in"source")public::common::log "source scripts";;"master")public::main::master;;"node")public::main::node;;*)echo "Usage:$0 --role master|node --mode renew|rotate./rotate.sh";;esac
}main "$@"

3.在任意Master节点上运行bash rotate-etcd.sh。

当看到命令行输出etcd CA and certs have succesfully rotated!时，表示所有Master节点上的证书和K8s证书已经轮转完成。

脚本执行过程中，会将etcd相关的服务端和客户端证书备份在如下目录中：