k8s cephfs(动态pvc)

官方参考文档：
GitHub - ceph/ceph-csi at v3.9.0

测试版本

Ceph Version	Ceph CSI Version	Container Orchestrator Name	Version Tested
v17.2.7	v3.9.0	Kubernetes	v1.25.6

安装Ceph-csi

Step 1 Download GitHub - ceph/ceph-csi at v3.9.0

root@sd-k8s-master-1:~# wget https://github.com/ceph/ceph-csi/archive/refs/tags/v3.9.0.zip
root@sd-k8s-master-1:~# unzip v3.9.0.zip

Step 2 创建csidriver

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csidriver.yaml 
#
# /!\ DO NOT MODIFY THIS FILE
#
# This file has been automatically generated by Ceph-CSI yamlgen.
# The source for the contents can be found in the api/deploy directory, make
# your modifications there.
#
---
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:name: "cephfs.csi.ceph.com"namespace: cephfs
spec:attachRequired: falsepodInfoOnMount: falsefsGroupPolicy: FileseLinuxMount: true
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csidriver.yaml

Step 3 为sidecar容器和节点插件部署RBAC：

root@sd-k8s-master-1:~# cd /root/ceph-csi-3.9.0/deploy/cephfs/kubernetes
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-provisioner-rbac.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:name: cephfs-csi-provisionernamespace: cephfs---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: cephfs-external-provisioner-runner
rules:- apiGroups: [""]resources: ["nodes"]verbs: ["get", "list", "watch"]- apiGroups: [""]resources: ["secrets"]verbs: ["get", "list", "watch"]- apiGroups: [""]resources: ["events"]verbs: ["list", "watch", "create", "update", "patch"]- apiGroups: [""]resources: ["persistentvolumes"]verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]- apiGroups: [""]resources: ["persistentvolumeclaims"]verbs: ["get", "list", "watch", "update"]- apiGroups: [""]resources: ["persistentvolumeclaims/status"]verbs: ["update", "patch"]- apiGroups: ["storage.k8s.io"]resources: ["storageclasses"]verbs: ["get", "list", "watch"]- apiGroups: ["snapshot.storage.k8s.io"]resources: ["volumesnapshots"]verbs: ["get", "list"]- apiGroups: ["snapshot.storage.k8s.io"]resources: ["volumesnapshots/status"]verbs: ["get", "list", "patch"]- apiGroups: ["snapshot.storage.k8s.io"]resources: ["volumesnapshotcontents"]verbs: ["get", "list", "watch", "update", "patch"]- apiGroups: ["snapshot.storage.k8s.io"]resources: ["volumesnapshotclasses"]verbs: ["get", "list", "watch"]- apiGroups: ["storage.k8s.io"]resources: ["csinodes"]verbs: ["get", "list", "watch"]- apiGroups: ["snapshot.storage.k8s.io"]resources: ["volumesnapshotcontents/status"]verbs: ["update", "patch"]- apiGroups: [""]resources: ["configmaps"]verbs: ["get"]- apiGroups: [""]resources: ["serviceaccounts"]verbs: ["get"]- apiGroups: [""]resources: ["serviceaccounts/token"]verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: cephfs-csi-provisioner-role
subjects:- kind: ServiceAccountname: cephfs-csi-provisionernamespace: cephfs
roleRef:kind: ClusterRolename: cephfs-external-provisioner-runnerapiGroup: rbac.authorization.k8s.io---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:# replace with non-cephfs namespace namenamespace: cephfsname: cephfs-external-provisioner-cfg
rules:- apiGroups: [""]resources: ["configmaps"]verbs: ["get", "list", "watch"]- apiGroups: ["coordination.k8s.io"]resources: ["leases"]verbs: ["get", "watch", "list", "delete", "update", "create"]---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: cephfs-csi-provisioner-role-cfg# replace with non-cephfs namespace namenamespace: cephfs
subjects:- kind: ServiceAccountname: cephfs-csi-provisioner# replace with non-cephfs namespace namenamespace: cephfs
roleRef:kind: Rolename: cephfs-external-provisioner-cfgapiGroup: rbac.authorization.k8s.io
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-nodeplugin-rbac.yaml 
---
apiVersion: v1
kind: ServiceAccount
metadata:name: cephfs-csi-nodepluginnamespace: cephfs
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: cephfs-csi-nodeplugin
rules:- apiGroups: [""]resources: ["secrets"]verbs: ["get"]- apiGroups: [""]resources: ["configmaps"]verbs: ["get"]- apiGroups: [""]resources: ["serviceaccounts"]verbs: ["get"]- apiGroups: [""]resources: ["serviceaccounts/token"]verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: cephfs-csi-nodeplugin
subjects:- kind: ServiceAccountname: cephfs-csi-nodeplugin# replace with non-cephfs namespace namenamespace: cephfs
roleRef:kind: ClusterRolename: cephfs-csi-nodepluginapiGroup: rbac.authorization.k8s.io
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-provisioner-rbac.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-nodeplugin-rbac.yaml

Step 4 编辑configmap文件

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-config-map.yaml
apiVersion: v1
kind: ConfigMap
data:config.json: |-[{"clusterID": "92ab6c78-7edc-11ee-aec4-5e807f521aec","monitors": ["10.220.9.13:6789","10.220.9.14:6789","10.220.9.15:6789"]}]
metadata:name: ceph-csi-confignamespace: cephfs
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat ceph-config.yaml 
apiVersion: v1
kind: ConfigMap
data:ceph.conf: |[global]auth_cluster_required = cephxauth_service_required = cephxauth_client_required = cephx# Workaround for http://tracker.ceph.com/issues/23446fuse_set_user_groups = false# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB# adding 'fuse_big_writes = true' option by default to override this limit# see https://github.com/ceph/ceph-csi/issues/1928fuse_big_writes = true# keyring is a required key and its value should be emptykeyring: |
metadata:name: ceph-confignamespace: cephfs
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-csi-config.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-config.yaml

Step 5 部署 CSI Sidecar 容器：
#注释ceph-csi-encryption-kms-config volume

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# sed -i 's/registry.k8s.io/k8s.dockerproxy.com/g' csi-cephfsplugin-provisioner.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# sed -i 's/registry.k8s.io/k8s.dockerproxy.com/g' csi-cephfsplugin.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-cephfsplugin-provisioner.yaml
---
kind: Service
apiVersion: v1
metadata:name: csi-cephfsplugin-provisionernamespace: cephfslabels:app: csi-metrics
spec:selector:app: csi-cephfsplugin-provisionerports:- name: http-metricsport: 8080protocol: TCPtargetPort: 8681---
kind: Deployment
apiVersion: apps/v1
metadata:name: csi-cephfsplugin-provisionernamespace: cephfs
spec:selector:matchLabels:app: csi-cephfsplugin-provisionerreplicas: 3template:metadata:labels:app: csi-cephfsplugin-provisionerspec:#      affinity:#        podAntiAffinity:#          requiredDuringSchedulingIgnoredDuringExecution:#            - labelSelector:#                matchExpressions:#                  - key: app#                    operator: In#                    values:#                      - csi-cephfsplugin-provisioner#              topologyKey: "kubernetes.io/hostname"serviceAccountName: cephfs-csi-provisionerpriorityClassName: system-cluster-criticalcontainers:- name: csi-provisionerimage: k8s.dockerproxy.com/sig-storage/csi-provisioner:v3.5.0args:- "--csi-address=$(ADDRESS)"- "--v=1"- "--timeout=150s"- "--leader-election=true"- "--retry-interval-start=500ms"- "--feature-gates=Topology=false"- "--feature-gates=HonorPVReclaimPolicy=true"- "--prevent-volume-mode-conversion=true"- "--extra-create-metadata=true"env:- name: ADDRESSvalue: unix:///csi/csi-provisioner.sockimagePullPolicy: "IfNotPresent"volumeMounts:- name: socket-dirmountPath: /csi- name: csi-resizerimage: k8s.dockerproxy.com/sig-storage/csi-resizer:v1.8.0args:- "--csi-address=$(ADDRESS)"- "--v=1"- "--timeout=150s"- "--leader-election"- "--retry-interval-start=500ms"- "--handle-volume-inuse-error=false"- "--feature-gates=RecoverVolumeExpansionFailure=true"env:- name: ADDRESSvalue: unix:///csi/csi-provisioner.sockimagePullPolicy: "IfNotPresent"volumeMounts:- name: socket-dirmountPath: /csi- name: csi-snapshotterimage: k8s.dockerproxy.com/sig-storage/csi-snapshotter:v6.2.2args:- "--csi-address=$(ADDRESS)"- "--v=1"- "--timeout=150s"- "--leader-election=true"- "--extra-create-metadata=true"env:- name: ADDRESSvalue: unix:///csi/csi-provisioner.sockimagePullPolicy: "IfNotPresent"volumeMounts:- name: socket-dirmountPath: /csi- name: csi-cephfspluginimage: quay.io/cephcsi/cephcsi:v3.9.0args:- "--nodeid=$(NODE_ID)"- "--type=cephfs"- "--controllerserver=true"- "--endpoint=$(CSI_ENDPOINT)"- "--v=5"- "--drivername=cephfs.csi.ceph.com"- "--pidlimit=-1"- "--enableprofiling=false"- "--setmetadata=true"env:- name: POD_IPvalueFrom:fieldRef:fieldPath: status.podIP- name: NODE_IDvalueFrom:fieldRef:fieldPath: spec.nodeName- name: CSI_ENDPOINTvalue: unix:///csi/csi-provisioner.sock- name: POD_NAMESPACEvalueFrom:fieldRef:fieldPath: metadata.namespace# - name: KMS_CONFIGMAP_NAME#   value: encryptionConfigimagePullPolicy: "IfNotPresent"volumeMounts:- name: socket-dirmountPath: /csi- name: host-sysmountPath: /sys- name: lib-modulesmountPath: /lib/modulesreadOnly: true- name: host-devmountPath: /dev- name: ceph-configmountPath: /etc/ceph/- name: ceph-csi-configmountPath: /etc/ceph-csi-config/- name: keys-tmp-dirmountPath: /tmp/csi/keys# - name: ceph-csi-encryption-kms-config
##                #   mountPath: /etc/ceph-csi-encryption-kms-config/- name: liveness-prometheusimage: quay.io/cephcsi/cephcsi:v3.9.0args:- "--type=liveness"- "--endpoint=$(CSI_ENDPOINT)"- "--metricsport=8681"- "--metricspath=/metrics"- "--polltime=60s"- "--timeout=3s"env:- name: CSI_ENDPOINTvalue: unix:///csi/csi-provisioner.sock- name: POD_IPvalueFrom:fieldRef:fieldPath: status.podIPvolumeMounts:- name: socket-dirmountPath: /csiimagePullPolicy: "IfNotPresent"volumes:- name: socket-diremptyDir: {medium: "Memory"}- name: host-syshostPath:path: /sys- name: lib-moduleshostPath:path: /lib/modules- name: host-devhostPath:path: /dev- name: ceph-configconfigMap:name: ceph-config- name: ceph-csi-configconfigMap:name: ceph-csi-config- name: keys-tmp-diremptyDir: {medium: "Memory"}# - name: ceph-csi-encryption-kms-config# configMap:# name: ceph-csi-encryption-kms-config
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-cephfsplugin.yaml
---
kind: DaemonSet
apiVersion: apps/v1
metadata:name: csi-cephfspluginnamespace: cephfs
spec:selector:matchLabels:app: csi-cephfsplugintemplate:metadata:labels:app: csi-cephfspluginspec:serviceAccountName: cephfs-csi-nodepluginpriorityClassName: system-node-criticalhostNetwork: truehostPID: true# to use e.g. Rook orchestrated cluster, and mons' FQDN is# resolved through k8s service, set dns policy to cluster firstdnsPolicy: ClusterFirstWithHostNetcontainers:- name: driver-registrar# This is necessary only for systems with SELinux, where# non-privileged sidecar containers cannot access unix domain socket# created by privileged CSI driver container.securityContext:privileged: trueallowPrivilegeEscalation: trueimage: k8s.dockerproxy.com/sig-storage/csi-node-driver-registrar:v2.8.0args:- "--v=1"- "--csi-address=/csi/csi.sock"- "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock"env:- name: KUBE_NODE_NAMEvalueFrom:fieldRef:fieldPath: spec.nodeNamevolumeMounts:- name: socket-dirmountPath: /csi- name: registration-dirmountPath: /registration- name: csi-cephfspluginsecurityContext:privileged: truecapabilities:add: ["SYS_ADMIN"]allowPrivilegeEscalation: trueimage: quay.io/cephcsi/cephcsi:v3.9.0args:- "--nodeid=$(NODE_ID)"- "--type=cephfs"- "--nodeserver=true"- "--endpoint=$(CSI_ENDPOINT)"- "--v=5"- "--drivername=cephfs.csi.ceph.com"- "--enableprofiling=false"# If topology based provisioning is desired, configure required# node labels representing the nodes topology domain# and pass the label names below, for CSI to consume and advertise# its equivalent topology domain# - "--domainlabels=failure-domain/region,failure-domain/zone"env:- name: POD_IPvalueFrom:fieldRef:fieldPath: status.podIP- name: NODE_IDvalueFrom:fieldRef:fieldPath: spec.nodeName- name: CSI_ENDPOINTvalue: unix:///csi/csi.sock- name: POD_NAMESPACEvalueFrom:fieldRef:fieldPath: metadata.namespace# - name: KMS_CONFIGMAP_NAME#   value: encryptionConfigimagePullPolicy: "IfNotPresent"volumeMounts:- name: socket-dirmountPath: /csi- name: mountpoint-dirmountPath: /var/lib/kubelet/podsmountPropagation: Bidirectional- name: plugin-dirmountPath: /var/lib/kubelet/pluginsmountPropagation: "Bidirectional"- name: host-sysmountPath: /sys- name: etc-selinuxmountPath: /etc/selinuxreadOnly: true- name: lib-modulesmountPath: /lib/modulesreadOnly: true- name: host-devmountPath: /dev- name: host-mountmountPath: /run/mount- name: ceph-configmountPath: /etc/ceph/- name: ceph-csi-configmountPath: /etc/ceph-csi-config/- name: keys-tmp-dirmountPath: /tmp/csi/keys- name: ceph-csi-mountinfomountPath: /csi/mountinfo# - name: ceph-csi-encryption-kms-config#   mountPath: /etc/ceph-csi-encryption-kms-config/- name: liveness-prometheussecurityContext:privileged: trueallowPrivilegeEscalation: trueimage: quay.io/cephcsi/cephcsi:v3.9.0args:- "--type=liveness"- "--endpoint=$(CSI_ENDPOINT)"- "--metricsport=8681"- "--metricspath=/metrics"- "--polltime=60s"- "--timeout=3s"env:- name: CSI_ENDPOINTvalue: unix:///csi/csi.sock- name: POD_IPvalueFrom:fieldRef:fieldPath: status.podIPvolumeMounts:- name: socket-dirmountPath: /csiimagePullPolicy: "IfNotPresent"volumes:- name: socket-dirhostPath:path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/type: DirectoryOrCreate- name: registration-dirhostPath:path: /var/lib/kubelet/plugins_registry/type: Directory- name: mountpoint-dirhostPath:path: /var/lib/kubelet/podstype: DirectoryOrCreate- name: plugin-dirhostPath:path: /var/lib/kubelet/pluginstype: Directory- name: host-syshostPath:path: /sys- name: etc-selinuxhostPath:path: /etc/selinux- name: lib-moduleshostPath:path: /lib/modules- name: host-devhostPath:path: /dev- name: host-mounthostPath:path: /run/mount- name: ceph-configconfigMap:name: ceph-config- name: ceph-csi-configconfigMap:name: ceph-csi-config- name: keys-tmp-diremptyDir: {medium: "Memory"}- name: ceph-csi-mountinfohostPath:path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/mountinfotype: DirectoryOrCreate#- name: ceph-csi-encryption-kms-config#  configMap:#    name: ceph-csi-encryption-kms-config
---
# This is a service to expose the liveness metrics
apiVersion: v1
kind: Service
metadata:name: csi-metrics-cephfspluginnamespace: cephfslabels:app: csi-metrics
spec:ports:- name: http-metricsport: 8080protocol: TCPtargetPort: 8681selector:app: csi-cephfsplugin
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-cephfsplugin-provisioner.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-cephfsplugin.yaml

Step 6 Ceph创建cephfs

[root@ceph01 ~]# ceph osd pool cephfs
[root@ceph01 ~]# ceph auth get-key client.admin
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

Step 7 创建ceph secret

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat ceph-sc.yaml 
apiVersion: v1
kind: Secret
metadata:name: csi-cephfs-secretnamespace: cephfs
stringData:userID: adminuserKey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxadminID: adminadminKey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-sc.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get secrets -n cephfs
NAME                TYPE     DATA   AGE
csi-cephfs-secret   Opaque   4      49m

Step 8 创建storageclass

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat ceph-storage-class.yaml 
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: csi-cephfs-sc
provisioner: cephfs.csi.ceph.com
parameters:clusterID: 92ab6c78-7edc-11ee-aec4-5e807f521aec fsName: cephfs          #cephfs名称，上面创建pool: cephfs_data           #cephfs pool名称#  mounter: fuse       #挂载方式csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secretcsi.storage.k8s.io/provisioner-secret-namespace: cephfscsi.storage.k8s.io/controller-expand-secret-name: csi-cephfs-secretcsi.storage.k8s.io/controller-expand-secret-namespace: cephfscsi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secretcsi.storage.k8s.io/node-stage-secret-namespace: cephfs
reclaimPolicy: Delete
allowVolumeExpansion: true
#mountOptions:# - discard
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-storage-class.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get storageclass
NAME                   PROVISIONER                                     RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
csi-cephfs-sc          cephfs.csi.ceph.com                             Delete          Immediate           true                   138m

Step 9 创建pvc测试

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat test-pvc.yaml 
apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: cephfs-test-pvcnamespace: cephfs
spec:accessModes:- ReadWriteManyresources:requests:storage: 1GistorageClassName: csi-cephfs-sc
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f test-pvc.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get pvc -n cephfs
NAME              STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS    AGE
cephfs-test-pvc   Bound    pvc-94236c61-a5b7-494d-89ad-e1e18eaad175   1Gi        RWX            csi-cephfs-sc   49m

Step 10 创建pod测试

root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat test-pod.yaml
---
apiVersion: v1
kind: Pod
metadata:name: test-pd
spec:terminationGracePeriodSeconds: 0containers:- image: harbor.zetyun.cn/gcp/nginx:1.25.3name: test-containervolumeMounts:- mountPath: /cachename: cache-volumevolumes:- name: cache-volumepersistentVolumeClaim:claimName: cephfs-test-pvc1  #填写pvc名称
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f test-pod.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get po test-pd 
NAME      READY   STATUS    RESTARTS   AGE
test-pd   1/1     Running   0          6m49s