OpenStack Cinder 对接 NFS 后端存储

准备好磁盘

[root@cloud ~]# fdisk /dev/nvme0n3Welcome to fdisk (util-linux 2.32.1).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.Device does not contain a recognized partition table.
Created a new DOS disklabel with disk identifier 0x1d9ab064.Command (m for help): n
Partition typep   primary (0 primary, 0 extended, 4 free)e   extended (container for logical partitions)
Select (default p):Using default response p.
Partition number (1-4, default 1):
First sector (2048-41943039, default 2048):
Last sector, +sectors or +size{K,M,G,T,P} (2048-41943039, default 41943039):Created a new partition 1 of type 'Linux' and of size 20 GiB.Command (m for help): w
The partition table has been altered.
Calling ioctl() to re-read partition table.
Syncing disks.[root@cloud ~]# fdisk -l
Disk /dev/nvme0n1: 40 GiB, 42949672960 bytes, 83886080 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x9243f6ebDevice         Boot    Start      End  Sectors  Size Id Type
/dev/nvme0n1p1 *        2048  1026047  1024000  500M 83 Linux
/dev/nvme0n1p2       1026048 17803263 16777216    8G 82 Linux swap / Solaris
/dev/nvme0n1p3      17803264 83886079 66082816 31.5G 83 LinuxDisk /dev/nvme0n2: 20 GiB, 21474836480 bytes, 41943040 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x87cd3c01Device         Boot Start      End  Sectors Size Id Type
/dev/nvme0n2p1       2048 41943039 41940992  20G 83 LinuxDisk /dev/nvme0n3: 20 GiB, 21474836480 bytes, 41943040 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x1d9ab064Device         Boot Start      End  Sectors Size Id Type
/dev/nvme0n3p1       2048 41943039 41940992  20G 83 Linux[root@cloud ~]# mkfs.xfs /dev/nvme0n3p1
meta-data=/dev/nvme0n3p1         isize=512    agcount=4, agsize=1310656 blks=                       sectsz=512   attr=2, projid32bit=1=                       crc=1        finobt=1, sparse=1, rmapbt=0=                       reflink=1
data     =                       bsize=4096   blocks=5242624, imaxpct=25=                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0, ftype=1
log      =internal log           bsize=4096   blocks=2560, version=2=                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0[root@cloud ~]# df -Th
Filesystem     Type      Size  Used Avail Use% Mounted on
devtmpfs       devtmpfs  1.9G     0  1.9G   0% /dev
tmpfs          tmpfs     2.0G     0  2.0G   0% /dev/shm
tmpfs          tmpfs     2.0G   10M  2.0G   1% /run
tmpfs          tmpfs     2.0G     0  2.0G   0% /sys/fs/cgroup
/dev/nvme0n1p3 xfs        32G   13G   20G  39% /
/dev/nvme0n1p1 ext4      477M  171M  278M  39% /boot
tmpfs          tmpfs     392M  1.2M  391M   1% /run/user/42
tmpfs          tmpfs     392M  4.6M  387M   2% /run/user/0
/dev/sr0       iso9660   7.7G  7.7G     0 100% /run/media/root/CentOS-8-2-2004-x86_64-dvd
/dev/nvme0n3p1 xfs        20G  175M   20G   1% /nfs_share

配置NFS服务

安装NFS

查询是否安装

[root@cloud ~]# rpm -qa |grep nfs
sssd-nfs-idmap-2.2.3-20.el8.x86_64
libnfsidmap-2.3.3-31.el8.x86_64
nfs-utils-2.3.3-31.el8.x86_64

如果没有安装，执行以下安装命令

[root@cloud ~]# yum install -y nfs-utils

启动NFS服务

[root@cloud ~]# systemctl start nfs-server.service
[root@cloud ~]# systemctl enable nfs-server.service
Created symlink /etc/systemd/system/multi-user.target.wants/nfs-server.service → /usr/lib/systemd/system/nfs-server.service.
[root@cloud ~]# systemctl status nfs-server.service
● nfs-server.service - NFS server and servicesLoaded: loaded (/usr/lib/systemd/system/nfs-server.service; enabled; vendor preset: disabled)Active: active (exited) since Thu 2023-08-17 18:56:44 CST; 11s agoMain PID: 2486 (code=exited, status=0/SUCCESS)Tasks: 0 (limit: 24696)Memory: 0BCGroup: /system.slice/nfs-server.serviceAug 17 18:56:44 cloud systemd[1]: Starting NFS server and services...
Aug 17 18:56:44 cloud systemd[1]: Started NFS server and services.

关闭防火墙及SELinux

[root@cloud ~]# setenforce 0
[root@cloud ~]# vim /etc/selinux/config
[root@cloud ~]# cat /etc/selinux/config# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of these three values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected.
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted[root@cloud ~]# systemctl stop firewalld.service
[root@cloud ~]# systemctl disable firewalld.service

创建共享目录

[root@cloud ~]# mkdir /nfs_share
[root@cloud ~]# mount /dev/nvme0n3p1 /nfs_share/
[root@cloud ~]# df -Th
Filesystem     Type      Size  Used Avail Use% Mounted on
devtmpfs       devtmpfs  1.9G     0  1.9G   0% /dev
tmpfs          tmpfs     2.0G     0  2.0G   0% /dev/shm
tmpfs          tmpfs     2.0G   10M  2.0G   1% /run
tmpfs          tmpfs     2.0G     0  2.0G   0% /sys/fs/cgroup
/dev/nvme0n1p3 xfs        32G   13G   20G  39% /
/dev/nvme0n1p1 ext4      477M  171M  278M  39% /boot
tmpfs          tmpfs     392M  1.2M  391M   1% /run/user/42
tmpfs          tmpfs     392M  4.6M  387M   2% /run/user/0
/dev/sr0       iso9660   7.7G  7.7G     0 100% /run/media/root/CentOS-8-2-2004-x86_64-dvd
/dev/nvme0n3p1 xfs        20G  175M   20G   1% /nfs_share
[root@cloud ~]# chmod 777 -R /nfs_share/

修改配置文件

[root@cloud ~]# vim /etc/exports
[root@cloud ~]# cat /etc/exports
/nfs_share 192.168.129.0/24(rw)

重启服务

[root@cloud ~]# systemctl restart nfs-server.service

客户端测试挂载

使用控制节点测试挂载

[root@controller ~]# rpm -qa | grep nfs
libnfsidmap-2.3.3-59.el8.x86_64
sssd-nfs-idmap-2.9.1-1.el8.x86_64
nfs-utils-2.3.3-59.el8.x86_64
[root@controller ~]# systemctl start nfs-utils.service 
[root@controller ~]# mkdir /nfs_test
[root@controller ~]# mount -t nfs 192.168.129.141:/nfs_share /nfs_test
[root@controller ~]# df -Th
Filesystem                 Type      Size  Used Avail Use% Mounted on
devtmpfs                   devtmpfs  1.9G     0  1.9G   0% /dev
tmpfs                      tmpfs     1.9G  4.0K  1.9G   1% /dev/shm
tmpfs                      tmpfs     1.9G   26M  1.9G   2% /run
tmpfs                      tmpfs     1.9G     0  1.9G   0% /sys/fs/cgroup
/dev/nvme0n1p3             xfs        92G  5.8G   86G   7% /
/dev/nvme0n1p1             xfs       794M  193M  602M  25% /boot
/dev/loop0                 ext4      1.9G   28K  1.7G   1% /srv/node/swiftloopback
tmpfs                      tmpfs     390M     0  390M   0% /run/user/0
192.168.129.141:/nfs_share nfs4       20G  175M   20G   1% /nfs_test[root@controller ~]# cd /nfs_test/
[root@controller nfs_test]# touch {a,b}{1,2}.txt
[root@controller nfs_test]# ls
a1.txt  a2.txt  b1.txt  b2.txt
[root@controller nfs_test]# umount /nfs_test 

控制节点配置NFS共享

创建文件并配置

[root@controller ~]# vim /etc/cinder/nfs_share
[root@controller ~]# cat /etc/cinder/nfs_share
192.168.129.141:/nfs_share

修改文件权限

[root@controller ~]# chown root:cinder /etc/cinder/nfs_share 
[root@controller ~]# chmod 0640 /etc/cinder/nfs_share 
[root@controller ~]# ll /etc/cinder/
total 204
-rw-r-----. 1 root   cinder   2391 Apr 27  2022 api-paste.ini
-rw-r-----. 1 root   cinder 189072 Aug  6 15:28 cinder.conf
-rw-r-----  1 root   cinder     27 Aug 17 17:33 nfs_share
-rw-r-----. 1 root   cinder    659 Apr 27  2022 resource_filters.json
-rw-r-----. 1 root   cinder    991 Apr 27  2022 rootwrap.conf
drwxr-xr-x. 2 root   root       30 Aug  6 15:24 rootwrap.d
drwxr-xr-x. 2 cinder root        6 Apr 27  2022 volumes

创建新的后端定义

增加NFS后端定义

修改参数 enabled_backends

## 修改之前建议先备份文件
[root@controller ~]# cp /etc/cinder/cinder.conf  /etc/cinder/cinder.conf.bak
[root@controller ~]# vim /etc/cinder/cinder.conf 436 enabled_backends=lvm,nfs

增加NFS后端参数

增加 nfs 后端参数，其中 lvm 默认存在，nfs 为新增

5261 [lvm]
5262 volume_backend_name=lvm
5263 volume_driver=cinder.volume.drivers.lvm.LVMVolumeDriver
5264 target_ip_address=192.168.129.185
5265 target_helper=lioadm
5266 volume_group=cinder-volumes
5267 volumes_dir=/var/lib/cinder/volumes
5268 
5269 [nfs]
5270 nfs_shares_config = /etc/cinder/nfs_share
5271 volume_driver = cinder.volume.drivers.nfs.NfsDriver
5272 volume_backend_name = nfsbackend

重启卷服务

[root@controller ~]# source keystonerc_admin 
[root@controller ~(keystone_admin)]# systemctl restart openstack-cinder-*

创建云硬盘类型

创建云硬盘类型

[root@controller ~(keystone_admin)]# cinder type-create nfstype
+--------------------------------------+---------+-------------+-----------+
| ID                                   | Name    | Description | Is_Public |
+--------------------------------------+---------+-------------+-----------+
| eb057bfe-8a7a-4a84-a9f7-6da60b44ce00 | nfstype | -           | True      |
+--------------------------------------+---------+-------------+-----------+
[root@controller ~(keystone_admin)]# cinder type-list
+--------------------------------------+-------------+---------------------+-----------+
| ID                                   | Name        | Description         | Is_Public |
+--------------------------------------+-------------+---------------------+-----------+
| 3a808ce2-ba85-4748-aa65-215c44539818 | iscsi       | -                   | True      |
| b54b4912-3d9a-47cc-a434-b760a74f60cc | __DEFAULT__ | Default Volume Type | True      |
| eb057bfe-8a7a-4a84-a9f7-6da60b44ce00 | nfstype     | -                   | True      |
+--------------------------------------+-------------+---------------------+-----------+

设置后端名称

[root@controller ~(keystone_admin)]# cinder type-key nfstype set volume_backend_name=nfsbackend
[root@controller ~(keystone_admin)]# cinder type-show nfstype
+---------------------------------+--------------------------------------+
| Property                        | Value                                |
+---------------------------------+--------------------------------------+
| description                     | None                                 |
| extra_specs                     | volume_backend_name : nfsbackend     |
| id                              | eb057bfe-8a7a-4a84-a9f7-6da60b44ce00 |
| is_public                       | True                                 |
| name                            | nfstype                              |
| os-volume-type-access:is_public | True                                 |
| qos_specs_id                    | None                                 |
+---------------------------------+--------------------------------------+

创建云硬盘

使用 NFS 类型创建云硬盘

[root@controller ~(keystone_admin)]# source keystonerc_cloud01 
[root@controller ~(keystone_cloud01)]# cinder create --display-name evs01 --volume-type nfstype 2
+------------------------------+--------------------------------------+
| Property                     | Value                                |
+------------------------------+--------------------------------------+
| attachments                  | []                                   |
| availability_zone            | nova                                 |
| bootable                     | false                                |
| consistencygroup_id          | None                                 |
| created_at                   | 2023-08-17T10:23:33.000000           |
| description                  | None                                 |
| encrypted                    | False                                |
| group_id                     | None                                 |
| id                           | 360aed1c-76f1-4640-9870-de9a3fbd5647 |
| metadata                     | {}                                   |
| multiattach                  | False                                |
| name                         | evs01                                |
| os-vol-tenant-attr:tenant_id | 1e3e8546921043b19b28a8ded2f22c82     |
| replication_status           | None                                 |
| service_uuid                 | None                                 |
| shared_targets               | True                                 |
| size                         | 2                                    |
| snapshot_id                  | None                                 |
| source_volid                 | None                                 |
| status                       | creating                             |
| updated_at                   | None                                 |
| user_id                      | a662677ef7474ba1b654a3460190f306     |
| volume_type                  | nfstype                              |
+------------------------------+--------------------------------------+

尝试挂载给主机

[root@controller ~(keystone_cloud01)]# cinder list
+--------------------------------------+-----------+-------+------+-------------+----------+-------------+
| ID                                   | Status    | Name  | Size | Volume Type | Bootable | Attached to |
+--------------------------------------+-----------+-------+------+-------------+----------+-------------+
| 360aed1c-76f1-4640-9870-de9a3fbd5647 | available | evs01 | 2    | nfstype     | false    |             |
+--------------------------------------+-----------+-------+------+-------------+----------+-------------+
[root@controller ~(keystone_cloud01)]# nova list
+--------------------------------------+---------+--------+------------+-------------+------------------------+
| ID                                   | Name    | Status | Task State | Power State | Networks               |
+--------------------------------------+---------+--------+------------+-------------+------------------------+
| 5ce35ff8-aa1f-4459-a5a6-50c7c3ea05bc | Linux01 | ACTIVE | -          | Running     | private=192.168.88.146 |
+--------------------------------------+---------+--------+------------+-------------+------------------------+
[root@controller ~(keystone_cloud01)]# nova volume-attach Linux01 360aed1c-76f1-4640-9870-de9a3fbd5647
+-----------------------+--------------------------------------+
| Property              | Value                                |
+-----------------------+--------------------------------------+
| delete_on_termination | False                                |
| device                | /dev/vdb                             |
| id                    | 360aed1c-76f1-4640-9870-de9a3fbd5647 |
| serverId              | 5ce35ff8-aa1f-4459-a5a6-50c7c3ea05bc |
| tag                   | -                                    |
| volumeId              | 360aed1c-76f1-4640-9870-de9a3fbd5647 |
+-----------------------+--------------------------------------+

V版可能存在一些bug，不对接也可以直接创建成功。

• END