bclinux aarch64 ceph 14.2.10 云主机 4节点 fio

ceph -s

由于是基于底层分布式存储的云主机，数据仅供参考

本地云盘性能 direct=1

1M读取 IOPS=134, BW=134MiB/s

[root@ceph-client rbd]# cd /
[root@ceph-client /]# fio -filename=fio.bin -direct=1 -iodepth 128 -thread -rw=read -ioengine=libaio -bs=1M -size=10G -numjobs=32 -runtime=120 -group_reporting -name=4krandwrite
4krandwrite: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
4krandwrite: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=0): [_(19),E(1),_(2),f(1),_(9)][100.0%][r=2785MiB/s,w=0KiB/s][r=2785,w=0 IOPS][eta 00m:00s]
4krandwrite: (groupid=0, jobs=32): err= 0: pid=3806: Wed Nov 22 10:56:01 2023
read: IOPS=134, BW=134MiB/s (141MB/s)(15.9GiB/121267msec)
slat (usec): min=53, max=3819.7k, avg=235588.96, stdev=197674.65
clat (usec): min=12, max=46434k, avg=29196153.18, stdev=11931935.17
lat (msec): min=23, max=46687, avg=29431.79, stdev=11971.50
clat percentiles (msec):
| 1.00th=[ 447], 5.00th=[ 5067], 10.00th=[11745], 20.00th=[17113],
| 30.00th=[17113], 40.00th=[17113], 50.00th=[17113], 60.00th=[17113],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( KiB/s): min= 1638, max=332494, per=3.02%, avg=4145.63, stdev=12434.40, samples=6949
iops : min= 1, max= 324, avg= 3.69, stdev=12.14, samples=6949
lat (usec) : 20=0.02%, 50=0.01%
lat (msec) : 4=0.01%, 10=0.05%, 20=0.04%, 50=0.07%, 100=0.16%
lat (msec) : 250=0.45%, 500=0.25%, 750=0.28%, 1000=0.34%
cpu : usr=0.01%, sys=1.69%, ctx=2133125, majf=2077771, minf=2209505
IO depths : 1=0.2%, 2=0.4%, 4=0.8%, 8=1.6%, 16=3.1%, 32=6.3%, >=64=87.6%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.7%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.3%
issued rwts: total=16256,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
READ: bw=134MiB/s (141MB/s), 134MiB/s-134MiB/s (141MB/s-141MB/s), io=15.9GiB (17.0GB), run=121267-121267msec

Disk stats (read/write):
dm-0: ios=58637/126, merge=0/0, ticks=894204/1724, in_queue=895928, util=64.46%, aggrios=2163930/92585, aggrmerge=1205635/3420710, aggrticks=3381169/744416, aggrin_queue=4097528, aggrutil=99.45%
vda: ios=2163930/92585, merge=1205635/3420710, ticks=3381169/744416, in_queue=4097528, util=99.45%

1M写入 IOPS=178, BW=178MiB/s

[root@ceph-client /]# fio -filename=fio.bin -direct=1 -iodepth 128 -thread -rw=write -ioengine=libaio -bs=1M -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 6 (f=5): [_(2),W(1),_(2),W(1),_(4),f(1),_(3),W(2),_(5),W(1),_(10)][4.5%][r=0KiB/s,w=2001MiB/s][r=0,w=2000 IOPS][eta 26m:00s]
test: (groupid=0, jobs=32): err= 0: pid=3994: Wed Nov 22 11:03:33 2023
write: IOPS=178, BW=178MiB/s (187MB/s)(10.5GiB/60515msec)
slat (usec): min=118, max=2027.3k, avg=177712.40, stdev=134335.60
clat (msec): min=8, max=36787, avg=21469.39, stdev=9112.76
lat (msec): min=99, max=37057, avg=21647.10, stdev=9124.88
clat percentiles (msec):
| 1.00th=[ 869], 5.00th=[ 3742], 10.00th=[ 8288], 20.00th=[13087],
| 30.00th=[16442], 40.00th=[17113], 50.00th=[17113], 60.00th=[17113],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( KiB/s): min= 1464, max=230760, per=2.48%, avg=4521.29, stdev=4896.85, samples=3054
iops : min= 1, max= 225, avg= 4.08, stdev= 4.81, samples=3054
lat (msec) : 10=0.04%, 20=0.23%, 50=0.02%, 100=0.02%, 250=0.06%
lat (msec) : 500=0.24%, 750=0.24%, 1000=0.27%
cpu : usr=0.18%, sys=2.07%, ctx=816156, majf=798957, minf=1476444
IO depths : 1=0.3%, 2=0.6%, 4=1.2%, 8=2.4%, 16=4.7%, 32=9.5%, >=64=81.3%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.5%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.5%
issued rwts: total=0,10795,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
WRITE: bw=178MiB/s (187MB/s), 178MiB/s-178MiB/s (187MB/s-187MB/s), io=10.5GiB (11.3GB), run=60515-60515msec

Disk stats (read/write):
dm-0: ios=6883/18901, merge=0/0, ticks=13208/527204, in_queue=540408, util=74.32%, aggrios=829492/85114, aggrmerge=1779613/2494328, aggrticks=1159221/1214615, aggrin_queue=2343400, aggrutil=97.23%
vda: ios=829492/85114, merge=1779613/2494328, ticks=1159221/1214615, in_queue=2343400, util=97.23%

4k随机写 IOPS=36.9k, BW=144MiB/s

[root@ceph-client /]# fio -filename=fio.bin -direct=1 -iodepth 128 -thread -rw=randwrite -ioengine=libaio -bs=4k -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 32 (f=32): [w(32)][100.0%][r=0KiB/s,w=150MiB/s][r=0,w=38.3k IOPS][eta 00m:00s]
test: (groupid=0, jobs=32): err= 0: pid=4046: Wed Nov 22 11:06:56 2023
write: IOPS=36.9k, BW=144MiB/s (151MB/s)(8664MiB/60046msec)
slat (usec): min=11, max=184132, avg=849.83, stdev=6720.42
clat (usec): min=1714, max=750041, avg=109891.48, stdev=79117.19
lat (usec): min=1752, max=750105, avg=110741.74, stdev=79445.06
clat percentiles (msec):
| 1.00th=[ 4], 5.00th=[ 7], 10.00th=[ 46], 20.00th=[ 55],
| 30.00th=[ 61], 40.00th=[ 67], 50.00th=[ 88], 60.00th=[ 110],
| 70.00th=[ 130], 80.00th=[ 165], 90.00th=[ 218], 95.00th=[ 268],
| 99.00th=[ 376], 99.50th=[ 418], 99.90th=[ 514], 99.95th=[ 558],
| 99.99th=[ 634]
bw ( KiB/s): min= 507, max=11920, per=3.12%, avg=4608.27, stdev=1448.99, samples=3838
iops : min= 126, max= 2980, avg=1151.99, stdev=362.25, samples=3838
lat (msec) : 2=0.01%, 4=2.50%, 10=4.53%, 20=0.48%, 50=6.06%
lat (msec) : 100=40.65%, 250=39.47%, 500=6.18%, 750=0.12%, 1000=0.01%
cpu : usr=0.57%, sys=2.80%, ctx=135347, majf=0, minf=32
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=0,2218102,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
WRITE: bw=144MiB/s (151MB/s), 144MiB/s-144MiB/s (151MB/s-151MB/s), io=8664MiB (9085MB), run=60046-60046msec

Disk stats (read/write):
dm-0: ios=1/2217368, merge=0/0, ticks=8/14325860, in_queue=14363632, util=100.00%, aggrios=5/2217948, aggrmerge=0/214, aggrticks=37/14302265, aggrin_queue=14355764, aggrutil=100.00%
vda: ios=5/2217948, merge=0/214, ticks=37/14302265, in_queue=14355764, util=100.00%

删除fio.bin再来一次，感觉性能有点夸张，居然跟1M顺序写差不多

[root@ceph-client /]# rm fio.bin
rm: remove regular file 'fio.bin'? y
[root@ceph-client /]# fio -filename=fio.bin -direct=1 -iodepth 128 -thread -rw=randwrite -ioengine=libaio -bs=4k -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
test: Laying out IO file (1 file / 10240MiB)
Jobs: 32 (f=32): [w(32)][100.0%][r=0KiB/s,w=108MiB/s][r=0,w=27.7k IOPS][eta 00m:00s]
test: (groupid=0, jobs=32): err= 0: pid=4098: Wed Nov 22 11:08:45 2023
write: IOPS=26.4k, BW=103MiB/s (108MB/s)(6204MiB/60091msec)
slat (usec): min=12, max=548134, avg=1168.42, stdev=15112.06
clat (usec): min=926, max=1422.0k, avg=153294.32, stdev=164294.54
lat (usec): min=1349, max=1423.0k, avg=154463.37, stdev=165043.87
clat percentiles (msec):
| 1.00th=[ 3], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 6],
| 30.00th=[ 11], 40.00th=[ 22], 50.00th=[ 132], 60.00th=[ 180],
| 70.00th=[ 224], 80.00th=[ 271], 90.00th=[ 376], 95.00th=[ 472],
| 99.00th=[ 676], 99.50th=[ 760], 99.90th=[ 961], 99.95th=[ 1045],
| 99.99th=[ 1284]
bw ( KiB/s): min= 8, max=16425, per=3.11%, avg=3287.64, stdev=2006.59, samples=3829
iops : min= 2, max= 4106, avg=821.69, stdev=501.66, samples=3829
lat (usec) : 1000=0.01%
lat (msec) : 2=0.10%, 4=4.33%, 10=24.81%, 20=9.99%, 50=4.43%
lat (msec) : 100=1.64%, 250=31.16%, 500=19.79%, 750=3.19%, 1000=0.48%
cpu : usr=0.45%, sys=3.40%, ctx=55393, majf=0, minf=32
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=0,1588171,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
WRITE: bw=103MiB/s (108MB/s), 103MiB/s-103MiB/s (108MB/s-108MB/s), io=6204MiB (6505MB), run=60091-60091msec

Disk stats (read/write):
dm-0: ios=0/1656046, merge=0/0, ticks=0/9813348, in_queue=9831376, util=100.00%, aggrios=0/1589818, aggrmerge=0/67626, aggrticks=0/7834686, aggrin_queue=7801752, aggrutil=100.00%
vda: ios=0/1589818, merge=0/67626, ticks=0/7834686, in_queue=7801752, util=100.00%

[root@ceph-client /]# fio -filename=fio.bin -direct=0 -iodepth 128 -thread -rw=write -ioengine=libaio -bs=1M -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 20 (f=20): [_(1),W(12),_(2),W(2),_(1),W(3),_(4),W(2),_(3),W(1),_(1)][9.2%][r=0KiB/s,w=794MiB/s][r=0,w=793 IOPS][eta 12m:01s]
test: (groupid=0, jobs=32): err= 0: pid=5578: Wed Nov 22 13:42:40 2023
write: IOPS=226, BW=226MiB/s (237MB/s)(13.5GiB/60973msec)
slat (usec): min=593, max=3411.1k, avg=139377.31, stdev=304670.31
clat (usec): min=14, max=59555k, avg=14735008.40, stdev=19498362.15
lat (usec): min=1410, max=59557k, avg=14874407.26, stdev=19591073.82
clat percentiles (msec):
| 1.00th=[ 414], 5.00th=[ 550], 10.00th=[ 592], 20.00th=[ 743],
| 30.00th=[ 911], 40.00th=[ 1116], 50.00th=[ 1636], 60.00th=[ 5403],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( KiB/s): min= 1368, max=434086, per=7.05%, avg=16317.41, stdev=44752.41, samples=1216
iops : min= 1, max= 423, avg=15.50, stdev=43.73, samples=1216
lat (usec) : 20=0.19%, 50=0.04%
lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.03%, 50=0.09%
lat (msec) : 100=0.14%, 250=0.15%, 500=0.62%, 750=19.06%, 1000=14.61%
cpu : usr=0.12%, sys=1.77%, ctx=302994, majf=274315, minf=832427
IO depths : 1=0.2%, 2=0.5%, 4=0.9%, 8=1.9%, 16=3.7%, 32=7.4%, >=64=85.4%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.7%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.3%
issued rwts: total=0,13782,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
WRITE: bw=226MiB/s (237MB/s), 226MiB/s-226MiB/s (237MB/s-237MB/s), io=13.5GiB (14.5GB), run=60973-60973msec

Disk stats (read/write):
dm-0: ios=10458/467392, merge=0/0, ticks=42160/9404388, in_queue=9449612, util=77.16%, aggrios=290643/68377, aggrmerge=802531/1752909, aggrticks=627265/1069223, aggrin_queue=1689784, aggrutil=96.89%
vda: ios=290643/68377, merge=802531/1752909, ticks=627265/1069223, in_queue=1689784, util=96.89%

本地云盘性能 direct=0

1M读取 IOPS=157, BW=157MiB/s

[root@ceph-client /]# fio -filename=fio.bin -direct=0 -iodepth 128 -thread -rw=read -ioengine=libaio -bs=1M -size=10G -numjobs=32 -runtime=120 -group_reporting -name=test
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 19 (f=17): [_(1),R(1),_(2),f(1),R(1),E(1),R(2),E(1),R(1),_(2),R(3),_(1),R(2),_(1),R(3),_(4),R(3),f(1),R(1)][8.4%][r=1992MiB/s,w=0KiB/s][r=1992,w=0 IOPS][eta 21m:59s]
test: (groupid=0, jobs=32): err= 0: pid=5512: Wed Nov 22 13:40:35 2023
read: IOPS=157, BW=157MiB/s (165MB/s)(18.6GiB/121093msec)
slat (usec): min=400, max=3052.7k, avg=202202.40, stdev=154758.98
clat (usec): min=10, max=52723k, avg=25017459.82, stdev=9134567.71
lat (msec): min=309, max=53077, avg=25219.67, stdev=9159.87
clat percentiles (msec):
| 1.00th=[ 2198], 5.00th=[10134], 10.00th=[13892], 20.00th=[17113],
| 30.00th=[17113], 40.00th=[17113], 50.00th=[17113], 60.00th=[17113],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( KiB/s): min= 1928, max=206801, per=2.74%, avg=4406.05, stdev=3084.99, samples=6968
iops : min= 1, max= 201, avg= 3.79, stdev= 3.04, samples=6968
lat (usec) : 20=0.06%, 50=0.04%, 500=0.01%, 750=0.01%, 1000=0.02%
lat (msec) : 2=0.01%, 4=0.01%, 50=0.02%, 500=0.04%, 750=0.10%
lat (msec) : 1000=0.11%
cpu : usr=0.01%, sys=2.41%, ctx=1661725, majf=1562525, minf=3005949
IO depths : 1=0.2%, 2=0.3%, 4=0.7%, 8=1.3%, 16=2.7%, 32=5.4%, >=64=89.4%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.2%
issued rwts: total=19035,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
READ: bw=157MiB/s (165MB/s), 157MiB/s-157MiB/s (165MB/s-165MB/s), io=18.6GiB (19.0GB), run=121093-121093msec

Disk stats (read/write):
dm-0: ios=423561/74, merge=0/0, ticks=3183952/276, in_queue=3184680, util=37.19%, aggrios=2022207/64773, aggrmerge=3010047/4255656, aggrticks=5337060/687635, aggrin_queue=5950512, aggrutil=98.76%
vda: ios=2022207/64773, merge=3010047/4255656, ticks=5337060/687635, in_queue=5950512, util=98.76%

1M写入 IOPS=226, BW=226MiB/s

[root@ceph-client /]# fio -filename=fio.bin -direct=0 -iodepth 128 -thread -rw=write -ioengine=libaio -bs=1M -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 20 (f=20): [_(1),W(12),_(2),W(2),_(1),W(3),_(4),W(2),_(3),W(1),_(1)][9.2%][r=0KiB/s,w=794MiB/s][r=0,w=793 IOPS][eta 12m:01s]
test: (groupid=0, jobs=32): err= 0: pid=5578: Wed Nov 22 13:42:40 2023
write: IOPS=226, BW=226MiB/s (237MB/s)(13.5GiB/60973msec)
slat (usec): min=593, max=3411.1k, avg=139377.31, stdev=304670.31
clat (usec): min=14, max=59555k, avg=14735008.40, stdev=19498362.15
lat (usec): min=1410, max=59557k, avg=14874407.26, stdev=19591073.82
clat percentiles (msec):
| 1.00th=[ 414], 5.00th=[ 550], 10.00th=[ 592], 20.00th=[ 743],
| 30.00th=[ 911], 40.00th=[ 1116], 50.00th=[ 1636], 60.00th=[ 5403],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( KiB/s): min= 1368, max=434086, per=7.05%, avg=16317.41, stdev=44752.41, samples=1216
iops : min= 1, max= 423, avg=15.50, stdev=43.73, samples=1216
lat (usec) : 20=0.19%, 50=0.04%
lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.03%, 50=0.09%
lat (msec) : 100=0.14%, 250=0.15%, 500=0.62%, 750=19.06%, 1000=14.61%
cpu : usr=0.12%, sys=1.77%, ctx=302994, majf=274315, minf=832427
IO depths : 1=0.2%, 2=0.5%, 4=0.9%, 8=1.9%, 16=3.7%, 32=7.4%, >=64=85.4%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.7%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.3%
issued rwts: total=0,13782,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
WRITE: bw=226MiB/s (237MB/s), 226MiB/s-226MiB/s (237MB/s-237MB/s), io=13.5GiB (14.5GB), run=60973-60973msec

4k随机写 IOPS=3842, BW=15.0MiB/s 比直接写慢！

[root@ceph-client /]# fio -filename=fio.bin -direct=0 -iodepth 128 -thread -rw=randwrite -ioengine=libaio -bs=4k -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 31 (f=31): [w(3),_(1),w(28)][70.1%][r=0KiB/s,w=0KiB/s][r=0,w=0 IOPS][eta 01m:00s]
test: (groupid=0, jobs=32): err= 0: pid=5806: Wed Nov 22 13:49:19 2023
write: IOPS=3842, BW=15.0MiB/s (15.7MB/s)(2119MiB/141199msec)
slat (usec): min=6, max=124444, avg=106.51, stdev=1603.48
clat (usec): min=5, max=156420, avg=14048.48, stdev=19617.06
lat (usec): min=79, max=156432, avg=14155.25, stdev=19727.73
clat percentiles (usec):
| 1.00th=[ 1647], 5.00th=[ 1713], 10.00th=[ 1745], 20.00th=[ 1795],
| 30.00th=[ 1860], 40.00th=[ 1942], 50.00th=[ 2040], 60.00th=[ 2180],
| 70.00th=[ 21103], 80.00th=[ 31851], 90.00th=[ 35914], 95.00th=[ 60031],
| 99.00th=[ 76022], 99.50th=[ 87557], 99.90th=[104334], 99.95th=[110625],
| 99.99th=[156238]
bw ( KiB/s): min= 6560, max=279832, per=100.00%, avg=36142.03, stdev=67910.21, samples=120
iops : min= 1640, max=69958, avg=9035.50, stdev=16977.55, samples=120
lat (usec) : 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01%
lat (msec) : 2=46.48%, 4=19.14%, 10=0.03%, 20=4.09%, 50=24.07%
lat (msec) : 100=6.01%, 250=0.16%
cpu : usr=0.05%, sys=0.43%, ctx=692170, majf=0, minf=33
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=0,542546,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
WRITE: bw=15.0MiB/s (15.7MB/s), 15.0MiB/s-15.0MiB/s (15.7MB/s-15.7MB/s), io=2119MiB (2222MB), run=141199-141199msec

Disk stats (read/write):
dm-0: ios=1/461403, merge=0/0, ticks=0/6020256, in_queue=6021368, util=98.89%, aggrios=7/451967, aggrmerge=0/8902, aggrticks=10/4466381, aggrin_queue=4467824, aggrutil=98.72%
vda: ios=7/451967, merge=0/8902, ticks=10/4466381, in_queue=4467824, util=98.72%

ceph块设备性能 direct=1

[root@ceph-client /]# cd /mnt/rbd/
[root@ceph-client rbd]# rm fio.bin
rm: remove regular file 'fio.bin'? y

1M顺序读 IOPS=221, BW=221MiB/s

[root@ceph-client rbd]# fio -filename=fio.bin -direct=1 -iodepth 128 -thread -rw=read -ioengine=libaio -bs=1M -size=10G -numjobs=32 -runtime=120 -group_reporting -name=test
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
test: Laying out IO file (1 file / 10240MiB)
Jobs: 22 (f=20): [R(1),_(1),R(4),_(1),f(1),R(1),_(1),R(4),_(2),R(7),_(1),R(2),_(3),R(1),f(1),_(1)][9.2%][r=1412MiB/s,w=0KiB/s][r=1412,w=0 IOPS][eta 20m:07s]
test: (groupid=0, jobs=32): err= 0: pid=4430: Wed Nov 22 11:14:25 2023
read: IOPS=221, BW=221MiB/s (232MB/s)(26.4GiB/122130msec)
slat (usec): min=11, max=7481.1k, avg=133901.85, stdev=276461.16
clat (msec): min=7, max=64289, avg=17420.34, stdev=20393.63
lat (msec): min=7, max=64667, avg=17554.33, stdev=20532.25
clat percentiles (msec):
| 1.00th=[ 118], 5.00th=[ 338], 10.00th=[ 575], 20.00th=[ 1167],
| 30.00th=[ 1737], 40.00th=[ 2366], 50.00th=[ 3675], 60.00th=[10671],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( KiB/s): min= 1490, max=216746, per=3.71%, avg=8392.69, stdev=20962.19, samples=5583
iops : min= 1, max= 211, avg= 7.63, stdev=20.54, samples=5583
lat (msec) : 10=0.01%, 20=0.01%, 50=0.14%, 100=0.61%, 250=2.41%
lat (msec) : 500=5.28%, 750=4.56%, 1000=4.59%
cpu : usr=0.01%, sys=1.81%, ctx=897924, majf=803006, minf=2274039
IO depths : 1=0.1%, 2=0.2%, 4=0.5%, 8=0.9%, 16=1.9%, 32=3.8%, >=64=92.5%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=26995,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
READ: bw=221MiB/s (232MB/s), 221MiB/s-221MiB/s (232MB/s-232MB/s), io=26.4GiB (28.3GB), run=122130-122130msec

Disk stats (read/write):
rbd0: ios=10063/6, merge=16376/0, ticks=19766613/10733, in_queue=19801672, util=99.79%

1M 顺序写 IOPS=132, BW=132MiB/s

[root@ceph-client rbd]# fio -filename=fio.bin -direct=1 -iodepth 128 -thread -rw=write -ioengine=libaio -bs=1M -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 20 (f=20): [_(1),W(1),_(1),W(3),_(4),W(1),_(3),W(9),_(3),W(6)][9.1%][r=0KiB/s,w=880MiB/s][r=0,w=879 IOPS][eta 12m:00s]
test: (groupid=0, jobs=32): err= 0: pid=4490: Wed Nov 22 11:18:18 2023
write: IOPS=132, BW=132MiB/s (139MB/s)(8033MiB/60656msec)
slat (usec): min=75, max=4124.5k, avg=238951.60, stdev=268363.98
clat (msec): min=26, max=48430, avg=27190.73, stdev=12415.33
lat (msec): min=158, max=50471, avg=27429.70, stdev=12398.47
clat percentiles (msec):
| 1.00th=[ 617], 5.00th=[ 2635], 10.00th=[ 6074], 20.00th=[16174],
| 30.00th=[17113], 40.00th=[17113], 50.00th=[17113], 60.00th=[17113],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( KiB/s): min= 1307, max=207615, per=2.57%, avg=3484.29, stdev=5079.97, samples=2367
iops : min= 1, max= 202, avg= 2.89, stdev= 4.99, samples=2367
lat (msec) : 50=0.07%, 100=0.10%, 250=0.26%, 500=0.26%, 750=0.61%
lat (msec) : 1000=0.36%
cpu : usr=0.10%, sys=2.46%, ctx=648608, majf=615598, minf=929413
IO depths : 1=0.4%, 2=0.8%, 4=1.6%, 8=3.2%, 16=6.4%, 32=12.7%, >=64=74.9%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.2%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.8%
issued rwts: total=0,8033,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
WRITE: bw=132MiB/s (139MB/s), 132MiB/s-132MiB/s (139MB/s-139MB/s), io=8033MiB (8423MB), run=60656-60656msec

Disk stats (read/write):
rbd0: ios=1/3815, merge=0/3991, ticks=19/9104718, in_queue=9118444, util=89.48%

4k随机写 IOPS=4321, BW=16.9MiB/s

[root@ceph-client rbd]# fio -filename=fio.bin -direct=1 -iodepth 128 -thread -rw=randwrite -ioengine=libaio -bs=4k -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 32 (f=32): [w(32)][100.0%][r=0KiB/s,w=10.3MiB/s][r=0,w=2640 IOPS][eta 00m:00s]
test: (groupid=0, jobs=32): err= 0: pid=5405: Wed Nov 22 13:33:11 2023
write: IOPS=4321, BW=16.9MiB/s (17.7MB/s)(1014MiB/60090msec)
slat (usec): min=6, max=737662, avg=7388.81, stdev=30969.91
clat (msec): min=23, max=2646, avg=929.65, stdev=371.76
lat (msec): min=32, max=2685, avg=937.04, stdev=373.83
clat percentiles (msec):
| 1.00th=[ 230], 5.00th=[ 393], 10.00th=[ 485], 20.00th=[ 609],
| 30.00th=[ 709], 40.00th=[ 802], 50.00th=[ 894], 60.00th=[ 986],
| 70.00th=[ 1099], 80.00th=[ 1234], 90.00th=[ 1435], 95.00th=[ 1603],
| 99.00th=[ 1938], 99.50th=[ 2056], 99.90th=[ 2299], 99.95th=[ 2400],
| 99.99th=[ 2567]
bw ( KiB/s): min= 8, max= 2424, per=3.14%, avg=543.14, stdev=291.75, samples=3763
iops : min= 2, max= 606, avg=135.72, stdev=72.95, samples=3763
lat (msec) : 50=0.01%, 100=0.15%, 250=1.06%, 500=9.97%, 750=23.35%
lat (msec) : 1000=26.69%
cpu : usr=0.06%, sys=0.23%, ctx=196151, majf=0, minf=32
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=0.4%, >=64=99.2%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=0,259671,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
WRITE: bw=16.9MiB/s (17.7MB/s), 16.9MiB/s-16.9MiB/s (17.7MB/s-17.7MB/s), io=1014MiB (1064MB), run=60090-60090msec

Disk stats (read/write):
rbd0: ios=0/258831, merge=0/34, ticks=0/15128214, in_queue=15181812, util=99.95%

此时一台osd负载情况

ceph块设备性能 direct=0

1M顺序读 IOPS=147, BW=148MiB/s

[root@ceph-client rbd]# fio -filename=fio.bin -direct=0 -iodepth 128 -thread -rw=read -ioengine=libaio -bs=1M -size=10G -numjobs=32 -runtime=120 -group_reporting -name=test
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
test: Laying out IO file (1 file / 10240MiB)
Jobs: 4 (f=2): [_(8),R(1),_(1),R(1),_(3),f(1),_(9),f(1),_(7)][4.1%][r=3353MiB/s,w=0KiB/s][r=3352,w=0 IOPS][eta 47m:04s]
test: (groupid=0, jobs=32): err= 0: pid=5977: Wed Nov 22 13:57:23 2023
read: IOPS=147, BW=148MiB/s (155MB/s)(17.5GiB/121234msec)
slat (usec): min=505, max=3964.0k, avg=214524.17, stdev=167542.54
clat (usec): min=13, max=45873k, avg=26424205.63, stdev=8964369.91
lat (msec): min=249, max=46152, avg=26638.73, stdev=8978.76
clat percentiles (msec):
| 1.00th=[ 2366], 5.00th=[10134], 10.00th=[15368], 20.00th=[17113],
| 30.00th=[17113], 40.00th=[17113], 50.00th=[17113], 60.00th=[17113],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( KiB/s): min= 1793, max=73288, per=2.70%, avg=4098.86, stdev=1849.55, samples=6900
iops : min= 1, max= 71, avg= 3.60, stdev= 1.86, samples=6900
lat (usec) : 20=0.06%, 50=0.04%, 100=0.02%, 500=0.01%, 750=0.01%
lat (usec) : 1000=0.02%
lat (msec) : 50=0.02%, 250=0.01%, 500=0.06%, 750=0.06%, 1000=0.12%
cpu : usr=0.01%, sys=2.21%, ctx=1800282, majf=1730680, minf=2859327
IO depths : 1=0.2%, 2=0.4%, 4=0.7%, 8=1.4%, 16=2.9%, 32=5.7%, >=64=88.8%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.2%
issued rwts: total=17942,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
READ: bw=148MiB/s (155MB/s), 148MiB/s-148MiB/s (155MB/s-155MB/s), io=17.5GiB (18.8GB), run=121234-121234msec

Disk stats (read/write):
rbd0: ios=12822/6, merge=2738/0, ticks=242098/93, in_queue=245060, util=45.88%

1M 顺序写 IOPS=105, BW=106MiB/s

[root@ceph-client rbd]# fio -filename=fio.bin -direct=0 -iodepth 128 -thread -rw=write -ioengine=libaio -bs=1M -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 19 (f=12): [_(1),W(4),_(3),f(2),W(1),f(1),W(4),_(2),W(1),f(1),_(3),f(1),_(1),W(2),_(3),f(2)][8.6%][r=0KiB/s,w=2574MiB/s][r=0,w=2573 IOPS][eta 12m:59s]
test: (groupid=0, jobs=32): err= 0: pid=6056: Wed Nov 22 14:01:43 2023
write: IOPS=105, BW=106MiB/s (111MB/s)(6433MiB/60870msec)
slat (usec): min=610, max=4076.8k, avg=298661.65, stdev=459340.49
clat (usec): min=13, max=58150k, avg=27218537.31, stdev=14880664.54
lat (msec): min=58, max=58539, avg=27517.21, stdev=14878.75
clat percentiles (msec):
| 1.00th=[ 384], 5.00th=[ 1368], 10.00th=[ 3239], 20.00th=[11208],
| 30.00th=[17113], 40.00th=[17113], 50.00th=[17113], 60.00th=[17113],
| 70.00th=[17113], 80.00th=[17113], 90.00th=[17113], 95.00th=[17113],
| 99.00th=[17113], 99.50th=[17113], 99.90th=[17113], 99.95th=[17113],
| 99.99th=[17113]
bw ( KiB/s): min= 1607, max=259035, per=4.10%, avg=4439.34, stdev=14032.13, samples=1301
iops : min= 1, max= 252, avg= 4.02, stdev=13.68, samples=1301
lat (usec) : 20=0.40%, 50=0.05%, 100=0.02%
lat (msec) : 2=0.03%, 100=0.09%, 250=0.25%, 500=0.56%, 750=1.12%
lat (msec) : 1000=0.99%
cpu : usr=0.08%, sys=1.20%, ctx=472217, majf=459065, minf=720564
IO depths : 1=0.5%, 2=1.0%, 4=2.0%, 8=4.0%, 16=8.0%, 32=15.9%, >=64=68.7%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=98.7%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=1.3%
issued rwts: total=0,6433,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128

Run status group 0 (all jobs):
WRITE: bw=106MiB/s (111MB/s), 106MiB/s-106MiB/s (111MB/s-111MB/s), io=6433MiB (6745MB), run=60870-60870msec

Disk stats (read/write):
rbd0: ios=1/3315, merge=0/2176, ticks=74/384402, in_queue=384540, util=77.11%

4k随机写 IOPS=3178, BW=12.4MiB/s

[root@ceph-client rbd]# fio -filename=fio.bin -direct=0 -iodepth 128 -thread -rw=randwrite -ioengine=libaio -bs=4k -size=10G -numjobs=32 -runtime=60 -group_reporting -name=test
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=128
...
fio-3.7
Starting 32 threads
Jobs: 30 (f=30): [w(9),_(1),w(17),_(1),w(4)][86.2%][r=0KiB/s,w=0KiB/s][r=0,w=0 IOPS][eta 00m:25s]
test: (groupid=0, jobs=32): err= 0: pid=6105: Wed Nov 22 14:05:06 2023
write: IOPS=3178, BW=12.4MiB/s (13.0MB/s)(1926MiB/155150msec)
slat (usec): min=5, max=211174, avg=239.38, stdev=3209.23
clat (usec): min=4, max=547960, avg=30903.21, stdev=53577.69
lat (usec): min=56, max=547970, avg=31142.84, stdev=53962.94
clat percentiles (usec):
| 1.00th=[ 1565], 5.00th=[ 1598], 10.00th=[ 1631], 20.00th=[ 1663],
| 30.00th=[ 1696], 40.00th=[ 1745], 50.00th=[ 1811], 60.00th=[ 1991],
| 70.00th=[ 12780], 80.00th=[ 68682], 90.00th=[ 95945], 95.00th=[152044],
| 99.00th=[219153], 99.50th=[258999], 99.90th=[320865], 99.95th=[362808],
| 99.99th=[501220]
bw ( KiB/s): min= 1064, max=263037, per=100.00%, avg=16316.94, stdev=49471.80, samples=240
iops : min= 266, max=65759, avg=4079.20, stdev=12367.88, samples=240
lat (usec) : 10=0.01%, 20=0.01%, 100=0.01%, 250=0.01%
lat (msec) : 2=60.21%, 4=7.62%, 10=1.93%, 20=0.91%, 50=2.86%
lat (msec) : 100=17.71%, 250=8.20%, 500=0.55%, 750=0.01%
cpu : usr=0.04%, sys=0.25%, ctx=260864, majf=0, minf=32
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=0,493164,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128