Redis分布式高可用集群搭建

2020年发布的Redis6.0是Redis自发布以来最大的升级，提供了多线程支持，ACLs细粒度的权限管理，SSL加密，客户端缓存，集群代理等多项重大更新

一、redis集群有三种方式：

主从复制，哨兵，集群（master-cluster）

1. 主从模式

主从关系，简单说是以主库为准，从库内容异步复制主数据库，从而达成主从内容基本一致的情况。master节点挂了以后，redis就不能对外提供写服务了，此模式不推荐

2. 哨兵

哨兵模式是基于主从模式的，比主从多了自动切换，系统更健壮，可用性更高 ，但是较难支持在线扩容，单机瓶颈尤为突出，此模式不推荐

3. 集群（master-cluster）

解决单机性能瓶颈，支持横向扩展，节点故障转移，真正实习系统高可用，此模式推荐集群（master-cluster）最低配置，两个副本三个分片，那么至少需要6个节点，压缩至两个节点一台机器，那么至少需要3台服务器，低于3台无法实现高可用集群，建议6台redis1	10.1.10.204	主	8001
redis2	10.1.10.205	主	8002
redis3	10.1.10.206	主	8003
redis4	10.1.10.204	备	8004
redis5	10.1.10.205	备	8005
redis6	10.1.10.206	备	8006

二、基于centos7操作系统操做

1. 关闭防火墙，三台机器都执行

systemctl stop firewalld
systemctl disable firewalld

2. hostname修改，三台机器都执行,这一步是为了在内网中三台服务器能相互连通

vi /etc/hosts
10.1.10.204 r1
10.1.10.205 r2
10.1.10.206 r3

3. 免登陆，三台机器都执行

ssh-keygen -t rsa
ssh-copy-id 10.1.10.204
ssh-copy-id 10.1.10.205
ssh-copy-id 10.1.10.206

4. 安装gcc并升级

yum install readline-devel pcre-devel openssl-devel gcc
yum -y install centos-release-scl
yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-binutils
scl enable devtoolset-9 bash
echo "source /opt/rh/devtoolset-9/enable" >>/etc/profile

5. 下载redis，在三台机器上都执行

mkdir -vp /u01/soft
cd /u01/soft/
wget https://download.redis.io/releases/redis-6.0.9.tar.gz 
tar zxf redis-6.0.9.tar.gz
cd redis-6.0.9
make MALLOC=libc
make PREFIX=/usr/local/redis install

6.创建目录

在204服务器执行:
mkdir /u01/redis-cluster
cd /u01/redis-cluster/
mkdir redis01
mkdir redis04
mkdir /usr/local/redis-cluster
cd /usr/local/redis-cluster
mkdir redis01
mkdir redis04
cp /u01/soft/redis-6.0.9/redis.conf /usr/local/redis-cluster/redis01
cp /u01/soft/redis-6.0.9/redis.conf /usr/local/redis-cluster/redis04
cp /u01/soft/redis-6.0.9/src/redis-server /usr/local/redis-cluster/redis01
cp /u01/soft/redis-6.0.9/src/redis-server /usr/local/redis-cluster/redis04在205服务器执行:
mkdir /u01/redis-cluster
cd /u01/redis-cluster/
mkdir redis02
mkdir redis05
mkdir /usr/local/redis-cluster
cd /usr/local/redis-cluster
mkdir redis02
mkdir redis05
cp /u01/soft/redis-6.0.9/redis.conf /usr/local/redis-cluster/redis02
cp /u01/soft/redis-6.0.9/redis.conf /usr/local/redis-cluster/redis05
cp /u01/soft/redis-6.0.9/src/redis-server /usr/local/redis-cluster/redis02
cp /u01/soft/redis-6.0.9/src/redis-server /usr/local/redis-cluster/redis05在206服务器执行:
mkdir /u01/redis-cluster
cd /u01/redis-cluster/
mkdir redis03
mkdir redis06
mkdir /usr/local/redis-cluster
cd /usr/local/redis-cluster
mkdir redis03
mkdir redis06
cp /u01/soft/redis-6.0.9/redis.conf /usr/local/redis-cluster/redis03
cp /u01/soft/redis-6.0.9/redis.conf /usr/local/redis-cluster/redis06
cp /u01/soft/redis-6.0.9/src/redis-server /usr/local/redis-cluster/redis03
cp /u01/soft/redis-6.0.9/src/redis-server /usr/local/redis-cluster/redis06

7. 修改配置，三台机器都要对应修改，对应开头的表格

$ vim redis01/redis.conf#是否开启保护模式，默认开启。要是配置里没有指定bind和密码。开启该参数后，redis只会本地进行访问，拒绝外部访问。
protected-mode no
#redis监听的端口号
port 8001
#是否在后台执行，yes：后台运行；no：不是后台运行
daemonize yes
#redis的进程文件
pidfile /var/run/redis_8001.pid
# 指定了记录日志的文件。空字符串的话，日志会打印到标准输出设备。后台运行的redis标准输出是/dev/null
logfile /u01/redis-cluster/redis01/redis_8001.log
# 指定本地数据库文件名，一般采用默认的 dump.rdb
# dbfilename /u01/redis-cluster/redis01/dump-8001.rdb
# 数据目录，数据库的写入会在这个目录。rdb、aof文件也会写在这个目录
dir /u01/redis-cluster/redis04/
# 集群
cluster-enabled yes
# 生成的node文件，不过不指定名称则直接为nodes.conf。在同一台服务器上启动第二个实列的时候就会报错。因为nodes.conf文件发生冲突
cluster-config-file nodes-8001.conf
appendonly yes
appendfilename "appendonly-8001.aof"

8. 启动并且创建集群

分别启动6个节点 
./redis-server redis.conf
创建集群 ./redis-cli --cluster create 10.1.10.204:8001 10.1.10.205:8002 10.1.10.206:8003 10.1.10.205:8005 10.1.10.206:8006 10.1.10.204:8004 --cluster-replicas 1

9. 集群节点信息

./redis-cli -h 10.1.10.204 -p 8004 cluster nodes                             
d148088d84486a5cc98d34df2003e80eac1c2c97 10.1.10.205:8005@18005 slave ed308cac5868f5284340fa96320453748d4c11c2 0 1605686204000 1 connected
70d624e48b69c0ab1c3f041bde83cca4ad3c2865 10.1.10.204:8004@18004 myself,slave 1a8e7da0488175a103c6b714e8e87410c326b868 0 1605686204000 3 connected
874a8cd728fc4134271863b6f71365d61da00659 10.1.10.206:8006@18006 slave 4bd9f8bac50539b6435c72ebc6b5a448ebbedcef 0 1605686204645 2 connected
ed308cac5868f5284340fa96320453748d4c11c2 10.1.10.204:8001@18001 master - 0 1605686205647 1 connected 0-5460
4bd9f8bac50539b6435c72ebc6b5a448ebbedcef 10.1.10.205:8002@18002 master - 0 1605686203000 2 connected 5461-10922
1a8e7da0488175a103c6b714e8e87410c326b868 10.1.10.206:8003@18003 master - 0 1605686204000 3 connected 10923-16383

10. 查看集群状态

./redis-cli -h 10.1.10.204 -p 8001 cluster info                              
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:1861
cluster_stats_messages_pong_sent:1866
cluster_stats_messages_sent:3727
cluster_stats_messages_ping_received:1861
cluster_stats_messages_pong_received:1861
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:3727

11. 查看节点内存、cpu、key数量等信息

./redis-cli -h 10.1.10.204 -p 8004 info                                      
# Server
redis_version:6.0.9
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:e0eeddaa7861f7a5
redis_mode:cluster
os:Linux 3.10.0-1127.el7.x86_64 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:9.3.1
process_id:8495
run_id:afe2d5b775ad69688a70dfbb9406b6549cd2bf7c
tcp_port:8004
uptime_in_seconds:2212
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:11850857
executable:/usr/local/redis-cluster/redis04/./redis-server
config_file:/usr/local/redis-cluster/redis04/redis.conf
io_threads_active:0# Clients
connected_clients:4
client_recent_max_input_buffer:8
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0# Memory
used_memory:2703376
used_memory_human:2.58M
used_memory_rss:18468864
used_memory_rss_human:17.61M
used_memory_peak:2703376
used_memory_peak_human:2.58M
used_memory_peak_perc:100.05%
used_memory_overhead:2570664
used_memory_startup:1460576
used_memory_dataset:132712
used_memory_dataset_perc:10.68%
allocator_allocated:2799904
allocator_active:3207168
allocator_resident:6090752
total_system_memory:16656355328
total_system_memory_human:15.51G
used_memory_lua:37888
used_memory_lua_human:37.00K
used_memory_scripts:0
used_memory_scripts_human:0B
number_of_cached_scripts:0
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.15
allocator_frag_bytes:407264
allocator_rss_ratio:1.90
allocator_rss_bytes:2883584
rss_overhead_ratio:3.03
rss_overhead_bytes:12378112
mem_fragmentation_ratio:7.00
mem_fragmentation_bytes:15828952
mem_not_counted_for_evict:0
mem_replication_backlog:1048576
mem_clients_slaves:0
mem_clients_normal:61504
mem_aof_buffer:8
mem_allocator:jemalloc-5.1.0
active_defrag_running:0
lazyfree_pending_objects:0# Persistence
loading:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1605684165
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:-1
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:0
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:0
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:2252800
module_fork_in_progress:0
module_fork_last_cow_size:0
aof_current_size:92
aof_base_size:92
aof_pending_rewrite:0
aof_buffer_length:0
aof_rewrite_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:0# Stats
total_connections_received:9
total_commands_processed:319
instantaneous_ops_per_sec:0
total_net_input_bytes:5222
total_net_output_bytes:433184
instantaneous_input_kbps:0.00
instantaneous_output_kbps:0.02
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:0
evicted_keys:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:705
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_reads_processed:327
total_writes_processed:2236
io_threaded_reads_processed:0
io_threaded_writes_processed:0# Replication
role:slave
master_host:10.1.10.206
master_port:8003
master_link_status:up
master_last_io_seconds_ago:4
master_sync_in_progress:0
slave_repl_offset:2982
slave_priority:100
slave_read_only:1
connected_slaves:0
master_replid:2f4df65f8ad840253afb0fb57cf51a4fdbc9eed3
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:2982
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:2982# CPU
used_cpu_sys:0.928872
used_cpu_user:1.269668
used_cpu_sys_children:0.001825
used_cpu_user_children:0.000000# Modules# Cluster
cluster_enabled:1# Keyspace

12. 集群代理

git clone https://github.com/artix75/redis-cluster-proxy
cd redis-cluster-proxy
make PREFIX=/usr/local/redis_cluster_proxy install 
./bin/redis-cluster-proxy -c ./proxy.conf