单节点的大数据集群运维脚本

vi /opt/bash/bigdata-operate-script.sh

#!/bin/bashsource ~/.bashrc
source /etc/profilehostname=bigdata#程序运行必要组件
important_components=("kafka" "clickhouse-server" "elasticsearch" "kibana")
#非必要组件
monitor_components=("prometheus" "node_exporter" "pushgateway" "kafka_exporter" "grafana")
#monitor脚本启动标志量
m_flag=0
#yarn上任务数
job_num_count=12
# Kafka 集群的地址
KAFKA_BROKER="$hostname:9092"#ES API
ES_API="http://$hostname:9200"
#Kibana API
KIBANA_API="http://$hostname:5601/api/status"
#如果需要，添加基本认证
ES_USER="ES用户名"
ES_PASS="ES密码"function start_component() {if ! systemctl is-active --quiet "$1"; thenecho "start $1"sudo systemctl start "$1"fi
}function check_status() {#检查HDFS状态#echo "========Check HDFS status========"if /usr/java/jdk1.8.0_212/bin/jps | grep -q NameNode && /usr/java/jdk1.8.0_212/bin/jps | grep -q DataNode && hadoop fs -ls / >/dev/null 2>&1; thenif hdfs dfsadmin -safemode get | grep -q ON; thenecho "HDFS in safe mode"elseecho "========HDFS is Running========"((m_flag++))fi       elseecho "--------HDFS is Not Running--------"fi#检查YARN状态#echo "========Check YARN status========"if /usr/java/jdk1.8.0_212/bin/jps | grep -q ResourceManager && /usr/java/jdk1.8.0_212/bin/jps | grep -q NodeManager && yarn node -list >/dev/null 2>&1; thenecho "========YARN is Running========"((m_flag++))elseecho "--------YARN is Not Running--------"fi#检查kafka状态#echo "========Check KAFKA status========"if kafka-topics.sh --bootstrap-server $KAFKA_BROKER --list >/dev/null 2>&1; thenecho "========Kafka cluster is Running========"((m_flag++))elseecho "--------Kafka cluster is Not Running--------"fi#检查clickhouse状态#echo "========Check clickhouse-server status========"if systemctl is-active --quiet clickhouse-server && clickhouse-client --port 9123 --user clickhouse --password clickhouse密码 --query "SELECT 1" >/dev/null 2>&1; thenecho "========ClickHouse-server service is Running========"((m_flag++))elseecho "--------ClickHouse-server service is Not Running--------"fi# 检查 Elasticsearch 服务状态if systemctl is-active --quiet elasticsearch && curl -s -u "$ES_USER:$ES_PASS" "$ES_API" >/dev/null 2>&1; thenecho "========Elasticsearch service is Running========"((m_flag++))elseecho "--------Elasticsearch service is Not Running--------"fi# 检查 Kibana 服务状态kibana_status=$(curl -s -u "$ES_USER:$ES_PASS" "$KIBANA_API" | /usr/local/bin/jq -r '.status.overall.state')if systemctl is-active --quiet kibana && [ "$kibana_status" == "green" ]; thenecho "========Kibana service is Running========"((m_flag++))elseecho "--------Kiaba service is Not Running--------"fi#检查监控组件状态for i in "${monitor_components[@]}"; doif systemctl is-active --quiet $i; thenecho "========$i service is Running========"elseecho "--------$i service is Not Running--------"fidone
}case $1 in
start)if ! /usr/java/jdk1.8.0_212/bin/jps | grep -q NameNode && ! hdfs dfsadmin -report >/dev/null 2>&1 && ! hadoop fs -ls / >/dev/null 2>&1; thenecho "start HDFS"/bin/bash /opt/module/hadoop-3.3.5/sbin/start-dfs.shfiif ! /usr/java/jdk1.8.0_212/bin/jps | grep -q ResourceManager && ! /usr/java/jdk1.8.0_212/bin/jps | grep -q NodeManager; thenecho "start Yarn"/bin/bash /opt/module/hadoop-3.3.5/sbin/start-yarn.shfifor i in "${important_components[@]}"; dostart_component "$i"donesleep 30for i in "${monitor_components[@]}"; dostart_component "$i"donesleep 20#检查组件状态check_status#判断是否满足启动任务条件if [ "$m_flag" -eq 6 ] && ! [ "$(curl -s http://$hostname:8088/ws/v1/cluster/metrics | /usr/local/bin/jq .clusterMetrics.appsRunning)" -eq $job_num_count ]; then/bin/bash /home/bigdata/job/dw3/monitor.shfi;;status)check_status;;stop)if /usr/java/jdk1.8.0_212/bin/jps | grep -q ResourceManager && /usr/java/jdk1.8.0_212/bin/jps | grep -q NodeManager; then#kill掉所有程序/bin/bash /home/bigdata/killAllApp.sh#判断程序是否都停掉if [ "$(curl -s http://$hostname:8088/ws/v1/cluster/metrics | /usr/local/bin/jq .clusterMetrics.appsRunning)" -eq "0" ]; then#关闭所有组件/bin/bash /opt/module/hadoop-3.3.5/sbin/stop-yarn.sh/bin/bash /opt/module/hadoop-3.3.5/sbin/stop-dfs.shfor i in "${important_components[@]}"; dosudo systemctl stop "$i"doneelseecho "Yarn上程序未停完，需手动停止"fififor i in "${monitor_components[@]}"; doif systemctl is-active --quiet $i; thenecho "========stop $i service========"sudo systemctl stop "$i"fidonecheck_status;;
*)echo "请输入合法的参数"echo "  start   启动所有组件集群"echo "  status  查看所有组件集群状态"echo "  stop    停止所有组件集群";;
esac

kill掉yarn所有带有ETL名字的任务

vi /home/bigdata/killAllApp.sh

#!/bin/bash
yarn application -list | grep 'ETL'| grep -v grep | awk '{print $1}' | xargs  yarn application -kill
echo -e "\033[1;33m============Yarn Application List============\033[0m"
yarn application -list