大数据集群软件启动脚本

大家也可以访问我的gitee仓库直接克隆源码大数据软件启动脚本

包括：hadoop，zookeeper，kafka，flume，hive，maxwell，dolphinscheduler，superset，集群一键启动，集群文件分发，集群执行命令，集群同步时间脚本

hadoop

 #!/bin/bash
 ​
 # hadoop集群启动
 ​
 if [ $# -eq 0 ]; then
     echo "Usage $0 start|stop"
 fi
 case $1 in
 ​
     "start"){
         echo " -------- 启动 集群 -------"
         echo " -------- 启动 hadoop集群 -------"
         ssh niit01 "source /etc/profile;start-dfs.sh"
         ssh niit03 "source /etc/profile;start-yarn.sh"
         ssh niit01 "source /etc/profile;mr-jobhistory-daemon.sh start historyserver"
     };;
     "stop"){
         echo " -------- 停止 集群 -------"
         echo " -------- 停止 hadoop集群 -------"
         ssh niit01 "source /etc/profile;mr-jobhistory-daemon.sh stop historyserver"
         ssh niit03 "source /etc/profile;stop-yarn.sh"
         ssh niit01 "source /etc/profile;stop-dfs.sh"
     };;
 esac
 ​

zookeeper

 #!/bin/bash
 ​
 # zookeeper集群启动脚本
 ​
 echo "Usage: $0 start|stop|status (zookeeper)"
 case $1 in
 "start"){
     for i in niit01 niit02 niit03
     do
         echo ---------- zookeeper $i 启动 ------------
         ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh start"
     done
 };;
 "stop"){
 for i in niit01 niit02 niit03
     do
         echo ---------- zookeeper $i 停止 ------------
         ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh stop"
     done
 };;
 "status"){
     for i in niit01 niit02 niit03
     do
         echo ---------- zookeeper $i 状态 ------------
         ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh status"
     done
 };;
 esac
 ​

kafka

 #!/bin/bash
 ​
 # kafka集群启动
 ​
 case $1 in
 "start"){
     for i in niit01 niit02 niit03
     do
         echo " --------启动 $i Kafka-------"
         ssh $i "/opt/onlineedu/kafka/bin/kafka-server-start.sh -daemon /opt/onlineedu/kafka/config/server.properties"
     done
 };;
 ​
 "stop"){
     for i in niit01 niit02 niit03
     do
         echo " --------停止 $i Kafka-------"
         ssh $i "/opt/onlineedu/kafka/bin/kafka-server-stop.sh stop"
     done
 };;
 esac
 ​

flume

 #! /bin/bash
 ​
 # flume采集通道1启动脚本
 ​
 case $1 in
 "start"){
     for i in niit01 niit02
     do
         echo " --------启动 $i 采集flume-------"
         ssh $i "nohup /opt/onlineedu/flume/bin/flume-ng agent --conf-file /opt/onlineedu/flume/job/file-flume-kafka.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/opt/onlineedu/flume/log1.txt 2>&1  &"
     done
 };;
 "stop"){
     for i in niit01 niit02
     do
         echo " --------停止 $i 采集flume-------"
         ssh $i "ps -ef | grep file-flume-kafka | grep -v grep |awk  '{print $2}' | xargs -n1 kill -9 "
     done
 };;
 esac
 ​

 #! /bin/bash
 ​
 # flume采集通道2启动脚本
 ​
 case $1 in
 "start"){
     for i in niit03
     do
         echo " --------启动 $i 消费flume-------"
         ssh $i "nohup /opt/onlineedu/flume/bin/flume-ng agent --conf-file /opt/onlineedu/flume/job/kafka-flume-hdfs.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/opt/onlineedu/flume/log2.txt 2>&1  &"
     done
 };;
 "stop"){
     for i in niit03
     do
         echo " --------停止 $i 消费flume-------"
         ssh $i "ps -ef | grep kafka-flume-hdfs | grep -v grep | awk '{print $2}' | xargs -n1 kill -9 "
     done
 };;
 esac
 ​

 #!/bin/bash
 ​
 # flume采集通道3启动脚本
 ​
 case $1 in
 "start")
         echo " --------启动 niit03 业务数据flume-------"
         ssh niit03 "nohup /opt/onlineedu/flume/bin/flume-ng agent -n a1 -c /opt/onlineedu/flume/conf -f /opt/onlineedu/flume/job/kafka_to_hdfs_db.conf >/opt/onlineedu/flume/log3.txt >/dev/null 2>&1 &"
 ;;
 "stop")
 ​
         echo " --------停止 niit03 业务数据flume-------"
         ssh niit03 "ps -ef | grep kafka_to_hdfs_db | grep -v grep |awk '{print $2}' | xargs -n1 kill"
 ;;
 esac
 ​

hive

 #!/bin/bash
 ​
 # hive启动
 ​
 HIVE_LOG_DIR=$HIVE_HOME/logs
 mkdir -p $HIVE_LOG_DIR
 #检查进程是否运行正常，参数1为进程名，参数2为进程端口
 ​
 function check_process()
 {
   pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
   ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
   echo $pid
   [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
 }
 ​
 function hive_start()
 {
   metapid=$(check_process HiveMetastore 9083)
   cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
   cmd=$cmd" sleep 4; hdfs dfsadmin -safemode wait >/dev/null 2>&1"
   [ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动"
   server2pid=$(check_process HiveServer2 10000)
   cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
   [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
 }
 ​
 function hive_stop()
 {
   metapid=$(check_process HiveMetastore 9083)
   [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
   server2pid=$(check_process HiveServer2 10000)
   [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
 }
 ​
 case $1 in
 "start")
   hive_start
   ;;
 "stop")
   hive_stop
   ;;
 "restart")
   hive_stop
   sleep 2
   hive_start
   ;;
 "status")
   check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常"
   check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常"
   ;;
 *)
   echo Invalid Args!
   echo 'Usage: '$(basename $0)' start|stop|restart|status (hiveserver2 & metastore)'
   ;;
 esac
 ​

maxwell

 #!/bin/bash
 ​
 # hive启动
 ​
 HIVE_LOG_DIR=$HIVE_HOME/logs
 mkdir -p $HIVE_LOG_DIR
 #检查进程是否运行正常，参数1为进程名，参数2为进程端口
 ​
 function check_process()
 {
   pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
   ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
   echo $pid
   [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
 }
 ​
 function hive_start()
 {
   metapid=$(check_process HiveMetastore 9083)
   cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
   cmd=$cmd" sleep 4; hdfs dfsadmin -safemode wait >/dev/null 2>&1"
   [ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动"
   server2pid=$(check_process HiveServer2 10000)
   cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
   [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
 }
 ​
 function hive_stop()
 {
   metapid=$(check_process HiveMetastore 9083)
   [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
   server2pid=$(check_process HiveServer2 10000)
   [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
 }
 ​
 case $1 in
 "start")
   hive_start
   ;;
 "stop")
   hive_stop
   ;;
 "restart")
   hive_stop
   sleep 2
   hive_start
   ;;
 "status")
   check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常"
   check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常"
   ;;
 *)
   echo Invalid Args!
   echo 'Usage: '$(basename $0)' start|stop|restart|status (hiveserver2 & metastore)'
   ;;
 esac
 ​

dolphinscheduler

#!/bin/bash

# dolphinscheduler集群启动

function node_start()
{
  /opt/onlineedu/dolphinscheduler/bin/start-all.sh  
  /home/niit/bin/xRun.sh jps
}

function node_stop()
{
  /opt/onlineedu/dolphinscheduler/bin/stop-all.sh
  /home/niit/bin/xRun.sh jps
}
case $1 in
"start")
  node_start
  ;;
"stop")
  node_stop
  ;;
"restart")
  node_stop
  sleep 10
  node_start
  ;;
*)
  
esac

#!/bin/bash

# dolphinscheduler单机版启动

function node_start()
{
  /opt/onlineedu/dolphinscheduler/bin/dolphinscheduler-daemon.sh start standalone-server   
  /home/niit/bin/xRun.sh jps
}

function node_stop()
{
  /opt/onlineedu/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop standalone-server
  /home/niit/bin/xRun.sh jps
}
case $1 in
"start")
  node_start
  ;;
"stop")
  node_stop
  ;;
"restart")
  node_stop
  sleep 10
  node_start
  ;;
*)
  
esac

superset

#!/bin/bash

# superset可视化启动

superset_status(){
    result=`ps -ef | awk '/gunicorn/ && !/awk/{print $2}' | wc -l`
    if [[ $result -eq 0 ]]; then
        return 0
    else
        return 1
    fi
}
superset_start(){
        source ~/.bashrc
        superset_status >/dev/null 2>&1
        if [[ $? -eq 0 ]]; then
            conda activate superset ; gunicorn --workers 5 --timeout 120 --bind niit01:8787 --daemon 'superset.app:create_app()'
        else
            echo "superset正在运行"
        fi

}

superset_stop(){
    superset_status >/dev/null 2>&1
    if [[ $? -eq 0 ]]; then
        echo "superset未在运行"
    else
        ps -ef | awk '/gunicorn/ && !/awk/{print $2}' | xargs kill -9
    fi
}


case $1 in
    start )
        echo "启动Superset"
        superset_start
    ;;
    stop )
        echo "停止Superset"
        superset_stop
    ;;
    restart )
        echo "重启Superset"
        superset_stop
        superset_start
    ;;
    status )
        superset_status >/dev/null 2>&1
        if [[ $? -eq 0 ]]; then
            echo "superset未在运行"
        else
            echo "superset正在运行"
        fi
esac

集群一键启动

#!/bin/bash

# 所有集群所有软件启动脚本

function node_start()
{
  /home/niit/bin/xCluster.sh start
  sleep 15
  /home/niit/bin/zk.sh start
  sleep 10
  /home/niit/bin/kf.sh start
  sleep 10
  /home/niit/bin/hv.sh start
  sleep 5
  /home/niit/bin/mxw.sh start
  sleep 5
  /home/niit/bin/f1.sh start
  sleep 5
  /home/niit/bin/f2.sh start
  sleep 5
  /home/niit/bin/f3.sh start
  sleep 5
  /home/niit/bin/dolphinscheduler-standalone.sh stop 
  sleep 5
  /home/niit/bin/superset.sh start
  sleep 5
  /home/niit/bin/xRun.sh jps
}

function node_stop()
{
  /home/niit/bin/mxw.sh stop
  sleep 5
  /home/niit/bin/f1.sh stop
  sleep 5
  /home/niit/bin/f2.sh stop
  sleep 5
  /home/niit/bin/f3.sh stop
  sleep 5
  /home/niit/bin/kf.sh stop
  sleep 10
  /home/niit/bin/kf.sh stop
  sleep 10
  /home/niit/bin/hv.sh stop
  sleep 5
  /home/niit/bin/xCluster.sh stop
  sleep 15
  /home/niit/bin/zk.sh stop
  sleep 10
  /home/niit/bin/superset.sh stop
  sleep 5
  /home/niit/bin/dolphinscheduler-standalone.sh stop 
  sleep 5
  /home/niit/bin/xRun.sh jps
}
case $1 in
"start")
  node_start
  ;;
"stop")
  node_stop
  ;;
"restart")
  node_stop
  sleep 10
  node_start
  ;;
*)
  
esac

集群文件分发

#!/bin/bash

# 集群分发文件脚本，普通用户权限

#1. 判断参数个数
if [ $# -lt 1 ]
then
    echo Not Enough Arguement!
    echo Usage: xSync.sh path1 path2..
    exit;
fi

#2. 遍历集群所有机器
for host in niit01 niit02 niit03
do
echo ==================== $host ====================
#3. 遍历所有目录，挨个发送

    for file in $@
    do
        #4 判断文件是否存在
        if [ -e $file ]
        then
            #5. 获取父目录
            pdir=$(cd -P $(dirname $file); pwd)

            #6. 获取当前文件的名称
            fname=$(basename $file)
            ssh $host "mkdir -p $pdir"
            rsync -av $pdir/$fname $host:$pdir
        else
            echo $file does not exists!
        fi
    done
done

#!/bin/bash

# 集群分发文件脚本，root权限

#1. 判断参数个数
if [ $# -lt 1 ]
then
    echo Not Enough Arguement!
    echo Usage: xSync.sh path1 path2 ...
    exit;
fi

#2. 遍历集群所有机器
for host in niit01 niit02 niit03
do
echo ==================== $host ====================
#3. 遍历所有目录，挨个发送

    for file in $@
    do
        #4 判断文件是否存在
        if [ -e $file ]
        then
            #5. 获取父目录
            pdir=$(cd -P $(dirname $file); pwd)

            #6. 获取当前文件的名称
            fname=$(basename $file)
            sudo ssh $host "mkdir -p $pdir"
            sudo rsync -av $pdir/$fname $host:$pdir
        else
            echo $file does not exists!
        fi
    done
done

集群执行命令

#/bin/bash

# 集群执行命令脚本

if (( $# == 0 ));then
    echo "Usage xRun.sh "<COMMAND>""
    exit 0
fi
for node in niit01 niit02 niit03
do
    echo "======== $node ========"
    echo "ssh $node $1"
    ssh $node "source /etc/profile;$1"
done

集群同步时间

#!/bin/bash

# 集群时间同步脚本，需要安装chrony到集群中

# 检查参数是否为空
if [ -z "$1" ]; then
        echo "Usage:  `basename $0` yyyy-MM-dd HH:mm:ss"
  exit 1
fi

# 使用date命令将时间字符串转换为日期和时间
# 如果转换失败，则说明时间字符串不合法
if ! date -d "$*" >/dev/null 2>&1; then
        echo "Wrong argument for $*"
        echo "Usage:  `basename $0` yyyy-MM-dd HH:mm:ss"
  exit 1
fi

echo ">>>>>>>>>>>> SYNC TIME START >>>>>>>>>>>>"
sum=-1

while [ $sum -ne 0 ]; do
  echo set time for niit01 to $1 '>>>'
  ssh niit01 "(sudo timedatectl set-ntp false && sudo date -s "$*" && sudo timedatectl set-ntp true)"
  ok1=$?
  echo sync time from niie02 to niit01 '>>>'
  ssh niit02 "(sudo timedatectl set-ntp false && sudo timedatectl set-ntp true)"
  ok2=$?
  echo sync time from niit03 to niit01 '>>>'
  ssh niit03 "(sudo timedatectl set-ntp false && sudo timedatectl set-ntp true)"
  ok3=$?
  sum=`expr $ok1 + $ok2 + $ok3`

  if [ $sum -eq 0 ]; then
    echo "<<<<<<<<<<<<< SYNC TIME END <<<<<<<<<<<<<"
    sleep 5
    xRun.sh date
  else
    echo "sync time failed, will try 10 senconds later"
    sleep 10
  fi
done