Ubuntu安装dolphinscheduler集群

用户201708
96 阅读5分钟

前置任务1:互相免密登录

1、各个节点分别安装ssh: sudo apt-get install ssh
2、# 各个节点删除原来的
   rm -rf ~/.ssh
   #各个节点生成公钥和私钥，输入命令，提示直接按enter就行了
      ssh-keygen -t  rsa
      cd ~/.ssh
  就会发现多了如下文件: id_rsa 和 id_rsa.pub 分别是私钥和公钥。
3、各个节点执行如下命令：
      cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
   各个节点将公钥添加到.ssh/authorized_keys，这时就能在本机免密码登陆了
4、各个节点然后将id_rsa.pub文件复制到其他主机上，这里我使用的是主机名
   scp /root/.ssh/id_rsa.pub  root@P-SH1CRAWLERAPP02:/root/
5、在132节点上执行1、2、3，然后将id_rsa.pub的内容添加到132的authorized_keys文件中，完成之后就能在本机免密码登录132了
   cat /root/id_rsa.pub >>  ~/.ssh/authorized_keys
其他节点操作 4，5

前置任务2:配置hostname

10.xx.xxx.61 P-SH1CRAWLERAPP01
10.xx.xxx.62 P-SH1CRAWLERAPP02
10.xx.xxx.63 P-SH1CRAWLERAPP03

前置任务3:安装JDK

前置任务4:安装docker-mysql

#  分发jar包
cp mysql-connector-j-9.2.0.jar ./apache-dolphinscheduler-3.2.1-bin/api-server/libs/
cp mysql-connector-j-9.2.0.jar /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/alert-server/libs/
cp mysql-connector-j-9.2.0.jar /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/master-server/libs/
cp mysql-connector-j-9.2.0.jar /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/worker-server/libs/
cp mysql-connector-j-9.2.0.jar /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/tools/libs/

#docker-compose文件

	version: '3'
	services:
	  mysql:
		# 镜像名
		image: mysql:latest
		# 容器名(以后的控制都通过这个)
		container_name: mysql
		# 重启策略
		restart: always
		environment:
		  # 时区上海
		  # TZ: Asia/Shanghai
		  # root 密码
		  MYSQL_ROOT_PASSWORD: Jmkx2022
		  # 映射端口
		ports:
		  - 3306:3306
		volumes:
		  - /data/mysql/mysql-datadir:/var/lib/mysql # 映射数据目录，宿主机:容器
		  - /data/mysql/conf.d:/etc/mysql/conf.d # 映射配置目录，宿主机:容器
		  - /data/mysql/mysql-log:/var/log/mysql  #映射日志目录，宿主机:容器
		  - /etc/localtime:/etc/localtime:ro   # 让容器的时钟与宿主机时钟同步，避免时间的问题，ro是read only的意思，就是只读。
		command:
		  --character-set-server=utf8mb4
		  --collation-server=utf8mb4_general_ci
		  --explicit_defaults_for_timestamp=true
		  --lower_case_table_names=1

在docker-mysql中

docker exec -it 0f21907666e6 /bin/bash 
docker-compose -f docker-compose.yml up -d mysql
docker stop docker_id

账号密码

grant all privileges on *.* to 'root'@'%' identified by 'Jmkx2022' with option;

#设置远程连接

[root@hadoop_zxy dolphinscheduler-1.3.4]# mysql -uroot -p密码

CREATE USER 'dolphinscheduler'@'%' IDENTIFIED BY 'dolphinscheduler123';
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'%';
mysql> CREATE USER 'dolphinscheduler'@'localhost' IDENTIFIED BY 'dolphinscheduler123';
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'localhost';
mysql> FLUSH PRIVILEGES;

初始化dolp的元数据，完成后去mysql查看表结构

sh /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/tools/bin/upgrade-schema.sh

前置任务5：安装zk

# 先改文件名 cp zoo_sample.cfg zoo.cfg
# vim文件    vim /data/zk/apache-zookeeper-3.8.4-bin/conf/zoo.cfg

dataDir=/data/zk/apache-zookeeper-3.8.4-bin/zdata

server.2=P-SH1CRAWLERAPP01:2888:3888
server.3=P-SH1CRAWLERAPP02:2888:3888
server.4=P-SH1CRAWLERAPP03:2888:3888

# cd /data/zk/apache-zookeeper-3.8.4-bin/bin
./zkServer.sh start

# 通过status查看启动状态。稍微有个等待时间
./zkServer.sh status

安装dolphinscheduler集群

1、vim $DS_HOME/bin/env/install_env.sh

#安装目录

#vim /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/bin/env/install_env.sh

#安装后目录

#vim /data/dolphinscheduler/bin/env/install_env.sh

# 将要部署任一DolphinScheduler服务的服务器主机名或ip列表
ips="P-SH1CRAWLERAPP01,P-SH1CRAWLERAPP02,P-SH1CRAWLERAPP03"
# ssh端口号
sshPort="22"
# master 所在主机名列表，必须是ips的子集
masters="P-SH1CRAWLERAPP01,P-SH1CRAWLERAPP02"
# worker主机名及队列，此处的ip必须在ips列表中
workers="P-SH1CRAWLERAPP01:default,P-SH1CRAWLERAPP02:default,P-SH1CRAWLERAPP03:default"
# 告警服务所在服务器主机名
alertServer="P-SH1CRAWLERAPP03"
# api服务所在服务器主机名
apiServers="P-SH1CRAWLERAPP01"
# DS 安装路径，如果不存在会创建
installPath="/data/dolphinscheduler/"
# 部署用户，任务执行服务是以sudo -u {linux-user} 切换不同Linux用户的方式来实现多租户运行作业，因此该用户必须有免密的sudo权限。
deployUser="root"
# 前文配置的所有节点的本地数据存储路径，需要确保部署用户拥有该目录的读写权限
dataBasedirPath="/data/dolphinscheduler/data"
# JAVA_HOME 路径
javaHome="/data/jdk/jdk1.8.0_171"
# apiServer的端口
apiServerPort="12345"
# 数据库类型
DATABASE_TYPE="mysql"
# 数据库 URL
SPRING_DATASOURCE_URL="jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler?useUnicode=true&allowPublicKeyRetrieval=true&characterEncoding=UTF-8"
# 数据库用户名
SPRING_DATASOURCE_USERNAME="dolphinscheduler"
# 数据库密码
SPRING_DATASOURCE_PASSWORD="dolphinscheduler123"
# 注册中心插件名称，DS 通过注册中心来确保集群配置的一致性
registryPluginName="zookeeper"
# 注册中心地址，即 Zookeeper 集群的地址
registryServers="P-SH1CRAWLERAPP01:2181,P-SH1CRAWLERAPP02:2181,P-SH1CRAWLERAPP03:2181"
# DS 在 Zookeeper 的结点名称
registryNamespace="dolphinscheduler"

2、vim $DS_HOME/bin/env/dolphinscheduler_env.sh

#vim /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/bin/env/dolphinscheduler_env.sh #vim /data/dolphinscheduler/bin/env/dolphinscheduler_env.sh

# 修改JAVA_HOME
export JAVA_HOME="/data/jdk/jdk1.8.0_171"
# 修改数据库连接信息
export DATABASE="mysql"
export SPRING_PROFILES_ACTIVE=${DATABASE}
export SPRING_DATASOURCE_URL="jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler?useUnicode=true&allowPublicKeyRetrieval=true&characterEncoding=UTF-8&useSSL=false"
export SPRING_DATASOURCE_USERNAME="dolphinscheduler"
export SPRING_DATASOURCE_PASSWORD="dolphinscheduler123"

# DolphinScheduler server related configuration
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}

# 修改zookeep连接地址
export REGISTRY_TYPE="zookeeper"
#export REGISTRY_ZOOKEEPER_CONNECT_STRING="P-SH1CRAWLERAPP01:2181,P-SH1CRAWLERAPP02:2181,P-SH1CRAWLERAPP03:2181"
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-P-SH1CRAWLERAPP01:2181,P-SH1CRAWLERAPP02:2181,P-SH1CRAWLERAPP03:2181}

3、master-server

#vi /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/master-server/conf/application.yaml

#vi /data/dolphinscheduler/master-server/conf/application.yaml

 datasource:
  driver-class-name: com.mysql.cj.jdbc.Driver
  url: jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler2204?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
  username: dolphinscheduler
  password: dolphinscheduler123

registry:
 type: zookeeper
 zookeeper:
  namespace: dolphinscheduler
  connect-string: P-SH1CRAWLERAPP01:2181,P-SH1CRAWLERAPP02:2181,P-SH1CRAWLERAPP03:2181

max-cpu-load-avg: 3
reserved-memory: 0.1
max-waiting-time: 150s

 datasource:
  driver-class-name: com.mysql.cj.jdbc.Driver
  url: jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler2204?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
  username: dolphinscheduler
  password: dolphinscheduler123

4、worker-server

#vi /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/worker-server/conf/application.yaml

#vi /data/dolphinscheduler/worker-server/conf/application.yaml

registry:
 type: zookeeper
 zookeeper:
  namespace: dolphinscheduler2204
  connect-string: P-SH1CRAWLERAPP01:2181,P-SH1CRAWLERAPP02:2181,P-SH1CRAWLERAPP03:2181
max-cpu-load-avg: 3
reserved-memory: 0.1
max-waiting-time: 150s

5、api-server

#vi /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/api-server/conf/application.yaml

#vi /data/dolphinscheduler/api-server/conf/application.yaml

datasource:
  driver-class-name: com.mysql.cj.jdbc.Driver
  url: jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler2204?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
  username: dolphinscheduler
  password: dolphinscheduler123

registry:
 type: zookeeper
 zookeeper:
  namespace: dolphinscheduler
  connect-string: P-SH1CRAWLERAPP01:2181,P-SH1CRAWLERAPP02:2181,P-SH1CRAWLERAPP03:2181

 datasource:
  driver-class-name: com.mysql.cj.jdbc.Driver
  url: jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler2204?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
  username: dolphinscheduler
  password: dolphinscheduler123

6、alert-server

#vi /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/alert-server/conf/application.yaml

#vi /data/dolphinscheduler/alert-server/conf/application.yaml

datasource:
  driver-class-name: com.mysql.cj.jdbc.Driver
  url: jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler2204?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
  username: dolphinscheduler
  password: dolphinscheduler123

registry:
 type: zookeeper
 zookeeper:
  namespace: dolphinscheduler
  connect-string: P-SH1CRAWLERAPP01:2181,P-SH1CRAWLERAPP02:2181,P-SH1CRAWLERAPP03:2181

 datasource:
  driver-class-name: com.mysql.cj.jdbc.Driver
  url: jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler2204?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
  username: dolphinscheduler
  password: dolphinscheduler123

7、tools

#vi /data/dolphinscheduler/apache-dolphinscheduler-3.2.2-bin/tools/conf/application.yaml

#vi /data/dolphinscheduler/tools/conf/application.yaml

 datasource:
  driver-class-name: com.mysql.cj.jdbc.Driver
  url: jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler2204?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
  username: dolphinscheduler
  password: dolphinscheduler123

 datasource:
  driver-class-name: com.mysql.cj.jdbc.Driver
  url: jdbc:mysql://P-SH1CRAWLERAPP01:3306/dolphinscheduler2204?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true

初次安装

bash ./bin/install.sh

浏览器访问地址 http://masterIP:12345/dolphinscheduler/ui 即可登录系统 UI。默认的用户名和密码是 admin/dolphinscheduler123

后续的再启动、操作

切换去 cd /data

# 一键停止集群所有服务
bash ./bin/stop-all.sh
 
# 一键开启集群所有服务
bash ./bin/start-all.sh
 
# 启停 Master
bash ./bin/dolphinscheduler-daemon.sh stop master-server
bash ./bin/dolphinscheduler-daemon.sh start master-server
 
# 启停 Worker
bash ./bin/dolphinscheduler-daemon.sh start worker-server
bash ./bin/dolphinscheduler-daemon.sh stop worker-server
 
# 启停 Api
bash ./bin/dolphinscheduler-daemon.sh start api-server
bash ./bin/dolphinscheduler-daemon.sh stop api-server
 
# 启停 Alert
bash ./bin/dolphinscheduler-daemon.sh start alert-server
bash ./bin/dolphinscheduler-daemon.sh stop alert-server
avatar
文章
阅读
粉丝