dolphinscheduler

dockercompose 部署 dolphinscheduler

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version: "3.8"

services:
  dolphinscheduler-postgresql:
    image: bitnami/postgresql:15.2.0
    ports:
      - "5432:5432"
    profiles: ["all", "schema"]
    environment:
      POSTGRESQL_USERNAME: root
      POSTGRESQL_PASSWORD: root
      POSTGRESQL_DATABASE: dolphinscheduler
    volumes:
      - dolphinscheduler-postgresql:/bitnami/postgresql
    healthcheck:
      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"]
      interval: 5s
      timeout: 60s
      retries: 120
    networks:
      - dolphinscheduler

  dolphinscheduler-zookeeper:
    image: bitnami/zookeeper:3.7.1
    profiles: ["all"]
    environment:
      ALLOW_ANONYMOUS_LOGIN: "yes"
      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
    volumes:
      - dolphinscheduler-zookeeper:/bitnami/zookeeper
    healthcheck:
      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"]
      interval: 5s
      timeout: 60s
      retries: 120
    networks:
      - dolphinscheduler

  dolphinscheduler-schema-initializer:
    image: ${HUB}/dolphinscheduler-tools:${TAG}
    env_file: .env
    profiles: ["schema"]
    command: [ tools/bin/upgrade-schema.sh ]
    depends_on:
      dolphinscheduler-postgresql:
        condition: service_healthy
    volumes:
      - dolphinscheduler-logs:/opt/dolphinscheduler/logs
      - dolphinscheduler-shared-local:/opt/soft
      - dolphinscheduler-resource-local:/dolphinscheduler
    networks:
      - dolphinscheduler

  dolphinscheduler-api:
    image: ${HUB}/dolphinscheduler-api:${TAG}
    ports:
      - "12345:12345"
      - "25333:25333"
    profiles: ["all"]
    env_file: .env
    healthcheck:
      test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
      dolphinscheduler-zookeeper:
        condition: service_healthy
    volumes:
      - dolphinscheduler-logs:/opt/dolphinscheduler/logs
      - dolphinscheduler-shared-local:/opt/soft
      - dolphinscheduler-resource-local:/dolphinscheduler
      - /root/datax:/root/datax
    networks:
      - dolphinscheduler

  dolphinscheduler-alert:
    image: ${HUB}/dolphinscheduler-alert-server:${TAG}
    profiles: ["all"]
    env_file: .env
    healthcheck:
      test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
      dolphinscheduler-zookeeper:
        condition: service_healthy
    volumes:
      - dolphinscheduler-logs:/opt/dolphinscheduler/logs
      - /root/datax:/root/datax
    networks:
      - dolphinscheduler

  dolphinscheduler-master:
    image: ${HUB}/dolphinscheduler-master:${TAG}
    profiles: ["all"]
    env_file: .env
    healthcheck:
      test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
      dolphinscheduler-zookeeper:
        condition: service_healthy
    volumes:
      - dolphinscheduler-logs:/opt/dolphinscheduler/logs
      - /root/datax:/root/datax
      - dolphinscheduler-shared-local:/opt/soft
    networks:
      - dolphinscheduler

  dolphinscheduler-worker:
    image: ${HUB}/dolphinscheduler-worker:${TAG}
    profiles: ["all"]
    env_file: .env
    healthcheck:
      test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
      dolphinscheduler-zookeeper:
        condition: service_healthy
    volumes:
      - dolphinscheduler-worker-data:/tmp/dolphinscheduler
      - dolphinscheduler-logs:/opt/dolphinscheduler/logs
      - dolphinscheduler-shared-local:/opt/soft
      - dolphinscheduler-resource-local:/dolphinscheduler
      - /root/datax:/root/datax
    networks:
      - dolphinscheduler


   

networks:
  dolphinscheduler:
    driver: bridge

volumes:
  dolphinscheduler-postgresql:
  dolphinscheduler-zookeeper:
  dolphinscheduler-worker-data:
  dolphinscheduler-logs:
  dolphinscheduler-shared-local:
  dolphinscheduler-resource-local:

2.集成python

有网络环境中，在官方镜像基础上封装python3环境

Docker exec -it docker_dolphinscheduler-worker_1 bash

apt update && apt install python3 -y

ln -s /usr/bin/python3 /usr/bin/python

 

docker commit docker_dolphinscheduler-worker_1 ghcr.io/apache/dolphinscheduler/dolphinscheduler-worker:latest

3.集成datax

解压datax.tar.gz到服务器。
chmod 777 datax/bin/datax.py  赋予datax/bin/datax.py最大权限

在dockercompose worker服务中增加一行映射，把宿主datax目录映射到容器中

docker-compose --profile all up -d

 

 

4.配置环境变量

安全中心—>环境管理—>创建环境

export PYTHON\_LAUNCHER=/usr/bin/python

export DATAX\_LAUNCHER=/root/datax/bin/datax.py

export JAVA\_HOME=/opt/java/openjdk

export PATH=$PYTHON_LAUNCHER:$JAVA\_HOME/bin:$DATAX_LAUNCHER:$PATH

 

5让Dolphinschedule源数据库支持Mysql

下载mysql驱动并放到对应libs目录中

如果使用 MySQL 需要手动下载 mysql-connector-java 驱动 (8.0.16) 并移动到 DolphinScheduler 的每个模块的 libs 目录下

 

在MySQL驱动包目录下，执行以下命令

如果用的是默认容器启动的zookeeper和PostgreSQL，会有两条Error，忽略即可

for container_name in docker ps |grep dolphinscheduler | awk '{print $1}';do docker cp ./mysql-connector-java-8.0.16.jar $container_name:/opt/dolphinscheduler/libs;done

 

6.修改docker_dolphinscheduler-worker的配置文件

容器内路径：/opt/dolphinscheduler/conf/ application.yaml 

可以先拷贝到本地开完再覆盖回去

 

 

docker-compose --profile all restart 重启服务

 

 

 

7.配置mysql源数据

源中心，创建源

 

创建项目

创建工作流