LTS分布式任务调度部署

llla
1,555 阅读2分钟

参考github

  1. 克隆代码

    git clone github.com/ltsopensour…

  2. 运行根目录下的sh build.sh或build.cmd脚本，会在dist目录下生成 lts-{version}-bin zip包，解压文件夹

下面是其目录结构，其中bin目录主要是JobTracker和LTS-Admin的启动脚本。
jobtracker 中是 JobTracker的配置文件和需要使用到的jar包，
lts-admin是LTS-Admin相关的war包和配置文件。 lts-{version}-bin的文件结构

-- lts-${version}-bin
    |-- bin
    |   |-- jobtracker.cmd
    |   |-- jobtracker.sh
    |   |-- lts-admin.cmd
    |   |-- lts-admin.sh
    |   |-- lts-monitor.cmd
    |   |-- lts-monitor.sh
    |   |-- tasktracker.sh
    |-- conf
    |   |-- log4j.properties
    |   |-- lts-admin.cfg
    |   |-- lts-monitor.cfg
    |   |-- readme.txt
    |   |-- tasktracker.cfg
    |   |-- zoo
    |       |-- jobtracker.cfg
    |       |-- log4j.properties
    |       |-- lts-monitor.cfg
    |-- lib
    |   |-- *.jar
    |-- war
        |-- jetty
        |   |-- lib
        |       |-- *.jar
        |-- lts-admin.war
  1. JobTracker启动

如果你想启动一个节点，直接修改下conf/zoo下的配置文件，然后运行 sh jobtracker.sh zoo start即可，如果你想启动两个JobTracker节点， 那么你需要拷贝一份zoo,譬如命名为zoo2,修改下zoo2下的配置文件， 然后运行sh jobtracker.sh zoo2 start即可。logs文件夹下生成 jobtracker-zoo.out日志

1.修改配置文件 lts-${version}-SNAPSHOT-bin/conf/zoo目录下的： jobtracker.cfg 、 lts-monitor.cfg ,主要修改zookeeper、mysql的配置信息

修改了registryAddress为redis, mysql配置

2.同上修改点

3.运行JobTracker

cd bin/ #进入lts-${version}-SNAPSHOT-bin/bin
sh jobtracker.sh zoo start

  1. LTS-Admin启动

修改conf/lts-monitor.cfg和conf/lts-admin.cfg下的配置， 然后运行bin下的sh lts-admin.sh或lts-admin.cmd脚本即可。 logs文件夹下会生成lts-admin.out日志，启动成功在日志中会打印出访问地址， 用户可以通过这个访问地址访问了

1.修改配置文件lts-1.7.1-SNAPSHOT-bin/conf目录下的：lts-admin.cfg 、 lts-monitor.cfg ，主要修改zookeeper、mysql的配置信息

2.运行LTS-Admin后台管理Web

cd bin/ #进入lts-${version}-SNAPSHOT-bin/bin
sh lts-admin.sh start

3.访问LTS-Admin后台管理Web

访问：http://172.20.48.141:8081/index.htm

1.启动TaskTracker和JobClient

  1. 配置
lts:
  taskTrackerNodeGroup: testtaskTracker
  remoting: mina
  tasktracker:
    cluster-name: test_cluster
    registry-address: redis://127.0.0.1:6379
    node-group: testtaskTracker
    configs:
      job:
        fail:
          store: mapdb
  jobclient:
      cluster-name: test_cluster
      registry-address: redis://127.0.0.1:6379
      node-group: testjobClient
      use-retry-client: true
      configs:
        job:
          fail:
            store: mapdb

  1. 注册定时Job
package com.example.lts.job.client;

import com.example.lts.job.LtsConfig;
import com.github.ltsopensource.core.domain.Job;
import com.github.ltsopensource.jobclient.JobClient;
import com.github.ltsopensource.jobclient.domain.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.Map;

@Component
public class JobRegister implements InitializingBean {

    @Autowired
    private LtsConfig ltsConfig;

    private Logger logger = LoggerFactory.getLogger(this.getClass());

    @Autowired
    private JobClient jobClient;

    @Override
    public void afterPropertiesSet() throws Exception {
        for (JobSchedule jobSchedule: JobSchedule.values()) {
            Job job = new Job();
            job.setTaskId(jobSchedule.getTaskId());
            job.setTaskTrackerNodeGroup(ltsConfig.getTaskTrackerNodeGroup());
            job.setNeedFeedback(jobSchedule.isNeedFeedback());
            job.setReplaceOnExist(jobSchedule.isReplaceOnExist());
            if (jobSchedule.isCronJob()) {
                job.setCronExpression(jobSchedule.getCronExpression());
            }
            Map<String, String> params = jobSchedule.getParams();
            if (params != null) {
                for (Map.Entry<String, String> param: params.entrySet()) {
                    job.setParam(param.getKey(), param.getValue());
                }
            }
            Response response = jobClient.submitJob(job);
        }

    }
}

读取配置（这里是用的枚举）

package com.example.lts.job.client;

import lombok.Getter;

import java.util.HashMap;
import java.util.Map;

@Getter
public enum JobSchedule {

    /** 测试Job */
    TEST_JOB("TEST_JOB", buildParams(new String[] { "key1" }, new String[] { "value1" }), false, true, true,
            "0 0/1 * * * ?");

    private String taskId;

    private Map<String, String> params = new HashMap<>();

    private boolean needFeedback;

    private boolean replaceOnExist;

    private boolean isCronJob;

    private String cronExpression;

    JobSchedule(String taskId, Map<String, String> params, boolean needFeedback, boolean replaceOnExist,
                boolean isCronJob, String cronExpression) {
        this.taskId = taskId;
        this.params = params;
        this.needFeedback = needFeedback;
        this.replaceOnExist = replaceOnExist;
        this.isCronJob = isCronJob;
        this.cronExpression = cronExpression;
    }

    private static Map<String, String> buildParams(String[] keys, String[] values) {
        Map<String, String> params = new HashMap<>();
        if (keys.length != values.length) {
            throw new RuntimeException("job参数键的个数和值的个数不一样");
        }
        for (int i = 0; i < keys.length; i++) {
            params.put(keys[i], values[i]);
        }
        return params;
    }

}

2.执行Job

  1. 添加执行方法(继承JobRunner)

在JobDistribute中run方法执行Job

package com.example.lts.job.task;

import com.github.ltsopensource.core.domain.Action;
import com.github.ltsopensource.core.domain.Job;
import com.github.ltsopensource.core.logger.Logger;
import com.github.ltsopensource.core.logger.LoggerFactory;
import com.github.ltsopensource.spring.boot.annotation.JobRunner4TaskTracker;
import com.github.ltsopensource.tasktracker.Result;
import com.github.ltsopensource.tasktracker.logger.BizLogger;
import com.github.ltsopensource.tasktracker.runner.JobContext;
import com.github.ltsopensource.tasktracker.runner.JobRunner;

@JobRunner4TaskTracker
public class JobDistribute implements JobRunner {
    private static final Logger LOGGER = LoggerFactory.getLogger(JobDistribute.class);

    @Override
    public Result run(JobContext jobContext) throws Throwable {
        try {
            BizLogger bizLogger = jobContext.getBizLogger();
            LOGGER.info("jobContext：" + jobContext);
            final Job job = jobContext.getJob();
            // 会发送到 LTS (JobTracker上)
            bizLogger.info("测试，业务日志");

            runJob(job);
        } catch (Exception e) {
            LOGGER.info("JOb_EXECUTE_FAILED", e);
            return new Result(Action.EXECUTE_FAILED, e.getMessage());
        }
        return new Result(Action.EXECUTE_SUCCESS, "EXECUTE_SUCCESS");
    }

    private void runJob(Job job) {
        final String taskId = job.getTaskId();
        if (JobInfo.valuesOfTaskId(taskId) != null) {
            final JobInfo jobInfo = JobInfo.valueOf(taskId);
            final String className = jobInfo.getClassName();
            try {
                final Class<?> jobClass = Class.forName(className);
                final Object jobInstance = jobClass.newInstance();
                SchedulerJob schedulerJob = (SchedulerJob) jobInstance;
                schedulerJob.perform(job);
            } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
                LOGGER.error(e);
            }
        } else {
            LOGGER.error("找不到task_id对应的job实现, taskId: " + taskId);
        }
    }
}

  1. 设置TaskTracker执行配置
package com.example.lts.job.task;

import com.google.common.collect.ImmutableMap;
import lombok.Getter;

@Getter
public enum JobInfo {

    /** 测试Job */
    TEST_JOB("TEST_JOB", "com.example.lts.job.task.jobs.TestJob");

    private static final ImmutableMap<String, JobInfo> CACHE;

    static {
        final ImmutableMap.Builder<String, JobInfo> builder = ImmutableMap.builder();
        for (JobInfo jobInfo: values()) {
            builder.put(jobInfo.getTaskId(), jobInfo);
        }
        CACHE = builder.build();
    }

    private String taskId;

    private String className;

    JobInfo(String taskId, String className) {
        this.taskId = taskId;
        this.className = className;
    }

    public static JobInfo valuesOfTaskId(String taskId) {
        return CACHE.get(taskId);
    }
}

  1. 手动提交Job（前提：Job已经提前在JobInfo中配置，无论是及时还定时）
package com.example.lts.job.client;

import com.example.lts.job.LtsConfig;
import com.github.ltsopensource.jobclient.JobClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

@Component
public class JobSubmiter {

    @Autowired
    private LtsConfig ltsConfig;

    @Autowired
    private JobClient jobClient;

//    public Response submitJob(JobVO jobVO) {
//        Job job = new Job();
//        job.setTaskId(jobVO.getTaskId());
//        job.setTaskTrackerNodeGroup(ltsConfig.getTaskTrackerNodeGroup());
//        job.setTriggerTime(new Date().getTime() + 1000);
//        final Map<String, String> params = jobVO.getParams();
//        if (params != null) {
//            for (Map.Entry<String, String> param: params.entrySet()) {
//                job.setParam(param.getKey(), param.getValue());
//            }
//        }
//        return jobClient.submitJob(job);
//    }
}
avatar
文章
阅读
粉丝