本文已参与「新人创作礼」活动，一起开启掘金创作之路。
JAVA的调用
OpenTSDB提供三种方式的读写操作：telnet、http、post，但官方并没提供JAVA版的API。在GitHub上有的Java的opentsdb-client ，才使得我能对openTSDB的读写操作进行封装，从而分享至此：github.com/shifeng258/… 在此项目上包装开发或将其打包成SDK使用均可。本人将用前一种方式进行的。 增加的一个统一调用类OpentsdbClient
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.ygsoft.opentsdb.client.ExpectResponse;
import com.ygsoft.opentsdb.client.HttpClient;
import com.ygsoft.opentsdb.client.HttpClientImpl;
import com.ygsoft.opentsdb.client.builder.MetricBuilder;
import com.ygsoft.opentsdb.client.request.Query;
import com.ygsoft.opentsdb.client.request.QueryBuilder;
import com.ygsoft.opentsdb.client.request.SubQueries;
import com.ygsoft.opentsdb.client.response.Response;
import com.ygsoft.opentsdb.client.response.SimpleHttpResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
/**
* Opentsdb读写工具类
*/
public class OpentsdbClient {
private static Logger log = LoggerFactory.getLogger(OpentsdbClient.class);
/**
* 取平均值的聚合器
*/
public static String AGGREGATOR_AVG = "avg";
/**
* 取累加值的聚合器
*/
public static String AGGREGATOR_SUM = "sum";
private HttpClient httpClient;
public OpentsdbClient(String opentsdbUrl) {
this.httpClient = new HttpClientImpl(opentsdbUrl);
}
/**
* 写入数据
* @param metric 指标
* @param timestamp 时间点
* @param value
* @param tagMap
* @return
* @throws Exception
*/
public boolean putData(String metric, Date timestamp, long value, Map tagMap) throws Exception {
long timsSecs = timestamp.getTime() / 1000;
return this.putData(metric, timsSecs, value, tagMap);
}
/**
* 写入数据
* @param metric 指标
* @param timestamp 时间点
* @param value
* @param tagMap
* @return
* @throws Exception
*/
public boolean putData(String metric, Date timestamp, double value, Map tagMap) throws Exception {
long timsSecs = timestamp.getTime() / 1000;
return this.putData(metric, timsSecs, value, tagMap);
}
/**
* 写入数据
* @param metric 指标
* @param timestamp 转化为秒的时间点
* @param value
* @param tagMap
* @return
* @throws Exception
*/
public boolean putData(String metric, long timestamp, long value, Map tagMap) throws Exception {
MetricBuilder builder = MetricBuilder.getInstance();
builder.addMetric(metric).setDataPoint(timestamp, value).addTags(tagMap);
try {
log.debug("write quest：{}", builder.build());
Response response = httpClient.pushMetrics(builder, ExpectResponse.SUMMARY);
log.debug("response.statusCode: {}", response.getStatusCode());
return response.isSuccess();
} catch (Exception e) {
log.error("put data to opentsdb error: ", e);
throw e;
}
}
/**
* 写入数据
* @param metric 指标
* @param timestamp 转化为秒的时间点
* @param value
* @param tagMap
* @return
* @throws Exception
*/
public boolean putData(String metric, long timestamp, double value, Map tagMap) throws Exception {
MetricBuilder builder = MetricBuilder.getInstance();
builder.addMetric(metric).setDataPoint(timestamp, value).addTags(tagMap);
try {
log.debug("write quest：{}", builder.build());
Response response = httpClient.pushMetrics(builder, ExpectResponse.SUMMARY);
log.debug("response.statusCode: {}", response.getStatusCode());
return response.isSuccess();
} catch (Exception e) {
log.error("put data to opentsdb error: ", e);
throw e;
}
}
/**
* 批量写入数据
* @param metric 指标
* @param timestamp 时间点
* @param value
* @param tagMap
* @return
* @throws Exception
*/
public boolean putData(JSONArray jsonArr) throws Exception {
return this.putDataBatch(jsonArr);
}
/**
* 批量写入数据
* @param metric 指标
* @param timestamp 转化为秒的时间点
* @param value
* @param tagMap
* @return
* @throws Exception
*/
public boolean putDataBatch(JSONArray jsonArr) throws Exception {
MetricBuilder builder = MetricBuilder.getInstance();
try {
for(int i = 0; i < jsonArr.size(); i++){
Map tagMap = new HashMap();
for(String key : jsonArr.getJSONObject(i).getJSONObject("tags").keySet()){
tagMap.put(key, jsonArr.getJSONObject(i).getJSONObject("tags").get(key));
}
String metric = jsonArr.getJSONObject(i).getString("metric").toString();
long timestamp = DateTimeUtil.parse(jsonArr.getJSONObject(i).getString("timestamp"), "yyyy/MM/dd HH:mm:ss").getTime() / 1000;
double value = Double.valueOf(jsonArr.getJSONObject(i).getString("value"));
builder.addMetric(metric).setDataPoint(timestamp, value).addTags(tagMap);
}
log.debug("write quest：{}", builder.build());
Response response = httpClient.pushMetrics(builder, ExpectResponse.SUMMARY);
log.debug("response.statusCode: {}", response.getStatusCode());
return response.isSuccess();
} catch (Exception e) {
log.error("put data to opentsdb error: ", e);
throw e;
}
}
/**
* 查询数据，返回的数据为json格式，结构为：
* "[
* " {
* " metric: mysql.innodb.row_lock_time,
* " tags: {
* " host: web01,
* " dc: beijing
* " },
* " aggregateTags: [],
* " dps: {
* " 1435716527: 1234,
* " 1435716529: 2345
* " }
* " },
* " {
* " metric: mysql.innodb.row_lock_time,
* " tags: {
* " host: web02,
* " dc: beijing
* " },
* " aggregateTags: [],
* " dps: {
* " 1435716627: 3456
* " }
* " }
* "]";
* @param metric 要查询的指标
* @param aggregator 查询的聚合类型, 如: OpentsdbClient.AGGREGATOR_AVG, OpentsdbClient.AGGREGATOR_SUM
* @param tagMap 查询的条件
* @param downsample 采样的时间粒度, 如: 1s,2m,1h,1d,2d
* @param startTime 查询开始时间,时间格式为yyyy/MM/dd HH:mm:ss
* @param endTime 查询结束时间,时间格式为yyyy/MM/dd HH:mm:ss
*/
public String getData(String metric, Map tagMap, String aggregator, String downsample, String startTime, String endTime) throws IOException {
QueryBuilder queryBuilder = QueryBuilder.getInstance();
Query query = queryBuilder.getQuery();
query.setStart(DateTimeUtil.parse(startTime, "yyyy/MM/dd HH:mm:ss").getTime() / 1000);
query.setEnd(DateTimeUtil.parse(endTime, "yyyy/MM/dd HH:mm:ss").getTime() / 1000);
List sqList = new ArrayList();
SubQueries sq = new SubQueries();
sq.addMetric(metric);
sq.addTag(tagMap);
sq.addAggregator(aggregator);
sq.setDownsample(downsample + "-" + aggregator);
sqList.add(sq);
query.setQueries(sqList);
try {
log.debug("query request：{}", queryBuilder.build()); //这行起到校验作用
SimpleHttpResponse spHttpResponse = httpClient.pushQueries(queryBuilder, ExpectResponse.DETAIL);
log.debug("response.content: {}", spHttpResponse.getContent());
if (spHttpResponse.isSuccess()) {
return spHttpResponse.getContent();
}
return null;
} catch (IOException e) {
log.error("get data from opentsdb error: ", e);
throw e;
}
}
/**
* 查询数据，返回tags与时序值的映射: Map>
* @param metric 要查询的指标
* @param aggregator 查询的聚合类型, 如: OpentsdbClient.AGGREGATOR_AVG, OpentsdbClient.AGGREGATOR_SUM
* @param tagMap 查询的条件
* @param downsample 采样的时间粒度, 如: 1s,2m,1h,1d,2d
* @param startTime 查询开始时间, 时间格式为yyyy/MM/dd HH:mm:ss
* @param endTime 查询结束时间, 时间格式为yyyy/MM/dd HH:mm:ss
* @param retTimeFmt 返回的结果集中，时间点的格式, 如：yyyy/MM/dd HH:mm:ss 或 yyyyMMddHH 等
* @return Map>
*/
public Map getData(String metric, Map tagMap, String aggregator, String downsample, String startTime, String endTime, String retTimeFmt) throws IOException {
String resContent = this.getData(metric, tagMap, aggregator, downsample, startTime, endTime);
return this.convertContentToMap(resContent, retTimeFmt);
}
public Map convertContentToMap(String resContent, String retTimeFmt) {
Map tagsValuesMap = new HashMap();
if (resContent == null || "".equals(resContent.trim())) {
return tagsValuesMap;
}
JSONArray array = (JSONArray) JSONObject.parse(resContent);
if (array != null) {
for (int i = 0; i < array.size(); i++) {
JSONObject obj = (JSONObject) array.get(i);
JSONObject tags = (JSONObject) obj.get("tags");
JSONObject dps = (JSONObject) obj.get("dps"); //timeValueMap.putAll(dps); Map timeValueMap = new HashMap(); for (Iterator it = dps.keySet().iterator(); it.hasNext(); ) { String timstamp = it.next(); Date datetime = new Date(Long.parseLong(timstamp)*1000); timeValueMap.put(DateTimeUtil.format(datetime, retTimeFmt), dps.get(timstamp)); } tagsValuesMap.put(tags.toString(), timeValueMap); } } return tagsValuesMap; }
}
}
return tagsValuesMap;
}
}
增加一个时间处理类
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class DateTimeUtil {
public static Date parse(String date,String fm){
Date res=null;
try {
SimpleDateFormat sft=new SimpleDateFormat(fm);
res=sft.parse(date);
} catch (ParseException e) {
e.printStackTrace();
}
return res;
}
}
为了建立HTTP的长连接，将PoolingHttpClient进行单例模式的修改（防止调用时频繁的实例化引起的资源消耗）
public class PoolingHttpClient {
...
/*单例模式修改*/
private static PoolingHttpClient poolingHttpClient;
private PoolingHttpClient() {
// Increase max total connection
connManager.setMaxTotal(maxTotalConnections);
// Increase default max connection per route
connManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
// config timeout
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.setConnectionRequestTimeout(waitTimeout)
.setSocketTimeout(readTimeout).build();
httpClient = HttpClients.custom()
.setKeepAliveStrategy(keepAliveStrategy)
.setConnectionManager(connManager)
.setDefaultRequestConfig(config).build();
// detect idle and expired connections and close them
IdleConnectionMonitorThread staleMonitor = new IdleConnectionMonitorThread(
connManager);
staleMonitor.start();
}
public static PoolingHttpClient getInstance() {
if (null == poolingHttpClient) {
//加锁保证线程安全
synchronized (PoolingHttpClient.class) {
if (null == poolingHttpClient) {
poolingHttpClient = new PoolingHttpClient();
}
}
}
return poolingHttpClient;
}
/*单例模式修改结束*/
public SimpleHttpResponse doPost(String url, String data)
throws IOException {
StringEntity requestEntity = new StringEntity(data);
HttpPost postMethod = new HttpPost(url);
postMethod.setEntity(requestEntity);
HttpResponse response = execute(postMethod);
int statusCode = response.getStatusLine().getStatusCode();
SimpleHttpResponse simpleResponse = new SimpleHttpResponse();
simpleResponse.setStatusCode(statusCode);
HttpEntity entity = response.getEntity();
if (entity != null) {
// should return: application/json; charset=UTF-8
String ctype = entity.getContentType().getValue();
String charset = getResponseCharset(ctype);
String content = EntityUtils.toString(entity, charset);
simpleResponse.setContent(content);
}
/*增加线程回收开始*/
EntityUtils.consume(entity);
postMethod.releaseConnection();
/*增加线程回收结束*/
return simpleResponse;
}
...
}
修改HttpClientImpl对单例模式的PoolingHttpClient调用
public class HttpClientImpl implements HttpClient {
...
//private PoolingHttpClient httpClient = new PoolingHttpClient();
private PoolingHttpClient httpClient = PoolingHttpClient.getInstance();
...
}
