需求，我们要重题库里面选择试题，生成试卷，然后上传到云上面，在传统的web中一份PDF文档的生成平均时长在50~55秒左右，
-
分离出需要处理的题目(60~120个，平均大约80个题目左右)
-
解析处理题目文本，对题目中的图片下载到本地，然后调用第三方工具生成PDF文档（耗时大约40~45秒）
-
将PDF文档上传到云空间进行存储（耗时大约9、10秒）
-
提供文档地址让用户去下载打印
所以我们要改进
-
架构改进之路
服务化，文档生成并行化，采用生产者消费者模式
-
文档处理的改进之路
考察业务特点：1、从容量为10万左右的题库中为每个学生抽取适合他的题目；2、每道题目都含有大量的图片需要下载到本地，和文字部分一起渲染；3、存在热点题目
1、解决方案：缓存避免重复工作、题目处理并行和异步化
2、如何实现
1）先检索缓存，新题目在生成时要考虑并发安全和充分利用Future
2）题目有更新时，怎么处理?
代码如下：
public class Consts {
public static final int CPU_COUNT = Runtime.getRuntime().availableProcessors();
public static final int QUESTION_COUNT_IN_DOC = 80;
public static final int SIZE_OF_QUESTION_BANK = 2000;
public static final int QUESTION_LENGTH = 2000;
}
public class CreatePendingDocs {
public static List<SrcDocVo> makePendingDoc(int count) {
Random random = new Random();
List<SrcDocVo> docList = new LinkedList<>();
for (int i = 0; i < count; i++) {
List<Integer> questionList = new LinkedList<>();
for (int j = 0; j < Consts.QUESTION_COUNT_IN_DOC; j++) {
int questionId = random.nextInt(Consts.SIZE_OF_QUESTION_BANK);
questionList.add(questionId);
}
SrcDocVo pendingDocVo = new SrcDocVo("pending_" + i, questionList);
docList.add(pendingDocVo);
}
return docList;
}
}
public class EncryptUtils {
/**
* @param strSrc 需要被加密的字符串
* @param encName 加密方式，有 MD5、SHA-1和SHA-256 这三种加密方式
* @return 返回加密后的字符串
*/
private static String EncryptStr(String strSrc, String encName) {
MessageDigest md = null;
String strDes = null;
byte[] bt = strSrc.getBytes();
try {
if (encName == null || encName.equals("")) {
encName = "MD5";
}
md = MessageDigest.getInstance(encName);
md.update(bt);
strDes = bytes2Hex(md.digest()); // to HexString
} catch (NoSuchAlgorithmException e) {
System.out.println("Invalid algorithm.");
return null;
}
return strDes;
}
/**
* @param str 需要被加密的字符串
* @return 对字符串str进行MD5加密后，将加密字符串返回
*/
public static String EncryptByMD5(String str) {
return EncryptStr(str, "MD5");
}
/**
* 该方法主要用于验证学生端的md5密码
*
* @param s the string want to encode
* @return the encoded String
*/
public static String to_MD5(String s) {
char hexDigits[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'a', 'b', 'c', 'd', 'e', 'f'};
try {
byte[] strTemp = s.getBytes();
MessageDigest mdTemp = MessageDigest.getInstance("MD5");
mdTemp.update(strTemp);
byte[] md = mdTemp.digest();
int j = md.length;
char str[] = new char[j * 2];
int k = 0;
for (int i = 0; i < j; i++) {
byte byte0 = md[i];
str[k++] = hexDigits[byte0 >>> 4 & 0xf];
str[k++] = hexDigits[byte0 & 0xf];
}
return new String(str);
} catch (Exception e) {
return null;
}
}
/**
* @param str 需要被加密的字符串
* @return 对字符串str进行SHA-1加密后，将加密字符串返回
*/
public static String EncryptBySHA1(String str) {
return EncryptStr(str, "SHA-1");
}
/**
* @param str 需要被加密的字符串
* @return 对字符串str进行SHA-256加密后，将加密字符串返回
*/
public static String EncryptBySHA256(String str) {
return EncryptStr(str, "SHA-256");
}
/**
* @param bts
* @return
*/
private static String bytes2Hex(byte[] bts) {
String des = "";
String tmp = null;
for (int i = 0; i < bts.length; i++) {
tmp = (Integer.toHexString(bts[i] & 0xFF));
if (tmp.length() == 1) {
des += "0";
}
des += tmp;
}
return des;
}
/**
* @param str
* @param key
* @return
*/
public static String union(String str, String key) {
int strLen = str.length();
int keyLen = key.length();
Character[] s = new Character[strLen + keyLen];
boolean flag = true;
int strIdx = 0;
int keyIdx = 0;
for (int i = 0; i < s.length; i++) {
if (flag) {
if (strIdx < strLen) {
s[i] = str.charAt(strIdx);
strIdx++;
}
if (keyIdx < keyLen) {
flag = false;
}
} else {
if (keyIdx < keyLen) {
s[i] = key.charAt(keyIdx);
keyIdx++;
}
if (strIdx < strLen) {
flag = true;
}
}
}
return StringUtils.join(s);
}
/**
* 加密str
*
* @param str 要加密的字符串
* @param key 加密的key
* @return
*/
public static String encrypt(String str, String key) {
if (str == null || str.length() == 0 || StringUtils.isBlank(key)) {
return encrypt(str);
}
return encrypt(union(str, key));
}
/**
* 先将str进行一次MD5加密，加密后再取加密后的字符串的第1、3、5个字符追加到加密串，再拿这个加密串进行加密
*
* @param str
* @return
* @throws NoSuchAlgorithmException
* @throws DigestException
*/
public static String encrypt(String str) {
String encryptStr = EncryptByMD5(str);
if (encryptStr != null) {
encryptStr = encryptStr + encryptStr.charAt(0) + encryptStr.charAt(2) + encryptStr.charAt(4);
encryptStr = EncryptByMD5(encryptStr);
}
return encryptStr;
}
}
public class SL_Busi {
public static void business(int i) {
try {
Thread.sleep(i);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
public class SL_QuestionBank {
private static ConcurrentHashMap<Integer, QuestionDbVo> questionBankMap = new ConcurrentHashMap<Integer, QuestionDbVo>();
private static ScheduledExecutorService updateQuestionBank = new ScheduledThreadPoolExecutor(Consts.CPU_COUNT * 2);
public static void initBank() {
for (int i = 0; i < Consts.SIZE_OF_QUESTION_BANK; i++) {
String questionContent = makeQuestionDetail(Consts.QUESTION_LENGTH);
questionBankMap.put(i, new QuestionDbVo(i, questionContent, EncryptUtils.EncryptBySHA1(questionContent)));
}
updateQuestionTimer();
}
private static void updateQuestionTimer() {
System.out.println("开始更新题库..............");
updateQuestionBank.scheduleAtFixedRate(new updateBank(), 15, 5, TimeUnit.SECONDS);
}
private static String makeQuestionDetail(int questionLength) {
String base = "abcdefghijklmnopqrstuvwxyz0123456789";
Random random = new Random();
StringBuffer sb = new StringBuffer();
for (int i = 0; i < questionLength; i++) {
int number = random.nextInt(base.length());
sb.append(base.charAt(number));
}
return sb.toString();
}
public static QuestionDbVo getQuestion(Integer questionId) {
SL_Busi.business(20);
return questionBankMap.get(questionId);
}
private static class updateBank implements Runnable {
@Override
public void run() {
Random random = new Random();
int questionId = random.nextInt(Consts.SIZE_OF_QUESTION_BANK);
String questionContent = makeQuestionDetail(Consts.QUESTION_LENGTH);
questionBankMap.put(questionId, new QuestionDbVo(questionId, questionContent, EncryptUtils.EncryptBySHA1(questionContent)));
System.out.println("题目【" + questionId + "】被跟新");
}
}
public static String getSha(int i) {
SL_Busi.business(10);
return questionBankMap.get(i).getSha();
}
}
public class BaseQuestionPorcesser {
public static String makeQuestion(Integer questionId, String detail) {
Random random = new Random();
SL_Busi.business(450 + random.nextInt(100));
return "CompleteQuestion[id=" + questionId + ",content=:" + detail + "]";
}
}
public class ParallerQstService {
private static ConcurrentHashMap<Integer, QuestionCacheVo> questionCache = new ConcurrentHashMap<>();
private static ConcurrentHashMap<Integer, Future<QuestionCacheVo>> processingQuestionCache = new ConcurrentHashMap<>();
private static ExecutorService makeQuestionService = Executors.newFixedThreadPool(Consts.CPU_COUNT * 2);
public static TaskResultVo makeQuestion(Integer questionId) {
QuestionCacheVo questionCacheVo = questionCache.get(questionId);
if (null == questionCacheVo) {
System.out.println("........题目" + questionId + "在缓存中不存在，准备启动");
return new TaskResultVo(getQstFuture(questionId));
} else {
String sha = SL_QuestionBank.getSha(questionId);
if (sha.equals(questionCache.get(questionId).getQuestionSha())) {
System.out.println("........题目" + questionId + "在缓存中存在，且为变化");
return new TaskResultVo(questionCacheVo.getQuestionDetail());
} else {
System.out.println("........题目" + questionId + "在缓存中存在，且发生变化");
return new TaskResultVo(getQstFuture(questionId));
}
}
}
private static Future<QuestionCacheVo> getQstFuture(Integer questionId) {
Future<QuestionCacheVo> questionCacheVoFuture = processingQuestionCache.get(questionId);
try {
if (null == questionCacheVoFuture) {
QuestionDbVo question = SL_QuestionBank.getQuestion(questionId);
QuestionTask questionTask = new QuestionTask(question, questionId);
FutureTask<QuestionCacheVo> ft = new FutureTask<>(questionTask);
questionCacheVoFuture = processingQuestionCache.putIfAbsent(questionId, ft);
if (null == questionCacheVoFuture) {
questionCacheVoFuture = ft;
System.out.println("有其他的线程刚刚启动了题目" + questionId + "的任务，本任务无需启动");
makeQuestionService.execute(ft);
} else {
System.out.println("成功启动了题目" + questionId + "的任务------------");
}
} else {
System.out.println("........题目" + questionId + "一存在任务去计算，无需重新生成");
}
} catch (Exception e) {
processingQuestionCache.remove(questionId);
throw e;
}
return questionCacheVoFuture;
}
private static class QuestionTask implements Callable<QuestionCacheVo> {
private final Integer questionId;
private final QuestionDbVo question;
public QuestionTask(QuestionDbVo question, Integer questionId) {
this.questionId = questionId;
this.question = question;
}
@Override
public QuestionCacheVo call() throws Exception {
try {
String s = BaseQuestionPorcesser.makeQuestion(questionId, SL_QuestionBank.getQuestion(questionId).getDetail());
String sha = question.getSha();
QuestionCacheVo questionCacheVo = new QuestionCacheVo(s, sha);
questionCache.put(questionId, questionCacheVo);
return questionCacheVo;
} finally {
processingQuestionCache.remove(questionId);
}
}
}
}
public class SingleQstService {
public static String makeQuestion(Integer questionId) {
return BaseQuestionPorcesser.makeQuestion(questionId, SL_QuestionBank.getQuestion(questionId).getDetail());
}
}
public class ProduceDocService {
public static String makeDoc(SrcDocVo doc) {
System.out.println("开始处理文档" + doc.getDocName());
StringBuffer sb = new StringBuffer();
for (Integer questionId : doc.getQuestionList()
) {
sb.append(SingleQstService.makeQuestion(questionId));
}
return "?complete_" + System.currentTimeMillis() + "_" + doc.getDocName() + ".pdf";
}
public static String uploadDoc(String loaclName) {
Random random = new Random();
SL_Busi.business(9000 + random.nextInt(400));
return "https://ww.baidu.com" + loaclName;
}
public static String makeDocAsync(SrcDocVo srcDocVo) throws ExecutionException, InterruptedException {
System.out.println("开始处理文档" + srcDocVo.getDocName());
Map<Integer, TaskResultVo> qstResultMap = new HashMap<>();
for (Integer questionId : srcDocVo.getQuestionList()
) {
qstResultMap.put(questionId, ParallerQstService.makeQuestion(questionId));
}
StringBuffer sb = new StringBuffer();
for (Integer questionId : srcDocVo.getQuestionList()
) {
TaskResultVo taskResultVo = qstResultMap.get(questionId);
sb.append(null == taskResultVo.getQuestionDetail() ?
taskResultVo.getQuestionFuture().get().getQuestionDetail()
: taskResultVo.getQuestionDetail());
}
return "?complete_" + System.currentTimeMillis() + "_" + srcDocVo.getDocName() + ".pdf";
}
}
public class QuestionCacheVo {
private final String questionDetail;
private final String questionSha;
public QuestionCacheVo(String questionDetail, String questionSha) {
this.questionDetail = questionDetail;
this.questionSha = questionSha;
}
public String getQuestionDetail() {
return questionDetail;
}
public String getQuestionSha() {
return questionSha;
}
}
public class QuestionDbVo {
private final int id;
private final String detail;
private final String sha;
public QuestionDbVo(int id, String detail, String sha) {
this.id = id;
this.detail = detail;
this.sha = sha;
}
public int getId() {
return id;
}
public String getDetail() {
return detail;
}
public String getSha() {
return sha;
}
}
public class SrcDocVo {
private final String docName;
private final List<Integer> questionList;
public SrcDocVo(String docName, List<Integer> questionList) {
this.docName = docName;
this.questionList = questionList;
}
public String getDocName() {
return docName;
}
public List<Integer> getQuestionList() {
return questionList;
}
}
public class TaskResultVo {
private final String questionDetail;
private final Future<QuestionCacheVo> questionFuture;
public TaskResultVo(String questionDetail, Future<QuestionCacheVo> questionFuture) {
this.questionDetail = questionDetail;
this.questionFuture = questionFuture;
}
public TaskResultVo(String questionDetail) {
this.questionDetail = questionDetail;
this.questionFuture = null;
}
public TaskResultVo(Future<QuestionCacheVo> questionFuture) {
this.questionFuture = questionFuture;
this.questionDetail = null;
}
public String getQuestionDetail() {
return questionDetail;
}
public Future<QuestionCacheVo> getQuestionFuture() {
return questionFuture;
}
}
public class RpcModeWeb {
private static ExecutorService docMakeService = Executors.newFixedThreadPool(Consts.CPU_COUNT * 2);
private static ExecutorService docUploadService = Executors.newFixedThreadPool(Consts.CPU_COUNT * 2);
private static CompletionService<String> docCs = new ExecutorCompletionService<>(docMakeService);
private static CompletionService<String> docUploadCs = new ExecutorCompletionService<>(docUploadService);
public static void main(String[] args) throws InterruptedException, ExecutionException {
System.out.println("题库开始初始化");
SL_QuestionBank.initBank();
System.out.println("题库初始化完成");
List<SrcDocVo> docList = CreatePendingDocs.makePendingDoc(60);
long startTotal = System.currentTimeMillis();
for (SrcDocVo doc : docList
) {
docCs.submit(new MakeDocTask(doc));
}
for (SrcDocVo doc : docList
) {
Future<String> take = docCs.take();
docUploadCs.submit(new UploadDocTask(take.get()));
}
for (SrcDocVo doc : docList
) {
docUploadCs.take().get();
}
System.out.println("-------共耗时" + (System.currentTimeMillis() - startTotal) + "-------");
}
private static class MakeDocTask implements Callable<String> {
private SrcDocVo srcDocVo;
private MakeDocTask(SrcDocVo srcDocVo) {
this.srcDocVo = srcDocVo;
}
@Override
public String call() throws Exception {
long start = System.currentTimeMillis();
// String localName = ProduceDocService.makeDoc(srcDocVo);
String localName = ProduceDocService.makeDocAsync(srcDocVo);
System.out.println("文档" + localName + "生成耗时：" + (System.currentTimeMillis() - start) + "ms");
return localName;
}
}
private static class UploadDocTask implements Callable<String> {
private String filePath;
private UploadDocTask(String filePath) {
this.filePath = filePath;
}
@Override
public String call() throws Exception {
long start = System.currentTimeMillis();
String remoteUrl = ProduceDocService.uploadDoc(filePath);
System.out.println("已上传至" + remoteUrl + "耗时：" + (System.currentTimeMillis() - start) + "ms");
return remoteUrl;
}
}
}
public class SingleWeb {
public static void main(String[] args) {
System.out.println("题库开始初始化");
SL_QuestionBank.initBank();
System.out.println("题库初始化完成");
List<SrcDocVo> docList = CreatePendingDocs.makePendingDoc(2);
long startTotal = System.currentTimeMillis();
for (SrcDocVo doc : docList
) {
System.out.println("开始处理文档：" + doc.getDocName() + "........");
long start = System.currentTimeMillis();
String localName = ProduceDocService.makeDoc(doc);
System.out.println("文档" + localName + "生成耗时：" + (System.currentTimeMillis() - start) + "ms");
start = System.currentTimeMillis();
String remoteUrl = ProduceDocService.uploadDoc(localName);
System.out.println("已上传至" + remoteUrl + "耗时：" + (System.currentTimeMillis() - start) + "ms");
}
System.out.println("-------共耗时" + (System.currentTimeMillis() - startTotal) + "-------");
}
}
