Apollo源码阅读（六）Apollo服务端-配置发布流程

一、配置发布整体流程

• Portal点击发布配置
• 调用AdminService持久化配置
• ConfigService扫描数据库里的更新的配置，通知客户端
• 客户端RemoteConfigRepository.onLongPollNotified获取到配置更新消息，通过trySync请求ConfigService获取配置

二、Portal发布配置

• com.ctrip.framework.apollo.portal.controller.ReleaseController#createRelease

@PreAuthorize(value = "@permissionValidator.hasReleaseNamespacePermission(#appId, #namespaceName, #env)")
@PostMapping(value = "/apps/{appId}/envs/{env}/clusters/{clusterName}/namespaces/{namespaceName}/releases")
public ReleaseDTO createRelease(@PathVariable String appId,
                                @PathVariable String env, @PathVariable String clusterName,
                                @PathVariable String namespaceName, @RequestBody NamespaceReleaseModel model) {
  // 紧急发布
  if (model.isEmergencyPublish() && !portalConfig.isEmergencyPublishAllowed(Env.valueOf(env))) {
    throw new BadRequestException(String.format("Env: %s is not supported emergency publish now", env));
  }
  // 这里调用AdminService发布配置
  ReleaseDTO createdRelease = releaseService.publish(model);
  // 发布ConfigPublishEvent事件，发送邮件
  ConfigPublishEvent event = ConfigPublishEvent.instance();
  publisher.publishEvent(event);
  return createdRelease;
}

• ReleaseService#publish

 public ReleaseDTO publish(NamespaceReleaseModel model) {
    Env env = model.getEnv();
    boolean isEmergencyPublish = model.isEmergencyPublish();
    String appId = model.getAppId();
    String clusterName = model.getClusterName();
    String namespaceName = model.getNamespaceName();
    String releaseBy = StringUtils.isEmpty(model.getReleasedBy()) ?
                       userInfoHolder.getUser().getUserId() : model.getReleasedBy();
    // 发布配置
    ReleaseDTO releaseDTO = releaseAPI.createRelease(appId, env, clusterName, namespaceName,
                                                     model.getReleaseTitle(), model.getReleaseComment(),
                                                     releaseBy, isEmergencyPublish);

    return releaseDTO;
  }

• AdminServiceAPI.ReleaseAPI#createRelease调用AdminService

public ReleaseDTO createRelease(String appId, Env env, String clusterName, String namespace,
       String releaseName, String releaseComment, String operator,
       boolean isEmergencyPublish) {
     HttpHeaders headers = new HttpHeaders();
     headers.setContentType(MediaType.parseMediaType(MediaType.APPLICATION_FORM_URLENCODED_VALUE + ";charset=UTF-8"));
     MultiValueMap<String, String> parameters = new LinkedMultiValueMap<>();
     parameters.add("name", releaseName);
     parameters.add("comment", releaseComment);
     parameters.add("operator", operator);
     parameters.add("isEmergencyPublish", String.valueOf(isEmergencyPublish));
     HttpEntity<MultiValueMap<String, String>> entity =
         new HttpEntity<>(parameters, headers);
     // 使用Apollo自己封装的RestTemplate
     ReleaseDTO response = restTemplate.post(
         env, "apps/{appId}/clusters/{clusterName}/namespaces/{namespaceName}/releases", entity,
         ReleaseDTO.class, appId, clusterName, namespace);
     return response;
   }

• RetryableRestTemplate#execute支持故障转移的RestTemplate

private <T> T execute(HttpMethod method, Env env, String path, Object request, Class<T> responseType,
                       Object... uriVariables) {
   if (path.startsWith("/")) {
     path = path.substring(1, path.length());
   }
   String uri = uriTemplateHandler.expand(path, uriVariables).getPath();
   // 1. 通过MetaServer(实际里面就是Eureka的服务发现)获取所有AdminService
   List<ServiceDTO> services = getAdminServices(env, ct);
   // 2. 循环服务列表执行请求
   for (ServiceDTO serviceDTO : services) {
     try {
       T result = doExecute(method, serviceDTO, path, request, responseType, uriVariables);
       return result;
     } catch (Throwable t) {
       if (canRetry(t, method)) { // 3. 如果满足条件，支持故障转移
         Tracer.logEvent(TracerEventType.API_RETRY, uri);
       } else {//biz exception rethrow
         throw t;
       }
     }
   }
   // 4. 所有AdminService都挂了抛出异常
   ServiceException e =
       new ServiceException(String.format("Admin servers are unresponsive. meta server address: %s, admin servers: %s",
               portalMetaDomainService.getDomain(env), services));
   throw e;
 }
  // post,delete,put请求在admin server处理超时情况下不做重试
 private boolean canRetry(Throwable e, HttpMethod method) {
   Throwable nestedException = e.getCause();
   if (method == HttpMethod.GET) {
     return nestedException instanceof SocketTimeoutException
            || nestedException instanceof HttpHostConnectException
            || nestedException instanceof ConnectTimeoutException;
   }
   return nestedException instanceof HttpHostConnectException
          || nestedException instanceof ConnectTimeoutException;
 }
 

## 三、AdminService持久化配置
### 3-1 主要流程

@Transactional @PostMapping("/apps/{appId}/clusters/{clusterName}/namespaces/{namespaceName}/releases") public ReleaseDTO publish(@PathVariable("appId") String appId, @PathVariable("clusterName") String clusterName, @PathVariable("namespaceName") String namespaceName, @RequestParam("name") String releaseName, @RequestParam(name = "comment", required = false) String releaseComment, @RequestParam("operator") String operator, @RequestParam(name = "isEmergencyPublish", defaultValue = "false") boolean isEmergencyPublish) { // 1. 查询namespace Namespace namespace = namespaceService.findOne(appId, clusterName, namespaceName); if (namespace == null) { throw new NotFoundException(); } // 2. 保存Release和ReleaseHistory Release release = releaseService.publish(namespace, releaseName, releaseComment, operator, isEmergencyPublish);

// 3. 灰度发布相关 - 如果是灰度发布，使用父namespace的clusterName Namespace parentNamespace = namespaceService.findParentNamespace(namespace); String messageCluster; if (parentNamespace != null) { messageCluster = parentNamespace.getClusterName(); } else { messageCluster = clusterName; } // 4. 保存ReleaseMessage，发布消息 messageSender.sendMessage(ReleaseMessageKeyGenerator.generate(appId, messageCluster, namespaceName), Topics.APOLLO_RELEASE_TOPIC); return BeanUtils.transform(ReleaseDTO.class, release); }

### 3-2 MessageSender(DatabaseMessageSender)的作用
- 持久化Message

@Override @Transactional public void sendMessage(String message, String channel) { // 1. 持久化 ReleaseMessage newMessage = releaseMessageRepository.save(new ReleaseMessage(message)); // 2. 把message的id放入一个阻塞队列 toClean.offer(newMessage.getId()); }

- 一个定时任务，清理非最新的message

@PostConstruct private void initialize() { cleanExecutorService.submit(() -> { while (!cleanStopped.get() && !Thread.currentThread().isInterrupted()) { try { Long rm = toClean.poll(1, TimeUnit.SECONDS); if (rm != null) { cleanMessage(rm); // 执行清理 } else { TimeUnit.SECONDS.sleep(5); // 5秒扫描一次 } } catch (Throwable ex) { Tracer.logError(ex); } } }); }

private void cleanMessage(Long id) { boolean hasMore = true; // 1. 根据id查询ReleaseMessage是否存在 ReleaseMessage releaseMessage = releaseMessageRepository.findById(id).orElse(null); if (releaseMessage == null) { return; } // 2. 对于相同 appId+clusterName+namespace 只保留最新的id的一条数据 while (hasMore && !Thread.currentThread().isInterrupted()) { List messages = releaseMessageRepository.findFirst100ByMessageAndIdLessThanOrderByIdAsc( releaseMessage.getMessage(), releaseMessage.getId()); releaseMessageRepository.deleteAll(messages); hasMore = messages.size() == 100; } }

  
## 四、ConfigService通知客户端
### 4-1 ReleaseMessageScanner扫描器注入，添加一些listener

@Bean public ReleaseMessageScanner releaseMessageScanner() { ReleaseMessageScanner releaseMessageScanner = new ReleaseMessageScanner(); //0. handle release message cache releaseMessageScanner.addMessageListener(releaseMessageServiceWithCache); //1. handle gray release rule releaseMessageScanner.addMessageListener(grayReleaseRulesHolder); //2. handle server cache releaseMessageScanner.addMessageListener(configService); releaseMessageScanner.addMessageListener(configFileController); //3. notify clients notificationControllerV2就是通知客户端配置发生更新的listener releaseMessageScanner.addMessageListener(notificationControllerV2); releaseMessageScanner.addMessageListener(notificationController); return releaseMessageScanner; }

### 4-2 ReleaseMessageScanner的初始化

@Override public void afterPropertiesSet() throws Exception { // 1. 定时任务时间间隔 默认1s databaseScanInterval = bizConfig.releaseMessageScanIntervalInMilli(); // 2. 获取ReleaseMessage中最大的id maxIdScanned = loadLargestMessageId(); // 3. 启动定时任务，每秒拉取需要通知的ReleaseMessage，通知所有ReleaseMessageListener executorService.scheduleWithFixedDelay((Runnable) () -> { scanMessages(); }, databaseScanInterval, databaseScanInterval, TimeUnit.MILLISECONDS);

}

  
### 4-3 ReleaseMessageScanner定时扫描逻辑

// 1 扫描比目前内存记录的id大的记录，500一批次 List releaseMessages = releaseMessageRepository.findFirst500ByIdGreaterThanOrderByIdAsc(maxIdScanned); if (CollectionUtils.isEmpty(releaseMessages)) { return false; } // 2 通知所有Listener // NotificationControllerV2.handleMessage 响应有配置变化的apollo客户端 fireMessageScanned(releaseMessages); int messageScanned = releaseMessages.size(); // 3 更新最大messageId maxIdScanned = releaseMessages.get(messageScanned - 1).getId(); return messageScanned == 500;

  
### 4-4 客户端LongPolling对应的ConfigService里的Controller

@GetMapping public DeferredResult<ResponseEntity<List>> pollNotification( @RequestParam(value = "appId") String appId, @RequestParam(value = "cluster") String cluster, @RequestParam(value = "notifications") String notificationsAsString, @RequestParam(value = "dataCenter", required = false) String dataCenter, @RequestParam(value = "ip", required = false) String clientIp) { List notifications = gson.fromJson(notificationsAsString, notificationsTypeReference); // DeferredResult包装 DeferredResultWrapper deferredResultWrapper = new DeferredResultWrapper(bizConfig.longPollingTimeoutInMilli()); Set namespaces = Sets.newHashSet(); Map<String, Long> clientSideNotifications = Maps.newHashMap(); Map<String, ApolloConfigNotification> filteredNotifications = filterNotifications(appId, notifications);

for (Map.Entry<String, ApolloConfigNotification> notificationEntry : filteredNotifications.entrySet()) { String normalizedNamespace = notificationEntry.getKey(); ApolloConfigNotification notification = notificationEntry.getValue(); namespaces.add(normalizedNamespace); clientSideNotifications.put(normalizedNamespace, notification.getNotificationId()); if (!Objects.equals(notification.getNamespaceName(), normalizedNamespace)) { deferredResultWrapper.recordNamespaceNameNormalizedResult(notification.getNamespaceName(), normalizedNamespace); } }

// 0. 根据appId, cluster, namespaces, dataCenter获取需要监听的配置 // key是namespace value是所有配置的key // 里面也包含了所有public的配置 Multimap<String, String> watchedKeysMap = watchKeysUtil.assembleAllWatchKeys(appId, cluster, namespaces, dataCenter);

Set watchedKeys = Sets.newHashSet(watchedKeysMap.values());

// 1. 设置DeferredResult超时后的处理方式 deferredResultWrapper .onTimeout(() -> logWatchedKeys(watchedKeys, "Apollo.LongPoll.TimeOutKeys")); // 2. 设置DeferredResult处理完成后的处理方法，移除所有watchKeys deferredResultWrapper.onCompletion(() -> { for (String key : watchedKeys) { deferredResults.remove(key, deferredResultWrapper); } logWatchedKeys(watchedKeys, "Apollo.LongPoll.CompletedKeys"); });

// 3. 向内存中的一个Map放入DeferredResult，等待其他线程执行setResult才会返回客户端，key是配置的key for (String key : watchedKeys) { this.deferredResults.put(key, deferredResultWrapper); }

List latestReleaseMessages = releaseMessageService.findLatestReleaseMessagesGroupByMessages(watchedKeys);

// 主动关闭数据库连接 entityManagerUtil.closeEntityManager(); // 获得新的 ApolloConfigNotification 通知数组 List newNotifications = getApolloConfigNotifications(namespaces, clientSideNotifications, watchedKeysMap, latestReleaseMessages); // 若有新的通知，直接设置结果 if (!CollectionUtils.isEmpty(newNotifications)) { deferredResultWrapper.setResult(newNotifications); }

return deferredResultWrapper.getResult(); }

 
 ### 4-5 对于没有最新通知的请求，是通过NotificationControllerV2.handleMessage设置Result的

@Override public void handleMessage(ReleaseMessage message, String channel) { // content = appId + cluster + namespace String content = message.getMessage(); // 获取namespace String changedNamespace = retrieveNamespaceFromReleaseMessage.apply(content); // 如果对于appId + cluster + namespace没有DeferredResult，直接返回 if (!deferredResults.containsKey(content)) { return; } // 创建一个新的list防止ConcurrentModificationException List results = Lists.newArrayList(deferredResults.get(content)); ApolloConfigNotification configNotification = new ApolloConfigNotification(changedNamespace, message.getId()); configNotification.addMessage(content, message.getId());

// 如果有很多客户端在等待配置通知，分批异步设置DeferredResult的result，防止太多客户端收到配置变化通知，同时请求ConfigService if (results.size() > bizConfig.releaseMessageNotificationBatch()) { largeNotificationBatchExecutorService.submit(() -> { for (int i = 0; i < results.size(); i++) { if (i > 0 && i % bizConfig.releaseMessageNotificationBatch() == 0) { TimeUnit.MILLISECONDS.sleep(bizConfig.releaseMessageNotificationBatchIntervalInMilli()); // 这里完成后，才会返回长轮询client results.get(i).setResult(configNotification); } }); return; } for (DeferredResultWrapper result : results) { // 这里完成后，才会返回长轮询client result.setResult(configNotification); } }

  
  
## 五、客户端更新配置
https://juejin.cn/post/6869315864982716423