量化平台中的实时行情处理架构设计

ekuoleung
0 阅读8分钟

实时行情处理是量化平台的核心基础设施之一。从数据源接入、数据清洗、数据聚合到数据分发，每个环节都需要精心设计，以确保数据的实时性、准确性和可靠性。

结论先行：EasyQuant 的实时行情处理采用微服务架构，支持多数据源接入、数据清洗、多周期聚合、实时分发，确保行情数据的高效处理和低延迟传输。

一、行情处理架构

1）行情处理流程

mermaid-drawing-2.png

2）核心组件

// market/MarketDataService.java

@Service
public class MarketDataService {
    
    private final List<MarketDataConnector> connectors;
    private final DataCleaner dataCleaner;
    private final BarAggregator barAggregator;
    private final DataStorage dataStorage;
    private final DataDistributor dataDistributor;
    
    /**
     * 启动行情服务
     */
    public void start() {
        // 启动所有连接器
        connectors.forEach(MarketDataConnector::connect);
        
        // 启动数据聚合器
        barAggregator.start();
        
        // 启动数据分发器
        dataDistributor.start();
    }
    
    /**
     * 停止行情服务
     */
    public void stop() {
        // 停止数据分发器
        dataDistributor.stop();
        
        // 停止数据聚合器
        barAggregator.stop();
        
        // 停止所有连接器
        connectors.forEach(MarketDataConnector::disconnect);
    }
}

二、行情接入

1）行情连接器接口

// market/MarketDataConnector.java

public interface MarketDataConnector {
    
    /**
     * 连接数据源
     */
    void connect();
    
    /**
     * 断开连接
     */
    void disconnect();
    
    /**
     * 是否已连接
     */
    boolean isConnected();
    
    /**
     * 订阅行情
     */
    void subscribe(List<String> symbols);
    
    /**
     * 取消订阅
     */
    void unsubscribe(List<String> symbols);
    
    /**
     * 添加行情监听器
     */
    void addListener(MarketDataListener listener);
    
    /**
     * 移除行情监听器
     */
    void removeListener(MarketDataListener listener);
}

// market/MarketDataListener.java

public interface MarketDataListener {
    
    /**
     * 收到 Tick
     */
    void onTick(Tick tick);
    
    /**
     * 收到 K 线
     */
    void onBar(Bar bar);
    
    /**
     * 连接状态变化
     */
    void onConnectionStateChanged(boolean connected);
}

2）WebSocket 行情连接器

// market/WebSocketMarketDataConnector.java

@Component
public class WebSocketMarketDataConnector implements MarketDataConnector {
    
    private final String url;
    private final String apiKey;
    
    private WebSocketSession session;
    private final List<MarketDataListener> listeners = new CopyOnWriteArrayList<>();
    private final Set<String> subscribedSymbols = ConcurrentHashMap.newKeySet();
    
    public WebSocketMarketDataConnector(
        @Value("${market.ws.url}") String url,
        @Value("${market.ws.api-key}") String apiKey
    ) {
        this.url = url;
        this.apiKey = apiKey;
    }
    
    @Override
    public void connect() {
        try {
            WebSocketContainer container = ContainerProvider.getWebSocketContainer();
            session = container.connectToServer(new Endpoint() {
                @Override
                public void onOpen(Session session, EndpointConfig config) {
                    log.info("WebSocket 连接成功");
                    notifyConnectionStateChanged(true);
                }
                
                @Override
                public void onClose(Session session, CloseReason closeReason) {
                    log.warn("WebSocket 连接关闭: {}", closeReason);
                    notifyConnectionStateChanged(false);
                }
                
                @Override
                public void onError(Session session, Throwable thr) {
                    log.error("WebSocket 错误", thr);
                    notifyConnectionStateChanged(false);
                }
            }, URI.create(url + "?api_key=" + apiKey));
            
            session.addMessageHandler(new MessageHandler.Whole<String>() {
                @Override
                public void onMessage(String message) {
                    handleMarketData(message);
                }
            });
            
        } catch (Exception e) {
            log.error("WebSocket 连接失败", e);
            throw new RuntimeException("WebSocket 连接失败", e);
        }
    }
    
    @Override
    public void disconnect() {
        if (session != null && session.isOpen()) {
            try {
                session.close();
            } catch (Exception e) {
                log.error("关闭 WebSocket 连接失败", e);
            }
        }
    }
    
    @Override
    public boolean isConnected() {
        return session != null && session.isOpen();
    }
    
    @Override
    public void subscribe(List<String> symbols) {
        if (!isConnected()) {
            throw new IllegalStateException("未连接");
        }
        
        symbols.forEach(subscribedSymbols::add);
        
        // 发送订阅请求
        try {
            String subscribeMessage = buildSubscribeMessage(symbols);
            session.getBasicRemote().sendText(subscribeMessage);
        } catch (Exception e) {
            log.error("发送订阅请求失败", e);
        }
    }
    
    @Override
    public void unsubscribe(List<String> symbols) {
        if (!isConnected()) {
            throw new IllegalStateException("未连接");
        }
        
        symbols.forEach(subscribedSymbols::remove);
        
        // 发送取消订阅请求
        try {
            String unsubscribeMessage = buildUnsubscribeMessage(symbols);
            session.getBasicRemote().sendText(unsubscribeMessage);
        } catch (Exception e) {
            log.error("发送取消订阅请求失败", e);
        }
    }
    
    @Override
    public void addListener(MarketDataListener listener) {
        listeners.add(listener);
    }
    
    @Override
    public void removeListener(MarketDataListener listener) {
        listeners.remove(listener);
    }
    
    private void handleMarketData(String message) {
        try {
            // 解析行情数据
            MarketData marketData = parseMarketData(message);
            
            // 通知监听器
            if (marketData instanceof Tick) {
                listeners.forEach(l -> l.onTick((Tick) marketData));
            } else if (marketData instanceof Bar) {
                listeners.forEach(l -> l.onBar((Bar) marketData));
            }
            
        } catch (Exception e) {
            log.error("处理行情数据失败: {}", message, e);
        }
    }
    
    private MarketData parseMarketData(String message) {
        // 解析 JSON 消息
        JsonObject json = JsonParser.parseString(message).getAsJsonObject();
        
        String type = json.get("type").getAsString();
        
        if ("tick".equals(type)) {
            return parseTick(json);
        } else if ("bar".equals(type)) {
            return parseBar(json);
        }
        
        throw new IllegalArgumentException("未知行情类型: " + type);
    }
    
    private Tick parseTick(JsonObject json) {
        return Tick.builder()
            .symbol(json.get("symbol").getAsString())
            .exchange(json.get("exchange").getAsString())
            .timestamp(Instant.parse(json.get("timestamp").getAsString()))
            .price(json.get("price").getAsBigDecimal())
            .volume(json.get("volume").getAsLong())
            .side(json.get("side").getAsString())
            .build();
    }
    
    private Bar parseBar(JsonObject json) {
        return Bar.builder()
            .symbol(json.get("symbol").getAsString())
            .exchange(json.get("exchange").getAsString())
            .timestamp(Instant.parse(json.get("timestamp").getAsString()))
            .openPrice(json.get("open").getAsBigDecimal())
            .highPrice(json.get("high").getAsBigDecimal())
            .lowPrice(json.get("low").getAsBigDecimal())
            .closePrice(json.get("close").getAsBigDecimal())
            .volume(json.get("volume").getAsLong())
            .build();
    }
    
    private String buildSubscribeMessage(List<String> symbols) {
        JsonObject message = new JsonObject();
        message.addProperty("action", "subscribe");
        message.add("symbols", new Gson().toJsonTree(symbols));
        return message.toString();
    }
    
    private String buildUnsubscribeMessage(List<String> symbols) {
        JsonObject message = new JsonObject();
        message.addProperty("action", "unsubscribe");
        message.add("symbols", new Gson().toJsonTree(symbols));
        return message.toString();
    }
    
    private void notifyConnectionStateChanged(boolean connected) {
        listeners.forEach(l -> l.onConnectionStateChanged(connected));
    }
}

三、数据清洗

1）数据清洗器

// market/DataCleaner.java

@Component
public class DataCleaner {
    
    private final List<DataValidator> validators;
    
    public DataCleaner(List<DataValidator> validators) {
        this.validators = validators;
    }
    
    /**
     * 清洗 Tick 数据
     */
    public Tick cleanTick(Tick tick) {
        // 验证数据
        for (DataValidator validator : validators) {
            ValidationResult result = validator.validate(tick);
            if (!result.isValid()) {
                log.warn("Tick 数据验证失败: {} - {}", 
                    tick.getSymbol(), result.getMessage());
                return null;
            }
        }
        
        // 标准化数据
        return normalizeTick(tick);
    }
    
    /**
     * 清洗 Bar 数据
     */
    public Bar cleanBar(Bar bar) {
        // 验证数据
        for (DataValidator validator : validators) {
            ValidationResult result = validator.validate(bar);
            if (!result.isValid()) {
                log.warn("Bar 数据验证失败: {} - {}", 
                    bar.getSymbol(), result.getMessage());
                return null;
            }
        }
        
        // 标准化数据
        return normalizeBar(bar);
    }
    
    private Tick normalizeTick(Tick tick) {
        // 标准化时间戳
        Instant normalizedTimestamp = normalizeTimestamp(tick.getTimestamp());
        
        // 标准化价格精度
        BigDecimal normalizedPrice = normalizePrice(tick.getPrice());
        
        return tick.toBuilder()
            .timestamp(normalizedTimestamp)
            .price(normalizedPrice)
            .build();
    }
    
    private Bar normalizeBar(Bar bar) {
        // 标准化时间戳
        Instant normalizedTimestamp = normalizeTimestamp(bar.getTimestamp());
        
        // 标准化价格精度
        BigDecimal normalizedOpen = normalizePrice(bar.getOpenPrice());
        BigDecimal normalizedHigh = normalizePrice(bar.getHighPrice());
        BigDecimal normalizedLow = normalizePrice(bar.getLowPrice());
        BigDecimal normalizedClose = normalizePrice(bar.getClosePrice());
        
        return bar.toBuilder()
            .timestamp(normalizedTimestamp)
            .openPrice(normalizedOpen)
            .highPrice(normalizedHigh)
            .lowPrice(normalizedLow)
            .closePrice(normalizedClose)
            .build();
    }
    
    private Instant normalizeTimestamp(Instant timestamp) {
        // 统一时间戳精度到毫秒
        return timestamp.truncatedTo(ChronoUnit.MILLIS);
    }
    
    private BigDecimal normalizePrice(BigDecimal price) {
        // 统一价格精度到 8 位小数
        return price.setScale(8, RoundingMode.HALF_UP);
    }
}

2）数据验证器

// market/DataValidator.java

public interface DataValidator {
    
    /**
     * 验证数据
     */
    ValidationResult validate(MarketData data);
}

// market/ValidationResult.java

@Data
@AllArgsConstructor
public class ValidationResult {
    
    private boolean valid;
    private String message;
    
    public static ValidationResult pass() {
        return new ValidationResult(true, null);
    }
    
    public static ValidationResult fail(String message) {
        return new ValidationResult(false, message);
    }
}

// market/PriceValidator.java

@Component
public class PriceValidator implements DataValidator {
    
    @Override
    public ValidationResult validate(MarketData data) {
        if (data instanceof Tick) {
            return validateTick((Tick) data);
        } else if (data instanceof Bar) {
            return validateBar((Bar) data);
        }
        
        return ValidationResult.fail("未知数据类型");
    }
    
    private ValidationResult validateTick(Tick tick) {
        // 检查价格是否为正数
        if (tick.getPrice().compareTo(BigDecimal.ZERO) <= 0) {
            return ValidationResult.fail("价格必须为正数");
        }
        
        // 检查价格是否合理
        if (tick.getPrice().compareTo(new BigDecimal("1000000")) > 0) {
            return ValidationResult.fail("价格异常");
        }
        
        return ValidationResult.pass();
    }
    
    private ValidationResult validateBar(Bar bar) {
        // 检查价格是否为正数
        if (bar.getOpenPrice().compareTo(BigDecimal.ZERO) <= 0 ||
            bar.getHighPrice().compareTo(BigDecimal.ZERO) <= 0 ||
            bar.getLowPrice().compareTo(BigDecimal.ZERO) <= 0 ||
            bar.getClosePrice().compareTo(BigDecimal.ZERO) <= 0) {
            return ValidationResult.fail("价格必须为正数");
        }
        
        // 检查高低价关系
        if (bar.getHighPrice().compareTo(bar.getLowPrice()) < 0) {
            return ValidationResult.fail("最高价不能低于最低价");
        }
        
        // 检查收盘价是否在高低价之间
        if (bar.getClosePrice().compareTo(bar.getHighPrice()) > 0 ||
            bar.getClosePrice().compareTo(bar.getLowPrice()) < 0) {
            return ValidationResult.fail("收盘价必须在最高价和最低价之间");
        }
        
        return ValidationResult.pass();
    }
}

// market/VolumeValidator.java

@Component
public class VolumeValidator implements DataValidator {
    
    @Override
    public ValidationResult validate(MarketData data) {
        if (data instanceof Tick) {
            return validateTick((Tick) data);
        } else if (data instanceof Bar) {
            return validateBar((Bar) data);
        }
        
        return ValidationResult.fail("未知数据类型");
    }
    
    private ValidationResult validateTick(Tick tick) {
        // 检查成交量是否为非负数
        if (tick.getVolume() < 0) {
            return ValidationResult.fail("成交量不能为负数");
        }
        
        return ValidationResult.pass();
    }
    
    private ValidationResult validateBar(Bar bar) {
        // 检查成交量是否为非负数
        if (bar.getVolume() < 0) {
            return ValidationResult.fail("成交量不能为负数");
        }
        
        return ValidationResult.pass();
    }
}

四、数据聚合

1）K 线聚合器

// market/BarAggregator.java

@Component
public class BarAggregator {
    
    private final Map<String, Map<TimeFrame, BarBuffer>> barBuffers = new ConcurrentHashMap<>();
    private final ScheduledExecutorService executor;
    
    public BarAggregator() {
        this.executor = Executors.newScheduledThreadPool(4);
    }
    
    /**
     * 启动聚合器
     */
    public void start() {
        // 每分钟执行一次聚合
        executor.scheduleAtFixedRate(
            this::aggregateBars,
            0,
            1,
            TimeUnit.MINUTES
        );
    }
    
    /**
     * 停止聚合器
     */
    public void stop() {
        executor.shutdown();
    }
    
    /**
     * 添加 Tick 到聚合缓冲区
     */
    public void addTick(Tick tick) {
        String symbol = tick.getSymbol();
        
        // 获取或创建标的的缓冲区
        Map<TimeFrame, BarBuffer> symbolBuffers = barBuffers
            .computeIfAbsent(symbol, k -> new ConcurrentHashMap<>());
        
        // 为每个时间周期添加 Tick
        for (TimeFrame timeFrame : TimeFrame.values()) {
            BarBuffer buffer = symbolBuffers.computeIfAbsent(
                timeFrame,
                k -> new BarBuffer(timeFrame)
            );
            buffer.addTick(tick);
        }
    }
    
    /**
     * 聚合 K 线
     */
    private void aggregateBars() {
        barBuffers.forEach((symbol, timeFrameBuffers) -> {
            timeFrameBuffers.forEach((timeFrame, buffer) -> {
                Bar bar = buffer.aggregate();
                if (bar != null) {
                    // 发布 K 线
                    publishBar(bar);
                }
            });
        });
    }
    
    private void publishBar(Bar bar) {
        // 发布 K 线到消息总线
        // ...
    }
}

// market/BarBuffer.java

public class BarBuffer {
    
    private final TimeFrame timeFrame;
    private final List<Tick> ticks = new ArrayList<>();
    private Instant currentWindowStart;
    
    public BarBuffer(TimeFrame timeFrame) {
        this.timeFrame = timeFrame;
        this.currentWindowStart = calculateWindowStart(Instant.now());
    }
    
    /**
     * 添加 Tick
     */
    public synchronized void addTick(Tick tick) {
        Instant tickWindowStart = calculateWindowStart(tick.getTimestamp());
        
        // 检查是否需要切换窗口
        if (!tickWindowStart.equals(currentWindowStart)) {
            // 先聚合当前窗口
            Bar bar = aggregate();
            if (bar != null) {
                publishBar(bar);
            }
            
            // 切换到新窗口
            currentWindowStart = tickWindowStart;
            ticks.clear();
        }
        
        ticks.add(tick);
    }
    
    /**
     * 聚合 K 线
     */
    public synchronized Bar aggregate() {
        if (ticks.isEmpty()) {
            return null;
        }
        
        // 计算开盘价（第一个 Tick 的价格）
        BigDecimal openPrice = ticks.get(0).getPrice();
        
        // 计算最高价
        BigDecimal highPrice = ticks.stream()
            .map(Tick::getPrice)
            .max(BigDecimal::compareTo)
            .orElse(openPrice);
        
        // 计算最低价
        BigDecimal lowPrice = ticks.stream()
            .map(Tick::getPrice)
            .min(BigDecimal::compareTo)
            .orElse(openPrice);
        
        // 计算收盘价（最后一个 Tick 的价格）
        BigDecimal closePrice = ticks.get(ticks.size() - 1).getPrice();
        
        // 计算成交量
        long volume = ticks.stream()
            .mapToLong(Tick::getVolume)
            .sum();
        
        // 计算 K 线时间戳
        Instant timestamp = currentWindowStart.plus(
            timeFrame.getDuration().minusSeconds(1)
        );
        
        return Bar.builder()
            .symbol(ticks.get(0).getSymbol())
            .exchange(ticks.get(0).getExchange())
            .timestamp(timestamp)
            .openPrice(openPrice)
            .highPrice(highPrice)
            .lowPrice(lowPrice)
            .closePrice(closePrice)
            .volume(volume)
            .build();
    }
    
    private Instant calculateWindowStart(Instant timestamp) {
        long epochSecond = timestamp.getEpochSecond();
        long windowSeconds = timeFrame.getDuration().getSeconds();
        
        long windowStartSecond = (epochSecond / windowSeconds) * windowSeconds;
        
        return Instant.ofEpochSecond(windowStartSecond);
    }
}

// market/TimeFrame.java

public enum TimeFrame {
    
    TICK_1S("1s", Duration.ofSeconds(1)),
    TICK_5S("5s", Duration.ofSeconds(5)),
    TICK_30S("30s", Duration.ofSeconds(30)),
    MIN_1("1m", Duration.ofMinutes(1)),
    MIN_5("5m", Duration.ofMinutes(5)),
    MIN_15("15m", Duration.ofMinutes(15)),
    MIN_30("30m", Duration.ofMinutes(30)),
    HOUR_1("1h", Duration.ofHours(1)),
    HOUR_4("4h", Duration.ofHours(4)),
    DAY_1("1d", Duration.ofDays(1));
    
    private final String code;
    private final Duration duration;
    
    TimeFrame(String code, Duration duration) {
        this.code = code;
        this.duration = duration;
    }
    
    public String getCode() {
        return code;
    }
    
    public Duration getDuration() {
        return duration;
    }
}

五、数据存储

1）数据存储服务

// market/DataStorage.java

@Service
public class DataStorage {
    
    private final TickRepository tickRepository;
    private final BarRepository barRepository;
    
    /**
     * 存储 Tick
     */
    public void storeTick(Tick tick) {
        tickRepository.save(tick);
    }
    
    /**
     * 批量存储 Tick
     */
    public void storeTicks(List<Tick> ticks) {
        tickRepository.saveAll(ticks);
    }
    
    /**
     * 存储 Bar
     */
    public void storeBar(Bar bar) {
        barRepository.save(bar);
    }
    
    /**
     * 批量存储 Bar
     */
    public void storeBars(List<Bar> bars) {
        barRepository.saveAll(bars);
    }
    
    /**
     * 查询最新 Tick
     */
    public List<Tick> getLatestTicks(String symbol, int limit) {
        return tickRepository.findLatestBySymbol(symbol, limit);
    }
    
    /**
     * 查询最新 Bar
     */
    public List<Bar> getLatestBars(String symbol, TimeFrame timeFrame, int limit) {
        return barRepository.findLatestBySymbolAndTimeFrame(symbol, timeFrame, limit);
    }
    
    /**
     * 查询历史 Bar
     */
    public List<Bar> getHistoricalBars(
        String symbol,
        TimeFrame timeFrame,
        Instant startTime,
        Instant endTime
    ) {
        return barRepository.findBySymbolAndTimeFrameAndTimestampBetween(
            symbol,
            timeFrame,
            startTime,
            endTime
        );
    }
}

六、数据分发

1）数据分发器

// market/DataDistributor.java

@Component
public class DataDistributor {
    
    private final Map<String, Set<WebSocketSession>> tickSubscribers = new ConcurrentHashMap<>();
    private final Map<String, Set<WebSocketSession>> barSubscribers = new ConcurrentHashMap<>();
    
    /**
     * 启动分发器
     */
    public void start() {
        // 启动定时任务，清理断开的连接
        Executors.newScheduledThreadPool(1).scheduleAtFixedRate(
            this::cleanupDisconnectedSessions,
            0,
            1,
            TimeUnit.MINUTES
        );
    }
    
    /**
     * 停止分发器
     */
    public void stop() {
        tickSubscribers.clear();
        barSubscribers.clear();
    }
    
    /**
     * 订阅 Tick
     */
    public void subscribeTick(String symbol, WebSocketSession session) {
        tickSubscribers.computeIfAbsent(symbol, k -> ConcurrentHashMap.newKeySet())
            .add(session);
    }
    
    /**
     * 订阅 Bar
     */
    public void subscribeBar(String symbol, WebSocketSession session) {
        barSubscribers.computeIfAbsent(symbol, k -> ConcurrentHashMap.newKeySet())
            .add(session);
    }
    
    /**
     * 取消订阅 Tick
     */
    public void unsubscribeTick(String symbol, WebSocketSession session) {
        Set<WebSocketSession> sessions = tickSubscribers.get(symbol);
        if (sessions != null) {
            sessions.remove(session);
        }
    }
    
    /**
     * 取消订阅 Bar
     */
    public void unsubscribeBar(String symbol, WebSocketSession session) {
        Set<WebSocketSession> sessions = barSubscribers.get(symbol);
        if (sessions != null) {
            sessions.remove(session);
        }
    }
    
    /**
     * 分发 Tick
     */
    public void distributeTick(Tick tick) {
        Set<WebSocketSession> sessions = tickSubscribers.get(tick.getSymbol());
        if (sessions != null) {
            String message = toJson(tick);
            sessions.forEach(session -> {
                if (session.isOpen()) {
                    try {
                        session.sendMessage(new TextMessage(message));
                    } catch (Exception e) {
                        log.error("发送 Tick 消息失败", e);
                    }
                }
            });
        }
    }
    
    /**
     * 分发 Bar
     */
    public void distributeBar(Bar bar) {
        Set<WebSocketSession> sessions = barSubscribers.get(bar.getSymbol());
        if (sessions != null) {
            String message = toJson(bar);
            sessions.forEach(session -> {
                if (session.isOpen()) {
                    try {
                        session.sendMessage(new TextMessage(message));
                    } catch (Exception e) {
                        log.error("发送 Bar 消息失败", e);
                    }
                }
            });
        }
    }
    
    private void cleanupDisconnectedSessions() {
        // 清理断开的连接
        tickSubscribers.forEach((symbol, sessions) -> {
            sessions.removeIf(session -> !session.isOpen());
        });
        
        barSubscribers.forEach((symbol, sessions) -> {
            sessions.removeIf(session -> !session.isOpen());
        });
    }
    
    private String toJson(Object obj) {
        return new Gson().toJson(obj);
    }
}

七、最佳实践总结

1）行情处理设计原则

  • 实时性：行情数据必须实时处理
  • 准确性：行情数据必须准确无误
  • 可靠性：行情服务必须稳定可靠
  • 可扩展性：支持多数据源和多周期

2）数据清洗原则

  • 完整性：确保数据完整
  • 一致性：确保数据一致
  • 准确性：确保数据准确
  • 及时性：确保数据及时

3）数据聚合原则

  • 高效性：聚合必须高效
  • 准确性：聚合必须准确
  • 实时性：聚合必须实时
  • 可配置性：聚合周期可配置

结语：实时行情处理是量化平台的基础设施

实时行情处理是量化平台的基础设施，它为策略引擎、前端展示、数据订阅等提供数据支持，是量化平台的核心能力。

关键优势总结：

  1. 多数据源支持：支持多个数据源接入
  2. 数据清洗：确保数据质量
  3. 多周期聚合：支持多个时间周期
  4. 实时分发：低延迟数据分发
  5. 高可靠性：稳定可靠的服务

对于正在构建量化平台的团队，完善的实时行情处理系统是必不可少的。

avatar
文章
阅读
粉丝