57-数据同步方案详解

数据同步方案详解

本章导读

数据同步是分布式系统中保证数据一致性的核心机制，无论是数据库主从复制、多数据中心同步，还是异构数据源集成，都离不开可靠的数据同步方案。本章将深入探讨同步复制、异步复制、增量同步、双向同步等关键技术。

学习目标：

• 目标1：掌握同步复制和异步复制的原理与实现
• 目标2：理解增量同步和变更数据捕获（CDC）技术
• 目标3：能够设计双向同步方案并处理数据冲突

前置知识：熟悉强一致性和最终一致性概念，了解数据库复制原理

阅读时长：约 40 分钟

一、知识概述

在分布式系统中，数据同步是保证数据一致性的核心机制。无论是主从复制、多数据中心同步，还是异构数据源集成，都需要可靠的数据同步方案。本文将深入探讨各种数据同步策略，包括同步/异步复制、增量同步、双向同步等技术。

数据同步的挑战

• 网络延迟：跨地域同步的网络延迟影响系统性能
• 数据冲突：并发更新导致的数据冲突问题
• 一致性保证：保证源和目标数据的最终一致
• 容错处理：处理网络故障、系统故障等异常情况

---

二、核心同步方案

2.1 同步复制

原理说明

同步复制要求主节点在提交事务前，将数据同步到从节点：

1. 主节点接收写请求
2. 主节点将数据发送到从节点
3. 等待从节点确认
4. 主节点提交事务

Java 实现

import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;

/**
 * 同步复制管理器
 */
public class SyncReplicationManager {
    
    private final String nodeId;
    private final List<ReplicaNode> replicas;
    private final ExecutorService executor;
    private final int writeQuorum;
    private final int timeout;
    
    // 本地数据存储
    private final Map<String, Object> dataStore;
    private final Map<String, Long> versionStore;
    private final AtomicLong currentVersion;
    
    public SyncReplicationManager(String nodeId, List<ReplicaNode> replicas, 
            int writeQuorum, int timeoutMs) {
        this.nodeId = nodeId;
        this.replicas = replicas;
        this.executor = Executors.newFixedThreadPool(replicas.size());
        this.writeQuorum = writeQuorum;
        this.timeout = timeoutMs;
        this.dataStore = new ConcurrentHashMap<>();
        this.versionStore = new ConcurrentHashMap<>();
        this.currentVersion = new AtomicLong(0);
    }
    
    /**
     * 写入数据（同步复制）
     */
    public WriteResult write(String key, Object value) {
        long version = currentVersion.incrementAndGet();
        WriteRequest request = new WriteRequest(
            UUID.randomUUID().toString(),
            key,
            value,
            version,
            System.currentTimeMillis()
        );
        
        // 1. 本地写入
        dataStore.put(key, value);
        versionStore.put(key, version);
        
        // 2. 同步到从节点
        AtomicInteger successCount = new AtomicInteger(1); // 本地算一个
        CountDownLatch latch = new CountDownLatch(replicas.size());
        
        for (ReplicaNode replica : replicas) {
            executor.submit(() -> {
                try {
                    boolean success = replica.replicate(request);
                    if (success) {
                        successCount.incrementAndGet();
                    }
                } catch (Exception e) {
                    System.err.println("[SyncReplication] Replication failed to " + 
                        replica.getNodeId() + ": " + e.getMessage());
                } finally {
                    latch.countDown();
                }
            });
        }
        
        // 3. 等待仲裁确认
        try {
            boolean completed = latch.await(timeout, TimeUnit.MILLISECONDS);
            
            if (!completed) {
                // 超时，回滚
                dataStore.remove(key);
                versionStore.remove(key);
                return WriteResult.timeout("Replication timeout");
            }
            
            if (successCount.get() < writeQuorum) {
                // 未达到仲裁，回滚
                dataStore.remove(key);
                versionStore.remove(key);
                return WriteResult.failure("Insufficient replicas, got " + 
                    successCount.get() + ", need " + writeQuorum);
            }
            
            return WriteResult.success(version);
            
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            return WriteResult.failure("Interrupted");
        }
    }
    
    /**
     * 读取数据
     */
    public ReadResult read(String key) {
        Object value = dataStore.get(key);
        Long version = versionStore.get(key);
        
        if (value == null) {
            return ReadResult.notFound();
        }
        
        return ReadResult.success(value, version);
    }
    
    /**
     * 接收复制请求（从节点视角）
     */
    public boolean receiveReplication(WriteRequest request) {
        try {
            // 检查版本
            Long currentVersion = versionStore.get(request.key);
            if (currentVersion != null && currentVersion >= request.version) {
                // 版本过旧，忽略
                return true;
            }
            
            // 写入数据
            dataStore.put(request.key, request.value);
            versionStore.put(request.key, request.version);
            
            System.out.println("[Node-" + nodeId + "] Received replication: " + 
                request.key + " v" + request.version);
            
            return true;
        } catch (Exception e) {
            return false;
        }
    }
    
    /**
     * 获取当前状态
     */
    public ReplicationStatus getStatus() {
        return new ReplicationStatus(
            nodeId,
            dataStore.size(),
            currentVersion.get(),
            replicas.stream().map(ReplicaNode::getNodeId).toList()
        );
    }
    
    public void shutdown() {
        executor.shutdown();
    }
}

/**
 * 副本节点
 */
interface ReplicaNode {
    String getNodeId();
    boolean replicate(WriteRequest request);
}

/**
 * 远程副本节点实现
 */
class RemoteReplicaNode implements ReplicaNode {
    
    private final String nodeId;
    private final String address;
    
    public RemoteReplicaNode(String nodeId, String address) {
        this.nodeId = nodeId;
        this.address = address;
    }
    
    @Override
    public String getNodeId() {
        return nodeId;
    }
    
    @Override
    public boolean replicate(WriteRequest request) {
        // 模拟远程调用
        // 实际中通过 RPC/HTTP 调用
        System.out.println("[Remote-" + nodeId + "] Replicating " + 
            request.key + " v" + request.version);
        return true;
    }
}

/**
 * 本地副本节点（用于测试）
 */
class LocalReplicaNode implements ReplicaNode {
    
    private final String nodeId;
    private final SyncReplicationManager replicaManager;
    
    public LocalReplicaNode(String nodeId, SyncReplicationManager replicaManager) {
        this.nodeId = nodeId;
        this.replicaManager = replicaManager;
    }
    
    @Override
    public String getNodeId() {
        return nodeId;
    }
    
    @Override
    public boolean replicate(WriteRequest request) {
        return replicaManager.receiveReplication(request);
    }
}

// 辅助类
record WriteRequest(String requestId, String key, Object value, long version, long timestamp) {}

record WriteResult(boolean success, String message, long version) {
    static WriteResult success(long version) {
        return new WriteResult(true, "Success", version);
    }
    
    static WriteResult failure(String message) {
        return new WriteResult(false, message, -1);
    }
    
    static WriteResult timeout(String message) {
        return new WriteResult(false, message, -1);
    }
}

record ReadResult(boolean found, Object value, Long version) {
    static ReadResult success(Object value, Long version) {
        return new ReadResult(true, value, version);
    }
    
    static ReadResult notFound() {
        return new ReadResult(false, null, null);
    }
}

record ReplicationStatus(String nodeId, int dataSize, long version, List<String> replicas) {}

// 同步复制演示
public class SyncReplicationDemo {
    public static void main(String[] args) {
        // 创建副本节点
        SyncReplicationManager replica1 = new SyncReplicationManager(
            "replica-1", Collections.emptyList(), 1, 5000);
        SyncReplicationManager replica2 = new SyncReplicationManager(
            "replica-2", Collections.emptyList(), 1, 5000);
        SyncReplicationManager replica3 = new SyncReplicationManager(
            "replica-3", Collections.emptyList(), 1, 5000);
        
        // 创建主节点
        List<ReplicaNode> replicas = Arrays.asList(
            new LocalReplicaNode("replica-1", replica1),
            new LocalReplicaNode("replica-2", replica2),
            new LocalReplicaNode("replica-3", replica3)
        );
        
        SyncReplicationManager primary = new SyncReplicationManager(
            "primary", replicas, 2, 5000); // W=2，至少2个节点确认
        
        // 写入数据
        System.out.println("=== Writing data ===");
        WriteResult result1 = primary.write("user:001", "张三");
        System.out.println("Write result: " + result1);
        
        WriteResult result2 = primary.write("user:002", "李四");
        System.out.println("Write result: " + result2);
        
        // 读取数据
        System.out.println("\n=== Reading data ===");
        System.out.println("Primary: " + primary.read("user:001"));
        System.out.println("Replica1: " + replica1.read("user:001"));
        System.out.println("Replica2: " + replica2.read("user:001"));
        
        // 关闭
        primary.shutdown();
        replica1.shutdown();
        replica2.shutdown();
        replica3.shutdown();
    }
}

---

2.2 异步复制

原理说明

异步复制在主节点提交事务后，异步地将数据同步到从节点：

1. 主节点接收写请求并提交
2. 立即返回成功响应
3. 后台异步同步到从节点

Java 实现

import java.util.*;
import java.util.concurrent.*;

/**
 * 异步复制管理器
 */
public class AsyncReplicationManager {
    
    private final String nodeId;
    private final List<ReplicaNode> replicas;
    private final ExecutorService executor;
    private final BlockingQueue<WriteRequest> replicationQueue;
    private final ScheduledExecutorService scheduler;
    
    // 本地数据存储
    private final Map<String, Object> dataStore;
    private final Map<String, Long> versionStore;
    private final AtomicLong currentVersion;
    
    // 复制状态
    private final Map<String, Long> replicaPositions;
    private volatile boolean running;
    
    public AsyncReplicationManager(String nodeId, List<ReplicaNode> replicas) {
        this.nodeId = nodeId;
        this.replicas = replicas;
        this.executor = Executors.newFixedThreadPool(replicas.size());
        this.replicationQueue = new LinkedBlockingQueue<>(10000);
        this.scheduler = Executors.newScheduledThreadPool(2);
        
        this.dataStore = new ConcurrentHashMap<>();
        this.versionStore = new ConcurrentHashMap<>();
        this.currentVersion = new AtomicLong(0);
        
        this.replicaPositions = new ConcurrentHashMap<>();
        for (ReplicaNode replica : replicas) {
            replicaPositions.put(replica.getNodeId(), 0L);
        }
        
        this.running = true;
        startReplicationWorkers();
        startMonitoringTask();
    }
    
    /**
     * 写入数据（异步复制）
     */
    public WriteResult write(String key, Object value) {
        long version = currentVersion.incrementAndGet();
        
        // 1. 本地写入
        dataStore.put(key, value);
        versionStore.put(key, version);
        
        // 2. 加入复制队列
        WriteRequest request = new WriteRequest(
            UUID.randomUUID().toString(),
            key,
            value,
            version,
            System.currentTimeMillis()
        );
        
        try {
            replicationQueue.put(request);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            return WriteResult.failure("Interrupted");
        }
        
        System.out.println("[AsyncReplication-" + nodeId + "] Wrote " + 
            key + " v" + version + ", queued for replication");
        
        return WriteResult.success(version);
    }
    
    /**
     * 读取数据
     */
    public ReadResult read(String key) {
        Object value = dataStore.get(key);
        Long version = versionStore.get(key);
        
        if (value == null) {
            return ReadResult.notFound();
        }
        
        return ReadResult.success(value, version);
    }
    
    /**
     * 启动复制工作线程
     */
    private void startReplicationWorkers() {
        for (ReplicaNode replica : replicas) {
            executor.submit(() -> {
                while (running && !Thread.currentThread().isInterrupted()) {
                    try {
                        WriteRequest request = replicationQueue.poll(1, TimeUnit.SECONDS);
                        if (request != null) {
                            replicateToNode(replica, request);
                        }
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                        break;
                    }
                }
            });
        }
    }
    
    /**
     * 复制到指定节点
     */
    private void replicateToNode(ReplicaNode replica, WriteRequest request) {
        int retries = 3;
        
        for (int i = 0; i < retries; i++) {
            try {
                boolean success = replica.replicate(request);
                
                if (success) {
                    replicaPositions.put(replica.getNodeId(), request.version);
                    return;
                }
            } catch (Exception e) {
                System.err.println("[AsyncReplication] Replication error to " + 
                    replica.getNodeId() + ": " + e.getMessage());
            }
            
            // 重试
            try {
                Thread.sleep(100 * (i + 1));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                return;
            }
        }
        
        System.err.println("[AsyncReplication] Failed to replicate to " + 
            replica.getNodeId() + " after " + retries + " retries");
    }
    
    /**
     * 启动监控任务
     */
    private void startMonitoringTask() {
        scheduler.scheduleAtFixedRate(() -> {
            long currentPos = currentVersion.get();
            
            System.out.println("\n[Monitor] Replication lag:");
            for (Map.Entry<String, Long> entry : replicaPositions.entrySet()) {
                long lag = currentPos - entry.getValue();
                System.out.println("  " + entry.getKey() + ": lag=" + lag);
            }
            System.out.println("  Queue size: " + replicationQueue.size());
            
        }, 5, 5, TimeUnit.SECONDS);
    }
    
    /**
     * 获取复制延迟
     */
    public Map<String, Long> getReplicationLag() {
        Map<String, Long> lag = new HashMap<>();
        long currentPos = currentVersion.get();
        
        for (Map.Entry<String, Long> entry : replicaPositions.entrySet()) {
            lag.put(entry.getKey(), currentPos - entry.getValue());
        }
        
        return lag;
    }
    
    /**
     * 接收复制请求
     */
    public boolean receiveReplication(WriteRequest request) {
        Long currentVersion = versionStore.get(request.key);
        if (currentVersion != null && currentVersion >= request.version) {
            return true;
        }
        
        dataStore.put(request.key, request.value);
        versionStore.put(request.key, request.version);
        
        return true;
    }
    
    public void shutdown() {
        running = false;
        executor.shutdown();
        scheduler.shutdown();
    }
}

/**
 * 批量异步复制
 */
public class BatchAsyncReplicationManager {
    
    private final String nodeId;
    private final List<ReplicaNode> replicas;
    private final ExecutorService executor;
    private final List<WriteRequest> writeBuffer;
    private final int batchSize;
    private final long flushInterval;
    private final ScheduledExecutorService scheduler;
    
    private final Map<String, Object> dataStore;
    private final Map<String, Long> versionStore;
    private final AtomicLong currentVersion;
    
    public BatchAsyncReplicationManager(String nodeId, List<ReplicaNode> replicas,
            int batchSize, long flushIntervalMs) {
        this.nodeId = nodeId;
        this.replicas = replicas;
        this.executor = Executors.newFixedThreadPool(replicas.size());
        this.writeBuffer = Collections.synchronizedList(new ArrayList<>());
        this.batchSize = batchSize;
        this.flushInterval = flushIntervalMs;
        this.scheduler = Executors.newScheduledThreadPool(1);
        
        this.dataStore = new ConcurrentHashMap<>();
        this.versionStore = new ConcurrentHashMap<>();
        this.currentVersion = new AtomicLong(0);
        
        startFlushTask();
    }
    
    /**
     * 写入数据
     */
    public WriteResult write(String key, Object value) {
        long version = currentVersion.incrementAndGet();
        
        dataStore.put(key, value);
        versionStore.put(key, version);
        
        WriteRequest request = new WriteRequest(
            UUID.randomUUID().toString(), key, value, version, System.currentTimeMillis()
        );
        
        writeBuffer.add(request);
        
        // 达到批次大小时立即刷新
        if (writeBuffer.size() >= batchSize) {
            flush();
        }
        
        return WriteResult.success(version);
    }
    
    /**
     * 启动定时刷新任务
     */
    private void startFlushTask() {
        scheduler.scheduleAtFixedRate(this::flush, 
            flushInterval, flushInterval, TimeUnit.MILLISECONDS);
    }
    
    /**
     * 刷新缓冲区
     */
    private synchronized void flush() {
        if (writeBuffer.isEmpty()) {
            return;
        }
        
        List<WriteRequest> batch = new ArrayList<>(writeBuffer);
        writeBuffer.clear();
        
        System.out.println("[BatchReplication] Flushing " + batch.size() + " requests");
        
        for (ReplicaNode replica : replicas) {
            executor.submit(() -> {
                for (WriteRequest request : batch) {
                    try {
                        replica.replicate(request);
                    } catch (Exception e) {
                        System.err.println("Replication error: " + e.getMessage());
                    }
                }
            });
        }
    }
    
    public ReadResult read(String key) {
        Object value = dataStore.get(key);
        Long version = versionStore.get(key);
        return value != null ? ReadResult.success(value, version) : ReadResult.notFound();
    }
    
    public void shutdown() {
        flush(); // 最后刷新一次
        executor.shutdown();
        scheduler.shutdown();
    }
}

// 异步复制演示
public class AsyncReplicationDemo {
    public static void main(String[] args) throws InterruptedException {
        // 创建副本节点
        SyncReplicationManager replica1 = new SyncReplicationManager(
            "replica-1", Collections.emptyList(), 1, 5000);
        SyncReplicationManager replica2 = new SyncReplicationManager(
            "replica-2", Collections.emptyList(), 1, 5000);
        
        // 创建主节点
        List<ReplicaNode> replicas = Arrays.asList(
            new LocalReplicaNode("replica-1", replica1),
            new LocalReplicaNode("replica-2", replica2)
        );
        
        AsyncReplicationManager primary = new AsyncReplicationManager("primary", replicas);
        
        // 写入数据
        System.out.println("=== Writing data ===");
        for (int i = 1; i <= 5; i++) {
            primary.write("key-" + i, "value-" + i);
        }
        
        // 等待复制完成
        Thread.sleep(2000);
        
        // 读取数据
        System.out.println("\n=== Reading data ===");
        System.out.println("Primary: " + primary.read("key-3"));
        System.out.println("Replica1: " + replica1.read("key-3"));
        System.out.println("Replica2: " + replica2.read("key-3"));
        
        // 关闭
        primary.shutdown();
        replica1.shutdown();
        replica2.shutdown();
    }
}

---

2.3 增量同步

原理说明

增量同步只同步变化的数据，而非全量数据：

1. 记录数据变更（通过 Binlog、Change Data Capture 等）
2. 解析变更事件
3. 将变更应用到目标系统

Java 实现

import java.util.*;
import java.util.concurrent.*;

/**
 * 变更数据捕获（CDC）
 */
public class ChangeDataCapture {
    
    private final String sourceName;
    private final BlockingQueue<ChangeEvent> eventQueue;
    private final List<ChangeEventConsumer> consumers;
    private final ExecutorService executor;
    private volatile boolean running;
    
    // 事件位置
    private final AtomicLong position;
    
    public ChangeDataCapture(String sourceName) {
        this.sourceName = sourceName;
        this.eventQueue = new LinkedBlockingQueue<>(10000);
        this.consumers = new CopyOnWriteArrayList<>();
        this.executor = Executors.newFixedThreadPool(4);
        this.running = true;
        this.position = new AtomicLong(0);
        
        startEventDispatcher();
    }
    
    /**
     * 记录变更事件
     */
    public void captureChange(String table, ChangeType type, 
            Map<String, Object> before, Map<String, Object> after) {
        
        long pos = position.incrementAndGet();
        
        ChangeEvent event = new ChangeEvent(
            pos,
            sourceName,
            table,
            type,
            before,
            after,
            System.currentTimeMillis()
        );
        
        try {
            eventQueue.put(event);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }
    
    /**
     * 注册消费者
     */
    public void registerConsumer(ChangeEventConsumer consumer) {
        consumers.add(consumer);
    }
    
    /**
     * 启动事件分发器
     */
    private void startEventDispatcher() {
        for (int i = 0; i < 4; i++) {
            executor.submit(() -> {
                while (running) {
                    try {
                        ChangeEvent event = eventQueue.poll(1, TimeUnit.SECONDS);
                        if (event != null) {
                            dispatchEvent(event);
                        }
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                        break;
                    }
                }
            });
        }
    }
    
    /**
     * 分发事件
     */
    private void dispatchEvent(ChangeEvent event) {
        for (ChangeEventConsumer consumer : consumers) {
            try {
                consumer.consume(event);
            } catch (Exception e) {
                System.err.println("[CDC] Consumer error: " + e.getMessage());
            }
        }
    }
    
    public long getPosition() {
        return position.get();
    }
    
    public void shutdown() {
        running = false;
        executor.shutdown();
    }
}

/**
 * 变更事件
 */
record ChangeEvent(
    long position,
    String source,
    String table,
    ChangeType type,
    Map<String, Object> before,
    Map<String, Object> after,
    long timestamp
) {}

enum ChangeType {
    INSERT, UPDATE, DELETE
}

/**
 * 变更事件消费者
 */
interface ChangeEventConsumer {
    void consume(ChangeEvent event);
}

/**
 * 增量同步管理器
 */
public class IncrementalSyncManager {
    
    private final String name;
    private final ChangeDataCapture cdc;
    private final Map<String, SyncTarget> targets;
    private final SyncPositionStore positionStore;
    private final ExecutorService executor;
    
    public IncrementalSyncManager(String name, ChangeDataCapture cdc) {
        this.name = name;
        this.cdc = cdc;
        this.targets = new ConcurrentHashMap<>();
        this.positionStore = new SyncPositionStore();
        this.executor = Executors.newCachedThreadPool();
        
        setupCDCConsumer();
    }
    
    /**
     * 添加同步目标
     */
    public void addTarget(String targetName, SyncTarget target) {
        targets.put(targetName, target);
    }
    
    /**
     * 设置 CDC 消费者
     */
    private void setupCDCConsumer() {
        cdc.registerConsumer(event -> {
            for (Map.Entry<String, SyncTarget> entry : targets.entrySet()) {
                String targetName = entry.getKey();
                SyncTarget target = entry.getValue();
                
                executor.submit(() -> {
                    try {
                        // 应用变更
                        target.applyChange(event);
                        
                        // 更新位置
                        positionStore.savePosition(name, targetName, event.position());
                        
                    } catch (Exception e) {
                        System.err.println("[IncrementalSync] Sync error to " + 
                            targetName + ": " + e.getMessage());
                    }
                });
            }
        });
    }
    
    /**
     * 获取同步位置
     */
    public long getSyncPosition(String targetName) {
        return positionStore.getPosition(name, targetName);
    }
    
    public void shutdown() {
        executor.shutdown();
    }
}

/**
 * 同步目标
 */
interface SyncTarget {
    void applyChange(ChangeEvent event) throws Exception;
}

/**
 * 数据库同步目标
 */
class DatabaseSyncTarget implements SyncTarget {
    
    private final String name;
    private final Map<String, Object> dataStore;
    
    public DatabaseSyncTarget(String name) {
        this.name = name;
        this.dataStore = new ConcurrentHashMap<>();
    }
    
    @Override
    public void applyChange(ChangeEvent event) throws Exception {
        String key = extractKey(event);
        
        switch (event.type()) {
            case INSERT:
            case UPDATE:
                dataStore.put(key, event.after());
                break;
            case DELETE:
                dataStore.remove(key);
                break;
        }
        
        System.out.println("[Target-" + name + "] Applied " + event.type() + 
            " on " + event.table() + ": " + key);
    }
    
    private String extractKey(ChangeEvent event) {
        Map<String, Object> data = event.after() != null ? event.after() : event.before();
        return event.table() + ":" + data.get("id");
    }
    
    public Object get(String key) {
        return dataStore.get(key);
    }
}

/**
 * 同步位置存储
 */
class SyncPositionStore {
    
    private final Map<String, Long> positions = new ConcurrentHashMap<>();
    
    public void savePosition(String syncName, String targetName, long position) {
        positions.put(syncName + ":" + targetName, position);
    }
    
    public long getPosition(String syncName, String targetName) {
        return positions.getOrDefault(syncName + ":" + targetName, 0L);
    }
}

// 增量同步演示
public class IncrementalSyncDemo {
    public static void main(String[] args) throws InterruptedException {
        // 创建 CDC
        ChangeDataCapture cdc = new ChangeDataCapture("source-db");
        
        // 创建同步管理器
        IncrementalSyncManager syncManager = new IncrementalSyncManager("main-sync", cdc);
        
        // 添加同步目标
        DatabaseSyncTarget target1 = new DatabaseSyncTarget("target-1");
        DatabaseSyncTarget target2 = new DatabaseSyncTarget("target-2");
        
        syncManager.addTarget("target-1", target1);
        syncManager.addTarget("target-2", target2);
        
        // 模拟数据变更
        System.out.println("=== Capturing changes ===");
        
        // 插入
        cdc.captureChange("users", ChangeType.INSERT, null, 
            Map.of("id", "user-001", "name", "张三", "email", "zhang@example.com"));
        
        // 更新
        cdc.captureChange("users", ChangeType.UPDATE, 
            Map.of("id", "user-001", "name", "张三"),
            Map.of("id", "user-001", "name", "李四", "email", "li@example.com"));
        
        // 插入另一条
        cdc.captureChange("users", ChangeType.INSERT, null, 
            Map.of("id", "user-002", "name", "王五", "email", "wang@example.com"));
        
        // 删除
        cdc.captureChange("users", ChangeType.DELETE, 
            Map.of("id", "user-002"), null);
        
        // 等待同步完成
        Thread.sleep(1000);
        
        // 检查结果
        System.out.println("\n=== Sync results ===");
        System.out.println("Target-1 position: " + syncManager.getSyncPosition("target-1"));
        System.out.println("Target-1 data: " + target1.get("users:user-001"));
        
        // 关闭
        cdc.shutdown();
        syncManager.shutdown();
    }
}

---

2.4 双向同步

原理说明

双向同步支持两个数据源之间的数据互相同步：

1. A 和 B 都可以写入数据
2. 变更自动同步到对方
3. 需要处理数据冲突

Java 实现

import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;

/**
 * 双向同步管理器
 */
public class BidirectionalSyncManager {
    
    private final String localNodeId;
    private final Map<String, Object> dataStore;
    private final Map<String, VectorClock> versionStore;
    private final List<BidirectionalSyncManager> peers;
    private final ExecutorService executor;
    
    // 冲突解决策略
    private final ConflictResolver conflictResolver;
    
    public BidirectionalSyncManager(String nodeId) {
        this.localNodeId = nodeId;
        this.dataStore = new ConcurrentHashMap<>();
        this.versionStore = new ConcurrentHashMap<>();
        this.peers = new CopyOnWriteArrayList<>();
        this.executor = Executors.newCachedThreadPool();
        this.conflictResolver = new LastWriteWinsResolver();
    }
    
    /**
     * 添加对等节点
     */
    public void addPeer(BidirectionalSyncManager peer) {
        peers.add(peer);
    }
    
    /**
     * 本地写入
     */
    public void put(String key, Object value) {
        // 获取当前版本时钟
        VectorClock currentClock = versionStore.get(key);
        if (currentClock == null) {
            currentClock = new VectorClock();
        }
        
        // 递增本地版本
        VectorClock newClock = currentClock.increment(localNodeId);
        
        // 写入数据和版本
        dataStore.put(key, value);
        versionStore.put(key, newClock);
        
        System.out.println("[Node-" + localNodeId + "] Local write: " + 
            key + " = " + value + " (" + newClock + ")");
        
        // 同步到对等节点
        syncToPeers(key, value, newClock);
    }
    
    /**
     * 本地读取
     */
    public Object get(String key) {
        return dataStore.get(key);
    }
    
    /**
     * 同步到对等节点
     */
    private void syncToPeers(String key, Object value, VectorClock clock) {
        SyncMessage message = new SyncMessage(
            localNodeId,
            key,
            value,
            clock,
            System.currentTimeMillis()
        );
        
        for (BidirectionalSyncManager peer : peers) {
            executor.submit(() -> peer.receiveSync(message));
        }
    }
    
    /**
     * 接收同步消息
     */
    public void receiveSync(SyncMessage message) {
        String key = message.key();
        Object remoteValue = message.value();
        VectorClock remoteClock = message.clock();
        
        // 获取本地版本
        VectorClock localClock = versionStore.get(key);
        Object localValue = dataStore.get(key);
        
        System.out.println("[Node-" + localNodeId + "] Received sync: " + 
            key + " from " + message.sourceNodeId());
        
        if (localClock == null) {
            // 本地不存在，直接写入
            dataStore.put(key, remoteValue);
            versionStore.put(key, remoteClock);
            return;
        }
        
        // 比较版本
        int comparison = localClock.compareTo(remoteClock);
        
        if (comparison < 0) {
            // 远程版本更新
            dataStore.put(key, remoteValue);
            versionStore.put(key, remoteClock);
            
        } else if (comparison > 0) {
            // 本地版本更新，忽略
            System.out.println("[Node-" + localNodeId + "] Local version newer, ignoring");
            
        } else {
            // 版本冲突
            Object resolved = conflictResolver.resolve(key, localValue, remoteValue, 
                localClock, remoteClock, message.sourceNodeId(), localNodeId);
            
            // 合并版本时钟
            VectorClock mergedClock = localClock.merge(remoteClock).increment(localNodeId);
            
            dataStore.put(key, resolved);
            versionStore.put(key, mergedClock);
            
            // 广播解决后的值
            if (!resolved.equals(localValue)) {
                syncToPeers(key, resolved, mergedClock);
            }
        }
    }
    
    public void shutdown() {
        executor.shutdown();
    }
}

/**
 * 向量时钟
 */
class VectorClock {
    
    private final Map<String, Long> clocks;
    
    public VectorClock() {
        this.clocks = new HashMap<>();
    }
    
    public VectorClock(Map<String, Long> clocks) {
        this.clocks = new HashMap<>(clocks);
    }
    
    /**
     * 递增指定节点的时钟
     */
    public VectorClock increment(String nodeId) {
        Map<String, Long> newClocks = new HashMap<>(clocks);
        newClocks.merge(nodeId, 1L, Long::sum);
        return new VectorClock(newClocks);
    }
    
    /**
     * 合并两个向量时钟
     */
    public VectorClock merge(VectorClock other) {
        Map<String, Long> newClocks = new HashMap<>(clocks);
        
        for (Map.Entry<String, Long> entry : other.clocks.entrySet()) {
            newClocks.merge(entry.getKey(), entry.getValue(), Math::max);
        }
        
        return new VectorClock(newClocks);
    }
    
    /**
     * 比较两个向量时钟
     * @return -1: this < other, 0: 并发, 1: this > other
     */
    public int compareTo(VectorClock other) {
        boolean thisGreater = false;
        boolean otherGreater = false;
        
        Set<String> allNodes = new HashSet<>();
        allNodes.addAll(clocks.keySet());
        allNodes.addAll(other.clocks.keySet());
        
        for (String node : allNodes) {
            long thisVal = clocks.getOrDefault(node, 0L);
            long otherVal = other.clocks.getOrDefault(node, 0L);
            
            if (thisVal > otherVal) {
                thisGreater = true;
            } else if (thisVal < otherVal) {
                otherGreater = true;
            }
        }
        
        if (thisGreater && !otherGreater) {
            return 1;
        } else if (otherGreater && !thisGreater) {
            return -1;
        } else {
            return 0; // 并发
        }
    }
    
    @Override
    public String toString() {
        return clocks.toString();
    }
}

/**
 * 同步消息
 */
record SyncMessage(
    String sourceNodeId,
    String key,
    Object value,
    VectorClock clock,
    long timestamp
) {}

/**
 * 冲突解决策略
 */
interface ConflictResolver {
    Object resolve(String key, Object localValue, Object remoteValue,
            VectorClock localClock, VectorClock remoteClock,
            String remoteNodeId, String localNodeId);
}

/**
 * 最后写入胜出策略
 */
class LastWriteWinsResolver implements ConflictResolver {
    
    @Override
    public Object resolve(String key, Object localValue, Object remoteValue,
            VectorClock localClock, VectorClock remoteClock,
            String remoteNodeId, String localNodeId) {
        
        System.out.println("[ConflictResolver] Conflict detected for " + key + 
            ", using Last-Write-Wins");
        
        // 使用字典序比较节点ID作为决胜条件
        return remoteNodeId.compareTo(localNodeId) > 0 ? remoteValue : localValue;
    }
}

/**
 * 合并策略
 */
class MergeResolver implements ConflictResolver {
    
    @Override
    public Object resolve(String key, Object localValue, Object remoteValue,
            VectorClock localClock, VectorClock remoteClock,
            String remoteNodeId, String localNodeId) {
        
        // 如果值是 Map，尝试合并
        if (localValue instanceof Map && remoteValue instanceof Map) {
            @SuppressWarnings("unchecked")
            Map<String, Object> localMap = new HashMap<>((Map<String, Object>) localValue);
            @SuppressWarnings("unchecked")
            Map<String, Object> remoteMap = (Map<String, Object>) remoteValue;
            
            localMap.putAll(remoteMap);
            return localMap;
        }
        
        // 否则使用 LWW
        return remoteNodeId.compareTo(localNodeId) > 0 ? remoteValue : localValue;
    }
}

// 双向同步演示
public class BidirectionalSyncDemo {
    public static void main(String[] args) throws InterruptedException {
        // 创建两个节点
        BidirectionalSyncManager nodeA = new BidirectionalSyncManager("A");
        BidirectionalSyncManager nodeB = new BidirectionalSyncManager("B");
        
        // 设置互相同步
        nodeA.addPeer(nodeB);
        nodeB.addPeer(nodeA);
        
        // Node A 写入
        System.out.println("=== Node A writes ===");
        nodeA.put("key1", "value-from-A");
        
        Thread.sleep(100);
        
        // Node B 写入
        System.out.println("\n=== Node B writes ===");
        nodeB.put("key2", "value-from-B");
        
        Thread.sleep(100);
        
        // Node B 更新 key1
        System.out.println("\n=== Node B updates key1 ===");
        nodeB.put("key1", "value-from-B-updated");
        
        Thread.sleep(100);
        
        // 检查结果
        System.out.println("\n=== Final state ===");
        System.out.println("Node A - key1: " + nodeA.get("key1"));
        System.out.println("Node A - key2: " + nodeA.get("key2"));
        System.out.println("Node B - key1: " + nodeB.get("key1"));
        System.out.println("Node B - key2: " + nodeB.get("key2"));
        
        // 关闭
        nodeA.shutdown();
        nodeB.shutdown();
    }
}

---

三、实战应用场景

3.1 数据库主从同步

/**
 * MySQL 主从同步模拟
 */
public class MySQLReplicationSimulator {
    
    private final MasterNode master;
    private final List<SlaveNode> slaves;
    
    public MySQLReplicationSimulator() {
        this.master = new MasterNode("master-1");
        this.slaves = new ArrayList<>();
    }
    
    public void addSlave(String slaveId) {
        SlaveNode slave = new SlaveNode(slaveId, master);
        slaves.add(slave);
        master.addSlave(slave);
    }
    
    /**
     * 主节点
     */
    static class MasterNode {
        
        private final String nodeId;
        private final List<SlaveNode> slaves;
        private final BinaryLog binlog;
        private final Map<String, Object> data;
        private final AtomicLong position;
        
        public MasterNode(String nodeId) {
            this.nodeId = nodeId;
            this.slaves = new CopyOnWriteArrayList<>();
            this.binlog = new BinaryLog();
            this.data = new ConcurrentHashMap<>();
            this.position = new AtomicLong(0);
        }
        
        public void addSlave(SlaveNode slave) {
            slaves.add(slave);
        }
        
        /**
         * 执行写操作
         */
        public void execute(String sql) {
            // 解析 SQL（简化）
            ParsedStatement stmt = parseSQL(sql);
            
            // 执行本地操作
            executeLocal(stmt);
            
            // 写入 Binlog
            long pos = position.incrementAndGet();
            BinlogEvent event = new BinlogEvent(
                pos,
                stmt.type(),
                stmt.table(),
                stmt.data(),
                System.currentTimeMillis()
            );
            binlog.write(event);
            
            System.out.println("[Master-" + nodeId + "] Executed: " + sql + 
                " (position: " + pos + ")");
            
            // 通知从节点
            notifySlaves(event);
        }
        
        private void executeLocal(ParsedStatement stmt) {
            String key = stmt.table() + ":" + stmt.data().get("id");
            switch (stmt.type()) {
                case INSERT, UPDATE -> data.put(key, stmt.data());
                case DELETE -> data.remove(key);
            }
        }
        
        private ParsedStatement parseSQL(String sql) {
            // 简化的 SQL 解析
            if (sql.contains("INSERT")) {
                return new ParsedStatement(StatementType.INSERT, "users", 
                    extractData(sql));
            } else if (sql.contains("UPDATE")) {
                return new ParsedStatement(StatementType.UPDATE, "users", 
                    extractData(sql));
            } else {
                return new ParsedStatement(StatementType.DELETE, "users", 
                    extractData(sql));
            }
        }
        
        private Map<String, Object> extractData(String sql) {
            // 简化：从 SQL 中提取数据
            return new HashMap<>();
        }
        
        private void notifySlaves(BinlogEvent event) {
            for (SlaveNode slave : slaves) {
                slave.receiveBinlogEvent(event);
            }
        }
        
        public Object get(String table, String id) {
            return data.get(table + ":" + id);
        }
        
        public BinlogEvent getBinlogEvent(long position) {
            return binlog.read(position);
        }
    }
    
    /**
     * 从节点
     */
    static class SlaveNode {
        
        private final String nodeId;
        private final MasterNode master;
        private final Map<String, Object> data;
        private long replicatedPosition;
        
        public SlaveNode(String nodeId, MasterNode master) {
            this.nodeId = nodeId;
            this.master = master;
            this.data = new ConcurrentHashMap<>();
            this.replicatedPosition = 0;
        }
        
        /**
         * 接收 Binlog 事件
         */
        public void receiveBinlogEvent(BinlogEvent event) {
            // 应用事件
            applyEvent(event);
            replicatedPosition = event.position();
            
            System.out.println("[Slave-" + nodeId + "] Replicated position: " + 
                replicatedPosition);
        }
        
        private void applyEvent(BinlogEvent event) {
            String key = event.table() + ":" + event.data().get("id");
            
            switch (event.type()) {
                case INSERT, UPDATE -> data.put(key, event.data());
                case DELETE -> data.remove(key);
            }
        }
        
        /**
         * 拉取复制（从节点主动拉取）
         */
        public void pullReplication() {
            while (true) {
                BinlogEvent event = master.getBinlogEvent(replicatedPosition + 1);
                if (event == null) {
                    break;
                }
                applyEvent(event);
                replicatedPosition = event.position();
            }
        }
        
        public Object get(String table, String id) {
            return data.get(table + ":" + id);
        }
        
        public long getReplicationLag() {
            // 返回与主节点的延迟
            return 0;
        }
    }
    
    // 辅助类
    static class BinaryLog {
        private final Map<Long, BinlogEvent> events = new ConcurrentHashMap<>();
        
        void write(BinlogEvent event) {
            events.put(event.position(), event);
        }
        
        BinlogEvent read(long position) {
            return events.get(position);
        }
    }
    
    record BinlogEvent(
        long position,
        StatementType type,
        String table,
        Map<String, Object> data,
        long timestamp
    ) {}
    
    enum StatementType {
        INSERT, UPDATE, DELETE
    }
    
    record ParsedStatement(
        StatementType type,
        String table,
        Map<String, Object> data
    ) {}
}

3.2 缓存同步

/**
 * 分布式缓存同步
 */
public class DistributedCacheSync {
    
    private final String nodeId;
    private final Map<String, CacheEntry> cache;
    private final List<DistributedCacheSync> peers;
    private final ExecutorService executor;
    
    // 失效队列
    private final BlockingQueue<InvalidationMessage> invalidationQueue;
    
    public DistributedCacheSync(String nodeId) {
        this.nodeId = nodeId;
        this.cache = new ConcurrentHashMap<>();
        this.peers = new CopyOnWriteArrayList<>();
        this.executor = Executors.newCachedThreadPool();
        this.invalidationQueue = new LinkedBlockingQueue<>();
        
        startInvalidationProcessor();
    }
    
    /**
     * 添加对等节点
     */
    public void addPeer(DistributedCacheSync peer) {
        peers.add(peer);
    }
    
    /**
     * 获取缓存
     */
    public Object get(String key) {
        CacheEntry entry = cache.get(key);
        if (entry == null) {
            return null;
        }
        
        if (entry.isExpired()) {
            cache.remove(key);
            return null;
        }
        
        return entry.value;
    }
    
    /**
     * 写入缓存
     */
    public void put(String key, Object value, long ttlMs) {
        CacheEntry entry = new CacheEntry(
            value,
            System.currentTimeMillis() + ttlMs,
            nodeId
        );
        
        cache.put(key, entry);
        
        // 通知其他节点失效
        broadcastInvalidation(key);
    }
    
    /**
     * 删除缓存
     */
    public void remove(String key) {
        cache.remove(key);
        broadcastInvalidation(key);
    }
    
    /**
     * 广播失效消息
     */
    private void broadcastInvalidation(String key) {
        InvalidationMessage message = new InvalidationMessage(
            nodeId,
            key,
            System.currentTimeMillis()
        );
        
        for (DistributedCacheSync peer : peers) {
            executor.submit(() -> peer.receiveInvalidation(message));
        }
    }
    
    /**
     * 接收失效消息
     */
    public void receiveInvalidation(InvalidationMessage message) {
        invalidationQueue.offer(message);
    }
    
    /**
     * 启动失效处理器
     */
    private void startInvalidationProcessor() {
        executor.submit(() -> {
            while (!Thread.currentThread().isInterrupted()) {
                try {
                    InvalidationMessage message = invalidationQueue.take();
                    
                    // 删除本地缓存
                    CacheEntry entry = cache.get(message.key());
                    if (entry != null && !entry.owner.equals(message.sourceNodeId())) {
                        cache.remove(message.key());
                        System.out.println("[Cache-" + nodeId + "] Invalidated: " + message.key());
                    }
                    
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
            }
        });
    }
    
    public void shutdown() {
        executor.shutdown();
    }
    
    // 内部类
    record CacheEntry(Object value, long expireTime, String owner) {
        boolean isExpired() {
            return System.currentTimeMillis() > expireTime;
        }
    }
    
    record InvalidationMessage(String sourceNodeId, String key, long timestamp) {}
}

// 缓存同步演示
public class CacheSyncDemo {
    public static void main(String[] args) throws InterruptedException {
        // 创建三个缓存节点
        DistributedCacheSync cache1 = new DistributedCacheSync("cache-1");
        DistributedCacheSync cache2 = new DistributedCacheSync("cache-2");
        DistributedCacheSync cache3 = new DistributedCacheSync("cache-3");
        
        // 设置互相同步
        cache1.addPeer(cache2);
        cache1.addPeer(cache3);
        cache2.addPeer(cache1);
        cache2.addPeer(cache3);
        cache3.addPeer(cache1);
        cache3.addPeer(cache2);
        
        // Cache-1 写入
        System.out.println("=== Cache-1 writes ===");
        cache1.put("user:001", "张三", 60000);
        cache1.put("user:002", "李四", 60000);
        
        Thread.sleep(100);
        
        // 检查同步
        System.out.println("\n=== Check sync ===");
        System.out.println("Cache-1 - user:001: " + cache1.get("user:001"));
        System.out.println("Cache-2 - user:001: " + cache2.get("user:001"));
        System.out.println("Cache-3 - user:001: " + cache3.get("user:001"));
        
        // Cache-2 更新
        System.out.println("\n=== Cache-2 updates ===");
        cache2.put("user:001", "王五", 60000);
        
        Thread.sleep(100);
        
        System.out.println("\n=== After update ===");
        System.out.println("Cache-1 - user:001: " + cache1.get("user:001"));
        System.out.println("Cache-2 - user:001: " + cache2.get("user:002"));
        
        // 关闭
        cache1.shutdown();
        cache2.shutdown();
        cache3.shutdown();
    }
}

---

四、总结与最佳实践

数据同步方案对比

方案	一致性	延迟	可用性	适用场景
同步复制	强一致	高	低	金融交易、关键数据
异步复制	最终一致	低	高	日志、非关键数据
增量同步	最终一致	低	高	大数据同步、数据仓库
双向同步	最终一致	低	高	多活架构、异地容灾

最佳实践

1. 选择合适的同步策略

   • 强一致需求：同步复制
   • 高可用需求：异步复制
   • 多活需求：双向同步

2. 监控同步延迟

   • 设置告警阈值
   • 及时发现同步问题

3. 处理数据冲突

   • 定义冲突解决策略
   • 使用向量时钟或时间戳

4. 容错设计

   • 重试机制
   • 死信队列
   • 人工介入流程

---

五、思考与练习

思考题

1. 基础题：同步复制和异步复制各有什么优缺点？如何根据业务场景选择？
2. 进阶题：双向同步中出现数据冲突时，有哪些常见的冲突解决策略？向量时钟如何帮助判断因果关系？
3. 实战题：设计一个跨地区的数据同步方案，要求支持三地数据中心、读写分离、故障自动切换。

编程练习

练习：实现一个基于CDC的数据同步系统，要求：

1. 监听数据库变更事件（INSERT/UPDATE/DELETE）
2. 将变更同步到目标数据库
3. 支持断点续传和故障恢复
4. 实现幂等性保证

章节关联

• 前置章节：《最终一致性方案详解》
• 后续章节：《短链系统设计详解》
• 扩展阅读：Debezium官方文档、Canal原理与实战

---

📝 下一章预告

下一章将进入系统设计案例篇，首先学习短链系统的设计，掌握短链生成算法、存储方案、高并发优化等核心技术。

---

本章完