16.2 太牛了！分库分表和智能分片竟然还能这样实现？
在分布式任务调度系统中，随着业务规模的增长，单一数据库往往无法满足海量数据存储和高并发访问的需求。分库分表技术是解决这一问题的关键手段。今天我们将深入探讨如何实现分库分表和智能分片策略。
分库分表架构设计
分库分表是一种将数据分散存储在多个数据库和表中的技术，能够有效提升系统的存储容量和并发处理能力。
package sharding
import (
"database/sql"
"fmt"
"hash/crc32"
"math"
"sort"
"strconv"
"strings"
"sync"
"time"
)
// ShardingConfig 分库分表配置
type ShardingConfig struct {
DatabaseCount int
TableCount int
ShardingKey string
ShardingType ShardingType
DatabasePrefix string
TablePrefix string
}
// ShardingType 分片类型
type ShardingType int
const (
ShardingTypeMod ShardingType = iota // 取模分片
ShardingTypeRange // 范围分片
ShardingTypeHash // 哈希分片
)
// ShardingManager 分库分表管理器
type ShardingManager struct {
config *ShardingConfig
databases map[string]*sql.DB
mutex sync.RWMutex
router *ShardingRouter
}
// ShardingRouter 分片路由
type ShardingRouter struct {
config *ShardingConfig
}
// NewShardingManager 创建分库分表管理器
func NewShardingManager(config *ShardingConfig) *ShardingManager {
router := &ShardingRouter{
config: config,
}
return &ShardingManager{
config: config,
databases: make(map[string]*sql.DB),
router: router,
}
}
// AddDatabase 添加数据库连接
func (sm *ShardingManager) AddDatabase(dbName string, db *sql.DB) {
sm.mutex.Lock()
defer sm.mutex.Unlock()
sm.databases[dbName] = db
}
// GetDatabase 获取数据库连接
func (sm *ShardingManager) GetDatabase(dbIndex int) (*sql.DB, error) {
sm.mutex.RLock()
defer sm.mutex.RUnlock()
dbName := fmt.Sprintf("%s_%d", sm.config.DatabasePrefix, dbIndex)
db, exists := sm.databases[dbName]
if !exists {
return nil, fmt.Errorf("database %s not found", dbName)
}
return db, nil
}
// Route 路由分片
func (sm *ShardingManager) Route(shardingValue interface{}) (*ShardLocation, error) {
return sm.router.Route(shardingValue)
}
// ShardLocation 分片位置
type ShardLocation struct {
DatabaseIndex int
TableIndex int
DatabaseName string
TableName string
}
// Route 路由分片
func (sr *ShardingRouter) Route(shardingValue interface{}) (*ShardLocation, error) {
var dbIndex, tableIndex int
switch sr.config.ShardingType {
case ShardingTypeMod:
dbIndex, tableIndex = sr.modSharding(shardingValue)
case ShardingTypeRange:
dbIndex, tableIndex = sr.rangeSharding(shardingValue)
case ShardingTypeHash:
dbIndex, tableIndex = sr.hashSharding(shardingValue)
default:
return nil, fmt.Errorf("unsupported sharding type: %v", sr.config.ShardingType)
}
location := &ShardLocation{
DatabaseIndex: dbIndex,
TableIndex: tableIndex,
DatabaseName: fmt.Sprintf("%s_%d", sr.config.DatabasePrefix, dbIndex),
TableName: fmt.Sprintf("%s_%d", sr.config.TablePrefix, tableIndex),
}
return location, nil
}
// modSharding 取模分片
func (sr *ShardingRouter) modSharding(shardingValue interface{}) (int, int) {
// 将分片值转换为整数
var value int64
switch v := shardingValue.(type) {
case int:
value = int64(v)
case int64:
value = v
case string:
// 对字符串进行哈希
value = int64(crc32.ChecksumIEEE([]byte(v)))
default:
// 默认使用字符串表示
value = int64(crc32.ChecksumIEEE([]byte(fmt.Sprintf("%v", v))))
}
// 计算数据库和表索引
dbIndex := int(value % int64(sr.config.DatabaseCount))
tableIndex := int((value / int64(sr.config.DatabaseCount)) % int64(sr.config.TableCount))
return dbIndex, tableIndex
}
// rangeSharding 范围分片
func (sr *ShardingRouter) rangeSharding(shardingValue interface{}) (int, int) {
var value int64
switch v := shardingValue.(type) {
case int:
value = int64(v)
case int64:
value = v
case string:
// 尝试解析为整数
if i, err := strconv.ParseInt(v, 10, 64); err == nil {
value = i
} else {
// 否则使用哈希
value = int64(crc32.ChecksumIEEE([]byte(v)))
}
default:
// 默认使用字符串表示
if s, ok := v.(fmt.Stringer); ok {
if i, err := strconv.ParseInt(s.String(), 10, 64); err == nil {
value = i
} else {
value = int64(crc32.ChecksumIEEE([]byte(s.String())))
}
} else {
value = int64(crc32.ChecksumIEEE([]byte(fmt.Sprintf("%v", v))))
}
}
// 假设我们按值范围分片，每1000000为一个范围
rangeSize := int64(1000000)
rangeIndex := int(value / rangeSize)
// 计算数据库和表索引
dbIndex := rangeIndex % sr.config.DatabaseCount
tableIndex := (rangeIndex / sr.config.DatabaseCount) % sr.config.TableCount
return dbIndex, tableIndex
}
// hashSharding 哈希分片
func (sr *ShardingRouter) hashSharding(shardingValue interface{}) (int, int) {
var strValue string
switch v := shardingValue.(type) {
case string:
strValue = v
default:
strValue = fmt.Sprintf("%v", v)
}
// 使用CRC32哈希
hashValue := int64(crc32.ChecksumIEEE([]byte(strValue)))
// 计算数据库和表索引
dbIndex := int(hashValue % int64(sr.config.DatabaseCount))
tableIndex := int((hashValue / int64(sr.config.DatabaseCount)) % int64(sr.config.TableCount))
return dbIndex, tableIndex
}
// CreateTable 创建分片表
func (sm *ShardingManager) CreateTable(tableName string) error {
// 为每个数据库和表组合创建表
for dbIndex := 0; dbIndex < sm.config.DatabaseCount; dbIndex++ {
db, err := sm.GetDatabase(dbIndex)
if err != nil {
return fmt.Errorf("failed to get database %d: %v", dbIndex, err)
}
for tableIndex := 0; tableIndex < sm.config.TableCount; tableIndex++ {
shardTableName := fmt.Sprintf("%s_%d", sm.config.TablePrefix, tableIndex)
sql := sm.generateCreateTableSQL(shardTableName)
_, err := db.Exec(sql)
if err != nil {
return fmt.Errorf("failed to create table %s in database %d: %v", shardTableName, dbIndex, err)
}
}
}
return nil
}
// generateCreateTableSQL 生成创建表SQL
func (sm *ShardingManager) generateCreateTableSQL(tableName string) string {
// 这里是一个示例，实际应用中需要根据具体需求调整
return fmt.Sprintf(`
CREATE TABLE IF NOT EXISTS %s (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
task_id VARCHAR(64) NOT NULL,
job_id VARCHAR(64) NOT NULL,
status VARCHAR(32) NOT NULL,
payload JSON,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_task_id (task_id),
INDEX idx_job_id (job_id),
INDEX idx_status (status),
INDEX idx_created_at (created_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
`, tableName)
}
// 使用示例
func ExampleShardingManager() {
// 创建分库分表配置
config := &ShardingConfig{
DatabaseCount: 4,
TableCount: 8,
ShardingKey: "task_id",
ShardingType: ShardingTypeMod,
DatabasePrefix: "task_db",
TablePrefix: "task_table",
}
// 创建分库分表管理器
sm := NewShardingManager(config)
// 模拟添加数据库连接
// 在实际应用中，这里会是真实的数据库连接
for i := 0; i < config.DatabaseCount; i++ {
dbName := fmt.Sprintf("%s_%d", config.DatabasePrefix, i)
// db, err := sql.Open("mysql", fmt.Sprintf("user:password@tcp(localhost:3306)/%s", dbName))
// if err != nil {
// panic(err)
// }
// sm.AddDatabase(dbName, db)
fmt.Printf("Added database: %s\n", dbName)
}
// 测试路由分片
testValues := []interface{}{
"task_001",
"task_002",
12345,
"job_999_task_888",
}
for _, value := range testValues {
location, err := sm.Route(value)
if err != nil {
fmt.Printf("Failed to route value %v: %v\n", value, err)
continue
}
fmt.Printf("Value %v routes to database %s (index %d), table %s (index %d)\n",
value, location.DatabaseName, location.DatabaseIndex, location.TableName, location.TableIndex)
}
fmt.Println("Sharding manager example completed")
}
智能分片算法
智能分片算法能够根据数据特征和访问模式动态调整分片策略，以达到最佳的负载均衡效果。
package intelligent
import (
"fmt"
"math"
"sort"
"sync"
"time"
)
// IntelligentSharding 智能分片
type IntelligentSharding struct {
shards map[string]*ShardInfo
accessStats map[string]*AccessStats
mu sync.RWMutex
rebalanceChan chan *RebalanceRequest
stopChan chan struct{}
}
// ShardInfo 分片信息
type ShardInfo struct {
ID string
Size int64
NodeID string
CreatedAt time.Time
LastRebalanced time.Time
}
// AccessStats 访问统计
type AccessStats struct {
ShardID string
ReadCount int64
WriteCount int64
LastAccess time.Time
AvgLatency time.Duration
ErrorCount int64
}
// RebalanceRequest 重平衡请求
type RebalanceRequest struct {
Reason RebalanceReason
Urgent bool
Timestamp time.Time
}
// RebalanceReason 重平衡原因
type RebalanceReason int
const (
RebalanceReasonSizeImbalance RebalanceReason = iota
RebalanceReasonAccessImbalance
RebalanceReasonNodeFailure
RebalanceReasonNewNode
)
// NewIntelligentSharding 创建智能分片
func NewIntelligentSharding() *IntelligentSharding {
is := &IntelligentSharding{
shards: make(map[string]*ShardInfo),
accessStats: make(map[string]*AccessStats),
rebalanceChan: make(chan *RebalanceRequest, 10),
stopChan: make(chan struct{}),
}
// 启动重平衡协程
go is.rebalanceWorker()
return is
}
// AddShard 添加分片
func (is *IntelligentSharding) AddShard(shard *ShardInfo) error {
is.mu.Lock()
defer is.mu.Unlock()
if _, exists := is.shards[shard.ID]; exists {
return fmt.Errorf("shard %s already exists", shard.ID)
}
is.shards[shard.ID] = shard
// 初始化访问统计
is.accessStats[shard.ID] = &AccessStats{
ShardID: shard.ID,
LastAccess: time.Now(),
}
return nil
}
// UpdateShardSize 更新分片大小
func (is *IntelligentSharding) UpdateShardSize(shardID string, size int64) error {
is.mu.Lock()
defer is.mu.Unlock()
shard, exists := is.shards[shardID]
if !exists {
return fmt.Errorf("shard %s not found", shardID)
}
oldSize := shard.Size
shard.Size = size
// 如果大小变化超过阈值，触发重平衡
if math.Abs(float64(size-oldSize)) > float64(oldSize)*0.2 { // 20%阈值
is.rebalanceChan <- &RebalanceRequest{
Reason: RebalanceReasonSizeImbalance,
Urgent: size > oldSize,
Timestamp: time.Now(),
}
}
return nil
}
// RecordAccess 记录访问
func (is *IntelligentSharding) RecordAccess(shardID string, isWrite bool, latency time.Duration, err error) error {
is.mu.Lock()
defer is.mu.Unlock()
stats, exists := is.accessStats[shardID]
if !exists {
return fmt.Errorf("shard %s not found", shardID)
}
stats.LastAccess = time.Now()
stats.AvgLatency = (stats.AvgLatency + latency) / 2 // 简化的平均延迟计算
if isWrite {
stats.WriteCount++
} else {
stats.ReadCount++
}
if err != nil {
stats.ErrorCount++
}
return nil
}
// AnalyzeAndRebalance 分析并重平衡
func (is *IntelligentSharding) AnalyzeAndRebalance() {
is.mu.RLock()
// 计算分片大小分布
var sizes []int64
sizeMap := make(map[string]int64)
for id, shard := range is.shards {
sizes = append(sizes, shard.Size)
sizeMap[id] = shard.Size
}
// 计算访问频率分布
var accessFreqs []int64
accessMap := make(map[string]int64)
for id, stats := range is.accessStats {
freq := stats.ReadCount + stats.WriteCount
accessFreqs = append(accessFreqs, freq)
accessMap[id] = freq
}
is.mu.RUnlock()
// 分析是否需要重平衡
if is.needRebalance(sizeMap, accessMap) {
is.rebalanceChan <- &RebalanceRequest{
Reason: RebalanceReasonAccessImbalance,
Urgent: false,
Timestamp: time.Now(),
}
}
}
// needRebalance 是否需要重平衡
func (is *IntelligentSharding) needRebalance(sizeMap, accessMap map[string]int64) bool {
if len(sizeMap) == 0 {
return false
}
// 计算平均值和标准差
var sizeSum, accessSum int64
for _, size := range sizeMap {
sizeSum += size
}
for _, access := range accessMap {
accessSum += access
}
sizeAvg := float64(sizeSum) / float64(len(sizeMap))
accessAvg := float64(accessSum) / float64(len(accessMap))
// 计算标准差
var sizeVariance, accessVariance float64
for _, size := range sizeMap {
sizeVariance += math.Pow(float64(size)-sizeAvg, 2)
}
for _, access := range accessMap {
accessVariance += math.Pow(float64(access)-accessAvg, 2)
}
sizeStdDev := math.Sqrt(sizeVariance / float64(len(sizeMap)))
accessStdDev := math.Sqrt(accessVariance / float64(len(accessMap)))
// 如果标准差超过平均值的30%，需要重平衡
return sizeStdDev > sizeAvg*0.3 || accessStdDev > accessAvg*0.3
}
// rebalanceWorker 重平衡工作协程
func (is *IntelligentSharding) rebalanceWorker() {
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
for {
select {
case req := <-is.rebalanceChan:
is.performRebalance(req)
case <-ticker.C:
// 定期分析
is.AnalyzeAndRebalance()
case <-is.stopChan:
return
}
}
}
// performRebalance 执行重平衡
func (is *IntelligentSharding) performRebalance(req *RebalanceRequest) {
is.mu.Lock()
defer is.mu.Unlock()
fmt.Printf("Performing rebalance due to %v (urgent: %v)\n", req.Reason, req.Urgent)
// 根据重平衡原因执行不同的策略
switch req.Reason {
case RebalanceReasonSizeImbalance:
is.rebalanceBySize()
case RebalanceReasonAccessImbalance:
is.rebalanceByAccess()
case RebalanceReasonNodeFailure:
is.rebalanceByNodeFailure()
case RebalanceReasonNewNode:
is.rebalanceByNewNode()
}
// 更新重平衡时间
now := time.Now()
for _, shard := range is.shards {
shard.LastRebalanced = now
}
}
// rebalanceBySize 按大小重平衡
func (is *IntelligentSharding) rebalanceBySize() {
// 按大小排序分片
type shardSize struct {
ID string
Size int64
}
var shardSizes []shardSize
for id, shard := range is.shards {
shardSizes = append(shardSizes, shardSize{ID: id, Size: shard.Size})
}
sort.Slice(shardSizes, func(i, j int) bool {
return shardSizes[i].Size > shardSizes[j].Size
})
// 将大分片拆分或迁移
fmt.Printf("Rebalancing by size, largest shards: %v\n", shardSizes[:int(math.Min(3, float64(len(shardSizes))))])
}
// rebalanceByAccess 按访问频率重平衡
func (is *IntelligentSharding) rebalanceByAccess() {
// 按访问频率排序分片
type shardAccess struct {
ID string
Count int64
}
var shardAccesses []shardAccess
for id, stats := range is.accessStats {
count := stats.ReadCount + stats.WriteCount
shardAccesses = append(shardAccesses, shardAccess{ID: id, Count: count})
}
sort.Slice(shardAccesses, func(i, j int) bool {
return shardAccesses[i].Count > shardAccesses[j].Count
})
// 将热点分片迁移或复制
fmt.Printf("Rebalancing by access, hottest shards: %v\n", shardAccesses[:int(math.Min(3, float64(len(shardAccesses))))])
}
// rebalanceByNodeFailure 按节点故障重平衡
func (is *IntelligentSharding) rebalanceByNodeFailure() {
fmt.Println("Rebalancing due to node failure")
// 实际应用中需要重新分配故障节点上的分片
}
// rebalanceByNewNode 按新增节点重平衡
func (is *IntelligentSharding) rebalanceByNewNode() {
fmt.Println("Rebalancing due to new node")
// 实际应用中需要将部分分片迁移到新节点
}
// Stop 停止智能分片
func (is *IntelligentSharding) Stop() {
close(is.stopChan)
}
// 使用示例
func ExampleIntelligentSharding() {
is := NewIntelligentSharding()
defer is.Stop()
// 添加分片
shards := []*ShardInfo{
{ID: "shard_1", Size: 1000, NodeID: "node_1", CreatedAt: time.Now()},
{ID: "shard_2", Size: 2000, NodeID: "node_1", CreatedAt: time.Now()},
{ID: "shard_3", Size: 500, NodeID: "node_2", CreatedAt: time.Now()},
{ID: "shard_4", Size: 1500, NodeID: "node_2", CreatedAt: time.Now()},
}
for _, shard := range shards {
if err := is.AddShard(shard); err != nil {
fmt.Printf("Failed to add shard %s: %v\n", shard.ID, err)
}
}
// 模拟访问记录
is.RecordAccess("shard_1", false, 10*time.Millisecond, nil)
is.RecordAccess("shard_1", true, 20*time.Millisecond, nil)
is.RecordAccess("shard_2", false, 5*time.Millisecond, nil)
is.RecordAccess("shard_3", true, 15*time.Millisecond, fmt.Errorf("error"))
// 更新分片大小
is.UpdateShardSize("shard_1", 3000) // 大小变化较大，触发重平衡
// 等待处理
time.Sleep(100 * time.Millisecond)
fmt.Println("Intelligent sharding example completed")
}
数据清理和归档策略
随着系统运行时间的增长，历史数据会占用大量存储空间。我们需要实现数据清理和归档策略来管理这些数据。
package cleanup
import (
"fmt"
"time"
)
// CleanupManager 数据清理管理器
type CleanupManager struct {
policies map[string]*CleanupPolicy
archivers map[string]*Archiver
}
// CleanupPolicy 清理策略
type CleanupPolicy struct {
Name string
TablePattern string
RetentionDays int
ArchiveBeforeCleanup bool
ArchiveDestination string
Enabled bool
LastRun time.Time
}
// Archiver 归档器
type Archiver struct {
Name string
Source string
Destination string
Format ArchiveFormat
Compress bool
}
// ArchiveFormat 归档格式
type ArchiveFormat int
const (
ArchiveFormatCSV ArchiveFormat = iota
ArchiveFormatJSON
ArchiveFormatParquet
)
// NewCleanupManager 创建数据清理管理器
func NewCleanupManager() *CleanupManager {
return &CleanupManager{
policies: make(map[string]*CleanupPolicy),
archivers: make(map[string]*Archiver),
}
}
// AddPolicy 添加清理策略
func (cm *CleanupManager) AddPolicy(policy *CleanupPolicy) error {
if _, exists := cm.policies[policy.Name]; exists {
return fmt.Errorf("policy %s already exists", policy.Name)
}
cm.policies[policy.Name] = policy
return nil
}
// AddArchiver 添加归档器
func (cm *CleanupManager) AddArchiver(archiver *Archiver) error {
if _, exists := cm.archivers[archiver.Name]; exists {
return fmt.Errorf("archiver %s already exists", archiver.Name)
}
cm.archivers[archiver.Name] = archiver
return nil
}
// RunCleanup 运行数据清理
func (cm *CleanupManager) RunCleanup(policyName string) error {
policy, exists := cm.policies[policyName]
if !exists {
return fmt.Errorf("policy %s not found", policyName)
}
if !policy.Enabled {
return fmt.Errorf("policy %s is disabled", policyName)
}
fmt.Printf("Running cleanup for policy: %s\n", policy.Name)
// 计算截止日期
cutoffDate := time.Now().Add(-time.Duration(policy.RetentionDays) * 24 * time.Hour)
fmt.Printf("Deleting data older than: %v\n", cutoffDate)
// 如果需要先归档
if policy.ArchiveBeforeCleanup {
if err := cm.archiveData(policy, cutoffDate); err != nil {
return fmt.Errorf("failed to archive data: %v", err)
}
}
// 执行清理操作
if err := cm.performCleanup(policy, cutoffDate); err != nil {
return fmt.Errorf("failed to perform cleanup: %v", err)
}
// 更新最后运行时间
policy.LastRun = time.Now()
return nil
}
// archiveData 归档数据
func (cm *CleanupManager) archiveData(policy *CleanupPolicy, cutoffDate time.Time) error {
fmt.Printf("Archiving data older than %v to %s\n", cutoffDate, policy.ArchiveDestination)
// 查找对应的归档器
archiver, exists := cm.archivers[policy.Name]
if !exists {
return fmt.Errorf("archiver for policy %s not found", policy.Name)
}
// 执行归档操作
fmt.Printf("Using archiver %s to archive data from %s to %s\n",
archiver.Name, archiver.Source, archiver.Destination)
// 模拟归档过程
time.Sleep(100 * time.Millisecond)
return nil
}
// performCleanup 执行清理
func (cm *CleanupManager) performCleanup(policy *CleanupPolicy, cutoffDate time.Time) error {
fmt.Printf("Deleting data older than %v from tables matching pattern %s\n",
cutoffDate, policy.TablePattern)
// 模拟清理过程
time.Sleep(100 * time.Millisecond)
return nil
}
// ScheduleCleanup 调度清理任务
func (cm *CleanupManager) ScheduleCleanup(policyName string, interval time.Duration) {
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for range ticker.C {
if err := cm.RunCleanup(policyName); err != nil {
fmt.Printf("Failed to run cleanup for policy %s: %v\n", policyName, err)
}
}
}()
}
// CleanupPolicyBuilder 清理策略构建器
type CleanupPolicyBuilder struct {
policy *CleanupPolicy
}
// NewCleanupPolicyBuilder 创建清理策略构建器
func NewCleanupPolicyBuilder(name string) *CleanupPolicyBuilder {
return &CleanupPolicyBuilder{
policy: &CleanupPolicy{
Name: name,
RetentionDays: 30, // 默认保留30天
Enabled: true,
}
}
}
// WithTablePattern 设置表模式
func (b *CleanupPolicyBuilder) WithTablePattern(pattern string) *CleanupPolicyBuilder {
b.policy.TablePattern = pattern
return b
}
// WithRetentionDays 设置保留天数
func (b *CleanupPolicyBuilder) WithRetentionDays(days int) *CleanupPolicyBuilder {
b.policy.RetentionDays = days
return b
}
// WithArchiveBeforeCleanup 设置清理前归档
func (b *CleanupPolicyBuilder) WithArchiveBeforeCleanup(destination string) *CleanupPolicyBuilder {
b.policy.ArchiveBeforeCleanup = true
b.policy.ArchiveDestination = destination
return b
}
// WithEnabled 设置启用状态
func (b *CleanupPolicyBuilder) WithEnabled(enabled bool) *CleanupPolicyBuilder {
b.policy.Enabled = enabled
return b
}
// Build 构建清理策略
func (b *CleanupPolicyBuilder) Build() *CleanupPolicy {
return b.policy
}
// 使用示例
func ExampleCleanupManager() {
cm := NewCleanupManager()
// 使用构建器创建清理策略
policy := NewCleanupPolicyBuilder("task_cleanup").
WithTablePattern("task_table_%").
WithRetentionDays(90).
WithArchiveBeforeCleanup("/archive/tasks").
WithEnabled(true).
Build()
// 添加策略
if err := cm.AddPolicy(policy); err != nil {
fmt.Printf("Failed to add policy: %v\n", err)
return
}
// 添加归档器
archiver := &Archiver{
Name: "task_archiver",
Source: "task_table_%",
Destination: "/archive/tasks",
Format: ArchiveFormatParquet,
Compress: true,
}
if err := cm.AddArchiver(archiver); err != nil {
fmt.Printf("Failed to add archiver: %v\n", err)
return
}
// 运行清理
if err := cm.RunCleanup("task_cleanup"); err != nil {
fmt.Printf("Failed to run cleanup: %v\n", err)
return
}
// 调度定期清理（每24小时）
cm.ScheduleCleanup("task_cleanup", 24*time.Hour)
fmt.Println("Cleanup manager example completed")
fmt.Println("Cleanup task scheduled to run every 24 hours")
}
总结
今天我们深入探讨了分布式任务调度系统中的分库分表和数据管理技术：
-
分库分表架构设计：
- 实现了取模、范围和哈希三种分片策略
- 设计了分片路由机制，能够根据分片键定位数据
- 支持自动创建分片表结构
-
智能分片算法：
- 实现了基于数据大小和访问频率的智能重平衡
- 支持多种重平衡触发条件（大小不平衡、访问不平衡等）
- 通过后台协程自动执行重平衡操作
-
数据清理和归档策略：
- 实现了可配置的数据清理策略
- 支持清理前归档重要数据
- 提供了策略构建器模式，简化配置过程
通过这些技术的组合应用，我们可以构建一个高可扩展、高性能的分布式任务调度系统，能够处理海量任务数据并保持良好的性能表现。在实际应用中，需要根据具体业务场景和数据特征选择合适的分片策略，并持续监控和优化系统性能。
