【kafka源码】LogSegment 日志段

提示：本文可能已过期,请点击原文查看：【kafka源码】LogSegment 日志段

推荐一款非常好用的kafka管理平台,kafka的灵魂伴侣
滴滴开源Logi-KafkaManager 一站式Kafka监控与管控平台

recoverLog

恢复日志段，恢复后返回下一个偏移量。此方法不需要将 IOException 转换为 KafkaStorageException，因为它只在所有日志加载之前调用。


  private def recoverLog(): Long = {
    // 如果有 .kafka_cleanshutdown文件。表示是正常关闭的；则跳过恢复阶段
    if (!hasCleanShutdownFile) {
      // 根据recovery-point-offset-checkpoint得到的值,算出哪些值是没有被写入到recovery-point-offset-checkpoint中的segment
      val unflushed = logSegments(this.recoveryPoint, Long.MaxValue).toIterator
      var truncated = false

      while (unflushed.hasNext && !truncated) {
        val segment = unflushed.next
        info(s"Recovering unflushed segment ${segment.baseOffset}")
        val truncatedBytes =
          try {
            recoverSegment(segment, leaderEpochCache)
          } catch {
            case _: InvalidOffsetException =>
              val startOffset = segment.baseOffset
              warn("Found invalid offset during recovery. Deleting the corrupt segment and " +
                s"creating an empty one with starting offset $startOffset")
              segment.truncateTo(startOffset)
          }
        if (truncatedBytes > 0) {
          // we had an invalid message, delete all remaining log
          warn(s"Corruption found in segment ${segment.baseOffset}, truncating to offset ${segment.readNextOffset}")
          removeAndDeleteSegments(unflushed.toList, asyncDelete = true)
          truncated = true
        }
      }
    }

    if (logSegments.nonEmpty) {
      val logEndOffset = activeSegment.readNextOffset
      if (logEndOffset < logStartOffset) {
        warn(s"Deleting all segments because logEndOffset ($logEndOffset) is smaller than logStartOffset ($logStartOffset). " +
          "This could happen if segment files were deleted from the file system.")
        removeAndDeleteSegments(logSegments, asyncDelete = true)
      }
    }

    if (logSegments.isEmpty) {
      // no existing segments, create a new mutable segment beginning at logStartOffset
      addSegment(LogSegment.open(dir = dir,
        baseOffset = logStartOffset,
        config,
        time = time,
        fileAlreadyExists = false,
        initFileSize = this.initFileSize,
        preallocate = config.preallocate))
    }

    recoveryPoint = activeSegment.readNextOffset
    recoveryPoint
  }

recover 恢复日志段

Broker 在启动时会从磁盘上加载所有日志段信息到内存中，并创建相应的 LogSegment 对象实例。

将给定的Segment执行恢复操作; 这将从日志文件重建索引并从日志和索引的末尾删除任何无效字节。

def recover(producerStateManager: ProducerStateManager, leaderEpochCache: Option[LeaderEpochFileCache] = None): Int = {
    offsetIndex.reset()
    timeIndex.reset()
    txnIndex.reset()
    var validBytes = 0
    var lastIndexEntry = 0
    maxTimestampSoFar = RecordBatch.NO_TIMESTAMP
    try {
      for (batch <- log.batches.asScala) {
        batch.ensureValid()
        ensureOffsetInRange(batch.lastOffset)

        // The max timestamp is exposed at the batch level, so no need to iterate the records
        if (batch.maxTimestamp > maxTimestampSoFar) {
          maxTimestampSoFar = batch.maxTimestamp
          offsetOfMaxTimestampSoFar = batch.lastOffset
        }

        // Build offset index
        if (validBytes - lastIndexEntry > indexIntervalBytes) {
          offsetIndex.append(batch.lastOffset, validBytes)
          timeIndex.maybeAppend(maxTimestampSoFar, offsetOfMaxTimestampSoFar)
          lastIndexEntry = validBytes
        }
        validBytes += batch.sizeInBytes()

        if (batch.magic >= RecordBatch.MAGIC_VALUE_V2) {
          leaderEpochCache.foreach { cache =>
            if (batch.partitionLeaderEpoch > 0 && cache.latestEpoch.forall(batch.partitionLeaderEpoch > _))
              cache.assign(batch.partitionLeaderEpoch, batch.baseOffset)
          }
          updateProducerState(producerStateManager, batch)
        }
      }
    } catch {
      case e@ (_: CorruptRecordException | _: InvalidRecordException) =>
        warn("Found invalid messages in log segment %s at byte offset %d: %s. %s"
          .format(log.file.getAbsolutePath, validBytes, e.getMessage, e.getCause))
    }
    val truncated = log.sizeInBytes - validBytes
    if (truncated > 0)
      debug(s"Truncated $truncated invalid bytes at the end of segment ${log.file.getAbsoluteFile} during recovery")

    //截取有效文件
    log.truncateTo(validBytes)
    offsetIndex.trimToValidSize()
    // A normally closed segment always appends the biggest timestamp ever seen into log segment, we do this as well.
    timeIndex.maybeAppend(maxTimestampSoFar, offsetOfMaxTimestampSoFar, skipFullCheck = true)
    timeIndex.trimToValidSize()
    truncated
  }

先初始化所有索引文件对象为空 .index、.timeindex、.txnindex ;
之后会开始遍历日志段中的所有消息集合或消息批次（RecordBatch）,确保其是合法的 ①.魔数值合法 ②.位移值没有越界, 最后一条消息的offset与日志段起始值offset差值必须是一个正整数
获取最大时间戳及所属消息位移、更新总消息字节数、更新Porducer状态和Leader Epoch缓存
然后添加索引项，时间索引根据实际情况判断是否添加，位移索引添加一个索引项。
遍历结束后，truncateTo文件截取；判断当前总字节数,和刚刚累计读取到的字节数(validBytes)，判断是否需要截取，比如上次宕机导致脏数据写入等等：根据当前累计读取到的数据大小去截取文件
修剪此段的.index、.timeindex文件,以仅适合有效条目，从文件中删除所有尾部未写入的字节。

kafka非常善于使用mmap技术和使用基于FileChannel接口使用的文件操作，这样可以很好的使用零拷贝技术，mmap(一种内存映射文件的方法)，能使其操作大文件的时候更加快速高效。