You’re never too old to set another goal or to dream a new dream.
FFmpeg 的 Demux 模块（解复用模块）负责从容器中分离音视频流、解析元数据并同步时间戳，是音视频解码与处理前的关键解复用过程，支撑转码、流媒体等应用的基础流程。本篇文章我们来深入分析下此模块的实现细节。 老样子，还是先通过一段demo code感受下demux模块的基本用法，然后再深入分析内部实现。
#include <libavformat/avformat.h>
int main(int argc, char *argv[]) {
AVFormatContext *fmt_ctx = NULL;
AVPacket pkt;
int ret, audio_stream_idx = -1;
// 打开输入文件
if ((ret = avformat_open_input(&fmt_ctx, "input.mp3", NULL, NULL)) < 0) {
char errbuf[128];
av_strerror(ret, errbuf, sizeof(errbuf));
fprintf(stderr, "无法打开文件: %s\n", errbuf);
return -1;
}
// 获取流信息
if ((ret = avformat_find_stream_info(fmt_ctx, NULL)) < 0) {
fprintf(stderr, "无法获取流信息\n");
avformat_close_input(&fmt_ctx);
return -1;
}
// 查找音频流索引
for (int i = 0; i < fmt_ctx->nb_streams; i++) {
if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
audio_stream_idx = i;
break;
}
}
if (audio_stream_idx == -1) {
fprintf(stderr, "未找到音频流\n");
avformat_close_input(&fmt_ctx);
return -1;
}
// 循环读取数据包
while (av_read_frame(fmt_ctx, &pkt) >= 0) {
if (pkt.stream_index == audio_stream_idx) {
printf("读取到音频数据包: 大小=%d, 时间戳=%lld\n",
pkt.size, pkt.pts);
// 此处可添加对音频数据的处理逻辑
}
av_packet_unref(&pkt); // 释放数据包
}
// 清理资源
avformat_close_input(&fmt_ctx);
return 0;
}
上面的代码看起来是非常简单易用的。下面提几个问题，我们带着问题深入看下代码，来理解其内部实现。
- demux模块内部如何支持解析不同格式的音频，视频文件的，如: MP3,OPUS,MP4,AVI等？
- avformat_find_stream_info到底find什么信息，为什么一般项目上都觉得这里耗时长，有什么优化手段？
avformat_open_input
graph TD
AVFormatContext --> AVFormatContext.iformat --> AVInputFormat
FFInputFormat --> AVInputFormat
AVFormatContext --> AVFormatContext.pb --> AVIOContext
int avformat_open_input(AVFormatContext **ps, const char *filename,
const AVInputFormat *fmt, AVDictionary **options)
{
AVFormatContext *s = *ps;
FFFormatContext *si;
AVDictionary *tmp = NULL;
ID3v2ExtraMeta *id3v2_extra_meta = NULL;
int ret = 0;
if (!s && !(s = avformat_alloc_context()))
return AVERROR(ENOMEM);
si = ffformatcontext(s);
//如果外部已经找到AVInputFormat，就直接赋值，在后续就不用找了。
if (fmt)
s->iformat = fmt;
if (options)
av_dict_copy(&tmp, *options, 0);
// 如果AVFormatContext是由外部传入，并且pb即AVIOContext不为NULL，
// 即由用户定制了一个私有的AVIOContext
if (s->pb) // must be before any goto fail
s->flags |= AVFMT_FLAG_CUSTOM_IO;
// 把option 设置到AVFormatContext
if ((ret = av_opt_set_dict(s, &tmp)) < 0)
goto fail;
// 赋值url
if (!(s->url = av_strdup(filename ? filename : ""))) {
ret = AVERROR(ENOMEM);
goto fail;
}
//重点函数，接下来细看
if ((ret = init_input(s, filename, &tmp)) < 0)
goto fail;
s->probe_score = ret;
.............................;
s->duration = s->start_time = AV_NOPTS_VALUE;
/* Allocate private data. */
if (ffifmt(s->iformat)->priv_data_size > 0) {
if (!(s->priv_data = av_mallocz(ffifmt(s->iformat)->priv_data_size))) {
ret = AVERROR(ENOMEM);
goto fail;
}
if (s->iformat->priv_class) {
*(const AVClass **) s->priv_data = s->iformat->priv_class;
av_opt_set_defaults(s->priv_data);
if ((ret = av_opt_set_dict(s->priv_data, &tmp)) < 0)
goto fail;
}
}
// id3v2为音视频container中的封面
if (s->pb)
ff_id3v2_read_dict(s->pb, &si->id3v2_meta, ID3v2_DEFAULT_MAGIC, &id3v2_extra_meta);
//执行read_header,此处需要重点注意，这里会创建stream
// 重点！！！， 重点！！！
if (ffifmt(s->iformat)->read_header)
if ((ret = ffifmt(s->iformat)->read_header(s)) < 0) {
if (ffifmt(s->iformat)->flags_internal & FF_INFMT_FLAG_INIT_CLEANUP)
goto close;
goto fail;
}
if (!s->metadata) {
s->metadata = si->id3v2_meta;
si->id3v2_meta = NULL;
} else if (si->id3v2_meta) {
av_log(s, AV_LOG_WARNING, "Discarding ID3 tags because more suitable tags were found.\n");
av_dict_free(&si->id3v2_meta);
}
....................id3v2的处理，先跳过...........................;
si->raw_packet_buffer_size = 0;
//等下重点分析
update_stream_avctx(s);
//这里需要注意，传入进来的options，其内部的选项如果被子模块消耗，则会从options中剔除。
if (options) {
av_dict_free(options);
*options = tmp;
}
*ps = s;
return 0;
close:
......................;
return ret;
}
下面这段代码比较重要,用于分配某个AVInputFormat的private空间，举例:
const FFInputFormat ff_mp3_demuxer = {
.p.name = "mp3",
.p.long_name = NULL_IF_CONFIG_SMALL("MP2/3 (MPEG audio layer 2/3)"),
.p.flags = AVFMT_GENERIC_INDEX,
.p.extensions = "mp2,mp3,m2a,mpa", /* XXX: use probe */
.p.priv_class = &demuxer_class,
.read_probe = mp3_read_probe,
.read_header = mp3_read_header,
.read_packet = mp3_read_packet,
.read_seek = mp3_seek,
.priv_data_size = sizeof(MP3DecContext),
};
下面这段代码就是在分配MP3DecContext这个ff_mp3_demuxer私有的数据。
/* Allocate private data. */
if (ffifmt(s->iformat)->priv_data_size > 0) {
if (!(s->priv_data = av_mallocz(ffifmt(s->iformat)->priv_data_size))) {
ret = AVERROR(ENOMEM);
goto fail;
}
if (s->iformat->priv_class) {
*(const AVClass **) s->priv_data = s->iformat->priv_class;
av_opt_set_defaults(s->priv_data);
if ((ret = av_opt_set_dict(s->priv_data, &tmp)) < 0)
goto fail;
}
}
接下来深入看下init_input
init_input中如何匹配demux？
static int init_input(AVFormatContext *s, const char *filename,
AVDictionary **options)
{
int ret;
AVProbeData pd = { filename, NULL, 0 };
int score = AVPROBE_SCORE_RETRY;
if (s->pb) {
s->flags |= AVFMT_FLAG_CUSTOM_IO;
//如果iformat为null，则查找AVInputFormat
if (!s->iformat)
return av_probe_input_buffer2(s->pb, &s->iformat, filename,
s, 0, s->format_probesize);
else if (s->iformat->flags & AVFMT_NOFILE)
av_log(s, AV_LOG_WARNING, "Custom AVIOContext makes no sense and "
"will be ignored with AVFMT_NOFILE format.\n");
return 0;
}
//从上面的代码可以看出，init_input主要干两件事情:
// 1. init s->pb即AVIOContext
// 2. 查找AVInputFormat
// 这里是指在外部已经初始化了AVIOContext来找对应的AVInputFormat
if ((s->iformat && s->iformat->flags & AVFMT_NOFILE) ||
(!s->iformat && (s->iformat = av_probe_input_format2(&pd, 0, &score))))
return score;
//针对常规的音视频数据，先调用io_open来初始化AVIOContext
if ((ret = s->io_open(s, &s->pb, filename, AVIO_FLAG_READ | s->avio_flags, options)) < 0)
return ret;
if (s->iformat)
return 0;
// 通过AVIOContext读取probe data并调用av_probe_input_format2再来匹配AVInputFormat
return av_probe_input_buffer2(s->pb, &s->iformat, filename,
s, 0, s->format_probesize);
}
首先来看下io_open是在哪里赋值的，指向哪个函数？ 其实是指向io_open_default
AVFormatContext *avformat_alloc_context(void)
{
FFFormatContext *const si = av_mallocz(sizeof(*si));
AVFormatContext *s;
if (!si)
return NULL;
s = &si->pub;
s->av_class = &av_format_context_class;
//赋值io_open
### s->io_open = io_open_default;
#### s->io_close2= io_close2_default;
av_opt_set_defaults(s);
si->pkt = av_packet_alloc();
si->parse_pkt = av_packet_alloc();
...............................;
return s;
}
ffio_open_whitelist函数中匹配protocol并初始化AVIOContext
static int io_open_default(AVFormatContext *s, AVIOContext **pb,
const char *url, int flags, AVDictionary **options)
{
int loglevel;
if (!strcmp(url, s->url) ||
s->iformat && !strcmp(s->iformat->name, "image2") ||
s->oformat && !strcmp(s->oformat->name, "image2")
) {
loglevel = AV_LOG_DEBUG;
} else
loglevel = AV_LOG_INFO;
........................................;
//此函数就是上一篇文章中讲解到avio_open2中调用的函数来初始化AVIOContext
return ffio_open_whitelist(pb, url, flags, &s->interrupt_callback,
options, s->protocol_whitelist,
s->protocol_blacklist);
}
av_probe_input_buffer2
此处重点看下这个函数，看看是如何通过url匹配到正确的AVInputFormat
int av_probe_input_buffer2(AVIOContext *pb, const AVInputFormat **fmt,
const char *filename, void *logctx,
unsigned int offset, unsigned int max_probe_size)
{
AVProbeData pd = { filename ? filename : "" };
uint8_t *buf = NULL;
int ret = 0, probe_size, buf_offset = 0;
int score = 0;
int ret2;
int eof = 0;
.....................................;
for (probe_size = PROBE_BUF_MIN; probe_size <= max_probe_size && !*fmt && !eof;
probe_size = FFMIN(probe_size << 1,
FFMAX(max_probe_size, probe_size + 1))) {
score = probe_size < max_probe_size ? AVPROBE_SCORE_RETRY : 0;
//分配probe buffer
if ((ret = av_reallocp(&buf, probe_size + AVPROBE_PADDING_SIZE)) < 0)
goto fail;
// 从avio中读取probe data
if ((ret = avio_read(pb, buf + buf_offset,
probe_size - buf_offset)) < 0) {
...........................;
}
buf_offset += ret;
if (buf_offset < offset)
continue;
pd.buf_size = buf_offset - offset;
pd.buf = &buf[offset];// 赋值到pd.buf
memset(pd.buf + pd.buf_size, 0, AVPROBE_PADDING_SIZE);
// 匹配正确的AVInputFormat
*fmt = av_probe_input_format2(&pd, 1, &score);
if (*fmt) {
/* This can only be true in the last iteration. */
if (score <= AVPROBE_SCORE_RETRY) {
av_log(logctx, AV_LOG_WARNING,
"Format %s detected only with low score of %d, "
"misdetection possible!\n", (*fmt)->name, score);
} else
av_log(logctx, AV_LOG_DEBUG,
"Format %s probed with size=%d and score=%d\n",
(*fmt)->name, probe_size, score);
}
}
if (!*fmt)
ret = AVERROR_INVALIDDATA;
..................................;
return ret < 0 ? ret : score;
}
const AVInputFormat *av_probe_input_format2(const AVProbeData *pd,
int is_opened, int *score_max)
{
int score_ret;
const AVInputFormat *fmt = av_probe_input_format3(pd, is_opened, &score_ret);
if (score_ret > *score_max) {
*score_max = score_ret;
return fmt;
} else
return NULL;
}
const AVInputFormat *av_probe_input_format3(const AVProbeData *pd,
int is_opened, int *score_ret)
{
AVProbeData lpd = *pd;
const AVInputFormat *fmt1 = NULL;
const AVInputFormat *fmt = NULL;
int score, score_max = 0;
void *i = 0;
const static uint8_t zerobuffer[AVPROBE_PADDING_SIZE];
.........................................;
while ((fmt1 = av_demuxer_iterate(&i))) {
if (fmt1->flags & AVFMT_EXPERIMENTAL)
continue;
if (!is_opened == !(fmt1->flags & AVFMT_NOFILE) && strcmp(fmt1->name, "image2"))
continue;
score = 0;
if (ffifmt(fmt1)->read_probe) {
//1. 通过每一个AVInputFormat的read_probe获取一个score
score = ffifmt(fmt1)->read_probe(&lpd);
if (score)
av_log(NULL, AV_LOG_TRACE, "Probing %s score:%d size:%d\n", fmt1->name, score, lpd.buf_size);
//2. 通过扩展名匹配，当扩展名匹配到之后，再根据id3 buffer的大小重新权衡下score
if (fmt1->extensions && av_match_ext(lpd.filename, fmt1->extensions)) {
switch (nodat) {
case NO_ID3:
score = FFMAX(score, 1);
break;
case ID3_GREATER_PROBE:
case ID3_ALMOST_GREATER_PROBE:
score = FFMAX(score, AVPROBE_SCORE_EXTENSION / 2 - 1);
break;
case ID3_GREATER_MAX_PROBE:
score = FFMAX(score, AVPROBE_SCORE_EXTENSION);
break;
}
}
} else if (fmt1->extensions) {
if (av_match_ext(lpd.filename, fmt1->extensions))
score = AVPROBE_SCORE_EXTENSION;
}
//3. 通过mime type来进行匹配
if (av_match_name(lpd.mime_type, fmt1->mime_type)) {
if (AVPROBE_SCORE_MIME > score) {
av_log(NULL, AV_LOG_DEBUG, "Probing %s score:%d increased to %d due to MIME type\n", fmt1->name, score, AVPROBE_SCORE_MIME);
score = AVPROBE_SCORE_MIME;
}
}
if (score > score_max) {
score_max = score;
fmt = fmt1;
} else if (score == score_max)
fmt = NULL;
}
if (nodat == ID3_GREATER_PROBE)
score_max = FFMIN(AVPROBE_SCORE_EXTENSION / 2 - 1, score_max);
*score_ret = score_max;
return fmt;
}
通过上述代码的分析，匹配规则是:
- 如果有read_probe函数，先通过read_probe获得一个score。如果同时获取到扩展名并扩展名匹配成功，则再根据ID3 buffer的大小修正下score
- 如果没有read_probe，直接进行扩展名匹配，扩展名匹配成功score = AVPROBE_SCORE_EXTENSION //50分
- 如果mime type匹配成功后，如果上述步骤分数小于AVPROBE_SCORE_MIME(75分)，则score为75.
当config的时候，config脚本会自动生成文件:demuxer_list.c,其内部定义了一个全局数组中定义了打开的demux：
static const FFInputFormat * const demuxer_list[] = {
&ff_aa_demuxer,
&ff_aac_demuxer,
&ff_aax_demuxer,
&ff_ac3_demuxer,
&ff_ac4_demuxer,
.......................;
}
const AVInputFormat *av_demuxer_iterate(void **opaque)
{
static const uintptr_t size = sizeof(demuxer_list)/sizeof(demuxer_list[0]) - 1;
uintptr_t i = (uintptr_t)*opaque;
const FFInputFormat *f = NULL;
uintptr_t tmp;
if (i < size) {
f = demuxer_list[i];
} else if (tmp = atomic_load_explicit(&indev_list_intptr, memory_order_relaxed)) {
const FFInputFormat *const *indev_list = (const FFInputFormat *const *)tmp;
f = indev_list[i - size];
}
if (f) {
*opaque = (void*)(i + 1);
return &f->p;
}
return NULL;
}
avformat_find_stream_info到底干了啥？
- 有些格式的在头部信息中找不到stream info，需要读一两包数据出来才有stream info，创建stream
- 有些codec信息需要解码一两针出来之后才能把信息回填到codecctx中用于计算fps，duration，start_time等
- 这个函数中进行主要是为了获取以上信息，如果你播放场景，音视频格式都时已知的，不需要ffmpeg解析什么额外信息，可以通过analyzeduration这个option来设置max_analyze_duration将探测数据量减小，缩短此函数的耗时。
graph TD
AVFormatContext --> .streams --> AVStream --> .time_base
AVStream --> .codecpar --> AVCodecParameters --> .extradata
AVCodecParameters --> AVCodecPara.sample_aspect_ratio
AVCodecParameters --> .framerate
AVStream --> .start_time
AVStream --> .duration
AVStream --> .avg_frame_rate
AVStream --> .sample_aspect_ratio
AVFormatContext --> AVFormatContext.start_time
AVFormatContext --> AVFormatContext.duration
此函数太长，我们来分段看下这个函数的源码。首先来看第一段:
int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
{
FFFormatContext *const si = ffformatcontext(ic);
int count = 0, ret = 0;
int64_t read_size;
AVPacket *pkt1 = si->pkt;
int64_t old_offset = avio_tell(ic->pb);
// new streams might appear, no options for those
int orig_nb_streams = ic->nb_streams;
int flush_codecs;
int64_t max_analyze_duration = ic->max_analyze_duration;
int64_t max_stream_analyze_duration;
int64_t max_subtitle_analyze_duration;
int64_t probesize = ic->probesize;
int eof_reached = 0;
int *missing_streams = av_opt_ptr(ic->iformat->priv_class, ic->priv_data, "missing_streams");
flush_codecs = probesize > 0;
av_opt_set_int(ic, "skip_clear", 1, AV_OPT_SEARCH_CHILDREN);
//计算最大分析数据时长:
//1. 如果AVFormatContext中设置了max_analyze_duration，就直接使用此值
//2. 如果没有设置，最大分析时长就是5s
//3.如果是flv格式，最大分析时长是90s
//4.如果是ts格式，最大分析时长是7s
max_stream_analyze_duration = max_analyze_duration;
max_subtitle_analyze_duration = max_analyze_duration;
if (!max_analyze_duration) {
max_stream_analyze_duration =
max_analyze_duration = 5*AV_TIME_BASE;
max_subtitle_analyze_duration = 30*AV_TIME_BASE;
if (!strcmp(ic->iformat->name, "flv"))
max_stream_analyze_duration = 90*AV_TIME_BASE;
if (!strcmp(ic->iformat->name, "mpeg") || !strcmp(ic->iformat->name, "mpegts"))
max_stream_analyze_duration = 7*AV_TIME_BASE;
}
//如果当前AVFormatContext支持I/O，就先打印下当前读到哪里，读了多少字节，seek了几次，
// 以及stream个数
//这些操作数据消耗和stream的创建一般是在reader_header中创建的。
if (ic->pb) {
FFIOContext *const ctx = ffiocontext(ic->pb);
av_log(ic, AV_LOG_DEBUG, "Before avformat_find_stream_info() pos: %"PRId64" bytes read:%"PRId64" seeks:%d nb_streams:%d\n",
avio_tell(ic->pb), ctx->bytes_read, ctx->seek_count, ic->nb_streams);
}
...............................................;
}
从上面可以看出avformat_find_stream_info在默认情况下要至少分析5s中的原始数据才会执行完成。所以会比较耗时。可以通过analyzeduration这个option来设置max_analyze_duration。接着看这个函数做了什么？
int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
{
.............................................;
for (unsigned i = 0; i < ic->nb_streams; i++) {
const AVCodec *codec;
AVDictionary *thread_opt = NULL;
AVStream *const st = ic->streams[i];
FFStream *const sti = ffstream(st);
AVCodecContext *const avctx = sti->avctx;
// 如果在read_header阶段没有找到想要的信息，request_probe就是-1，如果找到了就是大于0
// 这里的意思是如果有些demuxer在reader header阶段没有找到stream info，
// 这里就创建一个parser帮他们继续找这些信息
if (!sti->parser && !(ic->flags & AVFMT_FLAG_NOPARSE) && sti->request_probe <= 0) {
sti->parser = av_parser_init(st->codecpar->codec_id);
if (sti->parser) {
if (sti->need_parsing == AVSTREAM_PARSE_HEADERS) {
sti->parser->flags |= PARSER_FLAG_COMPLETE_FRAMES;
} else if (sti->need_parsing == AVSTREAM_PARSE_FULL_RAW) {
sti->parser->flags |= PARSER_FLAG_USE_CODEC_TS;
}
} else if (sti->need_parsing) {
av_log(ic, AV_LOG_VERBOSE, "parser not found for codec "
"%s, packets or times may be invalid.\n",
avcodec_get_name(st->codecpar->codec_id));
}
}
// 将codecpar中的信息同步到AVCodecContext
// 在avformat_new_stream会为stream分配一个AVCodecContext
ret = avcodec_parameters_to_context(avctx, st->codecpar);
if (ret < 0)
goto find_stream_info_err;
if (sti->request_probe <= 0)
sti->avctx_inited = 1;
//通过codec id找到对应的decoder
codec = find_probe_decoder(ic, st, st->codecpar->codec_id);
//设置thread个数，比如h264 decoder在多thread情况下不会去解析SPS, PPS这些信息
av_dict_set(options ? &options[i] : &thread_opt, "lowres", "0", 0);
if (ic->codec_whitelist)
av_dict_set(options ? &options[i] : &thread_opt, "codec_whitelist", ic->codec_whitelist, 0);
//如果是字幕 codec并且之前没有读到对应的信息，就打开decoder尝试获取信息
if (!has_codec_parameters(st, NULL) && sti->request_probe <= 0 ||
st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) {
if (codec && !avctx->codec)
if (avcodec_open2(avctx, codec, options ? &options[i] : &thread_opt) < 0)
av_log(ic, AV_LOG_WARNING,
"Failed to open codec in %s\n", __func__);
}
if (!options)
av_dict_free(&thread_opt);
}
.............................................;
}
上面这一段还是在准备阶段，还没有开始find info，接着往下看它要找什么info？
int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
{
.............................................;
for (;;) {
const AVPacket *pkt;
AVStream *st;
FFStream *sti;
AVCodecContext *avctx;
int analyzed_all_streams;
unsigned i;
//应用如果设置了interrupt_callback，这里调用下让用户快速退出避免卡在这里
if (ff_check_interrupt(&ic->interrupt_callback)) {
ret = AVERROR_EXIT;
av_log(ic, AV_LOG_DEBUG, "interrupted\n");
break;
}
// 这个for循环就是检查是否至少有一个stream仍缺少信息，然后利用变量i在下面继续判断
for (i = 0; i < ic->nb_streams; i++) {
AVStream *const st = ic->streams[i];
FFStream *const sti = ffstream(st);
int fps_analyze_framecount = 20;
int count;
if (!has_codec_parameters(st, NULL))
break;
................................
}
analyzed_all_streams = 0;
//如果stream缺失了，这种case下nb_streams应该是0
// 就把analyzed_all_streams赋值为1，当前只有flv有设置missing_streams这个option
if (!missing_streams || !*missing_streams)
if (i == ic->nb_streams) {
analyzed_all_streams = 1;
/* NOTE: If the format has no header, then we need to read some
* packets to get most of the streams, so we cannot stop here. */
if (!(ic->ctx_flags & AVFMTCTX_NOHEADER)) {
/* If we found the info for all the codecs, we can stop. */
ret = count;
av_log(ic, AV_LOG_DEBUG, "All info found\n");
flush_codecs = 0;
break;
}
}
/* We did not get all the codec info, but we read too much data. */
// 即使没有获取到所有codec信息，但读取了太多数据，这里就会退出
// probesize用户可以设置，不能小于PROBE_BUF_MIN，没有设置的话，默认值就是PROBE_BUF_MAX
// #define PROBE_BUF_MIN 2048
// #define PROBE_BUF_MAX (1 << 20)
if (read_size >= probesize) {
ret = count;
av_log(ic, AV_LOG_DEBUG,
"Probe buffer size limit of %"PRId64" bytes reached\n", probesize);
for (unsigned i = 0; i < ic->nb_streams; i++) {
AVStream *const st = ic->streams[i];
FFStream *const sti = ffstream(st);
if (!st->r_frame_rate.num &&
sti->info->duration_count <= 1 &&
st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
strcmp(ic->iformat->name, "image2"))
av_log(ic, AV_LOG_WARNING,
"Stream #%d: not enough frames to estimate rate; "
"consider increasing probesize\n", i);
}
break;
}
/* NOTE: A new stream can be added there if no header in file
* (AVFMTCTX_NOHEADER). */
// 调用read_frame_internal后可能会增加stream
ret = read_frame_internal(ic, pkt1);
if (ret == AVERROR(EAGAIN))
continue;
...............................;
st = ic->streams[pkt->stream_index];
sti = ffstream(st);
if (!(st->disposition & AV_DISPOSITION_ATTACHED_PIC))
read_size += pkt->size;
avctx = sti->avctx;
if (!sti->avctx_inited) {
ret = avcodec_parameters_to_context(avctx, st->codecpar);
if (ret < 0)
goto unref_then_goto_end;
sti->avctx_inited = 1;
}
//下面这一大段是计算FFStreamInfo中的值，此时还不太理解其内部变量的作用
if (pkt->dts != AV_NOPTS_VALUE && sti->codec_info_nb_frames > 1) {
/* check for non-increasing dts */
if (sti->info->fps_last_dts != AV_NOPTS_VALUE &&
sti->info->fps_last_dts >= pkt->dts) {
av_log(ic, AV_LOG_DEBUG,
"Non-increasing DTS in stream %d: packet %d with DTS "
"%"PRId64", packet %d with DTS %"PRId64"\n",
st->index, sti->info->fps_last_dts_idx,
sti->info->fps_last_dts, sti->codec_info_nb_frames,
pkt->dts);
sti->info->fps_first_dts =
sti->info->fps_last_dts = AV_NOPTS_VALUE;
}
if (sti->info->fps_last_dts != AV_NOPTS_VALUE &&
sti->info->fps_last_dts_idx > sti->info->fps_first_dts_idx &&
(pkt->dts - (uint64_t)sti->info->fps_last_dts) / 1000 >
(sti->info->fps_last_dts - (uint64_t)sti->info->fps_first_dts) /
(sti->info->fps_last_dts_idx - sti->info->fps_first_dts_idx)) {
av_log(ic, AV_LOG_WARNING,
"DTS discontinuity in stream %d: packet %d with DTS "
"%"PRId64", packet %d with DTS %"PRId64"\n",
st->index, sti->info->fps_last_dts_idx,
sti->info->fps_last_dts, sti->codec_info_nb_frames,
pkt->dts);
sti->info->fps_first_dts =
sti->info->fps_last_dts = AV_NOPTS_VALUE;
}
/* update stored dts values */
if (sti->info->fps_first_dts == AV_NOPTS_VALUE) {
sti->info->fps_first_dts = pkt->dts;
sti->info->fps_first_dts_idx = sti->codec_info_nb_frames;
}
sti->info->fps_last_dts = pkt->dts;
sti->info->fps_last_dts_idx = sti->codec_info_nb_frames;
}
//计算codec_info_duration，如果codec_info_duration超过了max_analyze_duration就退出
if (sti->codec_info_nb_frames > 1) {
int64_t t = 0;
int64_t limit;
if (st->time_base.den > 0)
t = av_rescale_q(sti->info->codec_info_duration, st->time_base, AV_TIME_BASE_Q);
if (st->avg_frame_rate.num > 0)
t = FFMAX(t, av_rescale_q(sti->codec_info_nb_frames, av_inv_q(st->avg_frame_rate), AV_TIME_BASE_Q));
if ( t == 0
&& sti->codec_info_nb_frames > 30
&& sti->info->fps_first_dts != AV_NOPTS_VALUE
&& sti->info->fps_last_dts != AV_NOPTS_VALUE) {
int64_t dur = av_sat_sub64(sti->info->fps_last_dts, sti->info->fps_first_dts);
t = FFMAX(t, av_rescale_q(dur, st->time_base, AV_TIME_BASE_Q));
}
if (analyzed_all_streams)
limit = max_analyze_duration;
else if (avctx->codec_type == AVMEDIA_TYPE_SUBTITLE)
limit = max_subtitle_analyze_duration;
else
limit = max_stream_analyze_duration;
if (t >= limit) {
av_log(ic, AV_LOG_VERBOSE, "max_analyze_duration %"PRId64" reached at %"PRId64" microseconds st:%d\n",
limit,
t, pkt->stream_index);
if (ic->flags & AVFMT_FLAG_NOBUFFER)
av_packet_unref(pkt1);
break;
}
if (pkt->duration > 0) {
const int fields = sti->codec_desc &&
(sti->codec_desc->props & AV_CODEC_PROP_FIELDS);
if (avctx->codec_type == AVMEDIA_TYPE_SUBTITLE && pkt->pts != AV_NOPTS_VALUE
&& st->start_time != AV_NOPTS_VALUE
&& pkt->pts >= st->start_time
&& (uint64_t)pkt->pts - st->start_time < INT64_MAX
) {
sti->info->codec_info_duration = FFMIN(pkt->pts - st->start_time, sti->info->codec_info_duration + pkt->duration);
} else
sti->info->codec_info_duration += pkt->duration;
sti->info->codec_info_duration_fields += sti->parser && sti->need_parsing
&& fields
? sti->parser->repeat_pict + 1 : 2;
}
}
//赋值frame_delay_evidence
if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
#if FF_API_R_FRAME_RATE
ff_rfps_add_frame(ic, st, pkt->dts);
#endif
if (pkt->dts != pkt->pts && pkt->dts != AV_NOPTS_VALUE && pkt->pts != AV_NOPTS_VALUE)
sti->info->frame_delay_evidence = 1;
}
//解析某些codec需要的一些信息，
// 如:* MJPEG: Huffman tables
// * rv10: additional flags
// * MPEG-4: global headers (they can be in the bitstream or here)
if (!sti->avctx->extradata) {
ret = extract_extradata(si, st, pkt);
if (ret < 0)
goto unref_then_goto_end;
}
// 有些情况下仍找不到一些信息，就需要把decoder打开解码一帧数据。避免使用太多内存和时间。
// 比如所，MPEG-4需要解码QuickTime
// 具体的解码器会往AVCodecContext中补充一些信息
try_decode_frame(ic, st, pkt,
(options && i < orig_nb_streams) ? &options[i] : NULL);
if (ic->flags & AVFMT_FLAG_NOBUFFER)
av_packet_unref(pkt1);
sti->codec_info_nb_frames++;
count++;
}
.......................................................;
}
经过上面的计算之后，主要是校准video的framerate和sample_aspect_ratio以及audio的disposition
for (unsigned i = 0; i < ic->nb_streams; i++) {
AVStream *const st = ic->streams[i];
FFStream *const sti = ffstream(st);
AVCodecContext *const avctx = sti->avctx;
if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
if (avctx->codec_id == AV_CODEC_ID_RAWVIDEO && !avctx->codec_tag && !avctx->bits_per_coded_sample) {
uint32_t tag= avcodec_pix_fmt_to_codec_tag(avctx->pix_fmt);
if (avpriv_pix_fmt_find(PIX_FMT_LIST_RAW, tag) == avctx->pix_fmt)
avctx->codec_tag= tag;
}
/* estimate average framerate if not set by demuxer */
if (sti->info->codec_info_duration_fields &&
!st->avg_frame_rate.num &&
sti->info->codec_info_duration) {
int best_fps = 0;
double best_error = 0.01;
AVRational codec_frame_rate = avctx->framerate;
if (sti->info->codec_info_duration >= INT64_MAX / st->time_base.num / 2||
sti->info->codec_info_duration_fields >= INT64_MAX / st->time_base.den ||
sti->info->codec_info_duration < 0)
continue;
av_reduce(&st->avg_frame_rate.num, &st->avg_frame_rate.den,
sti->info->codec_info_duration_fields * (int64_t) st->time_base.den,
sti->info->codec_info_duration * 2 * (int64_t) st->time_base.num, 60000);
/* Round guessed framerate to a "standard" framerate if it's
* within 1% of the original estimate. */
for (int j = 0; j < MAX_STD_TIMEBASES; j++) {
AVRational std_fps = { get_std_framerate(j), 12 * 1001 };
double error = fabs(av_q2d(st->avg_frame_rate) /
av_q2d(std_fps) - 1);
if (error < best_error) {
best_error = error;
best_fps = std_fps.num;
}
if (si->prefer_codec_framerate && codec_frame_rate.num > 0 && codec_frame_rate.den > 0) {
error = fabs(av_q2d(codec_frame_rate) /
av_q2d(std_fps) - 1);
if (error < best_error) {
best_error = error;
best_fps = std_fps.num;
}
}
}
if (best_fps)
av_reduce(&st->avg_frame_rate.num, &st->avg_frame_rate.den,
best_fps, 12 * 1001, INT_MAX);
}
if (!st->r_frame_rate.num) {
const AVCodecDescriptor *desc = sti->codec_desc;
AVRational mul = (AVRational){ desc && (desc->props & AV_CODEC_PROP_FIELDS) ? 2 : 1, 1 };
AVRational fr = av_mul_q(avctx->framerate, mul);
if (fr.num && fr.den && av_cmp_q(st->time_base, av_inv_q(fr)) <= 0) {
st->r_frame_rate = fr;
} else {
st->r_frame_rate.num = st->time_base.den;
st->r_frame_rate.den = st->time_base.num;
}
}
st->codecpar->framerate = avctx->framerate;
if (sti->display_aspect_ratio.num && sti->display_aspect_ratio.den) {
AVRational hw_ratio = { avctx->height, avctx->width };
st->sample_aspect_ratio = av_mul_q(sti->display_aspect_ratio,
hw_ratio);
}
} else if (avctx->codec_type == AVMEDIA_TYPE_AUDIO) {
if (!avctx->bits_per_coded_sample)
avctx->bits_per_coded_sample =
av_get_bits_per_sample(avctx->codec_id);
// set stream disposition based on audio service type
switch (avctx->audio_service_type) {
case AV_AUDIO_SERVICE_TYPE_EFFECTS:
st->disposition = AV_DISPOSITION_CLEAN_EFFECTS;
break;
case AV_AUDIO_SERVICE_TYPE_VISUALLY_IMPAIRED:
st->disposition = AV_DISPOSITION_VISUAL_IMPAIRED;
break;
case AV_AUDIO_SERVICE_TYPE_HEARING_IMPAIRED:
st->disposition = AV_DISPOSITION_HEARING_IMPAIRED;
break;
case AV_AUDIO_SERVICE_TYPE_COMMENTARY:
st->disposition = AV_DISPOSITION_COMMENT;
break;
case AV_AUDIO_SERVICE_TYPE_KARAOKE:
st->disposition = AV_DISPOSITION_KARAOKE;
break;
}
}
}
再接下来重新估算start_time和duration。以及把AVCodecContext中的信息同步到stream.AVCodecParameters
if (probesize)
estimate_timings(ic, old_offset);
/* update the stream parameters from the internal codec contexts */
for (unsigned i = 0; i < ic->nb_streams; i++) {
AVStream *const st = ic->streams[i];
FFStream *const sti = ffstream(st);
if (sti->avctx_inited) {
ret = avcodec_parameters_from_context(st->codecpar, sti->avctx);
if (ret < 0)
goto find_stream_info_err;
if (sti->avctx->rc_buffer_size > 0 || sti->avctx->rc_max_rate > 0 ||
sti->avctx->rc_min_rate) {
size_t cpb_size;
AVCPBProperties *props = av_cpb_properties_alloc(&cpb_size);
if (props) {
if (sti->avctx->rc_buffer_size > 0)
props->buffer_size = sti->avctx->rc_buffer_size;
if (sti->avctx->rc_min_rate > 0)
props->min_bitrate = sti->avctx->rc_min_rate;
if (sti->avctx->rc_max_rate > 0)
props->max_bitrate = sti->avctx->rc_max_rate;
if (!av_packet_side_data_add(&st->codecpar->coded_side_data,
&st->codecpar->nb_coded_side_data,
AV_PKT_DATA_CPB_PROPERTIES,
(uint8_t *)props, cpb_size, 0))
av_free(props);
}
}
}
av_read_frame
读取数据的过程就比较简单了，调用具体demuxer的read_packet读取AVPacket.
int av_read_frame(AVFormatContext *s, AVPacket *pkt)
--> int read_frame_internal(AVFormatContext *s, AVPacket *pkt)
--> int ff_read_packet(AVFormatContext *s, AVPacket *pkt)
---> int (*read_packet)(struct AVFormatContext *, AVPacket *pkt);
