上一篇中我们说到音视频流是是如何初始化的，这个小节我们讲音频解码及播放过程，其中会跳过音视频同步，放在后续小节进行分析。
音频流初始化好后会开启一个音频解码线程ff_audio_dec，其入口函数是audio_thread我们就从这里跟着走。
1.音频解码
在ijk中音频的解码只会走FFmpeg软解，之前有在小米6上简单测试过，音频的软解与硬解CPU消耗相差不大。在audio_thread函数中核心是调用decoder_decode_frame进行音频软解，再将解码后的Frame添加到VideoState的sampq中，等待播放。
static int audio_thread(void *arg)
{
FFPlayer *ffp = arg;
VideoState *is = ffp->is;
AVFrame *frame = av_frame_alloc();
Frame *af;
#if CONFIG_AVFILTER
int last_serial = -1;
int64_t dec_channel_layout;
int reconfigure;
#endif
int got_frame = 0;
AVRational tb;
int ret = 0;
int audio_accurate_seek_fail = 0;
int64_t audio_seek_pos = 0;
double frame_pts = 0;
double audio_clock = 0;
int64_t now = 0;
double samples_duration = 0;
int64_t deviation = 0;
int64_t deviation2 = 0;
int64_t deviation3 = 0;
if (!frame)
return AVERROR(ENOMEM);
do {
ffp_audio_statistic_l(ffp);
// 软解码
if ((got_frame = decoder_decode_frame(ffp, &is->auddec, frame, NULL)) < 0)
goto the_end;
// 删除了seek相关的代码，只看解码流程
if (got_frame) {
tb = (AVRational){1, frame->sample_rate};
// 将解码后的frame添加到 sampq 中等待播放
if (!(af = frame_queue_peek_writable(&is->sampq)))// 获取Frame
goto the_end;
// 填充数据
af->pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
af->pos = frame->pkt_pos;
af->serial = is->auddec.pkt_serial;
//时长
af->duration = av_q2d((AVRational){frame->nb_samples, frame->sample_rate});
// 将frame数据copy到af->frame中
av_frame_move_ref(af->frame, frame);
// 前移windex
frame_queue_push(&is->sampq);
}
} while (ret >= 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF);
the_end:
av_frame_free(&frame);
return ret;
}
下面看一下decoder_decode_frame软解码过程，其核心是FFmpeg的两个API：avcodec_send_packet和avcodec_receive_frame。
avcodec_send_packet是向解码器发送需要解码的AVPacket，而这个数据从哪儿来呢？是从read_thread中不断解封装出来，并放入到对应流的队列中，再通过packet_queue_get_or_buffering函数得到的。
avcodec_receive_frame是从解码器中获取解码后的AVFrame，这个结构体里面封装了可以进行渲染的音视频数据。
下面的代码其实就是执行的这些逻辑，只不过多处理了seek的情况。
static int decoder_decode_frame(FFPlayer *ffp, Decoder *d, AVFrame *frame, AVSubtitle *sub) {
int ret = AVERROR(EAGAIN);
for (;;) {
AVPacket pkt;
//一次seek一个序列，如果序列不相同，那么不能拿数据
if (d->queue->serial == d->pkt_serial) {
do {
//结束标志
if (d->queue->abort_request)
return -1;
switch (d->avctx->codec_type) {
case AVMEDIA_TYPE_VIDEO:
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
ffp->stat.vdps = SDL_SpeedSamplerAdd(&ffp->vdps_sampler, FFP_SHOW_VDPS_AVCODEC, "vdps[avcodec]");
if (ffp->decoder_reorder_pts == -1) {
frame->pts = frame->best_effort_timestamp;
} else if (!ffp->decoder_reorder_pts) {
frame->pts = frame->pkt_dts;
}
}
break;
case AVMEDIA_TYPE_AUDIO:
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
AVRational tb = (AVRational){1, frame->sample_rate};
if (frame->pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(frame->pts, av_codec_get_pkt_timebase(d->avctx), tb);
else if (d->next_pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
if (frame->pts != AV_NOPTS_VALUE) {
d->next_pts = frame->pts + frame->nb_samples;
d->next_pts_tb = tb;
}
}
break;
default:
break;
}
//如果EOF，设置d->finished
if (ret == AVERROR_EOF) {
d->finished = d->pkt_serial;
avcodec_flush_buffers(d->avctx);
return 0;
}
// 如果已经解码成功，返回数据
if (ret >= 0)
return 1;
} while (ret != AVERROR(EAGAIN));
}
do {
if (d->queue->nb_packets == 0)
SDL_CondSignal(d->empty_queue_cond);
// 处理EAGAIN的情况，不再从队列里面拿
if (d->packet_pending) {
av_packet_move_ref(&pkt, &d->pkt);
d->packet_pending = 0;
} else {
//从队列中获取AVPacket
if (packet_queue_get_or_buffering(ffp, d->queue, &pkt, &d->pkt_serial, &d->finished) < 0)
return -1;
}
//获取到的数据也必须是本次的serial，否则重新获取
} while (d->queue->serial != d->pkt_serial);
//如果发生了seek，需要执行解码器的flush操作
if (pkt.data == flush_pkt.data) {
avcodec_flush_buffers(d->avctx);
d->finished = 0;
d->next_pts = d->start_pts;
d->next_pts_tb = d->start_pts_tb;
} else {
if (d->avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
int got_frame = 0;
//解码字幕
ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, &pkt);
if (ret < 0) {
ret = AVERROR(EAGAIN);
} else {
if (got_frame && !pkt.data) {
d->packet_pending = 1;
av_packet_move_ref(&d->pkt, &pkt);
}
ret = got_frame ? 0 : (pkt.data ? AVERROR(EAGAIN) : AVERROR_EOF);
}
} else {
//填充packet数据进行解码
if (avcodec_send_packet(d->avctx, &pkt) == AVERROR(EAGAIN)) {
av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation.\n");
d->packet_pending = 1;
av_packet_move_ref(&d->pkt, &pkt);
}
}
av_packet_unref(&pkt);
}
}
}
packet_queue_get_or_buffering可以简单的理解为出队列的过程，从代码中可以看到在PacketQueue队列为空的情况下，会发生缓冲等待。
static int packet_queue_get_or_buffering(FFPlayer *ffp, PacketQueue *q, AVPacket *pkt, int *serial, int *finished)
{
assert(finished);
// packet_buffering 是否需要进行显式缓冲，默认为1
if (!ffp->packet_buffering)
return packet_queue_get(q, pkt, 1, serial);
while (1) {
// 非阻塞获取
int new_packet = packet_queue_get(q, pkt, 0, serial);
if (new_packet < 0)
return -1;
else if (new_packet == 0) {
//没有获取到packet，进行缓冲
if (q->is_buffer_indicator && !*finished)
ffp_toggle_buffering(ffp, 1);
// 阻塞直至获取到数据
new_packet = packet_queue_get(q, pkt, 1, serial);
if (new_packet < 0)
return -1;
}
//EOF的情况
if (*finished == *serial) {
av_packet_unref(pkt);
continue;
}
else
break;
}
return 1;
}
2.解封装
通过上面的解码步骤我们已经知道AVPackect如何使用的了，现在回过头再来看一下是如何生产的，其核心的API是FFmpeg中的av_read_frame，在调用这个函数获取到AVPacket后再分发到对应流的队列中。
static int read_thread(void *arg)
{
FFPlayer *ffp = arg;
VideoState *is = ffp->is;
AVFormatContext *ic = NULL;
int err, i, ret __unused;
int st_index[AVMEDIA_TYPE_NB];
AVPacket pkt1, *pkt = &pkt1;
int64_t stream_start_time;
int completed = 0;
int pkt_in_play_range = 0;
AVDictionaryEntry *t;
SDL_mutex *wait_mutex = SDL_CreateMutex();
int scan_all_pmts_set = 0;
int64_t pkt_ts;
int last_error = 0;
int64_t prev_io_tick_counter = 0;
int64_t io_tick_counter = 0;
int init_ijkmeta = 0;
for (;;) {
pkt->flags = 0;
// 从视频中读取AVPacket
ret = av_read_frame(ic, pkt);
// 清空当前解码帧缓冲区,防止旧数据干扰
if (pkt->flags & AV_PKT_FLAG_DISCONTINUITY) {
if (is->audio_stream >= 0) {
packet_queue_put(&is->audioq, &flush_pkt);
}
if (is->subtitle_stream >= 0) {
packet_queue_put(&is->subtitleq, &flush_pkt);
}
if (is->video_stream >= 0) {
packet_queue_put(&is->videoq, &flush_pkt);
}
}
/* check if packet is in play range specified by user, then queue, otherwise discard */
stream_start_time = ic->streams[pkt->stream_index]->start_time;
pkt_ts = pkt->pts == AV_NOPTS_VALUE ? pkt->dts : pkt->pts;
pkt_in_play_range = ffp->duration == AV_NOPTS_VALUE ||
(pkt_ts - (stream_start_time != AV_NOPTS_VALUE ? stream_start_time : 0)) *
av_q2d(ic->streams[pkt->stream_index]->time_base) -
(double)(ffp->start_time != AV_NOPTS_VALUE ? ffp->start_time : 0) / 1000000
<= ((double)ffp->duration / 1000000);
if (pkt->stream_index == is->audio_stream && pkt_in_play_range) {
// 放入音频Packet队列
packet_queue_put(&is->audioq, pkt);
} else if (pkt->stream_index == is->video_stream && pkt_in_play_range
&& !(is->video_st && (is->video_st->disposition & AV_DISPOSITION_ATTACHED_PIC))) {
// 放入视频Packet队列
packet_queue_put(&is->videoq, pkt);
} else if (pkt->stream_index == is->subtitle_stream && pkt_in_play_range) {
packet_queue_put(&is->subtitleq, pkt);
} else {
av_packet_unref(pkt);
}
}
}
3.AudioTrack播放音频
在上一篇中我们在打开音频流的过程中代码跟踪到了func_open_audio_output和audio_open函数，了解了ijk中有AudioTrack和OpenSL ES两种实现音频播放的实现，在func_open_audio_output中会配置相关音频操作接口。在audio_open会调用配置的接口SDL_AoutOpenAudio进行音频播放线程的初始化，我们先看AudioTrack方式的实现过程。
// 创建SDL_Aout
static SDL_Aout *func_open_audio_output(IJKFF_Pipeline *pipeline, FFPlayer *ffp)
{
SDL_Aout *aout = NULL;
if (ffp->use_oboe) {
aout = SDL_AoutAndroid_CreateForOboe();
} else if (ffp->opensles) {
// 配置成opensl接口
aout = SDL_AoutAndroid_CreateForOpenSLES();
} else {
// 配置成AudioTrack接口
aout = SDL_AoutAndroid_CreateForAudioTrack();
}
if (aout)
SDL_AoutSetStereoVolume(aout, pipeline->opaque->left_volume, pipeline->opaque->right_volume);
return aout;
}
// 配置AudioTrack接口
SDL_Aout *SDL_AoutAndroid_CreateForAudioTrack()
{
SDL_Aout *aout = SDL_Aout_CreateInternal(sizeof(SDL_Aout_Opaque));
if (!aout)
return NULL;
SDL_Aout_Opaque *opaque = aout->opaque;
opaque->wakeup_cond = SDL_CreateCond();
opaque->wakeup_mutex = SDL_CreateMutex();
opaque->speed = 1.0f;
aout->opaque_class = &g_audiotrack_class;
aout->free_l = aout_free_l;
aout->open_audio = aout_open_audio;
aout->pause_audio = aout_pause_audio;
aout->flush_audio = aout_flush_audio;
aout->set_volume = aout_set_volume;
aout->close_audio = aout_close_audio;
aout->func_get_audio_session_id = aout_get_audio_session_id;
aout->func_set_playback_rate = func_set_playback_rate;
return aout;
}
// 初始化音频播放线程
int SDL_AoutOpenAudio(SDL_Aout *aout, const SDL_AudioSpec *desired, SDL_AudioSpec *obtained)
{
if (aout && desired && aout->open_audio)
return aout->open_audio(aout, desired, obtained);
return -1;
}
// ijksdl_aout_android_audiotrack.c
static int aout_open_audio(SDL_Aout *aout, const SDL_AudioSpec *desired, SDL_AudioSpec *obtained)
{
// SDL_Aout_Opaque *opaque = aout->opaque;
JNIEnv *env = NULL;
if (JNI_OK != SDL_JNI_SetupThreadEnv(&env)) {
ALOGE("aout_open_audio: AttachCurrentThread: failed");
return -1;
}
return aout_open_audio_n(env, aout, desired, obtained);
}
创建AudioTrack
AudioTrack方式的实现是通过jni创建了一个Java层 AudioTrack对象，后续的播放操作都是通过控制这个Java对象来实现。
static int aout_open_audio_n(JNIEnv *env, SDL_Aout *aout, const SDL_AudioSpec *desired, SDL_AudioSpec *obtained)
{
assert(desired);
SDL_Aout_Opaque *opaque = aout->opaque;
opaque->spec = *desired;
// jni 创建AudioTrack
opaque->atrack = SDL_Android_AudioTrack_new_from_sdl_spec(env, desired);
if (!opaque->atrack) {
ALOGE("aout_open_audio_n: failed to new AudioTrcak()");
return -1;
}
opaque->buffer_size = SDL_Android_AudioTrack_get_min_buffer_size(opaque->atrack);
if (opaque->buffer_size <= 0) {
ALOGE("aout_open_audio_n: failed to getMinBufferSize()");
SDL_Android_AudioTrack_free(env, opaque->atrack);
opaque->atrack = NULL;
return -1;
}
opaque->buffer = malloc(opaque->buffer_size);
if (!opaque->buffer) {
ALOGE("aout_open_audio_n: failed to allocate buffer");
SDL_Android_AudioTrack_free(env, opaque->atrack);
opaque->atrack = NULL;
return -1;
}
if (obtained) {
SDL_Android_AudioTrack_get_target_spec(opaque->atrack, obtained);
SDLTRACE("audio target format fmt:0x%x, channel:0x%x", (int)obtained->format, (int)obtained->channels);
}
opaque->audio_session_id = SDL_Android_AudioTrack_getAudioSessionId(env, opaque->atrack);
ALOGI("audio_session_id = %d\n", opaque->audio_session_id);
opaque->pause_on = 1;
opaque->abort_request = 0;
//创建播放线程
opaque->audio_tid = SDL_CreateThreadEx(&opaque->_audio_tid, aout_thread, aout, "ff_aout_android");
if (!opaque->audio_tid) {
ALOGE("aout_open_audio_n: failed to create audio thread");
SDL_Android_AudioTrack_free(env, opaque->atrack);
opaque->atrack = NULL;
return -1;
}
return 0;
}
音频播放线程
音频播放线程中主要流程是通过opaque->spec.callback，获取可用播放的pcm，然后向AudioTrack对象写数据进行播放。那这个opaque->spec.callback是在哪儿赋值的呢？也是在audio_open函数中，所赋值为sdl_audio_callback函数。
static int aout_thread(void *arg)
{
SDL_Aout *aout = arg;
// SDL_Aout_Opaque *opaque = aout->opaque;
JNIEnv *env = NULL;
// attach jvm
if (JNI_OK != SDL_JNI_SetupThreadEnv(&env)) {
ALOGE("aout_thread: SDL_AndroidJni_SetupEnv: failed");
return -1;
}
return aout_thread_n(env, aout);
}
static int aout_thread_n(JNIEnv *env, SDL_Aout *aout)
{
SDL_Aout_Opaque *opaque = aout->opaque;
SDL_Android_AudioTrack *atrack = opaque->atrack;
SDL_AudioCallback audio_cblk = opaque->spec.callback;
void *userdata = opaque->spec.userdata;
uint8_t *buffer = opaque->buffer;
int copy_size = 256;
assert(atrack);
assert(buffer);
SDL_SetThreadPriority(SDL_THREAD_PRIORITY_HIGH);
if (!opaque->abort_request && !opaque->pause_on)
SDL_Android_AudioTrack_play(env, atrack);
while (!opaque->abort_request) {
SDL_LockMutex(opaque->wakeup_mutex);
//控制状态
if (!opaque->abort_request && opaque->pause_on) {
SDL_Android_AudioTrack_pause(env, atrack);
while (!opaque->abort_request && opaque->pause_on) {
SDL_CondWaitTimeout(opaque->wakeup_cond, opaque->wakeup_mutex, 1);
}
if (!opaque->abort_request && !opaque->pause_on) {
if (opaque->need_flush) {
opaque->need_flush = 0;
SDL_Android_AudioTrack_flush(env, atrack);
}
// 播放
SDL_Android_AudioTrack_play(env, atrack);
}
}
if (opaque->need_flush) {
opaque->need_flush = 0;
SDL_Android_AudioTrack_flush(env, atrack);
}
if (opaque->need_set_volume) {
opaque->need_set_volume = 0;
SDL_Android_AudioTrack_set_volume(env, atrack, opaque->left_volume, opaque->right_volume);
}
if (opaque->speed_changed) {
opaque->speed_changed = 0;
SDL_Android_AudioTrack_setSpeed(env, atrack, opaque->speed);
}
SDL_UnlockMutex(opaque->wakeup_mutex);
//获取pcm数据
audio_cblk(userdata, buffer, copy_size);
if (opaque->need_flush) {
SDL_Android_AudioTrack_flush(env, atrack);
opaque->need_flush = false;
}
if (opaque->need_flush) {
opaque->need_flush = 0;
SDL_Android_AudioTrack_flush(env, atrack);
} else {
// 写入pcm数据，进行播放
int written = SDL_Android_AudioTrack_write(env, atrack, buffer, copy_size);
if (written != copy_size) {
ALOGW("AudioTrack: not all data copied %d/%d", (int)written, (int)copy_size);
}
}
// TODO: 1 if callback return -1 or 0
}
SDL_Android_AudioTrack_free(env, atrack);
return 0;
}
pcm填充
上面我们已经说过，编码完成后音频AVFrame被放入到sampq中，可以预见到，播放的时候也是到这里去拿。通过查看代码可以确定就是如此，不过在填充pcm的时候，做了两处特殊处理：
1.判断格式是否与播放的相同，如果不同需要进行重采样。无论有没有重采样，获取出的pcm会先存放到audio_buf中，然后再copy到stream中。
2.按需进行填充，即外部传入需要多大，就填充多少，这块代码稍微复杂点，代码中已经添加上了注释。
static void sdl_audio_callback(void *opaque, Uint8 *stream, int len)
{
FFPlayer *ffp = opaque;
VideoState *is = ffp->is;
int audio_size, len1;
if (!ffp || !is) {
memset(stream, 0, len);
return;
}
ffp->audio_callback_time = av_gettime_relative();
if (ffp->pf_playback_rate_changed) {
ffp->pf_playback_rate_changed = 0;
#if defined(__ANDROID__)
if (!ffp->soundtouch_enable) {
SDL_AoutSetPlaybackRate(ffp->aout, ffp->pf_playback_rate);
}
#else
SDL_AoutSetPlaybackRate(ffp->aout, ffp->pf_playback_rate);
#endif
}
if (ffp->pf_playback_volume_changed) {
ffp->pf_playback_volume_changed = 0;
SDL_AoutSetPlaybackVolume(ffp->aout, ffp->pf_playback_volume);
}
// 一直循环到满足需要的大小
while (len > 0) {
/**
* 1.如果 audio_decode_frame 出来的数据大小等于需要的大小，那么在 len -= len1之后
* 不满足while循环条件，退出。
* 2.如果 audio_decode_frame 出来的数据大小大于需要的大小，那么只拷贝当前需要的数据量，
* 但会记住 pcm buffer拷贝到哪儿了，下次调用该方法获取pcm时 is->audio_buf_index >= is->audio_buf_size，不满足条件
* 这里是因为 is->audio_buf_index = 这次拷贝的大小（重置后），
* 所以不会调用 audio_decode_frame 去获取pcm，而是从 audio buffer中取
* 3.如果 audio_decode_frame 出来的数据大小小于需要的大小，那么在执行完一次len -= len1后，len > 0条件满足，
* 继续while循环，接着执行audio_decode_frame，填充一个新的pcm buffer然后再按条件执行1，2，3，直到满足需要的大小
*
*/
if (is->audio_buf_index >= is->audio_buf_size) {
audio_size = audio_decode_frame(ffp);
if (audio_size < 0) {
/* if error, just output silence */
is->audio_buf = NULL;
is->audio_buf_size = SDL_AUDIO_MIN_BUFFER_SIZE / is->audio_tgt.frame_size * is->audio_tgt.frame_size;
} else {
if (is->show_mode != SHOW_MODE_VIDEO)
update_sample_display(is, (int16_t *)is->audio_buf, audio_size);
//重置音频buffer大小
is->audio_buf_size = audio_size;
}
//重置index
is->audio_buf_index = 0;
}
//不为同一序列
if (is->auddec.pkt_serial != is->audioq.serial) {
is->audio_buf_index = is->audio_buf_size;
memset(stream, 0, len);
// stream += len;
// len = 0;
SDL_AoutFlushAudio(ffp->aout);
break;
}
//获取当前可用的buffer大小
len1 = is->audio_buf_size - is->audio_buf_index;
if (len1 > len)
len1 = len;//如果超过需要的，那么只拷贝需要的大小
//进行拷贝
if (!is->muted && is->audio_buf && is->audio_volume == SDL_MIX_MAXVOLUME)
memcpy(stream, (uint8_t *)is->audio_buf + is->audio_buf_index, len1);
else {
memset(stream, 0, len1);
if (!is->muted && is->audio_buf)
SDL_MixAudio(stream, (uint8_t *)is->audio_buf + is->audio_buf_index, len1, is->audio_volume);
}
//剩余需要拷贝长度
len -= len1;
//偏移起始点
stream += len1;
//向前推进buffer
is->audio_buf_index += len1;
}
//这里可能存在问题，按道理应该累加，因为可能是多次读取的结果
is->audio_write_buf_size = is->audio_buf_size - is->audio_buf_index;
/* Let's assume the audio driver that is used by SDL has two periods. */
if (!isnan(is->audio_clock)) {
//is->audio_clock 当前播放frame
//(double)(is->au dio_write_buf_size) / is->audio_tgt.bytes_per_sec 此次需要播放buffer的时长
//SDL_AoutGetLatencySeconds(ffp->aout) 底层播放延迟
set_clock_at(&is->audclk, is->audio_clock - (double)(is->audio_write_buf_size) / is->audio_tgt.bytes_per_sec - SDL_AoutGetLatencySeconds(ffp->aout), is->audio_clock_serial, ffp->audio_callback_time / 1000000.0);
sync_clock_to_slave(&is->extclk, &is->audclk);
}
if (!ffp->first_audio_frame_rendered) {
ffp->first_audio_frame_rendered = 1;
ffp_notify_msg1(ffp, FFP_MSG_AUDIO_RENDERING_START);
}
if (is->latest_audio_seek_load_serial == is->audio_clock_serial) {
int latest_audio_seek_load_serial = __atomic_exchange_n(&(is->latest_audio_seek_load_serial), -1, memory_order_seq_cst);
if (latest_audio_seek_load_serial == is->audio_clock_serial) {
if (ffp->av_sync_type == AV_SYNC_AUDIO_MASTER) {
ffp_notify_msg2(ffp, FFP_MSG_AUDIO_SEEK_RENDERING_START, 1);
} else {
ffp_notify_msg2(ffp, FFP_MSG_AUDIO_SEEK_RENDERING_START, 0);
}
}
}
if (ffp->render_wait_start && !ffp->start_on_prepared && is->pause_req) {
//暂停部分
while (is->pause_req && !is->abort_request) {
SDL_Delay(20);
}
}
}
static int audio_decode_frame(FFPlayer *ffp)
{
VideoState *is = ffp->is;
int data_size, resampled_data_size;
int64_t dec_channel_layout;
av_unused double audio_clock0;
int wanted_nb_samples;
Frame *af;
#if defined(__ANDROID__)
int translate_time = 1;
#endif
if (is->paused || is->step)
return -1;
if (ffp->sync_av_start && /* sync enabled */
is->video_st && /* has video stream */
!is->viddec.first_frame_decoded && /* not hot */
is->viddec.finished != is->videoq.serial) { /* not finished */
/* waiting for first video frame */
Uint64 now = SDL_GetTickHR();
if (now < is->viddec.first_frame_decoded_time ||
now > is->viddec.first_frame_decoded_time + 2000) {
is->viddec.first_frame_decoded = 1;
} else {
/* video pipeline is not ready yet */
return -1;
}
}
reload:
do {
// 读取音频Frame
if (!(af = frame_queue_peek_readable(&is->sampq)))
return -1;
frame_queue_next(&is->sampq);
} while (af->serial != is->audioq.serial);
data_size = av_samples_get_buffer_size(NULL, af->frame->channels,
af->frame->nb_samples,
af->frame->format, 1);
dec_channel_layout =
(af->frame->channel_layout && af->frame->channels == av_get_channel_layout_nb_channels(af->frame->channel_layout)) ?
af->frame->channel_layout : av_get_default_channel_layout(af->frame->channels);
wanted_nb_samples = synchronize_audio(is, af->frame->nb_samples);
//格式不同，需要进行重采样
if (af->frame->format != is->audio_src.fmt ||
dec_channel_layout != is->audio_src.channel_layout ||
af->frame->sample_rate != is->audio_src.freq ||
(wanted_nb_samples != af->frame->nb_samples && !is->swr_ctx)) {
AVDictionary *swr_opts = NULL;
swr_free(&is->swr_ctx);
is->swr_ctx = swr_alloc_set_opts(NULL,
is->audio_tgt.channel_layout, is->audio_tgt.fmt, is->audio_tgt.freq,
dec_channel_layout, af->frame->format, af->frame->sample_rate,
0, NULL);
if (!is->swr_ctx) {
av_log(NULL, AV_LOG_ERROR,
"Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!\n",
af->frame->sample_rate, av_get_sample_fmt_name(af->frame->format), af->frame->channels,
is->audio_tgt.freq, av_get_sample_fmt_name(is->audio_tgt.fmt), is->audio_tgt.channels);
return -1;
}
av_dict_copy(&swr_opts, ffp->swr_opts, 0);
if (af->frame->channel_layout == AV_CH_LAYOUT_5POINT1_BACK)
av_opt_set_double(is->swr_ctx, "center_mix_level", ffp->preset_5_1_center_mix_level, 0);
av_opt_set_dict(is->swr_ctx, &swr_opts);
av_dict_free(&swr_opts);
if (swr_init(is->swr_ctx) < 0) {
av_log(NULL, AV_LOG_ERROR,
"Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!\n",
af->frame->sample_rate, av_get_sample_fmt_name(af->frame->format), af->frame->channels,
is->audio_tgt.freq, av_get_sample_fmt_name(is->audio_tgt.fmt), is->audio_tgt.channels);
swr_free(&is->swr_ctx);
return -1;
}
is->audio_src.channel_layout = dec_channel_layout;
is->audio_src.channels = af->frame->channels;
is->audio_src.freq = af->frame->sample_rate;
is->audio_src.fmt = af->frame->format;
}
// 重采样进行转换
if (is->swr_ctx) {
const uint8_t **in = (const uint8_t **)af->frame->extended_data;
uint8_t **out = &is->audio_buf1;
int out_count = (int)((int64_t)wanted_nb_samples * is->audio_tgt.freq / af->frame->sample_rate + 256);
int out_size = av_samples_get_buffer_size(NULL, is->audio_tgt.channels, out_count, is->audio_tgt.fmt, 0);
int len2;
if (out_size < 0) {
av_log(NULL, AV_LOG_ERROR, "av_samples_get_buffer_size() failed\n");
return -1;
}
if (wanted_nb_samples != af->frame->nb_samples) {
if (swr_set_compensation(is->swr_ctx, (wanted_nb_samples - af->frame->nb_samples) * is->audio_tgt.freq / af->frame->sample_rate,
wanted_nb_samples * is->audio_tgt.freq / af->frame->sample_rate) < 0) {
av_log(NULL, AV_LOG_ERROR, "swr_set_compensation() failed\n");
return -1;
}
}
av_fast_malloc(&is->audio_buf1, &is->audio_buf1_size, out_size);
if (!is->audio_buf1)
return AVERROR(ENOMEM);
len2 = swr_convert(is->swr_ctx, out, out_count, in, af->frame->nb_samples);
if (len2 < 0) {
av_log(NULL, AV_LOG_ERROR, "swr_convert() failed\n");
return -1;
}
if (len2 == out_count) {
av_log(NULL, AV_LOG_WARNING, "audio buffer is probably too small\n");
if (swr_init(is->swr_ctx) < 0)
swr_free(&is->swr_ctx);
}
is->audio_buf = is->audio_buf1;
int bytes_per_sample = av_get_bytes_per_sample(is->audio_tgt.fmt);
//重新计算buffer大小
resampled_data_size = len2 * is->audio_tgt.channels * bytes_per_sample;
#if defined(__ANDROID__)
//使用soundtouch进行转换
if (ffp->soundtouch_enable && ffp->pf_playback_rate != 1.0f && !is->abort_request) {
av_fast_malloc(&is->audio_new_buf, &is->audio_new_buf_size, out_size * translate_time);
for (int i = 0; i < (resampled_data_size / 2); i++)
{
is->audio_new_buf[i] = (is->audio_buf1[i * 2] | (is->audio_buf1[i * 2 + 1] << 8));
}
int ret_len = ijk_soundtouch_translate(is->handle, is->audio_new_buf, (float)(ffp->pf_playback_rate), (float)(1.0f/ffp->pf_playback_rate),
resampled_data_size / 2, bytes_per_sample, is->audio_tgt.channels, af->frame->sample_rate);
if (ret_len > 0) {
is->audio_buf = (uint8_t*)is->audio_new_buf;
resampled_data_size = ret_len;
} else {
translate_time++;
goto reload;
}
}
#endif
} else {
// 不重采样
is->audio_buf = af->frame->data[0];
resampled_data_size = data_size;
}
audio_clock0 = is->audio_clock;
/* update the audio clock with the pts */
if (!isnan(af->pts))
//设置Audio的Pts 当前马上拿去播放frame的播放完成时间
is->audio_clock = af->pts + (double) af->frame->nb_samples / af->frame->sample_rate;
else
is->audio_clock = NAN;
//重新设置音频时钟的序列
is->audio_clock_serial = af->serial;
#ifdef FFP_SHOW_AUDIO_DELAY
{
static double last_clock;
printf("audio: delay=%0.3f clock=%0.3f clock0=%0.3f\n",
is->audio_clock - last_clock,
is->audio_clock, audio_clock0);
last_clock = is->audio_clock;
}
#endif
if (!is->auddec.first_frame_decoded) {
ALOGD("avcodec/Audio: first frame decoded\n");
ffp_notify_msg1(ffp, FFP_MSG_AUDIO_DECODED_START);
is->auddec.first_frame_decoded_time = SDL_GetTickHR();
is->auddec.first_frame_decoded = 1;
}
return resampled_data_size;
}
至此，音频的播放过程我们已经做了大致了解，下面画一个草图简单回顾一下。
AVPacket AVFrame pcm
音视频文件---->解封装--------------->解码-------------------->获取解码数据------------------------->播放------->声卡
av_read_frame avcodec_send_packet avcodec_receive_frame write
