内存方面
- 经常碰到解码过程中，野指针崩溃。尤其是当一帧解码失败时候。
- ffmepg有引用计数内存管理机制，对应_unref方法。
- 使用结束之后，尽量使用_unref方法释放内存，而避免直接_free
int av_packet_ref(AVPacket *dst, const AVPacket *src);
void av_packet_unref(AVPacket *pkt);
解码器碰到No start code is found.错误
[h264 @ 0x7fa318076e00] No start code is found.
[h264 @ 0x7fa318076e00] Error splitting the input into NAL units.
google之后，说没有使用下列方法
/**
* Fill the codec context based on the values from the supplied codec
* parameters. Any allocated fields in codec that have a corresponding field in
* par are freed and replaced with duplicates of the corresponding field in par.
* Fields in codec that do not have a counterpart in par are not touched.
*
* @return >= 0 on success, a negative AVERROR code on failure.
*/
int avcodec_parameters_to_context(AVCodecContext *codec,
const AVCodecParameters *par);
也就是说AVCodecContext有些值没有设置，导致解码失败。几经尝试之后，发现有两个属性是必要的。
uint8_t *extradata;
int extradata_size;
除非是自己编码的裸数据，mp4、flv等视频文件解封装之后，都会有该属性，对于后续解码很有必要。
static int h264_decode_frame(AVCodecContext *avctx, void *data,
int *got_frame, AVPacket *avpkt)
{
const uint8_t *buf = avpkt->data;
int buf_size = avpkt->size;
H264Context *h = avctx->priv_data;
AVFrame *pict = data;
int buf_index;
int ret;
h->flags = avctx->flags;
h->setup_finished = 0;
h->nb_slice_ctx_queued = 0;
ff_h264_unref_picture(h, &h->last_pic_for_ec);
/* end of stream, output what is still in the buffers */
if (buf_size == 0)
return send_next_delayed_frame(h, pict, got_frame, 0);
if (h->is_avc && av_packet_get_side_data(avpkt, AV_PKT_DATA_NEW_EXTRADATA, NULL)) {
int side_size;
uint8_t *side = av_packet_get_side_data(avpkt, AV_PKT_DATA_NEW_EXTRADATA, &side_size);
if (is_extra(side, side_size))
ff_h264_decode_extradata(side, side_size,
&h->ps, &h->is_avc, &h->nal_length_size,
avctx->err_recognition, avctx);
}
if(h->is_avc && buf_size >= 9 && buf[0]==1 && buf[2]==0 && (buf[4]&0xFC)==0xFC && (buf[5]&0x1F) && buf[8]==0x67){
if (is_extra(buf, buf_size))
return ff_h264_decode_extradata(buf, buf_size,
&h->ps, &h->is_avc, &h->nal_length_size,
avctx->err_recognition, avctx);
}
buf_index = decode_nal_units(h, buf, buf_size);
if (buf_index < 0)
return AVERROR_INVALIDDATA;
if (!h->cur_pic_ptr && h->nal_unit_type == H264_NAL_END_SEQUENCE) {
av_assert0(buf_index <= buf_size);
return send_next_delayed_frame(h, pict, got_frame, buf_index);
}
if (!(avctx->flags2 & AV_CODEC_FLAG2_CHUNKS) && (!h->cur_pic_ptr || !h->has_slice)) {
if (avctx->skip_frame >= AVDISCARD_NONREF ||
buf_size >= 4 && !memcmp("Q264", buf, 4))
return buf_size;
av_log(avctx, AV_LOG_ERROR, "no frame!\n");
return AVERROR_INVALIDDATA;
}
if (!(avctx->flags2 & AV_CODEC_FLAG2_CHUNKS) ||
(h->mb_y >= h->mb_height && h->mb_height)) {
if ((ret = ff_h264_field_end(h, &h->slice_ctx[0], 0)) < 0)
return ret;
/* Wait for second field. */
if (h->next_output_pic) {
ret = finalize_frame(h, pict, h->next_output_pic, got_frame);
if (ret < 0)
return ret;
}
}
av_assert0(pict->buf[0] || !*got_frame);
ff_h264_unref_picture(h, &h->last_pic_for_ec);
return get_consumed_bytes(buf_index, buf_size);
}
上述代码位于libcodec/h264dec.c下，是H.264核心的解码函数，可以看到，某些情况下在decode_nal_units() 获取NAL单元之前，先调用了ff_h264_decode_extradata()来处理extradata数据。主要功能是获取sps、pps、ps数据，位于libcodec/h264_parse.c下. avcc
int ff_h264_decode_extradata(const uint8_t *data, int size, H264ParamSets *ps,
int *is_avc, int *nal_length_size,
int err_recognition, void *logctx)
{
int ret;
if (!data || size <= 0)
return -1;
if (data[0] == 1) {
int i, cnt, nalsize;
const uint8_t *p = data;
*is_avc = 1;
if (size < 7) {
av_log(logctx, AV_LOG_ERROR, "avcC %d too short\n", size);
return AVERROR_INVALIDDATA;
}
// Decode sps from avcC
cnt = *(p + 5) & 0x1f; // Number of sps
p += 6;
for (i = 0; i < cnt; i++) {
nalsize = AV_RB16(p) + 2;
if (nalsize > size - (p - data))
return AVERROR_INVALIDDATA;
ret = decode_extradata_ps_mp4(p, nalsize, ps, err_recognition, logctx);
if (ret < 0) {
av_log(logctx, AV_LOG_ERROR,
"Decoding sps %d from avcC failed\n", i);
return ret;
}
p += nalsize;
}
// Decode pps from avcC
cnt = *(p++); // Number of pps
for (i = 0; i < cnt; i++) {
nalsize = AV_RB16(p) + 2;
if (nalsize > size - (p - data))
return AVERROR_INVALIDDATA;
ret = decode_extradata_ps_mp4(p, nalsize, ps, err_recognition, logctx);
if (ret < 0) {
av_log(logctx, AV_LOG_ERROR,
"Decoding pps %d from avcC failed\n", i);
return ret;
}
p += nalsize;
}
// Store right nal length size that will be used to parse all other nals
*nal_length_size = (data[4] & 0x03) + 1;
} else {
*is_avc = 0;
ret = decode_extradata_ps(data, size, ps, 0, logctx);
if (ret < 0)
return ret;
}
return size;
}
static int decode_extradata_ps(const uint8_t *data, int size, H264ParamSets *ps,
int is_avc, void *logctx)
{
H2645Packet pkt = { 0 };
int i, ret = 0;
ret = ff_h2645_packet_split(&pkt, data, size, logctx, is_avc, 2, AV_CODEC_ID_H264, 1);
if (ret < 0) {
ret = 0;
goto fail;
}
for (i = 0; i < pkt.nb_nals; i++) {
H2645NAL *nal = &pkt.nals[i];
switch (nal->type) {
case H264_NAL_SPS:
ret = ff_h264_decode_seq_parameter_set(&nal->gb, logctx, ps, 0);
if (ret < 0)
goto fail;
break;
case H264_NAL_PPS:
ret = ff_h264_decode_picture_parameter_set(&nal->gb, logctx, ps,
nal->size_bits);
if (ret < 0)
goto fail;
break;
default:
av_log(logctx, AV_LOG_VERBOSE, "Ignoring NAL type %d in extradata\n",
nal->type);
break;
}
}
fail:
ff_h2645_packet_uninit(&pkt);
return ret;
}
ff_h2645_packet_split()位于libcodec/h2645_parse.c下
int ff_h2645_packet_split(H2645Packet *pkt, const uint8_t *buf, int length,
void *logctx, int is_nalff, int nal_length_size,
enum AVCodecID codec_id, int small_padding)
{
GetByteContext bc;
int consumed, ret = 0;
int next_avc = is_nalff ? 0 : length;
int64_t padding = small_padding ? 0 : MAX_MBPAIR_SIZE;
bytestream2_init(&bc, buf, length);
av_fast_padded_malloc(&pkt->rbsp.rbsp_buffer, &pkt->rbsp.rbsp_buffer_alloc_size, length + padding);
if (!pkt->rbsp.rbsp_buffer)
return AVERROR(ENOMEM);
pkt->rbsp.rbsp_buffer_size = 0;
pkt->nb_nals = 0;
while (bytestream2_get_bytes_left(&bc) >= 4) {
H2645NAL *nal;
int extract_length = 0;
int skip_trailing_zeros = 1;
if (bytestream2_tell(&bc) == next_avc) {
int i = 0;
extract_length = get_nalsize(nal_length_size,
bc.buffer, bytestream2_get_bytes_left(&bc), &i, logctx);
if (extract_length < 0)
return extract_length;
bytestream2_skip(&bc, nal_length_size);
next_avc = bytestream2_tell(&bc) + extract_length;
} else {
int buf_index;
if (bytestream2_tell(&bc) > next_avc)
av_log(logctx, AV_LOG_WARNING, "Exceeded next NALFF position, re-syncing.\n");
/* search start code */
buf_index = find_next_start_code(bc.buffer, buf + next_avc);
bytestream2_skip(&bc, buf_index);
if (!bytestream2_get_bytes_left(&bc)) {
if (pkt->nb_nals > 0) {
// No more start codes: we discarded some irrelevant
// bytes at the end of the packet.
return 0;
} else {
av_log(logctx, AV_LOG_ERROR, "No start code is found.\n");
return AVERROR_INVALIDDATA;
}
}
extract_length = FFMIN(bytestream2_get_bytes_left(&bc), next_avc - bytestream2_tell(&bc));
if (bytestream2_tell(&bc) >= next_avc) {
/* skip to the start of the next NAL */
bytestream2_skip(&bc, next_avc - bytestream2_tell(&bc));
continue;
}
}
if (pkt->nals_allocated < pkt->nb_nals + 1) {
int new_size = pkt->nals_allocated + 1;
void *tmp = av_realloc_array(pkt->nals, new_size, sizeof(*pkt->nals));
if (!tmp)
return AVERROR(ENOMEM);
pkt->nals = tmp;
memset(pkt->nals + pkt->nals_allocated, 0,
(new_size - pkt->nals_allocated) * sizeof(*pkt->nals));
nal = &pkt->nals[pkt->nb_nals];
nal->skipped_bytes_pos_size = 1024; // initial buffer size
nal->skipped_bytes_pos = av_malloc_array(nal->skipped_bytes_pos_size, sizeof(*nal->skipped_bytes_pos));
if (!nal->skipped_bytes_pos)
return AVERROR(ENOMEM);
pkt->nals_allocated = new_size;
}
nal = &pkt->nals[pkt->nb_nals];
consumed = ff_h2645_extract_rbsp(bc.buffer, extract_length, &pkt->rbsp, nal, small_padding);
if (consumed < 0)
return consumed;
if (is_nalff && (extract_length != consumed) && extract_length)
av_log(logctx, AV_LOG_DEBUG,
"NALFF: Consumed only %d bytes instead of %d\n",
consumed, extract_length);
pkt->nb_nals++;
bytestream2_skip(&bc, consumed);
/* see commit 3566042a0 */
if (bytestream2_get_bytes_left(&bc) >= 4 &&
bytestream2_peek_be32(&bc) == 0x000001E0)
skip_trailing_zeros = 0;
nal->size_bits = get_bit_length(nal, skip_trailing_zeros);
ret = init_get_bits(&nal->gb, nal->data, nal->size_bits);
if (ret < 0)
return ret;
if (codec_id == AV_CODEC_ID_HEVC)
ret = hevc_parse_nal_header(nal, logctx);
else
ret = h264_parse_nal_header(nal, logctx);
if (ret <= 0 || nal->size <= 0) {
if (ret < 0) {
av_log(logctx, AV_LOG_ERROR, "Invalid NAL unit %d, skipping.\n",
nal->type);
}
pkt->nb_nals--;
}
}
return 0;
}
BOX TYPE avcc
0x61 0x76 0x63 0x43
对于mp4文件，以下是avcc具体数据，也就是extradata内容。
0x1 version
0x4d sps[1]
0x40 sps[2]
0x1e sps[3]
0b111111 6位reserved
0b11 2位h264中NALU长度，1+(size&3) = 4
0b111. 3位reserved
0b00001 sps个数1
sps 数据长度23
0x0 0x17
sps长度
0x67 0x4d 0x40 0x1e 0xe8 0x80 0x50 0x8 0x34 0x20 0x0 0x0 0x3 0x0 0x20 0x0 0x0 0x6 0x51 0xe2 0xc5 0xa2 0x40
0x1 pps个数
0x0 0x5 pps长度
pps数据
0x68 0xeb 0xec 0x4c 0x80