ImagePlayer为mtk实现的用来播放图片的播放器, 接入MediaPlayer中, 能实现4k播放
/vendor/mediatek/proprietary_tv/open/common/libraries/media/mmplayer/imgplayer
ImagePlayer接入
- 添加player_type枚举, IMG_PLAYER 为9
- 在MediaPlayerFactory registerBuiltinFactories中, new ImagePlayerFactory注册到sFactoryMap中
-
继承MediaPlayerFactory::IFactory, 实现ImagePlayerFactory, 重写需要source类型的scoreFactory及createPlayer方法. 这里可以看到ImagePlayerFactory 只重写了source类型为fd或者url类型的方法. 意味着只有这两个类型的播放,才可能选到ImagePlayer
class ImagePlayerFactory : public MediaPlayerFactory::IFactory { public: virtual float scoreFactory(const sp& /client/, const char* url, const sp &httpService, float curScore) { static const float kOurScore = 0.9;
if (kOurScore <= curScore) return 0.0; const char *ptr = strrchr(url, '/'); if (ptr != NULL) { ptr += 1; if ((strcasecmp(ptr, "video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_192kbps_44100hz") == 0) || (strcasecmp(ptr, "video_480x360_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz") == 0) || (strcasecmp(ptr, "video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_128kbps_44100hz") == 0) || (strcasecmp(ptr, "video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_192kbps_44100hz") == 0) || (strcasecmp(ptr, "video_176x144_3gp_h263_300kbps_25fps_aac_stereo_128kbps_22050hz") == 0)) { return 0.0; } }#ifdef SUPPORT_IMAGEPLAYER_DLOPEN if (ImagePlayerDrive::canBeUsed(url, httpService)) { #else if (ImagePlayer::canBeUsed(url, httpService)) { #endif return 1.0; }
return 0.0; } virtual float scoreFactory(const sp<IMediaPlayer>& /*client*/, int fd, int64_t /*offset*/, int64_t /*length*/, float curScore) { static const float kOurScore = 0.9; if (kOurScore <= curScore) return 0.0;#ifdef SUPPORT_IMAGEPLAYER_DLOPEN if (ImagePlayerDrive::canBeUsed(fd)) { #else if (ImagePlayer::canBeUsed(fd)) { #endif return 1.0; }
return 0.0; } virtual sp<MediaPlayerBase> createPlayer(pid_t /* pid */) {#ifdef SUPPORT_IMAGEPLAYER_DLOPEN return new ImagePlayerDrive(); #else return new ImagePlayer(); #endif } }; #endif
ImagePlayer实现
主要类继承调用关系如下:
ImagePlayer 使用流程:
-
创建MediaPlayer对象mMediaPlayer = new MediaPlayer()
-
设置播放文件 mMediaPlayer.setDataSource
-
通过mMediaPlayer.setDisplay()设置显示surface
-
mMediaPlayer.Prepare
-
mMediaPlayer.setParameter 设置图片的size和orientation等
-
开始播放mMediaPlayer.start
-
播放结束mMediaPlayer.stop
所有的接口调用,最终都走到ImagePlayer
ImagePlayer 创建流程
setDataSource流程
setVideoSurfaceTexture流程
Prepare 流程
setParameter流程
在app设置对应surface参数下来后, 就启动了解码流程, 并生成mDstImageDrawable
start流程
app调用start后, 直接调用show显示mDstImageDrawable
displayByMvop最终调用ANativeWindow方法将buffer 写到surface中
bool ImagePlayerImpl::displayByMvop(sp<ImageDrawable> &imageDrawable)
{
ALOGV("%s Enter", __FUNCTION__);
bool ret = false;
ANativeWindowBuffer* anb = NULL;
status_t err = native_window_dequeue_buffer_and_wait(mNativeWindow.get(), &anb);
if (err != OK) {
ALOGE("dequeueBuffer failed: %s (%d)", strerror(-err), -err);
} else {
#if defined(AN_O_BEFORE)
sp<GraphicBuffer> buf(new GraphicBuffer(anb, false));
#else
sp<GraphicBuffer> buf(GraphicBuffer::from(anb));
#endif
void* img = NULL;
#if defined(BUILD_PURE_AOSP) || !defined(AN_O_BEFORE)
err = buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN | GRALLOC_USAGE_PRIVATE_2, (void**)(&img));
#else
err = buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN, (void**)(&img));
#endif
if (err != NO_ERROR) {
ALOGE("error pushing image frame: lock failed: %s (%d)",
strerror(-err), -err);
goto error;
}
bool bGen4MvopSuccess;
bGen4MvopSuccess = imageDrawable->draw4Mvop(img);
if (bGen4MvopSuccess != true) {
err = UNKNOWN_ERROR;
ALOGE("error generate image frame");
goto error;
}
err = buf->unlock();
if (err != NO_ERROR) {
ALOGE("error pushing image frame: unlock failed: %s (%d)",
strerror(-err), -err);
goto error;
}
ALOGD("[%s] queueBuffer to surface", __FUNCTION__);
err = mNativeWindow->queueBuffer(mNativeWindow.get(),
buf->getNativeBuffer(), -1);
if (err != NO_ERROR) {
ALOGE("error pushing image frame: queueBuffer failed: %s (%d)",
strerror(-err), -err);
goto error;
}
ret = true;
error:
anb = NULL;
}
return ret;
}
stop流程类似,不详细介绍了
SkImageDrawable与HwImageDrawable
通过设置PROPERTY_IMGHWDEC_SW_SCALE属性, 可以决定使用SkImageDrawable 还是使用HwImageDrawable
#define PROPERTY_IMGHWDEC_SW_SCALE "vendor.mediatek.imghwdec.swscale"
SkImageDrawable的scale, rotate, crop等都使用软件做, 新建一个SkCanvas, 软件处理好的数据, 再画到SkCanvas上
sp<SkBitmapHolder> SkImageDrawable::doRotate(const sp<SkBitmapHolder> &srcBitmapHolder, float degrees)
{
const SkBitmap *srcBitmap = srcBitmapHolder->getSkData();
if (srcBitmap == NULL)
return NULL;
SkBitmap *devBitmap = new SkBitmap();
SkMatrix *matrix = new SkMatrix();
SkCanvas *canvas = NULL;
int sourceWidth = srcBitmap->width();
int sourceHeight = srcBitmap->height();
double radian = SkDegreesToRadians(degrees);
int dstWidth = sourceWidth * fabs(cos(radian)) + sourceHeight * fabs(sin(radian));
int dstHeight = sourceHeight * fabs(cos(radian)) + sourceWidth * fabs(sin(radian));
...
devBitmap->setInfo(SkImageInfo::Make(dstWidth, dstHeight, kN32_SkColorType, kOpaque_SkAlphaType));
devBitmap->allocPixels();
devBitmap->eraseARGB(255, 0, 0, 0);
canvas = new SkCanvas(*devBitmap);
matrix->postRotate(degrees, sourceWidth / 2, sourceHeight / 2);
matrix->postTranslate((dstWidth - sourceWidth) / 2, (dstHeight - sourceHeight) / 2);
canvas->concat(*matrix);
canvas->drawBitmap(*srcBitmap, 0, 0);
delete canvas;
delete matrix;
return new SkBitmapHolder(devBitmap);
}
HwImageDrawable的scale, rotate, crop等都使用硬件做, 在HwImageDrawable中只进行相关参数的计算
bool HwImageDrawable::rotate(float degrees)
{
ALOGV("%s degrees: %f", __FUNCTION__, degrees);
if (rotate2integer(degrees) % 90 != 0) {
ALOGE("%s bad degrees: %d(%f)", __FUNCTION__, rotate2integer(degrees), degrees);
return false;
}
mDrawParameters = doRotate(mDrawParameters, degrees);
dumpDrawParameters(__FUNCTION__, mDrawParameters);
return true;
}
HwImageDrawable::DrawParameters HwImageDrawable::doRotate(const HwImageDrawable::DrawParameters ¶meters, float degrees)
{
HwImageDrawable::DrawParameters dstDrawParameters = parameters;
dstDrawParameters.dstDrawMatrix = rotateMatrix(parameters.dstSurfaceSize, parameters.dstDrawMatrix, degrees);
dstDrawParameters.dstSurfaceSize = rotateSize(parameters.dstSurfaceSize, degrees);
dstDrawParameters.degrees = (float)rotate2integer(parameters.degrees + degrees);
return dstDrawParameters;
}
在draw4Mvop方法中,将buffer及参数传到mImgDispClient中进行相关scale, rotate等操作
bool HwImageDrawable::draw4Mvop(void *buffer)
{
bool ret = false;
sp<ImgDispClient> imgDispClient = mImgDispClient.promote();
if (imgDispClient != NULL) {
dumpDrawParameters(__FUNCTION__, mDrawParameters);
ImgDispClient::ConvertParameters convertParameters;
convertParameters.source = mDrawableBuffer->getBufferInfo();
convertParameters.crop.x = mDrawParameters.srcCropMatrix.x;
convertParameters.crop.y = mDrawParameters.srcCropMatrix.y;
convertParameters.crop.width = mDrawParameters.srcCropMatrix.width;
convertParameters.crop.height = mDrawParameters.srcCropMatrix.height;
convertParameters.angle = transRotateDegrees(mDrawParameters.degrees);
// only set width and height for destination
convertParameters.destination.width = mDrawParameters.dstSurfaceSize.width;
convertParameters.destination.height = mDrawParameters.dstSurfaceSize.height;
convertParameters.draw.x = mDrawParameters.dstDrawMatrix.x;
convertParameters.draw.y = mDrawParameters.dstDrawMatrix.y;
convertParameters.draw.width = mDrawParameters.dstDrawMatrix.width;
convertParameters.draw.height = mDrawParameters.dstDrawMatrix.height;
ret = imgDispClient->convertMvopHeader(buffer, convertParameters);
} else {
ALOGW("mImgDispClient promote failed");
}
return ret;
}
convertMvopHeader 最终调用到ImageDisp的ImageDisp_convert方法
//vendor/mediatek/proprietary_tv/open/common/libraries/media/mmplayer/imgplayer/ImageDisp.cpp
bool ImageDisp_convert(void *img, ImgDisp_ConvertParameters_t *pConvertParameters)
{
...
bRet = PortingLayer_DISP_Convert_Format(pConvertParameters, mOutputColorFmt);
...
return imageDisp_genDispFrameFormat(
dff,
pConvertParameters->destination.width,
alignedWidth,
pConvertParameters->destination.height,
u64PhyPaddr,
(unsigned char *)pPhyVaddr,
pConvertParameters->destination.width,
pConvertParameters->destination.height);
}
//vendor/mediatek/proprietary_tv/open/common/libraries/media/mmplayer/imgplayer/porting_layer/mi3/ImageDISP_Portinglayer_mi3.cpp
static inline bool OSD_Convert_Format_ex(const OSD_ConvertParameters parameters)
{
...
MI_OSD_BeginDraw();
do {
...
eRet = MI_OSD_FillRect(hDstSurface, &stRectAttr, &stFillRectBlitOpt, NULL /*&stWaitRenderJobId*/);
if (eRet != MI_OK) {
ALOGE("MI_OSD_FillRect failed, eRet:%d", eRet);
ret = false;
break;
}
}
...
eRet = MI_OSD_Bitblit(hSrcSurface, &stSrcRect, hDstSurface, &stDstRect, &stBlitOpt, &stWaitRenderJobId);
if (eRet != MI_OK) {
ALOGE("MI_OSD_Bitblit failed, eRet:%d", eRet);
ret = false;
break;
}
// Wait for bitblt operation done.
eRet = MI_OSD_WaitRenderDone(&stWaitRenderJobId);
if (eRet != MI_OK) {
ALOGE("MI_OSD_WaitRenderDone failed, eRet:%d", eRet);
ret = false;
break;
}
} while (0);
...
MI_OSD_EndDraw();
return ret;
}
SwImageDecoder与HwImageDecoder
通过设置PROPERTY_IMGHWDEC_SW_DEC属性,可以决定使用SwImageDecoder还是HwImageDecoder
#define PROPERTY_IMGHWDEC_SW_DEC "vendor.mediatek.imghwdec.swdec"
SwImageDecoder与HwImageDecoder的区别主要在于是硬件解码,还是软件解码
SwImageDecoder 主要用Skia库进行解码,使用MediaMetadataRetriever来获取图片帧
SkBitmap* SwImageDecoder::decode(SkCodecType *codec, int sampleSize, SkBaseStream *stream)
{
...
const SkImageInfo decodeInfo = SkImageInfo::Make(size.width(), size.height(), decodeColorType, alphaType);
SkBitmap *bitmap = new SkBitmap();
...
SkAndroidCodec::AndroidOptions codecOptions;
if (sampleSize > 0) {
codecOptions.fSampleSize = sampleSize;
}
SkCodec::Result result = codec->getAndroidPixels(decodeInfo, bitmap->getPixels(), bitmap->rowBytes(), &codecOptions);
...
}
HwImageDecoder使用硬件解码
sp<ImageDrawable> HwImageDecoder::onDecode(int sampleSize, sp<ImageStreamFactory> &streamFactory, const wp<ImgDispClient> &imgDispClient)
{
sp<ImageDrawable> imageDrawable = NULL;
do {
...
ALOGV("MS_HIDL_Image_HwDec_GetReadBuffer start");
if (false == MS_HIDL_Image_HwDec_GetReadBuffer(&stReadBufferInfo)) {
ALOGE("MS_HIDL_Image_HwDec_GetReadBuffer fail, go SW");
mForceSw = true;
break;
}
...
if (haveRead > 0 && mIsCancelDecode == false) {
...
if (false == MS_HIDL_Image_HwDec_Open(&stOpenInfo)) {
ALOGE("%s %d, MS_HIDL_Image_HwDec_Open fail go SW", __FUNCTION__, __LINE__);
mForceSw = true;
break;
}
ALOGD("%s open hw decoder done", __FUNCTION__);
if (false == MS_HIDL_Image_HwDec_Decode()) {
ALOGE("%s %d, MS_HIDL_Image_HwDec_Decode fail go SW", __FUNCTION__, __LINE__);
mForceSw = true;
MS_HIDL_Image_HwDec_Close();
break;
}
}
ImageHwDec_Status_e eDecStatus = ImageHwDec_STATUS_DECODING;
while (!mIsCancelDecode && eDecStatus == ImageHwDec_STATUS_DECODING) {
ImageHwDec_PhotoInfo_t stPhotoInfo;
eDecStatus = MS_HIDL_Image_HwDec_GetDecodeStatus();
if (eDecStatus == ImageHwDec_STATUS_DECODE_OK) {
ALOGD("%s hw decode done", __FUNCTION__);
bool bGetInfo = MS_HIDL_Image_HwDec_GetPhotoInfo(&stPhotoInfo);
...
}
经过HIDL调用后, 最终调用到Mi_Image接口
int32_t ImgHwDecWrapper::ImgHwDecWrapperDecode()
{
Status status;
if (!mInitDone) {
ALOGE("%s %d not init done", __FUNCTION__, __LINE__);
return -1;
}
status = mImgHwDec->ImageHwDec_decode();
if (status != Status::OK) {
ALOGE("%s %d ImageHwDec_decode failed, status: %d", __FUNCTION__, __LINE__, static_cast<int32_t>(status));
return -1;
}
return 0;
}
bool ImageHwDec_decode()
{
...
eRet = MI_IMGDEC_Decode(mImgdecHandle, stIMGInfo);
if (eRet != MI_OK) {
ALOGE("MI_IMGDEC_Decode failed\n");
return false;
}
...
}
解码流程, 以HEIF格式为例
HIEF格式介绍
HEIF/heic图片文件解析
ImagePlayer方式：
ImagePlayer -> SwImageDecoder -> SkCodec -> SkHeifCodec -> HeifDecoderImpl --> MediaMetadataRetriever::getImageRectAtIndex -> FrameDecoder::extractFrame
SwImageDecoder创建流程
std::unique_ptr<SkCodec> SkCodec::MakeFromStream(std::unique_ptr<SkStream> stream,
Result* outResult, SkPngChunkReader* chunkReader) {
...
for (DecoderProc proc : gDecoderProcs) {
if (proc.IsFormat(buffer, bytesRead)) {
return proc.MakeFromStream(std::move(stream), outResult);
}
}
...
}
static constexpr DecoderProc gDecoderProcs[] = {
#ifdef SK_HAS_JPEG_LIBRARY
{ SkJpegCodec::IsJpeg, SkJpegCodec::MakeFromStream },
#endif
#ifdef SK_HAS_WEBP_LIBRARY
{ SkWebpCodec::IsWebp, SkWebpCodec::MakeFromStream },
#endif
{ SkGifCodec::IsGif, SkGifCodec::MakeFromStream },
#ifdef SK_HAS_PNG_LIBRARY
{ SkIcoCodec::IsIco, SkIcoCodec::MakeFromStream },
#endif
{ SkBmpCodec::IsBmp, SkBmpCodec::MakeFromStream },
{ SkWbmpCodec::IsWbmp, SkWbmpCodec::MakeFromStream },
#ifdef SK_HAS_HEIF_LIBRARY
{ SkHeifCodec::IsHeif, SkHeifCodec::MakeFromStream },
#endif
};
status_t FrameDecoder::init(
int64_t frameTimeUs, size_t numFrames, int option, int colorFormat) {
...
sp<MediaCodec> decoder = MediaCodec::CreateByComponentName(
looper, mComponentName, &err);
if (decoder.get() == NULL || err != OK) {
ALOGW("Failed to instantiate decoder [%s]", mComponentName.c_str());
return (decoder.get() == NULL) ? NO_MEMORY : err;
}
...
err = decoder->configure(
videoFormat, NULL /* surface */, NULL /* crypto */, 0 /* flags */);
if (err != OK) {
ALOGW("configure returned error %d (%s)", err, asString(err));
decoder->release();
return err;
}
err = decoder->start();
if (err != OK) {
ALOGW("start returned error %d (%s)", err, asString(err));
decoder->release();
return err;
}
err = mSource->start();
if (err != OK) {
ALOGW("source failed to start: %d (%s)", err, asString(err));
decoder->release();
return err;
}
mDecoder = decoder;
return OK;
}
status_t FrameDecoder::extractInternal() {
status_t err = OK;
bool done = false;
size_t retriesLeft = kRetryCount;
do {
size_t index;
int64_t ptsUs = 0ll;
uint32_t flags = 0;
// Queue as many inputs as we possibly can, then block on dequeuing
// outputs. After getting each output, come back and queue the inputs
// again to keep the decoder busy.
while (mHaveMoreInputs) {
err = mDecoder->dequeueInputBuffer(&index, 0);
if (err != OK) {
ALOGV("Timed out waiting for input");
if (retriesLeft) {
err = OK;
}
break;
}
sp<MediaCodecBuffer> codecBuffer;
err = mDecoder->getInputBuffer(index, &codecBuffer);
if (err != OK) {
ALOGE("failed to get input buffer %zu", index);
break;
}
MediaBufferBase *mediaBuffer = NULL;
err = mSource->read(&mediaBuffer, &mReadOptions);
mReadOptions.clearSeekTo();
if (err != OK) {
mHaveMoreInputs = false;
if (!mFirstSample && err == ERROR_END_OF_STREAM) {
(void)mDecoder->queueInputBuffer(
index, 0, 0, 0, MediaCodec::BUFFER_FLAG_EOS);
err = OK;
} else {
ALOGW("Input Error: err=%d", err);
}
break;
}
...
}
while (err == OK) {
size_t offset, size;
// wait for a decoded buffer
err = mDecoder->dequeueOutputBuffer(
&index,
&offset,
&size,
&ptsUs,
&flags,
kBufferTimeOutUs);
if (err == INFO_FORMAT_CHANGED) {
ALOGV("Received format change");
err = mDecoder->getOutputFormat(&mOutputFormat);
} else if (err == INFO_OUTPUT_BUFFERS_CHANGED) {
ALOGV("Output buffers changed");
err = OK;
} else {
if (err == -EAGAIN /* INFO_TRY_AGAIN_LATER */ && --retriesLeft > 0) {
ALOGV("Timed-out waiting for output.. retries left = %zu", retriesLeft);
err = OK;
} else if (err == OK) {
// If we're seeking with CLOSEST option and obtained a valid targetTimeUs
// from the extractor, decode to the specified frame. Otherwise we're done.
ALOGV("Received an output buffer, timeUs=%lld", (long long)ptsUs);
sp<MediaCodecBuffer> videoFrameBuffer;
err = mDecoder->getOutputBuffer(index, &videoFrameBuffer);
if (err != OK) {
ALOGE("failed to get output buffer %zu", index);
break;
}
err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
mDecoder->releaseOutputBuffer(index);
} else {
ALOGW("Received error %d (%s) instead of output", err, asString(err));
done = true;
}
break;
}
}
} while (err == OK && !done);
if (err != OK) {
ALOGE("failed to get video frame (err %d)", err);
}
return err;
}
ImageView解码流程
ImageView的解码流程与ImagePlayer的类似
ImageView -> ImageDecoder -> SkCodec -> SkHeifCodec -> HeifDecoder --> getImageRectAtIndex -> FrameDecoder::extractFrame
Skia图片解析流程与图片编码原理初探
Android 图形显示系统（十五） Android Q Skia的绘制系统
