基于FFmpeg的Android播放器

基于FFmpeg的Android播放器

1. 前言

FFmpeg是一个最有名的开源的编解码库，实现了通常的编解码逻辑。它还能够根据平台特性，与平台自有的硬件编解码器进行适配。经过一段时间的学习后，我使用FFmpeg在Android上实现了一个简单的支持硬件解码的视频播放器。在此写下这篇博客记录关键知识点。

代码在此：Android-VideoPlayer

整体上，该工程是基于FFmpeg的，不仅是基于FFmpeg的解码能力，而且一些标志位等直接使用FFmpeg的，不再另外包装。

2. 编译相关组件库

要使用FFmpeg，需要先将FFmpeg库本身以及它所需要的一些第三方库编译成Android平台库。具体编译脚本和库我已经完成并放到了github上，相关build下面都有readme，记录了编译的事项和相关步骤。

• FFmpeg6.0：FFmpeg_build。基于最新的FFmpeg6.0编译，该版本的最大变化是支持了NDK的MediaCodec框架，在此之前，FFmpeg桥接Android硬件解码器的方法是通过反射调用到Java层的MediaCodec，需要将数据从Native拷贝到Java层，开销比较大，用起来也比较麻烦。另外一个重大更新是支持了av1-mediacodec，av1是一种比HEVC更高效的编码格式，体积小，但是编解码开销比较大，而且由于格式比较新，最近两三代的手机处理器才支持了av1的硬解，而编码器更是延后，今年苹果的A17 pro才支持了av1编码。但是这个格式将来一定会占有非常大的市场，目前从YouTube下载的HDR视频已经都是av1格式了。为了支持Android硬解，需要开启一些编译开关，最重要的是mediacodec后缀的那几个decoder，在FFmpeg源码目录运行./configure --list-decoders可以看到FFmpeg支持的所有的decoder。这个编译版本开启了非免费第三方库，不可用来商用，并且由于是练习，为了支持尽可能多的格式，没有进行过多裁切。
• libaom：aom_build， libaom是av1的编解码库，为了在老旧的cpu上支持av1解码，集成了该库。
• libx265 & libx264：x264_build，x265_build， H264与HEVC的编解码库，兜底用。
• fdk-aac & mp3lame：fdk-aac_build ，lame_build，aac与mp3的音频编解码库，一般都可以硬解。这里是为了编码做准备。

3. 解码器

目前，在Android系统中，支持的解码器就是那明确的几个。如果要使用硬件解码器，必须以名字来查询而不是codec_id。

#define HW_DEC_COUNT 7#define HW_DEC_H264 "h264_mediacodec"
#define HW_DEC_HEVC "hevc_mediacodec"
#define HW_DEC_VP8 "vp8_mediacodec"
#define HW_DEC_VP9 "vp9_mediacodec"
#define HW_DEC_AV1 "av1_mediacodec"
#define HW_DEC_MPEG2 "mpeg2_mediacodec"
#define HW_DEC_MPEG4 "mpeg4_mediacodec"const static AVCodecID HW_DECODERS[HW_DEC_COUNT] = {AVCodecID::AV_CODEC_ID_H264,AVCodecID::AV_CODEC_ID_HEVC,AVCodecID::AV_CODEC_ID_VP8,AVCodecID::AV_CODEC_ID_VP9,AVCodecID::AV_CODEC_ID_AV1,AVCodecID::AV_CODEC_ID_MPEG2VIDEO,AVCodecID::AV_CODEC_ID_MPEG4
};const static const char* HW_DECODER_NAMES[HW_DEC_COUNT] = {HW_DEC_H264,HW_DEC_HEVC,HW_DEC_VP8,HW_DEC_VP9,HW_DEC_AV1,HW_DEC_MPEG2,HW_DEC_MPEG4
};static bool supportHWDec(AVCodecID codecId) {for (AVCodecID id : HW_DECODERS) {if (id == codecId) {return true;}}return false;
}static const char* getHWDecName(AVCodecID codecId) {for (int i = 0; i < HW_DEC_COUNT; i++) {if (HW_DECODERS[i] == codecId) {return HW_DECODER_NAMES[i];}}return nullptr;
}

// 根据参数查找相应的decoder
bool FFmpegDecoder::init(AVCodecParameters *params, PreferCodecType preferType) {AVCodecID ffCodecID = AV_CODEC_ID_NONE;try {ffCodecID = AVCodecID(params->codec_id);} catch (...) {LOGE(TAG, "failed to convert %d to AVCodecID", params->codec_id);return false;}// 可以支持指定解码器类型。如果未指定，那就优先查找硬件解码器，找不到再去找软件解码器if (preferType == PreferCodecType::HW) {return findHWDecoder(params, ffCodecID);} else if (preferType == PreferCodecType::SW) {return findSWDecoder(params, ffCodecID);} else {if (findHWDecoder(params, ffCodecID)) {return true;}if (findSWDecoder(params, ffCodecID)) {return true;}return false;}
}// 查找硬件解码器
bool FFmpegDecoder::findHWDecoder(AVCodecParameters *params, AVCodecID codecId) {release();int ret;const char *hwDecName = getHWDecName(codecId);if (hwDecName == nullptr) {return false;}const AVCodec * aCodec = avcodec_find_decoder_by_name(hwDecName);if (aCodec == nullptr) {LOGE(TAG, "Can't find hw decoder for codec: {id = %d, hw_name = %s}", codecId, hwDecName);return false;} else {codec = const_cast<AVCodec *>(aCodec);}codecCtx = avcodec_alloc_context3(codec);if (!codecCtx) {LOGE(TAG, "failed to alloc codec context");return false;}ret = avcodec_parameters_to_context(codecCtx, params);if (ret < 0) {LOGE(TAG, "copy decoder params failed, err = %d", ret);return false;}for (int i = 0;;i++) {const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i);if (config == nullptr) {LOGE(TAG, "%s hw config is null", codec->name);break;}if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) &&config->device_type == AVHWDeviceType::AV_HWDEVICE_TYPE_MEDIACODEC) {// 该解码器支持硬件解码out_hw_pix_format = config->pix_fmt;hwPixFormat = config->pix_fmt;codecCtx->get_format = get_hw_format;if (initHWDecoder(codecCtx, AVHWDeviceType::AV_HWDEVICE_TYPE_MEDIACODEC) < 0) {LOGE(TAG, "initHWDecoder failed");return false;} else {break;}}}ret = avcodec_open2(codecCtx, codec, nullptr);if (ret < 0) {char buf[100];av_make_error_string(buf, 100, ret);LOGE(TAG, "open codec failed for %s, err = %s", codec->name, buf);return false;}codecType = CodecType::HW;return true;
}// 查找软件解码器
bool FFmpegDecoder::findSWDecoder(AVCodecParameters *params, AVCodecID codecId) {release();int ret;const AVCodec * aCodec = avcodec_find_decoder(codecId);if (aCodec == nullptr) {LOGE(TAG, "Can't find decoder for codecID %d", codecId);return false;}codec = const_cast<AVCodec *>(aCodec);codecCtx = avcodec_alloc_context3(codec);if (!codecCtx) {LOGE(TAG, "failed to alloc codec context");return false;}ret = avcodec_parameters_to_context(codecCtx, params);if (ret < 0) {LOGE(TAG, "copy decoder params failed, err = %d", ret);return false;}ret = avcodec_open2(codecCtx, codec, nullptr);if (ret < 0) {char buf[100];av_make_error_string(buf, 100, ret);LOGE(TAG, "open codec failed for %s, err = %s", codec->name, buf);return false;}codecType = CodecType::SW;return true;
}

4. 解码流程

解码其实很简单。首先是先创建方便使用的音频和视频帧结构体，这两个结构体分别是AudioFrame和VideoFrame。其实也就是额外包含了一些属性方便访问和使用，内容物还是AVFrame。

对于解码，其实流程比较固定。

• 打开文件，获取对应的FormatContext与Codec。
• 通过AVFormatContext解复用，获取音频和视频的AVPacket，并将其存放到对应音频解码和视频解码的同步队列中，等待解码线程取用。
• 音频和视频解码线程从各自的AVPacket同步队列中取出AVPacket，并进行解码。解码完成后获取AVFrame，然后将其设置到AudioFrame和VideoFrame中，并存放到对应的同步队列中，等待同步线程取用。
• 同步线程从AudioFrame和VideoFrame的同步队列中取出已经解码好的数据，根据各自的pts来决定将AudioFrame和VideoFrame送到输出的时机。

音视频同步逻辑：在视频播放的过程中，音频是连续不断的，而视频却有不同帧率，所以同步是基于音频的时间戳。当然有些视频没有音频内容，此时就需要独立时钟来作为同步时间戳。

对于seek功能的支持：当用户进行seek时，置一个seekFlag为true。在解复用阶段，如果seekFlag为true，那解复用就对文件进行seek。同时对音视频AVPacket同步队列都清空，并向其中各存放一个seek标志的AVPacket。然后继续从seek点开始解复用。解码阶段在读取解复用存放的特殊AVPacket时，就对解码器进行reset，清空其内部缓存，然后存放两个seek标志分别到AVFrame的同步队列。在同步阶段，如果同步线程读取到任何一个具有seek标志的AVFrame，就停止输送到输出，并等待另一个同步队列读取到具有seek标志的AVFrame。在音视频都读取到seek标志之前，所有的AVFrame都弃用。读取到之后，重新进行同步。

解复用：

/** Read packet data from source.* If the packet has some flags like STREAM_FLAG_SOUGHT, this packet won't* contain data.* */
void Player::readStreamLoop() {if (!formatCtx) {LOGE(TAG, "no format context");return;}int ret;bool pushSuccess = false;while (!stopReadFlag) {AVPacket *packet = av_packet_alloc();if (!packet) {LOGE(TAG, "av_packet_alloc failed");return;}if (seekFlag) {int64_t pts = (int64_t) (seekPtsMS / 1000.0f * AV_TIME_BASE);LOGD(TAG, "meet seek, time = %lld", pts);int streamIndex = -1;
//            if (audioStreamIndex >= 0) {
//                pts = (int64_t)(seekPtsMS / av_q2d(formatCtx->streams[audioStreamIndex]->time_base));
//                streamIndex = audioStreamIndex;
//            } else if (videoStreamIndex >= 0) {
//                pts = (int64_t)(seekPtsMS / av_q2d(formatCtx->streams[videoStreamIndex]->time_base));
//                streamIndex = videoStreamIndex;
//            }av_seek_frame(formatCtx, streamIndex, pts, AVSEEK_FLAG_BACKWARD);// put a empty packet width flag STREAM_FLAG_SOUGHTif (enableAudio) {audioPacketQueue.clear();PacketWrapper *p = playerContext.getEmptyPacketWrapper();p->flags = STREAM_FLAG_SOUGHT;audioPacketQueue.forcePush(p);
//                audioDecodeSeekFlag = true;}if (enableVideo) {videoPacketQueue.clear();PacketWrapper *p = playerContext.getEmptyPacketWrapper();p->flags = STREAM_FLAG_SOUGHT;videoPacketQueue.forcePush(p);
//                videoDecodeSeekFlag = true;}//            syncSeekFlag = true;seekFlag = false;}ret = av_read_frame(formatCtx, packet);if (ret == 0) {if (packet->stream_index == audioStreamIndex && enableAudio) {PacketWrapper *pw = playerContext.getEmptyPacketWrapper();pw->setParams(packet);if (videoPacketQueue.getSize() == 0) {audioPacketQueue.forcePush(pw);} else {pushSuccess = audioPacketQueue.push(pw);if (!pushSuccess) {audioPacketQueue.forcePush(pw);}}} else if (packet->stream_index == videoStreamIndex && enableVideo) {PacketWrapper *pw = playerContext.getEmptyPacketWrapper();pw->setParams(packet);if (audioPacketQueue.getSize() == 0) {videoPacketQueue.forcePush(pw);} else {pushSuccess = videoPacketQueue.push(pw);if (!pushSuccess) {videoPacketQueue.forcePush(pw);}}} else {av_packet_unref(packet);av_packet_free(&packet);}} else if (ret == AVERROR_EOF) {av_packet_free(&packet);packet = nullptr;if (enableAudio) {PacketWrapper *pw = playerContext.getEmptyPacketWrapper();audioPacketQueue.forcePush(pw);}if (enableVideo) {PacketWrapper *pw = playerContext.getEmptyPacketWrapper();videoPacketQueue.forcePush(pw);}} else if (ret < 0) {LOGE(TAG, "av_read_frame failed");av_packet_free(&packet);packet = nullptr;return;}}
}

音频解码：

void Player::decodeAudioLoop() {if (!formatCtx) {return;}if (!audioDecoder) {LOGE(TAG, "audio decoder is null");return;}int ret;optional<PacketWrapper *> packetOpt;PacketWrapper *pw = nullptr;AVFrame *frame = nullptr;AudioFrame *audioFrame = nullptr;while (!stopDecodeAudioFlag && enableAudio) {packetOpt = audioPacketQueue.pop();if (!packetOpt.has_value()) {LOGE(TAG, "audio packetOpt has no value");break;}pw = packetOpt.value();if ((pw->flags & STREAM_FLAG_SOUGHT) == STREAM_FLAG_SOUGHT) {LOGD(TAG, "decode audio, meet a seek frame");playerContext.recyclePacketWrapper(pw);pw = nullptr;audioFrameQueue.clear();audioDecoder->flush();audioFrame = playerContext.getEmptyAudioFrame();audioFrame->flags |= STREAM_FLAG_SOUGHT;audioFrameQueue.forcePush(audioFrame);audioFrame = nullptr;continue;}ret = audioDecoder->sendPacket(pw->avPacket);if (ret < 0) {LOGE(TAG, "audio decoder send packet failed, err = %d", ret);break;}while (true) {frame = av_frame_alloc();ret = audioDecoder->receiveFrame(frame);if (ret < 0) {av_frame_unref(frame);av_frame_free(&frame);frame = nullptr;break;}audioFrame = playerContext.getEmptyAudioFrame();audioFrame->setParams(frame, audioStreamMap[audioStreamIndex].sampleFormat,formatCtx->streams[audioStreamIndex]->time_base);if (!audioFrameQueue.push(audioFrame)) {audioFrameQueue.push(audioFrame, false);}// DON'T delete AVFrame here, it will be carried to output by AudioFrameaudioFrame = nullptr;frame = nullptr;}if (ret == AVERROR(EAGAIN)) {
//            LOGD(TAG, "audio stream again");continue;} else if (ret == AVERROR_EOF) {
//            LOGD(TAG, "audio stream meets eof");break;} else {
//            LOGE(TAG, "audio decoder error: %d", ret);break;}}if (pw) {playerContext.recyclePacketWrapper(pw);}if (frame) {av_frame_unref(frame);av_frame_free(&frame);frame = nullptr;}if (audioFrame) {audioFrameQueue.push(audioFrame, false);audioFrame = nullptr;}LOGD(TAG, "audio decode loop finish");}

视频解码：

void Player::decodeVideoLoop() {if (!formatCtx) {return;}if (!videoDecoder) {LOGE(TAG, "video decoder is null");return;}int ret;optional<PacketWrapper *> packetOpt;PacketWrapper *pw = nullptr;AVFrame *frame = nullptr;VideoFrame *videoFrame = nullptr;while (!stopDecodeVideoFlag && enableVideo) {packetOpt = videoPacketQueue.pop();if (!packetOpt.has_value()) {LOGE(TAG, "video packetOpt has no value");break;}pw = packetOpt.value();if ((pw->flags & STREAM_FLAG_SOUGHT) == STREAM_FLAG_SOUGHT) {LOGD(TAG, "decode video, meet a seek frame");playerContext.recyclePacketWrapper(pw);pw = nullptr;videoFrameQueue.clear();videoDecoder->flush();videoFrame = playerContext.getEmptyVideoFrame();videoFrame->flags |= STREAM_FLAG_SOUGHT;videoFrameQueue.forcePush(videoFrame);videoFrame = nullptr;continue;}ret = videoDecoder->sendPacket(pw->avPacket);if (ret < 0) {LOGE(TAG, "video decoder send packet failed, err = %d", ret);break;}while (true) {frame = av_frame_alloc();ret = videoDecoder->receiveFrame(frame);if (ret < 0) {av_frame_unref(frame);av_frame_free(&frame);frame = nullptr;break;}videoFrame = playerContext.getEmptyVideoFrame();videoFrame->setParams(frame, AVPixelFormat(frame->format),formatCtx->streams[videoStreamIndex]->time_base);if (!videoFrameQueue.push(videoFrame)) {videoFrameQueue.push(videoFrame, false);}// DON'T delete AVFrame, it will be carried to output by VideoFrame.videoFrame = nullptr;frame = nullptr;}if (ret == AVERROR(EAGAIN)) {
//            LOGD(TAG, "video stream again");continue;} else if (ret == AVERROR_EOF) {
//            LOGD(TAG, "video stream meets eof");break;} else {
//            LOGE(TAG, "video decoder error: %d", ret);break;}}if (pw) {playerContext.recyclePacketWrapper(pw);}if (frame) {av_frame_unref(frame);av_frame_free(&frame);frame = nullptr;}if (videoFrame) {videoFrameQueue.push(videoFrame, false);videoFrame = nullptr;}LOGD(TAG, "video decode loop finish");}

同步代码：

void Player::syncLoop() {chrono::system_clock::time_point lastAudioWriteTime;chrono::system_clock::time_point lastVideoWriteTime;int64_t lastAudioPts = -1;int64_t lastVideoPts = -1;if (stateListener != nullptr) {stateListener->playStateChanged(true);}AudioFrame *audioFrame = unPlayedAudioFrame;unPlayedAudioFrame = nullptr;VideoFrame *videoFrame = unPlayedVideoFrame;unPlayedVideoFrame = nullptr;while (!stopSyncFlag) {if (enableAudio && enableVideo) {if (audioFrame == nullptr) {optional<AudioFrame *> frameOpt = audioFrameQueue.pop();if (frameOpt.has_value()) {audioFrame = frameOpt.value();} else {break;}}if (videoFrame == nullptr) {optional<VideoFrame *> frameOpt = videoFrameQueue.pop();if (frameOpt.has_value()) {videoFrame = frameOpt.value();} else {break;}}if (lastAudioPts == -1) {lastAudioPts = audioFrame->pts;}if (lastVideoPts == -1) {lastVideoPts = videoFrame->pts;}if ((audioFrame->flags & STREAM_FLAG_SOUGHT) == STREAM_FLAG_SOUGHT&& (videoFrame->flags & STREAM_FLAG_SOUGHT) == STREAM_FLAG_SOUGHT) {LOGD(TAG, "sync, meet both audio and video seek frame");playerContext.recycleAudioFrame(audioFrame);audioFrame = nullptr;playerContext.recycleVideoFrame(videoFrame);videoFrame = nullptr;continue;} else if ((audioFrame->flags & STREAM_FLAG_SOUGHT) == STREAM_FLAG_SOUGHT) {LOGD(TAG, "sync, meet audio seek frame");playerContext.recycleVideoFrame(videoFrame);videoFrame = nullptr;continue;} else if ((videoFrame->flags & STREAM_FLAG_SOUGHT) == STREAM_FLAG_SOUGHT) {LOGD(TAG, "sync, meet video seek frame");playerContext.recycleAudioFrame(audioFrame);audioFrame = nullptr;continue;}int64_t audioOutputPts = audioFrame->getOutputPts();if (videoFrame->pts <= audioOutputPts) {lastVideoPts = videoFrame->pts;videoOutput->write(videoFrame);videoFrame = nullptr;} else {int64_t outputFrames = (videoFrame->pts + 3 - audioOutputPts) * 1.0f / 1000 * audioFrame->sampleRate;outputFrames = min((int64_t)(audioFrame->numFrames - audioFrame->outputStartIndex), outputFrames);audioFrame->outputFrameCount = outputFrames;lastAudioPts = audioOutputPts;audioOutput->write(audioFrame);audioFrame->outputStartIndex += audioFrame->outputFrameCount;if (audioFrame->outputStartIndex == audioFrame->numFrames) {playerContext.recycleAudioFrame(audioFrame);audioFrame = nullptr;}}if (stateListener != nullptr) {stateListener->progressChanged(lastAudioPts, false);}} else if (enableVideo) {if (videoFrame == nullptr) {optional<VideoFrame *> frameOpt = videoFrameQueue.pop();if (frameOpt.has_value()) {videoFrame = frameOpt.value();} else {break;}}lastVideoPts = videoFrame->pts;videoOutput->write(videoFrame);videoFrame = nullptr;this_thread::sleep_for(chrono::milliseconds(17));if (stateListener != nullptr) {stateListener->progressChanged(lastVideoPts, false);}} else if (enableAudio) {if (audioFrame == nullptr) {optional<AudioFrame *> frameOpt = audioFrameQueue.pop();if (frameOpt.has_value()) {audioFrame = frameOpt.value();} else {break;}}lastAudioPts = audioFrame->pts;audioOutput->write(audioFrame);playerContext.recycleAudioFrame(audioFrame);audioFrame = nullptr;if (stateListener != nullptr) {stateListener->progressChanged(lastAudioPts, false);}} else {LOGE(TAG, "both audio and video disabled, break");break;}}if (audioFrame) {unPlayedAudioFrame = audioFrame;}if (videoFrame) {unPlayedVideoFrame = videoFrame;}if (stateListener != nullptr) {stateListener->playStateChanged(false);}
}

5. 音频输出

不同的平台使用不同的音频框架进行音频输出。Android平台我选择使用oboe。对音频输出我定义了一下接口来统一各平台的调用。

//
// Created by 祖国瑞 on 2022/9/5.
//#ifndef ANDROID_VIDEOPLAYER_IAUDIOOUTPUT_H
#define ANDROID_VIDEOPLAYER_IAUDIOOUTPUT_H#include <stdlib.h>
#include "PlayerContext.h"
#include "AudioFrame.h"
extern "C" {
#include "FFmpeg/libavformat/avformat.h"
}class IAudioOutput {
public:IAudioOutput(PlayerContext *playerContext) {this->playerCtx = playerContext;}virtual bool create(int sampleRate, int channels, AVSampleFormat sampleFormat) = 0;virtual void release() = 0;virtual void start() = 0;virtual void stop() = 0;virtual bool write(AudioFrame *audioFrame) = 0;virtual void write(uint8_t *buffer, int framesPerChannel) = 0;protected:PlayerContext *playerCtx = nullptr;
};#endif //ANDROID_VIDEOPLAYER_IAUDIOOUTPUT_H

由于oboe使用比较简单，这里就不单独列出来了。需要注意的是，由于源文件采样率及采样格式多种多样，音频框架不一定支持。所以在输出之前，使用FFmpeg的avresample来重采样。

6. 视频输出（需要优化）

视频输出使用OpenGL ES，这里还有很多需要优化，比如EGL的格式目前是写死的，对于超出8bit长度的像素格式，还是统一转换为了float32并建立纹理，YUV的非planner数据（例如NV21格式，U和V是混合在一个数据平面中交替出现的）需要切分成三个纹理使用。这样很多工作都是由cpu完成的，开销比较大，而且速度较慢。接下来会探索Android的ANativeBuffer支持的format，它与OpenGL ES的纹理格式是对应的，尽量减少cpu工作。

视频输出同样也对各平台规定了一个接口：

//
// Created by 祖国瑞 on 2022/9/5.
//#ifndef ANDROID_VIDEOPLAYER_IVIDEOOUTPUT_H
#define ANDROID_VIDEOPLAYER_IVIDEOOUTPUT_H#include <stdlib.h>
#include "VideoFrame.h"
#include "PlayerContext.h"
#include "SizeMode.h"class IVideoOutput {
public:IVideoOutput(PlayerContext *playerContext) {this->playerCtx = playerContext;};virtual bool setFormat(AVPixelFormat pixelFormat, AVColorSpace colorSpace, bool isHDR) = 0;virtual bool create(void *surface) = 0;virtual void release() = 0;virtual void setScreenSize(int32_t width, int32_t height) = 0;virtual bool isReady() = 0;virtual void write(VideoFrame* frame) = 0;virtual void setSizeMode(SizeMode mode) = 0;protected:PlayerContext *playerCtx;AVPixelFormat srcPixelFormat = AVPixelFormat::AV_PIX_FMT_NONE;
};#endif //ANDROID_VIDEOPLAYER_IVIDEOOUTPUT_H

对于Android平台，我使用OpenGL ES来渲染。

首先，shader部分很简单，其主要作用就是yuv转rgb。但是由于GPU天然适合做大量简单计算，所以我们可以在shader里去最大化兼容各种像素格式，例如10bit int，16bit float等。以及各种大尾序小尾序等。

static const char *vertexShaderCode ="#version 300 es\n""layout (location = 0) in vec3 aPos;\n""layout (location = 1) in vec2 aTexCoord;\n""out vec2 TexCoord;\n""void main() {\n""    gl_Position = vec4(aPos, 1.0f);\n""    TexCoord = aTexCoord;\n""}\n";static const char *yuv2rgbShaderCode ="#version 300 es\n""precision mediump float;\n""uniform sampler2D tex_y;\n""uniform sampler2D tex_u;\n""uniform sampler2D tex_v;\n""in vec2 TexCoord;\n""out vec4 FragColor;\n""void main() {\n""    float y = texture(tex_y, TexCoord).r - 0.0625f;\n""    float u = texture(tex_u, TexCoord).r - 0.5f;\n""    float v = texture(tex_v, TexCoord).r - 0.5f;\n""    float r = 1.164f * y + 1.793f * v;\n""    float g = 1.164f * y - 0.213f * u - 0.533f * v;\n""    float b = 1.164f * y + 2.112f * u;\n""    //float a = texture(tex_y, TexCoord).r;\n""    FragColor = vec4(r, g, b, 1.0f);\n""}\n";static const char *yuv16ui2rgbShaderCode ="#version 300 es\n""precision mediump float;\n""uniform usampler2D tex_y;\n""uniform usampler2D tex_u;\n""uniform usampler2D tex_v;\n""in vec2 TexCoord;\n""out vec4 FragColor;\n""void main() {\n""    float y = float(texture(tex_y, TexCoord).r) - 0.0625f;\n""    float u = float(texture(tex_u, TexCoord).r) - 0.5f;\n""    float v = float(texture(tex_v, TexCoord).r) - 0.5f;\n""    float r = 1.164f * y + 1.793f * v;\n""    float g = 1.164f * y - 0.213f * u - 0.533f * v;\n""    float b = 1.164f * y + 2.112f * u;\n""    //float a = texture(tex_y, TexCoord).r;\n""    FragColor = vec4(r, g, b, 1.0f);\n""}\n";static const char *rgbShaderCode ="#version 300 se\n""\n""uniform sampler2D tex_rgb;\n""\n""in vec2 TexCoord;\n""\n""out vec4 FragColor;\n""\n""void main() {\n""    FragColor = texture(tex_rgb, TexCoord);\n""}\n";

然后就是像素处理了，我们需要把各种像素布局和像素格式的图片转换为OpenGL ES可以接受的texture，然后才能在shader里进行处理。

首先是根据像素格式来确定使用哪种shader。

bool GLESRender::setFormat(AVPixelFormat format, AVColorSpace colorSpace, bool isHDR) {LOGD(TAG, "setFormat");if (!eglWindow.isReady()) {LOGE(TAG, "eglWindow is not ready");return false;}pixelType = get_pixel_type(format);pixelLayout = get_pixel_layout(format);if (pixelType == PixelType::None || pixelLayout == PixelLayout::None) {LOGE(TAG, "unsupported pixel format: %d", format);return false;}// 根据像素格式来确定texture的格式等。注意这里GLES的format与GL的format并不完全一致，有很多是用不了的。// 你可以参考Android的硬件buffer format与GLES的format的对应关系。// 链接在此：https://developer.android.google.cn/ndk/reference/group/a-hardware-bufferif (pixelType == PixelType::RGB) {switch (format) {case AV_PIX_FMT_RGB24:glDataType = GL_UNSIGNED_BYTE;glInternalFormat = GL_RGB;glDataFormat = GL_UNSIGNED_BYTE;glSupportFormat = true;break;case AV_PIX_FMT_RGB565LE:glDataType = GL_UNSIGNED_SHORT_5_6_5;glInternalFormat = GL_RGB;glDataFormat = GL_UNSIGNED_SHORT_5_6_5;glSupportFormat = true;break;case AV_PIX_FMT_RGB444LE:glDataType = GL_UNSIGNED_SHORT_4_4_4_4;glInternalFormat = GL_RGB;glDataFormat = GL_UNSIGNED_SHORT_4_4_4_4;glSupportFormat = true;break;default:LOGE(TAG, "unsupported RGB format: %d", format);return false;}if (!shader.compileShader(vertexShaderCode, rgbShaderCode)) {LOGE(TAG, "format = RGB24, compile shader failed");return false;}} else if (pixelType == PixelType::YUV) {yuvCompDepth = get_yuv_comp_depth(format);if (yuvCompDepth < 0) {LOGE(TAG, "get_yuv_comp_depth failed, format = %d", format);return false;}const char *fragmentCode;if (yuvCompDepth <= 8) {glDataType = GL_UNSIGNED_BYTE;glInternalFormat = GL_LUMINANCE;glDataFormat = GL_LUMINANCE;fragmentCode = yuv2rgbShaderCode;} else if (yuvCompDepth <= 16) {glDataType = GL_FLOAT;glInternalFormat = GL_R32F;glDataFormat = GL_RED;fragmentCode = yuv2rgbShaderCode;} else {LOGE(TAG, "unsupported yuvCompDepth: %d", yuvCompDepth);return false;}if (!shader.compileShader(vertexShaderCode, fragmentCode)) {LOGE(TAG, "format = %d, compile shader failed", format);return false;}eglWindow.makeCurrent();LOGD(TAG, "yuvCompDepth = %d", yuvCompDepth);} else {LOGE(TAG, "unsupported pixel format: %d", format);return false;}this->format = format;this->colorSpace = colorSpace;this->isHDR = isHDR;LOGD(TAG, "setFormat: format = %d, pixType = %d, glDataType = 0x%x", format, pixelType, glDataType);if (!shader.isReady()) {LOGE(TAG, "shader is not ready");return false;}return true;
}

拿到视频数据后，某些yuv数据需要将其转换成三个独立的纹理，再给OpenGL ES去处理。最关键的信息就是像素类型、像素布局和像素深度。

enum class PixelType {None = -1,RGB,YUV
};// 像素布局。
enum class PixelLayout {None = -1,// 多数yuv数据(例如YUV420P)都是planner，YUV三种像素是独立存储的，各占一个平面。Planner, // 多数RGB数据(例如RGB565)都是packet，RGB三像素依次存储，混编在一个平面内。Packet,// 半平面，Android平台的NV21和NV12就是这种格式，Y像素单独存储到一个平面，UV数据依次存储，混编在一个平面Semi_Planner,
};// 获取某种yuv格式的像素深度。一般都是8bit，现在有很多HDR视频是10bit。
// 目前还没有RGB相关的该方法，因为RGB图片本来就可以直接转为纹理，不需要单独处理。如果之后出现了与ANativeBuffer的格式不兼容的RGB格式，
// 也需要类似的处理。
int get_yuv_comp_depth(AVPixelFormat format);

然后就是将yuv数据分别读取到三个像素buffer中，方便之后转换为纹理。

bool read_yuv_pixel(AVFrame *frame, AVPixelFormat format, int64_t width, int64_t height,uint8_t *yBuffer, int *yWidth, int *yHeight,uint8_t *uBuffer, int *uWidth, int *uHeight,uint8_t *vBuffer, int *vWidth, int *vHeight) {PixelLayout layout = get_pixel_layout(format);if (layout == PixelLayout::Planner) {return read_yuv_planner(frame, format, width, height, yBuffer, yWidth, yHeight, uBuffer, uWidth, uHeight, vBuffer, vWidth, vHeight);} else if (layout == PixelLayout::Packet) {return read_yuv_packet(frame, format, width, height, yBuffer, yWidth, yHeight, uBuffer, uWidth, uHeight, vBuffer, vWidth, vHeight);} else if (layout == PixelLayout::Semi_Planner) {return read_yuv_semi_planner(frame, format, width, height, yBuffer, yWidth, yHeight, uBuffer, uWidth, uHeight, vBuffer, vWidth, vHeight);} else {return false;}
}