C++音视频04：音视频编码、生成图片

视频编码

#include <libavutil/log.h>
#include <libavutil/opt.h>
#include <libavcodec/avcodec.h>static int encode(AVCodecContext *ctx, AVFrame *frame, AVPacket *pkt, FILE *out) {int ret = -1;ret = avcodec_send_frame(ctx, frame);if (ret < 0) {av_log(NULL, AV_LOG_ERROR, "Failed to send frame to encoder\n");goto _END;}while (ret >= 0) {ret = avcodec_receive_packet(ctx, pkt);// AVERROR(EAGAIN)表示出错了if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {return 0;} else if(ret < 0) {return -1;}fwrite(pkt->data, 1, pkt->size, out);av_packet_unref(pkt);}
_END:return 0;
}int main(int argc, char *argv[]) {char *dst = NULL;char *codecName = NULL;const AVCodec *codec = NULL;AVCodecContext *ctx = NULL;int ret = -1;FILE *f = NULL;AVFrame *frame = NULL;AVPacket *pkt = NULL;av_log_set_level(AV_LOG_DEBUG);// 1. 输入参数if (argc < 3) {av_log(NULL, AV_LOG_ERROR, "arguments must more than 3\n");goto _ERROR;}dst = argv[1];codecName = argv[2];// 2. 查找编码器codec = avcodec_find_encoder_by_name(codecName);if (!codec) {av_log(NULL, AV_LOG_ERROR, "don't find codec name %s\n", codecName);goto _ERROR;}// 3. 创建编码器上下文ctx = avcodec_alloc_context3(codec);if (!ctx) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 4. 设置编码器参数ctx->width = 640;ctx->height = 480;ctx->bit_rate = 500000;// 时间基和帧率ctx->time_base = (AVRational){1, 25};ctx->framerate = (AVRational){25, 1};// 设置每10帧一个gopctx->gop_size = 10;// 设置一个gop中最多有一个b帧ctx->max_b_frames = 1;// 设置yuv格式ctx->pix_fmt = AV_PIX_FMT_YUV422P;if (codec->id == AV_CODEC_ID_H264) {av_opt_set(ctx->priv_data, "preset", "slow", 0);}// 5. 编码器与编码器上下文绑定ret = avcodec_open2(ctx, codec, NULL);if (ret < 0) {av_log(ctx, AV_LOG_ERROR, "Don't open codec: %s", av_err2str(ret));goto _ERROR;}// 6. 创建输出文件f = fopen(dst, "wb");if (!f) {av_log(NULL, AV_LOG_ERROR, "Don't open file %s\n", dst);goto _ERROR;}// 7. 创建AVFrameframe = av_frame_alloc();if (!frame) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 必须赋值才能获取到bufferframe->width = ctx->width;frame->height = ctx->height;frame->format = ctx->pix_fmt;// frame中真正存储数据的位置不会被av_frame_alloc生成，还需要调用av_frame_get_buffer获取ret = av_frame_get_buffer(frame, 0);if (ret < 0) {av_log(NULL, AV_LOG_ERROR, "Could't not allocate the video frame\n");goto _ERROR;}// 8. 创建AVPacketpkt = av_packet_alloc();if (!pkt) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 9. 生成视频内容// 这里是虚拟生成25帧，即一秒钟的数据for (int i = 0; i < 25; ++i) {// 确保av_frame中的空间data可用，如果当前data域被锁定了，就会自动指定一个新的dataret = av_frame_make_writable(frame);if (ret < 0) {break;}// Y分量for (int y = 0; y < ctx->height; y++) {for (int x = 0; x < ctx->width; x++) {// data[0] 指的是yuv中的y，[frame->linesize[0] 指的是行的大小， x表示横轴位移]// x + y + i * 3表示Y分量随着x和y变化而渐变frame->data[0][y * frame->linesize[0] + x] = x + y + i * 3;}}// UV分量for (int y = 0; y < ctx->height; y++) {// 对应uv分量 422中都是y分量的一半，所以除以2for (int x = 0; x < ctx->width / 2; x++) {// U分量 128在U分量代表黑色， 后面的2是任意的frame->data[1][y * frame->linesize[1] + x] = 128 + y + i * 2;// V分量 64在V分量中代表黑色， 后面的5是任意的frame->data[2][y * frame->linesize[2] + x] = 64 + x + i * 5;}}frame->pts = i;// 10. 编码ret = encode(ctx, frame, pkt, f);if (ret == -1) {goto _ERROR;}}// 输出一个空包，用于刷新剩余数据encode(ctx, NULL, pkt, f);
_ERROR:if (ctx) {avcodec_free_context(&ctx);}if (frame) {av_frame_free(&frame);}if (pkt) {av_packet_free(&pkt);}if (f) {fclose(f);}return 0;
}

gcc -g -o encode_video encode_video.c `pkg-config --libs libavutil libavformat`./encode_video 1.h264 libx264

音频编码

#include <libavutil/log.h>
#include <libavutil/opt.h>
#include <libavcodec/avcodec.h>
#include <libavutil/samplefmt.h>static int select_best_sample_rate(const AVCodec *codec) {const int *p;int best_sample_rate = 0;if (!codec->supported_samplerates) {return 44100;}p = codec->supported_samplerates;// 找到一个离44100最近的采样率while (*p) {if (!best_sample_rate || abs(44100 - *p) < abs(44100 - best_sample_rate)) {best_sample_rate = *p;}p++;}return best_sample_rate;
}// ffmpeg源码自带
static int check_sample_fmt(const AVCodec *codec, enum AVSampleFormat sample_fmt)
{const enum AVSampleFormat *p = codec->sample_fmts;while (*p != AV_SAMPLE_FMT_NONE) {if (*p == sample_fmt)return 1;p++;}return 0;
}static int encode(AVCodecContext *ctx, AVFrame *frame, AVPacket *pkt, FILE *out) {int ret = -1;ret = avcodec_send_frame(ctx, frame);if (ret < 0) {av_log(NULL, AV_LOG_ERROR, "Failed to send frame to encoder\n");goto _END;}while (ret >= 0) {ret = avcodec_receive_packet(ctx, pkt);// AVERROR(EAGAIN)表示出错了if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {return 0;} else if(ret < 0) {return -1;}fwrite(pkt->data, 1, pkt->size, out);av_packet_unref(pkt);}
_END:return 0;
}int main(int argc, char *argv[]) {char *dst = NULL;char *codecName = NULL;const AVCodec *codec = NULL;AVCodecContext *ctx = NULL;int ret = -1;FILE *f = NULL;AVFrame *frame = NULL;AVPacket *pkt = NULL;uint16_t *samples = NULL;av_log_set_level(AV_LOG_DEBUG);// 1. 输入参数if (argc < 2) {av_log(NULL, AV_LOG_ERROR, "arguments must more than 2\n");goto _ERROR;}dst = argv[1];// codecName = argv[2];// 2. 查找编码器：两种方式，by_name可以使用外部的库，通过CODEC_ID的则使用FFMpeg自带的库// codec = avcodec_find_encoder_by_name("libx264");codec = avcodec_find_encoder_by_name("libfdk_aac");// codec = avcodec_find_encoder(AV_CODEC_ID_AAC); // 用这个会涉及下面注释的一系列修改if (!codec) {av_log(NULL, AV_LOG_ERROR, "don't find codec name %s\n", codecName);goto _ERROR;}// 3. 创建编码器上下文ctx = avcodec_alloc_context3(codec);if (!ctx) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 4. 设置编码器参数ctx->bit_rate = 64000;ctx->sample_fmt = AV_SAMPLE_FMT_S16;// AV_SAMPLE_FMT_FLTP，对于ffmpeg内部的编码器 用这个格式ret = check_sample_fmt(codec, AV_SAMPLE_FMT_S16); if (ret == 0) {av_log(NULL, AV_LOG_ERROR, "encoder do not support sample format\n");goto _ERROR;}// 自定义函数找到最接近44100的采样率ctx->sample_rate = select_best_sample_rate(codec);// 设置声道布局为立体声stereoav_channel_layout_copy(&ctx->ch_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO); // AV_CHANNEL_LAYOUT_MONO ffmpeg内部编码器修改成这个// 5. 编码器与编码器上下文绑定ret = avcodec_open2(ctx, codec, NULL);if (ret < 0) {av_log(ctx, AV_LOG_ERROR, "Don't open codec: %s", av_err2str(ret));goto _ERROR;}// 6. 创建输出文件f = fopen(dst, "wb");if (!f) {av_log(NULL, AV_LOG_ERROR, "Don't open file %s\n", dst);goto _ERROR;}// 7. 创建AVFrameframe = av_frame_alloc();if (!frame) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 必须赋值才能获取到bufferframe->nb_samples = ctx->frame_size;frame->format = ctx->sample_fmt;av_channel_layout_copy(&frame->ch_layout, &ctx->ch_layout);// frame中真正存储数据的位置不会被av_frame_alloc生成，还需要调用av_frame_get_buffer获取ret = av_frame_get_buffer(frame, 0);if (ret < 0) {av_log(NULL, AV_LOG_ERROR, "Could't not allocate the video frame\n");goto _ERROR;}// 8. 创建AVPacketpkt = av_packet_alloc();if (!pkt) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 9. 生成音频内容float t = 0.0;// 有空了解一下这个值float tincr = 2 * M_PI * 440 / ctx->sample_rate;for (int i = 0; i < 200; ++i) {ret = av_frame_make_writable(frame);if (ret < 0) {av_log(NULL, AV_LOG_ERROR, "do not allocate space\n");goto _ERROR;}samples = (uint16_t*)frame->data[0]; // FLTP是32位的 所以改成uint32, 下面所有2都得改成4// 2字节一个数据for (int j = 0; j < ctx->frame_size; ++j) {samples[2 * j] = (sin(t) * 10000);// 多声道的处理for (int k = 1; k < ctx->ch_layout.nb_channels; ++k) {samples[2 * j + k] = samples[2 * j];}t += tincr;}encode(ctx, frame, pkt, f);}// 输出一个空包，用于刷新剩余数据encode(ctx, NULL, pkt, f);
_ERROR:if (ctx) {avcodec_free_context(&ctx);}if (frame) {av_frame_free(&frame);}if (pkt) {av_packet_free(&pkt);}if (f) {fclose(f);}return 0;
}

gcc -g -o encode_audio encode_audio.c `pkg-config --libs libavutil libavformat`./encode_audio 1.aac

由于ffmpeg自带的aac格式是32位的AV_SAMPLE_FMT_FLTP，如果由libfdk_aac转成ffmpeg自带的aac，需要进行重采样成FLTP才能继续其他操作

生成图片

生成黑白色图片

先把视频解码成帧序列，然后将帧保存为图片

#include <libavformat/avformat.h>
#include <libavutil/log.h>
#include <libavutil/avutil.h>
#include <libavcodec/avcodec.h>static void savePic(unsigned char *buf, int lineSize, int width, int height, char *name) {FILE *f;f = fopen(name, "wb");// 写入头信息fprintf(f, "P5\n%d %d\n%d\n", width, height, 255);for (int i = 0; i < height; ++i) {fwrite(buf + i *lineSize, 1, width, f);}fclose(f);
}static int decode(AVCodecContext *ctx, AVFrame *frame, AVPacket *pkt, const char *fileName) {int ret = -1;char buf[1024];ret = avcodec_send_packet(ctx, pkt);if (ret < 0) {av_log(NULL, AV_LOG_ERROR, "Failed to send frame to encoder\n");goto _END;}while (ret >= 0) {ret = avcodec_receive_frame(ctx, frame);// AVERROR(EAGAIN)表示出错了if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {return 0;} else if(ret < 0) {return -1;}snprintf(buf, sizeof(buf), "%s-%d", fileName, ctx->frame_num);savePic(frame->data[0], frame->linesize[0], frame->width, frame->height, buf);if (pkt) {av_packet_unref(pkt);}}
_END:return 0;
}int main (int argc, char *argv[]) {// 1. 处理参数char *src;char *dst;int ret;int idx = -1;AVFormatContext *pFmtCtx = NULL;AVPacket *pkt = NULL;const AVCodec *codec = NULL;AVCodecContext *ctx = NULL;AVFrame *frame = NULL;AVStream *inStream = NULL;av_log_set_level(AV_LOG_DEBUG);if (argc < 3) {av_log(NULL, AV_LOG_ERROR, "argument must be more than 3\n");exit(-1);}src = argv[1];dst = argv[2];// 2. 打开多媒体文件ret = avformat_open_input(&pFmtCtx, src, NULL, NULL);if (ret < 0) {av_log(NULL, AV_LOG_ERROR, "%s\n", av_err2str(ret));exit(-1);}// 3. 从多媒体文件中找到音频流idx = av_find_best_stream(pFmtCtx, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0);if (idx < 0) {av_log(pFmtCtx, AV_LOG_ERROR, "Does not include audio stream\n");goto _ERROR;}inStream = pFmtCtx->streams[idx];// 4. 查找解码器codec = avcodec_find_decoder(inStream->codecpar->codec_id);if (!codec) {av_log(NULL, AV_LOG_ERROR, "don't find codec\n");goto _ERROR;}// 5. 创建解码器上下文ctx = avcodec_alloc_context3(codec);if (!ctx) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 拷贝编码参数到上下文对象avcodec_parameters_to_context(ctx, inStream->codecpar);// 5. 解码器与解码器上下文绑定ret = avcodec_open2(ctx, codec, NULL);if (ret < 0) {av_log(ctx, AV_LOG_ERROR, "Don't open codec: %s", av_err2str(ret));goto _ERROR;}// 6. 创建AVFrameframe = av_frame_alloc();if (!frame) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 7. 创建AVPacketpkt = av_packet_alloc();if (!pkt) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 8. 从源多媒体文件中读数据到目的文件中while (av_read_frame(pFmtCtx, pkt) >= 0) {if (pkt->stream_index == idx) {decode(ctx, frame, pkt, dst);}}decode(ctx, frame, NULL, dst);// 9. 将申请的资源释放掉
_ERROR:if (pFmtCtx) {avformat_close_input(&pFmtCtx);pFmtCtx = NULL;}    if (ctx) {avcodec_free_context(&ctx);ctx = NULL;}if (frame) {av_frame_free(&frame);frame = NULL;}if (pkt) {av_packet_free(&pkt);pkt = NULL;}return 0;
}

gcc -g -o gen_pic gen_pic.c `pkg-config --libs libavutil libavformat`./gen_pic ~/resource/1.mp4 out/out

生成彩色图片BMP

代码目前有bug尚未找出

#include <libavformat/avformat.h>
#include <libavutil/log.h>
#include <libavutil/avutil.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>#define WORD uint64_t
#define DWORD uint32_t
#define LONG int32_t
// 去网站上拷贝对应定义：https://learn.microsoft.com/zh-cn/windows/win32/api/wingdi/ns-wingdi-bitmapinfoheader
typedef struct tagBITMAPINFOHEADER {DWORD biSize;LONG  biWidth;LONG  biHeight;WORD  biPlanes;WORD  biBitCount;DWORD biCompression;DWORD biSizeImage;LONG  biXPelsPerMeter;LONG  biYPelsPerMeter;DWORD biClrUsed;DWORD biClrImportant;
} BITMAPINFOHEADER, *LPBITMAPINFOHEADER, *PBITMAPINFOHEADER;typedef struct tagBITMAPFILEHEADER {WORD  bfType;DWORD bfSize;WORD  bfReserved1;WORD  bfReserved2;DWORD bfOffBits;
} BITMAPFILEHEADER, *LPBITMAPFILEHEADER, *PBITMAPFILEHEADER;static void saveBMP(struct SwsContext *swsCtx,AVFrame *frame, int w, int h, char *name) {FILE *f = NULL;// 3是因为转换成24位的帧，刚好3个字节int dataSize = w * h * 3;// 1. 进行转换，将yuv frame 转成 BRG24 FrameAVFrame *frameBGR = av_frame_alloc();frameBGR->width = w;frameBGR->height = h;frameBGR->format = AV_PIX_FMT_BGR24;av_frame_get_buffer(frameBGR, 0);sws_scale(swsCtx, (const uint8_t *const *)frame->data,frame->linesize,0,frame->height,frameBGR->data, frameBGR->linesize);// 2. 构造 BITMAP INFO HEADERBITMAPINFOHEADER infoHeader;infoHeader.biSize = sizeof(BITMAPINFOHEADER);infoHeader.biWidth = w;infoHeader.biHeight = h * (-1); // 坐标与正常坐标相反infoHeader.biBitCount = 24;infoHeader.biCompression = 0;infoHeader.biSizeImage = 0;infoHeader.biClrImportant = 0;infoHeader.biClrUsed = 0;infoHeader.biXPelsPerMeter = 0;infoHeader.biYPelsPerMeter = 0;infoHeader.biPlanes = 0;// 3. 构造 BITMAP FILE HEADERBITMAPFILEHEADER fileHeader;fileHeader.bfType = 0x4d42; //  表示两个字符‘BM’fileHeader.bfSize = sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER) + dataSize;fileHeader.bfOffBits = sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER);// 4. 将数据写到文件f = fopen(name, "wb");fwrite(&fileHeader, sizeof(BITMAPFILEHEADER), 1, f);fwrite(&infoHeader, sizeof(BITMAPINFOHEADER), 1, f);fwrite(&frame->data[0], 1, dataSize, f);// 5. 释放资源   fclose(f);av_freep(&frameBGR->data[0]);av_free(frameBGR);}static void savePic(unsigned char *buf, int lineSize, int width, int height, char *name) {FILE *f;f = fopen(name, "wb");// 写入头信息fprintf(f, "P5\n%d %d\n%d\n", width, height, 255);for (int i = 0; i < height; ++i) {fwrite(buf + i *lineSize, 1, width, f);}fclose(f);
}static int decode(AVCodecContext *ctx, struct SwsContext *swsCtx,AVFrame *frame, AVPacket *pkt, const char *fileName) {int ret = -1;char buf[1024];ret = avcodec_send_packet(ctx, pkt);if (ret < 0) {av_log(NULL, AV_LOG_ERROR, "Failed to send frame to encoder\n");goto _END;}while (ret >= 0) {ret = avcodec_receive_frame(ctx, frame);// AVERROR(EAGAIN)表示出错了if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {return 0;} else if(ret < 0) {return -1;}snprintf(buf, sizeof(buf), "%s-%d.bmp", fileName, ctx->frame_num);// savePic(frame->data[0], frame->linesize[0], frame->width, frame->height, buf);saveBMP(swsCtx, frame, frame->width, frame->height, buf);// 释放前需要判空if (pkt) {av_packet_unref(pkt);}}
_END:return 0;
}int main (int argc, char *argv[]) {// 1. 处理参数char *src;char *dst;int ret;int idx = -1;AVFormatContext *pFmtCtx = NULL;AVPacket *pkt = NULL;const AVCodec *codec = NULL;AVCodecContext *ctx = NULL;AVFrame *frame = NULL;AVStream *inStream = NULL;struct SwsContext *swsCtx = NULL;av_log_set_level(AV_LOG_DEBUG);if (argc < 3) {av_log(NULL, AV_LOG_ERROR, "argument must be more than 3\n");exit(-1);}src = argv[1];dst = argv[2];// 2. 打开多媒体文件ret = avformat_open_input(&pFmtCtx, src, NULL, NULL);if (ret < 0) {av_log(NULL, AV_LOG_ERROR, "%s\n", av_err2str(ret));exit(-1);}// 3. 从多媒体文件中找到音频流idx = av_find_best_stream(pFmtCtx, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0);if (idx < 0) {av_log(pFmtCtx, AV_LOG_ERROR, "Does not include audio stream\n");goto _ERROR;}inStream = pFmtCtx->streams[idx];// 4. 查找解码器codec = avcodec_find_decoder(inStream->codecpar->codec_id);if (!codec) {av_log(NULL, AV_LOG_ERROR, "don't find codec\n");goto _ERROR;}// 5. 创建解码器上下文ctx = avcodec_alloc_context3(codec);if (!ctx) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 拷贝编码参数到上下文对象avcodec_parameters_to_context(ctx, inStream->codecpar);// 5. 解码器与解码器上下文绑定ret = avcodec_open2(ctx, codec, NULL);if (ret < 0) {av_log(ctx, AV_LOG_ERROR, "Don't open codec: %s", av_err2str(ret));goto _ERROR;}av_log(NULL, AV_LOG_INFO, "scale: %d, %d\n", ctx->width, ctx->height);// 5.1 获得SWS上下文sws_getCachedContext(NULL, ctx->width,         // src widthctx->height,        // src height AV_PIX_FMT_YUV420P,       // src pix fmt 有时候ctx->pix_fmt的格式是不对的640,         // dst width   这里的640 x 360 是因为需要按比例360,        // dst heightAV_PIX_FMT_BGR24,   // dst pix fmtSWS_BICUBIC,        // 后面研究NULL, NULL, NULL);// 6. 创建AVFrameframe = av_frame_alloc();if (!frame) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 7. 创建AVPacketpkt = av_packet_alloc();if (!pkt) {av_log(NULL, AV_LOG_ERROR, "No memory\n");goto _ERROR;}// 8. 从源多媒体文件中读数据到目的文件中while (av_read_frame(pFmtCtx, pkt) >= 0) {if (pkt->stream_index == idx) {decode(ctx, swsCtx, frame, pkt, dst);}}decode(ctx, swsCtx, frame, NULL, dst);// 9. 将申请的资源释放掉
_ERROR:if (pFmtCtx) {avformat_close_input(&pFmtCtx);pFmtCtx = NULL;}    if (ctx) {avcodec_free_context(&ctx);ctx = NULL;}if (frame) {av_frame_free(&frame);frame = NULL;}if (pkt) {av_packet_free(&pkt);pkt = NULL;}return 0;
}

gcc -g -o gen_pic gen_pic.c pkg-config --libs libavutil libavformat libswscale

./gen_pic ~/resource/1.mp4 out/out