Codec study notes Codec study notes 0

• Codec support on FFMPEG

registered codec is defined in libavcodec/allcodecs.c

extern AVCodec ff_h264_decoder;
extern AVCodec ff_hevc_decoder;
extern AVCodec ff_mpeg4_encoder;
extern AVCodec ff_mpeg4_decoder;
extern AVCodec ff_mjpeg_encoder;
extern AVCodec ff_mjpeg_decoder;
extern AVCodec ff_vp8_decoder;
extern AVCodec ff_vp9_decoder;
extern AVCodec ff_h263_decoder;

*h264dec

AVCodec {
    const char *name; //codec name
    const char *long_name;
    enum AVMediaType type;
    enum AVCodecID id;
    int capabilities;
    const AVRational *supported_framerates; ///< array of supported framerates, or NULL if any, array is terminated by {0,0}
    const enum AVPixelFormat *pix_fmts;     ///< array of supported pixel formats, or NULL if unknown, array is terminated by -1
    const int *supported_samplerates;       ///< array of supported audio samplerates, or NULL if unknown, array is terminated by 0
    const enum AVSampleFormat *sample_fmts; ///< array of supported sample formats, or NULL if unknown, array is terminated by -1
    const uint64_t *channel_layouts;         ///< array of support channel layouts, or NULL if unknown. array is terminated by 0
    uint8_t max_lowres;                     ///< maximum value for lowres supported by the decoder
    const AVClass *priv_class;              ///< AVClass for the private context
    const AVProfile *profiles;              ///< array of recognized profiles, or NULL if unknown, array is terminated by {FF_PROFILE_UNKNOWN}
 
    const char *wrapper_name; //usually AVCodec.name will be of the form "<codec_name>_<wrapper_name>").
 
    /*** No fields below this line are part of the public API. */
    int priv_data_size;
    struct AVCodec *next;
    /**
    int (*init_thread_copy)(AVCodecContext *);
    int (*update_thread_context)(AVCodecContext *dst, const AVCodecContext *src);
    /*  Private codec-specific defaults.*/
    const AVCodecDefault *defaults;
    void (*init_static_data)(struct AVCodec *codec);
    int (*init)(AVCodecContext *);
    int (*encode_sub)(AVCodecContext *, uint8_t *buf, int buf_size,
                      const struct AVSubtitle *sub);
    int (*encode2)(AVCodecContext *avctx, AVPacket *avpkt, const AVFrame *frame,
                   int *got_packet_ptr);
    int (*decode)(AVCodecContext *, void *outdata, int *outdata_size, AVPacket *avpkt);
    int (*close)(AVCodecContext *);
    int (*send_frame)(AVCodecContext *avctx, const AVFrame *frame);
    int (*receive_packet)(AVCodecContext *avctx, AVPacket *avpkt);
    int (*receive_frame)(AVCodecContext *avctx, AVFrame *frame);
    void (*flush)(AVCodecContext *);
    int caps_internal;
 
    /**
     * Decoding only, a comma-separated list of bitstream filters to apply to
     * packets before decoding.
     */
    const char *bsfs;
 
    /**
     * Array of pointers to hardware configurations supported by the codec,
     * or NULL if no hardware supported.  The array is terminated by a NULL
     * pointer.
     *
     * The user can only access this field via avcodec_get_hw_config().
     */
    const struct AVCodecHWConfigInternal **hw_configs;
 
    /**
     * List of supported codec_tags, terminated by FF_CODEC_TAGS_END.
     */
    const uint32_t *codec_tags;
}

hw accelator configure

const struct AVCodecHWConfigInternal **hw_configs = {
      AVCodecHWConfig public {
          enum AVPixelFormat pix_fmt {
             #defined in pixfmt.h in libavutil, AV_PIX_FMT_YUV420P
          }
          int methods; //AV_CODEC_HW_CONFIG_METHOD_xx defined in avcodec.h
          enum AVHWDeviceType device_type {
              AV_HWDEVICE_TYPE_NONE,
              AV_HWDEVICE_TYPE_VDPAU,
              AV_HWDEVICE_TYPE_CUDA,
              AV_HWDEVICE_TYPE_VAAPI,
              AV_HWDEVICE_TYPE_DXVA2,
              AV_HWDEVICE_TYPE_QSV,
              AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
              AV_HWDEVICE_TYPE_D3D11VA,
              AV_HWDEVICE_TYPE_DRM,
              AV_HWDEVICE_TYPE_OPENCL,
              AV_HWDEVICE_TYPE_MEDIACODEC,
              AV_HWDEVICE_TYPE_VULKAN,
          };
      };
 
      const AVHWAccel *hwaccel {
          const char *name;
          enum AVMediaType type; //AVMEDIA_TYPE_xxx
          enum AVCodecID id; //AV_CODEC_ID_xxx
          enum AVPixelFormat pix_fmt;
          int capabilities; //AV_HWACCEL_CODEC_CAP_*
          /* private */
          int (*alloc_frame)(AVCodecContext *avctx, AVFrame *frame);
          int (*start_frame)(AVCodecContext *avctx, const uint8_t *buf, uint32_t buf_size);
          int (*decode_params)(AVCodecContext *avctx, int type, const uint8_t *buf, uint32_t buf_size);
          int (*decode_slice)(AVCodecContext *avctx, const uint8_t *buf, uint32_t buf_size);
          int (*end_frame)(AVCodecContext *avctx);
          int frame_priv_data_size; /*  Size of per-frame hardware accelerator private data */
          void (*decode_mb)(struct MpegEncContext *s);
          int (*init)(AVCodecContext *avctx);
          int (*uninit)(AVCodecContext *avctx);
          /** Size of the private data to allocate in AVCodecInternal.hwaccel_priv_data.*/
          int priv_data_size;
          int caps_internal;
          int (*frame_params)(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx);
    }
}

example: h264

AVCodec ff_h264_decoder = {
    .name                  = "h264",
    .long_name             = NULL_IF_CONFIG_SMALL("H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10"),
    .type                  = AVMEDIA_TYPE_VIDEO,
    .id                    = AV_CODEC_ID_H264,
    .priv_data_size        = sizeof(H264Context),
    .init                  = h264_decode_init,
    .close                 = h264_decode_end,
    .decode                = h264_decode_frame,
    .capabilities          = /*AV_CODEC_CAP_DRAW_HORIZ_BAND |*/ AV_CODEC_CAP_DR1 |
                             AV_CODEC_CAP_DELAY | AV_CODEC_CAP_SLICE_THREADS |
                             AV_CODEC_CAP_FRAME_THREADS,
    .hw_configs            = (const AVCodecHWConfigInternal*[]) {
#if CONFIG_H264_DXVA2_HWACCEL
                               HWACCEL_DXVA2(h264),
#endif
#if CONFIG_H264_D3D11VA_HWACCEL
                               HWACCEL_D3D11VA(h264),
#endif
#if CONFIG_H264_D3D11VA2_HWACCEL
                               HWACCEL_D3D11VA2(h264),
#endif
#if CONFIG_H264_NVDEC_HWACCEL
                               HWACCEL_NVDEC(h264),
#endif
#if CONFIG_H264_VAAPI_HWACCEL
                               HWACCEL_VAAPI(h264),
#endif
#if CONFIG_H264_VDPAU_HWACCEL
                               HWACCEL_VDPAU(h264),
#endif
#if CONFIG_H264_VIDEOTOOLBOX_HWACCEL
                               HWACCEL_VIDEOTOOLBOX(h264),
#endif
                               NULL
                           },
    .caps_internal         = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_EXPORTS_CROPPING,
    .flush                 = flush_dpb,
    .init_thread_copy      = ONLY_IF_THREADS_ENABLED(decode_init_thread_copy),
    .update_thread_context = ONLY_IF_THREADS_ENABLED(ff_h264_update_thread_context),
    .profiles              = NULL_IF_CONFIG_SMALL(ff_h264_profiles),
    .priv_class            = &h264_class,
};

For DXVA2

{
    .public          = { 
	.pix_fmt     = AV_PIX_FMT_DXVA2_VLD, 
        .methods     = AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX | AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX | AV_CODEC_HW_CONFIG_METHOD_AD_HOC,
	.device_type = AV_HWDEVICE_TYPE_DXVA2,
    },
    .hwaccel         = {//ff_h264_dxva2_hwaccel, dxva2_h264.c
        .name           = "h264_dxva2",
        .type           = AVMEDIA_TYPE_VIDEO,
        .id             = AV_CODEC_ID_H264,
        .pix_fmt        = AV_PIX_FMT_DXVA2_VLD,
        .init           = ff_dxva2_decode_init,
        .uninit         = ff_dxva2_decode_uninit,
        .start_frame    = dxva2_h264_start_frame,
        .decode_slice   = dxva2_h264_decode_slice,
        .end_frame      = dxva2_h264_end_frame,
        .frame_params   = ff_dxva2_common_frame_params,
        .frame_priv_data_size = sizeof(struct dxva2_picture_context),
        .priv_data_size = sizeof(FFDXVASharedContext),			
    },

For D3D11VA

{
    .public          = { 
	.pix_fmt     = AV_PIX_FMT_DXVA2_VLD, 
        .methods     = AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX | AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX | AV_CODEC_HW_CONFIG_METHOD_AD_HOC,
	.device_type = AV_HWDEVICE_TYPE_DXVA2,
    },
    .hwaccel         = {//ff_h264_d3d11va_hwaccel, dxva2_h264.c
        .name           = "h264_d3d11va",
        .type           = AVMEDIA_TYPE_VIDEO,
        .id             = AV_CODEC_ID_H264,
        .pix_fmt        = AV_PIX_FMT_D3D11VA_VLD,
        .init           = ff_dxva2_decode_init,
        .uninit         = ff_dxva2_decode_uninit,
        .start_frame    = dxva2_h264_start_frame,
        .decode_slice   = dxva2_h264_decode_slice,
        .end_frame      = dxva2_h264_end_frame,
        .frame_params   = ff_dxva2_common_frame_params,
        .frame_priv_data_size = sizeof(struct dxva2_picture_context),
        .priv_data_size = sizeof(FFDXVASharedContext),		
    },

For NVDEC

const AVHWAccel ff_h264_nvdec_hwaccel = {
    .name                 = "h264_nvdec",
    .type                 = AVMEDIA_TYPE_VIDEO,
    .id                   = AV_CODEC_ID_H264,
    .pix_fmt              = AV_PIX_FMT_CUDA,
    .start_frame          = nvdec_h264_start_frame,
    .end_frame            = ff_nvdec_end_frame,
    .decode_slice         = nvdec_h264_decode_slice,
    .frame_params         = nvdec_h264_frame_params,
    .init                 = ff_nvdec_decode_init,
    .uninit               = ff_nvdec_decode_uninit,
    .priv_data_size       = sizeof(NVDECContext),
};

HW support <name> encoder 总结

libavcodec 加入 <name>_<codec>.c

add AVHWAccel ff_<codec>_<name>_hwaccel = { };
implement ff_<name>_decode_init/uninit/
implement ff_<name>2_common_frame_params
implement <name>_<codec>_start_frame/decode_slice/end_frame

add HW_CONFIG_HWACCEL macro in hwaccel.h

#define HWACCEL_<name>(codec) \
   HW_CONFIG_HWACCEL(1, 1, 1, pix_fmt,    <name>,        ff_<codec>_<name>_hwaccel)
       

add device type in AVHWDeviceType, in libavutil/hwcontext.h

AV_HWDEVICE_TYPE_<name>

add macro HWACCEL_<name> in libavcoder/<codec>dec.h (codec=h264,hevc,

  AVCodec ff_<codec>_decoder = {
#if CONFIG_<codec>_<name>_HWACCEL
        HWACCEL_<name>(<codec>),

=========================================================================

encoder

=========================================================================

libavcodec/omx.c
static const AVClass omx_h264enc_class = {

   .class_name = "h264_omx",
   .item_name  = av_default_item_name,
   .option     = options,
   .version    = LIBAVUTIL_VERSION_INT,

};
AVCodec ff_h264_omx_encoder = {

   .name             = "h264_omx",
   .long_name        = NULL_IF_CONFIG_SMALL("OpenMAX IL H.264 video encoder"),
   .type             = AVMEDIA_TYPE_VIDEO,
   .id               = AV_CODEC_ID_H264,
   .priv_data_size   = sizeof(OMXCodecContext),
   .init             = omx_encode_init,
   .encode2          = omx_encode_frame,
   .close            = omx_encode_end,
   .pix_fmts         = omx_encoder_pix_fmts,
   .capabilities     = AV_CODEC_CAP_DELAY,
   .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
   .priv_class       = &omx_h264enc_class,

};

static av_cold int omx_encode_init(AVCodecContext *avctx)
{
    omx_init(){
    static const char * const libnames[] = {
       /**** add our lib name  here ****/
#if CONFIG_OMX_RPI
        "/opt/vc/lib/libopenmaxil.so", "/opt/vc/lib/libbcm_host.so",
#else
        "libOMX_Core.so", NULL,
        "libOmxCore.so", NULL,
#endif
        NULL
    };
    s->host_init = dlsym(s->lib2, "bcm_host_init");
    s->ptr_Init                = dlsym_prefixed(s->lib, "OMX_Init", prefix);
    s->ptr_Deinit              = dlsym_prefixed(s->lib, "OMX_Deinit", prefix);
    s->ptr_ComponentNameEnum   = dlsym_prefixed(s->lib, "OMX_ComponentNameEnum", prefix);
    s->ptr_GetHandle           = dlsym_prefixed(s->lib, "OMX_GetHandle", prefix);
    s->ptr_FreeHandle          = dlsym_prefixed(s->lib, "OMX_FreeHandle", prefix);
    s->ptr_GetComponentsOfRole = dlsym_prefixed(s->lib, "OMX_GetComponentsOfRole", prefix);
    s->ptr_GetRolesOfComponent = dlsym_prefixed(s->lib, "OMX_GetRolesOfComponent", prefix);
 
}