static struct mp_image *d3d11va_new_ref(ID3D11VideoDecoderOutputView *view,
int w, int h)
{
if (!view)
return NULL;
struct d3d11va_surface *surface = talloc_zero(NULL, struct d3d11va_surface);
surface->surface = view;
ID3D11VideoDecoderOutputView_AddRef(surface->surface);
ID3D11VideoDecoderOutputView_GetResource(
surface->surface, (ID3D11Resource **)&surface->texture);
D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC surface_desc;
ID3D11VideoDecoderOutputView_GetDesc(surface->surface, &surface_desc);
surface->subindex = surface_desc.Texture2D.ArraySlice;
struct mp_image *mpi =
mp_image_new_custom_ref(NULL, surface, d3d11va_release_img);
if (!mpi)
abort();
mp_image_setfmt(mpi, IMGFMT_D3D11VA);
mp_image_set_size(mpi, w, h);
mpi->planes[0] = NULL;
mpi->planes[1] = (void *)surface->texture;
mpi->planes[2] = (void *)(intptr_t)surface->subindex;
mpi->planes[3] = (void *)surface->surface;
return mpi;
}
static struct mp_image *alloc_pool(void *pctx, int fmt, int w, int h)
{
struct vf_instance *vf = pctx;
struct vf_priv_s *p = vf->priv;
HRESULT hr;
ID3D11Texture2D *texture = NULL;
D3D11_TEXTURE2D_DESC texdesc = {
.Width = w,
.Height = h,
.Format = p->out_format,
.MipLevels = 1,
.ArraySize = 1,
.SampleDesc = { .Count = 1 },
.Usage = D3D11_USAGE_DEFAULT,
.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE,
.MiscFlags = p->out_shared ? D3D11_RESOURCE_MISC_SHARED : 0,
};
hr = ID3D11Device_CreateTexture2D(p->vo_dev, &texdesc, NULL, &texture);
if (FAILED(hr))
return NULL;
struct mp_image *mpi = mp_image_new_custom_ref(NULL, texture, release_tex);
if (!mpi)
abort();
mp_image_setfmt(mpi, p->out_params.imgfmt);
mp_image_set_size(mpi, w, h);
mpi->params.hw_subfmt = p->out_params.hw_subfmt;
mpi->planes[1] = (void *)texture;
mpi->planes[2] = (void *)(intptr_t)0;
return mpi;
}
static struct mp_image get_xv_buffer(struct vo *vo, int buf_index)
{
struct xvctx *ctx = vo->priv;
XvImage *xv_image = ctx->xvimage[buf_index];
struct mp_image img = {0};
mp_image_set_size(&img, ctx->image_width, ctx->image_height);
mp_image_setfmt(&img, ctx->image_format);
bool swapuv = ctx->xv_format == MP_FOURCC_YV12;
for (int n = 0; n < img.num_planes; n++) {
int sn = n > 0 && swapuv ? (n == 1 ? 2 : 1) : n;
img.planes[n] = xv_image->data + xv_image->offsets[sn];
img.stride[n] = xv_image->pitches[sn];
}
if (vo->params) {
struct mp_image_params params = *vo->params;
if (ctx->cached_csp)
params.colorspace = ctx->cached_csp;
mp_image_set_attributes(&img, ¶ms);
}
return img;
}
static struct mp_image *d3d11va_retrieve_image(struct lavc_ctx *s,
struct mp_image *img)
{
HRESULT hr;
struct priv *p = s->hwdec_priv;
ID3D11Texture2D *staging = p->decoder->staging;
if (img->imgfmt != IMGFMT_D3D11VA)
return img;
ID3D11Texture2D *texture = (void *)img->planes[1];
int subindex = (intptr_t)img->planes[2];
if (!texture) {
MP_ERR(p, "Failed to get Direct3D texture and surface from mp_image\n");
return img;
}
D3D11_TEXTURE2D_DESC texture_desc;
ID3D11Texture2D_GetDesc(texture, &texture_desc);
if (texture_desc.Width < img->w || texture_desc.Height < img->h) {
MP_ERR(p, "Direct3D11 texture smaller than mp_image dimensions\n");
return img;
}
// copy to the staging texture
ID3D11DeviceContext_CopySubresourceRegion(
p->device_ctx,
(ID3D11Resource *)staging, 0, 0, 0, 0,
(ID3D11Resource *)texture, subindex, NULL);
struct mp_image *sw_img = mp_image_pool_get(p->sw_pool,
p->decoder->mpfmt_decoded,
texture_desc.Width,
texture_desc.Height);
if (!sw_img) {
MP_ERR(p, "Failed to get %s surface from CPU pool\n",
mp_imgfmt_to_name(p->decoder->mpfmt_decoded));
return img;
}
// copy staging texture to the cpu mp_image
D3D11_MAPPED_SUBRESOURCE lock;
hr = ID3D11DeviceContext_Map(p->device_ctx, (ID3D11Resource *)staging,
0, D3D11_MAP_READ, 0, &lock);
if (FAILED(hr)) {
MP_ERR(p, "Failed to map D3D11 surface: %s\n", mp_HRESULT_to_str(hr));
talloc_free(sw_img);
return img;
}
copy_nv12(sw_img, lock.pData, lock.RowPitch, texture_desc.Height);
ID3D11DeviceContext_Unmap(p->device_ctx, (ID3D11Resource *)staging, 0);
mp_image_set_size(sw_img, img->w, img->h);
mp_image_copy_attributes(sw_img, img);
talloc_free(img);
return sw_img;
}
static struct mp_image *d3d11va_retrieve_image(struct lavc_ctx *s,
struct mp_image *img)
{
HRESULT hr;
struct priv *p = s->hwdec_priv;
ID3D11Texture2D *staging = p->decoder->staging;
ID3D11Texture2D *texture = d3d11_texture_in_mp_image(img);
ID3D11VideoDecoderOutputView *surface = d3d11_surface_in_mp_image(img);
if (!texture || !surface) {
MP_ERR(p, "Failed to get Direct3D texture and surface from mp_image\n");
return img;
}
D3D11_TEXTURE2D_DESC texture_desc;
ID3D11Texture2D_GetDesc(texture, &texture_desc);
if (texture_desc.Width < img->w || texture_desc.Height < img->h) {
MP_ERR(p, "Direct3D11 texture smaller than mp_image dimensions\n");
return img;
}
// copy to the staging texture
D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC surface_desc;
ID3D11VideoDecoderOutputView_GetDesc(surface, &surface_desc);
ID3D11DeviceContext_CopySubresourceRegion(
p->device_ctx,
(ID3D11Resource *)staging, 0, 0, 0, 0,
(ID3D11Resource *)texture, surface_desc.Texture2D.ArraySlice, NULL);
struct mp_image *sw_img = mp_image_pool_get(p->sw_pool,
p->decoder->mpfmt_decoded,
texture_desc.Width,
texture_desc.Height);
if (!sw_img) {
MP_ERR(p, "Failed to get %s surface from CPU pool\n",
mp_imgfmt_to_name(p->decoder->mpfmt_decoded));
return img;
}
// copy staging texture to the cpu mp_image
D3D11_MAPPED_SUBRESOURCE lock;
hr = ID3D11DeviceContext_Map(p->device_ctx, (ID3D11Resource *)staging,
0, D3D11_MAP_READ, 0, &lock);
if (FAILED(hr)) {
MP_ERR(p, "Failed to map D3D11 surface: %s\n", mp_HRESULT_to_str(hr));
talloc_free(sw_img);
return img;
}
copy_nv12(sw_img, lock.pData, lock.RowPitch, texture_desc.Height);
ID3D11DeviceContext_Unmap(p->device_ctx, (ID3D11Resource *)staging, 0);
mp_image_set_size(sw_img, img->w, img->h);
mp_image_copy_attributes(sw_img, img);
talloc_free(img);
return sw_img;
}
static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi)
{
if (mpi->fmt.flags & MP_IMGFLAG_HWACCEL) {
mp_image_set_size(mpi, vf->fmt_out.w, vf->fmt_out.h);
} else {
mp_image_crop(mpi, vf->priv->crop_x, vf->priv->crop_y,
vf->priv->crop_x + vf->priv->crop_w,
vf->priv->crop_y + vf->priv->crop_h);
}
return mpi;
}
static void copy_nv12(struct mp_image *dest, uint8_t *src_bits,
unsigned src_pitch, unsigned surf_height)
{
struct mp_image buf = {0};
mp_image_setfmt(&buf, IMGFMT_NV12);
mp_image_set_size(&buf, dest->w, dest->h);
buf.planes[0] = src_bits;
buf.stride[0] = src_pitch;
buf.planes[1] = src_bits + src_pitch * surf_height;
buf.stride[1] = src_pitch;
mp_image_copy_gpu(dest, &buf);
}
void mp_blur_rgba_sub_bitmap(struct sub_bitmap *d, double gblur)
{
struct mp_image *tmp1 = mp_image_alloc(IMGFMT_BGRA, d->w, d->h);
if (tmp1) { // on OOM, skip region
struct mp_image s = {0};
mp_image_setfmt(&s, IMGFMT_BGRA);
mp_image_set_size(&s, d->w, d->h);
s.stride[0] = d->stride;
s.planes[0] = d->bitmap;
mp_image_copy(tmp1, &s);
mp_image_sw_blur_scale(&s, tmp1, gblur);
}
talloc_free(tmp1);
}
static struct mp_image *dxva2_retrieve_image(struct lavc_ctx *s,
struct mp_image *img)
{
HRESULT hr;
struct priv *p = s->hwdec_priv;
IDirect3DSurface9 *surface = img->imgfmt == IMGFMT_DXVA2 ?
(IDirect3DSurface9 *)img->planes[3] : NULL;
if (!surface) {
MP_ERR(p, "Failed to get Direct3D surface from mp_image\n");
return img;
}
D3DSURFACE_DESC surface_desc;
IDirect3DSurface9_GetDesc(surface, &surface_desc);
if (surface_desc.Width < img->w || surface_desc.Height < img->h) {
MP_ERR(p, "Direct3D11 texture smaller than mp_image dimensions\n");
return img;
}
struct mp_image *sw_img = mp_image_pool_get(p->sw_pool,
p->mpfmt_decoded,
surface_desc.Width,
surface_desc.Height);
if (!sw_img) {
MP_ERR(p, "Failed to get %s surface from CPU pool\n",
mp_imgfmt_to_name(p->mpfmt_decoded));
return img;
}
D3DLOCKED_RECT lock;
hr = IDirect3DSurface9_LockRect(surface, &lock, NULL, D3DLOCK_READONLY);
if (FAILED(hr)) {
MP_ERR(p, "Unable to lock DXVA2 surface: %s\n",
mp_HRESULT_to_str(hr));
talloc_free(sw_img);
return img;
}
copy_nv12(sw_img, lock.pBits, lock.Pitch, surface_desc.Height);
IDirect3DSurface9_UnlockRect(surface);
mp_image_set_size(sw_img, img->w, img->h);
mp_image_copy_attributes(sw_img, img);
talloc_free(img);
return sw_img;
}
static struct mp_image map_vs_frame(struct vf_priv_s *p, const VSFrameRef *ref,
bool w)
{
const VSFormat *fmt = p->vsapi->getFrameFormat(ref);
struct mp_image img = {0};
mp_image_setfmt(&img, mp_from_vs(fmt->id));
mp_image_set_size(&img, p->vsapi->getFrameWidth(ref, 0),
p->vsapi->getFrameHeight(ref, 0));
for (int n = 0; n < img.num_planes; n++) {
if (w) {
img.planes[n] = p->vsapi->getWritePtr((VSFrameRef *)ref, n);
} else {
img.planes[n] = (uint8_t *)p->vsapi->getReadPtr(ref, n);
}
img.stride[n] = p->vsapi->getStride(ref, n);
}
return img;
}
// This actually returns dummy images, since vda_264 creates it's own AVFrames
// to wrap CVPixelBuffers in planes[3].
static struct mp_image *allocate_image(struct lavc_ctx *ctx, int fmt,
int w, int h)
{
struct priv *p = ctx->hwdec_priv;
if (fmt != IMGFMT_VDA)
return NULL;
if (w != p->vda_ctx.width || h != p->vda_ctx.height)
init_vda_decoder(ctx);
struct mp_image img = {0};
mp_image_setfmt(&img, fmt);
mp_image_set_size(&img, w, h);
// There is an `assert(!dst->f.buf[0])` in libavcodec/h264.c
// Setting the first plane to some dummy value allows to satisfy it
img.planes[0] = (void*)"dummy";
return mp_image_new_custom_ref(&img, NULL, NULL);
}
static struct mp_image *dxva2_new_ref(IDirectXVideoDecoder *decoder,
IDirect3DSurface9 *d3d9_surface,
int w, int h)
{
if (!decoder || !d3d9_surface)
return NULL;
struct dxva2_surface *surface = talloc_zero(NULL, struct dxva2_surface);
surface->surface = d3d9_surface;
IDirect3DSurface9_AddRef(surface->surface);
surface->decoder = decoder;
IDirectXVideoDecoder_AddRef(surface->decoder);
struct mp_image *mpi =
mp_image_new_custom_ref(NULL, surface, dxva2_release_img);
if (!mpi)
abort();
mp_image_setfmt(mpi, IMGFMT_DXVA2);
mp_image_set_size(mpi, w, h);
mpi->planes[3] = (void *)surface->surface;
return mpi;
}
static struct mp_image *dxva2_allocate_image(struct lavc_ctx *s,
int img_w, int img_h)
{
DXVA2Context *ctx = s->hwdec_priv;
int i, old_unused = -1;
for (i = 0; i < ctx->num_surfaces; i++) {
surface_info *info = &ctx->surface_infos[i];
if (!info->used && (old_unused == -1 || info->age < ctx->surface_infos[old_unused].age))
old_unused = i;
}
if (old_unused == -1) {
MP_ERR(ctx, "No free DXVA2 surface!\n");
return NULL;
}
i = old_unused;
DXVA2SurfaceWrapper *w = av_mallocz(sizeof(*w));
if (!w)
return NULL;
w->ctx = ctx;
w->surface = ctx->surfaces[i];;
IDirect3DSurface9_AddRef(w->surface);
w->decoder = ctx->decoder;
IDirectXVideoDecoder_AddRef(w->decoder);
ctx->surface_infos[i].used = 1;
ctx->surface_infos[i].age = ctx->surface_age++;
struct mp_image mpi = {0};
mp_image_setfmt(&mpi, IMGFMT_DXVA2);
mp_image_set_size(&mpi, img_w, img_h);
mpi.planes[3] = (void *)w->surface;
return mp_image_new_custom_ref(&mpi, w, dxva2_release_img);
}
static struct mp_image *dxva2_retrieve_image(struct lavc_ctx *s,
struct mp_image *img)
{
DXVA2Context *ctx = s->hwdec_priv;
LPDIRECT3DSURFACE9 surface = (LPDIRECT3DSURFACE9)img->planes[3];
D3DSURFACE_DESC surfaceDesc;
D3DLOCKED_RECT LockedRect;
HRESULT hr;
IDirect3DSurface9_GetDesc(surface, &surfaceDesc);
if (surfaceDesc.Width < img->w || surfaceDesc.Height < img->h)
return img;
struct mp_image *sw_img =
mp_image_pool_get(ctx->sw_pool, IMGFMT_NV12, surfaceDesc.Width, surfaceDesc.Height);
if (!sw_img)
return img;
hr = IDirect3DSurface9_LockRect(surface, &LockedRect, NULL, D3DLOCK_READONLY);
if (FAILED(hr)) {
MP_ERR(ctx, "Unable to lock DXVA2 surface\n");
talloc_free(sw_img);
return img;
}
copy_nv12(sw_img, LockedRect.pBits, LockedRect.Pitch, surfaceDesc.Height);
mp_image_set_size(sw_img, img->w, img->h);
mp_image_copy_attributes(sw_img, img);
IDirect3DSurface9_UnlockRect(surface);
talloc_free(img);
return sw_img;
}
// Attempt to initialize all pads. Return true if all are initialized, or
// false if more data is needed (or on error).
static bool init_pads(struct lavfi *c)
{
if (!c->graph)
goto error;
for (int n = 0; n < c->num_out_pads; n++) {
struct lavfi_pad *pad = c->out_pads[n];
if (pad->buffer)
continue;
const AVFilter *dst_filter = NULL;
if (pad->type == MP_FRAME_AUDIO) {
dst_filter = avfilter_get_by_name("abuffersink");
} else if (pad->type == MP_FRAME_VIDEO) {
dst_filter = avfilter_get_by_name("buffersink");
} else {
assert(0);
}
if (!dst_filter)
goto error;
char name[256];
snprintf(name, sizeof(name), "mpv_sink_%s", pad->name);
if (avfilter_graph_create_filter(&pad->buffer, dst_filter,
name, NULL, NULL, c->graph) < 0)
goto error;
if (avfilter_link(pad->filter, pad->filter_pad, pad->buffer, 0) < 0)
goto error;
}
for (int n = 0; n < c->num_in_pads; n++) {
struct lavfi_pad *pad = c->in_pads[n];
if (pad->buffer)
continue;
mp_frame_unref(&pad->in_fmt);
read_pad_input(c, pad);
// no input data, format unknown, can't init, wait longer.
if (!pad->pending.type)
return false;
if (mp_frame_is_data(pad->pending)) {
assert(pad->pending.type == pad->type);
pad->in_fmt = mp_frame_ref(pad->pending);
if (!pad->in_fmt.type)
goto error;
if (pad->in_fmt.type == MP_FRAME_VIDEO)
mp_image_unref_data(pad->in_fmt.data);
if (pad->in_fmt.type == MP_FRAME_AUDIO)
mp_aframe_unref_data(pad->in_fmt.data);
}
if (pad->pending.type == MP_FRAME_EOF && !pad->in_fmt.type) {
// libavfilter makes this painful. Init it with a dummy config,
// just so we can tell it the stream is EOF.
if (pad->type == MP_FRAME_AUDIO) {
struct mp_aframe *fmt = mp_aframe_create();
mp_aframe_set_format(fmt, AF_FORMAT_FLOAT);
mp_aframe_set_chmap(fmt, &(struct mp_chmap)MP_CHMAP_INIT_STEREO);
mp_aframe_set_rate(fmt, 48000);
pad->in_fmt = (struct mp_frame){MP_FRAME_AUDIO, fmt};
}
if (pad->type == MP_FRAME_VIDEO) {
struct mp_image *fmt = talloc_zero(NULL, struct mp_image);
mp_image_setfmt(fmt, IMGFMT_420P);
mp_image_set_size(fmt, 64, 64);
pad->in_fmt = (struct mp_frame){MP_FRAME_VIDEO, fmt};
}
}
if (pad->in_fmt.type != pad->type)
goto error;
AVBufferSrcParameters *params = av_buffersrc_parameters_alloc();
if (!params)
goto error;
pad->timebase = AV_TIME_BASE_Q;
char *filter_name = NULL;
if (pad->type == MP_FRAME_AUDIO) {
struct mp_aframe *fmt = pad->in_fmt.data;
params->format = af_to_avformat(mp_aframe_get_format(fmt));
params->sample_rate = mp_aframe_get_rate(fmt);
struct mp_chmap chmap = {0};
mp_aframe_get_chmap(fmt, &chmap);
params->channel_layout = mp_chmap_to_lavc(&chmap);
pad->timebase = (AVRational){1, mp_aframe_get_rate(fmt)};
filter_name = "abuffer";
} else if (pad->type == MP_FRAME_VIDEO) {
struct mp_image *fmt = pad->in_fmt.data;
params->format = imgfmt2pixfmt(fmt->imgfmt);
params->width = fmt->w;
params->height = fmt->h;
params->sample_aspect_ratio.num = fmt->params.p_w;
params->sample_aspect_ratio.den = fmt->params.p_h;
params->hw_frames_ctx = fmt->hwctx;
params->frame_rate = av_d2q(fmt->nominal_fps, 1000000);
filter_name = "buffer";
} else {
assert(0);
}
params->time_base = pad->timebase;
const AVFilter *filter = avfilter_get_by_name(filter_name);
if (filter) {
char name[256];
snprintf(name, sizeof(name), "mpv_src_%s", pad->name);
pad->buffer = avfilter_graph_alloc_filter(c->graph, filter, name);
}
if (!pad->buffer) {
av_free(params);
goto error;
}
int ret = av_buffersrc_parameters_set(pad->buffer, params);
av_free(params);
if (ret < 0)
goto error;
if (avfilter_init_str(pad->buffer, NULL) < 0)
goto error;
if (avfilter_link(pad->buffer, 0, pad->filter, pad->filter_pad) < 0)
goto error;
}
return true;
error:
MP_FATAL(c, "could not initialize filter pads\n");
c->failed = true;
mp_filter_internal_mark_failed(c->f);
return false;
}
mp_image *null_mp_image(uint imgfmt, int width, int height, void *arg, void(*free)(void*)) {
auto mpi = null_mp_image(arg, free);
mp_image_setfmt(mpi, imgfmt);
mp_image_set_size(mpi, width, height);
return mpi;
}
