// Create an MltInput for an image with the given format and dimensions.
static MltInput* create_input( mlt_properties properties, mlt_image_format format, int aspect_width, int aspect_height, int width, int height )
{
MltInput* input = new MltInput( format );
if ( format == mlt_image_rgb24a || format == mlt_image_opengl ) {
// TODO: Get the color space if available.
input->useFlatInput( FORMAT_RGBA_POSTMULTIPLIED_ALPHA, width, height );
}
else if ( format == mlt_image_rgb24 ) {
// TODO: Get the color space if available.
input->useFlatInput( FORMAT_RGB, width, height );
}
else if ( format == mlt_image_yuv420p ) {
ImageFormat image_format;
YCbCrFormat ycbcr_format;
get_format_from_properties( properties, &image_format, &ycbcr_format );
ycbcr_format.chroma_subsampling_x = ycbcr_format.chroma_subsampling_y = 2;
input->useYCbCrInput( image_format, ycbcr_format, width, height );
}
else if ( format == mlt_image_yuv422 ) {
ImageFormat image_format;
YCbCrFormat ycbcr_format;
get_format_from_properties( properties, &image_format, &ycbcr_format );
ycbcr_format.chroma_subsampling_x = 2;
ycbcr_format.chroma_subsampling_y = 1;
input->useYCbCrInput( image_format, ycbcr_format, width, height );
}
return input;
}
static void dispose_pixel_pointers( GlslChain *chain, mlt_service service, mlt_frame frame )
{
if ( service == (mlt_service) -1 ) {
mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
MltInput* input = chain->inputs[ producer ];
input->invalidate_pixel_data();
mlt_pool_release( GlslManager::get_input_pixel_pointer( producer, frame ) );
return;
}
mlt_service input_a = GlslManager::get_effect_input( service, frame );
dispose_pixel_pointers( chain, input_a, frame );
mlt_service input_b;
mlt_frame frame_b;
GlslManager::get_effect_secondary_input( service, frame, &input_b, &frame_b );
if ( input_b ) {
dispose_pixel_pointers( chain, input_b, frame_b );
}
GlslManager::get_effect_third_input( service, frame, &input_b, &frame_b );
if ( input_b ) {
dispose_pixel_pointers( chain, input_b, frame_b );
}
}
static Effect* build_movit_chain( mlt_service service, mlt_frame frame, GlslChain *chain )
{
if ( service == (mlt_service) -1 ) {
mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
MltInput* input = GlslManager::get_input( producer, frame );
GlslManager::set_input( producer, frame, NULL );
chain->effect_chain->add_input( input->get_input() );
chain->inputs.insert(std::make_pair( producer, input ) );
return input->get_input();
}
Effect* effect = GlslManager::get_effect( service, frame );
assert( effect );
GlslManager::set_effect( service, frame, NULL );
mlt_service input_a = GlslManager::get_effect_input( service, frame );
mlt_service input_b, input_c;
mlt_frame frame_b, frame_c;
GlslManager::get_effect_secondary_input( service, frame, &input_b, &frame_b );
GlslManager::get_effect_third_input( service, frame, &input_c, &frame_c );
Effect *effect_a = build_movit_chain( input_a, frame, chain );
if ( input_c && input_b ) {
Effect *effect_b = build_movit_chain( input_b, frame_b, chain );
Effect *effect_c = build_movit_chain( input_c, frame_c, chain );
chain->effect_chain->add_effect( effect, effect_a, effect_b, effect_c );
} else if ( input_b ) {
Effect *effect_b = build_movit_chain( input_b, frame_b, chain );
chain->effect_chain->add_effect( effect, effect_a, effect_b );
} else {
chain->effect_chain->add_effect( effect, effect_a );
}
chain->effects.insert(std::make_pair( service, effect ) );
return effect;
}
static int convert_image( mlt_frame frame, uint8_t **image, mlt_image_format *format, mlt_image_format output_format )
{
// Nothing to do!
if ( *format == output_format )
return 0;
mlt_properties properties = MLT_FRAME_PROPERTIES( frame );
mlt_log_debug( NULL, "filter_movit_convert: %s -> %s\n",
mlt_image_format_name( *format ), mlt_image_format_name( output_format ) );
// Use CPU if glsl not initialized or not supported.
GlslManager* glsl = GlslManager::get_instance();
if ( !glsl || !glsl->get_int("glsl_supported" ) )
return convert_on_cpu( frame, image, format, output_format );
// Do non-GL image conversions on a CPU-based image converter.
if ( *format != mlt_image_glsl && output_format != mlt_image_glsl && output_format != mlt_image_glsl_texture )
return convert_on_cpu( frame, image, format, output_format );
int error = 0;
int width = mlt_properties_get_int( properties, "width" );
int height = mlt_properties_get_int( properties, "height" );
int img_size = mlt_image_format_size( *format, width, height, NULL );
mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
mlt_service service = MLT_PRODUCER_SERVICE(producer);
GlslManager::get_instance()->lock_service( frame );
EffectChain* chain = GlslManager::get_chain( service );
MltInput* input = GlslManager::get_input( service );
if ( !chain || !input ) {
GlslManager::get_instance()->unlock_service( frame );
return 2;
}
if ( *format != mlt_image_glsl ) {
bool finalize_chain = false;
if ( output_format == mlt_image_glsl_texture ) {
// We might already have a texture from a previous conversion from mlt_image_glsl.
glsl_texture texture = (glsl_texture) mlt_properties_get_data( properties, "movit.convert.texture", NULL );
// XXX: requires a special property set on the frame by the app for now
// because we do not have reliable way to clear the texture property
// when a downstream filter has changed image.
if ( texture && mlt_properties_get_int( properties, "movit.convert.use_texture") ) {
*image = (uint8_t*) &texture->texture;
mlt_frame_set_image( frame, *image, 0, NULL );
mlt_properties_set_int( properties, "format", output_format );
*format = output_format;
GlslManager::get_instance()->unlock_service( frame );
return error;
} else {
// Use a separate chain to convert image in RAM to OpenGL texture.
// Use cached chain if available and compatible.
Mlt::Producer producer( mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) ) );
chain = (EffectChain*) producer.get_data( "movit.convert.chain" );
input = (MltInput*) producer.get_data( "movit.convert.input" );
int w = producer.get_int( "movit.convert.width" );
int h = producer.get_int( "movit.convert.height" );
mlt_image_format f = (mlt_image_format) producer.get_int( "movit.convert.format" );
if ( !chain || width != w || height != h || output_format != f ) {
chain = new EffectChain( width, height );
input = new MltInput( width, height );
chain->add_input( input );
chain->add_effect( new Mlt::VerticalFlip() );
finalize_chain = true;
producer.set( "movit.convert.chain", chain, 0, (mlt_destructor) delete_chain );
producer.set( "movit.convert.input", input, 0 );
producer.set( "movit.convert.width", width );
producer.set( "movit.convert.height", height );
producer.set( "movit.convert.format", output_format );
}
}
}
if ( *format == mlt_image_rgb24a || *format == mlt_image_opengl ) {
input->useFlatInput( chain, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, width, height );
input->set_pixel_data( *image );
}
else if ( *format == mlt_image_rgb24 ) {
input->useFlatInput( chain, FORMAT_RGB, width, height );
input->set_pixel_data( *image );
}
else if ( *format == mlt_image_yuv420p ) {
ImageFormat image_format;
YCbCrFormat ycbcr_format;
if ( 709 == mlt_properties_get_int( properties, "colorspace" ) ) {
image_format.color_space = COLORSPACE_REC_709;
image_format.gamma_curve = GAMMA_REC_709;
ycbcr_format.luma_coefficients = YCBCR_REC_709;
} else if ( 576 == mlt_properties_get_int( properties, "height" ) ) {
image_format.color_space = COLORSPACE_REC_601_625;
image_format.gamma_curve = GAMMA_REC_601;
ycbcr_format.luma_coefficients = YCBCR_REC_601;
} else {
image_format.color_space = COLORSPACE_REC_601_525;
image_format.gamma_curve = GAMMA_REC_601;
ycbcr_format.luma_coefficients = YCBCR_REC_601;
}
ycbcr_format.full_range = mlt_properties_get_int( properties, "force_full_luma" );
ycbcr_format.chroma_subsampling_x = ycbcr_format.chroma_subsampling_y = 2;
// TODO: make new frame properties set by producers
ycbcr_format.cb_x_position = ycbcr_format.cr_x_position = 0.0f;
ycbcr_format.cb_y_position = ycbcr_format.cr_y_position = 0.5f;
input->useYCbCrInput( chain, image_format, ycbcr_format, width, height );
input->set_pixel_data( *image );
}
else if ( *format == mlt_image_yuv422 ) {
ImageFormat image_format;
YCbCrFormat ycbcr_format;
if ( 709 == mlt_properties_get_int( properties, "colorspace" ) ) {
image_format.color_space = COLORSPACE_REC_709;
image_format.gamma_curve = GAMMA_REC_709;
ycbcr_format.luma_coefficients = YCBCR_REC_709;
} else if ( 576 == height ) {
image_format.color_space = COLORSPACE_REC_601_625;
image_format.gamma_curve = GAMMA_REC_601;
ycbcr_format.luma_coefficients = YCBCR_REC_601;
} else {
image_format.color_space = COLORSPACE_REC_601_525;
image_format.gamma_curve = GAMMA_REC_601;
ycbcr_format.luma_coefficients = YCBCR_REC_601;
}
ycbcr_format.full_range = mlt_properties_get_int( properties, "force_full_luma" );
ycbcr_format.chroma_subsampling_x = 2;
ycbcr_format.chroma_subsampling_y = 1;
// TODO: make new frame properties set by producers
ycbcr_format.cb_x_position = ycbcr_format.cr_x_position = 0.0f;
ycbcr_format.cb_y_position = ycbcr_format.cr_y_position = 0.5f;
input->useYCbCrInput( chain, image_format, ycbcr_format, width, height );
// convert chunky to planar
uint8_t* planar = (uint8_t*) mlt_pool_alloc( img_size );
yuv422_to_yuv422p( *image, planar, width, height );
input->set_pixel_data( planar );
mlt_frame_set_image( frame, planar, img_size, mlt_pool_release );
}
// Finalize the separate conversion chain if needed.
if ( finalize_chain )
chain->finalize();
}
if ( output_format != mlt_image_glsl ) {
glsl_fbo fbo = glsl->get_fbo( width, height );
if ( output_format == mlt_image_glsl_texture ) {
glsl_texture texture = glsl->get_texture( width, height, GL_RGBA );
glBindFramebuffer( GL_FRAMEBUFFER, fbo->fbo );
check_error();
glFramebufferTexture2D( GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture->texture, 0 );
check_error();
glBindFramebuffer( GL_FRAMEBUFFER, 0 );
check_error();
GlslManager::render( service, chain, fbo->fbo, width, height );
glFinish();
check_error();
glBindFramebuffer( GL_FRAMEBUFFER, 0 );
check_error();
*image = (uint8_t*) &texture->texture;
mlt_frame_set_image( frame, *image, 0, NULL );
mlt_properties_set_data( properties, "movit.convert.texture", texture, 0,
(mlt_destructor) GlslManager::release_texture, NULL );
mlt_properties_set_int( properties, "format", output_format );
*format = output_format;
}
else {
// Use a PBO to hold the data we read back with glReadPixels()
// (Intel/DRI goes into a slow path if we don't read to PBO)
GLenum gl_format = ( output_format == mlt_image_rgb24a || output_format == mlt_image_opengl )?
GL_RGBA : GL_RGB;
img_size = width * height * ( gl_format == GL_RGB? 3 : 4 );
glsl_pbo pbo = glsl->get_pbo( img_size );
glsl_texture texture = glsl->get_texture( width, height, gl_format );
if ( fbo && pbo && texture ) {
// Set the FBO
glBindFramebuffer( GL_FRAMEBUFFER, fbo->fbo );
check_error();
glFramebufferTexture2D( GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture->texture, 0 );
check_error();
glBindFramebuffer( GL_FRAMEBUFFER, 0 );
check_error();
GlslManager::render( service, chain, fbo->fbo, width, height );
// Read FBO into PBO
glBindBuffer( GL_PIXEL_PACK_BUFFER_ARB, pbo->pbo );
check_error();
glBufferData( GL_PIXEL_PACK_BUFFER_ARB, img_size, NULL, GL_STREAM_READ );
check_error();
glReadPixels( 0, 0, width, height, gl_format, GL_UNSIGNED_BYTE, BUFFER_OFFSET(0) );
check_error();
// Copy from PBO
uint8_t* buf = (uint8_t*) glMapBuffer( GL_PIXEL_PACK_BUFFER_ARB, GL_READ_ONLY );
check_error();
*image = (uint8_t*) mlt_pool_alloc( img_size );
mlt_frame_set_image( frame, *image, img_size, mlt_pool_release );
memcpy( *image, buf, img_size );
if ( output_format == mlt_image_yuv422 || output_format == mlt_image_yuv420p ) {
*format = mlt_image_rgb24;
error = convert_on_cpu( frame, image, format, output_format );
}
// Release PBO and FBO
glUnmapBuffer( GL_PIXEL_PACK_BUFFER_ARB );
check_error();
glBindBuffer( GL_PIXEL_PACK_BUFFER_ARB, 0 );
check_error();
glBindFramebuffer( GL_FRAMEBUFFER, 0 );
check_error();
glBindTexture( GL_TEXTURE_2D, 0 );
check_error();
mlt_properties_set_data( properties, "movit.convert.texture", texture, 0,
(mlt_destructor) GlslManager::release_texture, NULL);
mlt_properties_set_int( properties, "format", output_format );
*format = output_format;
}
else {
error = 1;
}
}
if ( fbo ) GlslManager::release_fbo( fbo );
}
else {
mlt_properties_set_int( properties, "format", output_format );
*format = output_format;
}
GlslManager::get_instance()->unlock_service( frame );
return error;
}
static int convert_image( mlt_frame frame, uint8_t **image, mlt_image_format *format, mlt_image_format output_format )
{
// Nothing to do!
if ( *format == output_format )
return 0;
mlt_properties properties = MLT_FRAME_PROPERTIES( frame );
mlt_log_debug( NULL, "filter_movit_convert: %s -> %s (%d)\n",
mlt_image_format_name( *format ), mlt_image_format_name( output_format ),
mlt_frame_get_position( frame ) );
// Use CPU if glsl not initialized or not supported.
GlslManager* glsl = GlslManager::get_instance();
if ( !glsl || !glsl->get_int("glsl_supported" ) )
return convert_on_cpu( frame, image, format, output_format );
// Do non-GL image conversions on a CPU-based image converter.
if ( *format != mlt_image_glsl && output_format != mlt_image_glsl && output_format != mlt_image_glsl_texture )
return convert_on_cpu( frame, image, format, output_format );
int error = 0;
int width = mlt_properties_get_int( properties, "width" );
int height = mlt_properties_get_int( properties, "height" );
GlslManager::get_instance()->lock_service( frame );
// If we're at the beginning of a series of Movit effects, store the input
// sent into the chain.
if ( output_format == mlt_image_glsl ) {
mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
mlt_profile profile = mlt_service_profile( MLT_PRODUCER_SERVICE( producer ) );
MltInput *input = create_input( properties, *format, profile->width, profile->height, width, height );
GlslManager::set_input( producer, frame, input );
uint8_t *img_copy = make_input_copy( *format, *image, width, height );
GlslManager::set_input_pixel_pointer( producer, frame, img_copy );
*image = (uint8_t *) -1;
mlt_frame_set_image( frame, *image, 0, NULL );
}
// If we're at the _end_ of a series of Movit effects, render the chain.
if ( *format == mlt_image_glsl ) {
mlt_service leaf_service = (mlt_service) *image;
if ( leaf_service == (mlt_service) -1 ) {
// Something on the way requested conversion to mlt_glsl,
// but never added an effect. Don't build a Movit chain;
// just do the conversion and we're done.
mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
MltInput *input = GlslManager::get_input( producer, frame );
*image = GlslManager::get_input_pixel_pointer( producer, frame );
*format = input->get_format();
delete input;
GlslManager::get_instance()->unlock_service( frame );
return convert_on_cpu( frame, image, format, output_format );
}
// Construct the chain unless we already have a good one.
finalize_movit_chain( leaf_service, frame );
// Set per-frame parameters now that we know which Effect instances to set them on.
// (finalize_movit_chain may already have done this, though, but twice doesn't hurt.)
GlslChain *chain = GlslManager::get_chain( leaf_service );
set_movit_parameters( chain, leaf_service, frame );
error = movit_render( chain->effect_chain, frame, format, output_format, width, height, image );
dispose_pixel_pointers( chain, leaf_service, frame );
}
// If we've been asked to render some frame directly to a texture (without any
// effects in-between), we create a new mini-chain to do so.
if ( *format != mlt_image_glsl && output_format == mlt_image_glsl_texture ) {
// We might already have a texture from a previous conversion from mlt_image_glsl.
glsl_texture texture = (glsl_texture) mlt_properties_get_data( properties, "movit.convert.texture", NULL );
// XXX: requires a special property set on the frame by the app for now
// because we do not have reliable way to clear the texture property
// when a downstream filter has changed image.
if ( texture && mlt_properties_get_int( properties, "movit.convert.use_texture") ) {
*image = (uint8_t*) &texture->texture;
mlt_frame_set_image( frame, *image, 0, NULL );
} else {
// Use a separate chain to convert image in RAM to OpenGL texture.
// Use cached chain if available and compatible.
Mlt::Producer producer( mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) ) );
EffectChain *chain = (EffectChain*) producer.get_data( "movit.convert.chain" );
MltInput *input = (MltInput*) producer.get_data( "movit.convert.input" );
int w = producer.get_int( "movit.convert.width" );
int h = producer.get_int( "movit.convert.height" );
mlt_image_format f = (mlt_image_format) producer.get_int( "movit.convert.format" );
if ( !chain || !input || width != w || height != h || *format != f ) {
chain = new EffectChain( width, height, GlslManager::get_instance()->get_resource_pool() );
input = create_input( properties, *format, width, height, width, height );
chain->add_input( input->get_input() );
chain->add_effect( new Mlt::VerticalFlip() );
ImageFormat movit_output_format;
movit_output_format.color_space = COLORSPACE_sRGB;
movit_output_format.gamma_curve = GAMMA_sRGB;
chain->add_output(movit_output_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
chain->set_dither_bits(8);
chain->finalize();
producer.set( "movit.convert.chain", chain, 0, (mlt_destructor) delete_chain );
producer.set( "movit.convert.input", input, 0, NULL );
producer.set( "movit.convert.width", width );
producer.set( "movit.convert.height", height );
producer.set( "movit.convert.format", *format );
}
if ( *format == mlt_image_yuv422 ) {
// We need to convert to planar, which make_input_copy() will do for us.
uint8_t *planar = make_input_copy( *format, *image, width, height );
input->set_pixel_data( planar );
error = movit_render( chain, frame, format, output_format, width, height, image );
mlt_pool_release( planar );
} else {
input->set_pixel_data( *image );
error = movit_render( chain, frame, format, output_format, width, height, image );
}
}
}
GlslManager::get_instance()->unlock_service( frame );
mlt_properties_set_int( properties, "format", output_format );
*format = output_format;
return error;
}
static void set_movit_parameters( GlslChain *chain, mlt_service service, mlt_frame frame )
{
if ( service == (mlt_service) -1 ) {
mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
MltInput* input = chain->inputs[ producer ];
input->set_pixel_data( GlslManager::get_input_pixel_pointer( producer, frame ) );
return;
}
Effect* effect = chain->effects[ service ];
mlt_service input_a = GlslManager::get_effect_input( service, frame );
set_movit_parameters( chain, input_a, frame );
mlt_service input_b;
mlt_frame frame_b;
GlslManager::get_effect_secondary_input( service, frame, &input_b, &frame_b );
if ( input_b ) {
set_movit_parameters( chain, input_b, frame_b );
}
GlslManager::get_effect_third_input( service, frame, &input_b, &frame_b );
if ( input_b ) {
set_movit_parameters( chain, input_b, frame_b );
}
mlt_properties properties = MLT_SERVICE_PROPERTIES( service );
int count = mlt_properties_count( properties );
for (int i = 0; i < count; ++i) {
const char *name = mlt_properties_get_name( properties, i );
if (strncmp(name, "_movit.parms.float.", strlen("_movit.parms.float.")) == 0 &&
mlt_properties_get_value( properties, i )) {
bool ok = effect->set_float(name + strlen("_movit.parms.float."),
mlt_properties_get_double( properties, name ));
assert(ok);
}
if (strncmp(name, "_movit.parms.int.", strlen("_movit.parms.int.")) == 0 &&
mlt_properties_get_value( properties, i )) {
bool ok = effect->set_int(name + strlen("_movit.parms.int."),
mlt_properties_get_int( properties, name ));
assert(ok);
}
if (strncmp(name, "_movit.parms.vec3.", strlen("_movit.parms.vec3.")) == 0 &&
strcmp(name + strlen(name) - 3, "[0]") == 0 &&
mlt_properties_get_value( properties, i )) {
float val[3];
char *name_copy = strdup(name);
char *index_char = name_copy + strlen(name_copy) - 2;
val[0] = mlt_properties_get_double( properties, name_copy );
*index_char = '1';
val[1] = mlt_properties_get_double( properties, name_copy );
*index_char = '2';
val[2] = mlt_properties_get_double( properties, name_copy );
index_char[-1] = '\0';
bool ok = effect->set_vec3(name_copy + strlen("_movit.parms.vec3."), val);
assert(ok);
free(name_copy);
}
if (strncmp(name, "_movit.parms.vec4.", strlen("_movit.parms.vec4.")) == 0 &&
strcmp(name + strlen(name) - 3, "[0]") == 0 &&
mlt_properties_get_value( properties, i )) {
float val[4];
char *name_copy = strdup(name);
char *index_char = name_copy + strlen(name_copy) - 2;
val[0] = mlt_properties_get_double( properties, name_copy );
*index_char = '1';
val[1] = mlt_properties_get_double( properties, name_copy );
*index_char = '2';
val[2] = mlt_properties_get_double( properties, name_copy );
*index_char = '3';
val[3] = mlt_properties_get_double( properties, name_copy );
index_char[-1] = '\0';
bool ok = effect->set_vec4(name_copy + strlen("_movit.parms.vec4."), val);
assert(ok);
free(name_copy);
}
}
}
