void jit_gl_hap_draw_frame(void *jitob, CVImageBufferRef frame)
{
t_jit_gl_hap * x = (t_jit_gl_hap*)jitob;
CFTypeID imageType = CFGetTypeID(frame);
OSType newPixelFormat;
if(x->validframe)
return;
if (imageType == CVPixelBufferGetTypeID()) {
// Update the texture
CVBufferRetain(frame);
if(x->buffer) {
CVPixelBufferUnlockBaseAddress(x->buffer, kCVPixelBufferLock_ReadOnly);
CVBufferRelease(x->buffer);
}
x->buffer = frame;
CVPixelBufferLockBaseAddress(x->buffer, kCVPixelBufferLock_ReadOnly);
x->dim[0] = CVPixelBufferGetWidth(x->buffer);
x->dim[1] = CVPixelBufferGetHeight(x->buffer);
newPixelFormat = CVPixelBufferGetPixelFormatType(x->buffer);
if(x->buffer && x->hap_format==JIT_GL_HAP_PF_HAP) {
size_t extraRight, extraBottom;
unsigned int bitsPerPixel;
size_t bytesPerRow;
size_t actualBufferSize;
CVPixelBufferGetExtendedPixels(x->buffer, NULL, &extraRight, NULL, &extraBottom);
x->roundedWidth = x->dim[0] + extraRight;
x->roundedHeight = x->dim[1] + extraBottom;
if (x->roundedWidth % 4 != 0 || x->roundedHeight % 4 != 0) {
x->validframe = 0;
return;
}
switch (newPixelFormat) {
case kHapPixelFormatTypeRGB_DXT1:
x->newInternalFormat = GL_COMPRESSED_RGB_S3TC_DXT1_EXT;
bitsPerPixel = 4;
break;
case kHapPixelFormatTypeRGBA_DXT5:
case kHapPixelFormatTypeYCoCg_DXT5:
x->newInternalFormat = GL_COMPRESSED_RGBA_S3TC_DXT5_EXT;
bitsPerPixel = 8;
break;
default:
// we don't support non-DXT pixel buffers
x->validframe = 0;
return;
break;
}
x->useshader = (newPixelFormat == kHapPixelFormatTypeYCoCg_DXT5);
bytesPerRow = (x->roundedWidth * bitsPerPixel) / 8;
x->newDataLength = bytesPerRow * x->roundedHeight; // usually not the full length of the buffer
actualBufferSize = CVPixelBufferGetDataSize(x->buffer);
// Check the buffer is as large as we expect it to be
if (x->newDataLength > actualBufferSize) {
x->validframe = 0;
return;
}
// If we got this far we're good to go
x->validframe = 1;
x->target = GL_TEXTURE_2D;
if(!x->flipped) {
jit_attr_setlong(x->texoutput, gensym("flip"), 1);
x->flipped = 1;
}
//x->drawhap = 1;
}
else if(x->buffer) {// && x->hap_format==JIT_GL_HAP_PF_HAP) {
if( newPixelFormat == k24RGBPixelFormat )
x->newInternalFormat = GL_RGB8;
else if( newPixelFormat == k32BGRAPixelFormat )
x->newInternalFormat = GL_RGBA8;
else {
x->validframe = 0;
return;
}
x->roundedWidth = x->dim[0];
x->roundedHeight = x->dim[1];
x->newDataLength = CVPixelBufferGetDataSize(x->buffer);
x->rowLength = CVPixelBufferGetBytesPerRow( x->buffer ) / (x->hap_format==JIT_GL_HAP_PF_RGB ? 3 : 4);
x->target = GL_TEXTURE_RECTANGLE_EXT;
if(!x->flipped) {
jit_attr_setlong(x->texoutput, gensym("flip"), 1);
x->flipped = 1;
}
x->validframe = 1;
}
}
else {
#ifdef MAC_VERSION
CGSize imageSize = CVImageBufferGetEncodedSize(frame);
bool flipped = CVOpenGLTextureIsFlipped(frame);
x->texture = CVOpenGLTextureGetName(frame);
x->useshader = 0;
x->dim[0] = (t_atom_long)imageSize.width;
x->dim[1] = (t_atom_long)imageSize.height;
x->validframe = 1;
x->target = GL_TEXTURE_RECTANGLE_ARB;
if(x->flipped!=flipped) {
jit_attr_setlong(x->texoutput, gensym("flip"), flipped);
x->flipped = flipped;
}
#endif
}
}
/*
* PsychQTGetTextureFromMovie() -- Create an OpenGL texture map from a specific videoframe from given movie object.
*
* win = Window pointer of onscreen window for which a OpenGL texture should be created.
* moviehandle = Handle to the movie object.
* checkForImage = true == Just check if new image available, false == really retrieve the image, blocking if necessary.
* timeindex = When not in playback mode, this allows specification of a requested frame by presentation time.
* If set to -1, or if in realtime playback mode, this parameter is ignored and the next video frame is returned.
* out_texture = Pointer to the Psychtoolbox texture-record where the new texture should be stored.
* presentation_timestamp = A ptr to a double variable, where the presentation timestamp of the returned frame should be stored.
*
* Returns true (1) on success, false (0) if no new image available, -1 if no new image available and there won't be any in future.
*/
int PsychQTGetTextureFromMovie(PsychWindowRecordType *win, int moviehandle, int checkForImage, double timeindex, PsychWindowRecordType *out_texture, double *presentation_timestamp)
{
static TimeValue myNextTimeCached = -2;
static TimeValue nextFramesTimeCached = -2;
TimeValue myCurrTime;
TimeValue myNextTime;
TimeValue nextFramesTime=0;
short myFlags;
OSType myTypes[1];
OSErr error = noErr;
Movie theMovie;
CVOpenGLTextureRef newImage = NULL;
QTVisualContextRef theMoviecontext;
unsigned int failcount=0;
float lowerLeft[2];
float lowerRight[2];
float upperRight[2];
float upperLeft[2];
GLuint texid;
Rect rect;
float rate;
double targetdelta, realdelta, frames;
PsychRectType outRect;
if (!PsychIsOnscreenWindow(win)) {
PsychErrorExitMsg(PsychError_user, "Need onscreen window ptr!!!");
}
// Activate OpenGL context of target window:
PsychSetGLContext(win);
// Explicitely disable Apple's Client storage extensions. For now they are not really useful to us.
glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE);
if (moviehandle < 0 || moviehandle >= PSYCH_MAX_MOVIES) {
PsychErrorExitMsg(PsychError_user, "Invalid moviehandle provided.");
}
if ((timeindex!=-1) && (timeindex < 0 || timeindex >= 10000.0)) {
PsychErrorExitMsg(PsychError_user, "Invalid timeindex provided.");
}
if (NULL == out_texture && !checkForImage) {
PsychErrorExitMsg(PsychError_internal, "NULL-Ptr instead of out_texture ptr passed!!!");
}
// Fetch references to objects we need:
theMovie = movieRecordBANK[moviehandle].theMovie;
theMoviecontext = movieRecordBANK[moviehandle].QTMovieContext;
if (theMovie == NULL) {
PsychErrorExitMsg(PsychError_user, "Invalid moviehandle provided. No movie associated with this handle.");
}
// Check if end of movie is reached. Rewind, if so...
if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag > 0) {
if (GetMovieRate(theMovie)>0) {
GoToBeginningOfMovie(theMovie);
} else {
GoToEndOfMovie(theMovie);
}
}
// Is movie actively playing (automatic async playback, possibly with synced sound)?
// If so, then we ignore the 'timeindex' parameter, because the automatic playback
// process determines which frames should be delivered to PTB when. This function will
// simply wait or poll for arrival/presence of a new frame that hasn't been fetched
// in previous calls.
if (0 == GetMovieRate(theMovie)) {
// Movie playback inactive. We are in "manual" mode: No automatic async playback,
// no synced audio output. The user just wants to manually fetch movie frames into
// textures for manual playback in a standard Matlab-loop.
// First pass - checking for new image?
if (checkForImage) {
// Image for specific point in time requested?
if (timeindex >= 0) {
// Yes. We try to retrieve the next possible image for requested timeindex.
myCurrTime = (TimeValue) ((timeindex * (double) GetMovieTimeScale(theMovie)) + 0.5f);
}
else {
// No. We just retrieve the next frame, given the current movie time.
myCurrTime = GetMovieTime(theMovie, NULL);
}
// Retrieve timeindex of the closest image sample after myCurrTime:
myFlags = nextTimeStep + nextTimeEdgeOK; // We want the next frame in the movie's media.
myTypes[0] = VisualMediaCharacteristic; // We want video samples.
GetMovieNextInterestingTime(theMovie, myFlags, 1, myTypes, myCurrTime, FloatToFixed(1), &myNextTime, &nextFramesTime);
error = GetMoviesError();
if (error != noErr) {
PsychErrorExitMsg(PsychError_internal, "Failed to fetch texture from movie for given timeindex!");
}
// Found useful event?
if (myNextTime == -1) {
if (PsychPrefStateGet_Verbosity() > 3) printf("PTB-WARNING: Bogus timevalue in movie track for movie %i. Trying to keep going.\n", moviehandle);
// No. Just push timestamp to current time plus a little bit in the hope
// this will get us unstuck:
myNextTime = myCurrTime + (TimeValue) 1;
nextFramesTime = (TimeValue) 0;
}
if (myNextTime != myNextTimeCached) {
// Set movies current time to myNextTime, so the next frame will be fetched from there:
SetMovieTimeValue(theMovie, myNextTime);
// nextFramesTime is the timeindex to which we need to advance for retrieval of next frame: (see code below)
nextFramesTime=myNextTime + nextFramesTime;
if (PsychPrefStateGet_Verbosity() > 5) printf("PTB-DEBUG: Current timevalue in movie track for movie %i is %lf secs.\n", moviehandle, (double) myNextTime / (double) GetMovieTimeScale(theMovie));
if (PsychPrefStateGet_Verbosity() > 5) printf("PTB-DEBUG: Next timevalue in movie track for movie %i is %lf secs.\n", moviehandle, (double) nextFramesTime / (double) GetMovieTimeScale(theMovie));
// Cache values for 2nd pass:
myNextTimeCached = myNextTime;
nextFramesTimeCached = nextFramesTime;
}
else {
// Somehow got stuck? Do nothing...
if (PsychPrefStateGet_Verbosity() > 5) printf("PTB-DEBUG: Seem to be a bit stuck at timevalue [for movie %i] of %lf secs. Nudging a bit forward...\n", moviehandle, (double) myNextTime / (double) GetMovieTimeScale(theMovie));
// Nudge the timeindex a bit forware in the hope that this helps:
SetMovieTimeValue(theMovie, GetMovieTime(theMovie, NULL) + 1);
}
}
else {
// This is the 2nd pass: Image fetching. Use cached values from first pass:
// Caching in a static works because we're always called immediately for 2nd
// pass after successfull return from 1st pass, and we're not multi-threaded,
// i.e., don't need to be reentrant or thread-safe here:
myNextTime = myNextTimeCached;
nextFramesTime = nextFramesTimeCached;
myNextTimeCached = -2;
}
}
else {
// myNextTime unavailable if in autoplayback-mode:
myNextTime=-1;
}
// Presentation timestamp requested?
if (presentation_timestamp) {
// Already available?
if (myNextTime==-1) {
// Retrieve the exact presentation timestamp of the retrieved frame (in movietime):
myFlags = nextTimeStep + nextTimeEdgeOK; // We want the next frame in the movie's media.
myTypes[0] = VisualMediaCharacteristic; // We want video samples.
// We search backward for the closest available image for the current time. Either we get the current time
// if we happen to fetch a frame exactly when it becomes ready, or we get a bit earlier timestamp, which is
// the optimal presentation timestamp for this frame:
GetMovieNextInterestingTime(theMovie, myFlags, 1, myTypes, GetMovieTime(theMovie, NULL), FloatToFixed(-1), &myNextTime, NULL);
}
// Convert pts (in Quicktime ticks) to pts in seconds since start of movie and return it:
*presentation_timestamp = (double) myNextTime / (double) GetMovieTimeScale(theMovie);
}
// Allow quicktime visual context task to do its internal bookkeeping and cleanup work:
if (theMoviecontext) QTVisualContextTask(theMoviecontext);
// Perform decompress-operation:
if (checkForImage) MoviesTask(theMovie, 0);
// Should we just check for new image? If so, just return availability status:
if (checkForImage) {
if (PSYCH_USE_QT_GWORLDS) {
// We use GWorlds. In this case we either suceed immediately due to the
// synchronous nature of GWorld rendering, or we fail completely at end
// of non-looping movie:
if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag == 0) {
// No new frame available and there won't be any in the future, because this is a non-looping
// movie that has reached its end.
return(-1);
}
// Is this the special case of a movie without video, but only sound? In that case,
// we always return a 'false' because there ain't no image to return.
if (movieRecordBANK[moviehandle].QTMovieGWorld == NULL) return(false);
// Success!
return(true);
}
// Code which uses QTVisualContextTasks...
if (QTVisualContextIsNewImageAvailable(theMoviecontext, NULL)) {
// New frame ready!
return(true);
}
else if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag == 0) {
// No new frame available and there won't be any in the future, because this is a non-looping
// movie that has reached its end.
return(-1);
}
else {
// No new frame available yet:
return(false);
}
}
if (!PSYCH_USE_QT_GWORLDS) {
// Blocking wait-code for non-GWorld mode:
// Try up to 1000 iterations for arrival of requested image data in wait-mode:
failcount=0;
while ((failcount < 1000) && !QTVisualContextIsNewImageAvailable(theMoviecontext, NULL)) {
PsychWaitIntervalSeconds(0.005);
MoviesTask(theMovie, 0);
failcount++;
}
// No new frame available and there won't be any in the future, because this is a non-looping
// movie that has reached its end.
if ((failcount>=1000) && IsMovieDone(theMovie) && (movieRecordBANK[moviehandle].loopflag == 0)) {
return(-1);
}
// Fetch new OpenGL texture with the new movie image frame:
error = QTVisualContextCopyImageForTime(theMoviecontext, kCFAllocatorDefault, NULL, &newImage);
if ((error!=noErr) || newImage == NULL) {
PsychErrorExitMsg(PsychError_internal, "OpenGL<->Quicktime texture fetch failed!!!");
}
// Disable client storage, if it was enabled:
glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE);
// Build a standard PTB texture record:
CVOpenGLTextureGetCleanTexCoords (newImage, lowerLeft, lowerRight, upperRight, upperLeft);
texid = CVOpenGLTextureGetName(newImage);
// Assign texture rectangle:
PsychMakeRect(outRect, upperLeft[0], upperLeft[1], lowerRight[0], lowerRight[1]);
// Set texture orientation as if it were an inverted Offscreen window: Upside-down.
out_texture->textureOrientation = (CVOpenGLTextureIsFlipped(newImage)) ? 3 : 4;
// Assign OpenGL texture id:
out_texture->textureNumber = texid;
// Store special texture object as part of the PTB texture record:
out_texture->targetSpecific.QuickTimeGLTexture = newImage;
}
else {
// Synchronous texture fetch code for GWorld rendering mode:
// At this point, the GWorld should contain the source image for creating a
// standard OpenGL texture:
// Disable client storage, if it was enabled:
glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE);
// Build a standard PTB texture record:
// Assign texture rectangle:
GetMovieBox(theMovie, &rect);
// Hack: Need to extend rect by 4 pixels, because GWorlds are 4 pixels-aligned via
// image row padding:
rect.right = rect.right + 4;
PsychMakeRect(out_texture->rect, rect.left, rect.top, rect.right, rect.bottom);
// Set NULL - special texture object as part of the PTB texture record:
out_texture->targetSpecific.QuickTimeGLTexture = NULL;
// Set texture orientation as if it were an inverted Offscreen window: Upside-down.
out_texture->textureOrientation = 3;
// Setup a pointer to our GWorld as texture data pointer:
out_texture->textureMemorySizeBytes = 0;
// Quicktime textures are aligned on 4 Byte boundaries:
out_texture->textureByteAligned = 4;
// Lock GWorld:
if(!LockPixels(GetGWorldPixMap(movieRecordBANK[moviehandle].QTMovieGWorld))) {
// Locking surface failed! We abort.
PsychErrorExitMsg(PsychError_internal, "PsychQTGetTextureFromMovie(): Locking GWorld pixmap surface failed!!!");
}
// This will retrieve an OpenGL compatible pointer to the GWorlds pixel data and assign it to our texmemptr:
out_texture->textureMemory = (GLuint*) GetPixBaseAddr(GetGWorldPixMap(movieRecordBANK[moviehandle].QTMovieGWorld));
// Let PsychCreateTexture() do the rest of the job of creating, setting up and
// filling an OpenGL texture with GWorlds content:
PsychCreateTexture(out_texture);
// Undo hack from above after texture creation: Now we need the real width of the
// texture for proper texture coordinate assignments in drawing code et al.
rect.right = rect.right - 4;
PsychMakeRect(outRect, rect.left, rect.top, rect.right, rect.bottom);
// Unlock GWorld surface. We do a glFinish() before, for safety reasons...
//glFinish();
UnlockPixels(GetGWorldPixMap(movieRecordBANK[moviehandle].QTMovieGWorld));
// Ready to use the texture... We're done.
}
// Normalize texture rectangle and assign it:
PsychNormalizeRect(outRect, out_texture->rect);
rate = FixedToFloat(GetMovieRate(theMovie));
// Detection of dropped frames: This is a heuristic. We'll see how well it works out...
if (rate && presentation_timestamp) {
// Try to check for dropped frames in playback mode:
// Expected delta between successive presentation timestamps:
targetdelta = 1.0f / (movieRecordBANK[moviehandle].fps * rate);
// Compute real delta, given rate and playback direction:
if (rate>0) {
realdelta = *presentation_timestamp - movieRecordBANK[moviehandle].last_pts;
if (realdelta<0) realdelta = 0;
}
else {
realdelta = -1.0 * (*presentation_timestamp - movieRecordBANK[moviehandle].last_pts);
if (realdelta<0) realdelta = 0;
}
frames = realdelta / targetdelta;
// Dropped frames?
if (frames > 1 && movieRecordBANK[moviehandle].last_pts>=0) {
movieRecordBANK[moviehandle].nr_droppedframes += (int) (frames - 1 + 0.5);
}
movieRecordBANK[moviehandle].last_pts = *presentation_timestamp;
}
// Manually advance movie time, if in fetch mode:
if (0 == GetMovieRate(theMovie)) {
// We are in manual fetch mode: Need to manually advance movie time to next
// media sample:
if (nextFramesTime == myNextTime) {
// Invalid value? Try to hack something that gets us unstuck:
myNextTime = GetMovieTime(theMovie, NULL);
nextFramesTime = myNextTime + (TimeValue) 1;
}
SetMovieTimeValue(theMovie, nextFramesTime);
}
// Check if end of movie is reached. Rewind, if so...
if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag > 0) {
if (GetMovieRate(theMovie)>0) {
GoToBeginningOfMovie(theMovie);
} else {
GoToEndOfMovie(theMovie);
}
}
return(TRUE);
}
