void jit_gl_hap_draw_frame(void *jitob, CVImageBufferRef frame) { t_jit_gl_hap * x = (t_jit_gl_hap*)jitob; CFTypeID imageType = CFGetTypeID(frame); OSType newPixelFormat; if(x->validframe) return; if (imageType == CVPixelBufferGetTypeID()) { // Update the texture CVBufferRetain(frame); if(x->buffer) { CVPixelBufferUnlockBaseAddress(x->buffer, kCVPixelBufferLock_ReadOnly); CVBufferRelease(x->buffer); } x->buffer = frame; CVPixelBufferLockBaseAddress(x->buffer, kCVPixelBufferLock_ReadOnly); x->dim[0] = CVPixelBufferGetWidth(x->buffer); x->dim[1] = CVPixelBufferGetHeight(x->buffer); newPixelFormat = CVPixelBufferGetPixelFormatType(x->buffer); if(x->buffer && x->hap_format==JIT_GL_HAP_PF_HAP) { size_t extraRight, extraBottom; unsigned int bitsPerPixel; size_t bytesPerRow; size_t actualBufferSize; CVPixelBufferGetExtendedPixels(x->buffer, NULL, &extraRight, NULL, &extraBottom); x->roundedWidth = x->dim[0] + extraRight; x->roundedHeight = x->dim[1] + extraBottom; if (x->roundedWidth % 4 != 0 || x->roundedHeight % 4 != 0) { x->validframe = 0; return; } switch (newPixelFormat) { case kHapPixelFormatTypeRGB_DXT1: x->newInternalFormat = GL_COMPRESSED_RGB_S3TC_DXT1_EXT; bitsPerPixel = 4; break; case kHapPixelFormatTypeRGBA_DXT5: case kHapPixelFormatTypeYCoCg_DXT5: x->newInternalFormat = GL_COMPRESSED_RGBA_S3TC_DXT5_EXT; bitsPerPixel = 8; break; default: // we don't support non-DXT pixel buffers x->validframe = 0; return; break; } x->useshader = (newPixelFormat == kHapPixelFormatTypeYCoCg_DXT5); bytesPerRow = (x->roundedWidth * bitsPerPixel) / 8; x->newDataLength = bytesPerRow * x->roundedHeight; // usually not the full length of the buffer actualBufferSize = CVPixelBufferGetDataSize(x->buffer); // Check the buffer is as large as we expect it to be if (x->newDataLength > actualBufferSize) { x->validframe = 0; return; } // If we got this far we're good to go x->validframe = 1; x->target = GL_TEXTURE_2D; if(!x->flipped) { jit_attr_setlong(x->texoutput, gensym("flip"), 1); x->flipped = 1; } //x->drawhap = 1; } else if(x->buffer) {// && x->hap_format==JIT_GL_HAP_PF_HAP) { if( newPixelFormat == k24RGBPixelFormat ) x->newInternalFormat = GL_RGB8; else if( newPixelFormat == k32BGRAPixelFormat ) x->newInternalFormat = GL_RGBA8; else { x->validframe = 0; return; } x->roundedWidth = x->dim[0]; x->roundedHeight = x->dim[1]; x->newDataLength = CVPixelBufferGetDataSize(x->buffer); x->rowLength = CVPixelBufferGetBytesPerRow( x->buffer ) / (x->hap_format==JIT_GL_HAP_PF_RGB ? 3 : 4); x->target = GL_TEXTURE_RECTANGLE_EXT; if(!x->flipped) { jit_attr_setlong(x->texoutput, gensym("flip"), 1); x->flipped = 1; } x->validframe = 1; } } else { #ifdef MAC_VERSION CGSize imageSize = CVImageBufferGetEncodedSize(frame); bool flipped = CVOpenGLTextureIsFlipped(frame); x->texture = CVOpenGLTextureGetName(frame); x->useshader = 0; x->dim[0] = (t_atom_long)imageSize.width; x->dim[1] = (t_atom_long)imageSize.height; x->validframe = 1; x->target = GL_TEXTURE_RECTANGLE_ARB; if(x->flipped!=flipped) { jit_attr_setlong(x->texoutput, gensym("flip"), flipped); x->flipped = flipped; } #endif } }
/* * PsychQTGetTextureFromMovie() -- Create an OpenGL texture map from a specific videoframe from given movie object. * * win = Window pointer of onscreen window for which a OpenGL texture should be created. * moviehandle = Handle to the movie object. * checkForImage = true == Just check if new image available, false == really retrieve the image, blocking if necessary. * timeindex = When not in playback mode, this allows specification of a requested frame by presentation time. * If set to -1, or if in realtime playback mode, this parameter is ignored and the next video frame is returned. * out_texture = Pointer to the Psychtoolbox texture-record where the new texture should be stored. * presentation_timestamp = A ptr to a double variable, where the presentation timestamp of the returned frame should be stored. * * Returns true (1) on success, false (0) if no new image available, -1 if no new image available and there won't be any in future. */ int PsychQTGetTextureFromMovie(PsychWindowRecordType *win, int moviehandle, int checkForImage, double timeindex, PsychWindowRecordType *out_texture, double *presentation_timestamp) { static TimeValue myNextTimeCached = -2; static TimeValue nextFramesTimeCached = -2; TimeValue myCurrTime; TimeValue myNextTime; TimeValue nextFramesTime=0; short myFlags; OSType myTypes[1]; OSErr error = noErr; Movie theMovie; CVOpenGLTextureRef newImage = NULL; QTVisualContextRef theMoviecontext; unsigned int failcount=0; float lowerLeft[2]; float lowerRight[2]; float upperRight[2]; float upperLeft[2]; GLuint texid; Rect rect; float rate; double targetdelta, realdelta, frames; PsychRectType outRect; if (!PsychIsOnscreenWindow(win)) { PsychErrorExitMsg(PsychError_user, "Need onscreen window ptr!!!"); } // Activate OpenGL context of target window: PsychSetGLContext(win); // Explicitely disable Apple's Client storage extensions. For now they are not really useful to us. glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE); if (moviehandle < 0 || moviehandle >= PSYCH_MAX_MOVIES) { PsychErrorExitMsg(PsychError_user, "Invalid moviehandle provided."); } if ((timeindex!=-1) && (timeindex < 0 || timeindex >= 10000.0)) { PsychErrorExitMsg(PsychError_user, "Invalid timeindex provided."); } if (NULL == out_texture && !checkForImage) { PsychErrorExitMsg(PsychError_internal, "NULL-Ptr instead of out_texture ptr passed!!!"); } // Fetch references to objects we need: theMovie = movieRecordBANK[moviehandle].theMovie; theMoviecontext = movieRecordBANK[moviehandle].QTMovieContext; if (theMovie == NULL) { PsychErrorExitMsg(PsychError_user, "Invalid moviehandle provided. No movie associated with this handle."); } // Check if end of movie is reached. Rewind, if so... if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag > 0) { if (GetMovieRate(theMovie)>0) { GoToBeginningOfMovie(theMovie); } else { GoToEndOfMovie(theMovie); } } // Is movie actively playing (automatic async playback, possibly with synced sound)? // If so, then we ignore the 'timeindex' parameter, because the automatic playback // process determines which frames should be delivered to PTB when. This function will // simply wait or poll for arrival/presence of a new frame that hasn't been fetched // in previous calls. if (0 == GetMovieRate(theMovie)) { // Movie playback inactive. We are in "manual" mode: No automatic async playback, // no synced audio output. The user just wants to manually fetch movie frames into // textures for manual playback in a standard Matlab-loop. // First pass - checking for new image? if (checkForImage) { // Image for specific point in time requested? if (timeindex >= 0) { // Yes. We try to retrieve the next possible image for requested timeindex. myCurrTime = (TimeValue) ((timeindex * (double) GetMovieTimeScale(theMovie)) + 0.5f); } else { // No. We just retrieve the next frame, given the current movie time. myCurrTime = GetMovieTime(theMovie, NULL); } // Retrieve timeindex of the closest image sample after myCurrTime: myFlags = nextTimeStep + nextTimeEdgeOK; // We want the next frame in the movie's media. myTypes[0] = VisualMediaCharacteristic; // We want video samples. GetMovieNextInterestingTime(theMovie, myFlags, 1, myTypes, myCurrTime, FloatToFixed(1), &myNextTime, &nextFramesTime); error = GetMoviesError(); if (error != noErr) { PsychErrorExitMsg(PsychError_internal, "Failed to fetch texture from movie for given timeindex!"); } // Found useful event? if (myNextTime == -1) { if (PsychPrefStateGet_Verbosity() > 3) printf("PTB-WARNING: Bogus timevalue in movie track for movie %i. Trying to keep going.\n", moviehandle); // No. Just push timestamp to current time plus a little bit in the hope // this will get us unstuck: myNextTime = myCurrTime + (TimeValue) 1; nextFramesTime = (TimeValue) 0; } if (myNextTime != myNextTimeCached) { // Set movies current time to myNextTime, so the next frame will be fetched from there: SetMovieTimeValue(theMovie, myNextTime); // nextFramesTime is the timeindex to which we need to advance for retrieval of next frame: (see code below) nextFramesTime=myNextTime + nextFramesTime; if (PsychPrefStateGet_Verbosity() > 5) printf("PTB-DEBUG: Current timevalue in movie track for movie %i is %lf secs.\n", moviehandle, (double) myNextTime / (double) GetMovieTimeScale(theMovie)); if (PsychPrefStateGet_Verbosity() > 5) printf("PTB-DEBUG: Next timevalue in movie track for movie %i is %lf secs.\n", moviehandle, (double) nextFramesTime / (double) GetMovieTimeScale(theMovie)); // Cache values for 2nd pass: myNextTimeCached = myNextTime; nextFramesTimeCached = nextFramesTime; } else { // Somehow got stuck? Do nothing... if (PsychPrefStateGet_Verbosity() > 5) printf("PTB-DEBUG: Seem to be a bit stuck at timevalue [for movie %i] of %lf secs. Nudging a bit forward...\n", moviehandle, (double) myNextTime / (double) GetMovieTimeScale(theMovie)); // Nudge the timeindex a bit forware in the hope that this helps: SetMovieTimeValue(theMovie, GetMovieTime(theMovie, NULL) + 1); } } else { // This is the 2nd pass: Image fetching. Use cached values from first pass: // Caching in a static works because we're always called immediately for 2nd // pass after successfull return from 1st pass, and we're not multi-threaded, // i.e., don't need to be reentrant or thread-safe here: myNextTime = myNextTimeCached; nextFramesTime = nextFramesTimeCached; myNextTimeCached = -2; } } else { // myNextTime unavailable if in autoplayback-mode: myNextTime=-1; } // Presentation timestamp requested? if (presentation_timestamp) { // Already available? if (myNextTime==-1) { // Retrieve the exact presentation timestamp of the retrieved frame (in movietime): myFlags = nextTimeStep + nextTimeEdgeOK; // We want the next frame in the movie's media. myTypes[0] = VisualMediaCharacteristic; // We want video samples. // We search backward for the closest available image for the current time. Either we get the current time // if we happen to fetch a frame exactly when it becomes ready, or we get a bit earlier timestamp, which is // the optimal presentation timestamp for this frame: GetMovieNextInterestingTime(theMovie, myFlags, 1, myTypes, GetMovieTime(theMovie, NULL), FloatToFixed(-1), &myNextTime, NULL); } // Convert pts (in Quicktime ticks) to pts in seconds since start of movie and return it: *presentation_timestamp = (double) myNextTime / (double) GetMovieTimeScale(theMovie); } // Allow quicktime visual context task to do its internal bookkeeping and cleanup work: if (theMoviecontext) QTVisualContextTask(theMoviecontext); // Perform decompress-operation: if (checkForImage) MoviesTask(theMovie, 0); // Should we just check for new image? If so, just return availability status: if (checkForImage) { if (PSYCH_USE_QT_GWORLDS) { // We use GWorlds. In this case we either suceed immediately due to the // synchronous nature of GWorld rendering, or we fail completely at end // of non-looping movie: if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag == 0) { // No new frame available and there won't be any in the future, because this is a non-looping // movie that has reached its end. return(-1); } // Is this the special case of a movie without video, but only sound? In that case, // we always return a 'false' because there ain't no image to return. if (movieRecordBANK[moviehandle].QTMovieGWorld == NULL) return(false); // Success! return(true); } // Code which uses QTVisualContextTasks... if (QTVisualContextIsNewImageAvailable(theMoviecontext, NULL)) { // New frame ready! return(true); } else if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag == 0) { // No new frame available and there won't be any in the future, because this is a non-looping // movie that has reached its end. return(-1); } else { // No new frame available yet: return(false); } } if (!PSYCH_USE_QT_GWORLDS) { // Blocking wait-code for non-GWorld mode: // Try up to 1000 iterations for arrival of requested image data in wait-mode: failcount=0; while ((failcount < 1000) && !QTVisualContextIsNewImageAvailable(theMoviecontext, NULL)) { PsychWaitIntervalSeconds(0.005); MoviesTask(theMovie, 0); failcount++; } // No new frame available and there won't be any in the future, because this is a non-looping // movie that has reached its end. if ((failcount>=1000) && IsMovieDone(theMovie) && (movieRecordBANK[moviehandle].loopflag == 0)) { return(-1); } // Fetch new OpenGL texture with the new movie image frame: error = QTVisualContextCopyImageForTime(theMoviecontext, kCFAllocatorDefault, NULL, &newImage); if ((error!=noErr) || newImage == NULL) { PsychErrorExitMsg(PsychError_internal, "OpenGL<->Quicktime texture fetch failed!!!"); } // Disable client storage, if it was enabled: glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE); // Build a standard PTB texture record: CVOpenGLTextureGetCleanTexCoords (newImage, lowerLeft, lowerRight, upperRight, upperLeft); texid = CVOpenGLTextureGetName(newImage); // Assign texture rectangle: PsychMakeRect(outRect, upperLeft[0], upperLeft[1], lowerRight[0], lowerRight[1]); // Set texture orientation as if it were an inverted Offscreen window: Upside-down. out_texture->textureOrientation = (CVOpenGLTextureIsFlipped(newImage)) ? 3 : 4; // Assign OpenGL texture id: out_texture->textureNumber = texid; // Store special texture object as part of the PTB texture record: out_texture->targetSpecific.QuickTimeGLTexture = newImage; } else { // Synchronous texture fetch code for GWorld rendering mode: // At this point, the GWorld should contain the source image for creating a // standard OpenGL texture: // Disable client storage, if it was enabled: glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE); // Build a standard PTB texture record: // Assign texture rectangle: GetMovieBox(theMovie, &rect); // Hack: Need to extend rect by 4 pixels, because GWorlds are 4 pixels-aligned via // image row padding: rect.right = rect.right + 4; PsychMakeRect(out_texture->rect, rect.left, rect.top, rect.right, rect.bottom); // Set NULL - special texture object as part of the PTB texture record: out_texture->targetSpecific.QuickTimeGLTexture = NULL; // Set texture orientation as if it were an inverted Offscreen window: Upside-down. out_texture->textureOrientation = 3; // Setup a pointer to our GWorld as texture data pointer: out_texture->textureMemorySizeBytes = 0; // Quicktime textures are aligned on 4 Byte boundaries: out_texture->textureByteAligned = 4; // Lock GWorld: if(!LockPixels(GetGWorldPixMap(movieRecordBANK[moviehandle].QTMovieGWorld))) { // Locking surface failed! We abort. PsychErrorExitMsg(PsychError_internal, "PsychQTGetTextureFromMovie(): Locking GWorld pixmap surface failed!!!"); } // This will retrieve an OpenGL compatible pointer to the GWorlds pixel data and assign it to our texmemptr: out_texture->textureMemory = (GLuint*) GetPixBaseAddr(GetGWorldPixMap(movieRecordBANK[moviehandle].QTMovieGWorld)); // Let PsychCreateTexture() do the rest of the job of creating, setting up and // filling an OpenGL texture with GWorlds content: PsychCreateTexture(out_texture); // Undo hack from above after texture creation: Now we need the real width of the // texture for proper texture coordinate assignments in drawing code et al. rect.right = rect.right - 4; PsychMakeRect(outRect, rect.left, rect.top, rect.right, rect.bottom); // Unlock GWorld surface. We do a glFinish() before, for safety reasons... //glFinish(); UnlockPixels(GetGWorldPixMap(movieRecordBANK[moviehandle].QTMovieGWorld)); // Ready to use the texture... We're done. } // Normalize texture rectangle and assign it: PsychNormalizeRect(outRect, out_texture->rect); rate = FixedToFloat(GetMovieRate(theMovie)); // Detection of dropped frames: This is a heuristic. We'll see how well it works out... if (rate && presentation_timestamp) { // Try to check for dropped frames in playback mode: // Expected delta between successive presentation timestamps: targetdelta = 1.0f / (movieRecordBANK[moviehandle].fps * rate); // Compute real delta, given rate and playback direction: if (rate>0) { realdelta = *presentation_timestamp - movieRecordBANK[moviehandle].last_pts; if (realdelta<0) realdelta = 0; } else { realdelta = -1.0 * (*presentation_timestamp - movieRecordBANK[moviehandle].last_pts); if (realdelta<0) realdelta = 0; } frames = realdelta / targetdelta; // Dropped frames? if (frames > 1 && movieRecordBANK[moviehandle].last_pts>=0) { movieRecordBANK[moviehandle].nr_droppedframes += (int) (frames - 1 + 0.5); } movieRecordBANK[moviehandle].last_pts = *presentation_timestamp; } // Manually advance movie time, if in fetch mode: if (0 == GetMovieRate(theMovie)) { // We are in manual fetch mode: Need to manually advance movie time to next // media sample: if (nextFramesTime == myNextTime) { // Invalid value? Try to hack something that gets us unstuck: myNextTime = GetMovieTime(theMovie, NULL); nextFramesTime = myNextTime + (TimeValue) 1; } SetMovieTimeValue(theMovie, nextFramesTime); } // Check if end of movie is reached. Rewind, if so... if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag > 0) { if (GetMovieRate(theMovie)>0) { GoToBeginningOfMovie(theMovie); } else { GoToEndOfMovie(theMovie); } } return(TRUE); }