PsychError SCREENSetOpenGLTextureFromMemPointer(void) { PsychWindowRecordType *windowRecord, *textureRecord; int w, h, d, testarg, upsidedown, glinternalformat, glexternaltype, glexternalformat; double doubleMemPtr; GLenum target = 0; w=h=d=-1; doubleMemPtr = 0; upsidedown = 0; //all subfunctions should have these two lines. PsychPushHelp(useString, synopsisString, seeAlsoString); if(PsychIsGiveHelp()){PsychGiveHelp();return(PsychError_none);}; PsychErrorExit(PsychCapNumInputArgs(11)); // The maximum number of inputs PsychErrorExit(PsychRequireNumInputArgs(5)); // The required number of inputs PsychErrorExit(PsychCapNumOutputArgs(2)); // The maximum number of outputs // Get the window record from the window record argument and get info from the window record PsychAllocInWindowRecordArg(kPsychUseDefaultArgPosition, TRUE, &windowRecord); // Get the texture record from the texture record argument. // Check if either none ( [] or '' ) or the special value zero was // provided as Psychtoolbox textureHandle. In that case, we create a new // empty texture record instead of reusing an existing one. testarg=0; PsychCopyInIntegerArg(2, FALSE, &testarg); if (testarg==0) { // No valid textureHandle provided. Create a new empty textureRecord. PsychCreateWindowRecord(&textureRecord); textureRecord->windowType=kPsychTexture; textureRecord->screenNumber = windowRecord->screenNumber; textureRecord->targetSpecific.contextObject = windowRecord->targetSpecific.contextObject; textureRecord->targetSpecific.deviceContext = windowRecord->targetSpecific.deviceContext; textureRecord->targetSpecific.glusercontextObject = windowRecord->targetSpecific.glusercontextObject; textureRecord->colorRange = windowRecord->colorRange; // Copy imaging mode flags from parent: textureRecord->imagingMode = windowRecord->imagingMode; // Mark it valid and return handle to userspace: PsychSetWindowRecordValid(textureRecord); } else { // None of the special values provided. We assume its a handle to a valid // and existing PTB texture and try to retrieve the textureRecord: PsychAllocInWindowRecordArg(2, TRUE, &textureRecord); } // Is it a textureRecord? if (!PsychIsTexture(textureRecord)) { PsychErrorExitMsg(PsychError_user, "You tried to set texture information on something else than a texture!"); } // Query double-encoded memory pointer: PsychCopyInDoubleArg(3, TRUE, &doubleMemPtr); // Query width: PsychCopyInIntegerArg(4, TRUE, &w); // Query height: PsychCopyInIntegerArg(5, TRUE, &h); // Query depth: PsychCopyInIntegerArg(6, TRUE, &d); // Query (optional) upsidedown - flag: PsychCopyInIntegerArg(7, FALSE, &upsidedown); // Query (optional) OpenGL texture target: PsychCopyInIntegerArg(8, FALSE, (int*) &target); // Query (optional) full format spec: glinternalformat = 0; PsychCopyInIntegerArg(9, FALSE, &glinternalformat); if (glinternalformat>0) { // Ok copy the (now non-optional) remaining format spec: PsychCopyInIntegerArg(10, TRUE, &glexternalformat); PsychCopyInIntegerArg(11, TRUE, &glexternaltype); } // Safety checks: if (doubleMemPtr == 0) { PsychErrorExitMsg(PsychError_user, "You tried to set invalid (NULL) imagePtr."); } if (w<=0) { PsychErrorExitMsg(PsychError_user, "You tried to set invalid (negative) texture width."); } if (h<=0) { PsychErrorExitMsg(PsychError_user, "You tried to set invalid (negative) texture height."); } if (d<=0) { PsychErrorExitMsg(PsychError_user, "You tried to set invalid (negative) texture depth."); } if (d>4) { PsychErrorExitMsg(PsychError_user, "You tried to set invalid (greater than four) texture depth."); } if (target!=0 && target!=GL_TEXTURE_RECTANGLE_EXT && target!=GL_TEXTURE_2D) { PsychErrorExitMsg(PsychError_user, "You tried to set invalid texture target."); } // Activate OpenGL rendering context of windowRecord and make it the active drawing target: PsychSetGLContext(windowRecord); PsychSetDrawingTarget(windowRecord); PsychTestForGLErrors(); // Ok, setup texture record for texture: PsychInitWindowRecordTextureFields(textureRecord); textureRecord->depth = d * 8; textureRecord->nrchannels = d; PsychMakeRect(textureRecord->rect, 0, 0, w, h); // Override texture target, if one was provided: if (target!=0) textureRecord->texturetarget = target; // Orientation is normally set to 2 - like an upright Offscreen window texture. // If upsidedown flag is set, then we do 3 - an upside down Offscreen window texture. textureRecord->textureOrientation = (upsidedown>0) ? 3 : 2; if (glinternalformat!=0) { textureRecord->textureinternalformat = glinternalformat; textureRecord->textureexternalformat = glexternalformat; textureRecord->textureexternaltype = glexternaltype; } // Setting memsize to zero prevents unwanted free() operation in PsychDeleteTexture... textureRecord->textureMemorySizeBytes = 0; // This will retrieve an OpenGL compatible pointer to the raw pixel data and assign it to our texmemptr: textureRecord->textureMemory = (GLuint*) PsychDoubleToPtr(doubleMemPtr); // printf("InTexPtr %p , %.20e", PsychDoubleToPtr(doubleMemPtr), doubleMemPtr); // Let PsychCreateTexture() do the rest of the job of creating, setting up and // filling an OpenGL texture with memory buffers image content: PsychCreateTexture(textureRecord); // Return new (or old) PTB textureHandle for this texture: PsychCopyOutDoubleArg(1, FALSE, textureRecord->windowIndex); PsychCopyOutRectArg(2, FALSE, textureRecord->rect); // Done. return(PsychError_none); }
PsychError SCREENMakeTexture(void) { int ix; PsychWindowRecordType *textureRecord; PsychWindowRecordType *windowRecord; PsychRectType rect; Boolean isImageMatrixBytes, isImageMatrixDoubles; int numMatrixPlanes, xSize, ySize, iters; unsigned char *byteMatrix; double *doubleMatrix; GLuint *texturePointer; GLubyte *texturePointer_b; GLfloat *texturePointer_f; double *rp, *gp, *bp, *ap; GLubyte *rpb, *gpb, *bpb, *apb; int usepoweroftwo, usefloatformat, assume_texorientation, textureShader; double optimized_orientation; Boolean bigendian; // Detect endianity (byte-order) of machine: ix=255; rpb=(GLubyte*) &ix; bigendian = ( *rpb == 255 ) ? FALSE : TRUE; ix = 0; rpb = NULL; if(PsychPrefStateGet_DebugMakeTexture()) //MARK #1 StoreNowTime(); //all subfunctions should have these two lines. PsychPushHelp(useString, synopsisString, seeAlsoString); if(PsychIsGiveHelp()){PsychGiveHelp();return(PsychError_none);}; //Get the window structure for the onscreen window. It holds the onscreein GL context which we will need in the //final step when we copy the texture from system RAM onto the screen. PsychErrorExit(PsychCapNumInputArgs(7)); PsychErrorExit(PsychRequireNumInputArgs(2)); PsychErrorExit(PsychCapNumOutputArgs(1)); //get the window record from the window record argument and get info from the window record PsychAllocInWindowRecordArg(kPsychUseDefaultArgPosition, TRUE, &windowRecord); if((windowRecord->windowType!=kPsychDoubleBufferOnscreen) && (windowRecord->windowType!=kPsychSingleBufferOnscreen)) PsychErrorExitMsg(PsychError_user, "MakeTexture called on something else than a onscreen window"); // Get optional texture orientation flag: assume_texorientation = 0; PsychCopyInIntegerArg(6, FALSE, &assume_texorientation); // Get optional texture shader handle: textureShader = 0; PsychCopyInIntegerArg(7, FALSE, &textureShader); //get the argument and sanity check it. isImageMatrixBytes=PsychAllocInUnsignedByteMatArg(2, kPsychArgAnything, &ySize, &xSize, &numMatrixPlanes, &byteMatrix); isImageMatrixDoubles=PsychAllocInDoubleMatArg(2, kPsychArgAnything, &ySize, &xSize, &numMatrixPlanes, &doubleMatrix); if(numMatrixPlanes > 4) PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Specified image matrix exceeds maximum depth of 4 layers"); if(ySize<1 || xSize <1) PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Specified image matrix must be at least 1 x 1 pixels in size"); if(! (isImageMatrixBytes || isImageMatrixDoubles)) PsychErrorExitMsg(PsychError_user, "Illegal argument type"); //not likely. // Is this a special image matrix which is already pre-transposed to fit our optimal format? if (assume_texorientation == 2) { // Yes. Swap xSize and ySize to take this into account: ix = xSize; xSize = ySize; ySize = ix; ix = 0; } // Build defining rect for this texture: PsychMakeRect(rect, 0, 0, xSize, ySize); // Copy in texture preferred draw orientation hint. We default to zero degrees, if // not provided. // This parameter is not yet used. It is silently ignorerd for now... optimized_orientation = 0; PsychCopyInDoubleArg(3, FALSE, &optimized_orientation); // Copy in special creation mode flag: It defaults to zero. If set to 1 then we // always create a power-of-two GL_TEXTURE_2D texture. This is useful if one wants // to create and use drifting gratings with no effort - texture wrapping is only available // for GL_TEXTURE_2D, not for non-pot types. It is also useful if the texture is to be // exported to external OpenGL code to simplify tex coords assignments. usepoweroftwo=0; PsychCopyInIntegerArg(4, FALSE, &usepoweroftwo); // Check if size constraints are fullfilled for power-of-two mode: if (usepoweroftwo & 1) { for(ix = 1; ix < xSize; ix*=2); if (ix!=xSize) { PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Power-of-two texture requested but width of imageMatrix is not a power of two!"); } for(ix = 1; ix < ySize; ix*=2); if (ix!=ySize) { PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Power-of-two texture requested but height of imageMatrix is not a power of two!"); } } // Check if creation of a floating point texture is requested? We default to non-floating point, // standard 8 bpc textures if this parameter is not provided. usefloatformat = 0; PsychCopyInIntegerArg(5, FALSE, &usefloatformat); if (usefloatformat<0 || usefloatformat>2) PsychErrorExitMsg(PsychError_user, "Invalid value for 'floatprecision' parameter provided! Valid values are 0 for 8bpc int, 1 for 16bpc float or 2 for 32bpc float."); if (usefloatformat && !isImageMatrixDoubles) { // Floating point texture requested. We only support this if our input is a double matrix, not // for uint8 matrices - converting them to float precision would be just a waste of ressources // without any benefit for precision. PsychErrorExitMsg(PsychError_user, "Creation of a floating point precision texture requested, but uint8 matrix provided! Only double matrices are acceptable for this mode."); } //Create a texture record. Really just a window record adapted for textures. PsychCreateWindowRecord(&textureRecord); //this also fills the window index field. textureRecord->windowType=kPsychTexture; // MK: We need to assign the screen number of the onscreen-window, so PsychCreateTexture() // can query the size of the screen/onscreen-window... textureRecord->screenNumber=windowRecord->screenNumber; textureRecord->depth=32; PsychCopyRect(textureRecord->rect, rect); //Allocate the texture memory and copy the MATLAB matrix into the texture memory. // MK: We only allocate the amount really needed for given format, aka numMatrixPlanes - Bytes per pixel. if (usefloatformat) { // Allocate a double for each color component and pixel: textureRecord->textureMemorySizeBytes= sizeof(double) * numMatrixPlanes * xSize * ySize; } else { // Allocate one byte per color component and pixel: textureRecord->textureMemorySizeBytes= numMatrixPlanes * xSize * ySize; } // MK: Allocate memory page-aligned... -> Helps Apple texture range extensions et al. if(PsychPrefStateGet_DebugMakeTexture()) //MARK #2 StoreNowTime(); textureRecord->textureMemory=malloc(textureRecord->textureMemorySizeBytes); if(PsychPrefStateGet_DebugMakeTexture()) //MARK #3 StoreNowTime(); texturePointer=textureRecord->textureMemory; // Does script explicitely request usage of a GL_TEXTURE_2D power-of-two texture? if (usepoweroftwo & 1) { // Enforce creation as a power-of-two texture: textureRecord->texturetarget=GL_TEXTURE_2D; } // Now the conversion routines that convert Matlab/Octave matrices into memory // buffers suitable for OpenGL: if (usefloatformat) { // Conversion routines for HDR 16 bpc or 32 bpc textures -- Slow path. // Our input is always double matrices... iters = xSize * ySize; // Our input buffer is always of GL_FLOAT precision: textureRecord->textureexternaltype = GL_FLOAT; texturePointer_f=(GLfloat*) texturePointer; if(numMatrixPlanes==1) { for(ix=0;ix<iters;ix++){ *(texturePointer_f++)= (GLfloat) *(doubleMatrix++); } textureRecord->depth=(usefloatformat==1) ? 16 : 32; textureRecord->textureinternalformat = (usefloatformat==1) ? GL_LUMINANCE_FLOAT16_APPLE : GL_LUMINANCE_FLOAT32_APPLE; textureRecord->textureexternalformat = GL_LUMINANCE; } if(numMatrixPlanes==2) { rp=(double*) ((psych_uint64) doubleMatrix); ap=(double*) ((psych_uint64) doubleMatrix + (psych_uint64) iters*sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_f++)= (GLfloat) *(rp++); *(texturePointer_f++)= (GLfloat) *(ap++); } textureRecord->depth=(usefloatformat==1) ? 32 : 64; textureRecord->textureinternalformat = (usefloatformat==1) ? GL_LUMINANCE_ALPHA_FLOAT16_APPLE : GL_LUMINANCE_ALPHA_FLOAT32_APPLE; textureRecord->textureexternalformat = GL_LUMINANCE_ALPHA; } if(numMatrixPlanes==3) { rp=(double*) ((psych_uint64) doubleMatrix); gp=(double*) ((psych_uint64) doubleMatrix + (psych_uint64) iters*sizeof(double)); bp=(double*) ((psych_uint64) gp + (psych_uint64) iters*sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_f++)= (GLfloat) *(rp++); *(texturePointer_f++)= (GLfloat) *(gp++); *(texturePointer_f++)= (GLfloat) *(bp++); } textureRecord->depth=(usefloatformat==1) ? 48 : 96; textureRecord->textureinternalformat = (usefloatformat==1) ? GL_RGB_FLOAT16_APPLE : GL_RGB_FLOAT32_APPLE; textureRecord->textureexternalformat = GL_RGB; } if(numMatrixPlanes==4) { rp=(double*) ((psych_uint64) doubleMatrix); gp=(double*) ((psych_uint64) doubleMatrix + (psych_uint64) iters*sizeof(double)); bp=(double*) ((psych_uint64) gp + (psych_uint64) iters*sizeof(double)); ap=(double*) ((psych_uint64) bp + (psych_uint64) iters*sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_f++)= (GLfloat) *(rp++); *(texturePointer_f++)= (GLfloat) *(gp++); *(texturePointer_f++)= (GLfloat) *(bp++); *(texturePointer_f++)= (GLfloat) *(ap++); } textureRecord->depth=(usefloatformat==1) ? 64 : 128; textureRecord->textureinternalformat = (usefloatformat==1) ? GL_RGBA_FLOAT16_APPLE : GL_RGBA_FLOAT32_APPLE; textureRecord->textureexternalformat = GL_RGBA; } // This is a special workaround for bugs in FLOAT16 texture creation on Mac OS/X 10.4.x and 10.5.x. // The OpenGL fails to properly flush very small values (< 1e-9) to zero when creating a FLOAT16 // type texture. Instead it seems to initialize with trash data, corrupting the texture. // Therefore, if FLOAT16 texture creation is requested, we loop over the whole input buffer and // set all values with magnitude smaller than 1e-9 to zero. Better safe than sorry... if(usefloatformat==1) { texturePointer_f=(GLfloat*) texturePointer; iters = iters * numMatrixPlanes; for(ix=0; ix<iters; ix++, texturePointer_f++) if(fabs((double) *texturePointer_f) < 1e-9) { *texturePointer_f = 0.0; } } // End of HDR conversion code... } else { // Standard LDR texture 8 bpc conversion routines -- Fast path. // Improved implementation: Takes 13 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==1){ texturePointer_b=(GLubyte*) texturePointer; iters=xSize*ySize; for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(doubleMatrix++); } textureRecord->depth=8; } // Improved version: Takes 3 ms on a 800x800 texture... // NB: Implementing memcpy manually by a for-loop takes 10 ms! This is a huge difference. // -> That's because memcpy on MacOS-X is implemented with hand-coded, highly tuned Assembler code for PowerPC. // -> It's always wise to use system-routines if available, instead of coding it by yourself! if(isImageMatrixBytes && numMatrixPlanes==1){ memcpy((void*) texturePointer, (void*) byteMatrix, xSize*ySize); textureRecord->depth=8; } // New version: Takes 33 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==2){ texturePointer_b=(GLubyte*) texturePointer; iters=xSize*ySize; rp=(double*) ((psych_uint64) doubleMatrix); ap=(double*) ((psych_uint64) doubleMatrix + (psych_uint64) iters*sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(ap++); } textureRecord->depth=16; } // New version: Takes 20 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==2){ texturePointer_b=(GLubyte*) texturePointer; iters=xSize*ySize; rpb=(GLubyte*) ((psych_uint64) byteMatrix); apb=(GLubyte*) ((psych_uint64) byteMatrix + (psych_uint64) iters); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(rpb++); *(texturePointer_b++)= *(apb++); } textureRecord->depth=16; } // Improved version: Takes 43 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==3){ texturePointer_b=(GLubyte*) texturePointer; iters=xSize*ySize; rp=(double*) ((psych_uint64) doubleMatrix); gp=(double*) ((psych_uint64) doubleMatrix + (psych_uint64) iters*sizeof(double)); bp=(double*) ((psych_uint64) gp + (psych_uint64) iters*sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(gp++); *(texturePointer_b++)= (GLubyte) *(bp++); } textureRecord->depth=24; } // Improved version: Takes 25 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==3){ texturePointer_b=(GLubyte*) texturePointer; iters=xSize*ySize; rpb=(GLubyte*) ((psych_uint64) byteMatrix); gpb=(GLubyte*) ((psych_uint64) byteMatrix + (psych_uint64) iters); bpb=(GLubyte*) ((psych_uint64) gpb + (psych_uint64) iters); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(rpb++); *(texturePointer_b++)= *(gpb++); *(texturePointer_b++)= *(bpb++); } textureRecord->depth=24; } // Improved version: Takes 55 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==4){ texturePointer_b=(GLubyte*) texturePointer; iters=xSize*ySize; rp=(double*) ((psych_uint64) doubleMatrix); gp=(double*) ((psych_uint64) doubleMatrix + (psych_uint64) iters*sizeof(double)); bp=(double*) ((psych_uint64) gp + (psych_uint64) iters*sizeof(double)); ap=(double*) ((psych_uint64) bp + (psych_uint64) iters*sizeof(double)); if (bigendian) { // Code for big-endian machines like PowerPC: for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(ap++); *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(gp++); *(texturePointer_b++)= (GLubyte) *(bp++); } } else { // Code for little-endian machines like Intel Pentium: for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(bp++); *(texturePointer_b++)= (GLubyte) *(gp++); *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(ap++); } } textureRecord->depth=32; } // Improved version: Takes 33 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==4){ texturePointer_b=(GLubyte*) texturePointer; iters=xSize*ySize; rpb=(GLubyte*) ((psych_uint64) byteMatrix); gpb=(GLubyte*) ((psych_uint64) byteMatrix + (psych_uint64) iters); bpb=(GLubyte*) ((psych_uint64) gpb + (psych_uint64) iters); apb=(GLubyte*) ((psych_uint64) bpb + (psych_uint64) iters); if (bigendian) { // Code for big-endian machines like PowerPC: for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(apb++); *(texturePointer_b++)= *(rpb++); *(texturePointer_b++)= *(gpb++); *(texturePointer_b++)= *(bpb++); } } else { // Code for little-endian machines like Intel Pentium: for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(bpb++); *(texturePointer_b++)= *(gpb++); *(texturePointer_b++)= *(rpb++); *(texturePointer_b++)= *(apb++); } } textureRecord->depth=32; } } // End of 8 bpc texture conversion code (fast-path for LDR textures) // The memory buffer now contains our texture data in a format ready to submit to OpenGL. // Assign parent window and copy its inheritable properties: PsychAssignParentWindow(textureRecord, windowRecord); // Texture orientation is zero aka transposed aka non-renderswapped. textureRecord->textureOrientation = ((assume_texorientation != 2) && (assume_texorientation != 3)) ? 0 : 2; // This is our best guess about the number of image channels: textureRecord->nrchannels = numMatrixPlanes; // Let's create and bind a new texture object and fill it with our new texture data. PsychCreateTexture(textureRecord); // Assign GLSL filter-/lookup-shaders if needed: PsychAssignHighPrecisionTextureShaders(textureRecord, windowRecord, usefloatformat, (usepoweroftwo & 2) ? 1 : 0); // User specified override shader for this texture provided? This is useful for // basic image processing and procedural texture shading: if (textureShader!=0) { // Assign provided shader as filtershader to this texture: We negate it so // that the texture blitter routines know this is a custom shader, not our // built in filter shader: textureRecord->textureFilterShader = -1 * textureShader; } // Texture ready. Mark it valid and return handle to userspace: PsychSetWindowRecordValid(textureRecord); PsychCopyOutDoubleArg(1, FALSE, textureRecord->windowIndex); // Swapping the texture to upright orientation requested? if (assume_texorientation == 1) { // Transform sourceRecord source texture into a normalized, upright texture if it isn't already in // that format. We require this standard orientation for simplified shader design. PsychSetShader(windowRecord, 0); PsychNormalizeTextureOrientation(textureRecord); } // Shall the texture be finally declared "normally oriented"? // This is either due to explicit renderswapping if assume_textureorientation == 1, // or because it was already pretransposed in Matlab/Octave if assume_textureorientation == 2, // or because user space tells us the texture is isotropic if assume_textureorientation == 3. if (assume_texorientation > 0) { // Yes. Label it as such: textureRecord->textureOrientation = 2; } if(PsychPrefStateGet_DebugMakeTexture()) //MARK #4 StoreNowTime(); return(PsychError_none); }
/* * PsychGSGetTextureFromMovie() -- Create an OpenGL texture map from a specific videoframe from given movie object. * * win = Window pointer of onscreen window for which a OpenGL texture should be created. * moviehandle = Handle to the movie object. * checkForImage = true == Just check if new image available, false == really retrieve the image, blocking if necessary. * timeindex = When not in playback mode, this allows specification of a requested frame by presentation time. * If set to -1, or if in realtime playback mode, this parameter is ignored and the next video frame is returned. * out_texture = Pointer to the Psychtoolbox texture-record where the new texture should be stored. * presentation_timestamp = A ptr to a double variable, where the presentation timestamp of the returned frame should be stored. * * Returns true (1) on success, false (0) if no new image available, -1 if no new image available and there won't be any in future. */ int PsychGSGetTextureFromMovie(PsychWindowRecordType *win, int moviehandle, int checkForImage, double timeindex, PsychWindowRecordType *out_texture, double *presentation_timestamp) { GstElement *theMovie; unsigned int failcount=0; double rate; double targetdelta, realdelta, frames; // PsychRectType outRect; GstBuffer *videoBuffer = NULL; gint64 bufferIndex; double deltaT = 0; GstEvent *event; if (!PsychIsOnscreenWindow(win)) { PsychErrorExitMsg(PsychError_user, "Need onscreen window ptr!!!"); } if (moviehandle < 0 || moviehandle >= PSYCH_MAX_MOVIES) { PsychErrorExitMsg(PsychError_user, "Invalid moviehandle provided."); } if ((timeindex!=-1) && (timeindex < 0 || timeindex >= 10000.0)) { PsychErrorExitMsg(PsychError_user, "Invalid timeindex provided."); } if (NULL == out_texture && !checkForImage) { PsychErrorExitMsg(PsychError_internal, "NULL-Ptr instead of out_texture ptr passed!!!"); } // Fetch references to objects we need: theMovie = movieRecordBANK[moviehandle].theMovie; if (theMovie == NULL) { PsychErrorExitMsg(PsychError_user, "Invalid moviehandle provided. No movie associated with this handle."); } // Allow context task to do its internal bookkeeping and cleanup work: PsychGSProcessMovieContext(movieRecordBANK[moviehandle].MovieContext, FALSE); // If this is a pure audio "movie" with no video tracks, we always return failed, // as those certainly don't have movie frames associated. if (movieRecordBANK[moviehandle].nrVideoTracks == 0) return((checkForImage) ? -1 : FALSE); // Get current playback rate: rate = movieRecordBANK[moviehandle].rate; // Is movie actively playing (automatic async playback, possibly with synced sound)? // If so, then we ignore the 'timeindex' parameter, because the automatic playback // process determines which frames should be delivered to PTB when. This function will // simply wait or poll for arrival/presence of a new frame that hasn't been fetched // in previous calls. if (0 == rate) { // Movie playback inactive. We are in "manual" mode: No automatic async playback, // no synced audio output. The user just wants to manually fetch movie frames into // textures for manual playback in a standard Matlab-loop. // First pass - checking for new image? if (checkForImage) { // Image for specific point in time requested? if (timeindex >= 0) { // Yes. We try to retrieve the next possible image for requested timeindex. // Seek to target timeindex: PsychGSSetMovieTimeIndex(moviehandle, timeindex, FALSE); } else { // No. We just retrieve the next frame, given the current position. // Nothing to do so far... } // Check for frame availability happens down there in the shared check code... } } // Should we just check for new image? If so, just return availability status: if (checkForImage) { PsychLockMutex(&movieRecordBANK[moviehandle].mutex); if ((((0 != rate) && movieRecordBANK[moviehandle].frameAvail) || ((0 == rate) && movieRecordBANK[moviehandle].preRollAvail)) && !gst_app_sink_is_eos(GST_APP_SINK(movieRecordBANK[moviehandle].videosink))) { // New frame available. Unlock and report success: //printf("PTB-DEBUG: NEW FRAME %d\n", movieRecordBANK[moviehandle].frameAvail); PsychUnlockMutex(&movieRecordBANK[moviehandle].mutex); return(true); } // Is this the special case of a movie without video, but only sound? In that case // we always return a 'false' because there ain't no image to return. We check this // indirectly - If the imageBuffer is NULL then the video callback hasn't been called. if (oldstyle && (NULL == movieRecordBANK[moviehandle].imageBuffer)) { PsychUnlockMutex(&movieRecordBANK[moviehandle].mutex); return(false); } // None available. Any chance there will be one in the future? if (gst_app_sink_is_eos(GST_APP_SINK(movieRecordBANK[moviehandle].videosink)) && movieRecordBANK[moviehandle].loopflag == 0) { // No new frame available and there won't be any in the future, because this is a non-looping // movie that has reached its end. PsychUnlockMutex(&movieRecordBANK[moviehandle].mutex); return(-1); } else { // No new frame available yet: PsychUnlockMutex(&movieRecordBANK[moviehandle].mutex); //printf("PTB-DEBUG: NO NEW FRAME\n"); return(false); } } // If we reach this point, then an image fetch is requested. If no new data // is available we shall block: PsychLockMutex(&movieRecordBANK[moviehandle].mutex); // printf("PTB-DEBUG: Blocking fetch start %d\n", movieRecordBANK[moviehandle].frameAvail); if (((0 != rate) && !movieRecordBANK[moviehandle].frameAvail) || ((0 == rate) && !movieRecordBANK[moviehandle].preRollAvail)) { // No new frame available. Perform a blocking wait: PsychTimedWaitCondition(&movieRecordBANK[moviehandle].condition, &movieRecordBANK[moviehandle].mutex, 10.0); // Recheck: if (((0 != rate) && !movieRecordBANK[moviehandle].frameAvail) || ((0 == rate) && !movieRecordBANK[moviehandle].preRollAvail)) { // Game over! Wait timed out after 10 secs. PsychUnlockMutex(&movieRecordBANK[moviehandle].mutex); printf("PTB-ERROR: No new video frame received after timeout of 10 seconds! Something's wrong. Aborting fetch.\n"); return(FALSE); } // At this point we should have at least one frame available. // printf("PTB-DEBUG: After blocking fetch start %d\n", movieRecordBANK[moviehandle].frameAvail); } // We're here with at least one frame available and the mutex lock held. // Preroll case is simple: movieRecordBANK[moviehandle].preRollAvail = 0; // Perform texture fetch & creation: if (oldstyle) { // Reset frame available flag: movieRecordBANK[moviehandle].frameAvail = 0; // This will retrieve an OpenGL compatible pointer to the pixel data and assign it to our texmemptr: out_texture->textureMemory = (GLuint*) movieRecordBANK[moviehandle].imageBuffer; } else { // Active playback mode? if (0 != rate) { // Active playback mode: One less frame available after our fetch: movieRecordBANK[moviehandle].frameAvail--; if (PsychPrefStateGet_Verbosity()>4) printf("PTB-DEBUG: Pulling from videosink, %d buffers avail...\n", movieRecordBANK[moviehandle].frameAvail); // Clamp frameAvail to queue lengths: if ((int) gst_app_sink_get_max_buffers(GST_APP_SINK(movieRecordBANK[moviehandle].videosink)) < movieRecordBANK[moviehandle].frameAvail) { movieRecordBANK[moviehandle].frameAvail = gst_app_sink_get_max_buffers(GST_APP_SINK(movieRecordBANK[moviehandle].videosink)); } // This will pull the oldest video buffer from the videosink. It would block if none were available, // but that won't happen as we wouldn't reach this statement if none were available. It would return // NULL if the stream would be EOS or the pipeline off, but that shouldn't ever happen: videoBuffer = gst_app_sink_pull_buffer(GST_APP_SINK(movieRecordBANK[moviehandle].videosink)); } else { // Passive fetch mode: Use prerolled buffers after seek: // These are available even after eos... videoBuffer = gst_app_sink_pull_preroll(GST_APP_SINK(movieRecordBANK[moviehandle].videosink)); } // We can unlock early, thanks to videosink's internal buffering: PsychUnlockMutex(&movieRecordBANK[moviehandle].mutex); if (videoBuffer) { // Assign pointer to videoBuffer's data directly: Avoids one full data copy compared to oldstyle method. out_texture->textureMemory = (GLuint*) GST_BUFFER_DATA(videoBuffer); // Assign pts presentation timestamp in pipeline stream time and convert to seconds: movieRecordBANK[moviehandle].pts = (double) GST_BUFFER_TIMESTAMP(videoBuffer) / (double) 1e9; if (GST_CLOCK_TIME_IS_VALID(GST_BUFFER_DURATION(videoBuffer))) deltaT = (double) GST_BUFFER_DURATION(videoBuffer) / (double) 1e9; bufferIndex = GST_BUFFER_OFFSET(videoBuffer); } else { printf("PTB-ERROR: No new video frame received in gst_app_sink_pull_buffer! Something's wrong. Aborting fetch.\n"); return(FALSE); } if (PsychPrefStateGet_Verbosity()>4) printf("PTB-DEBUG: ...done.\n"); } // Assign presentation_timestamp: if (presentation_timestamp) *presentation_timestamp = movieRecordBANK[moviehandle].pts; // Activate OpenGL context of target window: PsychSetGLContext(win); #if PSYCH_SYSTEM == PSYCH_OSX // Explicitely disable Apple's Client storage extensions. For now they are not really useful to us. glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE); #endif // Build a standard PTB texture record: PsychMakeRect(out_texture->rect, 0, 0, movieRecordBANK[moviehandle].width, movieRecordBANK[moviehandle].height); // Set NULL - special texture object as part of the PTB texture record: out_texture->targetSpecific.QuickTimeGLTexture = NULL; // Set texture orientation as if it were an inverted Offscreen window: Upside-down. out_texture->textureOrientation = 3; // We use zero client storage memory bytes: out_texture->textureMemorySizeBytes = 0; // Textures are aligned on 4 Byte boundaries because texels are RGBA8: out_texture->textureByteAligned = 4; // Assign texturehandle of our cached texture, if any, so it gets recycled now: out_texture->textureNumber = movieRecordBANK[moviehandle].cached_texture; // Let PsychCreateTexture() do the rest of the job of creating, setting up and // filling an OpenGL texture with content: PsychCreateTexture(out_texture); // After PsychCreateTexture() the cached texture object from our cache is used // and no longer available for recycling. We mark the cache as empty: // It will be filled with a new textureid for recycling if a texture gets // deleted in PsychMovieDeleteTexture().... movieRecordBANK[moviehandle].cached_texture = 0; // Detection of dropped frames: This is a heuristic. We'll see how well it works out... // TODO: GstBuffer videoBuffer provides special flags that should allow to do a more // robust job, although nothing's wrong with the current approach per se... if (rate && presentation_timestamp) { // Try to check for dropped frames in playback mode: // Expected delta between successive presentation timestamps: targetdelta = 1.0f / (movieRecordBANK[moviehandle].fps * rate); // Compute real delta, given rate and playback direction: if (rate > 0) { realdelta = *presentation_timestamp - movieRecordBANK[moviehandle].last_pts; if (realdelta < 0) realdelta = 0; } else { realdelta = -1.0 * (*presentation_timestamp - movieRecordBANK[moviehandle].last_pts); if (realdelta < 0) realdelta = 0; } frames = realdelta / targetdelta; // Dropped frames? if (frames > 1 && movieRecordBANK[moviehandle].last_pts >= 0) { movieRecordBANK[moviehandle].nr_droppedframes += (int) (frames - 1 + 0.5); } movieRecordBANK[moviehandle].last_pts = *presentation_timestamp; } // Unlock. if (oldstyle) { PsychUnlockMutex(&movieRecordBANK[moviehandle].mutex); } else { gst_buffer_unref(videoBuffer); videoBuffer = NULL; } // Manually advance movie time, if in fetch mode: if (0 == rate) { // We are in manual fetch mode: Need to manually advance movie to next // media sample: event = gst_event_new_step(GST_FORMAT_BUFFERS, 1, 1.0, TRUE, FALSE); gst_element_send_event(theMovie, event); // Block until seek completed, failed, or timeout of 30 seconds reached: gst_element_get_state(theMovie, NULL, NULL, (GstClockTime) (30 * 1e9)); } return(TRUE); }
/* * PsychQTGetTextureFromMovie() -- Create an OpenGL texture map from a specific videoframe from given movie object. * * win = Window pointer of onscreen window for which a OpenGL texture should be created. * moviehandle = Handle to the movie object. * checkForImage = true == Just check if new image available, false == really retrieve the image, blocking if necessary. * timeindex = When not in playback mode, this allows specification of a requested frame by presentation time. * If set to -1, or if in realtime playback mode, this parameter is ignored and the next video frame is returned. * out_texture = Pointer to the Psychtoolbox texture-record where the new texture should be stored. * presentation_timestamp = A ptr to a double variable, where the presentation timestamp of the returned frame should be stored. * * Returns true (1) on success, false (0) if no new image available, -1 if no new image available and there won't be any in future. */ int PsychQTGetTextureFromMovie(PsychWindowRecordType *win, int moviehandle, int checkForImage, double timeindex, PsychWindowRecordType *out_texture, double *presentation_timestamp) { static TimeValue myNextTimeCached = -2; static TimeValue nextFramesTimeCached = -2; TimeValue myCurrTime; TimeValue myNextTime; TimeValue nextFramesTime=0; short myFlags; OSType myTypes[1]; OSErr error = noErr; Movie theMovie; CVOpenGLTextureRef newImage = NULL; QTVisualContextRef theMoviecontext; unsigned int failcount=0; float lowerLeft[2]; float lowerRight[2]; float upperRight[2]; float upperLeft[2]; GLuint texid; Rect rect; float rate; double targetdelta, realdelta, frames; PsychRectType outRect; if (!PsychIsOnscreenWindow(win)) { PsychErrorExitMsg(PsychError_user, "Need onscreen window ptr!!!"); } // Activate OpenGL context of target window: PsychSetGLContext(win); // Explicitely disable Apple's Client storage extensions. For now they are not really useful to us. glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE); if (moviehandle < 0 || moviehandle >= PSYCH_MAX_MOVIES) { PsychErrorExitMsg(PsychError_user, "Invalid moviehandle provided."); } if ((timeindex!=-1) && (timeindex < 0 || timeindex >= 10000.0)) { PsychErrorExitMsg(PsychError_user, "Invalid timeindex provided."); } if (NULL == out_texture && !checkForImage) { PsychErrorExitMsg(PsychError_internal, "NULL-Ptr instead of out_texture ptr passed!!!"); } // Fetch references to objects we need: theMovie = movieRecordBANK[moviehandle].theMovie; theMoviecontext = movieRecordBANK[moviehandle].QTMovieContext; if (theMovie == NULL) { PsychErrorExitMsg(PsychError_user, "Invalid moviehandle provided. No movie associated with this handle."); } // Check if end of movie is reached. Rewind, if so... if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag > 0) { if (GetMovieRate(theMovie)>0) { GoToBeginningOfMovie(theMovie); } else { GoToEndOfMovie(theMovie); } } // Is movie actively playing (automatic async playback, possibly with synced sound)? // If so, then we ignore the 'timeindex' parameter, because the automatic playback // process determines which frames should be delivered to PTB when. This function will // simply wait or poll for arrival/presence of a new frame that hasn't been fetched // in previous calls. if (0 == GetMovieRate(theMovie)) { // Movie playback inactive. We are in "manual" mode: No automatic async playback, // no synced audio output. The user just wants to manually fetch movie frames into // textures for manual playback in a standard Matlab-loop. // First pass - checking for new image? if (checkForImage) { // Image for specific point in time requested? if (timeindex >= 0) { // Yes. We try to retrieve the next possible image for requested timeindex. myCurrTime = (TimeValue) ((timeindex * (double) GetMovieTimeScale(theMovie)) + 0.5f); } else { // No. We just retrieve the next frame, given the current movie time. myCurrTime = GetMovieTime(theMovie, NULL); } // Retrieve timeindex of the closest image sample after myCurrTime: myFlags = nextTimeStep + nextTimeEdgeOK; // We want the next frame in the movie's media. myTypes[0] = VisualMediaCharacteristic; // We want video samples. GetMovieNextInterestingTime(theMovie, myFlags, 1, myTypes, myCurrTime, FloatToFixed(1), &myNextTime, &nextFramesTime); error = GetMoviesError(); if (error != noErr) { PsychErrorExitMsg(PsychError_internal, "Failed to fetch texture from movie for given timeindex!"); } // Found useful event? if (myNextTime == -1) { if (PsychPrefStateGet_Verbosity() > 3) printf("PTB-WARNING: Bogus timevalue in movie track for movie %i. Trying to keep going.\n", moviehandle); // No. Just push timestamp to current time plus a little bit in the hope // this will get us unstuck: myNextTime = myCurrTime + (TimeValue) 1; nextFramesTime = (TimeValue) 0; } if (myNextTime != myNextTimeCached) { // Set movies current time to myNextTime, so the next frame will be fetched from there: SetMovieTimeValue(theMovie, myNextTime); // nextFramesTime is the timeindex to which we need to advance for retrieval of next frame: (see code below) nextFramesTime=myNextTime + nextFramesTime; if (PsychPrefStateGet_Verbosity() > 5) printf("PTB-DEBUG: Current timevalue in movie track for movie %i is %lf secs.\n", moviehandle, (double) myNextTime / (double) GetMovieTimeScale(theMovie)); if (PsychPrefStateGet_Verbosity() > 5) printf("PTB-DEBUG: Next timevalue in movie track for movie %i is %lf secs.\n", moviehandle, (double) nextFramesTime / (double) GetMovieTimeScale(theMovie)); // Cache values for 2nd pass: myNextTimeCached = myNextTime; nextFramesTimeCached = nextFramesTime; } else { // Somehow got stuck? Do nothing... if (PsychPrefStateGet_Verbosity() > 5) printf("PTB-DEBUG: Seem to be a bit stuck at timevalue [for movie %i] of %lf secs. Nudging a bit forward...\n", moviehandle, (double) myNextTime / (double) GetMovieTimeScale(theMovie)); // Nudge the timeindex a bit forware in the hope that this helps: SetMovieTimeValue(theMovie, GetMovieTime(theMovie, NULL) + 1); } } else { // This is the 2nd pass: Image fetching. Use cached values from first pass: // Caching in a static works because we're always called immediately for 2nd // pass after successfull return from 1st pass, and we're not multi-threaded, // i.e., don't need to be reentrant or thread-safe here: myNextTime = myNextTimeCached; nextFramesTime = nextFramesTimeCached; myNextTimeCached = -2; } } else { // myNextTime unavailable if in autoplayback-mode: myNextTime=-1; } // Presentation timestamp requested? if (presentation_timestamp) { // Already available? if (myNextTime==-1) { // Retrieve the exact presentation timestamp of the retrieved frame (in movietime): myFlags = nextTimeStep + nextTimeEdgeOK; // We want the next frame in the movie's media. myTypes[0] = VisualMediaCharacteristic; // We want video samples. // We search backward for the closest available image for the current time. Either we get the current time // if we happen to fetch a frame exactly when it becomes ready, or we get a bit earlier timestamp, which is // the optimal presentation timestamp for this frame: GetMovieNextInterestingTime(theMovie, myFlags, 1, myTypes, GetMovieTime(theMovie, NULL), FloatToFixed(-1), &myNextTime, NULL); } // Convert pts (in Quicktime ticks) to pts in seconds since start of movie and return it: *presentation_timestamp = (double) myNextTime / (double) GetMovieTimeScale(theMovie); } // Allow quicktime visual context task to do its internal bookkeeping and cleanup work: if (theMoviecontext) QTVisualContextTask(theMoviecontext); // Perform decompress-operation: if (checkForImage) MoviesTask(theMovie, 0); // Should we just check for new image? If so, just return availability status: if (checkForImage) { if (PSYCH_USE_QT_GWORLDS) { // We use GWorlds. In this case we either suceed immediately due to the // synchronous nature of GWorld rendering, or we fail completely at end // of non-looping movie: if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag == 0) { // No new frame available and there won't be any in the future, because this is a non-looping // movie that has reached its end. return(-1); } // Is this the special case of a movie without video, but only sound? In that case, // we always return a 'false' because there ain't no image to return. if (movieRecordBANK[moviehandle].QTMovieGWorld == NULL) return(false); // Success! return(true); } // Code which uses QTVisualContextTasks... if (QTVisualContextIsNewImageAvailable(theMoviecontext, NULL)) { // New frame ready! return(true); } else if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag == 0) { // No new frame available and there won't be any in the future, because this is a non-looping // movie that has reached its end. return(-1); } else { // No new frame available yet: return(false); } } if (!PSYCH_USE_QT_GWORLDS) { // Blocking wait-code for non-GWorld mode: // Try up to 1000 iterations for arrival of requested image data in wait-mode: failcount=0; while ((failcount < 1000) && !QTVisualContextIsNewImageAvailable(theMoviecontext, NULL)) { PsychWaitIntervalSeconds(0.005); MoviesTask(theMovie, 0); failcount++; } // No new frame available and there won't be any in the future, because this is a non-looping // movie that has reached its end. if ((failcount>=1000) && IsMovieDone(theMovie) && (movieRecordBANK[moviehandle].loopflag == 0)) { return(-1); } // Fetch new OpenGL texture with the new movie image frame: error = QTVisualContextCopyImageForTime(theMoviecontext, kCFAllocatorDefault, NULL, &newImage); if ((error!=noErr) || newImage == NULL) { PsychErrorExitMsg(PsychError_internal, "OpenGL<->Quicktime texture fetch failed!!!"); } // Disable client storage, if it was enabled: glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE); // Build a standard PTB texture record: CVOpenGLTextureGetCleanTexCoords (newImage, lowerLeft, lowerRight, upperRight, upperLeft); texid = CVOpenGLTextureGetName(newImage); // Assign texture rectangle: PsychMakeRect(outRect, upperLeft[0], upperLeft[1], lowerRight[0], lowerRight[1]); // Set texture orientation as if it were an inverted Offscreen window: Upside-down. out_texture->textureOrientation = (CVOpenGLTextureIsFlipped(newImage)) ? 3 : 4; // Assign OpenGL texture id: out_texture->textureNumber = texid; // Store special texture object as part of the PTB texture record: out_texture->targetSpecific.QuickTimeGLTexture = newImage; } else { // Synchronous texture fetch code for GWorld rendering mode: // At this point, the GWorld should contain the source image for creating a // standard OpenGL texture: // Disable client storage, if it was enabled: glPixelStorei(GL_UNPACK_CLIENT_STORAGE_APPLE, GL_FALSE); // Build a standard PTB texture record: // Assign texture rectangle: GetMovieBox(theMovie, &rect); // Hack: Need to extend rect by 4 pixels, because GWorlds are 4 pixels-aligned via // image row padding: rect.right = rect.right + 4; PsychMakeRect(out_texture->rect, rect.left, rect.top, rect.right, rect.bottom); // Set NULL - special texture object as part of the PTB texture record: out_texture->targetSpecific.QuickTimeGLTexture = NULL; // Set texture orientation as if it were an inverted Offscreen window: Upside-down. out_texture->textureOrientation = 3; // Setup a pointer to our GWorld as texture data pointer: out_texture->textureMemorySizeBytes = 0; // Quicktime textures are aligned on 4 Byte boundaries: out_texture->textureByteAligned = 4; // Lock GWorld: if(!LockPixels(GetGWorldPixMap(movieRecordBANK[moviehandle].QTMovieGWorld))) { // Locking surface failed! We abort. PsychErrorExitMsg(PsychError_internal, "PsychQTGetTextureFromMovie(): Locking GWorld pixmap surface failed!!!"); } // This will retrieve an OpenGL compatible pointer to the GWorlds pixel data and assign it to our texmemptr: out_texture->textureMemory = (GLuint*) GetPixBaseAddr(GetGWorldPixMap(movieRecordBANK[moviehandle].QTMovieGWorld)); // Let PsychCreateTexture() do the rest of the job of creating, setting up and // filling an OpenGL texture with GWorlds content: PsychCreateTexture(out_texture); // Undo hack from above after texture creation: Now we need the real width of the // texture for proper texture coordinate assignments in drawing code et al. rect.right = rect.right - 4; PsychMakeRect(outRect, rect.left, rect.top, rect.right, rect.bottom); // Unlock GWorld surface. We do a glFinish() before, for safety reasons... //glFinish(); UnlockPixels(GetGWorldPixMap(movieRecordBANK[moviehandle].QTMovieGWorld)); // Ready to use the texture... We're done. } // Normalize texture rectangle and assign it: PsychNormalizeRect(outRect, out_texture->rect); rate = FixedToFloat(GetMovieRate(theMovie)); // Detection of dropped frames: This is a heuristic. We'll see how well it works out... if (rate && presentation_timestamp) { // Try to check for dropped frames in playback mode: // Expected delta between successive presentation timestamps: targetdelta = 1.0f / (movieRecordBANK[moviehandle].fps * rate); // Compute real delta, given rate and playback direction: if (rate>0) { realdelta = *presentation_timestamp - movieRecordBANK[moviehandle].last_pts; if (realdelta<0) realdelta = 0; } else { realdelta = -1.0 * (*presentation_timestamp - movieRecordBANK[moviehandle].last_pts); if (realdelta<0) realdelta = 0; } frames = realdelta / targetdelta; // Dropped frames? if (frames > 1 && movieRecordBANK[moviehandle].last_pts>=0) { movieRecordBANK[moviehandle].nr_droppedframes += (int) (frames - 1 + 0.5); } movieRecordBANK[moviehandle].last_pts = *presentation_timestamp; } // Manually advance movie time, if in fetch mode: if (0 == GetMovieRate(theMovie)) { // We are in manual fetch mode: Need to manually advance movie time to next // media sample: if (nextFramesTime == myNextTime) { // Invalid value? Try to hack something that gets us unstuck: myNextTime = GetMovieTime(theMovie, NULL); nextFramesTime = myNextTime + (TimeValue) 1; } SetMovieTimeValue(theMovie, nextFramesTime); } // Check if end of movie is reached. Rewind, if so... if (IsMovieDone(theMovie) && movieRecordBANK[moviehandle].loopflag > 0) { if (GetMovieRate(theMovie)>0) { GoToBeginningOfMovie(theMovie); } else { GoToEndOfMovie(theMovie); } } return(TRUE); }
PsychError SCREENMakeTexture(void) { size_t ix, iters; PsychWindowRecordType *textureRecord; PsychWindowRecordType *windowRecord; PsychRectType rect; psych_bool isImageMatrixBytes, isImageMatrixDoubles; int numMatrixPlanes, xSize, ySize; unsigned char *byteMatrix; double *doubleMatrix; GLuint *texturePointer; GLubyte *texturePointer_b; GLfloat *texturePointer_f; double *rp, *gp, *bp, *ap; GLubyte *rpb, *gpb, *bpb, *apb; int usepoweroftwo, usefloatformat, assume_texorientation, textureShader; double optimized_orientation; psych_bool bigendian; psych_bool planar_storage = FALSE; // Detect endianity (byte-order) of machine: ix=255; rpb=(GLubyte*) &ix; bigendian = ( *rpb == 255 ) ? FALSE : TRUE; ix = 0; rpb = NULL; if(PsychPrefStateGet_DebugMakeTexture()) //MARK #1 StoreNowTime(); //all subfunctions should have these two lines. PsychPushHelp(useString, synopsisString, seeAlsoString); if(PsychIsGiveHelp()){PsychGiveHelp();return(PsychError_none);}; //Get the window structure for the onscreen window. It holds the onscreein GL context which we will need in the //final step when we copy the texture from system RAM onto the screen. PsychErrorExit(PsychCapNumInputArgs(7)); PsychErrorExit(PsychRequireNumInputArgs(2)); PsychErrorExit(PsychCapNumOutputArgs(1)); //get the window record from the window record argument and get info from the window record PsychAllocInWindowRecordArg(kPsychUseDefaultArgPosition, TRUE, &windowRecord); if((windowRecord->windowType!=kPsychDoubleBufferOnscreen) && (windowRecord->windowType!=kPsychSingleBufferOnscreen)) PsychErrorExitMsg(PsychError_user, "MakeTexture called on something else than a onscreen window"); // Get optional texture orientation flag: assume_texorientation = 0; PsychCopyInIntegerArg(6, FALSE, &assume_texorientation); // Get optional texture shader handle: textureShader = 0; PsychCopyInIntegerArg(7, FALSE, &textureShader); //get the argument and sanity check it. isImageMatrixBytes=PsychAllocInUnsignedByteMatArg(2, kPsychArgAnything, &ySize, &xSize, &numMatrixPlanes, &byteMatrix); isImageMatrixDoubles=PsychAllocInDoubleMatArg(2, kPsychArgAnything, &ySize, &xSize, &numMatrixPlanes, &doubleMatrix); if(numMatrixPlanes > 4) PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Specified image matrix exceeds maximum depth of 4 layers"); if(ySize<1 || xSize <1) PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Specified image matrix must be at least 1 x 1 pixels in size"); if(! (isImageMatrixBytes || isImageMatrixDoubles)) PsychErrorExitMsg(PsychError_user, "Illegal argument type"); //not likely. // Is this a special image matrix which is already pre-transposed to fit our optimal format? if (assume_texorientation == 2) { // Yes. Swap xSize and ySize to take this into account: ix = (size_t) xSize; xSize = ySize; ySize = (int) ix; ix = 0; } // Build defining rect for this texture: PsychMakeRect(rect, 0, 0, xSize, ySize); // Copy in texture preferred draw orientation hint. We default to zero degrees, if // not provided. // This parameter is not yet used. It is silently ignorerd for now... optimized_orientation = 0; PsychCopyInDoubleArg(3, FALSE, &optimized_orientation); // Copy in special creation mode flag: It defaults to zero. If set to 1 then we // always create a power-of-two GL_TEXTURE_2D texture. This is useful if one wants // to create and use drifting gratings with no effort - texture wrapping is only available // for GL_TEXTURE_2D, not for non-pot types. It is also useful if the texture is to be // exported to external OpenGL code to simplify tex coords assignments. usepoweroftwo=0; PsychCopyInIntegerArg(4, FALSE, &usepoweroftwo); // Check if size constraints are fullfilled for power-of-two mode: if (usepoweroftwo & 1) { for(ix = 1; ix < (size_t) xSize; ix*=2); if (ix != (size_t) xSize) { PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Power-of-two texture requested but width of imageMatrix is not a power of two!"); } for(ix = 1; ix < (size_t) ySize; ix*=2); if (ix != (size_t) ySize) { PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Power-of-two texture requested but height of imageMatrix is not a power of two!"); } } // Check if creation of a floating point texture is requested? We default to non-floating point, // standard 8 bpc textures if this parameter is not provided. usefloatformat = 0; PsychCopyInIntegerArg(5, FALSE, &usefloatformat); if (usefloatformat<0 || usefloatformat>2) PsychErrorExitMsg(PsychError_user, "Invalid value for 'floatprecision' parameter provided! Valid values are 0 for 8bpc int, 1 for 16bpc float or 2 for 32bpc float."); if (usefloatformat && !isImageMatrixDoubles) { // Floating point texture requested. We only support this if our input is a double matrix, not // for uint8 matrices - converting them to float precision would be just a waste of ressources // without any benefit for precision. PsychErrorExitMsg(PsychError_user, "Creation of a floating point precision texture requested, but uint8 matrix provided! Only double matrices are acceptable for this mode."); } //Create a texture record. Really just a window record adapted for textures. PsychCreateWindowRecord(&textureRecord); //this also fills the window index field. textureRecord->windowType=kPsychTexture; // MK: We need to assign the screen number of the onscreen-window, so PsychCreateTexture() // can query the size of the screen/onscreen-window... textureRecord->screenNumber=windowRecord->screenNumber; textureRecord->depth=32; PsychCopyRect(textureRecord->rect, rect); // Is texture storage in planar format explicitely requested by usercode? Do the gpu and its size // constraints on textures support planar storage for this image? // Can a proper planar -> interleaved remapping GLSL shader be generated and assigned for this texture? if ((usepoweroftwo == 4) && (numMatrixPlanes > 1) && (windowRecord->gfxcaps & kPsychGfxCapFBO) && !(PsychPrefStateGet_ConserveVRAM() & kPsychDontCacheTextures) && (ySize * numMatrixPlanes <= windowRecord->maxTextureSize) && PsychAssignPlanarTextureShaders(textureRecord, windowRecord, numMatrixPlanes)) { // Yes: Use the planar texture storage fast-path. planar_storage = TRUE; if (PsychPrefStateGet_Verbosity() > 6) printf("PTB-DEBUG: Using planar storage for %i layer texture of size %i x %i texels.\n", numMatrixPlanes, xSize, ySize); } else { planar_storage = FALSE; if (PsychPrefStateGet_Verbosity() > 7) printf("PTB-DEBUG: Using standard storage for %i layer texture of size %i x %i texels.\n", numMatrixPlanes, xSize, ySize); } //Allocate the texture memory and copy the MATLAB matrix into the texture memory. if (usefloatformat || (planar_storage && !isImageMatrixBytes)) { // Allocate a double for each color component and pixel: textureRecord->textureMemorySizeBytes = sizeof(double) * (size_t) numMatrixPlanes * (size_t) xSize * (size_t) ySize; } else { // Allocate one byte per color component and pixel: textureRecord->textureMemorySizeBytes = (size_t) numMatrixPlanes * (size_t) xSize * (size_t) ySize; } // We allocate our own intermediate conversion buffer unless this is // creation of a single-layer luminance8 integer texture from a single // layer uint8 input matrix and client storage is disabled. In that case, we can use a zero-copy path: if ((isImageMatrixBytes && (numMatrixPlanes == 1) && (!usefloatformat) && !(PsychPrefStateGet_ConserveVRAM() & kPsychDontCacheTextures)) || (isImageMatrixBytes && planar_storage)) { // Zero copy path: texturePointer = NULL; } else { // Allocate memory: if(PsychPrefStateGet_DebugMakeTexture()) StoreNowTime(); textureRecord->textureMemory = malloc(textureRecord->textureMemorySizeBytes); if(PsychPrefStateGet_DebugMakeTexture()) StoreNowTime(); texturePointer = textureRecord->textureMemory; } // Does script explicitely request usage of a GL_TEXTURE_2D power-of-two texture? if (usepoweroftwo & 1) { // Enforce creation as a power-of-two texture: textureRecord->texturetarget=GL_TEXTURE_2D; } // Now the conversion routines that convert Matlab/Octave matrices into memory // buffers suitable for OpenGL: if (planar_storage) { // Planar texture storage, backed by a LUMINANCE texture container: // Zero-Copy possible? Only for uint8 input -> uint8 output: if (texturePointer == NULL) { texturePointer = (GLuint*) byteMatrix; textureRecord->textureMemory = texturePointer; // Set size to zero, so PsychCreateTexture() does not free() our // input buffer: textureRecord->textureMemorySizeBytes = 0; // This is always a LUMINANCE8 texture, backing our planar uint8 texture: textureRecord->depth = 8 * numMatrixPlanes; textureRecord->textureexternaltype = GL_UNSIGNED_BYTE; textureRecord->textureexternalformat = GL_LUMINANCE; textureRecord->textureinternalformat = GL_LUMINANCE8; } else { // Some cast operation needed from double input format. // We always cast from double to float, potentially with // normalization and/or checking of value range. textureRecord->textureexternalformat = GL_LUMINANCE; if (usefloatformat) { // Floating point or other high precision format: textureRecord->depth = ((usefloatformat == 1) ? 16 : 32) * numMatrixPlanes; textureRecord->textureexternaltype = GL_FLOAT; textureRecord->textureinternalformat = (usefloatformat == 1) ? GL_LUMINANCE_FLOAT16_APPLE : GL_LUMINANCE_FLOAT32_APPLE; // Override for missing floating point texture support: Try to use 16 bit fixed point signed normalized textures [-1.0 ; 1.0] resolved at 15 bits: if ((usefloatformat == 1) && !(windowRecord->gfxcaps & kPsychGfxCapFPTex16)) textureRecord->textureinternalformat = GL_LUMINANCE16_SNORM; // Perform copy with double -> float cast: iters = (size_t) xSize * (size_t) ySize * (size_t) numMatrixPlanes; texturePointer_f = (GLfloat*) texturePointer; for(ix = 0; ix < iters; ix++) { *(texturePointer_f++) = (GLfloat) *(doubleMatrix++); } iters = (size_t) xSize * (size_t) ySize; } else { // 8 Bit format, but from double input matrix -> cast to uint8: textureRecord->depth = 8 * numMatrixPlanes; textureRecord->textureexternaltype = GL_UNSIGNED_BYTE; textureRecord->textureinternalformat = GL_LUMINANCE8; iters = (size_t) xSize * (size_t) ySize * (size_t) numMatrixPlanes; texturePointer_b = (GLubyte*) texturePointer; for(ix = 0; ix < iters; ix++) { *(texturePointer_b++) = (GLubyte) *(doubleMatrix++); } iters = (size_t) xSize * (size_t) ySize; } } } else if (usefloatformat) { // Conversion routines for HDR 16 bpc or 32 bpc textures -- Slow path. // Our input is always double matrices... iters = (size_t) xSize * (size_t) ySize; // Our input buffer is always of GL_FLOAT precision: textureRecord->textureexternaltype = GL_FLOAT; texturePointer_f=(GLfloat*) texturePointer; if(numMatrixPlanes==1) { for(ix=0;ix<iters;ix++){ *(texturePointer_f++)= (GLfloat) *(doubleMatrix++); } textureRecord->depth=(usefloatformat==1) ? 16 : 32; textureRecord->textureinternalformat = (usefloatformat==1) ? GL_LUMINANCE_FLOAT16_APPLE : GL_LUMINANCE_FLOAT32_APPLE; textureRecord->textureexternalformat = GL_LUMINANCE; // Override for missing floating point texture support: Try to use 16 bit fixed point signed normalized textures [-1.0 ; 1.0] resolved at 15 bits: if ((usefloatformat==1) && !(windowRecord->gfxcaps & kPsychGfxCapFPTex16)) textureRecord->textureinternalformat = GL_LUMINANCE16_SNORM; } if(numMatrixPlanes==2) { rp=(double*) ((size_t) doubleMatrix); ap=(double*) ((size_t) rp + (size_t) iters * sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_f++)= (GLfloat) *(rp++); *(texturePointer_f++)= (GLfloat) *(ap++); } textureRecord->depth=(usefloatformat==1) ? 32 : 64; textureRecord->textureinternalformat = (usefloatformat==1) ? GL_LUMINANCE_ALPHA_FLOAT16_APPLE : GL_LUMINANCE_ALPHA_FLOAT32_APPLE; textureRecord->textureexternalformat = GL_LUMINANCE_ALPHA; // Override for missing floating point texture support: Try to use 16 bit fixed point signed normalized textures [-1.0 ; 1.0] resolved at 15 bits: if ((usefloatformat==1) && !(windowRecord->gfxcaps & kPsychGfxCapFPTex16)) textureRecord->textureinternalformat = GL_LUMINANCE16_ALPHA16_SNORM; } if(numMatrixPlanes==3) { rp=(double*) ((size_t) doubleMatrix); gp=(double*) ((size_t) rp + (size_t) iters * sizeof(double)); bp=(double*) ((size_t) gp + (size_t) iters * sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_f++)= (GLfloat) *(rp++); *(texturePointer_f++)= (GLfloat) *(gp++); *(texturePointer_f++)= (GLfloat) *(bp++); } textureRecord->depth=(usefloatformat==1) ? 48 : 96; textureRecord->textureinternalformat = (usefloatformat==1) ? GL_RGB_FLOAT16_APPLE : GL_RGB_FLOAT32_APPLE; textureRecord->textureexternalformat = GL_RGB; // Override for missing floating point texture support: Try to use 16 bit fixed point signed normalized textures [-1.0 ; 1.0] resolved at 15 bits: if ((usefloatformat==1) && !(windowRecord->gfxcaps & kPsychGfxCapFPTex16)) textureRecord->textureinternalformat = GL_RGB16_SNORM; } if(numMatrixPlanes==4) { rp=(double*) ((size_t) doubleMatrix); gp=(double*) ((size_t) rp + (size_t) iters * sizeof(double)); bp=(double*) ((size_t) gp + (size_t) iters * sizeof(double)); ap=(double*) ((size_t) bp + (size_t) iters * sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_f++)= (GLfloat) *(rp++); *(texturePointer_f++)= (GLfloat) *(gp++); *(texturePointer_f++)= (GLfloat) *(bp++); *(texturePointer_f++)= (GLfloat) *(ap++); } textureRecord->depth=(usefloatformat==1) ? 64 : 128; textureRecord->textureinternalformat = (usefloatformat==1) ? GL_RGBA_FLOAT16_APPLE : GL_RGBA_FLOAT32_APPLE; textureRecord->textureexternalformat = GL_RGBA; // Override for missing floating point texture support: Try to use 16 bit fixed point signed normalized textures [-1.0 ; 1.0] resolved at 15 bits: if ((usefloatformat==1) && !(windowRecord->gfxcaps & kPsychGfxCapFPTex16)) textureRecord->textureinternalformat = GL_RGBA16_SNORM; } // End of HDR conversion code... } else { // Standard LDR texture 8 bpc conversion routines -- Fast path. iters = (size_t) xSize * (size_t) ySize; // Improved implementation: Takes 13 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==1){ texturePointer_b=(GLubyte*) texturePointer; for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(doubleMatrix++); } textureRecord->depth=8; } // Improved version: Takes 3 ms on a 800x800 texture... // NB: Implementing memcpy manually by a for-loop takes 10 ms! This is a huge difference. // -> That's because memcpy on MacOS-X is implemented with hand-coded, highly tuned Assembler code for PowerPC. // -> It's always wise to use system-routines if available, instead of coding it by yourself! if(isImageMatrixBytes && numMatrixPlanes==1) { if (texturePointer) { // Need to do a copy. Use optimized memcpy(): memcpy((void*) texturePointer, (void*) byteMatrix, iters); //texturePointer_b=(GLubyte*) texturePointer; //for(ix=0;ix<iters;ix++){ // *(texturePointer_b++) = *(byteMatrix++); //} } else { // Zero-Copy path. Just pass a pointer to our input matrix: texturePointer = (GLuint*) byteMatrix; textureRecord->textureMemory = texturePointer; // Set size to zero, so PsychCreateTexture() does not free() our // input buffer: textureRecord->textureMemorySizeBytes = 0; } textureRecord->depth=8; } // New version: Takes 33 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==2){ texturePointer_b=(GLubyte*) texturePointer; rp=(double*) ((size_t) doubleMatrix); ap=(double*) ((size_t) rp + (size_t) iters * sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(ap++); } textureRecord->depth=16; } // New version: Takes 20 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==2){ texturePointer_b=(GLubyte*) texturePointer; rpb=(GLubyte*) ((size_t) byteMatrix); apb=(GLubyte*) ((size_t) rpb + (size_t) iters); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(rpb++); *(texturePointer_b++)= *(apb++); } textureRecord->depth=16; } // Improved version: Takes 43 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==3){ texturePointer_b=(GLubyte*) texturePointer; rp=(double*) ((size_t) doubleMatrix); gp=(double*) ((size_t) rp + (size_t) iters * sizeof(double)); bp=(double*) ((size_t) gp + (size_t) iters * sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(gp++); *(texturePointer_b++)= (GLubyte) *(bp++); } textureRecord->depth=24; } // Improved version: Takes 25 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==3){ texturePointer_b=(GLubyte*) texturePointer; rpb=(GLubyte*) ((size_t) byteMatrix); gpb=(GLubyte*) ((size_t) rpb + (size_t) iters); bpb=(GLubyte*) ((size_t) gpb + (size_t) iters); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(rpb++); *(texturePointer_b++)= *(gpb++); *(texturePointer_b++)= *(bpb++); } textureRecord->depth=24; } // Improved version: Takes 55 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==4){ texturePointer_b=(GLubyte*) texturePointer; rp=(double*) ((size_t) doubleMatrix); gp=(double*) ((size_t) rp + (size_t) iters * sizeof(double)); bp=(double*) ((size_t) gp + (size_t) iters * sizeof(double)); ap=(double*) ((size_t) bp + (size_t) iters * sizeof(double)); if (bigendian) { // Code for big-endian machines like PowerPC: for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(ap++); *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(gp++); *(texturePointer_b++)= (GLubyte) *(bp++); } } else { // Code for little-endian machines like Intel Pentium: for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(bp++); *(texturePointer_b++)= (GLubyte) *(gp++); *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(ap++); } } textureRecord->depth=32; } // Improved version: Takes 33 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==4){ texturePointer_b=(GLubyte*) texturePointer; rpb=(GLubyte*) ((size_t) byteMatrix); gpb=(GLubyte*) ((size_t) rpb + (size_t) iters); bpb=(GLubyte*) ((size_t) gpb + (size_t) iters); apb=(GLubyte*) ((size_t) bpb + (size_t) iters); if (bigendian) { // Code for big-endian machines like PowerPC: for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(apb++); *(texturePointer_b++)= *(rpb++); *(texturePointer_b++)= *(gpb++); *(texturePointer_b++)= *(bpb++); } } else { // Code for little-endian machines like Intel Pentium: for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(bpb++); *(texturePointer_b++)= *(gpb++); *(texturePointer_b++)= *(rpb++); *(texturePointer_b++)= *(apb++); } } textureRecord->depth=32; } } // End of 8 bpc texture conversion code (fast-path for LDR textures) // Override for missing floating point texture support? if ((usefloatformat==1) && !(windowRecord->gfxcaps & kPsychGfxCapFPTex16)) { // Override enabled. Instead of a 16bpc float texture with 11 bit linear precision in the // range [-1.0 ; 1.0], we use a 16 bit signed normalized texture with a normalized value // range of [-1.0; 1.0], encoded with 1 bit sign and 15 bit magnitude. These textures have // an effective linear precision of 15 bits - better than 16 bpc float - but they are restricted // to a value range of [-1.0 ; 1.0], as opposed to 16 bpc float textures. Tell user about this // replacement at high verbosity levels: if (PsychPrefStateGet_Verbosity() > 4) printf("PTB-INFO:MakeTexture: Code requested 16 bpc float texture, but this is unsupported. Trying to use 16 bit snorm texture instead.\n"); // Signed normalized textures supported? Otherwise we bail... if (!(windowRecord->gfxcaps & kPsychGfxCapSNTex16)) { printf("PTB-ERROR:MakeTexture: Code requested 16 bpc floating point texture, but this is unsupported by this graphics card.\n"); printf("PTB-ERROR:MakeTexture: Tried to use 16 bit snorm texture instead, but failed as this is unsupported as well.\n"); PsychErrorExitMsg(PsychError_user, "Creation of 15 bit linear precision signed normalized texture failed. Not supported by your graphics hardware!"); } // Check value range of pixels. This will not work for out of [-1; 1] range values. texturePointer_f=(GLfloat*) texturePointer; iters = iters * (size_t) numMatrixPlanes; for (ix=0; ix<iters; ix++, texturePointer_f++) { if(fabs((double) *texturePointer_f) > 1.0) { // Game over! printf("PTB-ERROR:MakeTexture: Code requested 16 bpc floating point texture, but this is unsupported by this graphics card.\n"); printf("PTB-ERROR:MakeTexture: Tried to use 16 bit snorm texture instead, but failed because some pixels are outside the\n"); printf("PTB-ERROR:MakeTexture: representable range -1.0 to 1.0 for this texture type. Change your code or update your graphics hardware.\n"); PsychErrorExitMsg(PsychError_user, "Creation of 15 bit linear precision signed normalized texture failed due to out of [-1 ; +1] range pixel values!"); } } } // This is a special workaround for bugs in FLOAT16 texture creation on Mac OS/X 10.4.x and 10.5.x. // The OpenGL fails to properly flush very small values (< 1e-9) to zero when creating a FLOAT16 // type texture. Instead it seems to initialize with trash data, corrupting the texture. // Therefore, if FLOAT16 texture creation is requested, we loop over the whole input buffer and // set all values with magnitude smaller than 1e-9 to zero. Better safe than sorry... if ((usefloatformat==1) && (windowRecord->gfxcaps & kPsychGfxCapFPTex16)) { texturePointer_f=(GLfloat*) texturePointer; iters = iters * (size_t) numMatrixPlanes; for(ix=0; ix<iters; ix++, texturePointer_f++) if(fabs((double) *texturePointer_f) < 1e-9) { *texturePointer_f = 0.0; } } // The memory buffer now contains our texture data in a format ready to submit to OpenGL. // Assign parent window and copy its inheritable properties: PsychAssignParentWindow(textureRecord, windowRecord); // Texture orientation is zero aka transposed aka non-renderswapped. textureRecord->textureOrientation = ((assume_texorientation != 2) && (assume_texorientation != 3)) ? 0 : 2; // This is our best guess about the number of image channels: textureRecord->nrchannels = numMatrixPlanes; if (planar_storage) { // Setup special rect to fake PsychCreateTexture() into creating a luminance // texture numMatrixPlanes times the height (in rows) of the texture, to store the // numMatrixPlanes layers concatenated to each other. if (textureRecord->textureOrientation == 0) { // Normal case: Transposed storage. PsychMakeRect(&(textureRecord->rect[0]), 0, 0, xSize * numMatrixPlanes, ySize); } else { // Special case: Non-transposed or isotropic storage: PsychMakeRect(&(textureRecord->rect[0]), 0, 0, xSize, ySize * numMatrixPlanes); } // Create planar texture: PsychCreateTexture(textureRecord); // Restore rect and clientrect of texture to effective size: PsychMakeRect(&(textureRecord->rect[0]), 0, 0, xSize, ySize); PsychCopyRect(textureRecord->clientrect, textureRecord->rect); textureRecord->specialflags = kPsychPlanarTexture; } else { // Let's create and bind a new texture object and fill it with our new texture data. PsychCreateTexture(textureRecord); // Assign GLSL filter-/lookup-shaders if needed: PsychAssignHighPrecisionTextureShaders(textureRecord, windowRecord, usefloatformat, (usepoweroftwo & 2) ? 1 : 0); } // User specified override shader for this texture provided? This is useful for // basic image processing and procedural texture shading: if (textureShader!=0) { // Assign provided shader as filtershader to this texture: We negate it so // that the texture blitter routines know this is a custom shader, not our // built in filter shader: textureRecord->textureFilterShader = -1 * textureShader; } // Texture ready. Mark it valid and return handle to userspace: PsychSetWindowRecordValid(textureRecord); PsychCopyOutDoubleArg(1, FALSE, textureRecord->windowIndex); // Swapping the texture to upright orientation requested? if (assume_texorientation == 1) { // Transform sourceRecord source texture into a normalized, upright texture if it isn't already in // that format. We require this standard orientation for simplified shader design. PsychSetShader(windowRecord, 0); PsychNormalizeTextureOrientation(textureRecord); } // Shall the texture be finally declared "normally oriented"? // This is either due to explicit renderswapping if assume_textureorientation == 1, // or because it was already pretransposed in Matlab/Octave if assume_textureorientation == 2, // or because user space tells us the texture is isotropic if assume_textureorientation == 3. if (assume_texorientation > 0) { // Yes. Label it as such: textureRecord->textureOrientation = 2; } if(PsychPrefStateGet_DebugMakeTexture()) //MARK #4 StoreNowTime(); return(PsychError_none); }
PsychError SCREENOpenOffscreenWindow(void) { int screenNumber, depth, targetScreenNumber; PsychRectType rect; PsychColorType color; PsychWindowRecordType *exampleWindowRecord, *windowRecord, *targetWindow; psych_bool wasColorSupplied; char* texturePointer; size_t xSize, ySize, nbytes; psych_bool bigendian; GLubyte *rpb; int ix; GLenum fboInternalFormat; psych_bool needzbuffer; psych_bool overridedepth = FALSE; int usefloatformat = 0; int specialFlags = 0; int multiSample = 0; // Detect endianity (byte-order) of machine: ix=255; rpb=(GLubyte*) &ix; bigendian = ( *rpb == 255 ) ? FALSE : TRUE; ix = 0; rpb = NULL; //all sub functions should have these two lines PsychPushHelp(useString, synopsisString, seeAlsoString); if(PsychIsGiveHelp()){PsychGiveHelp();return(PsychError_none);}; //cap the number of inputs PsychErrorExit(PsychCapNumInputArgs(6)); //The maximum number of inputs PsychErrorExit(PsychCapNumOutputArgs(2)); //The maximum number of outputs //1-User supplies a window ptr 2-User supplies a screen number 3-User supplies rect and pixel size if(PsychIsWindowIndexArg(1)){ PsychAllocInWindowRecordArg(1, TRUE, &exampleWindowRecord); // Assign normalized copy of example windows rect -- Top-Left corner is always (0,0) PsychNormalizeRect(exampleWindowRecord->clientrect, rect); // We copy depth only from exampleWindow if it is a offscreen window (=texture). Copying from // onscreen windows doesn't make sense, e.g. depth=16 for onscreen means RGBA8 window, but it // would map onto a LUMINANCE+ALPHA texture for the offscreen window! We always use 32 bit RGBA8 // in such a case. depth=(PsychIsOffscreenWindow(exampleWindowRecord)) ? exampleWindowRecord->depth : 32; // unless it is a FBO backed onscreen window in imaging mode: Then we can use the depth from it. if (exampleWindowRecord->imagingMode & kPsychNeedFastBackingStore || exampleWindowRecord->imagingMode & kPsychNeedFastOffscreenWindows) depth = exampleWindowRecord->depth; targetScreenNumber=exampleWindowRecord->screenNumber; targetWindow=exampleWindowRecord; } else if(PsychIsScreenNumberArg(1)){ PsychCopyInScreenNumberArg(1, TRUE, &screenNumber); PsychGetScreenRect(screenNumber, rect); depth=32; // Always use RGBA8 in this case! See above... targetScreenNumber=screenNumber; targetWindow=NULL; } else if(PsychIsUnaffiliatedScreenNumberArg(1)){ //that means -1 or maybe also NaN if we add that option. // Default to a depth of 32 bpp: depth = 32; targetWindow = NULL; // Get first open onscreen window as target window: PsychFindScreenWindowFromScreenNumber(kPsychUnaffiliatedWindow, &targetWindow); if (targetWindow == NULL) PsychErrorExitMsg(PsychError_user, "Could not find any open onscreen window to act as parent for this offscreen window. Open an onscreen window first!"); targetScreenNumber = targetWindow->screenNumber; PsychGetScreenRect(targetScreenNumber, rect); } else { targetScreenNumber = 0; // Make compiler happy. PsychErrorExit(PsychError_invalidNumdex); } if (targetWindow==NULL) { // Get target window of screen: PsychFindScreenWindowFromScreenNumber(targetScreenNumber, &targetWindow); if (targetWindow == NULL) PsychErrorExitMsg(PsychError_user, "Could not find any open onscreen window to act as parent for this offscreen window. Open an onscreen window first!"); targetScreenNumber = targetWindow->screenNumber; } //Depth and rect argument supplied as arguments override those inherited from reference screen or window. //Note that PsychCopyIn* prefix means that value will not be overwritten if the arguments are not present. PsychCopyInRectArg(3,FALSE, rect); if (IsPsychRectEmpty(rect)) PsychErrorExitMsg(PsychError_user, "Invalid rect value provided: Empty rects are not allowed."); // Copy in optional depth: This gets overriden in many ways if imaging pipeline is on: if (PsychCopyInIntegerArg(4,FALSE, &depth)) overridedepth = TRUE; // If any of the no longer supported values 0, 1, 2 or 4 is provided, we // silently switch to 32 bits per pixel, which is the safest and fastest setting: if (depth==0 || depth==1 || depth==2 || depth==4) depth=32; // Final sanity check: if (!(targetWindow->imagingMode & kPsychNeedFastOffscreenWindows) && !(targetWindow->imagingMode & kPsychNeedFastBackingStore) && (depth==64 || depth==128)) { PsychErrorExitMsg(PsychError_user, "Invalid depth value provided. Must be 8 bpp, 16 bpp, 24 bpp or 32 bpp, unless you enable the imaging pipeline, which provides you with more options!"); } if (depth!=8 && depth!=16 && depth!=24 && depth!=32 && depth!=64 && depth!=128) { PsychErrorExitMsg(PsychError_user, "Invalid depth value provided. Must be 8 bpp, 16 bpp, 24 bpp, 32 bpp, or if imagingmode is enabled also 64 bpp or 128 bpp!"); } // If the imaging pipeline is enabled for the associated onscreen window and fast backing store, aka FBO's // is requested, then we only accept depths of at least 32 bit, i.e. RGBA windows. We override any lower // precision spec. This is because some common hardware only supports rendering to RGBA textures, not to // RGB, LA or Luminance textures. if ((targetWindow->imagingMode & kPsychNeedFastBackingStore || targetWindow->imagingMode & kPsychNeedFastOffscreenWindows) && (depth < 32)) depth = 32; // Find the color for the window background. wasColorSupplied=PsychCopyInColorArg(kPsychUseDefaultArgPosition, FALSE, &color); // If none provided, use a proper white-value for this window: if(!wasColorSupplied) PsychLoadColorStruct(&color, kPsychIndexColor, PsychGetWhiteValueFromWindow(targetWindow)); // Get the optional specialmode flag: PsychCopyInIntegerArg(5, FALSE, &specialFlags); // OpenGL-ES only supports GL_TEXTURE_2D targets, so enforce these via flags setting 1: if (PsychIsGLES(targetWindow)) specialFlags |= 1; // This command converts whatever color we got into RGBA format: PsychCoerceColorMode(&color); // printf("R=%i G=%i B=%i A=%i I=%i", color.value.rgba.r, color.value.rgba.g,color.value.rgba.b,color.value.rgba.a,color.value.index); // First allocate the offscreen window record to store stuff into. If we exit with an error PsychErrorExit() should // call PsychPurgeInvalidWindows which will clean up the window record. PsychCreateWindowRecord(&windowRecord); // This also fills the window index field. // This offscreen window is implemented as a Psychtoolbox texture: windowRecord->windowType=kPsychTexture; // We need to assign the screen number of the onscreen-window, so PsychCreateTexture() // can query the size of the screen/onscreen-window... windowRecord->screenNumber = targetScreenNumber; // Assign the computed depth: windowRecord->depth=depth; // Default number of channels: windowRecord->nrchannels=depth / 8; // Assign the computed rect, but normalize it to start with top-left at (0,0): PsychNormalizeRect(rect, windowRecord->rect); // Client rect of an offscreen window is always == rect of it: PsychCopyRect(windowRecord->clientrect, windowRecord->rect); // Until here no OpenGL commands executed. Now we need a valid context: Set targetWindow // as drawing target. This will perform neccessary context-switch and all backbuffer // backup/restore/whatever operations to make sure we can do what we want without // possibly screwing any offscreen windows and bindings: if (PsychIsOnscreenWindow(targetWindow) || PsychIsOffscreenWindow(targetWindow)) { // This is a possible on-/offscreen drawingtarget: PsychSetDrawingTarget(targetWindow); } else { // This must be a proxy-window object: Can't transition to it! // But we can safe-reset the current drawingtarget... PsychSetDrawingTarget((PsychWindowRecordType*) 0x1); // ...and then switch to the OpenGL context of the 'targetWindow' proxy object: PsychSetGLContext(targetWindow); // Ok, framebuffer and bindings are safe and disabled, context is set. We // should be safe to continue with the proxy... } // From here on we have a defined context and state. We can detach the drawing target whenever // we want, as everything is backed up somewhere for later reinit. // Create offscreen window either new style as FBO, or old style as texture: if ((targetWindow->imagingMode & kPsychNeedFastBackingStore) || (targetWindow->imagingMode & kPsychNeedFastOffscreenWindows)) { // Imaging mode for this window enabled: Use new way of creating the offscreen window: // We safely unbind any FBO bindings and drawingtargets: PsychSetDrawingTarget((PsychWindowRecordType*) 0x1); // Overriden for imagingmode: There we always have 4 channels... windowRecord->nrchannels=4; // Start off with standard 8 bpc fixed point: fboInternalFormat = GL_RGBA8; windowRecord->depth=32; usefloatformat = 0; // Need 16 bpc fixed point precision? if (targetWindow->imagingMode & kPsychNeed16BPCFixed) { fboInternalFormat = (targetWindow->gfxcaps & kPsychGfxCapSNTex16) ? GL_RGBA16_SNORM : GL_RGBA16; windowRecord->depth=64; usefloatformat = 0; } // Need 16 bpc floating point precision? if (targetWindow->imagingMode & kPsychNeed16BPCFloat) { fboInternalFormat = GL_RGBA_FLOAT16_APPLE; windowRecord->depth=64; usefloatformat = 1; } // Need 32 bpc floating point precision? if (targetWindow->imagingMode & kPsychNeed32BPCFloat) { fboInternalFormat = GL_RGBA_FLOAT32_APPLE; windowRecord->depth=128; usefloatformat = 2; } // Override depth value provided? if (overridedepth) { // Manual depth specified: Override with that depth: switch(depth) { case 32: fboInternalFormat = GL_RGBA8; windowRecord->depth=32; usefloatformat = 0; break; case 64: fboInternalFormat = GL_RGBA_FLOAT16_APPLE; windowRecord->depth=64; usefloatformat = 1; // Need fallback for lack of float 16 support? if (!(targetWindow->gfxcaps & kPsychGfxCapFPTex16) && !PsychIsGLES(targetWindow)) { // Yes. Try 16 bit signed normalized texture instead: if (PsychPrefStateGet_Verbosity() > 4) printf("PTB-INFO:OpenOffscreenWindow: Code requested 16 bpc float precision, but this is unsupported. Trying to use 15 bit snorm precision instead.\n"); fboInternalFormat = GL_RGBA16_SNORM; windowRecord->depth=64; usefloatformat = 0; if (!(targetWindow->gfxcaps & kPsychGfxCapSNTex16)) { printf("PTB-ERROR:OpenOffscreenWindow: Code requested 16 bpc float precision, but this is unsupported by this graphics card.\n"); printf("PTB-ERROR:OpenOffscreenWindow: Tried to use 16 bit snorm format instead, but failed as this is unsupported as well.\n"); } } break; case 128: fboInternalFormat = GL_RGBA_FLOAT32_APPLE; windowRecord->depth=128; usefloatformat = 2; break; default: fboInternalFormat = GL_RGBA8; windowRecord->depth=32; usefloatformat = 0; } } // Floating point framebuffer on OpenGL-ES requested? if (PsychIsGLES(targetWindow) && (usefloatformat > 0)) { // Yes. We only support 32 bpc float framebuffers with alpha-blending. On less supportive hardware we fail: if (!(targetWindow->gfxcaps & kPsychGfxCapFPTex32) || !(targetWindow->gfxcaps & kPsychGfxCapFPFBO32)) { PsychErrorExitMsg(PsychError_user, "Sorry, the requested offscreen window color resolution of 32 bpc floating point is not supported by your graphics card. Game over."); } // Supported. Upgrade requested format to 32 bpc float, whatever it was before: fboInternalFormat = GL_RGBA_FLOAT32_APPLE; windowRecord->depth=128; usefloatformat = 2; } // Do we need additional depth buffer attachments? needzbuffer = (PsychPrefStateGet_3DGfx()>0) ? TRUE : FALSE; // Copy in optional multiSample argument: It defaults to zero, aka multisampling disabled. PsychCopyInIntegerArg(6, FALSE, &multiSample); if (multiSample < 0) PsychErrorExitMsg(PsychError_user, "Invalid negative multiSample level provided!"); // Multisampled anti-aliasing requested? if (multiSample > 0) { // Yep. Supported by GPU? if (!(targetWindow->gfxcaps & kPsychGfxCapFBOMultisample)) { // No. We fall back to non-multisampled mode: multiSample = 0; // Tell user if warnings enabled: if (PsychPrefStateGet_Verbosity() > 1) { printf("PTB-WARNING: You requested stimulus anti-aliasing via multisampling by setting the multiSample parameter of Screen('OpenOffscreenWindow', ...) to a non-zero value.\n"); printf("PTB-WARNING: You also requested use of the imaging pipeline. Unfortunately, your combination of operating system, graphics hardware and driver does not\n"); printf("PTB-WARNING: support simultaneous use of the imaging pipeline and multisampled anti-aliasing.\n"); printf("PTB-WARNING: Will therefore continue without anti-aliasing...\n\n"); printf("PTB-WARNING: A driver upgrade may resolve this issue. Users of MacOS-X need at least OS/X 10.5.2 Leopard for support on recent ATI hardware.\n\n"); } } } // Allocate framebuffer object for this Offscreen window: if (!PsychCreateFBO(&(windowRecord->fboTable[0]), fboInternalFormat, needzbuffer, (int) PsychGetWidthFromRect(rect), (int) PsychGetHeightFromRect(rect), multiSample, specialFlags)) { // Failed! PsychErrorExitMsg(PsychError_user, "Creation of Offscreen window in imagingmode failed for some reason :("); } // Assign this FBO as drawBuffer for mono channel of our Offscreen window: windowRecord->drawBufferFBO[0] = 0; windowRecord->fboCount = 1; // Assign it as texture as well: windowRecord->textureNumber = windowRecord->fboTable[0]->coltexid; windowRecord->textureMemorySizeBytes = 0; windowRecord->textureMemory = NULL; windowRecord->texturetarget = (specialFlags & 0x1) ? GL_TEXTURE_2D : GL_TEXTURE_RECTANGLE_EXT; windowRecord->surfaceSizeBytes = (size_t) (PsychGetWidthFromRect(rect) * PsychGetHeightFromRect(rect) * (windowRecord->depth / 8)); // Set bpc for FBO backed offscreen window: windowRecord->bpc = (int) (windowRecord->depth / 4); // Initial setup done, continues below after some shared code... } else { // Traditional texture creation code: // Special case for alpha-channel: DBL_MAX signals maximum alpha // value requested. In our own code we need to manually map this to // the maximum uint8 alpha value of 255: if (color.value.rgba.a == DBL_MAX) color.value.rgba.a = 255; // Allocate the texture memory: // We only allocate the amount really needed for given format, aka numMatrixPlanes - Bytes per pixel. xSize = (size_t) PsychGetWidthFromRect(rect); ySize = (size_t) PsychGetHeightFromRect(rect); windowRecord->textureMemorySizeBytes = ((size_t) (depth/8)) * xSize * ySize; windowRecord->textureMemory = malloc(windowRecord->textureMemorySizeBytes); texturePointer=(char*) windowRecord->textureMemory; // printf("depth=%i xsize=%i ysize=%i mem=%i ptr=%p", depth, xSize, ySize, windowRecord->textureMemorySizeBytes, texturePointer); // Fill with requested background color: nbytes=0; switch (depth) { case 8: // Pure LUMINANCE texture: memset((void*) texturePointer, (int) color.value.rgba.r, windowRecord->textureMemorySizeBytes); break; case 16: // LUMINANCE + ALPHA while (nbytes < windowRecord->textureMemorySizeBytes) { *(texturePointer++) = (psych_uint8) color.value.rgba.r; *(texturePointer++) = (psych_uint8) color.value.rgba.a; nbytes+=2; } break; case 24: // RGB: while (nbytes < windowRecord->textureMemorySizeBytes) { *(texturePointer++) = (psych_uint8) color.value.rgba.r; *(texturePointer++) = (psych_uint8) color.value.rgba.g; *(texturePointer++) = (psych_uint8) color.value.rgba.b; nbytes+=3; } break; case 32: // RGBA if (bigendian) { // Code for big-endian machines, e.g., PowerPC: while (nbytes < windowRecord->textureMemorySizeBytes) { *(texturePointer++) = (psych_uint8) color.value.rgba.a; *(texturePointer++) = (psych_uint8) color.value.rgba.r; *(texturePointer++) = (psych_uint8) color.value.rgba.g; *(texturePointer++) = (psych_uint8) color.value.rgba.b; nbytes+=4; } } else { // Code for little-endian machines, e.g., IntelPC, IntelMAC, aka Pentium. while (nbytes < windowRecord->textureMemorySizeBytes) { *(texturePointer++) = (psych_uint8) color.value.rgba.b; *(texturePointer++) = (psych_uint8) color.value.rgba.g; *(texturePointer++) = (psych_uint8) color.value.rgba.r; *(texturePointer++) = (psych_uint8) color.value.rgba.a; nbytes+=4; } } break; } } // Shared setup code for FBO vs. non-FBO Offscreen windows: // Assign parent window and copy its inheritable properties: PsychAssignParentWindow(windowRecord, targetWindow); // Texture orientation is type 2 aka upright, non-transposed aka Offscreen window: windowRecord->textureOrientation = 2; if ((windowRecord->imagingMode & kPsychNeedFastBackingStore) || (windowRecord->imagingMode & kPsychNeedFastOffscreenWindows)) { // Last step for FBO backed Offscreen window: Clear it to its background color: PsychSetDrawingTarget(windowRecord); // Set default draw shader: PsychSetShader(windowRecord, -1); // Set background fill color: PsychSetGLColor(&color, windowRecord); // Setup alpha-blending: PsychUpdateAlphaBlendingFactorLazily(windowRecord); // Fullscreen fill of a non-onscreen window: PsychGLRect(windowRecord->rect); // Multisampling requested? If so, we need to enable it: if (multiSample > 0) { glEnable(GL_MULTISAMPLE); while (glGetError() != GL_NO_ERROR); } // Ready. Unbind it. PsychSetDrawingTarget(NULL); } else { // Old-style setup for non-FBO Offscreen windows: // Special texture format? if (specialFlags & 0x1) windowRecord->texturetarget = GL_TEXTURE_2D; // Let's create and bind a new texture object and fill it with our new texture data. PsychCreateTexture(windowRecord); } // Assign GLSL filter-/lookup-shaders if needed: PsychAssignHighPrecisionTextureShaders(windowRecord, targetWindow, usefloatformat, (specialFlags & 2) ? 1 : 0); // specialFlags setting 8? Disable auto-mipmap generation: if (specialFlags & 0x8) windowRecord->specialflags |= kPsychDontAutoGenMipMaps; // A specialFlags setting of 32? Protect texture against deletion via Screen('Close') without providing a explicit handle: if (specialFlags & 32) windowRecord->specialflags |= kPsychDontDeleteOnClose; // Window ready. Mark it valid and return handle to userspace: PsychSetWindowRecordValid(windowRecord); //Return the window index and the rect argument. PsychCopyOutDoubleArg(1, FALSE, windowRecord->windowIndex); PsychCopyOutRectArg(2, FALSE, rect); // Ready. return(PsychError_none); }
/* CHECKED TODO * PsychARGetTextureFromCapture() -- Create an OpenGL texturemap from a specific videoframe from given capture object. * * win = Window pointer of onscreen window for which a OpenGL texture should be created. * capturehandle = Handle to the capture object. * checkForImage = >0 == Just check if new image available, 0 == really retrieve the image, blocking if necessary. * 2 == Check for new image, block inside this function (if possible) if no image available. * * timeindex = This parameter is currently ignored and reserved for future use. * out_texture = Pointer to the Psychtoolbox texture-record where the new texture should be stored. * presentation_timestamp = A ptr to a double variable, where the presentation timestamp of the returned frame should be stored. * summed_intensity = An optional ptr to a double variable. If non-NULL, then sum of intensities over all channels is calculated and returned. * outrawbuffer = An optional ptr to a memory buffer of sufficient size. If non-NULL, the buffer will be filled with the captured raw image data, e.g., for use inside Matlab or whatever... * Returns Number of pending or dropped frames after fetch on success (>=0), -1 if no new image available yet, -2 if no new image available and there won't be any in future. */ int PsychARGetTextureFromCapture(PsychWindowRecordType *win, int capturehandle, int checkForImage, double timeindex, PsychWindowRecordType *out_texture, double *presentation_timestamp, double* summed_intensity, rawcapimgdata* outrawbuffer) { GLuint texid; int w, h; double targetdelta, realdelta, frames; unsigned int intensity = 0; unsigned int count, i, bpp; unsigned char* pixptr; psych_bool newframe = FALSE; double tstart, tend; unsigned int pixval, alphacount; int error; int nrdropped = 0; unsigned char* input_image = NULL; // Retrieve device record for handle: PsychVidcapRecordType* capdev = PsychGetARVidcapRecord(capturehandle); // Compute width and height for later creation of textures etc. Need to do this here, // so we can return the values for raw data retrieval: w=capdev->width; h=capdev->height; // Size of a single pixel in bytes: bpp = capdev->reqpixeldepth; // If a outrawbuffer struct is provided, we fill it with info needed to allocate a // sufficient memory buffer for returned raw image data later on: if (outrawbuffer) { outrawbuffer->w = w; outrawbuffer->h = h; outrawbuffer->depth = bpp; } // int waitforframe = (checkForImage > 1) ? 1:0; // Blocking wait for new image requested? // A checkForImage 4 means "no op" with the ARVideo capture engine: This is meant to drive // a movie recording engine, ie., grant processing time to it. Our ARVideo engine doesn't // support movie recording, so this is a no-op: if (checkForImage == 4) return(0); // Take start timestamp for timing stats: PsychGetAdjustedPrecisionTimerSeconds(&tstart); // Should we just check for new image? if (checkForImage) { // Reset current dropped count to zero: capdev->current_dropped = 0; if (capdev->grabber_active == 0) { // Grabber stopped. We'll never get a new image: return(-2); } // Check for image in polling mode: We capture in non-blocking mode: capdev->frame = ar2VideoGetImage(capdev->camera); // Ok, call succeeded. If the 'frame' pointer is non-NULL then there's a new frame ready and dequeued from DMA // ringbuffer. We'll return it on next non-poll invocation. Otherwise no new video data ready yet: capdev->frame_ready = (capdev->frame != NULL) ? 1 : 0; if (capdev->frame_ready) { // Store count of currently queued frames (in addition to the one just fetched). // This is an indication of how well the users script is keeping up with the video stream, // technically the number of frames that would need to be dropped to keep in sync with the // stream. // TODO: Think about this. ARVideo doesn't support a query for pending/dropped frames, so // we either need to live without this feature or think up something clever... capdev->current_dropped = (int) 0; // Ok, at least one new frame ready. If more than one frame has queued up and // we are in 'dropframes' mode, ie. we should always deliver the most recent available // frame, then we quickly fetch & discard all queued frames except the last one. while((capdev->dropframes) && ((int) capdev->current_dropped > 0)) { // We just poll - fetch the frames. As we know there are some queued frames, it // doesn't matter if we poll or block, but polling sounds like a bit less overhead // at the OS level: // First enqueue the recently dequeued buffer... if (ar2VideoCapNext(capdev->camera) != DC1394_SUCCESS) { PsychErrorExitMsg(PsychError_system, "Requeuing of discarded video frame failed while dropping frames (dropframes=1)!!!"); } // Then fetch the next one: if ((capdev->frame = ar2VideoGetImage(capdev->camera)) == NULL) { // Polling failed for some reason... PsychErrorExitMsg(PsychError_system, "Polling for new video frame failed while dropping frames (dropframes=1)!!!"); } } // Update stats for decompression: PsychGetAdjustedPrecisionTimerSeconds(&tend); // Increase counter of decompressed frames: capdev->nrframes++; // Update avg. decompress time: capdev->avg_decompresstime+=(tend - tstart); // Query capture timestamp in seconds: // TODO: ARVideo doesn't provide such a timestamp. For now we just return the current // system time as a lame replacement... // On Windows there would be uint64 capdev->camera->g_Timestamp PsychGetAdjustedPrecisionTimerSeconds(&(capdev->current_pts)); } // Return availability status: 0 = new frame ready for retrieval. -1 = No new frame ready yet. return((capdev->frame_ready) ? 0 : -1); } // This point is only reached if checkForImage == FALSE, which only happens // if a new frame is available in our buffer: // Presentation timestamp requested? if (presentation_timestamp) { // Return it: *presentation_timestamp = capdev->current_pts; } // Synchronous texture fetch: Copy content of capture buffer into a texture: // ========================================================================= // input_image points to the image buffer in our cam: input_image = (unsigned char*) (capdev->frame); // Do we want to do something with the image data and have a // scratch buffer for color conversion alloc'ed? if ((capdev->scratchbuffer) && ((out_texture) || (summed_intensity) || (outrawbuffer))) { // Yes. Perform color-conversion YUV->RGB from cameras DMA buffer // into the scratch buffer and set scratch buffer as source for // all further operations: memcpy(capdev->scratchbuffer, input_image, capdev->width * capdev->height * bpp); // Ok, at this point we should have a RGB8 texture image ready in scratch_buffer. // Set scratch buffer as our new image source for all further processing: input_image = (unsigned char*) capdev->scratchbuffer; } // Only setup if really a texture is requested (non-benchmarking mode): if (out_texture) { PsychMakeRect(out_texture->rect, 0, 0, w, h); // Set NULL - special texture object as part of the PTB texture record: out_texture->targetSpecific.QuickTimeGLTexture = NULL; // Set texture orientation as if it were an inverted Offscreen window: Upside-down. out_texture->textureOrientation = 3; #if PSYCH_SYSTEM == PSYCH_WINDOWS // On Windows in non RGB32 bit modes, set orientation to Upright: out_texture->textureOrientation = (capdev->reqpixeldepth == 4) ? 3 : 2; #endif // Setup a pointer to our buffer as texture data pointer: Setting memsize to zero // prevents unwanted free() operation in PsychDeleteTexture... out_texture->textureMemorySizeBytes = 0; // Set texture depth: Could be 8, 16, 24 or 32 bpp. out_texture->depth = capdev->reqpixeldepth * 8; // This will retrieve an OpenGL compatible pointer to the pixel data and assign it to our texmemptr: out_texture->textureMemory = (GLuint*) input_image; // Let PsychCreateTexture() do the rest of the job of creating, setting up and // filling an OpenGL texture with content: PsychCreateTexture(out_texture); // Ready to use the texture... } // Sum of pixel intensities requested? if(summed_intensity) { pixptr = (unsigned char*) input_image; count = w * h * bpp; for (i=0; i<count; i++) intensity+=(unsigned int) pixptr[i]; *summed_intensity = ((double) intensity) / w / h / bpp; } // Raw data requested? if (outrawbuffer) { // Copy it out: outrawbuffer->w = w; outrawbuffer->h = h; outrawbuffer->depth = bpp; count = (w * h * outrawbuffer->depth); memcpy(outrawbuffer->data, (const void *) input_image, count); } // Release the capture buffer. Return it to the DMA ringbuffer pool: if (ar2VideoCapNext(capdev->camera) != DC1394_SUCCESS) { PsychErrorExitMsg(PsychError_system, "Re-Enqueuing processed video frame failed."); } // Update total count of dropped (or pending) frames: capdev->nr_droppedframes += capdev->current_dropped; nrdropped = capdev->current_dropped; capdev->current_dropped = 0; // Timestamping: PsychGetAdjustedPrecisionTimerSeconds(&tend); // Increase counter of retrieved textures: capdev->nrgfxframes++; // Update average time spent in texture conversion: capdev->avg_gfxtime+=(tend - tstart); // We're successfully done! Return number of dropped (or pending in DMA ringbuffer) frames: return(nrdropped); }
PsychError SCREENMakeTexture(void) { int ix; PsychWindowRecordType *textureRecord; PsychWindowRecordType *windowRecord; PsychRectType rect; Boolean isImageMatrixBytes, isImageMatrixDoubles; int numMatrixPlanes, xSize, ySize; unsigned char *byteMatrix; double *doubleMatrix; GLuint *texturePointer; GLubyte *texturePointer_b; if(PsychPrefStateGet_DebugMakeTexture()) //MARK #1 StoreNowTime(); //all subfunctions should have these two lines. PsychPushHelp(useString, synopsisString, seeAlsoString); if(PsychIsGiveHelp()){PsychGiveHelp();return(PsychError_none);}; //Get the window structure for the onscreen window. It holds the onscreein GL context which we will need in the //final step when we copy the texture from system RAM onto the screen. PsychErrorExit(PsychCapNumInputArgs(2)); PsychErrorExit(PsychRequireNumInputArgs(2)); PsychErrorExit(PsychCapNumOutputArgs(1)); //get the window record from the window record argument and get info from the window record PsychAllocInWindowRecordArg(kPsychUseDefaultArgPosition, TRUE, &windowRecord); if((windowRecord->windowType!=kPsychDoubleBufferOnscreen) && (windowRecord->windowType!=kPsychSingleBufferOnscreen)) PsychErrorExitMsg(PsychError_user, "MakeTexture called on something else than a onscreen window"); //get the argument and sanity check it. isImageMatrixBytes=PsychAllocInUnsignedByteMatArg(2, kPsychArgAnything, &ySize, &xSize, &numMatrixPlanes, &byteMatrix); isImageMatrixDoubles=PsychAllocInDoubleMatArg(2, kPsychArgAnything, &ySize, &xSize, &numMatrixPlanes, &doubleMatrix); if(numMatrixPlanes > 4) PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Specified image matrix exceeds maximum depth of 4 layers"); if(ySize<1 || xSize <1) PsychErrorExitMsg(PsychError_inputMatrixIllegalDimensionSize, "Specified image matrix must be at least 1 x 1 pixels in size"); if(! (isImageMatrixBytes || isImageMatrixDoubles)) PsychErrorExitMsg(PsychError_user, "Illegal argument type"); //not likely. PsychMakeRect(rect, 0, 0, xSize, ySize); //Create a texture record. Really just a window recored adapted for textures. PsychCreateWindowRecord(&textureRecord); //this also fills the window index field. textureRecord->windowType=kPsychTexture; // MK: We need to assign the screen number of the onscreen-window, so PsychCreateTexture() // can query the size of the screen/onscreen-window... textureRecord->screenNumber=windowRecord->screenNumber; textureRecord->depth=32; PsychCopyRect(textureRecord->rect, rect); //Allocate the texture memory and copy the MATLAB matrix into the texture memory. // MK: We only allocate the amount really needed for given format, aka numMatrixPlanes - Bytes per pixel. textureRecord->textureMemorySizeBytes= numMatrixPlanes * xSize * ySize; // MK: Allocate memory page-aligned... -> Helps Apple texture range extensions et al. if(PsychPrefStateGet_DebugMakeTexture()) //MARK #2 StoreNowTime(); textureRecord->textureMemory=valloc(textureRecord->textureMemorySizeBytes); if(PsychPrefStateGet_DebugMakeTexture()) //MARK #3 StoreNowTime(); texturePointer=textureRecord->textureMemory; // Original implementation: Takes 80 ms on a 800x800 texture... /* if(isImageMatrixDoubles && numMatrixPlanes==1){ for(ix=0;ix<xSize;ix++){ for(iy=0;iy<ySize;iy++){ textureIndex=xSize*iy+ix; matrixIndex=iy + ySize * ix; textureRecord->textureMemory[textureIndex]= ((((((GLuint)255 << 8) | (GLuint)(doubleMatrix[matrixIndex])) << 8 ) | (GLuint)(doubleMatrix[matrixIndex]) ) << 8) | (GLuint)(doubleMatrix[matrixIndex]); } } } */ // Improved implementation: Takes 13 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==1){ texturePointer_b=(GLubyte*) texturePointer; int iters=xSize*ySize; for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(doubleMatrix++); } textureRecord->depth=8; } // Original implementation: Takes 30 ms on a 800x800 texture... /* if(isImageMatrixBytes && numMatrixPlanes==1){ for(ix=0;ix<xSize;ix++){ for(iy=0;iy<ySize;iy++){ textureIndex=xSize*iy+ix; matrixIndex=iy + ySize * ix; textureRecord->textureMemory[textureIndex]= ((((((GLuint)255 << 8) | (GLuint)(byteMatrix[matrixIndex])) << 8 ) | (GLuint)(byteMatrix[matrixIndex]) ) << 8) | (GLuint)(byteMatrix[matrixIndex]); } } } */ // Improved version: Takes 3 ms on a 800x800 texture... // NB: Implementing memcpy manually by a for-loop takes 10 ms! This is a huge difference. // -> That's because memcpy on MacOS-X is implemented with hand-coded, highly tuned Assembler code for PowerPC. // -> It's always wise to use system-routines if available, instead of coding it by yourself! if(isImageMatrixBytes && numMatrixPlanes==1){ memcpy((void*) texturePointer, (void*) byteMatrix, xSize*ySize); textureRecord->depth=8; } // New version: Takes 33 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==2){ texturePointer_b=(GLubyte*) texturePointer; int iters=xSize*ySize; double *rp, *ap; rp=(double*) ((unsigned long long) doubleMatrix); ap=(double*) ((unsigned long long) doubleMatrix + (unsigned long long) iters*sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(ap++); } textureRecord->depth=16; } // New version: Takes 20 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==2){ texturePointer_b=(GLubyte*) texturePointer; int iters=xSize*ySize; GLubyte *rp, *ap; rp=(GLubyte*) ((unsigned long long) byteMatrix); ap=(GLubyte*) ((unsigned long long) byteMatrix + (unsigned long long) iters); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(rp++); *(texturePointer_b++)= *(ap++); } textureRecord->depth=16; } // Original version: Takes 160 ms on a 800x800 texture... /* if(isImageMatrixDoubles && numMatrixPlanes==3){ for(ix=0;ix<xSize;ix++){ for(iy=0;iy<ySize;iy++){ textureIndex=xSize*iy+ix; redIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 0); greenIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 1); blueIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 2); red=(GLuint)doubleMatrix[redIndex]; green=(GLuint)doubleMatrix[greenIndex]; blue=(GLuint)doubleMatrix[blueIndex]; alpha=(GLuint)255; textureRecord->textureMemory[textureIndex]= (((((alpha << 8) | red) << 8 ) | green ) << 8) | blue; } } textureRecord->depth=24; } */ // Improved version: Takes 43 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==3){ texturePointer_b=(GLubyte*) texturePointer; int iters=xSize*ySize; double *rp, *gp, *bp; rp=(double*) ((unsigned long long) doubleMatrix); gp=(double*) ((unsigned long long) doubleMatrix + (unsigned long long) iters*sizeof(double)); bp=(double*) ((unsigned long long) gp + (unsigned long long) iters*sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(gp++); *(texturePointer_b++)= (GLubyte) *(bp++); } textureRecord->depth=24; } /* // Original version: Takes 94 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==3){ for(ix=0;ix<xSize;ix++){ for(iy=0;iy<ySize;iy++){ textureIndex=xSize*iy+ix; redIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 0); greenIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 1); blueIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 2); red=(GLuint)byteMatrix[redIndex]; green=(GLuint)byteMatrix[greenIndex]; blue=(GLuint)byteMatrix[blueIndex]; alpha=(GLuint)255; textureRecord->textureMemory[textureIndex]= (((((alpha << 8) | red) << 8 ) | green ) << 8) | blue; } } } */ // Improved version: Takes 25 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==3){ texturePointer_b=(GLubyte*) texturePointer; int iters=xSize*ySize; GLubyte *rp, *gp, *bp; rp=(GLubyte*) ((unsigned long long) byteMatrix); gp=(GLubyte*) ((unsigned long long) byteMatrix + (unsigned long long) iters); bp=(GLubyte*) ((unsigned long long) gp + (unsigned long long) iters); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(rp++); *(texturePointer_b++)= *(gp++); *(texturePointer_b++)= *(bp++); } textureRecord->depth=24; } // Original version: 190 ms on a 800x800 texture... /* if(isImageMatrixDoubles && numMatrixPlanes==4){ for(ix=0;ix<xSize;ix++){ for(iy=0;iy<ySize;iy++){ textureIndex=xSize*iy+ix; redIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 0); greenIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 1); blueIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 2); alphaIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 3); red=(GLuint)doubleMatrix[redIndex]; green=(GLuint)doubleMatrix[greenIndex]; blue=(GLuint)doubleMatrix[blueIndex]; alpha=(GLuint)doubleMatrix[alphaIndex]; textureRecord->textureMemory[textureIndex]= (((((alpha << 8) | red) << 8 ) | green ) << 8) | blue; } } } */ // Improved version: Takes 55 ms on a 800x800 texture... if(isImageMatrixDoubles && numMatrixPlanes==4){ texturePointer_b=(GLubyte*) texturePointer; int iters=xSize*ySize; double *rp, *gp, *bp, *ap; rp=(double*) ((unsigned long long) doubleMatrix); gp=(double*) ((unsigned long long) doubleMatrix + (unsigned long long) iters*sizeof(double)); bp=(double*) ((unsigned long long) gp + (unsigned long long) iters*sizeof(double)); ap=(double*) ((unsigned long long) bp + (unsigned long long) iters*sizeof(double)); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= (GLubyte) *(ap++); *(texturePointer_b++)= (GLubyte) *(rp++); *(texturePointer_b++)= (GLubyte) *(gp++); *(texturePointer_b++)= (GLubyte) *(bp++); } textureRecord->depth=32; } // Original version: Takes 125 ms on a 800x800 texture... /* if(isImageMatrixBytes && numMatrixPlanes==4){ for(ix=0;ix<xSize;ix++){ for(iy=0;iy<ySize;iy++){ textureIndex=xSize*iy+ix; redIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 0); greenIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 1); blueIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 2); alphaIndex=PsychIndexElementFrom3DArray(ySize, xSize, 3, iy, ix, 3); red=(GLuint)byteMatrix[redIndex]; green=(GLuint)byteMatrix[greenIndex]; blue=(GLuint)byteMatrix[blueIndex]; alpha=(GLuint)byteMatrix[alphaIndex]; textureRecord->textureMemory[textureIndex]= (((((alpha << 8) | red) << 8 ) | green ) << 8) | blue; } } } */ // Improved version: Takes 33 ms on a 800x800 texture... if(isImageMatrixBytes && numMatrixPlanes==4){ texturePointer_b=(GLubyte*) texturePointer; int iters=xSize*ySize; GLubyte *rp, *gp, *bp, *ap; rp=(GLubyte*) ((unsigned long long) byteMatrix); gp=(GLubyte*) ((unsigned long long) byteMatrix + (unsigned long long) iters); bp=(GLubyte*) ((unsigned long long) gp + (unsigned long long) iters); ap=(GLubyte*) ((unsigned long long) bp + (unsigned long long) iters); for(ix=0;ix<iters;ix++){ *(texturePointer_b++)= *(ap++); *(texturePointer_b++)= *(rp++); *(texturePointer_b++)= *(gp++); *(texturePointer_b++)= *(bp++); } textureRecord->depth=32; } // The memory buffer now contains our texture data in a format ready to submit to OpenGL. // Assign proper OpenGL-Renderingcontext to texture: // MK: Is this the proper way to do it??? textureRecord->targetSpecific.contextObject = windowRecord->targetSpecific.contextObject; // Let's create and bind a new texture object and fill it with our new texture data. PsychCreateTexture(textureRecord); // Texture ready. Mark it valid and return handle to userspace: PsychSetWindowRecordValid(textureRecord); PsychCopyOutDoubleArg(1, FALSE, textureRecord->windowIndex); if(PsychPrefStateGet_DebugMakeTexture()) //MARK #4 StoreNowTime(); return(PsychError_none); }