// Encode the rectangle using RRE inline UINT vncEncodeRRE::EncodeRect(BYTE *source, BYTE *dest, const rfb::Rect &rect) { int subrects = -1; const UINT rectW = rect.br.x - rect.tl.x; const UINT rectH = rect.br.y - rect.tl.y; // Create the rectangle header rfbFramebufferUpdateRectHeader *surh=(rfbFramebufferUpdateRectHeader *)dest; surh->r.x = (CARD16) rect.tl.x; surh->r.y = (CARD16) rect.tl.y; surh->r.w = (CARD16) (rectW); surh->r.h = (CARD16) (rectH); surh->r.x = Swap16IfLE(surh->r.x); surh->r.y = Swap16IfLE(surh->r.y); surh->r.w = Swap16IfLE(surh->r.w); surh->r.h = Swap16IfLE(surh->r.h); surh->encoding = Swap32IfLE(rfbEncodingRRE); // create a space big enough for the RRE encoded pixels if (m_bufflen < (rectW*rectH*m_remoteformat.bitsPerPixel / 8)) { if (m_buffer != NULL) { delete [] m_buffer; m_buffer = NULL; } m_buffer = new BYTE [rectW*rectH*m_remoteformat.bitsPerPixel/8+1]; if (m_buffer == NULL) return vncEncoder::EncodeRect(source, dest, rect); m_bufflen = rectW*rectH*m_remoteformat.bitsPerPixel/8; } // Translate the data into our new buffer Translate(source, m_buffer, rect); // Choose the appropriate encoding routine (for speed...) switch(m_remoteformat.bitsPerPixel) { case 8: subrects = subrectEncode8( m_buffer, dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader, rectW, rectH, m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader ); break; case 16: subrects = subrectEncode16( (CARD16 *)m_buffer, (CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader), rectW, rectH, m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader ); break; case 32: subrects = subrectEncode32( (CARD32 *)m_buffer, (CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader), rectW, rectH, m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader ); break; } // If we couldn't encode the rectangles then just send the data raw if (subrects < 0) return vncEncoder::EncodeRect(source, dest, rect); // Send the RREHeader rfbRREHeader *rreh=(rfbRREHeader *)(dest+sz_rfbFramebufferUpdateRectHeader); rreh->nSubrects = Swap32IfLE(subrects); // Return the amount of data sent return sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader + (m_remoteformat.bitsPerPixel / 8) + (subrects * (sz_rfbRectangle + m_remoteformat.bitsPerPixel / 8)); }
UINT vncEncodeCoRRE::EncodeSmallRect(BYTE *source, BYTE *dest, const rfb::Rect &rect) { int subrects = -1; const UINT rectW = rect.br.x - rect.tl.x; const UINT rectH = rect.br.y - rect.tl.y; // Create the rectangle header rfbFramebufferUpdateRectHeader *surh=(rfbFramebufferUpdateRectHeader *)dest; surh->r.x = (CARD16) rect.tl.x; surh->r.y = (CARD16) rect.tl.y; surh->r.w = (CARD16) (rectW); surh->r.h = (CARD16) (rectH); surh->r.x = Swap16IfLE(surh->r.x); surh->r.y = Swap16IfLE(surh->r.y); surh->r.w = Swap16IfLE(surh->r.w); surh->r.h = Swap16IfLE(surh->r.h); surh->encoding = Swap32IfLE(rfbEncodingCoRRE); // create a space big enough for the CoRRE encoded pixels if (m_bufflen < (rectW*rectH*m_remoteformat.bitsPerPixel / 8)) { if (m_buffer != NULL) { delete [] m_buffer; m_buffer = NULL; } m_buffer = new BYTE [rectW*rectH*m_remoteformat.bitsPerPixel/8+1]; if (m_buffer == NULL) return vncEncoder::EncodeRect(source, dest, rect); m_bufflen = rectW*rectH*m_remoteformat.bitsPerPixel/8; } // Translate the data into our new buffer Translate(source, m_buffer, rect); // The Buffer object will have ensured that the destination buffer is // big enough using RequiredBuffSize // Choose the appropriate encoding routine (for speed...) switch(m_remoteformat.bitsPerPixel) { case 8: subrects = subrectEncode8( m_buffer, dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader, rectW, rectH, m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader ); break; case 16: subrects = subrectEncode16( (CARD16 *)m_buffer, (CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader), rectW, rectH, m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader ); break; case 32: subrects = subrectEncode32( (CARD32 *)m_buffer, (CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader), rectW, rectH, m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader ); break; } // If we couldn't encode the rectangles then just send the data raw if (subrects < 0) return vncEncoder::EncodeRect(source, dest, rect); // Send the RREHeader rfbRREHeader *rreh=(rfbRREHeader *)(dest+sz_rfbFramebufferUpdateRectHeader); rreh->nSubrects = Swap32IfLE(subrects); // Calculate the size of the buffer produced return sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader + rreAfterBufLen; }
rfbBool rfbSendRectEncodingRRE(rfbClientPtr cl, int x, int y, int w, int h) { rfbFramebufferUpdateRectHeader rect; rfbRREHeader hdr; int nSubrects; int i; char *fbptr = (cl->scaledScreen->frameBuffer + (cl->scaledScreen->paddedWidthInBytes * y) + (x * (cl->scaledScreen->bitsPerPixel / 8))); int maxRawSize = (cl->scaledScreen->width * cl->scaledScreen->height * (cl->format.bitsPerPixel / 8)); if (cl->beforeEncBufSize < maxRawSize) { cl->beforeEncBufSize = maxRawSize; if (cl->beforeEncBuf == NULL) cl->beforeEncBuf = (char *)malloc(cl->beforeEncBufSize); else cl->beforeEncBuf = (char *)realloc(cl->beforeEncBuf, cl->beforeEncBufSize); } if (cl->afterEncBufSize < maxRawSize) { cl->afterEncBufSize = maxRawSize; if (cl->afterEncBuf == NULL) cl->afterEncBuf = (char *)malloc(cl->afterEncBufSize); else cl->afterEncBuf = (char *)realloc(cl->afterEncBuf, cl->afterEncBufSize); } (*cl->translateFn)(cl->translateLookupTable, &(cl->screen->serverFormat), &cl->format, fbptr, cl->beforeEncBuf, cl->scaledScreen->paddedWidthInBytes, w, h); switch (cl->format.bitsPerPixel) { case 8: nSubrects = subrectEncode8(cl, (uint8_t *)cl->beforeEncBuf, w, h); break; case 16: nSubrects = subrectEncode16(cl, (uint16_t *)cl->beforeEncBuf, w, h); break; case 32: nSubrects = subrectEncode32(cl, (uint32_t *)cl->beforeEncBuf, w, h); break; default: rfbLog("getBgColour: bpp %d?\n",cl->format.bitsPerPixel); return FALSE; } if (nSubrects < 0) { /* RRE encoding was too large, use raw */ return rfbSendRectEncodingRaw(cl, x, y, w, h); } rfbStatRecordEncodingSent(cl, rfbEncodingRRE, sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader + cl->afterEncBufLen, sz_rfbFramebufferUpdateRectHeader + w * h * (cl->format.bitsPerPixel / 8)); if (cl->ublen + sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader > UPDATE_BUF_SIZE) { if (!rfbSendUpdateBuf(cl)) return FALSE; } rect.r.x = Swap16IfLE(x); rect.r.y = Swap16IfLE(y); rect.r.w = Swap16IfLE(w); rect.r.h = Swap16IfLE(h); rect.encoding = Swap32IfLE(rfbEncodingRRE); memcpy(&cl->updateBuf[cl->ublen], (char *)&rect, sz_rfbFramebufferUpdateRectHeader); cl->ublen += sz_rfbFramebufferUpdateRectHeader; hdr.nSubrects = Swap32IfLE(nSubrects); memcpy(&cl->updateBuf[cl->ublen], (char *)&hdr, sz_rfbRREHeader); cl->ublen += sz_rfbRREHeader; for (i = 0; i < cl->afterEncBufLen;) { int bytesToCopy = UPDATE_BUF_SIZE - cl->ublen; if (i + bytesToCopy > cl->afterEncBufLen) { bytesToCopy = cl->afterEncBufLen - i; } memcpy(&cl->updateBuf[cl->ublen], &cl->afterEncBuf[i], bytesToCopy); cl->ublen += bytesToCopy; i += bytesToCopy; if (cl->ublen == UPDATE_BUF_SIZE) { if (!rfbSendUpdateBuf(cl)) return FALSE; } } return TRUE; }
// Encode the rectangle using RRE inline UINT vncEncodeRRE::EncodeRect(BYTE *source, BYTE *dest, const RECT &rect, int offx, int offy) { int subrects = -1; const UINT rectW = rect.right - rect.left; const UINT rectH = rect.bottom - rect.top; // Create the rectangle header rfbFramebufferUpdateRectHeader *surh=(rfbFramebufferUpdateRectHeader *)dest; surh->r.x = (CARD16) rect.left; surh->r.y = (CARD16) rect.top; surh->r.w = (CARD16) (rectW); surh->r.h = (CARD16) (rectH); surh->r.x = Swap16IfLE(surh->r.x - offx); surh->r.y = Swap16IfLE(surh->r.y - offy); surh->r.w = Swap16IfLE(surh->r.w); surh->r.h = Swap16IfLE(surh->r.h); surh->encoding = Swap32IfLE(rfbEncodingRRE); // create a space big enough for the RRE encoded pixels size_t rectSize = rectW * rectH * (m_remoteformat.bitsPerPixel / 8); if (m_bufflen < rectSize) { if (m_buffer != NULL) { delete [] m_buffer; m_buffer = NULL; } m_buffer = new BYTE [rectSize + 1]; if (m_buffer == NULL) return vncEncoder::EncodeRect(source, dest, rect, offx, offy); m_bufflen = rectSize; } // Translate the data into our new buffer Translate(source, m_buffer, rect); // Choose the appropriate encoding routine (for speed...) switch(m_remoteformat.bitsPerPixel) { case 8: subrects = subrectEncode8( m_buffer, dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader, rectW, rectH, m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader ); break; case 16: subrects = subrectEncode16( (CARD16 *)m_buffer, (CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader), rectW, rectH, m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader ); break; case 32: subrects = subrectEncode32( (CARD32 *)m_buffer, (CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader), rectW, rectH, m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader ); break; } // If we couldn't encode the rectangles then just send the data raw if (subrects < 0) return vncEncoder::EncodeRect(source, dest, rect, offx, offy); // Send the RREHeader rfbRREHeader *rreh=(rfbRREHeader *)(dest+sz_rfbFramebufferUpdateRectHeader); rreh->nSubrects = Swap32IfLE(subrects); // Update statistics for this rectangle. rectangleOverhead += sz_rfbFramebufferUpdateRectHeader; dataSize += ( rectW * rectH * m_remoteformat.bitsPerPixel) / 8; encodedSize += sz_rfbRREHeader + (m_remoteformat.bitsPerPixel / 8) + (subrects * (sz_rfbRectangle + m_remoteformat.bitsPerPixel / 8)); transmittedSize += sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader + (m_remoteformat.bitsPerPixel / 8) + (subrects * (sz_rfbRectangle + m_remoteformat.bitsPerPixel / 8)); // Return the amount of data sent return sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader + (m_remoteformat.bitsPerPixel / 8) + (subrects * (sz_rfbRectangle + m_remoteformat.bitsPerPixel / 8)); }