// Encode the rectangle using RRE
inline UINT
vncEncodeRRE::EncodeRect(BYTE *source, BYTE *dest, const rfb::Rect &rect)
{
int subrects = -1;
const UINT rectW = rect.br.x - rect.tl.x;
const UINT rectH = rect.br.y - rect.tl.y;
// Create the rectangle header
rfbFramebufferUpdateRectHeader *surh=(rfbFramebufferUpdateRectHeader *)dest;
surh->r.x = (CARD16) rect.tl.x;
surh->r.y = (CARD16) rect.tl.y;
surh->r.w = (CARD16) (rectW);
surh->r.h = (CARD16) (rectH);
surh->r.x = Swap16IfLE(surh->r.x);
surh->r.y = Swap16IfLE(surh->r.y);
surh->r.w = Swap16IfLE(surh->r.w);
surh->r.h = Swap16IfLE(surh->r.h);
surh->encoding = Swap32IfLE(rfbEncodingRRE);
// create a space big enough for the RRE encoded pixels
if (m_bufflen < (rectW*rectH*m_remoteformat.bitsPerPixel / 8))
{
if (m_buffer != NULL)
{
delete [] m_buffer;
m_buffer = NULL;
}
m_buffer = new BYTE [rectW*rectH*m_remoteformat.bitsPerPixel/8+1];
if (m_buffer == NULL)
return vncEncoder::EncodeRect(source, dest, rect);
m_bufflen = rectW*rectH*m_remoteformat.bitsPerPixel/8;
}
// Translate the data into our new buffer
Translate(source, m_buffer, rect);
// Choose the appropriate encoding routine (for speed...)
switch(m_remoteformat.bitsPerPixel)
{
case 8:
subrects = subrectEncode8(
m_buffer,
dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader,
rectW,
rectH,
m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader
);
break;
case 16:
subrects = subrectEncode16(
(CARD16 *)m_buffer,
(CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader),
rectW,
rectH,
m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader
);
break;
case 32:
subrects = subrectEncode32(
(CARD32 *)m_buffer,
(CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader),
rectW,
rectH,
m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader
);
break;
}
// If we couldn't encode the rectangles then just send the data raw
if (subrects < 0)
return vncEncoder::EncodeRect(source, dest, rect);
// Send the RREHeader
rfbRREHeader *rreh=(rfbRREHeader *)(dest+sz_rfbFramebufferUpdateRectHeader);
rreh->nSubrects = Swap32IfLE(subrects);
// Return the amount of data sent
return sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader +
(m_remoteformat.bitsPerPixel / 8) +
(subrects * (sz_rfbRectangle + m_remoteformat.bitsPerPixel / 8));
}
UINT
vncEncodeCoRRE::EncodeSmallRect(BYTE *source, BYTE *dest, const rfb::Rect &rect)
{
int subrects = -1;
const UINT rectW = rect.br.x - rect.tl.x;
const UINT rectH = rect.br.y - rect.tl.y;
// Create the rectangle header
rfbFramebufferUpdateRectHeader *surh=(rfbFramebufferUpdateRectHeader *)dest;
surh->r.x = (CARD16) rect.tl.x;
surh->r.y = (CARD16) rect.tl.y;
surh->r.w = (CARD16) (rectW);
surh->r.h = (CARD16) (rectH);
surh->r.x = Swap16IfLE(surh->r.x);
surh->r.y = Swap16IfLE(surh->r.y);
surh->r.w = Swap16IfLE(surh->r.w);
surh->r.h = Swap16IfLE(surh->r.h);
surh->encoding = Swap32IfLE(rfbEncodingCoRRE);
// create a space big enough for the CoRRE encoded pixels
if (m_bufflen < (rectW*rectH*m_remoteformat.bitsPerPixel / 8))
{
if (m_buffer != NULL)
{
delete [] m_buffer;
m_buffer = NULL;
}
m_buffer = new BYTE [rectW*rectH*m_remoteformat.bitsPerPixel/8+1];
if (m_buffer == NULL)
return vncEncoder::EncodeRect(source, dest, rect);
m_bufflen = rectW*rectH*m_remoteformat.bitsPerPixel/8;
}
// Translate the data into our new buffer
Translate(source, m_buffer, rect);
// The Buffer object will have ensured that the destination buffer is
// big enough using RequiredBuffSize
// Choose the appropriate encoding routine (for speed...)
switch(m_remoteformat.bitsPerPixel)
{
case 8:
subrects = subrectEncode8(
m_buffer,
dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader,
rectW,
rectH,
m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader
);
break;
case 16:
subrects = subrectEncode16(
(CARD16 *)m_buffer,
(CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader),
rectW,
rectH,
m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader
);
break;
case 32:
subrects = subrectEncode32(
(CARD32 *)m_buffer,
(CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader),
rectW,
rectH,
m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader
);
break;
}
// If we couldn't encode the rectangles then just send the data raw
if (subrects < 0)
return vncEncoder::EncodeRect(source, dest, rect);
// Send the RREHeader
rfbRREHeader *rreh=(rfbRREHeader *)(dest+sz_rfbFramebufferUpdateRectHeader);
rreh->nSubrects = Swap32IfLE(subrects);
// Calculate the size of the buffer produced
return sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader + rreAfterBufLen;
}
rfbBool
rfbSendRectEncodingRRE(rfbClientPtr cl,
int x,
int y,
int w,
int h)
{
rfbFramebufferUpdateRectHeader rect;
rfbRREHeader hdr;
int nSubrects;
int i;
char *fbptr = (cl->scaledScreen->frameBuffer + (cl->scaledScreen->paddedWidthInBytes * y)
+ (x * (cl->scaledScreen->bitsPerPixel / 8)));
int maxRawSize = (cl->scaledScreen->width * cl->scaledScreen->height
* (cl->format.bitsPerPixel / 8));
if (cl->beforeEncBufSize < maxRawSize) {
cl->beforeEncBufSize = maxRawSize;
if (cl->beforeEncBuf == NULL)
cl->beforeEncBuf = (char *)malloc(cl->beforeEncBufSize);
else
cl->beforeEncBuf = (char *)realloc(cl->beforeEncBuf, cl->beforeEncBufSize);
}
if (cl->afterEncBufSize < maxRawSize) {
cl->afterEncBufSize = maxRawSize;
if (cl->afterEncBuf == NULL)
cl->afterEncBuf = (char *)malloc(cl->afterEncBufSize);
else
cl->afterEncBuf = (char *)realloc(cl->afterEncBuf, cl->afterEncBufSize);
}
(*cl->translateFn)(cl->translateLookupTable,
&(cl->screen->serverFormat),
&cl->format, fbptr, cl->beforeEncBuf,
cl->scaledScreen->paddedWidthInBytes, w, h);
switch (cl->format.bitsPerPixel) {
case 8:
nSubrects = subrectEncode8(cl, (uint8_t *)cl->beforeEncBuf, w, h);
break;
case 16:
nSubrects = subrectEncode16(cl, (uint16_t *)cl->beforeEncBuf, w, h);
break;
case 32:
nSubrects = subrectEncode32(cl, (uint32_t *)cl->beforeEncBuf, w, h);
break;
default:
rfbLog("getBgColour: bpp %d?\n",cl->format.bitsPerPixel);
return FALSE;
}
if (nSubrects < 0) {
/* RRE encoding was too large, use raw */
return rfbSendRectEncodingRaw(cl, x, y, w, h);
}
rfbStatRecordEncodingSent(cl, rfbEncodingRRE,
sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader + cl->afterEncBufLen,
sz_rfbFramebufferUpdateRectHeader + w * h * (cl->format.bitsPerPixel / 8));
if (cl->ublen + sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader
> UPDATE_BUF_SIZE)
{
if (!rfbSendUpdateBuf(cl))
return FALSE;
}
rect.r.x = Swap16IfLE(x);
rect.r.y = Swap16IfLE(y);
rect.r.w = Swap16IfLE(w);
rect.r.h = Swap16IfLE(h);
rect.encoding = Swap32IfLE(rfbEncodingRRE);
memcpy(&cl->updateBuf[cl->ublen], (char *)&rect,
sz_rfbFramebufferUpdateRectHeader);
cl->ublen += sz_rfbFramebufferUpdateRectHeader;
hdr.nSubrects = Swap32IfLE(nSubrects);
memcpy(&cl->updateBuf[cl->ublen], (char *)&hdr, sz_rfbRREHeader);
cl->ublen += sz_rfbRREHeader;
for (i = 0; i < cl->afterEncBufLen;) {
int bytesToCopy = UPDATE_BUF_SIZE - cl->ublen;
if (i + bytesToCopy > cl->afterEncBufLen) {
bytesToCopy = cl->afterEncBufLen - i;
}
memcpy(&cl->updateBuf[cl->ublen], &cl->afterEncBuf[i], bytesToCopy);
cl->ublen += bytesToCopy;
i += bytesToCopy;
if (cl->ublen == UPDATE_BUF_SIZE) {
if (!rfbSendUpdateBuf(cl))
return FALSE;
}
}
return TRUE;
}
// Encode the rectangle using RRE
inline UINT
vncEncodeRRE::EncodeRect(BYTE *source, BYTE *dest, const RECT &rect, int offx, int offy)
{
int subrects = -1;
const UINT rectW = rect.right - rect.left;
const UINT rectH = rect.bottom - rect.top;
// Create the rectangle header
rfbFramebufferUpdateRectHeader *surh=(rfbFramebufferUpdateRectHeader *)dest;
surh->r.x = (CARD16) rect.left;
surh->r.y = (CARD16) rect.top;
surh->r.w = (CARD16) (rectW);
surh->r.h = (CARD16) (rectH);
surh->r.x = Swap16IfLE(surh->r.x - offx);
surh->r.y = Swap16IfLE(surh->r.y - offy);
surh->r.w = Swap16IfLE(surh->r.w);
surh->r.h = Swap16IfLE(surh->r.h);
surh->encoding = Swap32IfLE(rfbEncodingRRE);
// create a space big enough for the RRE encoded pixels
size_t rectSize = rectW * rectH * (m_remoteformat.bitsPerPixel / 8);
if (m_bufflen < rectSize)
{
if (m_buffer != NULL)
{
delete [] m_buffer;
m_buffer = NULL;
}
m_buffer = new BYTE [rectSize + 1];
if (m_buffer == NULL)
return vncEncoder::EncodeRect(source, dest, rect, offx, offy);
m_bufflen = rectSize;
}
// Translate the data into our new buffer
Translate(source, m_buffer, rect);
// Choose the appropriate encoding routine (for speed...)
switch(m_remoteformat.bitsPerPixel)
{
case 8:
subrects = subrectEncode8(
m_buffer,
dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader,
rectW,
rectH,
m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader
);
break;
case 16:
subrects = subrectEncode16(
(CARD16 *)m_buffer,
(CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader),
rectW,
rectH,
m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader
);
break;
case 32:
subrects = subrectEncode32(
(CARD32 *)m_buffer,
(CARD8 *)(dest+sz_rfbFramebufferUpdateRectHeader+sz_rfbRREHeader),
rectW,
rectH,
m_bufflen-sz_rfbFramebufferUpdateRectHeader-sz_rfbRREHeader
);
break;
}
// If we couldn't encode the rectangles then just send the data raw
if (subrects < 0)
return vncEncoder::EncodeRect(source, dest, rect, offx, offy);
// Send the RREHeader
rfbRREHeader *rreh=(rfbRREHeader *)(dest+sz_rfbFramebufferUpdateRectHeader);
rreh->nSubrects = Swap32IfLE(subrects);
// Update statistics for this rectangle.
rectangleOverhead += sz_rfbFramebufferUpdateRectHeader;
dataSize += ( rectW * rectH * m_remoteformat.bitsPerPixel) / 8;
encodedSize += sz_rfbRREHeader +
(m_remoteformat.bitsPerPixel / 8) +
(subrects * (sz_rfbRectangle + m_remoteformat.bitsPerPixel / 8));
transmittedSize += sz_rfbFramebufferUpdateRectHeader +
sz_rfbRREHeader +
(m_remoteformat.bitsPerPixel / 8) +
(subrects * (sz_rfbRectangle + m_remoteformat.bitsPerPixel / 8));
// Return the amount of data sent
return sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader +
(m_remoteformat.bitsPerPixel / 8) +
(subrects * (sz_rfbRectangle + m_remoteformat.bitsPerPixel / 8));
}
