Fix temporary decoder buffer sizes

Some of these were excessively large because of redundant factors in the size calculation.
aleksey-novikov · Oct 5, 2016 · 1349e42 · 1349e42
1 parent 31cad94
commit 1349e42
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 8 deletions.
diff --git a/common/rfb/ZRLEDecoder.cxx b/common/rfb/ZRLEDecoder.cxx
@@ -86,10 +86,9 @@ void ZRLEDecoder::decodeRect(const Rect& r, const void* buffer,
 {
   rdr::MemInStream is(buffer, buflen);
   const rfb::PixelFormat& pf = cp.pf();
-  rdr::U8* buf[64 * 64 * 4 * pf.bpp/8];
   switch (pf.bpp) {
-  case 8:  zrleDecode8 (r, &is, &zis, (rdr::U8*) buf, pf, pb); break;
-  case 16: zrleDecode16(r, &is, &zis, (rdr::U16*)buf, pf, pb); break;
+  case 8:  zrleDecode8 (r, &is, &zis, pf, pb); break;
+  case 16: zrleDecode16(r, &is, &zis, pf, pb); break;
   case 32:
     {
       Pixel maxPixel = pf.pixelFromRGB((rdr::U16)-1, (rdr::U16)-1, (rdr::U16)-1);
@@ -99,16 +98,16 @@ void ZRLEDecoder::decodeRect(const Rect& r, const void* buffer,
       if ((fitsInLS3Bytes && pf.isLittleEndian()) ||
           (fitsInMS3Bytes && pf.isBigEndian()))
       {
-        zrleDecode24A(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+        zrleDecode24A(r, &is, &zis, pf, pb);
       }
       else if ((fitsInLS3Bytes && pf.isBigEndian()) ||
                (fitsInMS3Bytes && pf.isLittleEndian()))
       {
-        zrleDecode24B(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+        zrleDecode24B(r, &is, &zis, pf, pb);
       }
       else
       {
-        zrleDecode32(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+        zrleDecode32(r, &is, &zis, pf, pb);
       }
       break;
     }

diff --git a/common/rfb/hextileDecode.h b/common/rfb/hextileDecode.h
@@ -44,7 +44,7 @@ static void HEXTILE_DECODE (const Rect& r, rdr::InStream* is,
   Rect t;
   PIXEL_T bg = 0;
   PIXEL_T fg = 0;
-  PIXEL_T buf[16 * 16 * 4];
+  PIXEL_T buf[16 * 16];
 
   for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 16) {
 

diff --git a/common/rfb/zrleDecode.h b/common/rfb/zrleDecode.h
@@ -47,12 +47,13 @@ namespace rfb {
 #endif
 
 void ZRLE_DECODE (const Rect& r, rdr::InStream* is,
-                  rdr::ZlibInStream* zis, PIXEL_T* buf,
+                  rdr::ZlibInStream* zis,
                   const PixelFormat& pf, ModifiablePixelBuffer* pb)
 {
   int length = is->readU32();
   zis->setUnderlying(is, length);
   Rect t;
+  PIXEL_T buf[64 * 64];
 
   for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 64) {