Member Controls

Wizards of SimPE & SimPE Bugs > Compression bug?
Bugreport       Priority ; State Open; SimPE Version 0.00; Operating System Windows
Afr0Link to postposted: Wed Feb 02, 2011 8:36 am

Member since:
 2007-08-07
Posts:
 3
Hi!
I'm trying to use SimPE's Compress() function, but it keeps insisting on filling the cData array with data beyond its length! And that happens in several places, too (I put in if-checks to see if I could leviate the problem, but the only thing that happened was that the data didn't seem to get compressed at all).



        private const uint m_MAGICNUMBER_QFS = 0xFB10;
        // some Compression Data
        private const int m_MAX_OFFSET = 0x20000;
        private const int m_MAX_COPY_COUNT = 0x404;
        //Compression strength
        private const int m_QFS_MAXITER = 0x80;



public byte[] Compress(byte[] data)
{
//try
//{
//return Comp(data, true);
#region Init Variables

//contains the latest offset for a combination of two characters
ArrayList[] cmpmap = new ArrayList[0x1000000];

//will contain the compressed Data
byte[] cdata = new byte[data.Length * 2];

//init some vars
int writeindex = 0;
int lastreadindex = 0;
ArrayList indexlist = null;
int copyoffset = 0;
int copycount = 0;
writeindex = 0;
int index = -1;
lastreadindex = 0;
byte[] retdata;
bool end = false;

#endregion

try
{
//begin main Compression Loop
while (index < data.Length - 3)
{
#region get all Compression Candidates (list of offsets for all occurances of the current 3 bytes)

do
{
index++;
if (index >= data.Length - 2)
{
end = true;
break;
}
int mapindex = data[index] | (data[index + 1] << 0x08) | (data[index + 2] << 0x10);

indexlist = cmpmap[mapindex];
if (indexlist == null)
{
indexlist = new ArrayList();
cmpmap[mapindex] = indexlist;
}

indexlist.Add(index);

} while (index < lastreadindex);
if (end) break;

#endregion

#region find the longest repeating byte sequence in the index List (for offset copy)

int offsetcopycount = 0;
int loopcount = 1;

while ((loopcount < indexlist.Count) && (loopcount < m_QFS_MAXITER))
{
int foundindex = (int)indexlist[(indexlist.Count - 1) - loopcount];
if ((index - foundindex) >= m_MAX_OFFSET)
break;

loopcount++;
copycount = 3;
while ((data.Length > index + copycount) && (data[index + copycount] == data[foundindex + copycount]) && (copycount < m_MAX_COPY_COUNT))
copycount++;

if (copycount > offsetcopycount)
{
int cof = index - foundindex;
offsetcopycount = copycount;
copyoffset = index - foundindex;
}
}

#endregion

#region Compression

//check if we can compress this
if (offsetcopycount < 3) offsetcopycount = 0;
else if ((offsetcopycount < 4) && (copyoffset > 0x400)) offsetcopycount = 0;
else if ((offsetcopycount < 5) && (copyoffset > 0x4000)) offsetcopycount = 0;

//this is offset-compressable? so do the compression
if (offsetcopycount > 0)
{
//plaincopy
while ((index - lastreadindex) > 3)
{
copycount = (index - lastreadindex);
while (copycount > 0x71) copycount -= 0x71;
copycount = copycount & 0xfc;
int realcopycount = (copycount >> 2);

/*if (writeindex >= cdata.Length)
break;*/

cdata[writeindex++] = (byte)(0xdf + realcopycount);
for (int i = 0; i < copycount; i++)
{
/*if (writeindex >= cdata.Length)
break;*/

cdata[writeindex++] = data[lastreadindex++];
}
}

//offsetcopy
copycount = index - lastreadindex;
copyoffset--;
if ((offsetcopycount <= 0xa) && (copyoffset < 0x400))
{
/*if (writeindex >= cdata.Length)
{
end = true;
break;
}*/

cdata[writeindex++] = (byte)((((copyoffset >> 3) & 0x60) | ((offsetcopycount - 3) << 2)) | copycount);
cdata[writeindex++] = (byte)(copyoffset & 0xff);
}
else if ((offsetcopycount <= 0x43) && (copyoffset < 0x4000))
{
cdata[writeindex++] = (byte)(0x80 | (offsetcopycount - 4));
cdata[writeindex++] = (byte)((copycount << 6) | (copyoffset >> 8));
cdata[writeindex++] = (byte)(copyoffset & 0xff);
}
else if ((offsetcopycount <= m_MAX_COPY_COUNT) && (copyoffset < m_MAX_OFFSET))
{
cdata[writeindex++] = (byte)(((0xc0 | ((copyoffset >> 0x0c) & 0x10)) + (((offsetcopycount - 5) >> 6) & 0x0c)) | copycount);
cdata[writeindex++] = (byte)((copyoffset >> 8) & 0xff);
cdata[writeindex++] = (byte)(copyoffset & 0xff);
cdata[writeindex++] = (byte)((offsetcopycount - 5) & 0xff);
}
else
{
copycount = 0;
offsetcopycount = 0;
}

//do the offset copy
for (int i = 0; i < copycount; i++)
cdata[writeindex++] = data[lastreadindex++];

lastreadindex += offsetcopycount;
}

#endregion
} //while (main Loop)

#region Add remaining Data

//add the End Record
index = data.Length;
lastreadindex = Math.Min(index, lastreadindex);

while ((index - lastreadindex) > 3)
{
copycount = (index - lastreadindex);
while (copycount > 0x71) copycount -= 0x71;
copycount = copycount & 0xfc;
int realcopycount = (copycount >> 2);

if (writeindex >= cdata.Length)
break;

cdata[writeindex++] = (byte)(0xdf + realcopycount);
for (int i = 0; i < copycount; i++)
cdata[writeindex++] = data[lastreadindex++];
}

copycount = index - lastreadindex;
if(writeindex < cdata.Length)
cdata[writeindex++] = (byte)(0xfc + copycount);

for (int i = 0; i < copycount; i++)
{
if (writeindex >= cdata.Length)
break;

cdata[writeindex++] = data[lastreadindex++];
}

#endregion

#region Trim Data & and add Header

//make a resulting Array of the apropriate size
retdata = new byte[writeindex + 18];

//0x01 = Compressed.
retdata[0] = 0x01;

//Decompressed size.
byte[] DecompressedSize = new byte[3];
DecompressedSize = BitConverter.GetBytes(data.Length);
for (int i = 0; i < 3; i++)
retdata[i + 1] = DecompressedSize[i];

//0x00 = Out Of Bounds character.
retdata[4] = 0x00;

//Stream body length.
byte[] sz = BitConverter.GetBytes((uint)(retdata.Length - 18));
for (int i = 0; i < 4; i++)
retdata[i + 5] = sz[i];

//Compressed size.
sz = BitConverter.GetBytes((uint)(retdata.Length));
for (int i = 0; i < 4; i++)
retdata[i + 9] = sz[i];

//Identification.
sz = BitConverter.GetBytes(m_MAGICNUMBER_QFS);
for (int i = 0; i < 2; i++)
retdata[i + 13] = sz[i];

//Decompressed size.
sz = BitConverter.GetBytes((uint)data.Length);
for (int i = 0; i < 3; i++)
retdata[i + 15] = sz[2 - i];

for (int i = 0; i < writeindex; i++)
retdata[i + 18] = cdata[i];

#endregion

return retdata;
}
finally
{
foreach (ArrayList a in cmpmap)
if (a != null) a.Clear();

cmpmap = null;
cdata = null;
retdata = null;
if (indexlist != null) indexlist.Clear();
indexlist = null;
}
//}
/*catch (Exception ex)
{
#if DEBUG
Helper.ExceptionMessage(ex);
#endif
//throw ex;
}*/

}

If any developers could tell me what I'm doing wrong, I'd appreciate it. I'm calling Compress() with a standard array of bytes containing the data of a *.png file.
pljonesLink to postposted: Thu Feb 03, 2011 8:27 pm
Avator for pljones

Member since:
 2005-04-02
From:
 London, UK
Posts:
 610
Sorry, the only help I can offer is that there's no point trying to compress a PNG -- the PNG compressor is much better at compressing image data than the package compressor is at compressing compressed image data.  You'll only make the data bigger than it already is.


viewthread, 0, 0, Compression-bug