size_t up_pack_step( void* outbuf_ptr, size_t outbuf_size, size_t* data_left ) {
lib_zstream_pack.next_out = outbuf_ptr;
lib_zstream_pack.avail_out = outbuf_size;
size_t input_size, chunk_left;
int flush, result;
input_size = lib_zstream_pack.avail_in;
flush = (lib_pack_left > 0) ? Z_NO_FLUSH : Z_FINISH;
result = z_deflate( &lib_zstream_pack, flush );
switch (result) {
case Z_OK:
chunk_left = lib_zstream_pack.avail_in;
lib_pack_left -= input_size - chunk_left;
break;
case Z_STREAM_END:
chunk_left = 0;
lib_pack_flushed = true;
break;
default:
lib_error = result;
return 0;
}
if (data_left != NULL) { *data_left = chunk_left; }
return outbuf_size - lib_zstream_pack.avail_out;
}
bool ON_CompressStream::In( ON__UINT64 size, const void* uncompressed_buffer )
{
if ( size <= 0 )
return true;
if ( 0 == m_implementation )
{
ErrorHandler();
return false;
}
if ( 0 == uncompressed_buffer )
{
ErrorHandler();
return false;
}
struct ON_ZlibImplementation* imp = (struct ON_ZlibImplementation*)m_implementation;
z_stream& strm = imp->m_strm;
if ( 0 != strm.avail_in || 0 != strm.next_in )
{
// strm.avail_in is always zero when we leave an ON_CompressStream function.
ErrorHandler();
return false;
}
const ON__UINT32 sizeof_out_buffer = (ON__UINT32)(sizeof(imp->m_zlib_out_buffer));
void* out_buffer = imp->m_zlib_out_buffer;
int zrc = Z_OK;
const ON__UINT64 max_sz = 0x7FFFFFF0;
bool rc = false;
ON__UINT32 deflate_output_count;
// counter prevents infinte loops if there is a bug in zlib return codes.
for( int counter = 512; counter > 0; counter-- )
{
// Call zlib's deflate function. It can either process
// more input from m_zlib.strm.next_in[], create more
// compressed output in m_zlib.strm.next_out[], or do both.
// provide storage for compressed stream output
strm.next_out = (z_Bytef*)out_buffer;
strm.avail_out = sizeof_out_buffer;
if ( strm.avail_in <= 0 )
{
if ( size <= 0 )
{
// finshed with uncompressed input
break;
}
// submit a portion of uncompressed_buffer to zlib
ON__UINT64 sz = (size > max_sz) ? max_sz : size;
m_in_size += sz;
m_in_crc = ON_CRC32(m_in_crc,(size_t)sz,uncompressed_buffer); // (size_t) cast is safe because sz <= max_sz = 0x7FFFFFF0
strm.next_in = (z_Bytef*)uncompressed_buffer;
strm.avail_in = (ON__UINT32)sz;
uncompressed_buffer = ((const unsigned char*)uncompressed_buffer) + sz;
size -= sz;
counter = 512; // added input - reset the counter that detects stalls
}
// calculate compression
ON__UINT32 avail_in0 = strm.avail_in;
ON__UINT32 avail_out0 = strm.avail_out;
zrc = z_deflate( &strm, Z_NO_FLUSH );
if ( zrc < 0 )
{
// Something went haywire - bail out.
ErrorHandler();
rc = false;
break;
}
if ( strm.avail_in < avail_in0 || strm.avail_out > avail_out0 )
{
// zlib did something
rc = true;
}
deflate_output_count = sizeof_out_buffer - strm.avail_out;
if ( deflate_output_count > 0 )
{
// The last call to deflate created compressed output.
// Send the output to compressed stream handler.
// Calculate the updated crc and size before we call
// the output handler because someday sombody will
// decide it's a good idea to modify the values
// in the buffer argument.
ON__UINT32 out_crc1 = ON_CRC32( m_out_crc, deflate_output_count, out_buffer);
ON__UINT64 out_size1 = m_out_size + deflate_output_count;
rc = (0 != m_out_callback_function)
? m_out_callback_function( m_out_callback_context, deflate_output_count, out_buffer )
: Out( m_out_callback_context, deflate_output_count, out_buffer );
if ( !rc )
break;
// Update compressed stream crc and size
m_out_crc = out_crc1;
m_out_size = out_size1;
counter = 512; // created output - reset counter that detects stalls
}
if ( size <= 0 && strm.avail_in <= 0 )
{
// no input left
break;
}
}
strm.avail_in = 0;
strm.next_in = 0;
strm.next_out = 0;
strm.avail_out = 0;
return rc;
}
bool ON_CompressStream::End()
{
if ( 0 == m_implementation )
{
ErrorHandler();
return false;
}
struct ON_ZlibImplementation* imp = (struct ON_ZlibImplementation*)m_implementation;
z_stream& strm = imp->m_strm;
if ( 0 != strm.avail_in || 0 != strm.next_in )
{
// strm.avail_in is always zero when we leave an ON_CompressStream function.
ErrorHandler();
return false;
}
const ON__UINT32 sizeof_out_buffer = (ON__UINT32)(sizeof(imp->m_zlib_out_buffer));
void* out_buffer = imp->m_zlib_out_buffer;
int zrc = Z_OK;
bool rc = false;
ON__UINT32 deflate_output_count;
// counter prevents infinte loops if there is a bug in zlib return codes.
for( int counter = 512; counter > 0; counter-- )
{
// provide storage for compressed stream output
strm.avail_in = 0;
strm.next_in = 0;
strm.next_out = (z_Bytef*)out_buffer;
strm.avail_out = sizeof_out_buffer;
// finish compression calculation
zrc = z_deflate( &strm, Z_FINISH );
if ( zrc < 0 )
{
// Something went haywire - bail out.
ErrorHandler();
rc = false;
break;
}
deflate_output_count = sizeof_out_buffer - strm.avail_out;
if ( deflate_output_count > 0 )
{
// The last call to deflate created compressed output.
// Send the output to compressed stream handler.
// Calculate the updated crc and size before we call
// the output handler because someday sombody will
// decide it's a good idea to modify the values
// in the buffer argument.
ON__UINT32 compressed_crc1 = ON_CRC32( m_out_crc, deflate_output_count, out_buffer);
ON__UINT64 compressed_size1 = m_out_size + ((ON__UINT64)deflate_output_count);
rc = (0 != m_out_callback_function)
? m_out_callback_function( m_out_callback_context, deflate_output_count, out_buffer )
: Out( m_out_callback_context, deflate_output_count, out_buffer );
if ( !rc )
break;
// Update compressed stream crc and size
m_out_crc = compressed_crc1;
m_out_size = compressed_size1;
counter = 512; // created output - reset counter that detects stalls
}
if ( Z_STREAM_END == zrc )
{
// no input left, all pending compressing is finished,
// and all compressed output has been returned.
rc = true;
break;
}
}
strm.avail_in = 0;
strm.next_in = 0;
strm.next_out = 0;
strm.avail_out = 0;
deflateEnd(&strm);
onfree(m_implementation);
m_implementation = 0;
return rc;
}
size_t ON_BinaryArchive::WriteDeflate( // returns number of bytes written
size_t sizeof___inbuffer, // sizeof uncompressed input data ( > 0 )
const void* in___buffer // uncompressed input data ( != NULL )
)
{
/*
In "standard" (in 2005) 32 bit code
sizeof(int) = 4 bytes,
sizeof(long) = 4 bytes,
sizeof(pointer) = 4 bytes, and
sizeof(size_t) = 4 bytes.
Theoretically I don't need to use multiple input buffer
chunks in case. But I'm paranoid and I will use multiple
input chunks when sizeof_inbuffer > 2GB in order to dodge
any potential zlib signed verses unsigned compare bugs or
having a signed int i++ roll over to a negative number.
In "standard" code that has 64 bit pointers
sizeof(int) >= 4 bytes, (it's 4 on MS VS2005)
sizeof(long) >= 4 bytes, (it's 4 on MS VS2005)
sizeof(pointer) = 8 bytes, and
sizeof(size_t) = 8 bytes.
So, I'm going to assume the ints and longs in the zlib code
are 4 bytes, but I could have sizeof_inbuffer > 4GB.
This means I have to use multiple input buffer chunks.
In this case I still use multiple input chunks when
sizeof_inbuffer > 2GB in order to dodge any potential zlib
signed verses unsigned compare bugs or having a signed
int i++ roll over to a negative number.
So, I set
const size_t max_avail = (largest signed 4 byte integer - 15)
and feed inflate and deflate buffers with size <= max_avail.
This information below is from the zlib 1.2.3 FAQ.
32. Can zlib work with greater than 4 GB of data?
Yes. inflate() and deflate() will process any amount of data correctly.
Each call of inflate() or deflate() is limited to input and output chunks
of the maximum value that can be stored in the compiler's "unsigned int"
type, but there is no limit to the number of chunks. Note however that the
strm.total_in and strm_total_out counters may be limited to 4 GB. These
counters are provided as a convenience and are not used internally by
inflate() or deflate(). The application can easily set up its own counters
updated after each call of inflate() or deflate() to count beyond 4 GB.
compress() and uncompress() may be limited to 4 GB, since they operate in a
single call. gzseek() and gztell() may be limited to 4 GB depending on how
zlib is compiled. See the zlibCompileFlags() function in zlib.h.
The word "may" appears several times above since there is a 4 GB limit
only if the compiler's "long" type is 32 bits. If the compiler's "long"
type is 64 bits, then the limit is 16 exabytes.
*/
const size_t max_avail = 0x7FFFFFF0;
// Compressed information is saved in a chunk.
bool rc = BeginWrite3dmChunk(TCODE_ANONYMOUS_CHUNK,0);
if ( !rc )
return false;
size_t out__count = 0;
int zrc = Z_OK;
size_t my_avail_in = sizeof___inbuffer;
unsigned char* my_next_in = (unsigned char*)in___buffer;
size_t d = my_avail_in;
if ( d > max_avail )
d = max_avail;
m_zlib.strm.next_in = my_next_in;
m_zlib.strm.avail_in = (unsigned int)d;
my_avail_in -= d;
my_next_in += d;
m_zlib.strm.next_out = m_zlib.buffer;
m_zlib.strm.avail_out = m_zlib.sizeof_x_buffer;
// counter guards prevents infinte loops if there is a bug in zlib return codes.
int counter = 512;
int flush = Z_NO_FLUSH;
size_t deflate_output_count = 0;
while( rc && counter > 0 )
{
// Call zlib's deflate function. It can either process
// more input from m_zlib.strm.next_in[], create more
// compressed output in m_zlib.strm.next_out[], or do both.
if ( 0 == my_avail_in && 0 == m_zlib.strm.avail_in )
{
// no uncompressed input is left - switch to finish mode
flush = Z_FINISH;
}
zrc = z_deflate( &m_zlib.strm, flush );
if ( zrc < 0 )
{
// Something went haywire - bail out.
ON_ERROR("ON_BinaryArchive::WriteDeflate - z_deflate failure");
rc = false;
break;
}
deflate_output_count = m_zlib.sizeof_x_buffer - m_zlib.strm.avail_out;
if ( deflate_output_count > 0 )
{
// The last call to deflate created output. Send
// this output to the archive.
rc = WriteChar( deflate_output_count, m_zlib.buffer );
if ( !rc )
break;
out__count += deflate_output_count;
m_zlib.strm.next_out = m_zlib.buffer;
m_zlib.strm.avail_out = m_zlib.sizeof_x_buffer;
}
if ( Z_FINISH == flush && Z_STREAM_END == zrc )
{
// no input left, all pending compressing is finished,
// and all compressed output has been returned.
break;
}
if ( my_avail_in > 0 && m_zlib.strm.avail_in < max_avail )
{
// inbuffer[] had more than max_zlib_avail_in bytes in it
// and I am feeding inbuffer[] to deflate in smaller chunks
// that the 32 bit integers in the zlib code can handle.
if ( 0 == m_zlib.strm.avail_in || 0 == m_zlib.strm.next_in )
{
// The call to deflate() used up all the input
// in m_zlib.strm.next_in[]. I can feed it another chunk
// from inbuffer[]
d = my_avail_in;
if ( d > max_avail )
d = max_avail;
m_zlib.strm.next_in = my_next_in;
m_zlib.strm.avail_in = (unsigned int)d;
}
else
{
// The call to deflate left some input in m_zlib.strm.next_in[],
// but I can increase m_zlib.strm.avail_in.
d = max_avail - m_zlib.strm.avail_in;
if ( d > my_avail_in )
d = my_avail_in;
m_zlib.strm.avail_in += (unsigned int)d;
}
my_avail_in -= d;
my_next_in += d;
}
else if ( 0 == deflate_output_count )
{
// no buffer changes this time
counter--;
}
if ( zrc != Z_OK )
{
break;
}
}
if ( !EndWrite3dmChunk() )
{
rc = false;
}
if ( 0 == counter )
{
rc = false;
}
return (rc ? out__count : 0);
}