/*Compress the input buffer into the output buffer, but switch the flush type in
* the middle of the compression to test what happens*/
int compress_swap_flush(uint8_t * data, uint32_t data_size, uint8_t * compressed_buf,
uint32_t * compressed_size, uint32_t flush_type)
{
int ret = IGZIP_COMP_OK;
struct isal_zstream stream;
struct isal_zstate *state = &stream.internal_state;
uint32_t partial_size;
#ifdef VERBOSE
printf("Starting Compress Swap Flush\n");
#endif
isal_deflate_init(&stream);
if (state->state != ZSTATE_NEW_HDR)
return COMPRESS_INCORRECT_STATE;
partial_size = rand() % (data_size + 1);
stream.flush = flush_type;
stream.avail_in = partial_size;
stream.next_in = data;
stream.avail_out = *compressed_size;
stream.next_out = compressed_buf;
stream.end_of_stream = 0;
ret =
isal_deflate_with_checks(&stream, data_size, *compressed_size, data, partial_size,
partial_size, compressed_buf, *compressed_size, 0);
if (ret)
return ret;
if (flush_type == NO_FLUSH)
flush_type = SYNC_FLUSH;
else
flush_type = NO_FLUSH;
stream.flush = flush_type;
stream.avail_in = data_size - partial_size;
stream.next_in = data + partial_size;
stream.end_of_stream = 1;
ret =
isal_deflate_with_checks(&stream, data_size, *compressed_size, data + partial_size,
data_size - partial_size, data_size, compressed_buf,
*compressed_size, 0);
if (ret == COMPRESS_GENERAL_ERROR)
return INVALID_FLUSH_ERROR;
*compressed_size = stream.total_out;
return ret;
}
/* Statelessly compress the input buffer into the output buffer */
int compress_stateless(uint8_t * data, uint32_t data_size, uint8_t * compressed_buf,
uint32_t * compressed_size)
{
int ret = IGZIP_COMP_OK;
struct isal_zstream stream;
create_rand_repeat_data((uint8_t *) & stream, sizeof(stream));
isal_deflate_init(&stream);
stream.avail_in = data_size;
stream.end_of_stream = 1;
stream.next_in = data;
stream.flush = NO_FLUSH;
stream.avail_out = *compressed_size;
stream.next_out = compressed_buf;
ret = isal_deflate_stateless(&stream);
/* verify the stream */
if (stream.next_in - data != stream.total_in ||
stream.total_in + stream.avail_in != data_size)
return COMPRESS_INPUT_STREAM_INTEGRITY_ERROR;
if (stream.next_out - compressed_buf != stream.total_out ||
stream.total_out + stream.avail_out != *compressed_size)
return COMPRESS_OUTPUT_STREAM_INTEGRITY_ERROR;
if (ret != IGZIP_COMP_OK) {
if (ret == STATELESS_OVERFLOW)
return COMPRESS_OUT_BUFFER_OVERFLOW;
else
return COMPRESS_GENERAL_ERROR;
}
if (!stream.end_of_stream) {
return COMPRESS_END_OF_STREAM_NOT_SET;
}
if (stream.avail_in != 0)
return COMPRESS_ALL_INPUT_FAIL;
*compressed_size = stream.total_out;
return ret;
}
int main(int argc, char *argv[])
{
uint8_t inbuf[BUF_SIZE], outbuf[BUF_SIZE];
FILE *in, *out;
if (argc != 3) {
fprintf(stderr, "Usage: igzip_sync_flush_example infile outfile\n");
exit(0);
}
in = fopen(argv[1], "rb");
if (!in) {
fprintf(stderr, "Can't open %s for reading\n", argv[1]);
exit(0);
}
out = fopen(argv[2], "wb");
if (!out) {
fprintf(stderr, "Can't open %s for writing\n", argv[2]);
exit(0);
}
printf("igzip_sync_flush_example\nWindow Size: %d K\n", IGZIP_HIST_SIZE / 1024);
fflush(0);
isal_deflate_init(&stream);
stream.end_of_stream = 0;
stream.flush = SYNC_FLUSH;
do {
if (stream.internal_state.state == ZSTATE_NEW_HDR) {
stream.avail_in = (uint32_t) fread(inbuf, 1, BUF_SIZE, in);
stream.end_of_stream = feof(in) ? 1 : 0;
stream.next_in = inbuf;
}
do {
stream.avail_out = BUF_SIZE;
stream.next_out = outbuf;
isal_deflate(&stream);
fwrite(outbuf, 1, BUF_SIZE - stream.avail_out, out);
} while (stream.avail_out == 0);
} while (stream.internal_state.state != ZSTATE_END);
fclose(out);
fclose(in);
printf("End of igzip_sync_flush_example\n\n");
return 0;
}
/* Compress the input data into the outbuffer in one call to isal_deflate */
int compress_single_pass(uint8_t * data, uint32_t data_size, uint8_t * compressed_buf,
uint32_t * compressed_size, uint32_t flush_type)
{
int ret = IGZIP_COMP_OK;
struct isal_zstream stream;
struct isal_zstate *state = &stream.internal_state;
#ifdef VERBOSE
printf("Starting Compress Single Pass\n");
#endif
create_rand_repeat_data((uint8_t *) & stream, sizeof(stream));
isal_deflate_init(&stream);
if (state->state != ZSTATE_NEW_HDR)
return COMPRESS_INCORRECT_STATE;
stream.flush = flush_type;
stream.avail_in = data_size;
stream.next_in = data;
stream.avail_out = *compressed_size;
stream.next_out = compressed_buf;
stream.end_of_stream = 1;
ret =
isal_deflate_with_checks(&stream, data_size, *compressed_size, data, data_size,
data_size, compressed_buf, *compressed_size, 0);
/* Check if the compression is completed */
if (state->state == ZSTATE_END)
*compressed_size = stream.total_out;
else if (flush_type == SYNC_FLUSH && stream.avail_out < 16)
ret = COMPRESS_OUT_BUFFER_OVERFLOW;
return ret;
}
int compress_file(void)
{
FILE *in = NULL, *out = NULL;
unsigned char *inbuf = NULL, *outbuf = NULL, *level_buf = NULL;
size_t inbuf_size, outbuf_size;
int level_size = 0;
struct isal_zstream stream;
struct isal_gzip_header gz_hdr;
int ret, success = 0;
char *infile_name = global_options.infile_name, *outfile_name =
global_options.outfile_name;
char *suffix = global_options.suffix;
size_t infile_name_len = global_options.infile_name_len;
size_t outfile_name_len = global_options.outfile_name_len;
size_t suffix_len = global_options.suffix_len;
int level = global_options.level;
if (suffix == NULL) {
suffix = default_suffixes[0];
suffix_len = default_suffixes_lens[0];
}
if (infile_name_len == stdin_file_name_len &&
memcmp(infile_name, stdin_file_name, infile_name_len) == 0) {
infile_name = NULL;
infile_name_len = 0;
}
if (outfile_name == NULL && infile_name != NULL && !global_options.use_stdout) {
outfile_name_len = infile_name_len + suffix_len;
outfile_name = malloc_safe(outfile_name_len + 1);
strcpy(outfile_name, infile_name);
strcat(outfile_name, suffix);
}
open_in_file(&in, infile_name);
if (in == NULL)
goto compress_file_cleanup;
if (infile_name_len != 0 && infile_name_len == outfile_name_len
&& strncmp(infile_name, outfile_name, infile_name_len) == 0) {
log_print(ERROR, "igzip: Error input and output file names must differ\n");
goto compress_file_cleanup;
}
open_out_file(&out, outfile_name);
if (out == NULL)
goto compress_file_cleanup;
inbuf_size = BLOCK_SIZE;
outbuf_size = BLOCK_SIZE;
inbuf = malloc_safe(inbuf_size);
outbuf = malloc_safe(outbuf_size);
level_size = level_size_buf[level];
level_buf = malloc_safe(level_size);
isal_gzip_header_init(&gz_hdr);
if (global_options.name == NAME_DEFAULT || global_options.name == YES_NAME) {
gz_hdr.time = get_posix_filetime(in);
gz_hdr.name = infile_name;
}
gz_hdr.os = UNIX;
gz_hdr.name_buf_len = infile_name_len + 1;
isal_deflate_init(&stream);
stream.avail_in = 0;
stream.flush = NO_FLUSH;
stream.level = level;
stream.level_buf = level_buf;
stream.level_buf_size = level_size;
stream.gzip_flag = IGZIP_GZIP_NO_HDR;
stream.next_out = outbuf;
stream.avail_out = outbuf_size;
isal_write_gzip_header(&stream, &gz_hdr);
do {
if (stream.avail_in == 0) {
stream.next_in = inbuf;
stream.avail_in =
fread_safe(stream.next_in, 1, inbuf_size, in, infile_name);
stream.end_of_stream = feof(in);
}
if (stream.next_out == NULL) {
stream.next_out = outbuf;
stream.avail_out = outbuf_size;
}
ret = isal_deflate(&stream);
if (ret != ISAL_DECOMP_OK) {
log_print(ERROR,
"igzip: Error encountered while compressing file %s\n",
infile_name);
goto compress_file_cleanup;
}
fwrite_safe(outbuf, 1, stream.next_out - outbuf, out, outfile_name);
stream.next_out = NULL;
} while (!feof(in) || stream.avail_out == 0);
success = 1;
compress_file_cleanup:
if (out != NULL && out != stdout)
fclose(out);
if (in != NULL && in != stdin) {
fclose(in);
if (success && global_options.remove)
remove(infile_name);
}
if (global_options.outfile_name == NULL && outfile_name != NULL)
free(outfile_name);
if (inbuf != NULL)
free(inbuf);
if (outbuf != NULL)
free(outbuf);
if (level_buf != NULL)
free(level_buf);
return (success == 0);
}
int isal_deflate_stateless(struct isal_zstream *stream)
{
uint8_t *next_in = stream->next_in;
const uint32_t avail_in = stream->avail_in;
uint8_t *next_out = stream->next_out;
const uint32_t avail_out = stream->avail_out;
uint32_t crc32 = 0;
uint32_t stored_len;
uint32_t dyn_min_len;
uint32_t min_len;
uint32_t select_stored_blk = 0;
if (avail_in == 0)
stored_len = STORED_BLK_HDR_BZ;
else
stored_len =
STORED_BLK_HDR_BZ * ((avail_in + STORED_BLK_MAX_BZ - 1) /
STORED_BLK_MAX_BZ) + avail_in;
/*
at least 1 byte compressed data in the case of empty dynamic block which only
contains the EOB
*/
dyn_min_len = stream->hufftables->deflate_hdr_count + 1;
#ifndef DEFLATE
dyn_min_len += gzip_hdr_bytes + gzip_trl_bytes + 1;
stored_len += gzip_hdr_bytes + gzip_trl_bytes;
#endif
min_len = dyn_min_len;
if (stored_len < dyn_min_len) {
min_len = stored_len;
select_stored_blk = 1;
}
/*
the output buffer should be no less than 8 bytes
while empty stored deflate block is 5 bytes only
*/
if (avail_out < min_len || stream->avail_out < 8)
return STATELESS_OVERFLOW;
if (!select_stored_blk) {
if (isal_deflate_int_stateless(stream, next_in, avail_in) == COMP_OK)
return COMP_OK;
}
if (avail_out < stored_len)
return STATELESS_OVERFLOW;
isal_deflate_init(stream);
stream->next_in = next_in;
stream->avail_in = avail_in;
stream->total_in = 0;
stream->next_out = next_out;
stream->avail_out = avail_out;
stream->total_out = 0;
#ifndef DEFLATE
crc32 = crc32_gzip(0x0, next_in, avail_in);
#endif
return write_stored_block_stateless(stream, stored_len, crc32);
}
int main(int argc, char *argv[])
{
FILE *in, *out = NULL;
unsigned char *inbuf, *outbuf;
int i, infile_size, iterations, outbuf_size;
if (argc > 3 || argc < 2) {
fprintf(stderr, "Usage: igzip_sync_flush_file_perf infile [outfile]\n"
"\t - Runs multiple iterations of igzip on a file to get more accurate time results.\n");
exit(0);
}
in = fopen(argv[1], "rb");
if (!in) {
fprintf(stderr, "Can't open %s for reading\n", argv[1]);
exit(0);
}
if (argc > 2) {
out = fopen(argv[2], "wb");
if (!out) {
fprintf(stderr, "Can't open %s for writing\n", argv[2]);
exit(0);
}
printf("outfile=%s\n", argv[2]);
}
printf("Window Size: %d K\n", HIST_SIZE);
printf("igzip_sync_flush_file_perf: \n");
fflush(0);
/* Allocate space for entire input file and
* output (assuming 1:1 max output size)
*/
infile_size = get_filesize(in);
if (infile_size != 0) {
outbuf_size = infile_size;
iterations = RUN_MEM_SIZE / infile_size;
} else {
outbuf_size = BUF_SIZE;
iterations = MIN_TEST_LOOPS;
}
if (iterations < MIN_TEST_LOOPS)
iterations = MIN_TEST_LOOPS;
inbuf = malloc(infile_size);
if (inbuf == NULL) {
fprintf(stderr, "Can't allocate input buffer memory\n");
exit(0);
}
outbuf = malloc(outbuf_size);
if (outbuf == NULL) {
fprintf(stderr, "Can't allocate output buffer memory\n");
exit(0);
}
printf("igzip_sync_flush_file_perf: %s %d iterations\n", argv[1], iterations);
/* Read complete input file into buffer */
stream.avail_in = (uint32_t) fread(inbuf, 1, infile_size, in);
if (stream.avail_in != infile_size) {
fprintf(stderr, "Couldn't fit all of input file into buffer\n");
exit(0);
}
struct perf start, stop;
perf_start(&start);
for (i = 0; i < iterations; i++) {
isal_deflate_init(&stream);
stream.end_of_stream = 0;
stream.flush = SYNC_FLUSH;
stream.next_in = inbuf;
stream.avail_in = infile_size / 2;
stream.next_out = outbuf;
stream.avail_out = outbuf_size / 2;
isal_deflate(&stream);
if (infile_size == 0)
continue;
stream.avail_in = infile_size - infile_size / 2;
stream.end_of_stream = 1;
stream.next_in = inbuf + stream.total_in;
stream.flush = SYNC_FLUSH;
stream.avail_out = infile_size - outbuf_size / 2;
stream.next_out = outbuf + stream.total_out;
isal_deflate(&stream);
if (stream.avail_in != 0)
break;
}
perf_stop(&stop);
if (stream.avail_in != 0) {
fprintf(stderr, "Could not compress all of inbuf\n");
exit(0);
}
printf(" file %s - in_size=%d out_size=%d iter=%d ratio=%3.1f%%\n", argv[1],
infile_size, stream.total_out, i, 100.0 * stream.total_out / infile_size);
printf("igzip_file: ");
perf_print(stop, start, (long long)infile_size * i);
if (argc > 2 && out) {
printf("writing %s\n", argv[2]);
fwrite(outbuf, 1, stream.total_out, out);
fclose(out);
}
fclose(in);
printf("End of igzip_sync_flush_file_perf\n\n");
fflush(0);
return 0;
}
/* Compress the input data into the output buffer where the input buffer and
* output buffer are randomly segmented to test state information for the
* compression*/
int compress_multi_pass(uint8_t * data, uint32_t data_size, uint8_t * compressed_buf,
uint32_t * compressed_size, uint32_t flush_type)
{
int ret = IGZIP_COMP_OK;
uint8_t *in_buf = NULL, *out_buf = NULL;
uint32_t in_size = 0, out_size = 0;
uint32_t in_processed = 0, out_processed = 0;
struct isal_zstream stream;
struct isal_zstate *state = &stream.internal_state;
uint32_t loop_count = 0;
#ifdef VERBOSE
printf("Starting Compress Multi Pass\n");
#endif
create_rand_repeat_data((uint8_t *) & stream, sizeof(stream));
isal_deflate_init(&stream);
if (state->state != ZSTATE_NEW_HDR)
return COMPRESS_INCORRECT_STATE;
stream.flush = flush_type;
stream.end_of_stream = 0;
/* These are set here to allow the loop to run correctly */
stream.avail_in = 0;
stream.avail_out = 0;
while (1) {
loop_count++;
/* Setup in buffer for next round of compression */
if (stream.avail_in == 0) {
if (flush_type != SYNC_FLUSH || state->state == ZSTATE_NEW_HDR) {
/* Randomly choose size of the next out buffer */
in_size = rand() % (data_size + 1);
/* Limit size of buffer to be smaller than maximum */
if (in_size >= data_size - in_processed) {
in_size = data_size - in_processed;
stream.end_of_stream = 1;
}
if (in_size != 0) {
if (in_buf != NULL) {
free(in_buf);
in_buf = NULL;
}
in_buf = malloc(in_size);
if (in_buf == NULL) {
ret = MALLOC_FAILED;
break;
}
memcpy(in_buf, data + in_processed, in_size);
in_processed += in_size;
stream.avail_in = in_size;
stream.next_in = in_buf;
}
}
}
/* Setup out buffer for next round of compression */
if (stream.avail_out == 0) {
/* Save compressed data inot compressed_buf */
if (out_buf != NULL) {
memcpy(compressed_buf + out_processed, out_buf,
out_size - stream.avail_out);
out_processed += out_size - stream.avail_out;
}
/* Randomly choose size of the next out buffer */
out_size = rand() % (*compressed_size + 1);
/* Limit size of buffer to be smaller than maximum */
if (out_size > *compressed_size - out_processed)
out_size = *compressed_size - out_processed;
if (out_size != 0) {
if (out_buf != NULL) {
free(out_buf);
out_buf = NULL;
}
out_buf = malloc(out_size);
if (out_buf == NULL) {
ret = MALLOC_FAILED;
break;
}
stream.avail_out = out_size;
stream.next_out = out_buf;
}
}
ret =
isal_deflate_with_checks(&stream, data_size, *compressed_size, in_buf,
in_size, in_processed, out_buf, out_size,
out_processed);
if (ret) {
if (ret == COMPRESS_OUT_BUFFER_OVERFLOW
|| ret == COMPRESS_INCORRECT_STATE)
memcpy(compressed_buf + out_processed, out_buf, out_size);
break;
}
/* Check if the compression is completed */
if (state->state == ZSTATE_END) {
memcpy(compressed_buf + out_processed, out_buf, out_size);
*compressed_size = stream.total_out;
break;
}
}
if (in_buf != NULL)
free(in_buf);
if (out_buf != NULL)
free(out_buf);
if (ret == COMPRESS_OUT_BUFFER_OVERFLOW && flush_type == SYNC_FLUSH
&& loop_count >= MAX_LOOPS)
ret = COMPRESS_LOOP_COUNT_OVERFLOW;
return ret;
}
