/**
* @brief Compress a buffer with an existing @ref SquashOptions
*
* @param codec The name of the codec to use
* @param[out] compressed Location to store the compressed data
* @param[in,out] compressed_length Location storing the size of the
* @a compressed buffer on input, replaced with the actual size of
* the compressed data
* @param uncompressed The uncompressed data
* @param uncompressed_length Length of the uncompressed data (in bytes)
* @param options Compression options, or *NULL* to use the defaults
* @return A status code
*/
SquashStatus
squash_compress_with_options (const char* codec,
size_t* compressed_length,
uint8_t compressed[SQUASH_ARRAY_PARAM(*compressed_length)],
size_t uncompressed_length,
const uint8_t uncompressed[SQUASH_ARRAY_PARAM(uncompressed_length)],
SquashOptions* options) {
SquashCodec* codec_real = squash_get_codec (codec);
if (codec_real == NULL)
return squash_error (SQUASH_NOT_FOUND);
return squash_codec_compress_with_options (codec_real,
compressed_length, compressed,
uncompressed_length, uncompressed,
options);
}
/**
* @brief Compress a buffer
*
* @param codec The codec to use
* @param[out] compressed Location to store the compressed data
* @param[in,out] compressed_length Location storing the size of the
* @a compressed buffer on input, replaced with the actual size of
* the compressed data
* @param uncompressed The uncompressed data
* @param uncompressed_length Length of the uncompressed data (in bytes)
* @param ... A variadic list of key/value option pairs, followed by
* *NULL*
* @return A status code
*/
SquashStatus squash_codec_compress (SquashCodec* codec,
size_t* compressed_length,
uint8_t compressed[SQUASH_ARRAY_PARAM(*compressed_length)],
size_t uncompressed_length,
const uint8_t uncompressed[SQUASH_ARRAY_PARAM(uncompressed_length)],
...) {
SquashOptions* options;
va_list ap;
assert (codec != NULL);
va_start (ap, uncompressed);
options = squash_options_newv (codec, ap);
va_end (ap);
return squash_codec_compress_with_options (codec,
compressed_length, compressed,
uncompressed_length, uncompressed,
options);
}
/**
* @brief Compress a buffer
*
* @param codec The name of the codec to use
* @param[out] compressed Location to store the compressed data
* @param[in,out] compressed_length Location storing the size of the
* @a compressed buffer on input, replaced with the actual size of
* the compressed data
* @param uncompressed The uncompressed data
* @param uncompressed_length Length of the uncompressed data (in bytes)
* @param ... A variadic list of key/value option pairs, followed by
* *NULL*
* @return A status code
*/
SquashStatus
squash_compress (const char* codec,
size_t* compressed_length,
uint8_t compressed[SQUASH_ARRAY_PARAM(*compressed_length)],
size_t uncompressed_length,
const uint8_t uncompressed[SQUASH_ARRAY_PARAM(uncompressed_length)],
...) {
SquashOptions* options;
va_list ap;
SquashCodec* codec_real = squash_get_codec (codec);
if (codec_real == NULL)
return squash_error (SQUASH_NOT_FOUND);
va_start (ap, uncompressed);
options = squash_options_newv (codec_real, ap);
va_end (ap);
return squash_codec_compress_with_options (codec_real,
compressed_length, compressed,
uncompressed_length, uncompressed,
options);
}
int main(int argc, char *argv[])
{
int ret;
off_t size = 0, size_c = 0;
size_t len, len_c;
FILE *in;
void *in_buf, *out_buf;
uint64_t full_len, out_len;
struct stat st;
void *in_map = NULL;
void *out_map = NULL;
int out_fd;
double time = 0.0;
#ifdef USE_MMAP
uint64_t out_mapped;
#endif
struct timespec start, stop, start_full, stop_full;
if (argc < 4 || argc > 6)
help();
if (argc == 5 || argc == 6)
assert(*argv[1] == 's');
if (strlen(argv[1]) != 1)
help();
//file handling
in = fopen(argv[2], "r");
assert(in);
out_fd = open(argv[3], O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
assert(out_fd != -1);
fstat(fileno(in), &st);
full_len = st.st_size;
#ifdef USE_MMAP
switch(argv[1][0]) {
case 'e' : out_mapped = full_len*2; break;
case 'd' : out_mapped = CHUNK_SIZE*8; break;
case 'm' : out_mapped = full_len; break;
}
#endif
//initialisation
#ifndef USE_MMAP_READ
in_buf = malloc(CHUNK_SIZE);
#else
in_map = mmap(NULL, full_len, PROT_READ, MAP_SHARED, fileno(in), 0);
if (in_map == MAP_FAILED) {
printf("WARNING: mmap failed for %s, using regular write\n", argv[2]);
in_map = NULL;
in_buf = malloc(2*CHUNK_SIZE);
}
else
in_buf = in_map;
#endif
#ifndef USE_MMAP_WRITE
out_buf = malloc(2*CHUNK_SIZE);
#else
ftruncate(out_fd, out_mapped);
out_map = mmap(NULL, out_mapped, PROT_WRITE, MAP_SHARED, out_fd, 0);
if (out_map == MAP_FAILED) {
printf("WARNING: mmap failed for %s, using regular write\n", argv[3]);
out_map = NULL;
out_buf = malloc(2*CHUNK_SIZE);
}
else
out_buf = out_map;
#endif
switch(argv[1][0]) {
#ifdef BBP_USE_SIMDCOMP
case 'e' :
clock_gettime(CLOCK_MONOTONIC, &start_full);
#ifndef USE_MMAP_READ
while ((len = fread(in_buf, 1, CHUNK_SIZE, in))) {
#else
for(in_buf=in_map;in_buf-in_map<full_len;in_buf+=CHUNK_SIZE) {
len = full_len - (in_buf-in_map);
if (len > CHUNK_SIZE)
len = CHUNK_SIZE;
if (len % (128*4))
break;
#endif
int b;
clock_gettime(CLOCK_MONOTONIC, &start);
//compression
for(b=0;b<BENCHMARK_ITERATIONS;b++)
len_c = compress(in_buf, len/4, out_buf);
clock_gettime(CLOCK_MONOTONIC, &stop);
time += ms_delta(start, stop);
size += len;
size_c += len_c;
if (!out_map) {
len = write(out_fd, out_buf, len_c);
assert(len == len_c);
}
else
out_buf += len_c;
}
out_len = size_c;
clock_gettime(CLOCK_MONOTONIC, &stop_full);
printf("compressed at %.3f MB/s / %.3fMB/s ratio %.2f\n",(float)size*BENCHMARK_ITERATIONS/1024/1024*1000/time, (float)size/1024/1024*1000/ms_delta(start_full, stop_full), (float)size/size_c);
printf("%.2f %.3f simdcomp\n",(float)size/size_c, (float)size*BENCHMARK_ITERATIONS/1024/1024*1000/time);
break;
#endif
#ifdef BBP_USE_SQUASH
case 's' :
assert(argc == 5 || argc == 6);
SquashCodec* codec = squash_get_codec(argv[4]);
SquashOptions *options;
assert(codec);
if (argc == 6) {
options = squash_options_new(codec, NULL);
ret = squash_options_parse_option(options, "level", argv[5]);
if (ret != SQUASH_OK) {
if (ret == SQUASH_BAD_PARAM) printf("ERROR: SQUASH: bad param\n");
if (ret == SQUASH_BAD_VALUE) printf("ERROR: SQUASH: bad value\n");
break;
}
}
else
options = NULL;
clock_gettime(CLOCK_MONOTONIC, &start_full);
#ifndef USE_MMAP_READ
while ((len = fread(in_buf, 1, CHUNK_SIZE, in))) {
#else
for(in_buf=in_map;in_buf-in_map<full_len;in_buf+=CHUNK_SIZE) {
len = full_len - (in_buf-in_map);
if (len > CHUNK_SIZE)
len = CHUNK_SIZE;
#endif
clock_gettime(CLOCK_MONOTONIC, &start);
//compression
len_c = 2*CHUNK_SIZE;
if (options)
squash_codec_compress_with_options(codec, &len_c, out_buf, len, in_buf, options);
else
ret = squash_compress(argv[4], &len_c, out_buf, len, in_buf, NULL);
if (ret != SQUASH_OK)
printf("ERROR: SQUASH (%s): %s\n",squash_status_to_string(ret), argv[4]);
clock_gettime(CLOCK_MONOTONIC, &stop);
time += ms_delta(start, stop);
size += len;
size_c += len_c;
if (!out_map) {
len = write(out_fd, out_buf, len_c);
assert(len == len_c);
}
else
out_buf += len_c;
}
out_len = size_c;
clock_gettime(CLOCK_MONOTONIC, &stop_full);
printf("compressed at %.3fMB/s / %.3fMB/s ratio %.2f\n",(float)size/1024/1024*1000/time, (float)size/1024/1024*1000/ms_delta(start_full, stop_full), (float)size/size_c);
printf("%.2f %.3f %s\n",(float)size/size_c, (float)size/1024/1024*1000/time, argv[4]);
break;
#endif
}
if (in_map)
munmap(in_map, full_len);
fclose(in);
if (out_map)
munmap(out_map, full_len*2);
ftruncate(out_fd, out_len);
close(out_fd);
return EXIT_SUCCESS;
}
static SquashStatus
squash_codec_process_file_with_options (SquashCodec* codec,
SquashStreamType stream_type,
FILE* output,
size_t output_length,
FILE* input,
SquashOptions* options) {
SquashStatus res = SQUASH_FAILED;
SquashStream* stream;
if (codec->impl.process_stream == NULL) {
/* Attempt to mmap the input and output. This short circuits the
whole SquashBufferStream hack when possible, which can be a
huge performance boost (50-100% for several codecs). */
SquashMappedFile* in_map;
in_map = squash_mapped_file_new (input, false);
if (in_map != NULL) {
SquashMappedFile* out_map;
size_t max_output_size;
if (stream_type == SQUASH_STREAM_COMPRESS) {
max_output_size = squash_codec_get_max_compressed_size (codec, in_map->data_length);
} else if (codec->impl.get_uncompressed_size != NULL) {
max_output_size = squash_codec_get_uncompressed_size (codec, in_map->data_length, in_map->data);
} else if (output_length != 0) {
max_output_size = output_length;
} else {
max_output_size = in_map->data_length - 1;
max_output_size |= max_output_size >> 1;
max_output_size |= max_output_size >> 2;
max_output_size |= max_output_size >> 4;
max_output_size |= max_output_size >> 8;
max_output_size |= max_output_size >> 16;
max_output_size++;
max_output_size <<= 3;
}
out_map = squash_mapped_file_new_full(output, false, 0, max_output_size);
if (out_map != NULL) {
size_t output_size = out_map->data_length;
if (stream_type == SQUASH_STREAM_COMPRESS) {
output_size = out_map->data_length;
res = squash_codec_compress_with_options (codec, &output_size, out_map->data, in_map->data_length, in_map->data, options);
} else {
do {
output_size = max_output_size;
res = squash_codec_decompress_with_options (codec, &output_size, out_map->data, in_map->data_length, in_map->data, options);
max_output_size <<= 1;
} while (SQUASH_UNLIKELY(res == SQUASH_BUFFER_FULL) && output_length == 0 && codec->impl.get_uncompressed_size == NULL);
}
squash_mapped_file_free (out_map);
out_map = NULL;
/* We know that the mmap was successful, so not checking
fileno() should be safe. */
if (ftruncate (fileno (output), (off_t) output_size) == -1)
res = SQUASH_FAILED;
if (fseek (output, output_size, SEEK_SET) == -1)
res = SQUASH_FAILED;
}
if (res != SQUASH_OK) {
size_t ipos = in_map->data_offset;
squash_mapped_file_free (in_map);
fseek (input, ipos, SEEK_SET);
} else {
squash_mapped_file_free (in_map);
}
in_map = NULL;
}
