static void
parse_block_list(char *str)
{
// It must be non-empty and not begin with a comma.
if (str[0] == '\0' || str[0] == ',')
message_fatal(_("%s: Invalid argument to --block-list"), str);
// Count the number of comma-separated strings.
size_t count = 1;
for (size_t i = 0; str[i] != '\0'; ++i)
if (str[i] == ',')
++count;
// Prevent an unlikely integer overflow.
if (count > SIZE_MAX / sizeof(uint64_t) - 1)
message_fatal(_("%s: Too many arguments to --block-list"),
str);
// Allocate memory to hold all the sizes specified.
// If --block-list was specified already, its value is forgotten.
free(opt_block_list);
opt_block_list = xmalloc((count + 1) * sizeof(uint64_t));
for (size_t i = 0; i < count; ++i) {
// Locate the next comma and replace it with \0.
char *p = strchr(str, ',');
if (p != NULL)
*p = '\0';
if (str[0] == '\0') {
// There is no string, that is, a comma follows
// another comma. Use the previous value.
//
// NOTE: We checked earler that the first char
// of the whole list cannot be a comma.
assert(i > 0);
opt_block_list[i] = opt_block_list[i - 1];
} else {
opt_block_list[i] = str_to_uint64("block-list", str,
0, UINT64_MAX);
// Zero indicates no more new Blocks.
if (opt_block_list[i] == 0) {
if (i + 1 != count)
message_fatal(_("0 can only be used "
"as the last element "
"in --block-list"));
opt_block_list[i] = UINT64_MAX;
}
}
str = p + 1;
}
// Terminate the array.
opt_block_list[count] = 0;
return;
}
extern void *xalloc(size_t size)
{
if (size == 0)
message_fatal("Unsupported request for zero byte allocation");
void *ptr = malloc(size);
if (ptr == NULL)
message_fatal("Unable to allocate memory");
return ptr;
}
extern void bitmap_write(struct bitmap *bitmap)
{
if (!output_path && is_tty_stdout())
message_fatal("Data cannot be written to a terminal");
FILE *fp = output_path ? fopen(output_path, "wb") : stdout;
if (fp == NULL)
message_fatal("Unable to open file for writing");
png_structp png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING,
NULL, NULL, NULL);
if (png_ptr == NULL)
message_fatal("Unable to create png struct");
png_infop info_ptr = png_create_info_struct(png_ptr);
if (info_ptr == NULL)
message_fatal("Unable to create png info struct");
if (setjmp(png_jmpbuf(png_ptr))) {
fclose(fp);
png_destroy_write_struct(&png_ptr, &info_ptr);
message_fatal("Error writing png file");
}
png_init_io(png_ptr, fp);
png_set_IHDR(png_ptr, info_ptr, bitmap->width, bitmap->height, 8,
PNG_COLOR_TYPE_PALETTE, PNG_INTERLACE_NONE,
PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
png_color *palette = palette_create();
png_set_PLTE(png_ptr, info_ptr, palette, 256);
png_bytep row_pointers[bitmap->height];
for (unsigned y = 0; y < bitmap->height; y++)
row_pointers[y] = bitmap->data + bitmap->width * y;
png_write_info(png_ptr, info_ptr);
png_write_image(png_ptr, row_pointers);
png_write_end(png_ptr, info_ptr);
palette_release(palette);
png_destroy_write_struct(&png_ptr, &info_ptr);
fclose(fp);
}
extern void bitmap_set_width(unsigned value)
{
if (value == 0)
message_fatal("Bitmap width must be positive");
arg_width = value;
}
extern void bitmap_set_height(unsigned value)
{
if (value == 0)
message_fatal("Bitmap height must be positive");
arg_height = value;
}
void set_period(double value)
{
if (value < 0.0)
message_fatal("Period must be positive");
period = value;
hist_min = 0.0;
hist_max = period;
}
extern void *
xrealloc(void *ptr, size_t size)
{
assert(size > 0);
ptr = realloc(ptr, size);
if (ptr == NULL)
message_fatal("%s", strerror(errno));
return ptr;
}
extern void
coder_add_filter(lzma_vli id, void *options)
{
if (filters_count == LZMA_FILTERS_MAX)
message_fatal(_("Maximum number of filters is four"));
filters[filters_count].id = id;
filters[filters_count].options = options;
++filters_count;
return;
}
extern void
suffix_set(const char *suffix)
{
// Empty suffix and suffixes having a directory separator are
// rejected. Such suffixes would break things later.
if (suffix[0] == '\0' || has_dir_sep(suffix))
message_fatal(_("%s: Invalid filename suffix"), optarg);
// Replace the old custom_suffix (if any) with the new suffix.
free(custom_suffix);
custom_suffix = xstrdup(suffix);
return;
}
extern void
list_file(const char *filename)
{
if (opt_format != FORMAT_XZ && opt_format != FORMAT_AUTO)
message_fatal(_("--list works only on .xz files "
"(--format=xz or --format=auto)"));
message_filename(filename);
if (filename == stdin_filename) {
message_error(_("--list does not support reading from "
"standard input"));
return;
}
// Unset opt_stdout so that io_open_src() won't accept special files.
// Set opt_force so that io_open_src() will follow symlinks.
opt_stdout = false;
opt_force = true;
file_pair *pair = io_open_src(filename);
if (pair == NULL)
return;
xz_file_info xfi = XZ_FILE_INFO_INIT;
if (!parse_indexes(&xfi, pair)) {
bool fail;
// We have three main modes:
// - --robot, which has submodes if --verbose is specified
// once or twice
// - Normal --list without --verbose
// - --list with one or two --verbose
if (opt_robot)
fail = print_info_robot(&xfi, pair);
else if (message_verbosity_get() <= V_WARNING)
fail = print_info_basic(&xfi, pair);
else
fail = print_info_adv(&xfi, pair);
// Update the totals that are displayed after all
// the individual files have been listed. Don't count
// broken files.
if (!fail)
update_totals(&xfi);
lzma_index_end(xfi.idx, NULL);
}
io_close(pair, false);
return;
}
extern void
coder_add_filter(lzma_vli id, void *options)
{
if (filters_count == LZMA_FILTERS_MAX)
message_fatal(_("Maximum number of filters is four"));
filters[filters_count].id = id;
filters[filters_count].options = options;
++filters_count;
// Setting a custom filter chain makes us forget the preset options.
// This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
// where the custom filter chain resets the preset level back to
// the default 6, making the example equivalent to "xz -6e".
preset_number = LZMA_PRESET_DEFAULT;
return;
}
extern void *
xrealloc(void *ptr, size_t size)
{
assert(size > 0);
// Save ptr so that we can free it if realloc fails.
// The point is that message_fatal ends up calling stdio functions
// which in some libc implementations might allocate memory from
// the heap. Freeing ptr improves the chances that there's free
// memory for stdio functions if they need it.
void *p = ptr;
ptr = realloc(ptr, size);
if (ptr == NULL) {
const int saved_errno = errno;
free(p);
message_fatal("%s", strerror(saved_errno));
}
return ptr;
}
void read_input(void)
{
FILE *in;
int i, j, *nbin, *nsim;
in = input ? fopen(input, "r") : stdin;
if (in == NULL)
message_fatal("Unable to open input file");
skip_comment(in);
if (fscanf(in, "%d", &sim_count) != 1)
message_fatal("Unable to read number of simulations");
if (sim_count <= 0)
message_fatal("Expected positive number of simulations");
bias_x = xmalloc(sim_count * sizeof(*bias_x));
bias_k = xmalloc(sim_count * sizeof(*bias_k));
log_nbin = xmalloc(bin_count * sizeof(*log_nbin));
log_nsim = xmalloc(sim_count * sizeof(*log_nsim));
nbin = xmalloc(bin_count * sizeof(*nbin));
nsim = xmalloc(sim_count * sizeof(*nsim));
for (i = 0; i < bin_count; i++)
nbin[i] = 0;
for (i = 0; i < sim_count; i++) {
double x;
int npt;
if (fscanf(in, "%lf %lf", &bias_x[i], &bias_k[i]) != 2)
message_fatal("Error reading bias info");
if (fscanf(in, "%d", &npt) != 1)
message_fatal("Error reading point count");
if (npt <= 0)
message_fatal("Expected positive number of points");
nsim[i] = 0;
for (j = 0; j < npt; j++) {
if (fscanf(in, "%lf", &x) != 1)
message_fatal("Error reading data points");
if (period > 0) {
if (x > 0)
while (x > period)
x -= period;
else
while (x < 0)
x += period;
}
if (x > hist_min && x < hist_max) {
nbin[bin_index(x)]++;
nsim[i]++;
}
}
message(V_VERBOSE, "Added %d of %d points from window %d",
nsim[i], npt, i + 1);
}
for (i = 0; i < bin_count; i++)
log_nbin[i] = log(nbin[i]);
for (i = 0; i < sim_count; i++)
log_nsim[i] = log(nsim[i]);
if (in != stdin)
fclose(in);
free(nbin);
free(nsim);
}
static void
parse_environment(args_info *args, char *argv0, const char *varname)
{
char *env = getenv(varname);
if (env == NULL)
return;
// We modify the string, so make a copy of it.
env = xstrdup(env);
// Calculate the number of arguments in env. argc stats at one
// to include space for the program name.
int argc = 1;
bool prev_was_space = true;
for (size_t i = 0; env[i] != '\0'; ++i) {
// NOTE: Cast to unsigned char is needed so that correct
// value gets passed to isspace(), which expects
// unsigned char cast to int. Casting to int is done
// automatically due to integer promotion, but we need to
// force char to unsigned char manually. Otherwise 8-bit
// characters would get promoted to wrong value if
// char is signed.
if (isspace((unsigned char)env[i])) {
prev_was_space = true;
} else if (prev_was_space) {
prev_was_space = false;
// Keep argc small enough to fit into a singed int
// and to keep it usable for memory allocation.
if (++argc == my_min(
INT_MAX, SIZE_MAX / sizeof(char *)))
message_fatal(_("The environment variable "
"%s contains too many "
"arguments"), varname);
}
}
// Allocate memory to hold pointers to the arguments. Add one to get
// space for the terminating NULL (if some systems happen to need it).
char **argv = xmalloc(((size_t)(argc) + 1) * sizeof(char *));
argv[0] = argv0;
argv[argc] = NULL;
// Go through the string again. Split the arguments using '\0'
// characters and add pointers to the resulting strings to argv.
argc = 1;
prev_was_space = true;
for (size_t i = 0; env[i] != '\0'; ++i) {
if (isspace((unsigned char)env[i])) {
prev_was_space = true;
env[i] = '\0';
} else if (prev_was_space) {
prev_was_space = false;
argv[argc++] = env + i;
}
}
// Parse the argument list we got from the environment. All non-option
// arguments i.e. filenames are ignored.
parse_real(args, argc, argv);
// Reset the state of the getopt_long() so that we can parse the
// command line options too. There are two incompatible ways to
// do it.
#ifdef HAVE_OPTRESET
// BSD
optind = 1;
optreset = 1;
#else
// GNU, Solaris
optind = 0;
#endif
// We don't need the argument list from environment anymore.
free(argv);
free(env);
return;
}
static void
parse_real(args_info *args, int argc, char **argv)
{
enum {
OPT_X86 = INT_MIN,
OPT_POWERPC,
OPT_IA64,
OPT_ARM,
OPT_ARMTHUMB,
OPT_SPARC,
OPT_DELTA,
OPT_LZMA1,
OPT_LZMA2,
OPT_SINGLE_STREAM,
OPT_NO_SPARSE,
OPT_FILES,
OPT_FILES0,
OPT_BLOCK_SIZE,
OPT_MEM_COMPRESS,
OPT_MEM_DECOMPRESS,
OPT_NO_ADJUST,
OPT_INFO_MEMORY,
OPT_ROBOT,
};
static const char short_opts[]
= "cC:defF:hHlkM:qQrS:tT:vVz0123456789";
static const struct option long_opts[] = {
// Operation mode
{ "compress", no_argument, NULL, 'z' },
{ "decompress", no_argument, NULL, 'd' },
{ "uncompress", no_argument, NULL, 'd' },
{ "test", no_argument, NULL, 't' },
{ "list", no_argument, NULL, 'l' },
// Operation modifiers
{ "keep", no_argument, NULL, 'k' },
{ "force", no_argument, NULL, 'f' },
{ "stdout", no_argument, NULL, 'c' },
{ "to-stdout", no_argument, NULL, 'c' },
{ "single-stream", no_argument, NULL, OPT_SINGLE_STREAM },
{ "no-sparse", no_argument, NULL, OPT_NO_SPARSE },
{ "suffix", required_argument, NULL, 'S' },
// { "recursive", no_argument, NULL, 'r' }, // TODO
{ "files", optional_argument, NULL, OPT_FILES },
{ "files0", optional_argument, NULL, OPT_FILES0 },
// Basic compression settings
{ "format", required_argument, NULL, 'F' },
{ "check", required_argument, NULL, 'C' },
{ "block-size", required_argument, NULL, OPT_BLOCK_SIZE },
{ "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS },
{ "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS },
{ "memlimit", required_argument, NULL, 'M' },
{ "memory", required_argument, NULL, 'M' }, // Old alias
{ "no-adjust", no_argument, NULL, OPT_NO_ADJUST },
{ "threads", required_argument, NULL, 'T' },
{ "extreme", no_argument, NULL, 'e' },
{ "fast", no_argument, NULL, '0' },
{ "best", no_argument, NULL, '9' },
// Filters
{ "lzma1", optional_argument, NULL, OPT_LZMA1 },
{ "lzma2", optional_argument, NULL, OPT_LZMA2 },
{ "x86", optional_argument, NULL, OPT_X86 },
{ "powerpc", optional_argument, NULL, OPT_POWERPC },
{ "ia64", optional_argument, NULL, OPT_IA64 },
{ "arm", optional_argument, NULL, OPT_ARM },
{ "armthumb", optional_argument, NULL, OPT_ARMTHUMB },
{ "sparc", optional_argument, NULL, OPT_SPARC },
{ "delta", optional_argument, NULL, OPT_DELTA },
// Other options
{ "quiet", no_argument, NULL, 'q' },
{ "verbose", no_argument, NULL, 'v' },
{ "no-warn", no_argument, NULL, 'Q' },
{ "robot", no_argument, NULL, OPT_ROBOT },
{ "info-memory", no_argument, NULL, OPT_INFO_MEMORY },
{ "help", no_argument, NULL, 'h' },
{ "long-help", no_argument, NULL, 'H' },
{ "version", no_argument, NULL, 'V' },
{ NULL, 0, NULL, 0 }
};
int c;
while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL))
!= -1) {
switch (c) {
// Compression preset (also for decompression if --format=raw)
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
coder_set_preset(c - '0');
break;
// --memlimit-compress
case OPT_MEM_COMPRESS:
parse_memlimit("memlimit-compress",
"memlimit-compress%", optarg,
true, false);
break;
// --memlimit-decompress
case OPT_MEM_DECOMPRESS:
parse_memlimit("memlimit-decompress",
"memlimit-decompress%", optarg,
false, true);
break;
// --memlimit
case 'M':
parse_memlimit("memlimit", "memlimit%", optarg,
true, true);
break;
// --suffix
case 'S':
suffix_set(optarg);
break;
case 'T':
// The max is from src/liblzma/common/common.h.
hardware_threads_set(str_to_uint64("threads",
optarg, 0, 16384));
break;
// --version
case 'V':
// This doesn't return.
message_version();
// --stdout
case 'c':
opt_stdout = true;
break;
// --decompress
case 'd':
opt_mode = MODE_DECOMPRESS;
break;
// --extreme
case 'e':
coder_set_extreme();
break;
// --force
case 'f':
opt_force = true;
break;
// --info-memory
case OPT_INFO_MEMORY:
// This doesn't return.
hardware_memlimit_show();
// --help
case 'h':
// This doesn't return.
message_help(false);
// --long-help
case 'H':
// This doesn't return.
message_help(true);
// --list
case 'l':
opt_mode = MODE_LIST;
break;
// --keep
case 'k':
opt_keep_original = true;
break;
// --quiet
case 'q':
message_verbosity_decrease();
break;
case 'Q':
set_exit_no_warn();
break;
case 't':
opt_mode = MODE_TEST;
break;
// --verbose
case 'v':
message_verbosity_increase();
break;
// --robot
case OPT_ROBOT:
opt_robot = true;
// This is to make sure that floating point numbers
// always have a dot as decimal separator.
setlocale(LC_NUMERIC, "C");
break;
case 'z':
opt_mode = MODE_COMPRESS;
break;
// Filter setup
case OPT_X86:
coder_add_filter(LZMA_FILTER_X86,
options_bcj(optarg));
break;
case OPT_POWERPC:
coder_add_filter(LZMA_FILTER_POWERPC,
options_bcj(optarg));
break;
case OPT_IA64:
coder_add_filter(LZMA_FILTER_IA64,
options_bcj(optarg));
break;
case OPT_ARM:
coder_add_filter(LZMA_FILTER_ARM,
options_bcj(optarg));
break;
case OPT_ARMTHUMB:
coder_add_filter(LZMA_FILTER_ARMTHUMB,
options_bcj(optarg));
break;
case OPT_SPARC:
coder_add_filter(LZMA_FILTER_SPARC,
options_bcj(optarg));
break;
case OPT_DELTA:
coder_add_filter(LZMA_FILTER_DELTA,
options_delta(optarg));
break;
case OPT_LZMA1:
coder_add_filter(LZMA_FILTER_LZMA1,
options_lzma(optarg));
break;
case OPT_LZMA2:
coder_add_filter(LZMA_FILTER_LZMA2,
options_lzma(optarg));
break;
// Other
// --format
case 'F': {
// Just in case, support both "lzma" and "alone" since
// the latter was used for forward compatibility in
// LZMA Utils 4.32.x.
static const struct {
char str[8];
enum format_type format;
} types[] = {
{ "auto", FORMAT_AUTO },
{ "xz", FORMAT_XZ },
{ "lzma", FORMAT_LZMA },
{ "alone", FORMAT_LZMA },
// { "gzip", FORMAT_GZIP },
// { "gz", FORMAT_GZIP },
{ "raw", FORMAT_RAW },
};
size_t i = 0;
while (strcmp(types[i].str, optarg) != 0)
if (++i == ARRAY_SIZE(types))
message_fatal(_("%s: Unknown file "
"format type"),
optarg);
opt_format = types[i].format;
break;
}
// --check
case 'C': {
static const struct {
char str[8];
lzma_check check;
} types[] = {
{ "none", LZMA_CHECK_NONE },
{ "crc32", LZMA_CHECK_CRC32 },
{ "crc64", LZMA_CHECK_CRC64 },
{ "sha256", LZMA_CHECK_SHA256 },
};
size_t i = 0;
while (strcmp(types[i].str, optarg) != 0) {
if (++i == ARRAY_SIZE(types))
message_fatal(_("%s: Unsupported "
"integrity "
"check type"), optarg);
}
// Use a separate check in case we are using different
// liblzma than what was used to compile us.
if (!lzma_check_is_supported(types[i].check))
message_fatal(_("%s: Unsupported integrity "
"check type"), optarg);
coder_set_check(types[i].check);
break;
}
case OPT_BLOCK_SIZE:
opt_block_size = str_to_uint64("block-size", optarg,
0, LZMA_VLI_MAX);
break;
case OPT_SINGLE_STREAM:
opt_single_stream = true;
break;
case OPT_NO_SPARSE:
io_no_sparse();
break;
case OPT_FILES:
args->files_delim = '\n';
// Fall through
case OPT_FILES0:
if (args->files_name != NULL)
message_fatal(_("Only one file can be "
"specified with `--files' "
"or `--files0'."));
if (optarg == NULL) {
args->files_name = (char *)stdin_filename;
args->files_file = stdin;
} else {
args->files_name = optarg;
args->files_file = fopen(optarg,
c == OPT_FILES ? "r" : "rb");
if (args->files_file == NULL)
message_fatal("%s: %s", optarg,
strerror(errno));
}
break;
case OPT_NO_ADJUST:
opt_auto_adjust = false;
break;
default:
message_try_help();
tuklib_exit(E_ERROR, E_ERROR, false);
}
}
return;
}
void set_tolerance(double value)
{
if (value < 0.0)
message_fatal("Tolerance must be positive");
tol = value;
}
void check_opts(void)
{
if (hist_min >= hist_max)
message_fatal("Histogram minimum must be less than maximum");
}
extern uint64_t
str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max)
{
uint64_t result = 0;
// Skip blanks.
while (*value == ' ' || *value == '\t')
++value;
// Accept special value "max". Supporting "min" doesn't seem useful.
if (strcmp(value, "max") == 0)
return max;
if (*value < '0' || *value > '9')
message_fatal(_("%s: Value is not a non-negative "
"decimal integer"), value);
do {
// Don't overflow.
if (result > UINT64_MAX / 10)
goto error;
result *= 10;
// Another overflow check
const uint32_t add = *value - '0';
if (UINT64_MAX - add < result)
goto error;
result += add;
++value;
} while (*value >= '0' && *value <= '9');
if (*value != '\0') {
// Look for suffix. Originally this supported both base-2
// and base-10, but since there seems to be little need
// for base-10 in this program, treat everything as base-2
// and also be more relaxed about the case of the first
// letter of the suffix.
uint64_t multiplier = 0;
if (*value == 'k' || *value == 'K')
multiplier = UINT64_C(1) << 10;
else if (*value == 'm' || *value == 'M')
multiplier = UINT64_C(1) << 20;
else if (*value == 'g' || *value == 'G')
multiplier = UINT64_C(1) << 30;
++value;
// Allow also e.g. Ki, KiB, and KB.
if (*value != '\0' && strcmp(value, "i") != 0
&& strcmp(value, "iB") != 0
&& strcmp(value, "B") != 0)
multiplier = 0;
if (multiplier == 0) {
message(V_ERROR, _("%s: Invalid multiplier suffix"),
value - 1);
message_fatal(_("Valid suffixes are `KiB' (2^10), "
"`MiB' (2^20), and `GiB' (2^30)."));
}
// Don't overflow here either.
if (result > UINT64_MAX / multiplier)
goto error;
result *= multiplier;
}
if (result < min || result > max)
goto error;
return result;
error:
message_fatal(_("Value of the option `%s' must be in the range "
"[%" PRIu64 ", %" PRIu64 "]"),
name, min, max);
}
void set_hist_max(double value)
{
if (period > 0)
message_fatal("Incompatible options -M and -p");
hist_max = value;
}
extern void
coder_set_compression_settings(void)
{
// The default check type is CRC64, but fallback to CRC32
// if CRC64 isn't supported by the copy of liblzma we are
// using. CRC32 is always supported.
if (check_default) {
check = LZMA_CHECK_CRC64;
if (!lzma_check_is_supported(check))
check = LZMA_CHECK_CRC32;
}
// Options for LZMA1 or LZMA2 in case we are using a preset.
static lzma_options_lzma opt_lzma;
if (filters_count == 0) {
// We are using a preset. This is not a good idea in raw mode
// except when playing around with things. Different versions
// of this software may use different options in presets, and
// thus make uncompressing the raw data difficult.
if (opt_format == FORMAT_RAW) {
// The message is shown only if warnings are allowed
// but the exit status isn't changed.
message(V_WARNING, _("Using a preset in raw mode "
"is discouraged."));
message(V_WARNING, _("The exact options of the "
"presets may vary between software "
"versions."));
}
// Get the preset for LZMA1 or LZMA2.
if (lzma_lzma_preset(&opt_lzma, preset_number))
message_bug();
// Use LZMA2 except with --format=lzma we use LZMA1.
filters[0].id = opt_format == FORMAT_LZMA
? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
filters[0].options = &opt_lzma;
filters_count = 1;
}
// Terminate the filter options array.
filters[filters_count].id = LZMA_VLI_UNKNOWN;
// If we are using the .lzma format, allow exactly one filter
// which has to be LZMA1.
if (opt_format == FORMAT_LZMA && (filters_count != 1
|| filters[0].id != LZMA_FILTER_LZMA1))
message_fatal(_("The .lzma format supports only "
"the LZMA1 filter"));
// If we are using the .xz format, make sure that there is no LZMA1
// filter to prevent LZMA_PROG_ERROR.
if (opt_format == FORMAT_XZ)
for (size_t i = 0; i < filters_count; ++i)
if (filters[i].id == LZMA_FILTER_LZMA1)
message_fatal(_("LZMA1 cannot be used "
"with the .xz format"));
// Print the selected filter chain.
message_filters_show(V_DEBUG, filters);
// Get the memory usage. Note that if --format=raw was used,
// we can be decompressing.
const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
uint64_t memory_usage;
if (opt_mode == MODE_COMPRESS) {
#ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) {
mt_options.threads = hardware_threads_get();
mt_options.block_size = opt_block_size;
mt_options.check = check;
memory_usage = lzma_stream_encoder_mt_memusage(
&mt_options);
if (memory_usage != UINT64_MAX)
message(V_DEBUG, _("Using up to %" PRIu32
" threads."),
mt_options.threads);
} else
#endif
{
memory_usage = lzma_raw_encoder_memusage(filters);
}
} else {
memory_usage = lzma_raw_decoder_memusage(filters);
}
if (memory_usage == UINT64_MAX)
message_fatal(_("Unsupported filter chain or filter options"));
// Print memory usage info before possible dictionary
// size auto-adjusting.
message_mem_needed(V_DEBUG, memory_usage);
if (opt_mode == MODE_COMPRESS) {
const uint64_t decmem = lzma_raw_decoder_memusage(filters);
if (decmem != UINT64_MAX)
message(V_DEBUG, _("Decompression will need "
"%s MiB of memory."), uint64_to_str(
round_up_to_mib(decmem), 0));
}
if (memory_usage <= memory_limit)
return;
// If --no-auto-adjust was used or we didn't find LZMA1 or
// LZMA2 as the last filter, give an error immediately.
// --format=raw implies --no-auto-adjust.
if (!opt_auto_adjust || opt_format == FORMAT_RAW)
memlimit_too_small(memory_usage);
assert(opt_mode == MODE_COMPRESS);
#ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && mt_options.threads > 1) {
// Try to reduce the number of threads before
// adjusting the compression settings down.
do {
// FIXME? The real single-threaded mode has
// lower memory usage, but it's not comparable
// because it doesn't write the size info
// into Block Headers.
if (--mt_options.threads == 0)
memlimit_too_small(memory_usage);
memory_usage = lzma_stream_encoder_mt_memusage(
&mt_options);
if (memory_usage == UINT64_MAX)
message_bug();
} while (memory_usage > memory_limit);
message(V_WARNING, _("Adjusted the number of threads "
"from %s to %s to not exceed "
"the memory usage limit of %s MiB"),
uint64_to_str(hardware_threads_get(), 0),
uint64_to_str(mt_options.threads, 1),
uint64_to_str(round_up_to_mib(
memory_limit), 2));
}
#endif
if (memory_usage <= memory_limit) {
return;
}
// Look for the last filter if it is LZMA2 or LZMA1, so we can make
// it use less RAM. With other filters we don't know what to do.
size_t i = 0;
while (filters[i].id != LZMA_FILTER_LZMA2
&& filters[i].id != LZMA_FILTER_LZMA1) {
if (filters[i].id == LZMA_VLI_UNKNOWN)
memlimit_too_small(memory_usage);
++i;
}
// Decrease the dictionary size until we meet the memory
// usage limit. First round down to full mebibytes.
lzma_options_lzma *opt = filters[i].options;
const uint32_t orig_dict_size = opt->dict_size;
opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
while (true) {
// If it is below 1 MiB, auto-adjusting failed. We could be
// more sophisticated and scale it down even more, but let's
// see if many complain about this version.
//
// FIXME: Displays the scaled memory usage instead
// of the original.
if (opt->dict_size < (UINT32_C(1) << 20))
memlimit_too_small(memory_usage);
memory_usage = lzma_raw_encoder_memusage(filters);
if (memory_usage == UINT64_MAX)
message_bug();
// Accept it if it is low enough.
if (memory_usage <= memory_limit)
break;
// Otherwise 1 MiB down and try again. I hope this
// isn't too slow method for cases where the original
// dict_size is very big.
opt->dict_size -= UINT32_C(1) << 20;
}
// Tell the user that we decreased the dictionary size.
message(V_WARNING, _("Adjusted LZMA%c dictionary size "
"from %s MiB to %s MiB to not exceed "
"the memory usage limit of %s MiB"),
filters[i].id == LZMA_FILTER_LZMA2
? '2' : '1',
uint64_to_str(orig_dict_size >> 20, 0),
uint64_to_str(opt->dict_size >> 20, 1),
uint64_to_str(round_up_to_mib(memory_limit), 2));
return;
}
void set_beta(double value)
{
if (value < 0.0)
message_fatal("Beta must be positive");
beta = value;
}
int
main(int argc, char **argv)
{
#if defined(_WIN32) && !defined(__CYGWIN__)
InitializeCriticalSection(&exit_status_cs);
#endif
// Set up the progname variable.
tuklib_progname_init(argv);
// Initialize the file I/O. This makes sure that
// stdin, stdout, and stderr are something valid.
io_init();
// Set up the locale and message translations.
tuklib_gettext_init(PACKAGE, LOCALEDIR);
// Initialize handling of error/warning/other messages.
message_init();
// Set hardware-dependent default values. These can be overriden
// on the command line, thus this must be done before args_parse().
hardware_init();
// Parse the command line arguments and get an array of filenames.
// This doesn't return if something is wrong with the command line
// arguments. If there are no arguments, one filename ("-") is still
// returned to indicate stdin.
args_info args;
args_parse(&args, argc, argv);
if (opt_mode != MODE_LIST && opt_robot)
message_fatal(_("Compression and decompression with --robot "
"are not supported yet."));
// Tell the message handling code how many input files there are if
// we know it. This way the progress indicator can show it.
if (args.files_name != NULL)
message_set_files(0);
else
message_set_files(args.arg_count);
// Refuse to write compressed data to standard output if it is
// a terminal.
if (opt_mode == MODE_COMPRESS) {
if (opt_stdout || (args.arg_count == 1
&& strcmp(args.arg_names[0], "-") == 0)) {
if (is_tty_stdout()) {
message_try_help();
tuklib_exit(E_ERROR, E_ERROR, false);
}
}
}
// Set up the signal handlers. We don't need these before we
// start the actual action and not in --list mode, so this is
// done after parsing the command line arguments.
//
// It's good to keep signal handlers in normal compression and
// decompression modes even when only writing to stdout, because
// we might need to restore O_APPEND flag on stdout before exiting.
// In --test mode, signal handlers aren't really needed, but let's
// keep them there for consistency with normal decompression.
if (opt_mode != MODE_LIST)
signals_init();
// coder_run() handles compression, decompression, and testing.
// list_file() is for --list.
void (*run)(const char *filename) = opt_mode == MODE_LIST
? &list_file : &coder_run;
// Process the files given on the command line. Note that if no names
// were given, args_parse() gave us a fake "-" filename.
for (size_t i = 0; i < args.arg_count && !user_abort; ++i) {
if (strcmp("-", args.arg_names[i]) == 0) {
// Processing from stdin to stdout. Check that we
// aren't writing compressed data to a terminal or
// reading it from a terminal.
if (opt_mode == MODE_COMPRESS) {
if (is_tty_stdout())
continue;
} else if (is_tty_stdin()) {
continue;
}
// It doesn't make sense to compress data from stdin
// if we are supposed to read filenames from stdin
// too (enabled with --files or --files0).
if (args.files_name == stdin_filename) {
message_error(_("Cannot read data from "
"standard input when "
"reading filenames "
"from standard input"));
continue;
}
// Replace the "-" with a special pointer, which is
// recognized by coder_run() and other things.
// This way error messages get a proper filename
// string and the code still knows that it is
// handling the special case of stdin.
args.arg_names[i] = (char *)stdin_filename;
}
// Do the actual compression or decompression.
run(args.arg_names[i]);
}
// If --files or --files0 was used, process the filenames from the
// given file or stdin. Note that here we don't consider "-" to
// indicate stdin like we do with the command line arguments.
if (args.files_name != NULL) {
// read_name() checks for user_abort so we don't need to
// check it as loop termination condition.
while (true) {
const char *name = read_name(&args);
if (name == NULL)
break;
// read_name() doesn't return empty names.
assert(name[0] != '\0');
run(name);
}
if (args.files_name != stdin_filename)
(void)fclose(args.files_file);
}
// All files have now been handled. If in --list mode, display
// the totals before exiting. We don't have signal handlers
// enabled in --list mode, so we don't need to check user_abort.
if (opt_mode == MODE_LIST) {
assert(!user_abort);
list_totals();
}
#ifndef NDEBUG
coder_free();
#endif
// If we have got a signal, raise it to kill the program instead
// of calling tuklib_exit().
signals_exit();
// Make a local copy of exit_status to keep the Windows code
// thread safe. At this point it is fine if we miss the user
// pressing C-c and don't set the exit_status to E_ERROR on
// Windows.
#if defined(_WIN32) && !defined(__CYGWIN__)
EnterCriticalSection(&exit_status_cs);
#endif
enum exit_status_type es = exit_status;
#if defined(_WIN32) && !defined(__CYGWIN__)
LeaveCriticalSection(&exit_status_cs);
#endif
// Suppress the exit status indicating a warning if --no-warn
// was specified.
if (es == E_WARNING && no_warn)
es = E_SUCCESS;
tuklib_exit(es, E_ERROR, message_verbosity_get() != V_SILENT);
}
