// There is no zlib gztruncate. Inflate it, truncate it, recompress it.
static int gztruncate(const char *path, off_t length, int compression)
{
int ret=1;
char tmp[16];
char *dest=NULL;
char *dest2=NULL;
snprintf(tmp, sizeof(tmp), ".%d", getpid());
if(!(dest=prepend(path, tmp))
|| !(dest2=prepend(dest, "-2"))
|| zlib_inflate(NULL, path, dest, NULL))
goto end;
if(truncate(dest, length))
{
logp("truncate of %s failed in %s\n", dest, __func__);
goto end;
}
if(compress_file(dest, dest2, compression))
goto end;
unlink(dest);
ret=do_rename(dest2, path);
end:
if(dest) unlink(dest);
if(dest2) unlink(dest2);
free_w(&dest);
free_w(&dest2);
return ret;
}
void compress(char* input_file, char* output_file) {
FILE* fp_in = fopen(input_file, "rb");
FILE* fp_out = fopen(output_file, "wb");
compress_file(fp_in, fp_out);
fclose(fp_in);
fclose(fp_out);
}
int compress_filename(const char *d, const char *file, const char *zfile, struct config *cconf)
{
char *fullfile=NULL;
char *fullzfile=NULL;
if(!(fullfile=prepend_s(d, file, strlen(file)))
|| !(fullzfile=prepend_s(d, zfile, strlen(zfile)))
|| compress_file(fullfile, fullzfile, cconf))
{
if(fullfile) free(fullfile);
if(fullzfile) free(fullzfile);
return -1;
}
return 0;
}
int compress_filename(const char *d,
const char *file, const char *zfile, int compression)
{
char *fullfile=NULL;
char *fullzfile=NULL;
if(!(fullfile=prepend_s(d, file))
|| !(fullzfile=prepend_s(d, zfile))
|| compress_file(fullfile, fullzfile, compression))
{
free_w(&fullfile);
free_w(&fullzfile);
return -1;
}
return 0;
}
static int compress(const char *input, const char *output) {
char *tmp = NULL;
FILE *in = NULL, *out = NULL;
size_t size_in = 0, size_out = 0;
int r = 1;
if (!output) {
size_t len = strlen(input);
output = tmp = malloc(len + 5);
if (!tmp) {
printf("Not enough memory");
return 1;
}
strcpy(tmp, input);
strcpy(tmp + len, ".lz4");
}
in = fopen(input, "rb");
if (!in) {
fprintf(stderr, "Failed to open input file %s: %s\n", input, strerror(errno));
goto cleanup;
}
out = fopen(output, "wb");
if (!out) {
fprintf(stderr, "Failed to open output file %s: %s\n", output, strerror(errno));
goto cleanup;
}
r = compress_file(in, out, &size_in, &size_out);
if (r == 0)
printf("%s: %zu → %zu bytes, %.1f%%\n",
input, size_in, size_out,
(double)size_out / size_in * 100);
cleanup:
if (in)
fclose(in);
if (out)
fclose(out);
free(tmp);
return r;
}
int
queue_message_commit(uint32_t msgid)
{
char msgpath[MAXPATHLEN];
char tmppath[MAXPATHLEN];
FILE *ifp = NULL;
FILE *ofp = NULL;
queue_message_incoming_path(msgid, msgpath, sizeof msgpath);
strlcat(msgpath, PATH_MESSAGE, sizeof(msgpath));
if (env->sc_queue_flags & QUEUE_COMPRESS) {
bsnprintf(tmppath, sizeof tmppath, "%s.comp", msgpath);
ifp = fopen(msgpath, "r");
ofp = fopen(tmppath, "w+");
if (ifp == NULL || ofp == NULL)
goto err;
if (! compress_file(ifp, ofp))
goto err;
fclose(ifp);
fclose(ofp);
ifp = NULL;
ofp = NULL;
if (rename(tmppath, msgpath) == -1) {
if (errno == ENOSPC)
return (0);
fatal("queue_message_commit: rename");
}
}
return env->sc_queue->message(QOP_COMMIT, &msgid);
err:
if (ifp)
fclose(ifp);
if (ofp)
fclose(ofp);
return 0;
}
int main(int argc, char** argv)
{
int i,
compression=1, // default action if no argument
decode=0;
char *input_filename=0,
*output_filename=0;
// Welcome message
fprintf(stderr, WELCOME_MESSAGE);
if (argc<2) { badusage(); return 1; }
for(i=1; i<argc; i++)
{
char* argument = argv[i];
char command = 0;
if(!argument) continue; // Protection if argument empty
if (argument[0]=='-') command++; // valid command trigger
// Select command
if (command)
{
argument += command;
// display help on usage
if( argument[0] =='h' )
{ usage(); return 0; }
// Forced Compression (default)
if( argument[0] =='c' )
{ compression=1; continue; }
// Forced Decoding
if( argument[0] =='d' )
{ decode=1; continue; }
}
// first provided filename is input
if (!input_filename) { input_filename=argument; continue; }
// second provided filename is output
if (!output_filename) { output_filename=argument; continue; }
}
// No input filename ==> Error
if(!input_filename) { badusage(); return 1; }
// No output filename
if (!output_filename) { badusage(); return 1; }
if (decode) return decode_file(input_filename, output_filename);
if (compression) return compress_file(input_filename, output_filename);
badusage();
return 0;
}
/*
* main
*
* Main procedure. Sets default values, parses arguments from command line,
* and calls uncompress or compress depending on the file linktype
*
*/
int main (int argc, char **argv)
{
FILE *fin;
lFILE *lfout;
struct pcap_file_header filehdr;
int swapped;
co_t co;
char filter[PROT_MAX_INDIVIDUAL_FILTER];
/* get the file names */
finname[0] = '\0';
foutname[0] = '\0';
parse_args (argc, argv);
/* open input file */
if ((finname[0] == '-' && finname[1] == '\0') || (finname[0] == '\0')) {
fin = stdin;
} else {
fin = fopen (finname, "r");
if (fin == NULL) {
fprintf (stderr, "Error: cannot open file %s\n", finname);
exit (1);
}
}
/* read the input header to know whether it's compressed or not */
if (pktd_fread_header (fin, &filehdr, &swapped, &co, filter) < 0) {
fprintf (stderr, "Error: file %s has wrong format\n", finname);
exit (1);
}
if (filehdr.version_minor == 5) {
/* pcap extended header */
printf ("Extended IP header contains mask: 0x%02x\n", co.ip_mask); /* XXX */
printf ("Extended TCP header contains mask: 0x%04x\n", co.tcp_mask); /* XXX */
printf ("Extended UDP header contains mask: 0x%02x\n", co.udp_mask); /* XXX */
printf ("Extended header contains offset: %i\n", co.rm_offset); /* XXX */
printf ("Extended header contains filter: %s\n", filter); /* XXX */
}
/* open output file */
if ((foutname[0] == '-' && foutname[1] == '\0') || (foutname[0] == '\0')) {
lfout = lfdopen (STDOUT_FILENO, 8192);
} else {
lfout = lfopen (foutname, 8192);
if (lfout == NULL) {
fprintf (stderr, "Error: cannot open file %s\n", foutname);
fclose (fin);
exit (1);
}
}
/* check if the file is compressed or not */
if (filehdr.linktype == DLT_COMPRESSED) {
uncompress_file (fin, lfout, &co);
} else {
compress_file (fin, lfout, &filehdr, swapped);
}
fclose (fin);
lfclose (lfout);
exit (0);
}
int
queue_message_commit(uint32_t msgid)
{
int r;
char msgpath[PATH_MAX];
char tmppath[PATH_MAX];
FILE *ifp = NULL;
FILE *ofp = NULL;
profile_enter("queue_message_commit");
queue_message_path(msgid, msgpath, sizeof(msgpath));
if (env->sc_queue_flags & QUEUE_COMPRESSION) {
bsnprintf(tmppath, sizeof tmppath, "%s.comp", msgpath);
ifp = fopen(msgpath, "r");
ofp = fopen(tmppath, "w+");
if (ifp == NULL || ofp == NULL)
goto err;
if (! compress_file(ifp, ofp))
goto err;
fclose(ifp);
fclose(ofp);
ifp = NULL;
ofp = NULL;
if (rename(tmppath, msgpath) == -1) {
if (errno == ENOSPC)
return (0);
unlink(tmppath);
log_warn("rename");
return (0);
}
}
if (env->sc_queue_flags & QUEUE_ENCRYPTION) {
bsnprintf(tmppath, sizeof tmppath, "%s.enc", msgpath);
ifp = fopen(msgpath, "r");
ofp = fopen(tmppath, "w+");
if (ifp == NULL || ofp == NULL)
goto err;
if (! crypto_encrypt_file(ifp, ofp))
goto err;
fclose(ifp);
fclose(ofp);
ifp = NULL;
ofp = NULL;
if (rename(tmppath, msgpath) == -1) {
if (errno == ENOSPC)
return (0);
unlink(tmppath);
log_warn("rename");
return (0);
}
}
r = handler_message_commit(msgid, msgpath);
profile_leave();
/* in case it's not done by the backend */
unlink(msgpath);
log_trace(TRACE_QUEUE,
"queue-backend: queue_message_commit(%08"PRIx32") -> %d",
msgid, r);
return (r);
err:
if (ifp)
fclose(ifp);
if (ofp)
fclose(ofp);
return 0;
}
static int uff_comp(int idx, char *filename)
{
putlog(LOG_BOTS, "*", "Compressing user file for %s.", dcc[idx].nick);
return compress_file(filename, compress_level);
}
/* ***************************************************************
File compression via the compression booster
*************************************************************** */
int main(int argc, char *argv[])
{
double getTime ( void );
void open_files(char *infile_name);
void lcp_file(void);
double end, start;
char *infile_name;
int c;
extern char *optarg;
extern int optind, opterr, optopt;
/* ------------- read command line options ----------- */
Verbose=0; // be quiet by default
Lambda = 1.0;
infile_name=NULL;
if(argc<2) {
fprintf(stderr, "Usage:\n\t%s infile ",argv[0]);
fprintf(stderr,"[-v] [-l lambda]\n");
fprintf(stderr,"\t-l lambda entropy estimator ");
fprintf(stderr,"(default %.2f)\n",Lambda);
fprintf(stderr,"\t-v verbose output\n\n");
exit(0);
}
opterr = 0;
while ((c=getopt(argc, argv, "l:v")) != -1) {
switch (c)
{
// case 'o':
// outfile_name = optarg; break;
case 'l':
Lambda=atof(optarg); break;
case 'v':
Verbose++; break;
case '?':
fprintf(stderr,"Unknown option: %c -main-\n", optopt);
exit(1);
}
}
if(optind<argc)
infile_name=argv[optind];
// ----- check input parameters--------------
if(infile_name==NULL) fatal_error("The input file name is required\n");
if(Verbose>2) {
fprintf(stderr,"\n*****************************************************");
fprintf(stderr,"\n bwtopt Ver 1.0\n");
fprintf(stderr,"Created on %s at %s from %s\n",__DATE__,__TIME__,__FILE__);
fprintf(stderr,"*****************************************************\n");
}
if(Verbose>1) {
fprintf(stderr,"Command line: ");
for(c=0;c<argc;c++)
fprintf(stderr,"%s ",argv[c]);
fprintf(stderr,"\n");
}
// ---- do the work ---------------------------
start = getTime();
open_files(infile_name);
compress_file();
end = getTime();
if(Verbose)
fprintf(stderr,"Elapsed time: %f seconds.\n", end-start);
return 0;
}
// a = length of struct bu array
// i = position to restore from
static int restore_manifest(struct bu *arr, int a, int i, const char *tmppath1, const char *tmppath2, regex_t *regex, int srestore, enum action act, const char *client, char **dir_for_notify, struct cntr *p1cntr, struct cntr *cntr, struct config *cconf)
{
int ret=0;
gzFile zp=NULL;
char *manifest=NULL;
char *datadir=NULL;
FILE *logfp=NULL;
char *logpath=NULL;
char *logpathz=NULL;
// For sending status information up to the server.
char status=STATUS_RESTORING;
if(act==ACTION_RESTORE) status=STATUS_RESTORING;
else if(act==ACTION_VERIFY) status=STATUS_VERIFYING;
if(
(act==ACTION_RESTORE && !(logpath=prepend_s(arr[i].path, "restorelog", strlen("restorelog"))))
|| (act==ACTION_RESTORE && !(logpathz=prepend_s(arr[i].path, "restorelog.gz", strlen("restorelog.gz"))))
|| (act==ACTION_VERIFY && !(logpath=prepend_s(arr[i].path, "verifylog", strlen("verifylog"))))
|| (act==ACTION_VERIFY && !(logpathz=prepend_s(arr[i].path, "verifylog.gz", strlen("verifylog.gz"))))
|| !(manifest=prepend_s(arr[i].path, "manifest.gz", strlen("manifest.gz"))))
{
log_and_send_oom(__FUNCTION__);
ret=-1;
}
else if(!(logfp=open_file(logpath, "ab")) || set_logfp(logfp, cconf))
{
char msg[256]="";
snprintf(msg, sizeof(msg),
"could not open log file: %s", logpath);
log_and_send(msg);
ret=-1;
}
*dir_for_notify=strdup(arr[i].path);
log_restore_settings(cconf, srestore);
// First, do a pass through the manifest to set up the counters.
// This is the equivalent of a phase1 scan during backup.
if(!ret && !(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
ret=-1;
}
else
{
int ars=0;
int quit=0;
struct sbuf sb;
init_sbuf(&sb);
while(!quit)
{
if((ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means end ok
quit++;
}
else
{
if((!srestore
|| check_srestore(cconf, sb.path))
&& check_regex(regex, sb.path))
{
do_filecounter(p1cntr, sb.cmd, 0);
if(sb.endfile)
do_filecounter_bytes(p1cntr,
strtoull(sb.endfile, NULL, 10));
/*
if(sb.cmd==CMD_FILE
|| sb.cmd==CMD_ENC_FILE
|| sb.cmd==CMD_METADATA
|| sb.cmd==CMD_ENC_METADATA
|| sb.cmd==CMD_VSS
|| sb.cmd==CMD_ENC_VSS
|| sb.cmd==CMD_VSS_T
|| sb.cmd==CMD_ENC_VSS_T
|| sb.cmd==CMD_EFS_FILE)
do_filecounter_bytes(p1cntr,
(unsigned long long)
sb.statp.st_size);
*/
}
}
free_sbuf(&sb);
}
free_sbuf(&sb);
gzclose_fp(&zp);
}
if(cconf->send_client_counters)
{
if(send_counters(client, p1cntr, cntr))
{
ret=-1;
}
}
// Now, do the actual restore.
if(!ret && !(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
ret=-1;
}
else
{
char cmd;
int s=0;
int quit=0;
size_t len=0;
struct sbuf sb;
// For out-of-sequence directory restoring so that the
// timestamps come out right:
int scount=0;
struct sbuf **sblist=NULL;
init_sbuf(&sb);
while(!quit)
{
int ars=0;
char *buf=NULL;
if(async_read_quick(&cmd, &buf, &len))
{
logp("read quick error\n");
ret=-1; quit++; break;
}
if(buf)
{
//logp("got read quick\n");
if(cmd==CMD_WARNING)
{
logp("WARNING: %s\n", buf);
do_filecounter(cntr, cmd, 0);
free(buf); buf=NULL;
continue;
}
else if(cmd==CMD_INTERRUPT)
{
// Client wanted to interrupt the
// sending of a file. But if we are
// here, we have already moved on.
// Ignore.
free(buf); buf=NULL;
continue;
}
else
{
logp("unexpected cmd from client: %c:%s\n", cmd, buf);
free(buf); buf=NULL;
ret=-1; quit++; break;
}
}
if((ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means end ok
quit++;
}
else
{
if((!srestore
|| check_srestore(cconf, sb.path))
&& check_regex(regex, sb.path)
&& restore_ent(client,
&sb, &sblist, &scount,
arr, a, i, tmppath1, tmppath2,
act, status, cconf,
cntr, p1cntr))
{
ret=-1;
quit++;
}
}
free_sbuf(&sb);
}
gzclose_fp(&zp);
// Restore any directories that are left in the list.
if(!ret) for(s=scount-1; s>=0; s--)
{
if(restore_sbuf(sblist[s], arr, a, i,
tmppath1, tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1;
break;
}
}
free_sbufs(sblist, scount);
if(!ret) ret=do_restore_end(act, cntr);
//print_endcounter(cntr);
print_filecounters(p1cntr, cntr, act);
reset_filecounter(p1cntr, time(NULL));
reset_filecounter(cntr, time(NULL));
}
set_logfp(NULL, cconf);
compress_file(logpath, logpathz, cconf);
if(manifest) free(manifest);
if(datadir) free(datadir);
if(logpath) free(logpath);
if(logpathz) free(logpathz);
return ret;
}
// a = length of struct bu array
// i = position to restore from
static int restore_manifest(struct bu *arr, int a, int i, const char *tmppath1, const char *tmppath2, regex_t *regex, enum action act, const char *client, struct cntr *p1cntr, struct cntr *cntr, struct config *cconf, bool all)
{
int ret=0;
gzFile zp=NULL;
char *manifest=NULL;
char *datadir=NULL;
FILE *logfp=NULL;
char *logpath=NULL;
char *logpathz=NULL;
// For sending status information up to the server.
char status=STATUS_RESTORING;
if(act==ACTION_RESTORE) status=STATUS_RESTORING;
else if(act==ACTION_VERIFY) status=STATUS_VERIFYING;
if(
(act==ACTION_RESTORE && !(logpath=prepend_s(arr[i].path, "restorelog", strlen("restorelog"))))
|| (act==ACTION_RESTORE && !(logpathz=prepend_s(arr[i].path, "restorelog.gz", strlen("restorelog.gz"))))
|| (act==ACTION_VERIFY && !(logpath=prepend_s(arr[i].path, "verifylog", strlen("verifylog"))))
|| (act==ACTION_VERIFY && !(logpathz=prepend_s(arr[i].path, "verifylog.gz", strlen("verifylog.gz"))))
|| !(manifest=prepend_s(arr[i].path, "manifest.gz", strlen("manifest.gz"))))
{
log_and_send("out of memory");
ret=-1;
}
else if(!(logfp=open_file(logpath, "ab")) || set_logfp(logfp))
{
char msg[256]="";
snprintf(msg, sizeof(msg),
"could not open log file: %s", logpath);
log_and_send(msg);
ret=-1;
}
else if(!(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
ret=-1;
}
else
{
char cmd;
int quit=0;
size_t len=0;
struct sbuf sb;
// For out-of-sequence directory restoring so that the
// timestamps come out right:
int s=0;
int scount=0;
struct sbuf **sblist=NULL;
init_sbuf(&sb);
while(!quit)
{
int ars=0;
char *buf=NULL;
if(async_read_quick(&cmd, &buf, &len))
{
logp("read quick error\n");
ret=-1; quit++; break;
}
if(buf)
{
//logp("got read quick\n");
if(cmd==CMD_WARNING)
{
logp("WARNING: %s\n", buf);
do_filecounter(cntr, cmd, 0);
free(buf); buf=NULL;
continue;
}
else if(cmd==CMD_INTERRUPT)
{
// Client wanted to interrupt the
// sending of a file. But if we are
// here, we have already moved on.
// Ignore.
free(buf); buf=NULL;
continue;
}
else
{
logp("unexpected cmd from client: %c:%s\n", cmd, buf);
free(buf); buf=NULL;
ret=-1; quit++; break;
}
}
if((ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means end ok
quit++;
}
else
{
if(check_regex(regex, sb.path))
{
// Check if we have any directories waiting
// to be restored.
for(s=scount-1; s>=0; s--)
{
if(is_subdir(sblist[s]->path, sb.path))
{
// We are still in a subdir.
//printf(" subdir (%s %s)\n", sblist[s]->path, sb.path);
break;
}
else
{
// Can now restore sblist[s]
// because nothing else is
// fiddling in a subdirectory.
if(restore_sbuf(sblist[s], arr,
a, i, tmppath1, tmppath2, act,
client, status,
p1cntr, cntr, cconf))
{
ret=-1; quit++;
break;
}
else if(del_from_sbuf_arr(
&sblist, &scount))
{
ret=-1; quit++;
break;
}
}
}
/* If it is a directory, need to remember it
and restore it later, so that the
permissions come out right. */
/* Meta data of directories will also have
the stat stuff set to be a directory,
so will also come out at the end. */
if(!ret && S_ISDIR(sb.statp.st_mode))
{
if(add_to_sbuf_arr(&sblist, &sb, &scount))
{
ret=-1; quit++;
}
// Wipe out sb, without freeing up
// all the strings inside it, which
// have been added to sblist.
init_sbuf(&sb);
}
else if(!ret && restore_sbuf(&sb, arr, a, i,
tmppath1, tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1; quit++;
}
}
}
free_sbuf(&sb);
}
gzclose_fp(&zp);
// Restore any directories that are left in the list.
if(!ret) for(s=scount-1; s>=0; s--)
{
if(restore_sbuf(sblist[s], arr, a, i,
tmppath1, tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1;
break;
}
}
free_sbufs(sblist, scount);
if(!ret && !all) ret=do_restore_end(act, cntr);
print_endcounter(cntr);
print_filecounters(p1cntr, cntr, act, 0);
reset_filecounter(p1cntr);
reset_filecounter(cntr);
}
set_logfp(NULL);
compress_file(logpath, logpathz, cconf);
if(manifest) free(manifest);
if(datadir) free(datadir);
if(logpath) free(logpath);
if(logpathz) free(logpathz);
return ret;
}
static int restore_manifest(struct asfd *asfd, struct bu *bu,
regex_t *regex, int srestore, enum action act, struct sdirs *sdirs,
char **dir_for_notify, struct conf **cconfs)
{
int ret=-1;
char *manifest=NULL;
char *logpath=NULL;
char *logpathz=NULL;
// For sending status information up to the server.
enum cntr_status cntr_status=CNTR_STATUS_RESTORING;
if(act==ACTION_RESTORE) cntr_status=CNTR_STATUS_RESTORING;
else if(act==ACTION_VERIFY) cntr_status=CNTR_STATUS_VERIFYING;
if((act==ACTION_RESTORE && get_logpaths(bu, "restorelog",
&logpath, &logpathz))
|| (act==ACTION_VERIFY && get_logpaths(bu, "verifylog",
&logpath, &logpathz))
|| !(manifest=prepend_s(bu->path,
get_protocol(cconfs)==PROTO_1?
"manifest.gz":"manifest")))
{
log_and_send_oom(asfd, __func__);
goto end;
}
if(log_fzp_set(logpath, cconfs))
{
char msg[256]="";
snprintf(msg, sizeof(msg),
"could not open log file: %s", logpath);
log_and_send(asfd, msg);
goto end;
}
*dir_for_notify=strdup_w(bu->path, __func__);
log_restore_settings(cconfs, srestore);
// First, do a pass through the manifest to set up cntr.
// This is the equivalent of a phase1 scan during backup.
if(setup_cntr(asfd, manifest,
regex, srestore, act, cntr_status, cconfs))
goto end;
if(get_int(cconfs[OPT_SEND_CLIENT_CNTR])
&& cntr_send(get_cntr(cconfs)))
goto end;
// Now, do the actual restore.
ret=actual_restore(asfd, bu, manifest,
regex, srestore, act, sdirs, cntr_status, cconfs);
end:
log_fzp_set(NULL, cconfs);
compress_file(logpath, logpathz, get_int(cconfs[OPT_COMPRESSION]));
if(manifest) free(manifest);
if(logpath) free(logpath);
if(logpathz) free(logpathz);
return ret;
}
int
tmain(int argc, tchar *argv[])
{
struct options options;
int opt_char;
int i;
int ret;
program_invocation_name = get_filename(argv[0]);
options.to_stdout = false;
options.decompress = is_gunzip();
options.force = false;
options.keep = false;
options.compression_level = 6;
options.suffix = T("gz");
while ((opt_char = tgetopt(argc, argv, optstring)) != -1) {
switch (opt_char) {
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
options.compression_level =
parse_compression_level(opt_char, toptarg);
if (options.compression_level == 0)
return -1;
break;
case 'c':
options.to_stdout = true;
break;
case 'd':
options.decompress = true;
break;
case 'f':
options.force = true;
break;
case 'h':
show_usage(stdout);
return 0;
case 'k':
options.keep = true;
break;
case 'S':
options.suffix = toptarg;
break;
case 'V':
show_version();
return 0;
default:
show_usage(stderr);
return 1;
}
}
argv += toptind;
argc -= toptind;
if (argc == 0) {
show_usage(stderr);
return 1;
}
for (i = 0; i < argc; i++) {
if (argv[i][0] == '-' && argv[i][1] == '\0') {
msg("This implementation of gzip does not yet "
"support reading from standard input.");
return 1;
}
}
ret = 0;
if (options.decompress) {
struct deflate_decompressor *d;
d = alloc_decompressor();
if (d == NULL)
return 1;
for (i = 0; i < argc; i++)
ret |= -decompress_file(d, argv[i], &options);
deflate_free_decompressor(d);
} else {
struct deflate_compressor *c;
c = alloc_compressor(options.compression_level);
if (c == NULL)
return 1;
for (i = 0; i < argc; i++)
ret |= -compress_file(c, argv[i], &options);
deflate_free_compressor(c);
}
/*
* If ret=0, there were no warnings or errors. Exit with status 0.
* If ret=2, there was at least one warning. Exit with status 2.
* Else, there was at least one error. Exit with status 1.
*/
if (ret != 0 && ret != 2)
ret = 1;
return ret;
}
int main(int argc, char *argv[])
{
int c;
char optstring[] = "hcdz0123456789o:S:kfqVvNnt";
int long_only_flag;
int ret = 0;
int bad_option = 0;
int bad_level = 0;
int bad_c = 0;
struct option long_options[] = {
{"help", no_argument, NULL, 'h'},
{"stdout", no_argument, NULL, 'c'},
{"to-stdout", no_argument, NULL, 'c'},
{"compress", no_argument, NULL, 'z'},
{"decompress", no_argument, NULL, 'd'},
{"uncompress", no_argument, NULL, 'd'},
{"keep", no_argument, NULL, 'k'},
{"rm", no_argument, &long_only_flag, RM},
{"suffix", no_argument, NULL, 'S'},
{"fast", no_argument, NULL, '1'},
{"best", no_argument, NULL, '0' + ISAL_DEF_MAX_LEVEL},
{"force", no_argument, NULL, 'f'},
{"quiet", no_argument, NULL, 'q'},
{"version", no_argument, NULL, 'V'},
{"verbose", no_argument, NULL, 'v'},
{"no-name", no_argument, NULL, 'n'},
{"name", no_argument, NULL, 'N'},
{"test", no_argument, NULL, 't'},
/* Possible future extensions
{"recursive, no_argument, NULL, 'r'},
{"list", no_argument, NULL, 'l'},
{"benchmark", optional_argument, NULL, 'b'},
{"benchmark_end", required_argument, NULL, 'e'},
{"threads", optional_argument, NULL, 'T'},
*/
{0, 0, 0, 0}
};
init_options(&global_options);
opterr = 0;
while ((c = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) {
if (c >= '0' && c <= '9') {
if (c > '0' + ISAL_DEF_MAX_LEVEL)
bad_level = 1;
else
global_options.level = c - '0';
continue;
}
switch (c) {
case 0:
switch (long_only_flag) {
case RM:
global_options.remove = true;
break;
default:
bad_option = 1;
bad_c = c;
break;
}
break;
case 'o':
global_options.outfile_name = optarg;
global_options.outfile_name_len = strlen(global_options.outfile_name);
break;
case 'c':
global_options.use_stdout = true;
break;
case 'z':
global_options.mode = COMPRESS_MODE;
break;
case 'd':
global_options.mode = DECOMPRESS_MODE;
break;
case 'S':
global_options.suffix = optarg;
global_options.suffix_len = strlen(global_options.suffix);
break;
case 'k':
global_options.remove = false;
break;
case 'f':
global_options.force = true;
break;
case 'q':
global_options.quiet_level++;
break;
case 'v':
global_options.verbose_level++;
break;
case 'V':
print_version();
return 0;
case 'N':
global_options.name = YES_NAME;
break;
case 'n':
global_options.name = NO_NAME;
break;
case 't':
global_options.test = TEST;
global_options.mode = DECOMPRESS_MODE;
break;
case 'h':
usage(0);
default:
bad_option = 1;
bad_c = optopt;
break;
}
}
if (bad_option) {
log_print(ERROR, "igzip: invalid option ");
if (bad_c)
log_print(ERROR, "-%c\n", bad_c);
else
log_print(ERROR, ("\n"));
usage(BAD_OPTION);
}
if (bad_level) {
log_print(ERROR, "igzip: invalid compression level\n");
usage(BAD_LEVEL);
}
if (global_options.outfile_name && optind < argc - 1) {
log_print(ERROR,
"igzip: An output file may be specified with only one input file\n");
return 0;
}
if (global_options.mode == COMPRESS_MODE) {
if (optind >= argc)
compress_file();
while (optind < argc) {
global_options.infile_name = argv[optind];
global_options.infile_name_len = strlen(global_options.infile_name);
ret |= compress_file();
optind++;
}
} else if (global_options.mode == DECOMPRESS_MODE) {
if (optind >= argc)
decompress_file();
while (optind < argc) {
global_options.infile_name = argv[optind];
global_options.infile_name_len = strlen(global_options.infile_name);
ret |= decompress_file();
optind++;
}
}
return ret;
}
int main(int argc, char *argv[])
{
struct timeval start_time, end_time;
struct sigaction handler;
double seconds,total_time; // for timers
bool lrzcat = false;
int c, i;
int hours,minutes;
extern int optind;
char *eptr; /* for environment */
control = &base_control;
initialise_control(control);
if (strstr(argv[0], "lrunzip"))
control->flags |= FLAG_DECOMPRESS;
else if (strstr(argv[0], "lrzcat")) {
control->flags |= FLAG_DECOMPRESS | FLAG_STDOUT;
lrzcat = true;
}
/* generate crc table */
CrcGenerateTable();
/* Get Preloaded Defaults from lrzip.conf
* Look in ., $HOME/.lrzip/, /etc/lrzip.
* If LRZIP=NOCONFIG is set, then ignore config
*/
eptr = getenv("LRZIP");
if (eptr == NULL)
read_config(control);
else if (!strstr(eptr,"NOCONFIG"))
read_config(control);
while ((c = getopt_long(argc, argv, "bcdDefghHiklL:nN:o:O:p:qS:tTUm:vVw:z?", long_options, &i)) != -1) {
switch (c) {
case 'b':
if (control->flags & FLAG_NOT_LZMA)
failure("Can only use one of -l, -b, -g, -z or -n\n");
control->flags |= FLAG_BZIP2_COMPRESS;
break;
case 'c':
control->flags |= FLAG_CHECK;
control->flags |= FLAG_HASH;
break;
case 'd':
control->flags |= FLAG_DECOMPRESS;
break;
case 'D':
control->flags &= ~FLAG_KEEP_FILES;
break;
case 'e':
control->flags |= FLAG_ENCRYPT;
break;
case 'f':
control->flags |= FLAG_FORCE_REPLACE;
break;
case 'g':
if (control->flags & FLAG_NOT_LZMA)
failure("Can only use one of -l, -b, -g, -z or -n\n");
control->flags |= FLAG_ZLIB_COMPRESS;
break;
case 'h':
case '?':
usage();
return -1;
case 'H':
control->flags |= FLAG_HASH;
break;
case 'i':
control->flags |= FLAG_INFO;
break;
case 'k':
control->flags |= FLAG_KEEP_BROKEN;
break;
case 'l':
if (control->flags & FLAG_NOT_LZMA)
failure("Can only use one of -l, -b, -g, -z or -n\n");
control->flags |= FLAG_LZO_COMPRESS;
break;
case 'L':
control->compression_level = atoi(optarg);
if (control->compression_level < 1 || control->compression_level > 9)
failure("Invalid compression level (must be 1-9)\n");
break;
case 'm':
control->ramsize = atol(optarg) * 1024 * 1024 * 100;
break;
case 'n':
if (control->flags & FLAG_NOT_LZMA)
failure("Can only use one of -l, -b, -g, -z or -n\n");
control->flags |= FLAG_NO_COMPRESS;
break;
case 'N':
control->nice_val = atoi(optarg);
if (control->nice_val < -20 || control->nice_val > 19)
failure("Invalid nice value (must be -20..19)\n");
break;
case 'o':
if (control->outdir)
failure("Cannot have -o and -O together\n");
if (unlikely(STDOUT))
failure("Cannot specify an output filename when outputting to stdout\n");
control->outname = optarg;
control->suffix = "";
break;
case 'O':
if (control->outname) /* can't mix -o and -O */
failure("Cannot have options -o and -O together\n");
if (unlikely(STDOUT))
failure("Cannot specify an output directory when outputting to stdout\n");
control->outdir = malloc(strlen(optarg) + 2);
if (control->outdir == NULL)
fatal("Failed to allocate for outdir\n");
strcpy(control->outdir,optarg);
if (strcmp(optarg+strlen(optarg) - 1, "/")) /* need a trailing slash */
strcat(control->outdir, "/");
break;
case 'p':
control->threads = atoi(optarg);
if (control->threads < 1)
failure("Must have at least one thread\n");
break;
case 'q':
control->flags &= ~FLAG_SHOW_PROGRESS;
break;
case 'S':
if (control->outname)
failure("Specified output filename already, can't specify an extension.\n");
if (unlikely(STDOUT))
failure("Cannot specify a filename suffix when outputting to stdout\n");
control->suffix = optarg;
break;
case 't':
if (control->outname)
failure("Cannot specify an output file name when just testing.\n");
if (!KEEP_FILES)
failure("Doubt that you want to delete a file when just testing.\n");
control->flags |= FLAG_TEST_ONLY;
break;
case 'T':
control->flags &= ~FLAG_THRESHOLD;
break;
case 'U':
control->flags |= FLAG_UNLIMITED;
break;
case 'v':
/* set verbosity flag */
if (!(control->flags & FLAG_VERBOSITY) && !(control->flags & FLAG_VERBOSITY_MAX))
control->flags |= FLAG_VERBOSITY;
else if ((control->flags & FLAG_VERBOSITY)) {
control->flags &= ~FLAG_VERBOSITY;
control->flags |= FLAG_VERBOSITY_MAX;
}
break;
case 'V':
print_output("lrzip version %s\n", PACKAGE_VERSION);
exit(0);
break;
case 'w':
control->window = atol(optarg);
break;
case 'z':
if (control->flags & FLAG_NOT_LZMA)
failure("Can only use one of -l, -b, -g, -z or -n\n");
control->flags |= FLAG_ZPAQ_COMPRESS;
break;
}
}
argc -= optind;
argv += optind;
if (control->outname && argc > 1)
failure("Cannot specify output filename with more than 1 file\n");
if (VERBOSE && !SHOW_PROGRESS) {
print_err("Cannot have -v and -q options. -v wins.\n");
control->flags |= FLAG_SHOW_PROGRESS;
}
if (UNLIMITED && control->window) {
print_err("If -U used, cannot specify a window size with -w.\n");
control->window = 0;
}
if (argc < 1)
control->flags |= FLAG_STDIN;
if (UNLIMITED && STDIN) {
print_err("Cannot have -U and stdin, unlimited mode disabled.\n");
control->flags &= ~FLAG_UNLIMITED;
}
setup_overhead(control);
/* Set the main nice value to half that of the backend threads since
* the rzip stage is usually the rate limiting step */
if (control->nice_val > 0 && !NO_COMPRESS) {
if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val / 2) == -1))
print_err("Warning, unable to set nice value\n");
} else {
if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val) == -1))
print_err("Warning, unable to set nice value\n");
}
/* One extra iteration for the case of no parameters means we will default to stdin/out */
for (i = 0; i <= argc; i++) {
if (i < argc)
control->infile = argv[i];
else if (!(i == 0 && STDIN))
break;
if (control->infile) {
if ((strcmp(control->infile, "-") == 0))
control->flags |= FLAG_STDIN;
else {
struct stat infile_stat;
stat(control->infile, &infile_stat);
if (unlikely(S_ISDIR(infile_stat.st_mode)))
failure("lrzip only works directly on FILES.\n"
"Use lrztar or pipe through tar for compressing directories.\n");
}
}
if (INFO && STDIN)
failure("Will not get file info from STDIN\n");
if ((control->outname && (strcmp(control->outname, "-") == 0)) ||
/* If no output filename is specified, and we're using
* stdin, use stdout */
(!control->outname && STDIN) || lrzcat ) {
control->flags |= FLAG_STDOUT;
control->outFILE = stdout;
control->msgout = stderr;
register_outputfile(control, control->msgout);
}
if (lrzcat) {
control->msgout = stderr;
control->outFILE = stdout;
register_outputfile(control, control->msgout);
}
if (!STDOUT) {
control->msgout = stdout;
register_outputfile(control, control->msgout);
}
if (STDIN)
control->inFILE = stdin;
/* Implement signal handler only once flags are set */
sigemptyset(&handler.sa_mask);
handler.sa_flags = 0;
handler.sa_handler = &sighandler;
sigaction(SIGTERM, &handler, 0);
sigaction(SIGINT, &handler, 0);
if (!FORCE_REPLACE) {
if (STDIN && isatty(fileno((FILE *)stdin))) {
print_err("Will not read stdin from a terminal. Use -f to override.\n");
usage();
exit (1);
}
if (!TEST_ONLY && STDOUT && isatty(fileno((FILE *)stdout))) {
print_err("Will not write stdout to a terminal. Use -f to override.\n");
usage();
exit (1);
}
}
if (CHECK_FILE) {
if (!DECOMPRESS) {
print_err("Can only check file written on decompression.\n");
control->flags &= ~FLAG_CHECK;
} else if (STDOUT) {
print_err("Can't check file written when writing to stdout. Checking disabled.\n");
control->flags &= ~FLAG_CHECK;
}
}
setup_ram(control);
show_summary();
gettimeofday(&start_time, NULL);
if (unlikely(STDIN && ENCRYPT))
failure("Unable to work from STDIN while reading password\n");
memcpy(&local_control, &base_control, sizeof(rzip_control));
if (DECOMPRESS || TEST_ONLY)
decompress_file(&local_control);
else if (INFO)
get_fileinfo(&local_control);
else
compress_file(&local_control);
/* compute total time */
gettimeofday(&end_time, NULL);
total_time = (end_time.tv_sec + (double)end_time.tv_usec / 1000000) -
(start_time.tv_sec + (double)start_time.tv_usec / 1000000);
hours = (int)total_time / 3600;
minutes = (int)(total_time / 60) % 60;
seconds = total_time - hours * 3600 - minutes * 60;
if (!INFO)
print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds);
}
return 0;
}
static int uff_comp(int idx, char *filename)
{
debug1("Compressing user file for %s.", dcc[idx].nick);
return compress_file(filename, compress_level);
}
int main( int argc, char** argv ) {
try {
indri::api::Parameters& param = indri::api::Parameters::instance();
param.loadCommandLine( argc, argv );
if( !param.exists("index") || !param.exists("input") || !param.exists("name") ) {
std::cerr << "makeprior usage: " << std::endl
<< " makeprior -index=myindex -input=myinputfile -name=priorname" << std::endl
<< " myindex: a valid Indri index " << std::endl
<< " myinputfile: a two column text file, where the first column contains docno values" << std::endl
<< " and the second column contains log probabilities (should be between -infinity and zero)" << std::endl
<< " name: the name of this prior (as you will reference it in queries, using the #prior(name) syntax)" << std::endl;
exit(-1);
}
std::string index = param["index"];
// get the total document count, including deleted documents.
indri::collection::Repository* _repository = new indri::collection::Repository();
_repository->openRead(index);
indri::collection::Repository::index_state indexes = _repository->indexes();
int documentCount = 0;
for( size_t i=0; i<indexes->size(); i++ ) {
indri::thread::ScopedLock lock( (*indexes)[i]->statisticsLock() );
documentCount += (int)(*indexes)[i]->documentCount();
}
delete _repository;
indri::api::QueryEnvironment env;
std::cout << "opening index: " << index << std::endl;
env.addIndex( index );
std::string input = param["input"];
std::string priorName = param["name"];
size_t memory = param.get( "memory", 50*1024*1024 );
// step one - convert file from docno/score format to binary format
indri::file::File unsortedBinary;
std::string unsortedName;
unsortedBinary.openTemporary( unsortedName );
std::cout << "converting to binary...";
std::cout.flush();
convert_docnoscore_to_binary( unsortedBinary, input, env );
std::cout << "finished" << std::endl;
// step two -- sort the binary version
indri::file::File uncompressedPrior;
std::string uncompressedPriorName;
uncompressedPrior.openTemporary( uncompressedPriorName );
std::cout << "sorting...";
std::cout.flush();
sort_file( uncompressedPrior, unsortedBinary, memory, documentCount );
std::cout << "finished";
unsortedBinary.close();
lemur_compat::remove( unsortedName.c_str() );
// step three -- check to see if it's compressable, if so, compress it
std::map<double, int> table;
indri::file::File compressedPrior;
std::string compressedPriorName;
compressedPrior.openTemporary( compressedPriorName );
indri::file::File& finalPrior = uncompressedPrior;
std::cout << "checking for compressability...";
std::cout.flush();
bool result = extract_compression_table( table, uncompressedPrior );
if( result ) {
std::cout << "yep" << std::endl;
// compress the file by using a lookup table
std::cout << "compressing...";
std::cout.flush();
compress_file( compressedPrior, uncompressedPrior, table );
std::cout << std::endl;
finalPrior = compressedPrior;
} else {
std::cout << "nope" << std::endl;
}
// step four -- install the prior in the index
std::cout << "installing...";
std::cout.flush();
install_prior( index, priorName, finalPrior );
std::cout << "finished" << std::endl;
// clean up
uncompressedPrior.close();
compressedPrior.close();
lemur_compat::remove( uncompressedPriorName.c_str() );
lemur_compat::remove( compressedPriorName.c_str() );
} catch( lemur::api::Exception& e ) {
LEMUR_ABORT(e);
}
return 0;
}
