inline std::vector<Monitor::Stat> Monitor::toc() {
std::vector<Monitor::Stat> results;
if (activated) {
activated = false;
for (auto* exe : exes) {
for (auto& arg : exe->arg_arrays) {
arg.WaitToRead();
}
for (auto& aux : exe->aux_arrays) {
aux.WaitToRead();
}
for (auto &pair : exe->arg_dict()) {
if (std::regex_match(pair.first, pattern)) {
stats.emplace_back(step, pair.first, stat_func(pair.second));
}
}
for (auto &pair : exe->aux_dict()) {
if (std::regex_match(pair.first, pattern)) {
stats.emplace_back(step, pair.first, stat_func(pair.second));
}
}
}
results.swap(stats);
}
++step;
return results;
}
/**
* Performs the send for a particular file/directory. If a directory is
* specified, get the list of files and call recursively for each.
* Returns 0 if a file was sent and none received it, 1 otherwise
*/
int send_file(const char *basedir, const char *filename,
const char *n_destfname, uint32_t group_id)
{
static uint16_t file_id = 1;
struct finfo_t finfo;
stat_struct statbuf;
char path[MAXPATHNAME], destpath[MAXPATHNAME];
int len, rval, fd, emptydir;
log(0, 0, "----- %s -----", filename);
len = snprintf(path, sizeof(path), "%s%c%s", basedir, PATH_SEP, filename);
if ((len >= sizeof(path)) || (len == -1)) {
log(0, 0, "Max pathname length exceeded: %s%c%s",
basedir, PATH_SEP, filename);
return 1;
}
if (follow_links) {
rval = stat_func(path, &statbuf);
} else {
rval = lstat_func(path, &statbuf);
}
if (rval == -1) {
syserror(0, 0, "Error getting file status for %s", filename);
return 1;
}
if (file_excluded(filename)) {
log(0, 0, "Skipping %s", filename);
return 1;
}
rval = 1;
if (S_ISREG(statbuf.st_mode)) {
if ((fd = open(path, OPENREAD, 0)) == -1) {
syserror(0, 0, "Error reading file %s", filename);
return 1;
}
close(fd);
memset(&finfo, 0, sizeof(struct finfo_t));
finfo.ftype = FTYPE_REG;
finfo.basedir = basedir;
finfo.filename = filename;
finfo.destfname = n_destfname;
finfo.group_id = group_id;
finfo.file_id = file_id++;
if (file_id == 0) {
file_id = 1;
}
finfo.size = statbuf.st_size;
finfo.blocks = (int32_t)((finfo.size / blocksize) +
(finfo.size % blocksize ? 1 :0));
finfo.sections = (finfo.blocks / (blocksize * 8)) +
(finfo.blocks % (blocksize * 8) ? 1 : 0);
finfo.naklist = calloc(finfo.blocks, 1);
finfo.deststate = calloc(destcount ? destcount : MAXDEST,
sizeof(struct deststate_t));
if ((finfo.naklist == NULL) || (finfo.deststate == NULL)) {
syserror(0, 0, "calloc failed!");
exit(1);
}
finfo.partial = 1;
rval = announce_phase(&finfo);
if (rval) {
rval = transfer_phase(&finfo);
}
free(finfo.deststate);
free(finfo.naklist);
#ifndef WINDOWS
} else if (S_ISLNK(statbuf.st_mode)) {
char linkname[MAXPATHNAME];
memset(linkname, 0, sizeof(linkname));
if (readlink(path, linkname, sizeof(linkname)-1) == -1) {
syserror(0, 0, "Failed to read symbolic link %s", path);
return 1;
}
// Both the file name and the link have to fit into a fileinfo_h.name
if (strlen(linkname) + strlen(filename) + 2 > MAXPATHNAME) {
log(0, 0, "Combined file name %s and link %s too long",
filename, linkname);
return 1;
}
memset(&finfo, 0, sizeof(struct finfo_t));
finfo.ftype = FTYPE_LINK;
finfo.basedir = basedir;
finfo.filename = filename;
finfo.destfname = n_destfname;
finfo.linkname = linkname;
finfo.group_id = group_id;
finfo.file_id = file_id++;
if (file_id == 0) {
file_id = 1;
}
finfo.deststate = calloc(destcount ? destcount : MAXDEST,
sizeof(struct deststate_t));
if (finfo.deststate == NULL) {
syserror(0, 0, "calloc failed!");
exit(1);
}
finfo.partial = 1;
rval = announce_phase(&finfo);
if (rval) {
rval = transfer_phase(&finfo);
}
#endif
} else if (S_ISDIR(statbuf.st_mode)) {
// read directory and do recursive send
#ifdef WINDOWS
intptr_t ffhandle;
struct _finddatai64_t ffinfo;
char dirglob[MAXPATHNAME];
snprintf(dirglob, sizeof(dirglob), "%s%c%s%c*", basedir, PATH_SEP,
filename, PATH_SEP);
if ((ffhandle = _findfirsti64(dirglob, &ffinfo)) == -1) {
syserror(0, 0, "Failed to open directory %s%c%s", basedir, PATH_SEP,
filename);
return 1;
}
emptydir = 1;
do {
len = snprintf(path, sizeof(path), "%s/%s", filename, ffinfo.name);
if ((len >= sizeof(path)) || (len == -1)) {
log(0, 0, "Max pathname length exceeded: %s/%s",
filename, ffinfo.name);
continue;
}
len = snprintf(destpath, sizeof(destpath), "%s/%s",
n_destfname, ffinfo.name);
if ((len >= sizeof(destpath)) || (len == -1)) {
log(0, 0, "Max pathname length exceeded: %s/%s",
n_destfname, ffinfo.name);
continue;
}
if (strcmp(ffinfo.name, ".") && strcmp(ffinfo.name, "..")) {
emptydir = 0;
if (!send_file(basedir, path, destpath, group_id)) {
rval = 0;
break;
}
}
} while (_findnexti64(ffhandle, &ffinfo) == 0);
_findclose(ffhandle);
#else
DIR *dir;
struct dirent *de;
char dirname[MAXPATHNAME];
snprintf(dirname, sizeof(dirname), "%s%c%s", basedir,PATH_SEP,filename);
if ((dir = opendir(dirname)) == NULL) {
syserror(0, 0, "Failed to open directory %s", dirname);
return 1;
}
// errno needs to be set to 0 before calling readdir, otherwise
// we'll report a false error when we exhaust the directory
emptydir = 1;
while ((errno = 0, de = readdir(dir)) != NULL) {
len = snprintf(path, sizeof(path), "%s/%s", filename, de->d_name);
if ((len >= sizeof(path)) || (len == -1)) {
log(0, 0, "Max pathname length exceeded: %s/%s",
filename, de->d_name);
continue;
}
len = snprintf(destpath, sizeof(destpath), "%s/%s",
n_destfname, de->d_name);
if ((len >= sizeof(destpath)) || (len == -1)) {
log(0, 0, "Max pathname length exceeded: %s/%s",
n_destfname, de->d_name);
continue;
}
if (strcmp(de->d_name, ".") && strcmp(de->d_name, "..")) {
emptydir = 0;
if (!send_file(basedir, path, destpath, group_id)) {
rval = 0;
break;
}
}
}
if (errno && (errno != ENOENT)) {
syserror(0, 0, "Failed to read directory %s", filename);
}
closedir(dir);
#endif
if (emptydir) {
memset(&finfo, 0, sizeof(struct finfo_t));
finfo.ftype = FTYPE_DIR;
finfo.basedir = basedir;
finfo.filename = filename;
finfo.destfname = n_destfname;
finfo.group_id = group_id;
finfo.file_id = file_id++;
if (file_id == 0) {
file_id = 1;
}
finfo.deststate = calloc(destcount ? destcount : MAXDEST,
sizeof(struct deststate_t));
if (finfo.deststate == NULL) {
syserror(0, 0, "calloc failed!");
exit(1);
}
finfo.partial = 1;
rval = announce_phase(&finfo);
if (rval) {
rval = transfer_phase(&finfo);
}
}
} else {
log(0, 0, "Skipping special file %s", filename);
}
return rval;
}
int main(int argc, char** argv) {
gchar outFile[1024];
gchar readBuf[READBUFZ];
gchar* extension = "bz2";
struct stat statBuf;
off_t fileSize = 0;
off_t partSize = 0;
ssize_t nread, written;
tp_args_t *tp_args = NULL;
int i, hours, min, sec, start;
int outFd, tmpFd;
time_t startTime, end, elapsed;
GThreadPool *tp;
gboolean anyErrors = FALSE;
FILE *infile;
FILE *outfile;
gchar* filen = NULL;
gint nthreads = NTHREADS;
gboolean verbose = FALSE;
gboolean useStdout = FALSE;
gboolean keepFile = FALSE;
gboolean useStream = FALSE;
gboolean useOMP = FALSE;
gboolean useGzip = FALSE;
gchar* outFileArg = NULL;
/* Initialize gthreads */
g_thread_init(NULL);
/* register the signal handler */
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = &_sig_handler;
sigaction(SIGINT, &sa, NULL);
startTime = time(NULL);
GOptionEntry entries[] = {
{ "threads", 't', 0, G_OPTION_ARG_INT, &nthreads,
"The number of threads to use.", NULL },
{ "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose,
"Show Progress", NULL },
{ "outfile", 'o', 0, G_OPTION_ARG_FILENAME, &outFileArg,
"The name of an output file to write to", NULL },
{ "stdout", 'c', 0, G_OPTION_ARG_NONE, &useStdout,
"Write data to standard out", NULL },
{ "stream", 's', 0, G_OPTION_ARG_NONE, &useStream,
"Compress using the stream method", NULL },
{ "omp", 'm', 0, G_OPTION_ARG_NONE, &useOMP,
"Compress using the openMP method", NULL },
{ "zip", 'z', 0, G_OPTION_ARG_NONE, &useGzip,
"Compress using the gzip algorithm", NULL },
{ "keep", 'k', 0, G_OPTION_ARG_NONE, &keepFile,
"Do not delete the input file when done", NULL },
{ NULL }
};
GError *error = NULL;
GOptionContext *context;
context = g_option_context_new ("<file> - muti threaded bzip2 compression utility\n\n <file> - The name of the file to compress or just use \"-\" to indicate \n\tstdandard input.");
g_option_context_add_main_entries(context, entries, NULL);
g_option_context_parse(context, &argc, &argv, &error);
g_option_context_free(context);
if (error != NULL) {
fprintf(stderr, "Error parsing arguments: %s\n", error->message);
exit(1);
}
if (nthreads <= 0) {
fprintf(stderr, "Invalid value for -t \"%d\". Must be greater than 0.\n", nthreads);
exit(1);
}
if (argc > 1)
filen = argv[1];
if (filen == NULL) {
fprintf(stderr, "You Must specify a <filename> arg\n");
exit(1);
}
if (useGzip)
extension = "gz";
if (useStdout && outFileArg) {
fprintf(stderr, "You can not specify -c and -o together\n");
exit(1);
}
if (useStdout) {
outfile = stdout;
}
if (strcmp(filen, "-") == 0) {
/* you must use the stream method if reading from stdin */
useStream = TRUE;
} else {
/* make sure the input file is readable */
if (!g_file_test(filen, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_REGULAR)) {
fprintf(stderr, "Invalid file: \"%s\". It must be a regular file (not a symlink or directory)\n", filen);
exit(1);
}
if (access(filen, R_OK) != 0) {
fprintf(stderr, "Invalid file: \"%s\". It is not readable\n", filen);
exit(1);
}
}
/* populate the name of the outfile */
if (!useStdout) {
if (outFileArg == NULL) {
if (strcmp(filen, "-") != 0) {
g_snprintf(&(outFile[0]), 1024, "%s.%s", filen, extension);
} else {
fprintf(stderr, "You must supply an output file argument (-o) if you are reading from stdin, or use -c to write to stdout (but don't forget to redirect the output!)\n");
exit(1);
}
} else {
g_snprintf(&(outFile[0]), 1024, "%s", outFileArg);
}
}
if (useStream) {
if (useGzip) {
fprintf(stderr, "Error you can not use gzip compression in stream mode\n");
exit(1);
}
/* Open the input file for streaming */
if (strcmp(filen, "-") == 0) {
infile = stdin;
} else {
infile = fopen(filen, "r");
if (infile == NULL) {
perror("Error opening file for reading");
exit(1);
}
}
/* open the output file for streaming */
if (useStdout) {
outfile = stdout; /* use stdout when using stdin unless -o is used */
} else {
outfile = fopen(outFile, "w");
if (outfile == NULL) {
perror("Error opening file for writing");
exit(1);
}
}
if(useOMP) {
#ifndef _OPENMP
fprintf(stderr, "Sorry OpenMP is not available\n");
#else
if (verbose) {
fprintf(stderr, "Using OpenMP :)\n");
}
#endif
omp_driver(verbose, nthreads,infile, outfile);
} else {
/* use the stream method to compress the file */
stream_driver(verbose, nthreads, infile, outfile);
}
if (!useStdout) {
fclose(outfile);
if (_recieved_SIGINT) {
remove(outFile);
}
}
if (strcmp(filen, "-") != 0) {
fclose(infile);
if (!keepFile && !useStdout && !_recieved_SIGINT) {
remove(filen);
}
}
} else {
/* use the split method */
tp_args = g_new0(tp_args_t, nthreads);
/* split it into n parts and spawn a thread for each part */
stat(filen, &statBuf);
fileSize = statBuf.st_size;
partSize = statBuf.st_size/nthreads;
/* fprintf(stderr, "inputfile: %s. Size=%lld partSize=%lld\n",
filen, fileSize, partSize);
*/
if (useGzip)
tp = g_thread_pool_new(file_read_func_zlib, NULL, nthreads, TRUE, NULL);
else
tp = g_thread_pool_new(file_read_func, NULL, nthreads, TRUE, NULL);
for (i = 0; i < nthreads; i++) {
/* set up each threads piece of the file */
tp_args[i].filen = filen;
tp_args[i].startPos = partSize * i;
tp_args[i].verbose = verbose;
if (i == nthreads -1)
tp_args[i].endPos = fileSize;
else
tp_args[i].endPos = (partSize * (i + 1));
getTmpFilen(&(tp_args[i].tmpFilen[0]), 1024, filen, i);
/* fprintf(stderr, "thread %d:\n outf=%s\n start=%lld\n end=%lld\n",
i, tp_args[i].tmpFilen, tp_args[i].startPos, tp_args[i].endPos);
*/
/* into the pool kids */
g_thread_pool_push(tp, &(tp_args[i]), NULL);
}
if (verbose) {
/* stat_func blocks untill all threads report that they are done */
stat_func(tp_args, nthreads);
}
/* wait until all threads are done */
g_thread_pool_free(tp, FALSE, TRUE);
if (verbose) {
end = time(NULL);
elapsed = end - startTime;
hours = elapsed/3600;
min = (elapsed - (hours * 3600))/60;
sec = (elapsed - (hours * 3600) - (min * 60));
fprintf(stderr, "zipping took %02d:%02d:%02d\n", hours, min, sec);
}
/* check for errors */
for (i = 0; i < nthreads && anyErrors == FALSE; i++) {
anyErrors = tp_args[i].error;
}
if (anyErrors) {
/* delete temp files and exit */
for (i = 0; i < nthreads; i++) {
remove(tp_args[i].tmpFilen);
}
exit(1);
}
/* now join the different parts together */
if (useStdout) {
outFd = fileno(stdout);
start = 0;
} else {
/* It's faster to just rename the first part rather then
copy its data. The rest of the parts need to be appended
in the correct order.
*/
rename(tp_args[0].tmpFilen, outFile);
outFd = open(outFile, O_WRONLY|O_APPEND, NULL);
start = 1;
}
for (i = start; i < nthreads; i++) {
/* open one of the temp files */
tmpFd = open(tp_args[i].tmpFilen, O_RDONLY);
if (verbose)
fprintf(stderr,"cat ing %s\n", tp_args[i].tmpFilen);
do {
/* copy the data from the temp file to the end of the output file */
nread = read(tmpFd, &(readBuf[0]), READBUFZ);
written = write(outFd, &(readBuf[0]), nread);
if (written != nread) {
fprintf(stderr, "Error: read %d bytes from %s but could only write %d bytes to %s\n",
nread, tp_args[i].tmpFilen, written, outFile);
} else {
/* fprintf(stderr, "Read %d bytes from %s, wrote %d byes to %s\n",
nread, tp_args[i].tmpFilen, written, outFile);
*/
}
} while (nread > 0);
close(tmpFd);
/* delete the temporty file */
remove(tp_args[i].tmpFilen);
}
if (!useStdout) {
close(outFd);
if (_recieved_SIGINT) {
remove(outFile);
}
}
g_free(tp_args);
if( !keepFile && !useStdout && !_recieved_SIGINT) {
remove(filen);
}
}
if (verbose) {
end = time(NULL);
elapsed = end - startTime;
hours = elapsed/3600;
min = (elapsed - (hours * 3600))/60;
sec = (elapsed - (hours * 3600) - (min * 60));
fprintf(stderr, "took %02d:%02d:%02d\n", hours, min, sec);
}
exit(0);
}
