Esempio n. 1
0
inline std::vector<Monitor::Stat> Monitor::toc() {
  std::vector<Monitor::Stat> results;
  if (activated) {
    activated = false;

    for (auto* exe : exes) {
      for (auto& arg : exe->arg_arrays) {
        arg.WaitToRead();
      }
      for (auto& aux : exe->aux_arrays) {
        aux.WaitToRead();
      }

      for (auto &pair : exe->arg_dict()) {
        if (std::regex_match(pair.first, pattern)) {
          stats.emplace_back(step, pair.first, stat_func(pair.second));
        }
      }
      for (auto &pair : exe->aux_dict()) {
        if (std::regex_match(pair.first, pattern)) {
          stats.emplace_back(step, pair.first, stat_func(pair.second));
        }
      }
    }
    results.swap(stats);
  }
  ++step;
  return results;
}
Esempio n. 2
0
/**
 * Performs the send for a particular file/directory.  If a directory is
 * specified, get the list of files and call recursively for each.
 * Returns 0 if a file was sent and none received it, 1 otherwise
 */
int send_file(const char *basedir, const char *filename,
              const char *n_destfname, uint32_t group_id)
{
    static uint16_t file_id = 1;
    struct finfo_t finfo;
    stat_struct statbuf;
    char path[MAXPATHNAME], destpath[MAXPATHNAME];
    int len, rval, fd, emptydir;

    log(0, 0, "----- %s -----", filename);
    len = snprintf(path, sizeof(path), "%s%c%s", basedir, PATH_SEP, filename);
    if ((len >= sizeof(path)) || (len == -1)) {
        log(0, 0, "Max pathname length exceeded: %s%c%s",
                   basedir, PATH_SEP, filename);
        return 1;
    }
    if (follow_links) {
        rval = stat_func(path, &statbuf);
    } else {
        rval = lstat_func(path, &statbuf);
    }
    if (rval == -1) {
        syserror(0, 0, "Error getting file status for %s", filename);
        return 1;
    }
    if (file_excluded(filename)) {
        log(0, 0, "Skipping %s", filename);
        return 1;
    }
    rval = 1;
    if (S_ISREG(statbuf.st_mode)) {
        if ((fd = open(path, OPENREAD, 0)) == -1) {
            syserror(0, 0, "Error reading file %s", filename);
            return 1;
        }
        close(fd);
        memset(&finfo, 0, sizeof(struct finfo_t));
        finfo.ftype = FTYPE_REG;
        finfo.basedir = basedir;
        finfo.filename = filename;
        finfo.destfname = n_destfname;
        finfo.group_id = group_id;
        finfo.file_id = file_id++;
        if (file_id == 0) {
            file_id = 1;
        }
        finfo.size = statbuf.st_size;
        finfo.blocks = (int32_t)((finfo.size / blocksize) +
                (finfo.size % blocksize ? 1 :0));
        finfo.sections = (finfo.blocks / (blocksize * 8)) +
                (finfo.blocks % (blocksize * 8) ? 1 : 0);
        finfo.naklist = calloc(finfo.blocks, 1);
        finfo.deststate = calloc(destcount ? destcount : MAXDEST,
                sizeof(struct deststate_t));
        if ((finfo.naklist == NULL) || (finfo.deststate == NULL)) {
            syserror(0, 0, "calloc failed!");
            exit(1);
        }
        finfo.partial = 1;
        rval = announce_phase(&finfo);
        if (rval) {
            rval = transfer_phase(&finfo);
        }
        free(finfo.deststate);
        free(finfo.naklist);
#ifndef WINDOWS
    } else if (S_ISLNK(statbuf.st_mode)) {
        char linkname[MAXPATHNAME];

        memset(linkname, 0, sizeof(linkname));
        if (readlink(path, linkname, sizeof(linkname)-1) == -1) {
            syserror(0, 0, "Failed to read symbolic link %s", path);
            return 1;
        }
        // Both the file name and the link have to fit into a fileinfo_h.name
        if (strlen(linkname) + strlen(filename) + 2 > MAXPATHNAME) {
            log(0, 0, "Combined file name %s and link %s too long",
                      filename, linkname);
            return 1;
        }
        memset(&finfo, 0, sizeof(struct finfo_t));
        finfo.ftype = FTYPE_LINK;
        finfo.basedir = basedir;
        finfo.filename = filename;
        finfo.destfname = n_destfname;
        finfo.linkname = linkname;
        finfo.group_id = group_id;
        finfo.file_id = file_id++;
        if (file_id == 0) {
            file_id = 1;
        }
        finfo.deststate = calloc(destcount ? destcount : MAXDEST,
                sizeof(struct deststate_t));
        if (finfo.deststate == NULL) {
            syserror(0, 0, "calloc failed!");
            exit(1);
        }
        finfo.partial = 1;
        rval = announce_phase(&finfo);
        if (rval) {
            rval = transfer_phase(&finfo);
        }
#endif
    } else if (S_ISDIR(statbuf.st_mode)) {
        // read directory and do recursive send
#ifdef WINDOWS
        intptr_t ffhandle;
        struct _finddatai64_t ffinfo;
        char dirglob[MAXPATHNAME];

        snprintf(dirglob, sizeof(dirglob), "%s%c%s%c*", basedir, PATH_SEP,
                                                        filename, PATH_SEP);
        if ((ffhandle = _findfirsti64(dirglob, &ffinfo)) == -1) {
            syserror(0, 0, "Failed to open directory %s%c%s", basedir, PATH_SEP,
                                                              filename);
            return 1;
        }
        emptydir = 1;
        do {
            len = snprintf(path, sizeof(path), "%s/%s", filename, ffinfo.name);
            if ((len >= sizeof(path)) || (len == -1)) {
                log(0, 0, "Max pathname length exceeded: %s/%s",
                           filename, ffinfo.name);
                continue;
            }
            len = snprintf(destpath, sizeof(destpath), "%s/%s",
                           n_destfname, ffinfo.name);
            if ((len >= sizeof(destpath)) || (len == -1)) {
                log(0, 0, "Max pathname length exceeded: %s/%s",
                           n_destfname, ffinfo.name);
                continue;
            }
            if (strcmp(ffinfo.name, ".") && strcmp(ffinfo.name, "..")) {
                emptydir = 0;
                if (!send_file(basedir, path, destpath, group_id)) {
                    rval = 0;
                    break;
                }
            }
        } while (_findnexti64(ffhandle, &ffinfo) == 0);
        _findclose(ffhandle);
#else
        DIR *dir;
        struct dirent *de;
        char dirname[MAXPATHNAME];

        snprintf(dirname, sizeof(dirname), "%s%c%s", basedir,PATH_SEP,filename);
        if ((dir = opendir(dirname)) == NULL) {
            syserror(0, 0, "Failed to open directory %s", dirname);
            return 1;
        }
        // errno needs to be set to 0 before calling readdir, otherwise
        // we'll report a false error when we exhaust the directory
        emptydir = 1;
        while ((errno = 0, de = readdir(dir)) != NULL) {
            len = snprintf(path, sizeof(path), "%s/%s", filename, de->d_name);
            if ((len >= sizeof(path)) || (len == -1)) {
                log(0, 0, "Max pathname length exceeded: %s/%s",
                           filename, de->d_name);
                continue;
            }
            len = snprintf(destpath, sizeof(destpath), "%s/%s",
                           n_destfname, de->d_name);
            if ((len >= sizeof(destpath)) || (len == -1)) {
                log(0, 0, "Max pathname length exceeded: %s/%s",
                           n_destfname, de->d_name);
                continue;
            }
            if (strcmp(de->d_name, ".") && strcmp(de->d_name, "..")) {
                emptydir = 0;
                if (!send_file(basedir, path, destpath, group_id)) {
                    rval = 0;
                    break;
                }
            }
        }
        if (errno && (errno != ENOENT)) {
            syserror(0, 0, "Failed to read directory %s", filename);
        }
        closedir(dir);
#endif
        if (emptydir) {
            memset(&finfo, 0, sizeof(struct finfo_t));
            finfo.ftype = FTYPE_DIR;
            finfo.basedir = basedir;
            finfo.filename = filename;
            finfo.destfname = n_destfname;
            finfo.group_id = group_id;
            finfo.file_id = file_id++;
            if (file_id == 0) {
                file_id = 1;
            }
            finfo.deststate = calloc(destcount ? destcount : MAXDEST,
                    sizeof(struct deststate_t));
            if (finfo.deststate == NULL) {
                syserror(0, 0, "calloc failed!");
                exit(1);
            }
            finfo.partial = 1;
            rval = announce_phase(&finfo);
            if (rval) {
                rval = transfer_phase(&finfo);
            }
        }
    } else {
        log(0, 0, "Skipping special file %s", filename);
    }
    return rval;
}
Esempio n. 3
0
int main(int argc, char** argv) {
  gchar outFile[1024];
  gchar readBuf[READBUFZ];
  gchar* extension = "bz2";
  struct stat statBuf;
  off_t fileSize = 0;
  off_t partSize = 0;
  ssize_t nread, written;
  tp_args_t *tp_args = NULL;
  int i, hours, min, sec, start;
  int outFd, tmpFd;
  time_t startTime, end, elapsed;
  GThreadPool *tp;
  gboolean anyErrors = FALSE;
  FILE *infile;
  FILE *outfile;

  gchar* filen = NULL;
  gint  nthreads = NTHREADS;
  gboolean verbose = FALSE;
  gboolean useStdout = FALSE;
  gboolean keepFile = FALSE;
  gboolean useStream = FALSE;
  gboolean useOMP = FALSE;
  gboolean useGzip = FALSE;
  gchar* outFileArg = NULL;

  /* Initialize gthreads */
  g_thread_init(NULL);

  /* register the signal handler */
  struct sigaction sa;
  memset(&sa, 0, sizeof(sa));
  sa.sa_handler = &_sig_handler;
  sigaction(SIGINT, &sa, NULL);
  
  startTime = time(NULL);
  GOptionEntry entries[] = {
    { "threads", 't', 0, G_OPTION_ARG_INT, &nthreads, 
      "The number of threads to use.", NULL },
    { "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose, 
      "Show Progress", NULL },
    { "outfile", 'o', 0, G_OPTION_ARG_FILENAME, &outFileArg, 
      "The name of an output file to write to", NULL },
    { "stdout", 'c', 0, G_OPTION_ARG_NONE, &useStdout, 
      "Write data to standard out", NULL },
    { "stream", 's', 0, G_OPTION_ARG_NONE, &useStream,
       "Compress using the stream method", NULL },
    { "omp", 'm', 0, G_OPTION_ARG_NONE, &useOMP,
       "Compress using the openMP method", NULL },
    { "zip", 'z', 0, G_OPTION_ARG_NONE, &useGzip,
       "Compress using the gzip algorithm", NULL },
    { "keep", 'k', 0, G_OPTION_ARG_NONE, &keepFile,
       "Do not delete the input file when done", NULL },
    { NULL }
  };
  
  GError *error = NULL;
  GOptionContext *context;
  
  context = g_option_context_new ("<file> - muti threaded bzip2 compression utility\n\n  <file> - The name of the file to compress or just use \"-\" to indicate \n\tstdandard input.");
  g_option_context_add_main_entries(context, entries, NULL);
  g_option_context_parse(context, &argc, &argv, &error);
  g_option_context_free(context);
  if (error != NULL) {
    fprintf(stderr, "Error parsing arguments: %s\n", error->message);
    exit(1);
  }

  if (nthreads <= 0) {
    fprintf(stderr, "Invalid value for -t \"%d\". Must be greater than 0.\n", nthreads);
    exit(1);
  }

  if (argc > 1)
    filen = argv[1];

  if (filen == NULL) {
    fprintf(stderr, "You Must specify a <filename> arg\n");
    exit(1);
  }

  if (useGzip)
    extension = "gz";

  if (useStdout && outFileArg) {
    fprintf(stderr, "You can not specify -c and -o together\n");
    exit(1);
    
  }

  if (useStdout) {
    outfile = stdout;
  } 

  if (strcmp(filen, "-") == 0) {
    /* you must use the stream method if reading from stdin */
    useStream = TRUE;
  } else {
    /* make sure the input file is readable */
    if (!g_file_test(filen, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_REGULAR)) {
      fprintf(stderr, "Invalid file: \"%s\". It must be a regular file (not a symlink or directory)\n", filen);
      exit(1);
    }
    
    if (access(filen, R_OK) != 0) {
      fprintf(stderr, "Invalid file: \"%s\". It is not readable\n", filen);
      exit(1);
    }
  }

  /* populate the name of the outfile */
  if (!useStdout) {
    if (outFileArg == NULL) {
      if (strcmp(filen, "-") != 0) {
        g_snprintf(&(outFile[0]), 1024, "%s.%s", filen, extension);
      } else {
        fprintf(stderr, "You must supply an output file argument (-o) if you are reading from stdin, or use -c to write to stdout (but don't forget to redirect the output!)\n");
        exit(1);
      }
    } else {
      g_snprintf(&(outFile[0]), 1024, "%s", outFileArg);
    }
  }
  
  if (useStream) {
    if (useGzip) {
      fprintf(stderr, "Error you can not use gzip compression in stream mode\n");
      exit(1);
    }
    /* Open the input file for streaming */
    if (strcmp(filen, "-") == 0) {
      infile = stdin;
    } else {
      infile = fopen(filen, "r");
      if (infile == NULL) {
        perror("Error opening file for reading");
        exit(1);
      }
    }

    /* open the output file for streaming */
    if (useStdout) {
      outfile = stdout; /* use stdout when using stdin unless -o is used */
    } else {
      outfile = fopen(outFile, "w");
      if (outfile == NULL) {
        perror("Error opening file for writing");
        exit(1);
      }
    }

    if(useOMP) {
#ifndef _OPENMP
      fprintf(stderr, "Sorry OpenMP is not available\n");
#else
      if (verbose) {
        fprintf(stderr, "Using OpenMP :)\n");
      }
#endif
      omp_driver(verbose, nthreads,infile, outfile);
    } else {
      /* use the stream method to compress the file */
      stream_driver(verbose, nthreads, infile, outfile);
    }

    if (!useStdout) {
      fclose(outfile);
      if (_recieved_SIGINT) {
        remove(outFile);
      }
    }

    if (strcmp(filen, "-") != 0) {
      fclose(infile);
      
      if (!keepFile && !useStdout && !_recieved_SIGINT) {
        remove(filen);
      }
    }
  } else  {
    /* use the split method */

    tp_args = g_new0(tp_args_t, nthreads);
    
    /* split it into n parts and spawn a thread for each part */
    stat(filen, &statBuf);
    fileSize = statBuf.st_size;
    partSize = statBuf.st_size/nthreads;
    
    /* fprintf(stderr, "inputfile: %s.  Size=%lld partSize=%lld\n",
       filen, fileSize, partSize);
    */
    if (useGzip)
      tp = g_thread_pool_new(file_read_func_zlib, NULL, nthreads, TRUE, NULL);
    else
      tp = g_thread_pool_new(file_read_func, NULL, nthreads, TRUE, NULL);
    
    for (i = 0; i < nthreads; i++) {
      /* set up each threads piece of the file */
      tp_args[i].filen = filen;
      tp_args[i].startPos = partSize * i;
      tp_args[i].verbose = verbose;
      if (i == nthreads -1)
        tp_args[i].endPos = fileSize;
      else
        tp_args[i].endPos = (partSize * (i + 1));
      
      getTmpFilen(&(tp_args[i].tmpFilen[0]), 1024, filen, i);
      
      /* fprintf(stderr, "thread %d:\n  outf=%s\n  start=%lld\n  end=%lld\n",
         i, tp_args[i].tmpFilen, tp_args[i].startPos, tp_args[i].endPos);
      */

      /* into the pool kids */
      g_thread_pool_push(tp, &(tp_args[i]), NULL);
    }
    
    
    if (verbose) {
      /* stat_func blocks untill all threads report that they are done */
      stat_func(tp_args, nthreads);
    }
    
    /* wait until all threads are done */
    g_thread_pool_free(tp, FALSE, TRUE);
    
    if (verbose) {
      end = time(NULL);
      elapsed = end - startTime;
      hours = elapsed/3600;
      min = (elapsed - (hours * 3600))/60;
      sec = (elapsed - (hours * 3600) - (min * 60));
      fprintf(stderr, "zipping took %02d:%02d:%02d\n", hours, min, sec);
    }
    
    /* check for errors */
    for (i = 0; i < nthreads && anyErrors == FALSE; i++) {
      anyErrors = tp_args[i].error;
    }
    
    if (anyErrors) {
      /* delete temp files and exit */
      for (i = 0; i < nthreads; i++) {
        remove(tp_args[i].tmpFilen);
      }
      exit(1);
    }
    /* now join the different parts together */
    if (useStdout) {
      outFd = fileno(stdout);
      start = 0;
    } else {
      /* It's faster to just rename the first part rather then 
         copy its data. The rest of the parts need to be appended
         in the correct order.
      */
      rename(tp_args[0].tmpFilen, outFile);
      outFd = open(outFile, O_WRONLY|O_APPEND, NULL);
      start = 1;
    }
    for (i = start; i < nthreads; i++) {
      /* open one of the temp files */
      tmpFd  = open(tp_args[i].tmpFilen, O_RDONLY);
      if (verbose)
        fprintf(stderr,"cat ing %s\n", tp_args[i].tmpFilen);
      
      do {
        /* copy the data from the temp file to the end of the output file */
        nread = read(tmpFd, &(readBuf[0]), READBUFZ);
        written = write(outFd, &(readBuf[0]), nread);
        if (written != nread) {
          fprintf(stderr, "Error: read %d bytes from %s but could only write %d bytes to %s\n",
                  nread, tp_args[i].tmpFilen, written, outFile);
        } else {
          /* fprintf(stderr, "Read %d bytes from %s, wrote %d byes to %s\n",
             nread, tp_args[i].tmpFilen, written, outFile);
          */
        }
      } while (nread > 0);
      
      close(tmpFd);
      /* delete the temporty file */
      remove(tp_args[i].tmpFilen);
    }
    
    if (!useStdout) {
      close(outFd);
      if (_recieved_SIGINT) {
        remove(outFile);
      }
    }
    
    g_free(tp_args);
    
    if( !keepFile && !useStdout && !_recieved_SIGINT) {
      remove(filen);
    }
  }
  
  if (verbose) {
    end = time(NULL);
    elapsed = end - startTime;
    hours = elapsed/3600;
    min = (elapsed - (hours * 3600))/60;
    sec = (elapsed - (hours * 3600) - (min * 60));
    fprintf(stderr, "took %02d:%02d:%02d\n", hours, min, sec);
  }

  exit(0);
}