Exemple #1
0
int main(int argc, char * argv[]) {
    try {
        // Note: Verbose mode logging is enabled (if requested) by options parse()
        opt.parse(argc, argv);
    } catch (exception &e) {
        cerr << "Error processing arguments: " << e.what() << endl;
        opt.printHelp(argv[0]);
        return 1;
    }

    if (opt.env()) {
        opt.setApiserverDxConfig();  // needed for 'ua --env' to report project name
        printEnvironmentInfo();
        return 0;
    }

    if (opt.version()) {
        cout << "Upload Agent Version: " << UAVERSION;
#if OLD_KERNEL_SUPPORT
        cout << " (old-kernel-support)";
#endif
        cout << endl
             << "git version: " << DXTOOLKIT_GITVERSION << endl
             << "libboost version: " << (BOOST_VERSION / 100000) << "." << ((BOOST_VERSION / 100) % 1000) << "." << (BOOST_VERSION % 100) << endl
             << "libcurl version: " << LIBCURL_VERSION_MAJOR << "." << LIBCURL_VERSION_MINOR << "." << LIBCURL_VERSION_PATCH << endl;
        return 0;
    } else if (opt.help() || opt.files.empty()) {
        opt.printHelp(argv[0]);
        return (opt.help()) ? 0 : 1;
    }

    setUserAgentString(); // also sets dx::config::USER_AGENT_STRING()
    DXLOG(logINFO) << "DNAnexus Upload Agent " << UAVERSION << " (git version: " << DXTOOLKIT_GITVERSION << ")";
    DXLOG(logINFO) << "Upload agent's User Agent string: '" << userAgentString << "'";
    DXLOG(logINFO) << "dxcpp's User Agent string: '" << dx::config::USER_AGENT_STRING() << "'";
    DXLOG(logINFO) << opt;

    try {
        opt.setApiserverDxConfig();
        opt.validate();

        /*
         * Check for updates, and terminate execution if necessary. This also
         * has the side effect of verifying that we can connect to the API
         * server, and that the authentication token is valid.
         */
        try {
            checkForUpdates();
        } catch (runtime_error &e) {
            cerr << endl << e.what() << endl;
            return 3;
        }
        if (!opt.doNotResume) {
            disallowDuplicateFiles(opt.files, opt.projects);
        }
    } catch (exception &e) {
        cerr << endl << "ERROR: " << e.what() << endl;
        return 1;
    }

    const bool anyImportAppToBeCalled = (opt.reads || opt.pairedReads || opt.mappings || opt.variants);

    chunksToCompress.setCapacity(opt.compressThreads);
    chunksToUpload.setCapacity(opt.uploadThreads);
    int exitCode = 0;
    try {
        curlInit(); // for curl requests to be made by upload chunk request

        NUMTRIES_g = opt.tries;

        vector<File> files;

        for (unsigned int i = 0; i < opt.files.size(); ++i) {
            DXLOG(logINFO) << "Getting MIME type for local file " << opt.files[i] << "...";
            string mimeType = getMimeType(opt.files[i]);
            DXLOG(logINFO) << "MIME type for local file " << opt.files[i] << " is '" << mimeType << "'.";
            bool toCompress;
            if (!opt.doNotCompress) {
                bool is_compressed = isCompressed(mimeType);
                toCompress = !is_compressed;
                if (is_compressed)
                    DXLOG(logINFO) << "File " << opt.files[i] << " is already compressed, so won't try to compress it any further.";
                else
                    DXLOG(logINFO) << "File " << opt.files[i] << " is not compressed, will compress it before uploading.";
            } else {
                toCompress = false;
            }
            if (toCompress) {
                mimeType = "application/x-gzip";
            }
            files.push_back(File(opt.files[i], opt.projects[i], opt.folders[i], opt.names[i], toCompress, !opt.doNotResume, mimeType, opt.chunkSize, i));
            totalChunks += files[i].createChunks(chunksToRead, opt.tries);
            cerr << endl;
        }

        if (opt.waitOnClose) {
            for (unsigned int i = 0; i < files.size(); ++i) {
                files[i].waitOnClose = true;
            }
        }

        // Create folders all at once (instead of one by one, above, where we
        // initialize the File objects).
        createFolders(opt.projects, opt.folders);

        // Take this point as the starting time for program operation
        // (to calculate average transfer speed)
        startTime = std::time(0);

        DXLOG(logINFO) << "Created " << totalChunks << " chunks.";

        createWorkerThreads(files);

        DXLOG(logINFO) << "Creating monitor thread..";
        boost::thread monitorThread(monitor);

        boost::thread uploadProgressThread;
        if (opt.progress) {
            DXLOG(logINFO) << "Creating Upload Progress thread..";
            uploadProgressThread = boost::thread(uploadProgress, boost::ref(files));
        }

        DXLOG(logINFO) << "Joining monitor thread...";
        monitorThread.join();
        DXLOG(logINFO) << "Monitor thread finished.";

        if (opt.progress) {
            DXLOG(logINFO) << "Joining Upload Progress thread..";
            keepShowingUploadProgress = false;
            uploadProgressThread.interrupt();
            uploadProgressThread.join();
            DXLOG(logINFO) << "Upload Progress thread finished.";
        }


        interruptWorkerThreads();
        joinWorkerThreads();

        while (!chunksFailed.empty()) {
            Chunk * c = chunksFailed.consume();
            c->log("Chunk failed", logERROR);
            markFileAsFailed(files, c->fileID);
        }
        if (opt.verbose) {
            cerr << endl;
        }
        for (unsigned int i = 0; i < files.size(); ++i) {
            if (files[i].failed) {
                cerr << "File \""<< files[i].localFile << "\" could not be uploaded." << endl;
            } else {
                cerr << "File \"" << files[i].localFile << "\" was uploaded successfully. Closing..." << endl;
                if (files[i].isRemoteFileOpen) {
                    files[i].close();
                }
            }
            if (files[i].failed)
                files[i].fileID = "failed";
        }

        DXLOG(logINFO) << "Waiting for files to be closed...";
        boost::thread waitOnCloseThread(waitOnClose, boost::ref(files));
        DXLOG(logINFO) << "Joining wait-on-close thread...";
        waitOnCloseThread.join();
        DXLOG(logINFO) << "Wait-on-close thread finished.";
        if (anyImportAppToBeCalled) {
            runImportApps(opt, files);
        }
        for (unsigned i = 0; i < files.size(); ++i) {
            cout << files[i].fileID;
            if (files[i].fileID == "failed")
                exitCode = 1;
            if (anyImportAppToBeCalled) {
                if (files[i].jobID == "failed")
                    exitCode = 1;
                cout << "\t" << files[i].jobID;
            }
            cout << endl;
        }
        curlCleanup();

        DXLOG(logINFO) << "Exiting.";
    } catch (bad_alloc &e) {
        boost::call_once(bad_alloc_once, boost::bind(&handle_bad_alloc, e));
    } catch (exception &e) {
        curlCleanup();
        cerr << endl << "ERROR: " << e.what() << endl;
        return 1;
    }

    return exitCode;
}
Exemple #2
0
int main(int argc, char * argv[]) {
  try {
    opt.parse(argc, argv);
  } catch (exception &e) {
    cerr << "Error processing arguments: " << e.what() << endl;
    opt.printHelp(argv[0]);
    return 1;
  }
  // Note: Verbose mode logging is now enabled by options parse()
  if (opt.env()) {
    printEnvironmentInfo();
    return 0;
  }
  if (opt.version()) {
    cout << "dx-verify-file Version: " << DX_VERIFY_FILE_VERSION << endl
         << "git version: " << DXTOOLKIT_GITVERSION << endl;
    return 0;
  } else if (opt.help()) {
    opt.printHelp(argv[0]);
    return 0;
  }

  LOG << "dx-verify-file" << DX_VERIFY_FILE_VERSION << " (git version: " << DXTOOLKIT_GITVERSION << ")" << endl;
  LOG << opt;
  try {
    opt.setApiserverDxConfig();
    opt.validate();
  } catch (exception &e) {
    cerr << "ERROR: " << e.what() << endl;
    opt.printHelp(argv[0]);
    return 1;
  }
 
  chunksToComputeMD5.setCapacity(opt.md5Threads);
  int exitCode = 0; 
  try {
    vector<File> files;

    for (unsigned int i = 0; i < opt.localFiles.size(); ++i) {
      files.push_back(File(opt.localFiles[i], opt.remoteFiles[i], files.size()));
      totalChunks += files[i].createChunks(chunksToRead);
    }

    LOG << "Created " << totalChunks << " chunks." << endl;
    
    createWorkerThreads(files);

    LOG << "Creating monitor thread.." << endl;
    boost::thread monitorThread(monitor);
    
    LOG << "Joining monitor thread..." << endl;
    monitorThread.join();
    LOG << "Monitor thread finished." << endl;

    interruptWorkerThreads();
    joinWorkerThreads();

    for (unsigned int i = 0; i < files.size(); ++i) {
      if (files[i].matchStatus != File::Status::FAILED_TO_MATCH_REMOTE_FILE) {
        cout << "identical" << endl;
      } else {
        exitCode = 4;
        cout << "mismatch" << endl;
      }
    }
    LOG << "Exiting." << endl;
  } catch (exception &e) {
    cerr << "ERROR: " << e.what() << endl;
    return 1;
  }

  return exitCode;
}