int main(int argc, char * argv[]) {
try {
// Note: Verbose mode logging is enabled (if requested) by options parse()
opt.parse(argc, argv);
} catch (exception &e) {
cerr << "Error processing arguments: " << e.what() << endl;
opt.printHelp(argv[0]);
return 1;
}
if (opt.env()) {
opt.setApiserverDxConfig(); // needed for 'ua --env' to report project name
printEnvironmentInfo();
return 0;
}
if (opt.version()) {
cout << "Upload Agent Version: " << UAVERSION;
#if OLD_KERNEL_SUPPORT
cout << " (old-kernel-support)";
#endif
cout << endl
<< "git version: " << DXTOOLKIT_GITVERSION << endl
<< "libboost version: " << (BOOST_VERSION / 100000) << "." << ((BOOST_VERSION / 100) % 1000) << "." << (BOOST_VERSION % 100) << endl
<< "libcurl version: " << LIBCURL_VERSION_MAJOR << "." << LIBCURL_VERSION_MINOR << "." << LIBCURL_VERSION_PATCH << endl;
return 0;
} else if (opt.help() || opt.files.empty()) {
opt.printHelp(argv[0]);
return (opt.help()) ? 0 : 1;
}
setUserAgentString(); // also sets dx::config::USER_AGENT_STRING()
DXLOG(logINFO) << "DNAnexus Upload Agent " << UAVERSION << " (git version: " << DXTOOLKIT_GITVERSION << ")";
DXLOG(logINFO) << "Upload agent's User Agent string: '" << userAgentString << "'";
DXLOG(logINFO) << "dxcpp's User Agent string: '" << dx::config::USER_AGENT_STRING() << "'";
DXLOG(logINFO) << opt;
try {
opt.setApiserverDxConfig();
opt.validate();
/*
* Check for updates, and terminate execution if necessary. This also
* has the side effect of verifying that we can connect to the API
* server, and that the authentication token is valid.
*/
try {
checkForUpdates();
} catch (runtime_error &e) {
cerr << endl << e.what() << endl;
return 3;
}
if (!opt.doNotResume) {
disallowDuplicateFiles(opt.files, opt.projects);
}
} catch (exception &e) {
cerr << endl << "ERROR: " << e.what() << endl;
return 1;
}
const bool anyImportAppToBeCalled = (opt.reads || opt.pairedReads || opt.mappings || opt.variants);
chunksToCompress.setCapacity(opt.compressThreads);
chunksToUpload.setCapacity(opt.uploadThreads);
int exitCode = 0;
try {
curlInit(); // for curl requests to be made by upload chunk request
NUMTRIES_g = opt.tries;
vector<File> files;
for (unsigned int i = 0; i < opt.files.size(); ++i) {
DXLOG(logINFO) << "Getting MIME type for local file " << opt.files[i] << "...";
string mimeType = getMimeType(opt.files[i]);
DXLOG(logINFO) << "MIME type for local file " << opt.files[i] << " is '" << mimeType << "'.";
bool toCompress;
if (!opt.doNotCompress) {
bool is_compressed = isCompressed(mimeType);
toCompress = !is_compressed;
if (is_compressed)
DXLOG(logINFO) << "File " << opt.files[i] << " is already compressed, so won't try to compress it any further.";
else
DXLOG(logINFO) << "File " << opt.files[i] << " is not compressed, will compress it before uploading.";
} else {
toCompress = false;
}
if (toCompress) {
mimeType = "application/x-gzip";
}
files.push_back(File(opt.files[i], opt.projects[i], opt.folders[i], opt.names[i], toCompress, !opt.doNotResume, mimeType, opt.chunkSize, i));
totalChunks += files[i].createChunks(chunksToRead, opt.tries);
cerr << endl;
}
if (opt.waitOnClose) {
for (unsigned int i = 0; i < files.size(); ++i) {
files[i].waitOnClose = true;
}
}
// Create folders all at once (instead of one by one, above, where we
// initialize the File objects).
createFolders(opt.projects, opt.folders);
// Take this point as the starting time for program operation
// (to calculate average transfer speed)
startTime = std::time(0);
DXLOG(logINFO) << "Created " << totalChunks << " chunks.";
createWorkerThreads(files);
DXLOG(logINFO) << "Creating monitor thread..";
boost::thread monitorThread(monitor);
boost::thread uploadProgressThread;
if (opt.progress) {
DXLOG(logINFO) << "Creating Upload Progress thread..";
uploadProgressThread = boost::thread(uploadProgress, boost::ref(files));
}
DXLOG(logINFO) << "Joining monitor thread...";
monitorThread.join();
DXLOG(logINFO) << "Monitor thread finished.";
if (opt.progress) {
DXLOG(logINFO) << "Joining Upload Progress thread..";
keepShowingUploadProgress = false;
uploadProgressThread.interrupt();
uploadProgressThread.join();
DXLOG(logINFO) << "Upload Progress thread finished.";
}
interruptWorkerThreads();
joinWorkerThreads();
while (!chunksFailed.empty()) {
Chunk * c = chunksFailed.consume();
c->log("Chunk failed", logERROR);
markFileAsFailed(files, c->fileID);
}
if (opt.verbose) {
cerr << endl;
}
for (unsigned int i = 0; i < files.size(); ++i) {
if (files[i].failed) {
cerr << "File \""<< files[i].localFile << "\" could not be uploaded." << endl;
} else {
cerr << "File \"" << files[i].localFile << "\" was uploaded successfully. Closing..." << endl;
if (files[i].isRemoteFileOpen) {
files[i].close();
}
}
if (files[i].failed)
files[i].fileID = "failed";
}
DXLOG(logINFO) << "Waiting for files to be closed...";
boost::thread waitOnCloseThread(waitOnClose, boost::ref(files));
DXLOG(logINFO) << "Joining wait-on-close thread...";
waitOnCloseThread.join();
DXLOG(logINFO) << "Wait-on-close thread finished.";
if (anyImportAppToBeCalled) {
runImportApps(opt, files);
}
for (unsigned i = 0; i < files.size(); ++i) {
cout << files[i].fileID;
if (files[i].fileID == "failed")
exitCode = 1;
if (anyImportAppToBeCalled) {
if (files[i].jobID == "failed")
exitCode = 1;
cout << "\t" << files[i].jobID;
}
cout << endl;
}
curlCleanup();
DXLOG(logINFO) << "Exiting.";
} catch (bad_alloc &e) {
boost::call_once(bad_alloc_once, boost::bind(&handle_bad_alloc, e));
} catch (exception &e) {
curlCleanup();
cerr << endl << "ERROR: " << e.what() << endl;
return 1;
}
return exitCode;
}
int main(int argc, char * argv[]) {
try {
opt.parse(argc, argv);
} catch (exception &e) {
cerr << "Error processing arguments: " << e.what() << endl;
opt.printHelp(argv[0]);
return 1;
}
// Note: Verbose mode logging is now enabled by options parse()
if (opt.env()) {
printEnvironmentInfo();
return 0;
}
if (opt.version()) {
cout << "dx-verify-file Version: " << DX_VERIFY_FILE_VERSION << endl
<< "git version: " << DXTOOLKIT_GITVERSION << endl;
return 0;
} else if (opt.help()) {
opt.printHelp(argv[0]);
return 0;
}
LOG << "dx-verify-file" << DX_VERIFY_FILE_VERSION << " (git version: " << DXTOOLKIT_GITVERSION << ")" << endl;
LOG << opt;
try {
opt.setApiserverDxConfig();
opt.validate();
} catch (exception &e) {
cerr << "ERROR: " << e.what() << endl;
opt.printHelp(argv[0]);
return 1;
}
chunksToComputeMD5.setCapacity(opt.md5Threads);
int exitCode = 0;
try {
vector<File> files;
for (unsigned int i = 0; i < opt.localFiles.size(); ++i) {
files.push_back(File(opt.localFiles[i], opt.remoteFiles[i], files.size()));
totalChunks += files[i].createChunks(chunksToRead);
}
LOG << "Created " << totalChunks << " chunks." << endl;
createWorkerThreads(files);
LOG << "Creating monitor thread.." << endl;
boost::thread monitorThread(monitor);
LOG << "Joining monitor thread..." << endl;
monitorThread.join();
LOG << "Monitor thread finished." << endl;
interruptWorkerThreads();
joinWorkerThreads();
for (unsigned int i = 0; i < files.size(); ++i) {
if (files[i].matchStatus != File::Status::FAILED_TO_MATCH_REMOTE_FILE) {
cout << "identical" << endl;
} else {
exitCode = 4;
cout << "mismatch" << endl;
}
}
LOG << "Exiting." << endl;
} catch (exception &e) {
cerr << "ERROR: " << e.what() << endl;
return 1;
}
return exitCode;
}