std::string ZLCurlNetworkManager::perform(const ZLExecutionData::Vector &dataList) const {
const ZLResource &errorResource = ZLResource::resource("dialog")["networkError"];
if (dataList.empty()) {
return errorResource["emptyLibrariesList"].value();
}
std::set<std::string> errors;
const std::string proxy = proxyHost() + ':' + proxyPort();
CURLM *handle = curl_multi_init();
std::map<CURL*,shared_ptr<ZLExecutionData> > handleToRequest;
for (ZLExecutionData::Vector::const_iterator it = dataList.begin(); it != dataList.end(); ++it) {
if (it->isNull() || !(*it)->isInstanceOf(ZLNetworkRequest::TYPE_ID)) {
continue;
}
ZLNetworkRequest &request = (ZLNetworkRequest&)**it;
const std::string err = doBeforeRequest(request);
if (!err.empty()) {
errors.insert(err);
continue;
}
CURL *easyHandle = curl_easy_init();
if (easyHandle != 0) {
handleToRequest[easyHandle] = *it;
setStandardOptions(easyHandle, proxy);
setRequestOptions(easyHandle, request);
curl_multi_add_handle(handle, easyHandle);
}
}
int counter;
CURLMcode res;
do {
res = curl_multi_perform(handle, &counter);
} while ((res == CURLM_CALL_MULTI_PERFORM) || (counter > 0));
CURLMsg *message;
do {
int queueSize;
message = curl_multi_info_read(handle, &queueSize);
if ((message != 0) && (message->msg == CURLMSG_DONE)) {
ZLNetworkRequest &request = (ZLNetworkRequest&)*handleToRequest[message->easy_handle];
const std::string &url = request.url();
CURLcode result = message->data.result;
bool doAfterResult = request.doAfter(result == CURLE_OK);
if (result == CURLE_OK && !doAfterResult) {
result = CURLE_WRITE_ERROR;
}
switch (result) {
case CURLE_OK:
break;
case CURLE_WRITE_ERROR:
if (!request.errorMessage().empty()) {
errors.insert(request.errorMessage());
} else {
errors.insert(ZLStringUtil::printf(errorResource["somethingWrongMessage"].value(), ZLNetworkUtil::hostFromUrl(url)));
}
break;
default:
errors.insert(ZLStringUtil::printf(errorResource["somethingWrongMessage"].value(), ZLNetworkUtil::hostFromUrl(url)));
break;
case CURLE_COULDNT_RESOLVE_PROXY:
errors.insert(ZLStringUtil::printf(errorResource["couldntResolveProxyMessage"].value(), proxyHost()));
break;
case CURLE_COULDNT_RESOLVE_HOST:
errors.insert(ZLStringUtil::printf(errorResource["couldntResolveHostMessage"].value(), ZLNetworkUtil::hostFromUrl(url)));
break;
case CURLE_COULDNT_CONNECT:
errors.insert(ZLStringUtil::printf(errorResource["couldntConnectMessage"].value(), ZLNetworkUtil::hostFromUrl(url)));
break;
case CURLE_OPERATION_TIMEDOUT:
errors.insert(errorResource["operationTimedOutMessage"].value());
break;
case CURLE_SSL_CONNECT_ERROR:
errors.insert(ZLStringUtil::printf(errorResource["sslConnectErrorMessage"].value(), curl_easy_strerror(CURLE_SSL_CONNECT_ERROR)));
break;
#if LIBCURL_VERSION_NUM > 0x071100
case CURLE_PEER_FAILED_VERIFICATION:
#else
case CURLE_SSL_PEER_CERTIFICATE:
#endif
errors.insert(ZLStringUtil::printf(errorResource["peerFailedVerificationMessage"].value(), ZLNetworkUtil::hostFromUrl(url)));
break;
case CURLE_SSL_CACERT:
errors.insert(ZLStringUtil::printf(errorResource["sslCertificateAuthorityMessage"].value(), ZLNetworkUtil::hostFromUrl(url)));
break;
case CURLE_SSL_CACERT_BADFILE:
errors.insert(ZLStringUtil::printf(errorResource["sslBadCertificateFileMessage"].value(), request.sslCertificate().Path));
break;
case CURLE_SSL_SHUTDOWN_FAILED:
errors.insert(ZLStringUtil::printf(errorResource["sslShutdownFailedMessage"].value(), ZLNetworkUtil::hostFromUrl(url)));
break;
}
}
} while ((message != 0) && (errors.size() < 3));
for (std::map<CURL*,shared_ptr<ZLExecutionData> >::const_iterator jt = handleToRequest.begin(); jt != handleToRequest.end(); ++jt) {
CURL *easyHandle = jt->first;
curl_multi_remove_handle(handle, easyHandle);
curl_easy_cleanup(easyHandle);
ZLNetworkRequest &request = (ZLNetworkRequest&)*jt->second;
clearRequestOptions(request);
}
handleToRequest.clear();
curl_multi_cleanup(handle);
std::string result;
for (std::set<std::string>::const_iterator et = errors.begin(); et != errors.end(); ++et) {
if (!result.empty()) {
result += '\n';
}
result += *et;
}
return result;
}
int main(int argc, char** argv)
{
const std::string program_name = "hts_AdapterTrimmer";
std::string app_description =
"Adapter Trimmer, trims off adapters by first overlapping paired-end reads and\n";
app_description += " trimming off overhangs which by definition are adapter sequence in standard\n";
app_description += " libraries. SE Reads are trimmed by overlapping the adapter-sequence and trimming off the overlap.";
try
{
/** Define and parse the program options
*/
namespace po = boost::program_options;
po::options_description standard = setStandardOptions();
// version|v ; help|h ; notes|N ; stats-file|L ; append-stats-file|A
po::options_description input = setInputOptions();
// read1-input|1 ; read2-input|2 ; singleend-input|U
// tab-input|T ; interleaved-input|I ; from-stdin|S
po::options_description output = setOutputOptions(program_name);
// force|F ; prefix|p ; gzip-output,g ; fastq-output|f
// tab-output|t ; interleaved-output|i ; unmapped-output|u ; to-stdout,O
po::options_description desc("Application Specific Options");
setDefaultParamsCutting(desc);
// no-orphans|n ; stranded|s ; min-length|m
setDefaultParamsOverlapping(desc);
// kmer|k ; kmer-offset|r ; max-mismatch-errorDensity|x
// check-lengths|c ; min-overlap|o
desc.add_options()
("no-fixbases,X", po::bool_switch()->default_value(false), "after trimming adapter, DO NOT use consensus sequence of paired reads, only trims adapter sequence");
desc.add_options()
("adapter-sequence,a", po::value<std::string>()->default_value("AGATCGGAAGAGCACACGTCTGAACTCCAGTCA"), "Primer sequence to trim in SE adapter trimming, default is truseq ht primer sequence");
po::options_description cmdline_options;
cmdline_options.add(standard).add(input).add(output).add(desc);
po::variables_map vm;
try
{
po::store(po::parse_command_line(argc, argv, cmdline_options), vm); // can throw
version_or_help(program_name, app_description, cmdline_options, vm);
po::notify(vm); // throws on error, so do after help in case
std::string statsFile(vm["stats-file"].as<std::string>());
std::string prefix(vm["prefix"].as<std::string>());
AdapterCounters counters(statsFile, vm["append-stats-file"].as<bool>(), program_name, vm["notes"].as<std::string>());
std::shared_ptr<OutputWriter> pe = nullptr;
std::shared_ptr<OutputWriter> se = nullptr;
outputWriters(pe, se, vm["fastq-output"].as<bool>(), vm["tab-output"].as<bool>(), vm["interleaved-output"].as<bool>(), vm["unmapped-output"].as<bool>(), vm["force"].as<bool>(), vm["gzip-output"].as<bool>(), vm["to-stdout"].as<bool>(), prefix );
if(vm.count("read1-input")) { // paired-end reads
if (vm["read1-input"].as<std::vector<std::string> >().size() != vm["read1-input"].as<std::vector<std::string> >().size()) {
throw std::runtime_error("must have same number of input files for read1 and read2");
}
InputReader<PairedEndRead, PairedEndReadFastqImpl> ifr(vm["read1-input"].as<std::vector<std::string> >(), vm["read2-input"].as<std::vector<std::string> >());
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
if (vm.count("interleaved-input")) { // interleaved pairs
InputReader<PairedEndRead, InterReadImpl> ifr(vm["interleaved-input"].as<std::vector<std::string > >());
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
if(vm.count("singleend-input")) { // single-end reads
InputReader<SingleEndRead, SingleEndReadFastqImpl> ifr(vm["singleend-input"].as<std::vector<std::string> >());
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
if(vm.count("tab-input")) { // tab_input
InputReader<ReadBase, TabReadImpl> ifr(vm["tab-input"].as<std::vector<std::string> > ());
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
if(vm["from-stdin"].as<bool>()) { // stdin
bi::stream<bi::file_descriptor_source> tabin {fileno(stdin), bi::close_handle};
InputReader<ReadBase, TabReadImpl> ifr(tabin);
helper_adapterTrimmer(ifr, pe, se, counters, vm["max-mismatch-errorDensity"].as<double>(), vm["max-mismatch"].as<size_t>(), vm["min-overlap"].as<size_t>(), vm["stranded"].as<bool>(), vm["min-length"].as<size_t>(), vm["check-lengths"].as<size_t>(), vm["kmer"].as<size_t>(), vm["kmer-offset"].as<size_t>(), vm["no-orphans"].as<bool>(), vm["no-fixbases"].as<bool>(), vm["adapter-sequence"].as<std::string>() );
}
// no input specified on the command line
if (!vm.count("read1-input") && !vm.count("interleaved-input") && !vm.count("singleend-input") && !vm.count("tab-input") && !vm["from-stdin"].as<bool>()) {
std::cerr << "ERROR: " << "Input file type absent from command line" << std::endl << std::endl;
version_or_help(program_name, app_description, cmdline_options, vm, true);
exit(ERROR_IN_COMMAND_LINE); //success
}
counters.write_out();
}
catch(po::error& e)
{
std::cerr << "ERROR: " << e.what() << std::endl << std::endl;
return ERROR_IN_COMMAND_LINE;
}
}
catch(std::exception& e)
{
std::cerr << "\n\tUnhandled Exception: "
<< e.what() << std::endl;
return ERROR_UNHANDLED_EXCEPTION;
}
return SUCCESS;
}
