IOAdapterId IOAdapterUtils::url2io(const GUrl& url) {
if( url.isVFSFile() ) {
return BaseIOAdapters::VFS_FILE;
}
if( url.isHyperLink() ) {
if( url.lastFileSuffix() == "gz") {
return BaseIOAdapters::GZIPPED_HTTP_FILE;
}
return BaseIOAdapters::HTTP_FILE;
}
if( url.lastFileSuffix() == "gz") {
return BaseIOAdapters::GZIPPED_LOCAL_FILE;
}
return BaseIOAdapters::LOCAL_FILE;
}
/***********************************************************************//**
* @brief Write XML text into URL
*
* @param[in] url Unified Resource Locator.
* @param[in] indent Text indentation (parameter ignored).
*
* Writes the text into the URL.
***************************************************************************/
void GXmlText::write(GUrl& url, const int& indent) const
{
// Write text
url.printf("%s", m_text.c_str());
// Return
return;
}
/***********************************************************************//**
* @brief Write XML text into URL
*
* @param[in] url Unified Resource Locator.
* @param[in] indent Text indentation (parameter ignored).
*
* Writes the text into the URL. Special characters are automatically
* transformed into predefined entities (e.g. ").
***************************************************************************/
void GXmlText::write(GUrl& url, const int& indent) const
{
// Write text
url.printf("%s", gammalib::str2xml(m_text).c_str());
// Return
return;
}
BAMImporterTask::BAMImporterTask(const GUrl& url, bool _useGui, const QVariantMap &hints) :
DocumentProviderTask(tr("BAM/SAM file import: %1").arg(url.fileName()), TaskFlags_NR_FOSCOE),
loadInfoTask(NULL),
loadBamInfoTask(NULL),
prepareToImportTask(NULL),
convertTask(NULL),
loadDocTask(NULL),
isSqliteDbTransit(false),
useGui(_useGui),
sam(hints.value(SAM_HINT, false).toBool()),
hints(hints),
hintedDbiRef(hints.value(DocumentFormat::DBI_REF_HINT).value<U2DbiRef>())
{
documentDescription = url.fileName();
loadInfoTask = new LoadInfoTask( url, sam );
addSubTask(loadInfoTask);
}
QString MultipleDocumentsReadingModeDialog::setupNewUrl() {
QString urlStr = newDocUrl->text();
if (urlStr.isEmpty()) {
GUrl url = urls.at(0);
urlStr = url.getURLString();
}
QFileInfo fi(urlStr);
urlStr = fi.dir().path();
QString extension4MergedDocument;
if (mergeMode->isChecked()) {
extension4MergedDocument = AppContext::getDocumentFormatRegistry()->getFormatById(BaseDocumentFormats::PLAIN_GENBANK)->getSupportedDocumentFileExtensions().first();
} else if (join2alignmentMode->isChecked()) {
extension4MergedDocument = AppContext::getDocumentFormatRegistry()->getFormatById(BaseDocumentFormats::CLUSTAL_ALN)->getSupportedDocumentFileExtensions().first();
}
return urlStr + "/merged_document." + extension4MergedDocument;
}
void BAMImporterTask::initPrepareToImportTask() {
GUrl srcUrl = loadInfoTask->getSourceUrl();
isSqliteDbTransit = hintedDbiRef.isValid() && SQLITE_DBI_ID != hintedDbiRef.dbiFactoryId;
if (!isSqliteDbTransit) {
localDbiRef = U2DbiRef(SQLITE_DBI_ID, srcUrl.dirPath() + QDir::separator() + srcUrl.fileName() + ".ugenedb");
} else {
const QString tmpDir = AppContext::getAppSettings()->getUserAppsSettings()->getCurrentProcessTemporaryDirPath("assembly_conversion") + QDir::separator();
QDir().mkpath(tmpDir);
const QString pattern = tmpDir + "XXXXXX.ugenedb";
QTemporaryFile *tempLocalDb = new QTemporaryFile(pattern, this);
tempLocalDb->open();
const QString filePath = tempLocalDb->fileName();
tempLocalDb->close();
SAFE_POINT_EXT(QFile::exists(filePath), setError(tr("Can't create a temporary database")), );
localDbiRef = U2DbiRef(SQLITE_DBI_ID, filePath);
}
QString refUrl;
bool convert = true;
if (useGui) {
QObjectScopedPointer<ConvertToSQLiteDialog> convertDialog = new ConvertToSQLiteDialog(loadInfoTask->getSourceUrl(), loadInfoTask->getInfo(), loadInfoTask->isSam());
convertDialog->hideAddToProjectOption();
const int rc = convertDialog->exec();
CHECK_EXT(!convertDialog.isNull(), setError("NULL dialog"), );
if (rc == QDialog::Accepted) {
localDbiRef = U2DbiRef(SQLITE_DBI_ID, convertDialog->getDestinationUrl().getURLString());
refUrl = convertDialog->getReferenceUrl();
} else {
convert = false;
stateInfo.setCanceled(true);
}
} else if (loadInfoTask->isSam() && loadInfoTask->getInfo().getHeader().getReferences().isEmpty()) {
void AceImportWidget::initSaveController(const GUrl& url) {
SaveDocumentControllerConfig config;
config.defaultFileName = url.getURLString() + EXTENSION;
config.defaultFormatId = BaseDocumentFormats::UGENEDB;
config.fileDialogButton = browseButton;
config.fileNameEdit = fileNameEdit;
config.parentWidget = this;
config.saveTitle = tr("Destination UGENEDB file");
const QList<DocumentFormatId> formats = QList<DocumentFormatId>() << BaseDocumentFormats::UGENEDB;
saveController = new SaveDocumentController(config, formats, this);
}
bool LocalFileAdapter::open(const GUrl& url, IOAdapterMode m) {
SAFE_POINT(!isOpen(), "Adapter is already opened!", false);
SAFE_POINT(f == NULL, "QFile is not null!", false);
if (url.isEmpty()) {
return false;
}
f = new QFile(url.getURLString());
QIODevice::OpenMode iomode;
switch (m) {
case IOAdapterMode_Read: iomode = QIODevice::ReadOnly; break;
case IOAdapterMode_Write: iomode = QIODevice::WriteOnly | QIODevice::Truncate; break;
case IOAdapterMode_Append: iomode = QIODevice::WriteOnly | QIODevice::Append; break;
}
bool res = f->open(iomode);
if (!res) {
delete f;
f = NULL;
return false;
}
fileSize = f->size();
return true;
}
bool VFSAdapter::open(const GUrl& _url, IOAdapterMode m) {
SAFE_POINT(!isOpen(), "Adapter is already opened!", false);
SAFE_POINT(buffer == NULL, "Buffers is not null!", false);
QString vfsPrefix = U2_VFS_URL_PREFIX;
// assume that all membuf adapters work with files in some vfs
if( !_url.getURLString().startsWith(vfsPrefix) ) {
return false; // not a file in vfs
}
VirtualFileSystemRegistry * vfsReg = AppContext::getVirtualFileSystemRegistry();
SAFE_POINT(vfsReg != NULL, "VirtualFileSystemRegistry not found!", false);
QStringList urlArgs = _url.getURLString().mid(vfsPrefix.size()).split(U2_VFS_FILE_SEPARATOR, QString::SkipEmptyParts);
if ( 2 != urlArgs.size() ) { // urlArgs - vfsname and filename
return false;
}
VirtualFileSystem * vfs = vfsReg->getFileSystemById( urlArgs[0] );
if( NULL == vfs ) {
return false; // no such vfs registered
}
if( !vfs->fileExists( urlArgs[1] ) ) {
if( IOAdapterMode_Read == m ) {
return false;
} else {
vfs->createFile( urlArgs[1], QByteArray() );
}
}
buffer = new QBuffer( &vfs->getFileByName( urlArgs[1] ) );
QIODevice::OpenMode iomode = m == IOAdapterMode_Read ? QIODevice::ReadOnly : QIODevice::WriteOnly | QIODevice::Truncate;
if (!buffer->open(iomode)) {
return false;
}
url = _url;
return true;
}
bool BgzipTask::checkBgzf(const GUrl &fileUrl) {
return bgzf_check_bgzf(fileUrl.getURLString().toLatin1().constData());
}
/***********************************************************************//**
* @brief Write element into URL
*
* @param[in] url Unified Resource Locator.
* @param[in] indent Text indentation.
*
* Writes the element into a Unified Resource Locator.
***************************************************************************/
void GXmlElement::write(GUrl& url, const int& indent) const
{
// Prepend indentation
for (int k = 0; k < indent; ++k) {
url.printf(" ");
}
// Write element name into URL
url.printf("<%s", m_name.c_str());
// Write attributes into URL
for (int k = 0; k < m_attr.size(); ++k) {
m_attr[k]->write(url);
}
// If there are no children then write an empty tag
if (is_empty()) {
url.printf(" />\n");
}
// ... otherwise finish start tag, write children and write end tag
else {
// Case A: The element contains a single text leaf
if ((m_nodes.size() == 1) && (m_nodes[0]->type() == NT_TEXT)) {
// Finish start tag
url.printf(">");
// Write text leaf
m_nodes[0]->write(url, 0);
// Write end tag
url.printf("</%s>\n", m_name.c_str());
}
// Case B: ... otherwise it contains markup
else {
// Finish start tag
url.printf(">\n");
// Write children in file
for (int i = 0; i < m_nodes.size(); ++i) {
m_nodes[i]->write(url, indent+g_indent);
if (m_nodes[i]->type() == NT_TEXT) {
url.printf("\n");
}
}
// Write end tag
for (int k = 0; k < indent; ++k) {
url.printf(" ");
}
url.printf("</%s>\n", m_name.c_str());
} // endelse: element contained markup
} // endelse: finished start tag
// Return
return;
}
void ConvertAssemblyToSamDialog::buildSamUrl(const GUrl &dbUrl) {
GUrl url = GUrlUtils::rollFileName(dbUrl.dirPath() + "/" + dbUrl.baseFileName() + ".sam", DocumentUtils::getNewDocFileNameExcludesHint());
ui->samPathEdit->setText(url.getURLString());
}
/***********************************************************************//**
* @brief Parse XML URL
*
* @param[in] url Unified Resource Locator.
*
* @exception GException::xml_syntax_error
* XML syntax error.
*
* Parses either a XML file or a XML text string and creates all associated
* nodes. The XML file is split into segments, made either of text or of
* tags.
***************************************************************************/
void GXml::parse(const GUrl& url)
{
// Initialise parser
int c;
bool in_markup = false;
bool in_comment = false;
std::string segment;
GXmlNode* current = &m_root;
// Main parsing loop
while ((c = url.get_char()) != EOF) {
// Convert special characters into line feeds
if (c == '\x85' || c == L'\x2028') {
if (in_markup) {
throw GException::xml_syntax_error(G_PARSE, segment,
"invalid character encountered");
}
else {
c = '\x0a';
}
}
// Skip all linefeeds (to avoid extra linefeeds in text segments)
if (c == '\x0a') {
continue;
}
// If we are not within a markup and if a markup is reached then
// add the text segment to the nodes and switch to in_markup mode
if (in_markup == false) {
// Markup start reached?
if (c == '<') {
// Add text segment to nodes (ignores empty segments)
process_text(¤t, segment);
// Prepare new segment and signal that we are within tag
segment.clear();
segment.append(1, (char)c);
in_markup = true;
}
// Markup stop encountered?
else if (c == '>') {
segment.append(1, (char)c);
throw GException::xml_syntax_error(G_PARSE, segment,
"unexpected closing bracket \">\" encountered");
}
// ... otherwise add character to segment
else {
segment.append(1, (char)c);
}
}
// If we are within a markup and if a markup end is reached then
// process the markup and switch to not in_tag mode
else {
// Markup stop reached?
if (c == '>') {
// Append character to segment
segment.append(1, (char)c);
// If we are in comment then check if this is the end of
// the comment
if (in_comment) {
int n = segment.length();
if (n > 2) {
if (segment.compare(n-3,3,"-->") == 0) {
in_comment = false;
}
}
}
// If we are not in the comment, then process markup
if (!in_comment) {
// Process markup
process_markup(¤t, segment);
// Prepare new segment and signal that we are not
// within markup
segment.clear();
in_markup = false;
}
}
// Markup start encountered?
else if (!in_comment && c == '<') {
// Append character to segment
segment.append(1, (char)c);
// If we encounter an opening bracket then throw an exception
throw GException::xml_syntax_error(G_PARSE, segment,
"unexpected opening bracket \"<\" encountered");
}
// ... otherwise add character to segment
else {
segment.append(1, (char)c);
if (!in_comment && segment == "<!--") {
in_comment = true;
}
}
}
} // endwhile: main parsing loop
// Process any pending segment
if (segment.size() > 0) {
if (in_markup) {
process_markup(¤t, segment);
}
else {
process_text(¤t, segment);
}
}
// Verify that we are back to the root node
if (current != &m_root) {
std::string message = "closing tag ";
GXmlElement* element = dynamic_cast<GXmlElement*>(current);
if (element != NULL) {
message += "for GXmlElement \""+element->name()+"\"";
}
message += " is missing";
throw GException::xml_syntax_error(G_PARSE, "", message);
}
// Return
return;
}
/**
* FASTQ format specification: http://maq.sourceforge.net/fastq.shtml
*/
static void load(IOAdapter* io, const U2DbiRef& dbiRef, const QVariantMap& hints, const GUrl& docUrl, QList<GObject*>& objects, U2OpStatus& os,
int gapSize, int predictedSize, QString& writeLockReason) {
DbiOperationsBlock opBlock(dbiRef, os);
CHECK_OP(os, );
Q_UNUSED(opBlock);
writeLockReason.clear();
bool merge = gapSize!=-1;
QByteArray sequence;
QByteArray qualityScores;
QStringList headers;
QSet<QString> uniqueNames;
QVector<U2Region> mergedMapping;
QByteArray gapSequence((merge ? gapSize : 0), 0);
sequence.reserve(predictedSize);
qualityScores.reserve(predictedSize);
// for lower case annotations
GObjectReference sequenceRef;
qint64 sequenceStart = 0;
U2SequenceImporter seqImporter(hints, true);
const QString folder = hints.value(DocumentFormat::DBI_FOLDER_HINT, U2ObjectDbi::ROOT_FOLDER).toString();
int seqNumber = 0;
int progressUpNum = 0;
const int objectsCountLimit = hints.contains(DocumentReadingMode_MaxObjectsInDoc) ? hints[DocumentReadingMode_MaxObjectsInDoc].toInt() : -1;
const bool settingsMakeUniqueName = !hints.value(DocumentReadingMode_DontMakeUniqueNames, false).toBool();
while (!os.isCoR()) {
//read header
QString sequenceName = readSequenceName(os, io, '@');
// check for eof while trying to read another FASTQ block
if (io->isEof()) {
break;
}
CHECK_OP_BREAK(os);
if(sequenceName.isEmpty()){
sequenceName = "Sequence";
}
if ((merge == false) || (seqNumber == 0)) {
QString objName = sequenceName;
if (settingsMakeUniqueName) {
objName = (merge) ? "Sequence" : TextUtils::variate(sequenceName, "_", uniqueNames);
objName.squeeze();
uniqueNames.insert(objName);
}
seqImporter.startSequence(dbiRef, folder, objName, false, os);
CHECK_OP_BREAK(os);
}
//read sequence
if (merge && sequence.length() > 0) {
seqImporter.addDefaultSymbolsBlock(gapSize,os);
sequenceStart += sequence.length();
sequenceStart+=gapSize;
CHECK_OP_BREAK(os);
}
sequence.clear();
readSequence(os, io, sequence);
MemoryLocker lSequence(os, qCeil(sequence.size()/(1000*1000)));
CHECK_OP_BREAK(os);
Q_UNUSED(lSequence);
seqImporter.addBlock(sequence.data(),sequence.length(),os);
CHECK_OP_BREAK(os);
QString qualSequenceName = readSequenceName(os, io, '+');
if (!qualSequenceName.isEmpty()) {
static const QString err = U2::FastqFormat::tr("Not a valid FASTQ file: %1, sequence name differs from quality scores name: %2 and %3");
CHECK_EXT_BREAK(sequenceName == qualSequenceName,
os.setError(err.arg(docUrl.getURLString()).arg(sequenceName).arg(qualSequenceName)));
}
// read qualities
qualityScores.clear();
readQuality(os, io, qualityScores, sequence.size());
CHECK_OP_BREAK(os);
static const QString err = U2::FastqFormat::tr("Not a valid FASTQ file: %1. Bad quality scores: inconsistent size.").arg(docUrl.getURLString());
CHECK_EXT_BREAK(sequence.length() == qualityScores.length(), os.setError(err));
seqNumber++;
progressUpNum++;
if (merge) {
headers.append(sequenceName);
mergedMapping.append(U2Region(sequenceStart, sequence.length() ));
}
else {
if (objectsCountLimit > 0 && objects.size() >= objectsCountLimit) {
os.setError(FastqFormat::tr("File \"%1\" contains too many sequences to be displayed. "
"However, you can process these data using instruments from the menu <i>Tools -> NGS data analysis</i> "
"or pipelines built with Workflow Designer.")
.arg(io->getURL().getURLString()));
break;
}
U2Sequence u2seq = seqImporter.finalizeSequenceAndValidate(os);
CHECK_OP_BREAK(os);
sequenceRef = GObjectReference(io->getURL().getURLString(), u2seq.visualName, GObjectTypes::SEQUENCE, U2EntityRef(dbiRef, u2seq.id));
U2SequenceObject* seqObj = new U2SequenceObject(u2seq.visualName, U2EntityRef(dbiRef, u2seq.id));
CHECK_EXT_BREAK(seqObj != NULL, os.setError("U2SequenceObject is NULL"));
seqObj->setQuality(DNAQuality(qualityScores));
objects << seqObj;
U1AnnotationUtils::addAnnotations(objects, seqImporter.getCaseAnnotations(), sequenceRef, NULL, hints);
}
if (PROGRESS_UPDATE_STEP == progressUpNum) {
progressUpNum = 0;
os.setProgress(io->getProgress());
}
}
CHECK_OP_EXT(os, qDeleteAll(objects); objects.clear(), );
bool emptyObjects = objects.isEmpty();
CHECK_EXT(!emptyObjects || merge, os.setError(Document::tr("Document is empty.")), );
SAFE_POINT(headers.size() == mergedMapping.size(), "headers <-> regions mapping failed!", );
if (!merge) {
return;
}
U2Sequence u2seq = seqImporter.finalizeSequenceAndValidate(os);
CHECK_OP(os,);
sequenceRef = GObjectReference(io->getURL().getURLString(), u2seq.visualName, GObjectTypes::SEQUENCE, U2EntityRef(dbiRef, u2seq.id));
U1AnnotationUtils::addAnnotations(objects, seqImporter.getCaseAnnotations(), sequenceRef, NULL, hints);
objects << new U2SequenceObject(u2seq.visualName, U2EntityRef(dbiRef, u2seq.id));
objects << DocumentFormatUtils::addAnnotationsForMergedU2Sequence(sequenceRef, dbiRef, headers, mergedMapping, hints);
if (headers.size() > 1) {
writeLockReason = DocumentFormat::MERGED_SEQ_LOCK;
}
}
void GTUtilsProject::openFiles(HI::GUITestOpStatus &os, const GUrl &path, const OpenFileSettings& s) {
openFiles(os, QList<QUrl>() << path.getURLString(), s);
GTUtilsTaskTreeView::waitTaskFinished(os);
}