void AutoImportWindow::setupWatcherPaths() {
ParametersConfiguration* conf = projectData.projectParameterData();
QString importImagesPath = conf->getValue("import_dir");
QString importAveragedFolder = conf->getValue("import_averages_folder");
QString importAlignedFolder = conf->getValue("import_aligned_folder");
QString importRawFolder = conf->getValue("import_raw_folder");
if(watcher_.directories().size() > 0) watcher_.removePaths(watcher_.directories());
QString currDir;
currDir = importImagesPath + '/' + importAveragedFolder;
if(QFileInfo(currDir).isDir()) watcher_.addPath(currDir);
currDir = importImagesPath + '/' + importAlignedFolder;
if(QFileInfo(currDir).isDir()) watcher_.addPath(currDir);
currDir = importImagesPath + '/' + importRawFolder;
if(QFileInfo(currDir).isDir()) watcher_.addPath(currDir);
}
QWidget* AutoImportWindow::setupInputFolderContainer() {
GroupContainer* container = new GroupContainer();
container->setTitle("Import Folder");
QFormLayout* layout = new QFormLayout;
layout->setHorizontalSpacing(10);
layout->setVerticalSpacing(0);
layout->setRowWrapPolicy(QFormLayout::DontWrapRows);
layout->setFieldGrowthPolicy(QFormLayout::ExpandingFieldsGrow);
layout->setFormAlignment(Qt::AlignHCenter | Qt::AlignTop);
layout->setLabelAlignment(Qt::AlignLeft);
BrowserWidget* filesDirBrowser_ = new BrowserWidget(BrowserWidget::BrowseType::DIRECTORY);
ParametersConfiguration* conf = projectData.projectParameterData();
QString importImagesPath = conf->getValue("import_dir");
filesDirBrowser_->setPath(importImagesPath);
setupWatcherPaths();
connect(filesDirBrowser_, &BrowserWidget::pathChanged, [ = ] (const QString & value){
conf->set("import_dir", value);
setupWatcherPaths();
analyzeImport();
});
layout->addRow(filesDirBrowser_);
QLabel* introLabel = new QLabel(introText());
introLabel->setWordWrap(true);
QPalette pal = introLabel->palette();
pal.setColor(QPalette::WindowText, Qt::darkGray);
introLabel->setPalette(pal);
layout->addRow(introLabel);
QCheckBox* restartCheck = new QCheckBox("Import new images in the import folder on start");
restartCheck->setChecked(ProjectPreferences().importRestartCheck());
connect(restartCheck, &QCheckBox::toggled, [ = ] (bool check){
ProjectPreferences().setImportRestartCheck(check);
});
layout->addRow(restartCheck);
layout->addRow(continuous);
deleteCheck = new QCheckBox("DELETE the original images in import folder after importing them");
deleteCheck->setChecked(ProjectPreferences().importDeleteCheck());
deleteLabel_->setVisible(deleteCheck->isChecked());
connect(deleteCheck, &QCheckBox::toggled, [ = ] (bool check){
ProjectPreferences().setImportDeleteCheck(check);
deleteLabel_->setVisible(check);
});
layout->addRow(deleteCheck);
container->setContainerLayout(layout);
return container;
}
void StatusViewer::load() {
//Load the data
if(image_) {
ParametersConfiguration* conf = image_->parameters();
magLabel_->setText(conf->getValue("CALCULATEDMAG"));
qvalLabel_->setText(conf->getValue("QVAL"));
phaseResLabel_->setText(conf->getValue("PHARES_SYM"));
numSpotsLabel_->setText(conf->getValue("PHARES_NUM_SPOTS"));
loadTable(binsTable_, conf, QStringList() << "RB" << "RP", QStringList() << "1" << "2" << "3" << "4" << "5" << "6", "_");
loadTable(qvalTable_, conf, QStringList() << "U1" << "U2" << "UMA" << "UMB",
QStringList() << "IQ1" << "IQ2" << "IQ3" << "IQ4" << "IQ5" << "IQ6", "_");
loadTable(tiltTable_, conf, QStringList() << "DEFOCUS" << "LATTICE" << "TTREFINE" << "MERGE",
QStringList() << "TLTAXIS" << "TLTANG" << "TAXA" << "TANGL", "_");
//Manually get the QVALs (as they are not in the format row_col)
qvalTable_->item(0, 6)->setText(conf->getValue("QVAL1"));
qvalTable_->item(1, 6)->setText(conf->getValue("QVAL2"));
qvalTable_->item(2, 6)->setText(conf->getValue("QVALMA"));
qvalTable_->item(3, 6)->setText(conf->getValue("QVALMB"));
}
}
void AutoImportWindow::importImage() {
if(continuous->isChecked()) {
analyzeImport(true);
}
if(toBeImported_.isEmpty()) {
finishExecution();
return;
}
if (imageExecuting_) {
QString numberExec = imageExecuting_->directory();
if (!numberExec.isEmpty()) {
if (dirToRowNumber_.keys().contains(numberExec)) {
if (dirToRowNumber_[numberExec] < resultsTable_->rowCount()) {
if (resultsTable_->item(dirToRowNumber_[numberExec], 0)) {
resultsTable_->item(dirToRowNumber_[numberExec], 0)->setIcon(ApplicationData::icon("process_done"));
}
}
}
}
}
QString number;
number = toBeImported_.keys().first();
QStringList files = toBeImported_[number];
toBeImported_.remove(number);
//Get the original file name used for search
QString baseName = files.first();
//Deduce parameters from baseName
QMap<QString, QString> fileNameParams = FileNameParserDialog::parseFileName(baseName);
QString importGroup_ = projectData.projectParameterData()->getValue("import_target_group");
QString importGroupSuffix = projectData.projectParameterData()->getValue("import_target_group_suffix");
//Add suffix to group name
QString suffix = "";
//case importGroupSuffix=1 => parameter = specimennumber
if(importGroupSuffix == "1") {
//If the file name contains this param take it from there, otherwise search master.cfg
if(fileNameParams.contains("specimennumber")) suffix = fileNameParams["specimennumber"];
else {
suffix = projectData.projectParameterData()->getValue("specimennumber");
}
}
if(suffix == "-") suffix = "";
importGroup_ = importGroup_ + suffix;
if (dirToRowNumber_.keys().contains(number)) {
if (dirToRowNumber_[number] < resultsTable_->rowCount()) {
if(resultsTable_->item(dirToRowNumber_[number], 0)) {
resultsTable_->item(dirToRowNumber_[number], 0)->setIcon(ApplicationData::icon("process_executing"));
resultsTable_->scrollToItem(resultsTable_->item(dirToRowNumber_[number], 0));
}
}
}
statusLabel_->setText(QString("Currently importing and %2 are in queue...").arg(toBeImported_.keys().size()));
projectData.projectParameterData()->set("import_imagenumber", number);
//Create dir
QDir workingDir = QDir(projectData.projectDir().canonicalPath() + "/" + importGroup_ + "/" + number);
//create import group
projectData.projectDir().mkpath(importGroup_);
QFile(projectData.projectDir().absolutePath() + "/merge").link("../2dx_master.cfg", projectData.projectDir().absolutePath() + "/" + importGroup_ + "/2dx_master.cfg");
//create Dir
projectData.projectDir().mkpath(importGroup_ + "/" + number);
workingDir.mkpath("proc");
workingDir.mkpath("LOGS");
//Copy Config File
projectData.projectParameterData()->saveAs(workingDir.canonicalPath() + "/2dx_image.cfg", true);
imageExecuting_ = projectData.addImage(importGroup_, number);
ParametersConfiguration* conf = imageExecuting_->parameters();
conf->set("imagenumber", number, false);
bool hasAveraged = false;
bool hasAligned = false;
bool hasRaw = false;
//Check for the averaged file
if(files.size() > 1 && !files[1].isEmpty()) {
conf->set("imagename", "image_2dx", false);
conf->set("nonmaskimagename", "image_2dx", false);
conf->set("imagename_original", files[1], false);
conf->set("import_original_time", QString::number(QFileInfo(files[1]).created().toMSecsSinceEpoch()), false);
scriptsToBeExecuted_.append("cp -f " + files[1] + " " + workingDir.canonicalPath() + "/" + "image_2dx.mrc");
if(deleteCheck->isChecked()) scriptsToBeExecuted_.append("rm -f " + files[1]);
hasAveraged = true;
}
//Check for aligned file
if (files.size() > 2 && !files[2].isEmpty()) {
conf->set("movie_stackname", "movie_aligned", false);
conf->set("movie_stackname_original", files[2], false);
conf->set("import_original_time", QString::number(QFileInfo(files[2]).created().toMSecsSinceEpoch()), false);
scriptsToBeExecuted_.append("cp -f " + files[2] + " " + workingDir.canonicalPath() + "/" + "movie_aligned.mrcs");
if(deleteCheck->isChecked()) scriptsToBeExecuted_.append("rm -f " + files[2]);
hasAligned = true;
}
//Check for raw file
if (files.size() > 3 && !files[3].isEmpty()) {
int rawOption = conf->getVariant("import_rawstack_type").toInt();
if(rawOption == 1) {
conf->set("import_rawstack", baseName + '.' + QFileInfo(files[3]).suffix(), false);
conf->set("import_rawstack_original", files[3], false);
conf->set("import_original_time", QString::number(QFileInfo(files[3]).created().toMSecsSinceEpoch()), false);
scriptsToBeExecuted_.append("cp -f " + files[3] + " " + workingDir.canonicalPath() + "/" + baseName + '.' + QFileInfo(files[3]).suffix());
if(deleteCheck->isChecked()) scriptsToBeExecuted_.append("rm -f " + files[3]);
hasRaw = true;
} else if (rawOption == 2) {
conf->set("import_rawstack", baseName + '.' + QFileInfo(files[3]).suffix(), false);
conf->set("import_rawstack_original", files[3], false);
conf->set("raw_gaincorrectedstack", "raw_gaincorrectedstack", false);
conf->set("raw_gaincorrectedstack_original", files[3], false);
conf->set("import_original_time", QString::number(QFileInfo(files[3]).created().toMSecsSinceEpoch()), false);
scriptsToBeExecuted_.append("cp -f " + files[3] + " " + workingDir.canonicalPath() + "/" + "raw_gaincorrectedstack.mrcs");
if(deleteCheck->isChecked()) scriptsToBeExecuted_.append("rm -f " + files[3]);
hasRaw = true;
}
}
//Check for defects list file
QString defectsFile = conf->getValue("import_defects_original");
if(QFileInfo(defectsFile).exists()) {
conf->set("import_defects", "../" + QFileInfo(defectsFile).fileName(), false);
scriptsToBeExecuted_.append("rsync -auvP " + defectsFile + " " + workingDir.canonicalPath() + "/../" + QFileInfo(defectsFile).fileName());
}
//Check for gain reference file
QString gainRefFile = conf->getValue("import_gainref_original");
if(QFileInfo(gainRefFile).exists()) {
conf->set("import_gainref", "../" + QFileInfo(gainRefFile).fileName(), false);
scriptsToBeExecuted_.append("rsync -auvP " + gainRefFile + " " + workingDir.canonicalPath() + "/../" + QFileInfo(gainRefFile).fileName());
}
//Set the parameters from filename
for(QString param : fileNameParams.keys()) {
if(parameterMaster.containsParameter(param)) {
conf->set(param, fileNameParams[param], false);
//qDebug() << "Parameter set: " << param << " => " << fileNameParams[param];
}
}
conf->setModified(true);
//Write to status folder if required
if(userPreferenceData.get("status_folder_update") == "y" && QFileInfo(userPreferenceData.get("status_folder")).isDir()) {
long currentMSecs = conf->getValue("import_original_time").toLong();
//Write the last imported data
QFile saveFile(userPreferenceData.get("status_folder") + "/last.txt");
long lastMSecs = 0;
if(saveFile.exists()) {
if (saveFile.open(QIODevice::ReadOnly | QIODevice::Text)) {
while (!saveFile.atEnd()) {
lastMSecs = QString(saveFile.readLine().simplified()).toLong();
}
saveFile.close();
}
saveFile.remove();
}
QString toBeWritten;
if(currentMSecs >= lastMSecs) toBeWritten = QString::number(currentMSecs);
else toBeWritten = QString::number(lastMSecs);
if (saveFile.open(QIODevice::WriteOnly | QIODevice::Text)) {
saveFile.write(toBeWritten.toLatin1());
}
saveFile.close();
//Write the time stamp in the last hour processed data
ProjectData::writeStatisticsToStatusFolder("last_recorded.txt", currentMSecs);
ProjectData::writeStatisticsToStatusFolder("last_imported.txt");
}
//register that this image was imported
ImportFolderSettings(QDir(conf->getValue("import_dir"))).addImportedImage(baseName, importGroup_ + "/" + number, hasAveraged, hasAligned, hasRaw);
//Add the scripts to be executed during import
QStringList scripts = importSelectorDialog.scriptPaths(ProjectPreferences().scripts("import"));
for(QString script : scripts) {
if(!script.isEmpty()) scriptsToBeExecuted_.append("SCRIPT:" + script);
}
continueExecution();
}
void AutoImportWindow::analyzeImport(bool force) {
if(currentlyExecuting_ && !force) {
// qDebug()<< "The import is already running, not analyzing the import folder for now!";
return;
}
QMutexLocker locker(&AutoImportWindow::mutex_);
dirToRowNumber_.clear();
rowToImagePaths_.clear();
toBeImported_.clear();
resultsTable_->setRowCount(0);
ParametersConfiguration* conf = projectData.projectParameterData();
QString importImagesPath = conf->getValue("import_dir");
QString importAveragedFolder = conf->getValue("import_averages_folder");
QString importAlignedFolder = conf->getValue("import_aligned_folder");
QString importRawFolder =conf->getValue("import_raw_folder");
QString importGroup =conf->getValue("import_target_group");
QStringList ignoreImagePattern = conf->getValue("import_ignore_strings").split(' ');
int imageNumberLength = conf->getVariant("import_numberlength").toInt();
int rawOption = conf->getVariant("import_rawstack_type").toInt();
if (importImagesPath.isEmpty() || !QFileInfo(importImagesPath).exists()) {
statusLabel_->setText("The import path does not exist (Please change on left)");
return;
}
int uid = projectData.projectParameterData()->getVariant("import_imagenumber").toInt();
bool addingAFile = false;
QDir importDir(importImagesPath);
ImportFolderSettings folderPreferences(importDir);
QStringList alreadyImportedBaseNames = folderPreferences.importedNames();
QStringList stackExtentions;
stackExtentions << "*.mrc" << "*.mrcs" << "*.tif" << "*.tiff";
QStringList avgExtentions;
avgExtentions << "*.mrc" << "*.tif" << "*.tiff";
//Get a list of all available files
QStringList fileNames = QDir(importImagesPath + "/" + importAveragedFolder).entryList(avgExtentions, QDir::Files | QDir::NoSymLinks);
fileNames.append(QDir(importImagesPath + "/" + importAlignedFolder).entryList(stackExtentions, QDir::Files | QDir::NoSymLinks));
fileNames.append(QDir(importImagesPath + "/" + importRawFolder).entryList(stackExtentions , QDir::Files | QDir::NoSymLinks));
fileNames.removeDuplicates();
QStringList baseNames;
for(QString image: fileNames) {
//get the basename and remove the to_be_ignored strings
QString baseName = QFileInfo(image).completeBaseName();
if (!ignoreImagePattern.isEmpty()) {
for (QString pattern : ignoreImagePattern) {
if (!pattern.trimmed().isEmpty()) baseName.remove(pattern.trimmed(), Qt::CaseInsensitive);
}
}
baseNames.append(baseName);
}
baseNames.removeDuplicates();
for (QString baseName : baseNames) {
bool copying = false;
bool processed = false;
bool hasAveraged = false;
bool hasAligned = false;
bool hasRaw = false;
QString dirName;
if(toBeImported_.keys().contains(baseName)) {
continue;
}
if(alreadyImportedBaseNames.contains(baseName)
&& QFileInfo(projectData.projectDir().canonicalPath() + '/' + folderPreferences.linkedDirectory(baseName) + "/2dx_image.cfg").exists()) {
processed = true;
dirName = folderPreferences.linkedDirectory(baseName);
hasAveraged = folderPreferences.hadAveraged(baseName);
hasAligned = folderPreferences.hadAligned(baseName);
hasRaw = folderPreferences.hadRaw(baseName);
}
else {
QString imageNumber = ProjectData::commitIntToStringLength(++uid, imageNumberLength);
while (QDir(projectData.projectDir().canonicalPath() + "/" + importGroup + "/" + imageNumber).exists()) {
imageNumber = ProjectData::commitIntToStringLength(++uid, imageNumberLength);
}
dirToRowNumber_.insert(imageNumber, resultsTable_->rowCount());
dirName = importGroup + "/" + imageNumber;
toBeImported_.insert(imageNumber, QStringList() << baseName);
//Search string for avg File
QStringList avgSearchStrings, stackSearchStrings;
for(QString ext : avgExtentions) avgSearchStrings.append(baseName + "*" + ext);
for(QString ext : stackExtentions) stackSearchStrings.append(baseName + "*" + ext);
//Check for averaged file
QString averagedFile;
if (QDir(importImagesPath + "/" + importAveragedFolder).exists()) {
QStringList possibleFiles = QDir(importImagesPath + "/" + importAveragedFolder).entryList(avgSearchStrings, QDir::Files | QDir::NoSymLinks);
if (!possibleFiles.isEmpty()) {
hasAveraged = true;
averagedFile = importImagesPath + "/" + importAveragedFolder + "/" + possibleFiles.first();
}
}
toBeImported_[imageNumber].append(averagedFile);
//Check for movie file
QString movieFile;
if (QDir(importImagesPath + "/" + importAlignedFolder).exists()) {
QStringList possibleFiles = QDir(importImagesPath + "/" + importAlignedFolder).entryList(stackSearchStrings, QDir::Files | QDir::NoSymLinks);
if (!possibleFiles.isEmpty()) {
hasAligned = true;
movieFile = importImagesPath + "/" + importAlignedFolder + "/" + possibleFiles.first();
}
}
toBeImported_[imageNumber].append(movieFile);
//Check for raw file
QString rawFile;
if (rawOption != 0) {
if (QDir(importImagesPath + "/" + importRawFolder).exists()) {
QStringList possibleFiles = QDir(importImagesPath + "/" + importRawFolder).entryList(stackSearchStrings, QDir::Files | QDir::NoSymLinks);
if (!possibleFiles.isEmpty()) {
hasRaw = true;
rawFile = importImagesPath + "/" + importRawFolder + "/" + possibleFiles.first();
}
}
}
toBeImported_[imageNumber].append(rawFile);
//Check if the file is still being copied
for(int i=1; i<toBeImported_[imageNumber].size(); ++i) {
if(!isSafeToCopy(toBeImported_[imageNumber][i])) {
copying = true;
addingAFile = true;
dirName = "Not yet ready";
toBeImported_.remove(imageNumber);
break;
}
}
}
rowToImagePaths_.append(dirName);
QTableWidgetItem *statusItem = new QTableWidgetItem();
statusItem->setFlags(statusItem->flags() ^ Qt::ItemIsEditable);
if(copying) statusItem->setIcon(ApplicationData::icon("import_copying"));
else if (!processed) statusItem->setIcon(ApplicationData::icon("process_wait"));
else statusItem->setIcon(ApplicationData::icon("process_done"));
QTableWidgetItem* imageItem = new QTableWidgetItem(baseName);
imageItem->setFlags(imageItem->flags() ^ Qt::ItemIsEditable);
QTableWidgetItem* numberItem = new QTableWidgetItem(dirName);
numberItem->setFlags(numberItem->flags() ^ Qt::ItemIsEditable);
QTableWidgetItem *averagedItem = new QTableWidgetItem();
averagedItem->setFlags(averagedItem->flags() ^ Qt::ItemIsEditable);
if (hasAveraged) averagedItem->setIcon(ApplicationData::icon("tick"));
else averagedItem->setIcon(ApplicationData::icon("cross"));
QTableWidgetItem *movieItem = new QTableWidgetItem();
movieItem->setFlags(movieItem->flags() ^ Qt::ItemIsEditable);
if (hasAligned) movieItem->setIcon(ApplicationData::icon("tick"));
else movieItem->setIcon(ApplicationData::icon("cross"));
QTableWidgetItem *rawItem = new QTableWidgetItem();
rawItem->setFlags(rawItem->flags() ^ Qt::ItemIsEditable);
if (hasRaw) rawItem->setIcon(ApplicationData::icon("tick"));
else rawItem->setIcon(ApplicationData::icon("cross"));
int row = resultsTable_->rowCount();
resultsTable_->insertRow(row);
resultsTable_->setItem(row, 0, statusItem);
resultsTable_->setItem(row, 1, numberItem);
resultsTable_->setItem(row, 2, imageItem);
resultsTable_->setItem(row, 3, averagedItem);
resultsTable_->setItem(row, 4, movieItem);
resultsTable_->setItem(row, 5, rawItem);
}
for (int i = 0; i < resultsTable_->columnCount(); ++i) resultsTable_->resizeColumnToContents(i);
if(resultsTable_->rowCount()!=0) statusLabel_->setText(tr("%1 image(s) found in folder of which %2 image(s) are to be imported").arg(resultsTable_->rowCount()).arg(toBeImported_.keys().size()));
else statusLabel_->setText("No such files could be found. (if not intended, please check the options again.)");
if(toBeImported_.isEmpty()) importButton_->setDisabled(true);
else importButton_->setEnabled(true);
resultsTable_->scrollToBottom();
}