/**
* \brief Main loop
*/
void Scanner::loop()
{
prctl(PR_SET_NAME, "fbitexp:Scanner\0", 0, 0, 0);
std::mutex mtx;
std::unique_lock<std::mutex> lock(mtx);
MSG_DEBUG(msg_module, "started");
while (!_done) {
/*
* This is before waiting because we want to check size on startup before
* any scan or add requests
*/
if (totalSize() > _max_size) {
removeDirs();
}
MSG_DEBUG(msg_module, "Total size: %s, Max: %s, Watermark: %s",
sizeToStr(totalSize()).c_str(), sizeToStr(_max_size).c_str(), sizeToStr(_watermark).c_str());
_cv.wait(lock, [&]{ return scanCount() > 0 || addCount() > 0 || _done || totalSize() > _max_size; });
if (_done) {
break;
}
/* Add dirs from queue */
if (addCount() > 0) {
addNewDirs();
}
/* Scan dirs in queue */
if (scanCount() > 0) {
rescanDirs();
}
}
MSG_DEBUG(msg_module, "closing thread");
}
bool DataFile::closeAndFinalize()
{
QMutexLocker ml(&mut);
if (!isOpen()) return false;
if (mode == Input) {
dataFile.close();
metaFile.close();
nChans = scanCt = sRate = 0;
mode = Undefined;
return true;
} else if (mode == Output) {
if (dfwt) delete dfwt, dfwt = 0;
// Output mode...
sha.Final();
params["sha1"] = /*sha.ReportHash().c_str()*/ "0";
params["fileTimeSecs"] = fileTimeSecs();
params["fileSizeBytes"] = dataFile.size();
params["createdBy"] = QString("%1").arg(VERSION_STR);
if (badData.count()) {
QString bdString;
QTextStream ts(&bdString);
int i = 0;
for (BadData::iterator it = badData.begin(); it != badData.end(); ++it, ++i) {
if (i) { ts << "; "; }
ts << (*it).first << "," << (*it).second;
}
ts.flush();
params["badData"] = bdString;
}
Debug() << fileName() << " closing after saving " << scanCount() << " scans @ " << (writeSpeedBytesSec()/1024.0/1024.0) << " MB/s avg";
dataFile.close();
QString mf = metaFile.fileName();
metaFile.close(); // close it.. we mostly reserved it in the FS.. however we did write to it if writeCommentToMetaFile() as called, otherwise we just reserved it on the FS
writeRateAvg_for_ui = writeRateAvg = 0.;
nWritesAvg = nWritesAvgMax = 0;
mode = Undefined;
return params.toFile(mf,true /* append since we may have written comments to metafile!*/);
}
return false; // not normally reached...
}
/**
* \brief Rescan directories
*/
void Scanner::rescanDirs()
{
Directory *dir;
std::string path;
while (scanCount() > 0) {
/* Get directory instance from path */
path = Directory::correctDirName(getNextScan());
if (path.empty()) {
continue;
}
dir = dirFromPath(path);
if (!dir) {
/* Directory not found in our tree */
MSG_WARNING(msg_module, "Cannot rescan %s, it's not part of this tree of it's too deep", path.c_str());
continue;
}
/* rescan directory */
dir->rescan();
}
}
/// Return a list of the names of all scans in this model:
QStringList AMScanSetModel::scanNames() const {
QStringList rv;
for(int i=0; i<scanCount(); i++)
rv << scanAt(i)->fullName();
return rv;
}
void AMExportController::continueScanExport()
{
if(state_ != Exporting)
return; // done, or paused. Don't keep going.
// 0. emit progress and signals
emit progressChanged(exportScanIndex_, scanCount());
// 1. Check for finished:
if(exportScanIndex_ >= scanCount()) {
emit stateChanged(state_ = Finished);
// Reset whether the exporter should overwrite files with matching filenames.
exporter_->setOverwriteOption(AMExporter::Default);
QString message = "Exported " % QString::number(succeededCount()) % " scans.";
if(failedCount())
message.append(" (" % QString::number(failedCount()) % " scans could not be exported.)");
AMErrorMon::report(AMErrorReport(this, AMErrorReport::Information, 0, message));
AMUser::user()->setLastExportDestination(destinationFolderPath());
AMUser::user()->storeToDb(AMUser::user()->database());
deleteLater();
return; // We're done!
}
try {
// 2. Load scan from db and check loaded successfully
AMScan* scan = 0;
AMDbObject* databaseObject = 0;
if(usingScanURLs_){
const QUrl& url = scanURLsToExport_.at(exportScanIndex_);
AMDatabase* db = 0;
QStringList path;
QString tableName;
int id = 0;
bool idOkay = false;
// parse the URL and make sure it's valid
if(!( url.scheme() == "amd" &&
(db = AMDatabase::database(url.host())) &&
(path = url.path().split('/', QString::SkipEmptyParts)).count() == 2 &&
(id = path.at(1).toInt(&idOkay)) > 0 &&
idOkay == true &&
(tableName = path.at(0)).isEmpty() == false
))
throw QString("The export system couldn't understand the scan URL '" % url.toString() % "', so this scan has not been exported.");
emit statusChanged(status_ = "Opening: " % url.toString());
databaseObject = AMDbObjectSupport::s()->createAndLoadObjectAt(db, tableName, id);
scan = qobject_cast<AMScan*>(databaseObject);
if(!scan) {
databaseObject->deleteLater();
throw QString("The export system couldn't load a scan out of the database (" % url.toString() % "), so this scan has not been exported.");
}
}
else if(usingScanObjects_) {
scan = scanObjectsToExport_.at(exportScanIndex_);
if(!scan)
throw QString("An invalid scan reference was provided, so this scan has not been exported.");
}
emit statusChanged(status_ = "Opening: " % scan->name()); // this is kinda pointless...
emit statusChanged(status_ = "Writing: " % scan->fullName());
// 3. Check that it can be exported with the exporter and option selected
if(!exporter_->isValidFor(scan, option_)) {
QString err("The exporter '" % exporter_->description() % "' and the template '" % option_->name() % "' are not compatible with this scan (" % scan->fullName() % "), so it has not been exported.");
emit statusChanged(status_ = err);
if (usingScanURLs_)
scan->deleteLater();
throw err;
}
// 4. Export
// 4.1 check and create the export folder for the current run
QString destinationFolderPathWithRun = destinationFolderPath_;
exporter_->setCurrentScan(scan); // we must set this, otherwise we can't get the name of the current run
QString currentRunExportFilePath = exporter_->currentRunExportFilePath();
if(currentRunExportFilePath.length() > 0) {
QDir exportDir;
exportDir.setCurrent(destinationFolderPath_);
if (!exportDir.entryList(QDir::AllDirs).contains(currentRunExportFilePath)) {
if(!exportDir.mkdir(currentRunExportFilePath)){
QString err("Could not create the export folder." % exportDir.absolutePath());
emit statusChanged(status_ = err);
throw err;
}
}
destinationFolderPathWithRun = destinationFolderPathWithRun % "/" % currentRunExportFilePath;
}
// 4.2 export
QString writtenFile = exporter_->exportScan(scan, destinationFolderPathWithRun, option_, exportScanIndex_);
if(writtenFile.isNull()) {
QString err("Export failed for scan '" % scan->fullName() % " to " % destinationFolderPathWithRun % "'.");
emit statusChanged(status_ = err);
if (usingScanURLs_)
scan->deleteLater();
throw err;
}
emit statusChanged(status_ = "Wrote: " % writtenFile);
succeededCount_++;
if (usingScanURLs_)
scan->deleteLater();
}
catch(QString errMsg) {
failedCount_++;
AMErrorMon::report(AMErrorReport(this, AMErrorReport::Alert, -1, errMsg));
}
// 5. increment exportScanIndex_ and re-schedule next one
exportScanIndex_++;
QTimer::singleShot(5, this, SLOT(continueScanExport()));
}
void AMExportController::continueAvailableDataSourceSearch()
{
if(searchScanIndex_ >= scanCount())
return; // We're done!
if(usingScanURLs_) {
const QUrl& url = scanURLsToExport_.at(searchScanIndex_++); // incrementing searchScanIndex_ here.
AMDatabase* db = 0;
QStringList path;
QString tableName;
int id = 0;
bool idOkay = false;
// parse the URL and make sure it's valid
if(url.scheme() == "amd" &&
(db = AMDatabase::database(url.host())) &&
(path = url.path().split('/', QString::SkipEmptyParts)).count() == 2 &&
(id = path.at(1).toInt(&idOkay)) > 0 &&
idOkay == true &&
(tableName = path.at(0)).isEmpty() == false
) {
// let's roll. Find all the raw data sources for this scan
QSqlQuery q = db->select(tableName % "_rawDataSources", "id2,table2", "id1='" % QString::number(id) % "'"); // note: checked that this is indeed using the index. Can go faster? Dunno.
q.exec();
while(q.next()) {
// get name, description, rank for this data source
QSqlQuery q2 = db->select( q.value(1).toString(),
"name,description,rank",
"id='" % q.value(0).toString() % "'");
q2.exec();
if(q2.next()) {
addFoundAvailableDataSource(q2.value(0).toString(), q2.value(1).toString(), q2.value(2).toInt());
}
}
// Find all the analyzed data sources for this scan
q = db->select(tableName % "_analyzedDataSources", "id2,table2", "id1='" % QString::number(id) % "'"); // note: checked that this is indeed using the index. Can go faster? Dunno.
q.exec();
while(q.next()) {
// get name, description, rank for this data source
QSqlQuery q2 = db->select( q.value(1).toString(),
"name,description,rank",
"id='" % q.value(0).toString() % "'");
q2.exec();
if(q2.next()) {
addFoundAvailableDataSource(q2.value(0).toString(), q2.value(1).toString(), q2.value(2).toInt());
}
}
}
}
else if(usingScanObjects_){
const AMScan *scan = scanObjectsToExport_.at(searchScanIndex_++);
const AMRawDataSourceSet *rawSources = scan->rawDataSources();
const AMRawDataSource *rds;
for(int x = 0; x < rawSources->count(); x++){
rds = rawSources->at(x);
addFoundAvailableDataSource(rds->name(), rds->description(), rds->rank());
}
const AMAnalyzedDataSourceSet *analyzedSources = scan->analyzedDataSources();
const AMAnalysisBlock *ads;
for(int x = 0; x < analyzedSources->count(); x++){
ads = analyzedSources->at(x);
addFoundAvailableDataSource(ads->name(), ads->description(), ads->rank());
}
}
// Schedule us to continue onto next scan. This 10ms timer might need to be adjusted for acceptable performance.
QTimer::singleShot(10, this, SLOT(continueAvailableDataSourceSearch()));
}
