void SearchOperation::doSearch( SearchData& searchData, qint64 initialLine )
{
const qint64 nbSourceLines = sourceLogData_->getNbLine();
int maxLength = 0;
int nbMatches = searchData.getNbMatches();
SearchResultArray currentList = SearchResultArray();
// Ensure no re-alloc will be done
currentList.reserve( nbLinesInChunk );
LOG(logDEBUG) << "Searching from line " << initialLine << " to " << nbSourceLines;
if (initialLine < startLine_) {
initialLine = startLine_;
}
const qint64 endLine = qMin(nbSourceLines, endLine_);
for ( qint64 i = initialLine; i < endLine; i += nbLinesInChunk ) {
if ( *interruptRequested_ )
break;
const int percentage = ( i - initialLine ) * 100 / ( endLine - initialLine );
emit searchProgressed( nbMatches, percentage );
const QStringList lines = sourceLogData_->getLines( i,
qMin( nbLinesInChunk, (int) ( endLine - i ) ) );
LOG(logDEBUG) << "Chunk starting at " << i <<
", " << lines.size() << " lines read.";
int j = 0;
for ( ; j < lines.size(); j++ ) {
if ( regexp_.match( lines[j] ).hasMatch() ) {
// FIXME: increase perf by removing temporary
const int length = sourceLogData_->getExpandedLineString(i+j).length();
if ( length > maxLength )
maxLength = length;
currentList.push_back( MatchingLine( i+j ) );
nbMatches++;
}
}
// After each block, copy the data to shared data
// and update the client
searchData.addAll( maxLength, currentList, i+j );
currentList.clear();
}
emit searchProgressed( nbMatches, 100 );
}
void SearchOperation::doSearch( SearchData& searchData, LineNumber initialLine )
{
const auto& config = Persistable::get<Configuration>();
const auto nbSourceLines = sourceLogData_->getNbLine();
LineLength maxLength = 0_length;
LinesCount nbMatches = searchData.getNbMatches();
const auto nbLinesInChunk = LinesCount( config.searchReadBufferSizeLines() );
LOG( logDEBUG ) << "Searching from line " << initialLine << " to " << nbSourceLines;
if ( initialLine < startLine_ ) {
initialLine = startLine_;
}
const auto endLine = qMin( LineNumber( nbSourceLines.get() ), endLine_ );
using namespace std::chrono;
high_resolution_clock::time_point t1 = high_resolution_clock::now();
QSemaphore searchCompleted;
QSemaphore blocksDone;
using SearchBlockQueue = moodycamel::BlockingConcurrentQueue<SearchBlockData>;
using ProcessMatchQueue = moodycamel::BlockingConcurrentQueue<PartialSearchResults>;
SearchBlockQueue searchBlockQueue;
ProcessMatchQueue processMatchQueue;
std::vector<QFuture<void>> matchers;
const auto matchingThreadsCount = [&config]()
{
if ( !config.useParallelSearch() ) {
return 1;
}
return qMax( 1, config.searchThreadPoolSize() == 0
? QThread::idealThreadCount() - 1
: static_cast<int>( config.searchThreadPoolSize() ) );
}();
LOG( logINFO ) << "Using " << matchingThreadsCount << " matching threads";
auto localThreadPool = std::make_unique<QThreadPool>();
localThreadPool->setMaxThreadCount( matchingThreadsCount + 2 );
const auto makeMatcher = [this, &searchBlockQueue, &processMatchQueue,
pool = localThreadPool.get()]( size_t index ) {
// copy and optimize regex for each thread
auto regexp = QRegularExpression{ regexp_.pattern(), regexp_.patternOptions() };
regexp.optimize();
return QtConcurrent::run( pool, [regexp, index, &searchBlockQueue, &processMatchQueue]() {
auto cToken = moodycamel::ConsumerToken{ searchBlockQueue };
auto pToken = moodycamel::ProducerToken{ processMatchQueue };
for ( ;; ) {
SearchBlockData blockData;
searchBlockQueue.wait_dequeue( cToken, blockData );
LOG( logDEBUG ) << "Searcher " << index << " " << blockData.chunkStart;
const auto lastBlock = blockData.lines.empty();
if ( !lastBlock ) {
blockData.results
= filterLines( regexp, blockData.lines, blockData.chunkStart );
}
LOG( logDEBUG ) << "Searcher " << index << " sending matches "
<< blockData.results.matchingLines.size();
processMatchQueue.enqueue( pToken, std::move( blockData.results ) );
if ( lastBlock ) {
LOG( logDEBUG ) << "Searcher " << index << " last block";
return;
}
}
} );
};
for ( int i = 0; i < matchingThreadsCount; ++i ) {
matchers.emplace_back( makeMatcher( i ) );
}
auto processMatches = QtConcurrent::run( localThreadPool.get(), [&]() {
auto cToken = moodycamel::ConsumerToken{ processMatchQueue };
size_t matchersDone = 0;
int reportedPercentage = 0;
auto reportedMatches = nbMatches;
LinesCount totalProcessedLines = 0_lcount;
const auto totalLines = endLine - initialLine;
for ( ;; ) {
PartialSearchResults matchResults;
processMatchQueue.wait_dequeue( cToken, matchResults );
LOG( logDEBUG ) << "Combining match results from " << matchResults.chunkStart;
if ( matchResults.processedLines.get() ) {
maxLength = qMax( maxLength, matchResults.maxLength );
nbMatches += LinesCount(
static_cast<LinesCount::UnderlyingType>( matchResults.matchingLines.size() ) );
const auto processedLines = LinesCount{ matchResults.chunkStart.get()
+ matchResults.processedLines.get() };
totalProcessedLines += matchResults.processedLines;
// After each block, copy the data to shared data
// and update the client
searchData.addAll( maxLength, matchResults.matchingLines, processedLines );
LOG( logDEBUG ) << "done Searching chunk starting at " << matchResults.chunkStart
<< ", " << matchResults.processedLines << " lines read.";
blocksDone.release( matchResults.processedLines.get() );
}
else {
matchersDone++;
}
const int percentage
= static_cast<int>( std::floor( 100.f * ( totalProcessedLines ).get() / totalLines.get() ) );
if ( percentage > reportedPercentage || nbMatches > reportedMatches ) {
emit searchProgressed( nbMatches, std::min( 99, percentage ), initialLine );
reportedPercentage = percentage;
reportedMatches = nbMatches;
}
if ( matchersDone == matchers.size() ) {
searchCompleted.release();
return;
}
}
} );
auto pToken = moodycamel::ProducerToken{ searchBlockQueue };
blocksDone.release( nbLinesInChunk.get() * ( static_cast<uint32_t>( matchers.size() ) + 1 ) );
for ( auto chunkStart = initialLine; chunkStart < endLine;
chunkStart = chunkStart + nbLinesInChunk ) {
if ( *interruptRequested_ )
break;
LOG( logDEBUG ) << "Reading chunk starting at " << chunkStart;
const auto linesInChunk
= LinesCount( qMin( nbLinesInChunk.get(), ( endLine - chunkStart ).get() ) );
auto lines = sourceLogData_->getLines( chunkStart, linesInChunk );
LOG( logDEBUG ) << "Sending chunk starting at " << chunkStart << ", " << lines.size()
<< " lines read.";
blocksDone.acquire( static_cast<uint32_t>( lines.size() ) );
searchBlockQueue.enqueue( pToken,
{ chunkStart, std::move( lines ), PartialSearchResults{} } );
LOG( logDEBUG ) << "Sent chunk starting at " << chunkStart << ", " << lines.size()
<< " lines read.";
}
for ( size_t i = 0; i < matchers.size(); ++i ) {
searchBlockQueue.enqueue( pToken,
{ endLine, std::vector<QString>{}, PartialSearchResults{} } );
}
searchCompleted.acquire();
high_resolution_clock::time_point t2 = high_resolution_clock::now();
auto duration = duration_cast<microseconds>( t2 - t1 ).count();
LOG( logINFO ) << "Searching done, took " << duration / 1000.f << " ms";
LOG( logINFO ) << "Searching perf "
<< static_cast<uint32_t>(
std::floor( 1000 * 1000.f * ( endLine - initialLine ).get() / duration ) )
<< " lines/s";
emit searchProgressed( nbMatches, 100, initialLine );
}