void DocsetRegistry::_runQuery(const QString &query)
{
m_cancellationToken.reset();
QList<Docset *> enabledDocsets;
const SearchQuery searchQuery = SearchQuery::fromString(query);
if (searchQuery.hasKeywords()) {
for (Docset *docset : qAsConst(m_docsets)) {
if (searchQuery.hasKeywords(docset->keywords()))
enabledDocsets << docset;
}
} else {
enabledDocsets = docsets();
}
QFuture<QList<SearchResult>> queryResultsFuture
= QtConcurrent::mappedReduced(enabledDocsets,
std::bind(&Docset::search,
std::placeholders::_1,
searchQuery.query(),
std::ref(m_cancellationToken)),
&MergeQueryResults);
QList<SearchResult> results = queryResultsFuture.result();
if (m_cancellationToken.isCanceled())
return;
std::sort(results.begin(), results.end());
if (m_cancellationToken.isCanceled())
return;
emit searchCompleted(results);
}
/**
* @brief ANNOTATION search testcase.
*/
void ColorSearchTest::annotationSearchTest()
{
SearchQuery query;
query.setDatasets({"../../../test/testdata/SingleFrameVideo"});
SearchObject obj;
obj.setSourceDataset("../../../test/testdata/SingleFrameVideo");
obj.setMedium(QDir("../../../test/testdata/SingleFrameVideo/SingleFrameVideo/").absolutePath());
obj.setMediumIndex(1);
RectangleAnnotation* ann = new RectangleAnnotation("11", "Person");
ann->setX(315);
ann->setY(53);
ann->setWidth(169);
ann->setHeight(570);
obj.setAnnotation(ann);
obj.setType(SearchObject::ANNOTATION);
query.setSearchObject(obj);
ColorSearch search;
QList<DataPacket*> in;
in.append(&query);
bool ok = search.setInputs(in);
QVERIFY(ok);
QList<DataPacket*> out = search.run();
QVERIFY(out.size() == 1);
}
void MainWindow::search(const SearchQuery &query)
{
if (query.isEmpty())
return;
ui->lineEdit->setText(query.toString());
ui->treeView->activated(ui->treeView->currentIndex());
}
bool SearchResult::getRelevance(SearchQuery& aQuery, RelevanceInfo& relevance_, const string& aLocalSearchToken) const noexcept {
if (!aLocalSearchToken.empty() && !matches(aQuery, aLocalSearchToken)) {
return false;
}
// Nothing to calculate with TTH searches
if (aQuery.root) {
relevance_.matchRelevance = 1;
relevance_.sourceScoreFactor = 0.01;
return true;
}
// Match path
SearchQuery::Recursion recursion;
ScopedFunctor([&] { aQuery.recursion = nullptr; });
if (!aQuery.matchesNmdcPath(path, recursion)) {
return false;
}
// Don't count the levels because they can't be compared with each others
auto matchRelevance = SearchQuery::getRelevanceScore(aQuery, 0, type == SearchResult::TYPE_DIRECTORY, getFileName());
double sourceScoreFactor = 0.01;
if (aQuery.recursion && aQuery.recursion->isComplete()) {
// There are subdirectories/files that have more matches than the main directory
// Don't give too much weight for those even if there are lots of sources
sourceScoreFactor = 0.001;
// We don't get the level scores so balance those here
matchRelevance = max(0.0, matchRelevance - (0.05 * aQuery.recursion->recursionLevel));
}
relevance_.matchRelevance = matchRelevance;
relevance_.sourceScoreFactor = sourceScoreFactor;
return true;
}
bool SearchResult::matches(SearchQuery& aQuery, const string& aLocalSearchToken) const noexcept {
if (!user.user->isNMDC()) {
// ADC
if (aLocalSearchToken != token) {
return false;
}
} else {
// NMDC results must be matched manually
// Exludes
if (aQuery.isExcluded(path)) {
return false;
}
if (aQuery.root && *aQuery.root != tth) {
return false;
}
}
// All clients can't handle this correctly
if (aQuery.itemType == SearchQuery::TYPE_FILE && type != SearchResult::TYPE_FILE) {
return false;
}
return true;
}
static SearchQuery* createRevComplQuery(SearchQuery* query, DNATranslation* transl) {
SAFE_POINT(query != NULL, "Query is null", NULL);
SAFE_POINT(transl != NULL, "Transl is null", NULL);
QByteArray reversed(query->constSequence());
TextUtils::reverse(reversed.data(), reversed.count());
DNASequence dnaSeq(QString("%1_rev").arg(query->getName()), reversed, NULL);
SearchQuery *rQu = new SearchQuery(&dnaSeq, query);
transl->translate(const_cast<char*>(rQu->constData()), rQu->length());
if (rQu->constSequence() == query->constSequence()) {
delete rQu; rQu = NULL;
return NULL;
}
query->setRevCompl(rQu);
return rQu;
}
void SearchRefreshService::HandleSearchResultsResponseReceived(const SearchQuery& query,
const std::vector<SearchResultModel>& results)
{
if (!m_searchResultsCleared)
{
m_searchResultsExist = true;
}
m_previousQueryLocationEcef = query.Location().ToECEF();
-- m_queriesPending;
Eegeo_ASSERT(m_queriesPending >= 0);
}
void WriteAlignedReadsSubTask::run() {
// ReadShortReadsSubTask can add new data what can lead to realloc. Noone can touch these vectors without sync
QReadLocker locker(&listM);
stateInfo.setProgress(0);
foreach(DataBunch* d, data) {
SAFE_POINT(NULL != d, "NULL data bunch", );
QVector<SearchQuery*> &queries = d->queries;
try {
SearchQuery *read = NULL;
SearchQuery *revCompl = NULL;
SearchQuery **q = queries.data();
int size = queries.size();
for (int i=0; i<size; i++) {
read = q[i];
revCompl = read->getRevCompl();
if (i<size-1 && revCompl == q[i+1]) {
continue;
}
if (NULL == revCompl && read->haveResult()) {
seqWriter->write(read, read->firstResult());
readsAligned++;
} else if (NULL != revCompl) {
int c = read->firstMCount();
int cRev = revCompl->firstMCount();
if (c <= cRev && c < INT_MAX) {
seqWriter->write(read, read->firstResult());
readsAligned++;
} else if (cRev < INT_MAX) {
seqWriter->write(revCompl, revCompl->firstResult());
readsAligned++;
}
}
}
} catch (QString exeptionMessage) {
setError(exeptionMessage);
}
}
/**
* @brief MEDIUM search testcase.
*/
void ColorSearchTest::mediumSearchTest()
{
SearchQuery query;
query.setDatasets({"../../../test/testdata/Fotos"});
SearchObject obj;
obj.setSourceDataset("../../../test/testdata/Fotos");
obj.setMedium(QDir("../../../test/testdata/Fotos/000_45.bmp").absolutePath());
obj.setMediumIndex(0);
obj.setType(SearchObject::MEDIUM);
query.setSearchObject(obj);
ColorSearch search;
QList<DataPacket*> in;
in.append(&query);
bool ok = search.setInputs(in);
QVERIFY(ok);
QList<DataPacket*> out = search.run();
QVERIFY(out.size() == 1);
}
void LLFloaterSearch::search(const SearchQuery &p)
{
if (! mWebBrowser || !p.validateBlock())
{
return;
}
// reset the god level warning as we're sending the latest state
getChildView("refresh_search")->setVisible(FALSE);
mSearchGodLevel = gAgent.getGodLevel();
// work out the subdir to use based on the requested category
LLSD subs;
if (mCategoryPaths.has(p.category))
{
subs["CATEGORY"] = mCategoryPaths[p.category].asString();
}
else
{
subs["CATEGORY"] = mCategoryPaths["all"].asString();
}
// add the search query string
subs["QUERY"] = LLURI::escape(p.query);
// add the permissions token that login.cgi gave us
// We use "search_token", and fallback to "auth_token" if not present.
LLSD search_token = LLLoginInstance::getInstance()->getResponse("search_token");
if (search_token.asString().empty())
{
search_token = LLLoginInstance::getInstance()->getResponse("auth_token");
}
subs["AUTH_TOKEN"] = search_token.asString();
// add the user's preferred maturity (can be changed via prefs)
std::string maturity;
if (gAgent.prefersAdult())
{
maturity = "42"; // PG,Mature,Adult
}
else if (gAgent.prefersMature())
{
maturity = "21"; // PG,Mature
}
else
{
maturity = "13"; // PG
}
subs["MATURITY"] = maturity;
// add the user's god status
subs["GODLIKE"] = gAgent.isGodlike() ? "1" : "0";
// get the search URL and expand all of the substitutions
// (also adds things like [LANGUAGE], [VERSION], [OS], etc.)
// <FS:AW opensim search support>
// std::string url = gSavedSettings.getString("SearchURL");
std::string url;
#ifdef HAS_OPENSIM_SUPPORT // <FS:AW optional opensim support>
std::string debug_url = gSavedSettings.getString("SearchURLDebug");
if (gSavedSettings.getBOOL("DebugSearch") && !debug_url.empty())
{
url = debug_url;
}
else if(LLGridManager::getInstance()->isInOpenSim())
{
url = LLLoginInstance::getInstance()->hasResponse("search")
? LLLoginInstance::getInstance()->getResponse("search").asString()
: gSavedSettings.getString("SearchURLOpenSim");
}
else // we are in SL or SL beta
#endif // HAS_OPENSIM_SUPPORT // <FS:AW optional opensim support>
{
url = gSavedSettings.getString("SearchURL");
}
// </FS:AW opensim search support>
url = LLWeb::expandURLSubstitutions(url, subs);
// and load the URL in the web view
mWebBrowser->navigateTo(url, "text/html");
}
/**
* Alright, the main point here is that when searching, a search string is most often found in
* the filename, not directory name, so we want to make that case faster. Also, we want to
* avoid changing StringLists unless we absolutely have to --> this should only be done if a string
* has been matched in the directory name. This new stringlist should also be used in all descendants,
* but not the parents...
*/
void ShareManager::Directory::search(SearchResultList& results, SearchQuery& query, size_t maxResults) const noexcept {
if(query.isExcluded(name))
return;
// Find any matches in the directory name and removed matched terms from the query.
StringSearch::List newTerms;
for(auto& term: query.include) {
if(term.match(name)) {
if(!newTerms.empty()) {
newTerms = query.include;
}
newTerms.erase(remove(newTerms.begin(), newTerms.end(), term), newTerms.end());
}
}
// auto const old = query.include;
if(!newTerms.empty()) {
query.include = newTerms;
}
if(query.include.empty() && query.ext.empty() && query.gt == 0) {
// We satisfied all the search words! Add the directory...
/// @todo send the directory hash when we have one
results.push_back(new SearchResult(SearchResult::TYPE_DIRECTORY, getSize(), getFullName(), TTHValue(string(39, 'A'))));
ShareManager::getInstance()->addHits(1);
}
if(!query.isDirectory) {
for(auto& i: files) {
if(!i.tth) { continue; }
// check the size
if(!(i.getSize() >= query.gt)) {
continue;
} else if(!(i.getSize() <= query.lt)) {
continue;
}
if(query.isExcluded(i.getName()))
continue;
// check if the name matches
auto j = query.include.begin();
for(; j != query.include.end() && j->match(i.getName()); ++j)
; // Empty
if(j != query.include.end())
continue;
// check extensions
if(!query.hasExt(i.getName()))
continue;
results.push_back(new SearchResult(SearchResult::TYPE_FILE, i.getSize(),
getFullName() + i.getName(), (i.tth)));
ShareManager::getInstance()->addHits(1);
if(results.size() >= maxResults) { return; }
}
}
for(auto& dir: directories) {
dir.second->search(results, query, maxResults);
if(results.size() >= maxResults) { return; }
}
}
SearchQuery::Recursion::Recursion(const SearchQuery& aSearch, const string& aName) noexcept : positions(aSearch.toPointList(aName)) {
if (aSearch.recursion && merge(positions, aSearch.recursion)) {
depthLen = aSearch.recursion->depthLen;
recursionLevel = aSearch.recursion->recursionLevel;
}
}
double SearchQuery::getRelevanceScore(const SearchQuery& aSearch, int aLevel, bool aIsDirectory, const string& aName) noexcept {
// get the level scores first
double scores = aLevel > 0 ? 9 / static_cast<double>(aLevel) : 10;
double maxPoints = 10;
auto positions = aSearch.getResultPositions(aName);
if (positions.empty()) {
// "Find and view NFO" in own list is performed without include terms, but we still want to prefer lower-level items
return scores / maxPoints;
}
dcassert(boost::find_if(positions, CompareFirst<size_t, int>(string::npos)) == positions.end());
// check the recursion level (ignore recurions if the last item was fully matched)
int recursionLevel = 0;
if (aSearch.recursion && aSearch.getLastIncludeMatches() != static_cast<int>(aSearch.include.count())) {
recursionLevel = aSearch.recursion->recursionLevel;
}
// Prefer sequential matches
bool isSorted = is_sorted(positions.begin(), positions.end());
if (isSorted) {
scores += 120;
}
maxPoints += 120;
// maximum points from SearchQuery::toPointList based on the include count
double maxPosPoints = (aSearch.include.count() * 20.0) + (20.0 * (recursionLevel+1));
// separators
double curPosPoints = 0;
for (auto i : positions) {
curPosPoints += i.second;
}
if (isSorted) {
scores += curPosPoints;
} else {
scores += (curPosPoints / maxPosPoints) * 10;
}
maxPoints += maxPosPoints;
// distance of the matched words (ignores missing separators)
if (isSorted) {
int minDistance = 0;
for (const auto& p : aSearch.include.getPatterns()) {
minDistance += p.size();
}
minDistance = minDistance + aSearch.include.count() - aSearch.include.getPatterns().back().size() - 1;
int extraDistance = (positions.back().first - positions.front().first) - minDistance;
scores += extraDistance > 0 ? max((1 / static_cast<double>(extraDistance)) * 20, 0.0) : 30;
}
maxPoints += 30;
// positions of the first pattern (prefer the beginning)
if (isSorted) {
auto startPos = positions[0].first;
scores += startPos > 0 ? (1 / static_cast<double>(startPos)) * 20 : 30;
}
maxPoints += 30;
// prefer directories
if (aIsDirectory) {
scores += 5;
}
maxPoints += 5;
// scale the points
scores = scores / maxPoints;
// drop results with no direct matches
if (recursionLevel > 0 && all_of(aSearch.getLastPositions().begin(), aSearch.getLastPositions().end(), [](size_t pos) { return pos == string::npos; })) {
scores = scores / (recursionLevel + 1);
}
return scores;
}
void ReadShortReadsSubTask::run() {
stateInfo.setProgress(0);
GTIMER(cvar, tvar, "ReadSubTask");
GenomeAlignerTask *parent = static_cast<GenomeAlignerTask*>(getParentTask());
if (!alignContext.bestMode) {
parent->pWriteTask->flush();
}
alignContext.cleanVectors();
dataBunch = new DataBunch();
if (isCanceled()) {
readingFinishedWakeAll();
return;
}
qint64 m = freeMemorySize;
taskLog.details(QString("Memory size is %1").arg(m));
bool alignReversed = settings.getCustomValue(GenomeAlignerTask::OPTION_ALIGN_REVERSED, true).toBool();
int qualityThreshold = settings.getCustomValue(GenomeAlignerTask::OPTION_QUAL_THRESHOLD, 0).toInt();
DNATranslation* transl = AppContext::getDNATranslationRegistry()->
lookupTranslation(BaseDNATranslationIds::NUCL_DNA_DEFAULT_COMPLEMENT);
alignContext.isReadingStarted = true;
bunchSize = 0;
int readNum = 0;
while(!seqReader->isEnd()) {
if (isCanceled()) {
readingFinishedWakeAll();
return;
}
SearchQuery *query = seqReader->read();
if (NULL == query) {
if (!seqReader->isEnd()) {
setError("Short-reads object type must be a sequence, but not a multiple alignment");
readingFinishedWakeAll();
return;
}
break;
}
++bunchSize;
if (!checkDnaQuality(query, qualityThreshold)) {
continue;
}
updateMinMaxReadLengths(alignContext, query->length());
int W = 0, q = 0;
int CMAX = alignContext.nMismatches;
if (!add(CMAX, W, q, readNum, query, parent)) {
delete query;
continue;
}
m -= query->memoryHint();
if (alignReversed) {
SearchQuery *rQu = createRevComplQuery(query, transl);
if (rQu) {
add(CMAX, W, q, readNum, rQu, parent);
m -= rQu->memoryHint();
}
}
qint64 alignContextMemoryHint = dataBunch->memoryHint();
if (m <= alignContextMemoryHint + prevMemoryHint) {
break;
}
SAFE_POINT(NULL != dataBunch, "No dataBunch",);
if (dataBunch->bitValuesV.size() > DROP_BUNCH_DATA_SIZE) {
dropToAlignContext();
readNum = 0;
alignContext.readShortReadsWait.wakeOne();
}
}
dropToAlignContext();
readingFinishedWakeAll();
}
int sbol::PartShop::searchCount(SearchQuery& q)
{
string url = parseURLDomain(resource);
/* Perform HTTP request */
string response;
CURL *curl;
CURLcode res;
/* In windows, this will init the winsock stuff */
curl_global_init(CURL_GLOBAL_ALL);
struct curl_slist *headers = NULL;
// headers = curl_slist_append(headers, "Accept: application/json");
headers = curl_slist_append(headers, "Content-Type: application/x-www-form-urlencoded");
// headers = curl_slist_append(headers, "charsets: utf-8");
/* get a curl handle */
curl = curl_easy_init();
/* Specify the GET parameters */
if(curl)
{
string parameters;
// Specify the type of SBOL object to search for
if (q["objectType"].size() == 1)
parameters = "objectType=" + parseClassName(q["objectType"].get()) + "&";
else
throw SBOLError(SBOL_ERROR_INVALID_ARGUMENT, "SearchQuery is invalid because it does not have an objectType specified");
// Get the search criteria, while ignoring special search parameters like objectType, offset, and limit
vector<string> search_criteria = q.getProperties();
auto i_ignore = std::find(std::begin(search_criteria), std::end(search_criteria), SBOL_IDENTITY);
search_criteria.erase(i_ignore);
i_ignore = std::find(std::begin(search_criteria), std::end(search_criteria), SBOL_URI "#objectType");
search_criteria.erase(i_ignore);
i_ignore = std::find(std::begin(search_criteria), std::end(search_criteria), SBOL_URI "#offset");
search_criteria.erase(i_ignore);
i_ignore = std::find(std::begin(search_criteria), std::end(search_criteria), SBOL_URI "#limit");
search_criteria.erase(i_ignore);
// Form GET request from the search criteria
for (auto & property_uri : search_criteria)
for (auto & property_val : q.getPropertyValues(property_uri))
{
if (property_val.length() > 0)
{
parameters += "<" + property_uri + ">=";
if (property_val.find("http") == 0)
parameters += "<" + property_val + ">&"; // encode property value as a URI
else
parameters += "'" + property_val + "'&"; // encode property value as a literal
}
}
encode_http(parameters);
parameters = url + "/searchCount/" + parameters;
/* First set the URL that is about to receive our GET. */
//curl_easy_setopt(curl, CURLOPT_URL, Config::getOption("validator_url").c_str());
curl_easy_setopt(curl, CURLOPT_URL, parameters.c_str());
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
/* Now specify the callback to read the response into string */
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
throw SBOLError(SBOL_ERROR_BAD_HTTP_REQUEST, "Attempt to validate online failed with " + string(curl_easy_strerror(res)));
/* always cleanup */
curl_easy_cleanup(curl);
}
curl_slist_free_all(headers);
curl_global_cleanup();
int count;
try
{
count = stoi(response);
}
catch (...)
{
throw SBOLError(SBOL_ERROR_BAD_HTTP_REQUEST, "Search failed with error message" + response);
}
return count;
};
// static
void LLFloaterSearch::search(const SearchQuery &p, LLMediaCtrl* mWebBrowser)
{
if (! mWebBrowser || !p.validateBlock())
{
return;
}
// work out the subdir to use based on the requested category
LLSD subs;
// declare a map that transforms a category name into
// the URL suffix that is used to search that category
static LLSD mCategoryPaths = LLSD::emptyMap();
if (mCategoryPaths.size() == 0)
{
mCategoryPaths["all"] = "search";
mCategoryPaths["people"] = "search/people";
mCategoryPaths["places"] = "search/places";
mCategoryPaths["events"] = "search/events";
mCategoryPaths["groups"] = "search/groups";
mCategoryPaths["wiki"] = "search/wiki";
mCategoryPaths["destinations"] = "destinations";
mCategoryPaths["classifieds"] = "classifieds";
}
if (mCategoryPaths.has(p.category))
{
subs["CATEGORY"] = mCategoryPaths[p.category].asString();
}
else
{
subs["CATEGORY"] = mCategoryPaths["all"].asString();
}
// add the search query string
subs["QUERY"] = LLURI::escape(p.query);
// add the permissions token that login.cgi gave us
// We use "search_token", and fallback to "auth_token" if not present.
LLSD search_token = LLUserAuth::getInstance()->getResponse("search_token");
if (search_token.asString().empty())
{
search_token = LLUserAuth::getInstance()->getResponse("auth_token");
}
subs["AUTH_TOKEN"] = search_token.asString();
// add the user's preferred maturity (can be changed via prefs)
std::string maturity;
if (gAgent.prefersAdult())
{
maturity = "42"; // PG,Mature,Adult
}
else if (gAgent.prefersMature())
{
maturity = "21"; // PG,Mature
}
else
{
maturity = "13"; // PG
}
subs["MATURITY"] = maturity;
// add the user's god status
subs["GODLIKE"] = gAgent.isGodlike() ? "1" : "0";
// get the search URL and expand all of the substitutions
// (also adds things like [LANGUAGE], [VERSION], [OS], etc.)
std::string url = gSavedSettings.getString("SearchURL");
url = LLWeb::expandURLSubstitutions(url, subs);
// and load the URL in the web view
mWebBrowser->navigateTo(url, "text/html");
}
