int main(int argc, char** argv) { QCoreApplication app(argc, argv); QNetworkAccessManager qnam; ProgramFetcher fetcher(&qnam, QUrl::fromLocalFile("programme-test.json")); QObject::connect(&fetcher, &ProgramFetcher::finished, [&app](const ProgramData& p) { debugOutput(p); Program prog = Program::fromProgramData(p); qDebug() << prog.days()[1].date(); qDebug() << prog.getDayByDate(QDate(2016, 4, 7))->title; app.quit(); }); fetcher.fetchAsync(); return app.exec(); }
TEST_F(CachingCorrectnessTest, PostToSameURLTwice) { ResourceRequest request1(KURL(ParsedURLString, kResourceURL)); request1.setHTTPMethod(HTTPNames::POST); ResourcePtr<Resource> resource1 = new Resource(ResourceRequest(request1.url()), Resource::Raw); resource1->setLoading(true); memoryCache()->add(resource1.get()); ResourceRequest request2(KURL(ParsedURLString, kResourceURL)); request2.setHTTPMethod(HTTPNames::POST); FetchRequest fetch2(request2, FetchInitiatorInfo()); ResourcePtr<Resource> resource2 = RawResource::fetchSynchronously(fetch2, fetcher()); EXPECT_EQ(resource2, memoryCache()->resourceForURL(request2.url())); EXPECT_NE(resource1, resource2); }
TEST_F(ClearKeyFetcherTest, Success) { ClearKeyFetcher fetcher(std::move(license_fetcher_)); EXPECT_EQ(OK, fetcher.Init()); uint64_t asset_id; std::vector<KeyFetcher::KeyInfo> keys; EXPECT_EQ(OK, fetcher.ObtainKey(ecm_, &asset_id, &keys)); EXPECT_EQ(2, keys.size()); EXPECT_EQ(0, keys[0].key_id); EXPECT_EQ(content_key_[0]->size(), keys[0].key_bytes->size()); EXPECT_EQ(0, memcmp(content_key_[0]->data(), keys[0].key_bytes->data(), content_key_[0]->size())); EXPECT_EQ(1, keys[1].key_id); EXPECT_EQ(content_key_[1]->size(), keys[1].key_bytes->size()); EXPECT_EQ(0, memcmp(content_key_[1]->data(), keys[1].key_bytes->data(), content_key_[1]->size())); }
int main(int argc, char *argv[]) { QGuiApplication app(argc, argv); Fetcher::Ptr fetcher(new Fetcher()); ScriptureModel scripture(fetcher); SearchModel search(fetcher); QQmlApplicationEngine engine; QQmlContext *context = engine.rootContext(); context->setContextProperty("scripture_model", &scripture); context->setContextProperty("search_model", &search); engine.load(QUrl(QStringLiteral("qrc:/main.qml"))); app.exec(); }
Resource* DocumentLoader::startPreload(Resource::Type type, FetchRequest& request) { Resource* resource = nullptr; switch (type) { case Resource::Image: resource = ImageResource::fetch(request, fetcher()); break; case Resource::Script: resource = ScriptResource::fetch(request, fetcher()); break; case Resource::CSSStyleSheet: resource = CSSStyleSheetResource::fetch(request, fetcher()); break; case Resource::Font: resource = FontResource::fetch(request, fetcher()); break; case Resource::Media: resource = RawResource::fetchMedia(request, fetcher()); break; case Resource::TextTrack: resource = RawResource::fetchTextTrack(request, fetcher()); break; case Resource::ImportResource: resource = RawResource::fetchImport(request, fetcher()); break; case Resource::Raw: resource = RawResource::fetch(request, fetcher()); break; default: NOTREACHED(); } // CSP layout tests verify that preloads are subject to access checks by // seeing if they are in the `preload started` list. Therefore do not add // them to the list if the load is immediately denied. if (resource && !resource->resourceError().isAccessCheck()) fetcher()->preloadStarted(resource); return resource; }
int main() { struct config conf; memset(&conf, 0, sizeof(conf)); // Lookup IP of target struct hostent *he = gethostbyname("128.138.200.81"); memset(&conf.sin, 0, sizeof(conf.sin)); conf.sin.sin_family = he->h_addrtype; conf.sin.sin_port = htons(443); conf.sin.sin_addr = *(((struct in_addr **)he->h_addr_list)[0]); // Dummy values memset(conf.prev_block_hash, 0xAA, 32); memset(conf.merkle_root, 0xBB, 32); memset(conf.difficulty, 0x00, 32); conf.difficulty[0] = 0x00; conf.difficulty[1] = 0x00; conf.difficulty[2] = 0x07; // Load RSA key FILE *fp = fopen("./pubkey.pem", "rb"); // Apparently, PEM_read_PUBKEY() doens't read PEM(?!?) PEM_read_RSA_PUBKEY(fp, &conf.public_key, NULL, NULL); fclose(fp); if (conf.public_key == NULL) { printf("Error couldn't read public key\n"); return -1; } // Open /dev/urandom conf.rand = fopen("/dev/urandom", "rb"); conf.base = event_base_new(); struct timeval one_sec = {1,0}; struct event *status_ev = event_new(conf.base, -1, EV_PERSIST, print_status, &conf); event_add(status_ev, &one_sec); fetcher(&conf, 1000); event_base_dispatch(conf.base); return 0; }
void DocumentLoader::startLoadingMainResource() { timing().markNavigationStart(); DCHECK(!m_mainResource); DCHECK_EQ(m_state, NotStarted); m_state = Provisional; if (maybeLoadEmpty()) return; DCHECK(timing().navigationStart()); // PlzNavigate: // The fetch has already started in the browser. Don't mark it again. if (!m_frame->settings()->browserSideNavigationEnabled()) { DCHECK(!timing().fetchStart()); timing().markFetchStart(); } DEFINE_STATIC_LOCAL( ResourceLoaderOptions, mainResourceLoadOptions, (DoNotBufferData, AllowStoredCredentials, ClientRequestedCredentials, CheckContentSecurityPolicy, DocumentContext)); FetchRequest fetchRequest(m_request, FetchInitiatorTypeNames::document, mainResourceLoadOptions); m_mainResource = RawResource::fetchMainResource(fetchRequest, fetcher(), m_substituteData); // PlzNavigate: // The final access checks are still performed here, potentially rejecting // the "provisional" load, but the browser side already expects the renderer // to be able to unconditionally commit. if (!m_mainResource || (m_frame->settings()->browserSideNavigationEnabled() && m_mainResource->errorOccurred())) { m_request = ResourceRequest(blankURL()); maybeLoadEmpty(); return; } // A bunch of headers are set when the underlying resource load begins, and // m_request needs to include those. Even when using a cached resource, we may // make some modification to the request, e.g. adding the referer header. m_request = m_mainResource->isLoading() ? m_mainResource->resourceRequest() : fetchRequest.resourceRequest(); m_mainResource->addClient(this); }
Resource* DocumentLoader::startPreload(Resource::Type type, FetchRequest& request) { Resource* resource = nullptr; switch (type) { case Resource::Image: resource = ImageResource::fetch(request, fetcher()); break; case Resource::Script: resource = ScriptResource::fetch(request, fetcher()); break; case Resource::CSSStyleSheet: resource = CSSStyleSheetResource::fetch(request, fetcher()); break; case Resource::Font: resource = FontResource::fetch(request, fetcher()); break; case Resource::Media: resource = RawResource::fetchMedia(request, fetcher()); break; case Resource::TextTrack: resource = RawResource::fetchTextTrack(request, fetcher()); break; case Resource::ImportResource: resource = RawResource::fetchImport(request, fetcher()); break; case Resource::LinkPreload: resource = RawResource::fetch(request, fetcher()); break; default: ASSERT_NOT_REACHED(); } if (resource) fetcher()->preloadStarted(resource); return resource; }
void MusicBrainzFetcherTest::testTitle() { Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Album, Tellico::Fetch::Title, m_fieldValues.value(QLatin1String("title"))); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::MusicBrainzFetcher(this)); Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); QCOMPARE(results.size(), 1); Tellico::Data::EntryPtr entry = results.at(0); QHashIterator<QString, QString> i(m_fieldValues); while(i.hasNext()) { i.next(); QString result = entry->field(i.key()).toLower(); QCOMPARE(result, i.value().toLower()); } QVERIFY(!entry->field(QLatin1String("track")).isEmpty()); QVERIFY(!entry->field(QLatin1String("cover")).isEmpty()); }
void EntryUpdateJobTest::testUpdate() { Tellico::Data::CollPtr coll(new Tellico::Data::BibtexCollection(true)); Tellico::Data::FieldPtr field(new Tellico::Data::Field("arxiv", "Arxiv ID")); coll->addField(field); Tellico::Data::EntryPtr entry(new Tellico::Data::Entry(coll)); entry->setField("arxiv", "hep-lat/0110180"); coll->addEntries(entry); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::ArxivFetcher(this)); // don't use 'this' as job parent, it crashes Tellico::EntryUpdateJob* job = new Tellico::EntryUpdateJob(0, entry, fetcher); connect(job, SIGNAL(result(KJob*)), &m_loop, SLOT(quit())); job->start(); m_loop.exec(); QCOMPARE(entry->field("title"), QLatin1String("Speeding up the Hybrid-Monte-Carlo algorithm for dynamical fermions")); }
void DBLPFetcherTest::testProceedings() { Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Bibtex, Tellico::Fetch::Keyword, QLatin1String("Chip and PIN is Broken")); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::DBLPFetcher(this)); Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); QCOMPARE(results.size(), 1); Tellico::Data::EntryPtr entry = results.at(0); QCOMPARE(entry->field(QLatin1String("title")), QLatin1String("Chip and PIN is Broken.")); QCOMPARE(entry->field(QLatin1String("author")), QLatin1String("Steven J. Murdoch; Saar Drimer; Ross J. Anderson; Mike Bond")); QCOMPARE(entry->field(QLatin1String("year")), QLatin1String("2010")); QCOMPARE(entry->field(QLatin1String("pages")), QLatin1String("433-446")); QCOMPARE(entry->field(QLatin1String("booktitle")), QLatin1String("IEEE Symposium on Security and Privacy")); QCOMPARE(entry->field(QLatin1String("url")), QLatin1String("http://dx.doi.org/10.1109/SP.2010.33")); QCOMPARE(entry->field(QLatin1String("doi")), QLatin1String("10.1109/SP.2010.33")); QCOMPARE(entry->field(QLatin1String("entry-type")), QLatin1String("inproceedings")); QCOMPARE(entry->field(QLatin1String("bibtex-key")), QLatin1String("MurdochDAB10")); }
void GoogleBookFetcherTest::testAuthor() { Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Book, Tellico::Fetch::Person, QLatin1String("Shelley Powers")); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::GoogleBookFetcher(this)); if(m_hasConfigFile) { KConfigGroup cg(&m_config, QLatin1String("GoogleBookTest")); fetcher->readConfig(cg, cg.name()); } Tellico::Data::EntryList results = DO_FETCH(fetcher, request); Tellico::Data::EntryPtr entry; foreach(Tellico::Data::EntryPtr testEntry, results) { if(testEntry->title() == QLatin1String("Practical RDF")) { entry = testEntry; break; } } QVERIFY(entry); compareEntry(entry); }
static TnyStream * tny_fs_stream_cache_get_stream (TnyStreamCache *self, const gchar *id, TnyStreamCacheOpenStreamFetcher fetcher, gpointer userdata) { TnyStream *result = NULL; TnyFsStreamCachePriv *priv; TnyCachedFile *cached_file; g_return_val_if_fail (TNY_IS_FS_STREAM_CACHE (self), NULL); g_return_val_if_fail (id, NULL); priv = TNY_FS_STREAM_CACHE_GET_PRIVATE (self); g_static_mutex_lock (priv->cache_lock); cached_file = g_hash_table_lookup (priv->cached_files, id); g_static_mutex_unlock (priv->cache_lock); if (cached_file) { result = tny_cached_file_get_stream (cached_file); } else { TnyStream *input_stream; gint64 expected_size = 0; input_stream = fetcher (self, &expected_size, userdata); if (input_stream == NULL) return NULL; if (expected_size <= get_available_size (TNY_FS_STREAM_CACHE (self))) { remove_old_files (TNY_FS_STREAM_CACHE (self), expected_size); cached_file = tny_cached_file_new (TNY_FS_STREAM_CACHE (self), id, expected_size, input_stream); g_static_mutex_lock (priv->cache_lock); g_hash_table_insert (priv->cached_files, g_strdup (id), cached_file); g_static_mutex_unlock (priv->cache_lock); result = tny_cached_file_get_stream (cached_file); } else { result = input_stream; } } return result; }
void MusicBrainzFetcherTest::testKeyword() { // the total test case ends up exceeding the throttle limit so pause for a second QTest::qWait(1000); Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Album, Tellico::Fetch::Keyword, m_fieldValues.value(QLatin1String("title"))); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::MusicBrainzFetcher(this)); Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); QCOMPARE(results.size(), 1); Tellico::Data::EntryPtr entry = results.at(0); QHashIterator<QString, QString> i(m_fieldValues); while(i.hasNext()) { i.next(); QString result = entry->field(i.key()).toLower(); QCOMPARE(result, i.value().toLower()); } QVERIFY(!entry->field(QLatin1String("track")).isEmpty()); QVERIFY(!entry->field(QLatin1String("cover")).isEmpty()); }
bool XSLStyleSheet::parseString(const String& source) { // Parse in a single chunk into an xmlDocPtr if (!m_stylesheetDocTaken) xmlFreeDoc(m_stylesheetDoc); m_stylesheetDocTaken = false; PageConsole* console = 0; Frame* frame = ownerDocument()->frame(); if (frame && frame->host()) console = &frame->host()->console(); XMLDocumentParserScope scope(fetcher(), XSLTProcessor::genericErrorFunc, XSLTProcessor::parseErrorFunc, console); XMLParserInput input(source); xmlParserCtxtPtr ctxt = xmlCreateMemoryParserCtxt(input.data(), input.size()); if (!ctxt) return 0; if (m_parentStyleSheet) { // The XSL transform may leave the newly-transformed document // with references to the symbol dictionaries of the style sheet // and any of its children. XML document disposal can corrupt memory // if a document uses more than one symbol dictionary, so we // ensure that all child stylesheets use the same dictionaries as their // parents. xmlDictFree(ctxt->dict); ctxt->dict = m_parentStyleSheet->m_stylesheetDoc->dict; xmlDictReference(ctxt->dict); } m_stylesheetDoc = xmlCtxtReadMemory(ctxt, input.data(), input.size(), finalURL().string().utf8().data(), input.encoding(), XML_PARSE_NOENT | XML_PARSE_DTDATTR | XML_PARSE_NOWARNING | XML_PARSE_NOCDATA); xmlFreeParserCtxt(ctxt); loadChildSheets(); return m_stylesheetDoc; }
void AnimenfoFetcherTest::testHachimitsu() { KConfig config(QFINDTESTDATA("tellicotest.config"), KConfig::SimpleConfig); QString groupName = QLatin1String("AnimeNfo.com"); if(!config.hasGroup(groupName)) { QSKIP("This test requires a config file.", SkipAll); } KConfigGroup cg(&config, groupName); Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Video, Tellico::Fetch::Keyword, "Hachimitsu to Clover"); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::AnimeNfoFetcher(this)); fetcher->readConfig(cg, cg.name()); Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); QCOMPARE(results.size(), 1); // the first entry had better be the right one Tellico::Data::EntryPtr entry = results.at(0); QVERIFY(entry); QCOMPARE(entry->field("title"), QLatin1String("Hachimitsu to Clover")); QCOMPARE(entry->field("year"), QLatin1String("2005")); QCOMPARE(entry->field("episodes"), QLatin1String("26")); QCOMPARE(entry->field("keyword"), QLatin1String("TV")); QCOMPARE(entry->field("genre"), QLatin1String("Comedy; Drama; Romance")); QCOMPARE(entry->field("studio"), QLatin1String("J.C.STAFF")); QCOMPARE(entry->field("origtitle"), QString::fromUtf8("ハチミツとクローバー")); QCOMPARE(entry->field("director"), QString::fromUtf8("Kasai Kenichi (カサヰ ケンイチ)")); QCOMPARE(entry->field("writer"), QString::fromUtf8("Kuroda Yosuke (黒田洋介)")); QCOMPARE(entry->field("alttitle"), QLatin1String("Honey and Clover")); QCOMPARE(entry->field("animenfo-rating"), QLatin1String("9")); QVERIFY(entry->field("plot").startsWith(QLatin1String("Takemoto, Mayama, and Morita are students"))); QVERIFY(!entry->field("cover").isEmpty()); QVERIFY(!entry->field("animenfo").isEmpty()); QStringList castList = Tellico::FieldFormat::splitTable(entry->field("cast")); QCOMPARE(castList.count(), 7); QCOMPARE(castList.at(0), QString::fromUtf8("Kudo Haruka (工藤晴香)::Hanamoto Hagumi")); }
void DBLPFetcherTest::testArticle() { Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Bibtex, Tellico::Fetch::Keyword, QLatin1String("Nontrivial independent sets of bipartite graphs")); Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::DBLPFetcher(this)); Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); QCOMPARE(results.size(), 1); Tellico::Data::EntryPtr entry = results.at(0); QCOMPARE(entry->field(QLatin1String("title")), QLatin1String("Nontrivial independent sets of bipartite graphs and cross-intersecting families.")); QCOMPARE(entry->field(QLatin1String("author")), QLatin1String("Jun Wang; Huajun Zhang")); QCOMPARE(entry->field(QLatin1String("year")), QLatin1String("2013")); QCOMPARE(entry->field(QLatin1String("pages")), QLatin1String("129-141")); QCOMPARE(entry->field(QLatin1String("volume")), QLatin1String("120")); QCOMPARE(entry->field(QLatin1String("number")), QLatin1String("1")); QCOMPARE(entry->field(QLatin1String("journal")), QLatin1String("J. Comb. Theory, Ser. A (JCT)")); QCOMPARE(entry->field(QLatin1String("booktitle")), QLatin1String("")); QCOMPARE(entry->field(QLatin1String("url")), QLatin1String("http://dx.doi.org/10.1016/j.jcta.2012.07.005")); QCOMPARE(entry->field(QLatin1String("doi")), QLatin1String("10.1016/j.jcta.2012.07.005")); QCOMPARE(entry->field(QLatin1String("entry-type")), QLatin1String("article")); QCOMPARE(entry->field(QLatin1String("bibtex-key")), QLatin1String("WangZ13")); }
//************************************************************************ //* LyricWiki * //************************************************************************ pfc::string_list_impl * provider_lyricwiki::lookup(unsigned p_index, metadb_handle_list_cref p_meta, threaded_process_status & p_status, abort_callback & p_abort) { TRACK_CALL_TEXT("provider_lyricwiki::lookup"); // Regular Expression Class CRegexpT<char> regexp; // Buffer pfc::string8 buff; pfc::string_list_impl * str_list = new pfc::string_list_impl; try { // Init fetcher curl_wrapper_simple fetcher(&m_config_item); for (t_size i = 0; i < p_meta.get_count(); ++i) { if (p_abort.is_aborting()) break; // Sleep how_to_sleep(i); // Clear buff buff.reset(); const metadb_handle_ptr & p = p_meta.get_item(i); if (p.is_empty()) { str_list->add_item(""); continue; } // Set progress pfc::string8_fast path = file_path_canonical(p->get_path()); // add subsong index? if (p->get_subsong_index() > 0) { path.add_string(" /index:"); path.add_string(pfc::format_uint(p->get_subsong_index())); } p_status.set_item_path(path); p_status.set_progress(i + 1, p_meta.get_count()); pfc::string8_fast artist, title; file_info_impl info; p->get_info(info); // Get count of artists t_size count = info.meta_get_count_by_name("artist"); // Get TITLE static_api_ptr_t<titleformat_compiler> compiler; service_ptr_t<titleformat_object> script; compiler->compile_safe(script, "$replace($caps2(%title%),' ','_')"); p->format_title(NULL, title, script, NULL); bool found = false; // Iterate through all artists listed for (int j = 0; j < count && !found; j++) { // Get Artist artist = info.meta_get("artist", j); artist.replace_char(' ', '_'); //Fetching from HTTP // Set HTTP Address pfc::string8_fast url("http://lyrics.wikia.com/index.php?title="); //URL = http://lyrics.wikia.com/index.php?title=Blackmore%27s_Night:I_Guess_It_Doesn%27t_Matter_Anymore&action=edit url += fetcher.quote(artist); url += ":"; url += fetcher.quote(title); // Get it now try { fetcher.fetch(url, buff); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_lyrics = "'lyricbox'(?P<instrumental>.*?)</div>(?P<lyrics>.*?)<!--"; // expression for extract lyrics regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE); int noGroup = regexp.GetNamedGroupNumber("lyrics"); int instGroup = regexp.GetNamedGroupNumber("instrumental"); MatchResult result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { int nStart = result.GetGroupStart(instGroup); int nEnd = result.GetGroupEnd(instGroup); pfc::string8_fast test(buff.get_ptr() + nStart, nEnd-nStart); if (test.find_first("Instrumental") != -1) { found = true; str_list->add_item("[Instrumental]"); continue; } nStart = result.GetGroupStart(noGroup); nEnd = result.GetGroupEnd(noGroup); pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart); convert_html_to_plain(lyric); if (lyric.get_length() > 0) { found = true; string_helper::remove_beginning_linebreaks(lyric); string_helper::remove_end_linebreaks(lyric); str_list->add_item(lyric); continue; } } } if (found) continue; else str_list->add_item(""); } } catch (pfc::exception & e) { console_error(e.what()); delete str_list; return NULL; } catch (...) { delete str_list; return NULL; } return str_list; }
Resource* MemoryCacheCorrectnessTestHelper::fetch() { ResourceRequest resourceRequest(KURL(ParsedURLString, kResourceURL)); resourceRequest.setRequestContext(WebURLRequest::RequestContextInternal); FetchRequest fetchRequest(resourceRequest, FetchInitiatorInfo()); return RawResource::fetch(fetchRequest, fetcher()); }
StatusWith<NearStage::CoveredInterval*> // GeoNear2DSphereStage::nextInterval(OperationContext* txn, WorkingSet* workingSet, Collection* collection) { // The search is finished if we searched at least once and all the way to the edge if (_currBounds.getInner() >= 0 && _currBounds.getOuter() == _fullBounds.getOuter()) { return StatusWith<CoveredInterval*>(NULL); } // // Setup the next interval // const NearStats* stats = getNearStats(); if (!stats->intervalStats.empty()) { const IntervalStats& lastIntervalStats = stats->intervalStats.back(); // TODO: Generally we want small numbers of results fast, then larger numbers later if (lastIntervalStats.numResultsBuffered < 300) _boundsIncrement *= 2; else if (lastIntervalStats.numResultsBuffered > 600) _boundsIncrement /= 2; } R2Annulus nextBounds(_currBounds.center(), _currBounds.getOuter(), min(_currBounds.getOuter() + _boundsIncrement, _fullBounds.getOuter())); bool isLastInterval = (nextBounds.getOuter() == _fullBounds.getOuter()); _currBounds = nextBounds; // // Setup the covering region and stages for this interval // IndexScanParams scanParams; scanParams.descriptor = _s2Index; scanParams.direction = 1; // We use a filter on the key. The filter rejects keys that don't intersect with the // annulus. An object that is in the annulus might have a key that's not in it and a key // that's in it. As such we can't just look at one key per object. // // This does force us to do our own deduping of results, though. scanParams.doNotDedup = true; scanParams.bounds = _nearParams.baseBounds; // Because the planner doesn't yet set up 2D index bounds, do it ourselves here const string s2Field = _nearParams.nearQuery.field; const int s2FieldPosition = getFieldPosition(_s2Index, s2Field); scanParams.bounds.fields[s2FieldPosition].intervals.clear(); OrderedIntervalList* coveredIntervals = &scanParams.bounds.fields[s2FieldPosition]; TwoDSphereKeyInRegionExpression* keyMatcher = new TwoDSphereKeyInRegionExpression(_currBounds, s2Field); ExpressionMapping::cover2dsphere(keyMatcher->getRegion(), _s2Index->infoObj(), coveredIntervals); // IndexScan owns the hash matcher IndexScan* scan = new IndexScanWithMatch(txn, scanParams, workingSet, keyMatcher); // FetchStage owns index scan FetchStage* fetcher(new FetchStage(workingSet, scan, _nearParams.filter, collection)); return StatusWith<CoveredInterval*>(new CoveredInterval(fetcher, true, nextBounds.getInner(), nextBounds.getOuter(), isLastInterval)); }
StatusWith<NearStage::CoveredInterval*> // GeoNear2DStage::nextInterval(OperationContext* txn, WorkingSet* workingSet, Collection* collection) { // The search is finished if we searched at least once and all the way to the edge if (_currBounds.getInner() >= 0 && _currBounds.getOuter() == _fullBounds.getOuter()) { return StatusWith<CoveredInterval*>(NULL); } // // Setup the next interval // const NearStats* stats = getNearStats(); if (!stats->intervalStats.empty()) { const IntervalStats& lastIntervalStats = stats->intervalStats.back(); // TODO: Generally we want small numbers of results fast, then larger numbers later if (lastIntervalStats.numResultsBuffered < 300) _boundsIncrement *= 2; else if (lastIntervalStats.numResultsBuffered > 600) _boundsIncrement /= 2; } _boundsIncrement = max(_boundsIncrement, min2DBoundsIncrement(_nearParams.nearQuery, _twoDIndex)); R2Annulus nextBounds(_currBounds.center(), _currBounds.getOuter(), min(_currBounds.getOuter() + _boundsIncrement, _fullBounds.getOuter())); const bool isLastInterval = (nextBounds.getOuter() == _fullBounds.getOuter()); _currBounds = nextBounds; // // Get a covering region for this interval // const CRS queryCRS = _nearParams.nearQuery.centroid.crs; auto_ptr<R2Region> coverRegion; if (FLAT == queryCRS) { // NOTE: Due to floating point math issues, FLAT searches of a 2D index need to treat // containment and distance separately. // Ex: (distance) 54.001 - 54 > 0.001, but (containment) 54 + 0.001 <= 54.001 // The idea is that a $near search with bounds is really a $within search, sorted by // distance. We attach a custom $within : annulus matcher to do the $within search, // and adjust max/min bounds slightly since, as above, containment does not mean the // distance calculation won't slightly overflow the boundary. // // The code below adjusts: // 1) Overall min/max bounds of the generated distance intervals to be more inclusive // 2) Bounds of the interval covering to be more inclusive // ... and later on we add the custom $within : annulus matcher. // // IMPORTANT: The *internal* interval distance bounds are *exact thresholds* - these // should not be adjusted. // TODO: Maybe integrate annuluses as a standard shape, and literally transform $near // internally into a $within query with $near just as sort. // Compute the maximum axis-aligned distance error const double epsilon = std::numeric_limits<double>::epsilon() * (max(abs(_fullBounds.center().x), abs(_fullBounds.center().y)) + _fullBounds.getOuter()); if (nextBounds.getInner() > 0 && nextBounds.getInner() == _fullBounds.getInner()) { nextBounds = R2Annulus(nextBounds.center(), max(0.0, nextBounds.getInner() - epsilon), nextBounds.getOuter()); } if (nextBounds.getOuter() > 0 && nextBounds.getOuter() == _fullBounds.getOuter()) { // We're at the max bound of the search, adjust interval maximum nextBounds = R2Annulus(nextBounds.center(), nextBounds.getInner(), nextBounds.getOuter() + epsilon); } // *Always* adjust the covering bounds to be more inclusive coverRegion.reset(new R2Annulus(nextBounds.center(), max(0.0, nextBounds.getInner() - epsilon), nextBounds.getOuter() + epsilon)); } else { invariant(SPHERE == queryCRS); // TODO: As above, make this consistent with $within : $centerSphere // Our intervals aren't in the same CRS as our index, so we need to adjust them coverRegion.reset(new R2Annulus(projectBoundsToTwoDDegrees(nextBounds))); } // // Setup the stages for this interval // IndexScanParams scanParams; scanParams.descriptor = _twoDIndex; scanParams.direction = 1; // We use a filter on the key. The filter rejects keys that don't intersect with the // annulus. An object that is in the annulus might have a key that's not in it and a key // that's in it. As such we can't just look at one key per object. // // This does force us to do our own deduping of results, though. scanParams.doNotDedup = true; // Scan bounds on 2D indexes are only over the 2D field - other bounds aren't applicable. // This is handled in query planning. scanParams.bounds = _nearParams.baseBounds; // The "2d" field is always the first in the index const string twoDFieldName = _nearParams.nearQuery.field; const int twoDFieldPosition = 0; OrderedIntervalList coveredIntervals; coveredIntervals.name = scanParams.bounds.fields[twoDFieldPosition].name; ExpressionMapping::cover2d(*coverRegion, _twoDIndex->infoObj(), internalGeoNearQuery2DMaxCoveringCells, &coveredIntervals); // Intersect the $near bounds we just generated into the bounds we have for anything else // in the scan (i.e. $within) IndexBoundsBuilder::intersectize(coveredIntervals, &scanParams.bounds.fields[twoDFieldPosition]); // These parameters are stored by the index, and so must be ok GeoHashConverter::Parameters hashParams; GeoHashConverter::parseParameters(_twoDIndex->infoObj(), &hashParams); MatchExpression* keyMatcher = new TwoDKeyInRegionExpression(coverRegion.release(), hashParams, twoDFieldName); // 2D indexes support covered search over additional fields they contain // TODO: Don't need to clone, can just attach to custom matcher above if (_nearParams.filter) { AndMatchExpression* andMatcher = new AndMatchExpression(); andMatcher->add(keyMatcher); andMatcher->add(_nearParams.filter->shallowClone()); keyMatcher = andMatcher; } // IndexScanWithMatch owns the matcher IndexScan* scan = new IndexScanWithMatch(txn, scanParams, workingSet, keyMatcher); MatchExpression* docMatcher = NULL; // FLAT searches need to add an additional annulus $within matcher, see above if (FLAT == queryCRS) { docMatcher = new TwoDPtInAnnulusExpression(_fullBounds, twoDFieldName); } // FetchStage owns index scan FetchStage* fetcher(new FetchStageWithMatch(workingSet, scan, docMatcher, collection)); return StatusWith<CoveredInterval*>(new CoveredInterval(fetcher, true, nextBounds.getInner(), nextBounds.getOuter(), isLastInterval)); }
void BackgroundSync::_produce(OperationContext* txn, executor::TaskExecutor* taskExecutor) { // this oplog reader does not do a handshake because we don't want the server it's syncing // from to track how far it has synced { stdx::unique_lock<stdx::mutex> lock(_mutex); if (_lastOpTimeFetched.isNull()) { // then we're initial syncing and we're still waiting for this to be set lock.unlock(); sleepsecs(1); // if there is no one to sync from return; } if (_replCoord->isWaitingForApplierToDrain() || _replCoord->getMemberState().primary() || inShutdownStrict()) { return; } } while (MONGO_FAIL_POINT(rsBgSyncProduce)) { sleepmillis(0); } // find a target to sync from the last optime fetched OpTime lastOpTimeFetched; { stdx::unique_lock<stdx::mutex> lock(_mutex); lastOpTimeFetched = _lastOpTimeFetched; _syncSourceHost = HostAndPort(); } OplogReader syncSourceReader; syncSourceReader.connectToSyncSource(txn, lastOpTimeFetched, _replCoord); // no server found if (syncSourceReader.getHost().empty()) { sleepsecs(1); // if there is no one to sync from return; } long long lastHashFetched; { stdx::lock_guard<stdx::mutex> lock(_mutex); if (_pause) { return; } lastOpTimeFetched = _lastOpTimeFetched; lastHashFetched = _lastFetchedHash; _syncSourceHost = syncSourceReader.getHost(); _replCoord->signalUpstreamUpdater(); } const Milliseconds oplogSocketTimeout(OplogReader::kSocketTimeout); // Prefer host in oplog reader to _syncSourceHost because _syncSourceHost may be cleared // if sync source feedback fails. const HostAndPort source = syncSourceReader.getHost(); syncSourceReader.resetConnection(); // no more references to oplog reader from here on. // If this status is not OK after the fetcher returns from wait(), // proceed to execute rollback Status remoteOplogStartStatus = Status::OK(); auto fetcherCallback = stdx::bind(&BackgroundSync::_fetcherCallback, this, stdx::placeholders::_1, stdx::placeholders::_3, stdx::cref(source), lastOpTimeFetched, lastHashFetched, &remoteOplogStartStatus); auto cmdObj = BSON("find" << nsToCollectionSubstring(rsOplogName) << "filter" << BSON("ts" << BSON("$gte" << lastOpTimeFetched.getTimestamp())) << "tailable" << true << "oplogReplay" << true << "awaitData" << true << "maxTimeMS" << int(fetcherMaxTimeMS.count())); Fetcher fetcher(taskExecutor, source, nsToDatabase(rsOplogName), cmdObj, fetcherCallback, rpc::makeEmptyMetadata()); auto scheduleStatus = fetcher.schedule(); if (!scheduleStatus.isOK()) { warning() << "unable to schedule fetcher to read remote oplog on " << source << ": " << scheduleStatus; return; } fetcher.wait(); // If the background sync is paused after the fetcher is started, we need to // re-evaluate our sync source and oplog common point. if (isPaused()) { return; } // Execute rollback if necessary. // Rollback is a synchronous operation that uses the task executor and may not be // executed inside the fetcher callback. if (!remoteOplogStartStatus.isOK()) { const int messagingPortTags = 0; ConnectionPool connectionPool(messagingPortTags); std::unique_ptr<ConnectionPool::ConnectionPtr> connection; auto getConnection = [&connection, &connectionPool, oplogSocketTimeout, source]() -> DBClientBase* { if (!connection.get()) { connection.reset(new ConnectionPool::ConnectionPtr( &connectionPool, source, Date_t::now(), oplogSocketTimeout)); }; return connection->get(); }; log() << "starting rollback: " << remoteOplogStartStatus; _rollback(txn, source, getConnection); stop(); } }
pfc::string8 provider_azlyrics::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort) { TRACK_CALL_TEXT("provider_azlyrics::lookup_one"); const float threshold = 0.8f; // Regular Expression Class CRegexpT<char> regexp; // Buffer pfc::string8 buff; try { TRACK_CALL_TEXT("Try"); // Init fetcher curl_wrapper_simple fetcher(&m_config_item); const metadb_handle_ptr & p = p_meta; if (p.is_empty()) { return ""; } pfc::string8_fast artist, title, album; static_api_ptr_t<titleformat_compiler> compiler; service_ptr_t<titleformat_object> script; file_info_impl info; p->get_info(info); // Get count of artists t_size count = info.meta_get_count_by_name("artist"); // Get TITLE title = info.meta_get("title", 0); // Iterate through all artists listed for (int j = 0; j < count; j++) { TRACK_CALL_TEXT("For"); // Get Artist artist = info.meta_get("artist", j); console::printf("%s - %s", artist, title); // Search the lyrics pfc::string8_fast url("http://search.azlyrics.com/search.php?q="); url += fetcher.quote(artist); url += "+"; url += fetcher.quote(title); // Get it now try { fetcher.fetch(url, buff); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } if (buff.get_length() == 0) continue; int resultStart = buff.find_first("<b>1.</b>"); int startUrl = buff.find_first("<a href=\"", resultStart) + 9; int endUrl = buff.find_first("\"", startUrl); url = pfc::string8_fast(buff.get_ptr()+startUrl, endUrl - startUrl); // Get it now try { fetcher.fetch(url, buff); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } if (buff.get_length() == 0) continue; const char * regex_lyrics = "<!-- END OF RINGTONE 1 -->\\s*?<b>\"(?P<title>.*?)\"</b><br>\\s*?<br>\\s\\s(?P<lyrics>.*?)\\s<br>"; // expression for extract lyrics regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE); int noGroup = regexp.GetNamedGroupNumber("lyrics"); int noTitle = regexp.GetNamedGroupNumber("title"); MatchResult result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { int nStart = result.GetGroupStart(noTitle); int nEnd = result.GetGroupEnd(noTitle); pfc::string8_fast songTitle(buff.get_ptr() +nStart, nEnd - nStart); int levDist = LD(title, title.get_length(), songTitle, songTitle.get_length()); float good = 1.0f - ((float)levDist / title.get_length()); if (good < threshold) return ""; nStart = result.GetGroupStart(noGroup); nEnd = result.GetGroupEnd(noGroup); pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart); convert_html_to_plain(lyric); if (lyric.get_length() > 0) { string_helper::remove_end_linebreaks(lyric); return lyric; } } } } catch (pfc::exception & e) { console_error(e.what()); return ""; } catch (...) { return ""; } return ""; }
//************************************************************************ //* AZ Lyrics * //************************************************************************ pfc::string_list_impl * provider_azlyrics::lookup(unsigned p_index, metadb_handle_list_cref p_meta, threaded_process_status & p_status, abort_callback & p_abort) { TRACK_CALL_TEXT("provider_azlyrics::lookup"); const float threshold = 0.8f; // Regular Expression Class CRegexpT<char> regexp; // Buffer pfc::string8 buff; pfc::string_list_impl * str_list = new pfc::string_list_impl; try { // Init fetcher curl_wrapper_simple fetcher(&m_config_item); for (t_size i = 0; i < p_meta.get_count(); ++i) { if (p_abort.is_aborting()) break; // Sleep how_to_sleep(i); // Clear buff buff.reset(); const metadb_handle_ptr & p = p_meta.get_item(i); if (p.is_empty()) { str_list->add_item(""); continue; } // Set progress pfc::string8_fast path = file_path_canonical(p->get_path()); // add subsong index? if (p->get_subsong_index() > 0) { path.add_string(" /index:"); path.add_string(pfc::format_uint(p->get_subsong_index())); } p_status.set_item_path(path); p_status.set_progress(i + 1, p_meta.get_count()); pfc::string8_fast artist, title; file_info_impl info; p->get_info(info); // Get count of artists t_size count = info.meta_get_count_by_name("artist"); // Get TITLE title = info.meta_get("title", 0); bool found = false; // Iterate through all artists listed for (int j = 0; j < count && !found; j++) { // Get Artist artist = info.meta_get("artist", j); // Search the lyrics pfc::string8_fast url("http://search.azlyrics.com/search.php?q="); url += fetcher.quote(artist); url += "+"; url += fetcher.quote(title); // Get it now try { fetcher.fetch(url, buff); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } int resultStart = buff.find_first("<b>1.</b>"); int startUrl = buff.find_first("<a href=\"", resultStart) + 9; int endUrl = buff.find_first("\"", startUrl); url = pfc::string8_fast(buff.get_ptr()+startUrl, endUrl - startUrl); // Get it now try { fetcher.fetch(url, buff); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_lyrics = "<!-- END OF RINGTONE 1 -->\\s*?<b>\"(?P<title>.*?)\"</b><br>\\s*?<br>\\s\\s(?P<lyrics>.*?)\\s<br>"; // expression for extract lyrics regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE); int noGroup = regexp.GetNamedGroupNumber("lyrics"); int noTitle = regexp.GetNamedGroupNumber("title"); // match MatchResult result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { int nStart = result.GetGroupStart(noTitle); int nEnd = result.GetGroupEnd(noTitle); pfc::string8_fast songTitle(buff.get_ptr() +nStart, nEnd - nStart); int levDist = LD(title, title.get_length(), songTitle, songTitle.get_length()); float good = 1.0f - ((float)levDist / title.get_length()); if (good < threshold) continue; nStart = result.GetGroupStart(noGroup); nEnd = result.GetGroupEnd(noGroup); pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart); convert_html_to_plain(lyric); if (lyric.get_length() > 0) { string_helper::remove_end_linebreaks(lyric); str_list->add_item(lyric); found = true; continue; } } } if (found) continue; else str_list->add_item(""); } } catch (pfc::exception & e) { console_error(e.what()); delete str_list; return NULL; } catch (...) { delete str_list; return NULL; } return str_list; }
pfc::string8 provider_darklyrics::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort) { TRACK_CALL_TEXT("provider_darklyrics::lookup_one"); const float threshold = 0.8f; const pfc::string8 site = "darklyrics.com"; // Regular Expression Class CRegexpT<char> regexp; MatchResult match; // Buffer pfc::string8 buff; try { // Init fetcher curl_wrapper_simple fetcher(&m_config_item); const metadb_handle_ptr & p = p_meta; if (p.is_empty()) { return ""; } pfc::string8_fast artist, title, album, keywords; file_info_impl info; p->get_info(info); // Get count of artists t_size count = info.meta_get_count_by_name("album"); if (count == 0) return ""; // Get Album album = info.meta_get("album", 0); count = info.meta_get_count_by_name("title"); if (count == 0) return ""; // Get TITLE title = info.meta_get("title", 0); count = info.meta_get_count_by_name("artist"); // Iterate through all artists listed for (int j = 0; j < count; j++) { // Get Artist artist = info.meta_get("artist", j); //Fetching from HTTP keywords = artist; keywords += "+"; keywords += album; keywords.replace_char(' ', '+'); // Get it now try { fetcher.fetch_googleluck(site, keywords, buff); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_ahref = "<a\\shref=\"#(?P<no>\\d+)\">(?P<text>.+?)</a>"; // expression for extract lyrics regexp.Compile(regex_ahref, IGNORECASE); // match MatchResult result = regexp.Match(buff.get_ptr()); int noGroup = regexp.GetNamedGroupNumber("no"); int textGroup = regexp.GetNamedGroupNumber("text"); int jump_to = 0; pfc::string8_fast compare = title; compare.insert_chars(0, ". "); float good; float best = 0.0f; while (result.IsMatched()) { int gStart = result.GetGroupStart(noGroup); int gEnd = result.GetGroupEnd(noGroup); pfc::string8_fast temp(buff.get_ptr() + gStart, gEnd - gStart); int no = StrToIntA(temp); gStart = result.GetGroupStart(textGroup); gEnd = result.GetGroupEnd(textGroup); temp = pfc::string8_fast(buff.get_ptr()+gStart, gEnd - gStart); int levDist = LD(compare, compare.get_length(), temp, temp.get_length()); good = 1.0f - (levDist / (float)compare.get_length()); if (good >= threshold && good > best) { jump_to = no; best = good; } result = regexp.Match(buff.get_ptr(),result.GetEnd()); } if (jump_to == 0) { continue; } char regex_lyrics[100]; sprintf(regex_lyrics, "<a\\s+name=%d><font*(.*?)</font*(.*?)>(?P<lyrics>.+?)<font", jump_to); // expression for extract lyrics regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE); noGroup = regexp.GetNamedGroupNumber("lyrics"); result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { int nStart = result.GetGroupStart(noGroup); int nEnd = result.GetGroupEnd(noGroup); pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart); convert_html_to_plain(lyric); if (lyric.get_length() > 0) { string_helper::remove_beginning_linebreaks(lyric); string_helper::remove_end_linebreaks(lyric); string_helper::remove_beginning(lyric, ' '); string_helper::remove_beginning(lyric, '\t'); return lyric; } } } } catch (pfc::exception & e) { console_error(e.what()); return ""; } catch (...) { return ""; } return ""; }
//************************************************************************ //* Dark Lyrics * //************************************************************************ pfc::string_list_impl * provider_darklyrics::lookup(unsigned p_index, metadb_handle_list_cref p_meta, threaded_process_status & p_status, abort_callback & p_abort) { TRACK_CALL_TEXT("provider_darklyrics::lookup"); const float threshold = 0.8f; const pfc::string8 site = "darklyrics.com"; // Regular Expression Class CRegexpT<char> regexp; MatchResult match; // Buffer pfc::string8 buff; pfc::string_list_impl * str_list = new pfc::string_list_impl; try { // Init fetcher curl_wrapper_simple fetcher(&m_config_item); for (t_size i = 0; i < p_meta.get_count(); ++i) { if (p_abort.is_aborting()) break; // Sleep how_to_sleep(i); // Clear buff buff.reset(); const metadb_handle_ptr & p = p_meta.get_item(i); if (p.is_empty()) { str_list->add_item(""); continue; } // Set progress pfc::string8_fast path = file_path_canonical(p->get_path()); // add subsong index? if (p->get_subsong_index() > 0) { path.add_string(" /index:"); path.add_string(pfc::format_uint(p->get_subsong_index())); } p_status.set_item_path(path); p_status.set_progress(i + 1, p_meta.get_count()); pfc::string8_fast artist, title, album, keywords; file_info_impl info; p->get_info(info); // Get count of artists t_size count = info.meta_get_count_by_name("album"); if (count == 0) continue; // Get Album album = info.meta_get("album", 0); count = info.meta_get_count_by_name("title"); if (count == 0) continue; // Get TITLE title = info.meta_get("title", 0); count = info.meta_get_count_by_name("artist"); bool found = false; // Iterate through all artists listed for (int j = 0; j < count && !found; j++) { // Get Artist artist = info.meta_get("artist", j); keywords = artist; keywords += "+"; keywords += album; keywords.replace_char(' ', '+'); // Get it now try { fetcher.fetch_googleluck(site, keywords, buff); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_ahref = "<a\\shref=\"#(?P<no>\\d+)\">(?P<text>.+?)</a>"; // expression for extract lyrics regexp.Compile(regex_ahref, IGNORECASE | SINGLELINE); // match MatchResult result = regexp.Match(buff.get_ptr()); int noGroup = regexp.GetNamedGroupNumber("no"); int textGroup = regexp.GetNamedGroupNumber("text"); int jump_to = 0; pfc::string8_fast compare = title; compare.insert_chars(0, ". "); float good; float best = 0.0f; while (result.IsMatched()) { int gStart = result.GetGroupStart(noGroup); int gEnd = result.GetGroupEnd(noGroup); pfc::string8_fast temp(buff.get_ptr() + gStart, gEnd - gStart); int no = StrToIntA(temp); gStart = result.GetGroupStart(textGroup); gEnd = result.GetGroupEnd(textGroup); temp = pfc::string8_fast(buff.get_ptr()+gStart, gEnd - gStart); if (temp.find_first(title) != -1) { jump_to = no; break; } int levDist = LD(compare, compare.get_length(), temp, temp.get_length()); good = 1.0f - (levDist / (float)compare.get_length()); if (good >= threshold && good > best) { jump_to = no; best = good; } result = regexp.Match(buff.get_ptr(),result.GetEnd()); } if (jump_to == 0) { continue; } char regex_lyrics[100]; sprintf(regex_lyrics, "<a\\s+name=%d><font*(.*?)</font*(.*?)>(?P<lyrics>.+?)<font", jump_to); // expression for extract lyrics regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE); noGroup = regexp.GetNamedGroupNumber("lyrics"); result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { int nStart = result.GetGroupStart(noGroup); int nEnd = result.GetGroupEnd(noGroup); pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart); convert_html_to_plain(lyric); if (lyric.get_length() > 0) { string_helper::remove_beginning_linebreaks(lyric); string_helper::remove_end_linebreaks(lyric); string_helper::remove_beginning(lyric, ' '); string_helper::remove_beginning(lyric, '\t'); console::print(lyric); str_list->add_item(lyric); found = true; continue; } } } if (found) continue; else str_list->add_item(""); } } catch (pfc::exception & e) { console_error(e.what()); delete str_list; return NULL; } catch (...) { delete str_list; return NULL; } return str_list; }
pfc::string8 provider_lyricsplugin::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort) { // Regular Expression Class CRegexpT<char> regexp; MatchResult match; // Buffer pfc::string8 buff; try { // Init fetcher curl_wrapper_simple fetcher(&m_config_item); // Clear buff buff.reset(); const metadb_handle_ptr & p = p_meta; if (p.is_empty()) { return ""; } pfc::string8_fast artist, title; static_api_ptr_t<titleformat_compiler> compiler; service_ptr_t<titleformat_object> script; file_info_impl info; p->get_info(info); // Get count of artists t_size count = info.meta_get_count_by_name("artist"); // Get TITLE compiler->compile_safe(script, "[%title%]"); p->format_title(NULL, title, script, NULL); bool found = false; // Iterate through all artists listed for (int j = 0; j < count && !found; j++) { // Get Artist artist = info.meta_get("artist", j); // Fetching from HTTP // Set HTTP Address pfc::string8_fast url("http://www.squirrelscript.net/mediamonkey/Lyricator/lyrics.php?artist="); // URL = http://www.squirrelscript.net/mediamonkey/Lyricator/lyrics.php?artist=<Artist>&title=<Title> url += fetcher.quote(artist); url += "&title="; url += fetcher.quote(title); // Get it now try { fetcher.fetch(url, buff); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_lyricbox = "<div\\s+id\\s*?=\\s*?\"lyrics\"\\s*?>[\\r\\n]*(.*?)[\\r\\n]*</div>"; // expression for extract lyrics regexp.Compile(regex_lyricbox, SINGLELINE); // match MatchResult result = regexp.Match(buff.get_ptr()); // Get Group if (result.IsMatched()) { int nStart = result.GetGroupStart(1); int nEnd = result.GetGroupEnd(1); int index; pfc::string8_fast lyric(buff.get_ptr() + nStart, nEnd - nStart); convert_html_to_plain(lyric); index = lyric.find_first("www.tunerankings.com"); if (index == 0) { continue; } else if (index != -1) { lyric.remove_chars(index, 20); } if (string_trim(lyric).get_length() > 0) { return lyric; } } } } catch (pfc::exception & e) { console_error(e.what()); return ""; } catch (...) { return ""; } return ""; }
pfc::string8 provider_leoslyrics::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort) { TRACK_CALL_TEXT("provider_leoslyrics::lookup_one"); // Regular Expression Class CRegexpT<char> regexp; // Buffer pfc::string8 buff; try { // Init fetcher curl_wrapper_simple fetcher(&m_config_item); const metadb_handle_ptr & p = p_meta; if (p.is_empty()) { return ""; } pfc::string8_fast artist, title, album; file_info_impl info; p->get_info(info); // Get count of artists t_size count = info.meta_get_count_by_name("artist"); // Get TITLE static_api_ptr_t<titleformat_compiler> compiler; service_ptr_t<titleformat_object> script; compiler->compile_safe(script, "%title%"); p->format_title(NULL, title, script, NULL); // Iterate through all artists listed for (int j = 0; j < count; j++) { // Get Artist artist = info.meta_get("artist", j); //Fetching from HTTP // Set HTTP Address pfc::string8_fast url("http://77.79.210.222/api_search.php?auth=LeosLyrics5&artist="); pfc::string8_fast host("api.leoslyrics.com"); //URL = http://77.79.210.222/api_search.php?auth=LeosLyrics5&artist=<artist>&songtitle=<title> url += fetcher.quote(artist); url += "&songtitle="; url += fetcher.quote(title); // Get it now try { fetcher.fetch_host(host, url, buff, p_abort); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_hid = "code=\"(?P<code>.).*?hid=\"(?P<hid>.*?)\""; // expression for extract lyrics regexp.Compile(regex_hid, IGNORECASE | SINGLELINE); pfc::string8_fast hid; int codeGroup = regexp.GetNamedGroupNumber("code"); int hidGroup = regexp.GetNamedGroupNumber("hid"); MatchResult result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { int nStart = result.GetGroupStart(codeGroup); int nEnd = result.GetGroupEnd(codeGroup); pfc::string8_fast code(buff.get_ptr() + nStart, nEnd-nStart); if (code.find_first("0") == -1) continue; nStart = result.GetGroupStart(hidGroup); nEnd = result.GetGroupEnd(hidGroup); hid = pfc::string8_fast(buff.get_ptr() + nStart, nEnd - nStart); url = "http://77.79.210.222/api_search.php?auth=LeosLyrics5&hid="; url += hid; //URL = http://77.79.210.222/api_search.php?auth=LeosLyrics5&hid=<songID> try { fetcher.fetch_host(host, url, buff, p_abort); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_hid = "<text>\\s(?P<lyrics>.*?)\\s</text>"; // expression for extract lyrics regexp.Compile(regex_hid, IGNORECASE | SINGLELINE); int lyricsGroup = regexp.GetNamedGroupNumber("lyrics"); result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { nStart = result.GetGroupStart(lyricsGroup); nEnd = result.GetGroupEnd(lyricsGroup); pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart); if (lyric.get_length() > 0) { string_helper::remove_beginning_linebreaks(lyric); string_helper::remove_end_linebreaks(lyric); return lyric; } } } } } catch (pfc::exception & e) { console_error(e.what()); return ""; } catch (...) { return ""; } return ""; }
//************************************************************************ //* Leo's Lyrics * //************************************************************************ pfc::string_list_impl * provider_leoslyrics::lookup(unsigned p_index, metadb_handle_list_cref p_meta, threaded_process_status & p_status, abort_callback & p_abort) { TRACK_CALL_TEXT("provider_leoslyrics::lookup"); // Regular Expression Class CRegexpT<char> regexp; // Buffer pfc::string8 buff; pfc::string_list_impl * str_list = new pfc::string_list_impl; try { // Init fetcher curl_wrapper_simple fetcher(&m_config_item); for (t_size i = 0; i < p_meta.get_count(); ++i) { if (p_abort.is_aborting()) break; // Sleep how_to_sleep(i); // Clear buff buff.reset(); const metadb_handle_ptr & p = p_meta.get_item(i); if (p.is_empty()) { str_list->add_item(""); continue; } // Set progress pfc::string8_fast path = file_path_canonical(p->get_path()); // add subsong index? if (p->get_subsong_index() > 0) { path.add_string(" /index:"); path.add_string(pfc::format_uint(p->get_subsong_index())); } p_status.set_item_path(path); p_status.set_progress(i + 1, p_meta.get_count()); pfc::string8_fast artist, title; file_info_impl info; p->get_info(info); // Get count of artists t_size count = info.meta_get_count_by_name("artist"); // Get TITLE static_api_ptr_t<titleformat_compiler> compiler; service_ptr_t<titleformat_object> script; compiler->compile_safe(script, "%title%"); p->format_title(NULL, title, script, NULL); bool found = false; // Iterate through all artists listed for (int j = 0; j < count && !found; j++) { // Get Artist artist = info.meta_get("artist", j); pfc::string8_fast url("http://77.79.210.222/api_search.php?auth=LeosLyrics5&artist="); pfc::string8_fast host("api.leoslyrics.com"); //URL = http://77.79.210.222/api_search.php?auth=LeosLyrics5&artist=<artist>&songtitle=<title> url += fetcher.quote(artist); url += "&songtitle="; url += fetcher.quote(title); // Get it now try { fetcher.fetch_host(host, url, buff, p_abort); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_hid = "code=\"(?P<code>.).*?hid=\"(?P<hid>.*?)\""; // expression for extract lyrics regexp.Compile(regex_hid, IGNORECASE | SINGLELINE); pfc::string8_fast hid; int codeGroup = regexp.GetNamedGroupNumber("code"); int hidGroup = regexp.GetNamedGroupNumber("hid"); MatchResult result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { int nStart = result.GetGroupStart(codeGroup); int nEnd = result.GetGroupEnd(codeGroup); pfc::string8_fast code(buff.get_ptr() + nStart, nEnd-nStart); if (code.find_first("0") == -1) continue; nStart = result.GetGroupStart(hidGroup); nEnd = result.GetGroupEnd(hidGroup); hid = pfc::string8_fast(buff.get_ptr() + nStart, nEnd - nStart); url = "http://77.79.210.222/api_search.php?auth=LeosLyrics5&hid="; url += hid; //URL = http://77.79.210.222/api_search.php?auth=LeosLyrics5&hid=<songID> try { fetcher.fetch_host(host, url, buff, p_abort); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_hid = "<text>\\s(?P<lyrics>.*?)\\s</text>"; // expression for extract lyrics regexp.Compile(regex_hid, IGNORECASE | SINGLELINE); int lyricsGroup = regexp.GetNamedGroupNumber("lyrics"); result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { nStart = result.GetGroupStart(lyricsGroup); nEnd = result.GetGroupEnd(lyricsGroup); pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart); if (lyric.get_length() > 0) { found = true; string_helper::remove_beginning_linebreaks(lyric); string_helper::remove_end_linebreaks(lyric); str_list->add_item(lyric); continue; } } } } if (found) continue; else str_list->add_item(""); } } catch (pfc::exception & e) { console_error(e.what()); delete str_list; return NULL; } catch (...) { delete str_list; return NULL; } return str_list; }
pfc::string8 provider_lyricwiki::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort) { TRACK_CALL_TEXT("provider_lyricwiki::lookup_one"); // Regular Expression Class CRegexpT<char> regexp; // Buffer pfc::string8 buff; try { // Init fetcher curl_wrapper_simple fetcher(&m_config_item); const metadb_handle_ptr & p = p_meta; if (p.is_empty()) { return ""; } pfc::string8_fast artist, title, album; file_info_impl info; p->get_info(info); // Get count of artists t_size count = info.meta_get_count_by_name("artist"); // Get TITLE static_api_ptr_t<titleformat_compiler> compiler; service_ptr_t<titleformat_object> script; compiler->compile_safe(script, "$replace($caps2(%title%),' ','_')"); p->format_title(NULL, title, script, NULL); // Iterate through all artists listed for (int j = 0; j < count; j++) { // Get Artist artist = info.meta_get("artist", j); artist.replace_char(' ', '_'); //Fetching from HTTP // Set HTTP Address pfc::string8_fast url("http://lyrics.wikia.com/index.php?title="); //URL = http://lyrics.wikia.com/index.php?title=Blackmore%27s_Night:I_Guess_It_Doesn%27t_Matter_Anymore&action=edit url += fetcher.quote(artist); url += ":"; url += fetcher.quote(title); // Get it now try { fetcher.fetch(url, buff); } catch (pfc::exception & e) { console_error(e.what()); continue; } catch (...) { continue; } const char * regex_lyrics = "'lyricbox'(?P<instrumental>.*?)</div>(?P<lyrics>.*?)<!--"; // expression for extract lyrics regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE); int noGroup = regexp.GetNamedGroupNumber("lyrics"); int instGroup = regexp.GetNamedGroupNumber("instrumental"); MatchResult result = regexp.Match(buff.get_ptr()); if (result.IsMatched()) { int nStart = result.GetGroupStart(instGroup); int nEnd = result.GetGroupEnd(instGroup); pfc::string8_fast test(buff.get_ptr() + nStart, nEnd-nStart); if (test.find_first("Instrumental") != -1) { return "[Instrumental]"; } nStart = result.GetGroupStart(noGroup); nEnd = result.GetGroupEnd(noGroup); pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart); convert_html_to_plain(lyric); if (lyric.get_length() > 0) { string_helper::remove_beginning_linebreaks(lyric); string_helper::remove_end_linebreaks(lyric); return lyric; } } } } catch (pfc::exception & e) { console_error(e.what()); return ""; } catch (...) { return ""; } return ""; }