int main(int argc, char** argv) {
QCoreApplication app(argc, argv);
QNetworkAccessManager qnam;
ProgramFetcher fetcher(&qnam, QUrl::fromLocalFile("programme-test.json"));
QObject::connect(&fetcher, &ProgramFetcher::finished, [&app](const ProgramData& p) {
debugOutput(p);
Program prog = Program::fromProgramData(p);
qDebug() << prog.days()[1].date();
qDebug() << prog.getDayByDate(QDate(2016, 4, 7))->title;
app.quit();
});
fetcher.fetchAsync();
return app.exec();
}
TEST_F(CachingCorrectnessTest, PostToSameURLTwice)
{
ResourceRequest request1(KURL(ParsedURLString, kResourceURL));
request1.setHTTPMethod(HTTPNames::POST);
ResourcePtr<Resource> resource1 = new Resource(ResourceRequest(request1.url()), Resource::Raw);
resource1->setLoading(true);
memoryCache()->add(resource1.get());
ResourceRequest request2(KURL(ParsedURLString, kResourceURL));
request2.setHTTPMethod(HTTPNames::POST);
FetchRequest fetch2(request2, FetchInitiatorInfo());
ResourcePtr<Resource> resource2 = RawResource::fetchSynchronously(fetch2, fetcher());
EXPECT_EQ(resource2, memoryCache()->resourceForURL(request2.url()));
EXPECT_NE(resource1, resource2);
}
TEST_F(ClearKeyFetcherTest, Success) {
ClearKeyFetcher fetcher(std::move(license_fetcher_));
EXPECT_EQ(OK, fetcher.Init());
uint64_t asset_id;
std::vector<KeyFetcher::KeyInfo> keys;
EXPECT_EQ(OK, fetcher.ObtainKey(ecm_, &asset_id, &keys));
EXPECT_EQ(2, keys.size());
EXPECT_EQ(0, keys[0].key_id);
EXPECT_EQ(content_key_[0]->size(), keys[0].key_bytes->size());
EXPECT_EQ(0, memcmp(content_key_[0]->data(),
keys[0].key_bytes->data(), content_key_[0]->size()));
EXPECT_EQ(1, keys[1].key_id);
EXPECT_EQ(content_key_[1]->size(), keys[1].key_bytes->size());
EXPECT_EQ(0, memcmp(content_key_[1]->data(),
keys[1].key_bytes->data(), content_key_[1]->size()));
}
int main(int argc, char *argv[])
{
QGuiApplication app(argc, argv);
Fetcher::Ptr fetcher(new Fetcher());
ScriptureModel scripture(fetcher);
SearchModel search(fetcher);
QQmlApplicationEngine engine;
QQmlContext *context = engine.rootContext();
context->setContextProperty("scripture_model", &scripture);
context->setContextProperty("search_model", &search);
engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
app.exec();
}
Resource* DocumentLoader::startPreload(Resource::Type type,
FetchRequest& request) {
Resource* resource = nullptr;
switch (type) {
case Resource::Image:
resource = ImageResource::fetch(request, fetcher());
break;
case Resource::Script:
resource = ScriptResource::fetch(request, fetcher());
break;
case Resource::CSSStyleSheet:
resource = CSSStyleSheetResource::fetch(request, fetcher());
break;
case Resource::Font:
resource = FontResource::fetch(request, fetcher());
break;
case Resource::Media:
resource = RawResource::fetchMedia(request, fetcher());
break;
case Resource::TextTrack:
resource = RawResource::fetchTextTrack(request, fetcher());
break;
case Resource::ImportResource:
resource = RawResource::fetchImport(request, fetcher());
break;
case Resource::Raw:
resource = RawResource::fetch(request, fetcher());
break;
default:
NOTREACHED();
}
// CSP layout tests verify that preloads are subject to access checks by
// seeing if they are in the `preload started` list. Therefore do not add
// them to the list if the load is immediately denied.
if (resource && !resource->resourceError().isAccessCheck())
fetcher()->preloadStarted(resource);
return resource;
}
int main()
{
struct config conf;
memset(&conf, 0, sizeof(conf));
// Lookup IP of target
struct hostent *he = gethostbyname("128.138.200.81");
memset(&conf.sin, 0, sizeof(conf.sin));
conf.sin.sin_family = he->h_addrtype;
conf.sin.sin_port = htons(443);
conf.sin.sin_addr = *(((struct in_addr **)he->h_addr_list)[0]);
// Dummy values
memset(conf.prev_block_hash, 0xAA, 32);
memset(conf.merkle_root, 0xBB, 32);
memset(conf.difficulty, 0x00, 32);
conf.difficulty[0] = 0x00;
conf.difficulty[1] = 0x00;
conf.difficulty[2] = 0x07;
// Load RSA key
FILE *fp = fopen("./pubkey.pem", "rb");
// Apparently, PEM_read_PUBKEY() doens't read PEM(?!?)
PEM_read_RSA_PUBKEY(fp, &conf.public_key, NULL, NULL);
fclose(fp);
if (conf.public_key == NULL) {
printf("Error couldn't read public key\n");
return -1;
}
// Open /dev/urandom
conf.rand = fopen("/dev/urandom", "rb");
conf.base = event_base_new();
struct timeval one_sec = {1,0};
struct event *status_ev = event_new(conf.base, -1, EV_PERSIST, print_status, &conf);
event_add(status_ev, &one_sec);
fetcher(&conf, 1000);
event_base_dispatch(conf.base);
return 0;
}
void DocumentLoader::startLoadingMainResource() {
timing().markNavigationStart();
DCHECK(!m_mainResource);
DCHECK_EQ(m_state, NotStarted);
m_state = Provisional;
if (maybeLoadEmpty())
return;
DCHECK(timing().navigationStart());
// PlzNavigate:
// The fetch has already started in the browser. Don't mark it again.
if (!m_frame->settings()->browserSideNavigationEnabled()) {
DCHECK(!timing().fetchStart());
timing().markFetchStart();
}
DEFINE_STATIC_LOCAL(
ResourceLoaderOptions, mainResourceLoadOptions,
(DoNotBufferData, AllowStoredCredentials, ClientRequestedCredentials,
CheckContentSecurityPolicy, DocumentContext));
FetchRequest fetchRequest(m_request, FetchInitiatorTypeNames::document,
mainResourceLoadOptions);
m_mainResource =
RawResource::fetchMainResource(fetchRequest, fetcher(), m_substituteData);
// PlzNavigate:
// The final access checks are still performed here, potentially rejecting
// the "provisional" load, but the browser side already expects the renderer
// to be able to unconditionally commit.
if (!m_mainResource || (m_frame->settings()->browserSideNavigationEnabled() &&
m_mainResource->errorOccurred())) {
m_request = ResourceRequest(blankURL());
maybeLoadEmpty();
return;
}
// A bunch of headers are set when the underlying resource load begins, and
// m_request needs to include those. Even when using a cached resource, we may
// make some modification to the request, e.g. adding the referer header.
m_request = m_mainResource->isLoading() ? m_mainResource->resourceRequest()
: fetchRequest.resourceRequest();
m_mainResource->addClient(this);
}
Resource* DocumentLoader::startPreload(Resource::Type type, FetchRequest& request)
{
Resource* resource = nullptr;
switch (type) {
case Resource::Image:
resource = ImageResource::fetch(request, fetcher());
break;
case Resource::Script:
resource = ScriptResource::fetch(request, fetcher());
break;
case Resource::CSSStyleSheet:
resource = CSSStyleSheetResource::fetch(request, fetcher());
break;
case Resource::Font:
resource = FontResource::fetch(request, fetcher());
break;
case Resource::Media:
resource = RawResource::fetchMedia(request, fetcher());
break;
case Resource::TextTrack:
resource = RawResource::fetchTextTrack(request, fetcher());
break;
case Resource::ImportResource:
resource = RawResource::fetchImport(request, fetcher());
break;
case Resource::LinkPreload:
resource = RawResource::fetch(request, fetcher());
break;
default:
ASSERT_NOT_REACHED();
}
if (resource)
fetcher()->preloadStarted(resource);
return resource;
}
void MusicBrainzFetcherTest::testTitle() {
Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Album, Tellico::Fetch::Title,
m_fieldValues.value(QLatin1String("title")));
Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::MusicBrainzFetcher(this));
Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1);
QCOMPARE(results.size(), 1);
Tellico::Data::EntryPtr entry = results.at(0);
QHashIterator<QString, QString> i(m_fieldValues);
while(i.hasNext()) {
i.next();
QString result = entry->field(i.key()).toLower();
QCOMPARE(result, i.value().toLower());
}
QVERIFY(!entry->field(QLatin1String("track")).isEmpty());
QVERIFY(!entry->field(QLatin1String("cover")).isEmpty());
}
void EntryUpdateJobTest::testUpdate() {
Tellico::Data::CollPtr coll(new Tellico::Data::BibtexCollection(true));
Tellico::Data::FieldPtr field(new Tellico::Data::Field("arxiv", "Arxiv ID"));
coll->addField(field);
Tellico::Data::EntryPtr entry(new Tellico::Data::Entry(coll));
entry->setField("arxiv", "hep-lat/0110180");
coll->addEntries(entry);
Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::ArxivFetcher(this));
// don't use 'this' as job parent, it crashes
Tellico::EntryUpdateJob* job = new Tellico::EntryUpdateJob(0, entry, fetcher);
connect(job, SIGNAL(result(KJob*)), &m_loop, SLOT(quit()));
job->start();
m_loop.exec();
QCOMPARE(entry->field("title"), QLatin1String("Speeding up the Hybrid-Monte-Carlo algorithm for dynamical fermions"));
}
void DBLPFetcherTest::testProceedings() {
Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Bibtex, Tellico::Fetch::Keyword,
QLatin1String("Chip and PIN is Broken"));
Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::DBLPFetcher(this));
Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1);
QCOMPARE(results.size(), 1);
Tellico::Data::EntryPtr entry = results.at(0);
QCOMPARE(entry->field(QLatin1String("title")), QLatin1String("Chip and PIN is Broken."));
QCOMPARE(entry->field(QLatin1String("author")), QLatin1String("Steven J. Murdoch; Saar Drimer; Ross J. Anderson; Mike Bond"));
QCOMPARE(entry->field(QLatin1String("year")), QLatin1String("2010"));
QCOMPARE(entry->field(QLatin1String("pages")), QLatin1String("433-446"));
QCOMPARE(entry->field(QLatin1String("booktitle")), QLatin1String("IEEE Symposium on Security and Privacy"));
QCOMPARE(entry->field(QLatin1String("url")), QLatin1String("http://dx.doi.org/10.1109/SP.2010.33"));
QCOMPARE(entry->field(QLatin1String("doi")), QLatin1String("10.1109/SP.2010.33"));
QCOMPARE(entry->field(QLatin1String("entry-type")), QLatin1String("inproceedings"));
QCOMPARE(entry->field(QLatin1String("bibtex-key")), QLatin1String("MurdochDAB10"));
}
void GoogleBookFetcherTest::testAuthor() {
Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Book, Tellico::Fetch::Person,
QLatin1String("Shelley Powers"));
Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::GoogleBookFetcher(this));
if(m_hasConfigFile) {
KConfigGroup cg(&m_config, QLatin1String("GoogleBookTest"));
fetcher->readConfig(cg, cg.name());
}
Tellico::Data::EntryList results = DO_FETCH(fetcher, request);
Tellico::Data::EntryPtr entry;
foreach(Tellico::Data::EntryPtr testEntry, results) {
if(testEntry->title() == QLatin1String("Practical RDF")) {
entry = testEntry;
break;
}
}
QVERIFY(entry);
compareEntry(entry);
}
static TnyStream *
tny_fs_stream_cache_get_stream (TnyStreamCache *self, const gchar *id,
TnyStreamCacheOpenStreamFetcher fetcher, gpointer userdata)
{
TnyStream *result = NULL;
TnyFsStreamCachePriv *priv;
TnyCachedFile *cached_file;
g_return_val_if_fail (TNY_IS_FS_STREAM_CACHE (self), NULL);
g_return_val_if_fail (id, NULL);
priv = TNY_FS_STREAM_CACHE_GET_PRIVATE (self);
g_static_mutex_lock (priv->cache_lock);
cached_file = g_hash_table_lookup (priv->cached_files, id);
g_static_mutex_unlock (priv->cache_lock);
if (cached_file) {
result = tny_cached_file_get_stream (cached_file);
} else {
TnyStream *input_stream;
gint64 expected_size = 0;
input_stream = fetcher (self, &expected_size, userdata);
if (input_stream == NULL)
return NULL;
if (expected_size <= get_available_size (TNY_FS_STREAM_CACHE (self))) {
remove_old_files (TNY_FS_STREAM_CACHE (self), expected_size);
cached_file = tny_cached_file_new (TNY_FS_STREAM_CACHE (self), id, expected_size, input_stream);
g_static_mutex_lock (priv->cache_lock);
g_hash_table_insert (priv->cached_files, g_strdup (id), cached_file);
g_static_mutex_unlock (priv->cache_lock);
result = tny_cached_file_get_stream (cached_file);
} else {
result = input_stream;
}
}
return result;
}
void MusicBrainzFetcherTest::testKeyword() {
// the total test case ends up exceeding the throttle limit so pause for a second
QTest::qWait(1000);
Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Album, Tellico::Fetch::Keyword,
m_fieldValues.value(QLatin1String("title")));
Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::MusicBrainzFetcher(this));
Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1);
QCOMPARE(results.size(), 1);
Tellico::Data::EntryPtr entry = results.at(0);
QHashIterator<QString, QString> i(m_fieldValues);
while(i.hasNext()) {
i.next();
QString result = entry->field(i.key()).toLower();
QCOMPARE(result, i.value().toLower());
}
QVERIFY(!entry->field(QLatin1String("track")).isEmpty());
QVERIFY(!entry->field(QLatin1String("cover")).isEmpty());
}
bool XSLStyleSheet::parseString(const String& source)
{
// Parse in a single chunk into an xmlDocPtr
if (!m_stylesheetDocTaken)
xmlFreeDoc(m_stylesheetDoc);
m_stylesheetDocTaken = false;
PageConsole* console = 0;
Frame* frame = ownerDocument()->frame();
if (frame && frame->host())
console = &frame->host()->console();
XMLDocumentParserScope scope(fetcher(), XSLTProcessor::genericErrorFunc, XSLTProcessor::parseErrorFunc, console);
XMLParserInput input(source);
xmlParserCtxtPtr ctxt = xmlCreateMemoryParserCtxt(input.data(), input.size());
if (!ctxt)
return 0;
if (m_parentStyleSheet) {
// The XSL transform may leave the newly-transformed document
// with references to the symbol dictionaries of the style sheet
// and any of its children. XML document disposal can corrupt memory
// if a document uses more than one symbol dictionary, so we
// ensure that all child stylesheets use the same dictionaries as their
// parents.
xmlDictFree(ctxt->dict);
ctxt->dict = m_parentStyleSheet->m_stylesheetDoc->dict;
xmlDictReference(ctxt->dict);
}
m_stylesheetDoc = xmlCtxtReadMemory(ctxt, input.data(), input.size(),
finalURL().string().utf8().data(), input.encoding(),
XML_PARSE_NOENT | XML_PARSE_DTDATTR | XML_PARSE_NOWARNING | XML_PARSE_NOCDATA);
xmlFreeParserCtxt(ctxt);
loadChildSheets();
return m_stylesheetDoc;
}
void AnimenfoFetcherTest::testHachimitsu() {
KConfig config(QFINDTESTDATA("tellicotest.config"), KConfig::SimpleConfig);
QString groupName = QLatin1String("AnimeNfo.com");
if(!config.hasGroup(groupName)) {
QSKIP("This test requires a config file.", SkipAll);
}
KConfigGroup cg(&config, groupName);
Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Video, Tellico::Fetch::Keyword, "Hachimitsu to Clover");
Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::AnimeNfoFetcher(this));
fetcher->readConfig(cg, cg.name());
Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1);
QCOMPARE(results.size(), 1);
// the first entry had better be the right one
Tellico::Data::EntryPtr entry = results.at(0);
QVERIFY(entry);
QCOMPARE(entry->field("title"), QLatin1String("Hachimitsu to Clover"));
QCOMPARE(entry->field("year"), QLatin1String("2005"));
QCOMPARE(entry->field("episodes"), QLatin1String("26"));
QCOMPARE(entry->field("keyword"), QLatin1String("TV"));
QCOMPARE(entry->field("genre"), QLatin1String("Comedy; Drama; Romance"));
QCOMPARE(entry->field("studio"), QLatin1String("J.C.STAFF"));
QCOMPARE(entry->field("origtitle"), QString::fromUtf8("ハチミツとクローバー"));
QCOMPARE(entry->field("director"), QString::fromUtf8("Kasai Kenichi (カサヰ ケンイチ)"));
QCOMPARE(entry->field("writer"), QString::fromUtf8("Kuroda Yosuke (黒田洋介)"));
QCOMPARE(entry->field("alttitle"), QLatin1String("Honey and Clover"));
QCOMPARE(entry->field("animenfo-rating"), QLatin1String("9"));
QVERIFY(entry->field("plot").startsWith(QLatin1String("Takemoto, Mayama, and Morita are students")));
QVERIFY(!entry->field("cover").isEmpty());
QVERIFY(!entry->field("animenfo").isEmpty());
QStringList castList = Tellico::FieldFormat::splitTable(entry->field("cast"));
QCOMPARE(castList.count(), 7);
QCOMPARE(castList.at(0), QString::fromUtf8("Kudo Haruka (工藤晴香)::Hanamoto Hagumi"));
}
void DBLPFetcherTest::testArticle() {
Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Bibtex, Tellico::Fetch::Keyword,
QLatin1String("Nontrivial independent sets of bipartite graphs"));
Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::DBLPFetcher(this));
Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1);
QCOMPARE(results.size(), 1);
Tellico::Data::EntryPtr entry = results.at(0);
QCOMPARE(entry->field(QLatin1String("title")), QLatin1String("Nontrivial independent sets of bipartite graphs and cross-intersecting families."));
QCOMPARE(entry->field(QLatin1String("author")), QLatin1String("Jun Wang; Huajun Zhang"));
QCOMPARE(entry->field(QLatin1String("year")), QLatin1String("2013"));
QCOMPARE(entry->field(QLatin1String("pages")), QLatin1String("129-141"));
QCOMPARE(entry->field(QLatin1String("volume")), QLatin1String("120"));
QCOMPARE(entry->field(QLatin1String("number")), QLatin1String("1"));
QCOMPARE(entry->field(QLatin1String("journal")), QLatin1String("J. Comb. Theory, Ser. A (JCT)"));
QCOMPARE(entry->field(QLatin1String("booktitle")), QLatin1String(""));
QCOMPARE(entry->field(QLatin1String("url")), QLatin1String("http://dx.doi.org/10.1016/j.jcta.2012.07.005"));
QCOMPARE(entry->field(QLatin1String("doi")), QLatin1String("10.1016/j.jcta.2012.07.005"));
QCOMPARE(entry->field(QLatin1String("entry-type")), QLatin1String("article"));
QCOMPARE(entry->field(QLatin1String("bibtex-key")), QLatin1String("WangZ13"));
}
//************************************************************************
//* LyricWiki *
//************************************************************************
pfc::string_list_impl * provider_lyricwiki::lookup(unsigned p_index, metadb_handle_list_cref p_meta, threaded_process_status & p_status, abort_callback & p_abort)
{
TRACK_CALL_TEXT("provider_lyricwiki::lookup");
// Regular Expression Class
CRegexpT<char> regexp;
// Buffer
pfc::string8 buff;
pfc::string_list_impl * str_list = new pfc::string_list_impl;
try
{
// Init fetcher
curl_wrapper_simple fetcher(&m_config_item);
for (t_size i = 0; i < p_meta.get_count(); ++i)
{
if (p_abort.is_aborting())
break;
// Sleep
how_to_sleep(i);
// Clear buff
buff.reset();
const metadb_handle_ptr & p = p_meta.get_item(i);
if (p.is_empty())
{
str_list->add_item("");
continue;
}
// Set progress
pfc::string8_fast path = file_path_canonical(p->get_path());
// add subsong index?
if (p->get_subsong_index() > 0)
{
path.add_string(" /index:");
path.add_string(pfc::format_uint(p->get_subsong_index()));
}
p_status.set_item_path(path);
p_status.set_progress(i + 1, p_meta.get_count());
pfc::string8_fast artist, title;
file_info_impl info;
p->get_info(info);
// Get count of artists
t_size count = info.meta_get_count_by_name("artist");
// Get TITLE
static_api_ptr_t<titleformat_compiler> compiler;
service_ptr_t<titleformat_object> script;
compiler->compile_safe(script, "$replace($caps2(%title%),' ','_')");
p->format_title(NULL, title, script, NULL);
bool found = false;
// Iterate through all artists listed
for (int j = 0; j < count && !found; j++)
{
// Get Artist
artist = info.meta_get("artist", j);
artist.replace_char(' ', '_');
//Fetching from HTTP
// Set HTTP Address
pfc::string8_fast url("http://lyrics.wikia.com/index.php?title=");
//URL = http://lyrics.wikia.com/index.php?title=Blackmore%27s_Night:I_Guess_It_Doesn%27t_Matter_Anymore&action=edit
url += fetcher.quote(artist);
url += ":";
url += fetcher.quote(title);
// Get it now
try
{
fetcher.fetch(url, buff);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_lyrics = "'lyricbox'(?P<instrumental>.*?)</div>(?P<lyrics>.*?)<!--";
// expression for extract lyrics
regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE);
int noGroup = regexp.GetNamedGroupNumber("lyrics");
int instGroup = regexp.GetNamedGroupNumber("instrumental");
MatchResult result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
int nStart = result.GetGroupStart(instGroup);
int nEnd = result.GetGroupEnd(instGroup);
pfc::string8_fast test(buff.get_ptr() + nStart, nEnd-nStart);
if (test.find_first("Instrumental") != -1)
{
found = true;
str_list->add_item("[Instrumental]");
continue;
}
nStart = result.GetGroupStart(noGroup);
nEnd = result.GetGroupEnd(noGroup);
pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart);
convert_html_to_plain(lyric);
if (lyric.get_length() > 0)
{
found = true;
string_helper::remove_beginning_linebreaks(lyric);
string_helper::remove_end_linebreaks(lyric);
str_list->add_item(lyric);
continue;
}
}
}
if (found)
continue;
else
str_list->add_item("");
}
}
catch (pfc::exception & e)
{
console_error(e.what());
delete str_list;
return NULL;
}
catch (...)
{
delete str_list;
return NULL;
}
return str_list;
}
Resource* MemoryCacheCorrectnessTestHelper::fetch() {
ResourceRequest resourceRequest(KURL(ParsedURLString, kResourceURL));
resourceRequest.setRequestContext(WebURLRequest::RequestContextInternal);
FetchRequest fetchRequest(resourceRequest, FetchInitiatorInfo());
return RawResource::fetch(fetchRequest, fetcher());
}
StatusWith<NearStage::CoveredInterval*> //
GeoNear2DSphereStage::nextInterval(OperationContext* txn,
WorkingSet* workingSet,
Collection* collection) {
// The search is finished if we searched at least once and all the way to the edge
if (_currBounds.getInner() >= 0 && _currBounds.getOuter() == _fullBounds.getOuter()) {
return StatusWith<CoveredInterval*>(NULL);
}
//
// Setup the next interval
//
const NearStats* stats = getNearStats();
if (!stats->intervalStats.empty()) {
const IntervalStats& lastIntervalStats = stats->intervalStats.back();
// TODO: Generally we want small numbers of results fast, then larger numbers later
if (lastIntervalStats.numResultsBuffered < 300)
_boundsIncrement *= 2;
else if (lastIntervalStats.numResultsBuffered > 600)
_boundsIncrement /= 2;
}
R2Annulus nextBounds(_currBounds.center(),
_currBounds.getOuter(),
min(_currBounds.getOuter() + _boundsIncrement,
_fullBounds.getOuter()));
bool isLastInterval = (nextBounds.getOuter() == _fullBounds.getOuter());
_currBounds = nextBounds;
//
// Setup the covering region and stages for this interval
//
IndexScanParams scanParams;
scanParams.descriptor = _s2Index;
scanParams.direction = 1;
// We use a filter on the key. The filter rejects keys that don't intersect with the
// annulus. An object that is in the annulus might have a key that's not in it and a key
// that's in it. As such we can't just look at one key per object.
//
// This does force us to do our own deduping of results, though.
scanParams.doNotDedup = true;
scanParams.bounds = _nearParams.baseBounds;
// Because the planner doesn't yet set up 2D index bounds, do it ourselves here
const string s2Field = _nearParams.nearQuery.field;
const int s2FieldPosition = getFieldPosition(_s2Index, s2Field);
scanParams.bounds.fields[s2FieldPosition].intervals.clear();
OrderedIntervalList* coveredIntervals = &scanParams.bounds.fields[s2FieldPosition];
TwoDSphereKeyInRegionExpression* keyMatcher =
new TwoDSphereKeyInRegionExpression(_currBounds, s2Field);
ExpressionMapping::cover2dsphere(keyMatcher->getRegion(),
_s2Index->infoObj(),
coveredIntervals);
// IndexScan owns the hash matcher
IndexScan* scan = new IndexScanWithMatch(txn, scanParams, workingSet, keyMatcher);
// FetchStage owns index scan
FetchStage* fetcher(new FetchStage(workingSet, scan, _nearParams.filter, collection));
return StatusWith<CoveredInterval*>(new CoveredInterval(fetcher,
true,
nextBounds.getInner(),
nextBounds.getOuter(),
isLastInterval));
}
StatusWith<NearStage::CoveredInterval*> //
GeoNear2DStage::nextInterval(OperationContext* txn,
WorkingSet* workingSet,
Collection* collection) {
// The search is finished if we searched at least once and all the way to the edge
if (_currBounds.getInner() >= 0 && _currBounds.getOuter() == _fullBounds.getOuter()) {
return StatusWith<CoveredInterval*>(NULL);
}
//
// Setup the next interval
//
const NearStats* stats = getNearStats();
if (!stats->intervalStats.empty()) {
const IntervalStats& lastIntervalStats = stats->intervalStats.back();
// TODO: Generally we want small numbers of results fast, then larger numbers later
if (lastIntervalStats.numResultsBuffered < 300)
_boundsIncrement *= 2;
else if (lastIntervalStats.numResultsBuffered > 600)
_boundsIncrement /= 2;
}
_boundsIncrement = max(_boundsIncrement,
min2DBoundsIncrement(_nearParams.nearQuery, _twoDIndex));
R2Annulus nextBounds(_currBounds.center(),
_currBounds.getOuter(),
min(_currBounds.getOuter() + _boundsIncrement,
_fullBounds.getOuter()));
const bool isLastInterval = (nextBounds.getOuter() == _fullBounds.getOuter());
_currBounds = nextBounds;
//
// Get a covering region for this interval
//
const CRS queryCRS = _nearParams.nearQuery.centroid.crs;
auto_ptr<R2Region> coverRegion;
if (FLAT == queryCRS) {
// NOTE: Due to floating point math issues, FLAT searches of a 2D index need to treat
// containment and distance separately.
// Ex: (distance) 54.001 - 54 > 0.001, but (containment) 54 + 0.001 <= 54.001
// The idea is that a $near search with bounds is really a $within search, sorted by
// distance. We attach a custom $within : annulus matcher to do the $within search,
// and adjust max/min bounds slightly since, as above, containment does not mean the
// distance calculation won't slightly overflow the boundary.
//
// The code below adjusts:
// 1) Overall min/max bounds of the generated distance intervals to be more inclusive
// 2) Bounds of the interval covering to be more inclusive
// ... and later on we add the custom $within : annulus matcher.
//
// IMPORTANT: The *internal* interval distance bounds are *exact thresholds* - these
// should not be adjusted.
// TODO: Maybe integrate annuluses as a standard shape, and literally transform $near
// internally into a $within query with $near just as sort.
// Compute the maximum axis-aligned distance error
const double epsilon = std::numeric_limits<double>::epsilon()
* (max(abs(_fullBounds.center().x), abs(_fullBounds.center().y))
+ _fullBounds.getOuter());
if (nextBounds.getInner() > 0 && nextBounds.getInner() == _fullBounds.getInner()) {
nextBounds = R2Annulus(nextBounds.center(),
max(0.0, nextBounds.getInner() - epsilon),
nextBounds.getOuter());
}
if (nextBounds.getOuter() > 0 && nextBounds.getOuter() == _fullBounds.getOuter()) {
// We're at the max bound of the search, adjust interval maximum
nextBounds = R2Annulus(nextBounds.center(),
nextBounds.getInner(),
nextBounds.getOuter() + epsilon);
}
// *Always* adjust the covering bounds to be more inclusive
coverRegion.reset(new R2Annulus(nextBounds.center(),
max(0.0, nextBounds.getInner() - epsilon),
nextBounds.getOuter() + epsilon));
}
else {
invariant(SPHERE == queryCRS);
// TODO: As above, make this consistent with $within : $centerSphere
// Our intervals aren't in the same CRS as our index, so we need to adjust them
coverRegion.reset(new R2Annulus(projectBoundsToTwoDDegrees(nextBounds)));
}
//
// Setup the stages for this interval
//
IndexScanParams scanParams;
scanParams.descriptor = _twoDIndex;
scanParams.direction = 1;
// We use a filter on the key. The filter rejects keys that don't intersect with the
// annulus. An object that is in the annulus might have a key that's not in it and a key
// that's in it. As such we can't just look at one key per object.
//
// This does force us to do our own deduping of results, though.
scanParams.doNotDedup = true;
// Scan bounds on 2D indexes are only over the 2D field - other bounds aren't applicable.
// This is handled in query planning.
scanParams.bounds = _nearParams.baseBounds;
// The "2d" field is always the first in the index
const string twoDFieldName = _nearParams.nearQuery.field;
const int twoDFieldPosition = 0;
OrderedIntervalList coveredIntervals;
coveredIntervals.name = scanParams.bounds.fields[twoDFieldPosition].name;
ExpressionMapping::cover2d(*coverRegion,
_twoDIndex->infoObj(),
internalGeoNearQuery2DMaxCoveringCells,
&coveredIntervals);
// Intersect the $near bounds we just generated into the bounds we have for anything else
// in the scan (i.e. $within)
IndexBoundsBuilder::intersectize(coveredIntervals,
&scanParams.bounds.fields[twoDFieldPosition]);
// These parameters are stored by the index, and so must be ok
GeoHashConverter::Parameters hashParams;
GeoHashConverter::parseParameters(_twoDIndex->infoObj(), &hashParams);
MatchExpression* keyMatcher =
new TwoDKeyInRegionExpression(coverRegion.release(),
hashParams,
twoDFieldName);
// 2D indexes support covered search over additional fields they contain
// TODO: Don't need to clone, can just attach to custom matcher above
if (_nearParams.filter) {
AndMatchExpression* andMatcher = new AndMatchExpression();
andMatcher->add(keyMatcher);
andMatcher->add(_nearParams.filter->shallowClone());
keyMatcher = andMatcher;
}
// IndexScanWithMatch owns the matcher
IndexScan* scan = new IndexScanWithMatch(txn, scanParams, workingSet, keyMatcher);
MatchExpression* docMatcher = NULL;
// FLAT searches need to add an additional annulus $within matcher, see above
if (FLAT == queryCRS) {
docMatcher = new TwoDPtInAnnulusExpression(_fullBounds, twoDFieldName);
}
// FetchStage owns index scan
FetchStage* fetcher(new FetchStageWithMatch(workingSet,
scan,
docMatcher,
collection));
return StatusWith<CoveredInterval*>(new CoveredInterval(fetcher,
true,
nextBounds.getInner(),
nextBounds.getOuter(),
isLastInterval));
}
void BackgroundSync::_produce(OperationContext* txn, executor::TaskExecutor* taskExecutor) {
// this oplog reader does not do a handshake because we don't want the server it's syncing
// from to track how far it has synced
{
stdx::unique_lock<stdx::mutex> lock(_mutex);
if (_lastOpTimeFetched.isNull()) {
// then we're initial syncing and we're still waiting for this to be set
lock.unlock();
sleepsecs(1);
// if there is no one to sync from
return;
}
if (_replCoord->isWaitingForApplierToDrain() || _replCoord->getMemberState().primary() ||
inShutdownStrict()) {
return;
}
}
while (MONGO_FAIL_POINT(rsBgSyncProduce)) {
sleepmillis(0);
}
// find a target to sync from the last optime fetched
OpTime lastOpTimeFetched;
{
stdx::unique_lock<stdx::mutex> lock(_mutex);
lastOpTimeFetched = _lastOpTimeFetched;
_syncSourceHost = HostAndPort();
}
OplogReader syncSourceReader;
syncSourceReader.connectToSyncSource(txn, lastOpTimeFetched, _replCoord);
// no server found
if (syncSourceReader.getHost().empty()) {
sleepsecs(1);
// if there is no one to sync from
return;
}
long long lastHashFetched;
{
stdx::lock_guard<stdx::mutex> lock(_mutex);
if (_pause) {
return;
}
lastOpTimeFetched = _lastOpTimeFetched;
lastHashFetched = _lastFetchedHash;
_syncSourceHost = syncSourceReader.getHost();
_replCoord->signalUpstreamUpdater();
}
const Milliseconds oplogSocketTimeout(OplogReader::kSocketTimeout);
// Prefer host in oplog reader to _syncSourceHost because _syncSourceHost may be cleared
// if sync source feedback fails.
const HostAndPort source = syncSourceReader.getHost();
syncSourceReader.resetConnection();
// no more references to oplog reader from here on.
// If this status is not OK after the fetcher returns from wait(),
// proceed to execute rollback
Status remoteOplogStartStatus = Status::OK();
auto fetcherCallback = stdx::bind(&BackgroundSync::_fetcherCallback,
this,
stdx::placeholders::_1,
stdx::placeholders::_3,
stdx::cref(source),
lastOpTimeFetched,
lastHashFetched,
&remoteOplogStartStatus);
auto cmdObj = BSON("find" << nsToCollectionSubstring(rsOplogName) << "filter"
<< BSON("ts" << BSON("$gte" << lastOpTimeFetched.getTimestamp()))
<< "tailable" << true << "oplogReplay" << true << "awaitData" << true
<< "maxTimeMS" << int(fetcherMaxTimeMS.count()));
Fetcher fetcher(taskExecutor,
source,
nsToDatabase(rsOplogName),
cmdObj,
fetcherCallback,
rpc::makeEmptyMetadata());
auto scheduleStatus = fetcher.schedule();
if (!scheduleStatus.isOK()) {
warning() << "unable to schedule fetcher to read remote oplog on " << source << ": "
<< scheduleStatus;
return;
}
fetcher.wait();
// If the background sync is paused after the fetcher is started, we need to
// re-evaluate our sync source and oplog common point.
if (isPaused()) {
return;
}
// Execute rollback if necessary.
// Rollback is a synchronous operation that uses the task executor and may not be
// executed inside the fetcher callback.
if (!remoteOplogStartStatus.isOK()) {
const int messagingPortTags = 0;
ConnectionPool connectionPool(messagingPortTags);
std::unique_ptr<ConnectionPool::ConnectionPtr> connection;
auto getConnection =
[&connection, &connectionPool, oplogSocketTimeout, source]() -> DBClientBase* {
if (!connection.get()) {
connection.reset(new ConnectionPool::ConnectionPtr(
&connectionPool, source, Date_t::now(), oplogSocketTimeout));
};
return connection->get();
};
log() << "starting rollback: " << remoteOplogStartStatus;
_rollback(txn, source, getConnection);
stop();
}
}
pfc::string8 provider_azlyrics::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort)
{
TRACK_CALL_TEXT("provider_azlyrics::lookup_one");
const float threshold = 0.8f;
// Regular Expression Class
CRegexpT<char> regexp;
// Buffer
pfc::string8 buff;
try
{
TRACK_CALL_TEXT("Try");
// Init fetcher
curl_wrapper_simple fetcher(&m_config_item);
const metadb_handle_ptr & p = p_meta;
if (p.is_empty())
{
return "";
}
pfc::string8_fast artist, title, album;
static_api_ptr_t<titleformat_compiler> compiler;
service_ptr_t<titleformat_object> script;
file_info_impl info;
p->get_info(info);
// Get count of artists
t_size count = info.meta_get_count_by_name("artist");
// Get TITLE
title = info.meta_get("title", 0);
// Iterate through all artists listed
for (int j = 0; j < count; j++)
{
TRACK_CALL_TEXT("For");
// Get Artist
artist = info.meta_get("artist", j);
console::printf("%s - %s", artist, title);
// Search the lyrics
pfc::string8_fast url("http://search.azlyrics.com/search.php?q=");
url += fetcher.quote(artist);
url += "+";
url += fetcher.quote(title);
// Get it now
try
{
fetcher.fetch(url, buff);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
if (buff.get_length() == 0)
continue;
int resultStart = buff.find_first("<b>1.</b>");
int startUrl = buff.find_first("<a href=\"", resultStart) + 9;
int endUrl = buff.find_first("\"", startUrl);
url = pfc::string8_fast(buff.get_ptr()+startUrl, endUrl - startUrl);
// Get it now
try
{
fetcher.fetch(url, buff);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
if (buff.get_length() == 0)
continue;
const char * regex_lyrics = "<!-- END OF RINGTONE 1 -->\\s*?<b>\"(?P<title>.*?)\"</b><br>\\s*?<br>\\s\\s(?P<lyrics>.*?)\\s<br>";
// expression for extract lyrics
regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE);
int noGroup = regexp.GetNamedGroupNumber("lyrics");
int noTitle = regexp.GetNamedGroupNumber("title");
MatchResult result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
int nStart = result.GetGroupStart(noTitle);
int nEnd = result.GetGroupEnd(noTitle);
pfc::string8_fast songTitle(buff.get_ptr() +nStart, nEnd - nStart);
int levDist = LD(title, title.get_length(), songTitle, songTitle.get_length());
float good = 1.0f - ((float)levDist / title.get_length());
if (good < threshold)
return "";
nStart = result.GetGroupStart(noGroup);
nEnd = result.GetGroupEnd(noGroup);
pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart);
convert_html_to_plain(lyric);
if (lyric.get_length() > 0)
{
string_helper::remove_end_linebreaks(lyric);
return lyric;
}
}
}
}
catch (pfc::exception & e)
{
console_error(e.what());
return "";
}
catch (...)
{
return "";
}
return "";
}
//************************************************************************
//* AZ Lyrics *
//************************************************************************
pfc::string_list_impl * provider_azlyrics::lookup(unsigned p_index, metadb_handle_list_cref p_meta, threaded_process_status & p_status, abort_callback & p_abort)
{
TRACK_CALL_TEXT("provider_azlyrics::lookup");
const float threshold = 0.8f;
// Regular Expression Class
CRegexpT<char> regexp;
// Buffer
pfc::string8 buff;
pfc::string_list_impl * str_list = new pfc::string_list_impl;
try
{
// Init fetcher
curl_wrapper_simple fetcher(&m_config_item);
for (t_size i = 0; i < p_meta.get_count(); ++i)
{
if (p_abort.is_aborting())
break;
// Sleep
how_to_sleep(i);
// Clear buff
buff.reset();
const metadb_handle_ptr & p = p_meta.get_item(i);
if (p.is_empty())
{
str_list->add_item("");
continue;
}
// Set progress
pfc::string8_fast path = file_path_canonical(p->get_path());
// add subsong index?
if (p->get_subsong_index() > 0)
{
path.add_string(" /index:");
path.add_string(pfc::format_uint(p->get_subsong_index()));
}
p_status.set_item_path(path);
p_status.set_progress(i + 1, p_meta.get_count());
pfc::string8_fast artist, title;
file_info_impl info;
p->get_info(info);
// Get count of artists
t_size count = info.meta_get_count_by_name("artist");
// Get TITLE
title = info.meta_get("title", 0);
bool found = false;
// Iterate through all artists listed
for (int j = 0; j < count && !found; j++)
{
// Get Artist
artist = info.meta_get("artist", j);
// Search the lyrics
pfc::string8_fast url("http://search.azlyrics.com/search.php?q=");
url += fetcher.quote(artist);
url += "+";
url += fetcher.quote(title);
// Get it now
try
{
fetcher.fetch(url, buff);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
int resultStart = buff.find_first("<b>1.</b>");
int startUrl = buff.find_first("<a href=\"", resultStart) + 9;
int endUrl = buff.find_first("\"", startUrl);
url = pfc::string8_fast(buff.get_ptr()+startUrl, endUrl - startUrl);
// Get it now
try
{
fetcher.fetch(url, buff);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_lyrics = "<!-- END OF RINGTONE 1 -->\\s*?<b>\"(?P<title>.*?)\"</b><br>\\s*?<br>\\s\\s(?P<lyrics>.*?)\\s<br>";
// expression for extract lyrics
regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE);
int noGroup = regexp.GetNamedGroupNumber("lyrics");
int noTitle = regexp.GetNamedGroupNumber("title");
// match
MatchResult result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
int nStart = result.GetGroupStart(noTitle);
int nEnd = result.GetGroupEnd(noTitle);
pfc::string8_fast songTitle(buff.get_ptr() +nStart, nEnd - nStart);
int levDist = LD(title, title.get_length(), songTitle, songTitle.get_length());
float good = 1.0f - ((float)levDist / title.get_length());
if (good < threshold)
continue;
nStart = result.GetGroupStart(noGroup);
nEnd = result.GetGroupEnd(noGroup);
pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart);
convert_html_to_plain(lyric);
if (lyric.get_length() > 0)
{
string_helper::remove_end_linebreaks(lyric);
str_list->add_item(lyric);
found = true;
continue;
}
}
}
if (found)
continue;
else
str_list->add_item("");
}
}
catch (pfc::exception & e)
{
console_error(e.what());
delete str_list;
return NULL;
}
catch (...)
{
delete str_list;
return NULL;
}
return str_list;
}
pfc::string8 provider_darklyrics::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort)
{
TRACK_CALL_TEXT("provider_darklyrics::lookup_one");
const float threshold = 0.8f;
const pfc::string8 site = "darklyrics.com";
// Regular Expression Class
CRegexpT<char> regexp;
MatchResult match;
// Buffer
pfc::string8 buff;
try
{
// Init fetcher
curl_wrapper_simple fetcher(&m_config_item);
const metadb_handle_ptr & p = p_meta;
if (p.is_empty())
{
return "";
}
pfc::string8_fast artist, title, album, keywords;
file_info_impl info;
p->get_info(info);
// Get count of artists
t_size count = info.meta_get_count_by_name("album");
if (count == 0)
return "";
// Get Album
album = info.meta_get("album", 0);
count = info.meta_get_count_by_name("title");
if (count == 0)
return "";
// Get TITLE
title = info.meta_get("title", 0);
count = info.meta_get_count_by_name("artist");
// Iterate through all artists listed
for (int j = 0; j < count; j++)
{
// Get Artist
artist = info.meta_get("artist", j); //Fetching from HTTP
keywords = artist;
keywords += "+";
keywords += album;
keywords.replace_char(' ', '+');
// Get it now
try
{
fetcher.fetch_googleluck(site, keywords, buff);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_ahref = "<a\\shref=\"#(?P<no>\\d+)\">(?P<text>.+?)</a>";
// expression for extract lyrics
regexp.Compile(regex_ahref, IGNORECASE);
// match
MatchResult result = regexp.Match(buff.get_ptr());
int noGroup = regexp.GetNamedGroupNumber("no");
int textGroup = regexp.GetNamedGroupNumber("text");
int jump_to = 0;
pfc::string8_fast compare = title;
compare.insert_chars(0, ". ");
float good;
float best = 0.0f;
while (result.IsMatched())
{
int gStart = result.GetGroupStart(noGroup);
int gEnd = result.GetGroupEnd(noGroup);
pfc::string8_fast temp(buff.get_ptr() + gStart, gEnd - gStart);
int no = StrToIntA(temp);
gStart = result.GetGroupStart(textGroup);
gEnd = result.GetGroupEnd(textGroup);
temp = pfc::string8_fast(buff.get_ptr()+gStart, gEnd - gStart);
int levDist = LD(compare, compare.get_length(), temp, temp.get_length());
good = 1.0f - (levDist / (float)compare.get_length());
if (good >= threshold && good > best)
{
jump_to = no;
best = good;
}
result = regexp.Match(buff.get_ptr(),result.GetEnd());
}
if (jump_to == 0)
{
continue;
}
char regex_lyrics[100];
sprintf(regex_lyrics, "<a\\s+name=%d><font*(.*?)</font*(.*?)>(?P<lyrics>.+?)<font", jump_to);
// expression for extract lyrics
regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE);
noGroup = regexp.GetNamedGroupNumber("lyrics");
result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
int nStart = result.GetGroupStart(noGroup);
int nEnd = result.GetGroupEnd(noGroup);
pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart);
convert_html_to_plain(lyric);
if (lyric.get_length() > 0)
{
string_helper::remove_beginning_linebreaks(lyric);
string_helper::remove_end_linebreaks(lyric);
string_helper::remove_beginning(lyric, ' ');
string_helper::remove_beginning(lyric, '\t');
return lyric;
}
}
}
}
catch (pfc::exception & e)
{
console_error(e.what());
return "";
}
catch (...)
{
return "";
}
return "";
}
//************************************************************************
//* Dark Lyrics *
//************************************************************************
pfc::string_list_impl * provider_darklyrics::lookup(unsigned p_index, metadb_handle_list_cref p_meta, threaded_process_status & p_status, abort_callback & p_abort)
{
TRACK_CALL_TEXT("provider_darklyrics::lookup");
const float threshold = 0.8f;
const pfc::string8 site = "darklyrics.com";
// Regular Expression Class
CRegexpT<char> regexp;
MatchResult match;
// Buffer
pfc::string8 buff;
pfc::string_list_impl * str_list = new pfc::string_list_impl;
try
{
// Init fetcher
curl_wrapper_simple fetcher(&m_config_item);
for (t_size i = 0; i < p_meta.get_count(); ++i)
{
if (p_abort.is_aborting())
break;
// Sleep
how_to_sleep(i);
// Clear buff
buff.reset();
const metadb_handle_ptr & p = p_meta.get_item(i);
if (p.is_empty())
{
str_list->add_item("");
continue;
}
// Set progress
pfc::string8_fast path = file_path_canonical(p->get_path());
// add subsong index?
if (p->get_subsong_index() > 0)
{
path.add_string(" /index:");
path.add_string(pfc::format_uint(p->get_subsong_index()));
}
p_status.set_item_path(path);
p_status.set_progress(i + 1, p_meta.get_count());
pfc::string8_fast artist, title, album, keywords;
file_info_impl info;
p->get_info(info);
// Get count of artists
t_size count = info.meta_get_count_by_name("album");
if (count == 0)
continue;
// Get Album
album = info.meta_get("album", 0);
count = info.meta_get_count_by_name("title");
if (count == 0)
continue;
// Get TITLE
title = info.meta_get("title", 0);
count = info.meta_get_count_by_name("artist");
bool found = false;
// Iterate through all artists listed
for (int j = 0; j < count && !found; j++)
{
// Get Artist
artist = info.meta_get("artist", j);
keywords = artist;
keywords += "+";
keywords += album;
keywords.replace_char(' ', '+');
// Get it now
try
{
fetcher.fetch_googleluck(site, keywords, buff);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_ahref = "<a\\shref=\"#(?P<no>\\d+)\">(?P<text>.+?)</a>";
// expression for extract lyrics
regexp.Compile(regex_ahref, IGNORECASE | SINGLELINE);
// match
MatchResult result = regexp.Match(buff.get_ptr());
int noGroup = regexp.GetNamedGroupNumber("no");
int textGroup = regexp.GetNamedGroupNumber("text");
int jump_to = 0;
pfc::string8_fast compare = title;
compare.insert_chars(0, ". ");
float good;
float best = 0.0f;
while (result.IsMatched())
{
int gStart = result.GetGroupStart(noGroup);
int gEnd = result.GetGroupEnd(noGroup);
pfc::string8_fast temp(buff.get_ptr() + gStart, gEnd - gStart);
int no = StrToIntA(temp);
gStart = result.GetGroupStart(textGroup);
gEnd = result.GetGroupEnd(textGroup);
temp = pfc::string8_fast(buff.get_ptr()+gStart, gEnd - gStart);
if (temp.find_first(title) != -1)
{
jump_to = no;
break;
}
int levDist = LD(compare, compare.get_length(), temp, temp.get_length());
good = 1.0f - (levDist / (float)compare.get_length());
if (good >= threshold && good > best)
{
jump_to = no;
best = good;
}
result = regexp.Match(buff.get_ptr(),result.GetEnd());
}
if (jump_to == 0)
{
continue;
}
char regex_lyrics[100];
sprintf(regex_lyrics, "<a\\s+name=%d><font*(.*?)</font*(.*?)>(?P<lyrics>.+?)<font", jump_to);
// expression for extract lyrics
regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE);
noGroup = regexp.GetNamedGroupNumber("lyrics");
result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
int nStart = result.GetGroupStart(noGroup);
int nEnd = result.GetGroupEnd(noGroup);
pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart);
convert_html_to_plain(lyric);
if (lyric.get_length() > 0)
{
string_helper::remove_beginning_linebreaks(lyric);
string_helper::remove_end_linebreaks(lyric);
string_helper::remove_beginning(lyric, ' ');
string_helper::remove_beginning(lyric, '\t');
console::print(lyric);
str_list->add_item(lyric);
found = true;
continue;
}
}
}
if (found)
continue;
else
str_list->add_item("");
}
}
catch (pfc::exception & e)
{
console_error(e.what());
delete str_list;
return NULL;
}
catch (...)
{
delete str_list;
return NULL;
}
return str_list;
}
pfc::string8 provider_lyricsplugin::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort)
{
// Regular Expression Class
CRegexpT<char> regexp;
MatchResult match;
// Buffer
pfc::string8 buff;
try
{
// Init fetcher
curl_wrapper_simple fetcher(&m_config_item);
// Clear buff
buff.reset();
const metadb_handle_ptr & p = p_meta;
if (p.is_empty())
{
return "";
}
pfc::string8_fast artist, title;
static_api_ptr_t<titleformat_compiler> compiler;
service_ptr_t<titleformat_object> script;
file_info_impl info;
p->get_info(info);
// Get count of artists
t_size count = info.meta_get_count_by_name("artist");
// Get TITLE
compiler->compile_safe(script, "[%title%]");
p->format_title(NULL, title, script, NULL);
bool found = false;
// Iterate through all artists listed
for (int j = 0; j < count && !found; j++)
{
// Get Artist
artist = info.meta_get("artist", j);
// Fetching from HTTP
// Set HTTP Address
pfc::string8_fast url("http://www.squirrelscript.net/mediamonkey/Lyricator/lyrics.php?artist=");
// URL = http://www.squirrelscript.net/mediamonkey/Lyricator/lyrics.php?artist=<Artist>&title=<Title>
url += fetcher.quote(artist);
url += "&title=";
url += fetcher.quote(title);
// Get it now
try
{
fetcher.fetch(url, buff);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_lyricbox = "<div\\s+id\\s*?=\\s*?\"lyrics\"\\s*?>[\\r\\n]*(.*?)[\\r\\n]*</div>";
// expression for extract lyrics
regexp.Compile(regex_lyricbox, SINGLELINE);
// match
MatchResult result = regexp.Match(buff.get_ptr());
// Get Group
if (result.IsMatched())
{
int nStart = result.GetGroupStart(1);
int nEnd = result.GetGroupEnd(1);
int index;
pfc::string8_fast lyric(buff.get_ptr() + nStart, nEnd - nStart);
convert_html_to_plain(lyric);
index = lyric.find_first("www.tunerankings.com");
if (index == 0)
{
continue;
}
else if (index != -1)
{
lyric.remove_chars(index, 20);
}
if (string_trim(lyric).get_length() > 0)
{
return lyric;
}
}
}
}
catch (pfc::exception & e)
{
console_error(e.what());
return "";
}
catch (...)
{
return "";
}
return "";
}
pfc::string8 provider_leoslyrics::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort)
{
TRACK_CALL_TEXT("provider_leoslyrics::lookup_one");
// Regular Expression Class
CRegexpT<char> regexp;
// Buffer
pfc::string8 buff;
try
{
// Init fetcher
curl_wrapper_simple fetcher(&m_config_item);
const metadb_handle_ptr & p = p_meta;
if (p.is_empty())
{
return "";
}
pfc::string8_fast artist, title, album;
file_info_impl info;
p->get_info(info);
// Get count of artists
t_size count = info.meta_get_count_by_name("artist");
// Get TITLE
static_api_ptr_t<titleformat_compiler> compiler;
service_ptr_t<titleformat_object> script;
compiler->compile_safe(script, "%title%");
p->format_title(NULL, title, script, NULL);
// Iterate through all artists listed
for (int j = 0; j < count; j++)
{
// Get Artist
artist = info.meta_get("artist", j);
//Fetching from HTTP
// Set HTTP Address
pfc::string8_fast url("http://77.79.210.222/api_search.php?auth=LeosLyrics5&artist=");
pfc::string8_fast host("api.leoslyrics.com");
//URL = http://77.79.210.222/api_search.php?auth=LeosLyrics5&artist=<artist>&songtitle=<title>
url += fetcher.quote(artist);
url += "&songtitle=";
url += fetcher.quote(title);
// Get it now
try
{
fetcher.fetch_host(host, url, buff, p_abort);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_hid = "code=\"(?P<code>.).*?hid=\"(?P<hid>.*?)\"";
// expression for extract lyrics
regexp.Compile(regex_hid, IGNORECASE | SINGLELINE);
pfc::string8_fast hid;
int codeGroup = regexp.GetNamedGroupNumber("code");
int hidGroup = regexp.GetNamedGroupNumber("hid");
MatchResult result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
int nStart = result.GetGroupStart(codeGroup);
int nEnd = result.GetGroupEnd(codeGroup);
pfc::string8_fast code(buff.get_ptr() + nStart, nEnd-nStart);
if (code.find_first("0") == -1)
continue;
nStart = result.GetGroupStart(hidGroup);
nEnd = result.GetGroupEnd(hidGroup);
hid = pfc::string8_fast(buff.get_ptr() + nStart, nEnd - nStart);
url = "http://77.79.210.222/api_search.php?auth=LeosLyrics5&hid=";
url += hid;
//URL = http://77.79.210.222/api_search.php?auth=LeosLyrics5&hid=<songID>
try
{
fetcher.fetch_host(host, url, buff, p_abort);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_hid = "<text>\\s(?P<lyrics>.*?)\\s</text>";
// expression for extract lyrics
regexp.Compile(regex_hid, IGNORECASE | SINGLELINE);
int lyricsGroup = regexp.GetNamedGroupNumber("lyrics");
result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
nStart = result.GetGroupStart(lyricsGroup);
nEnd = result.GetGroupEnd(lyricsGroup);
pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart);
if (lyric.get_length() > 0)
{
string_helper::remove_beginning_linebreaks(lyric);
string_helper::remove_end_linebreaks(lyric);
return lyric;
}
}
}
}
}
catch (pfc::exception & e)
{
console_error(e.what());
return "";
}
catch (...)
{
return "";
}
return "";
}
//************************************************************************
//* Leo's Lyrics *
//************************************************************************
pfc::string_list_impl * provider_leoslyrics::lookup(unsigned p_index, metadb_handle_list_cref p_meta, threaded_process_status & p_status, abort_callback & p_abort)
{
TRACK_CALL_TEXT("provider_leoslyrics::lookup");
// Regular Expression Class
CRegexpT<char> regexp;
// Buffer
pfc::string8 buff;
pfc::string_list_impl * str_list = new pfc::string_list_impl;
try
{
// Init fetcher
curl_wrapper_simple fetcher(&m_config_item);
for (t_size i = 0; i < p_meta.get_count(); ++i)
{
if (p_abort.is_aborting())
break;
// Sleep
how_to_sleep(i);
// Clear buff
buff.reset();
const metadb_handle_ptr & p = p_meta.get_item(i);
if (p.is_empty())
{
str_list->add_item("");
continue;
}
// Set progress
pfc::string8_fast path = file_path_canonical(p->get_path());
// add subsong index?
if (p->get_subsong_index() > 0)
{
path.add_string(" /index:");
path.add_string(pfc::format_uint(p->get_subsong_index()));
}
p_status.set_item_path(path);
p_status.set_progress(i + 1, p_meta.get_count());
pfc::string8_fast artist, title;
file_info_impl info;
p->get_info(info);
// Get count of artists
t_size count = info.meta_get_count_by_name("artist");
// Get TITLE
static_api_ptr_t<titleformat_compiler> compiler;
service_ptr_t<titleformat_object> script;
compiler->compile_safe(script, "%title%");
p->format_title(NULL, title, script, NULL);
bool found = false;
// Iterate through all artists listed
for (int j = 0; j < count && !found; j++)
{
// Get Artist
artist = info.meta_get("artist", j);
pfc::string8_fast url("http://77.79.210.222/api_search.php?auth=LeosLyrics5&artist=");
pfc::string8_fast host("api.leoslyrics.com");
//URL = http://77.79.210.222/api_search.php?auth=LeosLyrics5&artist=<artist>&songtitle=<title>
url += fetcher.quote(artist);
url += "&songtitle=";
url += fetcher.quote(title);
// Get it now
try
{
fetcher.fetch_host(host, url, buff, p_abort);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_hid = "code=\"(?P<code>.).*?hid=\"(?P<hid>.*?)\"";
// expression for extract lyrics
regexp.Compile(regex_hid, IGNORECASE | SINGLELINE);
pfc::string8_fast hid;
int codeGroup = regexp.GetNamedGroupNumber("code");
int hidGroup = regexp.GetNamedGroupNumber("hid");
MatchResult result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
int nStart = result.GetGroupStart(codeGroup);
int nEnd = result.GetGroupEnd(codeGroup);
pfc::string8_fast code(buff.get_ptr() + nStart, nEnd-nStart);
if (code.find_first("0") == -1)
continue;
nStart = result.GetGroupStart(hidGroup);
nEnd = result.GetGroupEnd(hidGroup);
hid = pfc::string8_fast(buff.get_ptr() + nStart, nEnd - nStart);
url = "http://77.79.210.222/api_search.php?auth=LeosLyrics5&hid=";
url += hid;
//URL = http://77.79.210.222/api_search.php?auth=LeosLyrics5&hid=<songID>
try
{
fetcher.fetch_host(host, url, buff, p_abort);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_hid = "<text>\\s(?P<lyrics>.*?)\\s</text>";
// expression for extract lyrics
regexp.Compile(regex_hid, IGNORECASE | SINGLELINE);
int lyricsGroup = regexp.GetNamedGroupNumber("lyrics");
result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
nStart = result.GetGroupStart(lyricsGroup);
nEnd = result.GetGroupEnd(lyricsGroup);
pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart);
if (lyric.get_length() > 0)
{
found = true;
string_helper::remove_beginning_linebreaks(lyric);
string_helper::remove_end_linebreaks(lyric);
str_list->add_item(lyric);
continue;
}
}
}
}
if (found)
continue;
else
str_list->add_item("");
}
}
catch (pfc::exception & e)
{
console_error(e.what());
delete str_list;
return NULL;
}
catch (...)
{
delete str_list;
return NULL;
}
return str_list;
}
pfc::string8 provider_lyricwiki::lookup_one(unsigned p_index, const metadb_handle_ptr & p_meta, threaded_process_status & p_status, abort_callback & p_abort)
{
TRACK_CALL_TEXT("provider_lyricwiki::lookup_one");
// Regular Expression Class
CRegexpT<char> regexp;
// Buffer
pfc::string8 buff;
try
{
// Init fetcher
curl_wrapper_simple fetcher(&m_config_item);
const metadb_handle_ptr & p = p_meta;
if (p.is_empty())
{
return "";
}
pfc::string8_fast artist, title, album;
file_info_impl info;
p->get_info(info);
// Get count of artists
t_size count = info.meta_get_count_by_name("artist");
// Get TITLE
static_api_ptr_t<titleformat_compiler> compiler;
service_ptr_t<titleformat_object> script;
compiler->compile_safe(script, "$replace($caps2(%title%),' ','_')");
p->format_title(NULL, title, script, NULL);
// Iterate through all artists listed
for (int j = 0; j < count; j++)
{
// Get Artist
artist = info.meta_get("artist", j);
artist.replace_char(' ', '_');
//Fetching from HTTP
// Set HTTP Address
pfc::string8_fast url("http://lyrics.wikia.com/index.php?title=");
//URL = http://lyrics.wikia.com/index.php?title=Blackmore%27s_Night:I_Guess_It_Doesn%27t_Matter_Anymore&action=edit
url += fetcher.quote(artist);
url += ":";
url += fetcher.quote(title);
// Get it now
try
{
fetcher.fetch(url, buff);
}
catch (pfc::exception & e)
{
console_error(e.what());
continue;
}
catch (...)
{
continue;
}
const char * regex_lyrics = "'lyricbox'(?P<instrumental>.*?)</div>(?P<lyrics>.*?)<!--";
// expression for extract lyrics
regexp.Compile(regex_lyrics, IGNORECASE | SINGLELINE);
int noGroup = regexp.GetNamedGroupNumber("lyrics");
int instGroup = regexp.GetNamedGroupNumber("instrumental");
MatchResult result = regexp.Match(buff.get_ptr());
if (result.IsMatched())
{
int nStart = result.GetGroupStart(instGroup);
int nEnd = result.GetGroupEnd(instGroup);
pfc::string8_fast test(buff.get_ptr() + nStart, nEnd-nStart);
if (test.find_first("Instrumental") != -1)
{
return "[Instrumental]";
}
nStart = result.GetGroupStart(noGroup);
nEnd = result.GetGroupEnd(noGroup);
pfc::string8 lyric(buff.get_ptr() + nStart, nEnd - nStart);
convert_html_to_plain(lyric);
if (lyric.get_length() > 0)
{
string_helper::remove_beginning_linebreaks(lyric);
string_helper::remove_end_linebreaks(lyric);
return lyric;
}
}
}
}
catch (pfc::exception & e)
{
console_error(e.what());
return "";
}
catch (...)
{
return "";
}
return "";
}
