static void BuildIndices(rssDatabase *db, const char *feedsFileURL)
{
url u;
urlconnection urlconn;
URLNewAbsolute(&u, feedsFileURL);
URLConnectionNew(&urlconn, &u);
if (urlconn.responseCode / 100 == 3) {
BuildIndices(db, urlconn.newUrl);
} else {
streamtokenizer st;
char remoteFileName[2048];
HashSetNew(&db->indices, sizeof(rssIndexEntry), kNumIndexEntryBuckets, IndexEntryHash, IndexEntryCompare, IndexEntryFree);
VectorNew(&db->previouslySeenArticles, sizeof(rssNewsArticle), NewsArticleFree, 0);
STNew(&st, urlconn.dataStream, kNewLineDelimiters, true);
while (STSkipUntil(&st, ":") != EOF) { // ignore everything up to the first selicolon of the line
STSkipOver(&st, ": "); // now ignore the semicolon and any whitespace directly after it
STNextToken(&st, remoteFileName, sizeof(remoteFileName));
ProcessFeed(db, remoteFileName);
}
printf("\n");
STDispose(&st);
}
URLConnectionDispose(&urlconn);
URLDispose(&u);
}
static void BuildIndices(const char *feedsFileURL ,rssFeedData * data)
{
url u;
urlconnection urlconn;
URLNewAbsolute(&u, feedsFileURL);
URLConnectionNew(&urlconn, &u);
if (urlconn.responseCode / 100 == 3) { // redirection, so recurse
BuildIndices(urlconn.newUrl, data);
} else {
streamtokenizer st;
char remoteDocumentURL[2048];
STNew(&st, urlconn.dataStream, kNewLineDelimiters, true);
while (STSkipUntil(&st, ":") != EOF) { // ignore everything up to the first selicolon of the line
STSkipOver(&st, ": "); // now ignore the semicolon and any whitespace directly after it
STNextToken(&st, remoteDocumentURL, sizeof(remoteDocumentURL));
ProcessFeed(remoteDocumentURL,data);
}
printf("\n");
STDispose(&st);
}
URLConnectionDispose(&urlconn);
URLDispose(&u);
}
static void ProcessFeed(const char *remoteDocumentName, hashset*stopWords,
hashset* wordHash, hashset *articlesSeen)
{
url u;
urlconnection urlconn;
URLNewAbsolute(&u, remoteDocumentName);
URLConnectionNew(&urlconn, &u);
switch (urlconn.responseCode) {
case 0: printf("Unable to connect to \"%s\". Ignoring...", u.serverName);
break;
case 200: PullAllNewsItems(&urlconn, stopWords, wordHash,articlesSeen);
break;
case 301:
case 302:
ProcessFeed(urlconn.newUrl, stopWords, wordHash,articlesSeen);
break;
default: printf("Connection to \"%s\" was established, but unable to retrieve \"%s\". [response code: %d, response message:\"%s\"]\n", u.serverName, u.fileName, urlconn.responseCode, urlconn.responseMessage);
break;
};
URLConnectionDispose(&urlconn);
URLDispose(&u);
}
static void BuildIndices(const char *feedsFileName, rssData *allData )
{
FILE *infile;
streamtokenizer st;
char remoteFileName[1024];
infile = fopen(feedsFileName, "r");
assert(infile != NULL);
STNew(&st, infile, kNewLineDelimiters, true);
while (STSkipUntil(&st, ":") != EOF) { // ignore everything up to the first selicolon of the line
STSkipOver(&st, ": "); // now ignore the semicolon and any whitespace directly after it
STNextToken(&st, remoteFileName, sizeof(remoteFileName));
ProcessFeed(remoteFileName, allData );
}
STDispose(&st);
fclose(infile);
printf("\n");
}
static void *process_feed(void *input)
{
ProcessFeed(ProcessMessage);
m9();
}
static unsigned __stdcall process_feed(void *ptr)
{
ProcessFeed(ProcessMessage);
}