static void BuildIndices(rssDatabase *db, const char *feedsFileURL)
{
url u;
urlconnection urlconn;
URLNewAbsolute(&u, feedsFileURL);
URLConnectionNew(&urlconn, &u);
if (urlconn.responseCode / 100 == 3) {
BuildIndices(db, urlconn.newUrl);
} else {
streamtokenizer st;
char remoteFileName[2048];
HashSetNew(&db->indices, sizeof(rssIndexEntry), kNumIndexEntryBuckets, IndexEntryHash, IndexEntryCompare, IndexEntryFree);
VectorNew(&db->previouslySeenArticles, sizeof(rssNewsArticle), NewsArticleFree, 0);
STNew(&st, urlconn.dataStream, kNewLineDelimiters, true);
while (STSkipUntil(&st, ":") != EOF) { // ignore everything up to the first selicolon of the line
STSkipOver(&st, ": "); // now ignore the semicolon and any whitespace directly after it
STNextToken(&st, remoteFileName, sizeof(remoteFileName));
ProcessFeed(db, remoteFileName);
}
printf("\n");
STDispose(&st);
}
URLConnectionDispose(&urlconn);
URLDispose(&u);
}
static void BuildIndices(const char *feedsFileURL ,rssFeedData * data)
{
url u;
urlconnection urlconn;
URLNewAbsolute(&u, feedsFileURL);
URLConnectionNew(&urlconn, &u);
if (urlconn.responseCode / 100 == 3) { // redirection, so recurse
BuildIndices(urlconn.newUrl, data);
} else {
streamtokenizer st;
char remoteDocumentURL[2048];
STNew(&st, urlconn.dataStream, kNewLineDelimiters, true);
while (STSkipUntil(&st, ":") != EOF) { // ignore everything up to the first selicolon of the line
STSkipOver(&st, ": "); // now ignore the semicolon and any whitespace directly after it
STNextToken(&st, remoteDocumentURL, sizeof(remoteDocumentURL));
ProcessFeed(remoteDocumentURL,data);
}
printf("\n");
STDispose(&st);
}
URLConnectionDispose(&urlconn);
URLDispose(&u);
}
static void ExtractElement(streamtokenizer *st, const char *htmlTag, char dataBuffer[], int bufferLength)
{
assert(htmlTag[strlen(htmlTag) - 1] == '>');
if (htmlTag[strlen(htmlTag) - 2] == '/') return; // e.g. <description/> would state that a description is not being supplied
STNextTokenUsingDifferentDelimiters(st, dataBuffer, bufferLength, "<");
RemoveEscapeCharacters(dataBuffer);
if (dataBuffer[0] == '<') strcpy(dataBuffer, ""); // e.g. <description></description> also means there's no description
STSkipUntil(st, ">");
STSkipOver(st, ">");
}
static void BuildIndices(const char *feedsFileName, rssData *allData )
{
FILE *infile;
streamtokenizer st;
char remoteFileName[1024];
infile = fopen(feedsFileName, "r");
assert(infile != NULL);
STNew(&st, infile, kNewLineDelimiters, true);
while (STSkipUntil(&st, ":") != EOF) { // ignore everything up to the first selicolon of the line
STSkipOver(&st, ": "); // now ignore the semicolon and any whitespace directly after it
STNextToken(&st, remoteFileName, sizeof(remoteFileName));
ProcessFeed(remoteFileName, allData );
}
STDispose(&st);
fclose(infile);
printf("\n");
}
