Ejemplo n.º 1
0
static void ScanArticle(streamtokenizer *st, void* userData)
{
  rssFeedData *data = userData;
  articleData* article = AddArticle(&data->articles,&data->item);
  
  int numWords = 0;
  char word[1024];
  char longestWord[1024] = {'\0'};

  while (STNextToken(st, word, sizeof(word))) {
    if (strcasecmp(word, "<") == 0) {
      SkipIrrelevantContent(st); // in html-utls.h
    } else {
      RemoveEscapeCharacters(word);
      if (WordIsWellFormed(word)) {
	char* dummy = word;//need this becouse cant do &word in c
	if(HashSetLookup(&data->stopWords,&dummy)==NULL){// skip stopwords

	  indexData *entry = addWordRecord(&data->indices, word);
	  indexWord(&entry->data,article);
	  
	  numWords++;
	  if (strlen(word) > strlen(longestWord))
	    strcpy(longestWord, word);
	}
      }
    }
  }

  printf("\tWe counted %d well-formed words [including duplicates].\n", numWords);
  printf("\tThe longest word scanned was \"%s\".", longestWord);
  if (strlen(longestWord) >= 15 && (strchr(longestWord, '-') == NULL)) 
    printf(" [Ooooo... long word!]");
  printf("\n");
}
Ejemplo n.º 2
0
static void ScanArticle(streamtokenizer *st, const char *articleTitle, const char *unused, const char *articleURL,
                        hashset *stopWords, hashset *wordCounts)
{
  int numWords = 0;
  char word[1024];
  char longestWord[1024] = {'\0'};

  while (STNextToken(st, word, sizeof(word))) {
    if (strcasecmp(word, "<") == 0) {
      SkipIrrelevantContent(st); // in html-utls.h
    } else {
      RemoveEscapeCharacters(word);
      char *dummy = word;
      void *pos = HashSetLookup(stopWords, &dummy);
      if (WordIsWellFormed(word) && pos == NULL) {
          //numWords++;
          //if (strlen(word) > strlen(longestWord))
          //  strcpy(longestWord, word);
          wordCountEnter(wordCounts, word, articleURL, articleTitle);
      }
    }
  }

  /*printf("\tWe counted %d well-formed words [including duplicates].\n", numWords);
  printf("\tThe longest word scanned was \"%s\".", longestWord);
  if (strlen(longestWord) >= 15 && (strchr(longestWord, '-') == NULL)) 
    printf(" [Ooooo... long word!]");
    printf("\n");*/
}
Ejemplo n.º 3
0
static void ExtractElement(streamtokenizer *st, const char *htmlTag, char dataBuffer[], int bufferLength)
{
  assert(htmlTag[strlen(htmlTag) - 1] == '>');
  if (htmlTag[strlen(htmlTag) - 2] == '/') return;    // e.g. <description/> would state that a description is not being supplied
  STNextTokenUsingDifferentDelimiters(st, dataBuffer, bufferLength, "<");
  RemoveEscapeCharacters(dataBuffer);
  if (dataBuffer[0] == '<') strcpy(dataBuffer, "");  // e.g. <description></description> also means there's no description
  STSkipUntil(st, ">");
  STSkipOver(st, ">");
}
Ejemplo n.º 4
0
static void ScanArticle(streamtokenizer *st, article *a, int articleIndex, rssData *allData )
{
  int numWords = 0;
  char word[1024];
  char longestWord[1024] = {'\0'};

  while (STNextToken(st, word, sizeof(word))) {
    if (strcasecmp(word, "<") == 0) {
      SkipIrrelevantContent(st); // in html-utls.h
    } else {
      RemoveEscapeCharacters(word);
      if (WordIsWellFormed(word)) {
	numWords++;	
	char *dummy = word;
	
	
	
	if ( WordNotInStopwords(&allData->stopwords, word)) { 
	  /**  Try looking up the word. If the word is not in the indices, create a new indexEntry 
	   *   initialized with the word and an empty vector and enter it into the hashset
	   */
	  indexEntry entry = {word}; 
	  indexEntry *found = HashSetLookup(&allData->indices, &entry);

	  if (found == NULL) {  
	    entry.word = strdup(dummy); 
	    VectorNew(&entry.articles, sizeof(wordcountEntry), NULL, 10); 
	    HashSetEnter(&allData->indices, &entry);
	  }

	  // now we act as if the entry was in the index all along
	  found  = (indexEntry*)HashSetLookup( &allData->indices, &entry); 	  
	  


	  UpdateIndices(&found->articles, articleIndex);

	}
	if (strlen(word) > strlen(longestWord))
	  strcpy(longestWord, word);
      }
    }
  }

  printf("\tWe counted %d well-formed words [including duplicates].\n", numWords);
  printf("\tThe longest word scanned was \"%s\".", longestWord);
  if (strlen(longestWord) >= 15 && (strchr(longestWord, '-') == NULL)) 
    printf(" [Ooooo... long word!]");
  printf("\n");
}
Ejemplo n.º 5
0
static void ScanArticle(streamtokenizer *st, article* a,
			hashset* stopWords, hashset* wordHash,
			hashset *articlesSeen)
{
  char word[1024];

  while (STNextToken(st, word, sizeof(word))) {
    if (strcasecmp(word, "<") == 0) {
      SkipIrrelevantContent(st); // in html-utls.h
    } else {
      RemoveEscapeCharacters(word);
      if (WordIsWellFormed(word)) {
	ProcessWellFormedWord(word,a,stopWords,wordHash,articlesSeen);
      }
    }
  }
}
Ejemplo n.º 6
0
static void ScanArticle(streamtokenizer *st, int articleID, hashset *indices, hashset *stopWords, pthread_mutex_t* indicesLock, pthread_mutex_t* stopWordsLock)
{
  char word[1024];

  while (STNextToken(st, word, sizeof(word))) {
    if (strcasecmp(word, "<") == 0) {
      SkipIrrelevantContent(st);
    } else {
      RemoveEscapeCharacters(word);
      pthread_mutex_lock(stopWordsLock);
      bool startIndexNow = WordIsWorthIndexing(word, stopWords);
      pthread_mutex_unlock(stopWordsLock);
      if (startIndexNow){
	
	pthread_mutex_lock(indicesLock);
	AddWordToIndices(indices, word, articleID);
	//printf("DONE INDEXING");
	pthread_mutex_unlock(indicesLock);
      }
    }
  }
}
Ejemplo n.º 7
0
static void ScanArticle(streamtokenizer *st, rssFeedData *data)
{
  articleData *article = addArticle(&data->articles, &data->rssItem);
  /*rssFeedItem *item = &(data->rssItem);
  char *articleTitle = item->title;
  char *articleURL = item->url;*/

  int numWords = 0;
  char word[1024];
  char longestWord[1024] = {'\0'};

  while (STNextToken(st, word, sizeof(word))) {
    if (strcasecmp(word, "<") == 0) {
      SkipIrrelevantContent(st); // in html-utls.h
    } else {
      RemoveEscapeCharacters(word);
      if (WordIsWellFormed(word)) {
	char *dummy = word;
	if (HashSetLookup(&(data->stopWords), &dummy)==NULL) {
		//not in stop list, index the word
		indexData *entry = addWordRecord(&data->indices, word);
		indexWord(&entry->counters, article);
		numWords++;
		if (strlen(word) > strlen(longestWord))
	  		strcpy(longestWord, word);
	}
      }
    }
  }

  printf("\tWe counted %d well-formed words [including duplicates].\n", numWords);
  printf("\tThe longest word scanned was \"%s\".", longestWord);
  if (strlen(longestWord) >= 15 && (strchr(longestWord, '-') == NULL)) 
    printf(" [Ooooo... long word!]");
  printf("\n");
}