void RSpatialIndex::addToIndex(int id, int pos, const RBox& bb) {
addToIndex(
id, pos,
bb.c1.x, bb.c1.y, bb.c1.z,
bb.c2.x, bb.c2.y, bb.c2.z
);
}
void RSpatialIndex::addToIndex(int id, const QList<RBox>& bbs) {
for (int pos = 0; pos < bbs.size(); ++pos) {
addToIndex(
id, pos,
bbs[pos]
);
}
}
/* public */
void
MCIndexSegmentSetMutualIntersector::setBaseSegments(SegmentString::ConstVect* segStrings)
{
// NOTE - mloskot: const qualifier is removed silently, dirty.
for(std::size_t i = 0, n = segStrings->size(); i < n; i++) {
const SegmentString* css = (*segStrings)[i];
SegmentString* ss = const_cast<SegmentString*>(css);
addToIndex(ss);
}
}
int recurseDirs(const char *path, ThreadManager *threadman) {
GValue message = {0,};
g_value_init(&message,G_TYPE_STRING);
g_value_set_string(&message,"busy");
g_object_set_property(G_OBJECT(threadman),"status", &message);
g_value_unset(&message);
if(path==NULL) {
return 1;
}
sleep(1);
GFile *dir = g_file_new_for_path(path);
GFileEnumerator *fileEnumerator;
GError *errorHandler=NULL;
fileEnumerator = g_file_enumerate_children(dir,"standard::*",G_FILE_QUERY_INFO_NONE,NULL,&errorHandler);
if(errorHandler!=NULL) {
fprintf(stderr,"ERROR: %s\n",errorHandler->message);
g_error_free(errorHandler);
return 1;
}
if(fileEnumerator!=NULL) {
GFileInfo *finfo = g_file_enumerator_next_file(fileEnumerator,NULL,NULL);
while(finfo!=NULL) {
if(!g_file_info_get_is_hidden(finfo)) {
const gchar *name;
char *fullPath;
name = g_file_info_get_name(finfo);
fullPath = strdup(path);
fullPath = realloc(fullPath,strlen(path)+3+strlen(name));
strcat(fullPath,"/");
strcat(fullPath,name);
if(g_file_info_get_file_type(finfo)==G_FILE_TYPE_DIRECTORY) {
int res = recurseDirs(fullPath,threadman);
if(res!=0) {
fprintf(stderr,"Error with %s\n",fullPath);
}
} else {
const gchar *type = g_file_info_get_attribute_string(finfo,G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE);
if(strcmp(type,"audio/mpeg")==0) {
addToIndex(name,fullPath,type);
}
}
free(fullPath);
}
finfo = g_file_enumerator_next_file(fileEnumerator,NULL,NULL);
}
g_object_unref(fileEnumerator);
}
return 0;
}
void IndexHandler::addPage(Page* nextPage)
{
//read in the keywords of the specific page
vector<string> temp = nextPage->getKeywords();
int size = temp.size();
//add pages with each keyword
for(int i = 0; i < size; i++)
{
addToIndex(nextPage, temp[i]);
}
}
///
/// Parsing function for creation of Indexer object. Takes words from file to index
/// and strips them of punctuation, converts all chars to lower case, then compares
/// with strings in skip file. It will finally store those that apply to these
/// guidelines, including their page number and line number.
///
void Indexer::parseFile()
{
std::ifstream inFile;
std::string inputLine;
inFile.open(inFileName);
if (!inFile.is_open())
{
std::cout << "No index file present with that name." << std::endl;
system("PAUSE");
exit(0);
}
while (!inFile.eof()) // First loop acquires a line from the text file.
{
std::getline(inFile, inputLine);
std::istringstream stringParse(inputLine);
incLineNmbr();
while (!stringParse.eof()) // Inner loop breaks line up into strings.
{
std::string token;
stringParse >> token;
if (token == "") // Check for empty strings.
continue;
if (token == "<newpage>")
{
incPageNmbr();
lineNmbr = 0;
break;
}
punctStrip(token); // Call function to remove punctuation
lowerCase(token); // Convert to lower case.
if (!wordsToSkip.skipWordcheck(token)) // Check for string in skipWords.
continue;
addToIndex(token);
}
}
}
/**
* Adds an item to the index.
*
* \param id ID of the item.
* \param x1 lower value of the first dimension.
* \param y1 lower value of the second dimension.
* \param z1 lower value of the third dimension.
* \param x2 higher value of the first dimension.
* \param y2 higher value of the second dimension.
* \param z2 higher value of the third dimension.
* \param dataLength Legth of the data in bytes or 0.
* \param data Pointer to the data associated with the item. The
* index makes a deep copy of the data, the caller is responsible
* for deleting the data. NULL indicates that the item data is stored
* externally (e.g. in a map or list).
*/
void RSpatialIndexNavel::addToIndex(
int id, int pos,
double x1, double y1, double z1,
double x2, double y2, double z2) {
//qDebug() << "RSpatialIndexNavel::addToIndex: id: " << id << ", pos: " << pos << ", " << x1 << "," << y1 << "/" << x2 << "," << y2;
//qDebug() << "\tbefore: " << *this;
addToIndex(
id, pos,
RSpatialIndexNavel::RSiRegion(
qMin(x1, x2), qMin(y1, y2), qMin(z1, z2),
qMax(x1, x2), qMax(y1, y2), qMax(z1, z2))
//dataLength,
//data
);
//qDebug() << "\tafter: " << *this << "\n\n";
}
void DhQGraphicsItemGroup::DvhaddToIndex() {
return addToIndex();
}
/**
* filters, transforms, and indexes file using ngrams to the index
*
* file name - name of file to process
* wikiindex - the judy arrays to store the index of the wiki in
*/
void indexWiki(char* inFileName, Pvoid_t *wikiIndex, int* articleCount) {
//-------------------- initialization --------------------//
bool articleIndex[lastNgram] = {0}; // boolean array of what trigrams are in an article
struct stemmer * currentStemmer = create_stemmer();
// file for writing the titles to
FILE* titleFile = NULL;
if (writeFiles) {
titleFile = fopen("title_file", "w");
if (NULL == titleFile) {
fprintf(stderr, "Error open title file: %m\n");
exit(1);
}
}
// initializes the libxml library
LIBXML_TEST_VERSION
xmlTextReaderPtr wikiReader; //the reader for the document
wikiReader = xmlReaderForFile(inFileName, NULL, XML_PARSE_RECOVER+XML_PARSE_NOWARNING+XML_PARSE_NOERROR+XML_PARSE_HUGE);
if (NULL == wikiReader) {
//fprintf(stderr, "%s %s\n", "Failed to open ", wikiFileName);
fprintf(stderr, "Error opening XML wiki: %m\n");
exit(1);
}
// for progress bar
int percent = 0;
long fileSize = getFileSize(inFileName);
// initialization for currentArticle and its componens
article currentArticle;
currentArticle.title = g_string_sized_new(256);
currentArticle.body = g_string_sized_new(786432); //768*1024
//-------------------- index the wiki --------------------//
optionalPrint ("%s", "Adding collection to index.\n");
optionalPrint ("%d", (int)(fileSize / 1048576));
optionalPrint (" MB in file\n");
displayProgressBar (xmlTextReaderByteConsumed(wikiReader), fileSize, &percent);
//prime the loop
currentArticle.title->len = 0;
currentArticle.body->len = 0;
xmlTextReaderRead(wikiReader);// at a <page> tag, drop in
xmlTextReaderRead(wikiReader);// at a <page> tag, drop in
// reads from xml file until file is finished, adds articles to index, and writes tittles to file
// processes one article per iteration
while (getArticle (wikiReader, ¤tArticle)) {
currentArticle.articleNumber = *articleCount;
*articleCount = *articleCount + 1;
// filter / transform text
removeMarkup(currentArticle.body);
stemText(currentArticle.body, currentStemmer); //ngramming.h
// index the text
indexText(currentArticle.body, articleIndex); //ngramming.h
addToIndex(articleIndex, wikiIndex, currentArticle.articleNumber);
//adds titles to title file
if (writeFiles) {fprintf(titleFile, "%s\n", currentArticle.title->str);}
//re-prime the loop
currentArticle.title->len = 0;
currentArticle.body->len = 0;
displayProgressBar (xmlTextReaderByteConsumed(wikiReader), fileSize, &percent);
}
optionalPrint ("\n%s", "Finished indexing. \n");
optionalPrint ("%lu", (long)(xmlTextReaderByteConsumed(wikiReader)/1048576));
optionalPrint ("MB consumed\n");
optionalPrint ("%d %s %d %s", *articleCount, "articles found", (int) currentArticle.body->allocated_len, "length allocated for article body\n");
// clean up of structures needed to process wiki
if (writeFiles) {fclose (titleFile);}
free_stemmer(currentStemmer);
xmlFreeTextReader(wikiReader);
xmlCleanupParser();
//g_string_free(currentArticle.title, TRUE);
//g_string_free(currentArticle.body, TRUE); //malloc fail if this is uncommented ?!
}
void DhQGraphicsEllipseItem::DvhaddToIndex() {
return addToIndex();
}
void RSpatialIndex::bulkLoad(const QList<int>& ids, const QList<QList<RBox> >& bbs) {
for (int i=0; i<ids.length() && i<bbs.length(); i++) {
addToIndex(ids[i], bbs[i]);
}
}
