uint8_t asfHeader::open(char *name) { _fd=fopen(name,"rb"); if(!_fd) { GUI_Error_HIG("File Error.","Cannot open file\n"); return 0; } myName=ADM_strdup(name); if(!getHeaders()) { return 0; } buildIndex(); fseeko(_fd,_dataStartOffset,SEEK_SET); _packet=new asfPacket(_fd,_nbPackets,_packetSize,&readQueue,_dataStartOffset); curSeq=1; if(_nbAudioTrack) { _curAudio=new asfAudio(this,_currentAudioStream); } return 1; }
int joinBucket() { // cerr << "---------------------------------------------------" << endl; int pairs = 0; bool selfjoin = stop.join_cardinality ==1 ? true : false ; int idx1 = SID_1 ; int idx2 = selfjoin ? SID_1 : SID_2 ; double low[2], high[2]; // for each tile (key) in the input stream try { std::vector<Geometry*> & poly_set_one = polydata[idx1]; std::vector<Geometry*> & poly_set_two = polydata[idx2]; int len1 = poly_set_one.size(); int len2 = poly_set_two.size(); if (len1 <= 0 || len2 <= 0) { return 0; } map<int,Geometry*> geom_polygons2; for (int j = 0; j < len2; j++) { geom_polygons2[j] = poly_set_two[j]; } // build spatial index for input polygons from idx2 bool ret = buildIndex(geom_polygons2); if (ret == false) { return -1; } // cerr << "len1 = " << len1 << endl; // cerr << "len2 = " << len2 << endl; for (int i = 0; i < len1; i++) { const Geometry* geom1 = poly_set_one[i]; const Envelope * env1 = geom1->getEnvelopeInternal(); low[0] = env1->getMinX(); low[1] = env1->getMinY(); high[0] = env1->getMaxX(); high[1] = env1->getMaxY(); /* Handle the buffer expansion for R-tree */ if (stop.JOIN_PREDICATE == ST_DWITHIN) { low[0] -= stop.expansion_distance; low[1] -= stop.expansion_distance; high[0] += stop.expansion_distance; high[1] += stop.expansion_distance; } Region r(low, high, 2); hits.clear(); MyVisitor vis; spidx->intersectsWithQuery(r, vis); //cerr << "j = " << j << " hits: " << hits.size() << endl; for (uint32_t j = 0 ; j < hits.size(); j++ ) { if (hits[j] == i && selfjoin) { continue; } const Geometry* geom2 = poly_set_two[hits[j]]; const Envelope * env2 = geom2->getEnvelopeInternal(); if (join_with_predicate(geom1, geom2, env1, env2, stop.JOIN_PREDICATE)) { ReportResult(i,hits[j]); pairs++; } } } } // end of try //catch (Tools::Exception& e) { catch (...) { std::cerr << "******ERROR******" << std::endl; //std::string s = e.what(); //std::cerr << s << std::endl; return -1; } // end of catch return pairs ; }
void Dbtup::execBUILD_INDX_IMPL_REQ(Signal* signal) { jamEntry(); #ifdef TIME_MEASUREMENT time_events= 0; tot_time_passed= 0; number_events= 1; #endif const BuildIndxImplReq* const req = (const BuildIndxImplReq*)signal->getDataPtr(); // get new operation BuildIndexPtr buildPtr; if (ERROR_INSERTED(4031) || ! c_buildIndexList.seizeFirst(buildPtr)) { jam(); BuildIndexRec buildRec; buildRec.m_request = *req; buildRec.m_errorCode = BuildIndxImplRef::Busy; if (ERROR_INSERTED(4031)) { CLEAR_ERROR_INSERT_VALUE; } buildIndexReply(signal, &buildRec); return; } buildPtr.p->m_request = *req; const BuildIndxImplReq* buildReq = &buildPtr.p->m_request; // check buildPtr.p->m_errorCode= BuildIndxImplRef::NoError; buildPtr.p->m_outstanding = 0; do { if (buildReq->tableId >= cnoOfTablerec) { jam(); buildPtr.p->m_errorCode= BuildIndxImplRef::InvalidPrimaryTable; break; } TablerecPtr tablePtr; tablePtr.i= buildReq->tableId; ptrCheckGuard(tablePtr, cnoOfTablerec, tablerec); if (tablePtr.p->tableStatus != DEFINED) { jam(); buildPtr.p->m_errorCode= BuildIndxImplRef::InvalidPrimaryTable; break; } // memory page format buildPtr.p->m_build_vs = (tablePtr.p->m_attributes[MM].m_no_of_varsize + tablePtr.p->m_attributes[MM].m_no_of_dynamic) > 0; if (DictTabInfo::isOrderedIndex(buildReq->indexType)) { jam(); const DLList<TupTriggerData>& triggerList = tablePtr.p->tuxCustomTriggers; TriggerPtr triggerPtr; triggerList.first(triggerPtr); while (triggerPtr.i != RNIL) { if (triggerPtr.p->indexId == buildReq->indexId) { jam(); break; } triggerList.next(triggerPtr); } if (triggerPtr.i == RNIL) { jam(); // trigger was not created ndbassert(false); buildPtr.p->m_errorCode = BuildIndxImplRef::InternalError; break; } buildPtr.p->m_indexId = buildReq->indexId; buildPtr.p->m_buildRef = DBTUX; AlterIndxImplReq* req = (AlterIndxImplReq*)signal->getDataPtrSend(); req->indexId = buildReq->indexId; req->senderRef = 0; req->requestType = AlterIndxImplReq::AlterIndexBuilding; EXECUTE_DIRECT(DBTUX, GSN_ALTER_INDX_IMPL_REQ, signal, AlterIndxImplReq::SignalLength); } else if(buildReq->indexId == RNIL) { jam(); // REBUILD of acc buildPtr.p->m_indexId = RNIL; buildPtr.p->m_buildRef = DBACC; } else { jam(); buildPtr.p->m_errorCode = BuildIndxImplRef::InvalidIndexType; break; } // set to first tuple position const Uint32 firstTupleNo = 0; buildPtr.p->m_fragNo= 0; buildPtr.p->m_pageId= 0; buildPtr.p->m_tupleNo= firstTupleNo; // start build bool offline = !!(buildReq->requestType&BuildIndxImplReq::RF_BUILD_OFFLINE); if (offline && m_max_parallel_index_build > 1) { jam(); buildIndexOffline(signal, buildPtr.i); } else { jam(); buildIndex(signal, buildPtr.i); } return; } while (0); // check failed buildIndexReply(signal, buildPtr.p); c_buildIndexList.release(buildPtr); }
AllDiffConstraint::AllDiffConstraint(WCSP *wcsp, EnumeratedVariable** scope_in, int arity_in) : FlowBasedGlobalConstraint(wcsp, scope_in, arity_in) { buildIndex(); }
void FuzzySearchImpl::prepare() { buildIndex(); }
int main(int argc, char const ** argv) { double startTime = 0; // Parse command line. FxFaidxOptions options; seqan::ArgumentParser::ParseResult res = parseArgs(options, argc, argv); if (res != seqan::ArgumentParser::PARSE_OK) return res == seqan::ArgumentParser::PARSE_ERROR; // 1 on errors, 0 otherwise // --------------------------------------------------------------------------- // Index I/O // --------------------------------------------------------------------------- // Load index, create if necessary. startTime = sysTime(); seqan::FaiIndex faiIndex; if (load(faiIndex, toCString(options.inFastaPath), toCString(options.inFaiPath)) != 0) { if (options.verbosity >= 2) std::cerr << "Building Index " << options.inFaiPath << " ..."; if (buildIndex(toCString(options.inFastaPath), toCString(options.inFaiPath), seqan::Fai()) != 0) { std::cerr << "Could not build FAI index at " << options.inFaiPath << " for FASTA file " << options.inFastaPath << "\n"; return 1; } if (load(faiIndex, toCString(options.inFastaPath), toCString(options.inFaiPath)) != 0) { std::cerr << "Could not load FAI index we just build.\n"; return 1; } } if (options.verbosity >= 3) std::cerr << "Took " << (startTime - sysTime()) << " s\n"; // --------------------------------------------------------------------------- // Parse and Fetch Regions. // --------------------------------------------------------------------------- if (empty(options.regions)) return 0; // Parse out regions. seqan::String<Region> regions; for (unsigned i = 0; i < length(options.regions); ++i) { Region region; if (!parseRegion(region, options.regions[i])) { std::cerr << "Could not parse region " << options.regions[i] << "\n"; return 1; } unsigned seqId; if (!getIdByName(faiIndex, region.seqName, seqId)) { std::cerr << "Unknown sequence for region " << options.regions[i] << "\n"; return 1; } region.seqId = seqId; if (region.seqId < 0 || (unsigned)region.seqId >= length(faiIndex.indexEntryStore)) { std::cerr << "Invalid region " << options.regions[i] << "\n"; return 1; } appendValue(regions, region); } // Open output file. std::ostream * outPtr = &std::cout; std::ofstream outF; if (!empty(options.outFastaPath)) { outF.open(toCString(options.outFastaPath), std::ios::binary | std::ios::out); if (!outF.good()) { std::cerr << "Could not open output file " << options.outFastaPath << "\n"; return 1; } } // Retrieve output infixes and write to result. for (unsigned i = 0; i < length(regions); ++i) { Region const & region = regions[i]; seqan::CharString id = options.regions[i]; seqan::CharString seq; unsigned beginPos = 0; if (region.beginPos > 0) beginPos = region.beginPos; unsigned endPos = sequenceLength(faiIndex, (unsigned)region.seqId); if (region.endPos > 0 && (unsigned)region.endPos < endPos) endPos = region.endPos; if (beginPos > endPos) endPos = beginPos; getSequenceInfix(seq, faiIndex, region.seqId, beginPos, endPos); if (writeRecord(*outPtr, id, seq, seqan::Fasta()) != 0) { std::cerr << "Could not write infix for region " << options.regions[i] << " to output.\n"; return 1; } } return 0; }
int main(int argc, char **argv) { if (argc != 6 && argc != 5) { cerr << "ERROR: Not enough arguments. Usage: " << argv[0] << " [geomid1] [geomid2] [partition_file] [prefixpath1] [prefixpath2]" << endl; return -1; } //int uid_idx = args_info.uid_arg; filename = argv[3]; GEOM_IDX = 2; JOIN_IDX = -1; char* stdinfilename = getenv("mapreduce_map_input_file"); char* prefix1 = argv[4]; char* prefix2 = NULL; if (argc == 6) { prefix2 = argv[5]; } if ( strstr(stdinfilename, prefix1) == NULL) { JOIN_IDX = 2; GEOM_IDX = atoi(argv[2]); } else { JOIN_IDX = 1; GEOM_IDX = atoi(argv[1]); } // cerr << "JOIN_IDX: " << JOIN_IDX << " Geom: " << GEOM_IDX <<endl; if (JOIN_IDX < 0) { cerr << "Invalid join index" << endl; return -1; } gf = new GeometryFactory(new PrecisionModel(),0); wkt_reader= new WKTReader(gf); // process input data // map<int,Geometry*> geom_polygons; string input_line; vector<string> fields; id_type id = 0; Geometry* geom ; genTiles(); bool ret = buildIndex(); if (ret == false) { cerr << "ERROR: Index building on tile structure has failed ." << std::endl; return 1 ; } else #ifndef NDEBUG cerr << "GRIDIndex Generated successfully." << endl; #endif cerr << "Reading input from stdin..." <<endl; while(cin && getline(cin, input_line) && !cin.eof()){ fields = parse(input_line); if (fields[GEOM_IDX].length() <2 ) { #ifndef NDEBUG cerr << "skipping record [" << id <<"]"<< endl; #endif continue ; // skip lines which has empty geometry } // try { geom = wkt_reader->read(fields[GEOM_IDX]); //} /*catch (...) { cerr << "WARNING: Record [id = " <<i << "] is not well formatted "<<endl; cerr << input_line << endl; continue ; }*/ // cout << input_line << endl; doQuery(geom); emitHits(geom, input_line); delete geom; } // cerr << "Number of tiles: " << geom_tiles.size() << endl; // build spatial index for input polygons cout.flush(); cerr.flush(); freeObjects(); return 0; // success }
void UTF8Utils::setStr(const string &s) { _str = s; buildIndex(); }
static int do_index(const char * filename, DIC * dic) { int i; int j; int k; struct stat sbuf; if (0 == strncmp(filename, dic->ename, strlen(dic->ename))) return 0; /* database (and database-temp file) is always pruned! */ if (isPruned(filename)) return 0; for (i=dic->deferredCount-1;i>=0;i--) { if (0 == strcmp(filename, dic->deferredTruncations[i])) { free(dic->deferredTruncations[i]); dic->deferredTruncations[i] = dic->deferredTruncations[dic->deferredCount-1]; GROW(dic->deferredTruncations, dic->deferredCount, dic->deferredCount-1); } } j = -1; for (i=DOODLE_getFileCount(dic->tree)-1;i>=0;i--) { if (0 == strcmp(filename, DOODLE_getFileAt(dic->tree,i)->filename)) { j = i; break; } } k = -1; for (i=0;i<dic->frPos;i++) { if (0 == strcmp(filename, dic->frNames[i])) { k=i; break; } } if (0 != lstat(filename, &sbuf)) { dic->log(dic->logContext, DOODLE_LOG_VERY_VERBOSE, _("Call to '%s' for file '%s' failed: %s\n"), "stat", filename, strerror(errno)); if (j != -1) { /* remove old keywords, file no longer there? */ GROW(dic->deferredTruncations, dic->deferredCount, dic->deferredCount + 1); dic->deferredTruncations[dic->deferredCount-1] = strdup(filename); } if (k != -1) { if (-1 == FAMCancelMonitor(&dic->fc, &dic->fr[k])) { dic->log(dic->logContext, DOODLE_LOG_CRITICAL, _("Call to '%s' for file '%s' failed: %s\n"), "FAMCancelMonitor", filename, FamErrlist[FAMErrno]); } free(dic->frNames[k]); dic->fr[k] = dic->fr[dic->frPos-1]; dic->frNames[k] = dic->frNames[dic->frPos-1]; dic->frNames[dic->frPos-1] = NULL; dic->frPos--; } return 0; } if ( (S_ISDIR(sbuf.st_mode)) && #ifdef S_ISLNK (! S_ISLNK(sbuf.st_mode)) && #endif #ifdef S_ISSOCK (! S_ISSOCK(sbuf.st_mode)) && #endif (k == -1) ) { char * fn; if (dic->frPos == dic->frSize) { unsigned int s = dic->frSize; GROW(dic->fr, dic->frSize, dic->frSize * 2); GROW(dic->frNames, s, s * 2); } dic->log(dic->logContext, DOODLE_LOG_VERY_VERBOSE, _("Will monitor directory '%s' for changes.\n"), filename); fn = STRDUP(filename); if (0 == FAMMonitorDirectory(&dic->fc, filename, &dic->fr[dic->frPos], fn)) { dic->frNames[dic->frPos] = fn; dic->frPos++; } else { dic->log(dic->logContext, DOODLE_LOG_CRITICAL, _("Call to '%s' for file '%s' failed: %s\n"), "FAMMonitorDirectory", filename, FamErrlist[FAMErrno]); free(fn); } } if (j != -1) { if (DOODLE_getFileAt(dic->tree,j)->mod_time == (unsigned int) sbuf.st_mtime) { return 0; /* already processed! */ } else { /* remove old keywords, file changed! */ /* we must do the new truncation now, so we also do all of those that were deferred */ GROW(dic->deferredTruncations, dic->deferredCount, dic->deferredCount + 2); dic->deferredTruncations[dic->deferredCount-2] = strdup(filename); DOODLE_tree_truncate_multiple(dic->tree, (const char**)dic->deferredTruncations); for (i=dic->deferredCount-2;i>=0;i--) free(dic->deferredTruncations[i]); GROW(dic->deferredTruncations, dic->deferredCount, 0); } } if (S_ISREG(sbuf.st_mode)) { dic->log(dic->logContext, DOODLE_LOG_VERY_VERBOSE, _("Processing file '%s'.\n"), filename); return buildIndex(dic->elist, NULL, filename, dic->tree, do_filenames); } return 0; }
bool SaveLoad_Playtoons::GameHandler::load(int16 dataVar, int32 size, int32 offset) { uint32 varSize; if (size < 0) { // Load a temporary sprite debugC(2, kDebugSaveLoad, "Loading temporary sprite %d at pos %d", size, offset); _tempSpriteHandler->load(dataVar, size, offset); return true; } varSize = SaveHandler::getVarSize(_vm); if (varSize == 0) return false; if (size == 0) { // Indicator to load all variables dataVar = 0; size = varSize; } if (((uint32) offset) < kPropsSize) { // Properties if (((uint32) (offset + size)) > kPropsSize) { warning("Wrong index size (%d, %d)", size, offset); return false; } _vm->_inter->_variables->copyFrom(dataVar, _props + offset, size); } else if (((uint32) offset) < kPropsSize + kIndexSize) { // Save index if (((uint32) size) != kIndexSize) { warning("Wrong index size (%d, %d)", size, offset); return false; } buildIndex(_vm->_inter->_variables->getAddressOff8(dataVar)); } else { // Save slot, whole variable block uint32 slot = _slotFile->getSlot(offset); int slotRem = _slotFile->getSlotRemainder(offset); debugC(2, kDebugSaveLoad, "Loading from slot %d", slot); if ((slot >= kSlotCount) || (slotRem != 0) || (dataVar != 0) || (((uint32) size) != varSize)) { warning("Invalid loading procedure (%d, %d, %d, %d, %d)", dataVar, size, offset, slot, slotRem); return false; } Common::String slotFile = _slotFile->build(slot); SaveReader *reader = 0; // New save, load directly reader = new SaveReader(2, slot, slotFile); SavePartInfo info(kSlotNameLength, (uint32) _vm->getGameType(), 0, _vm->getEndianness(), varSize); SavePartVars vars(_vm, varSize); if (!reader->load()) { delete reader; return false; } if (!reader->readPart(0, &info)) { delete reader; return false; } if (!reader->readPart(1, &vars)) { delete reader; return false; } // Get all variables if (!vars.writeInto(0, 0, varSize)) { delete reader; return false; } delete reader; } return true; }
UTF8Utils::UTF8Utils(const string &str) { _str = str; buildIndex(); }
void Dbtup::execBUILDINDXREQ(Signal* signal) { jamEntry(); #ifdef TIME_MEASUREMENT time_events= 0; tot_time_passed= 0; number_events= 1; #endif // get new operation BuildIndexPtr buildPtr; if (! c_buildIndexList.seize(buildPtr)) { jam(); BuildIndexRec buildRec; memcpy(buildRec.m_request, signal->theData, sizeof(buildRec.m_request)); buildRec.m_errorCode= BuildIndxRef::Busy; buildIndexReply(signal, &buildRec); return; } memcpy(buildPtr.p->m_request, signal->theData, sizeof(buildPtr.p->m_request)); // check buildPtr.p->m_errorCode= BuildIndxRef::NoError; do { const BuildIndxReq* buildReq= (const BuildIndxReq*)buildPtr.p->m_request; if (buildReq->getTableId() >= cnoOfTablerec) { jam(); buildPtr.p->m_errorCode= BuildIndxRef::InvalidPrimaryTable; break; } TablerecPtr tablePtr; tablePtr.i= buildReq->getTableId(); ptrCheckGuard(tablePtr, cnoOfTablerec, tablerec); if (tablePtr.p->tableStatus != DEFINED) { jam(); buildPtr.p->m_errorCode= BuildIndxRef::InvalidPrimaryTable; break; } // memory page format buildPtr.p->m_build_vs = tablePtr.p->m_attributes[MM].m_no_of_varsize > 0; if (DictTabInfo::isOrderedIndex(buildReq->getIndexType())) { jam(); const DLList<TupTriggerData>& triggerList = tablePtr.p->tuxCustomTriggers; TriggerPtr triggerPtr; triggerList.first(triggerPtr); while (triggerPtr.i != RNIL) { if (triggerPtr.p->indexId == buildReq->getIndexId()) { jam(); break; } triggerList.next(triggerPtr); } if (triggerPtr.i == RNIL) { jam(); // trigger was not created buildPtr.p->m_errorCode = BuildIndxRef::InternalError; break; } buildPtr.p->m_indexId = buildReq->getIndexId(); buildPtr.p->m_buildRef = DBTUX; } else if(buildReq->getIndexId() == RNIL) { jam(); // REBUILD of acc buildPtr.p->m_indexId = RNIL; buildPtr.p->m_buildRef = DBACC; } else { jam(); buildPtr.p->m_errorCode = BuildIndxRef::InvalidIndexType; break; } // set to first tuple position const Uint32 firstTupleNo = 0; buildPtr.p->m_fragNo= 0; buildPtr.p->m_pageId= 0; buildPtr.p->m_tupleNo= firstTupleNo; // start build buildIndex(signal, buildPtr.i); return; } while (0); // check failed buildIndexReply(signal, buildPtr.p); c_buildIndexList.release(buildPtr); }
void Query::startQuery() { int count = 0; //before starting query we have to get the index buildIndex(); cout << "Enter E to quit" << endl; string input = ""; string temp = ""; cin.ignore(); while(input != "E" || input != "e") { count = 0; //cin.ignore(); //http://www.codecogs.com/library/computing/stl/algorithms/set/set_union.php //good for parameters for a set cout << "To start a query please enter the words you would like to search for: " << endl; //cin.ignore(); getline(cin, input); if(input == "E" || input == "e") { break; } stringstream ss(input); while(ss >> temp) { if(temp != "AND" && temp != "OR" && temp != "NOT") transform(temp.begin(), temp.end(), temp.begin(), ::tolower); char* arr = new char[temp.length() + 1]; strcpy(arr, temp.c_str()); int x = stem(arr, 0, strlen(arr)-1); arr[x+1] = '\0'; temp = arr; searchWords.push_back(temp); //try to separate the string into vector or array count++; } Word * word1; Word * word2; Word * word3; vector<int> word1Pages; map <int, int> freqMap; map <int, int> freqMap2; int totalFrequency = 0; vector<int> totalWordFrequency; vector <Page*> pageTitleResults; vector<int> pageResults; int testCount = 0; if(count == 1) { word1 = table.returnWord(searchWords[0]); //word1 = tree.returnWord(searchWords[0]); if(word1 == NULL) { cout << searchWords[0] << " does not exist" << endl; } else { pageResults = word1->getPages(); for(int b = 0; b < pageResults.size(); b++) { int oPage = pageResults[b]; freqMap = word1->getInfo(); totalFrequency = freqMap[oPage]; totalWordFrequency.push_back(totalFrequency); //cout << "PAGE: " << pageResults[b] << " TOTAL FREQUENCY: " << totalFrequency << endl; } cout << endl; frequencySort(totalWordFrequency, pageResults); for(int i = 0; i < pageResults.size(); i++) { //cout << "FREQUENCY: " << totalWordFrequency[i] << " PAGE: " << pageResults[i] <<endl; } } } else if(searchWords[0] == "AND") { word1 = table.returnWord(searchWords[1]); word2 = table.returnWord(searchWords[2]); //word1 = tree.returnWord(searchWords[1]); //word2 = tree.returnWord(searchWords[2]); if(word1 == NULL || word2 == NULL) { cout << "word does not exist" << endl; } else { word1Pages = word1->getPages(); vector<int> word2Pages = word2->getPages(); pageResults = qAND(word1Pages, word2Pages); if(count == 5) { word3 = table.returnWord(searchWords[4]); //word3 = tree.returnWord(searchWords[4]); if(word3 == NULL) { cout << "word does not exist" << endl; } else { vector<int> word3Pages = word3->getPages(); pageResults = qNOT(pageResults, word3Pages); } //end else } } for(int b = 0; b < pageResults.size(); b++) { int oPage = pageResults[b]; if(oPage == 0) break; else { freqMap = word1->getInfo(); freqMap2 = word2->getInfo(); totalFrequency = freqMap[oPage] + freqMap2[oPage]; totalWordFrequency.push_back(totalFrequency); //cout << "PAGE: " << pageResults[b] << " TOTAL FREQUENCY: " << totalFrequency << endl; } } frequencySort(totalWordFrequency, pageResults); for(int i = 0; i < pageResults.size(); i++) { if(pageResults[i] == 0) break; //cout << "FREQUENCY: " << totalWordFrequency[i] << " PAGE: " << pageResults[i] <<endl; } }//end else else if(searchWords[0] == "OR") { word1 = table.returnWord(searchWords[1]); word2 = table.returnWord(searchWords[2]); //word1 = tree.returnWord(searchWords[1]); //word2 = tree.returnWord(searchWords[2]); if(word1 == NULL || word2 == NULL) { cout << "word does not exist" << endl; } else { word1Pages = word1->getPages(); vector<int> word2Pages = word2->getPages(); pageResults = qOR(word1Pages, word2Pages); if(count == 5) { word3 = table.returnWord(searchWords[4]); //word3 = tree.returnWord(searchWords[4]); if(word3 == NULL) { cout << "word does not exist" << endl; } else { vector<int> word3Pages = word3->getPages(); pageResults = qNOT(pageResults, word3Pages); }//end else } } for(int b = 0; b < pageResults.size(); b++) { int oPage = pageResults[b]; if(oPage == 0) break; else { freqMap = word1->getInfo(); freqMap2 = word2->getInfo(); totalFrequency = freqMap[oPage] + freqMap2[oPage]; totalWordFrequency.push_back(totalFrequency); //cout << "PAGE: " << pageResults[b] << " TOTAL FREQUENCY: " << totalFrequency << endl; } } frequencySort(totalWordFrequency, pageResults); for(int i = 0; i < pageResults.size(); i++) { if(pageResults[i] == 0) break; //cout << "FREQUENCY: " << totalWordFrequency[i] << " PAGE: " << pageResults[i] <<endl; } } else { word1 = table.returnWord(searchWords[0]); word2 = table.returnWord(searchWords[2]); //word1 = tree.returnWord(searchWords[0]); //word2 = tree.returnWord(searchWords[2]); if(word1 == NULL || word2 == NULL) { cout << "word does not exist" << endl; } else { word1Pages = word1->getPages(); vector<int> word2Pages = word2->getPages(); pageResults = qNOT(word1Pages, word2Pages); }//end else } Page* x; //frequencySort(totalWordFrequency, pageResults); if(pageResults.size() <= 15) { for(int i = 0; i < pageResults.size(); i++) //to return the top 15 items //for(int i = 0; i < 15; i++) { if(pageResults[i] == 0) break; cout << pageResults[i] << " "; x = pageIndex.returnObject(pageResults[i]); cout << x->getTitle() << endl; pageTitleResults.push_back(x); } } else { for(int i = 0; i < 15; i++) //to return the top 15 items //for(int i = 0; i < 15; i++) { if(pageResults[i] == 0) break; //cout << pageResults[i] << " "; x = pageIndex.returnObject(pageResults[i]); //cout << x->getTitle() << endl; pageTitleResults.push_back(x); } } cout << "Pages on which your search appears:" << endl << endl; //pageTitleResults.print3(cout); for(int s = 0; s < pageTitleResults.size(); s++) { cout << "Page: " << pageTitleResults[s]->getId() << " " << pageTitleResults[s]->getTitle() << endl; } cout << endl; Page* k; int pageChoice = 0; cout << "Please enter the page number of the page you would like to view " << endl; cout << "Enter -1 if you do not want to view a page" << endl; //cin >> pageChoice; string pc; getline(cin, pc); pageChoice = stoi(pc); if(pageChoice != -1) { for(int z = 0; z < pageTitleResults.size(); z++) { if(pageChoice == pageTitleResults[z]->getId()) { string name = ""; int fileNum = pageChoice % 100; stringstream fs; fs << fileNum; fs << ".txt"; fs >> name; string text = ""; string text1 = ""; string cmpString = "$#**%"; ifstream fin3(name); while(!fin3.eof()) { string pageTitle = ""; int pageId = 0; fin3 >> pageTitle; fin3.ignore(3); fin3 >> pageId; fin3.ignore(); getline(fin3, text1); while(cmpString != text1) { text += text1; getline(fin3, text1); } Page* p = new Page(pageTitle, pageId, text); topPageIndex.insert2(p); //cout << "PAGE TITLE: " << pageTitle << endl; //cout << "PAGE NUMBER:" << pageId << endl; //cout << "TEXT: " << text << endl; } fin3.close(); break; } else { //do nothing, keep looping until you find page number } } //topPageIndex.print4(cout); k = topPageIndex.returnObject(pageChoice); cout << k->getText() << endl; //cout << k->getText() << endl; }//end if statement
// add an element to the tree nodePtr buildIndex( rootNodePtr r, // root pointer size_t dim, // current dim size_t m, // current length of obs size_t * indexPtr, // pointer to obs indexes int useProb, // determine if we use probability to build an index double * prob, size_t * nodeIdentity ) { size_t i,K; size_t * indexLeftPtr = NULL; size_t * indexRightPtr = NULL; size_t indexLeftSize; size_t indexRightSize; double probSum = 0; nodePtr c = createNode(r); // record to the tree structure the new tree c->indexUsed = m; c->index = indexPtr; c->dim = dim; K = r->K; // do we have too many points? if(!useProb) { if( m <= r->leafSize ) { // save the final pointer locations for( i = 0; i < m; i++) { // go through each element of indexPtr and store a pointer to that indexPtr element in pointerIndex r->pointerIndex[ indexPtr[i] ] = &( indexPtr[i] ); r->nodeIndex[ indexPtr[i] ] = *nodeIdentity; // printf(" node assignment %d is %d\n", (int) indexPtr[i], (int) *nodeIdentity ); } *nodeIdentity = *nodeIdentity + 1; return c; } } else { // if using probSize we want to figure out how many samples per psu for( i = 0; i < m; i++) { probSum += prob[ indexPtr[i] ]; // go through each element of indexPtr and store a pointer to that indexPtr element in pointerIndex } if(probSum <= r->leafSize) { for( i = 0; i < m; i++) { r->pointerIndex[ indexPtr[i] ] = &( indexPtr[i] ); r->nodeIndex[ indexPtr[i] ] = *nodeIdentity; // printf(" node assignment %d is %d\n", (int) indexPtr[i], (int) *nodeIdentity ); } *nodeIdentity = *nodeIdentity + 1; return c; } #ifdef DEBUG_PROB printf("split!\n"); #endif } // if we are here we have too many points // create children // figure out our new dim // split data and give to children if( useProb ) { c->split = splitDataProb( r->data, c->index, &indexLeftPtr, &indexRightPtr, &indexLeftSize, &indexRightSize, m, K, dim, prob ); #ifdef DEBUG_PROB printf("Left Side Size = %d, Right Side Size = %d split = %f\n", (int) indexLeftSize, (int) indexRightSize, c->split); printf("Left\n:"); for(i=0; i < indexLeftSize; i++) printf("%d ", (int) indexLeftPtr[i]); printf("\nRight\n:"); for(i=0; i < indexRightSize; i++) printf("%d ", (int) indexRightPtr[i]); printf("\n"); #endif } else { c-> split = splitData( r->data, c->index, &indexLeftPtr, &indexRightPtr, &indexLeftSize, &indexRightSize, m, K, dim ); } free(c->index); c->index = NULL; // move current contents to new children c->left = buildIndex( r, (dim+1) % K, indexLeftSize , indexLeftPtr, useProb, prob, nodeIdentity); c->right = buildIndex( r, (dim+1) % K, indexRightSize, indexRightPtr, useProb, prob, nodeIdentity); return c; }
void DfaDbReadonlySegment::compressSingleKeyValue(ReadableSegment* input, DbContext* ctx) { llong prevId = -1, id = -1; llong logicRowNum = input->m_isDel.size(), newRowNum = 0; assert(logicRowNum > 0); auto tmpDir = m_segDir + ".tmp"; ColumnVec columns(m_schema->columnNum(), valvec_reserve()); valvec<byte> buf; StoreIteratorPtr iter(input->createStoreIterForward(ctx)); SortableStrVec keyVec, valueVec; const Schema& rowSchema = m_schema->getRowSchema(); const Schema& keySchema = m_schema->getIndexSchema(0); const Schema& valueSchema = m_schema->getColgroupSchema(1); std::unique_ptr<DictZipBlobStore::ZipBuilder> builder; FixedLenStorePtr store; if (valueSchema.should_use_FixedLenStore()) { store = new FixedLenStore(tmpDir, valueSchema); store->unneedsLock(); } else if (valueSchema.m_dictZipSampleRatio >= 0.0) { double sRatio = valueSchema.m_dictZipSampleRatio; double avgLen = double(input->dataInflateSize()) / logicRowNum; if ((sRatio > FLT_EPSILON) || (sRatio >= 0 && avgLen > 120)) { builder = createDictZipBlobStoreBuilder(valueSchema); } } std::mt19937_64 random; // (random.max() - random.min()) + 1 may overflow // do not +1 to avoid overflow uint64_t sampleUpperBound = random.min() + (random.max() - random.min()) * valueSchema.m_dictZipSampleRatio; size_t sampleLenSum = 0; valvec<byte_t> key, val; while (iter->increment(&id, &buf) && id < logicRowNum) { assert(id >= 0); assert(id < logicRowNum); assert(prevId < id); if (!m_isDel[id]) { rowSchema.parseRow(buf, &columns); keySchema.selectParent(columns, &key); valueSchema.selectParent(columns, &val); if (keySchema.getFixedRowLen() > 0) { keyVec.m_strpool.append(key); } else { keyVec.push_back(key); } if (builder) { if (random() < sampleUpperBound) { builder->addSample(val); sampleLenSum += val.size(); } } else { if (store) store->append(val, NULL); else valueVec.push_back(val); } newRowNum++; m_isDel.beg_end_set1(prevId+1, id); prevId = id; } } if (prevId != id) { assert(prevId < id); assert(m_isDel[id]); m_isDel.beg_end_set1(prevId+1, id); } llong inputRowNum = id + 1; assert(inputRowNum <= logicRowNum); if (inputRowNum < logicRowNum) { fprintf(stderr , "WARN: DfaDbReadonlySegment::compressSingleKeyValue(): realrows=%lld, m_isDel=%lld, some data have lost\n" , inputRowNum, logicRowNum); input->m_isDel.beg_end_set1(inputRowNum, logicRowNum); this->m_isDel.beg_end_set1(inputRowNum, logicRowNum); } m_delcnt = m_isDel.popcnt(); // recompute delcnt assert(newRowNum <= inputRowNum); assert(size_t(logicRowNum - newRowNum) == m_delcnt); if (builder) { iter->reset(); // free resources and seek to begin } else { iter = nullptr; } m_indices[0] = buildIndex(keySchema, keyVec); // memory heavy m_colgroups[0] = m_indices[0]->getReadableStore(); keyVec.clear(); if (builder) { assert(valueVec.m_index.size() == 0); assert(valueVec.m_strpool.size() == 0); std::lock_guard<std::mutex> lock(DictZip_reduceMemMutex()); auto fpath = tmpDir / ("colgroup-" + valueSchema.m_name + ".nlt"); emptyCheckProtect(sampleLenSum, val, *builder); builder->prepare(newRowNum, fpath.string()); prevId = -1; while (iter->increment(&id, &buf) && id < inputRowNum) { for (llong j = prevId+1; j < id; ++j) { if (!m_isDel[j]) { // j was deleted during compressing builder->addRecord(fstring()); // add an empty record } } prevId = id; if (!m_isDel[id]) { rowSchema.parseRow(buf, &columns); valueSchema.selectParent(columns, &val); builder->addRecord(val); } } iter = nullptr; m_colgroups[1] = new NestLoudsTrieStore(valueSchema, builder->finish( DictZipBlobStore::ZipBuilder::FinishFreeDict | DictZipBlobStore::ZipBuilder::FinishWriteDictFile )); } else if (store) { m_colgroups[1] = std::move(store); } else { m_colgroups[1] = this->buildStore(valueSchema, valueVec); } }
void BaseTrackCache::filterAndSort(const QSet<TrackId>& trackIds, const QString& searchQuery, const QString& extraFilter, const QString& orderByClause, const QList<SortColumn>& sortColumns, const int columnOffset, QHash<TrackId, int>* trackToIndex) { // Skip processing if there are no tracks to filter or sort. if (trackIds.size() == 0) { return; } if (!m_bIndexBuilt) { buildIndex(); } QStringList idStrings; // TODO(rryan) consider making this the data passed in and a separate // QVector for output QSet<TrackId> dirtyTracks; for (const auto& trackId: trackIds) { idStrings << trackId.toString(); if (m_dirtyTracks.contains(trackId)) { dirtyTracks.insert(trackId); } } std::unique_ptr<QueryNode> pQuery(parseQuery( searchQuery, extraFilter, idStrings)); QString filter = pQuery->toSql(); if (!filter.isEmpty()) { filter.prepend("WHERE "); } QString queryString = QString("SELECT %1 FROM %2 %3 %4") .arg(m_idColumn, m_tableName, filter, orderByClause); if (sDebug) { qDebug() << this << "select() executing:" << queryString; } QSqlQuery query(m_database); // This causes a memory savings since QSqlCachedResult (what QtSQLite uses) // won't allocate a giant in-memory table that we won't use at all. query.setForwardOnly(true); query.prepare(queryString); if (!query.exec()) { LOG_FAILED_QUERY(query); } int idColumn = query.record().indexOf(m_idColumn); int rows = query.size(); if (sDebug) { qDebug() << "Rows returned:" << rows; } m_trackOrder.resize(0); // keeps alocated memory trackToIndex->clear(); if (rows > 0) { trackToIndex->reserve(rows); m_trackOrder.reserve(rows); } while (query.next()) { TrackId trackId(query.value(idColumn)); (*trackToIndex)[trackId] = m_trackOrder.size(); m_trackOrder.append(trackId); } // At this point, the original set of tracks have been divided into two // pieces: those that should be in the result set and those that should // not. Unfortunately, due to TrackDAO caching, there may be tracks in // either category that are there incorrectly. We must look at all the dirty // tracks (within the original set, if specified) and evaluate whether they // would match or not match the given filter criteria. Once we correct the // membership of tracks in either set, we must then insertion-sort the // missing tracks into the resulting index list. if (dirtyTracks.size() == 0) { return; } for (TrackId trackId: dirtyTracks) { // Only get the track if it is in the cache. TrackPointer pTrack = lookupCachedTrack(trackId); if (!pTrack) { continue; } // The track should be in the result set if the search is empty or the // track matches the search. bool shouldBeInResultSet = searchQuery.isEmpty() || pQuery->match(pTrack); // If the track is in this result set. bool isInResultSet = trackToIndex->contains(trackId); if (shouldBeInResultSet) { // Track should be in result set... // Remove the track from the results first (we have to do this or it // will sort wrong). if (isInResultSet) { int index = (*trackToIndex)[trackId]; m_trackOrder.remove(index); // Don't update trackToIndex, since we do it below. } // Figure out where it is supposed to sort. The table is sorted by // the sort column, so we can binary search. int insertRow = findSortInsertionPoint( pTrack, sortColumns, columnOffset, m_trackOrder); if (sDebug) { qDebug() << this << "Insertion sort says it should be inserted at:" << insertRow; } // The track should sort at insertRow m_trackOrder.insert(insertRow, trackId); trackToIndex->clear(); // Fix the index. TODO(rryan) find a non-stupid way to do this. for (int i = 0; i < m_trackOrder.size(); ++i) { (*trackToIndex)[m_trackOrder[i]] = i; } } else if (isInResultSet) { // Track should not be in this result set, but it is. We need to // remove it. int index = (*trackToIndex)[trackId]; m_trackOrder.remove(index); trackToIndex->clear(); // Fix the index. TODO(rryan) find a non-stupid way to do this. for (int i = 0; i < m_trackOrder.size(); ++i) { (*trackToIndex)[m_trackOrder[i]] = i; } } } }
bool SaveLoad_v4::GameHandler::load(int16 dataVar, int32 size, int32 offset) { uint32 varSize = SaveHandler::getVarSize(_vm); if (varSize == 0) return false; if (size == 0) { // Indicator to load all variables dataVar = 0; size = varSize; } if (offset < 500) { // Global properties debugC(3, kDebugSaveLoad, "Loading global properties"); if ((size + offset) > 500) { warning("Wrong global properties list size (%d, %d)", size, offset); return false; } _vm->_inter->_variables->copyFrom(dataVar, _props + offset, size); } else if (offset == 500) { // Save index if (size != 1200) { warning("Requested index has wrong size (%d)", size); return false; } // Create/Fake the index buildIndex(_vm->_inter->_variables->getAddressOff8(dataVar)); } else { // Save slot, whole variable block uint32 slot = _slotFile->getSlot(offset); int slotRem = _slotFile->getSlotRemainder(offset); debugC(2, kDebugSaveLoad, "Loading from slot %d", slot); if ((slot >= kSlotCount) || (slotRem != 0) || (dataVar != 0) || (((uint32) size) != varSize)) { warning("Invalid saving procedure (%d, %d, %d, %d, %d)", dataVar, size, offset, slot, slotRem); return false; } _hasIndex = false; if (!createReader(slot)) return false; SavePartInfo info(kSlotNameLength, (uint32) _vm->getGameType(), 0, _vm->getEndianness(), varSize); SavePartVars vars(_vm, varSize); if (!_reader->readPart(0, &info)) return false; if (!_reader->readPart(1, &vars)) return false; // Get all variables if (!vars.writeInto(0, 0, varSize)) return false; _lastSlot = slot; } return true; }
RowLookupTable::RowLookupTable( const FeatureSet& fset ) : _maxY( -1 ), _minY ( -1 ) { buildIndex( fset ); }
bool SaveLoad_v2::GameHandler::load(int16 dataVar, int32 size, int32 offset) { uint32 varSize = SaveHandler::getVarSize(_vm); if (varSize == 0) return false; if (size == 0) { // Indicator to load all variables dataVar = 0; size = varSize; } if (offset == 0) { // Save index if (((uint32) size) != kIndexSize) { warning("Requested index has wrong size (%d)", size); return false; } // Create/Fake the index buildIndex(_vm->_inter->_variables->getAddressOff8(dataVar)); } else { // Save slot, whole variable block uint32 slot = _slotFile->getSlot(offset); int slotRem = _slotFile->getSlotRemainder(offset); debugC(2, kDebugSaveLoad, "Loading from slot %d", slot); if ((slot >= kSlotCount) || (slotRem != 0) || (dataVar != 0) || (((uint32) size) != varSize)) { warning("Invalid loading procedure (%d, %d, %d, %d, %d)", dataVar, size, offset, slot, slotRem); return false; } Common::String slotFile = _slotFile->build(slot); SaveReader *reader = 0; SaveConverter_v2 converter(_vm, slotFile); if (converter.isOldSave()) { // Old save, plug the converter in if (!converter.load()) return false; reader = new SaveReader(2, slot, converter); } else // New save, load directly reader = new SaveReader(2, slot, slotFile); SavePartInfo info(kSlotNameLength, (uint32) _vm->getGameType(), 0, _vm->getEndianness(), varSize); SavePartVars vars(_vm, varSize); if (!reader->load()) { delete reader; return false; } if (!reader->readPart(0, &info)) { delete reader; return false; } if (!reader->readPart(1, &vars)) { delete reader; return false; } // Get all variables if (!vars.writeInto(0, 0, varSize)) { delete reader; return false; } delete reader; } return true; }
RowLookupTable::RowLookupTable( const FeatureDescriptorExtractor& fset ) : _maxY( -1 ), _minY ( -1 ) { buildIndex( fset ); }
bool SaveLoad_v6::GameHandler::load(int16 dataVar, int32 size, int32 offset) { uint32 varSize = SaveHandler::getVarSize(_vm); if (varSize == 0) return false; if (size == 0) { // Indicator to load all variables dataVar = 0; size = varSize; } if (((uint32) offset) < kPropsSize) { // Properties refreshProps(); if (((uint32) (offset + size)) > kPropsSize) { warning("Wrong index size (%d, %d)", size, offset); return false; } _vm->_inter->_variables->copyFrom(dataVar, _props + offset, size); } else if (((uint32) offset) < kPropsSize + kIndexSize) { // Save index if (((uint32) size) != kIndexSize) { warning("Wrong index size (%d, %d)", size, offset); return false; } buildIndex(_vm->_inter->_variables->getAddressOff8(dataVar)); } else { // Save slot, whole variable block uint32 slot = _slotFile->getSlot(offset); int slotRem = _slotFile->getSlotRemainder(offset); debugC(2, kDebugSaveLoad, "Loading from slot %d", slot); if ((slot >= kSlotCount) || (slotRem != 0) || (dataVar != 0) || (((uint32) size) != varSize)) { warning("Invalid loading procedure (%d, %d, %d, %d, %d)", dataVar, size, offset, slot, slotRem); return false; } if (!createReader(slot)) return false; SavePartInfo info(kSlotNameLength, (uint32) _vm->getGameType(), 0, _vm->getEndianness(), varSize); SavePartVars vars(_vm, varSize); if (!_reader->load()) return false; if (!_reader->readPart(0, &info)) return false; if (!_reader->readPart(1, &vars)) return false; // Get all variables if (!vars.writeInto(0, 0, varSize)) return false; if (!_spriteHandler->set(_reader, 4)) return false; } return true; }
inline void setQ(unsigned int q){_q=q; buildIndex();}