vp_t
suggest(PhraseMap &pm, RMQ &st, std::string prefix, uint_t n = 16) {
pvpi_t phrases = pm.query(prefix);
// cerr<<"Got "<<phrases.second - phrases.first<<" candidate phrases from PhraseMap"<<endl;
uint_t first = phrases.first - pm.repr.begin();
uint_t last = phrases.second - pm.repr.begin();
if (first == last) {
return vp_t();
}
vp_t ret;
--last;
pqpr_t heap;
pui_t best = st.query_max(first, last);
heap.push(PhraseRange(first, last, best.first, best.second));
while (ret.size() < n && !heap.empty()) {
PhraseRange pr = heap.top();
heap.pop();
// cerr<<"Top phrase is at index: "<<pr.index<<endl;
// cerr<<"And is: "<<pm.repr[pr.index].first<<endl;
ret.push_back(pm.repr[pr.index]);
uint_t lower = pr.first;
uint_t upper = pr.index - 1;
// Prevent underflow
if (pr.index - 1 < pr.index && lower <= upper) {
// cerr<<"[1] adding to heap: "<<lower<<", "<<upper<<", "<<best.first<<", "<<best.second<<endl;
best = st.query_max(lower, upper);
heap.push(PhraseRange(lower, upper, best.first, best.second));
}
lower = pr.index + 1;
upper = pr.last;
// Prevent overflow
if (pr.index + 1 > pr.index && lower <= upper) {
// cerr<<"[2] adding to heap: "<<lower<<", "<<upper<<", "<<best.first<<", "<<best.second<<endl;
best = st.query_max(lower, upper);
heap.push(PhraseRange(lower, upper, best.first, best.second));
}
}
return ret;
}
vp_t
naive_suggest(PhraseMap& pm, RMQ& st, std::string prefix, uint_t n = 16) {
pvpi_t phrases = pm.query(prefix);
std::vector<uint_t> indexes;
vp_t ret;
while (phrases.first != phrases.second) {
indexes.push_back(phrases.first - pm.repr.begin());
++phrases.first;
}
while (ret.size() < n && !indexes.empty()) {
uint_t mi = 0;
for (size_t i = 1; i < indexes.size(); ++i) {
if (pm.repr[indexes[i]].weight > pm.repr[indexes[mi]].weight) {
mi = i;
}
}
ret.push_back(pm.repr[indexes[mi]]);
indexes.erase(indexes.begin() + mi);
}
return ret;
}
int
do_import(std::string file, int sorted, uint_t limit,
int &rnadded, int &rnlines) {
#if defined USE_CXX_IO
std::ifstream fin(file.c_str());
#else
FILE *fin = fopen(file.c_str(), "r");
#endif
int fd = open(file.c_str(), O_RDONLY);
// Potential race condition + not checking for return value
if_length = file_size(file.c_str());
DCERR("handle_import::file:"<<file<<endl);
if (!fin || !fd) {
return -IMPORT_FILE_NOT_FOUND;
}
else {
building = true;
int nlines = 0;
int foffset = 0;
if (if_mmap_addr) {
munmap(if_mmap_addr, if_length);
}
// mmap() the input file in
if_mmap_addr = (char*)mmap(NULL, if_length, PROT_READ, MAP_SHARED, fd, 0);
if (!if_mmap_addr) {
fclose(fin);
close(fd);
return -IMPORT_FILE_NOT_FOUND;
}
pm.repr.clear();
char buff[INPUT_LINE_SIZE];
while (
#if defined USE_CXX_IO
fin
#else
!feof(fin)
#endif
&& limit--) {
buff[0] = '\0';
#if defined USE_CXX_IO
fin.getline(buff, INPUT_LINE_SIZE);
const int llen = fin.gcount();
buff[INPUT_LINE_SIZE - 1] = '\0';
#else
char *got = fgets(buff, INPUT_LINE_SIZE, fin);
if (!got) {
break;
}
const int llen = strlen(buff);
if (llen && buff[llen-1] == '\n') {
buff[llen-1] = '\0';
}
#endif
++nlines;
int weight = 0;
std::string phrase;
StringProxy snippet;
InputLineParser(if_mmap_addr, foffset, buff, &weight, &phrase, &snippet).start_parsing();
foffset += llen;
if (!phrase.empty()) {
str_lowercase(phrase);
DCERR("Adding: "<<weight<<", "<<phrase<<", "<<std::string(snippet)<<endl);
pm.insert(weight, phrase, snippet);
}
}
fclose(fin);
pm.finalize(sorted);
vui_t weights;
for (size_t i = 0; i < pm.repr.size(); ++i) {
weights.push_back(pm.repr[i].weight);
}
st.initialize(weights);
rnadded = weights.size();
rnlines = nlines;
building = false;
}
return 0;
}
int
do_import(std::string file, uint_t limit,
int &rnadded, int &rnlines) {
bool is_input_sorted = true;
#if defined USE_CXX_IO
std::ifstream fin(file.c_str());
#else
FILE *fin = fopen(file.c_str(), "r");
#endif
int fd = open(file.c_str(), O_RDONLY);
DCERR("handle_import::file:" << file << "[fin: " << (!!fin) << ", fd: " << fd << "]" << endl);
if (!fin || fd == -1) {
perror("fopen");
return -IMPORT_FILE_NOT_FOUND;
}
else {
building = true;
int nlines = 0;
int foffset = 0;
if (if_mmap_addr) {
int r = munmap(if_mmap_addr, if_length);
if (r < 0) {
perror("munmap");
building = false;
return -IMPORT_MUNMAP_FAILED;
}
}
// Potential race condition + not checking for return value
if_length = file_size(file.c_str());
// mmap() the input file in
if_mmap_addr = (char*)mmap(NULL, if_length, PROT_READ, MAP_SHARED, fd, 0);
if (if_mmap_addr == MAP_FAILED) {
fprintf(stderr, "length: %llu, fd: %d\n", if_length, fd);
perror("mmap");
if (fin) { fclose(fin); }
if (fd != -1) { close(fd); }
building = false;
return -IMPORT_MMAP_FAILED;
}
pm.repr.clear();
char buff[INPUT_LINE_SIZE];
std::string prev_phrase;
while (!is_EOF(fin) && limit--) {
buff[0] = '\0';
int llen = -1;
get_line(fin, buff, INPUT_LINE_SIZE, llen);
if (llen == -1) {
break;
}
++nlines;
int weight = 0;
std::string phrase;
StringProxy snippet;
InputLineParser(if_mmap_addr, foffset, buff, &weight, &phrase, &snippet).start_parsing();
foffset += llen;
if (!phrase.empty()) {
str_lowercase(phrase);
DCERR("Adding: " << weight << ", " << phrase << ", " << std::string(snippet) << endl);
pm.insert(weight, phrase, snippet);
}
if (is_input_sorted && prev_phrase <= phrase) {
prev_phrase.swap(phrase);
} else if (is_input_sorted) {
is_input_sorted = false;
}
}
DCERR("Creating PhraseMap::Input is " << (!is_input_sorted ? "NOT " : "") << "sorted\n");
fclose(fin);
pm.finalize(is_input_sorted);
vui_t weights;
for (size_t i = 0; i < pm.repr.size(); ++i) {
weights.push_back(pm.repr[i].weight);
}
st.initialize(weights);
rnadded = weights.size();
rnlines = nlines;
building = false;
}
return 0;
}
int
test() {
PhraseMap pm;
pm.insert(1, "duckduckgo", "");
pm.insert(2, "duckduckgeese", "");
pm.insert(1, "duckduckgoose", "");
pm.insert(9, "duckduckgoo", "");
pm.insert(10, "duckgo", "");
pm.insert(3, "dukgo", "");
pm.insert(2, "luckkuckgo", "");
pm.insert(5, "chuckchuckgo", "");
pm.insert(15, "dilli - no one killed jessica", "");
pm.insert(11, "aaitbaar - no one killed jessica", "");
pm.finalize();
RMQ st;
vui_t weights;
for (size_t i = 0; i < pm.repr.size(); ++i) {
weights.push_back(pm.repr[i].weight);
}
st.initialize(weights);
cout<<"\n";
cout<<"suggest(\"d\"):\n"<<suggest(pm, st, "d")<<endl;
cout<<"naive_suggest(\"d\"):\n"<<naive_suggest(pm, st, "d")<<endl;
cout<<"\n";
cout<<"suggest(\"a\"):\n"<<suggest(pm, st, "a")<<endl;
cout<<"naive_suggest(\"a\"):\n"<<naive_suggest(pm, st, "a")<<endl;
cout<<"\n";
cout<<"suggest(\"b\"):\n"<<suggest(pm, st, "b")<<endl;
cout<<"naive_suggest(\"b\"):\n"<<naive_suggest(pm, st, "b")<<endl;
cout<<"\n";
cout<<"suggest(\"duck\"):\n"<<suggest(pm, st, "duck")<<endl;
cout<<"naive_suggest(\"duck\"):\n"<<naive_suggest(pm, st, "duck")<<endl;
cout<<"\n";
cout<<"suggest(\"k\"):\n"<<suggest(pm, st, "k")<<endl;
cout<<"naive_suggest(\"k\"):\n"<<naive_suggest(pm, st, "k")<<endl;
cout<<"\n";
cout<<"suggest(\"ka\"):\n"<<suggest(pm, st, "ka")<<endl;
cout<<"naive_suggest(\"ka\"):\n"<<naive_suggest(pm, st, "ka")<<endl;
cout<<"\n";
cout<<"suggest(\"c\"):\n"<<suggest(pm, st, "c")<<endl;
cout<<"naive_suggest(\"c\"):\n"<<naive_suggest(pm, st, "c")<<endl;
return 0;
}
