void initialize(vui_t const& elems) {
this->data = elems;
this->len = elems.size();
this->repr.clear();
DCERR("len: "<<this->len<<endl);
const size_t ntables = log2(this->len) + 1;
this->repr.resize(ntables);
DCERR("ntables: "<<ntables<<endl);
this->repr[0].resize(this->len);
for (size_t i = 0; i < this->len; ++i) {
// This is the identity mapping, since the MAX element in
// a range of length 1 is the element itself.
this->repr[0][i] = i;
}
for (size_t i = 1; i < ntables; ++i) {
/* The previous 'block size' */
const uint_t pbs = 1<<(i-1);
/* bs is the 'block size'. i.e. The number of elements
* from the data that are used to computed the max value
* and store it at repr[i][...].
*/
const uint_t bs = 1<<i;
/* The size of the vector at repr[i]. We need to resize it
* to this size.
*/
const size_t vsz = this->len - bs + 1;
DCERR("starting i: "<<i<<" bs: "<<bs<<endl);
this->repr[i].resize(vsz);
// cerr<<"i: "<<i<<", vsz: "<<vsz<<endl;
vui_t& curr = this->repr[i];
vui_t& prev = this->repr[i - 1];
for (size_t j = 0; j < vsz; ++j) {
// 'j' is the starting index of a block of size 'bs'
const uint_t prev_elem1 = data[prev[j]];
const uint_t prev_elem2 = data[prev[j+pbs]];
if (prev_elem1 > prev_elem2) {
curr[j] = prev[j];
}
else {
curr[j] = prev[j+pbs];
}
// cerr<<"curr["<<j<<"] = "<<curr[j].first<<endl;
}
// cerr<<"done with i: "<<i<<endl;
}
// cerr<<"initialize() completed"<<endl;
}
void
parse_options(int argc, char *argv[]) {
int c;
while (1) {
int option_index = 0;
static struct option long_options[] = {
{"file", 1, 0, 'f'},
{"port", 1, 0, 'p'},
{"sorted", 0, 0, 's'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
c = getopt_long(argc, argv, "f:p:sh",
long_options, &option_index);
if (c == -1)
break;
switch (c) {
case 0:
case 'f':
DCERR("File: "<<optarg<<endl);
ac_file = optarg;
break;
case 'p':
DCERR("Port: "<<optarg<<" ("<<atoi(optarg)<<")\n");
port = optarg;
break;
case 's':
DCERR("File is Sorted\n");
ac_sorted = true;
break;
case 'h':
opt_show_help = true;
break;
case '?':
cerr<<"ERROR::Invalid option: "<<optopt<<endl;
break;
}
}
}
void
on_snippet(const char *data, int len) {
if (len && this->psnippet_proxy) {
const char *base = this->mem_base + this->buff_offset +
(data - this->buff);
DCERR("on_snippet::base: "<<(void*)base<<", len: "<<len<<"\n");
this->psnippet_proxy->assign(base, len);
}
}
// validate the smbios table entry point
bool validateDMITableEntryPoint(
const smbiosLowlevel::dmi_table_entry_point *tempTEP,
bool strict,
ParseExceptionImpl &parseException
)
{
// This code checks for the following:
// entry point structure checksum : As per the specs
// anchor string : As per the specs
//
bool retval = true;
u8 checksum = 0;
const u8 *ptr = reinterpret_cast<const u8*>(tempTEP);
// don't overrun tempTEP if BIOS is buggy... (note sizeof() test here)
// added especially to deal with buggy Intel BIOS.
for( unsigned int i = 0; i < sizeof(*tempTEP); ++i )
{
// stupid stuff to avoid MVC++ .NET runtime exception check for cast to different size
checksum = (checksum + ptr[i]) & 0xFF;
}
ostringstream oss;
DCERR("_DMI_ anchor: " << tempTEP->anchor[0] << tempTEP->anchor[1] << tempTEP->anchor[2] << tempTEP->anchor[3] << tempTEP->anchor[4] << tempTEP->anchor[5] << endl);
if(memcmp(tempTEP->anchor,"_DMI_",5)!=0) // Checking intermediate anchor string
{
oss << _("Intermediate anchor string does not match. anchor string: %(dmi_anchor)s") << endl;
retval = false; // validation failed
}
DCERR("_DMI_ checksum: " << (int)checksum << endl);
if(checksum) // Checking entry point structure checksum
{
oss << _("Checksum check for table entry point should be zero. checksum: %(dmi_checksum)i ") << endl;
retval = false; // validation failed
}
parseException.setParameter("dmi_anchor", reinterpret_cast<const char *>(tempTEP->anchor));
parseException.setParameter("dmi_checksum", static_cast<int>(checksum));
return retval;
}
bool SmbiosMemoryStrategy::getSmbiosTable(const u8 **smbiosBuffer, smbiosLowlevel::smbios_table_entry_point *table_header, bool strict)
{
bool ret = false;
try
{
// allocates no mem
DCERR("trying SmbiosMemoryStrategy" << endl);
getSmbiosTableHeader(table_header, strict);
// allocates mem, but frees on exception
getSmbiosTableBuf(smbiosBuffer, *table_header);
if(smbiosBuffer)
ret = true;
}
catch( MARK_UNUSED const exception &e)
{
DCERR("got Exception: " << e.what() << endl);
}
DCERR(" ret for SmbiosMemoryStrategy is: " << ret << endl);
return ret;
}
void
on_snippet(const char *data, int len) {
if (len && this->psnippet_proxy) {
const char *base = this->mem_base + this->buff_offset +
(data - this->buff);
if (base < if_mmap_addr || base + len > if_mmap_addr + if_length) {
fprintf(stderr, "base: %p, if_mmap_addr: %p, if_mmap_addr+if_length: %p\n", base, if_mmap_addr, if_mmap_addr + if_length);
assert(base >= if_mmap_addr);
assert(base <= if_mmap_addr + if_length);
assert(base + len <= if_mmap_addr + if_length);
}
DCERR("on_snippet::base: "<<(void*)base<<", len: "<<len<<"\n");
this->psnippet_proxy->assign(base, len);
}
}
static void*
handle_suggest(enum mg_event event,
struct mg_connection *conn,
const struct mg_request_info *request_info) {
++nreq;
if (building) {
print_HTTP_response(conn, 412, "Busy");
return (void*)"";
}
std::string q = get_qs(request_info, "q");
std::string sn = get_qs(request_info, "n");
std::string cb = get_qs(request_info, "callback");
std::string type = get_qs(request_info, "type");
DCERR("handle_suggest::q:"<<q<<", sn:"<<sn<<", callback: "<<cb<<endl);
unsigned int n = sn.empty() ? NMAX : atoi(sn.c_str());
if (n > NMAX) {
n = NMAX;
}
const bool has_cb = !cb.empty();
if (has_cb && !is_valid_cb(cb)) {
print_HTTP_response(conn, 400, "Invalid Request");
return (void*)"";
}
print_HTTP_response(conn, 200, "OK");
str_lowercase(q);
vp_t results = suggest(pm, st, q, n);
/*
for (size_t i = 0; i < results.size(); ++i) {
mg_printf(conn, "%s:%d\n", results[i].first.c_str(), results[i].second);
}
*/
if (has_cb) {
mg_printf(conn, "%s(%s);\n", cb.c_str(), results_json(q, results, type).c_str());
}
else {
mg_printf(conn, "%s\n", results_json(q, results, type).c_str());
}
return (void*)"";
}
static void handle_suggest(client_t *client, parsed_url_t &url) {
++nreq;
std::string body;
headers_t headers;
headers["Cache-Control"] = "no-cache";
if (building) {
write_response(client, 412, "Busy", headers, body);
return;
}
std::string q = unescape_query(url.query["q"]);
std::string sn = url.query["n"];
std::string cb = unescape_query(url.query["callback"]);
std::string type = unescape_query(url.query["type"]);
DCERR("handle_suggest::q:"<<q<<", sn:"<<sn<<", callback: "<<cb<<endl);
unsigned int n = sn.empty() ? NMAX : atoi(sn.c_str());
if (n > NMAX) {
n = NMAX;
}
if (n < 1) {
n = 1;
}
const bool has_cb = !cb.empty();
str_lowercase(q);
vp_t results = suggest(pm, st, q, n);
/*
for (size_t i = 0; i < results.size(); ++i) {
mg_printf(conn, "%s:%d\n", results[i].first.c_str(), results[i].second);
}
*/
headers["Content-Type"] = "text/plain; charset=UTF-8";
if (has_cb) {
body = cb + "(" + results_json(q, results, type) + ");\n";
}
else {
body = results_json(q, results, type) + "\n";
}
write_response(client, 200, "OK", headers, body);
}
void serve_request(client_t *client) {
parsed_url_t url;
parse_URL(client->url, url);
std::string &request_uri = url.path;
DCERR("request_uri: " << request_uri << endl);
if (request_uri == "/face/suggest/") {
handle_suggest(client, url);
}
else if (request_uri == "/face/import/") {
handle_import(client, url);
}
else if (request_uri == "/face/export/") {
handle_export(client, url);
}
else if (request_uri == "/face/stats/") {
handle_stats(client, url);
}
else {
handle_invalid_request(client, url);
}
}
int
do_import(std::string file, int sorted, uint_t limit,
int &rnadded, int &rnlines) {
#if defined USE_CXX_IO
std::ifstream fin(file.c_str());
#else
FILE *fin = fopen(file.c_str(), "r");
#endif
int fd = open(file.c_str(), O_RDONLY);
// Potential race condition + not checking for return value
if_length = file_size(file.c_str());
DCERR("handle_import::file:"<<file<<endl);
if (!fin || !fd) {
return -IMPORT_FILE_NOT_FOUND;
}
else {
building = true;
int nlines = 0;
int foffset = 0;
if (if_mmap_addr) {
munmap(if_mmap_addr, if_length);
}
// mmap() the input file in
if_mmap_addr = (char*)mmap(NULL, if_length, PROT_READ, MAP_SHARED, fd, 0);
if (!if_mmap_addr) {
fclose(fin);
close(fd);
return -IMPORT_FILE_NOT_FOUND;
}
pm.repr.clear();
char buff[INPUT_LINE_SIZE];
while (
#if defined USE_CXX_IO
fin
#else
!feof(fin)
#endif
&& limit--) {
buff[0] = '\0';
#if defined USE_CXX_IO
fin.getline(buff, INPUT_LINE_SIZE);
const int llen = fin.gcount();
buff[INPUT_LINE_SIZE - 1] = '\0';
#else
char *got = fgets(buff, INPUT_LINE_SIZE, fin);
if (!got) {
break;
}
const int llen = strlen(buff);
if (llen && buff[llen-1] == '\n') {
buff[llen-1] = '\0';
}
#endif
++nlines;
int weight = 0;
std::string phrase;
StringProxy snippet;
InputLineParser(if_mmap_addr, foffset, buff, &weight, &phrase, &snippet).start_parsing();
foffset += llen;
if (!phrase.empty()) {
str_lowercase(phrase);
DCERR("Adding: "<<weight<<", "<<phrase<<", "<<std::string(snippet)<<endl);
pm.insert(weight, phrase, snippet);
}
}
fclose(fin);
pm.finalize(sorted);
vui_t weights;
for (size_t i = 0; i < pm.repr.size(); ++i) {
weights.push_back(pm.repr[i].weight);
}
st.initialize(weights);
rnadded = weights.size();
rnlines = nlines;
building = false;
}
return 0;
}
void
start_parsing() {
int i = 0;
int n = 0;
const char *p_start = NULL, *s_start = NULL;
int p_len = 0, s_len = 0;
while (this->buff[i]) {
char ch = this->buff[i];
DCERR("State: "<<this->state<<", read: "<<ch<<"\n");
switch (this->state) {
case ILP_BEFORE_NON_WS:
if (!isspace(ch)) {
this->state = ILP_WEIGHT;
}
else {
++i;
}
break;
case ILP_WEIGHT:
if (isdigit(ch)) {
n *= 10;
n += (ch - '0');
++i;
}
else {
this->state = ILP_BEFORE_PTAB;
on_weight(n);
}
break;
case ILP_BEFORE_PTAB:
if (ch == '\t') {
this->state = ILP_AFTER_PTAB;
}
++i;
break;
case ILP_AFTER_PTAB:
if (isspace(ch)) {
++i;
}
else {
p_start = this->buff + i;
this->state = ILP_PHRASE;
}
break;
case ILP_PHRASE:
// DCERR("State: ILP_PHRASE: "<<buff[i]<<endl);
if (ch != '\t') {
++p_len;
}
else {
// Note: Skip to ILP_SNIPPET since the snippet may
// start with a white-space that we wish to
// preserve.
//
// this->state = ILP_AFTER_STAB;
this->state = ILP_SNIPPET;
s_start = this->buff + i + 1;
}
++i;
break;
case ILP_AFTER_STAB:
if (isspace(ch)) {
this->state = ILP_SNIPPET;
s_start = this->buff + i;
}
else {
++i;
}
break;
case ILP_SNIPPET:
++i;
++s_len;
break;
};
}
on_phrase(p_start, p_len);
on_snippet(s_start, s_len);
}
// validate the smbios table entry point
bool validateSmbiosTableEntryPoint(
const smbiosLowlevel::smbios_table_entry_point *tempTEP,
bool strict,
ParseExceptionImpl &parseException
)
{
// This code checks for the following:
// entry point structure checksum : As per the specs
// smbios major version : As per the specs
// Intermediate anchor string : As per the specs
//
// This code does not check the following:
// intermediate checksum: the main checksum covers the
// entire area
// and should be sufficient, plus there is a
// possibility for
// BIOS bugs in this area.
//
// minor version: according to the spec, this parser should
// work
// with any change in minor version. The spec says this
// parser
// will break if major version changes, so we check
// that.
//
bool retval = true;
u8 checksum = 0;
const u8 *ptr = reinterpret_cast<const u8*>(tempTEP);
// don't overrun tempTEP if BIOS is buggy... (note sizeof() test here)
// added especially to deal with buggy Intel BIOS.
for( unsigned int i = 0; (i < static_cast<unsigned int>(tempTEP->eps_length)) && (i < sizeof(*tempTEP)); ++i )
{
// stupid stuff to avoid MVC++ .NET runtime exception check for cast to different size
checksum = (checksum + ptr[i]) & 0xFF;
}
ostringstream oss;
oss << _("validation of table entry point failed") << endl;
validateDMITableEntryPoint( &(tempTEP->dmi), strict, parseException );
DCERR("strict table checking: " << strict << endl);
DCERR("_SM_ checksum: " << (int)checksum << endl);
if(checksum) // Checking entry point structure checksum
{
oss << _("Checksum check for table entry point should be zero. checksum: %(checksum)i ") << endl;
retval = false; // validation failed
}
DCERR("major_ver: " << (int)tempTEP->major_ver << endl);
if(tempTEP->major_ver!=0x02) // Checking smbios major version
{
oss << _("Major version of table entry point should be 2: %(major_version)i") << endl;
retval = false; // validation failed
}
// Entry Point Length field is at least 0x1f.
DCERR("eps_length: " << (int)tempTEP->eps_length << endl);
if(tempTEP->eps_length < 0x0f)
{
oss << _("Entry Point Length field is at least 0x1f : %(eps_length)i") << endl;
retval = false; // validation failed
}
parseException.setParameter("checksum", static_cast<int>(checksum));
parseException.setParameter("major_version", static_cast<int>(tempTEP->major_ver));
parseException.setParameter("eps_length", static_cast<int>(tempTEP->eps_length));
parseException.setMessageString(oss.str());
return retval;
}
// allocates no memory, constructs no objects.
// can raise an exception
void SmbiosMemoryStrategy::getSmbiosTableHeader(smbiosLowlevel::smbios_table_entry_point *table_header, bool strict)
{
memory::IMemory *mem = memory::MemoryFactory::getFactory()->getSingleton();
unsigned long fp = E_BLOCK_START;
if( offset )
fp = offset;
ParseExceptionImpl parseException;
if( offset )
{
DCERR("SmbiosMemoryStrategy::getSmbiosTableHeader() using hardcoded offset: " << hex << offset << endl);
parseException.setMessageString(_("SMBIOS Header not found at offset: %(offsetValue)i"));
parseException.setParameter("offsetValue",offset);
}
else
{
DCERR("SmbiosMemoryStrategy::getSmbiosTableHeader() Memory scan for smbios table." << endl);
parseException.setMessageString(_("SMBIOS Header not found in search."));
}
// tell the memory subsystem that it can optimize here and
// keep memory open while we scan rather than open/close/open/close/...
// for each fillBuffer() call
//
// this would be safer if we used spiffy c++ raii technique here
mem->decReopenHint();
smbios_table_entry_point tempTEP;
memset(&tempTEP, 0, sizeof(tempTEP));
while ( (fp + sizeof(tempTEP)) < F_BLOCK_END)
{
mem->fillBuffer(
reinterpret_cast<u8 *>(&tempTEP),
fp,
sizeof(tempTEP)
);
// search for promising looking headers
// first, look for old-style DMI header
if (memcmp (&tempTEP, "_DMI_", 5) == 0)
{
DCERR("Found _DMI_ anchor. Trying to parse legacy DMI structure." << endl);
dmi_table_entry_point *dmiTEP = reinterpret_cast<dmi_table_entry_point *>(&tempTEP);
memmove(&(tempTEP.dmi), &dmiTEP, sizeof(dmi_table_entry_point));
// fake the rest of the smbios table entry point...
tempTEP.major_ver=2;
tempTEP.minor_ver=0;
if(validateDMITableEntryPoint(dmiTEP, strict, parseException))
{
DCERR("Found valid _DMI_ entry point at offset: " << fp << endl);
break;
}
}
// then, look for new-style smbios header. This will always
// occur before _DMI_ in memory
if (offset || (memcmp (&tempTEP, "_SM_", 4) == 0))
{
// if we are passed a hardcoded offset (EFI?), it is possible
// that machine doesnt have _SM_ anchor, but still has valid
// table. Just try validating the table and skip _SM_ check.
DCERR("Found _SM_ anchor or using hardcoded offset. Trying to parse Smbios Entry Point." << endl);
if(validateSmbiosTableEntryPoint(&tempTEP, strict, parseException))
{
DCERR("Found valid _SM_ entry point at offset: " << fp << endl);
break;
}
}
// previous if() would have broken out if we have a valid
// table header. if offset is set, then we are not supposed
// to be scanning through memory. We didn't find a table,
// so there is nothing to do but raise an exception.
if (offset)
{
// dont need memory optimization anymore
mem->incReopenHint();
throw parseException; // previously set up.
}
fp += 16;
}
// dont need memory optimization anymore
mem->incReopenHint();
// bad stuff happened if we got to here and fp > 0xFFFFFL
if ((fp + sizeof(tempTEP)) >= F_BLOCK_END)
throw parseException; // previously set up.
// found it. set offset for future reference (no need to search.)
offset = fp;
memcpy( const_cast<smbios_table_entry_point *>(table_header), &tempTEP, sizeof(*table_header) );
}
int
do_import(std::string file, uint_t limit,
int &rnadded, int &rnlines) {
bool is_input_sorted = true;
#if defined USE_CXX_IO
std::ifstream fin(file.c_str());
#else
FILE *fin = fopen(file.c_str(), "r");
#endif
int fd = open(file.c_str(), O_RDONLY);
DCERR("handle_import::file:" << file << "[fin: " << (!!fin) << ", fd: " << fd << "]" << endl);
if (!fin || fd == -1) {
perror("fopen");
return -IMPORT_FILE_NOT_FOUND;
}
else {
building = true;
int nlines = 0;
int foffset = 0;
if (if_mmap_addr) {
int r = munmap(if_mmap_addr, if_length);
if (r < 0) {
perror("munmap");
building = false;
return -IMPORT_MUNMAP_FAILED;
}
}
// Potential race condition + not checking for return value
if_length = file_size(file.c_str());
// mmap() the input file in
if_mmap_addr = (char*)mmap(NULL, if_length, PROT_READ, MAP_SHARED, fd, 0);
if (if_mmap_addr == MAP_FAILED) {
fprintf(stderr, "length: %llu, fd: %d\n", if_length, fd);
perror("mmap");
if (fin) { fclose(fin); }
if (fd != -1) { close(fd); }
building = false;
return -IMPORT_MMAP_FAILED;
}
pm.repr.clear();
char buff[INPUT_LINE_SIZE];
std::string prev_phrase;
while (!is_EOF(fin) && limit--) {
buff[0] = '\0';
int llen = -1;
get_line(fin, buff, INPUT_LINE_SIZE, llen);
if (llen == -1) {
break;
}
++nlines;
int weight = 0;
std::string phrase;
StringProxy snippet;
InputLineParser(if_mmap_addr, foffset, buff, &weight, &phrase, &snippet).start_parsing();
foffset += llen;
if (!phrase.empty()) {
str_lowercase(phrase);
DCERR("Adding: " << weight << ", " << phrase << ", " << std::string(snippet) << endl);
pm.insert(weight, phrase, snippet);
}
if (is_input_sorted && prev_phrase <= phrase) {
prev_phrase.swap(phrase);
} else if (is_input_sorted) {
is_input_sorted = false;
}
}
DCERR("Creating PhraseMap::Input is " << (!is_input_sorted ? "NOT " : "") << "sorted\n");
fclose(fin);
pm.finalize(is_input_sorted);
vui_t weights;
for (size_t i = 0; i < pm.repr.size(); ++i) {
weights.push_back(pm.repr[i].weight);
}
st.initialize(weights);
rnadded = weights.size();
rnlines = nlines;
building = false;
}
return 0;
}
void
start_parsing() {
int i = 0; // The current record byte-offset.
int n = 0; // Temporary buffer for numeric (integer) fields.
const char *p_start = NULL; // Beginning of the phrase.
const char *s_start = NULL; // Beginning of the snippet.
int p_len = 0; // Phrase Length.
int s_len = 0; // Snippet length.
while (this->buff[i]) {
char ch = this->buff[i];
DCERR("["<<this->state<<":"<<ch<<"]");
switch (this->state) {
case ILP_BEFORE_NON_WS:
if (!isspace(ch)) {
this->state = ILP_WEIGHT;
}
else {
++i;
}
break;
case ILP_WEIGHT:
if (isdigit(ch)) {
n *= 10;
n += (ch - '0');
++i;
}
else {
this->state = ILP_BEFORE_PTAB;
on_weight(n);
}
break;
case ILP_BEFORE_PTAB:
if (ch == '\t') {
this->state = ILP_AFTER_PTAB;
}
++i;
break;
case ILP_AFTER_PTAB:
if (isspace(ch)) {
++i;
}
else {
p_start = this->buff + i;
this->state = ILP_PHRASE;
}
break;
case ILP_PHRASE:
// DCERR("State: ILP_PHRASE: "<<buff[i]<<endl);
if (ch != '\t') {
++p_len;
}
else {
// Note: Skip to ILP_SNIPPET since the snippet may
// start with a white-space that we wish to
// preserve.
//
this->state = ILP_SNIPPET;
s_start = this->buff + i + 1;
}
++i;
break;
case ILP_SNIPPET:
++i;
++s_len;
break;
};
}
DCERR("\n");
on_phrase(p_start, p_len);
on_snippet(s_start, s_len);
}
