void InlineTextBox::attachLine()
{
if (!extracted())
return;
lineLayoutItem().attachTextBox(this);
}
void InlineTextBox::extractLine()
{
if (extracted())
return;
lineLayoutItem().extractTextBox(this);
}
void DownloadManager::extracted(int exitCode, QProcess::ExitStatus exitStatus)
{
Q_UNUSED(exitCode);
if(exitStatus == QProcess::NormalExit) {
QString dataDir = QStandardPaths::writableLocation(QStandardPaths::DataLocation);
QString path = dataDir + "/" + fileName + language_ + fileType;
QFile::remove(path);
emit extracted(language_);
}
}
void InlineElementBox::deleteLine()
{
if (!extracted()) {
if (is<RenderBox>(renderer()))
downcast<RenderBox>(renderer()).setInlineBoxWrapper(nullptr);
else if (is<RenderLineBreak>(renderer()))
downcast<RenderLineBreak>(renderer()).setInlineBoxWrapper(nullptr);
}
delete this;
}
// static
const bool SyntaxPtg::extract_PtgInt(std::wstring::const_iterator& first, std::wstring::const_iterator last, std::wstring& out_str)
{
static boost::wregex reg_differ(L"^\\d+[.:]"); // differ from PtgNum(5.55) and PtgRef(7:7)
boost::match_results<std::wstring::const_iterator> results;
if(!boost::regex_search(first, last, reg_differ))
{
static boost::wregex reg_int(L"^\\d+");
if(boost::regex_search(first, last, results, reg_int))
{
std::wstring extracted(first, results[0].second);
if(5 > extracted.length() || 5 == extracted.length() && L"65535" >= extracted)
{
out_str = extracted;
first = results[0].second;
return true;
}
}
}
return false;
}
static void colorExtraction(const cv::Mat& src, // input HSV image
cv::Mat* dst, // specified color extracted binarized image
const double hue_lower, const double hue_upper, // hue thresholds
const double sat_lower, const double sat_upper, // satulation thresholds
const double val_lower, const double val_upper) // value thresholds
{
/* create imput image copy */
cv::Mat input_img = src.clone();
*dst = cv::Scalar::all(0);
/*
ref:
http://qiita.com/crakaC/items/65fab9d0b0ac29e68ab6
*/
/* create LookUp Table */
cv::Mat lut(256, 1, CV_8UC3);
for (int i=0; i<256; i++)
{
lut.at<cv::Vec3b>(i)[0] = (IsRange(hue_lower, hue_upper, Actual_Hue(i))) ? 255 : 0;
lut.at<cv::Vec3b>(i)[1] = (IsRange(sat_lower, sat_upper, Actual_Sat(i))) ? 255 : 0;
lut.at<cv::Vec3b>(i)[2] = (IsRange(val_lower, val_upper, Actual_Val(i))) ? 255 : 0;
}
/* apply LUT to input image */
cv::Mat extracted(input_img.rows, input_img.cols, CV_8UC3);
LUT(input_img, lut, extracted);
/* divide image into each channel */
std::vector<cv::Mat> channels;
split(extracted, channels);
/* create mask */
bitwise_and(channels[0], channels[1], *dst);
bitwise_and(*dst, channels[2], *dst);
} /* static void colorExtraction() */
ArgumentList::ArgumentList(CellMatrix cells,
std::string ErrorId)
{
CellValue empty;
unsigned long rows = cells.RowsInStructure();
unsigned long columns = cells.ColumnsInStructure();
if (rows == 0)
throw(std::string("Argument List requires non empty cell matix ")+ErrorId);
if (!cells(0,0).IsAString())
throw(std::string("a structure name must be specified for argument list class ")+ErrorId);
else
{
StructureName = cells(0,0).StringValueLowerCase();
cells(0,0) = empty;
}
{for (unsigned long i=1; i < columns; i++)
if (!cells(0,i).IsEmpty() )
throw("An argument list should only have the structure name on the first line: "+StructureName+ " " + ErrorId);
}
ErrorId +=" "+StructureName;
{for (unsigned long i=1; i < rows; i++)
for (unsigned long j=0; j < columns; j++)
if (cells(i,j).IsError())
GenerateThrow("Error Cell passed in ",i,j);}
unsigned long row=1UL;
while (row < rows)
{
unsigned long rowsDown=1;
unsigned column = 0;
while (column < columns)
{
if (cells(row,column).IsEmpty())
{
// check nothing else in row
while (column< columns)
{
if (!cells(row,column).IsEmpty())
GenerateThrow("data or value where unexpected.",row, column);
++column;
}
}
else // we have data
{
if (!cells(row,column).IsAString())
GenerateThrow("data where name expected.", row, column);
std::string thisName(cells(row,column).StringValueLowerCase());
if (thisName =="")
GenerateThrow("empty name not permissible.", row, column);
if (rows == row+1)
GenerateThrow("No space where data expected below name", row, column);
cells(row,column).clear();
// weird syntax to satisfy VC6
CellValue* belowPtr = &cells(row+1,column);
CellValue& cellBelow = *belowPtr;
if (cellBelow.IsEmpty())
GenerateThrow("Data expected below name", row, column);
if (cellBelow.IsANumber())
{
add(thisName, cellBelow.NumericValue());
column++;
cellBelow=empty;
}
else
if (cellBelow.IsBoolean())
{
add(thisName, cellBelow.BooleanValue());
column++;
cellBelow=empty;
}
else // ok its a string
{
std::string stringVal = cellBelow.StringValueLowerCase();
if ( (cellBelow.StringValueLowerCase() == "list") ||
(cellBelow.StringValueLowerCase() == "matrix") ||
(cellBelow.StringValueLowerCase() == "cells") )
{
bool nonNumeric = false;
CellMatrix extracted(ExtractCells(cells,row+2,column,ErrorId,thisName,nonNumeric));
if (cellBelow.StringValueLowerCase() == "list")
{
ArgumentList value(extracted,ErrorId+":"+thisName);
addList(thisName, extracted); //note not value
}
if (cellBelow.StringValueLowerCase() == "cells")
{
add(thisName,extracted);
}
if (cellBelow.StringValueLowerCase() == "matrix")
{
if (nonNumeric)
throw("Non numerical value in matrix argument :"+thisName+ " "+ErrorId);
MJMatrix value(extracted.RowsInStructure(),extracted.ColumnsInStructure());
for (unsigned long i=0; i < extracted.RowsInStructure(); i++)
for (unsigned long j=0; j < extracted.ColumnsInStructure(); j++)
ChangingElement(value,i,j) = extracted(i,j);
add(thisName,value);
}
cellBelow = empty;
rowsDown = maxi(rowsDown,extracted.RowsInStructure()+2);
column+= extracted.ColumnsInStructure();
}
else // ok its an array or boring string
{
if (cellBelow.StringValueLowerCase() == "array"
||cellBelow.StringValueLowerCase() == "vector" )
{
cellBelow.clear();
if (row+2>= rows)
throw(ErrorId+" data expected below array "+thisName);
unsigned long size = cells(row+2,column);
cells(row+2,column).clear();
if (row+2+size>=rows)
throw(ErrorId+" more data expected below array "+thisName);
MyArray theArray(size);
for (unsigned long i=0; i < size; i++)
{
theArray[i] = cells(row+3+i,column);
cells(row+3+i,column).clear();
}
add(thisName,theArray);
rowsDown = maxi(rowsDown,size+2);
column+=1;
}
else
{
std::string value = cellBelow.StringValueLowerCase();
add(thisName,value);
column++;
cellBelow=empty;
}
}
}
}
}
row+=rowsDown+1;
}
{for (unsigned long i=0; i < rows; i++)
for (unsigned long j=0; j < columns; j++)
if (!cells(i,j).IsEmpty())
{
GenerateThrow("extraneous data "+ErrorId,i,j);
}}
}
bool GazeTracker::SixLEDs::associateGlints(const std::vector<cv::Point2d> &glints,
cv::Point2f pc,
const std::vector<cv::Point3d> &glints_3d) {
const size_t sz = glints.size();
if(sz <= 1 || sz >= 7) {
return false;
}
/************************************************************************
* The GroupManager requires an integer coordinate vector. Therefore
* the given double precision vector must be copied to such a container.
************************************************************************/
std::vector<cv::Point> extracted(sz);
{
std::vector<cv::Point>::iterator eit = extracted.begin();
std::vector<cv::Point2d>::const_iterator git = glints.begin();
while(eit != extracted.end()) {
eit->x = (int)(git->x + 0.5);
eit->y = (int)(git->y + 0.5);
++eit;
++git;
}
}
/************************************************************************
* try to identify the glints
************************************************************************/
group::GroupManager grp;
if(!grp.identify(extracted, pc)) {
return false;
}
/************************************************************************
* Initialise the leds vector with null points
************************************************************************/
{
cv::Point3d null_vector(0, 0, 0);
std::vector<LED>::iterator ledit = leds.begin();
while(ledit != leds.end()) {
ledit->setGlintDirection3D(null_vector);
ledit->setGlint2D(cv::Point2d());
ledit->setLabel(-1);
++ledit;
}
}
/***********************************************************************
* Ask the best configuration directly from the pattern finder
**********************************************************************/
{
const group::Configuration &config = grp.getBestConfiguration();
std::vector<group::Element>::const_iterator elit = config.elements.begin();
std::vector<group::Element>::const_iterator end = config.elements.end();
/************************************************************************
* Set the identified glints for the leds, yes it is quite funny that a
* container that supposedly stores LED info, also stores info about the
* glint. Deal with it.
************************************************************************/
int i = 0;
while(elit != end) {
const cv::Point3d &curGlint = glints_3d[i];
/*
* The elements in std::vector<...> elements, are in the same order
* as the std::vector<...> glints_3d
*/
leds[elit->label].setGlintDirection3D(curGlint); // direction vector to the glint
leds[elit->label].setGlint2D(glints[i]); // direction vector to the glint
leds[elit->label].setLabel(elit->label);
++elit;
++i;
}
}
return true;
}
int main(int argc, char *argv[])
{
int daemon = 0;
pid_t pid, sid;
std::string config_file = "./qextractor.conf";
const char* const short_options = "hdc:";
const struct option long_options[] = {
{ "help", 0, NULL, 'h' },
{ "daemon", 0, NULL, 'd' },
{ "conf", 1, NULL, 'c' },
{ NULL, 0, NULL, 0 }
};
int next_option;
do {
next_option = getopt_long (argc, argv, short_options,
long_options, NULL);
switch (next_option) {
case 'd':
daemon = 1;
break;
case 'h':
print_usage(stdout, 0);
break;
case 'c':
config_file = optarg;
break;
case -1:
break;
case '?':
print_usage(stderr, 1);
default:
print_usage(stderr, 1);
}
} while (next_option != -1);
if (daemon) {
pid = fork();
if (pid < 0) {
exit(EXIT_FAILURE);
} else if (pid > 0) {
exit(EXIT_SUCCESS);
}
umask(0);
sid = setsid();
if (sid < 0) {
exit(EXIT_FAILURE);
}
}
google::InitGoogleLogging(argv[0]);
QApplication app(argc, argv);
app.setApplicationName("qextractor");
signal(SIGHUP, signal_handler);
signal(SIGTERM, signal_handler);
signal(SIGINT, signal_handler);
signal(SIGQUIT, signal_handler);
int extract_url_limit = 0;
int parent_predict_limit = 45;
std::string url_queue_host = "localhost";
uint16_t url_queue_port = 19854;
std::string crawled_queue_name = "crawled";
std::string crawled_queue_host = "localhost";
uint16_t crawled_queue_port = 7676;
std::string extracted_queue_name = "extracted";
std::string extracted_queue_host = "localhost";
uint16_t extracted_queue_port = 7676;
std::string new_url_queue_name = "new_url";
std::string new_url_queue_host = "localhost";
uint16_t new_url_queue_port = 7676;
// html2vdom options
int enable_proxy = 0;
std::string proxy_host = "localhost";
uint16_t proxy_port = 3128;
// qstore
std::string url_seen_server = "localhost:9850";
std::string html_db_server = "localhost:9860";
std::string record_db_server = "localhost:9870";
std::string media_db_server = "localhost:9880";
// qurlfilter
std::string qurlfilter_conf_file = "./qurlfilter.conf";
// qthumbscale
std::string qthumbscale_cascade = "./qthumbscale.conf";
double qthumbscale_scale = 1.3;
qcontent::QContentConfig *config = qcontent::QContentConfig::get_instance();
if (!config->parse_file(config_file)) {
LOG(FATAL) << "Parse config file error";
exit(-1);
}
extract_url_limit = config->get_integer("qextractor.extract_url_limit", extract_url_limit);
LOG(INFO) << "extract url limit: " << extract_url_limit;
parent_predict_limit = config->get_integer("qextractor.parent_predict_limit", parent_predict_limit);
LOG(INFO) << "parent predict limit: " << parent_predict_limit;
url_queue_host = config->get_string("url_queue.host", url_queue_host);
url_queue_port = (uint16_t) config->get_integer("url_queue.port", url_queue_port);
crawled_queue_name = config->get_string("crawled_queue.name", crawled_queue_name);
crawled_queue_host = config->get_string("crawled_queue.host", crawled_queue_host);
crawled_queue_port = (uint16_t) config->get_integer("crawled_queue.port", crawled_queue_port);
extracted_queue_name = config->get_string("extracted_queue.name", extracted_queue_name);
extracted_queue_host = config->get_string("extracted_queue.host", extracted_queue_host);
extracted_queue_port = (uint16_t) config->get_integer("extracted_queue.port", extracted_queue_port);
new_url_queue_name = config->get_string("new_url_queue.name", new_url_queue_name);
new_url_queue_host = config->get_string("new_url_queue.host", new_url_queue_host);
new_url_queue_port = (uint16_t) config->get_integer("new_url_queue.port", new_url_queue_port);
enable_proxy = config->get_integer("qhtml2vdom.enable_proxy", enable_proxy);
proxy_host = config->get_string("qhtml2vdom.proxy_host", proxy_host);
proxy_port = config->get_integer("qhtml2vdom.enable_port", proxy_port);
/*
url_seen_server = config->get_string("qstore.url_seen_db_server", url_seen_server);
html_db_server = config->get_string("qstore.html_db_server", html_db_server);
record_db_server = config->get_string("qstore.record_db_server", record_db_server);
media_db_server = config->get_string("qstore.media_db_server", media_db_server);
*/
qurlfilter_conf_file = config->get_string("qurlfilter.conf_file", qurlfilter_conf_file);
qthumbscale_cascade = config->get_string("qthumbscale.cascade_conf", qthumbscale_cascade);
if (enable_proxy) {
LOG(INFO) << "enabe proxy " << proxy_host << ":" << proxy_port;
QNetworkProxy proxy(QNetworkProxy::HttpProxy, QString::fromUtf8(proxy_host.c_str()), (quint16)proxy_port);
QNetworkProxy::setApplicationProxy(proxy);
}
qcontent::UrlQueue urlqueue(url_queue_host, url_queue_port);
qcontent::HubQueue crawled(crawled_queue_host.c_str(), crawled_queue_port, crawled_queue_name, 10000);
qcontent::HubQueue extracted(extracted_queue_host.c_str(), extracted_queue_port, extracted_queue_name, 10000);
qcontent::HubQueue new_url(new_url_queue_host.c_str(), new_url_queue_port, new_url_queue_name, 10000);
qcontent::QHtml2Vdom qhtml2vdom;
//qcontent::QStore store(url_seen_server, html_db_server, record_db_server, media_db_server);
qcontent::QStore store(config);
qcontent::QUrlFilter qurlfilter(qurlfilter_conf_file);
qcontent::QThumbScale qthumb;
if (!qthumb.init(qthumbscale_cascade, qthumbscale_scale)) {
LOG(FATAL) << "init qthumbscale error";
}
std::map<std::string, std::string> qclassifier_map;
config->prefix_values_map("qclassifier", qclassifier_map);
qcontent::QPredictConfig qclassifier_config(qclassifier_map);
qcontent::QPredict qclassifier;
if (!qclassifier.init(&qclassifier_config)) {
LOG(FATAL) << "init qclassifier fatal";
}
if (!qclassifier.load_qpredict_model()) {
LOG(FATAL) << "qclassifier load fatal";
}
std::map<std::string, std::string> qpredict_map;
config->prefix_values_map("qpredict", qpredict_map);
qcontent::QPredictConfig qpredict_config(qpredict_map);
qcontent::QPredict qpredict;
if (!qpredict.init(&qpredict_config)) {
LOG(FATAL) << "init qpredict fatal";
}
if (!qpredict.load_qpredict_model()) {
LOG(FATAL) << "qpredict load fatal";
}
std::string titlefilter_stat_file = config->get_string("qcontentextractor.qtitlefilter.stat_file", "");
qcontent::QExtractor qextractor(&crawled, &extracted, &new_url, &urlqueue, \
&qhtml2vdom, &store, &qurlfilter, &qthumb, &qclassifier, &qpredict,
titlefilter_stat_file
);
if (extract_url_limit > 0) {
qextractor.set_extract_url_limit(extract_url_limit);
}
if (parent_predict_limit > 0) {
qextractor.set_parent_predict_limit(parent_predict_limit);
}
global_qextractor = &qextractor;
qextractor.start();
return 0;
}
