KURL::List MediaDirNotify::toMediaURL(const KURL &url)
{
kdDebug(1219) << "MediaDirNotify::toMediaURL(" << url << ")" << endl;
KURL::List result;
const TQPtrList<Medium> list = m_mediaList.list();
TQPtrList<Medium>::const_iterator it = list.begin();
TQPtrList<Medium>::const_iterator end = list.end();
for (; it!=end; ++it)
{
const Medium *m = *it;
KURL base = m->prettyBaseURL();
if ( base.isParentOf(url) )
{
TQString path = KURL::relativePath(base.path(),
url.path());
KURL new_url("media:/"+m->name()+"/"+path );
new_url.cleanPath();
result.append(new_url);
}
}
kdDebug(1219) << result << endl;
return result;
}
/**
* new_packetizer()
* * etch_packetizer public constructor
* @param itd i_transportdata interface owned by caller
* @param uri a URI string owned by caller.
* @param resources a resources map owned by caller.
*/
etch_packetizer* new_packetizer (i_transportdata* itd, wchar_t* uri, etch_resources* resxmap)
{
etch_url* url = new_url(uri);
etch_packetizer* packetizer = new_packetizer_a (itd, url, resxmap);
etch_object_destroy(url);
return packetizer;
}
/**
* @param s URL string to be parsed into its components.
*
* @returns A @c url object corresponding to the specified string.
*
* @throws boost::system::system_error Thrown when the URL string is invalid.
*/
static url from_string(const char* s)
{
boost::system::error_code ec;
url new_url(from_string(s, ec));
if (ec)
{
boost::system::system_error ex(ec);
boost::throw_exception(ex);
}
return new_url;
}
void PluginPart::requestURL(const QString &url, const QString &target)
{
kdDebug(1432) << "PluginPart::requestURL( url=" << url << ", target=" << target << endl;
KURL new_url(this->url(), url);
KParts::URLArgs args;
args.frameName = target;
args.setDoPost(false);
emit _extension->openURLRequest(new_url, args);
}
void PluginPart::postURL(const QString &url, const QString &target, const QByteArray &data, const QString &mime)
{
kdDebug(1432) << "PluginPart::postURL( url=" << url << ", target=" << target << endl;
KURL new_url(this->url(), url);
KParts::URLArgs args;
args.setDoPost(true);
args.frameName = target;
args.postData = data;
args.setContentType(mime);
emit _extension->openURLRequest(new_url, args);
}
static KIO::CopyJob *chooseAndPaste(const KURL &u, QMimeSource *data, const QValueVector< QCString > &formats, const QString &text, QWidget *widget,
bool clipboard)
{
QStringList formatLabels;
for(uint i = 0; i < formats.size(); ++i)
{
const QCString &fmt = formats[i];
KMimeType::Ptr mime = KMimeType::mimeType(fmt);
if(mime != KMimeType::defaultMimeTypePtr())
formatLabels.append(i18n("%1 (%2)").arg(mime->comment()).arg(fmt));
else
formatLabels.append(fmt);
}
QString dialogText(text);
if(dialogText.isEmpty())
dialogText = i18n("Filename for clipboard content:");
KIO::PasteDialog dlg(QString::null, dialogText, QString::null, formatLabels, widget, clipboard);
if(dlg.exec() != KDialogBase::Accepted)
return 0;
if(clipboard && dlg.clipboardChanged())
{
KMessageBox::sorry(widget, i18n("The clipboard has changed since you used 'paste': "
"the chosen data format is no longer applicable. "
"Please copy again what you wanted to paste."));
return 0;
}
const QString result = dlg.lineEditText();
const QCString chosenFormat = formats[dlg.comboItem()];
kdDebug() << " result=" << result << " chosenFormat=" << chosenFormat << endl;
KURL new_url(u);
new_url.addPath(result);
// if "data" came from QClipboard, then it was deleted already - by a nice 0-seconds timer
// In that case, get it again. Let's hope the user didn't copy something else meanwhile :/
if(clipboard)
{
data = QApplication::clipboard()->data();
}
const QByteArray ba = data->encodedData(chosenFormat);
return pasteDataAsyncTo(new_url, ba);
}
void GetFileUploadLinkRequest::requestSuccess(QNetworkReply& reply)
{
QString reply_content(reply.readAll());
do {
if (reply_content.size() <= 2)
break;
reply_content.remove(0, 1);
reply_content.chop(1);
QUrl new_url(reply_content);
if (!new_url.isValid())
break;
emit success(reply_content);
return;
} while (0);
emit failed(ApiError::fromHttpError(500));
}
int
main(int argc, char *argv[])
{
int x;
int result;
DATA D = new_data();
ARRAY urls = new_array();
CREW crew;
LINES *lines;
CLIENT *client;
pthread_t cease;
pthread_t timer;
pthread_attr_t scope_attr;
void *statusp;
sigset_t sigs;
sigemptyset(&sigs);
sigaddset(&sigs, SIGHUP);
sigaddset(&sigs, SIGINT);
sigaddset(&sigs, SIGALRM);
sigaddset(&sigs, SIGTERM);
sigprocmask(SIG_BLOCK, &sigs, NULL);
lines = xcalloc(1, sizeof *lines);
lines->index = 0;
lines->line = NULL;
memset(&my, 0, sizeof(struct CONFIG));
parse_rc_cmdline(argc, argv);
if (init_config() < 0) { /* defined in init.h */
exit( EXIT_FAILURE ); /* polly was a girl... */
}
parse_cmdline(argc, argv); /* defined above */
ds_module_check(); /* check config integ */
/**
* XXX: we should consider moving the following
* if-checks into the ds_module_check
*/
if (my.config) {
show_config(TRUE);
}
if (my.url != NULL) {
my.length = 1;
} else {
my.length = read_cfg_file(lines, my.file);
}
if (my.reps < 0) {
my.reps = my.length;
}
if (my.length == 0) {
display_help();
}
/* cookie is an EXTERN, defined in setup */
cookie = xcalloc(sizeof(COOKIE), 1);
cookie->first = NULL;
if ((result = pthread_mutex_init( &(cookie->mutex), NULL)) !=0) {
NOTIFY(FATAL, "pthread_mutex_init" );
}
/* memory allocation for threads and clients */
client = xcalloc(my.cusers, sizeof(CLIENT));
if ((crew = new_crew(my.cusers, my.cusers, FALSE)) == NULL) {
NOTIFY(FATAL, "unable to allocate memory for %d simulated browser", my.cusers);
}
/**
* determine the source of the url(s),
* command line or file, and add them
* to the urls struct.
*/
if (my.url != NULL) {
URL tmp = new_url(my.url);
url_set_ID(tmp, 0);
if (my.get && url_get_method(tmp) != POST && url_get_method(tmp) != PUT) {
url_set_method(tmp, my.method);
}
array_npush(urls, tmp, URLSIZE); // from cmd line
} else {
for (x = 0; x < my.length; x++) {
URL tmp = new_url(lines->line[x]);
url_set_ID(tmp, x);
array_npush(urls, tmp, URLSIZE);
}
}
/**
* display information about the siege
* to the user and prepare for verbose
* output if necessary.
*/
if (!my.get && !my.quiet) {
fprintf(stderr, "** ");
display_version(FALSE);
fprintf(stderr, "** Preparing %d concurrent users for battle.\n", my.cusers);
fprintf(stderr, "The server is now under siege...");
if (my.verbose) { fprintf(stderr, "\n"); }
}
/**
* record start time before spawning threads
* as the threads begin hitting the server as
* soon as they are created.
*/
data_set_start(D);
/**
* for each concurrent user, spawn a thread and
* loop until condition or pthread_cancel from the
* handler thread.
*/
pthread_attr_init(&scope_attr);
pthread_attr_setscope(&scope_attr, PTHREAD_SCOPE_SYSTEM);
#if defined(_AIX)
/* AIX, for whatever reason, defies the pthreads standard and *
* creates threads detached by default. (see pthread.h on AIX) */
pthread_attr_setdetachstate(&scope_attr, PTHREAD_CREATE_JOINABLE);
#endif
/**
* invoke OpenSSL's thread safety
*/
#ifdef HAVE_SSL
SSL_thread_setup();
#endif
/**
* create the signal handler and timer; the
* signal handler thread (cease) responds to
* ctrl-C (sigterm) and the timer thread sends
* sigterm to cease on time out.
*/
if ((result = pthread_create(&cease, NULL, (void*)sig_handler, (void*)crew)) < 0) {
NOTIFY(FATAL, "failed to create handler: %d\n", result);
}
if (my.secs > 0) {
if ((result = pthread_create(&timer, NULL, (void*)siege_timer, (void*)cease)) < 0) {
NOTIFY(FATAL, "failed to create handler: %d\n", result);
}
}
/**
* loop until my.cusers and create a corresponding thread...
*/
for (x = 0; x < my.cusers && crew_get_shutdown(crew) != TRUE; x++) {
client[x].id = x;
client[x].bytes = 0;
client[x].time = 0.0;
client[x].hits = 0;
client[x].code = 0;
client[x].ok200 = 0;
client[x].fail = 0;
client[x].urls = urls;
client[x].auth.www = 0;
client[x].auth.proxy = 0;
client[x].auth.type.www = BASIC;
client[x].auth.type.proxy = BASIC;
client[x].rand_r_SEED = urandom();
result = crew_add(crew, (void*)start_routine, &(client[x]));
if (result == FALSE) {
my.verbose = FALSE;
fprintf(stderr, "Unable to spawn additional threads; you may need to\n");
fprintf(stderr, "upgrade your libraries or tune your system in order\n");
fprintf(stderr, "to exceed %d users.\n", my.cusers);
NOTIFY(FATAL, "system resources exhausted");
}
} /* end of for pthread_create */
crew_join(crew, TRUE, &statusp);
#ifdef HAVE_SSL
SSL_thread_cleanup();
#endif
/**
* collect all the data from all the threads that
* were spawned by the run.
*/
for (x = 0; x < ((crew_get_total(crew) > my.cusers ||
crew_get_total(crew)==0 ) ? my.cusers : crew_get_total(crew)); x++) {
data_increment_count(D, client[x].hits);
data_increment_bytes(D, client[x].bytes);
data_increment_total(D, client[x].time);
data_increment_code (D, client[x].code);
data_increment_ok200(D, client[x].ok200);
data_increment_fail (D, client[x].fail);
data_set_highest (D, client[x].himark);
data_set_lowest (D, client[x].lomark);
client[x].rand_r_SEED = urandom();
} /* end of stats accumulation */
/**
* record stop time
*/
data_set_stop(D);
/**
* cleanup crew
*/
crew_destroy(crew);
for (x = 0; x < my.cusers; x++) {
// XXX: TODO
//digest_challenge_destroy(client[x].auth.wwwchlg);
//digest_credential_destroy(client[x].auth.wwwcred);
//digest_challenge_destroy(client[x].auth.proxychlg);
//digest_credential_destroy(client[x].auth.proxycred);
}
array_destroy(my.lurl);
xfree(client);
if (my.get) {
if (data_get_ok200(D) > 0) {
exit(EXIT_SUCCESS);
} else {
if (!my.quiet) echo("[done]\n");
exit(EXIT_FAILURE);
}
}
/**
* take a short nap for cosmetic effect
* this does NOT affect performance stats.
*/
pthread_usleep_np(10000);
if (my.verbose)
fprintf(stderr, "done.\n");
else
fprintf(stderr, "\b done.\n");
/**
* prepare and print statistics.
*/
if (my.failures > 0 && my.failed >= my.failures) {
fprintf(stderr, "%s aborted due to excessive socket failure; you\n", program_name);
fprintf(stderr, "can change the failure threshold in $HOME/.%src\n", program_name);
}
fprintf(stderr, "\nTransactions:\t\t%12u hits\n", data_get_count(D));
fprintf(stderr, "Availability:\t\t%12.2f %%\n", data_get_count(D)==0 ? 0 :
(double)data_get_count(D) /
(data_get_count(D)+my.failed)
*100
);
fprintf(stderr, "Elapsed time:\t\t%12.2f secs\n", data_get_elapsed(D));
fprintf(stderr, "Data transferred:\t%12.2f MB\n", data_get_megabytes(D)); /*%12llu*/
fprintf(stderr, "Response time:\t\t%12.3f secs\n", data_get_response_time(D));
fprintf(stderr, "Transaction rate:\t%12.2f trans/sec\n", data_get_transaction_rate(D));
fprintf(stderr, "Throughput:\t\t%12.2f MB/sec\n", data_get_throughput(D));
fprintf(stderr, "Concurrency:\t\t%12.2f\n", data_get_concurrency(D));
fprintf(stderr, "Successful transactions:%12u\n", data_get_code(D));
if (my.debug) {
fprintf(stderr, "HTTP OK received:\t%12u\n", data_get_ok200(D));
}
fprintf(stderr, "Failed transactions:\t%12u\n", my.failed);
fprintf(stderr, "Longest transaction:\t%12.3f\n", data_get_highest(D));
fprintf(stderr, "Shortest transaction:\t%12.3f\n", data_get_lowest(D));
fprintf(stderr, " \n");
if(my.mark) mark_log_file(my.markstr);
if(my.logging) log_transaction(D);
data_destroy(D);
if (my.url == NULL) {
for (x = 0; x < my.length; x++)
xfree(lines->line[x]);
xfree(lines->line);
xfree(lines);
} else {
xfree(lines->line);
xfree(lines);
}
pthread_mutex_destroy( &(cookie->mutex));
/**
* I should probably take a deeper look
* at cookie content to free it but at
* this point we're two lines from exit
*/
xfree (cookie);
xfree (my.url);
exit(EXIT_SUCCESS);
} /* end of int main **/
int main(int argc, char *argv[])
{
int daemon = 0;
pid_t pid, sid;
std::string config_file = "./qextractor.conf";
const char* const short_options = "hdc:";
const struct option long_options[] = {
{ "help", 0, NULL, 'h' },
{ "daemon", 0, NULL, 'd' },
{ "conf", 1, NULL, 'c' },
{ NULL, 0, NULL, 0 }
};
int next_option;
do {
next_option = getopt_long (argc, argv, short_options,
long_options, NULL);
switch (next_option) {
case 'd':
daemon = 1;
break;
case 'h':
print_usage(stdout, 0);
break;
case 'c':
config_file = optarg;
break;
case -1:
break;
case '?':
print_usage(stderr, 1);
default:
print_usage(stderr, 1);
}
} while (next_option != -1);
if (daemon) {
pid = fork();
if (pid < 0) {
exit(EXIT_FAILURE);
} else if (pid > 0) {
exit(EXIT_SUCCESS);
}
umask(0);
sid = setsid();
if (sid < 0) {
exit(EXIT_FAILURE);
}
}
google::InitGoogleLogging(argv[0]);
QApplication app(argc, argv);
app.setApplicationName("qextractor");
signal(SIGHUP, signal_handler);
signal(SIGTERM, signal_handler);
signal(SIGINT, signal_handler);
signal(SIGQUIT, signal_handler);
int extract_url_limit = 0;
int parent_predict_limit = 45;
std::string url_queue_host = "localhost";
uint16_t url_queue_port = 19854;
std::string crawled_queue_name = "crawled";
std::string crawled_queue_host = "localhost";
uint16_t crawled_queue_port = 7676;
std::string extracted_queue_name = "extracted";
std::string extracted_queue_host = "localhost";
uint16_t extracted_queue_port = 7676;
std::string new_url_queue_name = "new_url";
std::string new_url_queue_host = "localhost";
uint16_t new_url_queue_port = 7676;
// html2vdom options
int enable_proxy = 0;
std::string proxy_host = "localhost";
uint16_t proxy_port = 3128;
// qstore
std::string url_seen_server = "localhost:9850";
std::string html_db_server = "localhost:9860";
std::string record_db_server = "localhost:9870";
std::string media_db_server = "localhost:9880";
// qurlfilter
std::string qurlfilter_conf_file = "./qurlfilter.conf";
// qthumbscale
std::string qthumbscale_cascade = "./qthumbscale.conf";
double qthumbscale_scale = 1.3;
qcontent::QContentConfig *config = qcontent::QContentConfig::get_instance();
if (!config->parse_file(config_file)) {
LOG(FATAL) << "Parse config file error";
exit(-1);
}
extract_url_limit = config->get_integer("qextractor.extract_url_limit", extract_url_limit);
LOG(INFO) << "extract url limit: " << extract_url_limit;
parent_predict_limit = config->get_integer("qextractor.parent_predict_limit", parent_predict_limit);
LOG(INFO) << "parent predict limit: " << parent_predict_limit;
url_queue_host = config->get_string("url_queue.host", url_queue_host);
url_queue_port = (uint16_t) config->get_integer("url_queue.port", url_queue_port);
crawled_queue_name = config->get_string("crawled_queue.name", crawled_queue_name);
crawled_queue_host = config->get_string("crawled_queue.host", crawled_queue_host);
crawled_queue_port = (uint16_t) config->get_integer("crawled_queue.port", crawled_queue_port);
extracted_queue_name = config->get_string("extracted_queue.name", extracted_queue_name);
extracted_queue_host = config->get_string("extracted_queue.host", extracted_queue_host);
extracted_queue_port = (uint16_t) config->get_integer("extracted_queue.port", extracted_queue_port);
new_url_queue_name = config->get_string("new_url_queue.name", new_url_queue_name);
new_url_queue_host = config->get_string("new_url_queue.host", new_url_queue_host);
new_url_queue_port = (uint16_t) config->get_integer("new_url_queue.port", new_url_queue_port);
enable_proxy = config->get_integer("qhtml2vdom.enable_proxy", enable_proxy);
proxy_host = config->get_string("qhtml2vdom.proxy_host", proxy_host);
proxy_port = config->get_integer("qhtml2vdom.enable_port", proxy_port);
/*
url_seen_server = config->get_string("qstore.url_seen_db_server", url_seen_server);
html_db_server = config->get_string("qstore.html_db_server", html_db_server);
record_db_server = config->get_string("qstore.record_db_server", record_db_server);
media_db_server = config->get_string("qstore.media_db_server", media_db_server);
*/
qurlfilter_conf_file = config->get_string("qurlfilter.conf_file", qurlfilter_conf_file);
qthumbscale_cascade = config->get_string("qthumbscale.cascade_conf", qthumbscale_cascade);
if (enable_proxy) {
LOG(INFO) << "enabe proxy " << proxy_host << ":" << proxy_port;
QNetworkProxy proxy(QNetworkProxy::HttpProxy, QString::fromUtf8(proxy_host.c_str()), (quint16)proxy_port);
QNetworkProxy::setApplicationProxy(proxy);
}
qcontent::UrlQueue urlqueue(url_queue_host, url_queue_port);
qcontent::HubQueue crawled(crawled_queue_host.c_str(), crawled_queue_port, crawled_queue_name, 10000);
qcontent::HubQueue extracted(extracted_queue_host.c_str(), extracted_queue_port, extracted_queue_name, 10000);
qcontent::HubQueue new_url(new_url_queue_host.c_str(), new_url_queue_port, new_url_queue_name, 10000);
qcontent::QHtml2Vdom qhtml2vdom;
//qcontent::QStore store(url_seen_server, html_db_server, record_db_server, media_db_server);
qcontent::QStore store(config);
qcontent::QUrlFilter qurlfilter(qurlfilter_conf_file);
qcontent::QThumbScale qthumb;
if (!qthumb.init(qthumbscale_cascade, qthumbscale_scale)) {
LOG(FATAL) << "init qthumbscale error";
}
std::map<std::string, std::string> qclassifier_map;
config->prefix_values_map("qclassifier", qclassifier_map);
qcontent::QPredictConfig qclassifier_config(qclassifier_map);
qcontent::QPredict qclassifier;
if (!qclassifier.init(&qclassifier_config)) {
LOG(FATAL) << "init qclassifier fatal";
}
if (!qclassifier.load_qpredict_model()) {
LOG(FATAL) << "qclassifier load fatal";
}
std::map<std::string, std::string> qpredict_map;
config->prefix_values_map("qpredict", qpredict_map);
qcontent::QPredictConfig qpredict_config(qpredict_map);
qcontent::QPredict qpredict;
if (!qpredict.init(&qpredict_config)) {
LOG(FATAL) << "init qpredict fatal";
}
if (!qpredict.load_qpredict_model()) {
LOG(FATAL) << "qpredict load fatal";
}
std::string titlefilter_stat_file = config->get_string("qcontentextractor.qtitlefilter.stat_file", "");
qcontent::QExtractor qextractor(&crawled, &extracted, &new_url, &urlqueue, \
&qhtml2vdom, &store, &qurlfilter, &qthumb, &qclassifier, &qpredict,
titlefilter_stat_file
);
if (extract_url_limit > 0) {
qextractor.set_extract_url_limit(extract_url_limit);
}
if (parent_predict_limit > 0) {
qextractor.set_parent_predict_limit(parent_predict_limit);
}
global_qextractor = &qextractor;
qextractor.start();
return 0;
}
