int Spider_Executor::worker_work(void* param)
{
Job* job=(Job*)param;
if ( job!=NULL )
{
UrlPtr& url_ptr=job->url_ptr_;
int status_code=job->http_client_->recv_response(job->sock_,&url_ptr->response, url_ptr->length);
if ( status_code==200 )
{
if ( url_ptr->belong!=NULL && url_ptr->length>0 )
{
((Spider_WebSite*)url_ptr->belong)->process(url_ptr);
}
}
else if (status_code==302 && url_ptr->response!=NULL && strncmp(url_ptr->response,"http://",7)==0 )
{
UrlPtr new_url_ptr=create_url(url_ptr->response, url_ptr->type);
new_url_ptr->belong=url_ptr->belong;
Spider_Executor::instance().put_url(new_url_ptr);
}
else if( status_code==-1 || url_ptr->response==NULL )
{
//重新下载
UrlPtr new_url_ptr=create_url(url_ptr->url, url_ptr->type);
new_url_ptr->belong=url_ptr->belong;
Spider_Executor::instance().put_url(new_url_ptr);
}
}
return 0;
}
void
g_flickr_photosets_getList (GFlickr *f,
const gchar *user_id,
GFlickrListCb callback,
gpointer user_data)
{
/* Either we insert user_id or not */
gint params_no = (user_id == NULL) ? 1 : 2;
gchar *params[2];
params[0] = g_strdup_printf ("method=%s", FLICKR_PHOTOSETS_GETLIST_METHOD);
if (user_id != NULL)
params[1] = g_strdup_printf ("user_id=%s", user_id);
gchar *request = create_url (f, params, params_no);
free_params (params, params_no);
GFlickrData *gfd = g_slice_new (GFlickrData);
gfd->flickr = g_object_ref (f);
gfd->parse_xml = process_photosetslist_result;
gfd->list_cb = callback;
gfd->user_data = user_data;
read_url_async (f, request, gfd);
g_free (request);
}
void
g_flickr_photosets_getPhotos (GFlickr *f,
const gchar *photoset_id,
gint page,
GFlickrListCb callback,
gpointer user_data)
{
g_return_if_fail (G_IS_FLICKR (f));
g_return_if_fail (photoset_id);
gchar *params[6];
params[0] = g_strdup_printf ("photoset_id=%s", photoset_id);
params[1] = g_strdup ("extras=date_taken,owner_name,url_o,url_t,media");
params[2] = g_strdup ("media=photos");
params[3] = g_strdup_printf ("page=%d", page);
params[4] = g_strdup_printf ("per_page=%d", f->priv->per_page);
params[5] = g_strdup_printf ("method=%s", FLICKR_PHOTOSETS_GETPHOTOS_METHOD);
gchar *request = create_url (f, params, 6);
free_params (params, 6);
GFlickrData *gfd = g_slice_new (GFlickrData);
gfd->flickr = g_object_ref (f);
gfd->parse_xml = process_photosetsphotos_result;
gfd->list_cb = callback;
gfd->user_data = user_data;
read_url_async (f, request, gfd);
g_free (request);
}
audio_player::audio_player(std::string const& source, std::string const& root_path, QWidget* player_container
, ncl_widget* ncl, gntl::structure::composed::component_location
<std::string, std::string> location
, ncl::executor* executor_)
: common_player(executor_, location), sound_level_temporary(0)
, media_object(0), audio_output(0)
{
std::auto_ptr<audio_detail::MediaObject> media_object
(new audio_detail::MediaObject(player_container, ncl, this));
std::auto_ptr<Phonon::AudioOutput> audio_output(new Phonon::AudioOutput(player_container));
QObject::connect(media_object.get(), SIGNAL(stateChanged(Phonon::State, Phonon::State))
, media_object.get(), SLOT(stateChanged_impl(Phonon::State, Phonon::State)));
Phonon::Path phonon_path = Phonon::createPath(media_object.get(), audio_output.get());
if(phonon_path.isValid())
{
QUrl url = create_url(source, root_path);
Phonon::MediaSource media_source(url);
media_object->setCurrentSource(media_source);
media_object->play();
this->media_object = media_object.release();
this->audio_output = audio_output.release();
}
else
{
ncl->error_occurred("Error creating audio Path");
throw std::runtime_error("Error creating audio Path");
}
}
user *retrieve_graph_user(const char *access_token)
{
int id;
const char *name;
char *url = create_url("https://graph.facebook.com/me?fields=name&access_token=", access_token);
json_object *jobj;
json_object *name_obj;
json_object *id_obj;
user *u;
jobj = http_get_request_json(url);
json_object_object_get_ex(jobj, "name", &name_obj);
json_object_object_get_ex(jobj, "id", &id_obj);
name = json_object_get_string(name_obj);
id = (unsigned long) atoi(json_object_get_string(id_obj));
user *me = create_user(id, name);
json_object_put(jobj);
free(url);
return me;
}
/*******************************
get_response_page
return -1 on error
******************************/
static int get_response_page(internal_t * internal)
{
char * url;
time_t time_current;
url = create_url(internal);
if(url == NULL) {
printd(DEBUG_ERROR,"Can't get URL from Yandex engine\n");
return -1;
}
time_current = time(NULL);
while( time_current < time_last_request + TIME_BETWEEN_REQUEST ) {
printd(DEBUG_ERROR,"Waiting a bit before next request\n");
usleep(1000000);
time_current = time(NULL);
}
time_last_request = time(NULL);
if ( web_to_memory(url,internal->page) == -1 ) {
printd(DEBUG_ERROR,"web_to_memory error\n");
free(url);
return -1;
}
free(url);
internal->page_num++;
return 0;
}
void
g_flickr_tags_getHotList (GFlickr *f,
gint count,
GFlickrListCb callback,
gpointer user_data)
{
g_return_if_fail (G_IS_FLICKR (f));
gchar *params[2];
params[0] = g_strdup_printf ("count=%d", count);
params[1] = g_strdup_printf ("method=%s", FLICKR_TAGS_GETHOTLIST_METHOD);
gchar *request = create_url (f, params, 2);
free_params (params, 2);
GFlickrData *gfd = g_slice_new (GFlickrData);
gfd->flickr = g_object_ref (f);
gfd->parse_xml = process_taglist_result;
gfd->list_cb = callback;
gfd->user_data = user_data;
read_url_async (f, request, gfd);
g_free (request);
}
void
g_flickr_auth_checkToken (GFlickr *f,
const gchar *token,
GFlickrHashTableCb callback,
gpointer user_data)
{
gchar *request;
gchar *params[1];
g_return_if_fail (G_IS_FLICKR (f));
g_return_if_fail (token);
g_return_if_fail (callback);
params[0] = g_strdup_printf ("method=%s", FLICKR_OAUTH_CHECKTOKEN_METHOD);
request = create_url (f, params, 1);
free_params (params, 1);
GFlickrData *gfd = g_slice_new (GFlickrData);
gfd->flickr = g_object_ref (f);
gfd->parse_xml = process_token_result;
gfd->hashtable_cb = callback;
gfd->user_data = user_data;
read_url_async (f, request, gfd);
g_free (request);
}
void
g_flickr_photos_getInfo (GFlickr *f,
const gchar *photo_id,
GFlickrHashTableCb callback,
gpointer user_data)
{
g_return_if_fail (G_IS_FLICKR (f));
gchar *params[2];
params[0] = g_strdup_printf ("photo_id=%s", photo_id);
params[1] = g_strdup_printf ("method=%s", FLICKR_PHOTOS_GETINFO_METHOD);
gchar *request = create_url (f, params, 2);
free_params (params, 2);
GFlickrData *gfd = g_slice_new (GFlickrData);
gfd->flickr = g_object_ref (f);
gfd->parse_xml = process_photo_result;
gfd->hashtable_cb = callback;
gfd->user_data = user_data;
read_url_async (f, request, gfd);
g_free (request);
}
int m2x_client_get(m2x_context *ctx, const char *path, char **out)
{
curl_write_context *write_ctx;
char *url;
CURLcode res;
write_ctx = create_write_context(ctx);
url = create_url(ctx, path);
curl_easy_reset(ctx->curl);
curl_easy_setopt(ctx->curl, CURLOPT_URL, url);
m2x_free(url);
if (ctx->verbose) { curl_easy_setopt(ctx->curl, CURLOPT_VERBOSE, 1); }
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, write_ctx);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, write_callback);
res = curl_easy_perform(ctx->curl);
if (res != CURLE_OK) {
fprintf(stderr, "GET failed: %s\n",
curl_easy_strerror(res));
release_write_context(write_ctx, 1);
return res;
}
if (out != NULL) {
*out = write_ctx->p;
} else {
m2x_free(write_ctx->p);
}
release_write_context(write_ctx, 0);
return res;
}
void fill (void)
{
for ( int i = 0; i < UMAX; i++ )
{
U u (create_url (i));
uset_.insert (u.get ());
std::cout << "+" << std::flush;
cnt_++;
}
}
void Director::add_tr(std::ostringstream& ss, const std::string& path, const Entry& entry) {
if(entry.name() == ".")
return;
bool isFile = entry.is_file();
ss << "<tr>";
add_td(ss, "type", get_icon(entry));
add_td(ss, "file", create_url(path, entry.name()));
isFile ? add_td(ss, "size", human_readable_size(entry.size())) : add_td(ss, "size", "-");
isFile ? add_td(ss, "modified", "N/A") : add_td(ss, "modified", "-");
ss << "</tr>";
}
void
g_flickr_photos_search (GFlickr *f,
const gchar *user_id,
const gchar *text,
const gchar *tags,
gint page,
GFlickrListCb callback,
gpointer user_data)
{
g_return_if_fail (G_IS_FLICKR (f));
if (user_id == NULL) {
user_id = "";
}
if (text == NULL) {
text = "";
}
if (tags == NULL) {
tags = "";
}
gchar *params[8];
params[0] = g_strdup ("extras=date_taken,owner_name,url_0,url_t");
params[1] = g_strdup ("media=photos");
params[2] = g_strdup_printf ("user_id=%s", user_id);
params[3] = g_strdup_printf ("page=%d", page);
params[4] = g_strdup_printf ("per_page=%d", f->priv->per_page);
params[5] = g_strdup_printf ("tags=%s", tags);
params[6] = g_strdup_printf ("text=%s", text);
params[7] = g_strdup_printf ("method=%s", FLICKR_PHOTOS_SEARCH_METHOD);
/* Build the request */
gchar *request = create_url (f, params, 8);
free_params (params, 8);
GFlickrData *gfd = g_slice_new (GFlickrData);
gfd->flickr = g_object_ref (f);
gfd->parse_xml = process_photolist_result;
gfd->list_cb = callback;
gfd->user_data = user_data;
read_url_async (f, request, gfd);
g_free (request);
}
void roadmap_groups_show(void){
RMBrowserAttributes attrs;
int flags;
if (!roadmap_groups_feature_enabled())
return;
flags = BROWSER_FLAG_TITLE_BTN_LEFT1|BROWSER_FLAG_TITLE_BTN_LEFT2|BROWSER_FLAG_TITLE_BTN_RIGHT2;
attrs.on_close_cb = on_close_cb;
configure_browser( &attrs );
roadmap_browser_show_extended( create_url(), flags, &attrs );
}
void check (void)
{
for ( int i = UMAX - 1; i >= 0; i-- )
{
U u (create_url (i));
if ( uset_.find (u.get ()) != uset_.end () )
{
std::cout << "-" << std::flush;
uset_.erase (u.get ());
cnt_--;
}
else
{
std::cout << "." << std::flush;
}
}
}
/*******************************
get_response_page
return -1 on error
******************************/
static int get_response_page(internal_t * internal)
{
char * url;
url = create_url(internal);
if(url == NULL) {
printd(DEBUG_ERROR,"Can't get URL from Wikimedia engine\n");
return -1;
}
if ( web_to_memory(url,internal->page) == -1 ) {
printd(DEBUG_ERROR,"web_to_memory error\n");
free(url);
return -1;
}
free(url);
return 0;
}
int main()
{
/* toggle_bit */
ok(7 == toggle_bit(0b0101, 1));
ok(0b0100 == toggle_bit(0b0101, 0));
ok(0b0111 == toggle_bit(0b1111, 3));
/* split_url */
{
char *name, *suffix;
name = malloc(256*sizeof(char));
ok(0 == split_url(&name, &suffix, "github.com"));
ok(0 == strcmp(name, "github"));
ok(0 == strcmp(suffix, "com"));
ok(0 != split_url(&name, &suffix, "www.github.com"));
free(name);
}
/* is_valid_str */
ok(is_valid_str("github", 6));
ok(!is_valid_str("github.com", 10));
ok(is_valid_str("g1thub", 6));
/* bitsquat_char */
{
char c = 'a';
ok('a' != bitsquat_char(c));
}
/* random_loc */
{
size_t loc = random_loc("github");
ok(loc > 0 && loc < 6);
}
/* create_url */
{
struct Url m_url;
m_url = create_url("github.com");
ok(0 == strcmp(m_url.name, "github"));
ok(0 == strcmp(m_url.suffix, "com"));
free_url(m_url);
}
/* generate_entries and bitsquat_entries */
{
struct BSentries bs_entries;
struct Url url = create_url("github.com");
bs_entries = generate_entries(url, 3);
ok(3 == bs_entries.n_names);
ok(0 == strcmp(bs_entries.suffix, "com"));
ok(0 == strcmp(bs_entries.names[0], "github"));
bitsquat_entries(bs_entries);
ok(0 != strcmp(bs_entries.names[0], "github"));
free_url(url);
free_bs_entries(bs_entries);
}
return 0;
}
////////////////////////////////////////////////////////////////////////
// constructor
advertdirectory_cpi_impl::advertdirectory_cpi_impl (proxy * p,
cpi_info const & info,
saga::ini::ini const & glob_ini,
saga::ini::ini const & adap_ini,
TR1::shared_ptr <saga::adaptor> adaptor)
: saga::adaptors::v1_0::advert_directory_cpi <advertdirectory_cpi_impl> (p, info, adaptor, cpi::Noflags)
{
saga::url advert_url;
instance_data data (this);
advert_url = data->location_.clone();
std::string path;
int port;
path = advert_url.get_path();
if (path.empty())
path = "/"; // root only
std::string host(advert_url.get_host());
if (host.empty()) {
SAGA_OSSTREAM strm;
strm << "advert::advert_cpi_impl::init: "
"cannot handle advert entry name: "
<< advert_url.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::IncorrectURL);
}
std::string scheme(advert_url.get_scheme());
if (!scheme.empty() && scheme != "hbase" && scheme != "any") {
SAGA_OSSTREAM strm;
strm << "advert::advert_cpi_impl::init: "
"cannot handle advert entry name: " << advert_url.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::IncorrectURL);
}
port = advert_url.get_port();
boost::shared_ptr<TTransport> socket(new TSocket(host, port));
transport_ = boost::shared_ptr<TTransport>(new TBufferedTransport(socket));
boost::shared_ptr<TProtocol> protocol(new TBinaryProtocol(transport_));
client_ = new HbaseClient(protocol);
transport_->open();
saga::advert::flags mode = (saga::advert::flags)data->mode_;
if(path == "/") {
}
else if (((mode & saga::advert::Create) || (mode & saga::advert::CreateParents)) &&
(mode & saga::advert::Exclusive)) {
if(!url_exists(*client_, advert_url.get_url())) {
SAGA_OSSTREAM strm;
strm << "advert::advert_cpi_impl::init: "
"advert already exists: " << advert_url.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::AlreadyExists);
}
}
else if ((mode & saga::advert::Create) || (mode & saga::advert::CreateParents)) {
if(!url_exists(*client_, advert_url.get_url())) {
if(!create_url(*client_, advert_url.get_url(), true)) {
// failed to create url
SAGA_OSSTREAM strm;
strm << "advert::advert_cpi_impl::init: "
"advert couldn't create url: " << advert_url.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::AlreadyExists);
}
}
// no need to create this entry twice
data->mode_ &= ~(saga::advert::Create | saga::advert::CreateParents);
}
/*if(!url_exists(*client_, advert_url.get_url())) {
//here is where I am
SAGA_OSSTREAM strm;
strm << "advert::advert_cpi_impl::init: "
"advert does not exist: " << advert_url.get_url();
SAGA_ADAPTOR_THROW(SAGA_OSSTREAM_GETSTRING(strm), saga::BadParameter);
}*/
//SAGA_ADAPTOR_THROW ("Not Implemented", saga::NotImplemented);
}
Url::Url(const sfz::StringSlice& string) : Object<CFURLRef>(create_url(string)) {}
static gboolean
handle_request(GstHTTPClient *client)
{
MediaURL *url = NULL;
gchar *request;
gchar **headers;
GError *err = NULL;
gchar *line;
gsize len;
gsize pos;
GIOStatus ret;
int i;
client->gio = g_io_channel_unix_new(client->sock);
ret = g_io_channel_read_line(client->gio, &request, &len, &pos, &err);
if (ret == G_IO_STATUS_ERROR) {
fprintf(stderr, "read header error: %s\n", err->message);
g_error_free(err);
return FALSE;
}
if (ret != G_IO_STATUS_NORMAL || len < 1)
return FALSE;
request[pos] = 0; // terminate at the line terminator
headers = malloc(MAX_CLIENT_HEADERS * (sizeof (gchar *)));
headers[0] = 0;
for (i = 0; i < MAX_CLIENT_HEADERS-1;) {
ret = g_io_channel_read_line(client->gio, &line, &len, &pos, &err);
if (ret == G_IO_STATUS_ERROR) {
fprintf(stderr, "read header error: %s\n", err->message);
g_error_free(err);
break;
}
if (ret != G_IO_STATUS_NORMAL) {
g_free(request);
g_strfreev (headers);
return FALSE;
}
line[pos] = 0; // terminate at the line terminator
if (*line == 0) {
g_free(line);
break;
}
headers[i++] = line;
}
headers[i] = 0; // terminate the list
client->headers = headers;
if (*request) {
GST_DEBUG("request:%s\n", request);
url = create_url(request);
g_free(request);
GST_INFO ("client=%s:%d path='%s' query='%s'", client->peer_ip,
client->port, url->path, url->query);
if (strcmp(url->method, "GET") == 0 ||
strcmp(url->method, "POST") == 0)
{
client->media = gst_http_media_mapping_find(client->media_mapping,
url->path);
}
}
if (client->media) {
GstHTTPMedia *m = client->media;
if (m->pipeline_desc) {
char rfc1123[64];
time_t gmt;
GST_DEBUG_OBJECT(client, "pipeline mapping");
gmt = time (NULL);
strftime (rfc1123, 64, "%a, %d %b %Y %H:%M:%S GMT", gmtime (&gmt));
client_header(client);
/*
if (m->stream) {
gst_http_client_writeln(client,
"Content-Type: multipart/x-mixed-replace;boundary=%s",
m->boundary);
} else {
gst_http_client_writeln(client, "Content-Type: image/jpeg");
}
gst_http_client_writeln(client, "Expires: %s", rfc1123);
gst_http_client_writeln(client, "");
*/
if (strcmp(m->mimetype, "image/jpeg") == 0) {
gst_http_client_writeln(client, "Content-Type: %s", m->mimetype);
gst_http_client_write(client, "\r\n");
}
else if (strcmp(m->mimetype, "multipart/x-mixed-replace") == 0)
{
gst_http_client_writeln(client, "Content-Type: %s;boundary=%s",
m->mimetype, MULTIPART_BOUNDARY);
gst_http_client_writeln(client, "Expires: %s", rfc1123);
gst_http_client_write(client, "\r\n");
}
if (gst_http_media_play (m, client)) {
gst_http_client_writeln(client, "415 Unsupported Media Type");
gst_http_client_close(client, "unsupported");
}
goto out;
}
else if (m->func) {
GST_DEBUG_OBJECT(client, "got function mapping");
client_header(client);
if (m->func(url, client, m->data)) {
gst_http_client_close(client, "complete");
}
goto out;
}
}
gst_http_client_writeln(client, "404 Not Found");
gst_http_client_close(client, "not found");
out:
if (url) {
g_free(url->method);
g_free(url->path);
g_free(url->query);
g_strfreev(url->querys);
free(url);
}
return TRUE;
}
