static void
goto_url_action(struct session *ses,
unsigned char *(*get_url)(struct session *, unsigned char *, size_t))
{
unsigned char url[MAX_STR_LEN];
if (!get_url || !get_url(ses, url, sizeof(url)))
url[0] = 0;
dialog_goto_url(ses, url);
}
__attribute__ ((always_inline)) int GetC (int *c) {
A details_str;
if (!get_url (&details_str))
{
incr ();
return 1;
}
*c = get_time ();
return -1;
}
static int w_dbquery_avps(struct sip_msg* msg, char* query,
char* dest, char *url)
{
struct db_url *parsed_url;
if (get_url(msg, (struct db_url_container*)url, &parsed_url, 0) < 0) {
LM_ERR("failed to get db url\n");
return -1;
}
return ops_dbquery_avps ( msg, (pv_elem_t*)query,
parsed_url, (pvname_list_t*)dest);
}
static int w_async_dbquery_avps(struct sip_msg* msg, async_resume_module **rf,
void **rparam, char* query, char* dest, char* url)
{
struct db_url *parsed_url;
if (get_url(msg, (struct db_url_container*)url, &parsed_url, 1) < 0) {
LM_ERR("failed to get db url\n");
return -1;
}
return ops_async_dbquery(msg, rf, rparam, (pv_elem_t *)query,
parsed_url, (pvname_list_t *)dest);
}
int main (int argc, const char * argv[])
{
if (argc < 2) {
usage();
return 1;
}
std::string site_contents = get_url(argv[1]);
write_file("wget.out", site_contents);
return 0;
}
int main (int argc, char ** argv)
{
CURL * myHandle;
CURLcode success;
struct string s;
char * dest_dir;
if (argc == 1) {
printf("Usage: 4get [THREAD URL] [DESTINATION DIR]\n");
return 0;
}
if (argc == 2) {
dest_dir = "";
} else {
dest_dir = argv[2];
if (dest_dir[strlen(dest_dir)-1] != '/') {
strcat(dest_dir, "/");
}
}
mkdir(argv[2], 0777);
myHandle = curl_easy_init();
init_string(&s);
curl_easy_setopt(myHandle, CURLOPT_URL, argv[1]);
curl_easy_setopt(myHandle, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(myHandle, CURLOPT_WRITEDATA, &s);
success = curl_easy_perform(myHandle);
curl_easy_cleanup(myHandle);
char prev_url[255];
char url[255];
int is_image;
while ((s.ptr = strstr (s.ptr, "i.4cdn.org"))) {
memset(&url[0], 0, sizeof(url));
is_image = get_url(url, s.ptr);
if (is_image && !strcmp(url, prev_url))
get_image(url, dest_dir);
strcpy(prev_url, url);
s.ptr += 10;
}
free(s.ptr);
return EXIT_SUCCESS;
}
// https://wiki.london.hackspace.org.uk/view/Project:Tool_Access_Control/Solexious_Proposal#Get_card_permissions
int querycard(Card card)
{
char path[11 + 10 + 14 + 1];
int result = -100;
sprintf(path, "/%d/card/", acsettings.nodeid);
card.str(path + strlen(path));
result = get_url(path);
Serial.print("acserver said: ");
Serial.println(result);
return result;
}
void process_cmd(int sockcnfd,char *cmd)
{
char url[MAXSIZE]="./www";
char buf[MAXSIZE];
if(get_url(sockcnfd,cmd,url)!=-1){
printf("get url is %s\n",url);
if(is_noexit(url))
send_404(sockcnfd);
else if(is_dir(url))
re_ls(sockcnfd,url);
else
re_file(sockcnfd,url);
}
close(sockcnfd);
}
int main(int argc, char *argv[])
{
int ch;
char url[256], dump[256], proxy[256], method[32];
/*
const char *url1 = "http://127.0.0.1:80/";
HTTP_UTIL *req = http_util_req_new(url1, "POST");
printf(">>>host: %s\n", http_hdr_req_host(req->hdr_req));
acl_assert(req);
return (0);
*/
acl_init(); /* 初始化 acl 库 */
ACL_SAFE_STRNCPY(method, "GET", sizeof(method));
url[0] = 0;
dump[0] = 0;
proxy[0] = 0;
while ((ch = getopt(argc, argv, "hr:t:f:X:")) > 0) {
switch (ch) {
case 'h':
usage(argv[0]);
return (0);
case 'r':
ACL_SAFE_STRNCPY(url, optarg, sizeof(url));
break;
case 't':
ACL_SAFE_STRNCPY(method, optarg, sizeof(method));
break;
case 'f':
ACL_SAFE_STRNCPY(dump, optarg, sizeof(dump));
break;
case 'X':
ACL_SAFE_STRNCPY(proxy, optarg, sizeof(proxy));
break;
default:
break;
}
}
if (url[0] == 0) {
usage(argv[0]);
return (0);
}
get_url(method, url, proxy, dump);
return (0);
}
/*----------------------------------------------------------------------*/
int main(int argc, char *argv[]){
int rc;
FILE *outfile;
int index;
char *outfilename=NULL;
if(argc<2){
ERROR1("Format: %s <http://server[:port]/path> [-o <outputfile>] [-u [email protected]] [-t max_time] [-i inter_time]",
argv[0]);
}
/* parse args */
for(index=1; index<argc; index++){
if(strncmp(argv[index], "-t", 2)==0){
max_time = atoi(argv[++index]);
}
else if(strncmp(argv[index], "-i", 2)==0){
inter_time = atoi(argv[++index]);
}
else if(strncmp(argv[index], "-u", 2)==0){
useraddr = argv[++index];
}
else if(strncmp(argv[index], "-o", 2)==0){
outfilename = argv[++index];
}
}
/* install the handler */
signal(SIGALRM, alarm_handler);
alarm(max_time);
/* ### TO DO -- make it work okay with recursion */
/* open the file */
if(outfilename==NULL){
outfile = stdout;
}else{
outfile = fopen(outfilename, "w");
if(outfile==NULL){
ERROR1("Cannot open outfile %s", argv[2]);
}
}
rc = get_url(argv[1], outfile);
if(rc<0){
ERROR0("Cannot get page.");
}
}
/**
* grl_net_wc_request_with_headers_hash_async:
* @self: a #GrlNetWc instance
* @uri: The URI of the resource to request
* @headers: (allow-none) (element-type utf8 utf8): a set of additional HTTP
* headers for this request or %NULL to ignore
* @cancellable: (allow-none): a #GCancellable instance or %NULL to ignore
* @callback: The callback when the result is ready
* @user_data: User data set for the @callback
*
* Request the fetching of a web resource given the @uri. This request is
* asynchronous, thus the result will be returned within the @callback.
*
* Since: 0.2.2
* Rename to: grl_net_wc_request_with_headers_async
*/
void
grl_net_wc_request_with_headers_hash_async (GrlNetWc *self,
const char *uri,
GHashTable *headers,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
GSimpleAsyncResult *result;
result = g_simple_async_result_new (G_OBJECT (self),
callback,
user_data,
grl_net_wc_request_async);
get_url (self, uri, headers, G_ASYNC_RESULT (result), cancellable);
}
Promise fetch()
{
auto request = Http::Request::get(get_url());
request->set_headers({{"Accept",Archive::mimetype}});
return request->send().then([this, request]()
{
auto response = request->get_response();
if (response->ok())
{
if (response->has_body())
parse(response->get_response_text());
synced.trigger();
}
});
}
static HRESULT get_url_components(HTMLLocation *This, URL_COMPONENTSW *url)
{
const WCHAR *doc_url;
HRESULT hres;
hres = get_url(This, &doc_url);
if(FAILED(hres))
return hres;
if(!InternetCrackUrlW(doc_url, 0, 0, url)) {
FIXME("InternetCrackUrlW failed: 0x%08x\n", GetLastError());
SetLastError(0);
return E_FAIL;
}
return S_OK;
}
void xeta::Curl::perform()
{
static curlpp::easy handle;
try {
std::ofstream output( get_local_filename().c_str() );
handle.url( get_url() );
curlpp::ostream_trait o_trait( &output );
handle.m_body_storage.trait( &o_trait );
set_state( xeta::FileTransfer::Downloading );
handle.perform();
set_state( xeta::FileTransfer::Downloaded );
}
catch( curlpp::exception & e)
{
set_error( e.what() );
}
}
std::string mesos_http::make_uri(const std::string& path)
{
uri url = get_url();
std::string target_uri = url.get_scheme();
target_uri.append("://");
std::string user = url.get_user();
if(!user.empty())
{
target_uri.append(user).append(1, ':').append(url.get_password()).append(1, '@');
}
target_uri.append(url.get_host());
int port = url.get_port();
if(port)
{
target_uri.append(1, ':').append(std::to_string(port));
}
target_uri.append(path);
return target_uri;
}
static gboolean
favicon_web_done_idle_cb (gpointer vfav)
{
GdkPixbuf * pixbuf = NULL;
gboolean finished = FALSE;
struct favicon_data * fav = vfav;
if (fav->len > 0) /* we got something... try to make a pixbuf from it */
{
favicon_save_to_cache (fav->host, fav->contents, fav->len);
pixbuf = favicon_load_from_cache (fav->host);
finished = pixbuf != NULL;
}
if (!finished) /* no pixbuf yet... */
{
if (++fav->type == IMAGE_TYPES) /* failure */
{
finished = TRUE;
}
else /* keep trying */
{
char * url = get_url (fav->host, fav->type);
g_free (fav->contents);
fav->contents = NULL;
fav->len = 0;
tr_webRun (fav->session, url, NULL, NULL, favicon_web_done_cb, fav);
g_free (url);
}
}
if (finished)
{
fav->func (pixbuf, fav->data);
g_free (fav->host);
g_free (fav->contents);
g_free (fav);
}
return FALSE;
}
int Get_MultiTask::svc ()
{
ACE_DEBUG ((LM_INFO, ACE_TEXT ("(%P|%t) multitask started\n")));
ACE::HTTP::URL http_url;
const char* url = get_url ();
ACE_DEBUG ((LM_INFO, ACE_TEXT ("(%P|%t) parsing URL %C\n"), url));
if (http_url.parse (url))
{
ACE::HTTP::ClientRequestHandler rh;
http_url.open (rh);
if (rh.response_stream ())
ACE_DEBUG ((LM_INFO, ACE_TEXT ("(%P|%t) opened URL %C\n"), http_url.to_string ().c_str ()));
else
ACE_ERROR ((LM_ERROR, ACE_TEXT ("(%P|%t) failed to open URL %C\n"), http_url.to_string ().c_str ()));
wait_for_all ();
if (rh.response_stream ())
{
ACE::IOS::CString_OStream sos;
sos << rh.response_stream ().rdbuf ();
ACE_DEBUG ((LM_INFO, ACE_TEXT ("(%P|%t) result downloaded\n")));
if (!sos.bad ())
{
ACE_GUARD_RETURN (ACE_SYNCH_MUTEX,
guard_,
lock_,
0);
ACE_DEBUG ((LM_INFO, ACE_TEXT ("(%P|%t) marking result\n")));
--n_fails_;
}
}
shutdown ();
}
ACE_DEBUG ((LM_INFO, ACE_TEXT ("(%P|%t) task finished\n")));
return 0;
}
static gboolean
builder_source_git_download (BuilderSource *source,
gboolean update_vcs,
BuilderContext *context,
GError **error)
{
BuilderSourceGit *self = BUILDER_SOURCE_GIT (source);
g_autofree char *url = NULL;
url = get_url (self, context, error);
if (url == NULL)
return FALSE;
if (!git_mirror_repo (url,
update_vcs,
get_branch (self),
context,
error))
return FALSE;
return TRUE;
}
static void *url_worker(void *arg){
url_data *data = arg;
buf_data *bdata = malloc(sizeof(buf_data));
bdata->offset = 0;
bdata->size = 0;
bdata->buf = NULL;
bdata->postData = data->postData;
data->data = bdata;
if(data->httpType == CURL_HTTP_GET){
//printf("%s\n", data->url);
data->npcode = get_url(data->url, bdata, g_hash_table_lookup(g_FRManager.hash_table, data->instance));
}else{
data->npcode = post_url(data->url, bdata, g_hash_table_lookup(g_FRManager.hash_table, data->instance));
}
//TODO
gdk_threads_add_idle(stream_write, arg);
return NULL;
}
BOOL install_wine_gecko(void)
{
if(!*ARCH_STRING)
return FALSE;
/*
* Try to find Gecko .cab file in following order:
* - directory stored in GeckoCabDir value of HKCU/Wine/Software/MSHTML key
* - $datadir/gecko/
* - $INSTALL_DATADIR/wine/gecko/
* - /usr/share/wine/gecko/
* - download from URL stored in GeckoUrl value of HKCU/Wine/Software/MSHTML key
*/
if(!install_from_registered_dir()
&& !install_from_default_dir()
&& (url = get_url()))
DialogBoxW(hInst, MAKEINTRESOURCEW(ID_DWL_DIALOG), 0, installer_proc);
heap_free(url);
url = NULL;
return TRUE;
}
static gboolean
builder_source_git_extract (BuilderSource *source,
GFile *dest,
BuilderContext *context,
GError **error)
{
BuilderSourceGit *self = BUILDER_SOURCE_GIT (source);
g_autoptr(GFile) mirror_dir = NULL;
g_autofree char *mirror_dir_path = NULL;
g_autofree char *dest_path = NULL;
g_autofree char *url = NULL;
url = get_url (self, context, error);
if (url == NULL)
return FALSE;
mirror_dir = git_get_mirror_dir (url, context);
mirror_dir_path = g_file_get_path (mirror_dir);
dest_path = g_file_get_path (dest);
if (!git (NULL, NULL, error,
"clone", mirror_dir_path, dest_path, NULL))
return FALSE;
if (!git (dest, NULL, error,
"checkout", get_branch (self), NULL))
return FALSE;
if (!git_extract_submodule (url, dest, context, error))
return FALSE;
if (!git (dest, NULL, error,
"config", "--local", "remote.origin.url", url, NULL))
return FALSE;
return TRUE;
}
pfc::string8 get_track_info(const metadb_handle_ptr track, abort_callback &p_abort) {
pfc::string8 api_key = "803d3cbea0bbe50c61ab81c4fe5fe20f";
pfc::string8 url, page, artist_enc, title_enc;
const file_info *file_info;
const char *artist;
const char *title;
bool meta;
static_api_ptr_t<metadb> db;
db->database_lock();
meta = track->get_info_async_locked(file_info);
db->database_unlock();
if (meta && file_info->meta_exists("artist") &&
file_info->meta_exists("title")) {
artist = file_info->meta_get("artist", 0);
title = file_info->meta_get("title", 0);
} else {
throw pfc::exception("Unknown track");
}
pfc::urlEncode(artist_enc, artist);
pfc::urlEncode(title_enc, title);
url << "http://ws.audioscrobbler.com/2.0/?method=track.getInfo"
<< "&api_key=" << api_key << "&artist=" << artist_enc << "&track=" << title_enc;
console::print(url);
page = get_url(url, p_abort);
if (page.get_length() < 10) {
throw pfc::exception("Last.fm returned an empty page");
}
return page;
}
BOOL install_addon(addon_t addon_type)
{
if(!*ARCH_STRING)
return FALSE;
addon = addons_info+addon_type;
/*
* Try to find addon .msi file in following order:
* - directory stored in $dir_config_key value of HKCU/Wine/Software/$config_key key
* - $datadir/$addon_subdir/
* - $INSTALL_DATADIR/wine/$addon_subdir/
* - /usr/share/wine/$addon_subdir/
* - download from URL stored in $url_config_key value of HKCU/Wine/Software/$config_key key
*/
if(!install_from_registered_dir()
&& !install_from_default_dir()
&& (url = get_url()))
DialogBoxW(hInst, MAKEINTRESOURCEW(ID_DWL_DIALOG), 0, installer_proc);
heap_free(url);
url = NULL;
return TRUE;
}
TITANIUM_PROPERTY_GETTER(WebView, url)
{
return get_context().CreateString(get_url());
}
void k8s_event_handler::handle_json(Json::Value&& root)
{
/*if(g_logger.get_severity() >= sinsp_logger::SEV_TRACE)
{
g_logger.log(json_as_string(root), sinsp_logger::SEV_TRACE);
}*/
if(!m_state)
{
throw sinsp_exception("k8s_handler (" + get_id() + "), state is null for " + get_url() + ").");
}
const Json::Value& type = root["type"];
if(!type.isNull())
{
if(type.isConvertibleTo(Json::stringValue))
{
const Json::Value& kind = root["kind"];
if(!kind.isNull())
{
if(kind.isConvertibleTo(Json::stringValue))
{
std::string t = type.asString();
std::string k = kind.asString();
for(const Json::Value& item : root["items"])
{
msg_data data = get_msg_data(t, k, item);
std::string reason_type = data.get_reason_desc();
if(data.m_reason != k8s_component::COMPONENT_ADDED &&
data.m_reason != k8s_component::COMPONENT_MODIFIED &&
data.m_reason != k8s_component::COMPONENT_DELETED &&
data.m_reason != k8s_component::COMPONENT_ERROR)
{
g_logger.log(std::string("Unsupported K8S " + name() + " event reason: ") +
std::to_string(data.m_reason), sinsp_logger::SEV_ERROR);
continue;
}
/*if(g_logger.get_severity() >= sinsp_logger::SEV_TRACE)
{
g_logger.log("K8s handling event:\n" + json_as_string(item), sinsp_logger::SEV_TRACE);
}*/
if(handle_component(item, &data))
{
std::ostringstream os;
os << "K8s [" + reason_type + ", " << data.m_kind <<
", " << data.m_name << ", " << data.m_uid << "]";
g_logger.log(os.str(), sinsp_logger::SEV_INFO);
}
else if(!m_event_ignored)
{
g_logger.log("K8s: error occurred while handling " + reason_type +
" event for " + data.m_kind + ' ' + data.m_name + " [" +
data.m_uid + ']', sinsp_logger::SEV_ERROR);
}
m_event_ignored = false;
} // end for items
}
}
}
else
{
g_logger.log(std::string("K8S event type is not string."), sinsp_logger::SEV_ERROR);
}
}
else
{
g_logger.log(std::string("K8S event type is null."), sinsp_logger::SEV_ERROR);
}
}
int feed_main(match_data **feed_matches, int *feed_matches_counter) {
omnibet_match_data omnibet_match;
GSList *cookies;
char tmp_file[1024];
char tmp_file2[1024];
char tmp_file3[1024];
memset(&omnibet_match.match_time[0], '\0', sizeof(omnibet_match.match_time));
memset(&omnibet_match.team_home[0], '\0', sizeof(omnibet_match.team_home));
memset(&omnibet_match.team_away[0], '\0', sizeof(omnibet_match.team_away));
omnibet_match.score_home = 0;
omnibet_match.score_away = 0;
omnibet_match.stage = -1;
omnibet_match.skip = FALSE;
struct passwd *pw = getpwuid(getuid());
snprintf(&tmp_file[0], sizeof(tmp_file), "%s-%u", OMNIBET_FILENAME, pw->pw_uid);
snprintf(&tmp_file2[0], sizeof(tmp_file), "%s-%u-a", OMNIBET_FILENAME, pw->pw_uid);
// Get our cookie from main page
if (get_url(OMNIBET_URL1, OMNIBET_USER_AGENT, &tmp_file[0], NULL, &cookies))
return 1;
// Fetch actual page
if (get_url(OMNIBET_URL2, OMNIBET_USER_AGENT, &tmp_file[0], cookies, NULL)) {
if (cookies)
soup_cookies_free(cookies);
return 1;
}
if (cookies)
soup_cookies_free(cookies);
char *orig_xml = omnibet_load_file(&tmp_file[0]);
if (!orig_xml)
return 0;
char *fixed_xml2 = omnibet_replace(orig_xml, "<strong>", "<strong custom=livescore>");
if (!fixed_xml2)
return 0;
FILE *fp = fopen (&tmp_file2[0], "w");
if (!fp) {
printf("Cannot open output file!\n");
return 0;
}
fprintf(fp, "%s\n", fixed_xml2);
fclose(fp);
htmlDocPtr parser = htmlReadFile(&tmp_file2[0], OMNIBET_CHARSET,
HTML_PARSE_RECOVER |
//HTML_PARSE_NOBLANKS |
HTML_PARSE_NOERROR |
HTML_PARSE_NOWARNING |
#ifdef HAVE_MATE
HTML_PARSE_NOIMPLIED |
#endif
HTML_PARSE_COMPACT);
omnibet_walk_tree(xmlDocGetRootElement(parser), &omnibet_match, feed_matches, feed_matches_counter);
xmlFreeDoc(parser);
free(orig_xml);
free(fixed_xml2);
return 1;
}
int proxy_server()
{
int rx_bytes = 0, post_data_size = 0;
int lsock = 0, csock = 0, client_data_size = 0;
int result_size = 0, err_code = 0, id = 0;
int addrlen = sizeof(struct sockaddr_in);
struct sockaddr_in clientaddr;
char *buffer = NULL, *post_data = NULL;
char *host = NULL, *url = NULL;
char *query = NULL, *fifo_file = NULL;
char *get_data = NULL;
char *client_data = NULL, *headers = NULL;
char *server_ip = NULL;
memset((void *) &clientaddr,0,addrlen);
server_ip = config_get_server_ip();
/* Create TCP socket */
if((lsock = create_socket(server_ip,PROXY_PORT,SOCK_STREAM)) == SOCK_FAIL) {
glog("Failed to create TCP socket for proxy server",LOG_ERROR_TYPE);
if(server_ip) free(server_ip);
return EXIT_FAILURE;
}
if(server_ip) free(server_ip);
/* Infinite receive loop */
while(1) {
/* Accept incoming connection */
if((csock = accept(lsock,(struct sockaddr *) &clientaddr,(socklen_t *) &addrlen)) < 0) {
glog("Failed to accept TCP connection to proxy server",LOG_ERROR_TYPE);
if(buffer) free(buffer);
return EXIT_FAILURE;
}
if(!fork()) {
/* Receive client request */
if((buffer = receive(lsock,SOCK_STREAM,&rx_bytes,csock,&clientaddr)) == NULL) {
glog("Failed to read data from client request sent to proxy server",LOG_ERROR_TYPE);
exit(EXIT_FAILURE);
}
if(is_using_proxy(buffer)) {
/* Get the target's IP address */
host = get_host_name(buffer);
/* Get the target URL path */
url = get_url(buffer);
/* Get POST data, if any */
post_data = get_post_data(buffer,rx_bytes,&post_data_size);
/* Get HTTP headers from request */
headers = get_headers(buffer);
/* If the CONSOLE_HOST is requested, then display the Web console interface */
if(memcmp(host,CONSOLE_HOST,CONSOLE_HOST_SIZE) == 0) {
show_web_ui(csock,url);
close_socket(csock);
} else {
/* Make sure the requested host is in our clients list */
query = sqlite3_mprintf("SELECT id FROM %s WHERE strftime('%%s',callback_time) >= strftime('%%s','now') AND ip = %Q LIMIT 1",CLIENTS_TABLE,host);
sql_exec(query,&result_size,&err_code);
sqlite3_free(query);
if(result_size > 0) {
/* Don't allow requests for filtered file extensions */
if(!is_url_filtered(url)) {
fifo_file = create_fifo(host);
if(!fifo_file) {
glog("Failed to create fifo file",LOG_ERROR_TYPE);
} else {
/* Insert query into queue table */
query = sqlite3_mprintf("INSERT INTO %s (fifo,host,url,headers,pdata,sent) VALUES (%Q,%Q,%Q,%Q,%Q,0)",QUEUE_TABLE,fifo_file,host,url,headers,post_data);
sql_exec(query,&result_size,&err_code);
sqlite3_free(query);
if(err_code != SQLITE_OK) {
sql_log_error();
} else {
/* When the client data has returned, the callback server will write the ID of the callback to the FIFO */
id = read_from_fifo(fifo_file);
/* Extract the data from the DB */
get_data = sqlite3_mprintf("SELECT rdata FROM %s WHERE id = '%d'",QUEUE_TABLE,id);
client_data = sql_exec(get_data,&client_data_size,&err_code);
sqlite3_free(get_data);
if(err_code != SQLITE_OK) {
sql_log_error();
} else {
/* Write data to socket */
if(write(csock,client_data,client_data_size) != client_data_size) {
glog("Proxy socket write failed",LOG_ERROR_TYPE);
}
}
if(client_data) free(client_data);
/* Make sure the fifo gets deleted */
destroy_fifo(fifo_file);
}
}
}
}
}
}
/* Exit the child process */
close_socket(csock);
if(fifo_file) free(fifo_file);
if(buffer) free(buffer);
if(host) free(host);
if(url) free(url);
if(post_data) free(post_data);
if(headers) free(headers);
exit(EXIT_SUCCESS);
}
}
/* Close up shop */
close_socket(csock);
close_socket(lsock);
return EXIT_FAILURE;
}
static void
get_url (const char *url)
{
const char *name;
SoupMessage *msg;
const char *header;
FILE *output_file = NULL;
msg = soup_message_new (head ? "HEAD" : "GET", url);
soup_message_set_flags (msg, SOUP_MESSAGE_NO_REDIRECT);
if (loop) {
g_object_ref (msg);
soup_session_queue_message (session, msg, finished, loop);
g_main_loop_run (loop);
} else
soup_session_send_message (session, msg);
name = soup_message_get_uri (msg)->path;
if (!debug) {
if (msg->status_code == SOUP_STATUS_SSL_FAILED) {
GTlsCertificateFlags flags;
if (soup_message_get_https_status (msg, NULL, &flags))
g_print ("%s: %d %s (0x%x)\n", name, msg->status_code, msg->reason_phrase, flags);
else
g_print ("%s: %d %s (no handshake status)\n", name, msg->status_code, msg->reason_phrase);
} else if (!quiet || SOUP_STATUS_IS_TRANSPORT_ERROR (msg->status_code))
g_print ("%s: %d %s\n", name, msg->status_code, msg->reason_phrase);
}
if (SOUP_STATUS_IS_REDIRECTION (msg->status_code)) {
header = soup_message_headers_get_one (msg->response_headers,
"Location");
if (header) {
SoupURI *uri;
char *uri_string;
if (!debug && !quiet)
g_print (" -> %s\n", header);
uri = soup_uri_new_with_base (soup_message_get_uri (msg), header);
uri_string = soup_uri_to_string (uri, FALSE);
get_url (uri_string);
g_free (uri_string);
soup_uri_free (uri);
}
} else if (!head && SOUP_STATUS_IS_SUCCESSFUL (msg->status_code)) {
if (output_file_path) {
output_file = fopen (output_file_path, "w");
if (!output_file)
g_printerr ("Error trying to create file %s.\n", output_file_path);
} else if (!quiet)
output_file = stdout;
if (output_file) {
fwrite (msg->response_body->data,
1,
msg->response_body->length,
output_file);
if (output_file_path)
fclose (output_file);
}
}
}
int
main (int argc, char **argv)
{
GOptionContext *opts;
const char *url;
SoupURI *proxy_uri, *parsed;
GError *error = NULL;
SoupLogger *logger = NULL;
opts = g_option_context_new (NULL);
g_option_context_add_main_entries (opts, entries, NULL);
if (!g_option_context_parse (opts, &argc, &argv, &error)) {
g_printerr ("Could not parse arguments: %s\n",
error->message);
g_printerr ("%s",
g_option_context_get_help (opts, TRUE, NULL));
exit (1);
}
if (argc != 2) {
g_printerr ("%s",
g_option_context_get_help (opts, TRUE, NULL));
exit (1);
}
g_option_context_free (opts);
url = argv[1];
parsed = soup_uri_new (url);
if (!parsed) {
g_printerr ("Could not parse '%s' as a URL\n", url);
exit (1);
}
soup_uri_free (parsed);
session = g_object_new (SOUP_TYPE_SESSION,
SOUP_SESSION_SSL_CA_FILE, ca_file,
SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_CONTENT_DECODER,
SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_COOKIE_JAR,
SOUP_SESSION_USER_AGENT, "get ",
SOUP_SESSION_ACCEPT_LANGUAGE_AUTO, TRUE,
NULL);
if (ntlm)
soup_session_add_feature_by_type (session, SOUP_TYPE_AUTH_NTLM);
if (debug) {
logger = soup_logger_new (SOUP_LOGGER_LOG_BODY, -1);
soup_session_add_feature (session, SOUP_SESSION_FEATURE (logger));
g_object_unref (logger);
}
if (proxy) {
proxy_uri = soup_uri_new (proxy);
if (!proxy_uri) {
g_printerr ("Could not parse '%s' as URI\n",
proxy);
exit (1);
}
g_object_set (G_OBJECT (session),
SOUP_SESSION_PROXY_URI, proxy_uri,
NULL);
soup_uri_free (proxy_uri);
}
if (!synchronous)
loop = g_main_loop_new (NULL, TRUE);
get_url (url);
if (!synchronous)
g_main_loop_unref (loop);
return 0;
}
int main(int argc, char * argv[])
{
if (argc == 3) {
//check if 1st arg is file
if (!IsFile(argv[1])) {
fprintf(stderr, "ERROR: Sorry, %s is not a file.", argv[1]);
return 1;
}
//check if 2nd arg is directory
if (!IsDir(argv[2])) {
fprintf(stderr, "ERROR: Sorry, %s is not a directory.", argv[2]);
return 1;
}
// make indexer
HashTable * index = ReadFile(argv[1]);
if (index == NULL){
return 1;
}
char buffer[QUERY_MAX];
printf("QUERY>: ");
while( fgets(buffer, QUERY_MAX , stdin) != NULL ) /* break with ^D or ^Z */
{
WordNode * search_results = processQuery(buffer, index);
if (!search_results) {
// an error that must exit.
return 1;
}
if (search_results->head == NULL) {
printf("No results found.");
} else {
// order by frequency, mergesort returns the head (top-ranking url) of search_results
DocNode * doc = mergesort(search_results->head);
//set the search_results head to the doc, for freeing purposes.
search_results->head = doc;
// go through every doc node
while ( doc!= NULL){
char * docName = (char *)calloc(5,sizeof(char)); //assuming
int ret = snprintf(docName, 5, "%d", doc->docID);
if (ret == -1){
fprintf(stderr, "WOW you have tons of files. I cannot handle this. Sorry \n");
free(docName);
continue;
}
char * url = get_url(docName, argv[2]);
if (!url){
fprintf(stderr, "Getting the url of doc %s failed. \n", docName);
continue;
}
printf("Document ID: %d URL: %s", doc->docID, url);
free(url);
free(docName);
doc = doc->next;
}
}
//next iteration
printf("\nQUERY>: ");
free_list(search_results);
}
printf("done");
free_table(index);
return 0;
} else {
fprintf(stderr,"ERROR: Sorry, you entered in an incorrect number of input arguments. You must have 2 (index filename to read from, directory of html files)");
return 1;
}
}
