static int
descend_redirect_p (const char *redirected, const char *original, int depth,
struct url *start_url_parsed, struct hash_table *blacklist)
{
struct url *orig_parsed, *new_parsed;
struct urlpos *upos;
int success;
orig_parsed = url_parse (original, NULL);
assert (orig_parsed != NULL);
new_parsed = url_parse (redirected, NULL);
assert (new_parsed != NULL);
upos = xmalloc (sizeof (struct urlpos));
memset (upos, 0, sizeof (*upos));
upos->url = new_parsed;
success = download_child_p (upos, orig_parsed, depth,
start_url_parsed, blacklist);
url_free (orig_parsed);
url_free (new_parsed);
xfree (upos);
if (!success)
DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
return success;
}
int main() {
/* Repeatedly save and load the context, then compare it to the first one. */
urlctx *ctx = url_init(
"http://*gooogle*\n"
"ftp://fooo\n"
"*adwords\n"
"http*//*.php");
// Yes yes, insecure mktemp. This is a unit test.
char name[] = "/tmp/bintestXXXXXX";
mktemp(name);
if (url_save_optimized(ctx, name)) fail("save failed\n");
u32 i;
urlctx *tmp;
for (i = 0; i < 20; i++) {
tmp = url_init_file(name);
if (!tmp) fail("load failed\n");
if (url_save_optimized(tmp, name)) fail("save failed\n");
url_free(tmp);
}
tmp = url_init_file(name);
if (ctxcmp(ctx, tmp)) fail("compare failed\n");
url_free(ctx);
url_free(tmp);
unlink(name);
return 0;
}
URL_t*
check4proxies( URL_t *url ) {
URL_t *url_out = NULL;
if( url==NULL ) return NULL;
url_out = url_new( url->url );
if( !strcasecmp(url->protocol, "http_proxy") ) {
mp_msg(MSGT_NETWORK,MSGL_V,"Using HTTP proxy: http://%s:%d\n", url->hostname, url->port );
return url_out;
}
// Check if the http_proxy environment variable is set.
if( !strcasecmp(url->protocol, "http") ) {
char *proxy;
proxy = getenv("http_proxy");
if( proxy!=NULL ) {
// We got a proxy, build the URL to use it
int len;
char *new_url;
URL_t *tmp_url;
URL_t *proxy_url = url_new( proxy );
if( proxy_url==NULL ) {
mp_msg(MSGT_NETWORK,MSGL_WARN,
MSGTR_MPDEMUX_NW_InvalidProxySettingTryingWithout);
return url_out;
}
#ifdef HAVE_AF_INET6
if (network_ipv4_only_proxy && (gethostbyname(url->hostname)==NULL)) {
mp_msg(MSGT_NETWORK,MSGL_WARN,
MSGTR_MPDEMUX_NW_CantResolvTryingWithoutProxy);
url_free(proxy_url);
return url_out;
}
#endif
mp_msg(MSGT_NETWORK,MSGL_V,"Using HTTP proxy: %s\n", proxy_url->url );
len = strlen( proxy_url->hostname ) + strlen( url->url ) + 20; // 20 = http_proxy:// + port
new_url = malloc( len+1 );
if( new_url==NULL ) {
mp_msg(MSGT_NETWORK,MSGL_FATAL,MSGTR_MemAllocFailed);
url_free(proxy_url);
return url_out;
}
sprintf(new_url, "http_proxy://%s:%d/%s", proxy_url->hostname, proxy_url->port, url->url );
tmp_url = url_new( new_url );
if( tmp_url==NULL ) {
free( new_url );
url_free( proxy_url );
return url_out;
}
url_free( url_out );
url_out = tmp_url;
free( new_url );
url_free( proxy_url );
}
}
return url_out;
}
URL_t*
check4proxies( const URL_t *url ) {
URL_t *url_out = NULL;
if( url==NULL ) return NULL;
url_out = url_new( url->url );
if( !strcasecmp(url->protocol, "mp_http_proxy") ) {
mp_msg(MSGT_NETWORK,MSGL_V,"Using HTTP proxy: http://%s:%d\n", url->hostname, url->port );
return url_out;
}
// Check if the http_proxy environment variable is set.
if( !strcasecmp(url->protocol, "mp_http") ) {
char *proxy;
proxy = getenv("http_proxy");
if( proxy!=NULL ) {
// We got a proxy, build the URL to use it
char *new_url;
URL_t *tmp_url;
URL_t *proxy_url = url_new( proxy );
if( proxy_url==NULL ) {
mp_tmsg(MSGT_NETWORK,MSGL_WARN,
"Invalid proxy setting... Trying without proxy.\n");
return url_out;
}
#ifdef HAVE_AF_INET6
if (network_ipv4_only_proxy && (gethostbyname(url->hostname)==NULL)) {
mp_tmsg(MSGT_NETWORK,MSGL_WARN,
"Could not resolve remote hostname for AF_INET. Trying without proxy.\n");
url_free(proxy_url);
return url_out;
}
#endif
mp_msg(MSGT_NETWORK,MSGL_V,"Using HTTP proxy: %s\n", proxy_url->url );
new_url = get_http_proxy_url(proxy_url, url->url);
if( new_url==NULL ) {
mp_tmsg(MSGT_NETWORK,MSGL_FATAL,"Memory allocation failed.\n");
url_free(proxy_url);
return url_out;
}
tmp_url = url_new( new_url );
if( tmp_url==NULL ) {
free( new_url );
url_free( proxy_url );
return url_out;
}
url_free( url_out );
url_out = tmp_url;
free( new_url );
url_free( proxy_url );
}
}
return url_out;
}
static int open_s(stream_t *stream,int mode, void* opts, int* file_format) {
URL_t *url;
stream->streaming_ctrl = streaming_ctrl_new();
if( stream->streaming_ctrl==NULL ) {
return STREAM_ERROR;
}
stream->streaming_ctrl->bandwidth = network_bandwidth;
url = url_new(stream->url);
stream->streaming_ctrl->url = check4proxies(url);
url_free(url);
mp_msg(MSGT_OPEN, MSGL_INFO, MSGTR_MPDEMUX_ASF_InfoStreamASFURL, stream->url);
if((!strncmp(stream->url, "http", 4)) && (*file_format!=DEMUXER_TYPE_ASF && *file_format!=DEMUXER_TYPE_UNKNOWN)) {
streaming_ctrl_free(stream->streaming_ctrl);
stream->streaming_ctrl = NULL;
return STREAM_UNSUPPORTED;
}
if(asf_streaming_start(stream, file_format) < 0) {
mp_msg(MSGT_OPEN, MSGL_ERR, MSGTR_MPDEMUX_ASF_StreamingFailed);
streaming_ctrl_free(stream->streaming_ctrl);
stream->streaming_ctrl = NULL;
return STREAM_UNSUPPORTED;
}
if (*file_format != DEMUXER_TYPE_PLAYLIST)
*file_format = DEMUXER_TYPE_ASF;
stream->type = STREAMTYPE_STREAM;
fixup_network_stream_cache(stream);
return STREAM_OK;
}
int WebSession::Recv()
{
m_content = NULL;
m_contentLength = 0;
int r = socket_select_read(m_sock, SERVER_TIMEOUT);
if(r <= 0)
return 0==r ? ERROR_RECV_TIMEOUT : r;
r = http_server_recv(m_http);
if(r < 0)
return r;
void* url = url_parse(http_server_get_path(m_http));
for(int i=0; i<url_getparam_count(url); i++)
{
const char *name, *value;
if(0 != url_getparam(url, i, &name, &value))
continue;
m_params.insert(std::make_pair(std::string(name), std::string(value)));
}
m_path.assign(url_getpath(url));
url_free(url);
http_server_get_content(m_http, &m_content, &m_contentLength);
if(m_contentLength > 0 && m_contentLength < 2*1024)
{
printf("%s\n", (const char*)m_content);
}
return 0;
}
URL_t *url_new_with_proxy(const char *urlstr)
{
URL_t *url = url_new(urlstr);
URL_t *url_with_proxy = check4proxies(url);
url_free(url);
return url_with_proxy;
}
static bool http_request_init(http_request_t *req) {
url_t * url = url_parse(req->url);
char * uri = url_get_uri(url);
req->uri = uri ? strdup(uri) : strdup("/");
if(uri) {
free(uri);
}
http_conn_t * conn = http_conn_new(url->host, url->port ? url->port : 80);
assert(conn != NULL);
if(!conn) {
return false;
}
req->conn = conn;
url_free(url);
sstring_init(&req->header, 200);
sstring_init(&req->res_header, 512);
sstring_init(&req->response, 2048);
req->ht_headers = hash_table_new(20, free);
req->error = NULL;
return true;
}
static void
set_socks_server(const char *socks)
{
struct url *url;
enum socks_ver socksv;
url = url_tokenize(socks);
if (!url || !url->scheme) {
log_error("shim: bad socks server, %s", socks);
exit(1);
}
if (url->port < 0)
url->port = 1080;
if (!evutil_ascii_strcasecmp(url->scheme, "socks4"))
socksv = SOCKS_4;
else if (!evutil_ascii_strcasecmp(url->scheme, "socks4a"))
socksv = SOCKS_4a;
else {
log_error("shim: unknown socks version, %s",
url->scheme);
exit(1);
}
if (conn_set_socks_server(url->host, url->port, socksv) < 0)
exit(1);
url_free(url);
}
// tokenize a CONNECT host:port request
struct url *
url_connect_tokenize(const char *str)
{
struct url *url;
char *p;
int port;
url = mem_calloc(1, sizeof(*url));
p = strrchr(str, ':');
if (!p || p == str)
goto fail;
port = get_port(p + 1);
if (port < 0)
goto fail;
url->host = mem_strdup_n(str, p - str);
url->port = port;
return url;
fail:
url_free(url);
return NULL;
}
static int open_s2(stream_t *stream,int mode, void* opts, int* file_format) {
int seekable=0;
URL_t *url;
stream->streaming_ctrl = streaming_ctrl_new();
if( stream->streaming_ctrl==NULL ) {
return STREAM_ERROR;
}
stream->streaming_ctrl->bandwidth = network_bandwidth;
url = url_new(stream->url);
stream->streaming_ctrl->url = check4proxies(url);
url_free(url);
mp_msg(MSGT_OPEN, MSGL_V, "STREAM_HTTP(2), URL: %s\n", stream->url);
seekable = http_streaming_start(stream, file_format);
if(seekable < 0) {
if (stream->fd >= 0)
closesocket(stream->fd);
stream->fd = -1;
streaming_ctrl_free(stream->streaming_ctrl);
stream->streaming_ctrl = NULL;
return STREAM_UNSUPPORTED;
}
return fixup_open(stream, seekable);
}
void
streaming_ctrl_free( streaming_ctrl_t *streaming_ctrl ) {
if( streaming_ctrl==NULL ) return;
if( streaming_ctrl->url ) url_free( streaming_ctrl->url );
if( streaming_ctrl->buffer ) free( streaming_ctrl->buffer );
if( streaming_ctrl->data ) free( streaming_ctrl->data );
free( streaming_ctrl );
}
static int cddb_http_request(char *command,
int (*reply_parser)(HTTP_header_t*, cddb_data_t*),
cddb_data_t *cddb_data)
{
char request[4096];
int fd, ret = 0;
URL_t *url;
HTTP_header_t *http_hdr;
if (reply_parser == NULL || command == NULL || cddb_data == NULL)
return -1;
sprintf(request, "http://%s/~cddb/cddb.cgi?cmd=%s%s&proto=%d",
cddb_data->freedb_server, command, cddb_data->cddb_hello,
cddb_data->freedb_proto_level);
mp_msg(MSGT_OPEN, MSGL_INFO,"Request[%s]\n", request);
url = url_new(request);
if (url == NULL) {
mp_msg(MSGT_DEMUX, MSGL_ERR, MSGTR_MPDEMUX_CDDB_NotAValidURL);
return -1;
}
fd = http_send_request(url,0);
if (fd < 0) {
mp_msg(MSGT_DEMUX, MSGL_ERR,
MSGTR_MPDEMUX_CDDB_FailedToSendHTTPRequest);
return -1;
}
http_hdr = http_read_response(fd);
if (http_hdr == NULL) {
mp_msg(MSGT_DEMUX, MSGL_ERR,
MSGTR_MPDEMUX_CDDB_FailedToReadHTTPResponse);
return -1;
}
http_debug_hdr(http_hdr);
mp_msg(MSGT_OPEN, MSGL_INFO,"body=[%s]\n", http_hdr->body);
switch (http_hdr->status_code) {
case 200:
ret = reply_parser(http_hdr, cddb_data);
break;
case 400:
mp_msg(MSGT_DEMUX, MSGL_ERR, MSGTR_MPDEMUX_CDDB_HTTPErrorNOTFOUND);
break;
default:
mp_msg(MSGT_DEMUX, MSGL_ERR, MSGTR_MPDEMUX_CDDB_HTTPErrorUnknown);
}
http_free(http_hdr);
url_free(url);
return ret;
}
static int cddb_http_request(char *command,
int (*reply_parser)(HTTP_header_t*, cddb_data_t*),
cddb_data_t *cddb_data)
{
char request[4096];
int fd, ret = 0;
URL_t *url;
HTTP_header_t *http_hdr;
if (reply_parser == NULL || command == NULL || cddb_data == NULL)
return -1;
sprintf(request, "http://%s/~cddb/cddb.cgi?cmd=%s%s&proto=%d",
cddb_data->freedb_server, command, cddb_data->cddb_hello,
cddb_data->freedb_proto_level);
mp_msg(MSGT_OPEN, MSGL_INFO,"Request[%s]\n", request);
url = url_new(request);
if (url == NULL) {
mp_tmsg(MSGT_DEMUX, MSGL_ERR, "not a valid URL\n");
return -1;
}
fd = http_send_request(url,0);
if (fd < 0) {
mp_tmsg(MSGT_DEMUX, MSGL_ERR, "Failed to send the HTTP request.\n");
return -1;
}
http_hdr = http_read_response(fd);
if (http_hdr == NULL) {
mp_tmsg(MSGT_DEMUX, MSGL_ERR, "Failed to read the HTTP response.\n");
return -1;
}
http_debug_hdr(http_hdr);
mp_msg(MSGT_OPEN, MSGL_INFO,"body=[%s]\n", http_hdr->body);
switch (http_hdr->status_code) {
case 200:
ret = reply_parser(http_hdr, cddb_data);
break;
case 400:
mp_tmsg(MSGT_DEMUX, MSGL_ERR, "Not Found.\n");
break;
default:
mp_tmsg(MSGT_DEMUX, MSGL_ERR, "unknown error code\n");
}
http_free(http_hdr);
url_free(url);
return ret;
}
static void
bs_gocomics_hide (Evas_Object *layout)
{
url_free(mod->url);
ENNA_FREE(mod->comic_name);
ENNA_FREE(mod->comic_id);
ENNA_OBJECT_DEL(mod->list);
elm_object_part_content_unset(mod->layout, "service.browser.swallow");
ENNA_FREE(mod->path);
ENNA_FREE(mod);
}
bool URLOption::set(const char* value, buffer_t* error) {
url_t* u = NULL;
if (!(u = url_from_string(value,error)))
return false;
bool change = !url_eq(u,*store);
url_free(*store);
delete *store;
*store = u;
if (change)
sigOptionChange.emit(this);
return true;
}
bool
res_retrieve_file (const char *url, char **file, struct iri *iri)
{
struct iri *i = iri_new ();
uerr_t err;
char *robots_url = uri_merge (url, RES_SPECS_LOCATION);
int saved_ts_val = opt.timestamping;
int saved_sp_val = opt.spider, url_err;
struct url * url_parsed;
/* Copy server URI encoding for a possible IDNA transformation, no need to
encode the full URI in UTF-8 because "robots.txt" is plain ASCII */
set_uri_encoding (i, iri->uri_encoding, false);
i->utf8_encode = false;
logputs (LOG_VERBOSE, _("Loading robots.txt; please ignore errors.\n"));
*file = NULL;
opt.timestamping = false;
opt.spider = false;
url_parsed = url_parse (robots_url, &url_err, i, true);
if (!url_parsed)
{
char *error = url_error (robots_url, url_err);
logprintf (LOG_NOTQUIET, "%s: %s.\n", robots_url, error);
xfree (error);
err = URLERROR;
}
else
{
err = retrieve_url (url_parsed, robots_url, file, NULL, NULL, NULL,
false, i, false);
url_free(url_parsed);
}
opt.timestamping = saved_ts_val;
opt.spider = saved_sp_val;
xfree (robots_url);
iri_free (i);
if (err != RETROK && *file != NULL)
{
/* If the file is not retrieved correctly, but retrieve_url
allocated the file name, deallocate is here so that the
caller doesn't have to worry about it. */
xfree (*file);
*file = NULL;
}
return err == RETROK;
}
/**
* @brief Try to inherit logging settings from environment
* @param [in] ns pointer to logging namespace
* @return on success, zero is returned
* @retval -1 error occurred
*/
static int ll_ns_env(struct ll_namespace *ns)
{
assert(ns);
assert(ns->name);
const char *env;
/* get settings from LIBLOG|LIBLOG_%s variable */
if (*ns->name) {
char *s;
if (-1 == asprintf(&s, "LIBLOG_%s", ns->name)) {
return (-1);
}
env = getenv(s);
free(s);
} else {
env = getenv("LIBLOG");
}
if (!env) {
/* nothing to inherit */
return (-1);
}
/* logging level */
ns->level = atoi(env);
/* URI */
const char *uri = strchr(env, ',');
if (!uri) {
/* only logging level was set by environment variable? */
return (-1);
}
++ uri;
struct url *u;
if (!url_parse(uri, &u)) {
return (-1);
}
/* try to initialize logger */
int rc = ll_logger_open(u, ns);
url_free(u);
return (rc);
}
struct url_t *url_parse(const char *url) {
struct url_t *self = url_new();
const char *p = url;
if (!(p = parse_protocol(self, p))) {
url_free(self);
return NULL;
}
self->has_port = has_port(p);
if (!(p = parse_host(self, p))) {
url_free(self);
return NULL;
}
if (!(p = parse_port(self, p))) {
url_free(self);
return NULL;
}
self->path = strdup(p);
return self;
}
void streaming_ctrl_free(streaming_ctrl_t *streaming_ctrl) {
if (streaming_ctrl == NULL) return;
if (streaming_ctrl->url) url_free(streaming_ctrl->url);
if (streaming_ctrl->hc_buf) {
free(streaming_ctrl->hc_buf);
streaming_ctrl->hc_buf = NULL;
streaming_ctrl->hc_buf_ptr = NULL;
streaming_ctrl->hc_buf_end = NULL;
}
free(streaming_ctrl->buffer);
free(streaming_ctrl->data);
free(streaming_ctrl);
}
int main(int argc, char **argv)
{
struct url *url;
url = url_tokenize(argv[1]);
if (!url) {
printf("bad url!\n");
return 0;
}
printf("%s, %s, %d, %s\n",
url->scheme, url->host, url->port, url->path);
url_free(url);
return 0;
}
static int hls_server_onhttp(void* http, void* session, const char* method, const char* path)
{
// decode request uri
void* url = url_parse(path);
std::string s = url_getpath(url);
url_free(url);
path = s.c_str();
if (0 == strncmp(path, "/live/", 6))
{
std::vector<std::string> paths;
Split(path + 6, "/", paths);
if (strendswith(path, ".m3u8") && 1 == paths.size())
{
std::string app = paths[0].substr(0, paths[0].length() - 5);
if (s_playlists.find(app) == s_playlists.end())
{
hls_playlist_t* playlist = new hls_playlist_t();
playlist->file = app;
playlist->m3u8 = hls_m3u8_create(HLS_LIVE_NUM);
playlist->hls = hls_media_create(HLS_DURATION * 1000, hls_handler, playlist);
playlist->i = 0;
s_playlists[app] = playlist;
thread_create(&playlist->t, hls_server_worker, playlist);
}
return hls_server_m3u8(session, app);
}
else if (strendswith(path, ".ts") && 2 == paths.size())
{
if (s_playlists.find(paths[0]) != s_playlists.end())
{
return hls_server_ts(session, paths[0], paths[1]);
}
}
}
else if (0 == strncmp(path, "/vod/", 5))
{
if (path_testfile(path+5))
{
return hls_server_reply_file(session, path + 5);
}
}
return hls_server_reply(session, 404, "");
}
HISINFOPOST_API BOOL PASCAL confirmPres(const char *url, const char *pres_no, int post_type)
{
url_schema *urls = url_parse(url);
if (!urls)
return FALSE;
char data[100] = { 0 };
sprintf(data, "pres_no=%s&post_type=%d", pres_no, post_type);
static TCHAR hdrs[] = _T("Content-Type: text/html");
BOOL result = TRUE;
if (ERROR_SUCCESS != httpPost(urls, hdrs, data))
result = FALSE;
url_free(urls);
return result;
}
int
main (int argc, char **argv)
{
struct file f;
char *line;
int i;
int n;
n = 0;
for (i = 1; i < argc; i++) {
file_open (&f, argv[i]);
while ((line = file_readline (&f))) {
check_parse (line);
n++;
}
file_close (&f);
}
url_free (&url);
printf ("Tested %d urls.\n", n);
}
HISINFOPOST_API int PASCAL postPres(const char *url, const char *xml, int post_type, const char *encoding)
{
int rc = 0;
url_schema *urls = url_parse(url);
if (!urls)
return ERROR_URL_INVALID;
HWND hParent = ::GetForegroundWindow();
rc = DialogBox(hInstance, MAKEINTRESOURCE(IDD_DIALOG1), hParent, DlgProc);
if (rc == ACTION_COMMIT) {
int length = strlen(xml) + 100;
std::vector<char> data;
data.reserve(length);
char *p = &data[0];
length = sprintf(p, "post_type=%d&encoding=%s&data=%s", post_type, encoding, xml);
p[length] = '\0';
static TCHAR hdrs[] = _T("Content-Type: application/xml");
if (ERROR_SUCCESS != httpPost(urls, hdrs, p))
rc = ERROR_POST_FAILED;
}
url_free(urls);
return rc;
}
int
http_send_request( URL_t *url, off_t pos ) {
HTTP_header_t *http_hdr;
URL_t *server_url;
char str[256];
int fd = -1;
int ret;
int proxy = 0; // Boolean
http_hdr = http_new_header();
if( !strcasecmp(url->protocol, "http_proxy") ) {
proxy = 1;
server_url = url_new( (url->file)+1 );
http_set_uri( http_hdr, server_url->url );
} else {
server_url = url;
http_set_uri( http_hdr, server_url->file );
}
if (server_url->port && server_url->port != 80)
snprintf(str, 256, "Host: %s:%d", server_url->hostname, server_url->port );
else
snprintf(str, 256, "Host: %s", server_url->hostname );
http_set_field( http_hdr, str);
if (network_useragent)
{
snprintf(str, 256, "User-Agent: %s", network_useragent);
http_set_field(http_hdr, str);
}
else
http_set_field( http_hdr, "User-Agent: MPlayer/"VERSION);
if( strcasecmp(url->protocol, "noicyx") )
http_set_field(http_hdr, "Icy-MetaData: 1");
if(pos>0) {
// Extend http_send_request with possibility to do partial content retrieval
snprintf(str, 256, "Range: bytes=%"PRId64"-", (int64_t)pos);
http_set_field(http_hdr, str);
}
if (network_cookies_enabled) cookies_set( http_hdr, server_url->hostname, server_url->url );
http_set_field( http_hdr, "Connection: close");
http_add_basic_authentication( http_hdr, url->username, url->password );
if( http_build_request( http_hdr )==NULL ) {
goto err_out;
}
if( proxy ) {
if( url->port==0 ) url->port = 8080; // Default port for the proxy server
fd = connect2Server( url->hostname, url->port,1 );
url_free( server_url );
server_url = NULL;
} else {
if( server_url->port==0 ) server_url->port = 80; // Default port for the web server
fd = connect2Server( server_url->hostname, server_url->port,1 );
}
if( fd<0 ) {
goto err_out;
}
mp_msg(MSGT_NETWORK,MSGL_DBG2,"Request: [%s]\n", http_hdr->buffer );
ret = send( fd, http_hdr->buffer, http_hdr->buffer_size, 0 );
if( ret!=(int)http_hdr->buffer_size ) {
mp_msg(MSGT_NETWORK,MSGL_ERR,MSGTR_MPDEMUX_NW_ErrSendingHTTPRequest);
goto err_out;
}
http_free( http_hdr );
return fd;
err_out:
if (fd > 0) closesocket(fd);
http_free(http_hdr);
if (proxy && server_url)
url_free(server_url);
return -1;
}
static void
convert_links_in_hashtable (struct hash_table *downloaded_set,
int is_css,
int *file_count)
{
int i;
int cnt;
char **file_array;
cnt = 0;
if (downloaded_set)
cnt = hash_table_count (downloaded_set);
if (cnt == 0)
return;
file_array = alloca_array (char *, cnt);
string_set_to_array (downloaded_set, file_array);
for (i = 0; i < cnt; i++)
{
struct urlpos *urls, *cur_url;
char *url;
char *file = file_array[i];
/* Determine the URL of the file. get_urls_{html,css} will need
it. */
url = hash_table_get (dl_file_url_map, file);
if (!url)
{
DEBUGP (("Apparently %s has been removed.\n", file));
continue;
}
DEBUGP (("Scanning %s (from %s)\n", file, url));
/* Parse the file... */
urls = is_css ? get_urls_css_file (file, url) :
get_urls_html (file, url, NULL, NULL);
/* We don't respect meta_disallow_follow here because, even if
the file is not followed, we might still want to convert the
links that have been followed from other files. */
for (cur_url = urls; cur_url; cur_url = cur_url->next)
{
char *local_name;
struct url *u;
struct iri *pi;
if (cur_url->link_base_p)
{
/* Base references have been resolved by our parser, so
we turn the base URL into an empty string. (Perhaps
we should remove the tag entirely?) */
cur_url->convert = CO_NULLIFY_BASE;
continue;
}
/* We decide the direction of conversion according to whether
a URL was downloaded. Downloaded URLs will be converted
ABS2REL, whereas non-downloaded will be converted REL2ABS. */
pi = iri_new ();
set_uri_encoding (pi, opt.locale, true);
u = url_parse (cur_url->url->url, NULL, pi, true);
if (!u)
continue;
local_name = hash_table_get (dl_url_file_map, u->url);
/* Decide on the conversion type. */
if (local_name)
{
/* We've downloaded this URL. Convert it to relative
form. We do this even if the URL already is in
relative form, because our directory structure may
not be identical to that on the server (think `-nd',
`--cut-dirs', etc.) */
cur_url->convert = CO_CONVERT_TO_RELATIVE;
cur_url->local_name = xstrdup (local_name);
DEBUGP (("will convert url %s to local %s\n", u->url, local_name));
}
else
{
/* We haven't downloaded this URL. If it's not already
complete (including a full host name), convert it to
that form, so it can be reached while browsing this
HTML locally. */
if (!cur_url->link_complete_p)
cur_url->convert = CO_CONVERT_TO_COMPLETE;
cur_url->local_name = NULL;
DEBUGP (("will convert url %s to complete\n", u->url));
}
url_free (u);
iri_free (pi);
}
/* Convert the links in the file. */
convert_links (file, urls);
++*file_count;
/* Free the data. */
free_urlpos (urls);
}
}
uerr_t
retrieve_from_file (const char *file, bool html, int *count)
{
uerr_t status;
struct urlpos *url_list, *cur_url;
struct iri *iri = iri_new();
char *input_file, *url_file = NULL;
const char *url = file;
status = RETROK; /* Suppose everything is OK. */
*count = 0; /* Reset the URL count. */
/* sXXXav : Assume filename and links in the file are in the locale */
set_uri_encoding (iri, opt.locale, true);
set_content_encoding (iri, opt.locale);
if (url_valid_scheme (url))
{
int dt,url_err;
uerr_t status;
struct url *url_parsed = url_parse (url, &url_err, iri, true);
if (!url_parsed)
{
char *error = url_error (url, url_err);
logprintf (LOG_NOTQUIET, "%s: %s.\n", url, error);
xfree (error);
return URLERROR;
}
if (!opt.base_href)
opt.base_href = xstrdup (url);
status = retrieve_url (url_parsed, url, &url_file, NULL, NULL, &dt,
false, iri, true);
url_free (url_parsed);
if (!url_file || (status != RETROK))
return status;
if (dt & TEXTHTML)
html = true;
/* If we have a found a content encoding, use it.
* ( == is okay, because we're checking for identical object) */
if (iri->content_encoding != opt.locale)
set_uri_encoding (iri, iri->content_encoding, false);
/* Reset UTF-8 encode status */
iri->utf8_encode = opt.enable_iri;
xfree_null (iri->orig_url);
iri->orig_url = NULL;
input_file = url_file;
}
else
input_file = (char *) file;
url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
: get_urls_file (input_file));
xfree_null (url_file);
for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
{
char *filename = NULL, *new_file = NULL;
int dt;
struct iri *tmpiri = iri_dup (iri);
struct url *parsed_url = NULL;
if (cur_url->ignore_when_downloading)
continue;
if (opt.quota && total_downloaded_bytes > opt.quota)
{
status = QUOTEXC;
break;
}
parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
if ((opt.recursive || opt.page_requisites)
&& (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
{
int old_follow_ftp = opt.follow_ftp;
/* Turn opt.follow_ftp on in case of recursive FTP retrieval */
if (cur_url->url->scheme == SCHEME_FTP)
opt.follow_ftp = 1;
status = retrieve_tree (parsed_url ? parsed_url : cur_url->url,
tmpiri);
opt.follow_ftp = old_follow_ftp;
}
else
status = retrieve_url (parsed_url ? parsed_url : cur_url->url,
cur_url->url->url, &filename,
&new_file, NULL, &dt, opt.recursive, tmpiri,
true);
if (parsed_url)
url_free (parsed_url);
if (filename && opt.delete_after && file_exists_p (filename))
{
DEBUGP (("\
Removing file due to --delete-after in retrieve_from_file():\n"));
logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
if (unlink (filename))
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
dt &= ~RETROKF;
}
xfree_null (new_file);
xfree_null (filename);
iri_free (tmpiri);
}
uerr_t
retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
char **newloc, const char *refurl, int *dt, bool recursive,
struct iri *iri, bool register_status)
{
uerr_t result;
char *url;
bool location_changed;
bool iri_fallbacked = 0;
int dummy;
char *mynewloc, *proxy;
struct url *u = orig_parsed, *proxy_url;
int up_error_code; /* url parse error code */
char *local_file;
int redirection_count = 0;
bool post_data_suspended = false;
char *saved_post_data = NULL;
char *saved_post_file_name = NULL;
/* If dt is NULL, use local storage. */
if (!dt)
{
dt = &dummy;
dummy = 0;
}
url = xstrdup (origurl);
if (newloc)
*newloc = NULL;
if (file)
*file = NULL;
if (!refurl)
refurl = opt.referer;
redirected:
/* (also for IRI fallbacking) */
result = NOCONERROR;
mynewloc = NULL;
local_file = NULL;
proxy_url = NULL;
proxy = getproxy (u);
if (proxy)
{
struct iri *pi = iri_new ();
set_uri_encoding (pi, opt.locale, true);
pi->utf8_encode = false;
/* Parse the proxy URL. */
proxy_url = url_parse (proxy, &up_error_code, NULL, true);
if (!proxy_url)
{
char *error = url_error (proxy, up_error_code);
logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"),
proxy, error);
xfree (url);
xfree (error);
RESTORE_POST_DATA;
result = PROXERR;
goto bail;
}
if (proxy_url->scheme != SCHEME_HTTP && proxy_url->scheme != u->scheme)
{
logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
url_free (proxy_url);
xfree (url);
RESTORE_POST_DATA;
result = PROXERR;
goto bail;
}
}
if (u->scheme == SCHEME_HTTP
#ifdef HAVE_SSL
|| u->scheme == SCHEME_HTTPS
#endif
|| (proxy_url && proxy_url->scheme == SCHEME_HTTP))
{
result = http_loop (u, orig_parsed, &mynewloc, &local_file, refurl, dt,
proxy_url, iri);
}
else if (u->scheme == SCHEME_FTP)
{
/* If this is a redirection, temporarily turn off opt.ftp_glob
and opt.recursive, both being undesirable when following
redirects. */
bool oldrec = recursive, glob = opt.ftp_glob;
if (redirection_count)
oldrec = glob = false;
result = ftp_loop (u, &local_file, dt, proxy_url, recursive, glob);
recursive = oldrec;
/* There is a possibility of having HTTP being redirected to
FTP. In these cases we must decide whether the text is HTML
according to the suffix. The HTML suffixes are `.html',
`.htm' and a few others, case-insensitive. */
if (redirection_count && local_file && u->scheme == SCHEME_FTP)
{
if (has_html_suffix_p (local_file))
*dt |= TEXTHTML;
}
}
if (proxy_url)
{
url_free (proxy_url);
proxy_url = NULL;
}
location_changed = (result == NEWLOCATION || result == NEWLOCATION_KEEP_POST);
if (location_changed)
{
char *construced_newloc;
struct url *newloc_parsed;
assert (mynewloc != NULL);
if (local_file)
xfree (local_file);
/* The HTTP specs only allow absolute URLs to appear in
redirects, but a ton of boneheaded webservers and CGIs out
there break the rules and use relative URLs, and popular
browsers are lenient about this, so wget should be too. */
construced_newloc = uri_merge (url, mynewloc);
xfree (mynewloc);
mynewloc = construced_newloc;
/* Reset UTF-8 encoding state, keep the URI encoding and reset
the content encoding. */
iri->utf8_encode = opt.enable_iri;
set_content_encoding (iri, NULL);
xfree_null (iri->orig_url);
iri->orig_url = NULL;
/* Now, see if this new location makes sense. */
newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true);
if (!newloc_parsed)
{
char *error = url_error (mynewloc, up_error_code);
logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc),
error);
if (orig_parsed != u)
{
url_free (u);
}
xfree (url);
xfree (mynewloc);
xfree (error);
RESTORE_POST_DATA;
goto bail;
}
/* Now mynewloc will become newloc_parsed->url, because if the
Location contained relative paths like .././something, we
don't want that propagating as url. */
xfree (mynewloc);
mynewloc = xstrdup (newloc_parsed->url);
/* Check for max. number of redirections. */
if (++redirection_count > opt.max_redirect)
{
logprintf (LOG_NOTQUIET, _("%d redirections exceeded.\n"),
opt.max_redirect);
url_free (newloc_parsed);
if (orig_parsed != u)
{
url_free (u);
}
xfree (url);
xfree (mynewloc);
RESTORE_POST_DATA;
result = WRONGCODE;
goto bail;
}
xfree (url);
url = mynewloc;
if (orig_parsed != u)
{
url_free (u);
}
u = newloc_parsed;
/* If we're being redirected from POST, and we received a
redirect code different than 307, we don't want to POST
again. Many requests answer POST with a redirection to an
index page; that redirection is clearly a GET. We "suspend"
POST data for the duration of the redirections, and restore
it when we're done.
RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
specifically to preserve the method of the request.
*/
if (result != NEWLOCATION_KEEP_POST && !post_data_suspended)
SUSPEND_POST_DATA;
goto redirected;
}
/* Try to not encode in UTF-8 if fetching failed */
if (!(*dt & RETROKF) && iri->utf8_encode)
{
iri->utf8_encode = false;
if (orig_parsed != u)
{
url_free (u);
}
u = url_parse (origurl, NULL, iri, true);
if (u)
{
DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
url = xstrdup (u->url);
iri_fallbacked = 1;
goto redirected;
}
else
DEBUGP (("[Couldn't fallback to non-utf8 for %s\n", quote (url)));
}
if (local_file && u && *dt & RETROKF)
{
register_download (u->url, local_file);
if (!opt.spider && redirection_count && 0 != strcmp (origurl, u->url))
register_redirection (origurl, u->url);
if (*dt & TEXTHTML)
register_html (local_file);
if (*dt & TEXTCSS)
register_css (local_file);
}
if (file)
*file = local_file ? local_file : NULL;
else
xfree_null (local_file);
if (orig_parsed != u)
{
url_free (u);
}
if (redirection_count || iri_fallbacked)
{
if (newloc)
*newloc = url;
else
xfree (url);
}
else
{
if (newloc)
*newloc = NULL;
xfree (url);
}
RESTORE_POST_DATA;
bail:
if (register_status)
inform_exit_status (result);
return result;
}
int
http_send_request( URL_t *url, int64_t pos ) {
HTTP_header_t *http_hdr;
URL_t *server_url;
char str[256];
int fd = -1;
int ret;
int proxy = 0; // Boolean
http_hdr = http_new_header();
if( !strcasecmp(url->protocol, "http_proxy") ) {
proxy = 1;
server_url = url_new( (url->file)+1 );
if (!server_url) {
mp_msg(MSGT_NETWORK, MSGL_ERR, "Invalid URL '%s' to proxify\n", url->file+1);
goto err_out;
}
http_set_uri( http_hdr, server_url->noauth_url );
} else {
server_url = url;
http_set_uri( http_hdr, server_url->file );
}
if (server_url->port && server_url->port != 80)
snprintf(str, sizeof(str), "Host: %s:%d", server_url->hostname, server_url->port );
else
snprintf(str, sizeof(str), "Host: %s", server_url->hostname );
http_set_field( http_hdr, str);
if (network_useragent)
snprintf(str, sizeof(str), "User-Agent: %s", network_useragent);
else
snprintf(str, sizeof(str), "User-Agent: %s", mplayer_version);
http_set_field(http_hdr, str);
if (network_referrer) {
char *referrer = NULL;
size_t len = strlen(network_referrer) + 10;
// Check len to ensure we don't do something really bad in case of an overflow
if (len > 10)
referrer = malloc(len);
if (referrer == NULL) {
mp_msg(MSGT_NETWORK, MSGL_FATAL, MSGTR_MemAllocFailed);
} else {
snprintf(referrer, len, "Referer: %s", network_referrer);
http_set_field(http_hdr, referrer);
free(referrer);
}
}
if( strcasecmp(url->protocol, "noicyx") )
http_set_field(http_hdr, "Icy-MetaData: 1");
if(pos>0) {
// Extend http_send_request with possibility to do partial content retrieval
snprintf(str, sizeof(str), "Range: bytes=%"PRId64"-", (int64_t)pos);
http_set_field(http_hdr, str);
}
if (network_cookies_enabled) cookies_set( http_hdr, server_url->hostname, server_url->url );
if (network_http_header_fields) {
int i=0;
while (network_http_header_fields[i])
http_set_field(http_hdr, network_http_header_fields[i++]);
}
http_set_field( http_hdr, "Connection: close");
if (proxy)
http_add_basic_proxy_authentication(http_hdr, url->username, url->password);
http_add_basic_authentication(http_hdr, server_url->username, server_url->password);
if( http_build_request( http_hdr )==NULL ) {
goto err_out;
}
if( proxy ) {
if( url->port==0 ) url->port = 8080; // Default port for the proxy server
fd = connect2Server( url->hostname, url->port,1 );
url_free( server_url );
server_url = NULL;
} else {
if( server_url->port==0 ) server_url->port = 80; // Default port for the web server
fd = connect2Server( server_url->hostname, server_url->port,1 );
}
if( fd<0 ) {
goto err_out;
}
mp_msg(MSGT_NETWORK,MSGL_DBG2,"Request: [%s]\n", http_hdr->buffer );
ret = send( fd, http_hdr->buffer, http_hdr->buffer_size, DEFAULT_SEND_FLAGS );
if( ret!=(int)http_hdr->buffer_size ) {
mp_msg(MSGT_NETWORK,MSGL_ERR,MSGTR_MPDEMUX_NW_ErrSendingHTTPRequest);
goto err_out;
}
http_free( http_hdr );
return fd;
err_out:
if (fd > 0) closesocket(fd);
http_free(http_hdr);
if (proxy && server_url)
url_free(server_url);
return -1;
}