static void *
dot_create (long initial, long total)
{
struct dot_progress *dp = xmalloc (sizeof (struct dot_progress));
memset (dp, 0, sizeof (*dp));
dp->initial_length = initial;
dp->total_length = total;
if (dp->initial_length)
{
int dot_bytes = opt.dot_bytes;
long row_bytes = opt.dot_bytes * opt.dots_in_line;
int remainder = (int) (dp->initial_length % row_bytes);
long skipped = dp->initial_length - remainder;
if (skipped)
{
int skipped_k = (int) (skipped / 1024); /* skipped amount in K */
int skipped_k_len = numdigit (skipped_k);
if (skipped_k_len < 5)
skipped_k_len = 5;
/* Align the [ skipping ... ] line with the dots. To do
that, insert the number of spaces equal to the number of
digits in the skipped amount in K. */
logprintf (LOG_VERBOSE, _("\n%*s[ skipping %dK ]"),
2 + skipped_k_len, "", skipped_k);
}
logprintf (LOG_VERBOSE, "\n%5ldK", skipped / 1024);
for (; remainder >= dot_bytes; remainder -= dot_bytes)
{
if (dp->dots % opt.dot_spacing == 0)
logputs (LOG_VERBOSE, " ");
logputs (LOG_VERBOSE, ",");
++dp->dots;
}
assert (dp->dots < opt.dots_in_line);
dp->accumulated = remainder;
dp->rows = skipped / row_bytes;
}
return dp;
}
void
print_broken_links (void)
{
hash_table_iterator iter;
int num_elems;
if (!nonexisting_urls_set)
{
logprintf (LOG_NOTQUIET, _("Found no broken links.\n\n"));
return;
}
num_elems = hash_table_count (nonexisting_urls_set);
assert (num_elems > 0);
logprintf (LOG_NOTQUIET, ngettext("Found %d broken link.\n\n",
"Found %d broken links.\n\n", num_elems),
num_elems);
for (hash_table_iterate (nonexisting_urls_set, &iter);
hash_table_iter_next (&iter); )
{
/* Struct url_list *list; */
const char *url = (const char *) iter.key;
logprintf (LOG_NOTQUIET, _("%s\n"), url);
}
logputs (LOG_NOTQUIET, "\n");
}
static void
dot_finish (void *progress, double dltime)
{
struct dot_progress *dp = progress;
int dot_bytes = opt.dot_bytes;
long row_bytes = opt.dot_bytes * opt.dots_in_line;
int i;
log_set_flush (0);
if (dp->dots == 0)
logprintf (LOG_VERBOSE, "\n%5ldK", dp->rows * row_bytes / 1024);
for (i = dp->dots; i < opt.dots_in_line; i++)
{
if (i % opt.dot_spacing == 0)
logputs (LOG_VERBOSE, " ");
logputs (LOG_VERBOSE, " ");
}
if (dp->total_length)
{
print_percentage (dp->rows * row_bytes
+ dp->dots * dot_bytes
+ dp->accumulated,
dp->total_length);
}
{
long row_qty = dp->dots * dot_bytes + dp->accumulated;
if (dp->rows == dp->initial_length / row_bytes)
row_qty -= dp->initial_length % row_bytes;
print_download_speed (dp, row_qty, dltime);
}
logputs (LOG_VERBOSE, "\n\n");
log_set_flush (0);
xfree (dp);
}
bool
res_retrieve_file (const char *url, char **file, struct iri *iri)
{
struct iri *i = iri_new ();
uerr_t err;
char *robots_url = uri_merge (url, RES_SPECS_LOCATION);
int saved_ts_val = opt.timestamping;
int saved_sp_val = opt.spider, url_err;
struct url * url_parsed;
/* Copy server URI encoding for a possible IDNA transformation, no need to
encode the full URI in UTF-8 because "robots.txt" is plain ASCII */
set_uri_encoding (i, iri->uri_encoding, false);
i->utf8_encode = false;
logputs (LOG_VERBOSE, _("Loading robots.txt; please ignore errors.\n"));
*file = NULL;
opt.timestamping = false;
opt.spider = false;
url_parsed = url_parse (robots_url, &url_err, i, true);
if (!url_parsed)
{
char *error = url_error (robots_url, url_err);
logprintf (LOG_NOTQUIET, "%s: %s.\n", robots_url, error);
xfree (error);
err = URLERROR;
}
else
{
err = retrieve_url (url_parsed, robots_url, file, NULL, NULL, NULL,
false, i, false);
url_free(url_parsed);
}
opt.timestamping = saved_ts_val;
opt.spider = saved_sp_val;
xfree (robots_url);
iri_free (i);
if (err != RETROK && *file != NULL)
{
/* If the file is not retrieved correctly, but retrieve_url
allocated the file name, deallocate is here so that the
caller doesn't have to worry about it. */
xfree (*file);
*file = NULL;
}
return err == RETROK;
}
static void
dot_update (void *progress, long howmuch, double dltime)
{
struct dot_progress *dp = progress;
int dot_bytes = opt.dot_bytes;
long row_bytes = opt.dot_bytes * opt.dots_in_line;
log_set_flush (0);
dp->accumulated += howmuch;
for (; dp->accumulated >= dot_bytes; dp->accumulated -= dot_bytes)
{
if (dp->dots == 0)
logprintf (LOG_VERBOSE, "\n%5ldK", dp->rows * row_bytes / 1024);
if (dp->dots % opt.dot_spacing == 0)
logputs (LOG_VERBOSE, " ");
logputs (LOG_VERBOSE, ".");
++dp->dots;
if (dp->dots >= opt.dots_in_line)
{
long row_qty = row_bytes;
if (dp->rows == dp->initial_length / row_bytes)
row_qty -= dp->initial_length % row_bytes;
++dp->rows;
dp->dots = 0;
if (dp->total_length)
print_percentage (dp->rows * row_bytes, dp->total_length);
print_download_speed (dp, row_qty, dltime);
}
}
log_set_flush (1);
}
/* Retrieves the robots_filename from the root server directory, if
possible. Returns ROBOTSOK if robots were retrieved OK, and
NOROBOTS if robots could not be retrieved for any reason. */
static uerr_t
retrieve_robots (const char *url, const char *robots_filename)
{
int dt;
uerr_t err;
struct urlinfo *u;
u = robots_url (url, robots_filename);
logputs (LOG_VERBOSE, _("Loading robots.txt; please ignore errors.\n"));
err = retrieve_url (u->url, NULL, NULL, NULL, &dt);
freeurl (u, 1);
if (err == RETROK)
return ROBOTSOK;
else
return NOROBOTS;
}
int kvprintf( const char *format, va_list args ){
int slen = strlen( format );
int i = 0;
int signed_int;
unsigned unsigned_int;
char buf;
const char *str;
for ( i = 0; i < slen; i++ ){
if ( format[i] == '%' ){
switch( format[++i] ){
case '%':
buf = '%';
write_serial( &buf, 1 );
break;
case 'c':
buf = va_arg( args, int );
write_serial( &buf, 1 );
break;
case 's':
str = va_arg( args, char * );
logputs( str );
break;
case 'd':
signed_int = va_arg( args, int );
print_num( signed_int );
break;
case 'u':
unsigned_int = va_arg( args, unsigned );
print_num( unsigned_int );
break;
case 'x':
unsigned_int = va_arg( args, unsigned );
print_hex( unsigned_int );
break;
}
} else {
write_serial( format + i, 1 );
}
}
int
res_retrieve_file (const char *url, char **file)
{
uerr_t err;
char *robots_url = uri_merge (url, RES_SPECS_LOCATION);
logputs (LOG_VERBOSE, _("Loading robots.txt; please ignore errors.\n"));
*file = NULL;
err = retrieve_url (robots_url, file, NULL, NULL, NULL);
xfree (robots_url);
if (err != RETROK && *file != NULL)
{
/* If the file is not retrieved correctly, but retrieve_url
allocated the file name, deallocate is here so that the
caller doesn't have to worry about it. */
xfree (*file);
*file = NULL;
}
return err == RETROK;
}
struct address_list *
lookup_host (const char *host, int flags)
{
struct address_list *al;
int silent = flags & LH_SILENT;
int use_cache;
int numeric_address = 0;
double timeout = opt.dns_timeout;
#ifndef ENABLE_IPV6
/* If we're not using getaddrinfo, first check if HOST specifies a
numeric IPv4 address. Some implementations of gethostbyname
(e.g. the Ultrix one and possibly Winsock) don't accept
dotted-decimal IPv4 addresses. */
{
uint32_t addr_ipv4 = (uint32_t)inet_addr (host);
if (addr_ipv4 != (uint32_t) -1)
{
/* No need to cache host->addr relation, just return the
address. */
char *vec[2];
vec[0] = (char *)&addr_ipv4;
vec[1] = NULL;
return address_list_from_ipv4_addresses (vec);
}
}
#else /* ENABLE_IPV6 */
/* If we're using getaddrinfo, at least check whether the address is
already numeric, in which case there is no need to print the
"Resolving..." output. (This comes at no additional cost since
the is_valid_ipv*_address are already required for
url_parse.) */
{
const char *end = host + strlen (host);
if (is_valid_ipv4_address (host, end) || is_valid_ipv6_address (host, end))
numeric_address = 1;
}
#endif
/* Cache is normally on, but can be turned off with --no-dns-cache.
Don't cache passive lookups under IPv6. */
use_cache = opt.dns_cache;
#ifdef ENABLE_IPV6
if ((flags & LH_BIND) || numeric_address)
use_cache = 0;
#endif
/* Try to find the host in the cache so we don't need to talk to the
resolver. If LH_REFRESH is requested, remove HOST from the cache
instead. */
if (use_cache)
{
if (!(flags & LH_REFRESH))
{
al = cache_query (host);
if (al)
return al;
}
else
cache_remove (host);
}
/* No luck with the cache; resolve HOST. */
if (!silent && !numeric_address)
logprintf (LOG_VERBOSE, _("Resolving %s... "), escnonprint (host));
#ifdef ENABLE_IPV6
{
int err;
struct addrinfo hints, *res;
xzero (hints);
hints.ai_socktype = SOCK_STREAM;
if (opt.ipv4_only)
hints.ai_family = AF_INET;
else if (opt.ipv6_only)
hints.ai_family = AF_INET6;
else
/* We tried using AI_ADDRCONFIG, but removed it because: it
misinterprets IPv6 loopbacks, it is broken on AIX 5.1, and
it's unneeded since we sort the addresses anyway. */
hints.ai_family = AF_UNSPEC;
if (flags & LH_BIND)
hints.ai_flags |= AI_PASSIVE;
#ifdef AI_NUMERICHOST
if (numeric_address)
{
/* Where available, the AI_NUMERICHOST hint can prevent costly
access to DNS servers. */
hints.ai_flags |= AI_NUMERICHOST;
timeout = 0; /* no timeout needed when "resolving"
numeric hosts -- avoid setting up
signal handlers and such. */
}
#endif
err = getaddrinfo_with_timeout (host, NULL, &hints, &res, timeout);
if (err != 0 || res == NULL)
{
if (!silent)
logprintf (LOG_VERBOSE, _("failed: %s.\n"),
err != EAI_SYSTEM ? gai_strerror (err) : strerror (errno));
return NULL;
}
al = address_list_from_addrinfo (res);
freeaddrinfo (res);
if (!al)
{
logprintf (LOG_VERBOSE,
_("failed: No IPv4/IPv6 addresses for host.\n"));
return NULL;
}
/* Reorder addresses so that IPv4 ones (or IPv6 ones, as per
--prefer-family) come first. Sorting is stable so the order of
the addresses with the same family is undisturbed. */
if (al->count > 1 && opt.prefer_family != prefer_none)
stable_sort (al->addresses, al->count, sizeof (ip_address),
opt.prefer_family == prefer_ipv4
? cmp_prefer_ipv4 : cmp_prefer_ipv6);
}
#else /* not ENABLE_IPV6 */
{
struct hostent *hptr = gethostbyname_with_timeout (host, timeout);
if (!hptr)
{
if (!silent)
{
if (errno != ETIMEDOUT)
logprintf (LOG_VERBOSE, _("failed: %s.\n"),
host_errstr (h_errno));
else
logputs (LOG_VERBOSE, _("failed: timed out.\n"));
}
return NULL;
}
/* Do older systems have h_addr_list? */
al = address_list_from_ipv4_addresses (hptr->h_addr_list);
}
#endif /* not ENABLE_IPV6 */
/* Print the addresses determined by DNS lookup, but no more than
three. */
if (!silent && !numeric_address)
{
int i;
int printmax = al->count <= 3 ? al->count : 3;
for (i = 0; i < printmax; i++)
{
logprintf (LOG_VERBOSE, "%s",
pretty_print_address (al->addresses + i));
if (i < printmax - 1)
logputs (LOG_VERBOSE, ", ");
}
if (printmax != al->count)
logputs (LOG_VERBOSE, ", ...");
logputs (LOG_VERBOSE, "\n");
}
/* Cache the lookup information. */
if (use_cache)
cache_store (host, al);
return al;
}
/* Change the links in one file. LINKS is a list of links in the
document, along with their positions and the desired direction of
the conversion. */
static void
convert_links (const char *file, struct urlpos *links)
{
struct file_memory *fm;
FILE *fp;
const char *p;
downloaded_file_t downloaded_file_return;
struct urlpos *link;
int to_url_count = 0, to_file_count = 0;
logprintf (LOG_VERBOSE, _("Converting %s... "), file);
{
/* First we do a "dry run": go through the list L and see whether
any URL needs to be converted in the first place. If not, just
leave the file alone. */
int dry_count = 0;
struct urlpos *dry;
for (dry = links; dry; dry = dry->next)
if (dry->convert != CO_NOCONVERT)
++dry_count;
if (!dry_count)
{
logputs (LOG_VERBOSE, _("nothing to do.\n"));
return;
}
}
fm = wget_read_file (file);
if (!fm)
{
logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"),
file, strerror (errno));
return;
}
downloaded_file_return = downloaded_file (CHECK_FOR_FILE, file);
if (opt.backup_converted && downloaded_file_return)
write_backup_file (file, downloaded_file_return);
/* Before opening the file for writing, unlink the file. This is
important if the data in FM is mmaped. In such case, nulling the
file, which is what fopen() below does, would make us read all
zeroes from the mmaped region. */
if (unlink (file) < 0 && errno != ENOENT)
{
logprintf (LOG_NOTQUIET, _("Unable to delete %s: %s\n"),
quote (file), strerror (errno));
wget_read_file_free (fm);
return;
}
/* Now open the file for writing. */
fp = fopen (file, "wb");
if (!fp)
{
logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"),
file, strerror (errno));
wget_read_file_free (fm);
return;
}
/* Here we loop through all the URLs in file, replacing those of
them that are downloaded with relative references. */
p = fm->content;
for (link = links; link; link = link->next)
{
char *url_start = fm->content + link->pos;
if (link->pos >= fm->length)
{
DEBUGP (("Something strange is going on. Please investigate."));
break;
}
/* If the URL is not to be converted, skip it. */
if (link->convert == CO_NOCONVERT)
{
DEBUGP (("Skipping %s at position %d.\n", link->url->url, link->pos));
continue;
}
/* Echo the file contents, up to the offending URL's opening
quote, to the outfile. */
fwrite (p, 1, url_start - p, fp);
p = url_start;
switch (link->convert)
{
case CO_CONVERT_TO_RELATIVE:
/* Convert absolute URL to relative. */
{
char *newname = construct_relative (file, link->local_name);
char *quoted_newname = local_quote_string (newname,
link->link_css_p);
if (link->link_css_p)
p = replace_plain (p, link->size, fp, quoted_newname);
else if (!link->link_refresh_p)
p = replace_attr (p, link->size, fp, quoted_newname);
else
p = replace_attr_refresh_hack (p, link->size, fp, quoted_newname,
link->refresh_timeout);
DEBUGP (("TO_RELATIVE: %s to %s at position %d in %s.\n",
link->url->url, newname, link->pos, file));
xfree (newname);
xfree (quoted_newname);
++to_file_count;
break;
}
case CO_CONVERT_TO_COMPLETE:
/* Convert the link to absolute URL. */
{
char *newlink = link->url->url;
char *quoted_newlink = html_quote_string (newlink);
if (link->link_css_p)
p = replace_plain (p, link->size, fp, newlink);
else if (!link->link_refresh_p)
p = replace_attr (p, link->size, fp, quoted_newlink);
else
p = replace_attr_refresh_hack (p, link->size, fp, quoted_newlink,
link->refresh_timeout);
DEBUGP (("TO_COMPLETE: <something> to %s at position %d in %s.\n",
newlink, link->pos, file));
xfree (quoted_newlink);
++to_url_count;
break;
}
case CO_NULLIFY_BASE:
/* Change the base href to "". */
p = replace_attr (p, link->size, fp, "");
break;
case CO_NOCONVERT:
abort ();
break;
}
}
/* Output the rest of the file. */
if (p - fm->content < fm->length)
fwrite (p, 1, fm->length - (p - fm->content), fp);
fclose (fp);
wget_read_file_free (fm);
logprintf (LOG_VERBOSE, "%d-%d\n", to_file_count, to_url_count);
}
/* Loop through all files in metalink structure and retrieve them.
Returns RETROK if all files were downloaded.
Returns last retrieval error (from retrieve_url) if some files
could not be downloaded. */
uerr_t
retrieve_from_metalink (const metalink_t* metalink)
{
metalink_file_t **mfile_ptr;
uerr_t last_retr_err = RETROK; /* Store last encountered retrieve error. */
FILE *_output_stream = output_stream;
bool _output_stream_regular = output_stream_regular;
char *_output_document = opt.output_document;
DEBUGP (("Retrieving from Metalink\n"));
/* No files to download. */
if (!metalink->files)
return RETROK;
if (opt.output_document)
{
/* We cannot support output_document as we need to compute checksum
of downloaded file, and to remove it if the checksum is bad. */
logputs (LOG_NOTQUIET,
_("-O not supported for metalink download. Ignoring.\n"));
}
for (mfile_ptr = metalink->files; *mfile_ptr; mfile_ptr++)
{
metalink_file_t *mfile = *mfile_ptr;
metalink_resource_t **mres_ptr;
char *filename = NULL;
bool hash_ok = false;
uerr_t retr_err = METALINK_MISSING_RESOURCE;
/* -1 -> file should be rejected
0 -> could not verify
1 -> verified successfully */
char sig_status = 0;
output_stream = NULL;
DEBUGP (("Processing metalink file %s...\n", quote (mfile->name)));
/* Resources are sorted by priority. */
for (mres_ptr = mfile->resources; *mres_ptr; mres_ptr++)
{
metalink_resource_t *mres = *mres_ptr;
metalink_checksum_t **mchksum_ptr, *mchksum;
struct iri *iri;
struct url *url;
int url_err;
if (!RES_TYPE_SUPPORTED (mres->type))
{
logprintf (LOG_VERBOSE,
_("Resource type %s not supported, ignoring...\n"),
quote (mres->type));
continue;
}
retr_err = METALINK_RETR_ERROR;
/* If output_stream is not NULL, then we have failed on
previous resource and are retrying. Thus, remove the file. */
if (output_stream)
{
fclose (output_stream);
output_stream = NULL;
if (unlink (filename))
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
xfree (filename);
}
/* Parse our resource URL. */
iri = iri_new ();
set_uri_encoding (iri, opt.locale, true);
url = url_parse (mres->url, &url_err, iri, false);
if (!url)
{
char *error = url_error (mres->url, url_err);
logprintf (LOG_NOTQUIET, "%s: %s.\n", mres->url, error);
xfree (error);
inform_exit_status (URLERROR);
iri_free (iri);
continue;
}
else
{
/* Avoid recursive Metalink from HTTP headers. */
bool _metalink_http = opt.metalink_over_http;
/* Assure proper local file name regardless of the URL
of particular Metalink resource.
To do that we create the local file here and put
it as output_stream. We restore the original configuration
after we are finished with the file. */
output_stream = unique_create (mfile->name, true, &filename);
output_stream_regular = true;
/* Store the real file name for displaying in messages. */
opt.output_document = filename;
opt.metalink_over_http = false;
DEBUGP (("Storing to %s\n", filename));
retr_err = retrieve_url (url, mres->url, NULL, NULL,
NULL, NULL, opt.recursive, iri, false);
opt.metalink_over_http = _metalink_http;
}
url_free (url);
iri_free (iri);
if (retr_err == RETROK)
{
FILE *local_file;
/* Check the digest. */
local_file = fopen (filename, "rb");
if (!local_file)
{
logprintf (LOG_NOTQUIET, _("Could not open downloaded file.\n"));
continue;
}
for (mchksum_ptr = mfile->checksums; *mchksum_ptr; mchksum_ptr++)
{
char sha256[SHA256_DIGEST_SIZE];
char sha256_txt[2 * SHA256_DIGEST_SIZE + 1];
mchksum = *mchksum_ptr;
/* I have seen both variants... */
if (strcasecmp (mchksum->type, "sha256")
&& strcasecmp (mchksum->type, "sha-256"))
{
DEBUGP (("Ignoring unsupported checksum type %s.\n",
quote (mchksum->type)));
continue;
}
logprintf (LOG_VERBOSE, _("Computing checksum for %s\n"),
quote (mfile->name));
sha256_stream (local_file, sha256);
wg_hex_to_string (sha256_txt, sha256, SHA256_DIGEST_SIZE);
DEBUGP (("Declared hash: %s\n", mchksum->hash));
DEBUGP (("Computed hash: %s\n", sha256_txt));
if (!strcmp (sha256_txt, mchksum->hash))
{
logputs (LOG_VERBOSE,
_("Checksum matches.\n"));
hash_ok = true;
}
else
{
logprintf (LOG_NOTQUIET,
_("Checksum mismatch for file %s.\n"),
quote (mfile->name));
hash_ok = false;
}
/* Stop as soon as we checked the supported checksum. */
break;
} /* Iterate over available checksums. */
fclose (local_file);
local_file = NULL;
if (!hash_ok)
continue;
sig_status = 0; /* Not verified. */
#ifdef HAVE_GPGME
/* Check the crypto signature.
Note that the signtures from Metalink in XML will not be
parsed when using libmetalink version older than 0.1.3.
Metalink-over-HTTP is not affected by this problem. */
if (mfile->signature)
{
metalink_signature_t *msig = mfile->signature;
gpgme_error_t gpgerr;
gpgme_ctx_t gpgctx;
gpgme_data_t gpgsigdata, gpgdata;
gpgme_verify_result_t gpgres;
gpgme_signature_t gpgsig;
gpgme_protocol_t gpgprot = GPGME_PROTOCOL_UNKNOWN;
int fd = -1;
/* Initialize the library - as name suggests. */
gpgme_check_version (NULL);
/* Open data file. */
fd = open (filename, O_RDONLY);
if (fd == -1)
{
logputs (LOG_NOTQUIET,
_("Could not open downloaded file for signature "
"verification.\n"));
goto gpg_skip_verification;
}
/* Assign file descriptor to GPG data structure. */
gpgerr = gpgme_data_new_from_fd (&gpgdata, fd);
if (gpgerr != GPG_ERR_NO_ERROR)
{
logprintf (LOG_NOTQUIET,
"GPGME data_new_from_fd: %s\n",
gpgme_strerror (gpgerr));
goto gpg_skip_verification;
}
/* Prepare new GPGME context. */
gpgerr = gpgme_new (&gpgctx);
if (gpgerr != GPG_ERR_NO_ERROR)
{
logprintf (LOG_NOTQUIET,
"GPGME new: %s\n",
gpgme_strerror (gpgerr));
gpgme_data_release (gpgdata);
goto gpg_skip_verification;
}
DEBUGP (("Verifying signature %s:\n%s\n",
quote (msig->mediatype),
msig->signature));
/* Check signature type. */
if (!strcmp (msig->mediatype, "application/pgp-signature"))
gpgprot = GPGME_PROTOCOL_OpenPGP;
else /* Unsupported signature type. */
{
gpgme_release (gpgctx);
gpgme_data_release (gpgdata);
goto gpg_skip_verification;
}
gpgerr = gpgme_set_protocol (gpgctx, gpgprot);
if (gpgerr != GPG_ERR_NO_ERROR)
{
logprintf (LOG_NOTQUIET,
"GPGME set_protocol: %s\n",
gpgme_strerror (gpgerr));
gpgme_release (gpgctx);
gpgme_data_release (gpgdata);
goto gpg_skip_verification;
}
/* Load the signature. */
gpgerr = gpgme_data_new_from_mem (&gpgsigdata,
msig->signature,
strlen (msig->signature),
0);
if (gpgerr != GPG_ERR_NO_ERROR)
{
logprintf (LOG_NOTQUIET,
_("GPGME data_new_from_mem: %s\n"),
gpgme_strerror (gpgerr));
gpgme_release (gpgctx);
gpgme_data_release (gpgdata);
goto gpg_skip_verification;
}
/* Verify the signature. */
gpgerr = gpgme_op_verify (gpgctx, gpgsigdata, gpgdata, NULL);
if (gpgerr != GPG_ERR_NO_ERROR)
{
logprintf (LOG_NOTQUIET,
_("GPGME op_verify: %s\n"),
gpgme_strerror (gpgerr));
gpgme_data_release (gpgsigdata);
gpgme_release (gpgctx);
gpgme_data_release (gpgdata);
goto gpg_skip_verification;
}
/* Check the results. */
gpgres = gpgme_op_verify_result (gpgctx);
if (!gpgres)
{
logputs (LOG_NOTQUIET,
_("GPGME op_verify_result: NULL\n"));
gpgme_data_release (gpgsigdata);
gpgme_release (gpgctx);
gpgme_data_release (gpgdata);
goto gpg_skip_verification;
}
/* The list is null-terminated. */
for (gpgsig = gpgres->signatures; gpgsig; gpgsig = gpgsig->next)
{
DEBUGP (("Checking signature %s\n", gpgsig->fpr));
if (gpgsig->summary
& (GPGME_SIGSUM_VALID | GPGME_SIGSUM_GREEN))
{
logputs (LOG_VERBOSE,
_("Signature validation suceeded.\n"));
sig_status = 1;
break;
}
if (gpgsig->summary & GPGME_SIGSUM_RED)
{
logputs (LOG_NOTQUIET,
_("Invalid signature. Rejecting resource.\n"));
sig_status = -1;
break;
}
if (gpgsig->summary == 0
&& (gpgsig->status & 0xFFFF) == GPG_ERR_NO_ERROR)
{
logputs (LOG_VERBOSE,
_("Data matches signature, but signature "
"is not trusted.\n"));
}
if ((gpgsig->status & 0xFFFF) != GPG_ERR_NO_ERROR)
{
logprintf (LOG_NOTQUIET,
"GPGME: %s\n",
gpgme_strerror (gpgsig->status & 0xFFFF));
}
}
gpgme_data_release (gpgsigdata);
gpgme_release (gpgctx);
gpgme_data_release (gpgdata);
gpg_skip_verification:
if (fd != -1)
close (fd);
} /* endif (mfile->signature) */
#endif
/* Stop if file was downloaded with success. */
if (sig_status >= 0)
break;
} /* endif RETR_OK. */
} /* Iterate over resources. */
if (retr_err != RETROK)
{
logprintf (LOG_VERBOSE, _("Failed to download %s. Skipping resource.\n"),
quote (mfile->name));
}
else if (!hash_ok)
{
retr_err = METALINK_CHKSUM_ERROR;
logprintf (LOG_NOTQUIET,
_("File %s retrieved but checksum does not match. "
"\n"), quote (mfile->name));
}
#ifdef HAVE_GPGME
/* Signature will be only validated if hash check was successful. */
else if (sig_status < 0)
{
retr_err = METALINK_SIG_ERROR;
logprintf (LOG_NOTQUIET,
_("File %s retrieved but signature does not match. "
"\n"), quote (mfile->name));
}
#endif
last_retr_err = retr_err == RETROK ? last_retr_err : retr_err;
/* Remove the file if error encountered or if option specified.
Note: the file has been downloaded using *_loop. Therefore, it
is not necessary to keep the file for continuated download. */
if ((retr_err != RETROK || opt.delete_after)
&& filename != NULL && file_exists_p (filename))
{
logprintf (LOG_VERBOSE, _("Removing %s.\n"), quote (filename));
if (unlink (filename))
logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
}
if (output_stream)
{
fclose (output_stream);
output_stream = NULL;
}
xfree (filename);
} /* Iterate over files. */
/* Restore original values. */
opt.output_document = _output_document;
output_stream_regular = _output_stream_regular;
output_stream = _output_stream;
return last_retr_err;
}
struct address_list *
lookup_host (const char *host, int silent)
{
struct address_list *al = NULL;
uint32_t addr_ipv4;
ip_address addr;
/* First, try to check whether the address is already a numeric
address. */
#ifdef ENABLE_IPV6
if (inet_pton (AF_INET6, host, &addr) > 0)
return address_list_from_single (&addr);
#endif
addr_ipv4 = (uint32_t)inet_addr (host);
if (addr_ipv4 != (uint32_t)-1)
{
/* ADDR is defined to be in network byte order, which is what
this returns, so we can just copy it to STORE_IP. */
map_ipv4_to_ip ((ip4_address *)&addr_ipv4, &addr);
return address_list_from_single (&addr);
}
if (host_name_addresses_map)
{
al = hash_table_get (host_name_addresses_map, host);
if (al)
{
DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, al));
++al->refcount;
return al;
}
}
if (!silent)
logprintf (LOG_VERBOSE, _("Resolving %s... "), host);
/* Host name lookup goes on below. */
#ifdef HAVE_GETADDRINFO
{
struct addrinfo hints, *ai;
int err;
memset (&hints, 0, sizeof (hints));
if (ip_default_family == AF_INET)
hints.ai_family = AF_INET;
else
hints.ai_family = PF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
err = getaddrinfo_with_timeout (host, NULL, &hints, &ai, opt.dns_timeout);
if (err != 0 || ai == NULL)
{
if (!silent)
logprintf (LOG_VERBOSE, _("failed: %s.\n"),
err != EAI_SYSTEM ? gai_strerror (err) : strerror (errno));
return NULL;
}
al = address_list_from_addrinfo (ai);
freeaddrinfo (ai);
}
#else
{
struct hostent *hptr;
hptr = gethostbyname_with_timeout (host, opt.dns_timeout);
if (!hptr)
{
if (!silent)
{
if (errno != ETIMEDOUT)
logprintf (LOG_VERBOSE, _("failed: %s.\n"), herrmsg (h_errno));
else
logputs (LOG_VERBOSE, _("failed: timed out.\n"));
}
return NULL;
}
/* Do all systems have h_addr_list, or is it a newer thing? If
the latter, use address_list_from_single. */
al = address_list_from_vector (hptr->h_addr_list);
}
#endif
/* Print the addresses determined by DNS lookup, but no more than
three. */
if (!silent)
{
int i;
int printmax = al->count <= 3 ? al->count : 3;
for (i = 0; i < printmax; i++)
{
logprintf (LOG_VERBOSE, "%s",
pretty_print_address (al->addresses + i));
if (i < printmax - 1)
logputs (LOG_VERBOSE, ", ");
}
if (printmax != al->count)
logputs (LOG_VERBOSE, ", ...");
logputs (LOG_VERBOSE, "\n");
}
/* Cache the lookup information. */
if (opt.dns_cache)
cache_host_lookup (host, al);
return al;
}
/* Print `giving up', or `retrying', depending on the impending
action. N1 and N2 are the attempt number and the attempt limit. */
void
printwhat (int n1, int n2)
{
logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));
}
/* Retrieve the given URL. Decides which loop to call -- HTTP, FTP,
or simply copy it with file:// (#### the latter not yet
implemented!). */
uerr_t
retrieve_url (const char *origurl, char **file, char **newloc,
const char *refurl, int *dt)
{
uerr_t result;
char *url;
int location_changed, already_redirected, dummy;
int local_use_proxy;
char *mynewloc, *proxy;
struct urlinfo *u;
/* If dt is NULL, just ignore it. */
if (!dt)
dt = &dummy;
url = xstrdup (origurl);
if (newloc)
*newloc = NULL;
if (file)
*file = NULL;
already_redirected = 0;
again:
u = newurl ();
/* Parse the URL. RFC2068 requires `Location' to contain an
absoluteURI, but many sites break this requirement. #### We
should be liberal and accept a relative location, too. */
result = parseurl (url, u, already_redirected);
if (result != URLOK)
{
freeurl (u, 1);
logprintf (LOG_NOTQUIET, "%s: %s.\n", url, uerrmsg (result));
return result;
}
/* Set the referer. */
if (refurl)
u->referer = xstrdup (refurl);
else
u->referer = NULL;
local_use_proxy = USE_PROXY_P (u);
if (local_use_proxy)
{
struct urlinfo *pu = newurl ();
/* Copy the original URL to new location. */
memcpy (pu, u, sizeof (*u));
pu->proxy = NULL; /* A minor correction :) */
/* Initialize u to nil. */
memset (u, 0, sizeof (*u));
u->proxy = pu;
/* Get the appropriate proxy server, appropriate for the
current protocol. */
proxy = getproxy (pu->proto);
if (!proxy)
{
logputs (LOG_NOTQUIET, _("Could not find proxy host.\n"));
freeurl (u, 1);
return PROXERR;
}
/* Parse the proxy URL. */
result = parseurl (proxy, u, 0);
if (result != URLOK || u->proto != URLHTTP)
{
if (u->proto == URLHTTP)
logprintf (LOG_NOTQUIET, "Proxy %s: %s.\n", proxy, uerrmsg (result));
else
logprintf (LOG_NOTQUIET, _("Proxy %s: Must be HTTP.\n"), proxy);
freeurl (u, 1);
return PROXERR;
}
u->proto = URLHTTP;
}
assert (u->proto != URLFILE); /* #### Implement me! */
mynewloc = NULL;
if (u->proto == URLHTTP)
result = http_loop (u, &mynewloc, dt);
else if (u->proto == URLFTP)
{
/* If this is a redirection, we must not allow recursive FTP
retrieval, so we save recursion to oldrec, and restore it
later. */
int oldrec = opt.recursive;
if (already_redirected)
opt.recursive = 0;
result = ftp_loop (u, dt);
opt.recursive = oldrec;
/* There is a possibility of having HTTP being redirected to
FTP. In these cases we must decide whether the text is HTML
according to the suffix. The HTML suffixes are `.html' and
`.htm', case-insensitive.
#### All of this is, of course, crap. These types should be
determined through mailcap. */
if (already_redirected && u->local && (u->proto == URLFTP ))
{
char *suf = suffix (u->local);
if (suf && (!strcasecmp (suf, "html") || !strcasecmp (suf, "htm")))
*dt |= TEXTHTML;
FREE_MAYBE (suf);
}
}
location_changed = (result == NEWLOCATION);
if (location_changed)
{
/* Check for redirection to oneself. */
if (url_equal (url, mynewloc))
{
logprintf (LOG_NOTQUIET, _("%s: Redirection to itself.\n"),
mynewloc);
return WRONGCODE;
}
if (mynewloc)
{
free (url);
url = mynewloc;
}
freeurl (u, 1);
already_redirected = 1;
goto again;
}
if (file)
{
if (u->local)
*file = xstrdup (u->local);
else
*file = NULL;
}
freeurl (u, 1);
if (newloc)
*newloc = url;
else
free (url);
return result;
}
/* Show the dotted progress report of file loading. Called with
length and a flag to tell it whether to reset or not. It keeps the
offset information in static local variables.
Return value: 1 or 0, designating whether any dots have been drawn.
If the init argument is set, the routine will initialize.
If the res is non-zero, res/line_bytes lines are skipped
(meaning the appropriate number ok kilobytes), and the number of
"dots" fitting on the first line are drawn as ','. */
static int
show_progress (long res, long expected, enum spflags flags)
{
static long line_bytes;
static long offs;
static int ndot, nrow;
int any_output = 0;
if (flags == SP_FINISH)
{
if (expected)
{
int dot = ndot;
char *tmpstr = (char *)alloca (2 * opt.dots_in_line + 1);
char *tmpp = tmpstr;
for (; dot < opt.dots_in_line; dot++)
{
if (!(dot % opt.dot_spacing))
*tmpp++ = ' ';
*tmpp++ = ' ';
}
*tmpp = '\0';
logputs (LOG_VERBOSE, tmpstr);
print_percentage (nrow * line_bytes + ndot * opt.dot_bytes + offs,
expected);
}
logputs (LOG_VERBOSE, "\n\n");
return 0;
}
/* Temporarily disable flushing. */
opt.no_flush = 1;
/* init set means initialization. If res is set, it also means that
the retrieval is *not* done from the beginning. The part that
was already retrieved is not shown again. */
if (flags == SP_INIT)
{
/* Generic initialization of static variables. */
offs = 0L;
ndot = nrow = 0;
line_bytes = (long)opt.dots_in_line * opt.dot_bytes;
if (res)
{
if (res >= line_bytes)
{
nrow = res / line_bytes;
res %= line_bytes;
logprintf (LOG_VERBOSE,
_("\n [ skipping %dK ]"),
(int) ((nrow * line_bytes) / 1024));
ndot = 0;
}
}
logprintf (LOG_VERBOSE, "\n%5ldK ->", nrow * line_bytes / 1024);
}
/* Offset gets incremented by current value. */
offs += res;
/* While offset is >= opt.dot_bytes, print dots, taking care to
precede every 50th dot with a status message. */
for (; offs >= opt.dot_bytes; offs -= opt.dot_bytes)
{
if (!(ndot % opt.dot_spacing))
logputs (LOG_VERBOSE, " ");
any_output = 1;
logputs (LOG_VERBOSE, flags == SP_INIT ? "," : ".");
++ndot;
if (ndot == opt.dots_in_line)
{
ndot = 0;
++nrow;
if (expected)
print_percentage (nrow * line_bytes, expected);
logprintf (LOG_VERBOSE, "\n%5ldK ->", nrow * line_bytes / 1024);
}
}
/* Reenable flushing. */
opt.no_flush = 0;
if (any_output)
/* Force flush. #### Oh, what a kludge! */
logflush ();
return any_output;
}
