mget_iri_t *blacklist_add(mget_iri_t *iri)
{
if (!iri)
return NULL;
if (mget_iri_supported(iri)) {
mget_thread_mutex_lock(&mutex);
if (!blacklist) {
blacklist = mget_hashmap_create(128, -2, (unsigned int(*)(const void *))hash_iri, (int(*)(const void *, const void *))mget_iri_compare);
mget_hashmap_set_destructor(blacklist, (void(*)(void *, void *))_free_entry);
}
if (!mget_hashmap_contains(blacklist, iri)) {
// info_printf("Add to blacklist: %s\n",iri->uri);
mget_hashmap_put_noalloc(blacklist, iri, NULL); // use hashmap as a hashset (without value)
mget_thread_mutex_unlock(&mutex);
return iri;
}
mget_thread_mutex_unlock(&mutex);
}
mget_iri_free(&iri);
return NULL;
}
void queue_del(JOB *job)
{
if (job) {
debug_printf("queue_del %p\n", (void *)job);
// special handling for automatic robots.txt jobs
if (job->deferred) {
JOB new_job = { .iri = NULL };
if (job->host)
job->host->robot_job = NULL;
mget_iri_free(&job->iri);
// create a job for each deferred IRI
for (int it = 0; it < mget_vector_size(job->deferred); it++) {
new_job.iri = mget_vector_get(job->deferred, it);
new_job.local_filename = get_local_filename(new_job.iri);
queue_add_job(&new_job);
}
}
job_free(job);
mget_thread_mutex_lock(&mutex);
mget_list_remove(&queue, job);
mget_thread_mutex_unlock(&mutex);
}
void mget_ocsp_db_deinit(mget_ocsp_db_t *ocsp_db)
{
if (ocsp_db) {
mget_thread_mutex_lock(&ocsp_db->mutex);
mget_hashmap_free(&ocsp_db->fingerprints);
mget_hashmap_free(&ocsp_db->hosts);
mget_thread_mutex_unlock(&ocsp_db->mutex);
}
}
void mget_cookie_db_deinit(mget_cookie_db_t *cookie_db)
{
if (cookie_db) {
#ifdef WITH_LIBPSL
psl_free(cookie_db->psl);
cookie_db->psl = NULL;
#endif
mget_thread_mutex_lock(&cookie_db->mutex);
mget_vector_free(&cookie_db->cookies);
mget_thread_mutex_unlock(&cookie_db->mutex);
}
}
int mget_cookie_db_save(mget_cookie_db_t *cookie_db, const char *fname, int keep_session_cookies)
{
FILE *fp;
int it, ret = -1;
time_t now = time(NULL);
if (!cookie_db || !fname)
return -1;
info_printf(_("saving cookies to '%s'\n"), fname);
if ((fp = fopen(fname, "w"))) {
fputs("# HTTP cookie file\n", fp);
fputs("#Generated by Mget " PACKAGE_VERSION ". Edit at your own risk.\n\n", fp);
mget_thread_mutex_lock(&cookie_db->mutex);
for (it = 0; it < mget_vector_size(cookie_db->cookies) && !ferror(fp); it++) {
mget_cookie_t *cookie = mget_vector_get(cookie_db->cookies, it);
if (cookie->persistent) {
if (cookie->expires < now)
continue;
} else if (!keep_session_cookies)
continue;
fprintf(fp, "%s%s%s\t%s\t%s\t%s\t%"PRId64"\t%s\t%s\n",
cookie->http_only ? "#HttpOnly_" : "",
cookie->domain_dot ? "." : "", // compatibility, irrelevant since RFC 6562
cookie->domain,
cookie->host_only ? "FALSE" : "TRUE",
cookie->path, cookie->secure_only ? "TRUE" : "FALSE",
(int64_t)cookie->expires,
cookie->name, cookie->value);
}
mget_thread_mutex_unlock(&cookie_db->mutex);
if (!ferror(fp))
ret = 0;
if (fclose(fp))
ret = -1;
if (ret)
error_printf(_("Failed to write to cookie file '%s' (%d)\n"), fname, errno);
} else
error_printf(_("Failed to open cookie file '%s' (%d)\n"), fname, errno);
return ret;
}
JOB *queue_add_job(JOB *job)
{
if (job) {
JOB *jobp;
mget_thread_mutex_lock(&mutex);
jobp = mget_list_append(&queue, job, sizeof(JOB));
mget_thread_mutex_unlock(&mutex);
debug_printf("queue_add_job %p %s\n", (void *)jobp, job->iri->uri);
return jobp;
}
return NULL;
}
int mget_cookie_store_cookie(mget_cookie_db_t *cookie_db, mget_cookie_t *cookie)
{
mget_cookie_t *old;
int pos;
if (!cookie_db) {
mget_cookie_deinit(cookie);
return -1;
}
debug_printf("got cookie %s=%s\n", cookie->name, cookie->value);
if (!cookie->normalized) {
mget_cookie_deinit(cookie);
return -1;
}
if (mget_cookie_check_psl(cookie_db, cookie) != 0) {
debug_printf("cookie '%s' dropped, domain '%s' is a public suffix\n", cookie->name, cookie->domain);
mget_cookie_deinit(cookie);
return -1;
}
mget_thread_mutex_lock(&cookie_db->mutex);
old = mget_vector_get(cookie_db->cookies, pos = mget_vector_find(cookie_db->cookies, cookie));
if (old) {
debug_printf("replace old cookie %s=%s\n", cookie->name, cookie->value);
cookie->creation = old->creation;
mget_vector_replace(cookie_db->cookies, cookie, sizeof(*cookie), pos);
} else {
debug_printf("store new cookie %s=%s\n", cookie->name, cookie->value);
mget_vector_insert_sorted(cookie_db->cookies, cookie, sizeof(*cookie));
}
mget_thread_mutex_unlock(&cookie_db->mutex);
return 0;
}
char *mget_cookie_create_request_header(mget_cookie_db_t *cookie_db, const mget_iri_t *iri)
{
int it, init = 0;
time_t now = time(NULL);
mget_buffer_t buf;
if (!cookie_db || !iri)
return NULL;
debug_printf("cookie_create_request_header for host=%s path=%s\n",iri->host,iri->path);
mget_thread_mutex_lock(&cookie_db->mutex);
for (it = 0; it < mget_vector_size(cookie_db->cookies); it++) {
mget_cookie_t *cookie = mget_vector_get(cookie_db->cookies, it);
if (((!cookie->host_only && _domain_match(cookie->domain, iri->host)) ||
(cookie->host_only && !strcmp(cookie->domain, iri->host))) &&
(!cookie->expires || cookie->expires >= now) &&
(!cookie->secure_only || (cookie->secure_only && iri->scheme == MGET_IRI_SCHEME_HTTPS)) &&
_path_match(cookie->path, iri->path))
{
if (!init) {
mget_buffer_init(&buf, NULL, 128);
init = 1;
}
if (buf.length)
mget_buffer_printf_append2(&buf, "; %s=%s", cookie->name, cookie->value);
else
mget_buffer_printf_append2(&buf, "%s=%s", cookie->name, cookie->value);
}
}
mget_thread_mutex_unlock(&cookie_db->mutex);
return init ? buf.data : NULL;
}
void mget_ocsp_db_add_host(mget_ocsp_db_t *ocsp_db, mget_ocsp_t *ocsp)
{
if (!ocsp)
return;
if (!ocsp_db) {
mget_ocsp_free(ocsp);
return;
}
mget_thread_mutex_lock(&ocsp_db->mutex);
if (ocsp->maxage == 0) {
if (mget_hashmap_remove(ocsp_db->hosts, ocsp))
debug_printf("removed OCSP host %s\n", ocsp->key);
mget_ocsp_free(ocsp);
} else {
mget_ocsp_t *old = mget_hashmap_get(ocsp_db->hosts, ocsp);
if (old) {
if (old->mtime < ocsp->mtime) {
old->mtime = ocsp->mtime;
old->maxage = ocsp->maxage;
old->valid = ocsp->valid;
debug_printf("update OCSP host %s (maxage=%ld)\n", old->key, old->maxage);
}
mget_ocsp_free(ocsp);
} else {
// key and value are the same to make mget_hashmap_get() return old 'ocsp'
mget_hashmap_put_noalloc(ocsp_db->hosts, ocsp, ocsp);
debug_printf("add OCSP host %s (maxage=%ld)\n", ocsp->key, ocsp->maxage);
// no need to free anything here
}
}
mget_thread_mutex_unlock(&ocsp_db->mutex);
}
void blacklist_print(void)
{
mget_thread_mutex_lock(&mutex);
mget_hashmap_browse(blacklist, (int(*)(void *, const void *, void *))_blacklist_print, NULL);
mget_thread_mutex_unlock(&mutex);
}
void blacklist_free(void)
{
mget_thread_mutex_lock(&mutex);
mget_hashmap_free(&blacklist);
mget_thread_mutex_unlock(&mutex);
}
