mget_iri_t *blacklist_add(mget_iri_t *iri) { if (!iri) return NULL; if (mget_iri_supported(iri)) { mget_thread_mutex_lock(&mutex); if (!blacklist) { blacklist = mget_hashmap_create(128, -2, (unsigned int(*)(const void *))hash_iri, (int(*)(const void *, const void *))mget_iri_compare); mget_hashmap_set_destructor(blacklist, (void(*)(void *, void *))_free_entry); } if (!mget_hashmap_contains(blacklist, iri)) { // info_printf("Add to blacklist: %s\n",iri->uri); mget_hashmap_put_noalloc(blacklist, iri, NULL); // use hashmap as a hashset (without value) mget_thread_mutex_unlock(&mutex); return iri; } mget_thread_mutex_unlock(&mutex); } mget_iri_free(&iri); return NULL; }
void queue_del(JOB *job) { if (job) { debug_printf("queue_del %p\n", (void *)job); // special handling for automatic robots.txt jobs if (job->deferred) { JOB new_job = { .iri = NULL }; if (job->host) job->host->robot_job = NULL; mget_iri_free(&job->iri); // create a job for each deferred IRI for (int it = 0; it < mget_vector_size(job->deferred); it++) { new_job.iri = mget_vector_get(job->deferred, it); new_job.local_filename = get_local_filename(new_job.iri); queue_add_job(&new_job); } } job_free(job); mget_thread_mutex_lock(&mutex); mget_list_remove(&queue, job); mget_thread_mutex_unlock(&mutex); }
void mget_ocsp_db_deinit(mget_ocsp_db_t *ocsp_db) { if (ocsp_db) { mget_thread_mutex_lock(&ocsp_db->mutex); mget_hashmap_free(&ocsp_db->fingerprints); mget_hashmap_free(&ocsp_db->hosts); mget_thread_mutex_unlock(&ocsp_db->mutex); } }
void mget_cookie_db_deinit(mget_cookie_db_t *cookie_db) { if (cookie_db) { #ifdef WITH_LIBPSL psl_free(cookie_db->psl); cookie_db->psl = NULL; #endif mget_thread_mutex_lock(&cookie_db->mutex); mget_vector_free(&cookie_db->cookies); mget_thread_mutex_unlock(&cookie_db->mutex); } }
int mget_cookie_db_save(mget_cookie_db_t *cookie_db, const char *fname, int keep_session_cookies) { FILE *fp; int it, ret = -1; time_t now = time(NULL); if (!cookie_db || !fname) return -1; info_printf(_("saving cookies to '%s'\n"), fname); if ((fp = fopen(fname, "w"))) { fputs("# HTTP cookie file\n", fp); fputs("#Generated by Mget " PACKAGE_VERSION ". Edit at your own risk.\n\n", fp); mget_thread_mutex_lock(&cookie_db->mutex); for (it = 0; it < mget_vector_size(cookie_db->cookies) && !ferror(fp); it++) { mget_cookie_t *cookie = mget_vector_get(cookie_db->cookies, it); if (cookie->persistent) { if (cookie->expires < now) continue; } else if (!keep_session_cookies) continue; fprintf(fp, "%s%s%s\t%s\t%s\t%s\t%"PRId64"\t%s\t%s\n", cookie->http_only ? "#HttpOnly_" : "", cookie->domain_dot ? "." : "", // compatibility, irrelevant since RFC 6562 cookie->domain, cookie->host_only ? "FALSE" : "TRUE", cookie->path, cookie->secure_only ? "TRUE" : "FALSE", (int64_t)cookie->expires, cookie->name, cookie->value); } mget_thread_mutex_unlock(&cookie_db->mutex); if (!ferror(fp)) ret = 0; if (fclose(fp)) ret = -1; if (ret) error_printf(_("Failed to write to cookie file '%s' (%d)\n"), fname, errno); } else error_printf(_("Failed to open cookie file '%s' (%d)\n"), fname, errno); return ret; }
JOB *queue_add_job(JOB *job) { if (job) { JOB *jobp; mget_thread_mutex_lock(&mutex); jobp = mget_list_append(&queue, job, sizeof(JOB)); mget_thread_mutex_unlock(&mutex); debug_printf("queue_add_job %p %s\n", (void *)jobp, job->iri->uri); return jobp; } return NULL; }
int mget_cookie_store_cookie(mget_cookie_db_t *cookie_db, mget_cookie_t *cookie) { mget_cookie_t *old; int pos; if (!cookie_db) { mget_cookie_deinit(cookie); return -1; } debug_printf("got cookie %s=%s\n", cookie->name, cookie->value); if (!cookie->normalized) { mget_cookie_deinit(cookie); return -1; } if (mget_cookie_check_psl(cookie_db, cookie) != 0) { debug_printf("cookie '%s' dropped, domain '%s' is a public suffix\n", cookie->name, cookie->domain); mget_cookie_deinit(cookie); return -1; } mget_thread_mutex_lock(&cookie_db->mutex); old = mget_vector_get(cookie_db->cookies, pos = mget_vector_find(cookie_db->cookies, cookie)); if (old) { debug_printf("replace old cookie %s=%s\n", cookie->name, cookie->value); cookie->creation = old->creation; mget_vector_replace(cookie_db->cookies, cookie, sizeof(*cookie), pos); } else { debug_printf("store new cookie %s=%s\n", cookie->name, cookie->value); mget_vector_insert_sorted(cookie_db->cookies, cookie, sizeof(*cookie)); } mget_thread_mutex_unlock(&cookie_db->mutex); return 0; }
char *mget_cookie_create_request_header(mget_cookie_db_t *cookie_db, const mget_iri_t *iri) { int it, init = 0; time_t now = time(NULL); mget_buffer_t buf; if (!cookie_db || !iri) return NULL; debug_printf("cookie_create_request_header for host=%s path=%s\n",iri->host,iri->path); mget_thread_mutex_lock(&cookie_db->mutex); for (it = 0; it < mget_vector_size(cookie_db->cookies); it++) { mget_cookie_t *cookie = mget_vector_get(cookie_db->cookies, it); if (((!cookie->host_only && _domain_match(cookie->domain, iri->host)) || (cookie->host_only && !strcmp(cookie->domain, iri->host))) && (!cookie->expires || cookie->expires >= now) && (!cookie->secure_only || (cookie->secure_only && iri->scheme == MGET_IRI_SCHEME_HTTPS)) && _path_match(cookie->path, iri->path)) { if (!init) { mget_buffer_init(&buf, NULL, 128); init = 1; } if (buf.length) mget_buffer_printf_append2(&buf, "; %s=%s", cookie->name, cookie->value); else mget_buffer_printf_append2(&buf, "%s=%s", cookie->name, cookie->value); } } mget_thread_mutex_unlock(&cookie_db->mutex); return init ? buf.data : NULL; }
void mget_ocsp_db_add_host(mget_ocsp_db_t *ocsp_db, mget_ocsp_t *ocsp) { if (!ocsp) return; if (!ocsp_db) { mget_ocsp_free(ocsp); return; } mget_thread_mutex_lock(&ocsp_db->mutex); if (ocsp->maxage == 0) { if (mget_hashmap_remove(ocsp_db->hosts, ocsp)) debug_printf("removed OCSP host %s\n", ocsp->key); mget_ocsp_free(ocsp); } else { mget_ocsp_t *old = mget_hashmap_get(ocsp_db->hosts, ocsp); if (old) { if (old->mtime < ocsp->mtime) { old->mtime = ocsp->mtime; old->maxage = ocsp->maxage; old->valid = ocsp->valid; debug_printf("update OCSP host %s (maxage=%ld)\n", old->key, old->maxage); } mget_ocsp_free(ocsp); } else { // key and value are the same to make mget_hashmap_get() return old 'ocsp' mget_hashmap_put_noalloc(ocsp_db->hosts, ocsp, ocsp); debug_printf("add OCSP host %s (maxage=%ld)\n", ocsp->key, ocsp->maxage); // no need to free anything here } } mget_thread_mutex_unlock(&ocsp_db->mutex); }
void blacklist_print(void) { mget_thread_mutex_lock(&mutex); mget_hashmap_browse(blacklist, (int(*)(void *, const void *, void *))_blacklist_print, NULL); mget_thread_mutex_unlock(&mutex); }
void blacklist_free(void) { mget_thread_mutex_lock(&mutex); mget_hashmap_free(&blacklist); mget_thread_mutex_unlock(&mutex); }