void mget_robots_free(ROBOTS **robots)
{
if (robots && *robots) {
mget_vector_free(&(*robots)->paths);
mget_vector_free(&(*robots)->sitemaps);
}
}
void mget_cookie_free_public_suffixes(void)
{
// vec_browse(suffixes, (int (*)(void *))cookie_free_suffix);
mget_vector_free(&suffixes);
// vec_browse(suffix_exceptions, (int (*)(void *))cookie_free_suffix);
mget_vector_free(&suffix_exceptions);
}
void job_free(JOB *job)
{
if (job) {
mget_metalink_free(&job->metalink);
mget_vector_free(&job->parts);
mget_vector_clear_nofree(job->deferred);
mget_vector_free(&job->deferred);
xfree(job->local_filename);
}
}
void mget_cookie_free_cookies(void)
{
pthread_mutex_lock(&cookies_mutex);
mget_vector_browse(cookies, (int (*)(void *))mget_cookie_free_cookie);
mget_vector_free(&cookies);
pthread_mutex_unlock(&cookies_mutex);
}
void mget_cookie_db_deinit(mget_cookie_db_t *cookie_db)
{
if (cookie_db) {
#ifdef WITH_LIBPSL
psl_free(cookie_db->psl);
cookie_db->psl = NULL;
#endif
mget_thread_mutex_lock(&cookie_db->mutex);
mget_vector_free(&cookie_db->cookies);
mget_thread_mutex_unlock(&cookie_db->mutex);
}
}
static void test_vector(void)
{
struct ENTRY
*tmp,
txt_sorted[5] = { {""}, {"four"}, {"one"}, {"three"}, {"two"} },
*txt[countof(txt_sorted)];
mget_vector_t
*v = mget_vector_create(2, -2, (int(*)(const void *, const void *))compare_txt);
unsigned
it;
int
n;
// copy
for (it = 0; it < countof(txt); it++)
txt[it] = &txt_sorted[it];
// shuffle txt
for (it = 0; it < countof(txt); it++) {
n = rand() % countof(txt);
tmp = txt[n];
txt[n] = txt[it];
txt[it] = tmp;
}
for (it = 0; it < countof(txt); it++) {
mget_vector_insert_sorted(v, txt[it], sizeof(struct ENTRY));
}
for (it = 0; it < countof(txt); it++) {
struct ENTRY *e = mget_vector_get(v, it);
if (!strcmp(e->txt,txt_sorted[it].txt))
ok++;
else
failed++;
}
mget_vector_free(&v);
}
ROBOTS *mget_robots_parse(const char *data)
{
ROBOTS *robots;
ROBOTS_PATH path;
int collect = 0;
const char *p;
if (!data || !*data)
return NULL;
robots = xcalloc(1, sizeof (ROBOTS));
do {
if (collect < 2 && !strncasecmp(data, "User-agent:", 11)) {
if (!collect) {
for (data += 11; *data == ' ' || *data == '\t'; data++);
if (!strncasecmp(data, "mget", 4)) {
collect = 1;
}
else if (*data == '*') {
collect = 1;
}
} else
collect = 2;
}
else if (collect == 1 && !strncasecmp(data, "Disallow:", 9)) {
for (data += 9; *data == ' ' || *data == '\t'; data++);
if (*data == '\r' || *data == '\n' || !*data) {
// all allowed
mget_vector_free(&robots->paths);
collect = 2;
} else {
if (!robots->paths) {
robots->paths = mget_vector_create(32, -2, NULL);
mget_vector_set_destructor(robots->paths, (void(*)(void *))_free_path);
}
for (p = data; !isspace(*p); p++);
path.len = p - data;
path.path = strndup(data, path.len);
mget_vector_add(robots->paths, &path, sizeof(path));
}
}
else if (!strncasecmp(data, "Sitemap:", 8)) {
for (data += 8; *data==' ' || *data == '\t'; data++);
for (p = data; !isspace(*p); p++);
if (!robots->sitemaps)
robots->sitemaps = mget_vector_create(4, -2, NULL);
mget_vector_add_noalloc(robots->sitemaps, strndup(data, p - data));
}
if ((data = strchr(data, '\n')))
data++; // point to next line
} while (data && *data);
/*
for (int it = 0; it < mget_vector_size(robots->paths); it++) {
ROBOTS_PATH *path = mget_vector_get(robots->paths, it);
info_printf("path '%s'\n", path->path);
}
for (int it = 0; it < mget_vector_size(robots->sitemaps); it++) {
const char *sitemap = mget_vector_get(robots->sitemaps, it);
info_printf("sitemap '%s'\n", sitemap);
}
*/
return robots;
}
