char *get_domain(struct in_addr addr)
{
struct cond_domain *c;
if ((c = search_domain(addr, daemon->cond_domain)))
return c->domain;
return daemon->domain_suffix;
}
int is_rev_synth(int flag, struct all_addr *addr, char *name)
{
struct cond_domain *c;
if (flag & F_IPV4 && (c = search_domain(addr->addr.addr4, daemon->synth_domains)))
{
char *p;
*name = 0;
if (c->prefix)
strncpy(name, c->prefix, MAXDNAME - ADDRSTRLEN);
inet_ntop(AF_INET, &addr->addr.addr4, name + strlen(name), ADDRSTRLEN);
for (p = name; *p; p++)
if (*p == '.')
*p = '-';
strncat(name, ".", MAXDNAME);
strncat(name, c->domain, MAXDNAME);
return 1;
}
#ifdef HAVE_IPV6
if (flag & F_IPV6 && (c = search_domain6(&addr->addr.addr6, daemon->synth_domains)))
{
char *p;
*name = 0;
if (c->prefix)
strncpy(name, c->prefix, MAXDNAME - ADDRSTRLEN);
inet_ntop(AF_INET6, &addr->addr.addr6, name + strlen(name), ADDRSTRLEN);
/* IPv6 presentation address can start with ":", but valid domain names
cannot start with "-" so prepend a zero in that case. */
if (!c->prefix && *name == ':')
{
*name = '0';
inet_ntop(AF_INET6, &addr->addr.addr6, name+1, ADDRSTRLEN);
}
/* V4-mapped have periods.... */
for (p = name; *p; p++)
if (*p == ':' || *p == '.')
*p = '-';
strncat(name, ".", MAXDNAME);
strncat(name, c->domain, MAXDNAME);
return 1;
}
#endif
return 0;
}
char * search_email (char *data)
{
char *tmp;
char *name;
char *domain;
int ret,count;
struct email_info one_email;
tmp = data;
for ( ; *tmp !='\0';tmp++ )
{
if (*tmp == '@')
break;
}
if (tmp == data)
return NULL;
name = search_name(tmp,data);
if (name == NULL)
return NULL;
count = tmp - name ;
if (count > 32)
return NULL;
strncpy(one_email.name,name,count);
one_email.name[count] = 0;
ret = check_name(one_email.name,1);
if (!ret)
return NULL;
domain = search_domain(tmp);
if (domain == NULL)
return NULL;
count = domain - tmp - 1;
if (count > 64)
return NULL;
tmp ++;
strncpy(one_email.domain ,tmp,count);
one_email.domain[count] = 0;
ret = check_domain(one_email.domain,1);
if (!ret)
return NULL;
printf("name: %s@%s\n",one_email.name,one_email.domain);
}
void check_domain(void *mole,char *name,kdomain *root,int domain_type,int search_type){
kdomain *s_domain;
moleWorld *logMole = (moleWorld *) mole;
if((s_domain = search_domain(name,root,search_type)) != NULL){
if(s_domain->suspicious != domain_type){
s_domain->suspicious = domain_type;
useDB((void *)logMole,LIST_UPDATE,"domainList","type",domain_type,"name",name);
}
}
else{
useDB((void *)logMole,LIST_INSERT,"domainList",name,domain_type);
load_domain(name,root,domain_type);
}
}
int Spider_Url_Rinse::rinse_urls(UrlPtrVec& url_array)
{
Recursive_Lock lock(m_queue_mutex);
UrlPtrVec dns_url_array;
UrlPtrVec crawl_url_array;
for (unsigned int i=0 ;i<url_array.size();i++)
{
UrlPtr& object=url_array[i];
if ( !search_and_record_url(object) ) //去除重复的url
{ //没有检索过此URL
if ( search_domain(object->domain)) //检查Domain
{ //在dns数据库中
std::map<std::string,std::string>::iterator iter=m_dns_database.find(std::string(object->domain));
if ( iter!=m_dns_database.end() )
{
object->ip=strdup((iter->second).c_str());
crawl_url_array.push_back(object); //抓取URL
}
else
{
dns_url_array.push_back(object);
}
}
else
{ //不在数据库中
dns_url_array.push_back(object);
}
}
}
LLOG(L_DEBUG, "rinse_urls array size : %d", dns_url_array.size());
dns_parse(dns_url_array);
Spider_Executor::instance().put_urls(crawl_url_array);
return 0;
}
