static void test(void)
{
unsigned i;
for (i = 0; i < sizeof TestCase / sizeof TestCase[0]; i++) {
parse_frame pf = { PROT_DNS, T->len, T->txt, NULL };
printf("#%2u: ", i);
dump_chars(T->txt, T->len, stdout);
dns_parse(T->txt, T->len, &pf, NULL);
dump(&pf, 0, stdout);
fputc('\n', stdout);
T++;
}
}
int Spider_Url_Rinse::rinse_urls(UrlPtrVec& url_array)
{
Recursive_Lock lock(m_queue_mutex);
UrlPtrVec dns_url_array;
UrlPtrVec crawl_url_array;
for (unsigned int i=0 ;i<url_array.size();i++)
{
UrlPtr& object=url_array[i];
if ( !search_and_record_url(object) ) //去除重复的url
{ //没有检索过此URL
if ( search_domain(object->domain)) //检查Domain
{ //在dns数据库中
std::map<std::string,std::string>::iterator iter=m_dns_database.find(std::string(object->domain));
if ( iter!=m_dns_database.end() )
{
object->ip=strdup((iter->second).c_str());
crawl_url_array.push_back(object); //抓取URL
}
else
{
dns_url_array.push_back(object);
}
}
else
{ //不在数据库中
dns_url_array.push_back(object);
}
}
}
LLOG(L_DEBUG, "rinse_urls array size : %d", dns_url_array.size());
dns_parse(dns_url_array);
Spider_Executor::instance().put_urls(crawl_url_array);
return 0;
}
int main(int argc, char *argv[])
{
dnsPacket *dns, *dnsresponse;
int iResult, iReceived;
char *qdomain, *qserver, *response;
unsigned short qtype = 1, qclass = 1;
qdomain = ((qdomain = command_arg_get(0, '-', argc, argv)) != NULL) ? qdomain : DNS_DEFAULT_LOOKUP;
qserver = ((qserver = command_arg_get(1, '-', argc, argv)) != NULL) ? qserver : DNS_LOOKUP_SERVER;
// create our empty DNS packet
dns = dns_create();
// set the required header fields
dns->base->id = DNS_IDENTIFIER;
// use recursive query?
if (command_opt_set("-r", argc, argv))
{
dns->base->flags |= DNS_FLAGS_RD;
printf("+ Using recursive querying\n");
}
// use a different query type?
if (command_opt_set("-cname", argc, argv))
{
qtype = DNS_TYPE_CNAME;
printf("+ Using CNAME query type\n");
}
else if (command_opt_set("-ns", argc, argv))
{
qtype = DNS_TYPE_NS;
printf("+ Using NS query type\n");
}
else if (command_opt_set("-mx", argc, argv))
{
qtype = DNS_TYPE_MX;
printf("+ Using MX query type\n");
}
else if (command_opt_set("-ptr", argc, argv))
{
qtype = DNS_TYPE_PTR;
printf("+ Using PTR query type\n");
}
else if (command_opt_set("-aaaa", argc, argv))
{
qtype = DNS_TYPE_AAAA;
printf("+ Using AAAA query type\n");
}
// add our question section
dns_add_question(dns, qdomain, strlen(qdomain), qtype, qclass);
dns_hton(dns);
// initialize the network
network_init(qserver, DNS_PORT);
network_connect_udp();
// send the packet out
printf("Looking up %s using %s...\n", qdomain, qserver);
iResult = network_send((char *)dns->base, dns->size);
assert(iResult > 0);
// block until a response is ready
response = network_receive(&iReceived);
assert(iReceived > 0);
// load up the response packet
dnsresponse = dns_parse(response, iReceived);
assert(dnsresponse->base->flags & DNS_FLAGS_RESPONSE && dnsresponse->base->id == DNS_IDENTIFIER);
output_print_response(dnsresponse);
output_print_sections(dnsresponse);
}
