static void test(void) { unsigned i; for (i = 0; i < sizeof TestCase / sizeof TestCase[0]; i++) { parse_frame pf = { PROT_DNS, T->len, T->txt, NULL }; printf("#%2u: ", i); dump_chars(T->txt, T->len, stdout); dns_parse(T->txt, T->len, &pf, NULL); dump(&pf, 0, stdout); fputc('\n', stdout); T++; } }
int Spider_Url_Rinse::rinse_urls(UrlPtrVec& url_array) { Recursive_Lock lock(m_queue_mutex); UrlPtrVec dns_url_array; UrlPtrVec crawl_url_array; for (unsigned int i=0 ;i<url_array.size();i++) { UrlPtr& object=url_array[i]; if ( !search_and_record_url(object) ) //去除重复的url { //没有检索过此URL if ( search_domain(object->domain)) //检查Domain { //在dns数据库中 std::map<std::string,std::string>::iterator iter=m_dns_database.find(std::string(object->domain)); if ( iter!=m_dns_database.end() ) { object->ip=strdup((iter->second).c_str()); crawl_url_array.push_back(object); //抓取URL } else { dns_url_array.push_back(object); } } else { //不在数据库中 dns_url_array.push_back(object); } } } LLOG(L_DEBUG, "rinse_urls array size : %d", dns_url_array.size()); dns_parse(dns_url_array); Spider_Executor::instance().put_urls(crawl_url_array); return 0; }
int main(int argc, char *argv[]) { dnsPacket *dns, *dnsresponse; int iResult, iReceived; char *qdomain, *qserver, *response; unsigned short qtype = 1, qclass = 1; qdomain = ((qdomain = command_arg_get(0, '-', argc, argv)) != NULL) ? qdomain : DNS_DEFAULT_LOOKUP; qserver = ((qserver = command_arg_get(1, '-', argc, argv)) != NULL) ? qserver : DNS_LOOKUP_SERVER; // create our empty DNS packet dns = dns_create(); // set the required header fields dns->base->id = DNS_IDENTIFIER; // use recursive query? if (command_opt_set("-r", argc, argv)) { dns->base->flags |= DNS_FLAGS_RD; printf("+ Using recursive querying\n"); } // use a different query type? if (command_opt_set("-cname", argc, argv)) { qtype = DNS_TYPE_CNAME; printf("+ Using CNAME query type\n"); } else if (command_opt_set("-ns", argc, argv)) { qtype = DNS_TYPE_NS; printf("+ Using NS query type\n"); } else if (command_opt_set("-mx", argc, argv)) { qtype = DNS_TYPE_MX; printf("+ Using MX query type\n"); } else if (command_opt_set("-ptr", argc, argv)) { qtype = DNS_TYPE_PTR; printf("+ Using PTR query type\n"); } else if (command_opt_set("-aaaa", argc, argv)) { qtype = DNS_TYPE_AAAA; printf("+ Using AAAA query type\n"); } // add our question section dns_add_question(dns, qdomain, strlen(qdomain), qtype, qclass); dns_hton(dns); // initialize the network network_init(qserver, DNS_PORT); network_connect_udp(); // send the packet out printf("Looking up %s using %s...\n", qdomain, qserver); iResult = network_send((char *)dns->base, dns->size); assert(iResult > 0); // block until a response is ready response = network_receive(&iReceived); assert(iReceived > 0); // load up the response packet dnsresponse = dns_parse(response, iReceived); assert(dnsresponse->base->flags & DNS_FLAGS_RESPONSE && dnsresponse->base->id == DNS_IDENTIFIER); output_print_response(dnsresponse); output_print_sections(dnsresponse); }