int main(int argc, char *argv[])
{
char tag[1], attr[4], val[128];
CURL *curl;
HTMLSTREAMPARSER *hsp;
if (argc != 2) {
printf("Usage: %s URL\n", argv[0]);
return EXIT_FAILURE;
}
curl = curl_easy_init();
hsp = html_parser_init();
html_parser_set_tag_to_lower(hsp, 1);
html_parser_set_attr_to_lower(hsp, 1);
html_parser_set_tag_buffer(hsp, tag, sizeof(tag));
html_parser_set_attr_buffer(hsp, attr, sizeof(attr));
html_parser_set_val_buffer(hsp, val, sizeof(val)-1);
curl_easy_setopt(curl, CURLOPT_URL, argv[1]);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, hsp);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_perform(curl);
curl_easy_cleanup(curl);
html_parser_cleanup(hsp);
return EXIT_SUCCESS;
}
char* GetLinksByTag(char* htmlcontent, char* url, char * tag_name, char * attribute)
{
// Buffers for the stream parser.
char tag[8];
char attr[20];
char val[1024];
// Links string.
char* links = malloc(1);
size_t links_length = 0;
size_t i; // Loop iterator
char* base = malloc(strlen(url) + 1);
strcpy(base, url);
// Initialize the parser
HTMLSTREAMPARSER *hsp = html_parser_init( );
html_parser_set_tag_to_lower(hsp, 1);
html_parser_set_attr_to_lower(hsp, 1);
html_parser_set_tag_buffer(hsp, tag, sizeof(tag));
html_parser_set_attr_buffer(hsp, attr, sizeof(attr));
html_parser_set_val_buffer(hsp, val, sizeof(val)-1);
links[0] = '\0';
const size_t document_size = strlen(htmlcontent);
const size_t tag_length = strlen(tag_name);
const size_t attr_length = strlen(attribute);
// Loop over document
for (i = 0; i < document_size; i++) {
html_parser_char_parse(hsp, htmlcontent[i]);
// Detect different html base
if (html_parser_cmp_tag(hsp, "base", 4) && html_parser_cmp_attr(hsp, "href", 4) && html_parser_is_in(hsp, HTML_VALUE_ENDED)) {
html_parser_val(hsp)[html_parser_val_length(hsp)] = '\0';
base = realloc(base, strlen(html_parser_val(hsp)) + 1);
strcpy(base, html_parser_val(hsp));
}
if (html_parser_cmp_tag(hsp, tag_name, tag_length)) {
if (html_parser_cmp_attr(hsp, attribute, attr_length)) {
if (html_parser_is_in(hsp, HTML_VALUE_ENDED)) {
html_parser_val(hsp)[html_parser_val_length(hsp)] = '\0';
char* link = FormatLink(base, html_parser_val(hsp));
// Plus 1 byte for the newline
size_t link_length = strlen(link) + 1;
// Plus 1 byte for the null terminator
links = realloc(links, links_length + link_length + 1);
strcat(links, link);
strcat(links, "\n");
links_length += link_length;
free(link);
}
}
}
}
/*release the hsp*/
html_parser_cleanup(hsp);
free(base);
return links;
}
int main( int argc, char *argv[] )
{
if (argc<2)
{
printf( "Usage: %s <filename> {<filename>}\n\n", argv[0] );
exit(-1);
}
int paramnr;
html_parser_init();
for (paramnr=1; paramnr<argc; paramnr++)
{
FILE *file = fopen( argv[paramnr], "r" );
if (!file)
{
fprintf( stderr, "Could not open %s.\n", argv[1] );
html_parser_exit();
return -1;
}
// Get filesize:
struct stat fileinfo;
fstat( fileno( file ), &fileinfo );
// printf("Reading %i bytes...\n", fileinfo.st_size);
int size = fileinfo.st_size;
char *buf = (char*)malloc(sizeof(char)*size);
int i;
for (i=0; i<size;)
{
i+= fread( (void*)&(buf[i]), sizeof(char), size-i, file );
// printf("%i...\n", i);
}
char *title, *body;
html_parser_run( "http://YAHOOgroups.com/svada/index.html", buf, size, &title, &body, fn, NULL );
free(buf);
fclose(file);
printf("Title: %s\nBody:\n%.512s\n", title, body);
// printf("\n\033[1;34mTitle\033[0m: %s\n\033[1;34mBody\033[0m:\n%s\n", title, body);
free(title);
free(body);
}
html_parser_exit();
exit(0);
}
int main (int argc, char *argv[]) {
struct DocumentIndexFormat DocumentIndexPost;
int PopRankextern;
int PopRankintern;
int PopRanknoc;
int PopRanindex;
char ShortRank;
FILE *FH;
struct popl popextern;
struct popl popintern;
struct popl popnoc;
struct popl popindex;
uLong htmlBufferSize = 0;
char *htmlBuffer = NULL;
char *acl_allowbuffer = NULL;
char *acl_deniedbuffer = NULL;
char timebuf[26];
int optShowhtml = 0;
int optShowWords = 0;
int optSummary = 0;
int optAnchor = 0;
int optResource = 0;
int optPopRank = 0;
int optDelete = 0;
int optAdult = 0;
unsigned int DocID;
char *subname;
if (getenv("QUERY_STRING") == NULL) {
extern char *optarg;
extern int optind, opterr, optopt;
char c;
while ((c=getopt(argc,argv,"hwsarpdu"))!=-1) {
switch (c) {
case 'h':
optShowhtml = 1;
break;
case 'u':
optAdult = 1;
break;
case 'w':
optShowWords = 1;
break;
case 's':
optSummary = 1;
break;
case 'a':
optAnchor = 1;
break;
case 'p':
optPopRank = 1;
break;
case 'r':
optResource = 1;
break;
case 'd':
optDelete = 1;
break;
default:
exit(1);
}
}
--optind;
#ifdef DEBUG
printf("argc %i, optind %i\n",argc,optind);
#endif
if ((argc - optind)!= 3) {
printf("Dette programet gir info om en DocID\n\n\tUsage PageInfo DocID collection\n");
exit(1);
}
DocID = atol(argv[1 +optind]);
subname = argv[2 +optind];
}
else {
printf("Content-type: text/plain\n\n");
int res;
// Initialize the CGI lib
res = cgi_init();
// Was there an error initializing the CGI???
if (res != CGIERR_NONE) {
printf("Error # %d: %s<p>\n", res, cgilib_strerror(res));
fprintf(stderr,"Cgi-lib error.");
return -1;
}
if (cgi_getentrystr("subname") == NULL) {
fprintf(stderr,"Didn't recieve any subname.");
return -1;
}
else {
subname = cgi_getentrystr("subname");
}
if (cgi_getentrystr("DocID") == NULL) {
fprintf(stderr,"Didn't recieve any DocID.");
return -1;
}
else {
DocID = atol( cgi_getentrystr("DocID") );
}
}
html_parser_init();
printf("Showing data for Collection \"%s\", DocID %u\n\n",subname,DocID);
printf("Lot: %i\n",rLotForDOCid(DocID));
if (optDelete) {
memset(&DocumentIndexPost,'\0',sizeof(DocumentIndexPost));
DIWrite(&DocumentIndexPost,DocID,subname,NULL);
return 0;
}
if (DIRead_fmode(&DocumentIndexPost,DocID,subname,'s')) {
printf("Url: \"%s\"\nLanguage: %s (id: %s)\nOffensive code: %hu\nDocument type: %s\nTime tested sins last good crawl: %hu\nAdult weight: %hu\nResource size: %u\nIP Address: %u\nHtml size: %i\nImage size: %i\nUser ID: %i\nCrawler version: %f\nRepository pointer: %u\n",
DocumentIndexPost.Url,
getLangCode2(atoi(DocumentIndexPost.Sprok)),
DocumentIndexPost.Sprok,
DocumentIndexPost.Offensive_code,
DocumentIndexPost.Dokumenttype,
DocumentIndexPost.AntallFeiledeCrawl,
DocumentIndexPost.AdultWeight,
DocumentIndexPost.ResourceSize,
DocumentIndexPost.IPAddress,
DocumentIndexPost.htmlSize2,
DocumentIndexPost.imageSize,
DocumentIndexPost.userID,
DocumentIndexPost.clientVersion,
DocumentIndexPost.RepositoryPointer);
if (DocumentIndexPost.response == 200) {
printf("HTTP response: %hu\n",DocumentIndexPost.response);
}
else {
printf("HTTP response: \033[1;31m%hu\033[0m\n",DocumentIndexPost.response);
}
ctime_r((time_t *)&DocumentIndexPost.CrawleDato,timebuf);
timebuf[24] = '\0';
printf("Last crawled time: %u\n",DocumentIndexPost.CrawleDato);
printf("Last crawled time ISO: %s\n",timebuf);
printf("crc32: %u\n",DocumentIndexPost.crc32);
#ifdef BLACK_BOX
printf("Last seen Unix: %u\n",DocumentIndexPost.lastSeen);
printf("Last seen ISO: %s", ctime(&DocumentIndexPost.lastSeen));
#endif
printf("Nr of out links: %u\n",(unsigned int)DocumentIndexPost.nrOfOutLinks);
char *metadesc, *title, *body;
if (DocumentIndexPost.SummarySize == 0) {
printf("Summary: Don't have pre-parsed summery (summary size is 0)\n");
}
else if (rReadSummary(DocID,&metadesc, &title, &body,DocumentIndexPost.SummaryPointer,DocumentIndexPost.SummarySize,subname)) {
printf("\nSummary:\n");
printf("\tSummary pointer: %u\n\tSummary size: %hu\n",DocumentIndexPost.SummaryPointer,DocumentIndexPost.SummarySize);
printf("\tTitle from summary: \"%s\"\n\tMeta description from summary: \"%s\"\n",title,metadesc);
if (optSummary) {
printf("Summary body\n*******************\n%s\n*******************\n\n",body);
}
}
else {
printf("Don't have pre-parsed summery\n");
}
struct ReposetoryHeaderFormat ReposetoryHeader;
char *url, *attributes;
if (!rReadHtml(&htmlBuffer,&htmlBufferSize,DocumentIndexPost.RepositoryPointer,DocumentIndexPost.htmlSize2,DocID,subname,&ReposetoryHeader,&acl_allowbuffer,&acl_deniedbuffer,DocumentIndexPost.imageSize, &url, &attributes)) {
printf("rReadHtml: did not returne true!\n");
return;
}
printf("Entire url: %s\n", url);
#ifdef BLACK_BOX
printf("acl allow raw: \"%s\"\n",acl_allowbuffer);
printf("acl denied raw: \"%s\"\n",acl_deniedbuffer);
printf("acl allow resolved: \"%s\"\n",aclResolv(acl_allowbuffer));
printf("acl denied resolved: \"%s\"\n",aclResolv(acl_deniedbuffer));
printf("PopRank: %d\n", ReposetoryHeader.PopRank);
#endif
if (optShowhtml) {
printf("html uncompresed size %i\n",htmlBufferSize);
printf("html buff:\n*******************************\n");
fwrite(htmlBuffer,htmlBufferSize,1,stdout);
printf("\n*******************************\n\n");
}
if (optShowWords) {
printf("words:\n");
//run_html_parser( DocumentIndexPost.Url, htmlBuffer, htmlBufferSize, fn );
char *title, *body;
html_parser_run(url,htmlBuffer, htmlBufferSize,&title, &body,fn,NULL );
}
if (optResource) {
char buf[500000];
printf("Resource:\n");
printf("Ptr: 0x%x Len: %x\n", DocumentIndexPost.ResourcePointer, DocumentIndexPost.ResourceSize);
if (getResource(rLotForDOCid(DocID), subname, DocID, buf, sizeof(buf)) == 0) {
printf("\tDid not get any resource\n");
warn("");
} else {
printf("%s\n", buf);
}
}
printf("attributes:\"%s\"\n", attributes);
free(url);
free(attributes);
free(acl_allowbuffer);
free(acl_deniedbuffer);
}
else {
printf("Cant read post\n");
}
#ifndef BLACK_BOX
if (optAdult) {
int httpResponsCodes[nrOfHttpResponsCodes];
//char *title;
//char *body;
struct adultFormat *adult;
struct pagewordsFormat *pagewords = malloc(sizeof(struct pagewordsFormat));
int AdultWeight;
unsigned char langnr;
if ((adult = malloc(sizeof(struct adultFormat))) == NULL) {
perror("malloc argstruct.adult");
exit(1);
}
wordsInit(pagewords);
langdetectInit();
adultLoad(adult);
AdultWeight -1;
handelPage(pagewords,&ReposetoryHeader,htmlBuffer,htmlBufferSize,&title,&body);
wordsMakeRevIndex(pagewords,adult,&AdultWeight,&langnr);
printf("adult %i\n",AdultWeight);
}
if (optAnchor) {
int anchorBufferSize;
char *anchorBuffer;
anchorBufferSize = anchorRead(rLotForDOCid(DocID),subname,DocID,NULL,-1);
anchorBufferSize += 1;
anchorBuffer = malloc(anchorBufferSize);
anchorRead(rLotForDOCid(DocID),subname,DocID,anchorBuffer,anchorBufferSize);
printf("#######################################\nanchors:\n%s\n#######################################\n",anchorBuffer);
free(anchorBuffer);
}
if (optPopRank) {
popopen (&popindex,"/home/boitho/config/popindex");
PopRanindex = popRankForDocID(&popindex,DocID);
popclose(&popindex);
printf("popindex %i\n",PopRanindex);
if (popopen (&popextern,"/home/boitho/config/popextern")) {
PopRankextern = popRankForDocID(&popextern,DocID);
printf("PopRankextern: %i\n",PopRankextern);
popclose(&popextern);
}
if (popopen (&popintern,"/home/boitho/config/popintern")) {
PopRankintern = popRankForDocID(&popintern,DocID);
printf("PopRankintern %i\n",PopRankintern);
popclose(&popintern);
}
if (popopen (&popnoc,"/home/boitho/config/popnoc")) {
PopRanknoc = popRankForDocID(&popnoc,DocID);
printf("PopRanknoc %i\n",PopRanknoc);
popclose(&popnoc);
}
if (popopen (&popindex,"/home/boitho/config/popindex")) {
PopRanindex = popRankForDocID(&popindex,DocID);
printf("popindex %i\n",PopRanindex);
popclose(&popindex);
}
printf("PopRankextern: %i\nPopRankintern %i\nPopRanknoc %i\n",PopRankextern,PopRankintern,PopRanknoc);
int brank;
popopenMemArray_oneLot(subname,rLotForDOCid(DocID));
brank = popRankForDocIDMemArray(DocID);
printf("brank %i\n",brank);
//short rank
if ( (FH = fopen(SHORTPOPFILE,"rb")) == NULL ) {
perror("open");
}
else {
if ((fseek(FH,DocID* sizeof(ShortRank),SEEK_SET) == 0) && (fread(&ShortRank,sizeof(ShortRank),1,FH) != 0)) {
printf("Short rank %u\n",(unsigned char)ShortRank);
}
else {
printf("no hort rank avalibal\n");
};
fclose(FH);
}
} // if optPopRank
#endif
}
int main(int argc, char *argv[])
{
bblog_init("searchd");
bblog(CLEAN, "Initializing...");
int sockfd;
int runCount;
//int newsockfd;
socklen_t clilen;
struct sockaddr_in cli_addr, serv_addr;
FILE *LOGFILE;
//FILE *LOCK;
struct searchd_configFORMAT searchd_config;
struct config_t maincfg;
searchd_config.searchport = 0;
searchd_config.optLog = 0;
searchd_config.optMax = 0;
searchd_config.optSingle = 0;
searchd_config.optrankfile = NULL;
searchd_config.optPreOpen = 0;
searchd_config.optFastStartup = 0;
searchd_config.optCacheIndexes = 1;
searchd_config.optAlarm = 60;
// Needed for the speller to properly convert utf8 to wchar_t
setlocale(LC_ALL, "en_US.UTF-8");
/* Ignore collection updates for now */
signal(SIGUSR2, SIG_IGN);
/* And ignore spelling updates */
signal(SIGUSR1, SIG_IGN);
char c;
while ((c=getopt(argc,argv,"clp:m:b:vsofA:L:S:a:"))!=-1) {
switch (c) {
case 'p':
searchd_config.searchport = atoi(optarg);
bblog(CLEAN, "searchd: Option -p: Using port %i.",searchd_config.searchport);
break;
case 'l':
searchd_config.optLog = 1;
break;
case 'o':
searchd_config.optPreOpen = 1;
break;
case 'm':
searchd_config.optMax = atoi(optarg);
break;
case 'b':
searchd_config.optrankfile = optarg;
break;
case 'v': /* XXX: Remove now that we have severity in the logger? */
bblog(CLEAN, "searchd: Option -v: Verbose output.");
globalOptVerbose = 1;
break;
case 's':
bblog(INFO, "Option -s: Won't fork for new connections");
searchd_config.optSingle = 1;
break;
case 'f':
searchd_config.optFastStartup = 1;
break;
case 'c':
searchd_config.optCacheIndexes = 0;
break;
case 'A':
bblog_set_appenders(atoi(optarg));
break;
case 'L':
bblog_set_severity(atoi(optarg));
break;
case 'S':
spelling_min_freq = strtol(optarg, NULL, 10);
break;
case 'a':
searchd_config.optAlarm = atoi(optarg);
break;
default:
bblog(ERROR, "Unknown argument: %c", c);
errx(1, "Unknown argument: %c", c);
}
}
#ifdef BLACK_BOKS
bblog(CLEAN, "Blackbox mode (searchdbb)");
time_t starttime;
time(&starttime);
if (searchd_config.optLog) {
/* Only add file logging if syslog is disabled */
if ((bblog_get_appenders() & LOGGER_APPENDER_SYSLOG) == 0)
bblog_set_appenders(LOGGER_APPENDER_FILE|bblog_get_appenders());
}
/* Write pidfile */
FILE *pidfile = fopen(bfile("var/searchd.pid"), "w");
if (pidfile != NULL) {
fprintf(pidfile, "%d", getpid());
fclose(pidfile);
} else {
bblog(WARN, "Unable to write to pidfile");
}
bblog(CLEAN, "searchd: Starting. Time is %s",ctime(&starttime));
#endif
#ifdef DEBUG
bblog(DEBUGINFO, "searchd: Debug: argc %i, optind %i",argc,optind);
#endif
if (searchd_config.optrankfile == NULL) {
searchd_config.optrankfile = "Brank";
}
#ifdef WITH_SPELLING
if (searchd_config.optFastStartup != 1) {
if ((spelling = train(bfile("var/dictionarywords"))) == NULL) {
bblog(ERROR, "Can't init spelling.");
}
cache_spelling_keepalive(&spelling);
signal(SIGUSR1, cache_spelling_hup);
}
#endif
if (argc > optind) {
strncpy(servername,argv[optind], sizeof(servername) -1);
} else {
bblog(ERROR, "No hostname supplied");
errx(1, "You have to supply a hostname");
}
lotPreOpenStartl(&searchd_config.lotPreOpen.DocumentIndex,"DocumentIndex","www",searchd_config.optPreOpen);
lotPreOpenStartl(&searchd_config.lotPreOpen.Summary,"summary","www",searchd_config.optPreOpen);
#ifdef BLACK_BOKS
if (searchd_config.optCacheIndexes == 1) {
if (searchd_config.optFastStartup != 1) {
bblog(INFO, "Reading indexes");
cache_indexes(0);
bblog(INFO, "Cached indexes: %dMB, cached indexes: %d", indexcachescached[0]/(1024*1024), indexcachescached[1]);
preopen();
cache_fresh_lot_collection();
cache_indexes_keepalive();
signal(SIGUSR2, cache_indexes_hup);
}
else {
signal(SIGUSR2, SIG_IGN);
}
} else {
signal(SIGUSR2, SIG_IGN);
}
#endif
maincfg = maincfgopen();
if (searchd_config.searchport == 0) {
searchd_config.searchport = maincfg_get_int(&maincfg,"BSDPORT");
}
searchd_config.cmc_port = maincfg_get_int(&maincfg,"CMDPORT");
maincfgclose(&maincfg);
/***********************************************************************************/
//prøver å få fil lock. Bare en deamon kan kjøre avgangen
/*
#ifndef ALLOW_MULTIPLE_SEARCHD
if ((LOCK = fopen("/tmp/searchd.loc","w")) == NULL) {
perror("lock file");
exit(1);
}
if (flock(fileno(LOCK),LOCK_EX | LOCK_NB) != 0) {
if (errno == EWOULDBLOCK) {
printf("En annen prosses kjører allerede. Steng denne først.\n");
}
else {
perror("cant get lock file");
}
exit(1);
}
#endif
*/
/***********************************************************************************/
//#ifndef BLACK_BOKS
/* Initialize the configuration */
config_init(&cfg);
/* Load the file */
#ifdef DEBUG
bblog(DEBUGINFO, "searchd: Debug: Loading [%s] ...",bfile(cfg_searchd));
#endif
if (!config_read_file(&cfg, bfile(cfg_searchd))) {
bblog(ERROR, "config read failed: [%s]: %s at line %i",bfile(cfg_searchd),config_error_text(&cfg),config_error_line(&cfg));
exit(1);
}
//#endif
html_parser_init();
/*
#ifdef WITH_THREAD
pthread_t chld_thr;
printf("starting whth thread\n");
#else
printf("starting single thread version\n");
#endif
*/
bblog(CLEAN, "Servername: %s", servername);
//ToDo: må ha låsing her
if ((LOGFILE = bfopen("config/query.log","a")) == NULL) {
bblog_errno(ERROR, "%s", bfile("config/query.log"));
}
else {
fprintf(LOGFILE,"starting server %s\n",servername);
fclose(LOGFILE);
}
#ifdef BLACK_BOKS
// Initialiser thesaurus med ouput-filene fra 'build_thesaurus_*':
searchd_config.thesaurus_all = NULL;
#ifndef WITHOUT_THESAURUS
bblog(INFO, "init thesaurus");
searchd_config.thesaurus_all = NULL;
if (searchd_config.optFastStartup != 1) {
searchd_config.thesaurus_all = load_all_thesauruses(bfile("data/thesaurus/"));
if (searchd_config.thesaurus_all == NULL) {
bblog(ERROR, "Unable to open thesaurus. Disabling stemming");
} else {
bblog(INFO, "init thesaurus done");
}
}
#endif
bblog(INFO, "init file-extensions");
searchd_config.getfiletypep = fte_init(bfile("config/file_extensions.conf"));
if (searchd_config.getfiletypep == NULL) {
bblog(ERROR, "Unable to open file-extensions configuration file. Disabling file-extensions.");
}
bblog(INFO, "init attribute descriptions");
searchd_config.attrdescrp = adf_init(bfile("config/attribute_descriptions.conf"));
if (searchd_config.attrdescrp == NULL) {
bblog(ERROR, "Unable to open attribute descriptions configuration file. Disabling attribute descriptions.");
}
bblog(INFO, "init show-attributes");
char *warnings;
/*searchd_config.showattrp = show_attributes_init(bfile("config/show_attributes.conf"), &warnings);
if (searchd_config.showattrp == NULL)
{
fprintf(stderr, "searchd: ERROR!! Unable to open show-attributes configuration file. Disabling attributes.\n");
}
else if (warnings[0]!='\0')
{
fprintf(stderr, "searchd: ******************* Warnings reading show-attributes config: ********************\n");
fprintf(stderr, "%s", warnings);
fprintf(stderr, "searchd: *********************************************************************************\n");
}*/
#else
//starter opp
bblog(INFO, "Loading domain-ids...");
iintegerLoadMemArray2(&global_DomainIDs,"domainid",sizeof(unsigned short), "www");
//laster inn alle poprankene
bblog(INFO, "Loading pop MemArray...");
popopenMemArray2("www",searchd_config.optrankfile); // ToDo: hardkoder subname her, da vi ikke vet siden vi ikke her får et inn enda
bblog(INFO, "Loading adultWeight MemArray...");
adultWeightopenMemArray2("www"); // ToDo: hardkoder subname her, da vi ikke vet siden vi ikke her får et inn enda
#endif
IIndexInaliser();
#ifdef WITH_MEMINDEX
IIndexLoad();
#endif
if((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
bblog(ERROR, "Server error! Can't open stream socket.");
exit(1);
}
memset((char *) &serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
serv_addr.sin_port = htons(searchd_config.searchport);
bblog(INFO, "Will bind to port %i",searchd_config.searchport);
//seter at sokket kan rebrukes
int yes=1;
if (setsockopt(sockfd,SOL_SOCKET,SO_REUSEADDR,&yes,sizeof(int)) == -1) {
bblog_errno(ERROR, "setsockopt()");
exit(1);
}
if(bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) {
bblog_errno(ERROR, "Can't bind local address. Port %i", searchd_config.searchport);
exit(1);
}
/* set the level of thread concurrency we desire */
//thr_setconcurrency(5);
listen(sockfd, 5);
runCount = 0;
#ifndef NEW_CHILD_CATCHER
signal(SIGCLD, SIG_IGN); /* now I don't have to wait() for forked children! */
#else
{
// runarb 28 juni 2009:
// Hvis vi har verbose output så skal vi wait()e for våre barn, og vise prity print når de dør.
// desverre har det vært mye kød, der hovedprosessen blir hengene i sigchild_handler() og ikke kan
// fork'e flere barn. For å ungå dtte venter vi ikke på våre barn til vanlig.
if (globalOptVerbose) {
struct sigaction sa;
int ret;
sa.sa_sigaction = sigchild_handler;
sigemptyset(&sa.sa_mask);
sa.sa_flags = SA_SIGINFO;
ret = sigaction(SIGCHLD, &sa, 0);
if (ret) {
bblog_errno(ERROR, "sigaction()");
exit(1);
}
}
else {
signal(SIGCLD, SIG_IGN); /* now I don't have to wait() for forked children! */
}
}
#endif
bblog(CLEAN, "|------------------------------------------------------------------------------------------------|");
bblog(CLEAN, "|%-40s | %-11s | %-11s | %-11s | %-11s|","query", "TotaltTreff", "showabal", "filtered", "total_usecs");
bblog(CLEAN, "|------------------------------------------------------------------------------------------------|");
for((clilen = sizeof(cli_addr));;)
{
searchd_config.newsockfd = accept(sockfd, (struct sockaddr *) &cli_addr, &clilen);
if(searchd_config.newsockfd < 0) {
/* Just restart */
if (errno == EINTR)
continue;
bblog(WARN, "searchd: Server warning! Accept error");
}
else {
if (searchd_config.optSingle) {
do_chld((void *) &searchd_config);
}
else {
#ifdef DEBUG
bblog(DEBUGINFO, "Debug mode; will not fork to new process.");
do_chld((void *) &searchd_config);
#else
/*
#ifdef WITH_THREAD
//create a new thread to process the incomming request
//thr_create(NULL, 0, do_chld, (void *) searchd_config, THR_DETACHED, &chld_thr);
pthread_create(&chld_thr, NULL, do_chld, (void *) &searchd_config);
//the server is now free to accept another socket request
#else
do_chld((void *) &searchd_config);
#endif
*/
bblog(DEBUGINFO, "Forking new prosess.");
if (fork() == 0) { // this is the child process
close(sockfd); // child doesn't need the listener
do_chld((void *) &searchd_config);
close(searchd_config.newsockfd);
bblog(DEBUGINFO, "Terminating child.");
exit(0);
}
else {
close(searchd_config.newsockfd); // perent doesn't need the new socket
}
#endif
}
}
++runCount;
if ((searchd_config.optMax != 0) && (runCount >= searchd_config.optMax)) {
//venter på siste trå. Ikke helt optimalt dette, da vi kan ha flere tråer som kjører i paralell
/*
#ifdef WITH_THREAD
pthread_join(chld_thr, NULL);
#endif
*/
bblog(WARN, "have reached Max runs. Exiting...");
break;
}
}
html_parser_exit();
free(searchd_config.lotPreOpen.Summary);
free(searchd_config.lotPreOpen.DocumentIndex);
adf_destroy(searchd_config.attrdescrp);
fte_destroy(searchd_config.getfiletypep);
if (searchd_config.optFastStartup != 1) {
thesaurus_destroy(searchd_config.thesaurusp);
}
//freegjør spelling. Trekt, men kjekt av valgring kan finne ut om noe ikke her blirr frigjort.
if (searchd_config.optFastStartup != 1) {
untrain(&spelling);
}
return(0);
}
